def run_model(dbsession,modelrun):
mapping = create_resource_mapping(modelschemas[modelrun.model_name],modelrun)
modelrun.progress_state = PROGRESS_STATES['RUNNING']
dbsession.commit()
kwargs = {'dbsession':dbsession,'modelrun_id':modelrun.id}
kwargs.update(mapping)
try:
#run the model
model_modules[modelrun.model_name]['method'](event_emitter=ee,**kwargs)
output_mapping = resolve_output_map(mapping,modelschemas[modelrun.model_name])
for m in output_mapping:
if os.path.exists(output_mapping[m]['location']):
output_resource = ModelResource()
output_resource.resource_type=output_mapping[m]['type']
output_resource.resource_location = output_mapping[m]['location']
output_resource.resource_size = os.stat(output_resource.resource_location).st_size
modelrun.resources.append(output_resource)
modelrun.progress_state=PROGRESS_STATES['FINISHED']
logging.info('done running::{modelrun}'.format(modelrun=modelrun))
except:
logging.info('Erorr Happended while running model:{modelrun}'.format(modelrun=modelrun))
logging.info(traceback.format_exc())
modelrun.progress_state=PROGRESS_STATES['ERROR']
dbsession.commit()
def create_output_resources(modelrunner,output_map):
resources = []
for m in output_map:
if os.path.exists(output_map[m]):
obj = storage.container.upload_object(output_map[m],os.path.basename(output_map[m]))
output_resource = ModelResource()
output_resource.resource_type = modelrunner.get_resource_type_from_map(m,'outputs')
output_resource.resource_name = obj.name
output_resource.resource_size = obj.size
resources.append(output_resource)
return resources
def create_output_resources(modelrunner,output_map):
resources = []
for m in output_map:
if os.path.exists(output_map[m]):
obj = storage.container.upload_object(output_map[m],os.path.basename(output_map[m]))
output_resource = ModelResource()
output_resource.resource_type = modelrunner.get_resource_type_from_map(m,'outputs')
output_resource.resource_name = obj.name
output_resource.resource_size = obj.size
resources.append(output_resource)
return resources
def upload(id):
modelrun = ModelRun.query.get(id)
if modelrun:
# if modelrun.progress_state in
# [PROGRESS_STATES['NOT_STARTED'],PROGRESS_STATES['RUNNING']]:
file = request.files['file']
if file:
filename = secure_filename(file.filename)
resource_file = storage.upload(file)
resource_type = request.form['resource_type']
m = {
'modelrun_id': id,
'resource_type': resource_type,
'resource_name': resource_file.name,
#'resource_url': url_for('modelresource.download_resource_by_name', name=resource_file.name, _external=True),
'resource_size': resource_file.size
}
resource = ModelResource.create(**m)
msg = {"message": "Resource create for model run " +
str(id), 'resource': modelresource_serializer(resource)}
return jsonify(msg), 201
else:
err = {"message": "File parameter isn't provided"}
return jsonify(err), 400
else:
err = {"message": "Modelrun doesn't exist"}
return jsonify(err), 404
def upload(id):
modelrun = ModelRun.query.get(id)
if modelrun:
# if modelrun.progress_state in
# [PROGRESS_STATES['NOT_STARTED'],PROGRESS_STATES['RUNNING']]:
file = request.files['file']
if file:
filename = secure_filename(file.filename)
resource_file = storage.upload(file)
resource_type = request.form['resource_type']
m = {
'modelrun_id': id,
'resource_type': resource_type,
'resource_name': resource_file.name,
#'resource_url': url_for('modelresource.download_resource_by_name', name=resource_file.name, _external=True),
'resource_size': resource_file.size
}
resource = ModelResource.create(**m)
msg = {
"message": "Resource create for model run " + str(id),
'resource': modelresource_serializer(resource)
}
return jsonify(msg), 201
else:
err = {"message": "File parameter isn't provided"}
return jsonify(err), 400
else:
err = {"message": "Modelrun doesn't exist"}
return jsonify(err), 404
def upload_from_url(id):
'''
expects json. expects url,filename,resource_type
'''
modelrun = ModelRun.query.get(id)
if modelrun:
if modelrun.progress_state == PROGRESS_STATES['NOT_STARTED']:
try:
data = json.loads(request.get_data())
except ValueError:
return jsonify({'message': 'Please specify valid json'}), 400
if not ('url' in data and 'resource_type' in data
and 'filename' in data):
return jsonify({'message': 'Invalid Input Provided'}), 400
try:
filedata = requests.get(data['url'])
tmp_loc = os.path.join('/tmp/', data['filename'])
with app.open_instance_resource(tmp_loc, 'wb') as f:
f.write(filedata.content)
resource_file = storage.upload(tmp_loc)
resource_type = data['resource_type']
m = {
'modelrun_id': id,
'resource_type': resource_type,
'resource_name': resource_file.name,
#'resource_url': url_for('modelresource.download_resource_by_name', name=resource_file.name, _external=True),
'resource_size': resource_file.size
}
resource = ModelResource.create(**m)
return jsonify({
'message':
"Resource create for model run " + str(id),
'resource':
modelresource_serializer(resource)
}), 201
except Exception, e:
return jsonify({'message':
'Couldn\'t get file from url.'}), 400
else:
return jsonify({
'message':
'Uploading resources to new modelrun is permitted only'
}), 400
def upload_from_url(id):
'''
expects json. expects url,filename,resource_type
'''
modelrun = ModelRun.query.get(id)
if modelrun:
if modelrun.progress_state == PROGRESS_STATES['NOT_STARTED']:
try:
data = json.loads(request.get_data())
except ValueError:
return jsonify({'message': 'Please specify valid json'}), 400
if not ('url' in data and 'resource_type' in data and 'filename' in data):
return jsonify({'message': 'Invalid Input Provided'}), 400
try:
filedata = requests.get(data['url'])
tmp_loc = os.path.join('/tmp/', data['filename'])
with app.open_instance_resource(tmp_loc, 'wb') as f:
f.write(filedata.content)
resource_file = storage.upload(tmp_loc)
resource_type = data['resource_type']
m = {
'modelrun_id': id,
'resource_type': resource_type,
'resource_name': resource_file.name,
#'resource_url': url_for('modelresource.download_resource_by_name', name=resource_file.name, _external=True),
'resource_size': resource_file.size
}
resource = ModelResource.create(**m)
return jsonify({'message': "Resource create for model run " + str(id), 'resource': modelresource_serializer(resource)}), 201
except Exception, e:
return jsonify({'message': 'Couldn\'t get file from url.'}), 400
else:
return jsonify({'message': 'Uploading resources to new modelrun is permitted only'}), 400
