def post(self, jobID=None):
job = ParameterSweepJobWrapper.get_by_id(int(jobID))
logging.info(
"ParameterSweepVisualizationPage.post() jobID={0} path={1}".format(
jobID, job.outData))
reqType = self.request.get('reqType')
if reqType == 'redirectJupyterNotebook':
try:
#Check if notebook already exists, if not create one
notebook_filename = "{0}.ipynb".format(job.name)
local_path = os.path.relpath(
os.path.abspath(job.outData),
os.path.abspath(__file__ + '/../../../'))
notebook_file_path = os.path.abspath(
job.outData) + "/" + notebook_filename
if not os.path.isfile(notebook_file_path):
modelType = self.request.get('modelType')
if modelType == 'deterministic':
notebook_template_path = os.path.abspath(
__file__ + '/../../../jupyter_notebook_templates'
) + "/ParameterSweep_deterministic.ipynb"
elif modelType == 'stochastic':
notebook_template_path = os.path.abspath(
__file__ + '/../../../jupyter_notebook_templates'
) + "/ParameterSweep_stochastic.ipynb"
elif modelType == 'spatial':
notebook_template_path = os.path.abspath(
__file__ + '/../../../jupyter_notebook_templates'
) + "/ParameterSweep_spatial.ipynb"
else:
raise Exception(
"Error, '{0}' is not a valid modelType".format(
modelType))
logging.info("Creating {0} from {1}".format(
notebook_file_path, notebook_template_path))
shutil.copyfile(notebook_template_path, notebook_file_path)
if self.request.get('hostname') is not None:
logging.info('hostname = {0}'.format(
self.request.get('hostname')))
host = self.request.get('hostname')
else:
logging.info('hostname = localhost (default)')
host = 'localhost'
port = 9999
proto = 'http'
#
# return the url of the notebook
notebook_url = '{0}://{1}:{2}/notebooks/{3}/{4}'.format(
proto, host, port, local_path, notebook_filename)
self.redirect(notebook_url)
except Exception as e:
logging.error("Error in openJupyterNotebook: {0}".format(e))
self.response.write('Error: {0}'.format(e))
return
else:
self.get(int(jobID))
def get(self, jobID=None):
jobID = int(jobID)
initialData = {}
jobDb = ParameterSweepJobWrapper.get_by_id(jobID)
logging.info(
"ParameterSweepVisualizationPage.get() jobID={0} path={1}".format(
jobID, jobDb.outData))
initialData = jobDb.getJSON()
if jobDb.resource == 'qsub':
initialData['stdout'] = ""
initialData['stderr'] = ""
else:
try:
with open(os.path.join(jobDb.outData, 'stdout'), 'r') as f:
initialData['stdout'] = f.read()
with open(os.path.join(jobDb.outData, 'stderr'), 'r') as f:
initialData['stderr'] = f.read()
except IOError as e:
molnsConfigDb = db.GqlQuery(
"SELECT * FROM MolnsConfigWrapper WHERE user_id = :1",
self.user.user_id()).get()
initialData['data'] = {}
if not molnsConfigDb:
initialData[
'stdout'] = 'ERROR: could not lookup molnsConfigDb'
molnsConfig = molns.MOLNSConfig(
config_dir=molnsConfigDb.folder)
#TODO: Check if the molns service is active
try:
log = molns.MOLNSExec.job_logs([jobDb.molnsPID],
molnsConfig)
initialData['stdout'] = log['msg']
except (IOError, molns.MOLNSException) as e:
initialData['stdout'] = str(e)
if jobDb.resource == 'local' or jobDb.resource == 'qsub' or jobDb.output_stored:
try:
with open(os.path.join(jobDb.outData, 'results'), 'r') as f:
initialData['data'] = pickle.load(f)
initialData['status'] = 'Finished'
except IOError as e:
initialData['data'] = {}
#logging.error('*'*80)
#logging.error("{0}".format(**{'initialData' : json.dumps(initialData)}))
#logging.error("{0}".format(initialData))
#logging.error('*'*80)
self.render_response('parameter_sweep_visualization.html',
**{'initialData': json.dumps(initialData)})
def get(self, jobID = None):
jobID = int(jobID)
initialData = {}
jobDb = ParameterSweepJobWrapper.get_by_id(jobID)
logging.info("ParameterSweepVisualizationPage.get() jobID={0} path={1}".format(jobID, jobDb.outData))
initialData = jobDb.getJSON()
if jobDb.resource == 'qsub':
initialData['stdout'] = ""
initialData['stderr'] = ""
else:
try:
with open(os.path.join(jobDb.outData, 'stdout'), 'r') as f:
initialData['stdout'] = f.read()
with open(os.path.join(jobDb.outData, 'stderr'), 'r') as f:
initialData['stderr'] = f.read()
except IOError as e:
molnsConfigDb = db.GqlQuery("SELECT * FROM MolnsConfigWrapper WHERE user_id = :1", self.user.user_id()).get()
initialData['data'] = {}
if not molnsConfigDb:
initialData['stdout'] = 'ERROR: could not lookup molnsConfigDb'
molnsConfig = molns.MOLNSConfig(config_dir = molnsConfigDb.folder)
#TODO: Check if the molns service is active
try:
log = molns.MOLNSExec.job_logs([jobDb.molnsPID], molnsConfig)
initialData['stdout'] = log['msg']
except (IOError, molns.MOLNSException) as e:
initialData['stdout'] = str(e)
if jobDb.resource == 'local' or jobDb.resource == 'qsub' or jobDb.output_stored:
try:
with open(os.path.join(jobDb.outData, 'results'), 'r') as f:
initialData['data'] = pickle.load(f)
initialData['status'] = 'Finished'
except IOError as e:
initialData['data'] = {}
#logging.error('*'*80)
#logging.error("{0}".format(**{'initialData' : json.dumps(initialData)}))
#logging.error("{0}".format(initialData))
#logging.error('*'*80)
self.render_response('parameter_sweep_visualization.html', **{'initialData' : json.dumps(initialData)})
def post(self, jobID=None):
job = ParameterSweepJobWrapper.get_by_id(int(jobID))
logging.info("ParameterSweepVisualizationPage.post() jobID={0} path={1}".format(jobID, job.outData))
reqType = self.request.get('reqType')
if reqType == 'redirectJupyterNotebook':
try:
#Check if notebook already exists, if not create one
notebook_filename = "{0}.ipynb".format(job.name)
local_path = os.path.relpath(os.path.abspath(job.outData), os.path.abspath(__file__+'/../../../'))
notebook_file_path = os.path.abspath(job.outData) + "/" + notebook_filename
if not os.path.isfile(notebook_file_path):
modelType = self.request.get('modelType')
if modelType == 'deterministic':
notebook_template_path = os.path.abspath(__file__+'/../../../jupyter_notebook_templates')+"/ParameterSweep_deterministic.ipynb"
elif modelType == 'stochastic':
notebook_template_path = os.path.abspath(__file__+'/../../../jupyter_notebook_templates')+"/ParameterSweep_stochastic.ipynb"
elif modelType == 'spatial':
notebook_template_path = os.path.abspath(__file__+'/../../../jupyter_notebook_templates')+"/ParameterSweep_spatial.ipynb"
else:
raise Exception("Error, '{0}' is not a valid modelType".format(modelType))
logging.info("Creating {0} from {1}".format(notebook_file_path,notebook_template_path))
shutil.copyfile(notebook_template_path, notebook_file_path)
if self.request.get('hostname') is not None:
logging.info('hostname = {0}'.format(self.request.get('hostname')))
host = self.request.get('hostname')
else:
logging.info('hostname = localhost (default)')
host = 'localhost'
port = 9999
proto = 'http'
#
# return the url of the notebook
notebook_url = '{0}://{1}:{2}/notebooks/{3}/{4}'.format(proto,host,port,local_path,notebook_filename)
self.redirect(notebook_url)
except Exception as e:
logging.error("Error in openJupyterNotebook: {0}".format(e))
self.response.write('Error: {0}'.format(e))
return
else:
self.get(int(jobID))
def post(self):
reqType = self.request.get('reqType')
self.response.content_type = 'application/json'
if reqType == 'newJob':
# Run via Molns cloud
data = json.loads(self.request.get('data'))
self.user_data.set_selected(2)
job = db.GqlQuery(
"SELECT * FROM ParameterSweepJobWrapper WHERE user_id = :1 AND name = :2",
self.user.user_id(), data["jobName"].strip()).get()
if job != None:
self.response.write(
json.dumps({
"status": False,
"msg": "Job name must be unique"
}))
return
try:
result = self.runMolns(data=data)
return self.response.write(
json.dumps({
"status": True,
"msg": "Job launched",
"id": result.key().id()
}))
except Exception as e:
logging.exception(e)
result = {'status': False, 'msg': 'Error: {0}'.format(e)}
self.response.write(json.dumps(result))
return
elif reqType == 'newJobLocal':
logging.error("*" * 80)
logging.error("parametersweep.newJobLocal")
logging.error("*" * 80)
data = json.loads(self.request.get('data'))
self.user_data.set_selected(0)
job = db.GqlQuery(
"SELECT * FROM ParameterSweepJobWrapper WHERE user_id = :1 AND name = :2",
self.user.user_id(), data["jobName"].strip()).get()
if job != None:
logging.error(
"parametersweep.newJobLocal: error: Job name must be unique"
)
self.response.write(
json.dumps({
"status": False,
"msg": "Job name must be unique"
}))
return
try:
result = self.runLocal(data=data)
return self.response.write(
json.dumps({
"status": True,
"msg": "Job launched",
"id": result.key().id()
}))
except Exception as e:
logging.exception(e)
result = {'status': False, 'msg': 'Error: {0}'.format(e)}
self.response.write(json.dumps(result))
return
elif reqType == 'newJobQsub':
logging.error("*" * 80)
logging.error("parametersweep.newJobQsub")
logging.error("*" * 80)
data = json.loads(self.request.get('data'))
# cluster_node_info = self.user_data.get_cluster_node_info()[0]
# files = fileserver.FileManager.getFiles(self, 'clusterKeyFiles')
# cluster_ssh_key_info = {f['id']: {'id': f['id'], 'keyname': f['path']} for f in files}
cluster_info = dict()
received_cluster_info = json.loads(
self.request.get('cluster_info'))
cluster_info['ip_address'] = received_cluster_info['ip']
cluster_info['username'] = received_cluster_info['username']
cluster_info['ssh_key'] = fileserver.FileWrapper.get_by_id(
received_cluster_info['key_file_id']).storePath
self.user_data.set_selected(received_cluster_info['uuid'])
#logging.info("PARAMETER_SWEEP_CLUSTER_INFO = {0}".format(cluster_info))
#cluster_info = json.loads(self.request.get('cluster_info'))
job = db.GqlQuery(
"SELECT * FROM ParameterSweepJobWrapper WHERE user_id = :1 AND name = :2",
self.user.user_id(), data["jobName"].strip()).get()
if job != None:
logging.error(
"parametersweep.newJobQsub: error: Job name must be unique"
)
self.response.write(
json.dumps({
"status": False,
"msg": "Job name must be unique"
}))
return
try:
result = self.runQsub(data=data, cluster_info=cluster_info)
return self.response.write(
json.dumps({
"status": True,
"msg": "Job launched",
"id": result.key().id()
}))
except Exception as e:
logging.exception(e)
result = {'status': False, 'msg': 'Error: {0}'.format(e)}
self.response.write(json.dumps(result))
return
elif reqType == 'delJob':
jobID = json.loads(self.request.get('id'))
jobID = int(jobID)
job = ParameterSweepJobWrapper.get_by_id(jobID)
if job.user_id == self.user.user_id():
job.delete(self)
else:
self.response.write(
json.dumps({
"status":
False,
"msg":
"No permissions to delete this job (this should never happen)"
}))
return
elif reqType == 'getDataCloud':
try:
jobID = json.loads(self.request.get('id'))
job = ParameterSweepJobWrapper.get_by_id(int(jobID))
molnsConfigDb = db.GqlQuery(
"SELECT * FROM MolnsConfigWrapper WHERE user_id = :1",
self.user.user_id()).get()
if not molnsConfigDb:
return
molnsConfig = molns.MOLNSConfig(
config_dir=molnsConfigDb.folder)
try:
log = molns.MOLNSExec.job_logs([job.molnsPID], molnsConfig)
with open(os.path.join(job.outData, 'stdout'), 'w') as f:
f.write(log['msg'])
molns.MOLNSExec.fetch_job_results([
job.molnsPID, "results",
os.path.join(job.outData, 'results')
], molnsConfig)
job.output_stored = True
except (IOError, molns.MOLNSException) as e:
logging.info('Could not fetch results: {0}'.format(e))
# Save the updated status
job.put()
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
'status': True,
'msg': 'Job downloaded'
}))
return
except Exception as e:
traceback.print_exc()
self.response.write(
json.dumps({
"status": False,
"msg": "Error: {0}".format(e)
}))
return
elif reqType == 'getDataLocal':
jobID = json.loads(self.request.get('id'))
jobID = int(jobID)
job = ParameterSweepJobWrapper.get_by_id(jobID)
if not job.zipFileName:
szip = exportimport.SuperZip(os.path.abspath(
os.path.dirname(__file__) + '/../static/tmp/'),
preferredName=job.name + "_")
job.zipFileName = szip.getFileName()
szip.addParameterSweepJob(job, True)
szip.close()
# Save the updated status
job.put()
relpath = '/' + os.path.relpath(
job.zipFileName,
os.path.abspath(os.path.dirname(__file__) + '/../'))
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
'status': True,
'msg': 'Job prepared',
'url': relpath
}))
return
self.response.write(json.dumps({'status': True, 'msg': 'Success'}))
def runMolns(self, data):
self.user_data.set_selected(2)
modelDb = StochKitModelWrapper.get_by_id(data["modelID"])
path = os.path.abspath(os.path.dirname(__file__))
basedir = path + '/../'
dataDir = tempfile.mkdtemp(dir=basedir + 'output')
job = ParameterSweepJobWrapper()
job.user_id = self.user.user_id()
job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S")
job.name = data["jobName"]
job.inData = json.dumps(data)
job.modelName = modelDb.name
job.outData = dataDir
job.status = "Pending"
job.output_stored = False
# # execute cloud task
try:
template_filename = 'parametersweep_template_{0}.py'.format(
data['modelType'])
logging.error(
"parametersweep.runMolns() template_filename={0}".format(
template_filename))
logging.error("*" * 80)
with open(os.path.join(path, template_filename), 'r') as f:
template = f.read()
templateData = {
"name": modelDb.name,
"modelType": modelDb.type,
"species": modelDb.species,
"parameters": modelDb.parameters,
"reactions": modelDb.reactions,
"speciesSelect": data['speciesSelect'],
"maxTime": data['maxTime'],
"increment": data['increment'],
"trajectories": data['trajectories'],
"seed": data['seed'],
"parameterA": data['parameterA'],
"minValueA": data['minValueA'],
"maxValueA": data['maxValueA'],
"stepsA": data['stepsA'],
"logA": data['logA'],
"parameterB": data['parameterB'],
"minValueB": data['minValueB'],
"maxValueB": data['maxValueB'],
"stepsB": data['stepsB'],
"logB": data['logB'],
"variableCount": data['variableCount'],
"isSpatial": modelDb.isSpatial,
"isLocal": False
}
if modelDb.isSpatial:
try:
meshWrapperDb = mesheditor.MeshWrapper.get_by_id(
modelDb.spatial["mesh_wrapper_id"])
except Exception as e:
raise Exception(
"No Mesh file set. Choose one in the Mesh tab of the Model Editor"
)
try:
meshFileObj = fileserver.FileManager.getFile(
self, meshWrapperDb.meshFileId, noFile=False)
templateData["mesh"] = meshFileObj["data"]
except IOError as e:
raise Exception("Mesh file inaccessible. Try another mesh")
templateData[
'reaction_subdomain_assignments'] = modelDb.spatial[
"reactions_subdomain_assignments"]
templateData[
'species_subdomain_assignments'] = modelDb.spatial[
"species_subdomain_assignments"]
templateData[
'species_diffusion_coefficients'] = modelDb.spatial[
"species_diffusion_coefficients"]
templateData['initial_conditions'] = modelDb.spatial[
"initial_conditions"]
templateData['subdomains'] = meshWrapperDb.subdomains
program = os.path.join(dataDir, 'program.py')
with open(program, 'w') as f:
jsonString = json.dumps(templateData, indent=4, sort_keys=True)
# We've got to double escape the strings here cause of how we're substituting the JSON data in a source file
jsonString = jsonString.replace('\\', '\\\\')
f.write(template.replace('___JSON_STRING___', jsonString))
molnsConfigDb = db.GqlQuery(
"SELECT * FROM MolnsConfigWrapper WHERE user_id = :1",
self.user.user_id()).get()
if not molnsConfigDb:
raise Exception("Molns not initialized")
config = molns.MOLNSConfig(config_dir=molnsConfigDb.folder)
result = molns.MOLNSExec.start_job(
['EC2_controller', "python {0}".format(program)], config)
job.resource = "molns"
job.molnsPID = result['id']
job.put()
except Exception as e:
job.status = 'Failed'
job.delete(self)
raise
return job
def runQsub(self, data, cluster_info):
logging.error("*" * 80)
logging.error("parametersweep.runQsub() modelType={0}".format(
data['modelType']))
logging.error("*" * 80)
modelDb = StochKitModelWrapper.get_by_id(data["modelID"])
path = os.path.abspath(os.path.dirname(__file__))
basedir = path + '/../'
dataDir = tempfile.mkdtemp(dir=basedir + 'output')
job = ParameterSweepJobWrapper()
job.user_id = self.user.user_id()
job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S")
job.name = data["jobName"]
job.inData = json.dumps(data)
job.modelName = modelDb.name
job.outData = dataDir
job.status = "Pending"
job.output_stored = False
try:
templateData = {
"name": modelDb.name,
"modelType": modelDb.type,
"species": modelDb.species,
"parameters": modelDb.parameters,
"reactions": modelDb.reactions,
"speciesSelect": data['speciesSelect'],
"maxTime": data['maxTime'],
"increment": data['increment'],
"trajectories": data['trajectories'],
"seed": data['seed'],
"parameterA": data['parameterA'],
"minValueA": data['minValueA'],
"maxValueA": data['maxValueA'],
"stepsA": data['stepsA'],
"logA": data['logA'],
"parameterB": data['parameterB'],
"minValueB": data['minValueB'],
"maxValueB": data['maxValueB'],
"stepsB": data['stepsB'],
"logB": data['logB'],
"variableCount": data['variableCount'],
"isSpatial": modelDb.isSpatial,
"isLocal": True
}
if modelDb.isSpatial:
try:
meshWrapperDb = mesheditor.MeshWrapper.get_by_id(
modelDb.spatial["mesh_wrapper_id"])
except Exception as e:
logging.exception(e)
logging.error(
"No Mesh file set. Choose one in the Mesh tab of the Model Editor"
)
raise Exception(
"No Mesh file set. Choose one in the Mesh tab of the Model Editor"
)
try:
meshFileObj = fileserver.FileManager.getFile(
self, meshWrapperDb.meshFileId, noFile=False)
templateData["mesh"] = meshFileObj["data"]
except IOError as e:
logging.exception(e)
logging.error("Mesh file inaccessible. Try another mesh")
raise Exception("Mesh file inaccessible. Try another mesh")
templateData[
'reaction_subdomain_assignments'] = modelDb.spatial[
"reactions_subdomain_assignments"]
templateData[
'species_subdomain_assignments'] = modelDb.spatial[
"species_subdomain_assignments"]
templateData[
'species_diffusion_coefficients'] = modelDb.spatial[
"species_diffusion_coefficients"]
templateData['initial_conditions'] = modelDb.spatial[
"initial_conditions"]
templateData['subdomains'] = meshWrapperDb.subdomains
if data['modelType'] == "stochastic":
job.qsubHandle = pickle.dumps(
parametersweep_qsub.stochastic(templateData, cluster_info))
elif data['modelType'] == "deterministic":
job.qsubHandle = pickle.dumps(
parametersweep_qsub.deterministic(templateData,
cluster_info))
elif data['modelType'] == "spatial":
job.qsubHandle = pickle.dumps(
parametersweep_qsub.spatial(templateData, cluster_info))
else:
raise Exception(
"Trying to runQsub on unsupported modelType {0}".format(
data['modelType']))
job.resource = "qsub"
job.put()
except Exception as e:
logging.exception(e)
job.status = 'Failed'
job.delete(self)
raise
return job
def post(self):
reqType = self.request.get('reqType')
self.response.content_type = 'application/json'
if reqType == 'newJob':
data = json.loads(self.request.get('data'))
job = db.GqlQuery("SELECT * FROM ParameterSweepJobWrapper WHERE user_id = :1 AND name = :2",
self.user.user_id(),
data["jobName"].strip()).get()
if job != None:
self.response.write(json.dumps({"status" : False,
"msg" : "Job name must be unique"}))
return
try:
result = self.runMolns(data = data)
return self.response.write(json.dumps({
"status": True,
"msg": "Job launched",
"id": result.key().id()
}))
except Exception as e:
logging.exception(e)
result = {'status':False,
'msg':'Error: {0}'.format(e)}
self.response.write(json.dumps(result))
return
elif reqType == 'newJobLocal':
logging.error("*"*80)
logging.error("parametersweep.newJobLocal")
logging.error("*"*80)
data = json.loads(self.request.get('data'))
job = db.GqlQuery("SELECT * FROM ParameterSweepJobWrapper WHERE user_id = :1 AND name = :2",
self.user.user_id(),
data["jobName"].strip()).get()
if job != None:
logging.error("parametersweep.newJobLocal: error: Job name must be unique")
self.response.write(json.dumps({"status" : False,
"msg" : "Job name must be unique"}))
return
try:
result = self.runLocal(data = data)
return self.response.write(json.dumps({
"status": True,
"msg": "Job launched",
"id": result.key().id()
}))
except Exception as e:
logging.exception(e)
result = {'status':False,
'msg':'Error: {0}'.format(e)}
self.response.write(json.dumps(result))
return
elif reqType == 'delJob':
jobID = json.loads(self.request.get('id'))
jobID = int(jobID)
job = ParameterSweepJobWrapper.get_by_id(jobID)
if job.user_id == self.user.user_id():
job.delete(self)
else:
self.response.write(json.dumps({"status" : False,
"msg" : "No permissions to delete this job (this should never happen)"}))
return
elif reqType == 'getDataCloud':
try:
jobID = json.loads(self.request.get('id'))
job = ParameterSweepJobWrapper.get_by_id(int(jobID))
molnsConfigDb = db.GqlQuery("SELECT * FROM MolnsConfigWrapper WHERE user_id = :1", self.user.user_id()).get()
if not molnsConfigDb:
return
molnsConfig = molns.MOLNSConfig(config_dir = molnsConfigDb.folder)
try:
log = molns.MOLNSExec.job_logs([job.molnsPID], molnsConfig)
with open(os.path.join(job.outData, 'stdout'), 'w') as f:
f.write(log['msg'])
molns.MOLNSExec.fetch_job_results([job.molnsPID, "results", os.path.join(job.outData, 'results')], molnsConfig)
job.output_stored = True
except (IOError, molns.MOLNSException) as e:
logging.info('Could not fetch results: {0}'.format(e))
# Save the updated status
job.put()
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ 'status' : True,
'msg' : 'Job downloaded'}))
return
except Exception as e:
traceback.print_exc()
self.response.write(json.dumps({"status" : False,
"msg" : "Error: {0}".format(e)}))
return
elif reqType == 'getDataLocal':
jobID = json.loads(self.request.get('id'))
jobID = int(jobID)
job = ParameterSweepJobWrapper.get_by_id(jobID)
if not job.zipFileName:
szip = exportimport.SuperZip(os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/'), preferredName = job.name + "_")
job.zipFileName = szip.getFileName()
szip.addParameterSweepJob(job, True)
szip.close()
# Save the updated status
job.put()
relpath = '/' + os.path.relpath(job.zipFileName, os.path.abspath(os.path.dirname(__file__) + '/../'))
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ 'status' : True,
'msg' : 'Job prepared',
'url' : relpath }))
return
self.response.write(json.dumps({ 'status' : True,
'msg' : 'Success'}))
def runMolns(self, data):
modelDb = StochKitModelWrapper.get_by_id(data["modelID"])
path = os.path.abspath(os.path.dirname(__file__))
basedir = path + '/../'
dataDir = tempfile.mkdtemp(dir = basedir + 'output')
job = ParameterSweepJobWrapper()
job.user_id = self.user.user_id()
job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S")
job.name = data["jobName"]
job.inData = json.dumps(data)
job.modelName = modelDb.name
job.outData = dataDir
job.status = "Pending"
job.output_stored = False
# # execute cloud task
try:
with open(os.path.join(path, 'parametersweep_template.py'), 'r') as f:
template = f.read()
templateData = {
"name" : modelDb.name,
"modelType" : modelDb.type,
"species" : modelDb.species,
"parameters" : modelDb.parameters,
"reactions" : modelDb.reactions,
"speciesSelect" : data['speciesSelect'],
"maxTime" : data['maxTime'],
"increment" : data['increment'],
"trajectories" : data['trajectories'],
"seed" : data['seed'],
"parameterA" : data['parameterA'],
"minValueA" : data['minValueA'],
"maxValueA" : data['maxValueA'],
"stepsA" : data['stepsA'],
"logA" : data['logA'],
"parameterB" : data['parameterB'],
"minValueB" : data['minValueB'],
"maxValueB" : data['maxValueB'],
"stepsB" : data['stepsB'],
"logB" : data['logB'],
"variableCount" : data['variableCount'],
"isSpatial" : modelDb.isSpatial,
"isLocal" : False
}
if modelDb.isSpatial:
try:
meshWrapperDb = mesheditor.MeshWrapper.get_by_id(modelDb.spatial["mesh_wrapper_id"])
except Exception as e:
raise Exception("No Mesh file set. Choose one in the Mesh tab of the Model Editor")
try:
meshFileObj = fileserver.FileManager.getFile(self, meshWrapperDb.meshFileId, noFile = False)
templateData["mesh"] = meshFileObj["data"]
except IOError as e:
raise Exception("Mesh file inaccessible. Try another mesh")
templateData['reaction_subdomain_assignments'] = modelDb.spatial["reactions_subdomain_assignments"]
templateData['species_subdomain_assignments'] = modelDb.spatial["species_subdomain_assignments"]
templateData['species_diffusion_coefficients'] = modelDb.spatial["species_diffusion_coefficients"]
templateData['initial_conditions'] = modelDb.spatial["initial_conditions"]
templateData['subdomains'] = meshWrapperDb.subdomains
program = os.path.join(dataDir, 'program.py')
with open(program, 'w') as f:
jsonString = json.dumps(templateData, indent = 4, sort_keys = True)
# We've got to double escape the strings here cause of how we're substituting the JSON data in a source file
jsonString = jsonString.replace('\\', '\\\\')
f.write(template.replace('___JSON_STRING___', jsonString))
molnsConfigDb = db.GqlQuery("SELECT * FROM MolnsConfigWrapper WHERE user_id = :1", self.user.user_id()).get()
if not molnsConfigDb:
raise Exception("Molns not initialized")
config = molns.MOLNSConfig(config_dir=molnsConfigDb.folder)
result = molns.MOLNSExec.start_job(['EC2_controller', "python {0}".format(program)], config)
job.resource = "molns"
job.molnsPID = result['id']
job.put()
except Exception as e:
job.status='Failed'
job.delete(self)
raise
return job
def runQsub(self, data, cluster_info):
logging.error("*"*80)
logging.error("parametersweep.runQsub() modelType={0}".format(data['modelType']))
logging.error("*"*80)
modelDb = StochKitModelWrapper.get_by_id(data["modelID"])
path = os.path.abspath(os.path.dirname(__file__))
basedir = path + '/../'
dataDir = tempfile.mkdtemp(dir = basedir + 'output')
job = ParameterSweepJobWrapper()
job.user_id = self.user.user_id()
job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S")
job.name = data["jobName"]
job.inData = json.dumps(data)
job.modelName = modelDb.name
job.outData = dataDir
job.status = "Pending"
job.output_stored = False
try:
templateData = {
"name" : modelDb.name,
"modelType" : modelDb.type,
"species" : modelDb.species,
"parameters" : modelDb.parameters,
"reactions" : modelDb.reactions,
"speciesSelect" : data['speciesSelect'],
"maxTime" : data['maxTime'],
"increment" : data['increment'],
"trajectories" : data['trajectories'],
"seed" : data['seed'],
"parameterA" : data['parameterA'],
"minValueA" : data['minValueA'],
"maxValueA" : data['maxValueA'],
"stepsA" : data['stepsA'],
"logA" : data['logA'],
"parameterB" : data['parameterB'],
"minValueB" : data['minValueB'],
"maxValueB" : data['maxValueB'],
"stepsB" : data['stepsB'],
"logB" : data['logB'],
"variableCount" : data['variableCount'],
"isSpatial" : modelDb.isSpatial,
"isLocal" : True
}
if modelDb.isSpatial:
try:
meshWrapperDb = mesheditor.MeshWrapper.get_by_id(modelDb.spatial["mesh_wrapper_id"])
except Exception as e:
logging.exception(e)
logging.error("No Mesh file set. Choose one in the Mesh tab of the Model Editor")
raise Exception("No Mesh file set. Choose one in the Mesh tab of the Model Editor")
try:
meshFileObj = fileserver.FileManager.getFile(self, meshWrapperDb.meshFileId, noFile = False)
templateData["mesh"] = meshFileObj["data"]
except IOError as e:
logging.exception(e)
logging.error("Mesh file inaccessible. Try another mesh")
raise Exception("Mesh file inaccessible. Try another mesh")
templateData['reaction_subdomain_assignments'] = modelDb.spatial["reactions_subdomain_assignments"]
templateData['species_subdomain_assignments'] = modelDb.spatial["species_subdomain_assignments"]
templateData['species_diffusion_coefficients'] = modelDb.spatial["species_diffusion_coefficients"]
templateData['initial_conditions'] = modelDb.spatial["initial_conditions"]
templateData['subdomains'] = meshWrapperDb.subdomains
if data['modelType'] == "stochastic":
job.qsubHandle = pickle.dumps(parametersweep_qsub.stochastic(templateData, cluster_info))
elif data['modelType'] == "deterministic":
job.qsubHandle = pickle.dumps(parametersweep_qsub.deterministic(templateData, cluster_info))
elif data['modelType'] == "spatial":
job.qsubHandle = pickle.dumps(parametersweep_qsub.spatial(templateData, cluster_info))
else:
raise Exception("Trying to runQsub on unsupported modelType {0}".format(data['modelType']))
job.resource = "qsub"
job.put()
except Exception as e:
logging.exception(e)
job.status='Failed'
job.delete(self)
raise
return job
def runLocal(self, data):
self.user_data.set_selected(0)
logging.error("*"*80)
logging.error("parametersweep.runLocal() modelType={0}".format(data['modelType']))
logging.error("*"*80)
modelDb = StochKitModelWrapper.get_by_id(data["modelID"])
path = os.path.abspath(os.path.dirname(__file__))
basedir = path + '/../'
dataDir = tempfile.mkdtemp(dir = basedir + 'output')
job = ParameterSweepJobWrapper()
job.user_id = self.user.user_id()
job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S")
job.name = data["jobName"]
job.inData = json.dumps(data)
job.modelName = modelDb.name
job.outData = dataDir
job.status = "Pending"
job.output_stored = False
# # execute local task
try:
template_filename = 'parametersweep_template_{0}.py'.format(data['modelType'])
logging.error("parametersweep.runLocal() template_filename={0}".format(template_filename))
logging.error("*"*80)
with open(os.path.join(path,template_filename ), 'r') as f:
template = f.read()
templateData = {
"name" : modelDb.name,
"modelType" : modelDb.type,
"species" : modelDb.species,
"parameters" : modelDb.parameters,
"reactions" : modelDb.reactions,
"speciesSelect" : data['speciesSelect'],
"maxTime" : data['maxTime'],
"increment" : data['increment'],
"trajectories" : data['trajectories'],
"seed" : data['seed'],
"parameterA" : data['parameterA'],
"minValueA" : data['minValueA'],
"maxValueA" : data['maxValueA'],
"stepsA" : data['stepsA'],
"logA" : data['logA'],
"parameterB" : data['parameterB'],
"minValueB" : data['minValueB'],
"maxValueB" : data['maxValueB'],
"stepsB" : data['stepsB'],
"logB" : data['logB'],
"variableCount" : data['variableCount'],
"isSpatial" : modelDb.isSpatial,
"isLocal" : True
}
if modelDb.isSpatial:
try:
meshWrapperDb = mesheditor.MeshWrapper.get_by_id(modelDb.spatial["mesh_wrapper_id"])
except Exception as e:
logging.exception(e)
logging.error("No Mesh file set. Choose one in the Mesh tab of the Model Editor")
raise Exception("No Mesh file set. Choose one in the Mesh tab of the Model Editor")
try:
meshFileObj = fileserver.FileManager.getFile(self, meshWrapperDb.meshFileId, noFile = False)
templateData["mesh"] = meshFileObj["data"]
except IOError as e:
logging.exception(e)
logging.error("Mesh file inaccessible. Try another mesh")
raise Exception("Mesh file inaccessible. Try another mesh")
templateData['reaction_subdomain_assignments'] = modelDb.spatial["reactions_subdomain_assignments"]
templateData['species_subdomain_assignments'] = modelDb.spatial["species_subdomain_assignments"]
templateData['species_diffusion_coefficients'] = modelDb.spatial["species_diffusion_coefficients"]
templateData['initial_conditions'] = modelDb.spatial["initial_conditions"]
templateData['subdomains'] = meshWrapperDb.subdomains
program = os.path.join(dataDir, 'stochss_parametersweep_program.py')
with open(program, 'w') as f:
jsonString = json.dumps(templateData, indent = 4, sort_keys = True)
# We've got to double escape the strings here cause of how we're substituting the JSON data in a source file
jsonString = jsonString.replace('\\', '\\\\')
f.write(template.replace('___JSON_STRING___', jsonString))
#? molnsConfigDb = db.GqlQuery("SELECT * FROM MolnsConfigWrapper WHERE user_id = :1", self.user.user_id()).get()
#? if not molnsConfigDb:
#? raise Exception("Molns not initialized")
#?
#? config = molns.MOLNSConfig(config_dir=molnsConfigDb.folder)
#? result = molns.MOLNSExec.start_job(['EC2_controller', "python {0}".format(program)], config)
cmd = "python {0}".format(program)
logging.info('parametersweep.runLocal(): cmd={0}'.format(cmd))
logging.info('*'*80)
exstring = '{0}/backend/wrapper.py {1}/stdout {1}/stderr {1}/return_code {2}'.format(basedir, dataDir, cmd)
logging.info('parametersweep.runLocal(): exstring={0}'.format(exstring))
logging.info('*'*80)
handle = subprocess.Popen(exstring.split(), preexec_fn=os.setsid)
job.pid = handle.pid
logging.info("parametersweep.runLocal() job started pid={0}".format(job.pid))
logging.info('*'*80)
job.resource = "local"
job.put()
except Exception as e:
logging.exception(e)
job.status='Failed'
job.delete(self)
raise
return job
