def post(self): reqType = self.request.get('reqType') if reqType == "jobInfo": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) jsonJob = { "id": int(self.request.get('id')), "userId" : job.user_id, "jobName" : job.name, "startTime" : job.startTime, "indata" : json.loads(job.indata), "outData" : job.outData, "status" : job.status, "resource" : job.resource, "uuid": job.cloudDatabaseID, "output_stored": job.output_stored, "modelName" : job.modelName } if self.user.user_id() != job.user_id: self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(["Not the right user"])) if job.status == "Finished": if job.resource in backendservices.SUPPORTED_CLOUD_RESOURCES and job.outData is None: # Let the user decide if they want to download it self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ "status" : "Finished", "values" : [], "job" : jsonJob})) return outputdir = job.outData try: # Load all data from file in JSON format vhandle = open(outputdir + '/result/output.txt', 'r') values = { 'time' : [], 'trajectories' : {}, 'sensitivities' : {}, 'parameters' : {}} parameters = [] columnToList = [] for i, line in enumerate(vhandle): if i == 0: continue elif i == 1: names = line.split() parameterNames = [] for name in names: if ':' in name: specie, parameter = name.split(':') if parameter not in parameterNames: parameterNames.append(parameter) for name in names: if name == 'time': columnToList.append(values['time']) elif ':' in name: specie, parameter = name.split(':') if specie not in values['sensitivities']: values['sensitivities'][specie] = {} values['sensitivities'][specie][parameter] = [] # Make a new timeseries for sensitivity columnToList.append(values['sensitivities'][specie][parameter]) # Store a reference here for future use else: values['trajectories'][name] = [] # start a new timeseries for this name columnToList.append(values['trajectories'][name]) # Store a reference here for future use elif i == 2: parameters = map(float, line.split()) elif i == 3: for storage, value in zip(columnToList, map(float, line.split())): storage.append(value) elif i == 4: continue else: for storage, value in zip(columnToList, map(float, line.split())): storage.append(value) vhandle.close() values['parameters'] = dict(zip(parameterNames, parameters)) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ "status" : "Finished", "values" : values, "job" : jsonJob })) return except IOError as ioe: logging.error("caught error {0}".format(ioe)) job.status = "Failed" logging.error("put job.status = Failed") job.put() if job.status == "Failed": self.response.headers['Content-Type'] = 'application/json' stdout = '' stderr = '' try: fstdoutHandle = open(job.outData + '/stdout.log', 'r') stdout = fstdoutHandle.read() fstdoutHandle.close() fstderrHandle = open(job.outData + '/stderr.log', 'r') stderr = fstderrHandle.read() fstderrHandle.close() except IOError as ioe: logging.error("could not open error log files in {0}".format(job.outData)) self.response.write(json.dumps({ "status" : "Failed", "stdout" : stdout, "stderr" : stderr, "job" : jsonJob})) return # else self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ "status" : "asdfasdf" })) elif reqType == "getFromCloud": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) service = backendservices(self.user_data) service.fetchOutput(job) # Unpack it to its local output location os.system('tar -xf' +job.cloudDatabaseID+'.tar') job.outData = os.path.dirname(os.path.abspath(__file__))+'/../output/'+job.cloudDatabaseID job.outData = os.path.abspath(job.outData) # jsonJob["outData"] = job.outData # Clean up os.remove(job.cloudDatabaseID+'.tar') # Update the db entry job.put() self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : True, 'msg' : 'Job downloaded'})) return elif reqType == 'redirectJupyterNotebook': try: job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) #Check if notebook already exists, if not create one notebook_filename = "{0}.ipynb".format(job.name) local_path = os.path.relpath(os.path.abspath(job.outData), os.path.abspath(__file__+'/../../../')) notebook_file_path = os.path.abspath(job.outData) + "/" + notebook_filename notebook_template_path = os.path.abspath(__file__+'/../../../jupyter_notebook_templates')+"/Sensitivity.ipynb" if not os.path.isfile(notebook_file_path): logging.info("Creating {0} from {1}".format(notebook_file_path,notebook_template_path)) shutil.copyfile(notebook_template_path, notebook_file_path) if self.request.get('hostname') is not None: host = self.request.get('hostname') else: host = 'localhost' port = 9999 proto = 'http' # # return the url of the notebook notebook_url = '{0}://{1}:{2}/notebooks/{3}/{4}'.format(proto,host,port,local_path,notebook_filename) self.redirect(notebook_url) except Exception as e: logging.error("Error in openJupyterNotebook: {0}".format(e)) self.response.write('Error: {0}'.format(e)) return elif reqType == "getLocalData": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) if not job.zipFileName: szip = exportimport.SuperZip(os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/'), preferredName = job.name + "_") job.zipFileName = szip.getFileName() szip.addSensitivityJob(job, globalOp = True, ignoreStatus = True) szip.close() # Save the updated status job.put() relpath = os.path.relpath(job.zipFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : True, 'msg' : 'Job downloaded', 'url' : relpath })) return elif reqType == "delJob": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) if self.user.user_id() != job.user_id: self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(["Not the right user"])) return job.delete(self) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ "status" : True, "msg" : "Job deleted"})); elif reqType == "newJob": data = json.loads(self.request.get('data')) job = db.GqlQuery("SELECT * FROM SensitivityJobWrapper WHERE user_id = :1 AND name = :2", self.user.user_id(), data["jobName"].strip()).get() if job != None: self.response.write(json.dumps({"status" : False, "msg" : "Job name must be unique"})) return try: # Either local or cloud if data["resource"] == "local": job = self.runLocal(data) elif data["resource"] == "cloud": job = self.runCloud(data) else: raise Exception("Unknown resource {0}".format(data["resource"])) self.response.write(json.dumps( { "status" : True, "msg" : "Job launched", "id" : job.key().id() })) return except Exception as e: logging.exception(e) self.response.write(json.dumps({ "status" : False, "msg" : "Error: {0}".format(e) })) return else: self.response.write(json.dumps({"status" : False, "msg" : "No data submitted"}))
def runCloud(self, data): self.user_data.set_selected(1) job = SensitivityJobWrapper() service = backendservices(self.user_data) if not service.isOneOrMoreComputeNodesRunning(): raise Exception('No cloud computing resources found. (Have they been started?)') job.user_id = self.user.user_id() model = modeleditor.StochKitModelWrapper.get_by_id(data["id"]) job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.status = "Pending" job.modelName = model.name runtime = float(data["time"]) dt = float(data["increment"]) job.indata = json.dumps(data) parameters = [] for parameter in data['selections']["pc"]: if data['selections']["pc"][parameter]: parameters.append(parameter) stochkitmodel = model.createStochKitModel() # Wow, what a hack if stochkitmodel.units.lower() == 'population': document = stochkitmodel.serialize() stochkitmodel = StochMLDocument.fromString(document).toModel(model.name) for reactionN in stochkitmodel.getAllReactions(): reaction = stochkitmodel.getAllReactions()[reactionN] if reaction.massaction: if len(reaction.reactants) == 1 and reaction.reactants.values()[0] == 2: reaction.marate.setExpression(reaction.marate.expression + ' / 2') params = { "job_type": "sensitivity", "document": str( stochkitmodel.serialize() ), "paramstring": "stochkit_ode.py --sensi --parameters {0} -t {1} -i {2}".format( " ".join(parameters), runtime, int(runtime / dt) ), "bucketname": self.user_data.getBucketName() } # Send the task to the backend cloud_result = service.submit_cloud_task(params) # if not cloud_result["success"]: if not cloud_result["success"]: return None, cloud_result job.cloudDatabaseID = cloud_result["db_id"] job.celeryPID = cloud_result["celery_pid"] job.resource = cloud_result['resource'] job.outData = None job.zipFileName = None job.output_stored = 'True' job.put() return job
def runLocal(self, data): ''' ''' self.user_data.set_selected(0) job = SensitivityJobWrapper() job.resource = "local" job.user_id = self.user.user_id() model = modeleditor.StochKitModelWrapper.get_by_id(data["id"]) job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.modelName = model.name runtime = float(data["time"]) dt = float(data["increment"]) job.indata = json.dumps(data) path = os.path.abspath(os.path.dirname(__file__)) parameters = [] for parameter in data['selections']["pc"]: if data['selections']["pc"][parameter]: parameters.append(parameter) if len(parameters) == 0: raise Exception("At least one parameter must be selected"); basedir = path + '/../' dataDir = tempfile.mkdtemp(dir = basedir + 'output') job.outData = dataDir modelFileName = '{0}/{1}.xml'.format(job.outData, model.name) fmodelHandle = open(modelFileName, 'w') stochkitmodel = model.createStochKitModel() # Wow, what a hack if stochkitmodel.units.lower() == 'population': document = stochkitmodel.serialize() stochkitmodel = StochMLDocument.fromString(document).toModel(model.name) for reactionN in stochkitmodel.getAllReactions(): reaction = stochkitmodel.getAllReactions()[reactionN] if reaction.massaction: if len(reaction.reactants) == 1 and reaction.reactants.values()[0] == 2: reaction.marate.setExpression(reaction.marate.expression + ' / 2') fmodelHandle.write(stochkitmodel.serialize()) fmodelHandle.close() job.status = "Pending" args = "--sensi -m {0} --parameters {1} -t {2} --out-dir {3} -i {4}".format(modelFileName, " ".join(parameters), runtime, dataDir + '/result', int(runtime / dt)) ode = "{0}/../../ode/stochkit_ode.py {1}".format(path, args) exstring = '{0}/backend/wrapper.py {1}/stdout {1}/stderr {1}/return_code {2}'.format(basedir, dataDir, ode) handle = subprocess.Popen(exstring.split(), preexec_fn=os.setsid) job.pid = handle.pid job.put() return job
def post(self): reqType = self.request.get('reqType') if reqType == "jobInfo": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) jsonJob = { "id": int(self.request.get('id')), "userId": job.user_id, "jobName": job.name, "startTime": job.startTime, "indata": json.loads(job.indata), "outData": job.outData, "status": job.status, "resource": job.resource, "uuid": job.cloudDatabaseID, "output_stored": job.output_stored, "modelName": job.modelName } if self.user.user_id() != job.user_id: self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(["Not the right user"])) if job.status == "Finished": if job.resource in backendservices.SUPPORTED_CLOUD_RESOURCES and job.outData is None: # Let the user decide if they want to download it self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ "status": "Finished", "values": [], "job": jsonJob })) return outputdir = job.outData try: # Load all data from file in JSON format vhandle = open(outputdir + '/result/output.txt', 'r') values = { 'time': [], 'trajectories': {}, 'sensitivities': {}, 'parameters': {} } parameters = [] columnToList = [] for i, line in enumerate(vhandle): if i == 0: continue elif i == 1: names = line.split() parameterNames = [] for name in names: if ':' in name: specie, parameter = name.split(':') if parameter not in parameterNames: parameterNames.append(parameter) for name in names: if name == 'time': columnToList.append(values['time']) elif ':' in name: specie, parameter = name.split(':') if specie not in values['sensitivities']: values['sensitivities'][specie] = {} values[ 'sensitivities'][specie][parameter] = [ ] # Make a new timeseries for sensitivity columnToList.append( values['sensitivities'][specie] [parameter] ) # Store a reference here for future use else: values['trajectories'][name] = [ ] # start a new timeseries for this name columnToList.append( values['trajectories'][name] ) # Store a reference here for future use elif i == 2: parameters = map(float, line.split()) elif i == 3: for storage, value in zip(columnToList, map(float, line.split())): storage.append(value) elif i == 4: continue else: for storage, value in zip(columnToList, map(float, line.split())): storage.append(value) vhandle.close() values['parameters'] = dict(zip(parameterNames, parameters)) self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ "status": "Finished", "values": values, "job": jsonJob })) return except IOError as ioe: logging.error("caught error {0}".format(ioe)) job.status = "Failed" logging.error("put job.status = Failed") job.put() if job.status == "Failed": self.response.headers['Content-Type'] = 'application/json' stdout = '' stderr = '' try: fstdoutHandle = open(job.outData + '/stdout.log', 'r') stdout = fstdoutHandle.read() fstdoutHandle.close() fstderrHandle = open(job.outData + '/stderr.log', 'r') stderr = fstderrHandle.read() fstderrHandle.close() except IOError as ioe: logging.error( "could not open error log files in {0}".format( job.outData)) self.response.write( json.dumps({ "status": "Failed", "stdout": stdout, "stderr": stderr, "job": jsonJob })) return # else self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({"status": "asdfasdf"})) elif reqType == "getFromCloud": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) service = backendservices(self.user_data) service.fetchOutput(job) # Unpack it to its local output location os.system('tar -xf' + job.cloudDatabaseID + '.tar') job.outData = os.path.dirname(os.path.abspath( __file__)) + '/../output/' + job.cloudDatabaseID job.outData = os.path.abspath(job.outData) # jsonJob["outData"] = job.outData # Clean up os.remove(job.cloudDatabaseID + '.tar') # Update the db entry job.put() self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': True, 'msg': 'Job downloaded' })) return elif reqType == 'redirectJupyterNotebook': try: job = SensitivityJobWrapper.get_by_id( int(self.request.get('id'))) #Check if notebook already exists, if not create one notebook_filename = "{0}.ipynb".format(job.name) local_path = os.path.relpath( os.path.abspath(job.outData), os.path.abspath(__file__ + '/../../../')) notebook_file_path = os.path.abspath( job.outData) + "/" + notebook_filename notebook_template_path = os.path.abspath( __file__ + '/../../../jupyter_notebook_templates' ) + "/Sensitivity.ipynb" if not os.path.isfile(notebook_file_path): logging.info("Creating {0} from {1}".format( notebook_file_path, notebook_template_path)) shutil.copyfile(notebook_template_path, notebook_file_path) if self.request.get('hostname') is not None: host = self.request.get('hostname') else: host = 'localhost' port = 9999 proto = 'http' # # return the url of the notebook notebook_url = '{0}://{1}:{2}/notebooks/{3}/{4}'.format( proto, host, port, local_path, notebook_filename) self.redirect(notebook_url) except Exception as e: logging.error("Error in openJupyterNotebook: {0}".format(e)) self.response.write('Error: {0}'.format(e)) return elif reqType == "getLocalData": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) if not job.zipFileName: szip = exportimport.SuperZip(os.path.abspath( os.path.dirname(__file__) + '/../static/tmp/'), preferredName=job.name + "_") job.zipFileName = szip.getFileName() szip.addSensitivityJob(job, globalOp=True, ignoreStatus=True) szip.close() # Save the updated status job.put() relpath = os.path.relpath( job.zipFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': True, 'msg': 'Job downloaded', 'url': relpath })) return elif reqType == "delJob": job = SensitivityJobWrapper.get_by_id(int(self.request.get('id'))) if self.user.user_id() != job.user_id: self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(["Not the right user"])) return job.delete(self) self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ "status": True, "msg": "Job deleted" })) elif reqType == "newJob": data = json.loads(self.request.get('data')) job = db.GqlQuery( "SELECT * FROM SensitivityJobWrapper WHERE user_id = :1 AND name = :2", self.user.user_id(), data["jobName"].strip()).get() if job != None: self.response.write( json.dumps({ "status": False, "msg": "Job name must be unique" })) return try: # Either local or cloud if data["resource"] == "local": job = self.runLocal(data) elif data["resource"] == "cloud": job = self.runCloud(data) else: raise Exception("Unknown resource {0}".format( data["resource"])) self.response.write( json.dumps({ "status": True, "msg": "Job launched", "id": job.key().id() })) return except Exception as e: logging.exception(e) self.response.write( json.dumps({ "status": False, "msg": "Error: {0}".format(e) })) return else: self.response.write( json.dumps({ "status": False, "msg": "No data submitted" }))
def runCloud(self, data): self.user_data.set_selected(1) job = SensitivityJobWrapper() service = backendservices(self.user_data) if not service.isOneOrMoreComputeNodesRunning(): raise Exception( 'No cloud computing resources found. (Have they been started?)' ) job.user_id = self.user.user_id() model = modeleditor.StochKitModelWrapper.get_by_id(data["id"]) job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.status = "Pending" job.modelName = model.name runtime = float(data["time"]) dt = float(data["increment"]) job.indata = json.dumps(data) parameters = [] for parameter in data['selections']["pc"]: if data['selections']["pc"][parameter]: parameters.append(parameter) stochkitmodel = model.createStochKitModel() # Wow, what a hack if stochkitmodel.units.lower() == 'population': document = stochkitmodel.serialize() stochkitmodel = StochMLDocument.fromString(document).toModel( model.name) for reactionN in stochkitmodel.getAllReactions(): reaction = stochkitmodel.getAllReactions()[reactionN] if reaction.massaction: if len(reaction.reactants ) == 1 and reaction.reactants.values()[0] == 2: reaction.marate.setExpression( reaction.marate.expression + ' / 2') params = { "job_type": "sensitivity", "document": str(stochkitmodel.serialize()), "paramstring": "stochkit_ode.py --sensi --parameters {0} -t {1} -i {2}".format( " ".join(parameters), runtime, int(runtime / dt)), "bucketname": self.user_data.getBucketName() } # Send the task to the backend cloud_result = service.submit_cloud_task(params) # if not cloud_result["success"]: if not cloud_result["success"]: return None, cloud_result job.cloudDatabaseID = cloud_result["db_id"] job.celeryPID = cloud_result["celery_pid"] job.resource = cloud_result['resource'] job.outData = None job.zipFileName = None job.output_stored = 'True' job.put() return job
def runLocal(self, data): ''' ''' self.user_data.set_selected(0) job = SensitivityJobWrapper() job.resource = "local" job.user_id = self.user.user_id() model = modeleditor.StochKitModelWrapper.get_by_id(data["id"]) job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.modelName = model.name runtime = float(data["time"]) dt = float(data["increment"]) job.indata = json.dumps(data) path = os.path.abspath(os.path.dirname(__file__)) parameters = [] for parameter in data['selections']["pc"]: if data['selections']["pc"][parameter]: parameters.append(parameter) if len(parameters) == 0: raise Exception("At least one parameter must be selected") basedir = path + '/../' dataDir = tempfile.mkdtemp(dir=basedir + 'output') job.outData = dataDir modelFileName = '{0}/{1}.xml'.format(job.outData, model.name) fmodelHandle = open(modelFileName, 'w') stochkitmodel = model.createStochKitModel() # Wow, what a hack if stochkitmodel.units.lower() == 'population': document = stochkitmodel.serialize() stochkitmodel = StochMLDocument.fromString(document).toModel( model.name) for reactionN in stochkitmodel.getAllReactions(): reaction = stochkitmodel.getAllReactions()[reactionN] if reaction.massaction: if len(reaction.reactants ) == 1 and reaction.reactants.values()[0] == 2: reaction.marate.setExpression( reaction.marate.expression + ' / 2') fmodelHandle.write(stochkitmodel.serialize()) fmodelHandle.close() job.status = "Pending" args = "--sensi -m {0} --parameters {1} -t {2} --out-dir {3} -i {4}".format( modelFileName, " ".join(parameters), runtime, dataDir + '/result', int(runtime / dt)) ode = "{0}/../../ode/stochkit_ode.py {1}".format(path, args) exstring = '{0}/backend/wrapper.py {1}/stdout {1}/stderr {1}/return_code {2}'.format( basedir, dataDir, ode) handle = subprocess.Popen(exstring.split(), preexec_fn=os.setsid) job.pid = handle.pid job.put() return job