def runQsub(self, data, cluster_info): logging.error("*" * 80) logging.error("simulate.runQsub() modelType={0}".format( data['execType'])) logging.error("*" * 80) modelDb = StochKitModelWrapper.get_by_id(int(data["id"])) path = os.path.abspath(os.path.dirname(__file__)) basedir = path + '/../' dataDir = tempfile.mkdtemp(dir=basedir + 'output') job = SpatialJobWrapper() job.user_id = self.user.user_id() job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.indata = json.dumps(data) job.modelName = modelDb.name job.outData = dataDir job.status = "Pending" job.output_stored = "False" job.is_spatial = True try: templateData = { "name": modelDb.name, "modelType": modelDb.type, "species": modelDb.species, "parameters": modelDb.parameters, "reactions": modelDb.reactions, # "speciesSelect": data['speciesSelect'], "speciesSelect": data['selections'], # "maxTime": data['maxTime'], "maxTime": data['time'], "increment": data['increment'], # "trajectories": data['trajectories'], "trajectories": data['realizations'], "seed": data['seed'], "isSpatial": modelDb.isSpatial, "isLocal": True } if modelDb.isSpatial: try: meshWrapperDb = mesheditor.MeshWrapper.get_by_id( modelDb.spatial["mesh_wrapper_id"]) except Exception as e: logging.exception(e) logging.error( "No Mesh file set. Choose one in the Mesh tab of the Model Editor" ) raise Exception( "No Mesh file set. Choose one in the Mesh tab of the Model Editor" ) try: meshFileObj = fileserver.FileManager.getFile( self, meshWrapperDb.meshFileId, noFile=False) templateData["mesh"] = meshFileObj["data"] except IOError as e: logging.exception(e) logging.error("Mesh file inaccessible. Try another mesh") raise Exception("Mesh file inaccessible. Try another mesh") templateData[ 'reaction_subdomain_assignments'] = modelDb.spatial[ "reactions_subdomain_assignments"] templateData[ 'species_subdomain_assignments'] = modelDb.spatial[ "species_subdomain_assignments"] templateData[ 'species_diffusion_coefficients'] = modelDb.spatial[ "species_diffusion_coefficients"] templateData['initial_conditions'] = modelDb.spatial[ "initial_conditions"] templateData['subdomains'] = meshWrapperDb.subdomains if data['execType'] == "stochastic": job.qsubHandle = pickle.dumps( parametersweep_qsub.stochastic( templateData, cluster_info, not_full_parameter_sweep=True)) elif data['execType'] == "deterministic": job.qsubHandle = pickle.dumps( parametersweep_qsub.deterministic( templateData, cluster_info, not_full_parameter_sweep=True)) elif data['execType'] == "spatial": job.qsubHandle = pickle.dumps( parametersweep_qsub.spatial( templateData, cluster_info, not_full_parameter_sweep=True)) # else: raise Exception( "Trying to runQsub on unsupported modelType {0}".format( data['modelType'])) job.resource = "qsub" job.put() except Exception as e: logging.exception(e) job.status = 'Failed' #job.delete(self) raise return job
def runLocal(self, data): ''' Run a PyURDME run using local compute recources. ''' self.user_data.set_selected(0) ##### pymodel = self.construct_pyurdme_model(data) ##### simulation_algorithm = data[ 'algorithm'] # Don't trust this! I haven't implemented the algorithm selection for this yet simulation_realizations = data['realizations'] # If the seed is negative, this means choose a seed >= 0 randomly if int(data['seed']) < 0: random.seed() data['seed'] = random.randint(0, 2147483647) simulation_seed = data['seed'] ##### path = os.path.abspath(os.path.dirname(__file__)) basedir = path + '/../' dataDir = tempfile.mkdtemp(dir=basedir + 'output') job = SpatialJobWrapper() job.user_id = self.user.user_id() job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.indata = json.dumps(data) job.outData = dataDir job.modelName = pymodel.name job.resource = "local" job.status = "Running" model_file_pkl = "{0}/model_file.pkl".format(dataDir) result_dir = "{0}/results/".format(dataDir) os.makedirs(result_dir) # searilize the model and write it to a file in the data dir with open(model_file_pkl, 'w') as fd: pickle.dump(pymodel, fd) cmd = "{0}/../../pyurdme/pyurdme_wrapper.py {1} {2} {3} {4} {5}".format( path, model_file_pkl, result_dir, simulation_algorithm, simulation_realizations, simulation_seed) logging.info("cmd =\n{}".format(cmd)) exstring = '{0}/backend/wrapper.py {1}/stdout.log {1}/stderr.log {1}/return_code {2}'.format( basedir, dataDir, cmd) handle = subprocess.Popen(exstring, shell=True, preexec_fn=os.setsid) job.pid = int(handle.pid) job.put() return job
def runCloud(self, data): self.user_data.set_selected(1) service = backendservices(self.user_data) if not service.isOneOrMoreComputeNodesRunning(): raise Exception( 'No cloud computing resources found. (Have they been started?)' ) # If the seed is negative, this means choose a seed >= 0 randomly if int(data['seed']) < 0: random.seed() data['seed'] = random.randint(0, 2147483647) pymodel = self.construct_pyurdme_model(data) #logging.info('DATA: {0}'.format(data)) ##### cloud_params = { "job_type": "spatial", "simulation_algorithm": data['algorithm'], "simulation_realizations": data['realizations'], "simulation_seed": data['seed'], # "bucketname" : self.user_data.getBucketName(), #implys EC2, should be in backendservices "paramstring": '', } logging.debug('cloud_params = {}'.format(pprint.pformat(cloud_params))) cloud_params['document'] = pickle.dumps(pymodel) #logging.debug('PYURDME: {0}'.format(cloud_params['document'])) # Send the task to the backend cloud_result = service.submit_cloud_task(params=cloud_params) if not cloud_result["success"]: e = cloud_result["exception"] raise Exception("Cloud execution failed: {0}".format(e)) celery_task_id = cloud_result["celery_pid"] taskid = cloud_result["db_id"] job = SpatialJobWrapper() job.type = 'PyURDME Ensemble' job.user_id = self.user.user_id() job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.indata = json.dumps(data) job.outData = None # This is where the data should be locally, when we get data from cloud, it must be put here job.modelName = pymodel.name job.resource = cloud_result['resource'] job.cloudDatabaseID = taskid job.celeryPID = celery_task_id job.status = "Running" job.output_stored = "True" job.put() return job
def post(self): reqType = self.request.get('reqType') self.response.content_type = 'application/json' if reqType == 'newJob': data = json.loads(self.request.get('data')) logging.debug('data =\n{}'.format(pprint.pformat(data))) job = db.GqlQuery( "SELECT * FROM SpatialJobWrapper WHERE user_id = :1 AND name = :2", self.user.user_id(), data["jobName"].strip()).get() if job != None: self.response.write( json.dumps({ "status": False, "msg": "Job name must be unique" })) return try: if data["resource"] == "local": result = self.runLocal(data) elif data["resource"] == "cloud": result = self.runCloud(data) elif data["resource"] == "qsub": result = self.runQsubWrapper(data) else: raise Exception("Unknown resource {0}".format( data["resource"])) self.response.write( json.dumps({ "status": True, "msg": "Job launched", "id": result.key().id() })) return except Exception as e: logging.exception(e) result = {'status': False, 'msg': 'Error: {0}'.format(e)} self.response.write(json.dumps(result)) return elif reqType == 'stopJob': jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) try: job.stop(self) except Exception as e: logging.execption(e) self.response.write( json.dumps({ "status": False, "msg": "Error: {0}".format(e) })) return elif reqType == 'delJob': jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) try: job.delete(self) except Exception as e: logging.exception(e) self.response.write( json.dumps({ "status": False, "msg": "Error: {0}".format(e) })) return elif reqType == 'getDataCloud': try: jobID = json.loads(self.request.get('id')) job = SpatialJobWrapper.get_by_id(int(jobID)) service = backendservices(self.user_data) # Fetch service.fetchOutput(job) # Unpack os.system('tar -xf' + job.uuid + '.tar') # Record location job.outData = os.path.abspath( os.path.dirname(__file__)) + '/../output/' + job.uuid # Clean up os.remove(job.uuid + '.tar') # Save the updated status job.put() self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': True, 'msg': 'Job downloaded' })) return except Exception as e: traceback.print_exc() self.response.write( json.dumps({ "status": False, "msg": "Error: {0}".format(e) })) return elif reqType == 'getDataLocal': jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) if not job.zipFileName: szip = exportimport.SuperZip(os.path.abspath( os.path.dirname(__file__) + '/../static/tmp/'), preferredName=job.name + "_") job.zipFileName = szip.getFileName() szip.addSpatialJob(job, True) szip.close() # Save the updated status job.put() relpath = '/' + os.path.relpath( job.zipFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': True, 'msg': 'Job downloaded', 'url': relpath })) return elif reqType == 'openJupyterNotebook' or reqType == 'redirectJupyterNotebook': try: jobID = json.loads(self.request.get('id')) job = SpatialJobWrapper.get_by_id(int(jobID)) #Check if notebook already exists, if not create one notebook_filename = "{0}.ipynb".format(job.name) local_path = os.path.relpath( os.path.abspath(job.outData), os.path.abspath(__file__ + '/../../../')) notebook_file_path = os.path.abspath( job.outData) + "/" + notebook_filename notebook_template_path = os.path.abspath( __file__ + '/../../../jupyter_notebook_templates') + "/Spatial.ipynb" if not os.path.isfile(notebook_file_path): logging.info("Creating {0} from {1}".format( notebook_file_path, notebook_template_path)) shutil.copyfile(notebook_template_path, notebook_file_path) if self.request.get('hostname') is not None: host = self.request.get('hostname') else: host = 'localhost' port = 9999 proto = 'http' # # return the url of the notebook notebook_url = '{0}://{1}:{2}/notebooks/{3}/{4}'.format( proto, host, port, local_path, notebook_filename) if reqType == 'openJupyterNotebook': self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': True, 'msg': 'Notebook ready', 'url': notebook_url })) else: self.redirect(notebook_url) except Exception as e: logging.error("Error in openJupyterNotebook: {0}".format(e)) if reqType == 'openJupyterNotebook': self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': False, 'msg': 'error:{0}'.format(e) })) else: self.response.write('Error: {0}'.format(e)) return elif reqType == 'getVtkLocal': def zipdir(path, ziph, prefix): # ziph is zipfile handle for root, dirs, files in os.walk(path): for file in files: ziph.write( os.path.join(root, file), os.path.join( prefix, os.path.relpath(os.path.join(root, file), path))) jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) if not job.vtkFileName: try: tmpDir = None indata = json.loads(job.indata) tmpDir = tempfile.mkdtemp(dir=os.path.abspath( os.path.dirname(__file__) + '/../static/tmp/')) for trajectory in range(indata["realizations"]): resultFile = open( str(job.outData + '/results/result{0}'.format(trajectory))) result = pickle.load(resultFile) resultFile.close() for specie in result.model.listOfSpecies: result.export_to_vtk( specie, os.path.join( tmpDir, "trajectory_{0}".format(trajectory), "species_{0}".format(specie))) tmpFile = tempfile.NamedTemporaryFile(dir=os.path.abspath( os.path.dirname(__file__) + '/../static/tmp/'), prefix=job.name + "_", suffix='.zip', delete=False) zipf = zipfile.ZipFile(tmpFile, "w") zipdir(tmpDir, zipf, os.path.basename(tmpFile.name)) zipf.close() job.vtkFileName = tmpFile.name tmpFile.close() # Save the updated status job.put() finally: if tmpDir and os.path.exists(tmpDir): logging.info("Getting cleaned up") shutil.rmtree(tmpDir) relpath = '/' + os.path.relpath( job.vtkFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': True, 'msg': 'Job downloaded', 'url': relpath })) return elif reqType == 'getCsvLocal': def zipdir(path, ziph, prefix): # ziph is zipfile handle for root, dirs, files in os.walk(path): for file in files: ziph.write( os.path.join(root, file), os.path.join( prefix, os.path.relpath(os.path.join(root, file), path))) jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) if not job.csvFileName: try: tmpDir = None indata = json.loads(job.indata) tmpDir = tempfile.mkdtemp(dir=os.path.abspath( os.path.dirname(__file__) + '/../static/tmp/')) for trajectory in range(indata["realizations"]): resultFile = open( str(job.outData + '/results/result{0}'.format(trajectory))) result = pickle.load(resultFile) resultFile.close() result.export_to_csv( os.path.join( tmpDir, "trajectory_{0}".format(trajectory)).encode( 'ascii', 'ignore')) tmpFile = tempfile.NamedTemporaryFile(dir=os.path.abspath( os.path.dirname(__file__) + '/../static/tmp/'), prefix=job.name + "_", suffix='.zip', delete=False) zipf = zipfile.ZipFile(tmpFile, "w") zipdir(tmpDir, zipf, os.path.basename(tmpFile.name)) zipf.close() job.csvFileName = tmpFile.name tmpFile.close() # Save the updated status job.put() finally: if tmpDir and os.path.exists(tmpDir): logging.info("Getting cleaned up") shutil.rmtree(tmpDir) relpath = '/' + os.path.relpath( job.csvFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write( json.dumps({ 'status': True, 'msg': 'Job downloaded', 'url': relpath })) return self.response.write( json.dumps({ 'status': False, 'msg': 'Unknown Error processing request: no handler called' }))
def get(self): logging.info('GET self.request.body = {}'.format(self.request.body)) reqType = self.request.get('reqType') if reqType == 'getJobInfo': job = SpatialJobWrapper.get_by_id(int(self.request.get('id'))) if self.user.user_id() != job.user_id: self.response.headers['Content-Type'] = 'application/json' self.response.write({ "status": False, "msg": "Not the right user" }) result = {} stdout = '' stderr = '' complete = '' if job.outData is None: complete = 'yes' else: try: fstdoutHandle = open(str(job.outData + '/stdout.log'), 'r') stdout = fstdoutHandle.read() fstdoutHandle.close() fstderrHandle = open(str(job.outData + '/stderr.log'), 'r') stderr = fstderrHandle.read() fstderrHandle.close() if os.path.exists("{0}/results/complete".format( job.outData)): complete = 'yes' except IOError as e: traceback.print_exc() result['status'] = False result[ 'msg'] = 'Error running the simulation: stdout/stderr outputs missing.' result.update({ "id": int(self.request.get('id')), "jobStatus": job.status, "complete": complete, "resource": job.resource, "modelName": job.modelName, "outData": job.outData, "name": job.name, "uuid": job.cloudDatabaseID, "output_stored": job.output_stored, "stdout": stdout, "stderr": stderr, "indata": json.loads(job.indata) }) logging.debug("result =\n\n{}".format(pprint.pformat(result))) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result)) return elif reqType == 'getMeshData': try: job = SpatialJobWrapper.get_by_id(int(self.request.get('id'))) data = json.loads(self.request.get('data')) logging.debug("data = {}".format(data)) trajectory = data["trajectory"] timeIdx = data["timeIdx"] resultJS = {} #if not job.preprocessed or not os.path.exists(job.preprocessedDir): job.preprocess(trajectory) indir = job.preprocessedDir with open(os.path.join(indir, 'mesh.json'), 'r') as meshfile: mesh = json.load(meshfile) with open(os.path.join(indir, 'voxelTuples.json'), 'r') as voxelTuplesFile: voxelTuples = json.load(voxelTuplesFile) f = os.path.join(indir, 'result{0}'.format(trajectory)) with h5py.File(f, 'r') as dataFile: species = dataFile.keys() self.response.content_type = 'application/json' self.response.write( json.dumps({ "mesh": mesh, "voxelTuples": voxelTuples, "species": species })) except Exception as e: traceback.print_exc() result = {} result['status'] = False result['msg'] = 'Error: error fetching results {0}'.format(e) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result)) return elif reqType == 'getTimeSeriesData': try: job = SpatialJobWrapper.get_by_id(int(self.request.get('id'))) data = json.loads(self.request.get('data')) logging.debug( 'patial.get(onlyColorRange): data={0}'.format(data)) trajectory = data["trajectory"] sTime = data["timeStart"] eTime = data["timeEnd"] #TODO: what is the right value here? if eTime is None: eTime = 0 dataType = "population" if "showPopulation" in data and data[ "showPopulation"] else "concentration" resultJS = {} if job.preprocessed is None or trajectory not in job.preprocessed or not os.path.exists( job.preprocessedDir): job.preprocess(trajectory) f = os.path.join(job.preprocessedDir, 'result{0}'.format(trajectory)) limits = {} logging.debug( 'Spatial.get(onlyColorRange): sTime={0} eTime={0}'.format( sTime, eTime)) with h5py.File(f, 'r') as dataFile: dataTmp = {} colorTmp = {} for specie in dataFile.keys(): data2 = dataFile[specie][dataType][sTime:eTime + 1] dataTmp[specie] = data2 limits[specie] = { 'min': dataFile[specie][dataType].attrs['min'], 'max': dataFile[specie][dataType].attrs['max'] } cm.set_clim(dataFile[specie][dataType].attrs['min'], dataFile[specie][dataType].attrs['max']) rgbas = cm.to_rgba(data2, bytes=True).astype('uint32') rgbas = numpy.left_shift( rgbas[:, :, 0], 16) + numpy.left_shift( rgbas[:, :, 1], 8) + rgbas[:, :, 2] #rgbaInts = numpy.zeros((rgbas.shape[0], rgbas.shape[1])) #for i in range(rgbas.shape[0]): # for j in range(rgbas.shape[1]): # rgbaInts[i, j] = int('0x%02x%02x%02x' % tuple(rgbas[i, j][0:3]), 0) colorTmp[specie] = [] for i in range(rgbas.shape[0]): colorTmp[specie].append( list(rgbas[i].astype('int'))) colors = {} data = {} for i in range(abs(eTime - sTime + 1)): colors[sTime + i] = {} data[sTime + i] = {} for specie in dataFile.keys(): colors[sTime + i][specie] = colorTmp[specie][i] data[sTime + i][specie] = list(dataTmp[specie][i]) self.response.content_type = 'application/json' self.response.write( json.dumps({ "colors": colors, "raw": data, "limits": limits })) except Exception as e: traceback.print_exc() result = {} result['status'] = False result['msg'] = 'Error: error fetching results {0}'.format(e) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result)) return self.render_response('spatial.html')
def runCloud(self, data): service = backendservices(self.user_data) if not service.isOneOrMoreComputeNodesRunning(): raise Exception('No cloud computing resources found') # If the seed is negative, this means choose a seed >= 0 randomly if int(data['seed']) < 0: random.seed() data['seed'] = random.randint(0, 2147483647) pymodel = self.construct_pyurdme_model(data) #logging.info('DATA: {0}'.format(data)) ##### cloud_params = { "job_type": "spatial", "simulation_algorithm" : data['algorithm'], "simulation_realizations" : data['realizations'], "simulation_seed" : data['seed'], # "bucketname" : self.user_data.getBucketName(), #implys EC2, should be in backendservices "paramstring" : '', } logging.debug('cloud_params = {}'.format(pprint.pformat(cloud_params))) cloud_params['document'] = pickle.dumps(pymodel) #logging.debug('PYURDME: {0}'.format(cloud_params['document'])) # Send the task to the backend cloud_result = service.submit_cloud_task(params=cloud_params) if not cloud_result["success"]: e = cloud_result["exception"] raise Exception("Cloud execution failed: {0}".format(e)) celery_task_id = cloud_result["celery_pid"] taskid = cloud_result["db_id"] job = SpatialJobWrapper() job.type = 'PyURDME Ensemble' job.user_id = self.user.user_id() job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.indata = json.dumps(data) job.outData = None # This is where the data should be locally, when we get data from cloud, it must be put here job.modelName = pymodel.name job.resource = cloud_result['resource'] job.cloudDatabaseID = taskid job.celeryPID = celery_task_id job.status = "Running" job.output_stored = "True" job.put() return job
def get(self): logging.info('GET self.request.body = {}'.format(self.request.body)) reqType = self.request.get('reqType') if reqType == 'getJobInfo': job = SpatialJobWrapper.get_by_id(int(self.request.get('id'))) if self.user.user_id() != job.user_id: self.response.headers['Content-Type'] = 'application/json' self.response.write({ "status" : False, "msg" : "Not the right user" }) result = {} stdout = '' stderr = '' complete = '' if job.outData is None: complete = 'yes' else: try: fstdoutHandle = open(str(job.outData + '/stdout.log'), 'r') stdout = fstdoutHandle.read() fstdoutHandle.close() fstderrHandle = open(str(job.outData + '/stderr.log'), 'r') stderr = fstderrHandle.read() fstderrHandle.close() if os.path.exists("{0}/results/complete".format(job.outData)): complete = 'yes' except IOError as e: traceback.print_exc() result['status'] = False result['msg'] = 'Error running the simulation: stdout/stderr outputs missing.' result.update({"id" : int(self.request.get('id')), "jobStatus" : job.status, "complete" : complete, "resource" : job.resource, "modelName" : job.modelName, "outData" : job.outData, "name" : job.name, "uuid": job.cloudDatabaseID, "output_stored": job.output_stored, "stdout" : stdout, "stderr" : stderr, "indata" : json.loads(job.indata) }) logging.debug("result =\n\n{}".format(pprint.pformat(result))) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result)) return elif reqType == 'getMeshData': try: job = SpatialJobWrapper.get_by_id(int(self.request.get('id'))) data = json.loads(self.request.get('data')) logging.debug("data = {}".format(data)) trajectory = data["trajectory"] timeIdx = data["timeIdx"] resultJS = {} #if not job.preprocessed or not os.path.exists(job.preprocessedDir): job.preprocess(trajectory) indir = job.preprocessedDir with open(os.path.join(indir, 'mesh.json') ,'r') as meshfile: mesh = json.load(meshfile) with open(os.path.join(indir, 'voxelTuples.json') ,'r') as voxelTuplesFile: voxelTuples = json.load(voxelTuplesFile) f = os.path.join(indir, 'result{0}'.format(trajectory)) with h5py.File(f, 'r') as dataFile: species = dataFile.keys() self.response.content_type = 'application/json' self.response.write(json.dumps({ "mesh" : mesh, "voxelTuples" : voxelTuples, "species" : species })) except Exception as e: traceback.print_exc() result = {} result['status'] = False result['msg'] = 'Error: error fetching results {0}'.format(e) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result)) return elif reqType == 'getTimeSeriesData': try: job = SpatialJobWrapper.get_by_id(int(self.request.get('id'))) data = json.loads(self.request.get('data')) logging.debug('patial.get(onlyColorRange): data={0}'.format(data)) trajectory = data["trajectory"] sTime= data["timeStart"] eTime = data["timeEnd"] #TODO: what is the right value here? if eTime is None: eTime = 0 dataType = "population" if "showPopulation" in data and data["showPopulation"] else "concentration" resultJS = {} if job.preprocessed is None or trajectory not in job.preprocessed or not os.path.exists(job.preprocessedDir): job.preprocess(trajectory) f = os.path.join(job.preprocessedDir, 'result{0}'.format(trajectory)) limits = {} logging.debug('Spatial.get(onlyColorRange): sTime={0} eTime={0}'.format(sTime,eTime)) with h5py.File(f, 'r') as dataFile: dataTmp = {} colorTmp = {} for specie in dataFile.keys(): data2 = dataFile[specie][dataType][sTime:eTime + 1] dataTmp[specie] = data2 limits[specie] = { 'min' : dataFile[specie][dataType].attrs['min'], 'max' : dataFile[specie][dataType].attrs['max'] } cm.set_clim(dataFile[specie][dataType].attrs['min'], dataFile[specie][dataType].attrs['max']) rgbas = cm.to_rgba(data2, bytes = True).astype('uint32') rgbas = numpy.left_shift(rgbas[:, :, 0], 16) + numpy.left_shift(rgbas[:, :, 1], 8) + rgbas[:, :, 2] #rgbaInts = numpy.zeros((rgbas.shape[0], rgbas.shape[1])) #for i in range(rgbas.shape[0]): # for j in range(rgbas.shape[1]): # rgbaInts[i, j] = int('0x%02x%02x%02x' % tuple(rgbas[i, j][0:3]), 0) colorTmp[specie] = [] for i in range(rgbas.shape[0]): colorTmp[specie].append(list(rgbas[i].astype('int'))) colors = {} data = {} for i in range(abs(eTime - sTime + 1)): colors[sTime + i] = {} data[sTime + i] = {} for specie in dataFile.keys(): colors[sTime + i][specie] = colorTmp[specie][i] data[sTime + i][specie] = list(dataTmp[specie][i]) self.response.content_type = 'application/json' self.response.write(json.dumps( { "colors" : colors, "raw" : data, "limits" : limits } )) except Exception as e: traceback.print_exc() result = {} result['status'] = False result['msg'] = 'Error: error fetching results {0}'.format(e) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result)) return self.render_response('spatial.html')
def runLocal(self, data): ''' Run a PyURDME run using local compute recources. ''' ##### pymodel = self.construct_pyurdme_model(data) ##### simulation_algorithm = data['algorithm'] # Don't trust this! I haven't implemented the algorithm selection for this yet simulation_realizations = data['realizations'] # If the seed is negative, this means choose a seed >= 0 randomly if int(data['seed']) < 0: random.seed() data['seed'] = random.randint(0, 2147483647) simulation_seed = data['seed'] ##### path = os.path.abspath(os.path.dirname(__file__)) basedir = path + '/../' dataDir = tempfile.mkdtemp(dir = basedir + 'output') job = SpatialJobWrapper() job.user_id = self.user.user_id() job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.indata = json.dumps(data) job.outData = dataDir job.modelName = pymodel.name job.resource = "local" job.status = "Running" model_file_pkl = "{0}/model_file.pkl".format(dataDir) result_dir = "{0}/results/".format(dataDir) os.makedirs(result_dir) # searilize the model and write it to a file in the data dir with open(model_file_pkl, 'w') as fd: pickle.dump(pymodel, fd) cmd = "{0}/../../pyurdme/pyurdme_wrapper.py {1} {2} {3} {4} {5}".format(path, model_file_pkl, result_dir, simulation_algorithm, simulation_realizations, simulation_seed) logging.info("cmd =\n{}".format(cmd)) exstring = '{0}/backend/wrapper.py {1}/stdout.log {1}/stderr.log {1}/return_code {2}'.format(basedir, dataDir, cmd) handle = subprocess.Popen(exstring, shell=True, preexec_fn=os.setsid) job.pid = int(handle.pid) job.put() return job
def post(self): reqType = self.request.get('reqType') self.response.content_type = 'application/json' logging.error('spaital post reqType={0}'.format(reqType)) if reqType == 'newJob': data = json.loads(self.request.get('data')) logging.debug('data =\n{}'.format(pprint.pformat(data))) job = db.GqlQuery("SELECT * FROM SpatialJobWrapper WHERE user_id = :1 AND name = :2", self.user.user_id(), data["jobName"].strip()).get() if job != None: self.response.write(json.dumps({"status" : False, "msg" : "Job name must be unique"})) return try: if data["resource"] == "local": result = self.runLocal(data) elif data["resource"] == "cloud": result = self.runCloud(data) else: raise Exception("Unknown resource {0}".format(data["resource"])) self.response.write(json.dumps({"status" : True, "msg" : "Job launched", "id" : result.key().id()})) return except Exception as e: logging.exception(e) result = {'status':False, 'msg':'Error: {0}'.format(e)} self.response.write(json.dumps(result)) return elif reqType == 'stopJob': jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) try: job.stop(self) except Exception as e: logging.execption(e) self.response.write(json.dumps({"status" : False, "msg" : "Error: {0}".format(e)})) return elif reqType == 'delJob': jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) try: job.delete(self) except Exception as e: logging.exception(e) self.response.write(json.dumps({"status" : False, "msg" : "Error: {0}".format(e)})) return elif reqType == 'getDataCloud': try: jobID = json.loads(self.request.get('id')) job = SpatialJobWrapper.get_by_id(int(jobID)) service = backendservices(self.user_data) # Fetch service.fetchOutput(job) # Unpack os.system('tar -xf' +job.uuid+'.tar') # Record location job.outData = os.path.abspath(os.path.dirname(__file__))+'/../output/'+job.uuid # Clean up os.remove(job.uuid+'.tar') # Save the updated status job.put() self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : True, 'msg' : 'Job downloaded'})) return except Exception as e: traceback.print_exc() self.response.write(json.dumps({"status" : False, "msg" : "Error: {0}".format(e)})) return elif reqType == 'getDataLocal': jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) if not job.zipFileName: szip = exportimport.SuperZip(os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/'), preferredName = job.name + "_") job.zipFileName = szip.getFileName() szip.addSpatialJob(job, True) szip.close() # Save the updated status job.put() relpath = '/' + os.path.relpath(job.zipFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : True, 'msg' : 'Job downloaded', 'url' : relpath })) return elif reqType == 'openJupyterNotebook' or reqType == 'redirectJupyterNotebook': try: jobID = json.loads(self.request.get('id')) job = SpatialJobWrapper.get_by_id(int(jobID)) #Check if notebook already exists, if not create one notebook_filename = "{0}.ipynb".format(job.name) local_path = os.path.relpath(os.path.abspath(job.outData), os.path.abspath(__file__+'/../../../')) notebook_file_path = os.path.abspath(job.outData) + "/" + notebook_filename notebook_template_path = os.path.abspath(__file__+'/../../../jupyter_notebook_templates')+"/Spatial.ipynb" if not os.path.isfile(notebook_file_path): logging.info("Creating {0} from {1}".format(notebook_file_path,notebook_template_path)) shutil.copyfile(notebook_template_path, notebook_file_path) if self.request.get('hostname') is not None: host = self.request.get('hostname') else: host = 'localhost' port = 9999 proto = 'http' # # return the url of the notebook notebook_url = '{0}://{1}:{2}/notebooks/{3}/{4}'.format(proto,host,port,local_path,notebook_filename) if reqType == 'openJupyterNotebook': self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : True, 'msg' : 'Notebook ready', 'url' : notebook_url })) else: self.redirect(notebook_url) except Exception as e: logging.error("Error in openJupyterNotebook: {0}".format(e)) if reqType == 'openJupyterNotebook': self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : False, 'msg' : 'error:{0}'.format(e) })) else: self.response.write('Error: {0}'.format(e)) return elif reqType == 'getVtkLocal': def zipdir(path, ziph, prefix): # ziph is zipfile handle for root, dirs, files in os.walk(path): for file in files: ziph.write(os.path.join(root, file), os.path.join(prefix, os.path.relpath(os.path.join(root, file), path))) jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) if not job.vtkFileName: try: tmpDir = None indata = json.loads(job.indata) tmpDir = tempfile.mkdtemp(dir = os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/')) for trajectory in range(indata["realizations"]): resultFile = open(str(job.outData + '/results/result{0}'.format(trajectory))) result = pickle.load(resultFile) resultFile.close() for specie in result.model.listOfSpecies: result.export_to_vtk(specie, os.path.join(tmpDir, "trajectory_{0}".format(trajectory), "species_{0}".format(specie))) tmpFile = tempfile.NamedTemporaryFile(dir = os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/'), prefix = job.name + "_", suffix = '.zip', delete = False) zipf = zipfile.ZipFile(tmpFile, "w") zipdir(tmpDir, zipf, os.path.basename(tmpFile.name)) zipf.close() job.vtkFileName = tmpFile.name tmpFile.close() # Save the updated status job.put() finally: if tmpDir and os.path.exists(tmpDir): logging.info("Getting cleaned up") shutil.rmtree(tmpDir) relpath = '/' + os.path.relpath(job.vtkFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : True, 'msg' : 'Job downloaded', 'url' : relpath })) return elif reqType == 'getCsvLocal': def zipdir(path, ziph, prefix): # ziph is zipfile handle for root, dirs, files in os.walk(path): for file in files: ziph.write(os.path.join(root, file), os.path.join(prefix, os.path.relpath(os.path.join(root, file), path))) jobID = json.loads(self.request.get('id')) jobID = int(jobID) job = SpatialJobWrapper.get_by_id(jobID) if not job.csvFileName: try: tmpDir = None indata = json.loads(job.indata) tmpDir = tempfile.mkdtemp(dir=os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/')) for trajectory in range(indata["realizations"]): resultFile = open(str(job.outData + '/results/result{0}'.format(trajectory))) result = pickle.load(resultFile) resultFile.close() result.export_to_csv(os.path.join(tmpDir, "trajectory_{0}".format(trajectory)).encode('ascii', 'ignore')) tmpFile = tempfile.NamedTemporaryFile(dir = os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/'), prefix = job.name + "_", suffix = '.zip', delete = False) zipf = zipfile.ZipFile(tmpFile, "w") zipdir(tmpDir, zipf, os.path.basename(tmpFile.name)) zipf.close() job.csvFileName = tmpFile.name tmpFile.close() # Save the updated status job.put() finally: if tmpDir and os.path.exists(tmpDir): logging.info("Getting cleaned up") shutil.rmtree(tmpDir) relpath = '/' + os.path.relpath(job.csvFileName, os.path.abspath(os.path.dirname(__file__) + '/../')) self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'status' : True, 'msg' : 'Job downloaded', 'url' : relpath })) return self.response.write(json.dumps({ 'status' : False, 'msg' : 'Unknown Error processing request: no handler called'}))
def runQsub(self, data, cluster_info): logging.error("*" * 80) logging.error("simulate.runQsub() modelType={0}".format(data['execType'])) logging.error("*" * 80) modelDb = StochKitModelWrapper.get_by_id(int(data["id"])) path = os.path.abspath(os.path.dirname(__file__)) basedir = path + '/../' dataDir = tempfile.mkdtemp(dir=basedir + 'output') job = SpatialJobWrapper() job.user_id = self.user.user_id() job.startTime = time.strftime("%Y-%m-%d-%H-%M-%S") job.name = data["jobName"] job.indata = json.dumps(data) job.modelName = modelDb.name job.outData = dataDir job.status = "Pending" job.output_stored = "False" job.is_spatial = True try: templateData = { "name": modelDb.name, "modelType": modelDb.type, "species": modelDb.species, "parameters": modelDb.parameters, "reactions": modelDb.reactions, # "speciesSelect": data['speciesSelect'], "speciesSelect": data['selections'], # "maxTime": data['maxTime'], "maxTime": data['time'], "increment": data['increment'], # "trajectories": data['trajectories'], "trajectories": data['realizations'], "seed": data['seed'], "isSpatial": modelDb.isSpatial, "isLocal": True } if modelDb.isSpatial: try: meshWrapperDb = mesheditor.MeshWrapper.get_by_id(modelDb.spatial["mesh_wrapper_id"]) except Exception as e: logging.exception(e) logging.error("No Mesh file set. Choose one in the Mesh tab of the Model Editor") raise Exception("No Mesh file set. Choose one in the Mesh tab of the Model Editor") try: meshFileObj = fileserver.FileManager.getFile(self, meshWrapperDb.meshFileId, noFile=False) templateData["mesh"] = meshFileObj["data"] except IOError as e: logging.exception(e) logging.error("Mesh file inaccessible. Try another mesh") raise Exception("Mesh file inaccessible. Try another mesh") templateData['reaction_subdomain_assignments'] = modelDb.spatial["reactions_subdomain_assignments"] templateData['species_subdomain_assignments'] = modelDb.spatial["species_subdomain_assignments"] templateData['species_diffusion_coefficients'] = modelDb.spatial["species_diffusion_coefficients"] templateData['initial_conditions'] = modelDb.spatial["initial_conditions"] templateData['subdomains'] = meshWrapperDb.subdomains if data['execType'] == "stochastic": job.qsubHandle = pickle.dumps(parametersweep_qsub.stochastic(templateData, cluster_info, not_full_parameter_sweep=True)) elif data['execType'] == "deterministic": job.qsubHandle = pickle.dumps(parametersweep_qsub.deterministic(templateData, cluster_info, not_full_parameter_sweep=True)) elif data['execType'] == "spatial": job.qsubHandle = pickle.dumps(parametersweep_qsub.spatial(templateData, cluster_info, not_full_parameter_sweep=True))# else: raise Exception("Trying to runQsub on unsupported modelType {0}".format(data['modelType'])) job.resource = "qsub" job.put() except Exception as e: logging.exception(e) job.status = 'Failed' #job.delete(self) raise return job