def getJob(handler, job_id):
job = StochKitJobWrapper.get_by_id(job_id)
indata = json.loads(job.indata)
logging.debug('getJob() job.id = {0} job.indata = {1}'.format(job.key().id(), indata))
jsonJob = { "id" : job.key().id(),
"name" : job.name,
"stdout" : job.stdout,
"stderr" : job.stderr,
# These are things contained in the stochkit_job object
"type" : indata["type"],
"status" : job.status,
"startTime" : job.startTime,
"modelName" : job.modelName,
"output_stored": job.output_stored,
"output_location" : job.outData,
"zipFileName" : job.zipFileName,
"output_url" : job.outputURL,
"final_time" : indata["final_time"],
"increment" : indata["increment"],
"realizations" : indata["realizations"],
"exec_type" : indata["exec_type"],
"units" : indata["units"],
"resource" : job.resource,
"epsilon" : indata["epsilon"],
"threshold" : indata["threshold"],
"cloudDatabaseID" : job.cloudDatabaseID,
"seed" : indata["seed"],
"pid" : job.pid,
"result" : job.result }
return jsonJob
def getJob(handler, job_id):
job = StochKitJobWrapper.get_by_id(job_id)
indata = json.loads(job.indata)
logging.debug('getJob() job.id = {0} job.indata = {1}'.format(
job.key().id(), indata))
jsonJob = {
"id": job.key().id(),
"name": job.name,
"stdout": job.stdout,
"stderr": job.stderr,
# These are things contained in the stochkit_job object
"type": indata["type"],
"status": job.status,
"startTime": job.startTime,
"modelName": job.modelName,
"output_stored": job.output_stored,
"output_location": job.outData,
"zipFileName": job.zipFileName,
"output_url": job.outputURL,
"final_time": indata["final_time"],
"increment": indata["increment"],
"realizations": indata["realizations"],
"exec_type": indata["exec_type"],
"units": indata["units"],
"resource": job.resource,
"epsilon": indata["epsilon"],
"threshold": indata["threshold"],
"cloudDatabaseID": job.cloudDatabaseID,
"seed": indata["seed"],
"pid": job.pid,
"result": job.result
}
return jsonJob
def updateJob(handler, job):
jobWrap = StochKitJobWrapper.get_by_id(job_id)
jobWrap.user_id = handler.user.user_id()
jobWrap.put()
jsonJob = { "id" : jobWrap.key().id() }
return jsonJob
def updateJob(handler, job):
jobWrap = StochKitJobWrapper.get_by_id(job_id)
jobWrap.user_id = handler.user.user_id()
jobWrap.put()
jsonJob = {"id": jobWrap.key().id()}
return jsonJob
def post(self):
""" Assemble the input to StochKit2 and submit the job (locally or via cloud). """
reqType = self.request.get('reqType')
if reqType == 'getFromCloud':
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
service = backendservices(self.user_data)
service.fetchOutput(job)
# Unpack it to its local output location
os.system('tar -xf {0}.tar'.format(job.cloudDatabaseID))
job.outData = os.path.abspath('{0}/../output/{1}'.format(os.path.abspath(os.path.dirname(__file__)), job.cloudDatabaseID))
job.stdout = os.path.join(job.outData, '/stdout.log')
job.stderr = os.path.join(job.outData, '/stderr.log')
# Clean up
os.remove('{0}.tar'.format(job.cloudDatabaseID))
# Save the updated status
job.put()
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ 'status' : True,
'msg' : 'Job downloaded'}))
return
elif reqType == 'getDataLocal':
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
if not job.zipFileName:
szip = exportimport.SuperZip(os.path.abspath(os.path.dirname(__file__) + '/../static/tmp/'), preferredName = job.name + "_")
job.zipFileName = szip.getFileName()
szip.addStochKitJob(job, globalOp = True, ignoreStatus = True)
szip.close()
# Save the updated status
job.put()
relpath = os.path.relpath(job.zipFileName, os.path.abspath(os.path.dirname(__file__) + '/../'))
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ 'status' : True,
'msg' : 'Job downloaded',
'url' : relpath }))
return
elif reqType == 'delJob':
try:
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
if job.user_id == self.user.user_id():
job.delete(self)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ 'status' : True,
'msg' : "Job deleted from the datastore."}))
except Exception as e:
logging.exception(e)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ 'status' : False,
'msg' : "Error: {0}".format(e) }))
return
elif reqType == 'jobInfo':
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
indata = json.loads(job.indata)
if self.user.user_id() != job.user_id:
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(["Not the right user"]))
if job.status == "Finished":
try:
if (job.resource in backendservices.SUPPORTED_CLOUD_RESOURCES and job.output_stored == 'False') or (job.resource in backendservices.SUPPORTED_CLOUD_RESOURCES and job.outData is None):
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ "status" : "Finished",
"values" : [],
"job" : JobManager.getJob(self, job.key().id())}))
return
else:
outputdir = job.outData
# Load all data from file in JSON format
if indata['exec_type'] == 'stochastic':
tid = self.request.get('tid')
if tid != '' and tid != 'mean':
outfile = '/result/trajectories/trajectory{0}.txt'.format(tid)
vhandle = open(outputdir + outfile, 'r')
values = { 'time' : [], 'trajectories' : {} }
columnToList = []
for i, line in enumerate(vhandle):
if i == 0:
names = line.split()
for name in names:
if name == 'time':
columnToList.append(values['time'])
else:
values['trajectories'][name] = [] # start a new timeseries for this name
columnToList.append(values['trajectories'][name]) # Store a reference here for future use
else:
for storage, value in zip(columnToList, map(float, line.split())):
storage.append(value)
vhandle.close()
else:
outfile = '/result/stats/means.txt'
vhandle = open(outputdir + outfile, 'r')
values = { 'time' : [], 'trajectories' : {} }
columnToList = []
for i, line in enumerate(vhandle):
if i == 0:
names = line.split()
for name in names:
if name == 'time':
columnToList.append(values['time'])
else:
values['trajectories'][name] = [] # start a new timeseries for this name
columnToList.append(values['trajectories'][name]) # Store a reference here for future use
else:
for storage, value in zip(columnToList, map(float, line.split())):
storage.append(value)
vhandle.close()
else:
outfile = '/result/output.txt'
values = { 'time' : [], 'trajectories' : {} }
#if not os.path.isfile(outputdir + outfile):
vhandle = open(outputdir + outfile, 'r')
columnToList = []
for i, line in enumerate(vhandle):
if i == 0:
continue
elif i == 1:
names = line.split()
for name in names:
if name == 'time':
columnToList.append(values['time'])
else:
values['trajectories'][name] = [] # start a new timeseries for this name
columnToList.append(values['trajectories'][name]) # Store a reference here for future use
elif i == 2:
continue
elif i == 3:
for storage, value in zip(columnToList, map(float, line.split())):
storage.append(value)
elif i == 4:
continue
else:
for storage, value in zip(columnToList, map(float, line.split())):
storage.append(value)
vhandle.close()
self.response.headers['Content-Type'] = 'application/json'
result = {"status" : "Finished",
"values" : values,
"job" : JobManager.getJob(self, job.key().id())}
logging.debug("result = \n\n{}".format(result))
self.response.write(json.dumps(result))
return
except Exception as e:
traceback.print_exc()
job.status = "Failed"
job.put()
logging.error("Failed to parse output data. Assuming job failed and continuing")
if job.status == "Failed":
self.response.headers['Content-Type'] = 'application/json'
stdout = ""
stderr = ""
if job.outData is not None:
if os.path.isfile(job.outData + '/stdout'):
fstdoutHandle = open(job.outData + '/stdout', 'r')
else:
fstdoutHandle = open(job.outData + '/stdout.log', 'r')
stdout = fstdoutHandle.read()
fstdoutHandle.close()
if os.path.isfile(job.outData + '/stderr'):
fstderrHandle = open(job.outData + '/stderr', 'r')
else:
fstderrHandle = open(job.outData + '/stderr.log', 'r')
stderr = fstderrHandle.read()
fstderrHandle.close()
self.response.write(json.dumps({ "status" : "Failed",
"job" : JobManager.getJob(self, job.key().id()),
"stdout" : stdout,
"stderr" : stderr}))
else:
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps({ "status" : "asdfasfdfdsa" }))
else:
# Params is a dict that constains all response elements of the form
params = json.loads(self.request.get('data'))
self.response.headers['Content-Type'] = 'application/json'
job = db.GqlQuery("SELECT * FROM StochKitJobWrapper WHERE user_id = :1 AND name = :2",
self.user.user_id(), params["jobName"].strip()).get()
if job != None:
self.response.write(json.dumps({"status" : False,
"msg" : "Job name must be unique"}))
return
backend_services = backendservices(self.user_data)
# Create a stochhkit_job instance
try:
if params['resource'] == "local":
job = self.runStochKitLocal(params)
elif params['resource'] == 'cloud':
job = self.runCloud(params)
else:
raise Exception("Unknown resource {0}".format(params["resource"]))
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps( { "status" : True,
"msg" : "Job launched",
"id" : job.key().id() } ))
except Exception as e:
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps( { "status" : False,
"msg" : str(e) } ))
def deleteJob(handler, job_id):
job = StochKitJobWrapper.get_by_id(job_id)
job.delete()
def post(self):
""" Assemble the input to StochKit2 and submit the job (locally or via cloud). """
reqType = self.request.get('reqType')
if reqType == 'getFromCloud':
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
service = backendservices(self.user_data)
service.fetchOutput(job)
# Unpack it to its local output location
os.system('tar -xf {0}.tar'.format(job.cloudDatabaseID))
job.outData = os.path.abspath('{0}/../output/{1}'.format(
os.path.abspath(os.path.dirname(__file__)),
job.cloudDatabaseID))
job.stdout = os.path.join(job.outData, '/stdout.log')
job.stderr = os.path.join(job.outData, '/stderr.log')
# Clean up
os.remove('{0}.tar'.format(job.cloudDatabaseID))
# Save the updated status
job.put()
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
'status': True,
'msg': 'Job downloaded'
}))
return
elif reqType == 'getDataLocal':
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
if not job.zipFileName:
szip = exportimport.SuperZip(os.path.abspath(
os.path.dirname(__file__) + '/../static/tmp/'),
preferredName=job.name + "_")
job.zipFileName = szip.getFileName()
szip.addStochKitJob(job, globalOp=True, ignoreStatus=True)
szip.close()
# Save the updated status
job.put()
relpath = os.path.relpath(
job.zipFileName,
os.path.abspath(os.path.dirname(__file__) + '/../'))
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
'status': True,
'msg': 'Job downloaded',
'url': relpath
}))
return
elif reqType == 'delJob':
try:
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
if job.user_id == self.user.user_id():
job.delete(self)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
'status': True,
'msg': "Job deleted from the datastore."
}))
except Exception as e:
logging.exception(e)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
'status': False,
'msg': "Error: {0}".format(e)
}))
return
elif reqType == 'redirectJupyterNotebook':
try:
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
#Check if notebook already exists, if not create one
notebook_filename = "{0}.ipynb".format(job.name)
local_path = os.path.relpath(
os.path.abspath(job.outData),
os.path.abspath(__file__ + '/../../../'))
notebook_file_path = os.path.abspath(
job.outData) + "/" + notebook_filename
# TODO Deterministic or Stochastic template
indata = json.loads(job.indata)
if indata['exec_type'] == 'deterministic':
notebook_template_path = os.path.abspath(
__file__ + '/../../../jupyter_notebook_templates'
) + "/Deterministic.ipynb"
else:
notebook_template_path = os.path.abspath(
__file__ + '/../../../jupyter_notebook_templates'
) + "/Stochastic.ipynb"
if not os.path.isfile(notebook_file_path):
logging.info("Creating {0} from {1}".format(
notebook_file_path, notebook_template_path))
shutil.copyfile(notebook_template_path, notebook_file_path)
if self.request.get('hostname') is not None:
logging.info('hostname={0}'.format(
self.request.get('hostname')))
host = self.request.get('hostname')
else:
host = 'localhost'
port = 9999
proto = 'http'
#
# return the url of the notebook
notebook_url = '{0}://{1}:{2}/notebooks/{3}/{4}'.format(
proto, host, port, local_path, notebook_filename)
logging.info('redirect: {0}'.format(notebook_url))
self.redirect(notebook_url)
except Exception as e:
logging.error("Error in openJupyterNotebook: {0}".format(e))
self.response.write('Error: {0}'.format(e))
return
elif reqType == 'jobInfo':
job = StochKitJobWrapper.get_by_id(int(self.request.get('id')))
indata = json.loads(job.indata)
if self.user.user_id() != job.user_id:
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(["Not the right user"]))
if job.status == "Finished":
try:
if (job.resource
in backendservices.SUPPORTED_CLOUD_RESOURCES
and job.output_stored == 'False') or (
job.resource
in backendservices.SUPPORTED_CLOUD_RESOURCES
and job.outData is None):
self.response.headers[
'Content-Type'] = 'application/json'
self.response.write(
json.dumps({
"status":
"Finished",
"values": [],
"job":
JobManager.getJob(self,
job.key().id())
}))
return
else:
outputdir = job.outData
# Load all data from file in JSON format
if indata['exec_type'] == 'stochastic':
tid = self.request.get('tid')
if tid != '' and tid != 'mean':
outfile = '/result/trajectories/trajectory{0}.txt'.format(
tid)
vhandle = open(outputdir + outfile, 'r')
values = {'time': [], 'trajectories': {}}
columnToList = []
for i, line in enumerate(vhandle):
if i == 0:
names = line.split()
for name in names:
if name == 'time':
columnToList.append(
values['time'])
else:
values['trajectories'][name] = [
] # start a new timeseries for this name
columnToList.append(
values['trajectories']
[name]
) # Store a reference here for future use
else:
for storage, value in zip(
columnToList,
map(float, line.split())):
storage.append(value)
vhandle.close()
else:
outfile = '/result/stats/means.txt'
vhandle = open(outputdir + outfile, 'r')
values = {'time': [], 'trajectories': {}}
columnToList = []
for i, line in enumerate(vhandle):
if i == 0:
names = line.split()
for name in names:
if name == 'time':
columnToList.append(
values['time'])
else:
values['trajectories'][name] = [
] # start a new timeseries for this name
columnToList.append(
values['trajectories']
[name]
) # Store a reference here for future use
else:
for storage, value in zip(
columnToList,
map(float, line.split())):
storage.append(value)
vhandle.close()
else:
outfile = '/result/output.txt'
values = {'time': [], 'trajectories': {}}
#if not os.path.isfile(outputdir + outfile):
vhandle = open(outputdir + outfile, 'r')
columnToList = []
for i, line in enumerate(vhandle):
if i == 0:
continue
elif i == 1:
names = line.split()
for name in names:
if name == 'time':
columnToList.append(values['time'])
else:
values['trajectories'][name] = [
] # start a new timeseries for this name
columnToList.append(
values['trajectories'][name]
) # Store a reference here for future use
elif i == 2:
continue
elif i == 3:
for storage, value in zip(
columnToList,
map(float, line.split())):
storage.append(value)
elif i == 4:
continue
else:
for storage, value in zip(
columnToList,
map(float, line.split())):
storage.append(value)
vhandle.close()
self.response.headers['Content-Type'] = 'application/json'
result = {
"status": "Finished",
"values": values,
"job": JobManager.getJob(self,
job.key().id())
}
logging.debug("result = \n\n{}".format(result))
self.response.write(json.dumps(result))
return
except Exception as e:
traceback.print_exc()
job.status = "Failed"
job.put()
logging.error(
"Failed to parse output data. Assuming job failed and continuing"
)
if job.status == "Failed":
self.response.headers['Content-Type'] = 'application/json'
stdout = ""
stderr = ""
if job.outData is not None:
if os.path.isfile(job.outData + '/stdout'):
fstdoutHandle = open(job.outData + '/stdout', 'r')
else:
fstdoutHandle = open(job.outData + '/stdout.log', 'r')
stdout = fstdoutHandle.read()
fstdoutHandle.close()
if os.path.isfile(job.outData + '/stderr'):
fstderrHandle = open(job.outData + '/stderr', 'r')
else:
fstderrHandle = open(job.outData + '/stderr.log', 'r')
stderr = fstderrHandle.read()
fstderrHandle.close()
self.response.write(
json.dumps({
"status": "Failed",
"job": JobManager.getJob(self,
job.key().id()),
"stdout": stdout,
"stderr": stderr
}))
else:
raise Exception(
'This page should never be accessed if job is not Finished or Running (current state of job {0} : {1})'
.format(job.id, job.status))
traceback.print_exc()
print job.status
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({"status": "tttttttttasdfasfdfdsa"}))
else:
# Params is a dict that constains all response elements of the form
params = json.loads(self.request.get('data'))
self.response.headers['Content-Type'] = 'application/json'
job = db.GqlQuery(
"SELECT * FROM StochKitJobWrapper WHERE user_id = :1 AND name = :2",
self.user.user_id(), params["jobName"].strip()).get()
if job != None:
self.response.write(
json.dumps({
"status": False,
"msg": "Job name must be unique"
}))
return
backend_services = backendservices(self.user_data)
# Create a stochhkit_job instance
try:
if params['resource'] == "local":
job = self.runStochKitLocal(params)
elif params['resource'] == 'cloud':
job = self.runCloud(params)
elif params['resource'] == 'molns':
job = self.runMolns(params)
else:
raise Exception("Unknown resource {0}".format(
params["resource"]))
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
"status": True,
"msg": "Job launched",
"id": job.key().id()
}))
except Exception as e:
traceback.print_exc()
self.response.headers['Content-Type'] = 'application/json'
self.response.write(
json.dumps({
"status": False,
"msg": str(e)
}))
def deleteJob(handler, job_id):
job = StochKitJobWrapper.get_by_id(job_id)
job.delete()
