def calc(request, id=None):
"""
Get a list of calculations and report their id, status, job_type,
is_running, description, and a url where more detailed information
can be accessed.
Responses are in JSON.
"""
base_url = _get_base_url(request)
user = utils.get_user_data(request)
calc_data = _get_calcs(request.GET, user['name'], user['is_super'], id=id)
response_data = []
for hc_id, owner, status, job_type, is_running, desc in calc_data:
url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id)
response_data.append(
dict(id=hc_id, owner=owner, status=status, job_type=job_type,
is_running=is_running, description=desc, url=url))
# if id is specified the related dictionary is returned instead the list
if id is not None:
[response_data] = response_data
return HttpResponse(content=json.dumps(response_data),
content_type=JSON)
def run_calc(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
"""
callback_url = request.POST.get('callback_url')
foreign_calc_id = request.POST.get('foreign_calculation_id')
hazard_output_id = request.POST.get('hazard_output_id')
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_output_id or hazard_job_id:
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job.ini")
einfo, exctype, monitor = safely_call(
_prepare_job, (request, hazard_output_id, hazard_job_id, candidates))
if exctype:
tasks.update_calculation(callback_url, status="failed", einfo=einfo)
return HttpResponse(json.dumps(einfo.splitlines()),
content_type=JSON, status=500)
if not einfo:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]), content_type=JSON,
status=500)
temp_dir = os.path.dirname(einfo[0])
user = utils.get_user_data(request)
try:
job, _fut = submit_job(einfo[0], temp_dir, request.POST['database'],
user['name'], callback_url, foreign_calc_id,
hazard_output_id, hazard_job_id)
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = exc.args[0]
if isinstance(exc_msg, bytes):
exc_msg = exc_msg.decode('utf-8') # make it a unicode object
else:
assert isinstance(exc_msg, unicode), exc_msg
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
calc = oqe_models.OqJob.objects.get(pk=job.id)
response_data = vars(calc.get_oqparam())
response_data['job_id'] = job.id
response_data['status'] = calc.status
status = 200
return HttpResponse(content=json.dumps(response_data), content_type=JSON,
status=status)
def calc_results(request, calc_id):
"""
Get a summarized list of calculation results for a given ``calc_id``.
Result is a JSON array of objects containing the following attributes:
* id
* name
* type (hazard_curve, hazard_map, etc.)
* url (the exact url where the full result can be accessed)
"""
user = utils.get_user_data(request)
# If the specified calculation doesn't exist OR is not yet complete,
# throw back a 404.
try:
oqjob = oqe_models.OqJob.objects.get(id=calc_id)
if not user['is_super'] and oqjob.user_name != user['name']:
return HttpResponseNotFound()
if not oqjob.status == 'complete':
return HttpResponseNotFound()
except ObjectDoesNotExist:
return HttpResponseNotFound()
base_url = _get_base_url(request)
# NB: export_output has as keys the list (output_type, extension)
# so this returns an ordered map output_type -> extensions such as
# OrderedDict([('agg_loss_curve', ['xml', 'csv']), ...])
output_types = groupby(export_output, lambda oe: oe[0],
lambda oes: [e for o, e in oes])
results = oq_engine.get_outputs(calc_id)
if not results:
return HttpResponseNotFound()
response_data = []
for result in results:
try: # output from the old calculators
rtype = result.output_type
outtypes = output_types[rtype]
except KeyError:
try: # output from the datastore
rtype = result.ds_key
outtypes = output_types[rtype]
except KeyError:
continue # non-exportable outputs should not be shown
url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
datum = dict(
id=result.id, name=result.display_name, type=rtype,
outtypes=outtypes, url=url)
response_data.append(datum)
return HttpResponse(content=json.dumps(response_data))
def run_calc(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
If the request has the attribute `hazard_job_id`, the results of the
specified hazard calculations will be re-used as input by the risk
calculation.
The request also needs to contain the files needed to perform the
calculation. They can be uploaded as separate files, or zipped
together.
"""
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_job_id:
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
einfo, exctype, monitor = safely_call(
_prepare_job, (request, hazard_job_id, candidates))
if exctype:
return HttpResponse(json.dumps(einfo.splitlines()),
content_type=JSON, status=500)
if not einfo:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]), content_type=JSON,
status=500)
user = utils.get_user_data(request)
try:
job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id)
# restart the process pool at the end of each job
fut .add_done_callback(lambda f: TaskManager.restart())
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = str(exc)
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='created')
status = 200
return HttpResponse(content=json.dumps(response_data), content_type=JSON,
status=status)
def calc_results(request, calc_id):
"""
Get a summarized list of calculation results for a given ``calc_id``.
Result is a JSON array of objects containing the following attributes:
* id
* name
* type (hazard_curve, hazard_map, etc.)
* url (the exact url where the full result can be accessed)
"""
user = utils.get_user_data(request)
# If the specified calculation doesn't exist OR is not yet complete,
# throw back a 404.
try:
info = logs.dbcmd('calc_info', calc_id)
if user['acl_on'] and info['user_name'] != user['name']:
return HttpResponseNotFound()
except models.NotFound:
return HttpResponseNotFound()
base_url = _get_base_url(request)
# NB: export_output has as keys the list (output_type, extension)
# so this returns an ordered map output_type -> extensions such as
# OrderedDict([('agg_loss_curve', ['xml', 'csv']), ...])
output_types = groupby(export, lambda oe: oe[0],
lambda oes: [e for o, e in oes])
results = logs.dbcmd('get_outputs', calc_id)
if not results:
return HttpResponseNotFound()
response_data = []
for result in results:
try: # output from the datastore
rtype = result.ds_key
# Catalina asked to remove the .txt outputs (used for the GMFs)
outtypes = [ot for ot in output_types[rtype] if ot != 'txt']
except KeyError:
continue # non-exportable outputs should not be shown
url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
datum = dict(
id=result.id, name=result.display_name, type=rtype,
outtypes=outtypes, url=url)
response_data.append(datum)
return HttpResponse(content=json.dumps(response_data))
def calc_remove(request, calc_id):
"""
Remove the calculation id
"""
user = utils.get_user_data(request)['name']
try:
message = logs.dbcmd('del_calc', calc_id, user)
except dbapi.NotFound:
return HttpResponseNotFound()
if isinstance(message, list): # list of removed files
return HttpResponse(content=json.dumps(message),
content_type=JSON,
status=200)
else: # FIXME: the error is not passed properly to the javascript
logging.error(message)
return HttpResponse(content=message,
content_type='text/plain',
status=500)
def run_calc(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
If the request has the attribute `hazard_job_id`, the results of the
specified hazard calculations will be re-used as input by the risk
calculation.
The request also needs to contain the files needed to perform the
calculation. They can be uploaded as separate files, or zipped
together.
"""
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_job_id:
hazard_job_id = int(hazard_job_id)
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
einfo, exctype, monitor = safely_call(_prepare_job, (request, candidates))
if exctype:
return HttpResponse(json.dumps(einfo.splitlines()),
content_type=JSON, status=500)
if not einfo:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]), content_type=JSON,
status=500)
user = utils.get_user_data(request)
try:
job_id, _pid = submit_job(einfo[0], user['name'], hazard_job_id)
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = str(exc)
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='created')
status = 200
return HttpResponse(content=json.dumps(response_data), content_type=JSON,
status=status)
def calc_list(request, id=None):
# view associated to the endpoints /v1/calc/list and /v1/calc/:id/status
"""
Get a list of calculations and report their id, status, calculation_mode,
is_running, description, and a url where more detailed information
can be accessed. This is called several times by the Javascript.
Responses are in JSON.
"""
base_url = _get_base_url(request)
user = utils.get_user_data(request)
allowed_users = user['group_members'] or [user['name']]
calc_data = logs.dbcmd('get_calcs', request.GET, allowed_users,
user['acl_on'], id)
response_data = []
for hc_id, owner, status, calculation_mode, is_running, desc, pid \
in calc_data:
url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id)
abortable = False
if is_running:
try:
if (psutil.Process(pid).username() == psutil.Process(
os.getpid()).username()):
abortable = True
except psutil.NoSuchProcess:
pass
response_data.append(
dict(id=hc_id,
owner=owner,
calculation_mode=calculation_mode,
status=status,
is_running=bool(is_running),
description=desc,
url=url,
abortable=abortable))
# if id is specified the related dictionary is returned instead the list
if id is not None:
[response_data] = response_data
return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def run_calc(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
"""
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_job_id:
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
einfo, exctype, monitor = safely_call(
_prepare_job, (request, hazard_job_id, candidates))
if exctype:
return HttpResponse(json.dumps(einfo.splitlines()),
content_type=JSON, status=500)
if not einfo:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]), content_type=JSON,
status=500)
user = utils.get_user_data(request)
try:
job_id, _fut = submit_job(einfo[0], user['name'], hazard_job_id)
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = exc.args[0]
if isinstance(exc_msg, bytes):
exc_msg = exc_msg.decode('utf-8') # make it a unicode object
else:
assert isinstance(exc_msg, unicode), exc_msg
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='executing')
status = 200
return HttpResponse(content=json.dumps(response_data), content_type=JSON,
status=status)
def calc_remove(request, calc_id):
"""
Remove the calculation id
"""
user = utils.get_user_data(request)['name']
try:
message = logs.dbcmd('del_calc', calc_id, user)
except dbapi.NotFound:
return HttpResponseNotFound()
if 'success' in message:
return HttpResponse(content=json.dumps(message),
content_type=JSON, status=200)
elif 'error' in message:
logging.error(message['error'])
return HttpResponse(content=json.dumps(message),
content_type=JSON, status=403)
else:
# This is an untrapped server error
logging.error(message)
return HttpResponse(content=message,
content_type='text/plain', status=500)
def calc_abort(request, calc_id):
"""
Abort the given calculation, it is it running
"""
job = logs.dbcmd('get_job', calc_id)
if job is None:
message = {'error': 'Unknown job %s' % calc_id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if job.status not in ('executing', 'running'):
message = {'error': 'Job %s is not running' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
user = utils.get_user_data(request)
info = logs.dbcmd('calc_info', calc_id)
allowed_users = user['group_members'] or [user['name']]
if user['acl_on'] and info['user_name'] not in allowed_users:
message = {
'error': ('User %s has no permission to abort job %s' %
(info['user_name'], job.id))
}
return HttpResponse(content=json.dumps(message),
content_type=JSON,
status=403)
if job.pid: # is a spawned job
try:
os.kill(job.pid, signal.SIGTERM)
except Exception as exc:
logging.error(exc)
else:
logging.warn('Aborting job %d, pid=%d', job.id, job.pid)
logs.dbcmd('set_status', job.id, 'aborted')
message = {'success': 'Killing job %d' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
message = {'error': 'PID for job %s not found' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
def get_datastore(request, job_id):
"""
Download a full datastore file.
:param request:
`django.http.HttpRequest` object.
:param job_id:
The id of the requested datastore
:returns:
A `django.http.HttpResponse` containing the content
of the requested artifact, if present, else throws a 404
"""
user = utils.get_user_data(request)
username = user['name'] if user['acl_on'] else None
job = logs.dbcmd('get_job', int(job_id), username)
if job is None:
return HttpResponseNotFound()
fname = job.ds_calc_dir + '.hdf5'
response = FileResponse(FileWrapper(open(fname, 'rb')), content_type=HDF5)
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
return response