def calc_datastore(request, job_id):
"""
Download a full datastore file.
:param request:
`django.http.HttpRequest` object.
:param job_id:
The id of the requested datastore
:returns:
A `django.http.HttpResponse` containing the content
of the requested artifact, if present, else throws a 404
"""
job = logs.dbcmd('get_job', int(job_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
fname = job.ds_calc_dir + '.hdf5'
response = FileResponse(
FileWrapper(open(fname, 'rb')), content_type=HDF5)
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def calc_datastore(request, job_id):
"""
Download a full datastore file.
:param request:
`django.http.HttpRequest` object.
:param job_id:
The id of the requested datastore
:returns:
A `django.http.HttpResponse` containing the content
of the requested artifact, if present, else throws a 404
"""
job = logs.dbcmd('get_job', int(job_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
fname = job.ds_calc_dir + '.hdf5'
response = FileResponse(
FileWrapper(open(fname, 'rb')), content_type=HDF5)
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def calc_abort(request, calc_id):
"""
Abort the given calculation, it is it running
"""
job = logs.dbcmd('get_job', calc_id)
if job is None:
message = {'error': 'Unknown job %s' % calc_id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if job.status not in ('submitted', 'executing'):
message = {'error': 'Job %s is not running' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if not utils.user_has_permission(request, job.user_name):
message = {'error': ('User %s has no permission to abort job %s' %
(job.user_name, job.id))}
return HttpResponse(content=json.dumps(message), content_type=JSON,
status=403)
if job.pid: # is a spawned job
try:
os.kill(job.pid, signal.SIGINT)
except Exception as exc:
logging.error(exc)
else:
logging.warning('Aborting job %d, pid=%d', job.id, job.pid)
logs.dbcmd('set_status', job.id, 'aborted')
message = {'success': 'Killing job %d' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
message = {'error': 'PID for job %s not found' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
def calc_abort(request, calc_id):
"""
Abort the given calculation, it is it running
"""
job = logs.dbcmd('get_job', calc_id)
if job is None:
message = {'error': 'Unknown job %s' % calc_id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if job.status not in ('executing', 'running'):
message = {'error': 'Job %s is not running' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if not utils.user_has_permission(request, job.user_name):
message = {'error': ('User %s has no permission to abort job %s' %
(job.user_name, job.id))}
return HttpResponse(content=json.dumps(message), content_type=JSON,
status=403)
if job.pid: # is a spawned job
try:
os.kill(job.pid, signal.SIGTERM)
except Exception as exc:
logging.error(exc)
else:
logging.warn('Aborting job %d, pid=%d', job.id, job.pid)
logs.dbcmd('set_status', job.id, 'aborted')
message = {'success': 'Killing job %d' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
message = {'error': 'PID for job %s not found' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
def calc_oqparam(request, job_id):
"""
Return the calculation parameters as a JSON
"""
job = logs.dbcmd('get_job', int(job_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
oq = ds['oqparam']
return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def calc(request, calc_id):
"""
Get a JSON blob containing all of parameters for the given calculation
(specified by ``calc_id``). Also includes the current job status (
executing, complete, etc.).
"""
try:
info = logs.dbcmd('calc_info', calc_id)
if not utils.user_has_permission(request, info['user_name']):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(info), content_type=JSON)
def calc(request, calc_id):
"""
Get a JSON blob containing all of parameters for the given calculation
(specified by ``calc_id``). Also includes the current job status (
executing, complete, etc.).
"""
try:
info = logs.dbcmd('calc_info', calc_id)
if not utils.user_has_permission(request, info['user_name']):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(info), content_type=JSON)
def calc_oqparam(request, job_id):
"""
Return the calculation parameters as a JSON
"""
job = logs.dbcmd('get_job', int(job_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
oq = ds['oqparam']
return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(prefix=what.replace('/', '-'),
suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
aw = _extract(ds, what + query_string)
a = {}
for key, val in vars(aw).items():
key = str(key) # can be a numpy.bytes_
if key.startswith('_'):
continue
elif isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
elif isinstance(val, dict):
# this is hack: we are losing the values
a[key] = list(val)
else:
a[key] = val
numpy.savez_compressed(fname, **a)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(content='%s: %s\n%s' %
(exc.__class__.__name__, exc, tb),
content_type='text/plain',
status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def calc_results(request, calc_id):
"""
Get a summarized list of calculation results for a given ``calc_id``.
Result is a JSON array of objects containing the following attributes:
* id
* name
* type (hazard_curve, hazard_map, etc.)
* url (the exact url where the full result can be accessed)
"""
# If the specified calculation doesn't exist OR is not yet complete,
# throw back a 404.
try:
info = logs.dbcmd('calc_info', calc_id)
if not utils.user_has_permission(request, info['user_name']):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
base_url = _get_base_url(request)
# NB: export_output has as keys the list (output_type, extension)
# so this returns an ordered map output_type -> extensions such as
# {'agg_loss_curve': ['xml', 'csv'], ...}
output_types = groupby(export, lambda oe: oe[0],
lambda oes: [e for o, e in oes])
results = logs.dbcmd('get_outputs', calc_id)
if not results:
return HttpResponseNotFound()
response_data = []
for result in results:
try: # output from the datastore
rtype = result.ds_key
# Catalina asked to remove the .txt outputs (used for the GMFs)
outtypes = [ot for ot in output_types[rtype] if ot != 'txt']
except KeyError:
continue # non-exportable outputs should not be shown
url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
datum = dict(id=result.id,
name=result.display_name,
type=rtype,
outtypes=outtypes,
url=url,
size_mb=result.size_mb)
response_data.append(datum)
return HttpResponse(content=json.dumps(response_data))
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
aw = _extract(ds, what + query_string)
a = {}
for key, val in vars(aw).items():
key = str(key) # can be a numpy.bytes_
if isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
elif isinstance(val, dict):
# this is hack: we are losing the values
a[key] = list(val)
else:
a[key] = val
numpy.savez_compressed(fname, **a)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(
content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb),
content_type='text/plain', status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def calc_results(request, calc_id):
"""
Get a summarized list of calculation results for a given ``calc_id``.
Result is a JSON array of objects containing the following attributes:
* id
* name
* type (hazard_curve, hazard_map, etc.)
* url (the exact url where the full result can be accessed)
"""
# If the specified calculation doesn't exist OR is not yet complete,
# throw back a 404.
try:
info = logs.dbcmd('calc_info', calc_id)
if not utils.user_has_permission(request, info['user_name']):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
base_url = _get_base_url(request)
# NB: export_output has as keys the list (output_type, extension)
# so this returns an ordered map output_type -> extensions such as
# {'agg_loss_curve': ['xml', 'csv'], ...}
output_types = groupby(export, lambda oe: oe[0],
lambda oes: [e for o, e in oes])
results = logs.dbcmd('get_outputs', calc_id)
if not results:
return HttpResponseNotFound()
response_data = []
for result in results:
try: # output from the datastore
rtype = result.ds_key
# Catalina asked to remove the .txt outputs (used for the GMFs)
outtypes = [ot for ot in output_types[rtype] if ot != 'txt']
except KeyError:
continue # non-exportable outputs should not be shown
url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
datum = dict(
id=result.id, name=result.display_name, type=rtype,
outtypes=outtypes, url=url, size_mb=result.size_mb)
response_data.append(datum)
return HttpResponse(content=json.dumps(response_data))
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If setting.LOCKDOWN is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
# read the data and save them on a temporary .pik file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
obj = _extract(ds, what + query_string)
if inspect.isgenerator(obj):
array, attrs = 0, {k: _array(v) for k, v in obj}
elif hasattr(obj, '__toh5__'):
array, attrs = obj.__toh5__()
else: # assume obj is an array
array, attrs = obj, {}
a = {}
for key, val in attrs.items():
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
else:
a[key] = val
numpy.savez_compressed(fname, array=array, **a)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def hmap_png(request, calc_id, imt_id, poe_id):
"""
Get a PNG image with the relevant mean hazard map, if available
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
try:
from PIL import Image
response = HttpResponse(content_type="image/png")
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
arr = ds['png/hmap_%s_%s' % (imt_id, poe_id)][:]
Image.fromarray(arr).save(response, format='png')
return response
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(
content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb),
content_type='text/plain', status=500)
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
path = request.get_full_path()
n = len(request.path_info)
query_string = unquote_plus(path[n:])
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
obj = _extract(ds, what + query_string)
hdf5.save_npz(obj, fname)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(
content='%s: %s in %s\n%s' %
(exc.__class__.__name__, exc, path, tb),
content_type='text/plain', status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def calc_result(request, result_id):
"""
Download a specific result, by ``result_id``.
The common abstracted functionality for getting hazard or risk results.
:param request:
`django.http.HttpRequest` object. Can contain a `export_type` GET
param (the default is 'xml' if no param is specified).
:param result_id:
The id of the requested artifact.
:returns:
If the requested ``result_id`` is not available in the format
designated by the `export_type`.
Otherwise, return a `django.http.HttpResponse` containing the content
of the requested artifact.
Parameters for the GET request can include an `export_type`, such as 'xml',
'geojson', 'csv', etc.
"""
# If the result for the requested ID doesn't exist, OR
# the job which it is related too is not complete,
# throw back a 404.
try:
job_id, job_status, job_user, datadir, ds_key = logs.dbcmd(
'get_result', result_id)
if not utils.user_has_permission(request, job_user):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
etype = request.GET.get('export_type')
export_type = etype or DEFAULT_EXPORT_TYPE
tmpdir = tempfile.mkdtemp()
try:
exported = core.export_from_db((ds_key, export_type), job_id, datadir,
tmpdir)
except DataStoreExportError as exc:
# TODO: there should be a better error page
return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
content_type='text/plain',
status=500)
if not exported:
# Throw back a 404 if the exact export parameters are not supported
return HttpResponseNotFound(
'Nothing to export for export_type=%s, %s' % (export_type, ds_key))
elif len(exported) > 1:
# Building an archive so that there can be a single file download
archname = ds_key + '-' + export_type + '.zip'
zipfiles(exported, os.path.join(tmpdir, archname))
exported = os.path.join(tmpdir, archname)
else: # single file
exported = exported[0]
content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type,
DEFAULT_CONTENT_TYPE)
fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows
stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir))
response = FileResponse(stream, content_type=content_type)
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(exported))
return response
def calc_result(request, result_id):
"""
Download a specific result, by ``result_id``.
The common abstracted functionality for getting hazard or risk results.
:param request:
`django.http.HttpRequest` object. Can contain a `export_type` GET
param (the default is 'xml' if no param is specified).
:param result_id:
The id of the requested artifact.
:returns:
If the requested ``result_id`` is not available in the format
designated by the `export_type`.
Otherwise, return a `django.http.HttpResponse` containing the content
of the requested artifact.
Parameters for the GET request can include an `export_type`, such as 'xml',
'geojson', 'csv', etc.
"""
# If the result for the requested ID doesn't exist, OR
# the job which it is related too is not complete,
# throw back a 404.
try:
job_id, job_status, job_user, datadir, ds_key = logs.dbcmd(
'get_result', result_id)
if not utils.user_has_permission(request, job_user):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
etype = request.GET.get('export_type')
export_type = etype or DEFAULT_EXPORT_TYPE
tmpdir = tempfile.mkdtemp()
try:
exported = core.export_from_db(
(ds_key, export_type), job_id, datadir, tmpdir)
except DataStoreExportError as exc:
# TODO: there should be a better error page
return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
content_type='text/plain', status=500)
if not exported:
# Throw back a 404 if the exact export parameters are not supported
return HttpResponseNotFound(
'Nothing to export for export_type=%s, %s' % (export_type, ds_key))
elif len(exported) > 1:
# Building an archive so that there can be a single file download
archname = ds_key + '-' + export_type + '.zip'
zipfiles(exported, os.path.join(tmpdir, archname))
exported = os.path.join(tmpdir, archname)
else: # single file
exported = exported[0]
content_type = EXPORT_CONTENT_TYPE_MAP.get(
export_type, DEFAULT_CONTENT_TYPE)
fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows
stream.close = lambda: (
FileWrapper.close(stream), shutil.rmtree(tmpdir))
response = FileResponse(stream, content_type=content_type)
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(exported))
return response
