def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If setting.LOCKDOWN is true
only calculations owned by the current user can be retrieved.
"""
user = utils.get_user_data(request)
username = user['name'] if user['acl_on'] else None
job = logs.dbcmd('get_job', int(calc_id), username)
if job is None:
return HttpResponseNotFound()
# read the data and save them on a temporary .pik file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(prefix=what.replace('/', '-'),
suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = request.get_full_path()[n:]
obj = _extract(ds, what + query_string)
if inspect.isgenerator(obj):
array, attrs = 0, {k: _array(v) for k, v in obj}
elif hasattr(obj, '__toh5__'):
array, attrs = obj.__toh5__()
else: # assume obj is an array
array, attrs = obj, {}
numpy.savez_compressed(fname, array=array, **attrs)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
return response
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(prefix=what.replace('/', '-'),
suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
aw = _extract(ds, what + query_string)
a = {}
for key, val in vars(aw).items():
key = str(key) # can be a numpy.bytes_
if key.startswith('_'):
continue
elif isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
elif isinstance(val, dict):
# this is hack: we are losing the values
a[key] = list(val)
else:
a[key] = val
numpy.savez_compressed(fname, **a)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(content='%s: %s\n%s' %
(exc.__class__.__name__, exc, tb),
content_type='text/plain',
status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
aw = _extract(ds, what + query_string)
a = {}
for key, val in vars(aw).items():
key = str(key) # can be a numpy.bytes_
if isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
elif isinstance(val, dict):
# this is hack: we are losing the values
a[key] = list(val)
else:
a[key] = val
numpy.savez_compressed(fname, **a)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(
content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb),
content_type='text/plain', status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If setting.LOCKDOWN is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
# read the data and save them on a temporary .pik file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
obj = _extract(ds, what + query_string)
if inspect.isgenerator(obj):
array, attrs = 0, {k: _array(v) for k, v in obj}
elif hasattr(obj, '__toh5__'):
array, attrs = obj.__toh5__()
else: # assume obj is an array
array, attrs = obj, {}
a = {}
for key, val in attrs.items():
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
else:
a[key] = val
numpy.savez_compressed(fname, array=array, **a)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
path = request.get_full_path()
n = len(request.path_info)
query_string = unquote_plus(path[n:])
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
obj = _extract(ds, what + query_string)
hdf5.save_npz(obj, fname)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(
content='%s: %s in %s\n%s' %
(exc.__class__.__name__, exc, path, tb),
content_type='text/plain', status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def calc_result(request, result_id):
"""
Download a specific result, by ``result_id``.
The common abstracted functionality for getting hazard or risk results.
:param request:
`django.http.HttpRequest` object. Can contain a `export_type` GET
param (the default is 'xml' if no param is specified).
:param result_id:
The id of the requested artifact.
:returns:
If the requested ``result_id`` is not available in the format
designated by the `export_type`.
Otherwise, return a `django.http.HttpResponse` containing the content
of the requested artifact.
Parameters for the GET request can include an `export_type`, such as 'xml',
'geojson', 'csv', etc.
"""
# If the result for the requested ID doesn't exist, OR
# the job which it is related too is not complete,
# throw back a 404.
try:
job_id, job_status, job_user, datadir, ds_key = logs.dbcmd(
'get_result', result_id)
if not utils.user_has_permission(request, job_user):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
etype = request.GET.get('export_type')
export_type = etype or DEFAULT_EXPORT_TYPE
tmpdir = tempfile.mkdtemp()
try:
exported = core.export_from_db((ds_key, export_type), job_id, datadir,
tmpdir)
except DataStoreExportError as exc:
# TODO: there should be a better error page
return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
content_type='text/plain',
status=500)
if not exported:
# Throw back a 404 if the exact export parameters are not supported
return HttpResponseNotFound(
'Nothing to export for export_type=%s, %s' % (export_type, ds_key))
elif len(exported) > 1:
# Building an archive so that there can be a single file download
archname = ds_key + '-' + export_type + '.zip'
zipfiles(exported, os.path.join(tmpdir, archname))
exported = os.path.join(tmpdir, archname)
else: # single file
exported = exported[0]
content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type,
DEFAULT_CONTENT_TYPE)
fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows
stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir))
response = FileResponse(stream, content_type=content_type)
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(exported))
return response
def get_result(request, result_id):
"""
Download a specific result, by ``result_id``.
The common abstracted functionality for getting hazard or risk results.
:param request:
`django.http.HttpRequest` object. Can contain a `export_type` GET
param (the default is 'xml' if no param is specified).
:param result_id:
The id of the requested artifact.
:returns:
If the requested ``result_id`` is not available in the format
designated by the `export_type`.
Otherwise, return a `django.http.HttpResponse` containing the content
of the requested artifact.
Parameters for the GET request can include an `export_type`, such as 'xml',
'geojson', 'csv', etc.
"""
# If the result for the requested ID doesn't exist, OR
# the job which it is related too is not complete,
# throw back a 404.
try:
job_id, job_status, datadir, ds_key = logs.dbcmd(
'get_result', result_id)
except dbapi.NotFound:
return HttpResponseNotFound()
etype = request.GET.get('export_type')
export_type = etype or DEFAULT_EXPORT_TYPE
tmpdir = tempfile.mkdtemp()
try:
exported = core.export_from_db((ds_key, export_type), job_id, datadir,
tmpdir)
except DataStoreExportError as exc:
# TODO: there should be a better error page
return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
content_type='text/plain',
status=500)
if exported is None:
# Throw back a 404 if the exact export parameters are not supported
return HttpResponseNotFound('export_type=%s is not supported for %s' %
(export_type, ds_key))
content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type,
DEFAULT_CONTENT_TYPE)
bname = os.path.basename(exported)
if bname.startswith('.'):
# the "." is added by `export_from_db`, strip it
bname = bname[1:]
fname = 'output-%s-%s' % (result_id, bname)
# 'b' is needed when running the WebUI on Windows
stream = FileWrapper(open(exported, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir))
response = FileResponse(stream, content_type=content_type)
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
return response
def calc_result(request, result_id):
"""
Download a specific result, by ``result_id``.
The common abstracted functionality for getting hazard or risk results.
:param request:
`django.http.HttpRequest` object. Can contain a `export_type` GET
param (the default is 'xml' if no param is specified).
:param result_id:
The id of the requested artifact.
:returns:
If the requested ``result_id`` is not available in the format
designated by the `export_type`.
Otherwise, return a `django.http.HttpResponse` containing the content
of the requested artifact.
Parameters for the GET request can include an `export_type`, such as 'xml',
'geojson', 'csv', etc.
"""
# If the result for the requested ID doesn't exist, OR
# the job which it is related too is not complete,
# throw back a 404.
try:
job_id, job_status, job_user, datadir, ds_key = logs.dbcmd(
'get_result', result_id)
if not utils.user_has_permission(request, job_user):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
etype = request.GET.get('export_type')
export_type = etype or DEFAULT_EXPORT_TYPE
tmpdir = tempfile.mkdtemp()
try:
exported = core.export_from_db(
(ds_key, export_type), job_id, datadir, tmpdir)
except DataStoreExportError as exc:
# TODO: there should be a better error page
return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
content_type='text/plain', status=500)
if not exported:
# Throw back a 404 if the exact export parameters are not supported
return HttpResponseNotFound(
'Nothing to export for export_type=%s, %s' % (export_type, ds_key))
elif len(exported) > 1:
# Building an archive so that there can be a single file download
archname = ds_key + '-' + export_type + '.zip'
zipfiles(exported, os.path.join(tmpdir, archname))
exported = os.path.join(tmpdir, archname)
else: # single file
exported = exported[0]
content_type = EXPORT_CONTENT_TYPE_MAP.get(
export_type, DEFAULT_CONTENT_TYPE)
fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows
stream.close = lambda: (
FileWrapper.close(stream), shutil.rmtree(tmpdir))
response = FileResponse(stream, content_type=content_type)
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(exported))
return response
class FileSystemWorker(Worker):
def __init__(self, *args, **kwargs):
"""Builds some instance variables that will last the life of the
thread."""
Worker.__init__(self, *args, **kwargs)
self.root = os.path.abspath(self.app_info['document_root'])
self.display_index = self.app_info['display_index']
def serve_file(self, filepath, headers):
filestat = os.stat(filepath)
self.size = filestat.st_size
modtime = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(filestat.st_mtime))
self.headers.add_header('Last-Modified', modtime)
if headers.get('if_modified_since') == modtime:
# The browser cache is up-to-date, send a 304.
self.status = "304 Not Modified"
self.data = []
return
ct = mimetypes.guess_type(filepath)[0]
self.content_type = ct if ct else 'text/plain'
try:
f = open(filepath, 'rb')
self.headers['Pragma'] = 'cache'
self.headers['Cache-Control'] = 'private'
self.headers['Content-Length'] = str(self.size)
if self.etag:
self.headers.add_header('Etag', self.etag)
if self.expires:
self.headers.add_header('Expires', self.expires)
try:
# Implement 206 partial file support.
start, end = headers['range'].split('-')
start = 0 if not start.isdigit() else int(start)
end = self.size if not end.isdigit() else int(end)
if self.size < end or start < 0:
self.status = "214 Unsatisfiable Range Requested"
self.data = FileWrapper(f, CHUNK_SIZE)
else:
f.seek(start)
self.data = LimitingFileWrapper(f, CHUNK_SIZE, limit=end)
self.status = "206 Partial Content"
except:
self.data = FileWrapper(f, CHUNK_SIZE)
except IOError:
self.status = "403 Forbidden"
def serve_dir(self, pth, rpth):
def rel_path(path):
return os.path.normpath(path[len(self.root):] if path.startswith(self.root) else path)
if not self.display_index:
self.status = '404 File Not Found'
return b('')
else:
self.content_type = 'text/html'
dir_contents = [os.path.join(pth, x) for x in os.listdir(os.path.normpath(pth))]
dir_contents.sort()
dirs = [rel_path(x)+'/' for x in dir_contents if os.path.isdir(x)]
files = [rel_path(x) for x in dir_contents if os.path.isfile(x)]
self.data = [INDEX_HEADER % dict(path='/'+rpth)]
if rpth:
self.data += [INDEX_ROW % dict(name='(parent directory)', cls='dir parent', link='/'.join(rpth[:-1].split('/')[:-1]))]
self.data += [INDEX_ROW % dict(name=os.path.basename(x[:-1]), link=os.path.join(rpth, os.path.basename(x[:-1])).replace('\\', '/'), cls='dir') for x in dirs]
self.data += ['<tr><th>Files</th></tr>']
self.data += [INDEX_ROW % dict(name=os.path.basename(x), link=os.path.join(rpth, os.path.basename(x)).replace('\\', '/'), cls='file') for x in files]
self.data += [INDEX_FOOTER]
self.headers['Content-Length'] = self.size = str(sum([len(x) for x in self.data]))
self.status = '200 OK'
def run_app(self, conn):
self.status = "200 OK"
self.size = 0
self.expires = None
self.etag = None
self.content_type = 'text/plain'
self.content_length = None
if __debug__:
self.err_log.debug('Getting sock_file')
# Build our file-like object
sock_file = conn.makefile('rb',BUF_SIZE)
request = self.read_request_line(sock_file)
if request['method'].upper() not in ('GET', ):
self.status = "501 Not Implemented"
try:
# Get our file path
headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()])
rpath = request.get('path', '').lstrip('/')
filepath = os.path.join(self.root, rpath)
filepath = os.path.abspath(filepath)
if __debug__:
self.err_log.debug('Request for path: %s' % filepath)
self.closeConnection = headers.get('connection', 'close').lower() == 'close'
self.headers = Headers([('Date', formatdate(usegmt=True)),
('Server', HTTP_SERVER_SOFTWARE),
('Connection', headers.get('connection', 'close')),
])
if not filepath.lower().startswith(self.root.lower()):
# File must be within our root directory
self.status = "400 Bad Request"
self.closeConnection = True
elif not os.path.exists(filepath):
self.status = "404 File Not Found"
self.closeConnection = True
elif os.path.isdir(filepath):
self.serve_dir(filepath, rpath)
elif os.path.isfile(filepath):
self.serve_file(filepath, headers)
else:
# It exists but it's not a file or a directory????
# What is it then?
self.status = "501 Not Implemented"
self.closeConnection = True
h = self.headers
statcode, statstr = self.status.split(' ', 1)
statcode = int(statcode)
if statcode >= 400:
h.add_header('Content-Type', self.content_type)
self.data = [statstr]
# Build our output headers
header_data = HEADER_RESPONSE % (self.status, str(h))
# Send the headers
if __debug__:
self.err_log.debug('Sending Headers: %s' % repr(header_data))
self.conn.sendall(b(header_data))
for data in self.data:
self.conn.sendall(b(data))
if hasattr(self.data, 'close'):
self.data.close()
finally:
if __debug__:
self.err_log.debug('Finally closing sock_file')
sock_file.close()
class FileSystemWorker(Worker):
def __init__(self, *args, **kwargs):
"""Builds some instance variables that will last the life of the
thread."""
Worker.__init__(self, *args, **kwargs)
self.root = os.path.abspath(self.app_info['document_root'])
self.display_index = self.app_info['display_index']
def serve_file(self, filepath, headers):
filestat = os.stat(filepath)
self.size = filestat.st_size
modtime = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(filestat.st_mtime))
self.headers.add_header('Last-Modified', modtime)
if headers.get('if_modified_since') == modtime:
# The browser cache is up-to-date, send a 304.
self.status = "304 Not Modified"
self.data = []
return
ct = mimetypes.guess_type(filepath)[0]
self.content_type = ct if ct else 'text/plain'
try:
f = open(filepath, 'rb')
self.headers['Pragma'] = 'cache'
self.headers['Cache-Control'] = 'private'
self.headers['Content-Length'] = str(self.size)
if self.etag:
self.headers.add_header('Etag', self.etag)
if self.expires:
self.headers.add_header('Expires', self.expires)
try:
# Implement 206 partial file support.
start, end = headers['range'].split('-')
start = 0 if not start.isdigit() else int(start)
end = self.size if not end.isdigit() else int(end)
if self.size < end or start < 0:
self.status = "214 Unsatisfiable Range Requested"
self.data = FileWrapper(f, CHUNK_SIZE)
else:
f.seek(start)
self.data = LimitingFileWrapper(f, CHUNK_SIZE, limit=end)
self.status = "206 Partial Content"
except:
self.data = FileWrapper(f, CHUNK_SIZE)
except IOError:
self.status = "403 Forbidden"
def serve_dir(self, pth, rpth):
def rel_path(path):
return os.path.normpath(
path[len(self.root):] if path.startswith(self.root) else path)
if not self.display_index:
self.status = '404 File Not Found'
return b('')
else:
self.content_type = 'text/html'
dir_contents = [
os.path.join(pth, x) for x in os.listdir(os.path.normpath(pth))
]
dir_contents.sort()
dirs = [
rel_path(x) + '/' for x in dir_contents if os.path.isdir(x)
]
files = [rel_path(x) for x in dir_contents if os.path.isfile(x)]
self.data = [INDEX_HEADER % dict(path='/' + rpth)]
if rpth:
self.data += [
INDEX_ROW % dict(name='(parent directory)',
cls='dir parent',
link='/'.join(rpth[:-1].split('/')[:-1]))
]
self.data += [
INDEX_ROW %
dict(name=os.path.basename(x[:-1]),
link=os.path.join(rpth, os.path.basename(x[:-1])).replace(
'\\', '/'),
cls='dir') for x in dirs
]
self.data += ['<tr><th>Files</th></tr>']
self.data += [
INDEX_ROW %
dict(name=os.path.basename(x),
link=os.path.join(rpth, os.path.basename(x)).replace(
'\\', '/'),
cls='file') for x in files
]
self.data += [INDEX_FOOTER]
self.headers['Content-Length'] = self.size = str(
sum([len(x) for x in self.data]))
self.status = '200 OK'
def run_app(self, conn):
self.status = "200 OK"
self.size = 0
self.expires = None
self.etag = None
self.content_type = 'text/plain'
self.content_length = None
if __debug__:
self.err_log.debug('Getting sock_file')
# Build our file-like object
sock_file = conn.makefile('rb', BUF_SIZE)
request = self.read_request_line(sock_file)
if request['method'].upper() not in ('GET', ):
self.status = "501 Not Implemented"
try:
# Get our file path
headers = dict([(str(k.lower()), v)
for k, v in self.read_headers(sock_file).items()])
rpath = request.get('path', '').lstrip('/')
filepath = os.path.join(self.root, rpath)
filepath = os.path.abspath(filepath)
if __debug__:
self.err_log.debug('Request for path: %s' % filepath)
self.closeConnection = headers.get('connection',
'close').lower() == 'close'
self.headers = Headers([
('Date', formatdate(usegmt=True)),
('Server', HTTP_SERVER_SOFTWARE),
('Connection', headers.get('connection', 'close')),
])
if not filepath.lower().startswith(self.root.lower()):
# File must be within our root directory
self.status = "400 Bad Request"
self.closeConnection = True
elif not os.path.exists(filepath):
self.status = "404 File Not Found"
self.closeConnection = True
elif os.path.isdir(filepath):
self.serve_dir(filepath, rpath)
elif os.path.isfile(filepath):
self.serve_file(filepath, headers)
else:
# It exists but it's not a file or a directory????
# What is it then?
self.status = "501 Not Implemented"
self.closeConnection = True
h = self.headers
statcode, statstr = self.status.split(' ', 1)
statcode = int(statcode)
if statcode >= 400:
h.add_header('Content-Type', self.content_type)
self.data = [statstr]
# Build our output headers
header_data = HEADER_RESPONSE % (self.status, str(h))
# Send the headers
if __debug__:
self.err_log.debug('Sending Headers: %s' % repr(header_data))
self.conn.sendall(b(header_data))
for data in self.data:
self.conn.sendall(b(data))
if hasattr(self.data, 'close'):
self.data.close()
finally:
if __debug__:
self.err_log.debug('Finally closing sock_file')
sock_file.close()
