def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If setting.LOCKDOWN is true only calculations owned by the current user can be retrieved. """ user = utils.get_user_data(request) username = user['name'] if user['acl_on'] else None job = logs.dbcmd('get_job', int(calc_id), username) if job is None: return HttpResponseNotFound() # read the data and save them on a temporary .pik file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp(prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) n = len(request.path_info) query_string = request.get_full_path()[n:] obj = _extract(ds, what + query_string) if inspect.isgenerator(obj): array, attrs = 0, {k: _array(v) for k, v in obj} elif hasattr(obj, '__toh5__'): array, attrs = obj.__toh5__() else: # assume obj is an array array, attrs = obj, {} numpy.savez_compressed(fname, array=array, **attrs) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ('attachment; filename=%s' % os.path.basename(fname)) return response
def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true only calculations owned by the current user can be retrieved. """ job = logs.dbcmd('get_job', int(calc_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() try: # read the data and save them on a temporary .npz file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp(prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) n = len(request.path_info) query_string = unquote_plus(request.get_full_path()[n:]) aw = _extract(ds, what + query_string) a = {} for key, val in vars(aw).items(): key = str(key) # can be a numpy.bytes_ if key.startswith('_'): continue elif isinstance(val, str): # without this oq extract would fail a[key] = numpy.array(val.encode('utf-8')) elif isinstance(val, dict): # this is hack: we are losing the values a[key] = list(val) else: a[key] = val numpy.savez_compressed(fname, **a) except Exception as exc: tb = ''.join(traceback.format_tb(exc.__traceback__)) return HttpResponse(content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb), content_type='text/plain', status=500) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ('attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true only calculations owned by the current user can be retrieved. """ job = logs.dbcmd('get_job', int(calc_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() try: # read the data and save them on a temporary .npz file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp( prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) n = len(request.path_info) query_string = unquote_plus(request.get_full_path()[n:]) aw = _extract(ds, what + query_string) a = {} for key, val in vars(aw).items(): key = str(key) # can be a numpy.bytes_ if isinstance(val, str): # without this oq extract would fail a[key] = numpy.array(val.encode('utf-8')) elif isinstance(val, dict): # this is hack: we are losing the values a[key] = list(val) else: a[key] = val numpy.savez_compressed(fname, **a) except Exception as exc: tb = ''.join(traceback.format_tb(exc.__traceback__)) return HttpResponse( content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb), content_type='text/plain', status=500) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If setting.LOCKDOWN is true only calculations owned by the current user can be retrieved. """ job = logs.dbcmd('get_job', int(calc_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() # read the data and save them on a temporary .pik file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp( prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) n = len(request.path_info) query_string = unquote_plus(request.get_full_path()[n:]) obj = _extract(ds, what + query_string) if inspect.isgenerator(obj): array, attrs = 0, {k: _array(v) for k, v in obj} elif hasattr(obj, '__toh5__'): array, attrs = obj.__toh5__() else: # assume obj is an array array, attrs = obj, {} a = {} for key, val in attrs.items(): if isinstance(key, bytes): key = key.decode('utf-8') if isinstance(val, str): # without this oq extract would fail a[key] = numpy.array(val.encode('utf-8')) else: a[key] = val numpy.savez_compressed(fname, array=array, **a) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true only calculations owned by the current user can be retrieved. """ job = logs.dbcmd('get_job', int(calc_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() path = request.get_full_path() n = len(request.path_info) query_string = unquote_plus(path[n:]) try: # read the data and save them on a temporary .npz file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp( prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) obj = _extract(ds, what + query_string) hdf5.save_npz(obj, fname) except Exception as exc: tb = ''.join(traceback.format_tb(exc.__traceback__)) return HttpResponse( content='%s: %s in %s\n%s' % (exc.__class__.__name__, exc, path, tb), content_type='text/plain', status=500) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def calc_result(request, result_id): """ Download a specific result, by ``result_id``. The common abstracted functionality for getting hazard or risk results. :param request: `django.http.HttpRequest` object. Can contain a `export_type` GET param (the default is 'xml' if no param is specified). :param result_id: The id of the requested artifact. :returns: If the requested ``result_id`` is not available in the format designated by the `export_type`. Otherwise, return a `django.http.HttpResponse` containing the content of the requested artifact. Parameters for the GET request can include an `export_type`, such as 'xml', 'geojson', 'csv', etc. """ # If the result for the requested ID doesn't exist, OR # the job which it is related too is not complete, # throw back a 404. try: job_id, job_status, job_user, datadir, ds_key = logs.dbcmd( 'get_result', result_id) if not utils.user_has_permission(request, job_user): return HttpResponseForbidden() except dbapi.NotFound: return HttpResponseNotFound() etype = request.GET.get('export_type') export_type = etype or DEFAULT_EXPORT_TYPE tmpdir = tempfile.mkdtemp() try: exported = core.export_from_db((ds_key, export_type), job_id, datadir, tmpdir) except DataStoreExportError as exc: # TODO: there should be a better error page return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc), content_type='text/plain', status=500) if not exported: # Throw back a 404 if the exact export parameters are not supported return HttpResponseNotFound( 'Nothing to export for export_type=%s, %s' % (export_type, ds_key)) elif len(exported) > 1: # Building an archive so that there can be a single file download archname = ds_key + '-' + export_type + '.zip' zipfiles(exported, os.path.join(tmpdir, archname)) exported = os.path.join(tmpdir, archname) else: # single file exported = exported[0] content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type, DEFAULT_CONTENT_TYPE) fname = 'output-%s-%s' % (result_id, os.path.basename(exported)) stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir)) response = FileResponse(stream, content_type=content_type) response['Content-Disposition'] = ('attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(exported)) return response
def get_result(request, result_id): """ Download a specific result, by ``result_id``. The common abstracted functionality for getting hazard or risk results. :param request: `django.http.HttpRequest` object. Can contain a `export_type` GET param (the default is 'xml' if no param is specified). :param result_id: The id of the requested artifact. :returns: If the requested ``result_id`` is not available in the format designated by the `export_type`. Otherwise, return a `django.http.HttpResponse` containing the content of the requested artifact. Parameters for the GET request can include an `export_type`, such as 'xml', 'geojson', 'csv', etc. """ # If the result for the requested ID doesn't exist, OR # the job which it is related too is not complete, # throw back a 404. try: job_id, job_status, datadir, ds_key = logs.dbcmd( 'get_result', result_id) except dbapi.NotFound: return HttpResponseNotFound() etype = request.GET.get('export_type') export_type = etype or DEFAULT_EXPORT_TYPE tmpdir = tempfile.mkdtemp() try: exported = core.export_from_db((ds_key, export_type), job_id, datadir, tmpdir) except DataStoreExportError as exc: # TODO: there should be a better error page return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc), content_type='text/plain', status=500) if exported is None: # Throw back a 404 if the exact export parameters are not supported return HttpResponseNotFound('export_type=%s is not supported for %s' % (export_type, ds_key)) content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type, DEFAULT_CONTENT_TYPE) bname = os.path.basename(exported) if bname.startswith('.'): # the "." is added by `export_from_db`, strip it bname = bname[1:] fname = 'output-%s-%s' % (result_id, bname) # 'b' is needed when running the WebUI on Windows stream = FileWrapper(open(exported, 'rb')) stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir)) response = FileResponse(stream, content_type=content_type) response['Content-Disposition'] = ('attachment; filename=%s' % os.path.basename(fname)) return response
def calc_result(request, result_id): """ Download a specific result, by ``result_id``. The common abstracted functionality for getting hazard or risk results. :param request: `django.http.HttpRequest` object. Can contain a `export_type` GET param (the default is 'xml' if no param is specified). :param result_id: The id of the requested artifact. :returns: If the requested ``result_id`` is not available in the format designated by the `export_type`. Otherwise, return a `django.http.HttpResponse` containing the content of the requested artifact. Parameters for the GET request can include an `export_type`, such as 'xml', 'geojson', 'csv', etc. """ # If the result for the requested ID doesn't exist, OR # the job which it is related too is not complete, # throw back a 404. try: job_id, job_status, job_user, datadir, ds_key = logs.dbcmd( 'get_result', result_id) if not utils.user_has_permission(request, job_user): return HttpResponseForbidden() except dbapi.NotFound: return HttpResponseNotFound() etype = request.GET.get('export_type') export_type = etype or DEFAULT_EXPORT_TYPE tmpdir = tempfile.mkdtemp() try: exported = core.export_from_db( (ds_key, export_type), job_id, datadir, tmpdir) except DataStoreExportError as exc: # TODO: there should be a better error page return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc), content_type='text/plain', status=500) if not exported: # Throw back a 404 if the exact export parameters are not supported return HttpResponseNotFound( 'Nothing to export for export_type=%s, %s' % (export_type, ds_key)) elif len(exported) > 1: # Building an archive so that there can be a single file download archname = ds_key + '-' + export_type + '.zip' zipfiles(exported, os.path.join(tmpdir, archname)) exported = os.path.join(tmpdir, archname) else: # single file exported = exported[0] content_type = EXPORT_CONTENT_TYPE_MAP.get( export_type, DEFAULT_CONTENT_TYPE) fname = 'output-%s-%s' % (result_id, os.path.basename(exported)) stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows stream.close = lambda: ( FileWrapper.close(stream), shutil.rmtree(tmpdir)) response = FileResponse(stream, content_type=content_type) response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(exported)) return response
class FileSystemWorker(Worker): def __init__(self, *args, **kwargs): """Builds some instance variables that will last the life of the thread.""" Worker.__init__(self, *args, **kwargs) self.root = os.path.abspath(self.app_info['document_root']) self.display_index = self.app_info['display_index'] def serve_file(self, filepath, headers): filestat = os.stat(filepath) self.size = filestat.st_size modtime = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(filestat.st_mtime)) self.headers.add_header('Last-Modified', modtime) if headers.get('if_modified_since') == modtime: # The browser cache is up-to-date, send a 304. self.status = "304 Not Modified" self.data = [] return ct = mimetypes.guess_type(filepath)[0] self.content_type = ct if ct else 'text/plain' try: f = open(filepath, 'rb') self.headers['Pragma'] = 'cache' self.headers['Cache-Control'] = 'private' self.headers['Content-Length'] = str(self.size) if self.etag: self.headers.add_header('Etag', self.etag) if self.expires: self.headers.add_header('Expires', self.expires) try: # Implement 206 partial file support. start, end = headers['range'].split('-') start = 0 if not start.isdigit() else int(start) end = self.size if not end.isdigit() else int(end) if self.size < end or start < 0: self.status = "214 Unsatisfiable Range Requested" self.data = FileWrapper(f, CHUNK_SIZE) else: f.seek(start) self.data = LimitingFileWrapper(f, CHUNK_SIZE, limit=end) self.status = "206 Partial Content" except: self.data = FileWrapper(f, CHUNK_SIZE) except IOError: self.status = "403 Forbidden" def serve_dir(self, pth, rpth): def rel_path(path): return os.path.normpath(path[len(self.root):] if path.startswith(self.root) else path) if not self.display_index: self.status = '404 File Not Found' return b('') else: self.content_type = 'text/html' dir_contents = [os.path.join(pth, x) for x in os.listdir(os.path.normpath(pth))] dir_contents.sort() dirs = [rel_path(x)+'/' for x in dir_contents if os.path.isdir(x)] files = [rel_path(x) for x in dir_contents if os.path.isfile(x)] self.data = [INDEX_HEADER % dict(path='/'+rpth)] if rpth: self.data += [INDEX_ROW % dict(name='(parent directory)', cls='dir parent', link='/'.join(rpth[:-1].split('/')[:-1]))] self.data += [INDEX_ROW % dict(name=os.path.basename(x[:-1]), link=os.path.join(rpth, os.path.basename(x[:-1])).replace('\\', '/'), cls='dir') for x in dirs] self.data += ['<tr><th>Files</th></tr>'] self.data += [INDEX_ROW % dict(name=os.path.basename(x), link=os.path.join(rpth, os.path.basename(x)).replace('\\', '/'), cls='file') for x in files] self.data += [INDEX_FOOTER] self.headers['Content-Length'] = self.size = str(sum([len(x) for x in self.data])) self.status = '200 OK' def run_app(self, conn): self.status = "200 OK" self.size = 0 self.expires = None self.etag = None self.content_type = 'text/plain' self.content_length = None if __debug__: self.err_log.debug('Getting sock_file') # Build our file-like object sock_file = conn.makefile('rb',BUF_SIZE) request = self.read_request_line(sock_file) if request['method'].upper() not in ('GET', ): self.status = "501 Not Implemented" try: # Get our file path headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()]) rpath = request.get('path', '').lstrip('/') filepath = os.path.join(self.root, rpath) filepath = os.path.abspath(filepath) if __debug__: self.err_log.debug('Request for path: %s' % filepath) self.closeConnection = headers.get('connection', 'close').lower() == 'close' self.headers = Headers([('Date', formatdate(usegmt=True)), ('Server', HTTP_SERVER_SOFTWARE), ('Connection', headers.get('connection', 'close')), ]) if not filepath.lower().startswith(self.root.lower()): # File must be within our root directory self.status = "400 Bad Request" self.closeConnection = True elif not os.path.exists(filepath): self.status = "404 File Not Found" self.closeConnection = True elif os.path.isdir(filepath): self.serve_dir(filepath, rpath) elif os.path.isfile(filepath): self.serve_file(filepath, headers) else: # It exists but it's not a file or a directory???? # What is it then? self.status = "501 Not Implemented" self.closeConnection = True h = self.headers statcode, statstr = self.status.split(' ', 1) statcode = int(statcode) if statcode >= 400: h.add_header('Content-Type', self.content_type) self.data = [statstr] # Build our output headers header_data = HEADER_RESPONSE % (self.status, str(h)) # Send the headers if __debug__: self.err_log.debug('Sending Headers: %s' % repr(header_data)) self.conn.sendall(b(header_data)) for data in self.data: self.conn.sendall(b(data)) if hasattr(self.data, 'close'): self.data.close() finally: if __debug__: self.err_log.debug('Finally closing sock_file') sock_file.close()
class FileSystemWorker(Worker): def __init__(self, *args, **kwargs): """Builds some instance variables that will last the life of the thread.""" Worker.__init__(self, *args, **kwargs) self.root = os.path.abspath(self.app_info['document_root']) self.display_index = self.app_info['display_index'] def serve_file(self, filepath, headers): filestat = os.stat(filepath) self.size = filestat.st_size modtime = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(filestat.st_mtime)) self.headers.add_header('Last-Modified', modtime) if headers.get('if_modified_since') == modtime: # The browser cache is up-to-date, send a 304. self.status = "304 Not Modified" self.data = [] return ct = mimetypes.guess_type(filepath)[0] self.content_type = ct if ct else 'text/plain' try: f = open(filepath, 'rb') self.headers['Pragma'] = 'cache' self.headers['Cache-Control'] = 'private' self.headers['Content-Length'] = str(self.size) if self.etag: self.headers.add_header('Etag', self.etag) if self.expires: self.headers.add_header('Expires', self.expires) try: # Implement 206 partial file support. start, end = headers['range'].split('-') start = 0 if not start.isdigit() else int(start) end = self.size if not end.isdigit() else int(end) if self.size < end or start < 0: self.status = "214 Unsatisfiable Range Requested" self.data = FileWrapper(f, CHUNK_SIZE) else: f.seek(start) self.data = LimitingFileWrapper(f, CHUNK_SIZE, limit=end) self.status = "206 Partial Content" except: self.data = FileWrapper(f, CHUNK_SIZE) except IOError: self.status = "403 Forbidden" def serve_dir(self, pth, rpth): def rel_path(path): return os.path.normpath( path[len(self.root):] if path.startswith(self.root) else path) if not self.display_index: self.status = '404 File Not Found' return b('') else: self.content_type = 'text/html' dir_contents = [ os.path.join(pth, x) for x in os.listdir(os.path.normpath(pth)) ] dir_contents.sort() dirs = [ rel_path(x) + '/' for x in dir_contents if os.path.isdir(x) ] files = [rel_path(x) for x in dir_contents if os.path.isfile(x)] self.data = [INDEX_HEADER % dict(path='/' + rpth)] if rpth: self.data += [ INDEX_ROW % dict(name='(parent directory)', cls='dir parent', link='/'.join(rpth[:-1].split('/')[:-1])) ] self.data += [ INDEX_ROW % dict(name=os.path.basename(x[:-1]), link=os.path.join(rpth, os.path.basename(x[:-1])).replace( '\\', '/'), cls='dir') for x in dirs ] self.data += ['<tr><th>Files</th></tr>'] self.data += [ INDEX_ROW % dict(name=os.path.basename(x), link=os.path.join(rpth, os.path.basename(x)).replace( '\\', '/'), cls='file') for x in files ] self.data += [INDEX_FOOTER] self.headers['Content-Length'] = self.size = str( sum([len(x) for x in self.data])) self.status = '200 OK' def run_app(self, conn): self.status = "200 OK" self.size = 0 self.expires = None self.etag = None self.content_type = 'text/plain' self.content_length = None if __debug__: self.err_log.debug('Getting sock_file') # Build our file-like object sock_file = conn.makefile('rb', BUF_SIZE) request = self.read_request_line(sock_file) if request['method'].upper() not in ('GET', ): self.status = "501 Not Implemented" try: # Get our file path headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()]) rpath = request.get('path', '').lstrip('/') filepath = os.path.join(self.root, rpath) filepath = os.path.abspath(filepath) if __debug__: self.err_log.debug('Request for path: %s' % filepath) self.closeConnection = headers.get('connection', 'close').lower() == 'close' self.headers = Headers([ ('Date', formatdate(usegmt=True)), ('Server', HTTP_SERVER_SOFTWARE), ('Connection', headers.get('connection', 'close')), ]) if not filepath.lower().startswith(self.root.lower()): # File must be within our root directory self.status = "400 Bad Request" self.closeConnection = True elif not os.path.exists(filepath): self.status = "404 File Not Found" self.closeConnection = True elif os.path.isdir(filepath): self.serve_dir(filepath, rpath) elif os.path.isfile(filepath): self.serve_file(filepath, headers) else: # It exists but it's not a file or a directory???? # What is it then? self.status = "501 Not Implemented" self.closeConnection = True h = self.headers statcode, statstr = self.status.split(' ', 1) statcode = int(statcode) if statcode >= 400: h.add_header('Content-Type', self.content_type) self.data = [statstr] # Build our output headers header_data = HEADER_RESPONSE % (self.status, str(h)) # Send the headers if __debug__: self.err_log.debug('Sending Headers: %s' % repr(header_data)) self.conn.sendall(b(header_data)) for data in self.data: self.conn.sendall(b(data)) if hasattr(self.data, 'close'): self.data.close() finally: if __debug__: self.err_log.debug('Finally closing sock_file') sock_file.close()