def create_archive(sim): """Zip up the json file and its dependencies Args: sim (PKDict): parsed request Returns: py.path.Local: zip file name """ if not pkio.has_file_extension(sim.filename, ('zip', 'html')): raise sirepo.util.NotFound( 'unknown file type={}; expecting html or zip'.format(sim.filename) ) with simulation_db.tmp_dir() as d: want_zip = sim.filename.endswith('zip') f, c = _create_zip(sim, want_python=want_zip, out_dir=d) if want_zip: t = 'application/zip' else: f, t = _create_html(f, c) return sirepo.http_reply.gen_file_as_attachment( f, content_type=t, filename=sim.filename, )
def _create_zip(sim_type, sim_id, want_python): """Zip up the json file and its dependencies Args: sim_type (str): simulation type sim_id (str): simulation id want_python (bool): include template's python source? Returns: py.path.Local: zip file name """ from pykern import pkio from sirepo import simulation_db from sirepo.template import template_common #TODO(robnagler) need a lock with pkio.save_chdir(simulation_db.tmp_dir()): res = py.path.local(sim_id + '.zip') data = simulation_db.open_json_file(sim_type, sid=sim_id) if 'report' in data: del data['report'] files = template_common.lib_files(data) files.insert(0, simulation_db.sim_data_file(data.simulationType, sim_id)) if want_python: files.append(_python(data)) with zipfile.ZipFile( str(res), mode='w', compression=zipfile.ZIP_DEFLATED, allowZip64=True, ) as z: for f in files: z.write(str(f), f.basename) return res, data
def _create_zip(sim_type, sim_id, want_python): """Zip up the json file and its dependencies Args: sim_type (str): simulation type sim_id (str): simulation id want_python (bool): include template's python source? Returns: py.path.Local: zip file name """ from pykern import pkio from sirepo import simulation_db from sirepo.template import template_common #TODO(robnagler) need a lock with pkio.save_chdir(simulation_db.tmp_dir()): res = py.path.local(sim_id + '.zip') data = simulation_db.open_json_file(sim_type, sid=sim_id) files = template_common.lib_files(data) files.insert(0, simulation_db.sim_data_file(data.simulationType, sim_id)) if want_python: files.append(_python(data)) with zipfile.ZipFile( str(res), mode='w', compression=zipfile.ZIP_DEFLATED, allowZip64=True, ) as z: for f in files: z.write(str(f), f.basename) return res, data
def api_importFile(simulation_type): """ Args: simulation_type (str): which simulation type Params: file: file data folder: where to import to """ import sirepo.importer error = None f = None try: f = flask.request.files.get('file') if not f: raise sirepo.util.Error('must supply a file') req = http_request.parse_params( filename=f.filename, folder=flask.request.form.get('folder'), id=flask.request.form.get('simulationId'), template=True, type=simulation_type, ) req.file_stream = f.stream req.import_file_arguments = flask.request.form.get('arguments', '') def s(data): data.models.simulation.folder = req.folder data.models.simulation.isExample = False return _save_new_and_reply(data) if pkio.has_file_extension(req.filename, 'json'): data = sirepo.importer.read_json(req.file_stream.read(), req.type) #TODO(pjm): need a separate URI interface to importer, added exception for rs4pi for now # (dicom input is normally a zip file) elif pkio.has_file_extension(req.filename, 'zip') and req.type != 'rs4pi': data = sirepo.importer.read_zip(req.file_stream, sim_type=req.type) else: if not hasattr(req.template, 'import_file'): raise sirepo.util.Error('Only zip files are supported') with simulation_db.tmp_dir() as d: data = req.template.import_file(req, tmp_dir=d, reply_op=s) if 'error' in data: return http_reply.gen_json(data) return s(data) except werkzeug.exceptions.HTTPException: raise except sirepo.util.Reply: raise except Exception as e: pkdlog('{}: exception: {}', f and f.filename, pkdexc()) #TODO(robnagler) security issue here. Really don't want to report errors to user error = str(e.args) if hasattr(e, 'args') else str(e) return http_reply.gen_json({ 'error': error if error else 'An unknown error occurred', })
def get_data_file(run_dir, model, frame, **kwargs): if model == 'dicomAnimation4': with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f: return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream' tmp_dir = simulation_db.tmp_dir() filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir) with open(filename, 'rb') as f: dicom_data = f.read() pkio.unchecked_remove(tmp_dir) return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def get_data_file(run_dir, model, frame, **kwargs): if model == 'dicomAnimation4': with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f: return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream' tmp_dir = simulation_db.tmp_dir() filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir) with open (filename, 'rb') as f: dicom_data = f.read() pkio.unchecked_remove(tmp_dir) return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def app_import_file(simulation_type): f = flask.request.files['file'] error, data = sirepo.importer.import_python( f.read(), lib_dir=simulation_db.simulation_lib_dir(simulation_type), tmp_dir=simulation_db.tmp_dir(), user_filename=f.filename, ) if error: return flask.jsonify({'error': error}) return _save_new_and_reply(simulation_type, data)
def read_zip(stream, template=None): """Read zip file and store contents Args: stream (IO): file to read template (module): expected app Returns: dict: data """ from pykern import pkcollections from sirepo import simulation_db from sirepo.template import template_common import py.path import re import zipfile tmp = simulation_db.tmp_dir() data = None zipped = pkcollections.Dict() with zipfile.ZipFile(stream, 'r') as z: for i in z.infolist(): b = py.path.local(i.filename).basename c = z.read(i) if b.lower() == simulation_db.SIMULATION_DATA_FILE: assert not data, \ 'too many db files {} in archive'.format(b) data = read_json(c, template) if not template: import sirepo.template template = sirepo.template.import_module( data.simulationType) continue if re.match('__MACOSX', i.filename): continue #TODO(robnagler) ignore identical files hash assert not b in zipped, \ '{} duplicate file in archive'.format(i.filename) fn = tmp.join(b) with open(str(fn), 'wb') as f: f.write(c) zipped[b] = fn assert data, \ 'missing {} in archive'.format(simulation_db.SIMULATION_DATA_FILE) needed = pkcollections.Dict() for n in template_common.lib_files(data): assert n.basename in zipped or n.check(file=True, exists=True), \ 'auxiliary file {} missing in archive'.format(n.basename) needed[n.basename] = n lib_d = simulation_db.simulation_lib_dir(template.SIM_TYPE) for b, src in zipped.items(): if b in needed: src.copy(needed[b]) return data
def read_zip(stream, template=None): """Read zip file and store contents Args: stream (IO): file to read template (module): expected app Returns: dict: data """ from pykern import pkcollections from sirepo import simulation_db from sirepo.template import template_common import py.path import zipfile tmp = simulation_db.tmp_dir() data = None zipped = pkcollections.Dict() with zipfile.ZipFile(stream, 'r') as z: for i in z.infolist(): b = py.path.local(i.filename).basename c = z.read(i) if b.lower() == simulation_db.SIMULATION_DATA_FILE: assert not data, \ 'too many db files {} in archive'.format(b) data = read_json(c, template) if not template: import sirepo.template template = sirepo.template.import_module(data.simulationType) continue #TODO(robnagler) ignore identical files hash assert not b in zipped, \ '{} duplicate file in archive'.format(i.filename) fn = tmp.join(b) with open(str(fn), 'wb') as f: f.write(c) zipped[b] = fn assert data, \ 'missing {} in archive'.format(simulation_db.SIMULATION_DATA_FILE) needed = pkcollections.Dict() for n in template_common.lib_files(data): assert n.basename in zipped or n.check(file=True, exists=True), \ 'auxiliary file {} missing in archive'.format(n.basename) needed[n.basename] = n lib_d = simulation_db.simulation_lib_dir(template.SIM_TYPE) for b, src in zipped.items(): if b in needed: src.copy(needed[b]) return data
def app_import_file(simulation_type): f = flask.request.files['file'] arguments = str(flask.request.form['arguments']) pkdp('\n\tFile: {}\n\tArguments: {}', f.filename, arguments) error, data = sirepo.importer.import_python( f.read(), lib_dir=simulation_db.simulation_lib_dir(simulation_type), tmp_dir=simulation_db.tmp_dir(), user_filename=f.filename, arguments=arguments, ) if error: return flask.jsonify({'error': error}) return _save_new_and_reply(simulation_type, data)
def read_zip(stream, sim_type=None): """Read zip file and store contents Args: stream (IO): file to read sim_type (module): expected app Returns: dict: data """ from sirepo import simulation_db import sirepo.sim_data with simulation_db.tmp_dir() as tmp: data = None zipped = PKDict() with zipfile.ZipFile(stream, 'r') as z: for i in z.infolist(): b = pykern.pkio.py_path(i.filename).basename c = z.read(i) if b.lower() == simulation_db.SIMULATION_DATA_FILE: assert not data, \ 'too many db files {} in archive'.format(b) data = read_json(c, sim_type) continue if '__MACOSX' in i.filename: continue #TODO(robnagler) ignore identical files hash assert not b in zipped, \ '{} duplicate file in archive'.format(i.filename) zipped[b] = tmp.join(b) zipped[b].write(c, 'wb') assert data, \ 'missing {} in archive'.format(simulation_db.SIMULATION_DATA_FILE) needed = set() s = sirepo.sim_data.get_class(data.simulationType) for n in s.lib_file_basenames(data): #TODO(robnagler) this does not allow overwrites of lib files, # but it needs to be modularized if s.lib_file_exists(n): continue #TODO(robnagler) raise useralert instead of an assert assert n in zipped, \ 'auxiliary file={} missing in archive'.format(n) needed.add(n) for b, src in zipped.items(): if b in needed: src.copy(s.lib_file_write_path(b)) return data
def get_data_file(run_dir, model, frame, **kwargs): if model == 'dicomAnimation4': filename = filename = _parent_file(run_dir, _DOSE_DICOM_FILE) uri = RTDOSE_EXPORT_FILENAME with open(filename, mode='rb') as f: out = f.read() else: with simulation_db.tmp_dir() as tmp_dir: filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir) uri = RTSTRUCT_EXPORT_FILENAME with open(filename, mode='rb') as f: out = f.read() return PKDict( content=out, uri=uri, )
def api_importFile(simulation_type=None): """ Args: simulation_type (str): which simulation type Params: file: file data folder: where to import to """ import sirepo.importer error = None f = None try: template = simulation_type and sirepo.template.import_module( simulation_type) f = flask.request.files.get('file') assert f, \ ValueError('must supply a file') if pkio.has_file_extension(f.filename, 'json'): data = sirepo.importer.read_json(f.read(), template) #TODO(pjm): need a separate URI interface to importer, added exception for rs4pi for now # (dicom input is normally a zip file) elif pkio.has_file_extension(f.filename, 'zip') and simulation_type != 'rs4pi': data = sirepo.importer.read_zip(f.stream, template) else: assert simulation_type, \ 'simulation_type is required param for non-zip|json imports' assert hasattr(template, 'import_file'), \ ValueError('Only zip files are supported') data = template.import_file( flask.request, simulation_db.simulation_lib_dir(simulation_type), simulation_db.tmp_dir(), ) #TODO(robnagler) need to validate folder data.models.simulation.folder = flask.request.form['folder'] data.models.simulation.isExample = False return _save_new_and_reply(data) except Exception as e: pkdlog('{}: exception: {}', f and f.filename, pkdexc()) error = str(e.message) if hasattr(e, 'message') else str(e) return http_reply.gen_json({ 'error': error if error else 'An unknown error occurred', })
def api_getApplicationData(filename=None): """Get some data from the template Args: filename (str): if supplied, result is file attachment Returns: response: may be a file or JSON """ req = http_request.parse_post(template=True, filename=filename or None) with simulation_db.tmp_dir() as d: res = req.template.get_application_data(req.req_data, tmp_dir=d) if 'filename' in req: assert isinstance(res, pkconst.PY_PATH_LOCAL_TYPE), \ '{}: template did not return a file'.format(res) return http_reply.gen_file_as_attachment(res, filename=req.filename) return http_reply.gen_json(res)
def api_getApplicationData(filename=None): """Get some data from the template Args: filename (str): if supplied, result is file attachment Returns: response: may be a file or JSON """ req = http_request.parse_post(template=True, filename=filename or None) with simulation_db.tmp_dir() as d: assert 'method' in req.req_data res = req.template.get_application_data(req.req_data, tmp_dir=d) assert res != None, f'unhandled application data method: {req.req_data.method}' if 'filename' in req and isinstance(res, pkconst.PY_PATH_LOCAL_TYPE): return http_reply.gen_file_as_attachment( res, filename=req.filename, content_type=req.req_data.get('contentType', None)) return http_reply.gen_json(res)
def api_downloadDataFile(simulation_type, simulation_id, model, frame, suffix=None): #TODO(robnagler) validate suffix and frame req = sirepo.http_request.parse_params( id=simulation_id, model=model, type=simulation_type, check_sim_exists=True, ) s = suffix and sirepo.srschema.parse_name(suffix) t = None with simulation_db.tmp_dir() as d: # TODO(e-carlin): computeJobHash t = sirepo.job.DATA_FILE_ROOT.join(sirepo.job.unique_key()) t.mksymlinkto(d, absolute=True) try: r = _request( computeJobHash='unused', dataFileKey=t.basename, frame=int(frame), isParallel=False, req_data=req.req_data, suffix=s, ) assert not r.state == 'error', f'error state in request=={r}' f = d.listdir() if len(f) > 0: assert len(f) == 1, \ 'too many files={}'.format(f) return sirepo.http_reply.gen_file_as_attachment(f[0]) except requests.exceptions.HTTPError: #TODO(robnagler) HTTPError is too coarse a check pass finally: if t: pykern.pkio.unchecked_remove(t) raise sirepo.util.raise_not_found( 'frame={} not found {id} {type}'.format(frame, **req))
def api_uploadFile(simulation_type, simulation_id, file_type): f = flask.request.files['file'] req = http_request.parse_params( file_type=file_type, filename=f.filename, id=simulation_id, template=True, type=simulation_type, ) e = None in_use = None with simulation_db.tmp_dir() as d: t = d.join(req.filename) f.save(str(t)) if hasattr(req.template, 'validate_file'): # Note: validate_file may modify the file e = req.template.validate_file(req.file_type, t) if (not e and req.sim_data.lib_file_exists(req.filename) and not flask.request.form.get('confirm')): in_use = _simulations_using_file(req, ignore_sim_id=req.id) if in_use: e = 'File is in use in other simulations. Please confirm you would like to replace the file for all simulations.' if e: return http_reply.gen_json({ 'error': e, 'filename': req.filename, 'fileList': in_use, 'fileType': req.file_type, 'simulationId': req.id, }) t.rename(_lib_file_write_path(req)) return http_reply.gen_json({ 'filename': req.filename, 'fileType': req.file_type, 'simulationId': req.id, })
def api_importFile(simulation_type=None): """ Args: simulation_type (str): which simulation type Params: file: file data folder: where to import to """ import sirepo.importer error = None f = None try: template = simulation_type and sirepo.template.import_module(simulation_type) f = flask.request.files.get('file') assert f, \ ValueError('must supply a file') if pkio.has_file_extension(f.filename, 'json'): data = sirepo.importer.read_json(f.read(), template) elif pkio.has_file_extension(f.filename, 'zip'): data = sirepo.importer.read_zip(f.stream, template) else: assert simulation_type, \ 'simulation_type is required param for non-zip|json imports' data = template.import_file( flask.request, simulation_db.simulation_lib_dir(simulation_type), simulation_db.tmp_dir(), ) #TODO(robnagler) need to validate folder data.models.simulation.folder = flask.request.form['folder'] return _save_new_and_reply(data.simulationType, data) except Exception as e: pkdlog('{}: exception: {}', f and f.filename, pkdexc()) error = e.message if hasattr(e, 'message') else str(e) return _json_response({'error': error})
def app_import_file(simulation_type): template = sirepo.template.import_module(simulation_type) error, data = template.import_file(flask.request, simulation_db.simulation_lib_dir(simulation_type), simulation_db.tmp_dir()) if error: return _json_response({'error': error}) data['models']['simulation']['folder'] = flask.request.form['folder'] return _save_new_and_reply(simulation_type, data)