def create_archive(sim):
"""Zip up the json file and its dependencies
Args:
sim (PKDict): parsed request
Returns:
py.path.Local: zip file name
"""
if not pkio.has_file_extension(sim.filename, ('zip', 'html')):
raise sirepo.util.NotFound(
'unknown file type={}; expecting html or zip'.format(sim.filename)
)
with simulation_db.tmp_dir() as d:
want_zip = sim.filename.endswith('zip')
f, c = _create_zip(sim, want_python=want_zip, out_dir=d)
if want_zip:
t = 'application/zip'
else:
f, t = _create_html(f, c)
return sirepo.http_reply.gen_file_as_attachment(
f,
content_type=t,
filename=sim.filename,
)
def _create_zip(sim_type, sim_id, want_python):
"""Zip up the json file and its dependencies
Args:
sim_type (str): simulation type
sim_id (str): simulation id
want_python (bool): include template's python source?
Returns:
py.path.Local: zip file name
"""
from pykern import pkio
from sirepo import simulation_db
from sirepo.template import template_common
#TODO(robnagler) need a lock
with pkio.save_chdir(simulation_db.tmp_dir()):
res = py.path.local(sim_id + '.zip')
data = simulation_db.open_json_file(sim_type, sid=sim_id)
if 'report' in data:
del data['report']
files = template_common.lib_files(data)
files.insert(0, simulation_db.sim_data_file(data.simulationType, sim_id))
if want_python:
files.append(_python(data))
with zipfile.ZipFile(
str(res),
mode='w',
compression=zipfile.ZIP_DEFLATED,
allowZip64=True,
) as z:
for f in files:
z.write(str(f), f.basename)
return res, data
def _create_zip(sim_type, sim_id, want_python):
"""Zip up the json file and its dependencies
Args:
sim_type (str): simulation type
sim_id (str): simulation id
want_python (bool): include template's python source?
Returns:
py.path.Local: zip file name
"""
from pykern import pkio
from sirepo import simulation_db
from sirepo.template import template_common
#TODO(robnagler) need a lock
with pkio.save_chdir(simulation_db.tmp_dir()):
res = py.path.local(sim_id + '.zip')
data = simulation_db.open_json_file(sim_type, sid=sim_id)
files = template_common.lib_files(data)
files.insert(0, simulation_db.sim_data_file(data.simulationType,
sim_id))
if want_python:
files.append(_python(data))
with zipfile.ZipFile(
str(res),
mode='w',
compression=zipfile.ZIP_DEFLATED,
allowZip64=True,
) as z:
for f in files:
z.write(str(f), f.basename)
return res, data
def api_importFile(simulation_type):
"""
Args:
simulation_type (str): which simulation type
Params:
file: file data
folder: where to import to
"""
import sirepo.importer
error = None
f = None
try:
f = flask.request.files.get('file')
if not f:
raise sirepo.util.Error('must supply a file')
req = http_request.parse_params(
filename=f.filename,
folder=flask.request.form.get('folder'),
id=flask.request.form.get('simulationId'),
template=True,
type=simulation_type,
)
req.file_stream = f.stream
req.import_file_arguments = flask.request.form.get('arguments', '')
def s(data):
data.models.simulation.folder = req.folder
data.models.simulation.isExample = False
return _save_new_and_reply(data)
if pkio.has_file_extension(req.filename, 'json'):
data = sirepo.importer.read_json(req.file_stream.read(), req.type)
#TODO(pjm): need a separate URI interface to importer, added exception for rs4pi for now
# (dicom input is normally a zip file)
elif pkio.has_file_extension(req.filename,
'zip') and req.type != 'rs4pi':
data = sirepo.importer.read_zip(req.file_stream, sim_type=req.type)
else:
if not hasattr(req.template, 'import_file'):
raise sirepo.util.Error('Only zip files are supported')
with simulation_db.tmp_dir() as d:
data = req.template.import_file(req, tmp_dir=d, reply_op=s)
if 'error' in data:
return http_reply.gen_json(data)
return s(data)
except werkzeug.exceptions.HTTPException:
raise
except sirepo.util.Reply:
raise
except Exception as e:
pkdlog('{}: exception: {}', f and f.filename, pkdexc())
#TODO(robnagler) security issue here. Really don't want to report errors to user
error = str(e.args) if hasattr(e, 'args') else str(e)
return http_reply.gen_json({
'error':
error if error else 'An unknown error occurred',
})
def get_data_file(run_dir, model, frame, **kwargs):
if model == 'dicomAnimation4':
with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f:
return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream'
tmp_dir = simulation_db.tmp_dir()
filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir)
with open(filename, 'rb') as f:
dicom_data = f.read()
pkio.unchecked_remove(tmp_dir)
return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def get_data_file(run_dir, model, frame, **kwargs):
if model == 'dicomAnimation4':
with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f:
return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream'
tmp_dir = simulation_db.tmp_dir()
filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir)
with open (filename, 'rb') as f:
dicom_data = f.read()
pkio.unchecked_remove(tmp_dir)
return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def app_import_file(simulation_type):
f = flask.request.files['file']
error, data = sirepo.importer.import_python(
f.read(),
lib_dir=simulation_db.simulation_lib_dir(simulation_type),
tmp_dir=simulation_db.tmp_dir(),
user_filename=f.filename,
)
if error:
return flask.jsonify({'error': error})
return _save_new_and_reply(simulation_type, data)
def read_zip(stream, template=None):
"""Read zip file and store contents
Args:
stream (IO): file to read
template (module): expected app
Returns:
dict: data
"""
from pykern import pkcollections
from sirepo import simulation_db
from sirepo.template import template_common
import py.path
import re
import zipfile
tmp = simulation_db.tmp_dir()
data = None
zipped = pkcollections.Dict()
with zipfile.ZipFile(stream, 'r') as z:
for i in z.infolist():
b = py.path.local(i.filename).basename
c = z.read(i)
if b.lower() == simulation_db.SIMULATION_DATA_FILE:
assert not data, \
'too many db files {} in archive'.format(b)
data = read_json(c, template)
if not template:
import sirepo.template
template = sirepo.template.import_module(
data.simulationType)
continue
if re.match('__MACOSX', i.filename):
continue
#TODO(robnagler) ignore identical files hash
assert not b in zipped, \
'{} duplicate file in archive'.format(i.filename)
fn = tmp.join(b)
with open(str(fn), 'wb') as f:
f.write(c)
zipped[b] = fn
assert data, \
'missing {} in archive'.format(simulation_db.SIMULATION_DATA_FILE)
needed = pkcollections.Dict()
for n in template_common.lib_files(data):
assert n.basename in zipped or n.check(file=True, exists=True), \
'auxiliary file {} missing in archive'.format(n.basename)
needed[n.basename] = n
lib_d = simulation_db.simulation_lib_dir(template.SIM_TYPE)
for b, src in zipped.items():
if b in needed:
src.copy(needed[b])
return data
def read_zip(stream, template=None):
"""Read zip file and store contents
Args:
stream (IO): file to read
template (module): expected app
Returns:
dict: data
"""
from pykern import pkcollections
from sirepo import simulation_db
from sirepo.template import template_common
import py.path
import zipfile
tmp = simulation_db.tmp_dir()
data = None
zipped = pkcollections.Dict()
with zipfile.ZipFile(stream, 'r') as z:
for i in z.infolist():
b = py.path.local(i.filename).basename
c = z.read(i)
if b.lower() == simulation_db.SIMULATION_DATA_FILE:
assert not data, \
'too many db files {} in archive'.format(b)
data = read_json(c, template)
if not template:
import sirepo.template
template = sirepo.template.import_module(data.simulationType)
continue
#TODO(robnagler) ignore identical files hash
assert not b in zipped, \
'{} duplicate file in archive'.format(i.filename)
fn = tmp.join(b)
with open(str(fn), 'wb') as f:
f.write(c)
zipped[b] = fn
assert data, \
'missing {} in archive'.format(simulation_db.SIMULATION_DATA_FILE)
needed = pkcollections.Dict()
for n in template_common.lib_files(data):
assert n.basename in zipped or n.check(file=True, exists=True), \
'auxiliary file {} missing in archive'.format(n.basename)
needed[n.basename] = n
lib_d = simulation_db.simulation_lib_dir(template.SIM_TYPE)
for b, src in zipped.items():
if b in needed:
src.copy(needed[b])
return data
def app_import_file(simulation_type):
f = flask.request.files['file']
arguments = str(flask.request.form['arguments'])
pkdp('\n\tFile: {}\n\tArguments: {}', f.filename, arguments)
error, data = sirepo.importer.import_python(
f.read(),
lib_dir=simulation_db.simulation_lib_dir(simulation_type),
tmp_dir=simulation_db.tmp_dir(),
user_filename=f.filename,
arguments=arguments,
)
if error:
return flask.jsonify({'error': error})
return _save_new_and_reply(simulation_type, data)
def read_zip(stream, sim_type=None):
"""Read zip file and store contents
Args:
stream (IO): file to read
sim_type (module): expected app
Returns:
dict: data
"""
from sirepo import simulation_db
import sirepo.sim_data
with simulation_db.tmp_dir() as tmp:
data = None
zipped = PKDict()
with zipfile.ZipFile(stream, 'r') as z:
for i in z.infolist():
b = pykern.pkio.py_path(i.filename).basename
c = z.read(i)
if b.lower() == simulation_db.SIMULATION_DATA_FILE:
assert not data, \
'too many db files {} in archive'.format(b)
data = read_json(c, sim_type)
continue
if '__MACOSX' in i.filename:
continue
#TODO(robnagler) ignore identical files hash
assert not b in zipped, \
'{} duplicate file in archive'.format(i.filename)
zipped[b] = tmp.join(b)
zipped[b].write(c, 'wb')
assert data, \
'missing {} in archive'.format(simulation_db.SIMULATION_DATA_FILE)
needed = set()
s = sirepo.sim_data.get_class(data.simulationType)
for n in s.lib_file_basenames(data):
#TODO(robnagler) this does not allow overwrites of lib files,
# but it needs to be modularized
if s.lib_file_exists(n):
continue
#TODO(robnagler) raise useralert instead of an assert
assert n in zipped, \
'auxiliary file={} missing in archive'.format(n)
needed.add(n)
for b, src in zipped.items():
if b in needed:
src.copy(s.lib_file_write_path(b))
return data
def get_data_file(run_dir, model, frame, **kwargs):
if model == 'dicomAnimation4':
filename = filename = _parent_file(run_dir, _DOSE_DICOM_FILE)
uri = RTDOSE_EXPORT_FILENAME
with open(filename, mode='rb') as f:
out = f.read()
else:
with simulation_db.tmp_dir() as tmp_dir:
filename, _ = _generate_rtstruct_file(_parent_dir(run_dir),
tmp_dir)
uri = RTSTRUCT_EXPORT_FILENAME
with open(filename, mode='rb') as f:
out = f.read()
return PKDict(
content=out,
uri=uri,
)
def api_importFile(simulation_type=None):
"""
Args:
simulation_type (str): which simulation type
Params:
file: file data
folder: where to import to
"""
import sirepo.importer
error = None
f = None
try:
template = simulation_type and sirepo.template.import_module(
simulation_type)
f = flask.request.files.get('file')
assert f, \
ValueError('must supply a file')
if pkio.has_file_extension(f.filename, 'json'):
data = sirepo.importer.read_json(f.read(), template)
#TODO(pjm): need a separate URI interface to importer, added exception for rs4pi for now
# (dicom input is normally a zip file)
elif pkio.has_file_extension(f.filename,
'zip') and simulation_type != 'rs4pi':
data = sirepo.importer.read_zip(f.stream, template)
else:
assert simulation_type, \
'simulation_type is required param for non-zip|json imports'
assert hasattr(template, 'import_file'), \
ValueError('Only zip files are supported')
data = template.import_file(
flask.request,
simulation_db.simulation_lib_dir(simulation_type),
simulation_db.tmp_dir(),
)
#TODO(robnagler) need to validate folder
data.models.simulation.folder = flask.request.form['folder']
data.models.simulation.isExample = False
return _save_new_and_reply(data)
except Exception as e:
pkdlog('{}: exception: {}', f and f.filename, pkdexc())
error = str(e.message) if hasattr(e, 'message') else str(e)
return http_reply.gen_json({
'error':
error if error else 'An unknown error occurred',
})
def api_getApplicationData(filename=None):
"""Get some data from the template
Args:
filename (str): if supplied, result is file attachment
Returns:
response: may be a file or JSON
"""
req = http_request.parse_post(template=True, filename=filename or None)
with simulation_db.tmp_dir() as d:
res = req.template.get_application_data(req.req_data, tmp_dir=d)
if 'filename' in req:
assert isinstance(res, pkconst.PY_PATH_LOCAL_TYPE), \
'{}: template did not return a file'.format(res)
return http_reply.gen_file_as_attachment(res,
filename=req.filename)
return http_reply.gen_json(res)
def api_getApplicationData(filename=None):
"""Get some data from the template
Args:
filename (str): if supplied, result is file attachment
Returns:
response: may be a file or JSON
"""
req = http_request.parse_post(template=True, filename=filename or None)
with simulation_db.tmp_dir() as d:
assert 'method' in req.req_data
res = req.template.get_application_data(req.req_data, tmp_dir=d)
assert res != None, f'unhandled application data method: {req.req_data.method}'
if 'filename' in req and isinstance(res, pkconst.PY_PATH_LOCAL_TYPE):
return http_reply.gen_file_as_attachment(
res,
filename=req.filename,
content_type=req.req_data.get('contentType', None))
return http_reply.gen_json(res)
def api_downloadDataFile(simulation_type,
simulation_id,
model,
frame,
suffix=None):
#TODO(robnagler) validate suffix and frame
req = sirepo.http_request.parse_params(
id=simulation_id,
model=model,
type=simulation_type,
check_sim_exists=True,
)
s = suffix and sirepo.srschema.parse_name(suffix)
t = None
with simulation_db.tmp_dir() as d:
# TODO(e-carlin): computeJobHash
t = sirepo.job.DATA_FILE_ROOT.join(sirepo.job.unique_key())
t.mksymlinkto(d, absolute=True)
try:
r = _request(
computeJobHash='unused',
dataFileKey=t.basename,
frame=int(frame),
isParallel=False,
req_data=req.req_data,
suffix=s,
)
assert not r.state == 'error', f'error state in request=={r}'
f = d.listdir()
if len(f) > 0:
assert len(f) == 1, \
'too many files={}'.format(f)
return sirepo.http_reply.gen_file_as_attachment(f[0])
except requests.exceptions.HTTPError:
#TODO(robnagler) HTTPError is too coarse a check
pass
finally:
if t:
pykern.pkio.unchecked_remove(t)
raise sirepo.util.raise_not_found(
'frame={} not found {id} {type}'.format(frame, **req))
def api_uploadFile(simulation_type, simulation_id, file_type):
f = flask.request.files['file']
req = http_request.parse_params(
file_type=file_type,
filename=f.filename,
id=simulation_id,
template=True,
type=simulation_type,
)
e = None
in_use = None
with simulation_db.tmp_dir() as d:
t = d.join(req.filename)
f.save(str(t))
if hasattr(req.template, 'validate_file'):
# Note: validate_file may modify the file
e = req.template.validate_file(req.file_type, t)
if (not e and req.sim_data.lib_file_exists(req.filename)
and not flask.request.form.get('confirm')):
in_use = _simulations_using_file(req, ignore_sim_id=req.id)
if in_use:
e = 'File is in use in other simulations. Please confirm you would like to replace the file for all simulations.'
if e:
return http_reply.gen_json({
'error': e,
'filename': req.filename,
'fileList': in_use,
'fileType': req.file_type,
'simulationId': req.id,
})
t.rename(_lib_file_write_path(req))
return http_reply.gen_json({
'filename': req.filename,
'fileType': req.file_type,
'simulationId': req.id,
})
def api_importFile(simulation_type=None):
"""
Args:
simulation_type (str): which simulation type
Params:
file: file data
folder: where to import to
"""
import sirepo.importer
error = None
f = None
try:
template = simulation_type and sirepo.template.import_module(simulation_type)
f = flask.request.files.get('file')
assert f, \
ValueError('must supply a file')
if pkio.has_file_extension(f.filename, 'json'):
data = sirepo.importer.read_json(f.read(), template)
elif pkio.has_file_extension(f.filename, 'zip'):
data = sirepo.importer.read_zip(f.stream, template)
else:
assert simulation_type, \
'simulation_type is required param for non-zip|json imports'
data = template.import_file(
flask.request,
simulation_db.simulation_lib_dir(simulation_type),
simulation_db.tmp_dir(),
)
#TODO(robnagler) need to validate folder
data.models.simulation.folder = flask.request.form['folder']
return _save_new_and_reply(data.simulationType, data)
except Exception as e:
pkdlog('{}: exception: {}', f and f.filename, pkdexc())
error = e.message if hasattr(e, 'message') else str(e)
return _json_response({'error': error})
def app_import_file(simulation_type):
template = sirepo.template.import_module(simulation_type)
error, data = template.import_file(flask.request, simulation_db.simulation_lib_dir(simulation_type), simulation_db.tmp_dir())
if error:
return _json_response({'error': error})
data['models']['simulation']['folder'] = flask.request.form['folder']
return _save_new_and_reply(simulation_type, data)
def app_import_file(simulation_type):
template = sirepo.template.import_module(simulation_type)
error, data = template.import_file(flask.request, simulation_db.simulation_lib_dir(simulation_type), simulation_db.tmp_dir())
if error:
return _json_response({'error': error})
data['models']['simulation']['folder'] = flask.request.form['folder']
return _save_new_and_reply(simulation_type, data)
