async def _fastcgi_op(self, msg):
if msg.runDir:
_assert_run_dir_exists(pkio.py_path(msg.runDir))
if not self.fastcgi_cmd:
m = msg.copy()
m.jobCmd = 'fastcgi'
self._fastcgi_file = cfg.fastcgi_sock_dir.join(
f'sirepo_job_cmd-{cfg.agent_id:8}.sock', )
self._fastcgi_msg_q = sirepo.tornado.Queue(1)
pkio.unchecked_remove(self._fastcgi_file)
m.fastcgiFile = self._fastcgi_file
# Runs in an agent's directory and chdirs to real runDirs.
# Except in stateless_compute which doesn't interact with the db.
m.runDir = pkio.py_path()
# Kind of backwards, but it makes sense since we need to listen
# so _do_fastcgi can connect
self._fastcgi_remove_handler = tornado.netutil.add_accept_handler(
tornado.netutil.bind_unix_socket(str(self._fastcgi_file)),
self._fastcgi_accept,
)
# last thing, because of await: start fastcgi process
await self._cmd(m, send_reply=False)
self._fastcgi_msg_q.put_nowait(msg)
self.fastcgi_cmd.op_id = msg.opId
return None
def app_upload_file(simulation_type, simulation_id, file_type):
f = flask.request.files['file']
lib = simulation_db.simulation_lib_dir(simulation_type)
template = sirepo.template.import_module(simulation_type)
filename = werkzeug.secure_filename(f.filename)
if simulation_type == 'srw':
p = lib.join(filename)
else:
p = lib.join(werkzeug.secure_filename('{}.{}'.format(file_type, filename)))
err = None
if p.check():
err = 'file exists: {}'.format(filename)
if not err:
f.save(str(p))
err = template.validate_file(file_type, str(p))
if err:
pkio.unchecked_remove(p)
if err:
return _json_response({
'error': err,
'filename': filename,
'fileType': file_type,
'simulationId': simulation_id,
})
return _json_response({
'filename': filename,
'fileType': file_type,
'simulationId': simulation_id,
})
def test_build_clean():
"""Create a normal distribution"""
with _project_dir('pksetupunit1') as d:
check_call(['python', 'setup.py', 'sdist', '--formats=zip'])
archive = _assert_members(
['pksetupunit1', 'package_data', 'data1'],
['scripts', 'script1'],
['examples', 'example1.txt'],
['tests', 'mod2_test.py'],
)
check_call(['python', 'setup.py', 'build'])
dat = os.path.join('build', 'lib', 'pksetupunit1', 'package_data', 'data1')
assert os.path.exists(dat), \
'When package_data, installed in lib'
bin_dir = 'scripts-{}.{}'.format(*(sys.version_info[0:2]))
check_call(['python', 'setup.py', 'test'])
assert os.path.exists('tests/mod2_test.py')
check_call(['git', 'clean', '-dfx'])
assert not os.path.exists('build'), \
'When git clean runs, build directory should not exist'
check_call(['python', 'setup.py', 'sdist'])
pkio.unchecked_remove(archive)
_assert_members(
['!', 'tests', 'mod2_work', 'do_not_include_in_sdist.py'],
['tests', 'mod2_test.py'],
)
#TODO(robnagler) need another sentinel here
if os.environ.get('PKSETUP_PKDEPLOY_IS_DEV', False):
check_call(['python', 'setup.py', 'pkdeploy'])
def test_build_clean():
"""Create a normal distribution"""
from pykern import pkio
from pykern import pksetup
from pykern import pkunit
with _project_dir('pksetupunit1') as d:
check_call(['python', 'setup.py', 'sdist', '--formats=zip'])
archive = _assert_members(
['pksetupunit1', 'package_data', 'data1'],
['scripts', 'script1'],
['examples', 'example1.txt'],
['tests', 'mod2_test.py'],
)
check_call(['python', 'setup.py', 'build'])
dat = os.path.join('build', 'lib', 'pksetupunit1', 'package_data',
'data1')
assert os.path.exists(dat), \
'When building, package_data should be installed in lib'
bin_dir = 'scripts-{}.{}'.format(*(sys.version_info[0:2]))
check_call(['python', 'setup.py', 'test'])
assert os.path.exists('tests/mod2_test.py')
check_call(['git', 'clean', '-dfx'])
assert not os.path.exists('build'), \
'When git clean runs, build directory should not exist'
check_call(['python', 'setup.py', 'sdist'])
pkio.unchecked_remove(archive)
_assert_members(
['!', 'tests', 'mod2_work', 'do_not_include_in_sdist.py'],
['tests', 'mod2_test.py'],
)
#TODO(robnagler) need another sentinel here
if os.environ.get('PKSETUP_PKDEPLOY_IS_DEV', False):
check_call(['python', 'setup.py', 'pkdeploy'])
def _write_vti_file(filename, info, origin=None):
vti = copy.deepcopy(_VTI_TEMPLATE)
vti.spacing = [
info.PixelSpacing[0],
info.PixelSpacing[1],
info.SliceThickness,
]
vti.extent = [
0,
info.Columns - 1,
0,
info.Rows - 1,
0,
info.Count - 1,
]
vti.metadata = PKDict(
name='ct.vti' if 'PatientPosition' in info else 'dose.vti', )
for f in ('RescaleSlope', 'RescaleIntercept', 'DoseMax',
'DoseGridScaling'):
if f in info:
vti.metadata[f] = info[f]
vti.origin = origin or info.ImagePositionPatient
vti.pointData.arrays[0].data.size = info.Rows * info.Columns * info.Count
#vti.pointData.arrays[0].data.dataType = 'Int{}Array'.format(info.BitsAllocated)
vti.pointData.arrays[0].data.dataType = 'Uint16Array'
pkio.unchecked_remove(filename)
with ZipFile(filename, 'w') as vti_zip:
vti_zip.writestr('index.json', json.dumps(vti))
vti_zip.write(_PIXEL_DATA_FILE)
#pkdp('vti json: {}', vti)
pkio.unchecked_remove(_PIXEL_DATA_DIR)
def test_unchecked_remove():
"""Also tests mkdir_parent"""
from pykern import pkunit
from pykern import pkio
with pkunit.save_chdir_work():
fn = 'f1'
# Should not throw an exception
pkio.unchecked_remove(fn)
pkio.write_text(fn, 'hello')
pkio.unchecked_remove(fn)
assert not os.path.exists(fn), \
'When file removed, should be gone'
for f in ('d1', 'd2/d3'):
assert py.path.local(f) == pkio.mkdir_parent(f), \
'When mkdir_parent is called, returns path passed in'
assert os.path.exists('d1'), \
'When single directory, should exist'
assert os.path.exists('d2/d3'), \
'When nested directory, should exist'
with pytest.raises(AssertionError):
pkio.unchecked_remove('.')
with pytest.raises(AssertionError):
pkio.unchecked_remove(os.getcwd())
with pytest.raises(AssertionError):
pkio.unchecked_remove('/')
def write_parameters(data, run_dir, is_parallel):
# remove centrailzed geom files
pkio.unchecked_remove(_geom_file(data.simulationId), _dmp_file(data.simulationId))
pkio.write_text(
run_dir.join(template_common.PARAMETERS_PYTHON_FILE),
_generate_parameters_file(data),
)
def pytest_runtest_protocol(item, *args, **kwargs):
import signal
from pykern import pkunit
def _timeout(*args, **kwargs):
signal.signal(signal.SIGALRM, _timeout_failed)
signal.alarm(1)
pkunit.pkfail('MAX_CASE_RUN_SECS={} exceeded', MAX_CASE_RUN_SECS)
def _timeout_failed(*args, **kwargs):
import os
import sys
from pykern.pkdebug import pkdlog
pkdlog('failed to die after timeout (pkfail)')
os.killpg(os.getpgrp(), signal.SIGKILL)
# Seems to be the only way to get the module under test
m = item._request.module
is_new = m != pkunit.module_under_test
if is_new:
signal.signal(signal.SIGALRM, _timeout)
pkunit.module_under_test = m
signal.alarm(MAX_CASE_RUN_SECS)
if is_new:
from pykern import pkio
pkio.unchecked_remove(pkunit.work_dir())
def api_uploadFile(simulation_type, simulation_id, file_type):
f = flask.request.files['file']
filename = werkzeug.secure_filename(f.filename)
p = _lib_filepath(simulation_type, filename, file_type)
err = None
file_list = None
if p.check():
confirm = flask.request.form['confirm'] if 'confirm' in flask.request.form else None
if not confirm:
search_name = _lib_filename(simulation_type, filename, file_type)
file_list = _simulations_using_file(simulation_type, file_type, search_name, ignore_sim_id=simulation_id)
if file_list:
err = 'File is in use in other simulations. Please confirm you would like to replace the file for all simulations.'
if not err:
pkio.mkdir_parent_only(p)
f.save(str(p))
template = sirepo.template.import_module(simulation_type)
if hasattr(template, 'validate_file'):
err = template.validate_file(file_type, str(p))
if err:
pkio.unchecked_remove(p)
if err:
return http_reply.gen_json({
'error': err,
'filename': filename,
'fileList': file_list,
'fileType': file_type,
'simulationId': simulation_id,
})
return http_reply.gen_json({
'filename': filename,
'fileType': file_type,
'simulationId': simulation_id,
})
def app_upload_file(simulation_type, simulation_id, file_type):
f = flask.request.files['file']
lib = simulation_db.simulation_lib_dir(simulation_type)
template = sirepo.template.import_module(simulation_type)
filename = werkzeug.secure_filename(f.filename)
if simulation_type == 'srw':
p = lib.join(filename)
else:
p = lib.join(werkzeug.secure_filename('{}.{}'.format(file_type, filename)))
err = None
if p.check():
err = 'file exists: {}'.format(filename)
if not err:
f.save(str(p))
err = template.validate_file(file_type, str(p))
if err:
pkio.unchecked_remove(p)
if err:
return _json_response({
'error': err,
'filename': filename,
'fileType': file_type,
'simulationId': simulation_id,
})
return _json_response({
'filename': filename,
'fileType': file_type,
'simulationId': simulation_id,
})
def purge_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
list: directories removed (or to remove if confirm)
"""
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
uids = auth_db.all_uids()
now = datetime.datetime.utcnow()
to_remove = []
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
if simulation_db.uid_from_dir_name(d) in uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
to_remove.append(d)
if confirm:
pkio.unchecked_remove(*to_remove)
return to_remove
def run_background(cfg_dir):
res = {}
data = simulation_db.read_json(template_common.INPUT_BASE_NAME)
distribution = data['models']['bunch']['distribution']
run_with_mpi = distribution == 'lattice' or distribution == 'file'
try:
with pkio.save_chdir(cfg_dir):
if run_with_mpi:
mpi.run_script(
pkio.read_text(template_common.PARAMETERS_PYTHON_FILE))
else:
#TODO(pjm): MPI doesn't work with rsbeams distributions yet
exec(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE),
locals(), locals())
except Exception as e:
res = {
'error': str(e),
}
if run_with_mpi and 'error' in res:
text = pkio.read_text('mpi_run.out')
m = re.search(r'^Traceback .*?^\w*Error: (.*?)\n\n', text,
re.MULTILINE | re.DOTALL)
if m:
res['error'] = m.group(1)
# remove output file - write_result() will not overwrite an existing error output
pkio.unchecked_remove(
simulation_db.json_filename(template_common.OUTPUT_BASE_NAME))
simulation_db.write_result(res)
def test_build_clean():
"""Create a normal distribution"""
with _project_dir('pksetupunit1') as d:
check_call(['python', 'setup.py', 'sdist', '--formats=zip'])
archive = _assert_members(
['pksetupunit1', 'package_data', 'data1'],
['scripts', 'script1'],
['examples', 'example1.txt'],
['tests', 'mod2_test.py'],
)
check_call(['python', 'setup.py', 'build'])
dat = os.path.join('build', 'lib', 'pksetupunit1', 'package_data', 'data1')
assert os.path.exists(dat), \
'When package_data, installed in lib'
bin_dir = 'scripts-{}.{}'.format(*(sys.version_info[0:2]))
check_call(['python', 'setup.py', 'test'])
assert os.path.exists('tests/mod2_test.py')
check_call(['git', 'clean', '-dfx'])
assert not os.path.exists('build'), \
'When git clean runs, build directory should not exist'
check_call(['python', 'setup.py', 'sdist'])
pkio.unchecked_remove(archive)
_assert_members(
['!', 'tests', 'mod2_work', 'do_not_include_in_sdist.py'],
['tests', 'mod2_test.py'],
)
async def start_report_job(self, run_dir, jhash, backend, cmd, tmp_dir):
# First make sure there's no-one else using the run_dir
current_jhash, current_status = self.run_dir_status(run_dir)
if current_status is runner_client.JobStatus.RUNNING:
# Something's running.
if current_jhash == jhash:
# It's already the requested job, so we have nothing to
# do. Throw away the tmp_dir and move on.
pkdlog(
'job is already running; skipping (run_dir={}, jhash={}, tmp_dir={})',
run_dir,
jhash,
tmp_dir,
)
pkio.unchecked_remove(tmp_dir)
return
else:
# It's some other job. Better kill it before doing
# anything else.
# XX TODO: should we check some kind of sequence number
# here? I don't know how those work.
pkdlog(
'stale job is still running; killing it (run_dir={}, jhash={})',
run_dir,
jhash,
)
await self.kill_all(run_dir)
# Okay, now we have the dir to ourselves. Set up the new run_dir:
assert run_dir not in self.report_jobs
pkio.unchecked_remove(run_dir)
tmp_dir.rename(run_dir)
# Start the job:
report_job = await _BACKENDS[backend].start_report_job(run_dir, cmd)
# And update our records so we know it's running:
job_info = _JobInfo(
run_dir,
jhash,
runner_client.JobStatus.RUNNING,
report_job,
)
self.report_jobs[run_dir] = job_info
pkjson.dump_pretty(
{
'version': 1,
'backend': backend,
'backend_info': report_job.backend_info,
},
filename=run_dir.join(_RUNNER_INFO_BASENAME),
)
# And finally, start a background task to watch over it.
self._nursery.start_soon(
self._supervise_report_job,
run_dir,
jhash,
job_info,
)
def _gen_req(which, basename, domains, is_ca=False):
first = domains[0]
if not basename:
basename = first
basename = pkio.py_path(basename)
if is_ca:
# pathlen:0 means can only be used for signing certs, not for
# signing intermediate certs.
alt = '''x509_extensions = x509_req
[x509_req]
basicConstraints=critical,CA:true,pathlen:0'''
else:
# Must always provide subjectAltName except for is_ca
# see https://github.com/urllib3/urllib3/issues/497
# which points to RFC 2818:
# Although the use of the Common Name is existing practice, it is deprecated and
# Certification Authorities are encouraged to use the dNSName instead.
alt = '{}_extensions = v3_req\n[v3_req]\n{}'.format(
'req' if which == 'csr' else 'x509',
_alt_names(domains),
)
c = '''
[req]
distinguished_name = subj
prompt = no
{}
[subj]
C = US
ST = Colorado
L = Boulder
CN = {}'''.format(alt, first)
cfg = basename + '.cfg'
try:
cfg.write(c)
key = basename + KEY_EXT
out = basename + '.' + which
cmd = [
'openssl',
'req',
'-nodes',
'-newkey',
'rsa:2048',
'-keyout',
str(key),
'-out',
str(out),
'-config',
str(cfg),
]
if which == _CRT:
cmd += ['-x509'] + _signing_args()
_run(cmd)
finally:
pkio.unchecked_remove(cfg)
res = pkcollections.Dict(key=key)
res[which] = out
return res
def destroy(self):
self._terminating = True
if '_in_file' in self:
pkio.unchecked_remove(self.pkdel('_in_file'))
self._process.kill()
try:
self.cmds.remove(cmd)
except Exception:
pass
def destroy(self):
self._terminating = True
if '_in_file' in self:
pkio.unchecked_remove(self.pkdel('_in_file'))
self._process.kill()
try:
self.dispatcher.cmds.remove(self)
except ValueError:
pass
def tmp_dir():
"""Generates new, temporary directory
Returns:
py.path: directory to use for temporary work
"""
d = _random_id(_user_dir().join(_TMP_DIR))['path']
pkio.unchecked_remove(d)
return pkio.mkdir_parent(d)
def tmp_dir():
"""Generates new, temporary directory
Returns:
py.path: directory to use for temporary work
"""
d = _random_id(_user_dir().join(_TMP_DIR))['path']
pkio.unchecked_remove(d)
return pkio.mkdir_parent(d)
def _backup_db_and_prevent_upgrade_on_error():
b = sirepo.auth_db.db_filename() + '.bak'
sirepo.auth_db.db_filename().copy(b)
try:
yield
pkio.unchecked_remove(b)
except Exception:
pkdlog('original db={}', b)
_prevent_db_upgrade_file().ensure()
raise
def _clone(suffix):
base = bd + suffix
for cmd in [
['git', 'clone', '--quiet', '--mirror',
_GITHUB_URI + '/' + fn + suffix,
base],
['tar', 'cJf', base + '.txz', base],
]:
_shell(cmd)
pkio.unchecked_remove(base)
def get_data_file(run_dir, model, frame, **kwargs):
if model == 'dicomAnimation4':
with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f:
return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream'
tmp_dir = simulation_db.tmp_dir()
filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir)
with open (filename, 'rb') as f:
dicom_data = f.read()
pkio.unchecked_remove(tmp_dir)
return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def get_data_file(run_dir, model, frame, **kwargs):
if model == 'dicomAnimation4':
with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f:
return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream'
tmp_dir = simulation_db.tmp_dir()
filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir)
with open(filename, 'rb') as f:
dicom_data = f.read()
pkio.unchecked_remove(tmp_dir)
return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def _clone(suffix):
base = bd + suffix
for cmd in [
[
'git', 'clone', '--quiet', '--mirror',
_GITHUB_URI + '/' + fn + suffix, base
],
['tar', 'cJf', base + '.txz', base],
]:
_shell(cmd)
pkio.unchecked_remove(base)
def _remove_old_tmp_dirs():
pkdlog('scanning for stale tmp dirs')
count = 0
cutoff = time.time() - srdb.TMP_DIR_CLEANUP_TIME
for dirpath, dirnames, filenames in os.walk(srdb.root()):
if (dirpath.endswith(srdb.TMP_DIR_SUFFIX)
and os.stat(dirpath).st_mtime < cutoff):
pkdlog('removing stale tmp dir: {}', dirpath)
pkio.unchecked_remove(dirpath)
count += 1
pkdlog('finished scanning for stale tmp dirs ({} found)', count)
def test_remove_srw_report_dir(fc):
from pykern import pkio
from pykern import pkunit
import sirepo.srdb
m = 'intensityReport'
data = fc.sr_sim_data('NSLS-II ESM beamline')
fc.sr_run_sim(data, m)
g = pkio.sorted_glob(sirepo.srdb.root().join('user', fc.sr_uid, 'srw', '*', m))
pkunit.pkeq(1, len(g))
pkio.unchecked_remove(*g)
fc.sr_run_sim(data, m)
def _move_import_file(data):
sim = data['models']['simulation']
path = sim[_TMP_INPUT_FILE_FIELD]
del sim[_TMP_INPUT_FILE_FIELD]
if os.path.exists(path):
zip_path = _sim_file(sim['simulationId'], _ZIP_FILE_NAME)
os.rename(path, zip_path)
tmp_dir = _sim_file(sim['simulationId'], _TMP_ZIP_DIR)
zipfile.ZipFile(zip_path).extractall(tmp_dir)
_summarize_dicom_files(data, tmp_dir)
pkio.unchecked_remove(tmp_dir)
simulation_db.save_simulation_json(data)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.run_dir = pkio.py_path(self.msg.runDir)
self._is_compute = self.msg.jobCmd == 'compute'
if self._is_compute:
pkio.unchecked_remove(self.run_dir)
pkio.mkdir_parent(self.run_dir)
self._in_file = self._create_in_file()
self._process = _Process(self)
self._terminating = False
self._start_time = int(time.time())
self.jid = self.msg.computeJid
def _do_sbatch_status(msg, template):
s = pkio.py_path(msg.stopSentinel)
while True:
if s.exists():
if job.COMPLETED not in s.read():
# told to stop for an error or otherwise
return None
_write_parallel_status(msg, template, False)
pkio.unchecked_remove(s)
return PKDict(state=job.COMPLETED)
_write_parallel_status(msg, template, True)
time.sleep(msg.nextRequestSeconds)
def api_deleteFile():
req = http_request.parse_post(filename=True, file_type=True)
e = _simulations_using_file(req)
if len(e):
return http_reply.gen_json({
'error': 'File is in use in other simulations.',
'fileList': e,
'fileName': req.filename,
})
# Will not remove resource (standard) lib files
pkio.unchecked_remove(_lib_file_write_path(req))
return http_reply.gen_json_ok()
def _move_import_file(data):
sim = data['models']['simulation']
path = sim[_TMP_INPUT_FILE_FIELD]
del sim[_TMP_INPUT_FILE_FIELD]
if os.path.exists(path):
zip_path = _sim_file(sim['simulationId'], _ZIP_FILE_NAME)
os.rename(path, zip_path)
pkio.unchecked_remove(os.path.dirname(path))
tmp_dir = _sim_file(sim['simulationId'], _TMP_ZIP_DIR)
zipfile.ZipFile(zip_path).extractall(tmp_dir)
_summarize_dicom_files(data, tmp_dir)
pkio.unchecked_remove(tmp_dir)
simulation_db.save_simulation_json(data)
def simulation_run_dir(data, remove_dir=False):
"""Where to run the simulation
Args:
data (dict): contains simulationType and simulationId
remove_dir (bool): remove the directory [False]
Returns:
py.path: directory to run
"""
d = simulation_dir(data['simulationType'], parse_sid(data)).join(_report_dir(data))
if remove_dir:
pkio.unchecked_remove(d)
return d
def simulation_run_dir(data, remove_dir=False):
"""Where to run the simulation
Args:
data (dict): contains simulationType and simulationId
remove_dir (bool): remove the directory [False]
Returns:
py.path: directory to run
"""
d = simulation_dir(data['simulationType'], parse_sid(data)).join(_report_dir(data))
if remove_dir:
pkio.unchecked_remove(d)
return d
def host_db(self, channel, host):
res = Host()
v = PKDict(
rsconf_db=PKDict(
# Common defaults we allow overrides for
host_run_d='/srv',
run_u='vagrant',
root_u='root',
)
)
merge_dict(res, v)
#TODO(robnagler) optimize by caching default and channels
for l in LEVELS:
v = self.base[l]
if l != LEVELS[0]:
v = v.get(channel)
if not v:
continue
if l == LEVELS[2]:
v = v.get(host)
if not v:
continue
merge_dict(res, v)
host = host.lower()
v = PKDict(
rsconf_db=PKDict(
channel=channel,
db_d=self.db_d,
host=host,
proprietary_source_d=self.proprietary_source_d,
rpm_source_d=self.rpm_source_d,
secret_d=self.secret_d,
srv_d=self.srv_d,
srv_host_d=self.srv_host_d,
tmp_d=self.tmp_d.join(host),
# https://jnovy.fedorapeople.org/pxz/node1.html
# compression with 8 threads and max compression
# Useful (random) constants
compress_cmd='pxz -T8 -9',
)
)
v.rsconf_db.resource_paths = _init_resource_paths(v)
v.rsconf_db.local_files = _init_local_files(v)
pkio.unchecked_remove(v.rsconf_db.tmp_d)
pkio.mkdir_parent(v.rsconf_db.tmp_d)
merge_dict(res, v)
_assert_no_rsconf_db_values(res)
_update_paths(res)
pkdc('{}', res)
return res
def _purge_sim(db_file):
d = pkcollections.json_load_any(db_file)
# OPTIMIZATION: We assume the uids_of_paid_users doesn't change very
# frequently so we don't need to check again. A user could run a sim
# at anytime so we need to check that they haven't
if d.lastUpdateTime > _too_old:
return
if d.status == job.FREE_USER_PURGED:
return
p = sirepo.simulation_db.simulation_run_dir(d)
pkio.unchecked_remove(p)
d.status = job.FREE_USER_PURGED
cls.__db_write_file(d)
jids_purged.append(db_file.purebasename)
def api_deleteFile():
req = http_request.parse_json()
filename = werkzeug.secure_filename(req['fileName'])
search_name = _lib_filename(req['simulationType'], filename, req['fileType'])
err = _simulations_using_file(req['simulationType'], req['fileType'], search_name)
if len(err):
return http_reply.gen_json({
'error': 'File is in use in other simulations.',
'fileList': err,
'fileName': filename,
})
p = _lib_filepath(req['simulationType'], filename, req['fileType'])
pkio.unchecked_remove(p)
return http_reply.gen_json({})
def _link_or_unlink_proprietary_files(sim_type, should_link):
d = proprietary_code_dir(sim_type)
for e in simulation_db.examples(sim_type):
b = sim_data.get_class(sim_type).proprietary_lib_file_basename(e)
p = simulation_db.simulation_lib_dir(sim_type).join(b)
if not should_link:
pkio.unchecked_remove(p)
continue
try:
p.mksymlinkto(
d.join(b),
absolute=False,
)
except py.error.EEXIST:
pass
def pytest_runtest_protocol(item, *args, **kwargs):
"""Make sure work directory is empty for a module.
If `item` is in a module not seen before, it removes
the `pkunit.work_dir`.
Args:
item (Item): pytest test item (case)
Returns:
None: always so that the next hook runs the item.
"""
if not _uses_pykern:
return
from pykern import pkunit
# Seems to be the only way to get the module under test
m = item._request.module
is_new = m != pkunit.module_under_test
pkunit.module_under_test = m
if is_new:
from pykern import pkio
pkio.unchecked_remove(pkunit.work_dir())
def app_upload_file(simulation_type, simulation_id):
f = flask.request.files['file']
lib = simulation_db.simulation_lib_dir(simulation_type)
filename = werkzeug.secure_filename(f.filename)
p = lib.join(filename)
err = None
if p.check():
err = 'file exists: {}'.format(filename)
if not err:
f.save(str(p))
err = _validate_data_file(p)
if err:
pkio.unchecked_remove(p)
if err:
return flask.jsonify({
'error': err,
'filename': filename,
'simulationId': simulation_id,
})
return flask.jsonify({
'filename': filename,
'simulationId': simulation_id,
})
def purge_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
list: directories removed (or to remove if confirm)
"""
from pykern import pkio
from sirepo import server
from sirepo import simulation_db
from sirepo import api_auth
import datetime
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
uids = api_auth.all_uids()
now = datetime.datetime.utcnow()
to_remove = []
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
#TODO(pjm): need to skip special "src" user
if simulation_db.uid_from_dir_name(d) in uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
to_remove.append(d)
if confirm:
pkio.unchecked_remove(*to_remove)
return to_remove
def run_background(cfg_dir):
res = {}
data = simulation_db.read_json(template_common.INPUT_BASE_NAME)
distribution = data['models']['bunch']['distribution']
run_with_mpi = distribution == 'lattice' or distribution == 'file'
try:
with pkio.save_chdir(cfg_dir):
if run_with_mpi:
mpi.run_script(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE))
else:
#TODO(pjm): MPI doesn't work with rsbeams distributions yet
exec(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE), locals(), locals())
except Exception as e:
res = {
'error': str(e),
}
if run_with_mpi and 'error' in res:
text = pkio.read_text('mpi_run.out')
m = re.search(r'^Traceback .*?^\w*Error: (.*?)\n\n', text, re.MULTILINE|re.DOTALL)
if m:
res['error'] = m.group(1)
# remove output file - write_result() will not overwrite an existing error output
pkio.unchecked_remove(simulation_db.json_filename(template_common.OUTPUT_BASE_NAME))
simulation_db.write_result(res)
def remove_last_frame(run_dir):
for m in ('currentAnimation', 'fieldAnimation'):
files = _h5_file_list(run_dir, m)
if len(files) > 0:
pkio.unchecked_remove(files[-1])
def remove_last_frame(run_dir):
files = _h5_file_list(run_dir)
if len(files) > 0:
pkio.unchecked_remove(files[-1])
def delete_simulation(simulation_type, sid):
"""Deletes the simulation's directory.
"""
pkio.unchecked_remove(simulation_dir(simulation_type, sid))
def app_delete_simulation():
data = _json_input()
pkio.unchecked_remove(simulation_db.simulation_dir(data['simulationType'], data['simulationId']))
return '{}'
def _purge(self):
expires = datetime.datetime.utcnow() - cfg.keep_days
for d in pkio.sorted_glob('[0-9]' * len(self._date_d)):
t = datetime.datetime.utcfromtimestamp(d.stat().mtime)
if t < expires:
pkio.unchecked_remove(d)
