async def _fastcgi_op(self, msg): if msg.runDir: _assert_run_dir_exists(pkio.py_path(msg.runDir)) if not self.fastcgi_cmd: m = msg.copy() m.jobCmd = 'fastcgi' self._fastcgi_file = cfg.fastcgi_sock_dir.join( f'sirepo_job_cmd-{cfg.agent_id:8}.sock', ) self._fastcgi_msg_q = sirepo.tornado.Queue(1) pkio.unchecked_remove(self._fastcgi_file) m.fastcgiFile = self._fastcgi_file # Runs in an agent's directory and chdirs to real runDirs. # Except in stateless_compute which doesn't interact with the db. m.runDir = pkio.py_path() # Kind of backwards, but it makes sense since we need to listen # so _do_fastcgi can connect self._fastcgi_remove_handler = tornado.netutil.add_accept_handler( tornado.netutil.bind_unix_socket(str(self._fastcgi_file)), self._fastcgi_accept, ) # last thing, because of await: start fastcgi process await self._cmd(m, send_reply=False) self._fastcgi_msg_q.put_nowait(msg) self.fastcgi_cmd.op_id = msg.opId return None
def app_upload_file(simulation_type, simulation_id, file_type): f = flask.request.files['file'] lib = simulation_db.simulation_lib_dir(simulation_type) template = sirepo.template.import_module(simulation_type) filename = werkzeug.secure_filename(f.filename) if simulation_type == 'srw': p = lib.join(filename) else: p = lib.join(werkzeug.secure_filename('{}.{}'.format(file_type, filename))) err = None if p.check(): err = 'file exists: {}'.format(filename) if not err: f.save(str(p)) err = template.validate_file(file_type, str(p)) if err: pkio.unchecked_remove(p) if err: return _json_response({ 'error': err, 'filename': filename, 'fileType': file_type, 'simulationId': simulation_id, }) return _json_response({ 'filename': filename, 'fileType': file_type, 'simulationId': simulation_id, })
def test_build_clean(): """Create a normal distribution""" with _project_dir('pksetupunit1') as d: check_call(['python', 'setup.py', 'sdist', '--formats=zip']) archive = _assert_members( ['pksetupunit1', 'package_data', 'data1'], ['scripts', 'script1'], ['examples', 'example1.txt'], ['tests', 'mod2_test.py'], ) check_call(['python', 'setup.py', 'build']) dat = os.path.join('build', 'lib', 'pksetupunit1', 'package_data', 'data1') assert os.path.exists(dat), \ 'When package_data, installed in lib' bin_dir = 'scripts-{}.{}'.format(*(sys.version_info[0:2])) check_call(['python', 'setup.py', 'test']) assert os.path.exists('tests/mod2_test.py') check_call(['git', 'clean', '-dfx']) assert not os.path.exists('build'), \ 'When git clean runs, build directory should not exist' check_call(['python', 'setup.py', 'sdist']) pkio.unchecked_remove(archive) _assert_members( ['!', 'tests', 'mod2_work', 'do_not_include_in_sdist.py'], ['tests', 'mod2_test.py'], ) #TODO(robnagler) need another sentinel here if os.environ.get('PKSETUP_PKDEPLOY_IS_DEV', False): check_call(['python', 'setup.py', 'pkdeploy'])
def test_build_clean(): """Create a normal distribution""" from pykern import pkio from pykern import pksetup from pykern import pkunit with _project_dir('pksetupunit1') as d: check_call(['python', 'setup.py', 'sdist', '--formats=zip']) archive = _assert_members( ['pksetupunit1', 'package_data', 'data1'], ['scripts', 'script1'], ['examples', 'example1.txt'], ['tests', 'mod2_test.py'], ) check_call(['python', 'setup.py', 'build']) dat = os.path.join('build', 'lib', 'pksetupunit1', 'package_data', 'data1') assert os.path.exists(dat), \ 'When building, package_data should be installed in lib' bin_dir = 'scripts-{}.{}'.format(*(sys.version_info[0:2])) check_call(['python', 'setup.py', 'test']) assert os.path.exists('tests/mod2_test.py') check_call(['git', 'clean', '-dfx']) assert not os.path.exists('build'), \ 'When git clean runs, build directory should not exist' check_call(['python', 'setup.py', 'sdist']) pkio.unchecked_remove(archive) _assert_members( ['!', 'tests', 'mod2_work', 'do_not_include_in_sdist.py'], ['tests', 'mod2_test.py'], ) #TODO(robnagler) need another sentinel here if os.environ.get('PKSETUP_PKDEPLOY_IS_DEV', False): check_call(['python', 'setup.py', 'pkdeploy'])
def _write_vti_file(filename, info, origin=None): vti = copy.deepcopy(_VTI_TEMPLATE) vti.spacing = [ info.PixelSpacing[0], info.PixelSpacing[1], info.SliceThickness, ] vti.extent = [ 0, info.Columns - 1, 0, info.Rows - 1, 0, info.Count - 1, ] vti.metadata = PKDict( name='ct.vti' if 'PatientPosition' in info else 'dose.vti', ) for f in ('RescaleSlope', 'RescaleIntercept', 'DoseMax', 'DoseGridScaling'): if f in info: vti.metadata[f] = info[f] vti.origin = origin or info.ImagePositionPatient vti.pointData.arrays[0].data.size = info.Rows * info.Columns * info.Count #vti.pointData.arrays[0].data.dataType = 'Int{}Array'.format(info.BitsAllocated) vti.pointData.arrays[0].data.dataType = 'Uint16Array' pkio.unchecked_remove(filename) with ZipFile(filename, 'w') as vti_zip: vti_zip.writestr('index.json', json.dumps(vti)) vti_zip.write(_PIXEL_DATA_FILE) #pkdp('vti json: {}', vti) pkio.unchecked_remove(_PIXEL_DATA_DIR)
def test_unchecked_remove(): """Also tests mkdir_parent""" from pykern import pkunit from pykern import pkio with pkunit.save_chdir_work(): fn = 'f1' # Should not throw an exception pkio.unchecked_remove(fn) pkio.write_text(fn, 'hello') pkio.unchecked_remove(fn) assert not os.path.exists(fn), \ 'When file removed, should be gone' for f in ('d1', 'd2/d3'): assert py.path.local(f) == pkio.mkdir_parent(f), \ 'When mkdir_parent is called, returns path passed in' assert os.path.exists('d1'), \ 'When single directory, should exist' assert os.path.exists('d2/d3'), \ 'When nested directory, should exist' with pytest.raises(AssertionError): pkio.unchecked_remove('.') with pytest.raises(AssertionError): pkio.unchecked_remove(os.getcwd()) with pytest.raises(AssertionError): pkio.unchecked_remove('/')
def write_parameters(data, run_dir, is_parallel): # remove centrailzed geom files pkio.unchecked_remove(_geom_file(data.simulationId), _dmp_file(data.simulationId)) pkio.write_text( run_dir.join(template_common.PARAMETERS_PYTHON_FILE), _generate_parameters_file(data), )
def pytest_runtest_protocol(item, *args, **kwargs): import signal from pykern import pkunit def _timeout(*args, **kwargs): signal.signal(signal.SIGALRM, _timeout_failed) signal.alarm(1) pkunit.pkfail('MAX_CASE_RUN_SECS={} exceeded', MAX_CASE_RUN_SECS) def _timeout_failed(*args, **kwargs): import os import sys from pykern.pkdebug import pkdlog pkdlog('failed to die after timeout (pkfail)') os.killpg(os.getpgrp(), signal.SIGKILL) # Seems to be the only way to get the module under test m = item._request.module is_new = m != pkunit.module_under_test if is_new: signal.signal(signal.SIGALRM, _timeout) pkunit.module_under_test = m signal.alarm(MAX_CASE_RUN_SECS) if is_new: from pykern import pkio pkio.unchecked_remove(pkunit.work_dir())
def api_uploadFile(simulation_type, simulation_id, file_type): f = flask.request.files['file'] filename = werkzeug.secure_filename(f.filename) p = _lib_filepath(simulation_type, filename, file_type) err = None file_list = None if p.check(): confirm = flask.request.form['confirm'] if 'confirm' in flask.request.form else None if not confirm: search_name = _lib_filename(simulation_type, filename, file_type) file_list = _simulations_using_file(simulation_type, file_type, search_name, ignore_sim_id=simulation_id) if file_list: err = 'File is in use in other simulations. Please confirm you would like to replace the file for all simulations.' if not err: pkio.mkdir_parent_only(p) f.save(str(p)) template = sirepo.template.import_module(simulation_type) if hasattr(template, 'validate_file'): err = template.validate_file(file_type, str(p)) if err: pkio.unchecked_remove(p) if err: return http_reply.gen_json({ 'error': err, 'filename': filename, 'fileList': file_list, 'fileType': file_type, 'simulationId': simulation_id, }) return http_reply.gen_json({ 'filename': filename, 'fileType': file_type, 'simulationId': simulation_id, })
def purge_users(days=180, confirm=False): """Remove old users from db which have not registered. Args: days (int): maximum days of untouched files (old is mtime > days) confirm (bool): delete the directories if True (else don't delete) [False] Returns: list: directories removed (or to remove if confirm) """ days = int(days) assert days >= 1, \ '{}: days must be a positive integer' server.init() uids = auth_db.all_uids() now = datetime.datetime.utcnow() to_remove = [] for d in pkio.sorted_glob(simulation_db.user_dir_name('*')): if _is_src_dir(d): continue; if simulation_db.uid_from_dir_name(d) in uids: continue for f in pkio.walk_tree(d): if (now - now.fromtimestamp(f.mtime())).days <= days: break else: to_remove.append(d) if confirm: pkio.unchecked_remove(*to_remove) return to_remove
def run_background(cfg_dir): res = {} data = simulation_db.read_json(template_common.INPUT_BASE_NAME) distribution = data['models']['bunch']['distribution'] run_with_mpi = distribution == 'lattice' or distribution == 'file' try: with pkio.save_chdir(cfg_dir): if run_with_mpi: mpi.run_script( pkio.read_text(template_common.PARAMETERS_PYTHON_FILE)) else: #TODO(pjm): MPI doesn't work with rsbeams distributions yet exec(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE), locals(), locals()) except Exception as e: res = { 'error': str(e), } if run_with_mpi and 'error' in res: text = pkio.read_text('mpi_run.out') m = re.search(r'^Traceback .*?^\w*Error: (.*?)\n\n', text, re.MULTILINE | re.DOTALL) if m: res['error'] = m.group(1) # remove output file - write_result() will not overwrite an existing error output pkio.unchecked_remove( simulation_db.json_filename(template_common.OUTPUT_BASE_NAME)) simulation_db.write_result(res)
def test_build_clean(): """Create a normal distribution""" with _project_dir('pksetupunit1') as d: check_call(['python', 'setup.py', 'sdist', '--formats=zip']) archive = _assert_members( ['pksetupunit1', 'package_data', 'data1'], ['scripts', 'script1'], ['examples', 'example1.txt'], ['tests', 'mod2_test.py'], ) check_call(['python', 'setup.py', 'build']) dat = os.path.join('build', 'lib', 'pksetupunit1', 'package_data', 'data1') assert os.path.exists(dat), \ 'When package_data, installed in lib' bin_dir = 'scripts-{}.{}'.format(*(sys.version_info[0:2])) check_call(['python', 'setup.py', 'test']) assert os.path.exists('tests/mod2_test.py') check_call(['git', 'clean', '-dfx']) assert not os.path.exists('build'), \ 'When git clean runs, build directory should not exist' check_call(['python', 'setup.py', 'sdist']) pkio.unchecked_remove(archive) _assert_members( ['!', 'tests', 'mod2_work', 'do_not_include_in_sdist.py'], ['tests', 'mod2_test.py'], )
async def start_report_job(self, run_dir, jhash, backend, cmd, tmp_dir): # First make sure there's no-one else using the run_dir current_jhash, current_status = self.run_dir_status(run_dir) if current_status is runner_client.JobStatus.RUNNING: # Something's running. if current_jhash == jhash: # It's already the requested job, so we have nothing to # do. Throw away the tmp_dir and move on. pkdlog( 'job is already running; skipping (run_dir={}, jhash={}, tmp_dir={})', run_dir, jhash, tmp_dir, ) pkio.unchecked_remove(tmp_dir) return else: # It's some other job. Better kill it before doing # anything else. # XX TODO: should we check some kind of sequence number # here? I don't know how those work. pkdlog( 'stale job is still running; killing it (run_dir={}, jhash={})', run_dir, jhash, ) await self.kill_all(run_dir) # Okay, now we have the dir to ourselves. Set up the new run_dir: assert run_dir not in self.report_jobs pkio.unchecked_remove(run_dir) tmp_dir.rename(run_dir) # Start the job: report_job = await _BACKENDS[backend].start_report_job(run_dir, cmd) # And update our records so we know it's running: job_info = _JobInfo( run_dir, jhash, runner_client.JobStatus.RUNNING, report_job, ) self.report_jobs[run_dir] = job_info pkjson.dump_pretty( { 'version': 1, 'backend': backend, 'backend_info': report_job.backend_info, }, filename=run_dir.join(_RUNNER_INFO_BASENAME), ) # And finally, start a background task to watch over it. self._nursery.start_soon( self._supervise_report_job, run_dir, jhash, job_info, )
def _gen_req(which, basename, domains, is_ca=False): first = domains[0] if not basename: basename = first basename = pkio.py_path(basename) if is_ca: # pathlen:0 means can only be used for signing certs, not for # signing intermediate certs. alt = '''x509_extensions = x509_req [x509_req] basicConstraints=critical,CA:true,pathlen:0''' else: # Must always provide subjectAltName except for is_ca # see https://github.com/urllib3/urllib3/issues/497 # which points to RFC 2818: # Although the use of the Common Name is existing practice, it is deprecated and # Certification Authorities are encouraged to use the dNSName instead. alt = '{}_extensions = v3_req\n[v3_req]\n{}'.format( 'req' if which == 'csr' else 'x509', _alt_names(domains), ) c = ''' [req] distinguished_name = subj prompt = no {} [subj] C = US ST = Colorado L = Boulder CN = {}'''.format(alt, first) cfg = basename + '.cfg' try: cfg.write(c) key = basename + KEY_EXT out = basename + '.' + which cmd = [ 'openssl', 'req', '-nodes', '-newkey', 'rsa:2048', '-keyout', str(key), '-out', str(out), '-config', str(cfg), ] if which == _CRT: cmd += ['-x509'] + _signing_args() _run(cmd) finally: pkio.unchecked_remove(cfg) res = pkcollections.Dict(key=key) res[which] = out return res
def destroy(self): self._terminating = True if '_in_file' in self: pkio.unchecked_remove(self.pkdel('_in_file')) self._process.kill() try: self.cmds.remove(cmd) except Exception: pass
def destroy(self): self._terminating = True if '_in_file' in self: pkio.unchecked_remove(self.pkdel('_in_file')) self._process.kill() try: self.dispatcher.cmds.remove(self) except ValueError: pass
def tmp_dir(): """Generates new, temporary directory Returns: py.path: directory to use for temporary work """ d = _random_id(_user_dir().join(_TMP_DIR))['path'] pkio.unchecked_remove(d) return pkio.mkdir_parent(d)
def _backup_db_and_prevent_upgrade_on_error(): b = sirepo.auth_db.db_filename() + '.bak' sirepo.auth_db.db_filename().copy(b) try: yield pkio.unchecked_remove(b) except Exception: pkdlog('original db={}', b) _prevent_db_upgrade_file().ensure() raise
def _clone(suffix): base = bd + suffix for cmd in [ ['git', 'clone', '--quiet', '--mirror', _GITHUB_URI + '/' + fn + suffix, base], ['tar', 'cJf', base + '.txz', base], ]: _shell(cmd) pkio.unchecked_remove(base)
def get_data_file(run_dir, model, frame, **kwargs): if model == 'dicomAnimation4': with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f: return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream' tmp_dir = simulation_db.tmp_dir() filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir) with open (filename, 'rb') as f: dicom_data = f.read() pkio.unchecked_remove(tmp_dir) return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def get_data_file(run_dir, model, frame, **kwargs): if model == 'dicomAnimation4': with open(_parent_file(run_dir, _DOSE_DICOM_FILE)) as f: return RTDOSE_EXPORT_FILENAME, f.read(), 'application/octet-stream' tmp_dir = simulation_db.tmp_dir() filename, _ = _generate_rtstruct_file(_parent_dir(run_dir), tmp_dir) with open(filename, 'rb') as f: dicom_data = f.read() pkio.unchecked_remove(tmp_dir) return RTSTRUCT_EXPORT_FILENAME, dicom_data, 'application/octet-stream'
def _clone(suffix): base = bd + suffix for cmd in [ [ 'git', 'clone', '--quiet', '--mirror', _GITHUB_URI + '/' + fn + suffix, base ], ['tar', 'cJf', base + '.txz', base], ]: _shell(cmd) pkio.unchecked_remove(base)
def _remove_old_tmp_dirs(): pkdlog('scanning for stale tmp dirs') count = 0 cutoff = time.time() - srdb.TMP_DIR_CLEANUP_TIME for dirpath, dirnames, filenames in os.walk(srdb.root()): if (dirpath.endswith(srdb.TMP_DIR_SUFFIX) and os.stat(dirpath).st_mtime < cutoff): pkdlog('removing stale tmp dir: {}', dirpath) pkio.unchecked_remove(dirpath) count += 1 pkdlog('finished scanning for stale tmp dirs ({} found)', count)
def test_remove_srw_report_dir(fc): from pykern import pkio from pykern import pkunit import sirepo.srdb m = 'intensityReport' data = fc.sr_sim_data('NSLS-II ESM beamline') fc.sr_run_sim(data, m) g = pkio.sorted_glob(sirepo.srdb.root().join('user', fc.sr_uid, 'srw', '*', m)) pkunit.pkeq(1, len(g)) pkio.unchecked_remove(*g) fc.sr_run_sim(data, m)
def _move_import_file(data): sim = data['models']['simulation'] path = sim[_TMP_INPUT_FILE_FIELD] del sim[_TMP_INPUT_FILE_FIELD] if os.path.exists(path): zip_path = _sim_file(sim['simulationId'], _ZIP_FILE_NAME) os.rename(path, zip_path) tmp_dir = _sim_file(sim['simulationId'], _TMP_ZIP_DIR) zipfile.ZipFile(zip_path).extractall(tmp_dir) _summarize_dicom_files(data, tmp_dir) pkio.unchecked_remove(tmp_dir) simulation_db.save_simulation_json(data)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.run_dir = pkio.py_path(self.msg.runDir) self._is_compute = self.msg.jobCmd == 'compute' if self._is_compute: pkio.unchecked_remove(self.run_dir) pkio.mkdir_parent(self.run_dir) self._in_file = self._create_in_file() self._process = _Process(self) self._terminating = False self._start_time = int(time.time()) self.jid = self.msg.computeJid
def _do_sbatch_status(msg, template): s = pkio.py_path(msg.stopSentinel) while True: if s.exists(): if job.COMPLETED not in s.read(): # told to stop for an error or otherwise return None _write_parallel_status(msg, template, False) pkio.unchecked_remove(s) return PKDict(state=job.COMPLETED) _write_parallel_status(msg, template, True) time.sleep(msg.nextRequestSeconds)
def api_deleteFile(): req = http_request.parse_post(filename=True, file_type=True) e = _simulations_using_file(req) if len(e): return http_reply.gen_json({ 'error': 'File is in use in other simulations.', 'fileList': e, 'fileName': req.filename, }) # Will not remove resource (standard) lib files pkio.unchecked_remove(_lib_file_write_path(req)) return http_reply.gen_json_ok()
def _move_import_file(data): sim = data['models']['simulation'] path = sim[_TMP_INPUT_FILE_FIELD] del sim[_TMP_INPUT_FILE_FIELD] if os.path.exists(path): zip_path = _sim_file(sim['simulationId'], _ZIP_FILE_NAME) os.rename(path, zip_path) pkio.unchecked_remove(os.path.dirname(path)) tmp_dir = _sim_file(sim['simulationId'], _TMP_ZIP_DIR) zipfile.ZipFile(zip_path).extractall(tmp_dir) _summarize_dicom_files(data, tmp_dir) pkio.unchecked_remove(tmp_dir) simulation_db.save_simulation_json(data)
def simulation_run_dir(data, remove_dir=False): """Where to run the simulation Args: data (dict): contains simulationType and simulationId remove_dir (bool): remove the directory [False] Returns: py.path: directory to run """ d = simulation_dir(data['simulationType'], parse_sid(data)).join(_report_dir(data)) if remove_dir: pkio.unchecked_remove(d) return d
def host_db(self, channel, host): res = Host() v = PKDict( rsconf_db=PKDict( # Common defaults we allow overrides for host_run_d='/srv', run_u='vagrant', root_u='root', ) ) merge_dict(res, v) #TODO(robnagler) optimize by caching default and channels for l in LEVELS: v = self.base[l] if l != LEVELS[0]: v = v.get(channel) if not v: continue if l == LEVELS[2]: v = v.get(host) if not v: continue merge_dict(res, v) host = host.lower() v = PKDict( rsconf_db=PKDict( channel=channel, db_d=self.db_d, host=host, proprietary_source_d=self.proprietary_source_d, rpm_source_d=self.rpm_source_d, secret_d=self.secret_d, srv_d=self.srv_d, srv_host_d=self.srv_host_d, tmp_d=self.tmp_d.join(host), # https://jnovy.fedorapeople.org/pxz/node1.html # compression with 8 threads and max compression # Useful (random) constants compress_cmd='pxz -T8 -9', ) ) v.rsconf_db.resource_paths = _init_resource_paths(v) v.rsconf_db.local_files = _init_local_files(v) pkio.unchecked_remove(v.rsconf_db.tmp_d) pkio.mkdir_parent(v.rsconf_db.tmp_d) merge_dict(res, v) _assert_no_rsconf_db_values(res) _update_paths(res) pkdc('{}', res) return res
def _purge_sim(db_file): d = pkcollections.json_load_any(db_file) # OPTIMIZATION: We assume the uids_of_paid_users doesn't change very # frequently so we don't need to check again. A user could run a sim # at anytime so we need to check that they haven't if d.lastUpdateTime > _too_old: return if d.status == job.FREE_USER_PURGED: return p = sirepo.simulation_db.simulation_run_dir(d) pkio.unchecked_remove(p) d.status = job.FREE_USER_PURGED cls.__db_write_file(d) jids_purged.append(db_file.purebasename)
def api_deleteFile(): req = http_request.parse_json() filename = werkzeug.secure_filename(req['fileName']) search_name = _lib_filename(req['simulationType'], filename, req['fileType']) err = _simulations_using_file(req['simulationType'], req['fileType'], search_name) if len(err): return http_reply.gen_json({ 'error': 'File is in use in other simulations.', 'fileList': err, 'fileName': filename, }) p = _lib_filepath(req['simulationType'], filename, req['fileType']) pkio.unchecked_remove(p) return http_reply.gen_json({})
def _link_or_unlink_proprietary_files(sim_type, should_link): d = proprietary_code_dir(sim_type) for e in simulation_db.examples(sim_type): b = sim_data.get_class(sim_type).proprietary_lib_file_basename(e) p = simulation_db.simulation_lib_dir(sim_type).join(b) if not should_link: pkio.unchecked_remove(p) continue try: p.mksymlinkto( d.join(b), absolute=False, ) except py.error.EEXIST: pass
def pytest_runtest_protocol(item, *args, **kwargs): """Make sure work directory is empty for a module. If `item` is in a module not seen before, it removes the `pkunit.work_dir`. Args: item (Item): pytest test item (case) Returns: None: always so that the next hook runs the item. """ if not _uses_pykern: return from pykern import pkunit # Seems to be the only way to get the module under test m = item._request.module is_new = m != pkunit.module_under_test pkunit.module_under_test = m if is_new: from pykern import pkio pkio.unchecked_remove(pkunit.work_dir())
def app_upload_file(simulation_type, simulation_id): f = flask.request.files['file'] lib = simulation_db.simulation_lib_dir(simulation_type) filename = werkzeug.secure_filename(f.filename) p = lib.join(filename) err = None if p.check(): err = 'file exists: {}'.format(filename) if not err: f.save(str(p)) err = _validate_data_file(p) if err: pkio.unchecked_remove(p) if err: return flask.jsonify({ 'error': err, 'filename': filename, 'simulationId': simulation_id, }) return flask.jsonify({ 'filename': filename, 'simulationId': simulation_id, })
def purge_users(days=180, confirm=False): """Remove old users from db which have not registered. Args: days (int): maximum days of untouched files (old is mtime > days) confirm (bool): delete the directories if True (else don't delete) [False] Returns: list: directories removed (or to remove if confirm) """ from pykern import pkio from sirepo import server from sirepo import simulation_db from sirepo import api_auth import datetime days = int(days) assert days >= 1, \ '{}: days must be a positive integer' server.init() uids = api_auth.all_uids() now = datetime.datetime.utcnow() to_remove = [] for d in pkio.sorted_glob(simulation_db.user_dir_name('*')): if _is_src_dir(d): continue; #TODO(pjm): need to skip special "src" user if simulation_db.uid_from_dir_name(d) in uids: continue for f in pkio.walk_tree(d): if (now - now.fromtimestamp(f.mtime())).days <= days: break else: to_remove.append(d) if confirm: pkio.unchecked_remove(*to_remove) return to_remove
def run_background(cfg_dir): res = {} data = simulation_db.read_json(template_common.INPUT_BASE_NAME) distribution = data['models']['bunch']['distribution'] run_with_mpi = distribution == 'lattice' or distribution == 'file' try: with pkio.save_chdir(cfg_dir): if run_with_mpi: mpi.run_script(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE)) else: #TODO(pjm): MPI doesn't work with rsbeams distributions yet exec(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE), locals(), locals()) except Exception as e: res = { 'error': str(e), } if run_with_mpi and 'error' in res: text = pkio.read_text('mpi_run.out') m = re.search(r'^Traceback .*?^\w*Error: (.*?)\n\n', text, re.MULTILINE|re.DOTALL) if m: res['error'] = m.group(1) # remove output file - write_result() will not overwrite an existing error output pkio.unchecked_remove(simulation_db.json_filename(template_common.OUTPUT_BASE_NAME)) simulation_db.write_result(res)
def remove_last_frame(run_dir): for m in ('currentAnimation', 'fieldAnimation'): files = _h5_file_list(run_dir, m) if len(files) > 0: pkio.unchecked_remove(files[-1])
def remove_last_frame(run_dir): files = _h5_file_list(run_dir) if len(files) > 0: pkio.unchecked_remove(files[-1])
def delete_simulation(simulation_type, sid): """Deletes the simulation's directory. """ pkio.unchecked_remove(simulation_dir(simulation_type, sid))
def app_delete_simulation(): data = _json_input() pkio.unchecked_remove(simulation_db.simulation_dir(data['simulationType'], data['simulationId'])) return '{}'
def _purge(self): expires = datetime.datetime.utcnow() - cfg.keep_days for d in pkio.sorted_glob('[0-9]' * len(self._date_d)): t = datetime.datetime.utcfromtimestamp(d.stat().mtime) if t < expires: pkio.unchecked_remove(d)