def _cfg_uid(value): from sirepo import simulation_db if value and value == 'dev-no-validate' and pkconfig.channel_in_internal_test(): return value assert simulation_db.user_path(value).check(dir=True), \ 'uid={} does not exist'.format(value) return value
def upgrade(): """Upgrade the database""" from pykern import pkio from sirepo import simulation_db from sirepo import server import re def _inc(m): return m.group(1) + str(int(m.group(2)) + 1) server.init() for d in pkio.sorted_glob(simulation_db.user_path().join('*/warppba')): for fn in pkio.sorted_glob(d.join('*/sirepo-data.json')): with open(str(fn)) as f: t = f.read() for old, new in ( ('"WARP example laser simulation"', '"Laser-Plasma Wakefield"'), ('"Laser Pulse"', '"Laser-Plasma Wakefield"'), ('"WARP example electron beam simulation"', '"Electron Beam"'), ): if not old in t: continue t = t.replace(old, new) t = re.sub(r'(simulationSerial":\s+)(\d+)', _inc, t) break with open(str(fn), 'w') as f: f.write(t)
def _sim_path_from_id(sim_id): for f in glob.glob('{}/*/{}/{}/{}'.format( simulation_db.user_path(), _SIM_DATA.sim_type(), sim_id, simulation_db.SIMULATION_DATA_FILE, )): return f
def test_delete_user(): from pykern import pkio from pykern import pkunit from sirepo import auth_db from sirepo import simulation_db from sirepo import srunit from sirepo.sim_api import jupyterhublogin import sirepo.pkcli.admin import sirepo.srdb pkio.unchecked_remove(sirepo.srdb.root()) pkunit.data_dir().join('db').copy(sirepo.srdb.root()) sirepo.pkcli.admin.delete_user('IYgnLlSy') with auth_db.session_and_lock(): _is_empty(jupyterhublogin.JupyterhubUser.search_all_by()) _is_empty(auth_db.UserRegistration.search_all_by()) _is_empty(pkio.sorted_glob(jupyterhublogin.cfg.user_db_root_d.join('*'))) _is_empty(pkio.sorted_glob(simulation_db.user_path().join('*')))
def create_examples(): """Adds missing app examples to all users""" import sirepo.auth_db import sirepo.server sirepo.server.init() for d in pkio.sorted_glob(simulation_db.user_path().join('*')): if _is_src_dir(d): continue; uid = simulation_db.uid_from_dir_name(d) with sirepo.auth_db.session_and_lock(), \ auth.set_user_outside_of_http_request(uid): for sim_type in feature_config.cfg().sim_types: simulation_db.verify_app_directory(sim_type) names = [x.name for x in simulation_db.iterate_simulation_datafiles( sim_type, simulation_db.process_simulation_list, { 'simulation.isExample': True, })] for example in simulation_db.examples(sim_type): if example.models.simulation.name not in names: _create_example(example)
def upgrade_runner_to_job_db(db_dir): import sirepo.auth from pykern import pkio from pykern.pkcollections import PKDict from pykern.pkdebug import pkdp, pkdlog, pkdexc from sirepo import job from sirepo import simulation_db from sirepo import sim_data from sirepo import util import pykern.pkio import sirepo.template def _add_compute_status(run_dir, data): p = run_dir.join(job.RUNNER_STATUS_FILE) data.pkupdate( lastUpdateTime=int(p.mtime()), status=pkio.read_text(p), ) def _add_parallel_status(in_json, sim_data, run_dir, data): t = sirepo.template.import_module(data.simulationType) # pksetdefault emulates job_cmd._background_percent_complete data.parallelStatus = PKDict( t.background_percent_complete( sim_data.parse_model(in_json), run_dir, False, )).pksetdefault( lastUpdateTime=data.lastUpdateTime, frameCount=0, percentComplete=0.0, ) def _create_supervisor_state_file(run_dir): try: i, t = _load_in_json(run_dir) except Exception as e: if pykern.pkio.exception_is_not_found(e): return raise u = simulation_db.uid_from_dir_name(run_dir) sirepo.auth.cfg.logged_in_user = u c = sim_data.get_class(i.simulationType) d = PKDict( computeJid=c.parse_jid(i, u), computeJobHash=c.compute_job_hash( i), # TODO(e-carlin): Another user cookie problem computeJobSerial=t, computeJobStart=t, computeModel=c.compute_model(i), error=None, history=[], isParallel=c.is_parallel(i), simulationId=i.simulationId, simulationType=i.simulationType, uid=u, ) d.pkupdate( jobRunMode=job.PARALLEL if d.isParallel else job.SEQUENTIAL, nextRequestSeconds=c.poll_seconds(i), ) _add_compute_status(run_dir, d) if d.status not in (job.COMPLETED, job.CANCELED): return if d.isParallel: _add_parallel_status(i, c, run_dir, d) util.json_dump(d, path=_db_file(d.computeJid)) def _db_file(computeJid): return db_dir.join(computeJid + '.json') def _load_in_json(run_dir): p = simulation_db.json_filename( sirepo.template.template_common.INPUT_BASE_NAME, run_dir) c = simulation_db.read_json(p) return c, c.computeJobCacheKey.computeJobStart if \ c.get('computejobCacheKey') else \ int(p.mtime()) c = 0 db_dir = pkio.py_path(db_dir) pkio.mkdir_parent(db_dir) for f in pkio.walk_tree( simulation_db.user_path(), '^(?!.*src/).*/{}$'.format(sirepo.job.RUNNER_STATUS_FILE), ): try: _create_supervisor_state_file(pkio.py_path(f.dirname)) except Exception as e: c += 1 k = PKDict(run_dir=f) s = 'run_dir={run_dir}' if c < 50: k.stack = pkdexc() s += ' stack={stack}' else: k.error = getattr(e, 'args', []) or e s += ' error={error}' pkdlog(s, **k)