def _cfg_uid(value):
from sirepo import simulation_db
if value and value == 'dev-no-validate' and pkconfig.channel_in_internal_test():
return value
assert simulation_db.user_path(value).check(dir=True), \
'uid={} does not exist'.format(value)
return value
def upgrade():
"""Upgrade the database"""
from pykern import pkio
from sirepo import simulation_db
from sirepo import server
import re
def _inc(m):
return m.group(1) + str(int(m.group(2)) + 1)
server.init()
for d in pkio.sorted_glob(simulation_db.user_path().join('*/warppba')):
for fn in pkio.sorted_glob(d.join('*/sirepo-data.json')):
with open(str(fn)) as f:
t = f.read()
for old, new in (
('"WARP example laser simulation"',
'"Laser-Plasma Wakefield"'),
('"Laser Pulse"', '"Laser-Plasma Wakefield"'),
('"WARP example electron beam simulation"', '"Electron Beam"'),
):
if not old in t:
continue
t = t.replace(old, new)
t = re.sub(r'(simulationSerial":\s+)(\d+)', _inc, t)
break
with open(str(fn), 'w') as f:
f.write(t)
def _sim_path_from_id(sim_id):
for f in glob.glob('{}/*/{}/{}/{}'.format(
simulation_db.user_path(),
_SIM_DATA.sim_type(),
sim_id,
simulation_db.SIMULATION_DATA_FILE,
)):
return f
def test_delete_user():
from pykern import pkio
from pykern import pkunit
from sirepo import auth_db
from sirepo import simulation_db
from sirepo import srunit
from sirepo.sim_api import jupyterhublogin
import sirepo.pkcli.admin
import sirepo.srdb
pkio.unchecked_remove(sirepo.srdb.root())
pkunit.data_dir().join('db').copy(sirepo.srdb.root())
sirepo.pkcli.admin.delete_user('IYgnLlSy')
with auth_db.session_and_lock():
_is_empty(jupyterhublogin.JupyterhubUser.search_all_by())
_is_empty(auth_db.UserRegistration.search_all_by())
_is_empty(pkio.sorted_glob(jupyterhublogin.cfg.user_db_root_d.join('*')))
_is_empty(pkio.sorted_glob(simulation_db.user_path().join('*')))
def create_examples():
"""Adds missing app examples to all users"""
import sirepo.auth_db
import sirepo.server
sirepo.server.init()
for d in pkio.sorted_glob(simulation_db.user_path().join('*')):
if _is_src_dir(d):
continue;
uid = simulation_db.uid_from_dir_name(d)
with sirepo.auth_db.session_and_lock(), \
auth.set_user_outside_of_http_request(uid):
for sim_type in feature_config.cfg().sim_types:
simulation_db.verify_app_directory(sim_type)
names = [x.name for x in simulation_db.iterate_simulation_datafiles(
sim_type, simulation_db.process_simulation_list, {
'simulation.isExample': True,
})]
for example in simulation_db.examples(sim_type):
if example.models.simulation.name not in names:
_create_example(example)
def upgrade_runner_to_job_db(db_dir):
import sirepo.auth
from pykern import pkio
from pykern.pkcollections import PKDict
from pykern.pkdebug import pkdp, pkdlog, pkdexc
from sirepo import job
from sirepo import simulation_db
from sirepo import sim_data
from sirepo import util
import pykern.pkio
import sirepo.template
def _add_compute_status(run_dir, data):
p = run_dir.join(job.RUNNER_STATUS_FILE)
data.pkupdate(
lastUpdateTime=int(p.mtime()),
status=pkio.read_text(p),
)
def _add_parallel_status(in_json, sim_data, run_dir, data):
t = sirepo.template.import_module(data.simulationType)
# pksetdefault emulates job_cmd._background_percent_complete
data.parallelStatus = PKDict(
t.background_percent_complete(
sim_data.parse_model(in_json),
run_dir,
False,
)).pksetdefault(
lastUpdateTime=data.lastUpdateTime,
frameCount=0,
percentComplete=0.0,
)
def _create_supervisor_state_file(run_dir):
try:
i, t = _load_in_json(run_dir)
except Exception as e:
if pykern.pkio.exception_is_not_found(e):
return
raise
u = simulation_db.uid_from_dir_name(run_dir)
sirepo.auth.cfg.logged_in_user = u
c = sim_data.get_class(i.simulationType)
d = PKDict(
computeJid=c.parse_jid(i, u),
computeJobHash=c.compute_job_hash(
i), # TODO(e-carlin): Another user cookie problem
computeJobSerial=t,
computeJobStart=t,
computeModel=c.compute_model(i),
error=None,
history=[],
isParallel=c.is_parallel(i),
simulationId=i.simulationId,
simulationType=i.simulationType,
uid=u,
)
d.pkupdate(
jobRunMode=job.PARALLEL if d.isParallel else job.SEQUENTIAL,
nextRequestSeconds=c.poll_seconds(i),
)
_add_compute_status(run_dir, d)
if d.status not in (job.COMPLETED, job.CANCELED):
return
if d.isParallel:
_add_parallel_status(i, c, run_dir, d)
util.json_dump(d, path=_db_file(d.computeJid))
def _db_file(computeJid):
return db_dir.join(computeJid + '.json')
def _load_in_json(run_dir):
p = simulation_db.json_filename(
sirepo.template.template_common.INPUT_BASE_NAME, run_dir)
c = simulation_db.read_json(p)
return c, c.computeJobCacheKey.computeJobStart if \
c.get('computejobCacheKey') else \
int(p.mtime())
c = 0
db_dir = pkio.py_path(db_dir)
pkio.mkdir_parent(db_dir)
for f in pkio.walk_tree(
simulation_db.user_path(),
'^(?!.*src/).*/{}$'.format(sirepo.job.RUNNER_STATUS_FILE),
):
try:
_create_supervisor_state_file(pkio.py_path(f.dirname))
except Exception as e:
c += 1
k = PKDict(run_dir=f)
s = 'run_dir={run_dir}'
if c < 50:
k.stack = pkdexc()
s += ' stack={stack}'
else:
k.error = getattr(e, 'args', []) or e
s += ' error={error}'
pkdlog(s, **k)