def test_purge_users(monkeypatch):
from pykern.pkunit import pkeq, pkok
from pykern.pkdebug import pkdp
from pykern import pkio
from pykern import pkconfig
from sirepo import srunit
srunit.init_user_db()
from sirepo.pkcli import admin
from sirepo import simulation_db
from sirepo import api_auth
import datetime
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: no old users so empty')
pkdp(simulation_db.user_dir_name('*'))
g = simulation_db.user_dir_name('*')
dirs = list(pkio.sorted_glob(g))
pkeq(1, len(dirs), '{}: expecting exactly one user dir', g)
uid = dirs[0].basename
#TODO(robnagler) really want the db to be created, but need
# a test oauth class.
monkeypatch.setattr(api_auth, 'all_uids', lambda: [uid])
for f in pkio.walk_tree(dirs[0]):
f.setmtime(f.mtime() - 86400 * 2)
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: all users registered so no deletes')
monkeypatch.setattr(api_auth, 'all_uids', lambda: [])
res = admin.purge_users(days=1, confirm=False)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(dirs[0].check(dir=True), '{}: nothing deleted', res)
res = admin.purge_users(days=1, confirm=True)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(not dirs[0].check(dir=True), '{}: directory deleted', res)
def test_purge_users(monkeypatch):
from pykern.pkunit import pkeq, pkok
from pykern.pkdebug import pkdp
from pykern import pkio
from pykern import pkconfig
from sirepo import srunit
srunit.init_user_db()
from sirepo.pkcli import admin
from sirepo import simulation_db
from sirepo import api_auth
import datetime
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: no old users so empty')
pkdp(simulation_db.user_dir_name('*'))
g = simulation_db.user_dir_name('*')
dirs = list(pkio.sorted_glob(g))
pkeq(1, len(dirs), '{}: expecting exactly one user dir', g)
uid = dirs[0].basename
#TODO(robnagler) really want the db to be created, but need
# a test oauth class.
monkeypatch.setattr(api_auth, 'all_uids', lambda: [uid])
for f in pkio.walk_tree(dirs[0]):
f.setmtime(f.mtime() - 86400 * 2)
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: all users registered so no deletes')
monkeypatch.setattr(api_auth, 'all_uids', lambda: [])
res = admin.purge_users(days=1, confirm=False)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(dirs[0].check(dir=True), '{}: nothing deleted', res)
res = admin.purge_users(days=1, confirm=True)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(not dirs[0].check(dir=True), '{}: directory deleted', res)
def purge_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
list: directories removed (or to remove if confirm)
"""
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
uids = auth_db.all_uids()
now = datetime.datetime.utcnow()
to_remove = []
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
if simulation_db.uid_from_dir_name(d) in uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
to_remove.append(d)
if confirm:
pkio.unchecked_remove(*to_remove)
return to_remove
def create_examples():
"""Adds missing app examples to all users.
"""
from pykern import pkio
from sirepo import feature_config
from sirepo import server
from sirepo import simulation_db
from sirepo import cookie
server.init()
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
uid = simulation_db.uid_from_dir_name(d)
cookie.init_mock(uid)
for sim_type in feature_config.cfg.sim_types:
simulation_db.verify_app_directory(sim_type)
names = map(
lambda x: x['name'],
simulation_db.iterate_simulation_datafiles(sim_type, simulation_db.process_simulation_list, {
'simulation.isExample': True,
}))
for s in simulation_db.examples(sim_type):
if s.models.simulation.name not in names:
simulation_db.save_new_example(s)
def upgrade():
"""Upgrade the database"""
from pykern import pkio
from sirepo import simulation_db
from sirepo import server
import re
def _inc(m):
return m.group(1) + str(int(m.group(2)) + 1)
server.init()
for d in pkio.sorted_glob(simulation_db.user_dir_name().join('*/warppba')):
for fn in pkio.sorted_glob(d.join('*/sirepo-data.json')):
with open(str(fn)) as f:
t = f.read()
for old, new in (
('"WARP example laser simulation"',
'"Laser-Plasma Wakefield"'),
('"Laser Pulse"', '"Laser-Plasma Wakefield"'),
('"WARP example electron beam simulation"', '"Electron Beam"'),
):
if not old in t:
continue
t = t.replace(old, new)
t = re.sub(r'(simulationSerial":\s+)(\d+)', _inc, t)
break
with open(str(fn), 'w') as f:
f.write(t)
def upgrade():
"""Upgrade the database"""
from pykern import pkio
from sirepo import simulation_db
from sirepo import server
import re
def _inc(m):
return m.group(1) + str(int(m.group(2)) + 1)
server.init()
for d in pkio.sorted_glob(simulation_db.user_dir_name().join('*/warppba')):
for fn in pkio.sorted_glob(d.join('*/sirepo-data.json')):
with open(str(fn)) as f:
t = f.read()
for old, new in (
('"WARP example laser simulation"', '"Laser-Plasma Wakefield"'),
('"Laser Pulse"', '"Laser-Plasma Wakefield"'),
('"WARP example electron beam simulation"', '"Electron Beam"'),
):
if not old in t:
continue
t = t.replace(old, new)
t = re.sub(r'(simulationSerial":\s+)(\d+)', _inc, t)
break
with open(str(fn), 'w') as f:
f.write(t)
def _cfg_uid(value):
from sirepo import simulation_db
if value and value == 'dev-no-validate' and pkconfig.channel_in_internal_test(
):
return value
assert simulation_db.user_dir_name(value).check(dir=True), \
'uid={} does not exist'.format(value)
return value
def api_serverStatus():
"""Allow for remote monitoring of the web server status. Authentication
is done via Basic Auth. The username should be an existing sirepo uid.
The status checks that a simple simulation can complete successfully
within a short period of time.
"""
if not _basic_auth.authenticate():
return _basic_auth.challenge()
uid = flask.request.authorization.username
simulation_db.assert_id(uid)
if not simulation_db.user_dir_name(uid).exists():
raise RuntimeError('status user does not exist: {}'.format(uid))
cookie.set_user(uid)
_run_tests()
return http_reply.gen_json_ok({
'datetime':
datetime.datetime.utcnow().isoformat(),
})
def create_examples():
"""Adds missing app examples to all users.
"""
server.init()
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
uid = simulation_db.uid_from_dir_name(d)
auth.init_mock(uid)
for sim_type in feature_config.cfg.sim_types:
simulation_db.verify_app_directory(sim_type)
names = map(
lambda x: x['name'],
simulation_db.iterate_simulation_datafiles(sim_type, simulation_db.process_simulation_list, {
'simulation.isExample': True,
}))
for example in simulation_db.examples(sim_type):
if example.models.simulation.name not in names:
_create_example(example)
def create_examples():
"""Adds missing app examples to all users.
"""
server.init()
for d in pkio.sorted_glob(simulation_db.user_dir_name().join('*')):
if _is_src_dir(d):
continue
uid = simulation_db.uid_from_dir_name(d)
auth.set_user_for_utils(uid)
for sim_type in feature_config.cfg().sim_types:
simulation_db.verify_app_directory(sim_type)
names = [
x.name for x in simulation_db.iterate_simulation_datafiles(
sim_type, simulation_db.process_simulation_list, {
'simulation.isExample': True,
})
]
for example in simulation_db.examples(sim_type):
if example.models.simulation.name not in names:
_create_example(example)
def purge_guest_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
(list, list): dirs and uids of removed guest users (or to remove if confirm)
"""
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
from sirepo import srtime
guest_uids = auth.guest_uids()
now = srtime.utc_now()
dirs_and_uids = {}
for d in pkio.sorted_glob(simulation_db.user_dir_name().join('*')):
uid = simulation_db.uid_from_dir_name(d)
if _is_src_dir(d):
continue
if uid not in guest_uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
dirs_and_uids[d] = uid
if confirm:
pkio.unchecked_remove(*dirs_and_uids.keys())
auth_db.UserRegistration.delete_all_for_column_by_values(
'uid', dirs_and_uids.values())
return dirs_and_uids
def purge_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
list: directories removed (or to remove if confirm)
"""
from pykern import pkio
from sirepo import server
from sirepo import simulation_db
from sirepo import api_auth
import datetime
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
uids = api_auth.all_uids()
now = datetime.datetime.utcnow()
to_remove = []
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
#TODO(pjm): need to skip special "src" user
if simulation_db.uid_from_dir_name(d) in uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
to_remove.append(d)
if confirm:
pkio.unchecked_remove(*to_remove)
return to_remove
def _cfg_uid(value):
from sirepo import simulation_db
assert simulation_db.user_dir_name(value).check(dir=True), \
'uid={} does not exist'.format(value)
return value
def upgrade_runner_to_job_db(db_dir):
import sirepo.auth
from pykern import pkio
from pykern.pkcollections import PKDict
from pykern.pkdebug import pkdp, pkdlog, pkdexc
from sirepo import job
from sirepo import simulation_db
from sirepo import sim_data
from sirepo import util
import pykern.pkio
import sirepo.template
def _add_compute_status(run_dir, data):
p = run_dir.join(job.RUNNER_STATUS_FILE)
data.pkupdate(
lastUpdateTime=int(p.mtime()),
status=pkio.read_text(p),
)
def _add_parallel_status(in_json, sim_data, run_dir, data):
t = sirepo.template.import_module(data.simulationType)
# pksetdefault emulates job_cmd._background_percent_complete
data.parallelStatus = PKDict(
t.background_percent_complete(
sim_data.parse_model(in_json),
run_dir,
False,
)).pksetdefault(
lastUpdateTime=data.lastUpdateTime,
frameCount=0,
percentComplete=0.0,
)
def _create_supervisor_state_file(run_dir):
try:
i, t = _load_in_json(run_dir)
except Exception as e:
if pykern.pkio.exception_is_not_found(e):
return
raise
u = simulation_db.uid_from_dir_name(run_dir)
sirepo.auth.cfg.logged_in_user = u
c = sim_data.get_class(i.simulationType)
d = PKDict(
computeJid=c.parse_jid(i, u),
computeJobHash=c.compute_job_hash(
i), # TODO(e-carlin): Another user cookie problem
computeJobSerial=t,
computeJobStart=t,
error=None,
history=[],
isParallel=c.is_parallel(i),
simulationId=i.simulationId,
simulationType=i.simulationType,
uid=u,
)
d.pkupdate(
jobRunMode=job.PARALLEL if d.isParallel else job.SEQUENTIAL,
nextRequestSeconds=c.poll_seconds(i),
)
_add_compute_status(run_dir, d)
if d.status not in (job.COMPLETED, job.CANCELED):
return
if d.isParallel:
_add_parallel_status(i, c, run_dir, d)
util.json_dump(d, path=_db_file(d.computeJid))
def _db_file(computeJid):
return db_dir.join(computeJid + '.json')
def _load_in_json(run_dir):
p = simulation_db.json_filename(
sirepo.template.template_common.INPUT_BASE_NAME, run_dir)
c = simulation_db.read_json(p)
return c, c.computeJobCacheKey.computeJobStart if \
c.get('computejobCacheKey') else \
int(p.mtime())
c = 0
db_dir = pkio.py_path(db_dir)
pkio.mkdir_parent(db_dir)
for f in pkio.walk_tree(
simulation_db.user_dir_name(),
'^(?!.*src/).*/{}$'.format(sirepo.job.RUNNER_STATUS_FILE),
):
try:
_create_supervisor_state_file(pkio.py_path(f.dirname))
except Exception as e:
c += 1
k = PKDict(run_dir=f)
s = 'run_dir={run_dir}'
if c < 50:
k.stack = pkdexc()
s += ' stack={stack}'
else:
k.error = getattr(e, 'args', []) or e
s += ' error={error}'
pkdlog(s, **k)
def _get_dirs():
from pykern import pkio
from sirepo import simulation_db
g = simulation_db.user_dir_name().join('*')
return list(pkio.sorted_glob(g))