def _init():
from pykern import pkconfig
global _cfg
def b(msg, dev=False):
return (
pkconfig.channel_in('dev')
if dev else pkconfig.channel_in_internal_test(),
bool,
msg,
)
_cfg = pkconfig.init(
# No secrets should be stored here (see sirepo.job.agent_env)
api_modules=((), set, 'optional api modules, e.g. status'),
default_proprietary_sim_types=
(set(), set,
'codes where all users are authorized by default but that authorization can be revoked'
),
jspec=dict(derbenevskrinsky_force_formula=b(
'Include Derbenev-Skrinsky force formula'), ),
proprietary_sim_types=(set(), set, 'codes that require authorization'),
#TODO(robnagler) make this a sim_type config like srw and warpvnd
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
sim_types=(set(), set, 'simulation types (codes) to be imported'),
srw=dict(
app_url=('/en/xray-beamlines.html', str, 'URL for SRW link'),
beamline3d=b('Show 3D beamline plot'),
hide_guest_warning=b('Hide the guest warning in the UI', dev=True),
mask_in_toolbar=b('Show the mask element in toolbar'),
show_open_shadow=(
pkconfig.channel_in_internal_test(), bool,
'Show "Open as a New Shadow Simulation" menu item'),
show_rsopt_ml=(pkconfig.channel_in_internal_test(), bool,
'Show "Export ML Script" menu item'),
),
warpvnd=dict(
allow_3d_mode=(True, bool,
'Include 3D features in the Warp VND UI'),
display_test_boxes=b(
'Display test boxes to visualize 3D -> 2D projections'),
),
)
i = _cfg.proprietary_sim_types.intersection(
_cfg.default_proprietary_sim_types)
assert not i, \
f'{i}: cannot be in proprietary_sim_types and default_proprietary_sim_types'
s = set(_cfg.sim_types or
(PROD_FOSS_CODES if pkconfig.channel_in('prod') else _FOSS_CODES))
s.update(_cfg.proprietary_sim_types, _cfg.default_proprietary_sim_types)
for v in _DEPENDENT_CODES:
if v[0] in s:
s.add(v[1])
x = s.difference(VALID_CODES)
assert not x, \
'sim_type(s) invalid={} expected={}'.format(x, VALID_CODES)
_cfg.sim_types = frozenset(s)
return _cfg
def start():
#TODO(robnagler) commands need their own init hook like the server has
job.init()
global cfg
cfg = pkconfig.init(
agent_id=pkconfig.Required(str, 'id of this agent'),
fastcgi_sock_dir=(
pkio.py_path('/tmp'), pkio.py_path,
'directory of fastcfgi socket, must be less than 50 chars'),
start_delay=(0, pkconfig.parse_seconds,
'delay startup in internal_test mode'),
supervisor_uri=pkconfig.Required(
str,
'how to connect to the supervisor',
),
)
pkdlog('{}', cfg)
if pkconfig.channel_in_internal_test() and cfg.start_delay:
pkdlog('start_delay={}', cfg.start_delay)
time.sleep(cfg.start_delay)
i = tornado.ioloop.IOLoop.current()
d = _Dispatcher()
def s(*args):
return i.add_callback_from_signal(_terminate, d)
signal.signal(signal.SIGTERM, s)
signal.signal(signal.SIGINT, s)
i.spawn_callback(d.loop)
i.start()
def _cfg_uid(value):
from sirepo import simulation_db
if value and value == 'dev-no-validate' and pkconfig.channel_in_internal_test():
return value
assert simulation_db.user_path(value).check(dir=True), \
'uid={} does not exist'.format(value)
return value
def b(msg, dev=False):
return (
pkconfig.channel_in('dev')
if dev else pkconfig.channel_in_internal_test(),
bool,
msg,
)
async def _agent_start(self, op):
if self._agent_starting_timeout:
return
async with self._agent_start_lock:
# POSIT: we do not have to raise Awaited(), because
# this is the first thing an op waits on.
if self._agent_starting_timeout or self._websocket_ready.is_set():
return
try:
t = self.cfg.agent_starting_secs
if pkconfig.channel_in_internal_test():
x = op.msg.pkunchecked_nested_get(
'data.models.dog.favoriteTreat')
if x:
x = re.search(r'agent_start_delay=(\d+)', x)
if x:
self._agent_start_delay = int(x.group(1))
t += self._agent_start_delay
pkdlog('op={} agent_start_delay={}', op,
self._agent_start_delay)
pkdlog('{} {} await _do_agent_start', self, op)
# All awaits must be after this. If a call hangs the timeout
# handler will cancel this task
self._agent_starting_timeout = tornado.ioloop.IOLoop.current(
).call_later(
t,
self._agent_starting_timeout_handler,
)
# POSIT: Canceled errors aren't smothered by any of the below calls
await self.kill()
await self._do_agent_start(op)
except Exception as e:
pkdlog('{} error={} stack={}', self, e, pkdexc())
self.free_resources(internal_error='failure starting agent')
raise
def create_token(value):
if pkconfig.channel_in_internal_test() and cfg.create_token_secret:
v = base64.b32encode(
hashlib.sha256(
pkcompat.to_bytes(value + cfg.create_token_secret)).digest())
return pkcompat.from_bytes(v[:TOKEN_SIZE])
return random_base62(TOKEN_SIZE)
def _init():
from pykern import pkconfig
global _cfg
@pkconfig.parse_none
def _cfg_sim_types(value):
res = pkconfig.parse_set(value)
if not res:
return tuple(_codes())
for c in res:
assert c in _codes(), \
'invalid sim_type={}, expected one of={}'.format(c, _codes())
if 'jspec' in res:
res = set(res)
res.add('elegant')
return tuple(res)
def _codes():
return ALL_CODES if pkconfig.channel_in_internal_test() \
else NON_ALPHA_CODES
_cfg = pkconfig.init(
api_modules=((), set, 'optional api modules, e.g. status'),
job=(False, bool,
'[new] job execution architecture (replaces runner)'),
jspec=dict(derbenevskrinsky_force_formula=(
pkconfig.channel_in_internal_test(), bool,
'Include Derbenev-Skrinsky force forumla'), ),
#TODO(robnagler) make sim_type config
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
sim_types=(None, _cfg_sim_types,
'simulation types (codes) to be imported'),
srw=dict(
mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool,
'Show the mask element in toolbar'),
beamline3d=(pkconfig.channel_in_internal_test(), bool,
'Show 3D beamline plot'),
),
warpvnd=dict(
allow_3d_mode=(True, bool,
'Include 3D features in the Warp VND UI'),
display_test_boxes=(
pkconfig.channel_in_internal_test(), bool,
'Display test boxes to visualize 3D -> 2D projections'),
),
)
def _init():
if pkconfig.channel_in_internal_test():
from sirepo import uri_router
uri_router.register_api_module()
else:
global utc_now_as_float, utc_now
utc_now_as_float = time.time
utc_now = datetime.datetime.utcnow
def init():
if pkconfig.channel_in_internal_test():
return
global _initialized, utc_now_as_float, utc_now
if _initialized:
return
_initialized = True
utc_now_as_float = time.time
utc_now = datetime.datetime.utcnow
def utc_now_as_float():
"""Adjusted POSIX time as a float
Returns:
float: adjusted `time.time`
"""
assert pkconfig.channel_in_internal_test()
if _timedelta is None:
return time.time()
return to_timestamp(utc_now())
def init():
global _initialized, utc_now_as_int
if _initialized:
return
_initialized = True
if not pkconfig.channel_in_internal_test():
global utc_now_as_float, utc_now
utc_now_as_float = time.time
utc_now = datetime.datetime.utcnow
utc_now_as_int = lambda: int(utc_now_as_float())
def utc_now():
"""Adjusted UTC time as object
Returns:
datetime.datetime: adjusted `datetime.datetime.utcnow`
"""
assert pkconfig.channel_in_internal_test()
if _timedelta is None:
return datetime.datetime.utcnow()
return datetime.datetime.utcnow() + _timedelta
def _init():
from pykern import pkconfig
global _cfg
_cfg = pkconfig.init(
# No secrets should be stored here (see sirepo.job.agent_env)
api_modules=((), set, 'optional api modules, e.g. status'),
jspec=dict(
derbenevskrinsky_force_formula=(pkconfig.channel_in_internal_test(), bool, 'Include Derbenev-Skrinsky force formula'),
),
proprietary_sim_types=(set(), set, 'codes that require authorization'),
#TODO(robnagler) make this a sim_type config like srw and warpvnd
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
sim_types=(set(), set, 'simulation types (codes) to be imported'),
srw=dict(
mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'),
beamline3d=(pkconfig.channel_in_internal_test(), bool, 'Show 3D beamline plot'),
app_url=('/en/xray-beamlines.html', str, 'URL for SRW link'),
hide_guest_warning=(False, bool, 'Hide the guest warning in the UI'),
),
warpvnd=dict(
allow_3d_mode=(True, bool, 'Include 3D features in the Warp VND UI'),
display_test_boxes=(pkconfig.channel_in_internal_test(), bool, 'Display test boxes to visualize 3D -> 2D projections'),
),
)
s = set(
_cfg.sim_types or (
_FOSS_CODES if pkconfig.channel_in_internal_test() else _NON_ALPHA_FOSS_CODES
)
)
s.update(_cfg.proprietary_sim_types)
# jspec imports elegant, but elegant won't work if it is not a valid
# sim_type so need to include here. Need a better model of
# dependencies between codes.
if 'jspec' in s and 'elegant' not in s:
s.add('elegant')
x = s.difference(VALID_CODES)
assert not x, \
'sim_type(s) invalid={} expected={}'.format(x, VALID_CODES)
_cfg.sim_types = frozenset(s)
return _cfg
def api_adjustTime(days=None):
"""Shift the system time by days and get the adjusted time
Args:
days (str): must be integer. If None or 0, no adjustment.
"""
from sirepo import http_reply
assert pkconfig.channel_in_internal_test(), \
'API forbidden'
adjust_time(days)
return http_reply.gen_json_ok({
'adjustedNow':
utc_now().isoformat(),
'systemNow':
datetime.datetime.utcnow().isoformat(),
})
def _simulation_error(err, *args, **kwargs):
"""Something unexpected went wrong.
Parses ``err`` for error
Args:
err (str): exception or run_log
quiet (bool): don't write errors to log
Returns:
dict: error response
"""
if not kwargs.get('quiet'):
pkdlog('{}', ': '.join([str(a) for a in args] + ['error', err]))
m = re.search(_SUBPROCESS_ERROR_RE, str(err))
if m:
err = m.group(1)
if re.search(r'error exit\(-15\)', err):
err = 'Terminated'
elif not pkconfig.channel_in_internal_test():
err = 'unexpected error (see logs)'
return {'state': 'error', 'error': err}
def _simulation_error(err, *args, **kwargs):
"""Something unexpected went wrong.
Parses ``err`` for error
Args:
err (str): exception or run_log
quiet (bool): don't write errors to log
Returns:
dict: error response
"""
if not kwargs.get('quiet'):
pkdlog('{}', ': '.join([str(a) for a in args] + ['error', err]))
m = re.search(_SUBPROCESS_ERROR_RE, str(err))
if m:
err = m.group(1)
if re.search(r'error exit\(-15\)', err):
err = 'Terminated'
elif not pkconfig.channel_in_internal_test():
err = 'unexpected error (see logs)'
return {'state': 'error', 'error': err}
def _auth_state():
import sirepo.simulation_db
s = cookie.unchecked_get_value(_COOKIE_STATE)
v = pkcollections.Dict(
avatarUrl=None,
displayName=None,
guestIsOnlyMethod=not non_guest_methods,
isGuestUser=False,
isLoggedIn=_is_logged_in(s),
isLoginExpired=False,
jobRunModeMap=sirepo.simulation_db.JOB_RUN_MODE_MAP,
method=cookie.unchecked_get_value(_COOKIE_METHOD),
needCompleteRegistration=s == _STATE_COMPLETE_REGISTRATION,
roles=[],
userName=None,
visibleMethods=visible_methods,
)
if 'sbatch' in v.jobRunModeMap:
v.sbatchQueueMaxes=job.NERSC_QUEUE_MAX
u = cookie.unchecked_get_value(_COOKIE_USER)
if v.isLoggedIn:
if v.method == METHOD_GUEST:
# currently only method to expire login
v.displayName = _GUEST_USER_DISPLAY_NAME
v.isGuestUser = True
v.isLoginExpired = _METHOD_MODULES[METHOD_GUEST].is_login_expired()
v.needCompleteRegistration = False
v.visibleMethods = non_guest_methods
else:
r = auth_db.UserRegistration.search_by(uid=u)
if r:
v.displayName = r.display_name
v.roles = auth_db.UserRole.get_roles(u)
_plan(v)
_method_auth_state(v, u)
if pkconfig.channel_in_internal_test():
# useful for testing/debugging
v.uid = u
pkdc('state={}', v)
return v
def internal_build(self):
from rsconf import systemd
from rsconf.component import bop
# Must be after bop so bconf_f exists, and bop has to be on the machine
self.buildt.require_component('bop')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.bop_timer
for app_name in sorted(j2_ctx.bop.apps):
app_vars = bop.merge_app_vars(j2_ctx, app_name)
if pkconfig.channel_in_internal_test(
channel=j2_ctx.rsconf_db.channel):
self._add_initdb_spec(j2_ctx, z, app_vars)
if not app_name in z.spec:
continue
z.run_u = app_vars.run_u
z.bconf_f = app_vars.bconf_f
timers = z.spec[app_name]
for t in sorted(timers.keys()):
tv = timers[t]
z.bash_script = tv.bash_script
run_u = tv.get('run_u', z.run_u)
timer_name = '{}_{}'.format(app_name, t)
run_d = systemd.timer_prepare(
self,
j2_ctx,
#TODO(robnagler) time zone
on_calendar=tv.on_calendar,
service_name=timer_name,
)
run_f = run_d.join('run')
systemd.timer_enable(self,
j2_ctx=j2_ctx,
cmd=run_f,
run_u=run_u)
self.install_access(mode='500', owner=run_u)
self.install_resource('bop_timer/run.sh', j2_ctx, run_f)
def _cfg_sim_types(value):
res = pkconfig.parse_tuple(value)
if not res:
return _codes()
for c in res:
assert c in _codes(), \
'invalid sim_type={}, expected one of={}'.format(c, _codes())
return res
def _codes(want_all=pkconfig.channel_in('dev')):
return _ALL_CODES if want_all else _NON_DEV_CODES
cfg = pkconfig.init(
api_modules=((), tuple, 'optional api modules, e.g. bluesky'),
#TODO(robnagler) make sim_type config
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
sim_types=(None, _cfg_sim_types,
'simulation types (codes) to be imported'),
srw=dict(mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool,
'Show the mask element in toolbar'), ),
warpvnd=dict(
allow_3d_mode=(pkconfig.channel_in_internal_test(), bool,
'Include 3D features in the Warp VND UI'),
display_test_boxes=(
pkconfig.channel_in_internal_test(), bool,
'Display test boxes to visualize 3D -> 2D projections'),
),
)
def post(self):
assert pkconfig.channel_in_internal_test(), \
'You can only adjust time in internal test'
sirepo.srtime.adjust_time(pkjson.load_any(self.request.body).days)
self.write(PKDict())
@pkconfig.parse_none
def _cfg_sim_types(value):
res = pkconfig.parse_tuple(value)
if not res:
return _codes()
for c in res:
assert c in _codes(), \
'invalid sim_type={}, expected one of={}'.format(c, _codes())
return res
def _codes(want_all=pkconfig.channel_in('dev')):
return _ALL_CODES if want_all else _NON_DEV_CODES
cfg = pkconfig.init(
api_modules=((), tuple, 'optional api modules, e.g. bluesky'),
#TODO(robnagler) make sim_type config
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
sim_types=(None, _cfg_sim_types, 'simulation types (codes) to be imported'),
srw=dict(
mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'),
),
warpvnd=dict(
allow_3d_mode=(pkconfig.channel_in_internal_test(), bool, 'Include 3D features in the Warp VND UI'),
display_test_boxes=(pkconfig.channel_in_internal_test(), bool, 'Display test boxes to visualize 3D -> 2D projections'),
),
)
Returns:
dict: application specific config
"""
if sim_type not in cfg:
return {}
return pkcollections.map_to_dict(cfg[sim_type])
@pkconfig.parse_none
def _cfg_sim_types(value):
if not value:
return _codes()
user_specified_codes = tuple(value.split(':'))
for c in user_specified_codes:
assert c in _codes(), \
'{}: invalid sim_type, must be one of/combination of: {}'.format(c, _codes())
return user_specified_codes
def _codes(want_all=pkconfig.channel_in('dev')):
return _ALL_CODES if want_all else _NON_DEV_CODES
cfg = pkconfig.init(
srw=dict(mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool,
'Show the mask element in toolbar'), ),
sim_types=(None, _cfg_sim_types,
'simulation types (codes) to be imported'),
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
)
"""
if sim_type not in cfg:
return {}
return pkcollections.map_to_dict(cfg[sim_type])
@pkconfig.parse_none
def _cfg_sim_types(value):
if not value:
return _codes()
user_specified_codes = tuple(value.split(':'))
for c in user_specified_codes:
assert c in _codes(), \
'{}: invalid sim_type, must be one of/combination of: {}'.format(c, _codes())
return user_specified_codes
def _codes(want_all=pkconfig.channel_in('dev')):
return _ALL_CODES if want_all else _NON_DEV_CODES
cfg = pkconfig.init(
srw=dict(mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool,
'Show the mask element in toolbar'), ),
warpvnd=dict(particle_3d_report=(pkconfig.channel_in_internal_test(), bool,
'Show the 3d particle report'), ),
sim_types=(None, _cfg_sim_types,
'simulation types (codes) to be imported'),
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
)
def _codes():
return ALL_CODES if pkconfig.channel_in_internal_test() \
else NON_ALPHA_CODES
def _codes(want_all=None):
if want_all is None:
want_all = pkconfig.channel_in_internal_test()
return _ALL_CODES if want_all else _NON_ALPHA_CODES
_CPU_PERIOD_US = 100000
#: dump the slots whenever an update happens
_POOLS_DUMP_FILE = 'rsdockerspawner_pools.json'
#: Default user when no specific volume for user ['*']
_DEFAULT_USER_GROUP = 'everybody'
#: Name of the default pool when no user patches
_DEFAULT_POOL = _DEFAULT_USER_GROUP
#: Large time out for minimum allowed activity (effectively infinite)
_DEFAULT_MIN_ACTIVITY_HOURS = 1e6
#: Minimum five mins so we don't garbage collect too frequently
_MIN_MIN_ACTIVITY_SECS = 5.0 if pkconfig.channel_in_internal_test() else 300.0
#: Minimum number of processes available to the user not running in Jupyter
_MIN_NPROC_AVAIL = 512
#: User that won't match a legimate user
_DEFAULT_USER = '******'
#: Parameters set in create_object
_EXTRA_HOST_CONFIG = (
'cpu_period',
'cpu_quota',
'pids_limit',
'shm_size',
)
#: Configuration
cfg = None
def for_sim_type(sim_type):
"""Get cfg for simulation type
Args:
sim_type (str): srw, warp, etc.
Returns:
dict: application specific config
"""
if not sim_type in cfg:
return {}
return pkcollections.map_to_dict(cfg[sim_type])
@pkconfig.parse_none
def _cfg_bool(value):
"""Convert str to integer and then bool"""
if isinstance(value, str):
value = int(value)
return bool(value)
cfg = pkconfig.init(
srw=dict(
mask_in_toolbar=(pkconfig.channel_in_internal_test(), _cfg_bool, 'Show the mask element in toolbar'),
),
)
def for_sim_type(sim_type):
"""Get cfg for simulation type
Args:
sim_type (str): srw, warp, etc.
Returns:
dict: application specific config
"""
if sim_type not in cfg:
return {}
return pkcollections.map_to_dict(cfg[sim_type])
@pkconfig.parse_none
def _cfg_bool(value):
"""Convert str to integer and then bool"""
if isinstance(value, str):
value = int(value)
return bool(value)
cfg = pkconfig.init(srw=dict(
mask_in_toolbar=(pkconfig.channel_in_internal_test(), _cfg_bool,
'Show the mask element in toolbar'),
sample_in_toolbar=(pkconfig.channel_in_internal_test(), _cfg_bool,
'Show the sample element in toolbar'),
), )