def _init(): from pykern import pkconfig global _cfg def b(msg, dev=False): return ( pkconfig.channel_in('dev') if dev else pkconfig.channel_in_internal_test(), bool, msg, ) _cfg = pkconfig.init( # No secrets should be stored here (see sirepo.job.agent_env) api_modules=((), set, 'optional api modules, e.g. status'), default_proprietary_sim_types= (set(), set, 'codes where all users are authorized by default but that authorization can be revoked' ), jspec=dict(derbenevskrinsky_force_formula=b( 'Include Derbenev-Skrinsky force formula'), ), proprietary_sim_types=(set(), set, 'codes that require authorization'), #TODO(robnagler) make this a sim_type config like srw and warpvnd rs4pi_dose_calc=(False, bool, 'run the real dose calculator'), sim_types=(set(), set, 'simulation types (codes) to be imported'), srw=dict( app_url=('/en/xray-beamlines.html', str, 'URL for SRW link'), beamline3d=b('Show 3D beamline plot'), hide_guest_warning=b('Hide the guest warning in the UI', dev=True), mask_in_toolbar=b('Show the mask element in toolbar'), show_open_shadow=( pkconfig.channel_in_internal_test(), bool, 'Show "Open as a New Shadow Simulation" menu item'), show_rsopt_ml=(pkconfig.channel_in_internal_test(), bool, 'Show "Export ML Script" menu item'), ), warpvnd=dict( allow_3d_mode=(True, bool, 'Include 3D features in the Warp VND UI'), display_test_boxes=b( 'Display test boxes to visualize 3D -> 2D projections'), ), ) i = _cfg.proprietary_sim_types.intersection( _cfg.default_proprietary_sim_types) assert not i, \ f'{i}: cannot be in proprietary_sim_types and default_proprietary_sim_types' s = set(_cfg.sim_types or (PROD_FOSS_CODES if pkconfig.channel_in('prod') else _FOSS_CODES)) s.update(_cfg.proprietary_sim_types, _cfg.default_proprietary_sim_types) for v in _DEPENDENT_CODES: if v[0] in s: s.add(v[1]) x = s.difference(VALID_CODES) assert not x, \ 'sim_type(s) invalid={} expected={}'.format(x, VALID_CODES) _cfg.sim_types = frozenset(s) return _cfg
def start(): #TODO(robnagler) commands need their own init hook like the server has job.init() global cfg cfg = pkconfig.init( agent_id=pkconfig.Required(str, 'id of this agent'), fastcgi_sock_dir=( pkio.py_path('/tmp'), pkio.py_path, 'directory of fastcfgi socket, must be less than 50 chars'), start_delay=(0, pkconfig.parse_seconds, 'delay startup in internal_test mode'), supervisor_uri=pkconfig.Required( str, 'how to connect to the supervisor', ), ) pkdlog('{}', cfg) if pkconfig.channel_in_internal_test() and cfg.start_delay: pkdlog('start_delay={}', cfg.start_delay) time.sleep(cfg.start_delay) i = tornado.ioloop.IOLoop.current() d = _Dispatcher() def s(*args): return i.add_callback_from_signal(_terminate, d) signal.signal(signal.SIGTERM, s) signal.signal(signal.SIGINT, s) i.spawn_callback(d.loop) i.start()
def _cfg_uid(value): from sirepo import simulation_db if value and value == 'dev-no-validate' and pkconfig.channel_in_internal_test(): return value assert simulation_db.user_path(value).check(dir=True), \ 'uid={} does not exist'.format(value) return value
def b(msg, dev=False): return ( pkconfig.channel_in('dev') if dev else pkconfig.channel_in_internal_test(), bool, msg, )
async def _agent_start(self, op): if self._agent_starting_timeout: return async with self._agent_start_lock: # POSIT: we do not have to raise Awaited(), because # this is the first thing an op waits on. if self._agent_starting_timeout or self._websocket_ready.is_set(): return try: t = self.cfg.agent_starting_secs if pkconfig.channel_in_internal_test(): x = op.msg.pkunchecked_nested_get( 'data.models.dog.favoriteTreat') if x: x = re.search(r'agent_start_delay=(\d+)', x) if x: self._agent_start_delay = int(x.group(1)) t += self._agent_start_delay pkdlog('op={} agent_start_delay={}', op, self._agent_start_delay) pkdlog('{} {} await _do_agent_start', self, op) # All awaits must be after this. If a call hangs the timeout # handler will cancel this task self._agent_starting_timeout = tornado.ioloop.IOLoop.current( ).call_later( t, self._agent_starting_timeout_handler, ) # POSIT: Canceled errors aren't smothered by any of the below calls await self.kill() await self._do_agent_start(op) except Exception as e: pkdlog('{} error={} stack={}', self, e, pkdexc()) self.free_resources(internal_error='failure starting agent') raise
def create_token(value): if pkconfig.channel_in_internal_test() and cfg.create_token_secret: v = base64.b32encode( hashlib.sha256( pkcompat.to_bytes(value + cfg.create_token_secret)).digest()) return pkcompat.from_bytes(v[:TOKEN_SIZE]) return random_base62(TOKEN_SIZE)
def _init(): from pykern import pkconfig global _cfg @pkconfig.parse_none def _cfg_sim_types(value): res = pkconfig.parse_set(value) if not res: return tuple(_codes()) for c in res: assert c in _codes(), \ 'invalid sim_type={}, expected one of={}'.format(c, _codes()) if 'jspec' in res: res = set(res) res.add('elegant') return tuple(res) def _codes(): return ALL_CODES if pkconfig.channel_in_internal_test() \ else NON_ALPHA_CODES _cfg = pkconfig.init( api_modules=((), set, 'optional api modules, e.g. status'), job=(False, bool, '[new] job execution architecture (replaces runner)'), jspec=dict(derbenevskrinsky_force_formula=( pkconfig.channel_in_internal_test(), bool, 'Include Derbenev-Skrinsky force forumla'), ), #TODO(robnagler) make sim_type config rs4pi_dose_calc=(False, bool, 'run the real dose calculator'), sim_types=(None, _cfg_sim_types, 'simulation types (codes) to be imported'), srw=dict( mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'), beamline3d=(pkconfig.channel_in_internal_test(), bool, 'Show 3D beamline plot'), ), warpvnd=dict( allow_3d_mode=(True, bool, 'Include 3D features in the Warp VND UI'), display_test_boxes=( pkconfig.channel_in_internal_test(), bool, 'Display test boxes to visualize 3D -> 2D projections'), ), )
def _init(): if pkconfig.channel_in_internal_test(): from sirepo import uri_router uri_router.register_api_module() else: global utc_now_as_float, utc_now utc_now_as_float = time.time utc_now = datetime.datetime.utcnow
def init(): if pkconfig.channel_in_internal_test(): return global _initialized, utc_now_as_float, utc_now if _initialized: return _initialized = True utc_now_as_float = time.time utc_now = datetime.datetime.utcnow
def utc_now_as_float(): """Adjusted POSIX time as a float Returns: float: adjusted `time.time` """ assert pkconfig.channel_in_internal_test() if _timedelta is None: return time.time() return to_timestamp(utc_now())
def init(): global _initialized, utc_now_as_int if _initialized: return _initialized = True if not pkconfig.channel_in_internal_test(): global utc_now_as_float, utc_now utc_now_as_float = time.time utc_now = datetime.datetime.utcnow utc_now_as_int = lambda: int(utc_now_as_float())
def utc_now(): """Adjusted UTC time as object Returns: datetime.datetime: adjusted `datetime.datetime.utcnow` """ assert pkconfig.channel_in_internal_test() if _timedelta is None: return datetime.datetime.utcnow() return datetime.datetime.utcnow() + _timedelta
def _init(): from pykern import pkconfig global _cfg _cfg = pkconfig.init( # No secrets should be stored here (see sirepo.job.agent_env) api_modules=((), set, 'optional api modules, e.g. status'), jspec=dict( derbenevskrinsky_force_formula=(pkconfig.channel_in_internal_test(), bool, 'Include Derbenev-Skrinsky force formula'), ), proprietary_sim_types=(set(), set, 'codes that require authorization'), #TODO(robnagler) make this a sim_type config like srw and warpvnd rs4pi_dose_calc=(False, bool, 'run the real dose calculator'), sim_types=(set(), set, 'simulation types (codes) to be imported'), srw=dict( mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'), beamline3d=(pkconfig.channel_in_internal_test(), bool, 'Show 3D beamline plot'), app_url=('/en/xray-beamlines.html', str, 'URL for SRW link'), hide_guest_warning=(False, bool, 'Hide the guest warning in the UI'), ), warpvnd=dict( allow_3d_mode=(True, bool, 'Include 3D features in the Warp VND UI'), display_test_boxes=(pkconfig.channel_in_internal_test(), bool, 'Display test boxes to visualize 3D -> 2D projections'), ), ) s = set( _cfg.sim_types or ( _FOSS_CODES if pkconfig.channel_in_internal_test() else _NON_ALPHA_FOSS_CODES ) ) s.update(_cfg.proprietary_sim_types) # jspec imports elegant, but elegant won't work if it is not a valid # sim_type so need to include here. Need a better model of # dependencies between codes. if 'jspec' in s and 'elegant' not in s: s.add('elegant') x = s.difference(VALID_CODES) assert not x, \ 'sim_type(s) invalid={} expected={}'.format(x, VALID_CODES) _cfg.sim_types = frozenset(s) return _cfg
def api_adjustTime(days=None): """Shift the system time by days and get the adjusted time Args: days (str): must be integer. If None or 0, no adjustment. """ from sirepo import http_reply assert pkconfig.channel_in_internal_test(), \ 'API forbidden' adjust_time(days) return http_reply.gen_json_ok({ 'adjustedNow': utc_now().isoformat(), 'systemNow': datetime.datetime.utcnow().isoformat(), })
def _simulation_error(err, *args, **kwargs): """Something unexpected went wrong. Parses ``err`` for error Args: err (str): exception or run_log quiet (bool): don't write errors to log Returns: dict: error response """ if not kwargs.get('quiet'): pkdlog('{}', ': '.join([str(a) for a in args] + ['error', err])) m = re.search(_SUBPROCESS_ERROR_RE, str(err)) if m: err = m.group(1) if re.search(r'error exit\(-15\)', err): err = 'Terminated' elif not pkconfig.channel_in_internal_test(): err = 'unexpected error (see logs)' return {'state': 'error', 'error': err}
def _simulation_error(err, *args, **kwargs): """Something unexpected went wrong. Parses ``err`` for error Args: err (str): exception or run_log quiet (bool): don't write errors to log Returns: dict: error response """ if not kwargs.get('quiet'): pkdlog('{}', ': '.join([str(a) for a in args] + ['error', err])) m = re.search(_SUBPROCESS_ERROR_RE, str(err)) if m: err = m.group(1) if re.search(r'error exit\(-15\)', err): err = 'Terminated' elif not pkconfig.channel_in_internal_test(): err = 'unexpected error (see logs)' return {'state': 'error', 'error': err}
def _auth_state(): import sirepo.simulation_db s = cookie.unchecked_get_value(_COOKIE_STATE) v = pkcollections.Dict( avatarUrl=None, displayName=None, guestIsOnlyMethod=not non_guest_methods, isGuestUser=False, isLoggedIn=_is_logged_in(s), isLoginExpired=False, jobRunModeMap=sirepo.simulation_db.JOB_RUN_MODE_MAP, method=cookie.unchecked_get_value(_COOKIE_METHOD), needCompleteRegistration=s == _STATE_COMPLETE_REGISTRATION, roles=[], userName=None, visibleMethods=visible_methods, ) if 'sbatch' in v.jobRunModeMap: v.sbatchQueueMaxes=job.NERSC_QUEUE_MAX u = cookie.unchecked_get_value(_COOKIE_USER) if v.isLoggedIn: if v.method == METHOD_GUEST: # currently only method to expire login v.displayName = _GUEST_USER_DISPLAY_NAME v.isGuestUser = True v.isLoginExpired = _METHOD_MODULES[METHOD_GUEST].is_login_expired() v.needCompleteRegistration = False v.visibleMethods = non_guest_methods else: r = auth_db.UserRegistration.search_by(uid=u) if r: v.displayName = r.display_name v.roles = auth_db.UserRole.get_roles(u) _plan(v) _method_auth_state(v, u) if pkconfig.channel_in_internal_test(): # useful for testing/debugging v.uid = u pkdc('state={}', v) return v
def internal_build(self): from rsconf import systemd from rsconf.component import bop # Must be after bop so bconf_f exists, and bop has to be on the machine self.buildt.require_component('bop') j2_ctx = self.hdb.j2_ctx_copy() z = j2_ctx.bop_timer for app_name in sorted(j2_ctx.bop.apps): app_vars = bop.merge_app_vars(j2_ctx, app_name) if pkconfig.channel_in_internal_test( channel=j2_ctx.rsconf_db.channel): self._add_initdb_spec(j2_ctx, z, app_vars) if not app_name in z.spec: continue z.run_u = app_vars.run_u z.bconf_f = app_vars.bconf_f timers = z.spec[app_name] for t in sorted(timers.keys()): tv = timers[t] z.bash_script = tv.bash_script run_u = tv.get('run_u', z.run_u) timer_name = '{}_{}'.format(app_name, t) run_d = systemd.timer_prepare( self, j2_ctx, #TODO(robnagler) time zone on_calendar=tv.on_calendar, service_name=timer_name, ) run_f = run_d.join('run') systemd.timer_enable(self, j2_ctx=j2_ctx, cmd=run_f, run_u=run_u) self.install_access(mode='500', owner=run_u) self.install_resource('bop_timer/run.sh', j2_ctx, run_f)
def _cfg_sim_types(value): res = pkconfig.parse_tuple(value) if not res: return _codes() for c in res: assert c in _codes(), \ 'invalid sim_type={}, expected one of={}'.format(c, _codes()) return res def _codes(want_all=pkconfig.channel_in('dev')): return _ALL_CODES if want_all else _NON_DEV_CODES cfg = pkconfig.init( api_modules=((), tuple, 'optional api modules, e.g. bluesky'), #TODO(robnagler) make sim_type config rs4pi_dose_calc=(False, bool, 'run the real dose calculator'), sim_types=(None, _cfg_sim_types, 'simulation types (codes) to be imported'), srw=dict(mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'), ), warpvnd=dict( allow_3d_mode=(pkconfig.channel_in_internal_test(), bool, 'Include 3D features in the Warp VND UI'), display_test_boxes=( pkconfig.channel_in_internal_test(), bool, 'Display test boxes to visualize 3D -> 2D projections'), ), )
def post(self): assert pkconfig.channel_in_internal_test(), \ 'You can only adjust time in internal test' sirepo.srtime.adjust_time(pkjson.load_any(self.request.body).days) self.write(PKDict())
@pkconfig.parse_none def _cfg_sim_types(value): res = pkconfig.parse_tuple(value) if not res: return _codes() for c in res: assert c in _codes(), \ 'invalid sim_type={}, expected one of={}'.format(c, _codes()) return res def _codes(want_all=pkconfig.channel_in('dev')): return _ALL_CODES if want_all else _NON_DEV_CODES cfg = pkconfig.init( api_modules=((), tuple, 'optional api modules, e.g. bluesky'), #TODO(robnagler) make sim_type config rs4pi_dose_calc=(False, bool, 'run the real dose calculator'), sim_types=(None, _cfg_sim_types, 'simulation types (codes) to be imported'), srw=dict( mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'), ), warpvnd=dict( allow_3d_mode=(pkconfig.channel_in_internal_test(), bool, 'Include 3D features in the Warp VND UI'), display_test_boxes=(pkconfig.channel_in_internal_test(), bool, 'Display test boxes to visualize 3D -> 2D projections'), ), )
Returns: dict: application specific config """ if sim_type not in cfg: return {} return pkcollections.map_to_dict(cfg[sim_type]) @pkconfig.parse_none def _cfg_sim_types(value): if not value: return _codes() user_specified_codes = tuple(value.split(':')) for c in user_specified_codes: assert c in _codes(), \ '{}: invalid sim_type, must be one of/combination of: {}'.format(c, _codes()) return user_specified_codes def _codes(want_all=pkconfig.channel_in('dev')): return _ALL_CODES if want_all else _NON_DEV_CODES cfg = pkconfig.init( srw=dict(mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'), ), sim_types=(None, _cfg_sim_types, 'simulation types (codes) to be imported'), rs4pi_dose_calc=(False, bool, 'run the real dose calculator'), )
""" if sim_type not in cfg: return {} return pkcollections.map_to_dict(cfg[sim_type]) @pkconfig.parse_none def _cfg_sim_types(value): if not value: return _codes() user_specified_codes = tuple(value.split(':')) for c in user_specified_codes: assert c in _codes(), \ '{}: invalid sim_type, must be one of/combination of: {}'.format(c, _codes()) return user_specified_codes def _codes(want_all=pkconfig.channel_in('dev')): return _ALL_CODES if want_all else _NON_DEV_CODES cfg = pkconfig.init( srw=dict(mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'), ), warpvnd=dict(particle_3d_report=(pkconfig.channel_in_internal_test(), bool, 'Show the 3d particle report'), ), sim_types=(None, _cfg_sim_types, 'simulation types (codes) to be imported'), rs4pi_dose_calc=(False, bool, 'run the real dose calculator'), )
def _codes(): return ALL_CODES if pkconfig.channel_in_internal_test() \ else NON_ALPHA_CODES
def _codes(want_all=None): if want_all is None: want_all = pkconfig.channel_in_internal_test() return _ALL_CODES if want_all else _NON_ALPHA_CODES
_CPU_PERIOD_US = 100000 #: dump the slots whenever an update happens _POOLS_DUMP_FILE = 'rsdockerspawner_pools.json' #: Default user when no specific volume for user ['*'] _DEFAULT_USER_GROUP = 'everybody' #: Name of the default pool when no user patches _DEFAULT_POOL = _DEFAULT_USER_GROUP #: Large time out for minimum allowed activity (effectively infinite) _DEFAULT_MIN_ACTIVITY_HOURS = 1e6 #: Minimum five mins so we don't garbage collect too frequently _MIN_MIN_ACTIVITY_SECS = 5.0 if pkconfig.channel_in_internal_test() else 300.0 #: Minimum number of processes available to the user not running in Jupyter _MIN_NPROC_AVAIL = 512 #: User that won't match a legimate user _DEFAULT_USER = '******' #: Parameters set in create_object _EXTRA_HOST_CONFIG = ( 'cpu_period', 'cpu_quota', 'pids_limit', 'shm_size', )
#: Configuration cfg = None def for_sim_type(sim_type): """Get cfg for simulation type Args: sim_type (str): srw, warp, etc. Returns: dict: application specific config """ if not sim_type in cfg: return {} return pkcollections.map_to_dict(cfg[sim_type]) @pkconfig.parse_none def _cfg_bool(value): """Convert str to integer and then bool""" if isinstance(value, str): value = int(value) return bool(value) cfg = pkconfig.init( srw=dict( mask_in_toolbar=(pkconfig.channel_in_internal_test(), _cfg_bool, 'Show the mask element in toolbar'), ), )
def for_sim_type(sim_type): """Get cfg for simulation type Args: sim_type (str): srw, warp, etc. Returns: dict: application specific config """ if sim_type not in cfg: return {} return pkcollections.map_to_dict(cfg[sim_type]) @pkconfig.parse_none def _cfg_bool(value): """Convert str to integer and then bool""" if isinstance(value, str): value = int(value) return bool(value) cfg = pkconfig.init(srw=dict( mask_in_toolbar=(pkconfig.channel_in_internal_test(), _cfg_bool, 'Show the mask element in toolbar'), sample_in_toolbar=(pkconfig.channel_in_internal_test(), _cfg_bool, 'Show the sample element in toolbar'), ), )