def default_command():
global cfg
cfg = pkconfig.init(
debug=(pkconfig.channel_in('dev'), bool, 'run supervisor in debug mode'),
ip=(sirepo.job.DEFAULT_IP, str, 'ip to listen on'),
port=(sirepo.job.DEFAULT_PORT, int, 'what port to listen on'),
)
sirepo.job_supervisor.init()
pkio.mkdir_parent(sirepo.job.DATA_FILE_ROOT)
pkio.mkdir_parent(sirepo.job.LIB_FILE_ROOT)
app = tornado.web.Application(
[
(sirepo.job.AGENT_URI, _AgentMsg),
(sirepo.job.SERVER_URI, _ServerReq),
(sirepo.job.SERVER_PING_URI, _ServerPing),
(sirepo.job.DATA_FILE_URI + '/(.*)', _DataFileReq),
],
debug=cfg.debug,
static_path=sirepo.job.SUPERVISOR_SRV_ROOT.join(sirepo.job.LIB_FILE_URI),
# tornado expects a trailing slash
static_url_prefix=sirepo.job.LIB_FILE_URI + '/',
)
server = tornado.httpserver.HTTPServer(app, xheaders=True)
server.listen(cfg.port, cfg.ip)
signal.signal(signal.SIGTERM, _sigterm)
signal.signal(signal.SIGINT, _sigterm)
pkdlog('ip={} port={}', cfg.ip, cfg.port)
tornado.ioloop.IOLoop.current().start()
def init_class():
global cfg
cfg = pkconfig.init(
dev_volumes=(pkconfig.channel_in('dev'), bool,
'mount ~/.pyenv, ~/.local and ~/src for development'),
hosts=pkconfig.RequiredUnlessDev(tuple(), tuple, 'execution hosts'),
image=('radiasoft/sirepo', str, 'docker image to run all jobs'),
parallel=dict(
cores=(2, int, 'cores per parallel job'),
gigabytes=(1, int, 'gigabytes per parallel job'),
slots_per_host=(1, int, 'parallel slots per node'),
),
sequential=dict(
gigabytes=(1, int, 'gigabytes per sequential job'),
slots_per_host=(1, int, 'sequential slots per node'),
),
supervisor_uri=job.DEFAULT_SUPERVISOR_URI_DECL,
tls_dir=pkconfig.RequiredUnlessDev(None, _cfg_tls_dir,
'directory containing host certs'),
)
if not cfg.tls_dir or not cfg.hosts:
_init_dev_hosts()
_init_hosts()
return DockerDriver.init_class()
def init():
global cfg
if cfg:
return
cfg = pkconfig.init(
max_message_bytes=(int(2e8), pkconfig.parse_bytes,
'maximum message size throughout system'),
ping_interval_secs=(
2 * 60, pkconfig.parse_seconds,
'how long to wait between sending keep alive pings'),
ping_timeout_secs=(4 * 60, pkconfig.parse_seconds,
'how long to wait for a ping response'),
server_secret=(
'a very secret, secret',
str,
'shared secret between supervisor and server',
),
verify_tls=(not pkconfig.channel_in('dev'), bool,
'do not validate (self-signed) certs'),
)
global SUPERVISOR_SRV_ROOT, LIB_FILE_ROOT, DATA_FILE_ROOT
SUPERVISOR_SRV_ROOT = sirepo.srdb.root().join(SUPERVISOR_SRV_SUBDIR)
LIB_FILE_ROOT = SUPERVISOR_SRV_ROOT.join(LIB_FILE_URI[1:])
DATA_FILE_ROOT = SUPERVISOR_SRV_ROOT.join(DATA_FILE_URI[1:])
def init_class(cls, job_supervisor):
cls.cfg = pkconfig.init(
agent_starting_secs=(
cls._AGENT_STARTING_SECS_DEFAULT + 3,
int,
'how long to wait for agent start',
),
constrain_resources=(True, bool,
'apply --cpus and --memory constraints'),
dev_volumes=(pkconfig.channel_in('dev'), bool,
'mount ~/.pyenv, ~/.local and ~/src for development'),
hosts=pkconfig.RequiredUnlessDev(tuple(), tuple,
'execution hosts'),
idle_check_secs=pkconfig.ReplacedBy(
'sirepo.job_driver.idle_check_secs'),
image=('radiasoft/sirepo', str, 'docker image to run all jobs'),
parallel=dict(
cores=(2, int, 'cores per parallel job'),
gigabytes=(1, int, 'gigabytes per parallel job'),
slots_per_host=(1, int, 'parallel slots per node'),
),
sequential=dict(
gigabytes=(1, int, 'gigabytes per sequential job'),
slots_per_host=(1, int, 'sequential slots per node'),
),
supervisor_uri=job.DEFAULT_SUPERVISOR_URI_DECL,
tls_dir=pkconfig.RequiredUnlessDev(
None, _cfg_tls_dir, 'directory containing host certs'),
)
if not cls.cfg.tls_dir or not cls.cfg.hosts:
cls._init_dev_hosts()
cls._init_hosts(job_supervisor)
return cls
def _cfg():
import netrc
global cfg
n = None
p = pkcollections.Dict(
api_pause_seconds=(
0 if pkconfig.channel_in('dev') else 10,
int,
'pauses between backups',
),
exclude_re=(None, _cfg_exclude_re,
'regular expression to exclude a repo'),
keep_days=(
_cfg_keep_days(2),
_cfg_keep_days,
'how many days of backups to keep',
),
password=[None, str, 'github passsword'],
test_mode=(
pkconfig.channel_in('dev'),
pkconfig.parse_bool,
'only backs up test-github-backup repo',
),
user=[None, str, 'github user'],
)
try:
n = netrc.netrc().authenticators('github.com')
for i, k in (0, 'user'), (2, 'password'):
p[k][0] = n[i]
except Exception:
pass
cfg = pkconfig.init(**p)
assert cfg.test_mode or cfg.password is not None and cfg.user is not None, \
'user and password required unless test_mode'
def init_apis(*args, **kwargs):
def _init_model(base):
"""Creates User class bound to dynamic `db` variable"""
global AuthGithubUser, UserModel
class AuthGithubUser(base):
__tablename__ = 'auth_github_user_t'
oauth_id = sqlalchemy.Column(base.STRING_NAME, primary_key=True)
user_name = sqlalchemy.Column(base.STRING_NAME,
unique=True,
nullable=False)
uid = sqlalchemy.Column(base.STRING_ID, unique=True)
UserModel = AuthGithubUser
global cfg, AUTH_METHOD_VISIBLE
cfg = pkconfig.init(
callback_uri=(
None, str,
'Github callback URI (defaults to api_authGithubAuthorized)'),
key=pkconfig.Required(str, 'Github key'),
method_visible=(
True,
bool,
'github auth method is visible to users when it is an enabled method',
),
secret=pkconfig.Required(str, 'Github secret'),
)
AUTH_METHOD_VISIBLE = cfg.method_visible
auth_db.init_model(_init_model)
def init_apis(*args, **kwargs):
global cfg
cfg = pkconfig.init(
dst_db_root=(
pkio.py_path(sirepo.srdb.root()).join('jupyterhub'),
pkio.py_path,
'new jupyter user db',
),
rs_jupyter_migrate=(
False, bool,
'give user option to migrate data from jupyter.radiasoft.org'),
src_db_root=(
pkio.py_path('/var/empty'),
pkio.py_path,
'existing jupyter user db (ex /srv/jupyterhub)',
),
uri_root=('jupyter', str, 'the root uri of jupyterhub'),
)
sirepo.auth_db.init_model(_init_model)
sirepo.events.register({
'auth_logout': _event_auth_logout,
'end_api_call': _event_end_api_call,
'github_authorized': _event_github_authorized,
})
def _init_root():
global _cfg, _root
def _cfg_root(v):
"""Config value or root package's parent or cwd with `_DEFAULT_ROOT`"""
if not os.path.isabs(v):
pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be absolute')
if not os.path.isdir(v):
pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be a directory and exist')
return pkio.py_path(v)
_cfg = pkconfig.init(
root=(None, _cfg_root, 'where database resides'),
)
_root = _cfg.root
if _root:
return _root
assert pkconfig.channel_in('dev'), \
'SIREPO_SRDB_ROOT must be configured except in dev'
r = pkio.py_path(
sys.modules[pkinspect.root_package(_init_root)].__file__,
).dirpath().dirpath()
# Check to see if we are in our dev directory. This is a hack,
# but should be reliable.
if not r.join('requirements.txt').check():
# Don't run from an install directory
r = pkio.py_path('.')
_root = pkio.mkdir_parent(r.join(_DEFAULT_ROOT))
return _root
def init_class(cls, job_supervisor_module):
global job_supervisor
job_supervisor = job_supervisor_module
cls.cfg = pkconfig.init(
agent_log_read_sleep=(
5,
int,
'how long to wait before reading the agent log on start',
),
agent_starting_secs=(
cls._AGENT_STARTING_SECS_DEFAULT * 3,
int,
'how long to wait for agent start',
),
cores=(None, int, 'dev cores config'),
host=pkconfig.Required(str, 'host name for slum controller'),
host_key=pkconfig.Required(str, 'host key'),
run_slots=(1, int, 'number of concurrent OP_RUN for each user'),
shifter_image=(None, str, 'needed if using Shifter'),
sirepo_cmd=pkconfig.Required(str, 'how to run sirepo'),
srdb_root=pkconfig.Required(
_cfg_srdb_root,
'where to run job_agent, must include {sbatch_user}'),
supervisor_uri=job.DEFAULT_SUPERVISOR_URI_DECL,
)
cls._KNOWN_HOSTS = (
cls.cfg.host_key if cls.cfg.host in cls.cfg.host_key else
'{} {}'.format(cls.cfg.host, cls.cfg.host_key)).encode('ascii')
return cls
def init_apis(*args, **kwargs):
global cfg
cfg = pkconfig.init(
uid=pkconfig.Required(_cfg_uid,
'single user allowed to login with basic auth'),
password=pkconfig.Required(str, 'password for uid'),
)
def _init():
import sirepo.mpi
global SCHEMA_COMMON, cfg, JOB_RUN_MODE_MAP
cfg = pkconfig.init(
nfs_tries=(10, int, 'How many times to poll in hack_nfs_write_status'),
nfs_sleep=(0.5, float, 'Seconds sleep per hack_nfs_write_status poll'),
sbatch_display=(None, str, 'how to display sbatch cluster to user'),
tmp_dir=(None, pkio.py_path, 'Used by utilities (not regular config)'),
)
fn = STATIC_FOLDER.join('json/schema-common{}'.format(JSON_SUFFIX))
with open(str(fn)) as f:
SCHEMA_COMMON = json_load(f)
# In development, any schema update creates a new version
if pkconfig.channel_in('dev'):
SCHEMA_COMMON.version = max([
_timestamp(pkio.py_path(fn).mtime()) \
for fn in glob.glob(str(STATIC_FOLDER.join('json/*{}'.format(JSON_SUFFIX))))
])
else:
SCHEMA_COMMON.version = sirepo.__version__
JOB_RUN_MODE_MAP = PKDict(
sequential='Serial',
parallel='{} cores (SMP)'.format(sirepo.mpi.cfg.cores),
)
if cfg.sbatch_display:
JOB_RUN_MODE_MAP.sbatch = cfg.sbatch_display
def init_class(app, uwsgi):
global cfg, _hosts, _parallel_cores
if _hosts:
return
cfg = pkconfig.init(
hosts=(None, _cfg_hosts, 'execution hosts'),
image=('radiasoft/sirepo', str, 'docker image to run all jobs'),
tls_dir=(None, _cfg_tls_dir, 'directory containing host certs'),
)
if not cfg.tls_dir or not cfg.hosts:
_init_dev_hosts(app)
_hosts = pkcollections.Dict()
_parallel_cores = mpi.cfg.cores
# Require at least three levels to the domain name
# just to make the directory parsing easier.
for h in cfg.hosts:
d = cfg.tls_dir.join(h)
_hosts[h] = pkcollections.Dict(name=h, cmd_prefix=_cmd_prefix(h, d))
_init_host(h)
assert len(_hosts) > 0, \
'{}: no docker hosts found in directory'.format(_tls_d)
_init_hosts_slots_balance()
_init_slots()
_init_parse_jobs()
_init_slot_managers()
return DockerJob
def start():
#TODO(robnagler) commands need their own init hook like the server has
job.init()
global cfg
cfg = pkconfig.init(
agent_id=pkconfig.Required(str, 'id of this agent'),
fastcgi_sock_dir=(
pkio.py_path('/tmp'), pkio.py_path,
'directory of fastcfgi socket, must be less than 50 chars'),
start_delay=(0, pkconfig.parse_seconds,
'delay startup in internal_test mode'),
supervisor_uri=pkconfig.Required(
str,
'how to connect to the supervisor',
),
)
pkdlog('{}', cfg)
if pkconfig.channel_in_internal_test() and cfg.start_delay:
pkdlog('start_delay={}', cfg.start_delay)
time.sleep(cfg.start_delay)
i = tornado.ioloop.IOLoop.current()
d = _Dispatcher()
def s(*args):
return i.add_callback_from_signal(_terminate, d)
signal.signal(signal.SIGTERM, s)
signal.signal(signal.SIGINT, s)
i.spawn_callback(d.loop)
i.start()
def _cfg():
global __cfg
if not __cfg:
__cfg = pkconfig.init(
ip=('0.0.0.0', _cfg_ip, 'what IP address to open'),
jupyterhub_port=(8002, _cfg_port,
'port on which jupyterhub listens'),
jupyterhub_debug=(
True,
bool,
'turn on debugging for jupyterhub (hub, spawner, ConfigurableHTTPProxy)',
),
nginx_proxy_port=(8080, _cfg_port,
'port on which nginx_proxy listens'),
port=(8000, _cfg_port, 'port on which uwsgi or http listens'),
processes=(1, _cfg_int(1,
16), 'how many uwsgi processes to start'),
run_dir=(None, str, 'where to run the program (defaults db_dir)'),
# uwsgi got hung up with 1024 threads on a 4 core VM with 4GB
# so limit to 128, which is probably more than enough with
# this application.
threads=(10, _cfg_int(1, 128),
'how many uwsgi threads in each process'),
use_reloader=(pkconfig.channel_in('dev'), bool,
'use the Flask reloader'),
)
return __cfg
def init_apis(*args, **kwargs):
global cfg
cfg = pkconfig.init(
secret=pkconfig.Required(
str,
'Shared secret between Sirepo and BlueSky server',
),
)
def _init():
from pykern import pkconfig
global _cfg
def b(msg, dev=False):
return (
pkconfig.channel_in('dev')
if dev else pkconfig.channel_in_internal_test(),
bool,
msg,
)
_cfg = pkconfig.init(
# No secrets should be stored here (see sirepo.job.agent_env)
api_modules=((), set, 'optional api modules, e.g. status'),
default_proprietary_sim_types=
(set(), set,
'codes where all users are authorized by default but that authorization can be revoked'
),
jspec=dict(derbenevskrinsky_force_formula=b(
'Include Derbenev-Skrinsky force formula'), ),
proprietary_sim_types=(set(), set, 'codes that require authorization'),
#TODO(robnagler) make this a sim_type config like srw and warpvnd
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
sim_types=(set(), set, 'simulation types (codes) to be imported'),
srw=dict(
app_url=('/en/xray-beamlines.html', str, 'URL for SRW link'),
beamline3d=b('Show 3D beamline plot'),
hide_guest_warning=b('Hide the guest warning in the UI', dev=True),
mask_in_toolbar=b('Show the mask element in toolbar'),
show_open_shadow=(
pkconfig.channel_in_internal_test(), bool,
'Show "Open as a New Shadow Simulation" menu item'),
show_rsopt_ml=(pkconfig.channel_in_internal_test(), bool,
'Show "Export ML Script" menu item'),
),
warpvnd=dict(
allow_3d_mode=(True, bool,
'Include 3D features in the Warp VND UI'),
display_test_boxes=b(
'Display test boxes to visualize 3D -> 2D projections'),
),
)
i = _cfg.proprietary_sim_types.intersection(
_cfg.default_proprietary_sim_types)
assert not i, \
f'{i}: cannot be in proprietary_sim_types and default_proprietary_sim_types'
s = set(_cfg.sim_types or
(PROD_FOSS_CODES if pkconfig.channel_in('prod') else _FOSS_CODES))
s.update(_cfg.proprietary_sim_types, _cfg.default_proprietary_sim_types)
for v in _DEPENDENT_CODES:
if v[0] in s:
s.add(v[1])
x = s.difference(VALID_CODES)
assert not x, \
'sim_type(s) invalid={} expected={}'.format(x, VALID_CODES)
_cfg.sim_types = frozenset(s)
return _cfg
def _init():
global cfg
cfg = pkconfig.init(
lib_file_resource_only=(False, bool, 'used by utility programs'),
lib_file_list=(None, lambda v: pkio.read_text(v).split('\n'),
'directory listing of remote lib'),
lib_file_uri=(None, str, 'where to get files from when remote'),
)
def _init():
global SCHEMA_COMMON
with open(str(STATIC_FOLDER.join('json/schema-common{}'.format(JSON_SUFFIX)))) as f:
SCHEMA_COMMON = json_load(f)
global cfg
cfg = pkconfig.init(
nfs_tries=(10, int, 'How many times to poll in hack_nfs_write_status'),
nfs_sleep=(0.5, float, 'Seconds sleep per hack_nfs_write_status poll'),
)
def init_apis(app, *args, **kwargs):
global cfg
cfg = pkconfig.init(
key=pkconfig.Required(str, 'Github key'),
secret=pkconfig.Required(str, 'Github secret'),
callback_uri=(None, str, 'Github callback URI (defaults to api_authGithubAuthorized)'),
)
app.session_interface = _FlaskSessionInterface()
auth_db.init_model(app, _init_model)
def _init():
global cfg
if cfg:
return
p = os.environ.get('PYENV_ROOT')
if p:
p += '/versions/py2/bin/python'
else:
p = 'python2'
cfg = pkconfig.init(python_path=(p, str, 'python executable'), )
def init_apis(*args, **kwargs):
global cfg
cfg = pkconfig.init(
key=pkconfig.Required(str, 'Github key'),
secret=pkconfig.Required(str, 'Github secret'),
callback_uri=(
None, str,
'Github callback URI (defaults to api_authGithubAuthorized)'),
)
auth_db.init_model(_init_model)
def default_command():
global cfg
assert pkconfig.channel_in('dev'), \
'Only to be used in dev. channel={}'.format(pkconfig.cfg.channel)
cfg = pkconfig.init(proprietary_code_uri=(
f'file://{pathlib.Path.home()}/src/radiasoft/rsconf/proprietary',
str,
'root uri of proprietary codes files location',
), )
_proprietary_codes()
def _init():
global SCHEMA_COMMON, cfg
fn = STATIC_FOLDER.join('json/schema-common{}'.format(JSON_SUFFIX))
with open(str(fn)) as f:
SCHEMA_COMMON = json_load(f)
# In development, you can touch schema-common to get a new version
SCHEMA_COMMON.version = _timestamp(fn.mtime()) if pkconfig.channel_in('dev') else sirepo.__version__
cfg = pkconfig.init(
nfs_tries=(10, int, 'How many times to poll in hack_nfs_write_status'),
nfs_sleep=(0.5, float, 'Seconds sleep per hack_nfs_write_status poll'),
)
def _init():
global SCHEMA_COMMON, cfg
fn = STATIC_FOLDER.join('json/schema-common{}'.format(JSON_SUFFIX))
with open(str(fn)) as f:
SCHEMA_COMMON = json_load(f)
# In development, you can touch schema-common to get a new version
SCHEMA_COMMON.version = _timestamp(fn.mtime()) if pkconfig.channel_in('dev') else sirepo.__version__
cfg = pkconfig.init(
nfs_tries=(10, int, 'How many times to poll in hack_nfs_write_status'),
nfs_sleep=(0.5, float, 'Seconds sleep per hack_nfs_write_status poll'),
)
def init_apis(*args, **kwargs):
global cfg
cfg = pkconfig.init(
#TODO(robnagler) validate email
from_email=pkconfig.Required(str, 'From email address'),
from_name=pkconfig.Required(str, 'From display name'),
smtp_password=pkconfig.Required(str, 'SMTP auth password'),
smtp_server=pkconfig.Required(str, 'SMTP TLS server'),
smtp_user=pkconfig.Required(str, 'SMTP auth user'),
)
auth_db.init_model(_init_model)
def init_class():
global cfg
cfg = pkconfig.init(
slots=dict(
parallel=(1, int, 'max parallel slots'),
sequential=(1, int, 'max sequential slots'),
),
supervisor_uri=job.DEFAULT_SUPERVISOR_URI_DECL,
)
return LocalDriver.init_class()
def init_apis(*args, **kwargs):
global cfg
cfg = pkconfig.init(
mail_server=(None, str, 'Mail server'),
mail_username=(None, str, 'Mail user name'),
mail_password=(None, str, 'Mail password'),
mail_support_email=(None, str, 'Support email address'),
mail_recipient_email=(None, str, 'Email to receive registration messages'),
)
assert cfg.mail_server and cfg.mail_username and cfg.mail_password \
and cfg.mail_support_email and cfg.mail_recipient_email, \
'Missing mail config'
def init():
global _DB_DIR, cfg, _NEXT_REQUEST_SECONDS, job_driver
if _DB_DIR:
return
job.init()
from sirepo import job_driver
job_driver.init(pkinspect.this_module())
_DB_DIR = sirepo.srdb.root().join(_DB_SUBDIR)
cfg = pkconfig.init(
job_cache_secs=(300, int, 'when to re-read job state from disk'),
max_hours=dict(
analysis=(.04, float, 'maximum run-time for analysis job',),
parallel=(1, float, 'maximum run-time for parallel job (except sbatch)'),
parallel_premium=(2, float, 'maximum run-time for parallel job for premium user (except sbatch)'),
sequential=(.1, float, 'maximum run-time for sequential job'),
),
purge_free_after_days=(1000, int, 'how many days to wait before purging a free users simulation'),
purge_free_start=('02:00:00', str, 'time to first start purging free users simulations (%H:%M:%S)'),
purge_free_period=('01:00:00', str, 'how often to purge free users simulations after start time (%H:%M:%S)'),
sbatch_poll_secs=(15, int, 'how often to poll squeue and parallel status'),
)
_NEXT_REQUEST_SECONDS = PKDict({
job.PARALLEL: 2,
job.SBATCH: cfg.sbatch_poll_secs,
job.SEQUENTIAL: 1,
})
sirepo.auth_db.init(sirepo.srdb.root(), migrate_db_file=False)
if sirepo.simulation_db.user_dir_name().exists():
if not _DB_DIR.exists():
pkdlog('calling upgrade_runner_to_job_db path={}', _DB_DIR)
import subprocess
subprocess.check_call(
(
'pyenv',
'exec',
'sirepo',
'db',
'upgrade_runner_to_job_db',
_DB_DIR,
),
env=PKDict(os.environ).pkupdate(
PYENV_VERSION='py2',
SIREPO_AUTH_LOGGED_IN_USER='******',
),
)
else:
pykern.pkio.mkdir_parent(_DB_DIR)
tornado.ioloop.IOLoop.current().add_callback(
_ComputeJob.purge_free_simulations,
init=True,
)
def init(job_supervisor_module):
global cfg, _CLASSES, _DEFAULT_CLASS, job_supervisor
assert not cfg
job_supervisor = job_supervisor_module
cfg = pkconfig.init(modules=((_DEFAULT_MODULE, ), set,
'available job driver modules'), )
_CLASSES = PKDict()
p = pkinspect.this_module().__name__
for n in cfg.modules:
m = importlib.import_module(pkinspect.module_name_join((p, n)))
_CLASSES[n] = m.CLASS.init_class(job_supervisor)
_DEFAULT_CLASS = _CLASSES.get('docker') or _CLASSES.get(_DEFAULT_MODULE)
pkdlog('modules={}', sorted(_CLASSES.keys()))
def init_apis(app):
uri_router.register_api_module()
global cfg
cfg = pkconfig.init(
username=(None, str, 'Sirepo status user id'),
password=(None, str, 'Basic Auth password'),
)
if not cfg.username or not cfg.password:
raise RuntimeError('Missing Sirepo status config')
app.config['BASIC_AUTH_USERNAME'] = cfg.username
app.config['BASIC_AUTH_PASSWORD'] = cfg.password
global _basic_auth
_basic_auth = BasicAuth(app)
def _init():
global SCHEMA_COMMON, cfg
cfg = pkconfig.init(
nfs_tries=(10, int, 'How many times to poll in hack_nfs_write_status'),
nfs_sleep=(0.5, float, 'Seconds sleep per hack_nfs_write_status poll'),
sbatch_display=(None, str, 'how to display sbatch cluster to user'),
tmp_dir=(None, pkio.py_path, 'Used by utilities (not regular config)'),
)
fn = STATIC_FOLDER.join('json/schema-common{}'.format(JSON_SUFFIX))
with open(str(fn)) as f:
SCHEMA_COMMON = json_load(f)
# In development, you can touch schema-common to get a new version
SCHEMA_COMMON.version = _timestamp(fn.mtime()) if pkconfig.channel_in('dev') else sirepo.__version__
SCHEMA_COMMON.common.enum.JobRunMode = _init_JobRunMode()
def _init(app):
global _db
app.session_interface = _FlaskSessionInterface()
app.config.update(
SQLALCHEMY_DATABASE_URI='sqlite:///{}'.format(_db_filename(app)),
SQLALCHEMY_COMMIT_ON_TEARDOWN=True,
SQLALCHEMY_TRACK_MODIFICATIONS=False,
)
_db = SQLAlchemy(app, session_options=dict(autoflush=True))
global cfg
cfg = pkconfig.init(
github_key=(None, str, 'GitHub application key'),
github_secret=(None, str, 'GitHub application secret'),
github_callback_uri=(None, str, 'GitHub application callback URI'),
)
if not cfg.github_key or not cfg.github_secret:
raise RuntimeError('Missing GitHub oauth config')
def init_apis(app):
global cfg
cfg = pkconfig.init(
mail_server=(None, str, 'Mail server'),
mail_username=(None, str, 'Mail user name'),
mail_password=(None, str, 'Mail password'),
mail_support_email=(None, str, 'Support email address'),
mail_recipient_email=(None, str, 'Email to receive registration messages'),
)
assert cfg.mail_server and cfg.mail_username and cfg.mail_password \
and cfg.mail_support_email and cfg.mail_recipient_email, \
'Missing mail config'
app.config.update(
MAIL_USE_TLS=True,
MAIL_PORT=587,
MAIL_SERVER=cfg.mail_server,
MAIL_USERNAME=cfg.mail_username,
MAIL_PASSWORD=cfg.mail_password,
)
global _mail
_mail = flask_mail.Mail(app)
uri_router.register_api_module()
)
t = req.authNonce.split(_AUTH_NONCE_SEPARATOR)[0]
try:
t = int(t)
except ValueError as e:
util.raise_not_found(
'{}: auth_nonce prefix not an int: nonce={}',
t,
req.authNonce,
)
delta = now - t
if abs(delta) > _AUTH_NONCE_REPLAY_SECS:
util.raise_not_found(
'{}: auth_nonce time outside replay window={} now={} nonce={}',
t,
_AUTH_NONCE_REPLAY_SECS,
now,
req.authNonce,
)
def init_apis(app):
assert cfg.auth_secret, \
'sirepo_bluesky_auth_secret is not configured'
uri_router.register_api_module()
cfg = pkconfig.init(
auth_secret=(None, str, 'Shared secret between Sirepo and BlueSky server'),
)
pkconfig.append_load_path('sirepo')
import importlib
import os
import sys
from pykern.pkdebug import pkdc, pkdp
from pykern import pkcollections
from celery import Celery
celery = Celery('sirepo')
cfg = pkconfig.init(
broker_url=('amqp://guest@localhost//', str, 'Celery: queue broker url'),
celeryd_concurrency=(1, int, 'how many tasks to run in parallel'),
celeryd_task_time_limit=(3600, int, 'max run time for a task in seconds'),
)
celery.conf.update(
pkcollections.map_items(cfg, op=lambda k, v: (k.upper(), v)),
)
celery.conf.update(
CELERYD_LOG_COLOR=False,
CELERYD_MAX_TASKS_PER_CHILD=1,
CELERYD_PREFETCH_MULTIPLIER=1,
CELERYD_TASK_SOFT_TIME_LIMIT=celery.conf['CELERYD_TASK_TIME_LIMIT'] - 10,
CELERY_ACKS_LATE=True,
CELERY_RESULT_BACKEND = 'rpc',
CELERY_RESULT_PERSISTENT=True,
@pkconfig.parse_none
def _cfg_session_secret(value):
"""Reads file specified as config value"""
if not value:
assert pkconfig.channel_in('dev'), 'missing session secret configuration'
return 'dev dummy secret'
with open(value) as f:
return f.read()
def _init_maps():
import sirepo.cookie
res = {
'key': {
_ORIG_KEY: sirepo.cookie._COOKIE_USER,
},
'value': {}
}
if oauth_hook:
oauth_hook(res)
return res
cfg = pkconfig.init(
key=('sirepo_' + pkconfig.cfg.channel, str, 'Beaker: Name of the cookie key used to save the session under'),
secret=(None, _cfg_session_secret, 'Beaker: Used with the HMAC to ensure session integrity'),
)
if v is None:
return 999
return int(v)
cfg = pkconfig.init(
dict1=({
'd1': 'default1',
'd2': 'default2',
}, dict, 'first param is dict'),
list2=(['second1'], list, 'second param is list'),
p3=(1313, int, 'third param is int'),
p4=(None, int, 'fourth param is 10x p3'),
p5=(os.environ['HOME'], str, 'value of $HOME'),
p6=(None, _custom_p6, 'sixth param is a custom parser'),
list7=(['default7'], list, 'seventh param is a list '),
req8=pkconfig.Required(int, 'an eighth required parameter'),
sub_params9=dict(
sub9_1=(None, int, 'sub param is first of ninth group'),
sub9_2=dict(
sub9_2_1=(44, int, 'sub 9.2.1')
),
),
dynamic_default10=(None, _some_key, 'sub dynamic default by parsing None'),
bool1=(False, bool, 'a False boolean'),
bool2=(True, bool, 'a True boolean'),
bool3=(True, bool, 'a True boolean will be overriden'),
bool4=(False, bool, 'a False boolean will be overriden'),
)
if cfg.p4 is None:
cfg.p4 = cfg.p3 * 10
def _validate_serial(data):
"""Verify serial in data validates
Args:
data (dict): request with serial and possibly models
Returns:
object: None if all ok, or json response if invalid
"""
res = simulation_db.validate_serial(data)
if not res:
return None
return _json_response({
'state': 'error',
'error': 'invalidSerial',
'simulationData': res,
})
cfg = pkconfig.init(
beaker_session=dict(
key=('sirepo_{PYKERN_PKCONFIG_CHANNEL}', str, 'Beaker: Name of the cookie key used to save the session under'),
secret=(None, _cfg_session_secret, 'Beaker: Used with the HMAC to ensure session integrity'),
secure=(False, bool, 'Beaker: Whether or not the session cookie should be marked as secure'),
),
job_queue=('Background', runner.cfg_job_queue, 'how to run long tasks: Celery or Background'),
foreground_time_limit=(5 * 60, _cfg_time_limit, 'timeout for short (foreground) tasks'),
)
pass
def _cfg_job_class(value):
"""Return job queue class based on name
Args:
value (object): May be class or str.
Returns:
object: `Background` or `Celery` class.
"""
v = value.lower()
assert v in _JOB_CLASSES, \
'{}: invalid job_class, not in {}'.format(v, _JOB_CLASSES)
return v
cfg = pkconfig.init(
import_secs=(10, int, 'maximum runtime of backgroundImport'),
# default is set in init(), because of server.cfg.job_gueue
job_class=(
None,
_cfg_job_class,
'how to run jobs: {}'.format(', '.join(_JOB_CLASSES)),
),
parallel_secs=(3600, int, 'maximum runtime of serial job'),
sequential_secs=(300, int, 'maximum runtime of serial job'),
)
#: Configuration
cfg = None
def for_sim_type(sim_type):
"""Get cfg for simulation type
Args:
sim_type (str): srw, warp, etc.
Returns:
dict: application specific config
"""
if not sim_type in cfg:
return {}
return pkcollections.map_to_dict(cfg[sim_type])
@pkconfig.parse_none
def _cfg_bool(value):
"""Convert str to integer and then bool"""
if isinstance(value, str):
value = int(value)
return bool(value)
cfg = pkconfig.init(
srw=dict(
mask_in_toolbar=(pkconfig.channel_in_internal_test(), _cfg_bool, 'Show the mask element in toolbar'),
),
)
def _custom_p6(v):
import dateutil.parser
return dateutil.parser.parse(v)
@pkconfig.parse_none
def _some_key(v):
if v is None:
return 999
return int(v)
cfg = pkconfig.init(
dict1=({
'd1': 'default1',
'd2': 'default2',
}, dict, 'first param is dict'),
list2=(['second1'], list, 'second param is list'),
p3=(1313, int, 'third param is int'),
p4=('{P1_M1_P3}0', int, 'fourth param is 10x p3'),
p5=('{HOME}', str, 'value of $HOME'),
p6=(None, _custom_p6, 'sixth param is a custom parser'),
list7=(['default7'], list, 'seventh param is a list '),
req8=pkconfig.Required(int, 'an eighth required parameter'),
sub_params9=dict(
sub9_1=(None, int, 'sub param is first of ninth group'),
sub9_2=dict(
sub9_2_1=(44, int, 'sub 9.2.1')
),
),
dynamic_default10=(None, _some_key, 'sub dynamic default by parsing None'),
)
assert self.average is not None, \
'self.average is None and has not been initialized'
return self.average
def _Privy(object):
"""This is a private class that does nothing"""
pass
def _cfg_length(anything):
"""Configuration parser for any_length
Args:
anything (object): configured value
Returns:
int: value between 1 and 999
"""
anything = int(anything)
assert 0 < anything <= 999, \
'{}: any_length must be from 1 to 999'
return anything
# Finally we assign any length. Note that we include a trailing , at the
# end of every line in a list so that you don't have to remember to
# add the comma when you add another line.
cfg = pkconfig.init(
any_length=(1, _cfg_length, 'A length used by this module'),
)
pkconfig.append_load_path('sirepo')
from celery import Celery
from pykern import pkcollections
from pykern import pkio
from pykern import pksubprocess
from pykern.pkdebug import pkdc, pkdexc, pkdp, pkdlog
from sirepo.template import template_common
import py.path
celery = Celery('sirepo')
cfg = pkconfig.init(
broker_url=('amqp://guest@localhost//', str, 'Celery: queue broker url'),
celery_result_backend=('rpc://', str, 'configure db other than default'),
celeryd_concurrency=(1, int, 'how many worker processes to start'),
celeryd_task_time_limit=(3600, int, 'max run time for a task in seconds'),
)
celery.conf.update(
pkcollections.map_items(cfg, op=lambda k, v: (k.upper(), v)),
)
_SERIALIZER = 'json'
celery.conf.update(
CELERYD_LOG_COLOR=False,
CELERYD_MAX_TASKS_PER_CHILD=1,
CELERYD_PREFETCH_MULTIPLIER=1,
CELERYD_TASK_SOFT_TIME_LIMIT=celery.conf['CELERYD_TASK_TIME_LIMIT'] - 10,
CELERY_ACCEPT_CONTENT=[_SERIALIZER],
def _process_output(filename, model_data):
simulation_db.write_result(extract_report_data(filename, model_data))
def _run_srw():
#TODO(pjm): need to properly escape data values, untrusted from client
data = simulation_db.read_json(template_common.INPUT_BASE_NAME)
if data['report'] == 'mirrorReport':
#TODO(pjm): mirror report should use it's own jinja template
_process_output(_mirror_plot(data), data)
return
# This defines the main() function:
exec(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE), locals(), locals())
main()
_process_output(get_filename_for_model(data['report']), data)
def _cfg_int(lower, upper):
def wrapper(value):
v = int(value)
assert lower <= v <= upper, \
'value must be from {} to {}'.format(lower, upper)
return v
return wrapper
cfg = pkconfig.init(
particles_per_core=(5, int, 'particles for each core to process'),
)
@pkconfig.parse_none
def _cfg_sim_types(value):
res = pkconfig.parse_tuple(value)
if not res:
return _codes()
for c in res:
assert c in _codes(), \
'invalid sim_type={}, expected one of={}'.format(c, _codes())
return res
def _codes(want_all=pkconfig.channel_in('dev')):
return _ALL_CODES if want_all else _NON_DEV_CODES
cfg = pkconfig.init(
api_modules=((), tuple, 'optional api modules, e.g. bluesky'),
#TODO(robnagler) make sim_type config
rs4pi_dose_calc=(False, bool, 'run the real dose calculator'),
sim_types=(None, _cfg_sim_types, 'simulation types (codes) to be imported'),
srw=dict(
mask_in_toolbar=(pkconfig.channel_in_internal_test(), bool, 'Show the mask element in toolbar'),
),
warpvnd=dict(
allow_3d_mode=(pkconfig.channel_in_internal_test(), bool, 'Include 3D features in the Warp VND UI'),
display_test_boxes=(pkconfig.channel_in_internal_test(), bool, 'Display test boxes to visualize 3D -> 2D projections'),
),
)
outfile = v.si_fn
elif data['report'] == 'mirrorReport':
_process_output(_mirror_plot(data), data)
return
elif re.search('^watchpointReport', data['report']):
op = get_beamline_optics()
v.ws = True
outfile = v.ws_fni
else:
raise Exception('unknown report: {}'.format(data['report']))
if isinstance(mag, srwlib.SRWLGsnBm):
mag = None
srwl_bl.SRWLBeamline(_name=v.name, _mag_approx=mag).calc_all(v, op)
_process_output(outfile, data)
def _cfg_int(lower, upper):
def wrapper(value):
v = int(value)
assert lower <= v <= upper, \
'value must be from {} to {}'.format(lower, upper)
return v
return wrapper
cfg = pkconfig.init(
slave_processes=(1, int, 'cores to use for run_background slaves'),
particles_per_slave=(5, int, 'particles for each core to process'),
total_particles=(50000, int, 'total number of particles to process'),
)
return None
if isinstance(anything, _RE_TYPE):
return anything
return re.compile(anything, flags=re.IGNORECASE)
@pkconfig.parse_none
def _cfg_output(anything):
if anything is None:
return None
if hasattr(anything, 'write'):
return anything
return open(anything, 'w')
def _z(msg):
"""Useful for debugging this module"""
with open('/dev/tty', 'w') as f:
f.write(str(msg) + '\n')
cfg = pkconfig.init(
control=(None, _cfg_control, 'Pattern to match against pkdc messages'),
output=(None, _cfg_output, 'Where to write messages either as a "writable" or file name'),
redirect_logging=(False, bool, "Redirect Python's logging to output"),
want_pid_time=(False, bool, 'Display pid and time in messages'),
)
if cfg:
init()
raise
def run_script(script):
"""Execute python script with mpi.
Args:
script (str): python text
"""
abort = '''
from mpi4py import MPI
if MPI.COMM_WORLD.Get_rank():
import signal
signal.signal(signal.SIGTERM, lambda x, y: MPI.COMM_WORLD.Abort(1))
'''
n = re.sub(r'^from __future.*', abort, script, count=1, flags=re.MULTILINE)
script = abort + script if n == script else n
fn = 'mpi_run.py'
pkio.write_text(fn, script)
p = None
return run_program([sys.executable or 'python', fn])
cfg = pkconfig.init(
cores=(1, int, 'cores to use per run'),
slaves=(1, int, 'DEPRECATED: set $SIREPO_MPI_CORES'),
)
cfg.cores = max(cfg.cores, cfg.slaves)
def _cfg_int(lower, upper):
def wrapper(value):
v = int(value)
assert lower <= v <= upper, \
'value must be from {} to {}'.format(lower, upper)
return v
return wrapper
def _db_dir():
return pkio.mkdir_parent(cfg.db_dir)
def _run_dir():
return pkio.mkdir_parent(cfg.run_dir)
cfg = pkconfig.init(
db_dir=(None, _cfg_db_dir, 'where database resides'),
run_dir=('{SIREPO_PKCLI_SERVICE_DB_DIR}', str, 'where to run the program'),
port=(8000, _cfg_int(5001, 32767), 'port to listen on'),
processes=(1, _cfg_int(1, 16), 'how many uwsgi processes to start'),
# uwsgi got hung up with 1024 threads on a 4 core VM with 4GB
# so limit to 128, which is probably more than enough with
# this application.
threads=(10, _cfg_int(1, 128), 'how many uwsgi threads in each process'),
ip=('0.0.0.0', _cfg_ip, 'what IP address to open'),
)
return wrapper
def _cfg_ip(value):
try:
socket.inet_aton(value)
return value
except socket.error:
pkcli.command_error('{}: ip is not a valid IPv4 address', value)
def _run_dir():
from sirepo import server
if not isinstance(cfg.run_dir, type(py.path.local())):
cfg.run_dir = pkio.mkdir_parent(cfg.run_dir) if cfg.run_dir else server.cfg.db_dir.new()
return cfg.run_dir
cfg = pkconfig.init(
ip=('0.0.0.0', _cfg_ip, 'what IP address to open'),
nginx_proxy_port=(8080, _cfg_int(5001, 32767), 'port on which nginx_proxy listens'),
port=(8000, _cfg_int(5001, 32767), 'port on which uwsgi or http listens'),
processes=(1, _cfg_int(1, 16), 'how many uwsgi processes to start'),
run_dir=(None, str, 'where to run the program (defaults db_dir)'),
# uwsgi got hung up with 1024 threads on a 4 core VM with 4GB
# so limit to 128, which is probably more than enough with
# this application.
threads=(10, _cfg_int(1, 128), 'how many uwsgi threads in each process'),
)
# -*- coding: utf-8 -*-
u"""?
:copyright: Copyright (c) 2015 RadiaSoft LLC. All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern.pkdebug import pkdc, pkdp
from pykern import pkconfig
cfg = pkconfig.init(
anything=(13, int, 'an int param'),
)
def _cfg_exclude_re(anything):
if isinstance(anything, _RE_TYPE):
return anything
return re.compile(anything, flags=re.IGNORECASE)
def _cfg_keep_days(anything):
if isinstance(anything, datetime.timedelta):
return anything
return datetime.timedelta(days=int(anything))
def _shell(cmd):
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
cfg = pkconfig.init(
api_pause_seconds = (30, int, 'pauses between backups'),
exclude_re=(None, _cfg_exclude_re, 'regular expression to exclude a repo'),
keep_days=(
_cfg_keep_days(2),
_cfg_keep_days,
'how many days of backups to keep',
),
password=pkconfig.Required(str, 'github passsword'),
test_mode=(False, pkconfig.parse_bool, 'only backup this repo'),
user=pkconfig.Required(str, 'github user'),
)
"""
return _base_dir(_WORK_DIR_SUFFIX).ensure(dir=True)
def _base_dir(postfix):
"""Base name with directory.
Args:
postfix (str): what to append to base (``_data`` or ``_work``).
Returns:
py.path.local: base directory with postfix
"""
m = module_under_test or pkinspect.caller_module()
filename = py.path.local(m.__file__)
b = re.sub(r'_test$|^test_', '', filename.purebasename)
assert b != filename.purebasename, \
'{}: module name must end in _test'.format(filename)
return py.path.local(filename.dirname).join(b + postfix).realpath()
def _cfg_json(value):
from pykern import pkjson
if isinstance(value, pkcollections.Dict):
return value
return pkjson.load_any(value)
cfg = pkconfig.init(
aux=(pkcollections.Dict(), _cfg_json, 'extra values for tests for CI (e.g. Travis)'),
)
pkdlog('Cookie decoding failed: {} value={}', err, s)
def _serialize(self):
return _SERIALIZER_SEP.join(
itertools.chain.from_iterable(
[(k, self[k]) for k in sorted(self.keys())],
),
)
@pkconfig.parse_none
def _cfg_http_name(value):
assert re.search(r'^\w{1,32}$', value), \
'must be 1-32 word characters; http_name={}'.format(value)
return value
def _state():
return flask.g.sirepo_cookie
cfg = pkconfig.init(
http_name=('sirepo_' + pkconfig.cfg.channel, _cfg_http_name, 'Set-Cookie name'),
private_key=(None, str, 'urlsafe base64 encrypted 32-byte key'),
is_secure=(
not pkconfig.channel_in('dev'),
pkconfig.parse_bool,
'Add secure attriute to Set-Cookie',
)
)
