def __init__(self, kind, slots, *args, **kwargs):
super(_SlotManager, self).__init__(*args, **kwargs)
self.daemon = True
self.__event = threading.Event()
self.__kind = kind
self.__lock = threading.RLock()
self.__queued_jobs = []
self.__running_slots = pkcollections.Dict()
self.__available_slots = slots
assert slots, \
'{}: no available slots'.format(self.__kind)
self.__cname_prefix = _cname_join(self.__kind)
random.shuffle(self.__available_slots)
self.start()
def fixup_old_data(data):
for m in [
'bunch',
'bunchAnimation',
'bunchAnimation2',
'particle',
'simulationSettings',
'opticsReport',
'twissReport',
'twissReport2',
]:
if m not in data.models:
data.models[m] = pkcollections.Dict({})
template_common.update_model_defaults(data.models[m], m, _SCHEMA)
def host_init(j2_ctx, host):
from rsconf import db
jf = _sasl_password_path(j2_ctx)
if jf.check():
with jf.open() as f:
y = pkjson.load_any(f)
else:
y = pkcollections.Dict()
u = _SASL_PASSWORD_PREFIX + host
if not u in y:
y[u] = db.random_string()
pkjson.dump_pretty(y, filename=jf)
return u, y[u]
def _start(self):
"""Detach a process from the controlling terminal and run it in the
background as a daemon.
"""
#POSIT: jid is valid docker name (word chars and dash)
self.cname = _DOCKER_CONTAINER_PREFIX + self.jid
ctx = pkcollections.Dict(
kill_secs=_KILL_TIMEOUT_SECS,
run_dir=self.run_dir,
run_log=self.run_dir.join(template_common.RUN_LOG),
run_secs=self.__run_secs(),
sh_cmd=self.__sh_cmd(),
)
script = str(self.run_dir.join(_DOCKER_CONTAINER_PREFIX + 'run.sh'))
with open(str(script), 'wb') as f:
f.write(pkjinja.render_resource('runner/docker.sh', ctx))
cmd = [
'run',
#TODO(robnagler) configurable
'--cpus=1',
'--detach',
'--init',
'--log-driver=json-file',
# never should be large, just for output of the monitor
'--log-opt=max-size=1m',
'--memory=1g',
'--name=' + self.cname,
'--network=none',
'--rm',
'--ulimit=core=0',
#TODO(robnagler) this doesn't do anything
# '--ulimit=cpu=1',
'--ulimit=nofile={}'.format(_MAX_OPEN_FILES),
'--user='******'bash',
script,
]
self.cid = self.__docker(cmd)
pkdc(
'{}: started cname={} cid={} dir={} len_jobs={} cmd={}',
self.jid,
self.cname,
self.cid,
self.run_dir,
len(_job_map),
' '.join(cmd),
)
def api_root(simulation_type):
try:
sirepo.template.assert_sim_type(simulation_type)
except AssertionError:
pkdlog('{}: uri not found', simulation_type)
werkzeug.exceptions.abort(404)
if cfg.oauth_login:
from sirepo import oauth
values = oauth.set_default_state()
else:
values = pkcollections.Dict()
values.app_name = simulation_type
values.oauth_login = cfg.oauth_login
return _render_root_page('index', values)
def _list2dict(data_list):
"""
The function converts list of lists to a dictionary with keys from 1st elements and values from 3rd elements.
:param data_list: list of SRW parameters (e.g., 'appParam' in Sirepo's *.py files).
:return out_dict: dictionary with all parameters.
"""
out_dict = pkcollections.Dict()
for i in range(len(data_list)):
out_dict[data_list[i][0]] = data_list[i][2]
return out_dict
def init(app, **imports):
global MIME_TYPE, _RELOAD_JS_ROUTES, _app
_app = app
sirepo.util.setattr_imports(imports)
MIME_TYPE = pkcollections.Dict(
html='text/html',
js='application/javascript',
json=app.config.get('JSONIFY_MIMETYPE', 'application/json'),
py='text/x-python',
)
s = simulation_db.get_schema(sim_type=None)
_RELOAD_JS_ROUTES = frozenset(
(k for k, v in s.localRoutes.items() if v.get('requireReload')), )
def _parse_command_header(command):
res = _parse_command_line(pkcollections.Dict({}), command[0],
'type *name *label2')
for f in ('name', 'label2'):
# don't parse line numbers into name or label2
if f in res and re.search(r'^\d+$', res[f]):
del res[f]
if 'label2' in res:
if 'name' in res:
res['name'] = '{} {}'.format(res['name'], res['label2'])
else:
res['name'] = res['label2']
del res['label2']
return res
def _init_uris(app, simulation_db, sim_types):
global _default_route, _empty_route, srunit_uri, _api_to_route, _uri_to_route
assert not _default_route, \
'_init_uris called twice'
_uri_to_route = pkcollections.Dict()
_api_to_route = pkcollections.Dict()
for k, v in simulation_db.SCHEMA_COMMON.route.items():
r = _split_uri(v)
try:
r.func = _api_funcs[_FUNC_PREFIX + k]
except KeyError:
pkdc('not adding api, because module not registered: uri={}', v)
continue
sirepo.api_auth.assert_api_def(r.func)
r.decl_uri = v
r.name = k
assert not r.base_uri in _uri_to_route, \
'{}: duplicate end point; other={}'.format(v, _uri_to_route[r.base_uri])
_uri_to_route[r.base_uri] = r
_api_to_route[k] = r
if r.base_uri == '':
_default_route = r
if 'srunit' in v:
srunit_uri = v
assert _default_route, \
'missing default route'
_empty_route = _uri_to_route.en
_validate_root_redirect_uris(_uri_to_route, simulation_db)
app.add_url_rule('/<path:path>',
'_dispatch',
_dispatch,
methods=('GET', 'POST'))
app.add_url_rule('/',
'_dispatch_empty',
_dispatch_empty,
methods=('GET', 'POST'))
def api_authState():
s = cookie.unchecked_get_value(_COOKIE_STATE)
v = pkcollections.Dict(
avatarUrl=None,
displayName=None,
guestIsOnlyMethod=not non_guest_methods,
isGuestUser=False,
isLoggedIn=_is_logged_in(s),
isLoginExpired=False,
method=cookie.unchecked_get_value(_COOKIE_METHOD),
needCompleteRegistration=s == _STATE_COMPLETE_REGISTRATION,
userName=None,
visibleMethods=visible_methods,
)
u = cookie.unchecked_get_value(_COOKIE_USER)
if v.isLoggedIn:
if v.method == METHOD_GUEST:
# currently only method to expire login
v.displayName = _GUEST_USER_DISPLAY_NAME
v.isGuestUser = True
v.isLoginExpired = _METHOD_MODULES[METHOD_GUEST].is_login_expired()
v.needCompleteRegistration = False
v.visibleMethods = non_guest_methods
else:
r = auth_db.UserRegistration.search_by(uid=u)
if r:
v.displayName = r.display_name
_method_auth_state(v, u)
if pkconfig.channel_in('dev'):
# useful for testing/debugging
v.uid = u
pkdc('state={}', v)
return http_reply.render_static(
'auth-state',
'js',
pkcollections.Dict(auth_state=v),
)
def _slot_start(self, slot):
"""Have a slot so now ask docker to run the job
POSIT: Job locked by caller
"""
# __host is sentinel of the start attempt
self.__host = slot.host
ctx = pkcollections.Dict(
kill_secs=runner.KILL_TIMEOUT_SECS,
run_dir=self.run_dir,
run_log=self.run_dir.join(template_common.RUN_LOG),
run_secs=self.run_secs(),
sh_cmd=self.__sh_cmd(),
)
self.__image = _image()
script = str(self.run_dir.join('runner-docker.sh'))
with open(str(script), 'wb') as f:
f.write(pkjinja.render_resource('runner/docker.sh', ctx))
cmd = _RUN_PREFIX + (
'--cpus={}'.format(slot.cores),
'--detach',
#TODO(robnagler) other environ vars required?
'--env=SIREPO_MPI_CORES={}'.format(slot.cores),
'--init',
'--memory={}g'.format(slot.gigabytes),
'--name={}'.format(self.__cname),
'--network=none',
#TODO(robnagler) this doesn't do anything
# '--ulimit=cpu=1',
# do not use a user name, because that may not map inside the
# container properly. /etc/passwd on the host and guest are
# different.
'--user={}'.format(os.getuid()),
) + self.__volumes() + (
#TODO(robnagler) make this configurable per code (would be structured)
self.__image,
'bash',
script,
)
self.__cid = _cmd(slot.host, cmd)
simulation_db.write_status('running', self.run_dir)
pkdlog(
'{}: started slot={} cid={} dir={} cmd={}',
self.__cname,
slot,
self.__cid,
self.run_dir,
cmd,
)
def fixup_old_data(data):
if 'particleAnimation' not in data['models']:
data['models']['particleAnimation'] = pkcollections.Dict({
'reportType':
'w',
'renderCount':
'300',
})
if 'parameterAnimation' not in data['models']:
data['models']['parameterAnimation'] = pkcollections.Dict({
'reportType':
'wav-wmax',
})
if 'solenoidFile' not in data['models']['solenoid']:
data['models']['solenoid']['solenoidFile'] = ''
if 'beamDefinition' not in data['models']['beam']:
beam = data['models']['beam']
beam['beamDefinition'] = 'transverse_longitude'
beam['cstCompress'] = '0'
beam['transversalFile2d'] = ''
beam['transversalFile4d'] = ''
beam['longitudinalFile1d'] = ''
beam['longitudinalFile2d'] = ''
beam['cstFile'] = ''
def _fixup(obj):
"""Convert all objects to locale strings"""
if isinstance(obj, dict):
res = pkcollections.Dict()
for k in obj:
res[pkcompat.locale_str(k)] = _fixup(obj[k])
return res
if isinstance(obj, list):
res = []
for v in obj:
res.append(_fixup(v))
return res
if type(obj) == bytes or type(obj) == str and hasattr(obj, 'decode'):
return pkcompat.locale_str(obj)
return obj
def init(app, api_module, simulation_db):
"""Convert route map to dispatchable callables
Initializes `_uri_to_route` and adds a single flask route (`_dispatch`) to
dispatch based on the map.
Args:
app (Flask): flask app
api_module (module): where to get callables
"""
global _uri_to_route
if _uri_to_route:
# Already initialized
return
global _default_route, _empty_route, sr_unit_uri, _api_to_route
_uri_to_route = pkcollections.Dict()
_api_to_route = pkcollections.Dict()
for k, v in simulation_db.SCHEMA_COMMON.route.items():
r = _split_uri(v)
r.decl_uri = v
r.func = func_for_api(k, api_module)
r.name = k
assert not r.base_uri in _uri_to_route, \
'{}: duplicate end point; other={}'.format(v, routes[r.base_uri])
_uri_to_route[r.base_uri] = r
_api_to_route[k] = r
if r.base_uri == '':
_default_route = r
if 'sr_unit' in v:
sr_unit_uri = v
assert _default_route, \
'missing default route'
# 'light' is the homePage, not 'root'
_empty_route = _uri_to_route.light
app.add_url_rule('/<path:path>', '_dispatch', _dispatch, methods=('GET', 'POST'))
app.add_url_rule('/', '_dispatch_empty', _dispatch_empty, methods=('GET', 'POST'))
def _split_uri(uri):
"""Parse the URL for parameters
Args:
uri (str): full path with parameter args in flask format
Returns:
Dict: with base_uri, func, params, etc.
"""
parts = uri.split('/')
assert '' == parts.pop(0)
params = []
res = pkcollections.Dict(params=params)
in_optional = False
first = None
for p in parts:
m = _PARAM_RE.search(p)
if not m:
assert first is None, \
'{}: too many non-paramter components of uri'.format(uri)
first = p
continue
rp = pkcollections.Dict()
params.append(rp)
rp.optional = bool(m.group(1))
rp.name = m.group(2)
if rp.optional:
in_optional = True
else:
assert not in_optional, \
'{}: optional parameter ({}) followed by non-optional'.format(
uri,
rp.name,
)
res.base_uri = first or ''
return res
def gen_redirect_for_anchor(uri):
"""Redirect uri with an anchor using javascript
Safari browser doesn't support redirects with anchors so we do this
in all cases.
Args:
uri (str): where to redirect to
Returns:
flask.Response: reply object
"""
return render_static(
'javascript-redirect',
'html',
pkcollections.Dict(redirect_uri=uri),
)
def gen_json(value, pretty=False, response_kwargs=None):
"""Generate JSON flask response
Args:
value (dict): what to format
pretty (bool): pretty print [False]
Returns:
flask.Response: reply object
"""
app = flask.current_app
if not response_kwargs:
response_kwargs = pkcollections.Dict()
return app.response_class(simulation_db.generate_json(value,
pretty=pretty),
mimetype=MIME_TYPE.json,
**response_kwargs)
def update_j2_ctx(j2_ctx):
from rsconf.component import docker_registry
if not j2_ctx.docker_cache.host:
return False
addr = '{}:{}'.format(j2_ctx.docker_cache.host, _PORT)
j2_ctx.docker_cache.update(pkcollections.Dict(
http_addr=addr,
http_host='https://' + addr,
))
docker_registry.update_j2_ctx(j2_ctx)
assert not j2_ctx.docker_registry.get('http_host'), \
'{}: docker_cache and docker_registry.http_host both defined'.format(
j2_ctx.docker_registry.http_host,
)
return True
async def run_extract_job(self, run_dir, jhash, subcmd, arg):
pkdc('{} {}: {} {}', run_dir, jhash, subcmd, arg)
status = self.report_job_status(run_dir, jhash)
if status is runner_client.JobStatus.MISSING:
pkdlog('{} {}: report is missing; skipping extract job', run_dir,
jhash)
return {}
# figure out which backend and any backend-specific info
runner_info_file = run_dir.join(_RUNNER_INFO_BASENAME)
if runner_info_file.exists():
runner_info = pkjson.load_any(runner_info_file)
else:
# Legacy run_dir
runner_info = pkcollections.Dict(
version=1,
backend='local',
backend_info={},
)
assert runner_info.version == 1
# run the job
cmd = ['sirepo', 'extract', subcmd, arg]
result = await _BACKENDS[runner_info.backend].run_extract_job(
run_dir,
cmd,
runner_info.backend_info,
)
if result.stderr:
pkdlog(
'got output on stderr ({} {}):\n{}',
run_dir,
jhash,
result.stderr.decode('utf-8', errors='ignore'),
)
if result.returncode != 0:
pkdlog(
'failed with return code {} ({} {}), stdout:\n{}',
result.returncode,
run_dir,
subcmd,
result.stdout.decode('utf-8', errors='ignore'),
)
raise AssertionError
return pkjson.load_any(result.stdout)
def gen_redirect_for_anchor(uri, **kwargs):
"""Redirect uri with an anchor using javascript
Safari browser doesn't support redirects with anchors so we do this
in all cases. It also allows us to return sr_exception to the app
when we don't know if we can.
Args:
uri (str): where to redirect to
Returns:
flask.Response: reply object
"""
return render_static(
'javascript-redirect',
'html',
pkcollections.Dict(redirect_uri=uri, **kwargs),
)
def test_auth_hash():
from pykern import pkconfig
pkconfig.reset_state_for_testing({
'SIREPO_BLUESKY_AUTH_SECRET': 'a simple string is fine',
})
from sirepo import bluesky
from pykern import pkcollections
from pykern.pkunit import pkeq
req = pkcollections.Dict(
simulationType='xyz',
simulationId='1234',
authNonce='some random string',
)
bluesky.auth_hash(req)
pkeq('v1:-TEGBNOAt9dCTtCCvRD0WHtL_XaZR_lHM37cy6PePwE=', req.authHash)
def api_downloadDataFile(simulation_type, simulation_id, model, frame, suffix=None):
data = {
'simulationType': sirepo.template.assert_sim_type(simulation_type),
'simulationId': simulation_id,
'modelName': model,
}
options = pkcollections.Dict(data)
options.suffix = suffix
frame = int(frame)
template = sirepo.template.import_module(data)
if frame >= 0:
data['report'] = template.get_animation_name(data)
else:
data['report'] = model
run_dir = simulation_db.simulation_run_dir(data)
filename, content, content_type = template.get_data_file(run_dir, model, frame, options=options)
return _as_attachment(flask.make_response(content), content_type, filename)
def fixup_old_data(data):
for m in ('beamAnimation', 'beamHistogramAnimation', 'parameterAnimation', 'particleAnimation'):
if m not in data.models:
data.models[m] = pkcollections.Dict({})
template_common.update_model_defaults(data.models[m], m, _SCHEMA)
if 'solenoidFile' not in data['models']['solenoid']:
data['models']['solenoid']['solenoidFile'] = ''
if 'beamDefinition' not in data['models']['beam']:
beam = data['models']['beam']
beam['beamDefinition'] = 'transverse_longitude'
beam['cstCompress'] = '0'
beam['transversalFile2d'] = ''
beam['transversalFile4d'] = ''
beam['longitudinalFile1d'] = ''
beam['longitudinalFile2d'] = ''
beam['cstFile'] = ''
template_common.organize_example(data)
async def run_extract_job(run_dir, cmd, backend_info):
env = _subprocess_env()
# we're in py3 mode, and regular subprocesses will inherit our
# environment, so we have to manually switch back to py2 mode.
env['PYENV_VERSION'] = 'py2'
cmd = ['pyenv', 'exec'] + cmd
# When the next version of Trio is released, we'll be able to replace all
# this with a call to trio.run_process(...)
trio_process = trio.Process(
cmd,
cwd=run_dir,
start_new_session=True,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
)
try:
async def collect(stream, out_array):
while True:
data = await stream.receive_some(4096)
if not data:
break
out_array += data
stdout = bytearray()
stderr = bytearray()
async with trio.open_nursery() as nursery:
nursery.start_soon(collect, trio_process.stdout, stdout)
nursery.start_soon(collect, trio_process.stderr, stderr)
await trio_process.wait()
finally:
trio_process.kill()
await trio_process.aclose()
return pkcollections.Dict(
returncode=trio_process.returncode,
stdout=stdout,
stderr=stderr,
)
return stdout.decode('utf-8')
def internal_build(self):
from rsconf import systemd
from rsconf.component import logrotate
self.buildt.require_component('base_all')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.postgresql
z.run_d = j2_ctx.rsconf_db.host_run_d.join('postgresql')
z.conf_d = z.run_d.join('data')
z.conf_f = z.conf_d.join('postgresql.conf')
z.run_u = 'postgres'
z.log_filename = 'postgresql.log'
z.log_d = pkio.py_path('/var/log/postgresql')
# POSIT: pg_log is the default
z.log_f = z.log_d.join(z.log_filename)
self.install_access(mode='700', owner=z.run_u)
self.append_root_bash('rsconf_yum_install postgresql-server')
# Needs to be installed before main runs
self.install_directory(z.log_d)
self.append_root_bash_with_main(j2_ctx)
systemd.unit_prepare(self, j2_ctx, [z.run_d])
self.append_root_bash(
'rsconf_append "{}" "{}" || true'.format(
z.conf_f,
"include 'rsconf.conf'",
),
)
self.install_access(mode='400', owner=z.run_u)
kc = self.install_tls_key_and_crt(j2_ctx.rsconf_db.host, z.conf_d)
j2_ctx.setdefault('postgresql', pkcollections.Dict()).update(
ssl_cert_file=kc.crt.basename,
ssl_key_file=kc.key.basename,
)
self.install_resource(
'postgresql/rsconf.conf',
j2_ctx,
z.conf_d.join('rsconf.conf'),
)
self.install_resource(
'postgresql/pg_hba.conf', j2_ctx, z.conf_d.join('pg_hba.conf'))
self.install_access(mode='400', owner=j2_ctx.rsconf_db.root_u)
logrotate.install_conf(self, j2_ctx)
systemd.unit_enable(self, j2_ctx)
self.rsconf_service_restart()
def fixup_old_data(data):
if (float(data.fixup_old_version) < 20170703.000001
and 'geometricSource' in data.models):
g = data.models.geometricSource
x = g.cone_max
g.cone_max = g.cone_min
g.cone_min = x
for m in [
'initialIntensityReport',
'plotXYReport',
]:
if m not in data.models:
data.models[m] = pkcollections.Dict({})
template_common.update_model_defaults(data.models[m], m, _SCHEMA)
for m in data.models:
if template_common.is_watchpoint(m):
template_common.update_model_defaults(data.models[m],
'watchpointReport', _SCHEMA)
template_common.organize_example(data)
def report_info(data):
"""Read the run_dir and return cached_data.
Only a hit if the models between data and cache match exactly. Otherwise,
return cached data if it's there and valid.
Args:
data (dict): parameters identifying run_dir and models or reportParametersHash
Returns:
Dict: report parameters and hashes
"""
# Sets data['reportParametersHash']
rep = pkcollections.Dict(
cache_hit=False,
cached_data=None,
cached_hash=None,
job_id=job_id(data),
model_name=data['report'],
parameters_changed=False,
run_dir=simulation_run_dir(data),
)
rep.input_file = json_filename(template_common.INPUT_BASE_NAME,
rep.run_dir)
rep.job_status = read_status(rep.run_dir)
rep.req_hash = template_common.report_parameters_hash(data)
if not rep.run_dir.check():
return rep
#TODO(robnagler) Lock
try:
cd = read_json(rep.input_file)
rep.cached_hash = template_common.report_parameters_hash(cd)
rep.cached_data = cd
if rep.req_hash == rep.cached_hash:
rep.cache_hit = True
return rep
rep.parameters_changed = True
except IOError as e:
pkdlog('{}: ignore IOError: {} errno={}', rep.run_dir, e, e.errno)
except Exception as e:
pkdlog('{}: ignore other error: {}', rep.run_dir, e)
# No idea if cache is valid or not so throw away
return rep
def _setup_tls_host(self, j2_ctx, z):
import socket
import ipaddress
c, ca = _self_signed_crt(j2_ctx, j2_ctx.rsconf_db.host)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(_TLS_DIR)
self.install_access(mode='400', owner=z.run_u)
z.tls = pkcollections.Dict()
# just to match docker's documentation, not really important
for k, b in ('crt', 'cert'), ('key', 'key'):
z.tls[k] = _TLS_DIR.join(b + '.pem')
self.install_abspath(c[k], z.tls[k])
z.tls.ca_crt = _TLS_DIR.join('cacert.pem')
self.install_abspath(ca['crt'], z.tls.ca_crt)
z.tls.ip = socket.gethostbyname(z.tls_host)
assert ipaddress.ip_address(pkcompat.locale_str(z.tls.ip)).is_private, \
'tls_host={} is on public ip={}'.format(z.tls_host, z.tls.ip)
z.daemon_hosts.append('tcp://{}:{}'.format(z.tls.ip, _DAEMON_PORT))
def internal_build(self):
from rsconf.component import db_bkp
from rsconf import systemd
self.buildt.require_component('db_bkp')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.setdefault(
self.name,
pkcollections.Dict(run_u='comsol',
run_d=systemd.unit_run_d(j2_ctx, 'comsol')))
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.run_d)
db_bkp.install_script_and_subdir(
self,
j2_ctx,
# db_bkp runs as root as comsol user doesn't have shell
run_u=j2_ctx.rsconf_db.root_u,
run_d=z.run_d,
)
def _validate_name(data):
"""Validate and if necessary uniquify name
Args:
data (dict): what to validate
"""
starts_with = pkcollections.Dict()
s = data.models.simulation
n = s.name
for d in iterate_simulation_datafiles(
data.simulationType,
lambda res, _, d: res.append(d),
{'simulation.folder': s.folder},
):
n2 = d.models.simulation.name
if n2.startswith(n):
starts_with[n2] = d.models.simulation.simulationId
if n in starts_with and starts_with[n] != s.simulationId:
_validate_name_uniquify(data, starts_with)