def default_command(in_file):
"""Reads `in_file` passes to `msg.jobCmd`
Must be called in run_dir
Writes its output on stdout.
Args:
in_file (str): json parsed to msg
Returns:
str: json output of command, e.g. status msg
"""
try:
job.init()
f = pkio.py_path(in_file)
msg = pkjson.load_any(f)
#TODO(e-carlin): find common place to serialize/deserialize paths
msg.runDir = pkio.py_path(msg.runDir)
f.remove()
res = globals()['_do_' + msg.jobCmd](msg,
sirepo.template.import_module(
msg.simulationType))
if res is None:
return
r = PKDict(res).pksetdefault(state=job.COMPLETED)
except Exception as e:
r = PKDict(
state=job.ERROR,
error=e.sr_args.error
if isinstance(e, sirepo.util.UserAlert) else str(e),
stack=pkdexc(),
)
return pkjson.dump_pretty(r, pretty=False)
def _init_git():
"""Init git locally and to bitbucket"""
from pykern import pkcli
from pykern import pkio
import datetime
import re
import subprocess
title = pkio.py_path().basename
v = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')
name = 'sim-{}-{}'.format(pkio.py_path().basename, v).lower()
r, ctx = _git_api_request(
'post',
'repositories/{user}/{repo}',
dict(
repo=name,
json=dict(
scm='git',
is_private=True,
fork_policy='no_public_forks',
name=name,
),
),
)
repo_url = r.json()['links']['clone'][0]['href']
#TODO(robnagler) add README.md if not already there
subprocess.check_call(['git', 'init'])
subprocess.check_call(['git', 'remote', 'add', 'origin', repo_url])
subprocess.check_call(['git', 'config', 'user.name', ctx['user']])
if pkio.pkunit_prefix:
_pkunit_setup(ctx)
subprocess.check_call(['git', 'checkout', '-b', 'master'])
_out_dir()
_git_commit('init')
async def _fastcgi_op(self, msg):
if msg.runDir:
_assert_run_dir_exists(pkio.py_path(msg.runDir))
if not self.fastcgi_cmd:
m = msg.copy()
m.jobCmd = 'fastcgi'
self._fastcgi_file = cfg.fastcgi_sock_dir.join(
f'sirepo_job_cmd-{cfg.agent_id:8}.sock', )
self._fastcgi_msg_q = sirepo.tornado.Queue(1)
pkio.unchecked_remove(self._fastcgi_file)
m.fastcgiFile = self._fastcgi_file
# Runs in an agent's directory and chdirs to real runDirs.
# Except in stateless_compute which doesn't interact with the db.
m.runDir = pkio.py_path()
# Kind of backwards, but it makes sense since we need to listen
# so _do_fastcgi can connect
self._fastcgi_remove_handler = tornado.netutil.add_accept_handler(
tornado.netutil.bind_unix_socket(str(self._fastcgi_file)),
self._fastcgi_accept,
)
# last thing, because of await: start fastcgi process
await self._cmd(m, send_reply=False)
self._fastcgi_msg_q.put_nowait(msg)
self.fastcgi_cmd.op_id = msg.opId
return None
def _init_git():
"""Init git locally and to bitbucket"""
from pykern import pkcli
from pykern import pkio
import datetime
import re
import subprocess
title = pkio.py_path().basename
v = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')
name = 'sim-{}-{}'.format(pkio.py_path().basename, v).lower()
r, ctx = _git_api_request(
'post',
'repositories/{user}/{repo}',
dict(
repo=name,
json=dict(
scm='git',
is_private=True,
fork_policy='no_public_forks',
name=name,
),
),
)
repo_url = r.json()['links']['clone'][0]['href']
#TODO(robnagler) add README.md if not already there
subprocess.check_call(['git', 'init'])
subprocess.check_call(['git', 'remote', 'add', 'origin', repo_url])
subprocess.check_call(['git', 'config', 'user.name', ctx['user']])
if pkio.pkunit_prefix:
_pkunit_setup(ctx)
subprocess.check_call(['git', 'checkout', '-b', 'master'])
_out_dir()
_git_commit('init')
def _init_root(*args):
global _root
if args:
assert not cfg.root, \
'Cannot set both SIREPO_SRDB_ROOT ({}) and SIREPO_SERVER_DB_DIR ({})'.format(
cfg.root,
args[0],
)
cfg.root = args[0]
v = cfg.root
if v:
assert os.path.isabs(v), \
'{}: SIREPO_SRDB_ROOT must be absolute'.format(v)
assert os.path.isdir(v), \
'{}: SIREPO_SRDB_ROOT must be a directory and exist'.format(v)
v = pkio.py_path(v)
else:
assert pkconfig.channel_in('dev'), \
'SIREPO_SRDB_ROOT must be configured except in DEV'
fn = sys.modules[pkinspect.root_package(_init_root)].__file__
root = pkio.py_path(pkio.py_path(pkio.py_path(fn).dirname).dirname)
# Check to see if we are in our dev directory. This is a hack,
# but should be reliable.
if not root.join('requirements.txt').check():
# Don't run from an install directory
root = pkio.py_path.local('.')
v = pkio.mkdir_parent(root.join(_DEFAULT_ROOT))
_root = v
def dump_pretty(obj, filename=None, pretty=True, **kwargs):
"""Formats as json as string
If an object is not encoded by default, will call str() on the
object.
Args:
obj (object): any Python object
filename (str or py.path): where to write [None]
pretty (bool): pretty print [True]
kwargs (object): other arguments to `json.dumps`
Returns:
str: sorted and formatted JSON
"""
if pretty:
res = json.dumps(obj,
indent=4,
separators=(',', ': '),
sort_keys=True,
cls=Encoder,
**kwargs) + '\n'
else:
res = json.dumps(obj, separators=(',', ':'), cls=Encoder, **kwargs)
if filename:
from pykern import pkio
pkio.py_path(filename).write(res)
return res
def init_apis(*args, **kwargs):
global cfg
cfg = pkconfig.init(
dst_db_root=(
pkio.py_path(sirepo.srdb.root()).join('jupyterhub'),
pkio.py_path,
'new jupyter user db',
),
rs_jupyter_migrate=(
False, bool,
'give user option to migrate data from jupyter.radiasoft.org'),
src_db_root=(
pkio.py_path('/var/empty'),
pkio.py_path,
'existing jupyter user db (ex /srv/jupyterhub)',
),
uri_root=('jupyter', str, 'the root uri of jupyterhub'),
)
sirepo.auth_db.init_model(_init_model)
sirepo.events.register({
'auth_logout': _event_auth_logout,
'end_api_call': _event_end_api_call,
'github_authorized': _event_github_authorized,
})
def _init_root():
global _cfg, _root
def _cfg_root(v):
"""Config value or root package's parent or cwd with `_DEFAULT_ROOT`"""
if not os.path.isabs(v):
pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be absolute')
if not os.path.isdir(v):
pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be a directory and exist')
return pkio.py_path(v)
_cfg = pkconfig.init(
root=(None, _cfg_root, 'where database resides'),
)
_root = _cfg.root
if _root:
return _root
assert pkconfig.channel_in('dev'), \
'SIREPO_SRDB_ROOT must be configured except in dev'
r = pkio.py_path(
sys.modules[pkinspect.root_package(_init_root)].__file__,
).dirpath().dirpath()
# Check to see if we are in our dev directory. This is a hack,
# but should be reliable.
if not r.join('requirements.txt').check():
# Don't run from an install directory
r = pkio.py_path('.')
_root = pkio.mkdir_parent(r.join(_DEFAULT_ROOT))
return _root
def internal_build(self):
from rsconf import systemd
from rsconf.component import network
from rsconf.component import logrotate
from rsconf.component import bop
self.buildt.require_component('base_all')
jc = self.hdb.j2_ctx_copy()
z = jc.spamd
conf_d = pkio.py_path('/etc/mail/spamassassin')
nc = self.buildt.get_component('network')
z.sa_update_keys_d = conf_d.join('sa-update-keys')
z.trusted_networks = ' '.join(nc.trusted_nets())
watch = bop.install_perl_rpms(self, jc) + [conf_d]
z.run_d = systemd.custom_unit_prepare(self, jc, watch)
z.log_postrotate_f = z.run_d.join('log_postrotate')
socket_d = pkio.py_path('/run/spamd')
z.socket_path = pkio.py_path('/run/spamd/spamd.sock')
self.install_access(mode='755', owner=jc.rsconf_db.run_u)
self.install_directory(conf_d)
self.install_directory(socket_d)
self.install_access(mode='500')
self.install_resource('spamd/log_postrotate.sh', jc,
z.log_postrotate_f)
self.install_access(mode='444')
self.install_resource(
'spamd/spamc.conf',
jc,
conf_d.join('spamc.conf'),
)
logrotate.install_conf(self, jc)
self.append_root_bash_with_main(jc)
systemd.custom_unit_enable(self, jc)
def get_data_file(run_dir, model, frame, **kwargs):
data = simulation_db.read_json(
run_dir.join(template_common.INPUT_BASE_NAME))
path = pkio.py_path(
_SIM_DATA.lib_file_for_sim(data, _FRAME_FILENAME[frame]))
if not path.exists() and frame == _SCHEMA.constants.dose2FrameId:
# no alternate dose exists, use main dose instead
path = pkio.py_path(_SIM_DATA.lib_file_for_sim(data, RTDOSE_FILE))
return PKDict(filename=path)
def copy_related_files(data, source_path, target_path):
# copy results and log for the long-running simulations
for m in ('animation', ):
# copy any simulation output
s = pkio.py_path(source_path).join(m)
if not s.exists():
continue
t = pkio.py_path(target_path).join(m)
pkio.mkdir_parent(str(t))
for f in pkio.sorted_glob('*'):
f.copy(t)
def test_read_all():
from pykern import pkio
from pykern import pkjson
from pykern import pkunit
from pykern.pkunit import pkok, pkeq, pkre
from pykern.pkdebug import pkdp
from pykern.pkcli import rsmanifest
import re
with pkunit.save_chdir_work(is_pkunit_prefix=True) as d:
rsmanifest.add_code(
'code1',
version='1.1',
uri='http://x.com',
source_d='/tmp',
pyenv='py2',
)
v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version
pkjson.dump_pretty(
{'version': v, 'image': {'type': 'docker'}},
filename=rsmanifest.CONTAINER_FILE,
)
m = rsmanifest.read_all()
pkeq(v, m.version)
pkeq('docker', m.image.type)
pkeq('1.1', m.codes.py2.code1.version)
async def run_extract_job(run_dir, cmd, backend_info):
# no output redirection here - we want to let the docker daemon collect it
quoted_bash_cmd = ' '.join(shlex.quote(c) for c in cmd)
# We never run more than one extract job at a time in a given run_dir, so
# no need to give them unique names.
working_dir = pkio.py_path(backend_info.working_dir)
container = await _make_container(
run_dir, working_dir, quoted_bash_cmd, 'extract',
)
try:
result = await _container_wait(container)
stdout = await trio.run_sync_in_worker_thread(
functools.partial(
container.logs,
stdout=True,
stderr=False,
)
)
stderr = await trio.run_sync_in_worker_thread(
functools.partial(
container.logs,
stdout=False,
stderr=True,
)
)
return pkcollections.Dict(
returncode=result['StatusCode'],
stdout=stdout,
stderr=stderr,
)
finally:
await trio.run_sync_in_worker_thread(container.remove)
def _flash_run_command_and_parse_log_on_error(
cls,
command,
work_dir,
log_file,
regex,
):
p = pkio.py_path(log_file)
with pkio.open_text(p.ensure(), mode='r+') as l:
try:
subprocess.run(
command,
check=True,
cwd=work_dir,
stderr=l,
stdout=l,
)
except subprocess.CalledProcessError as e:
l.seek(0)
c = l.read()
m = [x.group().strip() for x in re.finditer(
regex,
c,
re.MULTILINE,
)]
if m:
r = '\n'.join(m)
else:
r = c.splitlines()[-1]
raise sirepo.util.UserAlert(
r,
'{}',
e
)
def _do_compute(msg, template):
msg.runDir = pkio.py_path(msg.runDir)
with msg.runDir.join(template_common.RUN_LOG).open('w') as run_log:
p = subprocess.Popen(
_do_prepare_simulation(msg, template).cmd,
stdout=run_log,
stderr=run_log,
)
while True:
for j in range(20):
time.sleep(.1)
r = p.poll()
i = r is None
if not i:
break
if msg.isParallel:
# TODO(e-carlin): This has a potential to fail. We likely
# don't want the job to fail in this case
_write_parallel_status(msg, template, i)
if i:
continue
if r != 0:
return PKDict(state=job.ERROR,
error='non zero returncode={}'.format(r))
return PKDict(state=job.COMPLETED)
def merge_app_vars(j2_ctx, app_name):
z = j2_ctx.bop
z.setdefault('is_production', j2_ctx.rsconf_db.channel == 'prod')
z.is_test = not z.is_production
z.run_u = j2_ctx.rsconf_db.run_u
db.merge_dict(z, j2_ctx[app_name])
z.app_name = app_name
# all apps are secured by TLS now
z.can_secure = True
z.setdefault('bconf_aux', '')
z.setdefault('block_goto', False)
z.setdefault('client_max_body_size', _DEFAULT_CLIENT_MAX_BODY_SIZE)
z.source_code_d = SOURCE_CODE_D
# Assumed by PetShop's crm-ticket-deletion.btest
z.want_status_email = z.perl_root == PETSHOP_ROOT
if z.is_test:
z.http_host = _domain(j2_ctx, z.vhosts[0])[0]
z.mail_host = z.http_host
z.run_d = systemd.unit_run_d(j2_ctx, app_name)
z.conf_f = z.run_d.join('httpd.conf')
z.bconf_f = z.run_d.join('bivio.bconf')
z.initdb_f = z.run_d.join('initdb.sh')
z.initdb_cmd = 'source {}'.format(z.initdb_f)
z.log_postrotate_f = z.run_d.join('reload')
z.httpd_cmd = "/usr/sbin/httpd -d '{}' -f '{}'".format(z.run_d, z.conf_f)
z.local_file_root_d = pkio.py_path('/var/www/facades')
return z
def start():
#TODO(robnagler) commands need their own init hook like the server has
job.init()
global cfg
cfg = pkconfig.init(
agent_id=pkconfig.Required(str, 'id of this agent'),
fastcgi_sock_dir=(
pkio.py_path('/tmp'), pkio.py_path,
'directory of fastcfgi socket, must be less than 50 chars'),
start_delay=(0, pkconfig.parse_seconds,
'delay startup in internal_test mode'),
supervisor_uri=pkconfig.Required(
str,
'how to connect to the supervisor',
),
)
pkdlog('{}', cfg)
if pkconfig.channel_in_internal_test() and cfg.start_delay:
pkdlog('start_delay={}', cfg.start_delay)
time.sleep(cfg.start_delay)
i = tornado.ioloop.IOLoop.current()
d = _Dispatcher()
def s(*args):
return i.add_callback_from_signal(_terminate, d)
signal.signal(signal.SIGTERM, s)
signal.signal(signal.SIGINT, s)
i.spawn_callback(d.loop)
i.start()
def __init__(self, *args, **kwargs):
super(T, self).__init__(*args, **kwargs)
self.root_d = pkio.py_path(cfg.root_d)
self.db_d = self.root_d.join(DB_SUBDIR)
self.proprietary_source_d = self.root_d.join(PROPRIETARY_SUBDIR)
self.rpm_source_d = self.root_d.join(RPM_SUBDIR)
self.tmp_d = self.root_d.join(TMP_SUBDIR)
self.secret_d = self.db_d.join(SECRET_SUBDIR)
self.srv_d = self.root_d.join(SRV_SUBDIR)
self.srv_host_d = self.srv_d.join(HOST_SUBDIR)
self.base = PKDict()
f = None
try:
for d in self.db_d, self.secret_d:
for f in pkio.sorted_glob(d.join(ZERO_YML)):
v = pkyaml.load_str(
pkjinja.render_file(
f,
self.base,
strict_undefined=True,
),
)
merge_dict(self.base, v)
except Exception:
pkdlog('error rendering db={}', f)
raise
def _init():
import sirepo.mpi
global SCHEMA_COMMON, cfg, JOB_RUN_MODE_MAP
cfg = pkconfig.init(
nfs_tries=(10, int, 'How many times to poll in hack_nfs_write_status'),
nfs_sleep=(0.5, float, 'Seconds sleep per hack_nfs_write_status poll'),
sbatch_display=(None, str, 'how to display sbatch cluster to user'),
tmp_dir=(None, pkio.py_path, 'Used by utilities (not regular config)'),
)
fn = STATIC_FOLDER.join('json/schema-common{}'.format(JSON_SUFFIX))
with open(str(fn)) as f:
SCHEMA_COMMON = json_load(f)
# In development, any schema update creates a new version
if pkconfig.channel_in('dev'):
SCHEMA_COMMON.version = max([
_timestamp(pkio.py_path(fn).mtime()) \
for fn in glob.glob(str(STATIC_FOLDER.join('json/*{}'.format(JSON_SUFFIX))))
])
else:
SCHEMA_COMMON.version = sirepo.__version__
JOB_RUN_MODE_MAP = PKDict(
sequential='Serial',
parallel='{} cores (SMP)'.format(sirepo.mpi.cfg.cores),
)
if cfg.sbatch_display:
JOB_RUN_MODE_MAP.sbatch = cfg.sbatch_display
def _prepare_hosts(self, jc, z):
nc = self.buildt.get_component('network')
z.ip, _ = nc.ip_and_net_for_host(jc.rsconf_db.host)
if not z.is_first:
return
res = []
for h in z.hosts:
ip, net = nc.ip_and_net_for_host(h)
if 'net' in z:
assert net == z.net, \
'net={} for host={} not on same net={}'.format(
net,
h,
z.net,
)
else:
z.net = net
s = self._gen_keys(jc, z, h)
res.append(
pkcollections.Dict(
host=h,
ip=ip,
host_key_pub=pkio.read_text(s.host_key_pub_f).rstrip(),
# replace this (first) host with current host (h) so we don't
# have to recompute
identity_f=pkio.py_path(
str(z.guest.identity_f).replace(jc.rsconf_db.host,
h), )), )
# Convert to CIDR
z.net = str(z.net.name)
z.max_slots = z.slots_per_host * len(res)
z.hosts_sorted = sorted(res, key=lambda x: x.ip)
def api_copyNonSessionSimulation():
req = http_request.parse_post(id=True, template=True)
simulation_db.verify_app_directory(req.type)
src = pkio.py_path(
simulation_db.find_global_simulation(
req.type,
req.id,
checked=True,
), )
data = simulation_db.open_json_file(
req.type,
src.join(simulation_db.SIMULATION_DATA_FILE),
)
data.pkdel('report')
data.models.simulation.isExample = False
data.models.simulation.outOfSessionSimulationId = req.id
res = _save_new_and_reply(data)
sirepo.sim_data.get_class(req.type).lib_files_from_other_user(
data,
simulation_db.lib_dir_from_sim_dir(src),
)
target = simulation_db.simulation_dir(req.type,
data.models.simulation.simulationId)
#TODO(robnagler) does not work, supervisor needs to be notified to
# copy the simulation state.
# if hasattr(req.template, 'copy_related_files'):
# req.template.copy_related_files(data, str(src), str(target))
return res
def _sym(old, new_base=None):
old = pkio.py_path(old)
if not new_base:
new_base = old.basename
assert old.check(), \
'{}: does not exist'.format(old)
srv.join(new_base).mksymlinkto(old, absolute=False)
def api_staticFile(path_info=None):
"""flask.send_from_directory for static folder.
Uses safe_join which doesn't allow indexing, paths with '..',
or absolute paths.
Args:
path_info (str): relative path to join
Returns:
flask.Response: flask.send_from_directory response
"""
if not path_info:
raise util.raise_not_found('empty path info')
p = pkio.py_path(
flask.safe_join(str(simulation_db.STATIC_FOLDER), path_info))
r = None
if _google_tag_manager and re.match(r'^en/[^/]+html$', path_info):
return http_reply.headers_for_cache(
flask.make_response(
_google_tag_manager_re.sub(
_google_tag_manager,
pkio.read_text(p),
), ),
path=p,
)
if re.match(r'^html/[^/]+html$', path_info):
return http_reply.render_html(p)
return flask.send_file(p, conditional=True)
def _do_compute(msg, template):
msg.runDir = pkio.py_path(msg.runDir)
with msg.runDir.join(template_common.RUN_LOG).open('w') as run_log:
p = subprocess.Popen(
_do_prepare_simulation(msg, template).cmd,
stdout=run_log,
stderr=run_log,
)
while True:
for j in range(20):
time.sleep(.1)
r = p.poll()
i = r is None
if not i:
break
if msg.isParallel:
# TODO(e-carlin): This has a potential to fail. We likely
# don't want the job to fail in this case
_write_parallel_status(msg, template, i)
if i:
continue
return _on_do_compute_exit(
r == 0,
msg.isParallel,
template,
msg.runDir,
)
def _cfg_root(v):
"""Config value or root package's parent or cwd with `_DEFAULT_ROOT`"""
if not os.path.isabs(v):
pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be absolute')
if not os.path.isdir(v):
pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be a directory and exist')
return pkio.py_path(v)
def test_read_all():
from pykern import pkio
from pykern import pkjson
from pykern import pkunit
from pykern.pkunit import pkok, pkeq, pkre
from pykern.pkdebug import pkdp
from pykern.pkcli import rsmanifest
import re
with pkunit.save_chdir_work(is_pkunit_prefix=True) as d:
rsmanifest.add_code(
'code1',
version='1.1',
uri='http://x.com',
source_d='/tmp',
pyenv='py2',
)
v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version
pkjson.dump_pretty(
{
'version': v,
'image': {
'type': 'docker'
}
},
filename=rsmanifest.CONTAINER_FILE,
)
m = rsmanifest.read_all()
pkeq(v, m.version)
pkeq('docker', m.image.type)
pkeq('1.1', m.codes.py2.code1.version)
def test_py_path():
from pykern import pkunit
from pykern import pkio
from pykern.pkunit import pkeq
with pkunit.save_chdir_work():
d = pkunit.data_dir()
pkeq(d, pkio.py_path(d))
def _find(paths):
from pykern import pkio
import re
i = re.compile(r'(?:_work|_data)/')
res = []
cwd = pkio.py_path()
for t in paths or ('tests', ):
t = pkio.py_path(t)
if t.check(file=True):
res.append(str(cwd.bestrelpath(t)))
continue
for p in pkio.walk_tree(t, re.compile(r'_test\.py$')):
p = str(cwd.bestrelpath(p))
if not i.search(p):
res.append(p)
return res
def _gen_req(which, basename, domains, is_ca=False):
first = domains[0]
if not basename:
basename = first
basename = pkio.py_path(basename)
if is_ca:
# pathlen:0 means can only be used for signing certs, not for
# signing intermediate certs.
alt = '''x509_extensions = x509_req
[x509_req]
basicConstraints=critical,CA:true,pathlen:0'''
else:
# Must always provide subjectAltName except for is_ca
# see https://github.com/urllib3/urllib3/issues/497
# which points to RFC 2818:
# Although the use of the Common Name is existing practice, it is deprecated and
# Certification Authorities are encouraged to use the dNSName instead.
alt = '{}_extensions = v3_req\n[v3_req]\n{}'.format(
'req' if which == 'csr' else 'x509',
_alt_names(domains),
)
c = '''
[req]
distinguished_name = subj
prompt = no
{}
[subj]
C = US
ST = Colorado
L = Boulder
CN = {}'''.format(alt, first)
cfg = basename + '.cfg'
try:
cfg.write(c)
key = basename + KEY_EXT
out = basename + '.' + which
cmd = [
'openssl',
'req',
'-nodes',
'-newkey',
'rsa:2048',
'-keyout',
str(key),
'-out',
str(out),
'-config',
str(cfg),
]
if which == _CRT:
cmd += ['-x509'] + _signing_args()
_run(cmd)
finally:
pkio.unchecked_remove(cfg)
res = pkcollections.Dict(key=key)
res[which] = out
return res
def _pkunit_setup(ctx):
from pykern import pkio
import subprocess
f = pkio.py_path('git-credentials')
f.write('https://{user}:{pass}@{host}'.format(**ctx))
f.chmod(0600)
subprocess.check_call(['git', 'config', 'credential.helper', 'cache'])
subprocess.check_call(['git', 'config', 'credential.helper', 'store --file ' + str(f)])
def __slots_from_dump(cls, pool_name):
p = pkio.py_path(_POOLS_DUMP_FILE)
if not p.exists():
return PKDict()
slots = PKDict()
for s in pkjson.load_any(p).pkunchecked_nested_get(
f'{pool_name}.slots', ) or []:
slots[s.cname] = s
return slots
def _cfg_tls_dir(value):
if not value:
assert pkconfig.channel_in('dev'), \
'required config'
return None
res = pkio.py_path(value)
assert res.check(dir=True), \
'directory does not exist; value={}'.format(value)
return res
def run(cfg_dir):
cfg_dir = pkio.py_path(cfg_dir)
data = simulation_db.read_json(template_common.INPUT_BASE_NAME)
if data['report'] == 'doseCalculation':
_run_dose_calculation(data, cfg_dir)
elif data['report'] == 'dvhReport':
_run_dvh(data, cfg_dir)
else:
raise RuntimeError('unknown report: {}'.format(data['report']))
def run(cfg_dir):
cfg_dir = pkio.py_path(cfg_dir)
data = simulation_db.read_json(template_common.INPUT_BASE_NAME)
if data['report'] == 'doseCalculation':
_run_dose_calculation(data, cfg_dir)
elif data['report'] == 'dvhReport':
_run_dvh(data, cfg_dir)
else:
raise AssertionError('unknown report: {}'.format(data['report']))
def dump_pretty(obj, filename=None, pretty=True):
"""Formats as json as string
Args:
obj (object): any Pyton object
filename (str or py.path): where to write [None]
pretty (bool): pretty print [True]
Returns:
str: sorted and formatted JSON
"""
from pykern import pkio
import json
import py.path
if pretty:
res = json.dumps(obj, indent=4, separators=(',', ': '), sort_keys=True) + '\n'
else:
res = json.dumps(obj)
if filename:
pkio.py_path(filename).write(res)
return res
def _call(args):
"""Run a command with the proper local python and path environment
Args:
args (tuple): what to run (flags and all)
"""
from pykern import pkio
import subprocess
import os
ub = pkio.py_path(_PYTHON_USER_BASE)
env = os.environ.copy()
env['PATH'] = str(ub.join('bin')) + ':' + env['PATH']
env['PYTHONUSERBASE'] = str(ub)
subprocess.check_call(args, env=env)
def restore(git_txz):
"""Restores the git directory (only) to a new directory with the .git.txz suffix
"""
m = re.search('(([^/]+)\.git)\.txz$', git_txz)
if not m:
raise ValueError(git_txz, ': does not end in .git.txz')
git_txz = pkio.py_path(git_txz)
d = m.group(2)
pkdc('restore: {}', d)
g = m.group(1)
with pkio.save_chdir(d, mkdir=True):
_shell(['tar', 'xJf', str(git_txz)])
os.rename(g, '.git')
_shell(['git', 'config', 'core.bare', 'false'])
_shell(['git', 'config', 'core.logallrefupdates', 'true'])
_shell(['git', 'checkout'])
def test_add_code():
from pykern import pkio
from pykern import pkjson
from pykern import pkunit
from pykern.pkunit import pkok, pkeq, pkre
from pykern.pkdebug import pkdp
from pykern.pkcli import rsmanifest
import re
with pkunit.save_chdir_work(is_pkunit_prefix=True) as d:
rsmanifest.add_code('A', 'b', 'c', 'd', pyenv='v')
j = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE).read())
pkok(20170101.0 < float(j.version), 'version must be after 2017')
pkeq('A', j.codes.v.a.name)
pkeq('b', j.codes.v.a.version)
rsmanifest.add_code('a', 'bb', 'cc', 'dd')
j = pkjson.load_any(pkio.expand_user_path(rsmanifest.USER_FILE).read())
pkeq('A', j.codes.v.a.name)
pkeq('a', j.codes[''].a.name)
pkeq('bb', j.codes[''].a.version)
pkre('20.*T.*Z', j.codes[''].a.installed)
def _out_dir():
from pykern import pkio
p = pkio.py_path(_OUT_DIR).ensure_dir()
p.join('.gitignore').write('*\n!.gitignore\n')
def _init_python_user_base():
"""Ensure all python_user_base files are committed"""
from pykern import pkio
ub = pkio.py_path(_PYTHON_USER_BASE).ensure_dir()
ub.join('.gitignore').write('!*\n')
def test_get_data_file():
from sirepo import srunit
from pykern import pkunit
from pykern import pkio
import sdds
fc = srunit.flask_client()
fc.get('/elegant')
data = fc.sr_post(
'listSimulations',
{'simulationType': 'elegant', 'search': {'simulationName': 'fourDipoleCSR'}},
)
data = data[0].simulation
data = fc.sr_get(
'simulationData',
params=dict(
pretty='1',
simulation_id=data.simulationId,
simulation_type='elegant',
),
)
run = fc.sr_post(
'runSimulation',
dict(
forceRun=False,
models=data.models,
report='bunchReport1',
simulationId=data.models.simulation.simulationId,
simulationType=data.simulationType,
),
)
for _ in range(10):
run = fc.sr_post(
'runStatus',
run.nextRequest
)
if run.state == 'completed':
break
time.sleep(1)
else:
pkunit.pkfail('runStatus: failed to complete: {}', run)
resp = fc.sr_get(
'downloadDataFile',
dict(
simulation_type=data.simulationType,
simulation_id=data.models.simulation.simulationId,
model='bunchReport1',
frame='-1',
suffix='csv',
),
raw_response=True,
)
rows = csv.reader(StringIO.StringIO(resp.get_data()))
assert len(list(rows)) == 5001
resp = fc.sr_get(
'downloadDataFile',
dict(
simulation_type=data.simulationType,
simulation_id=data.models.simulation.simulationId,
model='bunchReport1',
frame='-1',
),
raw_response=True,
)
m = re.search(r'attachment; filename="([^"]+)"', resp.headers['Content-Disposition'])
with pkunit.save_chdir_work():
path = pkio.py_path(m.group(1))
with open(str(path), 'w') as f:
f.write(resp.get_data())
assert sdds.sddsdata.InitializeInput(0, str(path)) == 1, \
'{}: sdds failed to open'.format(path)
# Verify we can read something
assert 0 <= len(sdds.sddsdata.GetColumnNames(0))
sdds.sddsdata.Terminate(0)
def _parent_file(cfg_dir, filename):
return str(pkio.py_path(cfg_dir.dirname).join(filename))
import sirepo.template
import threading
import time
import werkzeug.exceptions
#: Json files
JSON_SUFFIX = '.json'
#: Schema common values, e.g. version
SCHEMA_COMMON = None
#: Simulation file name is globally unique to avoid collisions with simulation output
SIMULATION_DATA_FILE = 'sirepo-data' + JSON_SUFFIX
#: The root of the pkresource tree (package_data)
RESOURCE_FOLDER = pkio.py_path(pkresource.filename(''))
#: Where server files and static files are found
STATIC_FOLDER = RESOURCE_FOLDER.join('static')
#: Verify ID
_IS_PARALLEL_RE = re.compile('animation', re.IGNORECASE)
#: How to find examples in resources
_EXAMPLE_DIR = 'examples'
#: Valid characters in ID
_ID_CHARS = numconv.BASE62
#: length of ID
_ID_LEN = 8
