def copy_lib_files(data, source, target):
"""Copy auxiliary files to target
Args:
data (dict): simulation db
target (py.path): destination directory
"""
for f in lib_files(data, source):
path = target.join(f.basename)
pkio.mkdir_parent_only(path)
if not path.exists():
if not f.exists():
sim_resource = resource_dir(data.simulationType)
r = sim_resource.join(f.basename)
# the file doesn't exist in the simulation lib, check the resource lib
if r.exists():
pkio.mkdir_parent_only(f)
r.copy(f)
else:
pkdlog('No file in lib or resource: {}', f)
continue
if source:
# copy files from another session
f.copy(path)
else:
# symlink into the run directory
path.mksymlinkto(f, absolute=False)
def api_uploadFile(simulation_type, simulation_id, file_type):
f = flask.request.files['file']
filename = werkzeug.secure_filename(f.filename)
p = _lib_filepath(simulation_type, filename, file_type)
err = None
file_list = None
if p.check():
confirm = flask.request.form['confirm'] if 'confirm' in flask.request.form else None
if not confirm:
search_name = _lib_filename(simulation_type, filename, file_type)
file_list = _simulations_using_file(simulation_type, file_type, search_name, ignore_sim_id=simulation_id)
if file_list:
err = 'File is in use in other simulations. Please confirm you would like to replace the file for all simulations.'
if not err:
pkio.mkdir_parent_only(p)
f.save(str(p))
template = sirepo.template.import_module(simulation_type)
if hasattr(template, 'validate_file'):
err = template.validate_file(file_type, str(p))
if err:
pkio.unchecked_remove(p)
if err:
return http_reply.gen_json({
'error': err,
'filename': filename,
'fileList': file_list,
'fileType': file_type,
'simulationId': simulation_id,
})
return http_reply.gen_json({
'filename': filename,
'fileType': file_type,
'simulationId': simulation_id,
})
def _before_request(fc):
target = srunit.server.app.sirepo_db_dir.join('beaker',
'container_file',
filename[0:1],
filename[0:2], filename)
pkio.mkdir_parent_only(target)
shutil.copy(str(pkunit.data_dir().join(filename)), str(target))
def _find_tls_crt(j2_ctx, domain):
from rsconf.pkcli import tls
from rsconf import db
d = db.secret_path(j2_ctx, TLS_SECRET_SUBDIR, visibility='global')
for crt, domains in j2_ctx.component.tls_crt.items():
if domain in domains:
return d.join(crt), domains
for s in (
domain,
domain.replace('.', '_'),
'.'.join([_WILDCARD_TLS] + domain.split('.')[1:]),
# sirepo.com is in wildcard cert star.sirepo.com
_WILDCARD_TLS + '.' + domain,
):
src = d.join(s)
# due to dots in domain, we can't use ext=
if (src + tls.KEY_EXT).check():
return src, domain
assert j2_ctx.component.tls_crt_create, \
f'{domain}: tls crt for domain not found'
src = d.join(domain)
pkio.mkdir_parent_only(src)
tls.gen_self_signed_crt(str(src), *domain)
return src, domain
def _create_example(example):
data = simulation_db.save_new_example(example)
# ensure all datafiles for the new example exist in the sim lib dir
for f in template_common.lib_files(data):
if not f.exists():
r = template_common.resource_dir(data.simulationType).join(f.basename)
assert r.exists(), 'Example missing resource file: {}'.format(f)
pkio.mkdir_parent_only(f)
r.copy(f)
def _before_request(fc):
target = srunit.server._app.sirepo_db_dir.join('beaker',
'container_file',
filename[0:1],
filename[0:2], filename)
pkio.mkdir_parent_only(target)
shutil.copy(str(pkunit.data_dir().join(filename)), str(target))
for h in header.split('; '):
x = h.split('=')
fc.set_cookie('', *x)
def _crt_create(basename, op):
from rsconf.pkcli import tls
res = pkcollections.Dict(
key=basename + tls.KEY_EXT,
crt=basename + tls.CRT_EXT,
)
if res.crt.check():
return res
pkio.mkdir_parent_only(res.crt)
return op()
def secret_path(hdb, filename, visibility=None, qualifier=None):
if visibility:
assert visibility in VISIBILITY_LIST, \
'{}: invalid visibility, must be {}'.format(
visibility,
VISIBILITY_LIST,
)
else:
visibility = VISIBILITY_DEFAULT
p = [] if visibility == VISIBILITY_GLOBAL else [qualifier or hdb.rsconf_db[visibility]]
p.append(filename)
res = hdb.rsconf_db.secret_d.join(*p)
pkio.mkdir_parent_only(res)
return res
def import_file(req, tmp_dir=None, **kwargs):
if not pkio.has_file_extension(req.filename, 'zip'):
raise sirepo.util.UserAlert('unsupported import filename: {}'.format(filename))
#TODO(pjm): writing to simulation lib for now, tmp_dir will get removed after this request
filepath = str(simulation_db.simulation_lib_dir(SIM_TYPE).join(_ZIP_FILE_NAME))
pkio.mkdir_parent_only(filepath)
with open(filepath, 'wb') as f:
f.write(req.file_stream.read())
data = simulation_db.default_data(SIM_TYPE)
data['models']['simulation']['name'] = req.filename
data['models']['simulation'][_TMP_INPUT_FILE_FIELD] = filepath
# more processing occurs in prepare_for_client() via:
# import_file => _save_new_and_reply => api_simulationData => prepare_for_client
return data
def tls_key_and_crt(j2_ctx, domain):
from rsconf.pkcli import tls
from rsconf import db
src, domains = _find_tls_crt(j2_ctx, domain)
src_key = src + tls.KEY_EXT
src_crt = src + tls.CRT_EXT
if not src_crt.check():
assert j2_ctx.component.tls_crt_create, \
'{}: missing crt for: {}'.format(src_crt, domain)
pkio.mkdir_parent_only(src_crt)
# https://stackoverflow.com/a/42730929
# Cannot pass (basename=str(src), *domains)
tls.gen_self_signed_crt(str(src), *domains)
assert src_key.check(), \
'{}: missing key for: {}'.format(src_key, domain)
return PKDict(key=src_key, crt=src_crt)
def init_tree(name, author, author_email, description, license, url):
"""Setup a project tree with: docs, tests, etc., and checkin to git.
Creates: setup.py, index.rst, project dir, <name>_console.py, etc.
Overwrites files if they exist without checking.
Args:
name (str): short name of the project, e.g. ``pykern``.
author (str): copyright holder, e.g. ``RadiaSoft LLC``
author_email (str): how to reach author, e.g. ``[email protected]``
description (str): one-line summary of project
license (str): url of license
url (str): website for project, e.g. http://pykern.org
"""
assert os.path.isdir('.git'), \
'Must be run from the root directory of the repo'
assert not os.path.isdir(name), \
'{}: already exists, only works on fresh repos'.format(name)
assert name == py.path.local().basename, \
'{}: name must be the name of the current directory'.format(name)
license = license.lower()
base = pkresource.filename('projex')
values = copy.deepcopy(DEFAULTS)
values.update({
'name': name,
'author': author,
'description': description,
'author_email': author_email,
'url': url,
'license': _license(license, 0),
'classifier_license': _license(license, 1),
})
values['copyright_license_rst'] = values['copyright_license_rst'].format(**values)
suffix_re = r'\.jinja$'
for src in pkio.walk_tree(base, file_re=suffix_re):
dst = py.path.local(src).relto(str(base))
dst = dst.replace('projex', name).replace('dot-', '.')
dst = re.sub(suffix_re, '', dst)
pkio.mkdir_parent_only(dst)
_render(src, values, output=dst)
src = py.path.local(pkresource.filename('projex-licenses'))
src = src.join(license + '.jinja')
_render(src, values, output='LICENSE')
def setup_cluster(compt, hosts, tls_d, run_u, j2_ctx):
from rsconf import db
compt.install_access(mode='700', owner=run_u)
compt.install_directory(tls_d)
b = db.secret_path(j2_ctx,
compt.name + '_' + _TLS_BASENAME,
visibility='host')
pkio.mkdir_parent_only(b)
for h in hosts:
c, ca = _self_signed_crt(j2_ctx, h)
c = _signed_crt(j2_ctx, ca, b.join(h))
d = tls_d.join(h)
compt.install_access(mode='700')
compt.install_directory(d)
compt.install_access(mode='400')
# POSIT: sirepo.runner.docker uses {cacert,cert,key}.pem
compt.install_abspath(ca.crt, d.join('cacert.pem'))
compt.install_abspath(c.crt, d.join('cert.pem'))
compt.install_abspath(c.key, d.join('key.pem'))
def _bash_append_and_dst(self,
host_path,
ignore_exists=False,
file_contents=None,
file_src=None):
dst = self.hdb.build.dst_d.join(host_path)
if dst.check():
if ignore_exists:
return None
raise AssertionError('{}: dst already exists'.format(dst))
pkio.mkdir_parent_only(dst)
md5 = None
if file_src:
assert file_contents is None, \
'{}: do not pass both file_contents and file_src'.format(host_path)
file_contents = file_src.read_binary()
if file_contents:
md5 = _md5(self._write_binary(dst, file_contents))
self._bash_append(host_path, md5=md5)
return dst
def _flash_create_sim_files(cls, data, run_dir):
import sirepo.mpi
import sirepo.template.flash
subprocess.check_output(
[
'tar',
'--extract',
'--gunzip',
f'--file={cls._flash_src_tarball_basename()}',
f'--directory={run_dir}',
],
stderr=subprocess.STDOUT,
)
s = run_dir.join(cls.sim_type())
d = []
if data.models.problemFiles.archive:
for i, r in cls._flash_extract_problem_files_archive(
run_dir.join(cls._flash_problem_files_archive_basename(data)),
):
b = pkio.py_path(i.filename).basename
#TODO(pjm): zip file also includes required datafiles
# if not re.match(r'(\w+\.F90)|(Makefile)', b):
# continue
p = cls.flash_simulation_unit_file_path(run_dir, data, b)
pkio.mkdir_parent_only(p)
pkio.write_text(p, r())
d.append(p.basename)
cls._flash_check_datafiles(data, d)
sirepo.template.flash.generate_config_file(run_dir, data)
t = s.join(cls.schema().constants.flashAppName)
c = sirepo.template.flash.setup_command(data)
pkdc('setup_command={}', ' '.join(c))
cls._flash_run_command_and_parse_log_on_error(
c,
s,
cls._SETUP_LOG,
r'(.*PPDEFINE.*$)|(^\s+\*.*$(\n\w+.*)?)',
)
flash_schema = flash_parser.SetupParameterParser(
run_dir.join(cls.sim_type(), cls.schema().constants.flashAppName)
).generate_schema()
cls._add_default_views(flash_schema)
pkio.write_text(
run_dir.join(cls.SETUP_PARAMS_SCHEMA_FILE),
pkjson.dump_pretty(PKDict(
flashSchema=flash_schema,
))
)
datafiles = flash_schema.enum.SetupDatafiles
cls._flash_run_command_and_parse_log_on_error(
['make', f'-j{sirepo.mpi.cfg.cores}'],
t,
cls._COMPILE_LOG,
r'^(?:Error): (.*)',
)
for c, b in PKDict({
v: v for v in [f[0] for f in datafiles]
}).pkupdate(
# POSIT: values match cls._sim_file_basenames
flash4=cls.flash_exe_basename(data),
).items():
p = t.join(c)
cls.delete_sim_file(cls._flash_file_prefix(b), data)
cls.put_sim_file(p, b, data)
if p.check(link=1):
p.copy(run_dir.join(b))
else:
p.move(run_dir.join(b))
def _before_request(fc):
target = srunit.server.app.sirepo_db_dir.join(
'beaker', 'container_file', filename[0:1], filename[0:2], filename)
pkio.mkdir_parent_only(target)
shutil.copy(str(pkunit.data_dir().join(filename)), str(target))
def default_command():
import rsconf.component
import rsconf.component.rsconf
import rsconf.pkcli.tls
root_d = db.cfg.root_d
if root_d.check():
return '{}: already exists'.format(root_d)
srv = pkio.mkdir_parent(root_d.join(db.SRV_SUBDIR))
def _sym(old, new_base=None):
old = pkio.py_path(old)
if not new_base:
new_base = old.basename
assert old.check(), \
'{}: does not exist'.format(old)
srv.join(new_base).mksymlinkto(old, absolute=False)
# ssh-keygen -q -N '' -C rsconf -t rsa -b 4096 -f /var/tmp/foo
# -- don't need this
db_d = pkio.mkdir_parent(root_d.join(db.DB_SUBDIR))
secret_d = pkio.mkdir_parent(db_d.join(db.SECRET_SUBDIR))
nginx_d = pkio.mkdir_parent(root_d.join(NGINX_SUBDIR))
boot_hdb = pkcollections.Dict(rsconf_db=pkcollections.Dict(
secret_d=secret_d,
channel='dev',
))
j2_ctx = pkcollections.Dict(
all_host='v9.radia.run',
group=grp.getgrgid(os.getgid())[0],
host='v4.radia.run',
master='v3.radia.run',
port=2916,
root_d=root_d,
srv_d=str(srv),
uid=os.getuid(),
user=pwd.getpwuid(os.getuid())[0],
worker5_host='v5.radia.run',
worker6_host='v6.radia.run',
)
hosts = [h for h in j2_ctx.values() if str(h).endswith('.radia.run')]
# bootstrap
j2_ctx.update(boot_hdb)
j2_ctx.rsconf_db.http_host = 'http://{}:{}'.format(j2_ctx.master,
j2_ctx.port)
j2_ctx.bkp = pkcollections.Dict(primary=j2_ctx.host)
j2_ctx.passwd_f = rsconf.component.rsconf.passwd_secret_f(j2_ctx)
for h in hosts:
_add_host(j2_ctx, srv, h)
_sym('~/src/radiasoft/download/bin/install.sh', 'index.html')
_sym(pkresource.filename('rsconf/rsconf.sh'), 'rsconf.sh')
dev_d = pkio.py_path(pkresource.filename('dev'))
for f in pkio.walk_tree(dev_d):
if str(f).endswith('~') or str(f).startswith('#'):
continue
x = str(f.relto(dev_d))
if not ('local/' in x and x.endswith('.sh.jinja')):
x = re.sub('.jinja$', '', x)
dst = root_d.join(x)
pkio.mkdir_parent_only(dst)
if f.basename == dst.basename:
f.copy(dst)
else:
pkjinja.render_file(f, j2_ctx, output=dst, strict_undefined=True)
n = []
for e in 'rpm', 'proprietary':
d = pkio.py_path(root_d.dirname).join(e)
pkio.mkdir_parent(d)
root_d.join(e).mksymlinkto(d, absolute=False)
n.append(str(d))
subprocess.check_call(['bash', str(secret_d.join('setup_dev.sh')), *n], )
# dev only, really insecure, but makes consistent builds easy
_sym('~/src/radiasoft')
_sym('~/src/biviosoftware')
for h in hosts:
# needed to be able to talk setup certs for registry so we can
# pull private images from all hosts. Only used in dev, because
# private registry doesn't protect against pushes from these hosts.
if h != j2_ctx.master:
subprocess.check_call(
['rsconf', 'host', 'init_docker_registry', h])
tls_d = secret_d.join(rsconf.component.TLS_SECRET_SUBDIR)
tls_d.ensure(dir=True)
for h in (
'jupyter.' + j2_ctx.all_host,
'jupyter.' + j2_ctx.host,
j2_ctx.all_host,
j2_ctx.master,
j2_ctx.worker5_host,
j2_ctx.worker6_host,
):
rsconf.pkcli.tls.gen_self_signed_crt(
tls_d.join(h),
h,
)
rsconf.pkcli.tls.gen_self_signed_crt(
tls_d.join('star.' + j2_ctx.host),
'*.' + j2_ctx.host,
j2_ctx.host,
)
