def test_walk_tree():
"""Creates looks in data_dir"""
with pkunit.save_chdir_work() as pwd:
for f in ('d1/d7', 'd2/d3', 'd4/d5/d6'):
pkio.mkdir_parent(f)
expect = []
for f in ['d1/d7/f1', 'd4/d5/f2', 'd2/d3/f3']:
pkio.write_text(f, '')
expect.append(py.path.local(f))
assert sorted(expect) == list(pkio.walk_tree('.')), \
'When walking tree, should only return files'
assert [expect[2]] == list(pkio.walk_tree('.', 'f3')), \
'When walking tree with file_re, should only return matching files'
assert [expect[0]] == list(pkio.walk_tree('.', '^d1')), \
'When walking tree with file_re, file to match does not include dir being searched'
def test_purge_users(monkeypatch):
from pykern.pkunit import pkeq, pkok
from pykern.pkdebug import pkdp
from pykern import pkio
from pykern import pkconfig
from sirepo import srunit
srunit.init_auth_db(sim_types='myapp')
from sirepo.pkcli import admin
from sirepo import simulation_db
from sirepo import auth_db
import datetime
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: no old users so empty')
g = simulation_db.user_dir_name('*')
dirs = list(pkio.sorted_glob(g))
pkeq(1, len(dirs), '{}: expecting exactly one user dir', g)
uid = dirs[0].basename
#TODO(robnagler) really want the db to be created, but need
# a test oauth class.
monkeypatch.setattr(auth_db, 'all_uids', lambda: [uid])
for f in pkio.walk_tree(dirs[0]):
f.setmtime(f.mtime() - 86400 * 2)
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: all users registered so no deletes')
monkeypatch.setattr(auth_db, 'all_uids', lambda: [])
res = admin.purge_users(days=1, confirm=False)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(dirs[0].check(dir=True), '{}: nothing deleted', res)
res = admin.purge_users(days=1, confirm=True)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(not dirs[0].check(dir=True), '{}: directory deleted', res)
def _dicom_files(dirname):
files = PKDict(
ctmap=PKDict(),
rtdose=None,
rtstruct=None,
position=None,
additional_rtdose=[],
)
for path in pkio.walk_tree(dirname):
if not pkio.has_file_extension(str(path), 'dcm'):
continue
v = pydicom.dcmread(str(path),
specific_tags=[
'SOPClassUID',
'InstanceNumber',
'PatientPosition',
])
if v.SOPClassUID == _DICOM_CLASS.CT_IMAGE:
files.ctmap[int(v.InstanceNumber)] = str(path)
files.position = v.PatientPosition
elif v.SOPClassUID == _DICOM_CLASS.RT_DOSE:
if files.rtdose:
files.additional_rtdose.append(str(path))
else:
files.rtdose = str(path)
elif v.SOPClassUID == _DICOM_CLASS.RT_STRUCT:
files.rtstruct = str(path)
assert files.rtdose and files.rtstruct, 'Missing RTSTRUCT and/or RTDOSE'
return files
def purge_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
list: directories removed (or to remove if confirm)
"""
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
uids = auth_db.all_uids()
now = datetime.datetime.utcnow()
to_remove = []
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
if simulation_db.uid_from_dir_name(d) in uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
to_remove.append(d)
if confirm:
pkio.unchecked_remove(*to_remove)
return to_remove
def test_purge_users(monkeypatch):
from pykern.pkunit import pkeq, pkok
from pykern.pkdebug import pkdp
from pykern import pkio
from pykern import pkconfig
from sirepo import srunit
srunit.init_user_db()
from sirepo.pkcli import admin
from sirepo import simulation_db
from sirepo import api_auth
import datetime
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: no old users so empty')
pkdp(simulation_db.user_dir_name('*'))
g = simulation_db.user_dir_name('*')
dirs = list(pkio.sorted_glob(g))
pkeq(1, len(dirs), '{}: expecting exactly one user dir', g)
uid = dirs[0].basename
#TODO(robnagler) really want the db to be created, but need
# a test oauth class.
monkeypatch.setattr(api_auth, 'all_uids', lambda: [uid])
for f in pkio.walk_tree(dirs[0]):
f.setmtime(f.mtime() - 86400 * 2)
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: all users registered so no deletes')
monkeypatch.setattr(api_auth, 'all_uids', lambda: [])
res = admin.purge_users(days=1, confirm=False)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(dirs[0].check(dir=True), '{}: nothing deleted', res)
res = admin.purge_users(days=1, confirm=True)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(not dirs[0].check(dir=True), '{}: directory deleted', res)
def _check_file(exists=True):
pkunit.pkeq(
[_proprietary_file] if exists else [],
[
x.basename
for x in pkio.walk_tree(fc.sr_user_dir(), _proprietary_file)
],
)
def examples(app):
files = pkio.walk_tree(
pkresource.filename(_EXAMPLE_DIR_FORMAT.format(app)),
re.escape(JSON_SUFFIX) + '$',
)
#TODO(robnagler) Need to update examples statically before build
# and assert on build
return [open_json_file(app, path=str(f)) for f in files]
def examples(app):
files = pkio.walk_tree(
pkresource.filename(_EXAMPLE_DIR_FORMAT.format(app)),
re.escape(JSON_SUFFIX) + '$',
)
#TODO(robnagler) Need to update examples statically before build
# and assert on build
return [open_json_file(app, path=str(f)) for f in files]
def _init_local_files(values):
v = values.rsconf_db
r = v.db_d.join(LOCAL_SUBDIR)
res = PKDict()
for l in LEVELS[0], v.channel, v.host:
d = r.join(l)
for f in pkio.walk_tree(d):
res['/' + d.bestrelpath(f)] = f
return res
def examples(app):
files = pkio.walk_tree(
sirepo.sim_data.get_class(app).resource_path(_EXAMPLE_DIR),
re.escape(JSON_SUFFIX) + '$',
)
#TODO(robnagler) Need to update examples statically before build
# and assert on build
# example data is not fixed-up to avoid performance problems when searching examples by name
# fixup occurs during save_new_example()
return [open_json_file(app, path=str(f), fixup=False) for f in files]
def examples(app):
files = pkio.walk_tree(
template_common.resource_dir(app).join(_EXAMPLE_DIR),
re.escape(JSON_SUFFIX) + '$',
)
#TODO(robnagler) Need to update examples statically before build
# and assert on build
# example data is not fixed-up to avoid performance problems when searching examples by name
# fixup occurs during save_new_example()
return [open_json_file(app, path=str(f), fixup=False) for f in files]
def test_walk_tree_and_sorted_glob():
"""Looks in work_dir"""
from pykern import pkunit
from pykern import pkio
with pkunit.save_chdir_work() as pwd:
for f in ('d1/d7', 'd2/d3', 'd4/d5/d6'):
pkio.mkdir_parent(f)
expect = []
for f in ['d1/d7/f1', 'd4/d5/f2', 'd2/d3/f3']:
pkio.write_text(f, '')
expect.append(py.path.local(f))
assert sorted(expect) == list(pkio.walk_tree('.')), \
'When walking tree, should only return files'
assert [expect[2]] == list(pkio.walk_tree('.', 'f3')), \
'When walking tree with file_re, should only return matching files'
assert [expect[0]] == list(pkio.walk_tree('.', '^d1')), \
'When walking tree with file_re, file to match does not include dir being searched'
assert pkio.sorted_glob('*[42]') == [py.path.local(f) for f in ('d2', 'd4')]
def _extract_series_frames(simulation, dicom_dir):
#TODO(pjm): give user a choice between multiple study/series if present
selected_series = None
frames = {}
dicom_dose = None
rt_struct_path = None
res = {
'description': '',
}
for path in pkio.walk_tree(dicom_dir):
if pkio.has_file_extension(str(path), 'dcm'):
plan = dicom.read_file(str(path))
if plan.SOPClassUID == _DICOM_CLASS['RT_STRUCT']:
rt_struct_path = str(path)
elif plan.SOPClassUID == _DICOM_CLASS['RT_DOSE']:
res['dicom_dose'] = _summarize_rt_dose(simulation, plan)
plan.save_as(_dose_dicom_filename(simulation))
if plan.SOPClassUID != _DICOM_CLASS['CT_IMAGE']:
continue
orientation = _float_list(plan.ImageOrientationPatient)
if not (_EXPECTED_ORIENTATION == orientation).all():
continue
if not selected_series:
selected_series = plan.SeriesInstanceUID
res['StudyInstanceUID'] = plan.StudyInstanceUID
res['PixelSpacing'] = plan.PixelSpacing
if hasattr(plan, 'SeriesDescription'):
res['description'] = plan.SeriesDescription
if selected_series != plan.SeriesInstanceUID:
continue
info = {
'pixels': np.float32(plan.pixel_array),
'shape': plan.pixel_array.shape,
'ImagePositionPatient': _string_list(plan.ImagePositionPatient),
'ImageOrientationPatient': _float_list(plan.ImageOrientationPatient),
'PixelSpacing': _float_list(plan.PixelSpacing),
}
for f in ('FrameOfReferenceUID', 'StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID'):
info[f] = getattr(plan, f)
z = _frame_id(info['ImagePositionPatient'][2])
info['frameId'] = z
if z in frames:
raise RuntimeError('duplicate frame with z coord: {}'.format(z))
_scale_pixel_data(plan, info['pixels'])
frames[z] = info
if not selected_series:
raise RuntimeError('No series found with {} orientation'.format(_EXPECTED_ORIENTATION))
if rt_struct_path:
res['regionsOfInterest'] = _summarize_rt_structure(simulation, dicom.read_file(rt_struct_path), frames.keys())
sorted_frames = []
res['frames'] = sorted_frames
for z in sorted(_float_list(frames.keys())):
sorted_frames.append(frames[_frame_id(z)])
return res
def _extract_series_frames(simulation, dicom_dir):
#TODO(pjm): give user a choice between multiple study/series if present
selected_series = None
frames = {}
dicom_dose = None
rt_struct_path = None
res = {
'description': '',
}
for path in pkio.walk_tree(dicom_dir):
if pkio.has_file_extension(str(path), 'dcm'):
plan = dicom.read_file(str(path))
if plan.SOPClassUID == _DICOM_CLASS['RT_STRUCT']:
rt_struct_path = str(path)
elif plan.SOPClassUID == _DICOM_CLASS['RT_DOSE']:
res['dicom_dose'] = _summarize_rt_dose(simulation, plan)
plan.save_as(_dose_dicom_filename(simulation))
if plan.SOPClassUID != _DICOM_CLASS['CT_IMAGE']:
continue
orientation = _float_list(plan.ImageOrientationPatient)
if not (_EXPECTED_ORIENTATION == orientation).all():
continue
if not selected_series:
selected_series = plan.SeriesInstanceUID
res['StudyInstanceUID'] = plan.StudyInstanceUID
res['PixelSpacing'] = plan.PixelSpacing
if hasattr(plan, 'SeriesDescription'):
res['description'] = plan.SeriesDescription
if selected_series != plan.SeriesInstanceUID:
continue
info = {
'pixels': np.float32(plan.pixel_array),
'shape': plan.pixel_array.shape,
'ImagePositionPatient': _string_list(plan.ImagePositionPatient),
'ImageOrientationPatient': _float_list(plan.ImageOrientationPatient),
'PixelSpacing': _float_list(plan.PixelSpacing),
}
for f in ('FrameOfReferenceUID', 'StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID'):
info[f] = getattr(plan, f)
z = _frame_id(info['ImagePositionPatient'][2])
info['frameId'] = z
if z in frames:
raise RuntimeError('duplicate frame with z coord: {}'.format(z))
_scale_pixel_data(plan, info['pixels'])
frames[z] = info
if not selected_series:
raise RuntimeError('No series found with {} orientation'.format(_EXPECTED_ORIENTATION))
if rt_struct_path:
res['regionsOfInterest'] = _summarize_rt_structure(simulation, dicom.read_file(rt_struct_path), frames.keys())
sorted_frames = []
res['frames'] = sorted_frames
for z in sorted(_float_list(frames.keys())):
sorted_frames.append(frames[_frame_id(z)])
return res
def _find(paths):
from pykern import pkio
import re
i = re.compile(r'(?:_work|_data)/')
res = []
cwd = pkio.py_path()
for t in paths or ('tests', ):
t = pkio.py_path(t)
if t.check(file=True):
res.append(str(cwd.bestrelpath(t)))
continue
for p in pkio.walk_tree(t, re.compile(r'_test\.py$')):
p = str(cwd.bestrelpath(p))
if not i.search(p):
res.append(p)
return res
def init_tree(name, author, author_email, description, license, url):
"""Setup a project tree with: docs, tests, etc., and checkin to git.
Creates: setup.py, index.rst, project dir, <name>_console.py, etc.
Overwrites files if they exist without checking.
Args:
name (str): short name of the project, e.g. ``pykern``.
author (str): copyright holder, e.g. ``RadiaSoft LLC``
author_email (str): how to reach author, e.g. ``[email protected]``
description (str): one-line summary of project
license (str): url of license
url (str): website for project, e.g. http://pykern.org
"""
assert os.path.isdir('.git'), \
'Must be run from the root directory of the repo'
assert not os.path.isdir(name), \
'{}: already exists, only works on fresh repos'.format(name)
assert name == py.path.local().basename, \
'{}: name must be the name of the current directory'.format(name)
license = license.lower()
base = pkresource.filename('projex')
values = copy.deepcopy(DEFAULTS)
values.update({
'name': name,
'author': author,
'description': description,
'author_email': author_email,
'url': url,
'license': _license(license, 0),
'classifier_license': _license(license, 1),
})
values['copyright_license_rst'] = values['copyright_license_rst'].format(**values)
suffix_re = r'\.jinja$'
for src in pkio.walk_tree(base, file_re=suffix_re):
dst = py.path.local(src).relto(str(base))
dst = dst.replace('projex', name).replace('dot-', '.')
dst = re.sub(suffix_re, '', dst)
pkio.mkdir_parent_only(dst)
_render(src, values, output=dst)
src = py.path.local(pkresource.filename('projex-licenses'))
src = src.join(license + '.jinja')
_render(src, values, output='LICENSE')
def purge_guest_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
(list, list): dirs and uids of removed guest users (or to remove if confirm)
"""
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
from sirepo import srtime
guest_uids = auth.guest_uids()
now = srtime.utc_now()
dirs_and_uids = {}
for d in pkio.sorted_glob(simulation_db.user_dir_name().join('*')):
uid = simulation_db.uid_from_dir_name(d)
if _is_src_dir(d):
continue
if uid not in guest_uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
dirs_and_uids[d] = uid
if confirm:
pkio.unchecked_remove(*dirs_and_uids.keys())
auth_db.UserRegistration.delete_all_for_column_by_values(
'uid', dirs_and_uids.values())
return dirs_and_uids
def test_purge_users(monkeypatch):
from pykern.pkunit import pkeq, pkok
from pykern.pkdebug import pkdp
from pykern import pkio
from sirepo import sr_unit
# Need to initialize first
sr_unit.init_user_db()
from sirepo.pkcli import admin
from sirepo import simulation_db
from sirepo import server
import datetime
#TODO(pjm): tried pkconfig.reset_state_for_testing() but couldn't override bool to False
server.cfg.oauth_login = False
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: no old users so empty')
pkdp(simulation_db.user_dir_name('*'))
g = simulation_db.user_dir_name('*')
dirs = list(pkio.sorted_glob(g))
pkeq(1, len(dirs), '{}: expecting exactly one user dir', g)
uid = dirs[0].basename
#TODO(robnagler) really want the db to be created, but need
# a test oauth class.
monkeypatch.setattr(server, 'all_uids', lambda: [uid])
for f in pkio.walk_tree(dirs[0]):
f.setmtime(f.mtime() - 86400 * 2)
res = admin.purge_users(days=1, confirm=False)
pkeq([], res, '{}: all users registered so no deletes')
monkeypatch.setattr(server, 'all_uids', lambda: [])
res = admin.purge_users(days=1, confirm=False)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(dirs[0].check(dir=True), '{}: nothing deleted', res)
res = admin.purge_users(days=1, confirm=True)
pkeq(dirs, res, '{}: no users registered so one delete', res)
pkok(not dirs[0].check(dir=True), '{}: directory deleted', res)
def _search_and_parse(dirs):
files = []
for d in dirs:
files.extend(list(pkio.walk_tree(d, file_re=r'index.txt$')))
files.reverse()
res = {
'images': [],
'links': [],
'titles': [],
'words': {},
}
for f, fi in zip(files, range(len(files))):
s = _DIR_RE.search(f.dirname)
assert s, '{}: non-date dirname'.format(f)
root = py.path.local(s.group(1))
y, m, d = s.group(2, 3, 4)
_add_words(res['words'], fi, [y, y + m, y + m + d])
w, i, t = _index_parse(str(f), fi)
res['links'].append(root.bestrelpath(py.path.local(f.dirname)))
res['images'].append(_thumb(i))
res['titles'].append('{}/{}/{}{}'.format(m, d, y, t))
_add_words(res['words'], fi, w)
return res
def _search_and_parse(dirs):
files = []
for d in dirs:
files.extend(list(pkio.walk_tree(d, file_re=r'(?:^|/)index.txt$')))
files.reverse()
res = {
'images': [],
'links': [],
'titles': [],
'words': {},
}
for f, fi in zip(files, range(len(files))):
s = _DIR_RE.search(f.dirname)
assert s, '{}: non-date dirname'.format(f)
root = py.path.local(s.group(1))
y, m, d = s.group(2, 3, 4)
_add_words(res['words'], fi, [y, y + m, y + m + d])
w, i, t = _index_parse(str(f), fi)
res['links'].append(root.bestrelpath(py.path.local(f.dirname)))
res['images'].append(_thumb(i))
res['titles'].append('{}/{}/{}{}'.format(m, d, y, t))
_add_words(res['words'], fi, w)
return res
def purge_users(days=180, confirm=False):
"""Remove old users from db which have not registered.
Args:
days (int): maximum days of untouched files (old is mtime > days)
confirm (bool): delete the directories if True (else don't delete) [False]
Returns:
list: directories removed (or to remove if confirm)
"""
from pykern import pkio
from sirepo import server
from sirepo import simulation_db
from sirepo import api_auth
import datetime
days = int(days)
assert days >= 1, \
'{}: days must be a positive integer'
server.init()
uids = api_auth.all_uids()
now = datetime.datetime.utcnow()
to_remove = []
for d in pkio.sorted_glob(simulation_db.user_dir_name('*')):
if _is_src_dir(d):
continue;
#TODO(pjm): need to skip special "src" user
if simulation_db.uid_from_dir_name(d) in uids:
continue
for f in pkio.walk_tree(d):
if (now - now.fromtimestamp(f.mtime())).days <= days:
break
else:
to_remove.append(d)
if confirm:
pkio.unchecked_remove(*to_remove)
return to_remove
def _h5_file_list(run_dir, model_name):
return pkio.walk_tree(
run_dir.join('diags/xzsolver/hdf5' if model_name == 'currentAnimation' else 'diags/fields/electric'),
r'\.h5$',
)
def upgrade_runner_to_job_db(db_dir):
import sirepo.auth
from pykern import pkio
from pykern.pkcollections import PKDict
from pykern.pkdebug import pkdp, pkdlog, pkdexc
from sirepo import job
from sirepo import simulation_db
from sirepo import sim_data
from sirepo import util
import pykern.pkio
import sirepo.template
def _add_compute_status(run_dir, data):
p = run_dir.join(job.RUNNER_STATUS_FILE)
data.pkupdate(
lastUpdateTime=int(p.mtime()),
status=pkio.read_text(p),
)
def _add_parallel_status(in_json, sim_data, run_dir, data):
t = sirepo.template.import_module(data.simulationType)
# pksetdefault emulates job_cmd._background_percent_complete
data.parallelStatus = PKDict(
t.background_percent_complete(
sim_data.parse_model(in_json),
run_dir,
False,
)).pksetdefault(
lastUpdateTime=data.lastUpdateTime,
frameCount=0,
percentComplete=0.0,
)
def _create_supervisor_state_file(run_dir):
try:
i, t = _load_in_json(run_dir)
except Exception as e:
if pykern.pkio.exception_is_not_found(e):
return
raise
u = simulation_db.uid_from_dir_name(run_dir)
sirepo.auth.cfg.logged_in_user = u
c = sim_data.get_class(i.simulationType)
d = PKDict(
computeJid=c.parse_jid(i, u),
computeJobHash=c.compute_job_hash(
i), # TODO(e-carlin): Another user cookie problem
computeJobSerial=t,
computeJobStart=t,
error=None,
history=[],
isParallel=c.is_parallel(i),
simulationId=i.simulationId,
simulationType=i.simulationType,
uid=u,
)
d.pkupdate(
jobRunMode=job.PARALLEL if d.isParallel else job.SEQUENTIAL,
nextRequestSeconds=c.poll_seconds(i),
)
_add_compute_status(run_dir, d)
if d.status not in (job.COMPLETED, job.CANCELED):
return
if d.isParallel:
_add_parallel_status(i, c, run_dir, d)
util.json_dump(d, path=_db_file(d.computeJid))
def _db_file(computeJid):
return db_dir.join(computeJid + '.json')
def _load_in_json(run_dir):
p = simulation_db.json_filename(
sirepo.template.template_common.INPUT_BASE_NAME, run_dir)
c = simulation_db.read_json(p)
return c, c.computeJobCacheKey.computeJobStart if \
c.get('computejobCacheKey') else \
int(p.mtime())
c = 0
db_dir = pkio.py_path(db_dir)
pkio.mkdir_parent(db_dir)
for f in pkio.walk_tree(
simulation_db.user_dir_name(),
'^(?!.*src/).*/{}$'.format(sirepo.job.RUNNER_STATUS_FILE),
):
try:
_create_supervisor_state_file(pkio.py_path(f.dirname))
except Exception as e:
c += 1
k = PKDict(run_dir=f)
s = 'run_dir={run_dir}'
if c < 50:
k.stack = pkdexc()
s += ' stack={stack}'
else:
k.error = getattr(e, 'args', []) or e
s += ' error={error}'
pkdlog(s, **k)
def default_command():
"""Find all dirs and try to fix"""
for f in pkio.walk_tree('.', file_re=r'index.txt$'):
with pkio.save_chdir(f.dirname):
_one_dir()
def _check_run_dir(should_exist=0):
f = pkio.walk_tree(fc.sr_user_dir(), file_re=m)
pkunit.pkeq(should_exist, len(f), 'incorrect file count')
def _h5_file_list(run_dir):
return pkio.walk_tree(
run_dir.join('hdf5'),
r'\.h5$',
)
def examples(app):
files = pkio.walk_tree(
pkresource.filename(_EXAMPLE_DIR_FORMAT.format(app)),
re.escape(JSON_SUFFIX) + '$',
)
return [open_json_file(app, str(f)) for f in files]
def _h5_file_list(run_dir):
return pkio.walk_tree(
run_dir.join('hdf5'),
r'\.h5$',
)
def default_command():
import rsconf.component
import rsconf.component.rsconf
import rsconf.pkcli.tls
root_d = db.cfg.root_d
if root_d.check():
return '{}: already exists'.format(root_d)
srv = pkio.mkdir_parent(root_d.join(db.SRV_SUBDIR))
def _sym(old, new_base=None):
old = pkio.py_path(old)
if not new_base:
new_base = old.basename
assert old.check(), \
'{}: does not exist'.format(old)
srv.join(new_base).mksymlinkto(old, absolute=False)
# ssh-keygen -q -N '' -C rsconf -t rsa -b 4096 -f /var/tmp/foo
# -- don't need this
db_d = pkio.mkdir_parent(root_d.join(db.DB_SUBDIR))
secret_d = pkio.mkdir_parent(db_d.join(db.SECRET_SUBDIR))
nginx_d = pkio.mkdir_parent(root_d.join(NGINX_SUBDIR))
boot_hdb = pkcollections.Dict(rsconf_db=pkcollections.Dict(
secret_d=secret_d,
channel='dev',
))
j2_ctx = pkcollections.Dict(
all_host='v9.radia.run',
group=grp.getgrgid(os.getgid())[0],
host='v4.radia.run',
master='v3.radia.run',
port=2916,
root_d=root_d,
srv_d=str(srv),
uid=os.getuid(),
user=pwd.getpwuid(os.getuid())[0],
worker5_host='v5.radia.run',
worker6_host='v6.radia.run',
)
hosts = [h for h in j2_ctx.values() if str(h).endswith('.radia.run')]
# bootstrap
j2_ctx.update(boot_hdb)
j2_ctx.rsconf_db.http_host = 'http://{}:{}'.format(j2_ctx.master,
j2_ctx.port)
j2_ctx.bkp = pkcollections.Dict(primary=j2_ctx.host)
j2_ctx.passwd_f = rsconf.component.rsconf.passwd_secret_f(j2_ctx)
for h in hosts:
_add_host(j2_ctx, srv, h)
_sym('~/src/radiasoft/download/bin/install.sh', 'index.html')
_sym(pkresource.filename('rsconf/rsconf.sh'), 'rsconf.sh')
dev_d = pkio.py_path(pkresource.filename('dev'))
for f in pkio.walk_tree(dev_d):
if str(f).endswith('~') or str(f).startswith('#'):
continue
x = str(f.relto(dev_d))
if not ('local/' in x and x.endswith('.sh.jinja')):
x = re.sub('.jinja$', '', x)
dst = root_d.join(x)
pkio.mkdir_parent_only(dst)
if f.basename == dst.basename:
f.copy(dst)
else:
pkjinja.render_file(f, j2_ctx, output=dst, strict_undefined=True)
n = []
for e in 'rpm', 'proprietary':
d = pkio.py_path(root_d.dirname).join(e)
pkio.mkdir_parent(d)
root_d.join(e).mksymlinkto(d, absolute=False)
n.append(str(d))
subprocess.check_call(['bash', str(secret_d.join('setup_dev.sh')), *n], )
# dev only, really insecure, but makes consistent builds easy
_sym('~/src/radiasoft')
_sym('~/src/biviosoftware')
for h in hosts:
# needed to be able to talk setup certs for registry so we can
# pull private images from all hosts. Only used in dev, because
# private registry doesn't protect against pushes from these hosts.
if h != j2_ctx.master:
subprocess.check_call(
['rsconf', 'host', 'init_docker_registry', h])
tls_d = secret_d.join(rsconf.component.TLS_SECRET_SUBDIR)
tls_d.ensure(dir=True)
for h in (
'jupyter.' + j2_ctx.all_host,
'jupyter.' + j2_ctx.host,
j2_ctx.all_host,
j2_ctx.master,
j2_ctx.worker5_host,
j2_ctx.worker6_host,
):
rsconf.pkcli.tls.gen_self_signed_crt(
tls_d.join(h),
h,
)
rsconf.pkcli.tls.gen_self_signed_crt(
tls_d.join('star.' + j2_ctx.host),
'*.' + j2_ctx.host,
j2_ctx.host,
)
def _h5_file_list(run_dir, model_name):
return pkio.walk_tree(
run_dir.join('diags/xzsolver/hdf5' if model_name ==
'currentAnimation' else 'diags/fields/electric'),
r'\.h5$',
)
def examples(app):
files = pkio.walk_tree(
pkresource.filename(_EXAMPLE_DIR_FORMAT.format(app)),
re.escape(JSON_SUFFIX) + '$',
)
return [open_json_file(app, str(f)) for f in files]
