def internal_build_compile(self):
from rsconf import db
from rsconf import systemd
from rsconf.component import docker_registry
self.buildt.require_component('docker')
jc, z = self.j2_ctx_init()
self.__run_d = systemd.docker_unit_prepare(self, jc)
z.update(
user_d=self._jupyterhub_db().join(_USER_SUBDIR),
jupyter_docker_image=docker_registry.absolute_image(
jc, z.jupyter_docker_image,
),
run_u=jc.rsconf_db.run_u,
jupyter_run_u=jc.rsconf_db.run_u,
)
z.setdefault('http_timeout', 30);
z.setdefault('template_vars', {});
self._network(z, jc)
z.admin_users_str = _list_to_str(z.admin_users)
z.template_d = self.__run_d.join(_TEMPLATE_D)
z.home_d = db.user_home_path(jc, z.jupyter_run_u)
z.cookie_secret_hex = self.secret_path_value(
self._COOKIE_SECRET,
gen_secret=lambda: db.random_string(length=64, is_hex=True),
visibility='channel',
)[0]
z.proxy_auth_token = self.secret_path_value(
self._PROXY_AUTH,
gen_secret=lambda: db.random_string(length=64, is_hex=True),
visibility='channel',
)[0]
self._rsdockerspawner(z)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.github_bkp
z.run_u = j2_ctx.rsconf_db.run_u
run_d = systemd.timer_prepare(self, j2_ctx, on_calendar=z.on_calendar)
z.db_d = run_d.join('db')
z.run_f = run_d.join('run')
env = pkcollections.Dict(
PYKERN_PKCLI_GITHUB_EXCLUDE_RE=z.exclude_re,
PYKERN_PKCLI_GITHUB_PASSWORD=z.password,
PYKERN_PKCLI_GITHUB_TEST_MODE=z.setdefault('test_mode', False),
PYKERN_PKCLI_GITHUB_USER=z.user,
)
systemd.docker_unit_enable(
self,
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
j2_ctx=j2_ctx,
env=env,
run_u=z.run_u,
cmd=z.run_f,
)
self.install_access(mode='500', owner=z.run_u)
self.install_resource('github_bkp/run.sh', j2_ctx, z.run_f)
self.install_access(mode='700')
self.install_directory(z.db_d)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
import pykern.pkdebug
def _env_ok(k):
if k.startswith('PYKERN_') or k.startswith(
'SIREPO_FEATURE_CONFIG_'):
return not pykern.pkdebug.SECRETS_RE.search(k)
return (k == _SECRET
or k.startswith('SIREPO_PKCLI_TEST_HTTP_SERVER'))
jc, z = self.j2_ctx_init()
e = PKDict()
for k, v in self.buildt.get_component('sirepo').sirepo_unit_env(
self).items():
if _env_ok(k):
e[k] = v
assert e.get(_SECRET), \
f'{_SECRET} not found in sirepo config'
systemd.timer_prepare(self, jc, on_calendar=z.on_calendar)
e.pksetdefault(
SIREPO_PKCLI_TEST_HTTP_SERVER_URI=f'https://{jc.sirepo.vhost}', )
systemd.docker_unit_enable(
self,
cmd='sirepo test_http',
env=e,
image=docker_registry.absolute_image(jc, jc.sirepo.docker_image),
j2_ctx=jc,
run_u=jc.rsconf_db.run_u,
)
def _enable_docker(self):
from rsconf import systemd
from rsconf.component import docker_registry
z = self.j2_ctx[self.name]
systemd.docker_unit_enable(
self,
self.j2_ctx,
cmd="bash -l -c 'jupyterhub -f {}'".format(self.__conf_f),
image=docker_registry.absolute_image(self.j2_ctx, z.docker_image),
run_u=z.run_u,
)
def _enable_docker(self):
from rsconf import systemd
from rsconf.component import docker_registry
def _env_ok(elem):
for p in (
# TODO(e-carlin): all of auth and smtp is really too much.
# We just need auth_methods for checking if the cookies is
# valid. But bringing in methods then initializes the methods
# so we have to bring in their full config.
'SIREPO_AUTH',
'SIREPO_COOKIE',
'SIREPO_FEATURE_PROPRIETARY_SIM',
'SIREPO_SIM_API_JUPYTERHUBLOGIN',
'SIREPO_SMTP',
'SIREPO_SRDB',
):
if elem[0].startswith(f'{p}'):
return True
return False
z = self.j2_ctx[self.name]
systemd.docker_unit_enable(
self,
self.j2_ctx,
cmd="bash -l -c 'jupyterhub -f {}'".format(self.__conf_f),
env=PKDict(
filter(
_env_ok,
self.buildt.get_component('sirepo').sirepo_unit_env(
self).items(),
),
**pkconfig.to_environ(
['*'],
values=dict(sirepo=dict(feature_config=dict(
sim_types=set(('jupyterhublogin', ))), )),
),
),
image=docker_registry.absolute_image(self.j2_ctx, z.docker_image),
run_u=z.run_u,
volumes=[
self._jupyterhub_db(),
self.j2_ctx.sirepo.srdb.root,
],
)
def internal_build_write(self):
from rsconf import systemd
from rsconf.component import docker_registry
jc = self.jc
z = jc.mpi_worker
self.install_access(mode='700', owner=z.run_u)
# Need to make sure host_d exists, even though it isn't ours
for d in z.host_d, z.host.conf_d, z.host.ssh_d:
self.install_directory(d)
self.install_access(mode='400')
self.install_resource(z.host.sshd_config, jc)
for k, v in z.secrets.items():
self.install_abspath(v, z.host[k])
if z.is_first:
for f in 'ssh_config', 'known_hosts':
self.install_resource(z.host[f], jc)
self.install_access(mode='700')
self.install_directory(z.host.bin_d)
self.install_access(mode='500')
self.install_resource(z.host.bin_d.join('rsmpi.sh'), jc)
x = [
[z.host_d, z.guest_d],
# SECURITY: no modifications to run_d
[z.run_d, z.run_d, 'ro'],
]
for k in sorted(z.volumes.keys()):
g = str(z.volumes[k]).format(username=z.user)
assert g.startswith(str(z.guest_d) + '/'), \
'mount={} must start with guest_d={}'.format(g, z.guest_d)
k = k.format(username=z.user)
x.append([k, g])
systemd.docker_unit_enable(
self,
jc,
image=docker_registry.absolute_image(jc, z.docker_image),
volumes=x,
cmd="/usr/sbin/sshd -D -f '{}'".format(z.guest.sshd_config),
)
def internal_build(self):
from rsconf import systemd
from rsconf.component import db_bkp
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.rs_mariadb
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.conf_f = z.run_d.join('my.cnf')
z.db_d = z.run_d.join('db')
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
[z.conf_f, '/etc/my.cnf'],
# hardwired in some places, not sure where, and hardwired
# in Dockefile so just mount as an alias
[z.db_d, '/var/lib/mysql'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
# find in path (probably /usr/sbin, but might be /usr/libexec)
cmd='mysqld',
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400')
self.install_resource('rs_mariadb/my.cnf', j2_ctx, z.conf_f)
db_bkp.install_script_and_subdir(
self,
j2_ctx,
run_u=z.run_u,
run_d=z.run_d,
)
self.append_root_bash_with_main(j2_ctx)
self.rsconf_service_restart()
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.owncloud_redis
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.conf_f = z.run_d.join('redis.conf')
z.db_d = z.run_d.join('db')
systemd.docker_unit_enable(
self,
j2_ctx,
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
# find in path (probably /usr/local/bin, but might be /usr/bin)
cmd="redis-server '{}'".format(z.conf_f),
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400')
self.install_resource('owncloud_redis/redis.conf', j2_ctx, z.conf_f)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
from rsconf.component import logrotate
from rsconf.component import nginx
self.buildt.require_component('docker', 'nginx', 'rs_mariadb')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.owncloud
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.db_d = z.run_d.join('db')
z.run_f = z.run_d.join('run')
z.apache_conf_f = z.run_d.join('apache.conf')
z.apache_envvars_f = z.run_d.join('envvars')
# Created dynamically every run
z.apache_run_d = '/tmp/apache2'
z.conf_d = z.db_d.join('config')
z.conf_f = z.conf_d.join('config.php')
z.files_d = z.db_d.join('files')
z.init_conf_f = z.run_d.join('init_config.php')
z.log_d = z.run_d.join('log')
z.sessions_d = z.db_d.join('sessions')
z.log_postrotate_f = z.run_d.join('reload')
# 127.0.0.1 assumes docker --network=host
# If you connect to "localhost" (not 127.0.0.1) mysql fails to connect,
# because the socket isn't there, instead of trying TCP (port supplied).
# mysql -h localhost -P 7011 -u owncloud
# ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
z.db_host = '{}:{}'.format(
j2_ctx.rs_mariadb.ip,
j2_ctx.rs_mariadb.port,
)
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
# /mnt/data/sessions is hardwired in
# /etc/php/7.0/mods-available/owncloud.ini. We could overrided
# but this is just as easy, and likely to make other things work
# that are hardwired.
[z.db_d, '/mnt/data'],
[z.conf_d, '/var/www/owncloud/config'],
[
z.apache_conf_f,
'/etc/apache2/sites-enabled/000-default.conf'
],
[z.apache_envvars_f, '/etc/apache2/envvars'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
cmd=z.run_f,
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
for d in z.db_d, z.log_d, z.files_d, z.sessions_d, z.conf_d:
self.install_directory(d)
self.install_access(mode='400')
self.install_resource('owncloud/init_config.php', j2_ctx,
z.init_conf_f)
self.install_resource('owncloud/apache.conf', j2_ctx, z.apache_conf_f)
self.install_resource('owncloud/apache_envvars', j2_ctx,
z.apache_envvars_f)
self.install_access(mode='500')
self.install_resource('owncloud/run.sh', j2_ctx, z.run_f)
self.install_resource('owncloud/reload.sh', j2_ctx, z.log_postrotate_f)
logrotate.install_conf(self, j2_ctx)
nginx.install_vhost(
self,
vhost=z.domain,
backend_host='127.0.0.1',
backend_port=z.port,
j2_ctx=j2_ctx,
)
self.append_root_bash_with_main(j2_ctx)
def docker_unit_enable(compt, j2_ctx, image, cmd, env=None, volumes=None, after=None, run_u=None, ports=None):
"""Must be last call"""
from rsconf.component import docker_registry
def _extra_run_flags():
try:
c = z.pknested_get('extra_run_flags.' + compt.name)
except KeyError:
return ''
return ' '.join(
(f'--{k}={v}' for k, v in c.items()),
)
z = j2_ctx.systemd
if env is None:
env = PKDict()
if 'TZ' not in env:
# Tested on CentOS 7, and it does have the localtime stat problem
# https://blog.packagecloud.io/eng/2017/02/21/set-environment-variable-save-thousands-of-system-calls/
env['TZ'] = ':/etc/localtime'
image = docker_registry.absolute_image(j2_ctx, image)
z.update(
after=_after(after),
extra_run_flags=_extra_run_flags(),
service_exec=cmd,
exports='\n'.join(
["export '{}={}'".format(k, env[k]) for k in sorted(env.keys())],
),
image=image,
run_u=run_u or j2_ctx.rsconf_db.run_u,
)
volumes = _tuple_arg(volumes)
run_d_in_volumes = False
for v in volumes:
if v[0] == z.run_d:
run_d_in_volumes = True
if not run_d_in_volumes:
volumes.insert(0, (str(z.run_d), str(z.run_d)))
z.volumes = _colon_format('-v', volumes)
z.network = _colon_format('-p', _tuple_arg(ports))
if not z.network:
z.network = '--network=host'
scripts = ('cmd', 'env', 'remove', 'start', 'stop')
compt.install_access(mode='700', owner=z.run_u)
compt.install_directory(z.run_d)
compt.install_access(mode='500')
for s in scripts:
z[s] = z.run_d.join(s)
if not cmd:
z.cmd = ''
for s in scripts:
if z[s]:
compt.install_resource('systemd/docker_' + s, j2_ctx, z[s])
# See Poettering's omniscience about what's good for all of us here:
# https://github.com/systemd/systemd/issues/770
# These files should be 400, since there's no value in making them public.
compt.install_access(mode='444', owner=j2_ctx.rsconf_db.root_u)
if z.is_timer:
compt.install_resource(
'systemd/timer_unit',
j2_ctx,
z.timer_f,
)
compt.install_resource(
'systemd/docker_unit',
j2_ctx,
z.service_f,
)
compt.append_root_bash(
"rsconf_service_docker_pull '{}' '{}'".format(z.image, z.service_name),
)
unit_enable(compt, j2_ctx)
def internal_build_compile(self):
from rsconf.component import docker_registry
from rsconf import db
self.__docker_unit_enable_after = []
self.__uwsgi_docker_vols = []
self.buildt.require_component('docker', 'nginx', 'db_bkp')
jc, z = self.j2_ctx_init()
self.__run_d = systemd.docker_unit_prepare(self, jc)
d = self.__run_d.join(_DB_SUBDIR)
self.j2_ctx_pksetdefault(dict(
sirepo=dict(
cookie=dict(
http_name=lambda: 'sirepo_{}'.format(jc.rsconf_db.channel),
private_key=lambda: self.secret_path_value(
_COOKIE_PRIVATE_KEY,
gen_secret=lambda: pkcompat.from_bytes(
base64.urlsafe_b64encode(os.urandom(32)),
),
visibility='channel',
)[0],
),
docker_image=docker_registry.absolute_image(jc, z.docker_image),
feature_config=dict(
api_modules=[],
job=True,
default_proprietary_sim_types=tuple(),
proprietary_sim_types=tuple(),
),
job=dict(
max_message_bytes='200m',
),
pkcli=dict(
service=dict(
ip='0.0.0.0',
run_dir=self.__run_d,
),
),
srdb=dict(root=d),
),
pykern=dict(
pkdebug=dict(
redirect_logging=True,
want_pid_time=True,
),
pkconfig=dict(channel=jc.rsconf_db.channel),
),
))
self._comsol(z)
self._jupyterhublogin(z)
self.j2_ctx_pksetdefault(dict(
sirepo=dict(
client_max_body_size=pkconfig.parse_bytes(z.job.max_message_bytes),
job_driver=dict(modules=['docker']),
job=dict(
server_secret=lambda: self.secret_path_value(
_SERVER_SECRET,
gen_secret=lambda: pkcompat.from_bytes(
base64.urlsafe_b64encode(os.urandom(32)),
),
visibility='channel',
)[0],
verify_tls=lambda: jc.pykern.pkconfig.channel != 'dev',
),
pkcli=dict(
job_supervisor=dict(
ip='127.0.0.1',
port=8001,
),
),
),
))
z.pksetdefault(job_api=PKDict)
# server connects locally only so go direct to tornado.
# supervisor has different uri to pass to agents.
z.job_api.supervisor_uri = 'http://{}:{}'.format(
z.pkcli.job_supervisor.ip,
z.pkcli.job_supervisor.port,
)
self._set_sirepo_config('sirepo_job_supervisor')
#TODO(robnagler) remove when deployed to alpha
z.job.supervisor_uri = z.job_api.supervisor_uri
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
from rsconf.component import logrotate
from rsconf.component import nginx
self.buildt.require_component('docker', 'nginx', 'rs_mariadb')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.wordpress
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.srv_d = z.run_d.join('srv')
z.run_f = z.run_d.join('run')
z.apache_conf_f = z.run_d.join('apache.conf')
z.apache_envvars_f = z.run_d.join('envvars')
z.setdefault('client_max_body_size', _DEFAULT_CLIENT_MAX_BODY_SIZE)
# Created dynamically every run
z.apache_run_d = '/tmp/apache2'
z.log_d = z.run_d.join('log')
z.ip = '127.0.0.1'
z.log_postrotate_f = z.run_d.join('reload')
z.setdefault('num_servers', 4)
# 127.0.0.1 assumes docker --network=host
# If you connect to "localhost" (not 127.0.0.1) mysql fails to connect,
# because the socket isn't there, instead of trying TCP (port supplied).
# mysql -h localhost -P 7011 -u wordpress
# ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
z.db_host = '{}:{}'.format(
j2_ctx.rs_mariadb.ip,
j2_ctx.rs_mariadb.port,
)
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
[z.apache_conf_f, '/etc/apache2/sites-enabled/000-default.conf'],
[z.apache_envvars_f, '/etc/apache2/envvars'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
cmd=z.run_f,
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
for d in z.log_d, z.srv_d:
self.install_directory(d)
for h, v in z.vhosts.items():
v.srv_d = z.srv_d.join(h)
self.install_directory(v.srv_d)
self.install_access(mode='400')
self.install_resource('wordpress/apache_envvars', j2_ctx, z.apache_envvars_f)
self.install_resource('wordpress/apache.conf', j2_ctx, z.apache_conf_f)
self.install_access(mode='500')
self.install_resource('wordpress/run.sh', j2_ctx, z.run_f)
self.install_resource('wordpress/reload.sh', j2_ctx, z.log_postrotate_f)
logrotate.install_conf(self, j2_ctx)
for h, v in z.vhosts.items():
nginx.install_vhost(
self,
vhost=h,
backend_host=z.ip,
backend_port=v.port,
j2_ctx=j2_ctx,
)
self.append_root_bash_with_main(j2_ctx)
