def internal_build_write(self):
from rsconf.component import db_bkp
from rsconf.component import nginx
from rsconf.component import docker
jc = self.j2_ctx
z = jc[self.name]
self._install_dirs_and_files()
nginx.install_vhost(
self,
vhost=z.vhost,
backend_host=jc.rsconf_db.host,
backend_port=z.pkcli.service_port,
j2_ctx=jc,
)
systemd.docker_unit_enable(
self,
jc,
image=z.docker_image,
env=self.sirepo_unit_env(),
cmd='sirepo service uwsgi',
after=self.__docker_unit_enable_after,
#TODO(robnagler) wanted by nginx
volumes=self.__uwsgi_docker_vols,
)
db_bkp.install_script_and_subdir(
self,
jc,
run_u=jc.rsconf_db.run_u,
run_d=self.__run_d,
)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.github_bkp
z.run_u = j2_ctx.rsconf_db.run_u
run_d = systemd.timer_prepare(self, j2_ctx, on_calendar=z.on_calendar)
z.db_d = run_d.join('db')
z.run_f = run_d.join('run')
env = pkcollections.Dict(
PYKERN_PKCLI_GITHUB_EXCLUDE_RE=z.exclude_re,
PYKERN_PKCLI_GITHUB_PASSWORD=z.password,
PYKERN_PKCLI_GITHUB_TEST_MODE=z.setdefault('test_mode', False),
PYKERN_PKCLI_GITHUB_USER=z.user,
)
systemd.docker_unit_enable(
self,
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
j2_ctx=j2_ctx,
env=env,
run_u=z.run_u,
cmd=z.run_f,
)
self.install_access(mode='500', owner=z.run_u)
self.install_resource('github_bkp/run.sh', j2_ctx, z.run_f)
self.install_access(mode='700')
self.install_directory(z.db_d)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
import pykern.pkdebug
def _env_ok(k):
if k.startswith('PYKERN_') or k.startswith(
'SIREPO_FEATURE_CONFIG_'):
return not pykern.pkdebug.SECRETS_RE.search(k)
return (k == _SECRET
or k.startswith('SIREPO_PKCLI_TEST_HTTP_SERVER'))
jc, z = self.j2_ctx_init()
e = PKDict()
for k, v in self.buildt.get_component('sirepo').sirepo_unit_env(
self).items():
if _env_ok(k):
e[k] = v
assert e.get(_SECRET), \
f'{_SECRET} not found in sirepo config'
systemd.timer_prepare(self, jc, on_calendar=z.on_calendar)
e.pksetdefault(
SIREPO_PKCLI_TEST_HTTP_SERVER_URI=f'https://{jc.sirepo.vhost}', )
systemd.docker_unit_enable(
self,
cmd='sirepo test_http',
env=e,
image=docker_registry.absolute_image(jc, jc.sirepo.docker_image),
j2_ctx=jc,
run_u=jc.rsconf_db.run_u,
)
def _enable_docker(self):
from rsconf import systemd
from rsconf.component import docker_registry
z = self.j2_ctx[self.name]
systemd.docker_unit_enable(
self,
self.j2_ctx,
cmd="bash -l -c 'jupyterhub -f {}'".format(self.__conf_f),
image=docker_registry.absolute_image(self.j2_ctx, z.docker_image),
run_u=z.run_u,
)
def internal_build(self):
from rsconf import systemd
# https://docs.docker.com/registry/configuration/
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
run_d = systemd.docker_unit_prepare(self, j2_ctx)
assert update_j2_ctx(j2_ctx), \
'no registry host'
conf_f = run_d.join('config.yml')
volumes = [
[conf_f, _GLOBAL_CONF],
]
systemd.docker_unit_enable(
self,
j2_ctx,
# Specify pull from docker.io directly to avoid registry not yet running
image=REGISTRY_IMAGE,
cmd=None,
after=['docker.service'],
run_u=j2_ctx.docker_registry.run_u,
volumes=volumes,
)
kc = self.install_tls_key_and_crt(j2_ctx.docker_registry.host, run_d)
db_d = run_d.join(_DB_SUBDIR)
j2_ctx.docker_registry.update(
auth_htpasswd_path=run_d.join('passwd'),
conf_f=conf_f,
db_d=db_d,
http_tls_certificate=kc.crt,
http_tls_key=kc.key,
)
j2_ctx.docker_registry.http_secret = self.secret_path_value(
_HTTP_SECRET_F,
lambda: db.random_string(length=64),
visibility=_HTTP_SECRET_VISIBILITY,
)[0]
self.install_secret_path(
_PASSWD_SECRET_F,
j2_ctx.docker_registry.auth_htpasswd_path,
gen_secret=db.random_string,
visibility=_PASSWD_VISIBILITY,
)
self.install_access(mode='700', owner=j2_ctx.docker_registry.run_u)
self.install_directory(db_d)
self.install_access(mode='400', owner=j2_ctx.docker_registry.run_u)
self.install_resource('docker_registry/config.yml', j2_ctx,
j2_ctx.docker_registry.conf_f)
self.rsconf_service_restart()
def _enable_docker(self):
from rsconf import systemd
from rsconf.component import docker_registry
def _env_ok(elem):
for p in (
# TODO(e-carlin): all of auth and smtp is really too much.
# We just need auth_methods for checking if the cookies is
# valid. But bringing in methods then initializes the methods
# so we have to bring in their full config.
'SIREPO_AUTH',
'SIREPO_COOKIE',
'SIREPO_FEATURE_PROPRIETARY_SIM',
'SIREPO_SIM_API_JUPYTERHUBLOGIN',
'SIREPO_SMTP',
'SIREPO_SRDB',
):
if elem[0].startswith(f'{p}'):
return True
return False
z = self.j2_ctx[self.name]
systemd.docker_unit_enable(
self,
self.j2_ctx,
cmd="bash -l -c 'jupyterhub -f {}'".format(self.__conf_f),
env=PKDict(
filter(
_env_ok,
self.buildt.get_component('sirepo').sirepo_unit_env(
self).items(),
),
**pkconfig.to_environ(
['*'],
values=dict(sirepo=dict(feature_config=dict(
sim_types=set(('jupyterhublogin', ))), )),
),
),
image=docker_registry.absolute_image(self.j2_ctx, z.docker_image),
run_u=z.run_u,
volumes=[
self._jupyterhub_db(),
self.j2_ctx.sirepo.srdb.root,
],
)
def internal_build_write(self):
from rsconf import systemd
from rsconf.component import docker_registry
jc = self.jc
z = jc.mpi_worker
self.install_access(mode='700', owner=z.run_u)
# Need to make sure host_d exists, even though it isn't ours
for d in z.host_d, z.host.conf_d, z.host.ssh_d:
self.install_directory(d)
self.install_access(mode='400')
self.install_resource(z.host.sshd_config, jc)
for k, v in z.secrets.items():
self.install_abspath(v, z.host[k])
if z.is_first:
for f in 'ssh_config', 'known_hosts':
self.install_resource(z.host[f], jc)
self.install_access(mode='700')
self.install_directory(z.host.bin_d)
self.install_access(mode='500')
self.install_resource(z.host.bin_d.join('rsmpi.sh'), jc)
x = [
[z.host_d, z.guest_d],
# SECURITY: no modifications to run_d
[z.run_d, z.run_d, 'ro'],
]
for k in sorted(z.volumes.keys()):
g = str(z.volumes[k]).format(username=z.user)
assert g.startswith(str(z.guest_d) + '/'), \
'mount={} must start with guest_d={}'.format(g, z.guest_d)
k = k.format(username=z.user)
x.append([k, g])
systemd.docker_unit_enable(
self,
jc,
image=docker_registry.absolute_image(jc, z.docker_image),
volumes=x,
cmd="/usr/sbin/sshd -D -f '{}'".format(z.guest.sshd_config),
)
def internal_build(self):
from rsconf import systemd
from rsconf.component import db_bkp
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.rs_mariadb
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.conf_f = z.run_d.join('my.cnf')
z.db_d = z.run_d.join('db')
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
[z.conf_f, '/etc/my.cnf'],
# hardwired in some places, not sure where, and hardwired
# in Dockefile so just mount as an alias
[z.db_d, '/var/lib/mysql'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
# find in path (probably /usr/sbin, but might be /usr/libexec)
cmd='mysqld',
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400')
self.install_resource('rs_mariadb/my.cnf', j2_ctx, z.conf_f)
db_bkp.install_script_and_subdir(
self,
j2_ctx,
run_u=z.run_u,
run_d=z.run_d,
)
self.append_root_bash_with_main(j2_ctx)
self.rsconf_service_restart()
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.owncloud_redis
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.conf_f = z.run_d.join('redis.conf')
z.db_d = z.run_d.join('db')
systemd.docker_unit_enable(
self,
j2_ctx,
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
# find in path (probably /usr/local/bin, but might be /usr/bin)
cmd="redis-server '{}'".format(z.conf_f),
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400')
self.install_resource('owncloud_redis/redis.conf', j2_ctx, z.conf_f)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
# https://docs.docker.com/registry/configuration/
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
assert update_j2_ctx(j2_ctx), \
'no docker_cache.host in hdb'
z = j2_ctx.docker_cache
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
# Has to run as root, because 3rd party container
z.run_u = j2_ctx.rsconf_db.root_u
z.conf_f = z.run_d.join('config.yml')
z.db_d = z.run_d.join(_DB_SUBDIR)
volumes = [
[z.conf_f, _GLOBAL_CONF],
]
systemd.docker_unit_enable(
self,
j2_ctx,
image=docker_registry.REGISTRY_IMAGE,
cmd=None,
after=['docker.service'],
run_u=z.run_u,
volumes=volumes,
)
kc = self.install_tls_key_and_crt(z.host, z.run_d)
z.update(
http_tls_certificate=kc.crt,
http_tls_key=kc.key,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400', owner=z.run_u)
self.install_resource('docker_cache/config.yml', j2_ctx, z.conf_f)
self.rsconf_service_restart()
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
from rsconf.component import logrotate
from rsconf.component import nginx
self.buildt.require_component('docker', 'nginx', 'rs_mariadb')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.owncloud
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.db_d = z.run_d.join('db')
z.run_f = z.run_d.join('run')
z.apache_conf_f = z.run_d.join('apache.conf')
z.apache_envvars_f = z.run_d.join('envvars')
# Created dynamically every run
z.apache_run_d = '/tmp/apache2'
z.conf_d = z.db_d.join('config')
z.conf_f = z.conf_d.join('config.php')
z.files_d = z.db_d.join('files')
z.init_conf_f = z.run_d.join('init_config.php')
z.log_d = z.run_d.join('log')
z.sessions_d = z.db_d.join('sessions')
z.log_postrotate_f = z.run_d.join('reload')
# 127.0.0.1 assumes docker --network=host
# If you connect to "localhost" (not 127.0.0.1) mysql fails to connect,
# because the socket isn't there, instead of trying TCP (port supplied).
# mysql -h localhost -P 7011 -u owncloud
# ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
z.db_host = '{}:{}'.format(
j2_ctx.rs_mariadb.ip,
j2_ctx.rs_mariadb.port,
)
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
# /mnt/data/sessions is hardwired in
# /etc/php/7.0/mods-available/owncloud.ini. We could overrided
# but this is just as easy, and likely to make other things work
# that are hardwired.
[z.db_d, '/mnt/data'],
[z.conf_d, '/var/www/owncloud/config'],
[
z.apache_conf_f,
'/etc/apache2/sites-enabled/000-default.conf'
],
[z.apache_envvars_f, '/etc/apache2/envvars'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
cmd=z.run_f,
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
for d in z.db_d, z.log_d, z.files_d, z.sessions_d, z.conf_d:
self.install_directory(d)
self.install_access(mode='400')
self.install_resource('owncloud/init_config.php', j2_ctx,
z.init_conf_f)
self.install_resource('owncloud/apache.conf', j2_ctx, z.apache_conf_f)
self.install_resource('owncloud/apache_envvars', j2_ctx,
z.apache_envvars_f)
self.install_access(mode='500')
self.install_resource('owncloud/run.sh', j2_ctx, z.run_f)
self.install_resource('owncloud/reload.sh', j2_ctx, z.log_postrotate_f)
logrotate.install_conf(self, j2_ctx)
nginx.install_vhost(
self,
vhost=z.domain,
backend_host='127.0.0.1',
backend_port=z.port,
j2_ctx=j2_ctx,
)
self.append_root_bash_with_main(j2_ctx)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
from rsconf.component import logrotate
from rsconf.component import nginx
self.buildt.require_component('docker', 'nginx', 'rs_mariadb')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.wordpress
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.srv_d = z.run_d.join('srv')
z.run_f = z.run_d.join('run')
z.apache_conf_f = z.run_d.join('apache.conf')
z.apache_envvars_f = z.run_d.join('envvars')
z.setdefault('client_max_body_size', _DEFAULT_CLIENT_MAX_BODY_SIZE)
# Created dynamically every run
z.apache_run_d = '/tmp/apache2'
z.log_d = z.run_d.join('log')
z.ip = '127.0.0.1'
z.log_postrotate_f = z.run_d.join('reload')
z.setdefault('num_servers', 4)
# 127.0.0.1 assumes docker --network=host
# If you connect to "localhost" (not 127.0.0.1) mysql fails to connect,
# because the socket isn't there, instead of trying TCP (port supplied).
# mysql -h localhost -P 7011 -u wordpress
# ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
z.db_host = '{}:{}'.format(
j2_ctx.rs_mariadb.ip,
j2_ctx.rs_mariadb.port,
)
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
[z.apache_conf_f, '/etc/apache2/sites-enabled/000-default.conf'],
[z.apache_envvars_f, '/etc/apache2/envvars'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
cmd=z.run_f,
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
for d in z.log_d, z.srv_d:
self.install_directory(d)
for h, v in z.vhosts.items():
v.srv_d = z.srv_d.join(h)
self.install_directory(v.srv_d)
self.install_access(mode='400')
self.install_resource('wordpress/apache_envvars', j2_ctx, z.apache_envvars_f)
self.install_resource('wordpress/apache.conf', j2_ctx, z.apache_conf_f)
self.install_access(mode='500')
self.install_resource('wordpress/run.sh', j2_ctx, z.run_f)
self.install_resource('wordpress/reload.sh', j2_ctx, z.log_postrotate_f)
logrotate.install_conf(self, j2_ctx)
for h, v in z.vhosts.items():
nginx.install_vhost(
self,
vhost=h,
backend_host=z.ip,
backend_port=v.port,
j2_ctx=j2_ctx,
)
self.append_root_bash_with_main(j2_ctx)