def internal_build_compile(self):
from rsconf import db
from rsconf import systemd
from rsconf.component import docker_registry
self.buildt.require_component('docker')
jc, z = self.j2_ctx_init()
self.__run_d = systemd.docker_unit_prepare(self, jc)
z.update(
user_d=self._jupyterhub_db().join(_USER_SUBDIR),
jupyter_docker_image=docker_registry.absolute_image(
jc, z.jupyter_docker_image,
),
run_u=jc.rsconf_db.run_u,
jupyter_run_u=jc.rsconf_db.run_u,
)
z.setdefault('http_timeout', 30);
z.setdefault('template_vars', {});
self._network(z, jc)
z.admin_users_str = _list_to_str(z.admin_users)
z.template_d = self.__run_d.join(_TEMPLATE_D)
z.home_d = db.user_home_path(jc, z.jupyter_run_u)
z.cookie_secret_hex = self.secret_path_value(
self._COOKIE_SECRET,
gen_secret=lambda: db.random_string(length=64, is_hex=True),
visibility='channel',
)[0]
z.proxy_auth_token = self.secret_path_value(
self._PROXY_AUTH,
gen_secret=lambda: db.random_string(length=64, is_hex=True),
visibility='channel',
)[0]
self._rsdockerspawner(z)
def internal_build(self):
from rsconf import systemd
# https://docs.docker.com/registry/configuration/
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
run_d = systemd.docker_unit_prepare(self, j2_ctx)
assert update_j2_ctx(j2_ctx), \
'no registry host'
conf_f = run_d.join('config.yml')
volumes = [
[conf_f, _GLOBAL_CONF],
]
systemd.docker_unit_enable(
self,
j2_ctx,
# Specify pull from docker.io directly to avoid registry not yet running
image=REGISTRY_IMAGE,
cmd=None,
after=['docker.service'],
run_u=j2_ctx.docker_registry.run_u,
volumes=volumes,
)
kc = self.install_tls_key_and_crt(j2_ctx.docker_registry.host, run_d)
db_d = run_d.join(_DB_SUBDIR)
j2_ctx.docker_registry.update(
auth_htpasswd_path=run_d.join('passwd'),
conf_f=conf_f,
db_d=db_d,
http_tls_certificate=kc.crt,
http_tls_key=kc.key,
)
j2_ctx.docker_registry.http_secret = self.secret_path_value(
_HTTP_SECRET_F,
lambda: db.random_string(length=64),
visibility=_HTTP_SECRET_VISIBILITY,
)[0]
self.install_secret_path(
_PASSWD_SECRET_F,
j2_ctx.docker_registry.auth_htpasswd_path,
gen_secret=db.random_string,
visibility=_PASSWD_VISIBILITY,
)
self.install_access(mode='700', owner=j2_ctx.docker_registry.run_u)
self.install_directory(db_d)
self.install_access(mode='400', owner=j2_ctx.docker_registry.run_u)
self.install_resource('docker_registry/config.yml', j2_ctx,
j2_ctx.docker_registry.conf_f)
self.rsconf_service_restart()
def internal_build_compile(self):
from rsconf import systemd
# nfs_client is required, because host_d is likely on nfs
self.buildt.require_component('docker', 'nfs_client', 'network')
self.jc = self.hdb.j2_ctx_copy()
jc = self.jc
z = jc.mpi_worker
self._find_cluster(jc, z)
z.host_d = z.host_root_d.join(z.user)
z.setdefault('volumes', {})
z.secrets = self._gen_keys(jc, z, jc.rsconf_db.host)
for x in 'guest', 'host':
z[x] = self._gen_paths(jc, z, z.get(x + '_d'))
z.run_u = jc.rsconf_db.run_u
# Only additional config for the server is the sshd config.
# ssh_config and known_hosts are not read by sshd so don't
# trigger a restart.
z.run_d = systemd.docker_unit_prepare(self,
jc,
watch_files=[z.host.ssh_d])
self._prepare_hosts(jc, z)
def internal_build(self):
from rsconf import systemd
from rsconf.component import db_bkp
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.rs_mariadb
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.conf_f = z.run_d.join('my.cnf')
z.db_d = z.run_d.join('db')
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
[z.conf_f, '/etc/my.cnf'],
# hardwired in some places, not sure where, and hardwired
# in Dockefile so just mount as an alias
[z.db_d, '/var/lib/mysql'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
# find in path (probably /usr/sbin, but might be /usr/libexec)
cmd='mysqld',
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400')
self.install_resource('rs_mariadb/my.cnf', j2_ctx, z.conf_f)
db_bkp.install_script_and_subdir(
self,
j2_ctx,
run_u=z.run_u,
run_d=z.run_d,
)
self.append_root_bash_with_main(j2_ctx)
self.rsconf_service_restart()
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.owncloud_redis
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.conf_f = z.run_d.join('redis.conf')
z.db_d = z.run_d.join('db')
systemd.docker_unit_enable(
self,
j2_ctx,
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
# find in path (probably /usr/local/bin, but might be /usr/bin)
cmd="redis-server '{}'".format(z.conf_f),
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400')
self.install_resource('owncloud_redis/redis.conf', j2_ctx, z.conf_f)
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
# https://docs.docker.com/registry/configuration/
self.buildt.require_component('docker')
j2_ctx = self.hdb.j2_ctx_copy()
assert update_j2_ctx(j2_ctx), \
'no docker_cache.host in hdb'
z = j2_ctx.docker_cache
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
# Has to run as root, because 3rd party container
z.run_u = j2_ctx.rsconf_db.root_u
z.conf_f = z.run_d.join('config.yml')
z.db_d = z.run_d.join(_DB_SUBDIR)
volumes = [
[z.conf_f, _GLOBAL_CONF],
]
systemd.docker_unit_enable(
self,
j2_ctx,
image=docker_registry.REGISTRY_IMAGE,
cmd=None,
after=['docker.service'],
run_u=z.run_u,
volumes=volumes,
)
kc = self.install_tls_key_and_crt(z.host, z.run_d)
z.update(
http_tls_certificate=kc.crt,
http_tls_key=kc.key,
)
self.install_access(mode='700', owner=z.run_u)
self.install_directory(z.db_d)
self.install_access(mode='400', owner=z.run_u)
self.install_resource('docker_cache/config.yml', j2_ctx, z.conf_f)
self.rsconf_service_restart()
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
from rsconf.component import logrotate
from rsconf.component import nginx
self.buildt.require_component('docker', 'nginx', 'rs_mariadb')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.owncloud
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.db_d = z.run_d.join('db')
z.run_f = z.run_d.join('run')
z.apache_conf_f = z.run_d.join('apache.conf')
z.apache_envvars_f = z.run_d.join('envvars')
# Created dynamically every run
z.apache_run_d = '/tmp/apache2'
z.conf_d = z.db_d.join('config')
z.conf_f = z.conf_d.join('config.php')
z.files_d = z.db_d.join('files')
z.init_conf_f = z.run_d.join('init_config.php')
z.log_d = z.run_d.join('log')
z.sessions_d = z.db_d.join('sessions')
z.log_postrotate_f = z.run_d.join('reload')
# 127.0.0.1 assumes docker --network=host
# If you connect to "localhost" (not 127.0.0.1) mysql fails to connect,
# because the socket isn't there, instead of trying TCP (port supplied).
# mysql -h localhost -P 7011 -u owncloud
# ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
z.db_host = '{}:{}'.format(
j2_ctx.rs_mariadb.ip,
j2_ctx.rs_mariadb.port,
)
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
# /mnt/data/sessions is hardwired in
# /etc/php/7.0/mods-available/owncloud.ini. We could overrided
# but this is just as easy, and likely to make other things work
# that are hardwired.
[z.db_d, '/mnt/data'],
[z.conf_d, '/var/www/owncloud/config'],
[
z.apache_conf_f,
'/etc/apache2/sites-enabled/000-default.conf'
],
[z.apache_envvars_f, '/etc/apache2/envvars'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
cmd=z.run_f,
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
for d in z.db_d, z.log_d, z.files_d, z.sessions_d, z.conf_d:
self.install_directory(d)
self.install_access(mode='400')
self.install_resource('owncloud/init_config.php', j2_ctx,
z.init_conf_f)
self.install_resource('owncloud/apache.conf', j2_ctx, z.apache_conf_f)
self.install_resource('owncloud/apache_envvars', j2_ctx,
z.apache_envvars_f)
self.install_access(mode='500')
self.install_resource('owncloud/run.sh', j2_ctx, z.run_f)
self.install_resource('owncloud/reload.sh', j2_ctx, z.log_postrotate_f)
logrotate.install_conf(self, j2_ctx)
nginx.install_vhost(
self,
vhost=z.domain,
backend_host='127.0.0.1',
backend_port=z.port,
j2_ctx=j2_ctx,
)
self.append_root_bash_with_main(j2_ctx)
def internal_build_compile(self):
from rsconf.component import docker_registry
from rsconf import db
self.__docker_unit_enable_after = []
self.__uwsgi_docker_vols = []
self.buildt.require_component('docker', 'nginx', 'db_bkp')
jc, z = self.j2_ctx_init()
self.__run_d = systemd.docker_unit_prepare(self, jc)
d = self.__run_d.join(_DB_SUBDIR)
self.j2_ctx_pksetdefault(dict(
sirepo=dict(
cookie=dict(
http_name=lambda: 'sirepo_{}'.format(jc.rsconf_db.channel),
private_key=lambda: self.secret_path_value(
_COOKIE_PRIVATE_KEY,
gen_secret=lambda: pkcompat.from_bytes(
base64.urlsafe_b64encode(os.urandom(32)),
),
visibility='channel',
)[0],
),
docker_image=docker_registry.absolute_image(jc, z.docker_image),
feature_config=dict(
api_modules=[],
job=True,
default_proprietary_sim_types=tuple(),
proprietary_sim_types=tuple(),
),
job=dict(
max_message_bytes='200m',
),
pkcli=dict(
service=dict(
ip='0.0.0.0',
run_dir=self.__run_d,
),
),
srdb=dict(root=d),
),
pykern=dict(
pkdebug=dict(
redirect_logging=True,
want_pid_time=True,
),
pkconfig=dict(channel=jc.rsconf_db.channel),
),
))
self._comsol(z)
self._jupyterhublogin(z)
self.j2_ctx_pksetdefault(dict(
sirepo=dict(
client_max_body_size=pkconfig.parse_bytes(z.job.max_message_bytes),
job_driver=dict(modules=['docker']),
job=dict(
server_secret=lambda: self.secret_path_value(
_SERVER_SECRET,
gen_secret=lambda: pkcompat.from_bytes(
base64.urlsafe_b64encode(os.urandom(32)),
),
visibility='channel',
)[0],
verify_tls=lambda: jc.pykern.pkconfig.channel != 'dev',
),
pkcli=dict(
job_supervisor=dict(
ip='127.0.0.1',
port=8001,
),
),
),
))
z.pksetdefault(job_api=PKDict)
# server connects locally only so go direct to tornado.
# supervisor has different uri to pass to agents.
z.job_api.supervisor_uri = 'http://{}:{}'.format(
z.pkcli.job_supervisor.ip,
z.pkcli.job_supervisor.port,
)
self._set_sirepo_config('sirepo_job_supervisor')
#TODO(robnagler) remove when deployed to alpha
z.job.supervisor_uri = z.job_api.supervisor_uri
def internal_build(self):
from rsconf import systemd
from rsconf.component import docker_registry
from rsconf.component import logrotate
from rsconf.component import nginx
self.buildt.require_component('docker', 'nginx', 'rs_mariadb')
j2_ctx = self.hdb.j2_ctx_copy()
z = j2_ctx.wordpress
z.run_u = j2_ctx.rsconf_db.run_u
z.run_d = systemd.docker_unit_prepare(self, j2_ctx)
z.srv_d = z.run_d.join('srv')
z.run_f = z.run_d.join('run')
z.apache_conf_f = z.run_d.join('apache.conf')
z.apache_envvars_f = z.run_d.join('envvars')
z.setdefault('client_max_body_size', _DEFAULT_CLIENT_MAX_BODY_SIZE)
# Created dynamically every run
z.apache_run_d = '/tmp/apache2'
z.log_d = z.run_d.join('log')
z.ip = '127.0.0.1'
z.log_postrotate_f = z.run_d.join('reload')
z.setdefault('num_servers', 4)
# 127.0.0.1 assumes docker --network=host
# If you connect to "localhost" (not 127.0.0.1) mysql fails to connect,
# because the socket isn't there, instead of trying TCP (port supplied).
# mysql -h localhost -P 7011 -u wordpress
# ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
z.db_host = '{}:{}'.format(
j2_ctx.rs_mariadb.ip,
j2_ctx.rs_mariadb.port,
)
systemd.docker_unit_enable(
self,
j2_ctx,
volumes=[
[z.apache_conf_f, '/etc/apache2/sites-enabled/000-default.conf'],
[z.apache_envvars_f, '/etc/apache2/envvars'],
],
image=docker_registry.absolute_image(j2_ctx, z.docker_image),
cmd=z.run_f,
run_u=z.run_u,
)
self.install_access(mode='700', owner=z.run_u)
for d in z.log_d, z.srv_d:
self.install_directory(d)
for h, v in z.vhosts.items():
v.srv_d = z.srv_d.join(h)
self.install_directory(v.srv_d)
self.install_access(mode='400')
self.install_resource('wordpress/apache_envvars', j2_ctx, z.apache_envvars_f)
self.install_resource('wordpress/apache.conf', j2_ctx, z.apache_conf_f)
self.install_access(mode='500')
self.install_resource('wordpress/run.sh', j2_ctx, z.run_f)
self.install_resource('wordpress/reload.sh', j2_ctx, z.log_postrotate_f)
logrotate.install_conf(self, j2_ctx)
for h, v in z.vhosts.items():
nginx.install_vhost(
self,
vhost=h,
backend_host=z.ip,
backend_port=v.port,
j2_ctx=j2_ctx,
)
self.append_root_bash_with_main(j2_ctx)