def internal_build_compile(self): from rsconf import db from rsconf import systemd from rsconf.component import docker_registry self.buildt.require_component('docker') jc, z = self.j2_ctx_init() self.__run_d = systemd.docker_unit_prepare(self, jc) z.update( user_d=self._jupyterhub_db().join(_USER_SUBDIR), jupyter_docker_image=docker_registry.absolute_image( jc, z.jupyter_docker_image, ), run_u=jc.rsconf_db.run_u, jupyter_run_u=jc.rsconf_db.run_u, ) z.setdefault('http_timeout', 30); z.setdefault('template_vars', {}); self._network(z, jc) z.admin_users_str = _list_to_str(z.admin_users) z.template_d = self.__run_d.join(_TEMPLATE_D) z.home_d = db.user_home_path(jc, z.jupyter_run_u) z.cookie_secret_hex = self.secret_path_value( self._COOKIE_SECRET, gen_secret=lambda: db.random_string(length=64, is_hex=True), visibility='channel', )[0] z.proxy_auth_token = self.secret_path_value( self._PROXY_AUTH, gen_secret=lambda: db.random_string(length=64, is_hex=True), visibility='channel', )[0] self._rsdockerspawner(z)
def internal_build(self): from rsconf import systemd from rsconf.component import docker_registry self.buildt.require_component('docker') j2_ctx = self.hdb.j2_ctx_copy() z = j2_ctx.github_bkp z.run_u = j2_ctx.rsconf_db.run_u run_d = systemd.timer_prepare(self, j2_ctx, on_calendar=z.on_calendar) z.db_d = run_d.join('db') z.run_f = run_d.join('run') env = pkcollections.Dict( PYKERN_PKCLI_GITHUB_EXCLUDE_RE=z.exclude_re, PYKERN_PKCLI_GITHUB_PASSWORD=z.password, PYKERN_PKCLI_GITHUB_TEST_MODE=z.setdefault('test_mode', False), PYKERN_PKCLI_GITHUB_USER=z.user, ) systemd.docker_unit_enable( self, image=docker_registry.absolute_image(j2_ctx, z.docker_image), j2_ctx=j2_ctx, env=env, run_u=z.run_u, cmd=z.run_f, ) self.install_access(mode='500', owner=z.run_u) self.install_resource('github_bkp/run.sh', j2_ctx, z.run_f) self.install_access(mode='700') self.install_directory(z.db_d)
def internal_build(self): from rsconf import systemd from rsconf.component import docker_registry import pykern.pkdebug def _env_ok(k): if k.startswith('PYKERN_') or k.startswith( 'SIREPO_FEATURE_CONFIG_'): return not pykern.pkdebug.SECRETS_RE.search(k) return (k == _SECRET or k.startswith('SIREPO_PKCLI_TEST_HTTP_SERVER')) jc, z = self.j2_ctx_init() e = PKDict() for k, v in self.buildt.get_component('sirepo').sirepo_unit_env( self).items(): if _env_ok(k): e[k] = v assert e.get(_SECRET), \ f'{_SECRET} not found in sirepo config' systemd.timer_prepare(self, jc, on_calendar=z.on_calendar) e.pksetdefault( SIREPO_PKCLI_TEST_HTTP_SERVER_URI=f'https://{jc.sirepo.vhost}', ) systemd.docker_unit_enable( self, cmd='sirepo test_http', env=e, image=docker_registry.absolute_image(jc, jc.sirepo.docker_image), j2_ctx=jc, run_u=jc.rsconf_db.run_u, )
def _enable_docker(self): from rsconf import systemd from rsconf.component import docker_registry z = self.j2_ctx[self.name] systemd.docker_unit_enable( self, self.j2_ctx, cmd="bash -l -c 'jupyterhub -f {}'".format(self.__conf_f), image=docker_registry.absolute_image(self.j2_ctx, z.docker_image), run_u=z.run_u, )
def _enable_docker(self): from rsconf import systemd from rsconf.component import docker_registry def _env_ok(elem): for p in ( # TODO(e-carlin): all of auth and smtp is really too much. # We just need auth_methods for checking if the cookies is # valid. But bringing in methods then initializes the methods # so we have to bring in their full config. 'SIREPO_AUTH', 'SIREPO_COOKIE', 'SIREPO_FEATURE_PROPRIETARY_SIM', 'SIREPO_SIM_API_JUPYTERHUBLOGIN', 'SIREPO_SMTP', 'SIREPO_SRDB', ): if elem[0].startswith(f'{p}'): return True return False z = self.j2_ctx[self.name] systemd.docker_unit_enable( self, self.j2_ctx, cmd="bash -l -c 'jupyterhub -f {}'".format(self.__conf_f), env=PKDict( filter( _env_ok, self.buildt.get_component('sirepo').sirepo_unit_env( self).items(), ), **pkconfig.to_environ( ['*'], values=dict(sirepo=dict(feature_config=dict( sim_types=set(('jupyterhublogin', ))), )), ), ), image=docker_registry.absolute_image(self.j2_ctx, z.docker_image), run_u=z.run_u, volumes=[ self._jupyterhub_db(), self.j2_ctx.sirepo.srdb.root, ], )
def internal_build_write(self): from rsconf import systemd from rsconf.component import docker_registry jc = self.jc z = jc.mpi_worker self.install_access(mode='700', owner=z.run_u) # Need to make sure host_d exists, even though it isn't ours for d in z.host_d, z.host.conf_d, z.host.ssh_d: self.install_directory(d) self.install_access(mode='400') self.install_resource(z.host.sshd_config, jc) for k, v in z.secrets.items(): self.install_abspath(v, z.host[k]) if z.is_first: for f in 'ssh_config', 'known_hosts': self.install_resource(z.host[f], jc) self.install_access(mode='700') self.install_directory(z.host.bin_d) self.install_access(mode='500') self.install_resource(z.host.bin_d.join('rsmpi.sh'), jc) x = [ [z.host_d, z.guest_d], # SECURITY: no modifications to run_d [z.run_d, z.run_d, 'ro'], ] for k in sorted(z.volumes.keys()): g = str(z.volumes[k]).format(username=z.user) assert g.startswith(str(z.guest_d) + '/'), \ 'mount={} must start with guest_d={}'.format(g, z.guest_d) k = k.format(username=z.user) x.append([k, g]) systemd.docker_unit_enable( self, jc, image=docker_registry.absolute_image(jc, z.docker_image), volumes=x, cmd="/usr/sbin/sshd -D -f '{}'".format(z.guest.sshd_config), )
def internal_build(self): from rsconf import systemd from rsconf.component import db_bkp from rsconf.component import docker_registry self.buildt.require_component('docker') j2_ctx = self.hdb.j2_ctx_copy() z = j2_ctx.rs_mariadb z.run_u = j2_ctx.rsconf_db.run_u z.run_d = systemd.docker_unit_prepare(self, j2_ctx) z.conf_f = z.run_d.join('my.cnf') z.db_d = z.run_d.join('db') systemd.docker_unit_enable( self, j2_ctx, volumes=[ [z.conf_f, '/etc/my.cnf'], # hardwired in some places, not sure where, and hardwired # in Dockefile so just mount as an alias [z.db_d, '/var/lib/mysql'], ], image=docker_registry.absolute_image(j2_ctx, z.docker_image), # find in path (probably /usr/sbin, but might be /usr/libexec) cmd='mysqld', run_u=z.run_u, ) self.install_access(mode='700', owner=z.run_u) self.install_directory(z.db_d) self.install_access(mode='400') self.install_resource('rs_mariadb/my.cnf', j2_ctx, z.conf_f) db_bkp.install_script_and_subdir( self, j2_ctx, run_u=z.run_u, run_d=z.run_d, ) self.append_root_bash_with_main(j2_ctx) self.rsconf_service_restart()
def internal_build(self): from rsconf import systemd from rsconf.component import docker_registry self.buildt.require_component('docker') j2_ctx = self.hdb.j2_ctx_copy() z = j2_ctx.owncloud_redis z.run_u = j2_ctx.rsconf_db.run_u z.run_d = systemd.docker_unit_prepare(self, j2_ctx) z.conf_f = z.run_d.join('redis.conf') z.db_d = z.run_d.join('db') systemd.docker_unit_enable( self, j2_ctx, image=docker_registry.absolute_image(j2_ctx, z.docker_image), # find in path (probably /usr/local/bin, but might be /usr/bin) cmd="redis-server '{}'".format(z.conf_f), run_u=z.run_u, ) self.install_access(mode='700', owner=z.run_u) self.install_directory(z.db_d) self.install_access(mode='400') self.install_resource('owncloud_redis/redis.conf', j2_ctx, z.conf_f)
def internal_build(self): from rsconf import systemd from rsconf.component import docker_registry from rsconf.component import logrotate from rsconf.component import nginx self.buildt.require_component('docker', 'nginx', 'rs_mariadb') j2_ctx = self.hdb.j2_ctx_copy() z = j2_ctx.owncloud z.run_u = j2_ctx.rsconf_db.run_u z.run_d = systemd.docker_unit_prepare(self, j2_ctx) z.db_d = z.run_d.join('db') z.run_f = z.run_d.join('run') z.apache_conf_f = z.run_d.join('apache.conf') z.apache_envvars_f = z.run_d.join('envvars') # Created dynamically every run z.apache_run_d = '/tmp/apache2' z.conf_d = z.db_d.join('config') z.conf_f = z.conf_d.join('config.php') z.files_d = z.db_d.join('files') z.init_conf_f = z.run_d.join('init_config.php') z.log_d = z.run_d.join('log') z.sessions_d = z.db_d.join('sessions') z.log_postrotate_f = z.run_d.join('reload') # 127.0.0.1 assumes docker --network=host # If you connect to "localhost" (not 127.0.0.1) mysql fails to connect, # because the socket isn't there, instead of trying TCP (port supplied). # mysql -h localhost -P 7011 -u owncloud # ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2) z.db_host = '{}:{}'.format( j2_ctx.rs_mariadb.ip, j2_ctx.rs_mariadb.port, ) systemd.docker_unit_enable( self, j2_ctx, volumes=[ # /mnt/data/sessions is hardwired in # /etc/php/7.0/mods-available/owncloud.ini. We could overrided # but this is just as easy, and likely to make other things work # that are hardwired. [z.db_d, '/mnt/data'], [z.conf_d, '/var/www/owncloud/config'], [ z.apache_conf_f, '/etc/apache2/sites-enabled/000-default.conf' ], [z.apache_envvars_f, '/etc/apache2/envvars'], ], image=docker_registry.absolute_image(j2_ctx, z.docker_image), cmd=z.run_f, run_u=z.run_u, ) self.install_access(mode='700', owner=z.run_u) for d in z.db_d, z.log_d, z.files_d, z.sessions_d, z.conf_d: self.install_directory(d) self.install_access(mode='400') self.install_resource('owncloud/init_config.php', j2_ctx, z.init_conf_f) self.install_resource('owncloud/apache.conf', j2_ctx, z.apache_conf_f) self.install_resource('owncloud/apache_envvars', j2_ctx, z.apache_envvars_f) self.install_access(mode='500') self.install_resource('owncloud/run.sh', j2_ctx, z.run_f) self.install_resource('owncloud/reload.sh', j2_ctx, z.log_postrotate_f) logrotate.install_conf(self, j2_ctx) nginx.install_vhost( self, vhost=z.domain, backend_host='127.0.0.1', backend_port=z.port, j2_ctx=j2_ctx, ) self.append_root_bash_with_main(j2_ctx)
def docker_unit_enable(compt, j2_ctx, image, cmd, env=None, volumes=None, after=None, run_u=None, ports=None): """Must be last call""" from rsconf.component import docker_registry def _extra_run_flags(): try: c = z.pknested_get('extra_run_flags.' + compt.name) except KeyError: return '' return ' '.join( (f'--{k}={v}' for k, v in c.items()), ) z = j2_ctx.systemd if env is None: env = PKDict() if 'TZ' not in env: # Tested on CentOS 7, and it does have the localtime stat problem # https://blog.packagecloud.io/eng/2017/02/21/set-environment-variable-save-thousands-of-system-calls/ env['TZ'] = ':/etc/localtime' image = docker_registry.absolute_image(j2_ctx, image) z.update( after=_after(after), extra_run_flags=_extra_run_flags(), service_exec=cmd, exports='\n'.join( ["export '{}={}'".format(k, env[k]) for k in sorted(env.keys())], ), image=image, run_u=run_u or j2_ctx.rsconf_db.run_u, ) volumes = _tuple_arg(volumes) run_d_in_volumes = False for v in volumes: if v[0] == z.run_d: run_d_in_volumes = True if not run_d_in_volumes: volumes.insert(0, (str(z.run_d), str(z.run_d))) z.volumes = _colon_format('-v', volumes) z.network = _colon_format('-p', _tuple_arg(ports)) if not z.network: z.network = '--network=host' scripts = ('cmd', 'env', 'remove', 'start', 'stop') compt.install_access(mode='700', owner=z.run_u) compt.install_directory(z.run_d) compt.install_access(mode='500') for s in scripts: z[s] = z.run_d.join(s) if not cmd: z.cmd = '' for s in scripts: if z[s]: compt.install_resource('systemd/docker_' + s, j2_ctx, z[s]) # See Poettering's omniscience about what's good for all of us here: # https://github.com/systemd/systemd/issues/770 # These files should be 400, since there's no value in making them public. compt.install_access(mode='444', owner=j2_ctx.rsconf_db.root_u) if z.is_timer: compt.install_resource( 'systemd/timer_unit', j2_ctx, z.timer_f, ) compt.install_resource( 'systemd/docker_unit', j2_ctx, z.service_f, ) compt.append_root_bash( "rsconf_service_docker_pull '{}' '{}'".format(z.image, z.service_name), ) unit_enable(compt, j2_ctx)
def internal_build_compile(self): from rsconf.component import docker_registry from rsconf import db self.__docker_unit_enable_after = [] self.__uwsgi_docker_vols = [] self.buildt.require_component('docker', 'nginx', 'db_bkp') jc, z = self.j2_ctx_init() self.__run_d = systemd.docker_unit_prepare(self, jc) d = self.__run_d.join(_DB_SUBDIR) self.j2_ctx_pksetdefault(dict( sirepo=dict( cookie=dict( http_name=lambda: 'sirepo_{}'.format(jc.rsconf_db.channel), private_key=lambda: self.secret_path_value( _COOKIE_PRIVATE_KEY, gen_secret=lambda: pkcompat.from_bytes( base64.urlsafe_b64encode(os.urandom(32)), ), visibility='channel', )[0], ), docker_image=docker_registry.absolute_image(jc, z.docker_image), feature_config=dict( api_modules=[], job=True, default_proprietary_sim_types=tuple(), proprietary_sim_types=tuple(), ), job=dict( max_message_bytes='200m', ), pkcli=dict( service=dict( ip='0.0.0.0', run_dir=self.__run_d, ), ), srdb=dict(root=d), ), pykern=dict( pkdebug=dict( redirect_logging=True, want_pid_time=True, ), pkconfig=dict(channel=jc.rsconf_db.channel), ), )) self._comsol(z) self._jupyterhublogin(z) self.j2_ctx_pksetdefault(dict( sirepo=dict( client_max_body_size=pkconfig.parse_bytes(z.job.max_message_bytes), job_driver=dict(modules=['docker']), job=dict( server_secret=lambda: self.secret_path_value( _SERVER_SECRET, gen_secret=lambda: pkcompat.from_bytes( base64.urlsafe_b64encode(os.urandom(32)), ), visibility='channel', )[0], verify_tls=lambda: jc.pykern.pkconfig.channel != 'dev', ), pkcli=dict( job_supervisor=dict( ip='127.0.0.1', port=8001, ), ), ), )) z.pksetdefault(job_api=PKDict) # server connects locally only so go direct to tornado. # supervisor has different uri to pass to agents. z.job_api.supervisor_uri = 'http://{}:{}'.format( z.pkcli.job_supervisor.ip, z.pkcli.job_supervisor.port, ) self._set_sirepo_config('sirepo_job_supervisor') #TODO(robnagler) remove when deployed to alpha z.job.supervisor_uri = z.job_api.supervisor_uri
def internal_build(self): from rsconf import systemd from rsconf.component import docker_registry from rsconf.component import logrotate from rsconf.component import nginx self.buildt.require_component('docker', 'nginx', 'rs_mariadb') j2_ctx = self.hdb.j2_ctx_copy() z = j2_ctx.wordpress z.run_u = j2_ctx.rsconf_db.run_u z.run_d = systemd.docker_unit_prepare(self, j2_ctx) z.srv_d = z.run_d.join('srv') z.run_f = z.run_d.join('run') z.apache_conf_f = z.run_d.join('apache.conf') z.apache_envvars_f = z.run_d.join('envvars') z.setdefault('client_max_body_size', _DEFAULT_CLIENT_MAX_BODY_SIZE) # Created dynamically every run z.apache_run_d = '/tmp/apache2' z.log_d = z.run_d.join('log') z.ip = '127.0.0.1' z.log_postrotate_f = z.run_d.join('reload') z.setdefault('num_servers', 4) # 127.0.0.1 assumes docker --network=host # If you connect to "localhost" (not 127.0.0.1) mysql fails to connect, # because the socket isn't there, instead of trying TCP (port supplied). # mysql -h localhost -P 7011 -u wordpress # ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2) z.db_host = '{}:{}'.format( j2_ctx.rs_mariadb.ip, j2_ctx.rs_mariadb.port, ) systemd.docker_unit_enable( self, j2_ctx, volumes=[ [z.apache_conf_f, '/etc/apache2/sites-enabled/000-default.conf'], [z.apache_envvars_f, '/etc/apache2/envvars'], ], image=docker_registry.absolute_image(j2_ctx, z.docker_image), cmd=z.run_f, run_u=z.run_u, ) self.install_access(mode='700', owner=z.run_u) for d in z.log_d, z.srv_d: self.install_directory(d) for h, v in z.vhosts.items(): v.srv_d = z.srv_d.join(h) self.install_directory(v.srv_d) self.install_access(mode='400') self.install_resource('wordpress/apache_envvars', j2_ctx, z.apache_envvars_f) self.install_resource('wordpress/apache.conf', j2_ctx, z.apache_conf_f) self.install_access(mode='500') self.install_resource('wordpress/run.sh', j2_ctx, z.run_f) self.install_resource('wordpress/reload.sh', j2_ctx, z.log_postrotate_f) logrotate.install_conf(self, j2_ctx) for h, v in z.vhosts.items(): nginx.install_vhost( self, vhost=h, backend_host=z.ip, backend_port=v.port, j2_ctx=j2_ctx, ) self.append_root_bash_with_main(j2_ctx)