def _add_dockerd_services(manifest, tm_env):
"""Configure docker daemon services."""
# add dockerd service
(_uid, proid_gid) = _get_user_uid_gid(manifest['proid'])
dockerd_svc = {
'name': 'dockerd',
'proid': 'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': (
'exec {dockerd} --add-runtime docker-runc={docker_runtime}'
' --default-runtime=docker-runc'
' --exec-opt native.cgroupdriver=cgroupfs --bridge=none'
' --ip-forward=false --ip-masq=false --iptables=false'
' --cgroup-parent=/docker -G {gid}'
' --insecure-registry {registry} --add-registry {registry}'
).format(
dockerd=subproc.resolve('dockerd'),
docker_runtime=subproc.resolve('docker_runtime'),
gid=proid_gid,
registry=_get_docker_registry(tm_env)
),
'root': True,
'environ': [],
'config': None,
'downed': False,
'trace': False,
}
manifest['services'].append(dockerd_svc)
def _add_ssh_system_service(manifest):
"""Configures sshd services in the container."""
mkdir = subproc.resolve('mkdir')
chmod = subproc.resolve('chmod')
sshd = subproc.resolve('sshd')
sshd_svc = {
'name':
'sshd',
'proid':
None,
'restart': {
'limit': 5,
'interval': 60,
},
# TODO: this needs to be moved to sshd template.
'command':
'{mkdir} -p /var/empty/sshd && '
'{chmod} 0755 /var/empty/sshd && '
'exec {sshd} -D -f /etc/ssh/sshd_config '
'-p $TREADMILL_ENDPOINT_SSH'.format(mkdir=mkdir,
chmod=chmod,
sshd=sshd)
}
manifest['system_services'].append(sshd_svc)
ssh_endpoint = {
'name': 'ssh',
'port': 0,
'type': 'infra',
}
manifest['endpoints'].append(ssh_endpoint)
def run(self):
"""Exec into the tree."""
s6_envdir = subproc.resolve('s6_envdir')
utils.sane_execvp(s6_envdir, [
s6_envdir, self.paths.env_dir,
subproc.resolve('s6_svscan'), self.paths.svscan_tree_dir
])
def _generate_dockerd_service(tm_env, manifest):
"""Configure docker daemon services."""
# add dockerd service
ulimits = dockerutils.init_ulimit()
default_ulimit = dockerutils.fmt_ulimit_to_flag(ulimits)
# we disable advanced network features
command = ('exec'
' {dockerd}'
' -H tcp://127.0.0.1:2375'
' --authorization-plugin=authz'
' --add-runtime docker-runc={docker_runtime}'
' --default-runtime=docker-runc'
' --exec-opt native.cgroupdriver=cgroupfs'
' --bridge=none'
' --ip-forward=false'
' --ip-masq=false'
' --iptables=false'
' --cgroup-parent=docker'
' --block-registry="*"'
' {default_ulimit}').format(
dockerd=subproc.resolve('dockerd'),
docker_runtime=subproc.resolve('docker_runtime'),
default_ulimit=default_ulimit,
)
tls_conf = _get_tls_conf(tm_env)
if tls_conf['ca_cert']:
command += (' --tlsverify'
' --tlscacert={ca_cert}').format(
ca_cert=tls_conf['ca_cert'], )
if tls_conf['host_cert'] or tls_conf['host_key']:
# NOTE: host_cert/host_key come in pair.
command += (' --tlscert={host_cert}'
' --tlskey={host_key}').format(
host_cert=tls_conf['host_cert'],
host_key=tls_conf['host_key'])
# we only allow pull image from specified registry
for registry_name in _get_docker_registry(tm_env, manifest['environment']):
command += (' --add-registry {registry}').format(
registry=registry_name)
dockerd_svc = {
'name': 'dockerd',
'proid': 'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': command,
'root': True,
'environ': [],
'config': None,
'downed': False,
'trace': False,
}
return dockerd_svc
def test_resolve(self):
"""Test resolve.
"""
self.assertEqual(subproc.resolve('foo'), 'bar')
if os.name == 'nt':
self.assertEqual(subproc.resolve('xxx'), r'\x\y\z\xxx')
self.assertEqual(subproc.resolve('xxx_d'), r'\x\y\z')
else:
self.assertEqual(subproc.resolve('xxx'), '/x/y/z/xxx')
self.assertEqual(subproc.resolve('xxx_d'), '/x/y/z')
def add_linux_services(manifest):
"""Configure linux standard services."""
# Configures sshd services in the container.
sshd_svc = {
'name':
'sshd',
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command':
('exec {sshd} -D -f /etc/ssh/sshd_config'
' -p $TREADMILL_ENDPOINT_SSH').format(sshd=subproc.resolve('sshd')),
'root':
True,
'environ': [],
'config':
None,
'downed':
False,
'trace':
False,
}
manifest['services'].append(sshd_svc)
ssh_endpoint = {
'name': 'ssh',
'proto': 'tcp',
'port': 0,
'type': 'infra',
}
manifest['endpoints'].append(ssh_endpoint)
def add_linux_services(manifest):
"""Configure linux standard services."""
# Configures sshd services in the container.
sshd_svc = {
'name':
'sshd',
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': ('unset KRB5_KTNAME;unset KRB5CCNAME;'
'exec {sshd} -D -f /etc/ssh/sshd_config'
' -p {sshd_port}').format(sshd=subproc.resolve('sshd'),
sshd_port=_SSHD_PORT),
'root':
True,
'environ': [],
'config':
None,
'downed':
False,
'trace':
False,
}
manifest['services'].append(sshd_svc)
ssh_endpoint = {
'name': 'ssh',
'proto': 'tcp',
'port': _SSHD_PORT,
'type': 'infra',
}
manifest['endpoints'].append(ssh_endpoint)
def _prepare_ldpreload(container_dir, app):
"""Add mandatory ldpreloads to the container environment.
"""
etc_dir = os.path.join(container_dir, 'overlay', 'etc')
fs.mkdir_safe(etc_dir)
new_ldpreload = os.path.join(etc_dir, 'ld.so.preload')
try:
shutil.copyfile('/etc/ld.so.preload', new_ldpreload)
except IOError as err:
if err.errno != errno.ENOENT:
raise
_LOGGER.info('/etc/ld.so.preload not found, creating empty.')
utils.touch(new_ldpreload)
ldpreloads = []
if app.ephemeral_ports.tcp or app.ephemeral_ports.udp:
treadmill_bind_preload = subproc.resolve('treadmill_bind_preload.so')
ldpreloads.append(treadmill_bind_preload)
if not ldpreloads:
return
_LOGGER.info('Configuring /etc/ld.so.preload: %r', ldpreloads)
with open(new_ldpreload, 'a') as f:
f.write('\n'.join(ldpreloads) + '\n')
def _configure_locker(tkt_spool_dir, scandir, cell, celluser):
"""Configure ticket forwarding service."""
if os.path.exists(os.path.join(scandir, cell)):
return
_LOGGER.info('Configuring ticket locker: %s/%s', scandir, cell)
name = cell
realms = krb5.get_host_realm(sysinfo.hostname())
krb5ccname = 'FILE:{tkt_spool_dir}/{celluser}@{realm}'.format(
tkt_spool_dir=tkt_spool_dir,
celluser=celluser,
realm=realms[0],
)
supervisor.create_service(
scandir,
name=name,
app_run_script=(
'{treadmill}/bin/treadmill sproc '
'tickets locker --tkt-spool-dir {tkt_spool_dir}'.format(
treadmill=subproc.resolve('treadmill'),
tkt_spool_dir=tkt_spool_dir)),
userid='root',
environ_dir=os.path.join(scandir, name, 'env'),
environ={
'KRB5CCNAME': krb5ccname,
'TREADMILL_CELL': cell,
},
downed=False,
trace=None,
monitor_policy=None)
def _s6(exe):
"""Resolve s6 exe."""
s6_dir = subproc.resolve('s6')
if not s6_dir:
return None
return os.path.join(s6_dir, 'bin', exe.replace('_', '-'))
def _generate_dockerd_service(tm_env):
"""Configure docker daemon services."""
# add dockerd service
# we disable advanced network features
command = (
'exec'
' {dockerd}'
' -H tcp://127.0.0.1:2375'
' --authorization-plugin=authz'
' --add-runtime docker-runc={docker_runtime}'
' --default-runtime=docker-runc'
' --exec-opt native.cgroupdriver=cgroupfs'
' --bridge=none'
' --ip-forward=false'
' --ip-masq=false'
' --iptables=false'
' --cgroup-parent=docker'
' --block-registry="*"'
).format(
dockerd=subproc.resolve('dockerd'),
docker_runtime=subproc.resolve('docker_runtime'),
)
# we only allow pull image from specified registry
for registry_name in _get_docker_registry(tm_env):
command += (
' --insecure-registry {registry}'
' --add-registry {registry}'
).format(registry=registry_name)
dockerd_svc = {
'name': 'dockerd',
'proid': 'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': command,
'root': True,
'environ': [],
'config': None,
'downed': False,
'trace': False,
}
return dockerd_svc
def _get_docker_run_cmd(name,
image,
uidgid=None,
commands=None,
use_shell=True):
"""Get docker run cmd from raw command
"""
tpl = ('exec $TREADMILL/bin/treadmill sproc docker'
' --name {name}'
' --envdirs /env,/docker/env,/services/{name}/env')
# FIXME: hardcode volumes for now
treadmill_bind = subproc.resolve('treadmill_bind_distro')
volumes = [
('/var/log', '/var/log', 'rw'),
('/var/spool', '/var/spool', 'rw'),
('/var/tmp', '/var/tmp', 'rw'),
('/docker/etc/hosts', '/etc/hosts', 'ro'),
('/docker/etc/passwd', '/etc/passwd', 'ro'),
('/docker/etc/group', '/etc/group', 'ro'),
('/env', '/env', 'ro'),
(treadmill_bind, TREADMILL_BIND_PATH, 'ro'),
]
# TODO: we need better way to handle volume list including cases for NFS
# If host has krb5.conf, we suppose container always mounts it inside now
if os.path.exists('/etc/krb5.conf'):
volumes.append(('/etc/krb5.conf', '/etc/krb5.conf', 'ro'))
for volume in volumes:
tpl += ' --volume {source}:{dest}:{mode}'.format(source=volume[0],
dest=volume[1],
mode=volume[2])
if uidgid is not None:
tpl += ' --user {uidgid}'.format(uidgid=uidgid)
tpl += ' --image {image}'
# put entrypoint and image in the last
if commands is not None:
commands = shlex.split(commands)
if not use_shell:
tpl += ' --entrypoint {entrypoint}'
entrypoint = commands.pop(0)
else:
entrypoint = None
if commands:
tpl += ' -- {cmds}'
else:
commands = []
entrypoint = None
return tpl.format(name=name,
image=shlex.quote(image),
entrypoint=entrypoint,
cmds=' '.join((shlex.quote(cmd) for cmd in commands)))
def _add_cleanup_app(self, path):
"""Configure a new cleanup app.
"""
name = os.path.basename(path)
if name.startswith('.'):
_LOGGER.warning('Ignore %s', name)
return
cleaning_link = os.path.join(self.tm_env.cleaning_dir, name)
if os.path.islink(cleaning_link):
_LOGGER.warning('Cleaning app already configured %s', name)
return
cleanup_link = os.path.join(self.tm_env.cleanup_dir, name)
if not os.path.islink(cleanup_link):
_LOGGER.info('Ignore - not a link: %s', cleanup_link)
return
_LOGGER.info('Configure cleaning app: %s', name)
bin_name = 'scripts' if os.name == 'nt' else 'bin'
command = ('{treadmill}/{bin}/treadmill sproc cleanup instance'
' --approot {tm_root}'
' {instance}').format(
treadmill=subproc.resolve('treadmill'),
bin=bin_name,
tm_root=self.tm_env.root,
instance=name)
if os.name == 'posix':
command = 'exec ' + command
supervisor.create_service(
self.tm_env.cleanup_apps_dir,
name=name,
app_run_script=command,
userid='root',
monitor_policy={
'limit': 5,
'interval': 60,
'tombstone': {
'path': self.tm_env.cleanup_tombstone_dir,
'id': name,
},
'skip_path': os.path.join(self.tm_env.cleanup_dir, name)
},
log_run_script=None,
)
fs.symlink_safe(cleaning_link,
os.path.join(self.tm_env.cleanup_apps_dir, name))
_LOGGER.debug('Cleanup app %s ready', name)
self._refresh_supervisor()
def check(aliases):
"""Check aliases."""
subproc.load_packages(aliases.split(':'))
for exe in subproc.get_aliases():
success = True
fullpath = subproc.resolve(exe)
if fullpath:
print('{:<30}{:<10}{}'.format(exe, 'ok', fullpath))
else:
print('{:<30}{:<10}'.format(exe, 'fail'))
def _get_docker_run_cmd(name, image,
uidgid=None,
commands=None,
use_shell=True):
"""Get docker run cmd from raw command
"""
tpl = (
'exec $TREADMILL/bin/treadmill sproc docker'
' --name {name}'
' --envdirs /env,/docker/env,/services/{name}/env'
)
# FIXME: hardcode volumes for now
treadmill_bind = subproc.resolve('treadmill_bind_distro')
volumes = [
('/var/log', '/var/log', 'rw'),
('/var/spool', '/var/spool', 'rw'),
('/var/tmp', '/var/tmp', 'rw'),
('/var/run/docker.sock', '/var/run/docker.sock', 'rw'),
('/docker/etc/hosts', '/etc/hosts', 'ro'),
('/env', '/env', 'ro'),
(treadmill_bind, TREADMILL_BIND_PATH, 'ro'),
]
for volume in volumes:
tpl += ' --volume {source}:{dest}:{mode}'.format(
source=volume[0],
dest=volume[1],
mode=volume[2]
)
if uidgid is not None:
tpl += ' --user {uidgid}'.format(uidgid=uidgid)
tpl += ' --image {image}'
# put entrypoint and image in the last
if commands is not None:
commands = shlex.split(commands)
if not use_shell:
tpl += ' --entrypoint {entrypoint}'
entrypoint = commands.pop(0)
else:
entrypoint = None
if commands:
tpl += ' -- {cmds}'
else:
commands = []
entrypoint = None
return tpl.format(
name=name,
image=image,
entrypoint=entrypoint,
cmds=' '.join((shlex.quote(cmd) for cmd in commands))
)
def _get_docker_run_cmd(name, image,
args=None, command=None, uidgid=None):
"""Get docker run cmd from raw command
"""
# TODO: hardode volume for now
treadmill_bind = subproc.resolve('treadmill_bind_distro')
# XXX: hardode volume for now
volumes = [
('/var/tmp', '/var/tmp', 'rw'),
('/var/spool', '/var/spool', 'rw'),
('/docker/etc/hosts', '/etc/hosts', 'ro'),
('/env', '/env', 'ro'),
(treadmill_bind, TREADMILL_BIND_PATH, 'ro'),
]
tpl = (
'exec $TREADMILL/bin/treadmill sproc docker'
' --name {name}'
' --envdirs /env,/docker/env,/services/{name}/env'
)
for volume in volumes:
tpl += ' --volume {source}:{dest}:{mode}'.format(
source=volume[0],
dest=volume[1],
mode=volume[2]
)
if uidgid is not None:
tpl += ' --user {uidgid}'.format(uidgid=uidgid)
# put entrypoint and image in the last
if command is not None:
tpl += ' --entrypoint {entrypoint}'
command = shlex.quote(command)
tpl += ' --image {image}'
# create args str in treadmill sproc docker
if args is not None:
tpl += ' -- {cmd}'
args_str = ' '.join([shlex.quote(arg) for arg in args])
else:
args_str = None
return tpl.format(
name=name,
image=image,
entrypoint=command,
cmd=args_str,
)
def _generate_dockerd_service(docker_cfg):
"""Configure docker daemon services."""
# add dockerd service
# we disable advanced network features
command = 'exec {dockerd}'.format(dockerd=subproc.resolve('dockerd'))
extra_cmd_params = [''] # Start with a space
# configure ulimits.
ulimits = dockerutils.get_ulimits()
# Format rich dictionary to dockerd-compatible cli flags.
# Do not respect "soft" limit as dockerd has a known issue when comparing
# finite vs infinite values; will error on {Soft=0, Hard=-1}
for flag in ulimits:
extra_cmd_params.append('--default-ulimit')
extra_cmd_params.append('{}={}:{}'.format(flag['Name'], flag['Hard'],
flag['Hard']))
# We block all registries and only allow image pulls from our configured
# registries.
extra_cmd_params.append('--block-registry="*"')
for registry in docker_cfg['registries']:
extra_cmd_params.append('--add-registry')
extra_cmd_params.append(registry['host'])
if registry.get('insecure', False):
extra_cmd_params.append('--insecure-registry')
extra_cmd_params.append(registry['host'])
command += ' '.join(extra_cmd_params)
_LOGGER.info('dockerd cmd: %s', command)
dockerd_svc = {
'name': 'dockerd',
'proid': 'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': command,
'root': True,
'environ': [],
'config': None,
'downed': False,
'trace': False,
}
return dockerd_svc
def configure(self, manifest):
_LOGGER.info('Configuring krb5keytab.')
unique_name = appcfg.manifest_unique_name(manifest)
appdir = os.path.join(self._tm_env.apps_dir, unique_name, 'data',
'root')
krb5keytab_svc = {
'name':
'krb5keytab',
'restart': {
'limit': 5,
'interval': 60,
},
'proid':
'root',
'root':
True,
'command': ('{treadmill}/bin/treadmill --debug admin krb5keytab'
' --owner {user}'
' --principal {user}/{hostname}'
' --keytab {appdir}/var/spool/keytabs/{user}'
' --cachedir /tmp'
' --lockdir /tmp'
'; exec sleep inf').format(
treadmill=subproc.resolve('treadmill'),
user=manifest['proid'],
hostname=sysinfo.hostname(),
appdir=appdir,
),
'environ': [{
'name':
'KRB5CCNAME',
'value':
os.path.expandvars('FILE:${TREADMILL_HOST_TICKET}'),
}],
'config':
None,
'downed':
False,
'trace':
False,
'logger':
's6.logger.run',
}
manifest['system_services'].append(krb5keytab_svc)
def test_resolve(self):
"""Test resolve.
"""
self.assertEqual(subproc.resolve('foo'), 'bar')
if os.name != 'nt':
self.assertEqual(subproc.resolve('lib'), '/x/$LIB/lib.so')
if os.name == 'nt':
self.assertEqual(subproc.resolve('xxx'), '\\x\\y\\z\\xxx')
self.assertEqual(subproc.resolve('xxx_d'), '\\x\\y\\z')
else:
self.assertEqual(subproc.resolve('xxx'), '/x/y/z/xxx')
self.assertEqual(subproc.resolve('xxx_d'), '/x/y/z')
def create_docker_environ_dir(container_dir, root_dir, app):
"""Creates environ dir for docker"""
env_dir = os.path.join(container_dir, _CONTAINER_DOCKER_ENV_DIR)
env = {}
treadmill_bind_preload_so = os.path.basename(
subproc.resolve('treadmill_bind_preload.so'))
if app.ephemeral_ports.tcp or app.ephemeral_ports.udp:
env['LD_PRELOAD'] = os.path.join(_manifest.TREADMILL_BIND_PATH, '$LIB',
treadmill_bind_preload_so)
supervisor.create_environ_dir(env_dir, env)
# Bind the environ directory in the container volume
fs.mkdir_safe(os.path.join(root_dir, _CONTAINER_DOCKER_ENV_DIR))
fs_linux.mount_bind(root_dir,
os.path.join(os.sep, _CONTAINER_DOCKER_ENV_DIR),
source=os.path.join(container_dir,
_CONTAINER_DOCKER_ENV_DIR),
recursive=False,
read_only=True)
def _get_docker_run_cmd(name, image, command=None, uidgid=None):
"""Get docker run cmd from raw command
"""
# TODO: hardode volume for now
treadmill_bind = subproc.resolve('treadmill_bind_distro')
# XXX: hardode volume for now
volumes = [
('/var/tmp', '/var/tmp', 'rw'),
('/var/spool', '/var/spool', 'rw'),
('/docker/etc/hosts', '/etc/hosts', 'ro'),
('/env', '/env', 'ro'),
(treadmill_bind, TREADMILL_BIND_PATH, 'ro'),
]
tpl = ('exec {python} -m treadmill sproc'
' docker'
' --name {name}'
' --envdirs /env,/docker/env,/services/{name}/env'
' --image {image}')
for volume in volumes:
tpl += ' --volume {source}:{dest}:{mode}'.format(source=volume[0],
dest=volume[1],
mode=volume[2])
if uidgid is not None:
tpl += ' --user {uidgid}'.format(uidgid=uidgid)
if command is not None:
tpl += ' -- {cmd}'
return tpl.format(
python=sys.executable,
name=name,
image=image,
cmd=command,
)
def _add_ssh_system_service(manifest):
"""Configures sshd services in the container."""
sshd_svc = {
'name':
'sshd',
'proid':
None,
'restart': {
'limit': 5,
'interval': 60,
},
'command':
'%s -D -f /etc/ssh/sshd_config '
'-p $TREADMILL_ENDPOINT_SSH' % (subproc.resolve('sshd'))
}
manifest['system_services'].append(sshd_svc)
ssh_endpoint = {
'name': 'ssh',
'port': 0,
'type': 'infra',
}
manifest['endpoints'].append(ssh_endpoint)
def _prepare_daemon_conf(confdir, daemon_conf):
"""Write down a dockerd `daemon.json` config file.
"""
conf = {
'authorization-plugins': ['authz'],
'bridge': 'none',
'cgroup-parent': 'docker',
'default-runtime': 'docker-runc',
'exec-opt': ['native.cgroupdriver=cgroupfs'],
'hosts': ['tcp://127.0.0.1:2375'],
'ip-forward': False,
'ip-masq': False,
'iptables': False,
'ipv6': False,
'runtimes': {
'docker-runc': {
'path': subproc.resolve('docker_runtime'),
},
},
}
conf.update(daemon_conf)
with open(os.path.join(confdir, 'daemon.json'), 'w') as f:
json.dump(conf, fp=f)
def create_supervision_tree(container_dir, app):
"""Creates s6 supervision tree."""
# Disable R0915: Too many statements
# pylint: disable=R0915
root_dir = os.path.join(container_dir, 'root')
# Services and sys directories will be restored when container restarts
# with data retention on existing volume.
#
# Sys directories will be removed. Services directory will stay, which
# present a danger of accumulating restart counters in finished files.
#
# TODO:
#
# It is rather arbitrary how restart counts should work when data is
# restored, but most likely services are "restart always" policy, so it
# will not affect them.
services_dir = os.path.join(container_dir, 'services')
sys_dir = os.path.join(container_dir, 'sys')
if os.path.exists(sys_dir):
_LOGGER.info('Deleting existing sys dir: %s', sys_dir)
shutil.rmtree(sys_dir)
app_json = os.path.join(root_dir, 'app.json')
# Create /services directory for the supervisor
svcdir = os.path.join(root_dir, 'services')
fs.mkdir_safe(svcdir)
fs.mkdir_safe(services_dir)
fs.mount_bind(root_dir, '/services', services_dir)
root_pw = pwd.getpwnam('root')
proid_pw = pwd.getpwnam(app.proid)
# Create .s6-svscan directories for svscan finish
sys_svscandir = os.path.join(sys_dir, '.s6-svscan')
fs.mkdir_safe(sys_svscandir)
svc_svscandir = os.path.join(services_dir, '.s6-svscan')
fs.mkdir_safe(svc_svscandir)
# svscan finish scripts to wait on all services
utils.create_script(os.path.join(sys_svscandir, 'finish'),
'svscan.finish',
timeout=6000)
utils.create_script(os.path.join(svc_svscandir, 'finish'),
'svscan.finish',
timeout=5000)
for svc in app.services:
if getattr(svc, 'root', False):
svc_user = '******'
svc_home = root_pw.pw_dir
svc_shell = root_pw.pw_shell
else:
svc_user = app.proid
svc_home = proid_pw.pw_dir
svc_shell = proid_pw.pw_shell
supervisor.create_service(
services_dir,
svc_user,
svc_home,
svc_shell,
svc.name,
svc.command,
env=app.environment,
down=True,
envdirs=['/environ/app', '/environ/sys'],
as_root=True,
)
_create_logrun(os.path.join(services_dir, svc.name))
for svc in app.system_services:
supervisor.create_service(
services_dir,
'root',
root_pw.pw_dir,
root_pw.pw_shell,
svc.name,
svc.command,
env=app.environment,
down=False,
envdirs=['/environ/sys'],
as_root=True,
)
_create_logrun(os.path.join(services_dir, svc.name))
# Vring services
for cell in app.vring.cells:
fs.mkdir_safe(os.path.join(sys_dir, 'vring.%s' % cell))
cmd = '%s sproc --zookeeper - --cell %s vring %s' % (
treadmill.TREADMILL_BIN, cell, app_json)
utils.create_script(os.path.join(sys_dir, 'vring.%s' % cell, 'run'),
'supervisor.run_sys',
cmd=cmd)
_create_logrun(os.path.join(sys_dir, 'vring.%s' % cell))
# Create endpoint presence service
presence_monitor_cmd = '%s sproc presence monitor %s %s' % (
treadmill.TREADMILL_BIN, app_json, container_dir)
presence_register_cmd = '%s sproc presence register %s %s' % (
treadmill.TREADMILL_BIN, app_json, container_dir)
shadow_etc = os.path.join(container_dir, 'overlay', 'etc')
host_aliases_cmd = '%s sproc host-aliases --aliases-dir %s %s %s' % (
treadmill.TREADMILL_BIN,
os.path.join(shadow_etc, 'host-aliases'),
os.path.join(shadow_etc, 'hosts.original'),
os.path.join(shadow_etc, 'hosts'),
)
_create_sysrun(sys_dir, 'monitor', presence_monitor_cmd)
_create_sysrun(sys_dir, 'register', presence_register_cmd)
_create_sysrun(sys_dir, 'hostaliases', host_aliases_cmd)
cmd = None
args = None
if hasattr(app, 'command'):
cmd = app.command
if hasattr(app, 'args'):
args = app.args
if not cmd:
cmd = subproc.resolve('s6_svscan')
if not args:
args = ['/services']
_create_sysrun(sys_dir,
'start_container',
'%s %s %s -m -p -i %s %s' %
(subproc.resolve('chroot'), root_dir,
subproc.resolve('pid1'), cmd, ' '.join(args)),
down=True)
supervisor._service_wait(svcroot, '-u', '-o')
expected_cmd = [
's6-svwait', '-u', '-t', '0', '-o', svcroot + '/a', svcroot + '/b'
]
actual_cmd = treadmill.subproc.check_call.call_args[0][0]
self.assertItemsEqual(expected_cmd, actual_cmd)
treadmill.subproc.check_call.assert_called_with(actual_cmd)
treadmill.subproc.check_call.reset_mock()
supervisor._service_wait(svcroot, '-u', '-o', subset=['a'])
treadmill.subproc.check_call.assert_called_with(
['s6-svwait', '-u', '-t', '0', '-o', svcroot + '/a'])
treadmill.subproc.check_call.reset_mock()
supervisor._service_wait(svcroot, '-u', '-o', subset={'a': 1})
treadmill.subproc.check_call.assert_called_with(
['s6-svwait', '-u', '-t', '0', '-o', svcroot + '/a'])
treadmill.subproc.check_call.reset_mock()
supervisor._service_wait(svcroot, '-u', '-o', subset=[])
self.assertFalse(treadmill.subproc.check_call.called)
if __name__ == '__main__':
CONFIG_PATTERN = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'etc', '*.config'))
EXE_CONFIGS = glob.glob(CONFIG_PATTERN)
os.environ['TREADMILL_EXE_WHITELIST'] = ':'.join(EXE_CONFIGS)
os.environ['PATH'] = ':'.join(os.environ['PATH'].split(':') +
[os.path.join(subproc.resolve('s6'), 'bin')])
unittest.main()
def _add_linux_system_services(tm_env, manifest):
"""Configure linux system services."""
container_svcdir = supervisor.open_service(os.path.join(
tm_env.apps_dir, appcfg.manifest_unique_name(manifest)),
existing=False)
container_data_dir = container_svcdir.data_dir
if 'vring' in manifest:
# Add the Vring daemon services
for cell in manifest['vring']['cells']:
vring = {
'name':
'vring.%s' % cell,
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': ('exec {tm} sproc'
' --zookeeper {zkurl}'
' --cell {cell}'
' vring'
' --approot {tm_root}'
' {manifest}').format(tm=dist.TREADMILL_BIN,
zkurl=manifest['zookeeper'],
cell=cell,
tm_root=tm_env.root,
manifest=os.path.join(
container_data_dir,
'state.json')),
'environ': [
{
'name':
'KRB5CCNAME',
'value':
os.path.expandvars('FILE:${TREADMILL_HOST_TICKET}'),
},
],
'config':
None,
'downed':
False,
'trace':
False,
}
manifest['system_services'].append(vring)
# Create ticket refresh and container/endpoint presence service
register_presence = {
'name':
'register',
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': ('exec {tm} sproc'
' --zookeeper {zkurl}'
' --cell {cell}'
' presence register'
' --approot {tm_root}'
' {manifest} {container_dir}').format(
tm=dist.TREADMILL_BIN,
zkurl=manifest['zookeeper'],
cell=manifest['cell'],
tm_root=tm_env.root,
manifest=os.path.join(container_data_dir,
'state.json'),
container_dir=container_data_dir),
'environ': [
{
'name': 'KRB5CCNAME',
'value': os.path.expandvars('FILE:${TREADMILL_HOST_TICKET}'),
},
{
'name': 'TREADMILL_ALIASES_PATH',
'value': os.getenv('TREADMILL_ALIASES_PATH'),
},
],
'config':
None,
'downed':
False,
'trace':
False,
}
manifest['system_services'].append(register_presence)
# Create container /etc/hosts manager service
run_overlay = os.path.join(container_data_dir, 'overlay', 'run')
etc_overlay = os.path.join(container_data_dir, 'overlay', 'etc')
hostaliases = {
'name':
'hostaliases',
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': ('exec {tm} sproc'
' --cell {cell}'
' host-aliases'
' --aliases-dir {aliases_dir}'
' {hosts_original} {hosts_container}').format(
tm=dist.TREADMILL_BIN,
cell=manifest['cell'],
aliases_dir=os.path.join(
run_overlay,
'host-aliases',
),
hosts_original=os.path.join('/', 'etc', 'hosts'),
hosts_container=os.path.join(etc_overlay, 'hosts'),
),
'environ': [],
'downed':
False,
'trace':
False,
}
manifest['system_services'].append(hostaliases)
# Create the user app top level supervisor
start_container = {
'name':
'start_container',
'proid':
'root',
'restart': {
'limit': 0,
'interval': 60,
},
'command': ('exec {chroot} {container_dir}/root'
' {pid1} -m -p -i'
' {svscan} -s /services').format(
chroot=subproc.resolve('chroot'),
container_dir=container_data_dir,
pid1=subproc.resolve('pid1'),
svscan=subproc.resolve('s6_svscan'),
),
'environ': [],
'config':
None,
'downed':
True,
'trace':
False,
}
manifest['system_services'].append(start_container)
# Create the services monitor service
monitor = {
'name':
'monitor',
'proid':
'root',
'restart':
None, # Monitor should not be monitored
'command':
('exec {tm} sproc'
' --cell {cell}'
' monitor services'
' --approot {tm_root}'
' -c {container_dir}'
' -s {services_opts}').format(
tm=dist.TREADMILL_BIN,
cell=manifest['cell'],
tm_root=tm_env.root,
container_dir=container_svcdir.directory,
# This adds all services beside monitor itself
services_opts=' -s'.join([
os.path.join(container_data_dir, 'sys', s['name'])
for s in manifest['system_services']
] + [
os.path.join(container_data_dir, 'services', s['name'])
for s in manifest['services']
])),
'environ': [
{
'name': 'KRB5CCNAME',
'value': os.path.expandvars('FILE:${TREADMILL_HOST_TICKET}'),
},
{
'name': 'TREADMILL_ALIASES_PATH',
'value': os.getenv('TREADMILL_ALIASES_PATH'),
},
],
'config':
None,
'downed':
False,
'trace':
False,
}
manifest['system_services'].append(monitor)
def test_resolve(self):
"""Test resolve.
"""
self.assertEqual(subproc.resolve('foo'), 'bar')
self.assertEqual(subproc.resolve('xxx'), '/x/y/z/xxx')
self.assertEqual(subproc.resolve('xxx_d'), '/x/y/z')
def add_linux_system_services(tm_env, manifest):
"""Configure linux system services."""
unique_name = appcfg.manifest_unique_name(manifest)
container_svcdir = supervisor.open_service(os.path.join(
tm_env.apps_dir, unique_name),
existing=False)
container_data_dir = container_svcdir.data_dir
if 'vring' in manifest:
# Add the Vring daemon services
for cell in manifest['vring']['cells']:
vring = {
'name':
'vring.%s' % cell,
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command':
('exec {treadmill}/bin/treadmill sproc'
' --zookeeper {zkurl}'
' --cell {cell}'
' vring'
' --approot {tm_root}'
' {manifest}').format(treadmill=subproc.resolve('treadmill'),
zkurl=manifest['zookeeper'],
cell=cell,
tm_root=tm_env.root,
manifest=os.path.join(
container_data_dir, 'state.json')),
'environ': [
{
'name':
'KRB5CCNAME',
'value':
os.path.expandvars('FILE:${TREADMILL_HOST_TICKET}'),
},
],
'config':
None,
'downed':
False,
'trace':
False,
}
manifest['system_services'].append(vring)
# Create ticket refresh and container/endpoint presence service
register_presence = {
'name':
'register',
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': ('exec {treadmill}/bin/treadmill sproc'
' --zookeeper {zkurl}'
' --cell {cell}'
' presence register'
' {manifest} {container_dir}').format(
treadmill=subproc.resolve('treadmill'),
zkurl=manifest['zookeeper'],
cell=manifest['cell'],
manifest=os.path.join(container_data_dir,
'state.json'),
container_dir=container_data_dir),
'environ': [
{
'name': 'KRB5CCNAME',
'value': os.path.expandvars('FILE:${TREADMILL_HOST_TICKET}'),
},
{
'name': 'TREADMILL_ALIASES_PATH',
'value': os.getenv('TREADMILL_ALIASES_PATH'),
},
],
'config':
None,
'downed':
False,
'trace':
False,
}
manifest['system_services'].append(register_presence)
# Create container /etc/hosts manager service
run_overlay = os.path.join(container_data_dir, 'overlay', 'run')
etc_overlay = os.path.join(container_data_dir, 'overlay', 'etc')
hostaliases = {
'name':
'hostaliases',
'proid':
'root',
'restart': {
'limit': 5,
'interval': 60,
},
'command': ('exec {treadmill}/bin/treadmill sproc'
' --cell {cell}'
' host-aliases'
' --aliases-dir {aliases_dir}'
' {hosts_original} {hosts_container}').format(
treadmill=subproc.resolve('treadmill'),
cell=manifest['cell'],
aliases_dir=os.path.join(
run_overlay,
'host-aliases',
),
hosts_original=os.path.join('/', 'etc', 'hosts'),
hosts_container=os.path.join(etc_overlay, 'hosts'),
),
'environ': [],
'downed':
False,
'trace':
False,
}
manifest['system_services'].append(hostaliases)
# Create the user app top level supervisor
#
# Reset environment variables set by treadmill to default values.
start_container = {
'name':
'start_container',
'proid':
'root',
'restart': {
'limit': 0,
'interval': 60,
},
'command': ('exec'
' {pid1} -i -m -p'
' --propagation slave'
' {treadmill}/bin/treadmill sproc'
' --cgroup /apps/{unique_name}/services'
' --cell {cell}'
' start-container'
' --container-root {container_dir}/root'
' {manifest}').format(
treadmill=subproc.resolve('treadmill'),
unique_name=unique_name,
cell=manifest['cell'],
pid1=subproc.resolve('pid1'),
container_dir=container_data_dir,
manifest=os.path.join(container_data_dir,
'state.json'),
),
'environ': [],
'config':
None,
'downed':
True,
'trace':
False,
}
manifest['system_services'].append(start_container)
def run(tm_env, container_dir, manifest, watchdog, terminated):
"""Creates container environment and prepares to exec root supervisor.
The function is intended to be invoked from 'run' script and never
returns.
:param tm_env:
Treadmill application environment
:type tm_env:
`appenv.AppEnvironment`
:param container_dir:
Full path to the container
:type container_dir:
``str``
:param manifest:
App manifest.
:type manifest:
``dict``
:param watchdog:
App run watchdog.
:type watchdog:
``treadmill.watchdog``
:param terminated:
Flag where terminated signal will accumulate.
:param terminated:
``set``
:returns:
This function never returns
"""
with lc.LogContext(_LOGGER, os.path.basename(container_dir),
lc.ContainerAdapter) as log:
# R0915: Need to refactor long function into smaller pieces.
# R0912: Too many branches
#
# pylint: disable=R0915,R0912
log.logger.info('Running %r', container_dir)
# Allocate dynamic ports
#
# Ports are taken from ephemeral range, by binding to socket to port 0.
#
# Sockets are then put into global list, so that they are not closed
# at gc time, and address remains in use for the lifetime of the
# supervisor.
sockets = _allocate_network_ports(tm_env.host_ip, manifest)
unique_name = appcfg.manifest_unique_name(manifest)
# First wait for the network device to be ready
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network'))
app_network = network_client.wait(unique_name)
manifest['network'] = app_network
# FIXME(boysson): backward compatibility for TM 2.0. Remove in 3.0
manifest['vip'] = {
'ip0': app_network['gateway'],
'ip1': app_network['vip'],
}
# Save the manifest with allocated vip and ports in the state
state_file = os.path.join(container_dir, _STATE_YML)
with tempfile.NamedTemporaryFile(dir=container_dir,
delete=False,
mode='w') as temp_file:
yaml.dump(manifest, stream=temp_file)
# chmod for the file to be world readable.
os.fchmod(
temp_file.fileno(),
stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
os.rename(temp_file.name, state_file)
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest)
if not app.shared_network:
_unshare_network(tm_env, app)
# Create root directory structure (chroot base).
# container_dir/<subdir>
root_dir = os.path.join(container_dir, 'root')
# chroot_dir/<subdir>
# FIXME(boysson): env_dir should be in a well defined location (part
# of the container "API").
env_dir = os.path.join(root_dir, 'environ')
# Create and format the container root volumne
_create_root_dir(tm_env, container_dir, root_dir, app)
# NOTE: below here, MOUNT namespace is private
# FIXME(boysson): Lots of things are still reading this file.
# Copy updated state manifest as app.yml in the
# container_dir so it is visible in chrooted env.
shutil.copy(state_file, os.path.join(root_dir, _APP_YML))
_create_environ_dir(env_dir, app)
# Create the supervision tree
_create_supervision_tree(container_dir, app)
# Set app limits before chroot.
_share_cgroup_info(app, root_dir)
ldpreloads = []
if app.ephemeral_ports.tcp or app.ephemeral_ports.udp:
treadmill_bind_preload = subproc.resolve(
'treadmill_bind_preload.so')
ldpreloads.append(treadmill_bind_preload)
_prepare_ldpreload(root_dir, ldpreloads)
def _bind(src, tgt):
"""Helper function to bind source to target in the same root"""
# FIXME(boysson): This name mount_bind() have counter-intuitive
# arguments ordering.
src_path = os.path.join(root_dir, src)
if os.path.exists(src_path):
fs.mount_bind(root_dir,
tgt,
target=src_path,
bind_opt='--bind')
# Override the /etc/resolv.conf, so that container always uses
# dnscache.
_bind('.etc/resolv.conf', '/etc/resolv.conf')
_bind('.etc/hosts', '/etc/hosts')
if ldpreloads:
# Override /etc/ld.so.preload to enforce necessary system hooks
_bind('.etc/ld.so.preload', '/etc/ld.so.preload')
# If network is shared, close ephermal sockets before starting the
# supervisor, as these ports will be use be container apps.
if app.shared_network:
for socket_ in sockets:
socket_.close()
# Override pam.d sshd stack with special sshd pam that unshares
# network.
_bind('.etc/pam.d/sshd.shared_network', '/etc/pam.d/sshd')
# else:
# # Override pam.d sshd stack.
# _bind('.etc/pam.d/sshd', '/etc/pam.d/sshd')
watchdog.remove()
if not terminated:
sys_dir = os.path.join(container_dir, 'sys')
supervisor.exec_root_supervisor(sys_dir)
def configure(tm_env, event, runtime):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)/
- apps/
- (app unique name)/
- data/
- app_start
- app.json
- manifest.yml
env/
- TREADMILL_*
run
finish
log/
- run
The 'run' script is responsible for creating container environment
and starting the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# Load the app from the event
try:
manifest_data = load_runtime_manifest(tm_env, event, runtime)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception('No event to load: %r', event)
return None
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Write the actual container start script
if os.name == 'nt':
run_script = '{treadmill}/scripts/treadmill sproc run .'.format(
treadmill=subproc.resolve('treadmill'),
)
else:
run_script = 'exec {treadmill}/bin/treadmill sproc run ../'.format(
treadmill=subproc.resolve('treadmill'),
)
# Create the service for that container
container_svc = supervisor.create_service(
tm_env.apps_dir,
name=uniq_name,
app_run_script=run_script,
userid='root',
downed=False,
monitor_policy={
'limit': 0,
'interval': 60,
'tombstone': {
'uds': False,
'path': tm_env.running_tombstone_dir,
'id': app.name
}
},
environ={},
environment=app.environment
)
data_dir = container_svc.data_dir
# Copy the original event as 'manifest.yml' in the container dir
try:
shutil.copyfile(
event,
os.path.join(data_dir, 'manifest.yml')
)
except IOError as err:
# File is gone, cleanup.
if err.errno == errno.ENOENT:
shutil.rmtree(container_svc.directory)
_LOGGER.exception('Event gone: %r', event)
return None
else:
raise
# Store the app.json in the container directory
fs.write_safe(
os.path.join(data_dir, appcfg.APP_JSON),
lambda f: f.writelines(
utils.json_genencode(manifest_data)
),
mode='w',
permission=0o644
)
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(
instanceid=app.name,
uniqueid=app.uniqueid
)
)
return container_svc.directory