def _cleanup(tm_env, container_dir, app):
"""Cleanup a container that actually ran.
"""
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
# Create service clients
cgroup_client = tm_env.svc_cgroup.make_client(
os.path.join(container_dir, 'resources', 'cgroups'))
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'resources', 'localdisk'))
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'resources', 'network'))
presence_client = tm_env.svc_presence.make_client(
os.path.join(container_dir, 'resources', 'presence'))
# Unregister presence
presence_client.delete(app.name)
# Destroy the volume
try:
localdisk_client.delete(unique_name)
except (IOError, OSError) as err:
if err.errno == errno.ENOENT:
pass
else:
raise
if not app.shared_network:
_cleanup_network(tm_env, container_dir, app, network_client)
# Add metrics to archive
rrd_file = os.path.join(
tm_env.metrics_dir, 'apps',
'{name}-{instanceid}-{uniqueid}.rrd'.format(
name=app.app,
instanceid=app.task,
uniqueid=app.uniqueid,
))
rrdutils.flush_noexc(rrd_file)
_copy_metrics(rrd_file, container_dir)
# Cleanup our cgroup resources
try:
cgroup_client.delete(unique_name)
except (IOError, OSError) as err:
if err.errno == errno.ENOENT:
pass
else:
raise
try:
runtime.archive_logs(tm_env, appcfg.app_unique_name(app),
container_dir)
except Exception: # pylint: disable=W0703
_LOGGER.exception('Unexpected exception storing local logs.')
def test__create_root_dir(self):
"""Test creation on the container root directory."""
# Access protected module _create_root_dir
# pylint: disable=W0212
app = utils.to_obj({
'type': 'native',
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'dev',
'disk': '100G',
})
app_unique_name = appcfg.app_unique_name(app)
container_dir = os.path.join(self.root, 'apps', app_unique_name)
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.wait.return_value = localdisk
treadmill.runtime.linux._run._create_root_dir(self.tm_env,
container_dir,
'/some/root_dir', app)
treadmill.fs.create_filesystem.assert_called_with('/dev/foo')
unshare.unshare.assert_called_with(unshare.CLONE_NEWNS)
treadmill.fs.mount_filesystem('/dev/foo', '/some/root_dir')
def test__create_root_dir(self):
"""Test creation on the container root directory."""
# Access protected module _create_root_dir
# pylint: disable=W0212
app = utils.to_obj(
{
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'dev',
'disk': '100G',
}
)
app_unique_name = appcfg.app_unique_name(app)
container_dir = os.path.join(self.root, 'apps', app_unique_name)
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.wait.return_value = localdisk
treadmill.runtime.linux._run._create_root_dir(self.tm_env,
container_dir,
'/some/root_dir',
app)
treadmill.fs.chroot_init.assert_called_with('/some/root_dir')
treadmill.fs.create_filesystem.assert_called_with('/dev/foo')
treadmill.fs.mount_filesystem('/dev/foo', '/some/root_dir')
treadmill.fs.make_rootfs.assert_called_with('/some/root_dir',
'myproid')
treadmill.fs.configure_plugins.assert_called_with(
self.root,
'/some/root_dir',
app
)
shutil.rmtree.assert_called_with(
'/some/root_dir/.etc'
)
shutil.copytree.assert_called_with(
os.path.join(self.tm_env.root, 'etc'),
'/some/root_dir/.etc'
)
shutil.copyfile.assert_has_calls([
mock.call('/etc/hosts', '/some/root_dir/.etc/hosts'),
mock.call('/etc/hosts', '/some/root_dir/.etc/hosts.original'),
])
treadmill.subproc.check_call.assert_has_calls([
mock.call(
[
'mount', '-n', '--bind',
os.path.join(self.tm_env.root, 'etc/resolv.conf'),
'/etc/resolv.conf'
]
)
])
def apply_cgroup_limits(tm_env, container_dir, manifest):
"""Configures cgroups and limits.
:param tm_env:
Treadmill application environment
:type tm_env:
`appenv.AppEnvironment`
:param container_dir:
Full path to the container
:type container_dir:
``str``
:param manifest:
App manifest.
:type manifest:
``dict``
"""
app = utils.to_obj(manifest)
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
# Setup the service clients
cgroup_client = tm_env.svc_cgroup.make_client(
os.path.join(container_dir, 'cgroups'))
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk'))
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network'))
# Cgroup
cgroup_req = {
'memory': app.memory,
'cpu': app.cpu,
}
# Local Disk
localdisk_req = {
'size': app.disk,
}
# Network
network_req = {
'environment': app.environment,
}
cgroup_client.put(unique_name, cgroup_req)
localdisk_client.put(unique_name, localdisk_req)
if not app.shared_network:
network_client.put(unique_name, network_req)
app_cgroups = cgroup_client.wait(unique_name)
_LOGGER.info('Joining cgroups: %r', app_cgroups)
for subsystem, cgrp in app_cgroups.items():
cgroups.join(subsystem, cgrp)
def _create_root_dir(tm_env, container_dir, root_dir, app):
"""Prepares chrooted environment and creates all mountpoints.
:param tm_env:
Treadmill application environment
:type tm_env:
`appenv.AppEnvironment`
"""
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
# First wait for the block device to be ready
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk'))
localdisk = localdisk_client.wait(unique_name)
already_initialized = fs.test_filesystem(localdisk['block_dev'])
if not already_initialized:
# Format the block device
fs.create_filesystem(localdisk['block_dev'])
_LOGGER.info('Creating container root directory: %s', root_dir)
fs.chroot_init(root_dir)
fs.mount_filesystem(localdisk['block_dev'], root_dir)
fs.make_rootfs(root_dir, app.proid)
fs.configure_plugins(tm_env.root, root_dir, app)
# Ensure .etc directory is clean in case of volume restore.
try:
shutil.rmtree(os.path.join(root_dir, '.etc'))
except OSError as err:
if err.errno != errno.ENOENT:
raise
shutil.copytree(os.path.join(tm_env.root, 'etc'),
os.path.join(root_dir, '.etc'))
shutil.copyfile('/etc/hosts', os.path.join(root_dir, '.etc/hosts'))
shutil.copyfile('/etc/hosts', os.path.join(root_dir,
'.etc/hosts.original'))
hosts_aliases = os.path.join(root_dir, '.etc', 'hosts-aliases')
fs.mkdir_safe(hosts_aliases)
pwnam = pwd.getpwnam(app.proid)
os.chown(hosts_aliases, pwnam.pw_uid, pwnam.pw_gid)
# Always use our own resolv.conf. Safe to rbind, as we are running in
# private mount subsystem by now.
resolv_conf_path = os.path.join(tm_env.root, 'etc/resolv.conf')
if os.path.exists(resolv_conf_path):
subproc.check_call(
['mount', '-n', '--bind', resolv_conf_path, '/etc/resolv.conf'])
def kill(self):
app = runtime.load_app(self._service.data_dir, runtime.STATE_JSON)
if not app:
return
name = appcfg.app_unique_name(app)
try:
client = self._get_client()
container = client.containers.get(name)
container.kill()
except docker.errors.NotFound:
pass
def _create_container(tm_env, conf, client, app):
"""Create docker container from given app.
"""
ports = {}
for endpoint in app.endpoints:
port_key = '{0}/{1}'.format(endpoint.port, endpoint.proto)
ports[port_key] = endpoint.real_port
_log_port_mapping_config(ports)
# app.image contains a uri which starts with docker://
image_name = app.image[9:]
client.images.pull(image_name)
name = appcfg.app_unique_name(app)
container_args = {
'image': image_name,
'name': name,
'environment': _create_environ(app),
'entrypoint': app.command,
'command': app.args,
'detach': True,
'tty': True,
'ports': ports,
'network': conf.get('network', 'nat'),
# 1024 is max number of shares for docker
'cpu_shares': int(
(app.cpu / (multiprocessing.cpu_count() * 100.0)) * 1024),
'mem_limit': app.memory,
'storage_opt': {
'size': app.disk
},
'ulimits': [
{'core': 'unlimited'},
{'nofile': 32768},
],
}
if os.name == 'nt':
_get_gmsa(tm_env, client, app, container_args)
try:
# The container might exist already
# TODO: start existing container with different ports
container = client.containers.get(name)
container.remove(force=True)
except docker.errors.NotFound:
pass
return client.containers.create(**container_args)
def _finish(self):
app = runtime.load_app(self._service.data_dir, runtime.STATE_JSON)
if app:
client = self._get_client()
container = state = None
name = appcfg.app_unique_name(app)
try:
container = client.containers.get(name)
state = container.attrs.get('State')
except docker.errors.NotFound:
pass
if container is not None:
try:
container.remove(force=True)
except docker.errors.APIError:
_LOGGER.error('Failed to remove %s', container.id)
aborted = _check_aborted(self._service.data_dir)
if aborted is not None:
app_abort.report_aborted(self._tm_env,
app.name,
why=aborted.get('why'),
payload=aborted.get('payload'))
elif state is not None:
if state.get('OOMKilled', False):
event = events.KilledTraceEvent(
instanceid=app.name,
is_oom=True,
)
else:
event = events.FinishedTraceEvent(instanceid=app.name,
rc=state.get(
'ExitCode', 256),
signal=0,
payload=state)
trace.post(self._tm_env.app_events_dir, event)
if os.name == 'nt':
credential_spec.cleanup(name, client)
try:
runtime.archive_logs(self._tm_env, name,
self._service.data_dir)
except Exception: # pylint: disable=W0703
_LOGGER.exception('Unexpected exception storing local logs.')
def share_cgroup_info(app, root_dir):
"""Shares subset of cgroup tree with the container."""
# Bind /cgroup/memory inside chrooted environment to /cgroup/.../memory
# of the container.
unique_name = appcfg.app_unique_name(app)
cgrp = os.path.join('treadmill', 'apps', unique_name)
# FIXME: This should be removed and proper cgroups should be
# exposed (readonly). This is so that tools that
# (correctly) read /proc/self/cgroups can access cgroup
# data.
shared_subsystems = ['memory']
for subsystem in shared_subsystems:
fs.mkdir_safe(os.path.join(root_dir, 'cgroup', subsystem))
fs.mount_bind(root_dir, os.path.join('/cgroup', subsystem),
cgroups.makepath(subsystem, cgrp))
def _create_root_dir(tm_env, container_dir, root_dir, app):
"""Prepares chrooted environment."""
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
# First wait for the block device to be ready
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk'))
localdisk = localdisk_client.wait(unique_name)
already_initialized = fs.test_filesystem(localdisk['block_dev'])
if not already_initialized:
# Format the block device
fs.create_filesystem(localdisk['block_dev'])
_LOGGER.info('Creating container root directory: %s', root_dir)
# Creates directory that will serve as new root.
fs.mkdir_safe(fs.norm_safe(root_dir))
# Unshare the mount namespace
unshare.unshare(unshare.CLONE_NEWNS)
# Mount the container root volume
fs.mount_filesystem(localdisk['block_dev'], root_dir)
def test__create_root_dir(self):
"""Test creation on the container root directory."""
# Access protected module _create_root_dir
# pylint: disable=W0212
app = utils.to_obj({
'type': 'native',
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'dev',
'disk': '100G',
})
app_unique_name = appcfg.app_unique_name(app)
container_dir = '/some/dir'
localdisk = {
'block_dev': '/dev/foo',
}
treadmill.runtime.linux._run._create_root_dir(container_dir, localdisk)
treadmill.fs.linux.blk_fs_create.assert_called_with('/dev/foo')
unshare.unshare.assert_called_with(unshare.CLONE_NEWNS)
treadmill.fs.linux.mount_filesystem.assert_called_with(
'/dev/foo', os.path.join(container_dir, 'root'), fs_type='ext4')
def configure(tm_env, event, runtime):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)/
- apps/
- (app unique name)/
- data/
- app_start
- app.json
- manifest.yml
- policy.json
env/
- TREADMILL_*
run
finish
log/
- run
The 'run' script is responsible for creating container environment
and starting the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# Load the app from the event
try:
manifest_data = app_manifest.load(tm_env, event, runtime)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception('No event to load: %r', event)
return
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Write the actual container start script
if os.name == 'nt':
run_script = ' '.join(
[sys.executable, '-m', 'treadmill.ms', 'sproc', 'run', '.'])
else:
run_script = ' '.join(
['exec', dist.TREADMILL_BIN, 'sproc', 'run', '../'])
# Create the service for that container
container_svc = supervisor.create_service(tm_env.apps_dir,
name=uniq_name,
app_run_script=run_script,
userid='root',
downed=False,
monitor_policy={
'limit': 0,
'interval': 60
},
environ={},
environment=app.environment)
data_dir = container_svc.data_dir
# Copy the original event as 'manifest.yml' in the container dir
shutil.copyfile(event, os.path.join(data_dir, 'manifest.yml'))
# Store the app.json in the container directory
fs.write_safe(os.path.join(data_dir, appcfg.APP_JSON),
lambda f: f.writelines(utils.json_genencode(manifest_data)),
mode='w',
permission=0o644)
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(instanceid=app.name,
uniqueid=app.uniqueid))
return container_svc.directory
def _cleanup(tm_env, zkclient, container_dir, app):
"""Cleanup a container that actually ran.
"""
# Too many branches.
#
# pylint: disable=R0912
rootdir = os.path.join(container_dir, 'root')
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
# Create service clients
cgroup_client = tm_env.svc_cgroup.make_client(
os.path.join(container_dir, 'cgroups'))
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk'))
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network'))
# Make sure all processes are killed
# FIXME(boysson): Should we use `kill_apps_in_cgroup` instead?
_kill_apps_by_root(rootdir)
# Setup the archive filename that will hold this container's data
filetime = utils.datetime_utcnow().strftime('%Y%m%d_%H%M%S%f')
archive_filename = os.path.join(
container_dir, '{instance_name}_{hostname}_{timestamp}.tar'.format(
instance_name=appcfg.appname_task_id(app.name),
hostname=sysinfo.hostname(),
timestamp=filetime))
# Tar up container root filesystem if archive list is in manifest
try:
localdisk = localdisk_client.get(unique_name)
fs.archive_filesystem(localdisk['block_dev'], rootdir,
archive_filename, app.archive)
except services.ResourceServiceError:
_LOGGER.warning('localdisk never allocated')
except subprocess.CalledProcessError:
_LOGGER.exception('Unable to archive root device of %r', unique_name)
except: # pylint: disable=W0702
_LOGGER.exception('Unknown exception while archiving %r', unique_name)
# Destroy the volume
try:
localdisk = localdisk_client.delete(unique_name)
except (IOError, OSError) as err:
if err.errno == errno.ENOENT:
pass
else:
raise
if not app.shared_network:
_cleanup_network(tm_env, app, network_client)
# Add metrics to archive
rrd_file = os.path.join(
tm_env.metrics_dir, 'apps',
'{name}-{instanceid}-{uniqueid}.rrd'.format(
name=app.app,
instanceid=app.task,
uniqueid=app.uniqueid,
))
rrdutils.flush_noexc(rrd_file)
_copy_metrics(rrd_file, container_dir)
# Cleanup our cgroup resources
try:
cgroup_client.delete(unique_name)
except (IOError, OSError) as err:
if err.errno == errno.ENOENT:
pass
else:
raise
try:
_archive_logs(tm_env, container_dir)
except Exception: # pylint: disable=W0703
_LOGGER.exception('Unexpected exception storing local logs.')
# Append or create the tarball with folders outside of container
# Compress and send the tarball to HCP
try:
archive_filename = fs.tar(sources=container_dir,
target=archive_filename,
compression='gzip').name
_send_container_archive(zkclient, app, archive_filename)
except: # pylint: disable=W0702
_LOGGER.exception("Failed to update archive")
def create_supervision_tree(tm_env, container_dir, root_dir, app,
cgroups_path):
"""Creates s6 supervision tree."""
uniq_name = appcfg.app_unique_name(app)
ctl_uds = os.path.join(os.sep, 'run', 'tm_ctl')
tombstone_ctl_uds = os.path.join(ctl_uds, 'tombstone')
sys_dir = os.path.join(container_dir, 'sys')
try:
old_system_services = [
svc_name for svc_name in os.listdir(sys_dir)
if (not svc_name.startswith('.')
and os.path.isdir(os.path.join(sys_dir, svc_name)))
]
except FileNotFoundError:
old_system_services = []
new_system_services = [svc_def.name for svc_def in app.system_services]
for svc_name in set(old_system_services) - set(new_system_services):
_LOGGER.info('Removing old system service: %s', svc_name)
fs.rmtree_safe(os.path.join(sys_dir, svc_name))
sys_scandir = supervisor.create_scan_dir(
sys_dir,
finish_timeout=6000,
wait_cgroups=cgroups_path,
)
for svc_def in app.system_services:
if svc_def.restart is not None:
monitor_policy = {
'limit': svc_def.restart.limit,
'interval': svc_def.restart.interval,
'tombstone': {
'uds': False,
'path': tm_env.services_tombstone_dir,
'id': '{},{}'.format(uniq_name, svc_def.name)
}
}
else:
monitor_policy = None
supervisor.create_service(
sys_scandir,
name=svc_def.name,
app_run_script=svc_def.command,
userid='root',
environ_dir=os.path.join(container_dir, _CONTAINER_ENV_DIR),
environ={envvar.name: envvar.value
for envvar in svc_def.environ},
environment=app.environment,
downed=svc_def.downed,
trace=None,
monitor_policy=monitor_policy)
sys_scandir.write()
services_dir = os.path.join(container_dir, 'services')
services_scandir = supervisor.create_scan_dir(services_dir,
finish_timeout=5000)
for svc_def in app.services:
if svc_def.restart is not None:
monitor_policy = {
'limit': svc_def.restart.limit,
'interval': svc_def.restart.interval,
'tombstone': {
'uds': True,
'path': tombstone_ctl_uds,
'id': '{},{}'.format(uniq_name, svc_def.name)
}
}
else:
monitor_policy = None
if svc_def.trace is not None:
trace = {
'instanceid': app.name,
'uniqueid': app.uniqueid,
'service': svc_def.name,
'path': os.path.join(ctl_uds, 'appevents')
}
else:
trace = None
logger_template = getattr(svc_def, 'logger', 's6.app-logger.run')
_LOGGER.info('Using logger: %s', logger_template)
supervisor.create_service(
services_scandir,
name=svc_def.name,
app_run_script=svc_def.command,
userid=svc_def.proid,
environ_dir='/' + _CONTAINER_ENV_DIR,
environ={envvar.name: envvar.value
for envvar in svc_def.environ},
environment=app.environment,
downed=svc_def.downed,
trace=trace if svc_def.trace else None,
log_run_script=logger_template,
monitor_policy=monitor_policy)
services_scandir.write()
# Bind the service directory in the container volume
fs.mkdir_safe(os.path.join(root_dir, 'services'))
fs_linux.mount_bind(root_dir,
os.path.join(os.sep, 'services'),
source=os.path.join(container_dir, 'services'),
recursive=False,
read_only=False)
# Bind the ctrl directory in the container volume which has all the
# unix domain sockets to communicate outside the container to treadmill
fs.mkdir_safe(os.path.join(root_dir, 'run', 'tm_ctl'))
fs_linux.mount_bind(root_dir,
os.path.join(os.sep, 'run', 'tm_ctl'),
source=tm_env.ctl_dir,
recursive=False,
read_only=False)
def _unshare_network(tm_env, container_dir, app):
"""Configures private app network.
:param ``appenv.AppEnvironment`` tm_env:
Treadmill application environment
"""
unique_name = appcfg.app_unique_name(app)
# Configure DNAT rules while on host network.
for endpoint in app.endpoints:
_LOGGER.info('Creating DNAT rule: %s:%s -> %s:%s',
app.network.external_ip,
endpoint.real_port,
app.network.vip,
endpoint.port)
dnatrule = firewall.DNATRule(proto=endpoint.proto,
dst_ip=app.network.external_ip,
dst_port=endpoint.real_port,
new_ip=app.network.vip,
new_port=endpoint.port)
snatrule = firewall.SNATRule(proto=endpoint.proto,
src_ip=app.network.vip,
src_port=endpoint.port,
new_ip=app.network.external_ip,
new_port=endpoint.real_port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
tm_env.rules.create_rule(chain=iptables.POSTROUTING_SNAT,
rule=snatrule,
owner=unique_name)
# See if this container requires vring service
if app.vring:
_LOGGER.debug('adding %r to VRing set', app.network.vip)
iptables.add_ip_set(
iptables.SET_VRING_CONTAINERS,
app.network.vip
)
# See if this was an "infra" endpoint and if so add it to the whitelist
# set.
if getattr(endpoint, 'type', None) == 'infra':
_LOGGER.debug('adding %s:%s to infra services set',
app.network.vip, endpoint.port)
iptables.add_ip_set(
iptables.SET_INFRA_SVC,
'{ip},{proto}:{port}'.format(
ip=app.network.vip,
proto=endpoint.proto,
port=endpoint.port,
)
)
for port in app.ephemeral_ports.tcp:
_LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s',
app.network.external_ip, port,
app.network.vip, port)
dnatrule = firewall.DNATRule(proto='tcp',
dst_ip=app.network.external_ip,
dst_port=port,
new_ip=app.network.vip,
new_port=port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
# Treat ephemeral ports as infra, consistent with current prodperim
# behavior.
iptables.add_ip_set(iptables.SET_INFRA_SVC,
'{ip},tcp:{port}'.format(ip=app.network.vip,
port=port))
for port in app.ephemeral_ports.udp:
_LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s',
app.network.external_ip, port,
app.network.vip, port)
dnatrule = firewall.DNATRule(proto='udp',
dst_ip=app.network.external_ip,
dst_port=port,
new_ip=app.network.vip,
new_port=port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
# Treat ephemeral ports as infra, consistent with current prodperim
# behavior.
iptables.add_ip_set(iptables.SET_INFRA_SVC,
'{ip},udp:{port}'.format(ip=app.network.vip,
port=port))
# configure passthrough while on main network.
if getattr(app, 'passthrough', None):
_LOGGER.info('adding passthrough for: %r', app.passthrough)
# Resolve all the hosts (+dedup)
new_ips = {
socket.gethostbyname(host)
for host in app.passthrough
}
# Create a passthrough rule from each of the source IP to the
# container IP and record these source IP in a set.
for ipaddr in new_ips:
passthroughrule = firewall.PassThroughRule(
src_ip=ipaddr,
dst_ip=app.network.vip,
)
tm_env.rules.create_rule(chain=iptables.PREROUTING_PASSTHROUGH,
rule=passthroughrule,
owner=unique_name)
# configure exception filter rules
try:
firewall_plugin = plugin_manager.load(
'treadmill.firewall.plugins', 'firewall'
)
firewall_plugin.apply_exception_rules(tm_env, container_dir, app)
except Exception: # pylint: disable=W0703
_LOGGER.exception(
'Error in firewall plugin, skip applying firewall exception rules.'
)
service_ip = None
if app.shared_ip:
service_ip = app.network.external_ip
# Unshare network and create virtual device
newnet.create_newnet(app.network.veth,
app.network.vip,
app.network.gateway,
service_ip)
def _cleanup_network(tm_env, container_dir, app, network_client):
"""Cleanup the network part of a container.
"""
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
try:
app_network = network_client.get(unique_name)
except services.ResourceServiceError:
_LOGGER.warning('network never allocated')
return
if app_network is None:
_LOGGER.info('Network resource already freed')
return
# Unconfigure passthrough
if hasattr(app, 'passthrough'):
_LOGGER.info('Deleting passthrough for: %r', app.passthrough)
# Resolve all the hosts
# FIXME: There is no guarantie the hosts will resolve to
# the same IPs as they did during creation.
ips = set([socket.gethostbyname(host) for host in app.passthrough])
for ip in ips:
tm_env.rules.unlink_rule(
chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(src_ip=ip,
dst_ip=app_network['vip']),
owner=unique_name,
)
if app.vring:
# Mark the container's IP as VRing enabled
_LOGGER.debug('removing %r from VRing set', app_network['vip'])
iptables.rm_ip_set(iptables.SET_VRING_CONTAINERS, app_network['vip'])
for endpoint in app.endpoints:
tm_env.rules.unlink_rule(
chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto=endpoint.proto,
dst_ip=app_network['external_ip'],
dst_port=endpoint.real_port,
new_ip=app_network['vip'],
new_port=endpoint.port),
owner=unique_name,
)
tm_env.rules.unlink_rule(
chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto=endpoint.proto,
src_ip=app_network['vip'],
src_port=endpoint.port,
new_ip=app_network['external_ip'],
new_port=endpoint.real_port),
owner=unique_name,
)
# See if this was an "infra" endpoint and if so remove it
# from the whitelist set.
if getattr(endpoint, 'type', None) == 'infra':
_LOGGER.debug('removing %s:%s from infra services set',
app_network['vip'], endpoint.port)
iptables.rm_ip_set(
iptables.SET_INFRA_SVC, '{ip},{proto}:{port}'.format(
ip=app_network['vip'],
proto=endpoint.proto,
port=endpoint.port,
))
_cleanup_ephemeral_ports(tm_env, unique_name, app_network['external_ip'],
app_network['vip'], app.ephemeral_ports.tcp,
'tcp')
_cleanup_ephemeral_ports(tm_env, unique_name, app_network['external_ip'],
app_network['vip'], app.ephemeral_ports.udp,
'udp')
_cleanup_exception_rules(tm_env, container_dir, app)
# Terminate any entries in the conntrack table
iptables.flush_cnt_conntrack_table(app_network['vip'])
# Cleanup network resources
network_client.delete(unique_name)
def test__unshare_network_simple(self):
"""Tests unshare network sequence.
"""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'name':
'proid.test#0',
'uniqueid':
'ID1234',
'environment':
'dev',
'network': {
'veth': 'id1234.0',
'vip': '192.168.1.1',
'gateway': '192.168.254.254',
'external_ip': '172.31.81.67',
},
'shared_ip':
True,
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'endpoints': [{
'real_port': '5007',
'proto': 'tcp',
'port': '22',
'type': 'infra'
}, {
'real_port': '5013',
'proto': 'udp',
'port': '12345'
}],
'vring': {
'some': 'data'
}
})
app_unique_name = appcfg.app_unique_name(app)
treadmill.runtime.linux._run._unshare_network(self.tm_env,
'test_container_dir',
app)
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.1.1'),
mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.1.1,tcp:22'),
],
any_order=True)
self.tm_env.rules.create_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port='5007',
new_ip='192.168.1.1',
new_port='22'),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.1.1',
src_port='22',
new_ip='172.31.81.67',
new_port='5007'),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port='5013',
new_ip='192.168.1.1',
new_port='12345'),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='udp',
src_ip='192.168.1.1',
src_port='12345',
new_ip='172.31.81.67',
new_port='5013'),
owner=app_unique_name)
],
any_order=True)
self.assertEqual(self.tm_env.rules.create_rule.call_count, 4)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.1.1',
'192.168.254.254',
'172.31.81.67',
)
def test__unshare_network_complex(self):
"""Test unshare network advanced sequence (ephemeral/passthrough)."""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'name':
'myproid.test#0',
'environment':
'dev',
'uniqueid':
'ID1234',
'network': {
'veth': 'id1234.0',
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'external_ip': '172.31.81.67',
},
'shared_ip':
False,
'endpoints': [{
'name': 'ssh',
'port': 54321,
'real_port': 54321,
'type': 'infra',
'proto': 'tcp',
}, {
'name': 'test2',
'port': 54322,
'real_port': 54322,
'proto': 'udp',
}],
'ephemeral_ports': {
'tcp': [10000, 10001, 10002],
'udp': [],
},
'passthrough': [
'xxx',
'yyy',
'zzz',
],
'vring': {
'some': 'data'
}
})
app_unique_name = appcfg.app_unique_name(app)
hosts_to_ip = {
'xxx': '4.4.4.4',
'yyy': '5.5.5.5',
'zzz': '5.5.5.5',
}
socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h]
self.tm_env.rules.get_rules.return_value = set()
treadmill.runtime.linux._run._unshare_network(self.tm_env,
'test_container_dir',
app)
self.tm_env.rules.create_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=54322,
new_ip='192.168.0.2',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='udp',
src_ip='192.168.0.2',
src_port=54322,
new_ip='172.31.81.67',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10000,
new_ip='192.168.0.2',
new_port=10000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10001,
new_ip='192.168.0.2',
new_port=10001),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10002,
new_ip='192.168.0.2',
new_port=10002),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('4.4.4.4', '192.168.0.2'),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('5.5.5.5', '192.168.0.2'),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.create_rule.call_count, 9)
# Check that infra services + ephemeral ports are in the same set.
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10000'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10001'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10002'),
],
any_order=True)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.0.2',
'192.168.254.254',
None,
)
def get_cgroup_path(app):
"""Gets the path of the cgroup."""
unique_name = appcfg.app_unique_name(app)
cgrp = os.path.join('treadmill', 'apps', unique_name, 'services')
return cgrp
def configure(tm_env, event):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)
- apps
- (app unique name)
- app.yml
run
finish
log/run
The 'run' script is responsible for creating container environment
and starting svscan inside the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# R0915: Need to refactor long function into smaller pieces.
#
# pylint: disable=R0915
# Load the app from the event
try:
manifest_data = app_manifest.load(tm_env, event)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception("No event to load: %r", event)
return
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Check the identity we are going to run as
_check_identity(app.proid)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Create the app's running directory
container_dir = os.path.join(tm_env.apps_dir, uniq_name)
if not fs.mkdir_safe(container_dir):
_LOGGER.info('Resuming container %r', uniq_name)
# Copy the event as 'manifest.yml' in the container dir
shutil.copyfile(event, os.path.join(container_dir, 'manifest.yml'))
# Store the app int the container_dir
app_yml = os.path.join(container_dir, _APP_YML)
with open(app_yml, 'w') as f:
yaml.dump(manifest_data, stream=f)
# Mark the container as defaulting to down state
utils.touch(os.path.join(container_dir, 'down'))
# Generate the supervisor's run script
app_run_cmd = ' '.join(
[treadmill.TREADMILL_BIN, 'sproc', 'run', container_dir])
utils.create_script(os.path.join(container_dir, 'run'),
'supervisor.run_no_log',
cmd=app_run_cmd)
fs.mkdir_safe(os.path.join(container_dir, 'log'))
utils.create_script(os.path.join(container_dir, 'log', 'run'),
'logger.run')
# Unique name for the link, based on creation time.
cleanup_link = os.path.join(tm_env.cleanup_dir, uniq_name)
if os.name == 'nt':
finish_cmd = '%%COMSPEC%% /C mklink /D %s %s' % \
(cleanup_link, container_dir)
else:
finish_cmd = '/bin/ln -snvf %s %s' % (container_dir, cleanup_link)
utils.create_script(os.path.join(container_dir, 'finish'),
'supervisor.finish',
service=app.name,
proid=None,
cmds=[finish_cmd])
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(instanceid=app.name,
uniqueid=app.uniqueid))
return container_dir
def configure(tm_env, event, runtime):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)/
- apps/
- (app unique name)/
- data/
- app_start
- app.json
- manifest.yml
env/
- TREADMILL_*
run
finish
log/
- run
The 'run' script is responsible for creating container environment
and starting the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# Load the app from the event
try:
manifest_data = load_runtime_manifest(tm_env, event, runtime)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception('No event to load: %r', event)
return None
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Write the actual container start script
if os.name == 'nt':
run_script = '{treadmill}/scripts/treadmill sproc run .'.format(
treadmill=subproc.resolve('treadmill'),
)
else:
run_script = 'exec {treadmill}/bin/treadmill sproc run ../'.format(
treadmill=subproc.resolve('treadmill'),
)
# Create the service for that container
container_svc = supervisor.create_service(
tm_env.apps_dir,
name=uniq_name,
app_run_script=run_script,
userid='root',
downed=False,
monitor_policy={
'limit': 0,
'interval': 60,
'tombstone': {
'uds': False,
'path': tm_env.running_tombstone_dir,
'id': app.name
}
},
environ={},
environment=app.environment
)
data_dir = container_svc.data_dir
# Copy the original event as 'manifest.yml' in the container dir
try:
shutil.copyfile(
event,
os.path.join(data_dir, 'manifest.yml')
)
except IOError as err:
# File is gone, cleanup.
if err.errno == errno.ENOENT:
shutil.rmtree(container_svc.directory)
_LOGGER.exception('Event gone: %r', event)
return None
else:
raise
# Store the app.json in the container directory
fs.write_safe(
os.path.join(data_dir, appcfg.APP_JSON),
lambda f: f.writelines(
utils.json_genencode(manifest_data)
),
mode='w',
permission=0o644
)
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(
instanceid=app.name,
uniqueid=app.uniqueid
)
)
return container_svc.directory
