def test__create_root_dir(self):
"""Test creation on the container root directory."""
# Access protected module _create_root_dir
# pylint: disable=W0212
app = utils.to_obj({
'type': 'native',
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'dev',
'disk': '100G',
})
app_unique_name = appcfg.app_unique_name(app)
container_dir = os.path.join(self.root, 'apps', app_unique_name)
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.wait.return_value = localdisk
treadmill.runtime.linux._run._create_root_dir(self.tm_env,
container_dir,
'/some/root_dir', app)
treadmill.fs.create_filesystem.assert_called_with('/dev/foo')
unshare.unshare.assert_called_with(unshare.CLONE_NEWNS)
treadmill.fs.mount_filesystem('/dev/foo', '/some/root_dir')
def test__unshare_network_simple(self):
"""Tests unshare network sequence.
"""
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj({
'name':
'proid.test#0',
'uniqueid':
'ID1234',
'environment':
'dev',
'network': {
'veth': 'id1234.0',
'vip': '192.168.1.1',
'gateway': '192.168.254.254',
},
'host_ip':
'172.31.81.67',
'shared_ip':
True,
'ephemeral_ports': [],
'endpoints': [{
'real_port': '5007',
'proto': 'tcp',
'port': '22',
'type': 'infra'
}, {
'real_port': '5013',
'proto': 'udp',
'port': '12345'
}],
})
app_unique_name = appmgr.app_unique_name(app)
appmgr.run._unshare_network(self.app_env, app)
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.1.1,tcp:22'),
])
self.app_env.rules.create_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', '5007',
'192.168.1.1', '22'),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('udp', '172.31.81.67', '5013',
'192.168.1.1', '12345'),
owner=app_unique_name)
],
any_order=True)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.1.1',
'192.168.254.254',
'172.31.81.67',
)
def test__create_root_dir(self):
"""Test creation on the container root directory."""
# Access protected module _create_root_dir
# pylint: disable=W0212
app = utils.to_obj(
{
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'dev',
'disk': '100G',
}
)
app_unique_name = appcfg.app_unique_name(app)
container_dir = os.path.join(self.root, 'apps', app_unique_name)
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.wait.return_value = localdisk
treadmill.runtime.linux._run._create_root_dir(self.tm_env,
container_dir,
'/some/root_dir',
app)
treadmill.fs.chroot_init.assert_called_with('/some/root_dir')
treadmill.fs.create_filesystem.assert_called_with('/dev/foo')
treadmill.fs.mount_filesystem('/dev/foo', '/some/root_dir')
treadmill.fs.make_rootfs.assert_called_with('/some/root_dir',
'myproid')
treadmill.fs.configure_plugins.assert_called_with(
self.root,
'/some/root_dir',
app
)
shutil.rmtree.assert_called_with(
'/some/root_dir/.etc'
)
shutil.copytree.assert_called_with(
os.path.join(self.tm_env.root, 'etc'),
'/some/root_dir/.etc'
)
shutil.copyfile.assert_has_calls([
mock.call('/etc/hosts', '/some/root_dir/.etc/hosts'),
mock.call('/etc/hosts', '/some/root_dir/.etc/hosts.original'),
])
treadmill.subproc.check_call.assert_has_calls([
mock.call(
[
'mount', '-n', '--bind',
os.path.join(self.tm_env.root, 'etc/resolv.conf'),
'/etc/resolv.conf'
]
)
])
def apply_cgroup_limits(tm_env, container_dir, manifest):
"""Configures cgroups and limits.
:param tm_env:
Treadmill application environment
:type tm_env:
`appenv.AppEnvironment`
:param container_dir:
Full path to the container
:type container_dir:
``str``
:param manifest:
App manifest.
:type manifest:
``dict``
"""
app = utils.to_obj(manifest)
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
# Setup the service clients
cgroup_client = tm_env.svc_cgroup.make_client(
os.path.join(container_dir, 'cgroups'))
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk'))
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network'))
# Cgroup
cgroup_req = {
'memory': app.memory,
'cpu': app.cpu,
}
# Local Disk
localdisk_req = {
'size': app.disk,
}
# Network
network_req = {
'environment': app.environment,
}
cgroup_client.put(unique_name, cgroup_req)
localdisk_client.put(unique_name, localdisk_req)
if not app.shared_network:
network_client.put(unique_name, network_req)
app_cgroups = cgroup_client.wait(unique_name)
_LOGGER.info('Joining cgroups: %r', app_cgroups)
for subsystem, cgrp in app_cgroups.items():
cgroups.join(subsystem, cgrp)
def _load_config(config_file):
"""Load the linux runtime configuration.
"""
cp = configparser.SafeConfigParser()
with io.open(config_file) as f:
cp.readfp(f) # pylint: disable=deprecated-method
conf = {
'host_mount_whitelist':
cp.get('linux', 'host_mount_whitelist', fallback='').split(',')
}
return utils.to_obj(conf)
def test_to_obj(self):
"""Tests dict to namedtuple conversion."""
obj = utils.to_obj({'a': 1, 'b': 2, 'c': 3}, 'foo')
self.assertEquals(1, obj.a)
self.assertEquals(2, obj.b)
self.assertEquals(3, obj.c)
obj = utils.to_obj({'a': 1, 'b': [1, 2, 3], 'c': 3}, 'foo')
self.assertEquals(1, obj.a)
self.assertEquals([1, 2, 3], obj.b)
self.assertEquals(3, obj.c)
obj = utils.to_obj({'a': 1, 'b': {'d': 5}, 'c': 3}, 'foo')
self.assertEquals(1, obj.a)
self.assertEquals(5, obj.b.d)
self.assertEquals(3, obj.c)
obj = utils.to_obj({'a': [1, {'d': 5}, 3], 'b': 33}, 'foo')
self.assertEquals(1, obj.a[0])
self.assertEquals(5, obj.a[1].d)
self.assertEquals(3, obj.a[2])
self.assertEquals(33, obj.b)
def save_app(manifest, container_dir, app_json=STATE_JSON):
"""Saves app manifest and freezes to object."""
# Save the manifest with allocated vip and ports in the state
state_file = os.path.join(container_dir, app_json)
with tempfile.NamedTemporaryFile(dir=container_dir, delete=False,
mode='w') as temp_file:
json.dump(manifest, temp_file)
# chmod for the file to be world readable.
os.fchmod(temp_file.fileno(),
stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
os.rename(temp_file.name, state_file)
# Freeze the app data into a namedtuple object
return utils.to_obj(manifest)
def save_app(manifest, container_dir, app_json=STATE_JSON):
"""Saves app manifest and freezes to object."""
# Save the manifest with allocated vip and ports in the state
#
state_file = os.path.join(container_dir, app_json)
fs.write_safe(
state_file,
lambda f: f.writelines(utils.json_genencode(manifest)),
mode='w',
# chmod for the file to be world readable.
permission=0o644)
# Freeze the app data into a namedtuple object
return utils.to_obj(manifest)
def load_app(container_dir, app_json=STATE_JSON):
"""Load app manifest as object."""
manifest_file = os.path.join(container_dir, app_json)
try:
manifest = app_manifest.read(manifest_file)
_LOGGER.debug('Manifest: %r', manifest)
return utils.to_obj(manifest)
except IOError as err:
if err.errno != errno.ENOENT:
raise
_LOGGER.info('Manifest file does not exist: %s', manifest_file)
return None
def _load_app(container_dir, app_yml):
"""Load app from original manifest, pre-configured."""
manifest_file = os.path.join(container_dir, app_yml)
try:
manifest = app_manifest.read(manifest_file)
_LOGGER.debug('Manifest: %r', manifest)
return utils.to_obj(manifest)
except IOError as err:
if err.errno != errno.ENOENT:
raise
_LOGGER.critical('Manifest file does not exit: %r', manifest_file)
return None
def setUp(self):
# Access protected module _base_service
# pylint: disable=W0212
self.container_dir = tempfile.mkdtemp()
self.root = tempfile.mkdtemp(dir=self.container_dir)
self.services_tombstone_dir = os.path.join(self.root, 'tombstone')
self.tm_env = mock.Mock(
root=self.root,
services_tombstone_dir=self.services_tombstone_dir,
ctl_dir=os.path.join(self.root, 'ctl'),
svc_cgroup=mock.Mock(
spec_set=treadmill.services._base_service.ResourceService,
),
svc_localdisk=mock.Mock(
spec_set=treadmill.services._base_service.ResourceService,
),
svc_network=mock.Mock(
spec_set=treadmill.services._base_service.ResourceService,
),
rules=mock.Mock(
spec_set=treadmill.rulefile.RuleMgr,
),
)
self.app = utils.to_obj(
{
'type': 'native',
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'dev',
'disk': '100G',
'endpoints': [
{
'name': 'ssh',
'port': 47299,
'proto': 'tcp',
'real_port': 47299,
'type': 'infra'
}
],
'shared_network': False,
'ephemeral_ports': {
'tcp': 1,
'udp': 0
},
'docker': False
}
)
def load_app_safe(container, container_dir, app_json=STATE_JSON):
"""Load app manifest as object.
If app manifest is corrupted or invalid, return object with key attributes.
"""
try:
return load_app(container_dir, app_json=app_json)
except ValueError as err:
_LOGGER.error('Manifest file is corrupted or invalid: %s', err)
appname = appcfg.app_name(container)
return utils.to_obj({
'name': appname,
'app': appcfg.appname_basename(appname),
'task': appcfg.appname_task_id(appname),
'uniqueid': appcfg.app_unique_id(container),
})
def test__share_cgroup_info(self):
"""Test sharing of cgroup information with the container."""
# Access protected module _share_cgroup_info
# pylint: disable=W0212
app = utils.to_obj({
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
})
treadmill.appmgr.run._share_cgroup_info(app, '/some/root_dir')
# Check that cgroup mountpoint exists inside the container.
treadmill.fs.mkdir_safe.assert_has_calls(
[mock.call('/some/root_dir/cgroup/memory')])
treadmill.fs.mount_bind.assert_has_calls([
mock.call('/some/root_dir', '/cgroup/memory', '/test/cgroup/path')
])
def test__create_root_dir(self):
"""Test creation on the container root directory."""
# Access protected module _create_root_dir
# pylint: disable=W0212
app = utils.to_obj({
'type': 'native',
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'dev',
'disk': '100G',
})
app_unique_name = appcfg.app_unique_name(app)
container_dir = '/some/dir'
localdisk = {
'block_dev': '/dev/foo',
}
treadmill.runtime.linux._run._create_root_dir(container_dir, localdisk)
treadmill.fs.linux.blk_fs_create.assert_called_with('/dev/foo')
unshare.unshare.assert_called_with(unshare.CLONE_NEWNS)
treadmill.fs.linux.mount_filesystem.assert_called_with(
'/dev/foo', os.path.join(container_dir, 'root'), fs_type='ext4')
def test_run(self):
"""Tests appmgr.run sequence, which will result in supervisor exec.
"""
# access protected module _allocate_network_ports
# pylint: disable=w0212
manifest = {
'shared_network': False,
'ephemeral_ports': 3,
'passthrough': [
'xxx',
'yyy',
'zzz'
],
'memory': '100M',
'host_ip': '172.31.81.67',
'uniqueid': 'ID1234',
'services': [
{
'command': '/bin/true',
'restart_count': 3,
'name': 'web_server'
}
],
'disk': '100G',
'tickets': True,
'name': 'proid.myapp#0',
'system_services': [],
'environment': 'dev',
'proid': 'foo',
'endpoints': [
{
'name': 'http',
'port': 8000
},
{
'name': 'port0',
'port': 0
},
{
'type': 'infra',
'name': 'ssh',
'port': 0
}
],
'cpu': '100%'
}
treadmill.appmgr.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-0-0000000ID1234'
app_dir = os.path.join(self.root, 'apps', app_unique_name)
os.makedirs(app_dir)
mock_nwrk_client = self.app_env.svc_network.make_client.return_value
network = {
'vip': '2.2.2.2',
'gateway': '1.1.1.1',
'veth': 'testveth.0',
}
mock_nwrk_client.wait.return_value = network
def _fake_allocate_network_ports(_ip, manifest):
"""Mimick inplace manifest modification in _allocate_network_ports.
"""
manifest['ephemeral_ports'] = ['1', '2', '3']
return mock.DEFAULT
treadmill.appmgr.run._allocate_network_ports.side_effect = \
_fake_allocate_network_ports
mock_watchdog = mock.Mock()
treadmill.subproc.BINARIES['treadmill_bind_preload.so'] = (
'/some/$LIB/treadmill_bind_preload.so')
app_run.run(
self.app_env, app_dir, mock_watchdog, terminated=()
)
# Check that port allocation is correctly called.
# XXX(boysson): potential mock bug: assert_call expects the vip since
# manifest is modified in place even though the vip are
# allocated *after*.
manifest['vip'] = {
'ip0': '1.1.1.1',
'ip1': '2.2.2.2',
}
manifest['network'] = network
manifest['ephemeral_ports'] = ['1', '2', '3']
treadmill.appmgr.run._allocate_network_ports.assert_called_with(
'172.31.81.67', manifest,
)
# Make sure, post modification, that the manifest is readable by other.
st = os.stat(os.path.join(app_dir, 'state.yml'))
self.assertTrue(st.st_mode & stat.S_IRUSR)
self.assertTrue(st.st_mode & stat.S_IRGRP)
self.assertTrue(st.st_mode & stat.S_IROTH)
self.assertTrue(st.st_mode & stat.S_IWUSR)
self.assertFalse(st.st_mode & stat.S_IWGRP)
self.assertFalse(st.st_mode & stat.S_IWOTH)
# State yml is what is copied in the container
shutil.copy.assert_called_with(
os.path.join(app_dir, 'state.yml'),
os.path.join(app_dir, 'root', 'app.yml'),
)
# Network unshare
app = utils.to_obj(manifest)
treadmill.appmgr.run._unshare_network.assert_called_with(
self.app_env, app
)
# Create root dir
treadmill.appmgr.run._create_root_dir.assert_called_with(
self.app_env,
app_dir,
os.path.join(app_dir, 'root'),
app,
)
# XXX(boysson): Missing environ_dir/manifest_dir tests
# Create supervision tree
treadmill.appmgr.run._create_supervision_tree.assert_called_with(
app_dir,
self.app_env.app_events_dir,
app
)
treadmill.appmgr.run._share_cgroup_info.assert_called_with(
app,
os.path.join(app_dir, 'root'),
)
# Ephemeral LDPRELOAD
treadmill.appmgr.run._prepare_ldpreload.assert_called_with(
os.path.join(app_dir, 'root'),
['/some/$LIB/treadmill_bind_preload.so']
)
# Misc bind mounts
treadmill.fs.mount_bind.assert_has_calls([
mock.call(
os.path.join(app_dir, 'root'),
'/etc/resolv.conf',
bind_opt='--bind',
target=os.path.join(app_dir, 'root/.etc/resolv.conf')
),
mock.call(
os.path.join(app_dir, 'root'),
'/etc/ld.so.preload',
bind_opt='--bind',
target=os.path.join(app_dir, 'root/.etc/ld.so.preload')
),
mock.call(
os.path.join(app_dir, 'root'),
'/etc/pam.d/sshd',
bind_opt='--bind',
target=os.path.join(app_dir, 'root/treadmill/etc/pam.d/sshd')
),
])
self.assertTrue(mock_watchdog.remove.called)
def test__unshare_network_complex(self):
"""Test unshare network advanced sequence (ephemeral/passthrough)."""
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj(
{
'name': 'myproid.test#0',
'environment': 'dev',
'uniqueid': 'ID1234',
'network': {
'veth': 'id1234.0',
'vip': '192.168.0.2',
'gateway': '192.168.254.254'
},
'shared_ip': False,
'endpoints': [
{
'name': 'ssh',
'port': 54321,
'real_port': 54321,
'type': 'infra',
}
],
'ephemeral_ports': [
10000,
10001,
10002,
],
'passthrough': [
'xxx',
'yyy',
'zzz',
],
}
)
app_unique_name = appmgr.app_unique_name(app)
hosts_to_ip = {
'xxx': '4.4.4.4',
'yyy': '5.5.5.5',
'zzz': '5.5.5.5',
}
socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h]
self.app_env.rules.get_rules.return_value = set()
treadmill.appmgr.run._unshare_network(
self.app_env,
app
)
self.app_env.rules.create_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('172.31.81.67', 54321,
'192.168.0.2', 54321),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10000,
'192.168.0.2', 10000),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10001,
'192.168.0.2', 10001),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10002,
'192.168.0.2', 10002),
owner=app_unique_name),
mock.call(rule=firewall.PassThroughRule('4.4.4.4',
'192.168.0.2'),
owner=app_unique_name),
mock.call(rule=firewall.PassThroughRule('5.5.5.5',
'192.168.0.2'),
owner=app_unique_name),
])
# Check that infra services + ephemeral ports are in the same set.
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10000'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10001'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10002'),
])
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.0.2',
'192.168.254.254',
None,
)
def test__create_supervision_tree(self):
"""Test creation of the supervision tree."""
# pylint: disable=W0212
treadmill.subproc.BINARIES = {
'chroot': '/bin/ls',
'pid1': '/bin/ls',
}
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj(
{
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'prod',
'services': [
{
'name': 'command1',
'command': '/path/to/command',
'restart_count': 3,
}, {
'name': 'command2',
'command': '/path/to/other/command',
'restart_count': -1,
}
],
'system_services': [
{
'name': 'command3',
'command': '/path/to/sbin/command',
'restart_count': 1,
}, {
'name': 'command4',
'command': '/path/to/other/sbin/command',
'restart_count': -1,
}
],
'vring': {
'cells': ['a', 'b']
},
}
)
base_dir = '/some/dir'
events_dir = '/some/dir/appevents'
treadmill.appmgr.run._create_supervision_tree(
base_dir,
events_dir,
app,
)
treadmill.fs.mkdir_safe.assert_has_calls([
mock.call('/some/dir/root/services'),
mock.call('/some/dir/services'),
mock.call('/some/dir/services/command1/log'),
mock.call('/some/dir/services/command2/log'),
mock.call('/some/dir/services/command3/log'),
mock.call('/some/dir/services/command4/log'),
mock.call('/some/dir/sys/vring.a'),
mock.call('/some/dir/sys/vring.a/log'),
mock.call('/some/dir/sys/vring.b'),
mock.call('/some/dir/sys/vring.b/log'),
mock.call('/some/dir/sys/monitor'),
mock.call('/some/dir/sys/monitor/log'),
mock.call('/some/dir/sys/register'),
mock.call('/some/dir/sys/register/log'),
mock.call('/some/dir/sys/start_container'),
mock.call('/some/dir/sys/start_container/log'),
])
treadmill.fs.mount_bind.assert_called_with(
'/some/dir/root', '/services', '/some/dir/services',
)
pwd.getpwnam.assert_has_calls(
[
mock.call('myproid'),
mock.call('root')
],
any_order=True
)
treadmill.supervisor.create_service.assert_has_calls([
# user services
mock.call('/some/dir/services',
'myproid',
mock.ANY, mock.ANY,
'command1',
'/path/to/command',
as_root=True,
down=True,
envdir='/environ',
env='prod'),
mock.call('/some/dir/services',
'myproid',
mock.ANY, mock.ANY,
'command2',
'/path/to/other/command',
as_root=True,
down=True,
envdir='/environ',
env='prod'),
# system services
mock.call('/some/dir/services',
'root',
mock.ANY, mock.ANY,
'command3',
'/path/to/sbin/command',
as_root=True,
down=False,
envdir='/environ',
env='prod'),
mock.call('/some/dir/services',
'root',
mock.ANY, mock.ANY,
'command4',
'/path/to/other/sbin/command',
as_root=True,
down=False,
envdir='/environ',
env='prod')
])
treadmill.utils.create_script.assert_has_calls([
mock.call('/some/dir/services/command1/log/run', 'logger.run'),
mock.call('/some/dir/services/command2/log/run', 'logger.run'),
mock.call('/some/dir/services/command3/log/run', 'logger.run'),
mock.call('/some/dir/services/command4/log/run', 'logger.run'),
mock.call('/some/dir/sys/vring.a/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/vring.a/log/run',
'logger.run'),
mock.call('/some/dir/sys/vring.b/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/vring.b/log/run',
'logger.run'),
mock.call('/some/dir/sys/monitor/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/monitor/log/run',
'logger.run'),
mock.call('/some/dir/sys/register/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/register/log/run',
'logger.run'),
mock.call(
'/some/dir/sys/start_container/run',
'supervisor.run_sys',
cmd=('/bin/ls /some/dir/root /bin/ls '
'-m -p -i s6-svscan /services')
),
mock.call('/some/dir/sys/start_container/log/run',
'logger.run'),
])
treadmill.utils.touch.assert_has_calls([
mock.call('/some/dir/sys/start_container/down'),
])
def configure(tm_env, event):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)
- apps
- (app unique name)
- app.yml
run
finish
The 'run' script is responsible for creating container environment
and starting svscan inside the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# R0915: Need to refactor long function into smaller pieces.
#
# pylint: disable=R0915
# Load the app from the event
try:
manifest_data = app_manifest.load(tm_env, event)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception("No event to load: %r", event)
return
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Check the identity we are going to run as. It needs to exists on the host
# or we will fail later on as we try to seteuid.
try:
pwd.getpwnam(app.proid)
except KeyError:
_LOGGER.exception('Unable to find proid %r in passwd database.',
app.proid)
raise
# Generate a unique name for the app
uniq_name = appmgr.app_unique_name(app)
# Create the app's running directory
container_dir = os.path.join(tm_env.apps_dir, uniq_name)
# We assume it is a 'resume' if the container directory already exists.
is_resume = False
try:
os.makedirs(container_dir)
except OSError as err:
if err.errno == errno.EEXIST:
_LOGGER.info('Resuming container %r', uniq_name)
is_resume = True
else:
raise
# Copy the event as 'manifest.yml' in the container dir
shutil.copyfile(
event,
os.path.join(container_dir, 'manifest.yml')
)
# Setup the service clients
cgroup_client = tm_env.svc_cgroup.make_client(
os.path.join(container_dir, 'cgroups')
)
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk')
)
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network')
)
# Store the app int the container_dir
app_yml = os.path.join(container_dir, _APP_YML)
with open(app_yml, 'w') as f:
yaml.dump(manifest_data, stream=f)
# Generate resources requests
# Cgroup
cgroup_req = {
'memory': app.memory,
'cpu': app.cpu,
}
# Local Disk
localdisk_req = {
'size': app.disk,
}
# Network
network_req = {
'environment': app.environment,
}
if not is_resume:
cgroup_client.create(uniq_name, cgroup_req)
localdisk_client.create(uniq_name, localdisk_req)
else:
cgroup_client.update(uniq_name, cgroup_req)
localdisk_client.update(uniq_name, localdisk_req)
if not app.shared_network:
if not is_resume:
network_client.create(uniq_name, network_req)
else:
network_client.update(uniq_name, network_req)
# Mark the container as defaulting to down state
utils.touch(os.path.join(container_dir, 'down'))
# Generate the supervisor's run script
app_run_cmd = ' '.join([
os.path.join(treadmill.TREADMILL, 'bin', 'treadmill'),
'sproc', 'run', container_dir
])
run_out_file = os.path.join(container_dir, 'run.out')
utils.create_script(os.path.join(container_dir, 'run'),
'supervisor.run_no_log',
log_out=run_out_file,
cmd=app_run_cmd)
_init_log_file(run_out_file,
os.path.join(tm_env.apps_dir, "%s.run.out" % uniq_name))
# Unique name for the link, based on creation time.
cleanup_link = os.path.join(tm_env.cleanup_dir, uniq_name)
finish_cmd = '/bin/ln -snvf %s %s' % (container_dir, cleanup_link)
utils.create_script(os.path.join(container_dir, 'finish'),
'supervisor.finish',
service=app.name, proid=None,
cmds=[finish_cmd])
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(
instanceid=app.name,
uniqueid=app.uniqueid
)
)
return container_dir
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
# Access protected module _kill_apps_by_root
# pylint: disable=W0212
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'host_ip':
'172.31.81.67',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/tmp/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
}
treadmill.appmgr.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.app_env.svc_cgroup.make_client.return_value
mock_ld_client = self.app_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.app_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.app_env.apps_dir, app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'exitinfo'), 'w') as f:
f.write(yaml.dump({'service': 'web_server', 'rc': 0, 'sig': 0}))
mock_zkclient = kazoo.client.KazooClient()
app_finish.finish(self.app_env, mock_zkclient, app_dir)
self.app_env.watchdogs.create.assert_called_with(
'treadmill.appmgr.finish-' + app_unique_name, '5m', mock.ANY)
treadmill.subproc.check_call.assert_has_calls([
mock.call([
's6-svc',
'-d',
app_dir,
]),
mock.call([
's6-svwait',
'-d',
app_dir,
]),
])
# All resource service clients are properly created
self.app_env.svc_cgroup.make_client.assert_called_with(
os.path.join(app_dir, 'cgroups'))
self.app_env.svc_localdisk.make_client.assert_called_with(
os.path.join(app_dir, 'localdisk'))
self.app_env.svc_network.make_client.assert_called_with(
os.path.join(app_dir, 'network'))
treadmill.appmgr.finish._kill_apps_by_root.assert_called_with(
os.path.join(app_dir, 'root'))
# Verify that we tested the archiving for the app root volume
treadmill.fs.archive_filesystem.assert_called_with(
'/dev/foo', os.path.join(app_dir, 'root'),
os.path.join(app_dir, '001_xxx.xx.com_20150122_141436537918.tar'),
mock.ANY)
# Verify that the file is uploaded by Uploader
app = utils.to_obj(manifest)
treadmill.appmgr.finish._send_container_archive.assert_called_with(
mock_zkclient,
app,
os.path.join(app_dir,
'001_xxx.xx.com_20150122_141436537918.tar.gz'),
)
# Verify that the app folder was deleted
self.assertFalse(os.path.exists(app_dir))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.app_env.rules.unlink_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 5000,
'192.168.0.2', 8000),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 54321,
'192.168.0.2', 54321),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 45024,
'192.168.0.2', 45024),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('udp', '172.31.81.67', 62422,
'192.168.0.2', 62422),
owner=app_unique_name),
])
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
])
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'sig': 0,
'rc': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd'))
def run(tm_env, container_dir, manifest, watchdog, terminated):
"""Creates container environment and prepares to exec root supervisor.
The function is intended to be invoked from 'run' script and never
returns.
:param tm_env:
Treadmill application environment
:type tm_env:
`appenv.AppEnvironment`
:param container_dir:
Full path to the container
:type container_dir:
``str``
:param manifest:
App manifest.
:type manifest:
``dict``
:param watchdog:
App run watchdog.
:type watchdog:
``treadmill.watchdog``
:param terminated:
Flag where terminated signal will accumulate.
:param terminated:
``set``
:returns:
This function never returns
"""
with lc.LogContext(_LOGGER, os.path.basename(container_dir),
lc.ContainerAdapter) as log:
# R0915: Need to refactor long function into smaller pieces.
# R0912: Too many branches
#
# pylint: disable=R0915,R0912
log.logger.info('Running %r', container_dir)
# Allocate dynamic ports
#
# Ports are taken from ephemeral range, by binding to socket to port 0.
#
# Sockets are then put into global list, so that they are not closed
# at gc time, and address remains in use for the lifetime of the
# supervisor.
sockets = _allocate_network_ports(tm_env.host_ip, manifest)
unique_name = appcfg.manifest_unique_name(manifest)
# First wait for the network device to be ready
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network'))
app_network = network_client.wait(unique_name)
manifest['network'] = app_network
# FIXME(boysson): backward compatibility for TM 2.0. Remove in 3.0
manifest['vip'] = {
'ip0': app_network['gateway'],
'ip1': app_network['vip'],
}
# Save the manifest with allocated vip and ports in the state
state_file = os.path.join(container_dir, _STATE_YML)
with tempfile.NamedTemporaryFile(dir=container_dir,
delete=False,
mode='w') as temp_file:
yaml.dump(manifest, stream=temp_file)
# chmod for the file to be world readable.
os.fchmod(
temp_file.fileno(),
stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
os.rename(temp_file.name, state_file)
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest)
if not app.shared_network:
_unshare_network(tm_env, app)
# Create root directory structure (chroot base).
# container_dir/<subdir>
root_dir = os.path.join(container_dir, 'root')
# chroot_dir/<subdir>
# FIXME(boysson): env_dir should be in a well defined location (part
# of the container "API").
env_dir = os.path.join(root_dir, 'environ')
# Create and format the container root volumne
_create_root_dir(tm_env, container_dir, root_dir, app)
# NOTE: below here, MOUNT namespace is private
# FIXME(boysson): Lots of things are still reading this file.
# Copy updated state manifest as app.yml in the
# container_dir so it is visible in chrooted env.
shutil.copy(state_file, os.path.join(root_dir, _APP_YML))
_create_environ_dir(env_dir, app)
# Create the supervision tree
_create_supervision_tree(container_dir, app)
# Set app limits before chroot.
_share_cgroup_info(app, root_dir)
ldpreloads = []
if app.ephemeral_ports.tcp or app.ephemeral_ports.udp:
treadmill_bind_preload = subproc.resolve(
'treadmill_bind_preload.so')
ldpreloads.append(treadmill_bind_preload)
_prepare_ldpreload(root_dir, ldpreloads)
def _bind(src, tgt):
"""Helper function to bind source to target in the same root"""
# FIXME(boysson): This name mount_bind() have counter-intuitive
# arguments ordering.
src_path = os.path.join(root_dir, src)
if os.path.exists(src_path):
fs.mount_bind(root_dir,
tgt,
target=src_path,
bind_opt='--bind')
# Override the /etc/resolv.conf, so that container always uses
# dnscache.
_bind('.etc/resolv.conf', '/etc/resolv.conf')
_bind('.etc/hosts', '/etc/hosts')
if ldpreloads:
# Override /etc/ld.so.preload to enforce necessary system hooks
_bind('.etc/ld.so.preload', '/etc/ld.so.preload')
# If network is shared, close ephermal sockets before starting the
# supervisor, as these ports will be use be container apps.
if app.shared_network:
for socket_ in sockets:
socket_.close()
# Override pam.d sshd stack with special sshd pam that unshares
# network.
_bind('.etc/pam.d/sshd.shared_network', '/etc/pam.d/sshd')
# else:
# # Override pam.d sshd stack.
# _bind('.etc/pam.d/sshd', '/etc/pam.d/sshd')
watchdog.remove()
if not terminated:
sys_dir = os.path.join(container_dir, 'sys')
supervisor.exec_root_supervisor(sys_dir)
def test__create_supervision_tree(self):
"""Test creation of the supervision tree."""
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'proid':
'myproid',
'name':
'myproid.test#0',
'uniqueid':
'ID1234',
'environment':
'prod',
'services': [{
'name': 'command1',
'command': '/path/to/command',
'restart': {
'limit': 3,
'interval': 60,
},
}, {
'name': 'command2',
'command': '/path/to/other/command',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'system_services': [{
'name': 'command3',
'command': '/path/to/sbin/command',
'restart': {
'limit': 5,
'interval': 60,
},
}, {
'name': 'command4',
'command': '/path/to/other/sbin/command',
'restart': {
'limit': 5,
'interval': 60,
},
}],
'vring': {
'cells': ['a', 'b']
},
})
base_dir = '/some/dir'
native.create_supervision_tree(base_dir, app)
treadmill.fs.mkdir_safe.assert_has_calls([
mock.call('/some/dir/root/services'),
mock.call('/some/dir/services'),
mock.call('/some/dir/sys/.s6-svscan'),
mock.call('/some/dir/services/.s6-svscan'),
mock.call('/some/dir/services/command1/log'),
mock.call('/some/dir/services/command2/log'),
mock.call('/some/dir/services/command3/log'),
mock.call('/some/dir/services/command4/log'),
mock.call('/some/dir/sys/vring.a'),
mock.call('/some/dir/sys/vring.a/log'),
mock.call('/some/dir/sys/vring.b'),
mock.call('/some/dir/sys/vring.b/log'),
mock.call('/some/dir/sys/monitor'),
mock.call('/some/dir/sys/monitor/log'),
mock.call('/some/dir/sys/register'),
mock.call('/some/dir/sys/register/log'),
mock.call('/some/dir/sys/hostaliases'),
mock.call('/some/dir/sys/hostaliases/log'),
mock.call('/some/dir/sys/start_container'),
mock.call('/some/dir/sys/start_container/log'),
])
treadmill.fs.mount_bind.assert_called_with(
'/some/dir/root',
'/services',
'/some/dir/services',
)
pwd.getpwnam.assert_has_calls(
[mock.call('myproid'), mock.call('root')], any_order=True)
treadmill.supervisor.create_service.assert_has_calls([
# user services
mock.call('/some/dir/services',
'myproid',
mock.ANY,
mock.ANY,
'command1',
'/path/to/command',
as_root=True,
down=True,
envdirs=['/environ/app', '/environ/sys'],
env='prod'),
mock.call('/some/dir/services',
'myproid',
mock.ANY,
mock.ANY,
'command2',
'/path/to/other/command',
as_root=True,
down=True,
envdirs=['/environ/app', '/environ/sys'],
env='prod'),
# system services
mock.call('/some/dir/services',
'root',
mock.ANY,
mock.ANY,
'command3',
'/path/to/sbin/command',
as_root=True,
down=False,
envdirs=['/environ/sys'],
env='prod'),
mock.call('/some/dir/services',
'root',
mock.ANY,
mock.ANY,
'command4',
'/path/to/other/sbin/command',
as_root=True,
down=False,
envdirs=['/environ/sys'],
env='prod')
])
treadmill.utils.create_script.assert_has_calls([
mock.call('/some/dir/sys/.s6-svscan/finish',
'svscan.finish',
timeout=mock.ANY),
mock.call('/some/dir/services/.s6-svscan/finish',
'svscan.finish',
timeout=mock.ANY),
mock.call('/some/dir/services/command1/log/run', 'logger.run'),
mock.call('/some/dir/services/command2/log/run', 'logger.run'),
mock.call('/some/dir/services/command3/log/run', 'logger.run'),
mock.call('/some/dir/services/command4/log/run', 'logger.run'),
mock.call('/some/dir/sys/vring.a/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/vring.a/log/run', 'logger.run'),
mock.call('/some/dir/sys/vring.b/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/vring.b/log/run', 'logger.run'),
mock.call('/some/dir/sys/monitor/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/monitor/log/run', 'logger.run'),
mock.call('/some/dir/sys/register/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/register/log/run', 'logger.run'),
mock.call('/some/dir/sys/hostaliases/run',
'supervisor.run_sys',
cmd=mock.ANY),
mock.call('/some/dir/sys/hostaliases/log/run', 'logger.run'),
mock.call('/some/dir/sys/start_container/run',
'supervisor.run_sys',
cmd=('/bin/chroot /some/dir/root /path/to/pid1 '
'-m -p -i /path/to/s6-svscan /services')),
mock.call('/some/dir/sys/start_container/log/run', 'logger.run'),
])
treadmill.utils.touch.assert_has_calls([
mock.call('/some/dir/sys/start_container/down'),
])
def test__unshare_network_simple(self):
"""Tests unshare network sequence.
"""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'name':
'proid.test#0',
'uniqueid':
'ID1234',
'environment':
'dev',
'network': {
'veth': 'id1234.0',
'vip': '192.168.1.1',
'gateway': '192.168.254.254',
'external_ip': '172.31.81.67',
},
'shared_ip':
True,
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'endpoints': [{
'real_port': '5007',
'proto': 'tcp',
'port': '22',
'type': 'infra'
}, {
'real_port': '5013',
'proto': 'udp',
'port': '12345'
}],
'vring': {
'some': 'data'
}
})
app_unique_name = appcfg.app_unique_name(app)
treadmill.runtime.linux._run._unshare_network(self.tm_env,
'test_container_dir',
app)
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.1.1'),
mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.1.1,tcp:22'),
],
any_order=True)
self.tm_env.rules.create_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port='5007',
new_ip='192.168.1.1',
new_port='22'),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.1.1',
src_port='22',
new_ip='172.31.81.67',
new_port='5007'),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port='5013',
new_ip='192.168.1.1',
new_port='12345'),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='udp',
src_ip='192.168.1.1',
src_port='12345',
new_ip='172.31.81.67',
new_port='5013'),
owner=app_unique_name)
],
any_order=True)
self.assertEqual(self.tm_env.rules.create_rule.call_count, 4)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.1.1',
'192.168.254.254',
'172.31.81.67',
)
def test__create_supervision_tree(self):
"""Test creation of the supervision tree."""
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj(
{
'type': 'native',
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'prod',
'services': [
{
'name': 'command1',
'proid': 'test',
'command': '/path/to/command',
'restart': {
'limit': 3,
'interval': 60,
},
'environ': {},
'config': None,
'downed': False,
'trace': True,
},
{
'name': 'command2',
'proid': 'test',
'command': '/path/to/other/command',
'restart': {
'limit': 3,
'interval': 60,
},
'environ': {},
'config': None,
'downed': False,
'trace': True,
}
],
'system_services': [
{
'name': 'command3',
'proid': 'root',
'command': '/path/to/sbin/command',
'restart': {
'limit': 5,
'interval': 60,
},
'environ': {},
'config': None,
'downed': True,
'trace': False,
},
{
'name': 'command4',
'proid': 'root',
'command': '/path/to/other/sbin/command',
'restart': None, # not monitored
'environ': {},
'config': None,
'downed': False,
'trace': False,
}
],
'vring': {
'cells': ['a', 'b']
},
}
)
base_dir = '/some/dir'
mock_service_dir = mock.create_autospec(supervisor.ScanDir)
treadmill.supervisor.create_scan_dir.return_value =\
mock_service_dir
native.create_supervision_tree(
self.tm_env,
base_dir,
os.path.join(base_dir, 'root'),
app,
'/test/cgroup/path'
)
treadmill.supervisor.create_scan_dir.assert_has_calls([
mock.call(
os.path.join(base_dir, 'sys'),
finish_timeout=6000,
wait_cgroups='/test/cgroup/path'
),
mock.call().write(),
mock.call(
os.path.join(base_dir, 'services'),
finish_timeout=5000,
),
mock.call().write(),
])
treadmill.supervisor.create_service.assert_has_calls([
# system services
mock.call(mock_service_dir,
name='command3',
app_run_script='/path/to/sbin/command',
userid='root',
environ_dir='/some/dir/env',
environ={},
environment='prod',
downed=True,
monitor_policy={
'limit': 5,
'interval': 60,
'tombstone': {
'uds': False,
'path': self.services_tombstone_dir,
'id': 'myproid.test-0-0000000ID1234,command3'
}
},
trace=None),
mock.call(mock_service_dir,
name='command4',
app_run_script='/path/to/other/sbin/command',
userid='root',
environ_dir='/some/dir/env',
environ={},
environment='prod',
downed=False,
monitor_policy=None,
trace=None),
# user services
mock.call(mock_service_dir,
name='command1',
app_run_script='/path/to/command',
userid='test',
environ_dir='/env',
environ={},
environment='prod',
downed=False,
monitor_policy={
'limit': 3,
'interval': 60,
'tombstone': {
'uds': True,
'path': '/run/tm_ctl/tombstone',
'id': 'myproid.test-0-0000000ID1234,command1'
}
},
log_run_script='s6.app-logger.run',
trace={
'instanceid': 'myproid.test#0',
'uniqueid': 'ID1234',
'service': 'command1',
'path': '/run/tm_ctl/appevents'
}),
mock.call(mock_service_dir,
name='command2',
app_run_script='/path/to/other/command',
userid='test',
environ_dir='/env',
environ={},
environment='prod',
downed=False,
monitor_policy={
'limit': 3,
'interval': 60,
'tombstone': {
'uds': True,
'path': '/run/tm_ctl/tombstone',
'id': 'myproid.test-0-0000000ID1234,command2'
}
},
log_run_script='s6.app-logger.run',
trace={
'instanceid': 'myproid.test#0',
'uniqueid': 'ID1234',
'service': 'command2',
'path': '/run/tm_ctl/appevents'
})
])
self.assertEqual(2, mock_service_dir.write.call_count)
treadmill.fs.linux.mount_bind.assert_has_calls([
mock.call('/some/dir/root', '/services',
source='/some/dir/services',
read_only=False, recursive=False),
mock.call('/some/dir/root', '/run/tm_ctl',
source=os.path.join(self.root, 'ctl'),
read_only=False, recursive=False),
])
treadmill.fs.rmtree_safe.assert_called_once_with(
'/some/dir/sys/command0'
)
def test__unshare_network_complex(self):
"""Test unshare network advanced sequence (ephemeral/passthrough)."""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'name':
'myproid.test#0',
'environment':
'dev',
'uniqueid':
'ID1234',
'network': {
'veth': 'id1234.0',
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'external_ip': '172.31.81.67',
},
'shared_ip':
False,
'endpoints': [{
'name': 'ssh',
'port': 54321,
'real_port': 54321,
'type': 'infra',
'proto': 'tcp',
}, {
'name': 'test2',
'port': 54322,
'real_port': 54322,
'proto': 'udp',
}],
'ephemeral_ports': {
'tcp': [10000, 10001, 10002],
'udp': [],
},
'passthrough': [
'xxx',
'yyy',
'zzz',
],
'vring': {
'some': 'data'
}
})
app_unique_name = appcfg.app_unique_name(app)
hosts_to_ip = {
'xxx': '4.4.4.4',
'yyy': '5.5.5.5',
'zzz': '5.5.5.5',
}
socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h]
self.tm_env.rules.get_rules.return_value = set()
treadmill.runtime.linux._run._unshare_network(self.tm_env,
'test_container_dir',
app)
self.tm_env.rules.create_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=54322,
new_ip='192.168.0.2',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='udp',
src_ip='192.168.0.2',
src_port=54322,
new_ip='172.31.81.67',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10000,
new_ip='192.168.0.2',
new_port=10000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10001,
new_ip='192.168.0.2',
new_port=10001),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10002,
new_ip='192.168.0.2',
new_port=10002),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('4.4.4.4', '192.168.0.2'),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('5.5.5.5', '192.168.0.2'),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.create_rule.call_count, 9)
# Check that infra services + ephemeral ports are in the same set.
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10000'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10001'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10002'),
],
any_order=True)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.0.2',
'192.168.254.254',
None,
)
def test_run(self):
"""Tests linux.run sequence, which will result in supervisor exec.
"""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
manifest = {
'type':
'native',
'shared_network':
False,
'ephemeral_ports': {
'tcp': 3,
'udp': 0,
},
'passthrough': ['xxx', 'yyy', 'zzz'],
'memory':
'100M',
'uniqueid':
'ID1234',
'services': [{
'name': 'web_server',
'command': '/bin/true',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'disk':
'100G',
'tickets':
True,
'name':
'proid.myapp#0',
'system_services': [],
'environment':
'dev',
'proid':
'foo',
'endpoints': [{
'name': 'http',
'port': 8000
}, {
'name': 'port0',
'port': 0
}, {
'type': 'infra',
'name': 'ssh',
'port': 0
}],
'cpu':
'100%'
}
app_unique_name = 'proid.myapp-0-0000000ID1234'
app_dir = os.path.join(self.root, 'apps', app_unique_name)
os.makedirs(app_dir)
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
cgroups = {
'cpu': '/some/path',
'cpuacct': '/some/other/path',
'memory': '/mem/path',
'blkio': '/io/path',
}
mock_cgroup_client.wait.return_value = cgroups
network = {
'vip': '2.2.2.2',
'gateway': '1.1.1.1',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.wait.return_value = network
def _fake_allocate_network_ports(_ip, manifest):
"""Mimick inplace manifest modification in allocate_network_ports.
"""
manifest['ephemeral_ports'] = {'tcp': ['1', '2', '3']}
return mock.DEFAULT
treadmill.runtime.allocate_network_ports.side_effect = \
_fake_allocate_network_ports
mock_image = mock.Mock()
treadmill.runtime.linux.image.get_image_repo.return_value = mock_image
mock_runtime_config = mock.Mock()
mock_runtime_config.host_mount_whitelist = []
app_run.run(self.tm_env, mock_runtime_config, app_dir, manifest)
mock_cgroup_client.put.assert_called_with(app_unique_name, {
'memory': '100M',
'cpu': '100%',
})
mock_ld_client.put.assert_called_with(app_unique_name, {
'size': '100G',
})
mock_nwrk_client.put.assert_called_with(app_unique_name, {
'environment': 'dev',
})
mock_cgroup_client.wait.assert_called_with(app_unique_name)
mock_ld_client.wait.assert_called_with(app_unique_name)
mock_nwrk_client.wait.assert_called_with(app_unique_name)
# Check that port allocation is correctly called.
manifest['network'] = network
manifest['ephemeral_ports'] = {'tcp': ['1', '2', '3']}
treadmill.runtime.allocate_network_ports\
.assert_called_with(
'172.31.81.67', manifest,
)
# Make sure, post modification, that the manifest is readable by other.
st = os.stat(os.path.join(app_dir, 'state.json'))
self.assertTrue(st.st_mode & stat.S_IRUSR)
self.assertTrue(st.st_mode & stat.S_IRGRP)
self.assertTrue(st.st_mode & stat.S_IROTH)
self.assertTrue(st.st_mode & stat.S_IWUSR)
self.assertFalse(st.st_mode & stat.S_IWGRP)
self.assertFalse(st.st_mode & stat.S_IWOTH)
app = utils.to_obj(manifest)
treadmill.runtime.linux._run._unshare_network.assert_called_with(
self.tm_env, app_dir, app)
# Create root dir
treadmill.runtime.linux._run._create_root_dir.assert_called_with(
app_dir, mock_ld_client.wait.return_value)
def test_sysdir_cleanslate(self):
"""Verifies that sys directories are always clean slate."""
# Disable access to protected member warning.
#
# pylint: disable=W0212
base_dir = os.path.join(self.root, 'some/dir')
events_dir = os.path.join(base_dir, 'appevents')
fs.mkdir_safe(base_dir)
app = utils.to_obj(
{
'proid': 'myproid',
'name': 'myproid.test#0',
'uniqueid': 'ID1234',
'environment': 'prod',
'services': [
{
'name': 'command1',
'command': '/path/to/command',
'restart_count': 3,
}, {
'name': 'command2',
'command': '/path/to/other/command',
'restart_count': -1,
}
],
'system_services': [
{
'name': 'command3',
'command': '/path/to/sbin/command',
'restart_count': 1,
}, {
'name': 'command4',
'command': '/path/to/other/sbin/command',
'restart_count': -1,
}
],
'vring': {
'cells': [],
},
}
)
treadmill.appmgr.run._create_supervision_tree(
base_dir,
events_dir,
app,
)
self.assertTrue(os.path.exists(os.path.join(base_dir, 'sys')))
with open(os.path.join(base_dir, 'sys', 'toberemoved'), 'w+') as _f:
pass
self.assertTrue(
os.path.exists(os.path.join(base_dir, 'sys', 'toberemoved')))
treadmill.appmgr.run._create_supervision_tree(
base_dir,
events_dir,
app,
)
self.assertTrue(os.path.exists(os.path.join(base_dir, 'sys')))
self.assertFalse(
os.path.exists(os.path.join(base_dir, 'sys', 'toberemoved')))
def configure(tm_env, event, runtime):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)/
- apps/
- (app unique name)/
- data/
- app_start
- app.json
- manifest.yml
env/
- TREADMILL_*
run
finish
log/
- run
The 'run' script is responsible for creating container environment
and starting the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# Load the app from the event
try:
manifest_data = load_runtime_manifest(tm_env, event, runtime)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception('No event to load: %r', event)
return None
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Write the actual container start script
if os.name == 'nt':
run_script = '{treadmill}/scripts/treadmill sproc run .'.format(
treadmill=subproc.resolve('treadmill'),
)
else:
run_script = 'exec {treadmill}/bin/treadmill sproc run ../'.format(
treadmill=subproc.resolve('treadmill'),
)
# Create the service for that container
container_svc = supervisor.create_service(
tm_env.apps_dir,
name=uniq_name,
app_run_script=run_script,
userid='root',
downed=False,
monitor_policy={
'limit': 0,
'interval': 60,
'tombstone': {
'uds': False,
'path': tm_env.running_tombstone_dir,
'id': app.name
}
},
environ={},
environment=app.environment
)
data_dir = container_svc.data_dir
# Copy the original event as 'manifest.yml' in the container dir
try:
shutil.copyfile(
event,
os.path.join(data_dir, 'manifest.yml')
)
except IOError as err:
# File is gone, cleanup.
if err.errno == errno.ENOENT:
shutil.rmtree(container_svc.directory)
_LOGGER.exception('Event gone: %r', event)
return None
else:
raise
# Store the app.json in the container directory
fs.write_safe(
os.path.join(data_dir, appcfg.APP_JSON),
lambda f: f.writelines(
utils.json_genencode(manifest_data)
),
mode='w',
permission=0o644
)
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(
instanceid=app.name,
uniqueid=app.uniqueid
)
)
return container_svc.directory
def configure(tm_env, event):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)
- apps
- (app unique name)
- app.yml
run
finish
log/run
The 'run' script is responsible for creating container environment
and starting svscan inside the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# R0915: Need to refactor long function into smaller pieces.
#
# pylint: disable=R0915
# Load the app from the event
try:
manifest_data = app_manifest.load(tm_env, event)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception("No event to load: %r", event)
return
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Check the identity we are going to run as
_check_identity(app.proid)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Create the app's running directory
container_dir = os.path.join(tm_env.apps_dir, uniq_name)
if not fs.mkdir_safe(container_dir):
_LOGGER.info('Resuming container %r', uniq_name)
# Copy the event as 'manifest.yml' in the container dir
shutil.copyfile(event, os.path.join(container_dir, 'manifest.yml'))
# Store the app int the container_dir
app_yml = os.path.join(container_dir, _APP_YML)
with open(app_yml, 'w') as f:
yaml.dump(manifest_data, stream=f)
# Mark the container as defaulting to down state
utils.touch(os.path.join(container_dir, 'down'))
# Generate the supervisor's run script
app_run_cmd = ' '.join(
[treadmill.TREADMILL_BIN, 'sproc', 'run', container_dir])
utils.create_script(os.path.join(container_dir, 'run'),
'supervisor.run_no_log',
cmd=app_run_cmd)
fs.mkdir_safe(os.path.join(container_dir, 'log'))
utils.create_script(os.path.join(container_dir, 'log', 'run'),
'logger.run')
# Unique name for the link, based on creation time.
cleanup_link = os.path.join(tm_env.cleanup_dir, uniq_name)
if os.name == 'nt':
finish_cmd = '%%COMSPEC%% /C mklink /D %s %s' % \
(cleanup_link, container_dir)
else:
finish_cmd = '/bin/ln -snvf %s %s' % (container_dir, cleanup_link)
utils.create_script(os.path.join(container_dir, 'finish'),
'supervisor.finish',
service=app.name,
proid=None,
cmds=[finish_cmd])
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(instanceid=app.name,
uniqueid=app.uniqueid))
return container_dir
def setUp(self):
# Access protected module _base_service
# pylint: disable=W0212
self.root = tempfile.mkdtemp()
self.container_dir = os.path.join(self.root, 'apps', 'test')
self.data_dir = os.path.join(self.container_dir, 'data')
fs.mkdir_safe(self.data_dir)
with io.open(os.path.join(self.container_dir, 'type'), 'w') as f:
f.write('longrun')
self.events_dir = os.path.join(self.root, 'appevents')
os.mkdir(self.events_dir)
self.configs_dir = os.path.join(self.root, 'configs')
os.mkdir(self.configs_dir)
self.tm_env = mock.Mock(
root=self.root,
app_events_dir=self.events_dir,
configs_dir=self.configs_dir,
rules=mock.Mock(spec_set=rulefile.RuleMgr, ),
)
self.manifest = {
'cpu':
50,
'disk':
'20G',
'memory':
'512M',
'cell':
'test',
'app':
'proid.app',
'task':
'001',
'name':
'proid.app#001',
'uniqueid':
'abcdefghijklm',
'identity':
None,
'identity_group':
None,
'proid':
'proid',
'environment':
'dev',
'endpoints': [
{
'name': 'ep1',
'port': '80',
'real_port': '12345',
'proto': 'tcp'
},
],
'ephemeral_ports': {
'tcp': [54321],
'udp': []
},
'environ': [
{
'name': 'key1',
'value': 'value1'
},
{
'name': 'TREADMILL_MEMORY',
'value': 'should not be here'
},
],
'image':
'docker://test',
'command':
'cmd',
'args': []
}
self.app = utils.to_obj(self.manifest)
def configure(tm_env, event, runtime):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)/
- apps/
- (app unique name)/
- data/
- app_start
- app.json
- manifest.yml
- policy.json
env/
- TREADMILL_*
run
finish
log/
- run
The 'run' script is responsible for creating container environment
and starting the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# Load the app from the event
try:
manifest_data = app_manifest.load(tm_env, event, runtime)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception('No event to load: %r', event)
return
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Write the actual container start script
if os.name == 'nt':
run_script = ' '.join(
[sys.executable, '-m', 'treadmill.ms', 'sproc', 'run', '.'])
else:
run_script = ' '.join(
['exec', dist.TREADMILL_BIN, 'sproc', 'run', '../'])
# Create the service for that container
container_svc = supervisor.create_service(tm_env.apps_dir,
name=uniq_name,
app_run_script=run_script,
userid='root',
downed=False,
monitor_policy={
'limit': 0,
'interval': 60
},
environ={},
environment=app.environment)
data_dir = container_svc.data_dir
# Copy the original event as 'manifest.yml' in the container dir
shutil.copyfile(event, os.path.join(data_dir, 'manifest.yml'))
# Store the app.json in the container directory
fs.write_safe(os.path.join(data_dir, appcfg.APP_JSON),
lambda f: f.writelines(utils.json_genencode(manifest_data)),
mode='w',
permission=0o644)
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(instanceid=app.name,
uniqueid=app.uniqueid))
return container_svc.directory
