def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
# Access protected module _kill_apps_by_root
# pylint: disable=W0212
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'host_ip':
'172.31.81.67',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/tmp/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
}
treadmill.appmgr.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.app_env.svc_cgroup.make_client.return_value
mock_ld_client = self.app_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.app_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.app_env.apps_dir, app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'exitinfo'), 'w') as f:
f.write(yaml.dump({'service': 'web_server', 'rc': 0, 'sig': 0}))
mock_zkclient = kazoo.client.KazooClient()
app_finish.finish(self.app_env, mock_zkclient, app_dir)
self.app_env.watchdogs.create.assert_called_with(
'treadmill.appmgr.finish-' + app_unique_name, '5m', mock.ANY)
treadmill.subproc.check_call.assert_has_calls([
mock.call([
's6-svc',
'-d',
app_dir,
]),
mock.call([
's6-svwait',
'-d',
app_dir,
]),
])
# All resource service clients are properly created
self.app_env.svc_cgroup.make_client.assert_called_with(
os.path.join(app_dir, 'cgroups'))
self.app_env.svc_localdisk.make_client.assert_called_with(
os.path.join(app_dir, 'localdisk'))
self.app_env.svc_network.make_client.assert_called_with(
os.path.join(app_dir, 'network'))
treadmill.appmgr.finish._kill_apps_by_root.assert_called_with(
os.path.join(app_dir, 'root'))
# Verify that we tested the archiving for the app root volume
treadmill.fs.archive_filesystem.assert_called_with(
'/dev/foo', os.path.join(app_dir, 'root'),
os.path.join(app_dir, '001_xxx.xx.com_20150122_141436537918.tar'),
mock.ANY)
# Verify that the file is uploaded by Uploader
app = utils.to_obj(manifest)
treadmill.appmgr.finish._send_container_archive.assert_called_with(
mock_zkclient,
app,
os.path.join(app_dir,
'001_xxx.xx.com_20150122_141436537918.tar.gz'),
)
# Verify that the app folder was deleted
self.assertFalse(os.path.exists(app_dir))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.app_env.rules.unlink_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 5000,
'192.168.0.2', 8000),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 54321,
'192.168.0.2', 54321),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 45024,
'192.168.0.2', 45024),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('udp', '172.31.81.67', 62422,
'192.168.0.2', 62422),
owner=app_unique_name),
])
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
])
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'sig': 0,
'rc': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd'))
def test_finish_no_resources(self):
"""Test app finish on directory when all resources are already freed.
"""
# Access protected module _kill_apps_by_root
# pylint: disable=W0212
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'host_ip':
'172.31.81.67',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/tmp/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'services': [{
'command': '/bin/false',
'restart_count': 3,
'name': 'web_server'
}],
}
treadmill.appmgr.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.app_env.svc_cgroup.make_client.return_value
mock_ld_client = self.app_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.app_env.svc_network.make_client.return_value
# All resource managers return None
mock_cgroup_client.get.return_value = None
mock_ld_client.get.return_value = None
mock_nwrk_client.get.return_value = None
app_dir = os.path.join(self.app_env.apps_dir, app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'exitinfo'), 'w') as f:
f.write(yaml.dump({'service': 'web_server', 'rc': 0, 'sig': 0}))
mock_zkclient = kazoo.client.KazooClient()
app_finish.finish(self.app_env, mock_zkclient, app_dir)
self.app_env.watchdogs.create.assert_called_with(
'treadmill.appmgr.finish-' + app_unique_name, '5m', mock.ANY)
treadmill.subproc.check_call.assert_has_calls([
mock.call([
's6-svc',
'-d',
app_dir,
], ),
mock.call([
's6-svwait',
'-d',
app_dir,
], ),
])
self.app_env.svc_cgroup.make_client.assert_called_with(
os.path.join(app_dir, 'cgroups'))
self.app_env.svc_localdisk.make_client.assert_called_with(
os.path.join(app_dir, 'localdisk'))
self.app_env.svc_network.make_client.assert_called_with(
os.path.join(app_dir, 'network'))
treadmill.appmgr.finish._kill_apps_by_root.assert_called_with(
os.path.join(app_dir, 'root'))
# Verify that the app folder was deleted
self.assertFalse(os.path.exists(app_dir))
# Cleanup the network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'sig': 0,
'rc': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd'))
def test_finish_no_manifest(self):
"""Test app finish on directory with no app.yml.
"""
app_finish.finish(self.app_env, None, self.root)
def test_finish_aborted(self):
"""Tests container finish procedure when node is aborted.
"""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'host_ip':
'172.31.81.67',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/tmp/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'ephemeral_ports': {
'tcp': [],
'udp': [],
}
}
treadmill.appmgr.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_ld_client = self.app_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
mock_nwrk_client = self.app_env.svc_network.make_client.return_value
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.root, 'apps', app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'aborted'), 'w') as aborted:
aborted.write('something went wrong')
mock_zkclient = kazoo.client.KazooClient()
app_finish.finish(self.app_env, mock_zkclient, app_dir)
treadmill.appevents.post(
mock.ANY,
events.AbortedTraceEvent(instanceid='proid.myapp#001',
why=None,
payload={
'why': 'something went wrong',
'node': 'hostname',
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd'))
def test_finish_no_manifest(self):
"""Test app finish on directory with no app.yml.
"""
app_env = appmgr.AppEnvironment(root=self.root)
app_finish.finish(app_env, None, self.root)
def test_finish_error(self):
"""Tests container finish procedure when app is improperly finished."""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'host_ip':
'172.31.81.67',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000001234',
'archive': ['/var/tmp/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000
}],
'services': [{
'command': '/bin/false',
'restart_count': 3,
'name': 'web_server'
}],
}
treadmill.appmgr.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000001234'
mock_ld_client = self.app_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
mock_nwrk_client = self.app_env.svc_network.make_client.return_value
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.app_env.apps_dir, app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'exitinfo'), 'w') as f:
f.write(yaml.dump({'service': 'web_server', 'rc': 1, 'sig': 3}))
kazoo.client.KazooClient.exists.return_value = True
kazoo.client.KazooClient.get_children.return_value = []
app_finish.finish(self.app_env, kazoo.client.KazooClient(), app_dir)
treadmill.appevents.post.assert_called_with(
mock.ANY, 'proid.myapp#001', 'finished', '1.3', {
'sig': 3,
'service': 'web_server',
'rc': 1
})
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.app_env.metrics_dir, 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd'))