def test_finish_oom_event(self):
"""Test posting oom event.
"""
app_unique_name = 'proid.myapp-001-0000000ID1234'
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
fs.mkdir_safe(data_dir)
utils.touch(os.path.join(data_dir, 'oom'))
app_finish.finish(self.tm_env, app_dir)
args, _kwargs = treadmill.appevents.post.call_args
_events_dir, event = args
self.assertIsInstance(event, events.KilledTraceEvent)
def test_terminated_no_event(self):
"""Test that event won't be posted if container is terminated/evicted.
"""
app_unique_name = 'proid.myapp-001-0000000ID1234'
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
fs.mkdir_safe(data_dir)
utils.touch(os.path.join(data_dir, 'terminated'))
# exitinfo file should be ignored.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.write('{"service": "web_server", "return_code": 0, "signal": 0}')
app_finish.finish(self.tm_env, app_dir)
treadmill.appevents.post.assert_not_called()
def test_finish_aborted_event(self):
"""Test posting aborted event.
"""
app_unique_name = 'proid.myapp-001-0000000ID1234'
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
fs.mkdir_safe(data_dir)
with io.open(os.path.join(data_dir, 'aborted'), 'w') as f:
f.write('{"why": "reason", "payload": "test"}')
app_finish.finish(self.tm_env, app_dir)
args, _kwargs = treadmill.appevents.post.call_args
_events_dir, event = args
self.assertIsInstance(event, events.AbortedTraceEvent)
def test_finish_exitinfo_event(self):
"""Test posting finished event.
"""
app_unique_name = 'proid.myapp-001-0000000ID1234'
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
fs.mkdir_safe(data_dir)
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.write('{"service": "web_server", "return_code": 0, "signal": 0}')
app_finish.finish(self.tm_env, app_dir)
args, _kwargs = treadmill.appevents.post.call_args
_events_dir, event = args
self.assertIsInstance(event, events.FinishedTraceEvent)
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
# Access protected module _kill_apps_by_root
# pylint: disable=W0212
manifest = {
'app': 'proid.myapp',
'cell': 'test',
'cpu': '100%',
'disk': '100G',
'environment': 'dev',
'memory': '100M',
'name': 'proid.myapp#001',
'proid': 'foo',
'shared_network': False,
'task': '001',
'uniqueid': '0000000ID1234',
'archive': [
'/var/tmp/treadmill'
],
'endpoints': [
{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
},
{
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}
],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'services': [
{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}
],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'exitinfo'), 'w') as f:
f.write(yaml.dump({'service': 'web_server', 'rc': 0, 'sig': 0}))
mock_zkclient = kazoo.client.KazooClient()
mock_watchdog = mock.Mock()
app_finish.finish(self.tm_env, mock_zkclient, app_dir, mock_watchdog)
treadmill.subproc.check_call.assert_has_calls(
[
mock.call(
[
's6_svc',
'-d',
app_dir,
]
),
mock.call(
[
's6_svwait',
'-d',
app_dir,
]
),
]
)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(app_dir, 'cgroups')
)
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(app_dir, 'localdisk')
)
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(app_dir, 'network')
)
treadmill.runtime.linux._finish._kill_apps_by_root.assert_called_with(
os.path.join(app_dir, 'root')
)
# Verify that we tested the archiving for the app root volume
treadmill.fs.archive_filesystem.assert_called_with(
'/dev/foo',
os.path.join(app_dir, 'root'),
os.path.join(app_dir,
'001_xxx.xx.com_20150122_141436537918.tar'),
mock.ANY
)
# Verify that the file is uploaded by Uploader
app = utils.to_obj(manifest)
treadmill.runtime.linux._finish._send_container_archive\
.assert_called_with(
mock_zkclient,
app,
os.path.join(app_dir,
'001_xxx.xx.com_20150122_141436537918.tar.gz'),
)
# Verify that the app folder was deleted
self.assertFalse(os.path.exists(app_dir))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.tm_env.rules.unlink_rule.assert_has_calls(
[
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(
proto='tcp',
dst_ip='172.31.81.67', dst_port=5000,
new_ip='192.168.0.2', new_port=8000
),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(
proto='tcp',
src_ip='192.168.0.2', src_port=8000,
new_ip='172.31.81.67', new_port=5000
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(
proto='tcp',
dst_ip='172.31.81.67', dst_port=54321,
new_ip='192.168.0.2', new_port=54321
),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(
proto='tcp',
src_ip='192.168.0.2', src_port=54321,
new_ip='172.31.81.67', new_port=54321
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(
proto='tcp',
dst_ip='172.31.81.67', dst_port=45024,
new_ip='192.168.0.2', new_port=45024
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(
proto='udp',
dst_ip='172.31.81.67', dst_port=62422,
new_ip='192.168.0.2', new_port=62422
),
owner=app_unique_name),
],
any_order=True
)
self.assertEqual(self.tm_env.rules.unlink_rule.call_count, 6)
treadmill.iptables.rm_ip_set.assert_has_calls(
[
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
mock.call(treadmill.iptables.SET_VRING_CONTAINERS,
'192.168.0.2'),
],
any_order=True
)
self.assertEqual(treadmill.iptables.rm_ip_set.call_count, 4)
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(
instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'sig': 0,
'rc': 0
}
)
)
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd')
)
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd')
)
self.assertTrue(mock_watchdog.remove.called)
def test_finish_no_manifest(self):
"""Test app finish on directory with no app.json.
"""
app_finish.finish(self.tm_env, None, self.root, mock.Mock())
def test_finish_aborted(self):
"""Tests container finish procedure when node is aborted.
"""
manifest = {
'app': 'proid.myapp',
'cell': 'test',
'cpu': '100%',
'disk': '100G',
'environment': 'dev',
'host_ip': '172.31.81.67',
'memory': '100M',
'name': 'proid.myapp#001',
'proid': 'foo',
'shared_network': False,
'task': '001',
'uniqueid': '0000000ID1234',
'archive': [
'/var/tmp/treadmill'
],
'endpoints': [
{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}
],
'services': [
{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}
],
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.root, 'apps', app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'aborted'), 'w') as aborted:
aborted.write('something went wrong')
mock_zkclient = kazoo.client.KazooClient()
mock_watchdog = mock.Mock()
app_finish.finish(
self.tm_env, mock_zkclient, app_dir, mock_watchdog
)
treadmill.appevents.post(
mock.ANY,
events.AbortedTraceEvent(
instanceid='proid.myapp#001',
why=None,
payload={
'why': 'something went wrong',
'node': 'hostname',
}
)
)
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd')
)
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd')
)
self.assertTrue(mock_watchdog.remove.called)
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'passthrough': [
'8.8.8.8',
'9.9.9.9',
],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 0,
'signal': 0
}, ))
app_finish.finish(self.tm_env, app_dir)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'cgroups'))
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'localdisk'))
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'network'))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.tm_env.rules.unlink_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5000,
new_ip='192.168.0.2',
new_port=8000),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=8000,
new_ip='172.31.81.67',
new_port=5000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=45024,
new_ip='192.168.0.2',
new_port=45024),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=62422,
new_ip='192.168.0.2',
new_port=62422),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='8.8.8.8',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='9.9.9.9',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.unlink_rule.call_count, 8)
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
],
any_order=True)
self.assertEqual(treadmill.iptables.rm_ip_set.call_count, 4)
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.iptables.flush_cnt_conntrack_table.assert_called_with(
'192.168.0.2')
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'signal': 0,
'return_code': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()
def test_finish_error(self):
"""Tests container finish procedure when app is improperly finished."""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000001234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000001234'
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 1,
'signal': 3
}, ))
app_finish.finish(self.tm_env, app_dir)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=1,
signal=3,
payload={
'service': 'web_server',
'signal': 3,
'return_code': 1,
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.tm_env.metrics_dir, 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()
def test_finish_no_manifest(self):
"""Test app finish on directory with no app.yml.
"""
app_finish.finish(self.tm_env, None, self.root)
