def _post_exit_event(tm_env, appname, exitinfo):
"""Post finished event based on exit info."""
trace.post(
tm_env.app_events_dir,
events.FinishedTraceEvent(instanceid=appname,
rc=exitinfo.get('return_code', 256),
signal=exitinfo.get('signal', 256),
payload=exitinfo))
def test__finish(self):
"""Tests docker runtime finish."""
# Access to a protected member
# pylint: disable=W0212
if os.name == 'nt':
treadmill.ad.credential_spec.cleanup = mock.Mock()
treadmill.runtime.save_app(self.manifest, self.data_dir)
client = mock.MagicMock()
docker.from_env.return_value = client
container = mock.MagicMock()
client.containers.get.return_value = container
type(container).attrs = mock.PropertyMock(return_value={
'State': {
'OOMKilled': False,
'ExitCode': 5
}
})
docker_runtime = runtime.DockerRuntime(
self.tm_env, self.container_dir, {'version': '1.24'}
)
docker_runtime._finish()
container.remove.assert_called_with(force=True)
treadmill.trace.post.assert_called_with(
self.tm_env.app_events_dir,
events.FinishedTraceEvent(
instanceid='proid.app#001',
rc=5,
signal=0,
payload=mock.ANY
)
)
treadmill.trace.post.reset_mock()
type(container).attrs = mock.PropertyMock(return_value={
'State': {
'OOMKilled': True
}
})
docker_runtime._finish()
treadmill.trace.post.assert_called_with(
self.tm_env.app_events_dir,
events.KilledTraceEvent(
instanceid='proid.app#001',
is_oom=True,
)
)
treadmill.runtime.archive_logs.assert_called()
def _finish(self):
app = runtime.load_app(self._service.data_dir, runtime.STATE_JSON)
if app:
client = self._get_client()
container = state = None
name = appcfg.app_unique_name(app)
try:
container = client.containers.get(name)
state = container.attrs.get('State')
except docker.errors.NotFound:
pass
if container is not None:
try:
container.remove(force=True)
except docker.errors.APIError:
_LOGGER.error('Failed to remove %s', container.id)
aborted = _check_aborted(self._service.data_dir)
if aborted is not None:
app_abort.report_aborted(self._tm_env,
app.name,
why=aborted.get('why'),
payload=aborted.get('payload'))
elif state is not None:
if state.get('OOMKilled', False):
event = events.KilledTraceEvent(
instanceid=app.name,
is_oom=True,
)
else:
event = events.FinishedTraceEvent(instanceid=app.name,
rc=state.get(
'ExitCode', 256),
signal=0,
payload=state)
trace.post(self._tm_env.app_events_dir, event)
if os.name == 'nt':
credential_spec.cleanup(name, client)
try:
runtime.archive_logs(self._tm_env, name,
self._service.data_dir)
except Exception: # pylint: disable=W0703
_LOGGER.exception('Unexpected exception storing local logs.')
def test_finished(self, stdout_mock):
"""Test printing Finished event.
"""
event = events.FinishedTraceEvent(timestamp=1,
source='tests',
instanceid='proid.foo#123',
rc=1,
signal=2,
payload={'foo': 'bar'})
self.trace_printer.process(event)
self.assertEqual(
stdout_mock.getvalue(), 'Thu, 01 Jan 1970 00:00:01+0000 - '
'proid.foo#123 finished on tests\n')
def test_finished(self):
"""Finished event operations.
"""
event = events.FinishedTraceEvent(
timestamp=1,
source='tests',
instanceid='proid.foo#123',
rc=1,
signal=2,
payload={'foo': 'bar'}
)
self.assertEqual(
event.to_dict(),
{
'event_type': 'finished',
'timestamp': 1,
'source': 'tests',
'instanceid': 'proid.foo#123',
'rc': 1,
'signal': 2,
'payload': {'foo': 'bar'},
}
)
self.assertEqual(
event.to_data(),
(
1,
'tests',
'proid.foo#123',
'finished',
'1.2',
{'foo': 'bar'},
)
)
self.assertEqual(
event,
events.FinishedTraceEvent.from_data(
timestamp=1,
source='tests',
instanceid='proid.foo#123',
event_type='finished',
event_data='1.2',
payload={'foo': 'bar'}
)
)
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'passthrough': [
'8.8.8.8',
'9.9.9.9',
],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 0,
'signal': 0
}, ))
app_finish.finish(self.tm_env, app_dir)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'cgroups'))
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'localdisk'))
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'network'))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.tm_env.rules.unlink_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5000,
new_ip='192.168.0.2',
new_port=8000),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=8000,
new_ip='172.31.81.67',
new_port=5000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=45024,
new_ip='192.168.0.2',
new_port=45024),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=62422,
new_ip='192.168.0.2',
new_port=62422),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='8.8.8.8',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='9.9.9.9',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.unlink_rule.call_count, 8)
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
],
any_order=True)
self.assertEqual(treadmill.iptables.rm_ip_set.call_count, 4)
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.iptables.flush_cnt_conntrack_table.assert_called_with(
'192.168.0.2')
treadmill.trace.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'signal': 0,
'return_code': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()
def test_finish_no_resources(self):
"""Test app finish on directory when all resources are already freed.
"""
# Access to a protected member _finish of a client class
# pylint: disable=W0212
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'services': [{
'command': '/bin/false',
'restart_count': 3,
'name': 'web_server'
}],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
# All resource managers return None
mock_cgroup_client.get.return_value = None
mock_ld_client.get.return_value = None
mock_nwrk_client.get.return_value = None
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 0,
'signal': 0
}, ))
treadmill.runtime.linux._finish.finish(self.tm_env, app_dir)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'cgroups'))
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'localdisk'))
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'network'))
# Cleanup the network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
treadmill.trace.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'signal': 0,
'return_code': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()
def test_finish_error(self):
"""Tests container finish procedure when app is improperly finished."""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000001234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000001234'
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 1,
'signal': 3
}, ))
app_finish.finish(self.tm_env, app_dir)
treadmill.trace.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=1,
signal=3,
payload={
'service': 'web_server',
'signal': 3,
'return_code': 1,
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.tm_env.metrics_dir, 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()