def _post_exit_event(tm_env, appname, exitinfo): """Post finished event based on exit info.""" trace.post( tm_env.app_events_dir, events.FinishedTraceEvent(instanceid=appname, rc=exitinfo.get('return_code', 256), signal=exitinfo.get('signal', 256), payload=exitinfo))
def test__finish(self): """Tests docker runtime finish.""" # Access to a protected member # pylint: disable=W0212 if os.name == 'nt': treadmill.ad.credential_spec.cleanup = mock.Mock() treadmill.runtime.save_app(self.manifest, self.data_dir) client = mock.MagicMock() docker.from_env.return_value = client container = mock.MagicMock() client.containers.get.return_value = container type(container).attrs = mock.PropertyMock(return_value={ 'State': { 'OOMKilled': False, 'ExitCode': 5 } }) docker_runtime = runtime.DockerRuntime( self.tm_env, self.container_dir, {'version': '1.24'} ) docker_runtime._finish() container.remove.assert_called_with(force=True) treadmill.trace.post.assert_called_with( self.tm_env.app_events_dir, events.FinishedTraceEvent( instanceid='proid.app#001', rc=5, signal=0, payload=mock.ANY ) ) treadmill.trace.post.reset_mock() type(container).attrs = mock.PropertyMock(return_value={ 'State': { 'OOMKilled': True } }) docker_runtime._finish() treadmill.trace.post.assert_called_with( self.tm_env.app_events_dir, events.KilledTraceEvent( instanceid='proid.app#001', is_oom=True, ) ) treadmill.runtime.archive_logs.assert_called()
def _finish(self): app = runtime.load_app(self._service.data_dir, runtime.STATE_JSON) if app: client = self._get_client() container = state = None name = appcfg.app_unique_name(app) try: container = client.containers.get(name) state = container.attrs.get('State') except docker.errors.NotFound: pass if container is not None: try: container.remove(force=True) except docker.errors.APIError: _LOGGER.error('Failed to remove %s', container.id) aborted = _check_aborted(self._service.data_dir) if aborted is not None: app_abort.report_aborted(self._tm_env, app.name, why=aborted.get('why'), payload=aborted.get('payload')) elif state is not None: if state.get('OOMKilled', False): event = events.KilledTraceEvent( instanceid=app.name, is_oom=True, ) else: event = events.FinishedTraceEvent(instanceid=app.name, rc=state.get( 'ExitCode', 256), signal=0, payload=state) trace.post(self._tm_env.app_events_dir, event) if os.name == 'nt': credential_spec.cleanup(name, client) try: runtime.archive_logs(self._tm_env, name, self._service.data_dir) except Exception: # pylint: disable=W0703 _LOGGER.exception('Unexpected exception storing local logs.')
def test_finished(self, stdout_mock): """Test printing Finished event. """ event = events.FinishedTraceEvent(timestamp=1, source='tests', instanceid='proid.foo#123', rc=1, signal=2, payload={'foo': 'bar'}) self.trace_printer.process(event) self.assertEqual( stdout_mock.getvalue(), 'Thu, 01 Jan 1970 00:00:01+0000 - ' 'proid.foo#123 finished on tests\n')
def test_finished(self): """Finished event operations. """ event = events.FinishedTraceEvent( timestamp=1, source='tests', instanceid='proid.foo#123', rc=1, signal=2, payload={'foo': 'bar'} ) self.assertEqual( event.to_dict(), { 'event_type': 'finished', 'timestamp': 1, 'source': 'tests', 'instanceid': 'proid.foo#123', 'rc': 1, 'signal': 2, 'payload': {'foo': 'bar'}, } ) self.assertEqual( event.to_data(), ( 1, 'tests', 'proid.foo#123', 'finished', '1.2', {'foo': 'bar'}, ) ) self.assertEqual( event, events.FinishedTraceEvent.from_data( timestamp=1, source='tests', instanceid='proid.foo#123', event_type='finished', event_data='1.2', payload={'foo': 'bar'} ) )
def test_finish(self): """Tests container finish procedure and freeing of the resources. """ manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000ID1234', 'archive': ['/var/lib/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000, 'proto': 'tcp', }, { 'port': 54321, 'type': 'infra', 'name': 'ssh', 'real_port': 54321, 'proto': 'tcp', }], 'ephemeral_ports': { 'tcp': [45024], 'udp': [62422], }, 'passthrough': [ '8.8.8.8', '9.9.9.9', ], 'services': [{ 'name': 'web_server', 'command': '/bin/false', 'restart': { 'limit': 3, 'interval': 60, }, }], 'vring': { 'some': 'settings' } } treadmill.appcfg.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000ID1234' mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value mock_nwrk_client = self.tm_env.svc_network.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.get.return_value = localdisk network = { 'vip': '192.168.0.2', 'gateway': '192.168.254.254', 'veth': 'testveth.0', 'external_ip': '172.31.81.67', } mock_nwrk_client.get.return_value = network app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name) data_dir = os.path.join(app_dir, 'data') # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(data_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f: f.writelines( utils.json_genencode( { 'service': 'web_server', 'return_code': 0, 'signal': 0 }, )) app_finish.finish(self.tm_env, app_dir) # All resource service clients are properly created self.tm_env.svc_cgroup.make_client.assert_called_with( os.path.join(data_dir, 'resources', 'cgroups')) self.tm_env.svc_localdisk.make_client.assert_called_with( os.path.join(data_dir, 'resources', 'localdisk')) self.tm_env.svc_network.make_client.assert_called_with( os.path.join(data_dir, 'resources', 'network')) # Cleanup the block device mock_ld_client.delete.assert_called_with(app_unique_name) # Cleanup the cgroup resource mock_cgroup_client.delete.assert_called_with(app_unique_name) # Cleanup network resources mock_nwrk_client.get.assert_called_with(app_unique_name) self.tm_env.rules.unlink_rule.assert_has_calls([ mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port=5000, new_ip='192.168.0.2', new_port=8000), owner=app_unique_name), mock.call(chain=iptables.POSTROUTING_SNAT, rule=firewall.SNATRule(proto='tcp', src_ip='192.168.0.2', src_port=8000, new_ip='172.31.81.67', new_port=5000), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port=54321, new_ip='192.168.0.2', new_port=54321), owner=app_unique_name), mock.call(chain=iptables.POSTROUTING_SNAT, rule=firewall.SNATRule(proto='tcp', src_ip='192.168.0.2', src_port=54321, new_ip='172.31.81.67', new_port=54321), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port=45024, new_ip='192.168.0.2', new_port=45024), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='udp', dst_ip='172.31.81.67', dst_port=62422, new_ip='192.168.0.2', new_port=62422), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_PASSTHROUGH, rule=firewall.PassThroughRule( src_ip='8.8.8.8', dst_ip='192.168.0.2', ), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_PASSTHROUGH, rule=firewall.PassThroughRule( src_ip='9.9.9.9', dst_ip='192.168.0.2', ), owner=app_unique_name), ], any_order=True) self.assertEqual(self.tm_env.rules.unlink_rule.call_count, 8) treadmill.iptables.rm_ip_set.assert_has_calls([ mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:54321'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:45024'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,udp:62422'), mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'), ], any_order=True) self.assertEqual(treadmill.iptables.rm_ip_set.call_count, 4) mock_nwrk_client.delete.assert_called_with(app_unique_name) treadmill.iptables.flush_cnt_conntrack_table.assert_called_with( '192.168.0.2') treadmill.trace.post.assert_called_with( mock.ANY, events.FinishedTraceEvent(instanceid='proid.myapp#001', rc=0, signal=0, payload={ 'service': 'web_server', 'signal': 0, 'return_code': 0 })) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd'), os.path.join(data_dir, 'metrics.rrd')) treadmill.runtime.archive_logs.assert_called()
def test_finish_no_resources(self): """Test app finish on directory when all resources are already freed. """ # Access to a protected member _finish of a client class # pylint: disable=W0212 manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000ID1234', 'archive': ['/var/lib/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000 }, { 'port': 54321, 'type': 'infra', 'name': 'ssh', 'real_port': 54321 }], 'ephemeral_ports': { 'tcp': [45024], 'udp': [62422], }, 'services': [{ 'command': '/bin/false', 'restart_count': 3, 'name': 'web_server' }], 'vring': { 'some': 'settings' } } treadmill.appcfg.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000ID1234' mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value mock_nwrk_client = self.tm_env.svc_network.make_client.return_value # All resource managers return None mock_cgroup_client.get.return_value = None mock_ld_client.get.return_value = None mock_nwrk_client.get.return_value = None app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name) data_dir = os.path.join(app_dir, 'data') # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(data_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f: f.writelines( utils.json_genencode( { 'service': 'web_server', 'return_code': 0, 'signal': 0 }, )) treadmill.runtime.linux._finish.finish(self.tm_env, app_dir) # All resource service clients are properly created self.tm_env.svc_cgroup.make_client.assert_called_with( os.path.join(data_dir, 'resources', 'cgroups')) self.tm_env.svc_localdisk.make_client.assert_called_with( os.path.join(data_dir, 'resources', 'localdisk')) self.tm_env.svc_network.make_client.assert_called_with( os.path.join(data_dir, 'resources', 'network')) # Cleanup the network resources mock_nwrk_client.get.assert_called_with(app_unique_name) # Cleanup the block device mock_ld_client.delete.assert_called_with(app_unique_name) # Cleanup the cgroup resource mock_cgroup_client.delete.assert_called_with(app_unique_name) treadmill.trace.post.assert_called_with( mock.ANY, events.FinishedTraceEvent(instanceid='proid.myapp#001', rc=0, signal=0, payload={ 'service': 'web_server', 'signal': 0, 'return_code': 0 })) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd'), os.path.join(data_dir, 'metrics.rrd')) treadmill.runtime.archive_logs.assert_called()
def test_finish_error(self): """Tests container finish procedure when app is improperly finished.""" manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000001234', 'archive': ['/var/lib/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000, 'proto': 'tcp', }], 'services': [{ 'name': 'web_server', 'command': '/bin/false', 'restart': { 'limit': 3, 'interval': 60, }, }], 'ephemeral_ports': { 'tcp': [], 'udp': [], }, 'vring': { 'some': 'settings' } } treadmill.appcfg.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000001234' mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.get.return_value = localdisk mock_nwrk_client = self.tm_env.svc_network.make_client.return_value network = { 'vip': '192.168.0.2', 'gateway': '192.168.254.254', 'veth': 'testveth.0', 'external_ip': '172.31.81.67', } mock_nwrk_client.get.return_value = network app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name) data_dir = os.path.join(app_dir, 'data') # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(data_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f: f.writelines( utils.json_genencode( { 'service': 'web_server', 'return_code': 1, 'signal': 3 }, )) app_finish.finish(self.tm_env, app_dir) treadmill.trace.post.assert_called_with( mock.ANY, events.FinishedTraceEvent(instanceid='proid.myapp#001', rc=1, signal=3, payload={ 'service': 'web_server', 'signal': 3, 'return_code': 1, })) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.tm_env.metrics_dir, 'apps', app_unique_name + '.rrd'), os.path.join(data_dir, 'metrics.rrd')) treadmill.runtime.archive_logs.assert_called()