def start_container(container_root, manifest): """Treadmill container boot process. """ _LOGGER.info('Initializing container: %s', container_root) app = app_manifest.read(manifest) try: pivot_root.make_root(container_root) os.chdir('/') except Exception as err: # pylint: disable=broad-except event = traceevents.AbortedTraceEvent( instanceid=app['name'], why=app_abort.AbortedReason.PIVOT_ROOT.value, payload=str(err), ) _abort(event, container_root) # reraise err to exit start_container raise err # XXX: Debug info _LOGGER.debug('Current mounts: %s', pprint.pformat(fs_linux.list_mounts())) # Clean the environ # TODO: Remove me once clean environment management is merged in. os.environ.pop('PYTHONPATH', None) os.environ.pop('LC_ALL', None) os.environ.pop('LANG', None) # Clear aliases path. os.environ.pop('TREADMILL_ALIASES_PATH', None) subproc.safe_exec(['s6_svscan', '-s', '/services'])
def start_container(ctx, container_root, manifest): """Treadmill container boot process. """ _LOGGER.info('Initializing container: %s', container_root) app = app_manifest.read(manifest) cgroup = ctx.obj.get('CGROUP') try: # if cgroups set, we need to remount cgroup path # so that from cgroup directory we only see container pids # <container_root>/sys/fs/cgroup/memory => # /sys/fs/cgroup/memory/treadmill/apps/<app-inst-unique>/services if cgroup: remount_cgroup(container_root, cgroup, ctx.obj['ROOT_CGROUP']) pivot_root.make_root(container_root) os.chdir('/') except Exception as err: # pylint: disable=broad-except event = traceevents.AbortedTraceEvent( instanceid=app['name'], why=app_abort.AbortedReason.PIVOT_ROOT.value, payload=str(err), ) _abort(event, container_root) # reraise err to exit start_container raise err # XXX: Debug info _LOGGER.debug('Current mounts: %s', pprint.pformat(fs_linux.list_mounts())) subproc.safe_exec(['/services/{}'.format(supervisor.SVC_INIT_FILE)])
def _abort_task(self, appname, exception): """Set task into aborted state in case of scheduling error.""" if self.app_events_dir: trace.post( self.app_events_dir, app_events.AbortedTraceEvent( instanceid=appname, why=app_abort.AbortedReason.SCHEDULER.value, payload=exception))
def report_aborted(tm_env, instance, why=None, payload=None): """Report an aborted instance. Called when aborting after failed configure step or from cleanup. """ if payload is not None: payload = str(payload) trace.post( tm_env.app_events_dir, events.AbortedTraceEvent(instanceid=instance, why=_why_str(why), payload=payload))
def test_aborted(self, stdout_mock): """Test printing Aborted event. """ event = events.AbortedTraceEvent(timestamp=1, source='tests', instanceid='proid.foo#123', why='unknown', payload='test') self.trace_printer.process(event) self.assertEqual( stdout_mock.getvalue(), 'Thu, 01 Jan 1970 00:00:01+0000 - ' 'proid.foo#123 aborted on tests [reason: unknown]\n')
def test_aborted(self): """Aborted event operations. """ event = events.AbortedTraceEvent( timestamp=1, source='tests', instanceid='proid.foo#123', why='reason', payload='test' ) self.assertEqual( event.to_dict(), { 'event_type': 'aborted', 'timestamp': 1, 'source': 'tests', 'instanceid': 'proid.foo#123', 'why': 'reason', 'payload': 'test', } ) self.assertEqual( event.to_data(), ( 1, 'tests', 'proid.foo#123', 'aborted', 'reason', 'test', ) ) self.assertEqual( event, events.AbortedTraceEvent.from_data( timestamp=1, source='tests', instanceid='proid.foo#123', event_type='aborted', event_data='reason', payload='test' ) )
def test_report_aborted(self): """Tests report abort sequence.""" context.GLOBAL.zk.url = 'zookeeper://xxx@hhh:123/treadmill/mycell' treadmill.zkutils.connect.return_value = kazoo.client.KazooClient() kazoo.client.KazooClient.get_children.return_value = [] kazoo.client.KazooClient.exists.return_value = True kazoo.client.KazooClient.create.reset() kazoo.client.KazooClient.delete.reset() app_abort.report_aborted(self.tm_env, 'proid.myapp#001', why=app_abort.AbortedReason.TICKETS, payload='test') treadmill.trace.post.assert_called_with( mock.ANY, events.AbortedTraceEvent( instanceid='proid.myapp#001', why='tickets', payload='test', ), )
def test_post(self): """Test trace.post.""" # Disable W0212(protected-access) # pylint: disable=W0212 zkclient_mock = mock.Mock() zkclient_mock.get_children.return_value = [] publisher = events_publisher.EventsPublisher( zkclient_mock, app_events_dir=self.app_events_dir, server_events_dir=self.server_events_dir ) trace.post( self.app_events_dir, app_events.PendingTraceEvent( instanceid='foo.bar#123', why='created', ) ) path = os.path.join( self.app_events_dir, '100,foo.bar#123,pending,created' ) self.assertTrue(os.path.exists(path)) publisher._on_created(path, app_zk.publish) zkclient_mock.create.assert_called_once_with( '/trace/007B/foo.bar#123,100,baz,pending,created', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ) zkclient_mock.reset_mock() trace.post( self.app_events_dir, app_events.PendingDeleteTraceEvent( instanceid='foo.bar#123', why='deleted' ) ) path = os.path.join( self.app_events_dir, '100,foo.bar#123,pending_delete,deleted' ) self.assertTrue(os.path.exists(path)) publisher._on_created(path, app_zk.publish) zkclient_mock.create.assert_called_once_with( '/trace/007B/foo.bar#123,100,baz,pending_delete,deleted', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ) zkclient_mock.reset_mock() trace.post( self.app_events_dir, app_events.AbortedTraceEvent( instanceid='foo.bar#123', why='test' ) ) path = os.path.join( self.app_events_dir, '100,foo.bar#123,aborted,test' ) self.assertTrue(os.path.exists(path)) publisher._on_created(path, app_zk.publish) self.assertEqual(zkclient_mock.create.call_args_list, [ mock.call( '/trace/007B/foo.bar#123,100,baz,aborted,test', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ), mock.call( '/finished/foo.bar#123', json.dumps({ 'data': 'test', 'host': 'baz', 'state': 'aborted', 'when': '100' }, sort_keys=True).encode(), makepath=True, ephemeral=False, acl=mock.ANY, sequence=False ) ]) zkclient_mock.reset_mock() trace.post( self.server_events_dir, server_events.ServerStateTraceEvent( servername='test.xx.com', state='up' ) ) path = os.path.join( self.server_events_dir, '100,test.xx.com,server_state,up' ) self.assertTrue(os.path.exists(path)) publisher._on_created(path, server_zk.publish) zkclient_mock.create.assert_called_once_with( '/server-trace/005D/test.xx.com,100,baz,server_state,up', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ) zkclient_mock.reset_mock() trace.post( self.server_events_dir, server_events.ServerBlackoutTraceEvent( servername='test.xx.com' ) ) path = os.path.join( self.server_events_dir, '100,test.xx.com,server_blackout,' ) self.assertTrue(os.path.exists(path)) publisher._on_created(path, server_zk.publish) zkclient_mock.create.assert_called_once_with( '/server-trace/005D/test.xx.com,100,baz,server_blackout,', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY )
def test_post_zk(self): """Test trace.post_zk.""" zkclient_mock = mock.Mock() zkclient_mock.get_children.return_value = [] trace.post_zk( zkclient_mock, app_events.PendingTraceEvent( instanceid='foo.bar#123', why='created', payload='' ) ) zkclient_mock.create.assert_called_once_with( '/trace/007B/foo.bar#123,100,baz,pending,created', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ) zkclient_mock.reset_mock() trace.post_zk( zkclient_mock, app_events.PendingDeleteTraceEvent( instanceid='foo.bar#123', why='deleted' ) ) zkclient_mock.create.assert_called_once_with( '/trace/007B/foo.bar#123,100,baz,pending_delete,deleted', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ) zkclient_mock.reset_mock() trace.post_zk( zkclient_mock, app_events.AbortedTraceEvent( instanceid='foo.bar#123', why='test' ) ) zkclient_mock.create.assert_has_calls([ mock.call( '/trace/007B/foo.bar#123,100,baz,aborted,test', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ), mock.call( '/finished/foo.bar#123', json.dumps({ 'data': 'test', 'state': 'aborted', 'when': '100', 'host': 'baz' }, sort_keys=True).encode(), ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ) ]) zkclient_mock.reset_mock() trace.post_zk( zkclient_mock, server_events.ServerStateTraceEvent( servername='test.xx.com', state='up' ) ) zkclient_mock.create.assert_called_once_with( '/server-trace/005D/test.xx.com,100,baz,server_state,up', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY ) zkclient_mock.reset_mock() trace.post_zk( zkclient_mock, server_events.ServerBlackoutTraceEvent( servername='test.xx.com' ) ) zkclient_mock.create.assert_called_once_with( '/server-trace/005D/test.xx.com,100,baz,server_blackout,', b'', ephemeral=False, makepath=True, sequence=False, acl=mock.ANY )
def test_finish_aborted(self): """Tests container finish procedure when node is aborted. """ manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'host_ip': '172.31.81.67', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000ID1234', 'archive': ['/var/lib/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000, 'proto': 'tcp', }], 'services': [{ 'name': 'web_server', 'command': '/bin/false', 'restart': { 'limit': 3, 'interval': 60, }, }], 'ephemeral_ports': { 'tcp': [], 'udp': [], }, 'vring': { 'some': 'settings' } } treadmill.appcfg.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000ID1234' mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.get.return_value = localdisk mock_nwrk_client = self.tm_env.svc_network.make_client.return_value network = { 'vip': '192.168.0.2', 'gateway': '192.168.254.254', 'veth': 'testveth.0', 'external_ip': '172.31.81.67', } mock_nwrk_client.get.return_value = network app_dir = os.path.join(self.root, 'apps', app_unique_name) data_dir = os.path.join(app_dir, 'data') # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(data_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with io.open(os.path.join(data_dir, 'aborted'), 'w') as aborted: aborted.write('{"why": "reason", "payload": "test"}') app_finish.finish(self.tm_env, app_dir) treadmill.trace.post.assert_called_with( mock.ANY, events.AbortedTraceEvent(instanceid='proid.myapp#001', why='reason', payload='test')) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd'), os.path.join(data_dir, 'metrics.rrd')) treadmill.trace.post.reset() with io.open(os.path.join(data_dir, 'aborted'), 'w') as aborted: aborted.write('') app_finish.finish(self.tm_env, app_dir) treadmill.trace.post.assert_called_with( mock.ANY, events.AbortedTraceEvent(instanceid='proid.myapp#001', why='unknown', payload=None)) treadmill.runtime.archive_logs.assert_called()