def start_container(container_root, manifest):
"""Treadmill container boot process.
"""
_LOGGER.info('Initializing container: %s', container_root)
app = app_manifest.read(manifest)
try:
pivot_root.make_root(container_root)
os.chdir('/')
except Exception as err: # pylint: disable=broad-except
event = traceevents.AbortedTraceEvent(
instanceid=app['name'],
why=app_abort.AbortedReason.PIVOT_ROOT.value,
payload=str(err),
)
_abort(event, container_root)
# reraise err to exit start_container
raise err
# XXX: Debug info
_LOGGER.debug('Current mounts: %s',
pprint.pformat(fs_linux.list_mounts()))
# Clean the environ
# TODO: Remove me once clean environment management is merged in.
os.environ.pop('PYTHONPATH', None)
os.environ.pop('LC_ALL', None)
os.environ.pop('LANG', None)
# Clear aliases path.
os.environ.pop('TREADMILL_ALIASES_PATH', None)
subproc.safe_exec(['s6_svscan', '-s', '/services'])
def start_container(ctx, container_root, manifest):
"""Treadmill container boot process.
"""
_LOGGER.info('Initializing container: %s', container_root)
app = app_manifest.read(manifest)
cgroup = ctx.obj.get('CGROUP')
try:
# if cgroups set, we need to remount cgroup path
# so that from cgroup directory we only see container pids
# <container_root>/sys/fs/cgroup/memory =>
# /sys/fs/cgroup/memory/treadmill/apps/<app-inst-unique>/services
if cgroup:
remount_cgroup(container_root, cgroup, ctx.obj['ROOT_CGROUP'])
pivot_root.make_root(container_root)
os.chdir('/')
except Exception as err: # pylint: disable=broad-except
event = traceevents.AbortedTraceEvent(
instanceid=app['name'],
why=app_abort.AbortedReason.PIVOT_ROOT.value,
payload=str(err),
)
_abort(event, container_root)
# reraise err to exit start_container
raise err
# XXX: Debug info
_LOGGER.debug('Current mounts: %s',
pprint.pformat(fs_linux.list_mounts()))
subproc.safe_exec(['/services/{}'.format(supervisor.SVC_INIT_FILE)])
def _abort_task(self, appname, exception):
"""Set task into aborted state in case of scheduling error."""
if self.app_events_dir:
trace.post(
self.app_events_dir,
app_events.AbortedTraceEvent(
instanceid=appname,
why=app_abort.AbortedReason.SCHEDULER.value,
payload=exception))
def report_aborted(tm_env, instance, why=None, payload=None):
"""Report an aborted instance.
Called when aborting after failed configure step or from cleanup.
"""
if payload is not None:
payload = str(payload)
trace.post(
tm_env.app_events_dir,
events.AbortedTraceEvent(instanceid=instance,
why=_why_str(why),
payload=payload))
def test_aborted(self, stdout_mock):
"""Test printing Aborted event.
"""
event = events.AbortedTraceEvent(timestamp=1,
source='tests',
instanceid='proid.foo#123',
why='unknown',
payload='test')
self.trace_printer.process(event)
self.assertEqual(
stdout_mock.getvalue(), 'Thu, 01 Jan 1970 00:00:01+0000 - '
'proid.foo#123 aborted on tests [reason: unknown]\n')
def test_aborted(self):
"""Aborted event operations.
"""
event = events.AbortedTraceEvent(
timestamp=1,
source='tests',
instanceid='proid.foo#123',
why='reason',
payload='test'
)
self.assertEqual(
event.to_dict(),
{
'event_type': 'aborted',
'timestamp': 1,
'source': 'tests',
'instanceid': 'proid.foo#123',
'why': 'reason',
'payload': 'test',
}
)
self.assertEqual(
event.to_data(),
(
1,
'tests',
'proid.foo#123',
'aborted',
'reason',
'test',
)
)
self.assertEqual(
event,
events.AbortedTraceEvent.from_data(
timestamp=1,
source='tests',
instanceid='proid.foo#123',
event_type='aborted',
event_data='reason',
payload='test'
)
)
def test_report_aborted(self):
"""Tests report abort sequence."""
context.GLOBAL.zk.url = 'zookeeper://xxx@hhh:123/treadmill/mycell'
treadmill.zkutils.connect.return_value = kazoo.client.KazooClient()
kazoo.client.KazooClient.get_children.return_value = []
kazoo.client.KazooClient.exists.return_value = True
kazoo.client.KazooClient.create.reset()
kazoo.client.KazooClient.delete.reset()
app_abort.report_aborted(self.tm_env,
'proid.myapp#001',
why=app_abort.AbortedReason.TICKETS,
payload='test')
treadmill.trace.post.assert_called_with(
mock.ANY,
events.AbortedTraceEvent(
instanceid='proid.myapp#001',
why='tickets',
payload='test',
),
)
def test_post(self):
"""Test trace.post."""
# Disable W0212(protected-access)
# pylint: disable=W0212
zkclient_mock = mock.Mock()
zkclient_mock.get_children.return_value = []
publisher = events_publisher.EventsPublisher(
zkclient_mock,
app_events_dir=self.app_events_dir,
server_events_dir=self.server_events_dir
)
trace.post(
self.app_events_dir,
app_events.PendingTraceEvent(
instanceid='foo.bar#123',
why='created',
)
)
path = os.path.join(
self.app_events_dir, '100,foo.bar#123,pending,created'
)
self.assertTrue(os.path.exists(path))
publisher._on_created(path, app_zk.publish)
zkclient_mock.create.assert_called_once_with(
'/trace/007B/foo.bar#123,100,baz,pending,created',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
zkclient_mock.reset_mock()
trace.post(
self.app_events_dir,
app_events.PendingDeleteTraceEvent(
instanceid='foo.bar#123',
why='deleted'
)
)
path = os.path.join(
self.app_events_dir, '100,foo.bar#123,pending_delete,deleted'
)
self.assertTrue(os.path.exists(path))
publisher._on_created(path, app_zk.publish)
zkclient_mock.create.assert_called_once_with(
'/trace/007B/foo.bar#123,100,baz,pending_delete,deleted',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
zkclient_mock.reset_mock()
trace.post(
self.app_events_dir,
app_events.AbortedTraceEvent(
instanceid='foo.bar#123',
why='test'
)
)
path = os.path.join(
self.app_events_dir, '100,foo.bar#123,aborted,test'
)
self.assertTrue(os.path.exists(path))
publisher._on_created(path, app_zk.publish)
self.assertEqual(zkclient_mock.create.call_args_list, [
mock.call(
'/trace/007B/foo.bar#123,100,baz,aborted,test',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
),
mock.call(
'/finished/foo.bar#123',
json.dumps({
'data': 'test',
'host': 'baz',
'state': 'aborted',
'when': '100'
}, sort_keys=True).encode(),
makepath=True,
ephemeral=False,
acl=mock.ANY,
sequence=False
)
])
zkclient_mock.reset_mock()
trace.post(
self.server_events_dir,
server_events.ServerStateTraceEvent(
servername='test.xx.com',
state='up'
)
)
path = os.path.join(
self.server_events_dir, '100,test.xx.com,server_state,up'
)
self.assertTrue(os.path.exists(path))
publisher._on_created(path, server_zk.publish)
zkclient_mock.create.assert_called_once_with(
'/server-trace/005D/test.xx.com,100,baz,server_state,up',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
zkclient_mock.reset_mock()
trace.post(
self.server_events_dir,
server_events.ServerBlackoutTraceEvent(
servername='test.xx.com'
)
)
path = os.path.join(
self.server_events_dir, '100,test.xx.com,server_blackout,'
)
self.assertTrue(os.path.exists(path))
publisher._on_created(path, server_zk.publish)
zkclient_mock.create.assert_called_once_with(
'/server-trace/005D/test.xx.com,100,baz,server_blackout,',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
def test_post_zk(self):
"""Test trace.post_zk."""
zkclient_mock = mock.Mock()
zkclient_mock.get_children.return_value = []
trace.post_zk(
zkclient_mock,
app_events.PendingTraceEvent(
instanceid='foo.bar#123',
why='created',
payload=''
)
)
zkclient_mock.create.assert_called_once_with(
'/trace/007B/foo.bar#123,100,baz,pending,created',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
zkclient_mock.reset_mock()
trace.post_zk(
zkclient_mock,
app_events.PendingDeleteTraceEvent(
instanceid='foo.bar#123',
why='deleted'
)
)
zkclient_mock.create.assert_called_once_with(
'/trace/007B/foo.bar#123,100,baz,pending_delete,deleted',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
zkclient_mock.reset_mock()
trace.post_zk(
zkclient_mock,
app_events.AbortedTraceEvent(
instanceid='foo.bar#123',
why='test'
)
)
zkclient_mock.create.assert_has_calls([
mock.call(
'/trace/007B/foo.bar#123,100,baz,aborted,test',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
),
mock.call(
'/finished/foo.bar#123',
json.dumps({
'data': 'test',
'state': 'aborted',
'when': '100',
'host': 'baz'
}, sort_keys=True).encode(),
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
])
zkclient_mock.reset_mock()
trace.post_zk(
zkclient_mock,
server_events.ServerStateTraceEvent(
servername='test.xx.com',
state='up'
)
)
zkclient_mock.create.assert_called_once_with(
'/server-trace/005D/test.xx.com,100,baz,server_state,up',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
zkclient_mock.reset_mock()
trace.post_zk(
zkclient_mock,
server_events.ServerBlackoutTraceEvent(
servername='test.xx.com'
)
)
zkclient_mock.create.assert_called_once_with(
'/server-trace/005D/test.xx.com,100,baz,server_blackout,',
b'',
ephemeral=False, makepath=True, sequence=False,
acl=mock.ANY
)
def test_finish_aborted(self):
"""Tests container finish procedure when node is aborted.
"""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'host_ip':
'172.31.81.67',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.root, 'apps', app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'aborted'), 'w') as aborted:
aborted.write('{"why": "reason", "payload": "test"}')
app_finish.finish(self.tm_env, app_dir)
treadmill.trace.post.assert_called_with(
mock.ANY,
events.AbortedTraceEvent(instanceid='proid.myapp#001',
why='reason',
payload='test'))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.trace.post.reset()
with io.open(os.path.join(data_dir, 'aborted'), 'w') as aborted:
aborted.write('')
app_finish.finish(self.tm_env, app_dir)
treadmill.trace.post.assert_called_with(
mock.ANY,
events.AbortedTraceEvent(instanceid='proid.myapp#001',
why='unknown',
payload=None))
treadmill.runtime.archive_logs.assert_called()
