def test_finish_no_resources(self):
"""Test app finish on directory when all resources are already freed.
"""
# Access to a protected member _finish of a client class
# pylint: disable=W0212
manifest = {
'app': 'proid.myapp',
'cell': 'test',
'cpu': '100%',
'disk': '100G',
'environment': 'dev',
'memory': '100M',
'name': 'proid.myapp#001',
'proid': 'foo',
'shared_network': False,
'task': '001',
'uniqueid': '0000000ID1234',
'archive': [
'/var/lib/treadmill'
],
'endpoints': [
{
'port': 8000,
'name': 'http',
'real_port': 5000
},
{
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321
}
],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'services': [
{
'command': '/bin/false',
'restart_count': 3,
'name': 'web_server'
}
],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
# All resource managers return None
mock_cgroup_client.get.return_value = None
mock_ld_client.get.return_value = None
mock_nwrk_client.get.return_value = None
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{'service': 'web_server', 'return_code': 0, 'signal': 0},
)
)
treadmill.runtime.linux._finish.finish(self.tm_env, app_dir)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'cgroups')
)
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'localdisk')
)
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'network')
)
# Cleanup the network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(
instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'signal': 0,
'return_code': 0
}
)
)
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd')
)
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd')
)
treadmill.runtime.archive_logs.assert_called()
def configure(tm_env, event, runtime):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)/
- apps/
- (app unique name)/
- data/
- app_start
- app.json
- manifest.yml
env/
- TREADMILL_*
run
finish
log/
- run
The 'run' script is responsible for creating container environment
and starting the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# Load the app from the event
try:
manifest_data = load_runtime_manifest(tm_env, event, runtime)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception('No event to load: %r', event)
return None
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Write the actual container start script
if os.name == 'nt':
run_script = '{python} -m treadmill sproc run .'.format(
python=sys.executable)
else:
run_script = 'exec {python} -m treadmill sproc run ../'.format(
python=sys.executable)
# Create the service for that container
container_svc = supervisor.create_service(
tm_env.apps_dir,
name=uniq_name,
app_run_script=run_script,
userid='root',
downed=False,
monitor_policy={
'limit': 0,
'interval': 60,
'tombstone': {
'uds': False,
'path': tm_env.running_tombstone_dir,
'id': app.name
}
},
environ={},
environment=app.environment)
data_dir = container_svc.data_dir
# Copy the original event as 'manifest.yml' in the container dir
try:
shutil.copyfile(event, os.path.join(data_dir, 'manifest.yml'))
except IOError as err:
# File is gone, cleanup.
if err.errno == errno.ENOENT:
shutil.rmtree(container_svc.directory)
_LOGGER.exception('Event gone: %r', event)
return None
else:
raise
# Store the app.json in the container directory
fs.write_safe(os.path.join(data_dir, appcfg.APP_JSON),
lambda f: f.writelines(utils.json_genencode(manifest_data)),
mode='w',
permission=0o644)
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(instanceid=app.name,
uniqueid=app.uniqueid))
return container_svc.directory
def test_finish_error(self):
"""Tests container finish procedure when app is improperly finished."""
manifest = {
'app': 'proid.myapp',
'cell': 'test',
'cpu': '100%',
'disk': '100G',
'environment': 'dev',
'memory': '100M',
'name': 'proid.myapp#001',
'proid': 'foo',
'shared_network': False,
'task': '001',
'uniqueid': '0000000001234',
'archive': [
'/var/lib/treadmill'
],
'endpoints': [
{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}
],
'services': [
{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}
],
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000001234'
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{'service': 'web_server', 'return_code': 1, 'signal': 3},
)
)
app_finish.finish(self.tm_env, app_dir)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(
instanceid='proid.myapp#001',
rc=1,
signal=3,
payload={
'service': 'web_server',
'signal': 3,
'return_code': 1,
}
)
)
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd')
)
shutil.copy.assert_called_with(
os.path.join(self.tm_env.metrics_dir, 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd')
)
treadmill.runtime.archive_logs.assert_called()
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'passthrough': [
'8.8.8.8',
'9.9.9.9',
],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 0,
'signal': 0
}, ))
app_finish.finish(self.tm_env, app_dir)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'cgroups'))
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'localdisk'))
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'network'))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.tm_env.rules.unlink_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5000,
new_ip='192.168.0.2',
new_port=8000),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=8000,
new_ip='172.31.81.67',
new_port=5000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=45024,
new_ip='192.168.0.2',
new_port=45024),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=62422,
new_ip='192.168.0.2',
new_port=62422),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='8.8.8.8',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='9.9.9.9',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.unlink_rule.call_count, 8)
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
],
any_order=True)
self.assertEqual(treadmill.iptables.rm_ip_set.call_count, 4)
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.iptables.flush_cnt_conntrack_table.assert_called_with(
'192.168.0.2')
treadmill.trace.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'signal': 0,
'return_code': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()