def _get_current_passthrough_rules(chain):
"""Extract all PassThrough rules from iptables.
:param ``str`` chain:
Iptables chain to process.
:returns:
``set([PassThroughRule])`` -- Set of rules.
"""
rules = set()
if chain is None:
chain = PREROUTING_PASSTHROUGH
for line in _iptables_output('nat', '-S', chain).splitlines():
match = _PASSTHROUGH_RULE_RE.match(line.strip())
if match:
data = match.groupdict()
rule = firewall.PassThroughRule(src_ip=data['src_ip'],
dst_ip=data['dst_ip'])
rules.add(rule)
return rules
def test_passthrough_missing_rule(self):
"""Tests PassThrough setup when new rule needs to be created."""
treadmill.iptables.get_current_passthrough_rules.return_value = \
self.passthrough_rules
missing_rule = firewall.PassThroughRule(src_ip='10.197.19.20',
dst_ip='192.168.2.2'),
passthroughs = self.passthrough_rules | set([missing_rule, ])
iptables.configure_passthrough_rules(
passthroughs,
iptables.PREROUTING_PASSTHROUGH
)
treadmill.iptables.add_passthrough_rule.assert_called_with(
missing_rule,
chain=iptables.PREROUTING_PASSTHROUGH
)
self.assertEquals(
0, treadmill.iptables.delete_passthrough_rule.call_count
)
def test_passthrough_extra_rule(self):
"""Tests PassThrough setup when rule needs to be removed."""
treadmill.iptables.get_current_passthrough_rules.return_value = \
self.passthrough_rules
extra_rule = firewall.PassThroughRule(src_ip='10.197.19.19',
dst_ip='192.168.2.2')
passthroughs = self.passthrough_rules - set([extra_rule, ])
iptables.configure_passthrough_rules(
passthroughs,
iptables.PREROUTING_PASSTHROUGH
)
self.assertEquals(
0, treadmill.iptables.add_passthrough_rule.call_count
)
treadmill.iptables.delete_passthrough_rule.assert_called_with(
extra_rule,
chain=iptables.PREROUTING_PASSTHROUGH
)
def setUp(self):
# Pylint warning re accessing protected class member.
# pylint: disable=W0212
self.root = tempfile.mkdtemp()
self.rules_dir = os.path.join(self.root, 'rules')
self.apps_dir = os.path.join(self.root, 'apps')
os.makedirs(self.rules_dir)
os.makedirs(self.apps_dir)
self.rules = rulefile.RuleMgr(self.rules_dir, self.apps_dir)
self.natrule = firewall.DNATRule('1.1.1.1', 123, '2.2.2.2', 234)
self.natfile = rulefile.RuleMgr._filenameify(self.natrule)
self.natuid = '1234'
with open(os.path.join(self.apps_dir, self.natuid), 'w'):
pass
self.passthroughrule = firewall.PassThroughRule('3.3.3.3', '4.4.4.4')
self.passthroughfile = rulefile.RuleMgr._filenameify(
self.passthroughrule,
)
self.passthroughuid = '4321'
with open(os.path.join(self.apps_dir, self.passthroughuid), 'w'):
pass
def test__unshare_network_complex(self):
"""Test unshare network advanced sequence (ephemeral/passthrough)."""
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj(
{
'name': 'myproid.test#0',
'environment': 'dev',
'uniqueid': 'ID1234',
'network': {
'veth': 'id1234.0',
'vip': '192.168.0.2',
'gateway': '192.168.254.254'
},
'shared_ip': False,
'endpoints': [
{
'name': 'ssh',
'port': 54321,
'real_port': 54321,
'type': 'infra',
}
],
'ephemeral_ports': [
10000,
10001,
10002,
],
'passthrough': [
'xxx',
'yyy',
'zzz',
],
}
)
app_unique_name = appmgr.app_unique_name(app)
hosts_to_ip = {
'xxx': '4.4.4.4',
'yyy': '5.5.5.5',
'zzz': '5.5.5.5',
}
socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h]
self.app_env.rules.get_rules.return_value = set()
treadmill.appmgr.run._unshare_network(
self.app_env,
app
)
self.app_env.rules.create_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('172.31.81.67', 54321,
'192.168.0.2', 54321),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10000,
'192.168.0.2', 10000),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10001,
'192.168.0.2', 10001),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10002,
'192.168.0.2', 10002),
owner=app_unique_name),
mock.call(rule=firewall.PassThroughRule('4.4.4.4',
'192.168.0.2'),
owner=app_unique_name),
mock.call(rule=firewall.PassThroughRule('5.5.5.5',
'192.168.0.2'),
owner=app_unique_name),
])
# Check that infra services + ephemeral ports are in the same set.
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10000'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10001'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10002'),
])
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.0.2',
'192.168.254.254',
None,
)
def _unshare_network(tm_env, container_dir, app):
"""Configures private app network.
:param ``appenv.AppEnvironment`` tm_env:
Treadmill application environment
"""
unique_name = appcfg.app_unique_name(app)
# Configure DNAT rules while on host network.
for endpoint in app.endpoints:
_LOGGER.info('Creating DNAT rule: %s:%s -> %s:%s',
app.network.external_ip,
endpoint.real_port,
app.network.vip,
endpoint.port)
dnatrule = firewall.DNATRule(proto=endpoint.proto,
dst_ip=app.network.external_ip,
dst_port=endpoint.real_port,
new_ip=app.network.vip,
new_port=endpoint.port)
snatrule = firewall.SNATRule(proto=endpoint.proto,
src_ip=app.network.vip,
src_port=endpoint.port,
new_ip=app.network.external_ip,
new_port=endpoint.real_port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
tm_env.rules.create_rule(chain=iptables.POSTROUTING_SNAT,
rule=snatrule,
owner=unique_name)
# See if this container requires vring service
if app.vring:
_LOGGER.debug('adding %r to VRing set', app.network.vip)
iptables.add_ip_set(
iptables.SET_VRING_CONTAINERS,
app.network.vip
)
# See if this was an "infra" endpoint and if so add it to the whitelist
# set.
if getattr(endpoint, 'type', None) == 'infra':
_LOGGER.debug('adding %s:%s to infra services set',
app.network.vip, endpoint.port)
iptables.add_ip_set(
iptables.SET_INFRA_SVC,
'{ip},{proto}:{port}'.format(
ip=app.network.vip,
proto=endpoint.proto,
port=endpoint.port,
)
)
for port in app.ephemeral_ports.tcp:
_LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s',
app.network.external_ip, port,
app.network.vip, port)
dnatrule = firewall.DNATRule(proto='tcp',
dst_ip=app.network.external_ip,
dst_port=port,
new_ip=app.network.vip,
new_port=port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
# Treat ephemeral ports as infra, consistent with current prodperim
# behavior.
iptables.add_ip_set(iptables.SET_INFRA_SVC,
'{ip},tcp:{port}'.format(ip=app.network.vip,
port=port))
for port in app.ephemeral_ports.udp:
_LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s',
app.network.external_ip, port,
app.network.vip, port)
dnatrule = firewall.DNATRule(proto='udp',
dst_ip=app.network.external_ip,
dst_port=port,
new_ip=app.network.vip,
new_port=port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
# Treat ephemeral ports as infra, consistent with current prodperim
# behavior.
iptables.add_ip_set(iptables.SET_INFRA_SVC,
'{ip},udp:{port}'.format(ip=app.network.vip,
port=port))
# configure passthrough while on main network.
if getattr(app, 'passthrough', None):
_LOGGER.info('adding passthrough for: %r', app.passthrough)
# Resolve all the hosts (+dedup)
new_ips = {
socket.gethostbyname(host)
for host in app.passthrough
}
# Create a passthrough rule from each of the source IP to the
# container IP and record these source IP in a set.
for ipaddr in new_ips:
passthroughrule = firewall.PassThroughRule(
src_ip=ipaddr,
dst_ip=app.network.vip,
)
tm_env.rules.create_rule(chain=iptables.PREROUTING_PASSTHROUGH,
rule=passthroughrule,
owner=unique_name)
# configure exception filter rules
try:
firewall_plugin = plugin_manager.load(
'treadmill.firewall.plugins', 'firewall'
)
firewall_plugin.apply_exception_rules(tm_env, container_dir, app)
except Exception: # pylint: disable=W0703
_LOGGER.exception(
'Error in firewall plugin, skip applying firewall exception rules.'
)
service_ip = None
if app.shared_ip:
service_ip = app.network.external_ip
# Unshare network and create virtual device
newnet.create_newnet(app.network.veth,
app.network.vip,
app.network.gateway,
service_ip)
def _cleanup_network(tm_env, app, network_client):
"""Cleanup the network part of a container.
"""
# Generate a unique name for the app
unique_name = appmgr.app_unique_name(app)
try:
app_network = network_client.get(unique_name)
except services.ResourceServiceError:
_LOGGER.warning('network never allocated')
return
if app_network is None:
_LOGGER.info('Network resource already freed')
return
# Unconfigure passthrough
if hasattr(app, 'passthrough'):
_LOGGER.info('Deleting passthrough for: %r', app.passthrough)
# Resolve all the hosts
# FIXME: There is no guarantie the hosts will resolve to
# the same IPs as they did during creation.
ips = set([socket.gethostbyname(host) for host in app.passthrough])
for ip in ips:
tm_env.rules.unlink_rule(
rule=firewall.PassThroughRule(src_ip=ip,
dst_ip=app_network['vip']),
owner=unique_name,
)
for endpoint in app.endpoints:
tm_env.rules.unlink_rule(
rule=firewall.DNATRule(proto=endpoint.proto,
orig_ip=app.host_ip,
orig_port=endpoint.real_port,
new_ip=app_network['vip'],
new_port=endpoint.port),
owner=unique_name,
)
# See if this was an "infra" endpoint and if so remove it
# from the whitelist set.
if getattr(endpoint, 'type', None) == 'infra':
_LOGGER.debug('removing %s:%s from infra services set',
app_network['vip'], endpoint.port)
iptables.rm_ip_set(
iptables.SET_INFRA_SVC, '{ip},{proto}:{port}'.format(
ip=app_network['vip'],
proto=endpoint.proto,
port=endpoint.port,
))
_cleanup_ports(tm_env, unique_name, app_network['vip'],
app.ephemeral_ports.tcp, 'tcp')
_cleanup_ports(tm_env, unique_name, app_network['vip'],
app.ephemeral_ports.udp, 'udp')
# Terminate any entries in the conntrack table
iptables.flush_conntrack_table(app_network['vip'])
# Cleanup network resources
network_client.delete(unique_name)
def test__unshare_network_complex(self):
"""Test unshare network advanced sequence (ephemeral/passthrough)."""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'name':
'myproid.test#0',
'environment':
'dev',
'uniqueid':
'ID1234',
'network': {
'veth': 'id1234.0',
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'external_ip': '172.31.81.67',
},
'shared_ip':
False,
'endpoints': [{
'name': 'ssh',
'port': 54321,
'real_port': 54321,
'type': 'infra',
'proto': 'tcp',
}, {
'name': 'test2',
'port': 54322,
'real_port': 54322,
'proto': 'udp',
}],
'ephemeral_ports': {
'tcp': [10000, 10001, 10002],
'udp': [],
},
'passthrough': [
'xxx',
'yyy',
'zzz',
],
'vring': {
'some': 'data'
}
})
app_unique_name = appcfg.app_unique_name(app)
hosts_to_ip = {
'xxx': '4.4.4.4',
'yyy': '5.5.5.5',
'zzz': '5.5.5.5',
}
socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h]
self.tm_env.rules.get_rules.return_value = set()
treadmill.runtime.linux._run._unshare_network(self.tm_env,
'test_container_dir',
app)
self.tm_env.rules.create_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=54322,
new_ip='192.168.0.2',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='udp',
src_ip='192.168.0.2',
src_port=54322,
new_ip='172.31.81.67',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10000,
new_ip='192.168.0.2',
new_port=10000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10001,
new_ip='192.168.0.2',
new_port=10001),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10002,
new_ip='192.168.0.2',
new_port=10002),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('4.4.4.4', '192.168.0.2'),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('5.5.5.5', '192.168.0.2'),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.create_rule.call_count, 9)
# Check that infra services + ephemeral ports are in the same set.
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10000'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10001'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10002'),
],
any_order=True)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.0.2',
'192.168.254.254',
None,
)
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'passthrough': [
'8.8.8.8',
'9.9.9.9',
],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 0,
'signal': 0
}, ))
app_finish.finish(self.tm_env, app_dir)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'cgroups'))
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'localdisk'))
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'network'))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.tm_env.rules.unlink_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5000,
new_ip='192.168.0.2',
new_port=8000),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=8000,
new_ip='172.31.81.67',
new_port=5000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=45024,
new_ip='192.168.0.2',
new_port=45024),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=62422,
new_ip='192.168.0.2',
new_port=62422),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='8.8.8.8',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='9.9.9.9',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.unlink_rule.call_count, 8)
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
],
any_order=True)
self.assertEqual(treadmill.iptables.rm_ip_set.call_count, 4)
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.iptables.flush_cnt_conntrack_table.assert_called_with(
'192.168.0.2')
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'signal': 0,
'return_code': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()
def get_rule(rulespec):
"""Parse a forwarding rule spec into a usable firewall rule.
:param ``str`` rulespec:
Forward rule in string form
:returns:
tuple(Chain, ``DNATRule`` | ``SNATRule`` | ``PassThroughRule``) |
``None`` -- A tuple of a chain and a firewall rule object. If
parsing failed, returns ``None``
"""
match = _DNAT_FILE_RE.match(rulespec)
if match:
data = match.groupdict()
return (
data['chain'],
firewall.DNATRule(
proto=data['proto'],
src_ip=(
data['src_ip'] if data['src_ip'] != _ANY else None
),
src_port=(
data['src_port'] if data['src_port'] != _ANY else None
),
dst_ip=(
data['dst_ip'] if data['dst_ip'] != _ANY else None
),
dst_port=(
data['dst_port'] if data['dst_port'] != _ANY else None
),
new_ip=data['new_ip'],
new_port=data['new_port']
)
)
match = _SNAT_FILE_RE.match(rulespec)
if match:
data = match.groupdict()
return (
data['chain'],
firewall.SNATRule(
proto=data['proto'],
src_ip=(
data['src_ip'] if data['src_ip'] != _ANY else None
),
src_port=(
data['src_port'] if data['src_port'] != _ANY else None
),
dst_ip=(
data['dst_ip'] if data['dst_ip'] != _ANY else None
),
dst_port=(
data['dst_port'] if data['dst_port'] != _ANY else None
),
new_ip=data['new_ip'],
new_port=data['new_port']
)
)
match = _PASSTHROUGH_FILE_RE.match(rulespec)
if match:
data = match.groupdict()
return (
data['chain'],
firewall.PassThroughRule(data['src_ip'],
data['dst_ip'])
)
return None
