def setUp(self):
# Note: These two match the content of NAT_TABLE_SAVE
self.dnat_rules = set([
firewall.DNATRule('udp',
'172.31.81.67', 5002,
'192.168.1.13', 8000),
firewall.DNATRule('tcp',
'172.31.81.67', 5000,
'192.168.0.11', 8000),
firewall.DNATRule('tcp',
'172.31.81.67', 5003,
'192.168.1.13', 22),
firewall.DNATRule('tcp',
'172.31.81.67', 5001,
'192.168.0.11', 22),
])
self.snat_rules = set([
firewall.SNATRule('udp',
'192.168.0.3', 22,
'172.31.81.67', 5001),
])
self.passthrough_rules = set([
firewall.PassThroughRule(src_ip='10.197.19.18',
dst_ip='192.168.3.2'),
firewall.PassThroughRule(src_ip='10.197.19.19',
dst_ip='192.168.2.2'),
])
def test_dnat_extra_rule(self):
"""Tests DNAT setup when rule needs to be removed."""
# Disable protected-access: Test access protected members.
# pylint: disable=protected-access
treadmill.iptables._get_current_dnat_rules.return_value = (
self.dnat_rules |
set([
firewall.DNATRule('tcp',
'172.31.81.67', 5004,
'192.168.2.15', 22),
])
)
desired_rules = (
self.dnat_rules
)
iptables.configure_dnat_rules(
desired_rules,
iptables.PREROUTING_DNAT
)
self.assertEqual(0, treadmill.iptables.add_dnat_rule.call_count)
treadmill.iptables.delete_dnat_rule.assert_called_with(
firewall.DNATRule('tcp',
'172.31.81.67', 5004,
'192.168.2.15', 22),
chain=iptables.PREROUTING_DNAT,
)
def setUp(self):
# Note: These two match the content of NAT_TABLE_SAVE
self.dnat_rules = set([
firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67', dst_port=5002,
new_ip='192.168.1.13', new_port=8000),
firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67', dst_port=5000,
new_ip='192.168.0.11', new_port=8000),
firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67', dst_port=5003,
new_ip='192.168.1.13', new_port=22),
firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67', dst_port=5001,
new_ip='192.168.0.11', new_port=22),
])
self.snat_rules = set([
firewall.SNATRule(proto='udp',
src_ip='192.168.0.3', src_port=22,
new_ip='172.31.81.67', new_port=5001),
])
self.passthrough_rules = set([
firewall.PassThroughRule(src_ip='10.197.19.18',
dst_ip='192.168.3.2'),
firewall.PassThroughRule(src_ip='10.197.19.19',
dst_ip='192.168.2.2'),
])
self.iptables_state = _test_data('iptables_state.save')
self.iptables_empty_state = _test_data('iptables_empty_state.save')
self.iptables_filter_state = _test_data('iptables_filter_state.save')
self.ipset_state = _test_data('ipset_state.save')
self.nat_table_save = _test_data('iptables_test_nat_table.save')
def test_dnat_missing_rule(self):
"""Tests DNAT setup when new rule needs to be created."""
treadmill.iptables.get_current_nat_rules.return_value = \
self.dnat_rules | self.snat_rules
desired_rules = (
self.dnat_rules | self.snat_rules |
set([
firewall.DNATRule('tcp',
'172.31.81.67', 5004,
'192.168.2.15', 22),
firewall.SNATRule('tcp',
'172.31.81.67', 5004,
'192.168.2.15', 22),
])
)
iptables.configure_nat_rules(
desired_rules,
iptables.PREROUTING_DNAT
)
treadmill.iptables.add_dnat_rule.assert_called_with(
firewall.DNATRule('tcp',
'172.31.81.67', 5004,
'192.168.2.15', 22),
chain=iptables.PREROUTING_DNAT
)
treadmill.iptables.add_snat_rule.assert_called_with(
firewall.SNATRule('tcp',
'172.31.81.67', 5004,
'192.168.2.15', 22),
chain=iptables.PREROUTING_DNAT
)
self.assertEquals(0, treadmill.iptables.delete_dnat_rule.call_count)
self.assertEquals(0, treadmill.iptables.delete_snat_rule.call_count)
def setUp(self):
# Pylint warning re accessing protected class member.
# pylint: disable=W0212
self.root = tempfile.mkdtemp()
self.rules_dir = os.path.join(self.root, 'rules')
self.apps_dir = os.path.join(self.root, 'apps')
os.makedirs(self.rules_dir)
os.makedirs(self.apps_dir)
self.rules = rulefile.RuleMgr(self.rules_dir, self.apps_dir)
self.tcpdnatrule = firewall.DNATRule(
proto='tcp',
dst_ip='1.1.1.1', dst_port=123,
new_ip='2.2.2.2', new_port=234
)
self.tcpdnatfile = rulefile.RuleMgr._filenameify(
'SOME_CHAIN',
self.tcpdnatrule
)
self.tcpdnatuid = '1234'
with io.open(os.path.join(self.apps_dir, self.tcpdnatuid), 'w'):
pass
self.udpdnatrule = firewall.DNATRule(
proto='udp',
dst_ip='1.1.1.1', dst_port=123,
new_ip='2.2.2.2', new_port=234
)
self.udpdnatfile = rulefile.RuleMgr._filenameify(
'SOME_CHAIN',
self.udpdnatrule
)
self.udpdnatuid = '2345'
with io.open(os.path.join(self.apps_dir, self.udpdnatuid), 'w'):
pass
self.udpsnatrule = firewall.SNATRule(
proto='udp',
src_ip='1.1.1.1', src_port=123,
new_ip='2.2.2.2', new_port=234
)
self.udpsnatfile = rulefile.RuleMgr._filenameify(
'SOME_CHAIN',
self.udpsnatrule
)
self.udpsnatuid = '3456'
with io.open(os.path.join(self.apps_dir, self.udpsnatuid), 'w'):
pass
self.passthroughrule = firewall.PassThroughRule('3.3.3.3', '4.4.4.4')
self.passthroughfile = rulefile.RuleMgr._filenameify(
'SOME_CHAIN',
self.passthroughrule,
)
self.passthroughuid = '4321'
with io.open(os.path.join(self.apps_dir, self.passthroughuid), 'w'):
pass
def test__unshare_network_simple(self):
"""Tests unshare network sequence.
"""
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj({
'name':
'proid.test#0',
'uniqueid':
'ID1234',
'environment':
'dev',
'network': {
'veth': 'id1234.0',
'vip': '192.168.1.1',
'gateway': '192.168.254.254',
},
'host_ip':
'172.31.81.67',
'shared_ip':
True,
'ephemeral_ports': [],
'endpoints': [{
'real_port': '5007',
'proto': 'tcp',
'port': '22',
'type': 'infra'
}, {
'real_port': '5013',
'proto': 'udp',
'port': '12345'
}],
})
app_unique_name = appmgr.app_unique_name(app)
appmgr.run._unshare_network(self.app_env, app)
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.1.1,tcp:22'),
])
self.app_env.rules.create_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', '5007',
'192.168.1.1', '22'),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('udp', '172.31.81.67', '5013',
'192.168.1.1', '12345'),
owner=app_unique_name)
],
any_order=True)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.1.1',
'192.168.254.254',
'172.31.81.67',
)
def setUp(self):
# Note: These two match the content of NAT_TABLE_SAVE
self.dnat_rules = set([
firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=5002,
new_ip='192.168.1.13',
new_port=8000),
firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5000,
new_ip='192.168.0.11',
new_port=8000),
firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5003,
new_ip='192.168.1.13',
new_port=22),
firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5001,
new_ip='192.168.0.11',
new_port=22),
])
self.snat_rules = set([
firewall.SNATRule(proto='udp',
src_ip='192.168.0.3',
src_port=22,
new_ip='172.31.81.67',
new_port=5001),
])
self.passthrough_rules = set([
firewall.PassThroughRule(src_ip='10.197.19.18',
dst_ip='192.168.3.2'),
firewall.PassThroughRule(src_ip='10.197.19.19',
dst_ip='192.168.2.2'),
])
with io.open(self.IPTABLES_STATE) as f:
self.iptables_state = f.read()
with io.open(self.IPTABLES_EMPTY_STATE) as f:
self.iptables_empty_state = f.read()
with io.open(self.IPTABLES_FILTER_STATE) as f:
self.iptables_filter_state = f.read()
with io.open(self.IPTABLES_FILTER_DROP_STATE) as f:
self.iptables_filter_drop_state = f.read()
with io.open(self.IPSET_STATE) as f:
self.ipset_state = f.read()
with io.open(self.NAT_TABLE_SAVE) as f:
self.nat_table_save = f.read()
def _get_current_dnat_rules(chain):
"""Extract all DNAT rules in chain from iptables.
:param ``str`` chain:
Iptables chain to process.
:returns:
``set([DNATRule])`` -- Set of rules.
"""
if chain is None:
chain = PREROUTING_DNAT
rules = set()
for line in _iptables_output('nat', '-S', chain).splitlines():
dnat_match = _DNAT_RULE_RE.match(line.strip())
if dnat_match:
data = dnat_match.groupdict()
rule = firewall.DNATRule(proto=data['proto'],
dst_ip=data['dst_ip'],
dst_port=data['dst_port'],
src_ip=data['src_ip'],
src_port=data['src_port'],
new_ip=data['new_ip'],
new_port=data['new_port'])
rules.add(rule)
continue
return rules
def test_delete_dnat_rule(self):
"""Test dnat rule deletion."""
iptables.delete_dnat_rule(
firewall.DNATRule('1.1.1.1', 123, '2.2.2.2', 345), 'SOME_RULE')
treadmill.iptables.delete_raw_rule.assert_called_with(
'nat', 'SOME_RULE', ('-d 1.1.1.1 -p tcp -m tcp --dport 123'
' -j DNAT --to-destination 2.2.2.2:345'))
def test_delete_raw_rule(self):
"""Test deleting an iptable rule."""
iptables.delete_dnat_rule(
firewall.DNATRule('1.1.1.1', 123, '2.2.2.2', 345), 'SOME_RULE')
treadmill.subproc.check_call.assert_called_with([
'iptables', '-t', 'nat', '-D', 'SOME_RULE', '-d', '1.1.1.1', '-p',
'tcp', '-m', 'tcp', '--dport', '123', '-j', 'DNAT',
'--to-destination', '2.2.2.2:345'
])
def test_add_dnat_rule(self):
"""Test dnat rule addition."""
iptables.add_dnat_rule(firewall.DNATRule('tcp', '1.1.1.1', 123,
'2.2.2.2', 345),
'SOME_RULE',
safe=True)
treadmill.iptables.add_raw_rule.assert_called_with(
'nat', 'SOME_RULE', ('-d 1.1.1.1 -p tcp -m tcp --dport 123'
' -j DNAT --to-destination 2.2.2.2:345'),
True)
def test_delete_dnat_rule(self):
"""Test dnat rule deletion."""
iptables.delete_dnat_rule(
firewall.DNATRule(proto='tcp',
dst_ip='1.1.1.1',
dst_port=123,
new_ip='2.2.2.2',
new_port=345), 'SOME_RULE')
treadmill.iptables.delete_raw_rule.assert_called_with(
'nat', 'SOME_RULE',
('-s 0.0.0.0/0 -d 1.1.1.1 -p tcp -m tcp --dport 123'
' -j DNAT --to-destination 2.2.2.2:345'))
def test_delete_rule_nonexist(self):
"""Test dnat rule deleting when the rule does not exist."""
treadmill.subproc.check_call.side_effect = \
subprocess.CalledProcessError(returncode=1, output='', cmd='')
iptables.delete_dnat_rule(
firewall.DNATRule('1.1.1.1', 123, '2.2.2.2', 345), 'SOME_RULE')
treadmill.subproc.check_call.assert_called_with([
'iptables', '-t', 'nat', '-D', 'SOME_RULE', '-d', '1.1.1.1', '-p',
'tcp', '-m', 'tcp', '--dport', '123', '-j', 'DNAT',
'--to-destination', '2.2.2.2:345'
])
def get_rule(rulespec):
"""Parse a forwarding rule spec into a usable firewall rule.
:param ``str`` rulespec:
Forward rule in string form
:returns:
tuple(Chain, ``DNATRule`` | ``SNATRule`` | ``PassThroughRule``) |
``None`` -- A tuple of a chain and a firewall rule object. If
parsing failed, returns ``None``
"""
match = _DNAT_FILE_RE.match(rulespec)
if match:
data = match.groupdict()
return (data['chain'],
firewall.DNATRule(
proto=data['proto'],
src_ip=(data['src_ip']
if data['src_ip'] != _ANY else None),
src_port=(data['src_port']
if data['src_port'] != _ANY else None),
dst_ip=(data['dst_ip']
if data['dst_ip'] != _ANY else None),
dst_port=(data['dst_port']
if data['dst_port'] != _ANY else None),
new_ip=data['new_ip'],
new_port=data['new_port']))
match = _SNAT_FILE_RE.match(rulespec)
if match:
data = match.groupdict()
return (data['chain'],
firewall.SNATRule(
proto=data['proto'],
src_ip=(data['src_ip']
if data['src_ip'] != _ANY else None),
src_port=(data['src_port']
if data['src_port'] != _ANY else None),
dst_ip=(data['dst_ip']
if data['dst_ip'] != _ANY else None),
dst_port=(data['dst_port']
if data['dst_port'] != _ANY else None),
new_ip=data['new_ip'],
new_port=data['new_port']))
match = _PASSTHROUGH_FILE_RE.match(rulespec)
if match:
data = match.groupdict()
return (data['chain'],
firewall.PassThroughRule(data['src_ip'], data['dst_ip']))
return None
def test_dnat_missing_rule(self):
"""Tests DNAT setup when new rule needs to be created."""
treadmill.iptables.get_current_dnat_rules.return_value = \
self.dnat_rules
missing_rule = firewall.DNATRule('172.31.81.67', 5004, '192.168.2.15',
22)
redirects = self.dnat_rules | set([
missing_rule,
])
iptables.configure_dnat_rules(redirects, iptables.PREROUTING_DNAT)
treadmill.iptables.add_dnat_rule.assert_called_with(
missing_rule, chain=iptables.PREROUTING_DNAT)
self.assertEquals(0, treadmill.iptables.delete_dnat_rule.call_count)
def _cleanup_ports(tm_env, unique_name, vip, ports, proto):
"""Cleanup firewall rules for ports."""
for port in ports:
# We treat ephemeral ports as infra, consistent with current
# prodperim behavior.
iptables.rm_ip_set(
iptables.SET_INFRA_SVC, '{ip},{proto}:{port}'.format(ip=vip,
proto=proto,
port=port))
dnatrule = firewall.DNATRule(proto=proto,
orig_ip=tm_env.host_ip,
orig_port=port,
new_ip=vip,
new_port=port)
tm_env.rules.unlink_rule(rule=dnatrule, owner=unique_name)
def test_dnat_extra_rule(self):
"""Tests DNAT setup when rule needs to be removed."""
treadmill.iptables.get_current_dnat_rules.return_value = \
self.dnat_rules
extra_rule = firewall.DNATRule('tcp', '172.31.81.67', 5003,
'192.168.1.13', 22)
redirects = self.dnat_rules - set([
extra_rule,
])
iptables.configure_dnat_rules(redirects, iptables.PREROUTING_DNAT)
self.assertEquals(0, treadmill.iptables.add_dnat_rule.call_count)
treadmill.iptables.delete_dnat_rule.assert_called_with(
extra_rule,
chain=iptables.PREROUTING_DNAT,
)
def setUp(self):
# Pylint warning re accessing protected class member.
# pylint: disable=W0212
self.root = tempfile.mkdtemp()
self.rules_dir = os.path.join(self.root, 'rules')
self.apps_dir = os.path.join(self.root, 'apps')
os.makedirs(self.rules_dir)
os.makedirs(self.apps_dir)
self.rules = rulefile.RuleMgr(self.rules_dir, self.apps_dir)
self.natrule = firewall.DNATRule('1.1.1.1', 123, '2.2.2.2', 234)
self.natfile = rulefile.RuleMgr._filenameify(self.natrule)
self.natuid = '1234'
with open(os.path.join(self.apps_dir, self.natuid), 'w'):
pass
self.passthroughrule = firewall.PassThroughRule('3.3.3.3', '4.4.4.4')
self.passthroughfile = rulefile.RuleMgr._filenameify(
self.passthroughrule,
)
self.passthroughuid = '4321'
with open(os.path.join(self.apps_dir, self.passthroughuid), 'w'):
pass
def test__unshare_network_complex(self):
"""Test unshare network advanced sequence (ephemeral/passthrough)."""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'name':
'myproid.test#0',
'environment':
'dev',
'uniqueid':
'ID1234',
'network': {
'veth': 'id1234.0',
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'external_ip': '172.31.81.67',
},
'shared_ip':
False,
'endpoints': [{
'name': 'ssh',
'port': 54321,
'real_port': 54321,
'type': 'infra',
'proto': 'tcp',
}, {
'name': 'test2',
'port': 54322,
'real_port': 54322,
'proto': 'udp',
}],
'ephemeral_ports': {
'tcp': [10000, 10001, 10002],
'udp': [],
},
'passthrough': [
'xxx',
'yyy',
'zzz',
],
'vring': {
'some': 'data'
}
})
app_unique_name = appcfg.app_unique_name(app)
hosts_to_ip = {
'xxx': '4.4.4.4',
'yyy': '5.5.5.5',
'zzz': '5.5.5.5',
}
socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h]
self.tm_env.rules.get_rules.return_value = set()
treadmill.runtime.linux._run._unshare_network(self.tm_env,
'test_container_dir',
app)
self.tm_env.rules.create_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=54322,
new_ip='192.168.0.2',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='udp',
src_ip='192.168.0.2',
src_port=54322,
new_ip='172.31.81.67',
new_port=54322),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10000,
new_ip='192.168.0.2',
new_port=10000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10001,
new_ip='192.168.0.2',
new_port=10001),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=10002,
new_ip='192.168.0.2',
new_port=10002),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('4.4.4.4', '192.168.0.2'),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule('5.5.5.5', '192.168.0.2'),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.create_rule.call_count, 9)
# Check that infra services + ephemeral ports are in the same set.
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10000'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10001'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10002'),
],
any_order=True)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.0.2',
'192.168.254.254',
None,
)
def test__unshare_network_complex(self):
"""Test unshare network advanced sequence (ephemeral/passthrough)."""
# Access protected module _create_supervision_tree
# pylint: disable=W0212
app = utils.to_obj(
{
'name': 'myproid.test#0',
'environment': 'dev',
'uniqueid': 'ID1234',
'network': {
'veth': 'id1234.0',
'vip': '192.168.0.2',
'gateway': '192.168.254.254'
},
'shared_ip': False,
'endpoints': [
{
'name': 'ssh',
'port': 54321,
'real_port': 54321,
'type': 'infra',
}
],
'ephemeral_ports': [
10000,
10001,
10002,
],
'passthrough': [
'xxx',
'yyy',
'zzz',
],
}
)
app_unique_name = appmgr.app_unique_name(app)
hosts_to_ip = {
'xxx': '4.4.4.4',
'yyy': '5.5.5.5',
'zzz': '5.5.5.5',
}
socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h]
self.app_env.rules.get_rules.return_value = set()
treadmill.appmgr.run._unshare_network(
self.app_env,
app
)
self.app_env.rules.create_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('172.31.81.67', 54321,
'192.168.0.2', 54321),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10000,
'192.168.0.2', 10000),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10001,
'192.168.0.2', 10001),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('172.31.81.67', 10002,
'192.168.0.2', 10002),
owner=app_unique_name),
mock.call(rule=firewall.PassThroughRule('4.4.4.4',
'192.168.0.2'),
owner=app_unique_name),
mock.call(rule=firewall.PassThroughRule('5.5.5.5',
'192.168.0.2'),
owner=app_unique_name),
])
# Check that infra services + ephemeral ports are in the same set.
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10000'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10001'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:10002'),
])
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.0.2',
'192.168.254.254',
None,
)
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/lib/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'passthrough': [
'8.8.8.8',
'9.9.9.9',
],
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
'vring': {
'some': 'settings'
}
}
treadmill.appcfg.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.tm_env.svc_cgroup.make_client.return_value
mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.tm_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
'external_ip': '172.31.81.67',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(data_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(data_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.writelines(
utils.json_genencode(
{
'service': 'web_server',
'return_code': 0,
'signal': 0
}, ))
app_finish.finish(self.tm_env, app_dir)
# All resource service clients are properly created
self.tm_env.svc_cgroup.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'cgroups'))
self.tm_env.svc_localdisk.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'localdisk'))
self.tm_env.svc_network.make_client.assert_called_with(
os.path.join(data_dir, 'resources', 'network'))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.tm_env.rules.unlink_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=5000,
new_ip='192.168.0.2',
new_port=8000),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=8000,
new_ip='172.31.81.67',
new_port=5000),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=54321,
new_ip='192.168.0.2',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.0.2',
src_port=54321,
new_ip='172.31.81.67',
new_port=54321),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port=45024,
new_ip='192.168.0.2',
new_port=45024),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port=62422,
new_ip='192.168.0.2',
new_port=62422),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='8.8.8.8',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_PASSTHROUGH,
rule=firewall.PassThroughRule(
src_ip='9.9.9.9',
dst_ip='192.168.0.2',
),
owner=app_unique_name),
],
any_order=True)
self.assertEqual(self.tm_env.rules.unlink_rule.call_count, 8)
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'),
],
any_order=True)
self.assertEqual(treadmill.iptables.rm_ip_set.call_count, 4)
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.iptables.flush_cnt_conntrack_table.assert_called_with(
'192.168.0.2')
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'signal': 0,
'return_code': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(data_dir, 'metrics.rrd'))
treadmill.runtime.archive_logs.assert_called()
def test_finish(self):
"""Tests container finish procedure and freeing of the resources.
"""
# Access protected module _kill_apps_by_root
# pylint: disable=W0212
manifest = {
'app':
'proid.myapp',
'cell':
'test',
'cpu':
'100%',
'disk':
'100G',
'environment':
'dev',
'host_ip':
'172.31.81.67',
'memory':
'100M',
'name':
'proid.myapp#001',
'proid':
'foo',
'shared_network':
False,
'task':
'001',
'uniqueid':
'0000000ID1234',
'archive': ['/var/tmp/treadmill'],
'endpoints': [{
'port': 8000,
'name': 'http',
'real_port': 5000,
'proto': 'tcp',
}, {
'port': 54321,
'type': 'infra',
'name': 'ssh',
'real_port': 54321,
'proto': 'tcp',
}],
'ephemeral_ports': {
'tcp': [45024],
'udp': [62422],
},
'services': [{
'name': 'web_server',
'command': '/bin/false',
'restart': {
'limit': 3,
'interval': 60,
},
}],
}
treadmill.appmgr.manifest.read.return_value = manifest
app_unique_name = 'proid.myapp-001-0000000ID1234'
mock_cgroup_client = self.app_env.svc_cgroup.make_client.return_value
mock_ld_client = self.app_env.svc_localdisk.make_client.return_value
mock_nwrk_client = self.app_env.svc_network.make_client.return_value
localdisk = {
'block_dev': '/dev/foo',
}
mock_ld_client.get.return_value = localdisk
network = {
'vip': '192.168.0.2',
'gateway': '192.168.254.254',
'veth': 'testveth.0',
}
mock_nwrk_client.get.return_value = network
app_dir = os.path.join(self.app_env.apps_dir, app_unique_name)
# Create content in app root directory, verify that it is archived.
fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx'))
fs.mkdir_safe(os.path.join(app_dir, 'services'))
# Simulate daemontools finish script, marking the app is done.
with open(os.path.join(app_dir, 'exitinfo'), 'w') as f:
f.write(yaml.dump({'service': 'web_server', 'rc': 0, 'sig': 0}))
mock_zkclient = kazoo.client.KazooClient()
app_finish.finish(self.app_env, mock_zkclient, app_dir)
self.app_env.watchdogs.create.assert_called_with(
'treadmill.appmgr.finish-' + app_unique_name, '5m', mock.ANY)
treadmill.subproc.check_call.assert_has_calls([
mock.call([
's6-svc',
'-d',
app_dir,
]),
mock.call([
's6-svwait',
'-d',
app_dir,
]),
])
# All resource service clients are properly created
self.app_env.svc_cgroup.make_client.assert_called_with(
os.path.join(app_dir, 'cgroups'))
self.app_env.svc_localdisk.make_client.assert_called_with(
os.path.join(app_dir, 'localdisk'))
self.app_env.svc_network.make_client.assert_called_with(
os.path.join(app_dir, 'network'))
treadmill.appmgr.finish._kill_apps_by_root.assert_called_with(
os.path.join(app_dir, 'root'))
# Verify that we tested the archiving for the app root volume
treadmill.fs.archive_filesystem.assert_called_with(
'/dev/foo', os.path.join(app_dir, 'root'),
os.path.join(app_dir, '001_xxx.xx.com_20150122_141436537918.tar'),
mock.ANY)
# Verify that the file is uploaded by Uploader
app = utils.to_obj(manifest)
treadmill.appmgr.finish._send_container_archive.assert_called_with(
mock_zkclient,
app,
os.path.join(app_dir,
'001_xxx.xx.com_20150122_141436537918.tar.gz'),
)
# Verify that the app folder was deleted
self.assertFalse(os.path.exists(app_dir))
# Cleanup the block device
mock_ld_client.delete.assert_called_with(app_unique_name)
# Cleanup the cgroup resource
mock_cgroup_client.delete.assert_called_with(app_unique_name)
# Cleanup network resources
mock_nwrk_client.get.assert_called_with(app_unique_name)
self.app_env.rules.unlink_rule.assert_has_calls([
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 5000,
'192.168.0.2', 8000),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 54321,
'192.168.0.2', 54321),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 45024,
'192.168.0.2', 45024),
owner=app_unique_name),
mock.call(rule=firewall.DNATRule('udp', '172.31.81.67', 62422,
'192.168.0.2', 62422),
owner=app_unique_name),
])
treadmill.iptables.rm_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:54321'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,tcp:45024'),
mock.call(treadmill.iptables.SET_INFRA_SVC,
'192.168.0.2,udp:62422'),
])
mock_nwrk_client.delete.assert_called_with(app_unique_name)
treadmill.appevents.post.assert_called_with(
mock.ANY,
events.FinishedTraceEvent(instanceid='proid.myapp#001',
rc=0,
signal=0,
payload={
'service': 'web_server',
'sig': 0,
'rc': 0
}))
treadmill.rrdutils.flush_noexc.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'))
shutil.copy.assert_called_with(
os.path.join(self.root, 'metrics', 'apps',
app_unique_name + '.rrd'),
os.path.join(app_dir, 'metrics.rrd'))
def _unshare_network(tm_env, container_dir, app):
"""Configures private app network.
:param ``appenv.AppEnvironment`` tm_env:
Treadmill application environment
"""
unique_name = appcfg.app_unique_name(app)
# Configure DNAT rules while on host network.
for endpoint in app.endpoints:
_LOGGER.info('Creating DNAT rule: %s:%s -> %s:%s',
app.network.external_ip,
endpoint.real_port,
app.network.vip,
endpoint.port)
dnatrule = firewall.DNATRule(proto=endpoint.proto,
dst_ip=app.network.external_ip,
dst_port=endpoint.real_port,
new_ip=app.network.vip,
new_port=endpoint.port)
snatrule = firewall.SNATRule(proto=endpoint.proto,
src_ip=app.network.vip,
src_port=endpoint.port,
new_ip=app.network.external_ip,
new_port=endpoint.real_port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
tm_env.rules.create_rule(chain=iptables.POSTROUTING_SNAT,
rule=snatrule,
owner=unique_name)
# See if this container requires vring service
if app.vring:
_LOGGER.debug('adding %r to VRing set', app.network.vip)
iptables.add_ip_set(
iptables.SET_VRING_CONTAINERS,
app.network.vip
)
# See if this was an "infra" endpoint and if so add it to the whitelist
# set.
if getattr(endpoint, 'type', None) == 'infra':
_LOGGER.debug('adding %s:%s to infra services set',
app.network.vip, endpoint.port)
iptables.add_ip_set(
iptables.SET_INFRA_SVC,
'{ip},{proto}:{port}'.format(
ip=app.network.vip,
proto=endpoint.proto,
port=endpoint.port,
)
)
for port in app.ephemeral_ports.tcp:
_LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s',
app.network.external_ip, port,
app.network.vip, port)
dnatrule = firewall.DNATRule(proto='tcp',
dst_ip=app.network.external_ip,
dst_port=port,
new_ip=app.network.vip,
new_port=port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
# Treat ephemeral ports as infra, consistent with current prodperim
# behavior.
iptables.add_ip_set(iptables.SET_INFRA_SVC,
'{ip},tcp:{port}'.format(ip=app.network.vip,
port=port))
for port in app.ephemeral_ports.udp:
_LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s',
app.network.external_ip, port,
app.network.vip, port)
dnatrule = firewall.DNATRule(proto='udp',
dst_ip=app.network.external_ip,
dst_port=port,
new_ip=app.network.vip,
new_port=port)
tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT,
rule=dnatrule,
owner=unique_name)
# Treat ephemeral ports as infra, consistent with current prodperim
# behavior.
iptables.add_ip_set(iptables.SET_INFRA_SVC,
'{ip},udp:{port}'.format(ip=app.network.vip,
port=port))
# configure passthrough while on main network.
if getattr(app, 'passthrough', None):
_LOGGER.info('adding passthrough for: %r', app.passthrough)
# Resolve all the hosts (+dedup)
new_ips = {
socket.gethostbyname(host)
for host in app.passthrough
}
# Create a passthrough rule from each of the source IP to the
# container IP and record these source IP in a set.
for ipaddr in new_ips:
passthroughrule = firewall.PassThroughRule(
src_ip=ipaddr,
dst_ip=app.network.vip,
)
tm_env.rules.create_rule(chain=iptables.PREROUTING_PASSTHROUGH,
rule=passthroughrule,
owner=unique_name)
# configure exception filter rules
try:
firewall_plugin = plugin_manager.load(
'treadmill.firewall.plugins', 'firewall'
)
firewall_plugin.apply_exception_rules(tm_env, container_dir, app)
except Exception: # pylint: disable=W0703
_LOGGER.exception(
'Error in firewall plugin, skip applying firewall exception rules.'
)
service_ip = None
if app.shared_ip:
service_ip = app.network.external_ip
# Unshare network and create virtual device
newnet.create_newnet(app.network.veth,
app.network.vip,
app.network.gateway,
service_ip)
def run(routing, endpoints, discovery, rulemgr, ip_owner, rules_owner):
"""Manage ring rules based on discovery info.
:param routing:
The map between logical endpoint name and internal container port that
is used for this endpoint.
:param endpoints:
The set of endpoints to monitor.
:param discovery:
The treadmill.discovery object/iterator. Loop over discovery never
ends, and it yields results in a form:
appname:endpoint hostname:port
appname:endpoint
Absense of hostname:port indicates that given endpoint no longer
exists.
:param ``RuleMgr`` rulemgr:
Firewall rule manager instance.
:param ``str`` rules_owner:
Unique name of the container owning all the rules.
:param ``str`` ip_owner:
IP of the container owning of the VRing.
"""
local_host = sysinfo.hostname()
local_ip = socket.gethostbyname(local_host)
_LOGGER.info('Starting vring: %r %r %r %r %r', local_host, ip_owner,
rules_owner, routing, endpoints)
# Add reflective rules back to the container
for endpoint in endpoints:
dnat_rule = firewall.DNATRule(proto=routing[endpoint]['proto'],
src_ip=ip_owner,
dst_ip=local_ip,
dst_port=routing[endpoint]['port'],
new_ip=ip_owner,
new_port=routing[endpoint]['port'])
rulemgr.create_rule(chain=iptables.VRING_DNAT,
rule=dnat_rule,
owner=rules_owner)
vring_state = {}
for (app, hostport) in discovery.iteritems():
# app is in the form appname:endpoint. We care only about endpoint
# name.
_name, proto, endpoint = app.split(':')
# Ignore if endpoint is not in routing (only interested in endpoints
# that are in routing table).
if endpoint not in endpoints:
continue
private_port = int(routing[endpoint]['port'])
if hostport:
host, public_port = hostport.split(':')
if host == local_host:
continue
try:
ipaddr = socket.gethostbyname(host)
except socket.gaierror as err:
_LOGGER.warning('Error resolving %r(%s), skipping.', host, err)
continue
public_port = int(public_port)
vring_route = (proto, ipaddr, public_port)
_LOGGER.info('add vring route: %r', vring_route)
vring_state[app] = vring_route
dnat_rule = firewall.DNATRule(proto=proto,
src_ip=ip_owner,
dst_ip=ipaddr,
dst_port=private_port,
new_ip=ipaddr,
new_port=public_port)
snat_rule = firewall.SNATRule(proto=proto,
src_ip=ipaddr,
src_port=public_port,
dst_ip=ip_owner,
new_ip=ipaddr,
new_port=private_port)
rulemgr.create_rule(chain=iptables.VRING_DNAT,
rule=dnat_rule,
owner=rules_owner)
rulemgr.create_rule(chain=iptables.VRING_SNAT,
rule=snat_rule,
owner=rules_owner)
else:
vring_route = vring_state.pop(app, None)
if not vring_route:
continue
_LOGGER.info('del vring route: %r', vring_route)
proto, ipaddr, public_port = vring_route
dnat_rule = firewall.DNATRule(proto=proto,
src_ip=ip_owner,
dst_ip=ipaddr,
dst_port=private_port,
new_ip=ipaddr,
new_port=public_port)
snat_rule = firewall.SNATRule(
proto=proto,
src_ip=ipaddr,
src_port=public_port,
dst_ip=ip_owner,
new_ip=ipaddr,
new_port=private_port,
)
rulemgr.unlink_rule(chain=iptables.VRING_DNAT,
rule=dnat_rule,
owner=rules_owner)
rulemgr.unlink_rule(chain=iptables.VRING_SNAT,
rule=snat_rule,
owner=rules_owner)
def test__unshare_network_simple(self):
"""Tests unshare network sequence.
"""
# Disable W0212: Access to a protected member
# pylint: disable=W0212
app = utils.to_obj({
'type':
'native',
'name':
'proid.test#0',
'uniqueid':
'ID1234',
'environment':
'dev',
'network': {
'veth': 'id1234.0',
'vip': '192.168.1.1',
'gateway': '192.168.254.254',
'external_ip': '172.31.81.67',
},
'shared_ip':
True,
'ephemeral_ports': {
'tcp': [],
'udp': [],
},
'endpoints': [{
'real_port': '5007',
'proto': 'tcp',
'port': '22',
'type': 'infra'
}, {
'real_port': '5013',
'proto': 'udp',
'port': '12345'
}],
'vring': {
'some': 'data'
}
})
app_unique_name = appcfg.app_unique_name(app)
treadmill.runtime.linux._run._unshare_network(self.tm_env,
'test_container_dir',
app)
treadmill.iptables.add_ip_set.assert_has_calls([
mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.1.1'),
mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.1.1,tcp:22'),
],
any_order=True)
self.tm_env.rules.create_rule.assert_has_calls([
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='tcp',
dst_ip='172.31.81.67',
dst_port='5007',
new_ip='192.168.1.1',
new_port='22'),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='tcp',
src_ip='192.168.1.1',
src_port='22',
new_ip='172.31.81.67',
new_port='5007'),
owner=app_unique_name),
mock.call(chain=iptables.PREROUTING_DNAT,
rule=firewall.DNATRule(proto='udp',
dst_ip='172.31.81.67',
dst_port='5013',
new_ip='192.168.1.1',
new_port='12345'),
owner=app_unique_name),
mock.call(chain=iptables.POSTROUTING_SNAT,
rule=firewall.SNATRule(proto='udp',
src_ip='192.168.1.1',
src_port='12345',
new_ip='172.31.81.67',
new_port='5013'),
owner=app_unique_name)
],
any_order=True)
self.assertEqual(self.tm_env.rules.create_rule.call_count, 4)
treadmill.newnet.create_newnet.assert_called_with(
'id1234.0',
'192.168.1.1',
'192.168.254.254',
'172.31.81.67',
)
def _cleanup_network(tm_env, app, network_client):
"""Cleanup the network part of a container.
"""
# Generate a unique name for the app
unique_name = appmgr.app_unique_name(app)
try:
app_network = network_client.get(unique_name)
except services.ResourceServiceError:
_LOGGER.warning('network never allocated')
return
if app_network is None:
_LOGGER.info('Network resource already freed')
return
# Unconfigure passthrough
if hasattr(app, 'passthrough'):
_LOGGER.info('Deleting passthrough for: %r', app.passthrough)
# Resolve all the hosts
# FIXME: There is no guarantie the hosts will resolve to
# the same IPs as they did during creation.
ips = set([socket.gethostbyname(host) for host in app.passthrough])
for ip in ips:
tm_env.rules.unlink_rule(
rule=firewall.PassThroughRule(src_ip=ip,
dst_ip=app_network['vip']),
owner=unique_name,
)
for endpoint in app.endpoints:
tm_env.rules.unlink_rule(
rule=firewall.DNATRule(proto=endpoint.proto,
orig_ip=app.host_ip,
orig_port=endpoint.real_port,
new_ip=app_network['vip'],
new_port=endpoint.port),
owner=unique_name,
)
# See if this was an "infra" endpoint and if so remove it
# from the whitelist set.
if getattr(endpoint, 'type', None) == 'infra':
_LOGGER.debug('removing %s:%s from infra services set',
app_network['vip'], endpoint.port)
iptables.rm_ip_set(
iptables.SET_INFRA_SVC, '{ip},{proto}:{port}'.format(
ip=app_network['vip'],
proto=endpoint.proto,
port=endpoint.port,
))
_cleanup_ports(tm_env, unique_name, app_network['vip'],
app.ephemeral_ports.tcp, 'tcp')
_cleanup_ports(tm_env, unique_name, app_network['vip'],
app.ephemeral_ports.udp, 'udp')
# Terminate any entries in the conntrack table
iptables.flush_conntrack_table(app_network['vip'])
# Cleanup network resources
network_client.delete(unique_name)
