def test_intersect_rules():
r1 = [
Rule(src=ipset('1.0.0.0/24'),
dst=ipset('0.0.0.0/0'),
app='app',
name='r1'),
]
r2 = [
Rule(src=ipset('1.0.0.10', '1.0.0.14'),
dst=ipset('0.0.0.0/0'),
app='app',
name='r2'),
]
r3 = [
Rule(src=ipset('1.0.0.11', '1.0.0.14'),
dst=ipset('0.0.0.0/0'),
app='app',
name='r3'),
]
eq_(
process.intersect_rules([r1, r2, r3], ipset('0.0.0.0/0'),
ipset('2.0.0.0/8')), [
Rule(src=ipset('1.0.0.14'),
dst=ipset('2.0.0.0/8'),
app='app',
name='r1+r2+r3'),
])
def test_overlapping_rules():
lga_rules = {
'app': [
Rule(ipset('1.1.0.0/16'), ipset('2.0.0.0/8'), 'app', 'lga'),
]
}
ord_rules = {
'app': [
Rule(ipset('1.0.0.0/8'), ipset('2.1.0.0/16'), 'app', 'ord'),
]
}
address_spaces = {
'lga': ipset('1.0.0.0/8'),
'ord': ipset('2.0.0.0/8'),
}
sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules}
with no_simplify():
result = process.combine(address_spaces, routes, sources)
eq_(
result,
{
'app': [
# takes the intersection of both rules:
Rule(ipset('1.1.0.0/16'), ipset('2.1.0.0/16'), 'app',
'lga+ord'),
]
})
def test_combine():
address_spaces = {
'ten':
ipset('10.0.0.0/8'),
'twenty':
ipset('20.0.0.0/8'),
'unmanaged':
ipset('0.0.0.0/0') - ipset('10.0.0.0/8') - ipset('20.0.0.0/8'),
}
routes = {
('ten', 'ten'): ['fw1.ten'],
('ten', 'twenty'): ['fw1.ten', 'fw1.twenty'],
('ten', 'unmanaged'): ['fw1.ten'],
('twenty', 'ten'): ['fw1.ten', 'fw1.twenty'],
('twenty', 'twenty'): ['fw1.twenty'],
('twenty', 'unmanaged'): ['fw1.twenty'],
('unmanaged', 'ten'): ['fw1.ten'],
('unmanaged', 'twenty'): ['fw1.twenty'],
('unmanaged', 'unmanaged'): [],
}
sources = {
'fw1.ten': RULES_10,
'fw1.twenty': RULES_20,
}
res = process.combine(address_spaces, routes, sources)
res['http'].sort()
eq_(
res,
{
'http':
sorted([
Rule(src=ipset('10.10.0.0/16'),
dst=ipset('10.20.0.0/16', '30.20.0.0/16'),
app='http',
name='10->10+10->30'),
Rule(src=ipset('20.10.0.0/16'),
dst=ipset('20.20.0.0/16', '30.20.0.0/16'),
app='http',
name='20->20+20->30'),
Rule(src=ipset('30.10.0.0/16'),
dst=ipset('10.20.0.0/16', '20.20.0.0/16'),
app='http',
name='30->10+30->20'),
# note that only the intersection of these flows makes it through
Rule(src=ipset('10.20.0.0/16'),
dst=ipset('20.20.0.0/16'),
app='http',
name='10->20'),
Rule(src=ipset('20.20.0.0/16'),
dst=ipset('10.20.0.0/16'),
app='http',
name='20->10'),
]),
})
def test_one_address_space():
rules = {
'app': [
Rule(ipset('1.2.3.4'), ipset('1.7.7.7'), 'app', 'p2p'),
Rule(ipset('1.2.5.0/24'), ipset('1.7.7.7'), 'app', 'net'),
]
}
with no_simplify():
result = process.combine({'nyc': ipset('1.0.0.0/8')},
{('nyc', 'nyc'): ['fw1.nyc']},
{'fw1.nyc': rules})
eq_(sorted(result), sorted(rules))
def test_rules_from_to_intersection():
rules = [
Rule(src=ipset('0.0.0.0/7'),
dst=ipset('2.0.1.0/24'),
app='app',
name='r'),
]
eq_(process.rules_from_to(rules, ipset('1.0.0.0/8'), ipset('2.0.0.0/8')), [
Rule(src=ipset('1.0.0.0/8'),
dst=ipset('2.0.1.0/24'),
app='app',
name='r'),
])
def process_rules(app_map, policies, zone_nets, policies_by_zone_pair,
src_per_policy, dst_per_policy):
logger.info("processing rules")
# turn policies into a list of Rules (permit only), limited by zone,
# that do not overlap. The tricky bit here is processing policies in
# order and accounting for denies. We do this once for each
# (from_zone, to_zone, app) tuple. The other tricky bit is handling
# the application "any", which we treat as including all applications
# used anywhere, and also record in a special "@@other" app.
rules_by_app = {}
all_apps = set(itertools.chain(*[p.applications for p in policies]))
all_apps = all_apps | set(app_map.keys())
all_apps.discard('any')
global_policies = policies_by_zone_pair.get((None, None), [])
for from_zone, to_zone in itertools.product(zone_nets, zone_nets):
zpolicies = sorted(policies_by_zone_pair.get((from_zone, to_zone), []),
key=lambda p: p.sequence)
zpolicies += global_policies
logger.debug(" from-zone %s to-zone %s (%d policies)", from_zone,
to_zone, len(zpolicies))
rule_count = 0
apps = set(itertools.chain(*[p.applications for p in zpolicies]))
if 'any' in apps:
apps = all_apps
for app in apps | set(['@@other']):
mapped_app = app_map[app]
# for each app, count down the IP pairs that have not matched a
# rule yet, starting with the zones' IP spaces. This simulates sequential
# processing of the policies.
remaining_pairs = IPPairs(
(zone_nets[from_zone], zone_nets[to_zone]))
rules = rules_by_app.setdefault(mapped_app, [])
for pol in zpolicies + global_policies:
if app not in pol.applications and 'any' not in pol.applications:
continue
src = src_per_policy[pol]
dst = dst_per_policy[pol]
# if the policy is a "permit", add rules for each
# src/destination pair
if pol.action == 'permit':
for s, d in remaining_pairs:
s = s & src
d = d & dst
if len(s) and len(d):
rules.append(Rule(s, d, mapped_app, pol.name))
rule_count += 1
# regardless, consider this src/dst pair matched
remaining_pairs = remaining_pairs - IPPairs((src, dst))
# if we've matched everything, we're done
if not remaining_pairs:
break
logger.debug(" from-zone %s to-zone %s => %d rules", from_zone,
to_zone, rule_count)
# only include @@other if it's used
if not rules_by_app['@@other']:
del rules_by_app['@@other']
# simplify and return the result
return simplify_rules(rules_by_app)
def test_limited_by_space():
lax_rules = {'app': []}
lga_rules = {
'app': [
# /7 covers both lax and lga
Rule(ipset('0.0.0.0/7'), ipset('2.0.0.0/8'), 'app', 'lga'),
]
}
ord_rules = {
'app': [
Rule(ipset('0.0.0.0/7'), ipset('2.0.0.0/8'), 'app', 'ord'),
]
}
address_spaces = {
'lax': ipset('0.0.0.0/8'),
'lga': ipset('1.0.0.0/8'),
'ord': ipset('2.0.0.0/8'),
}
routes = {
('lax', 'lax'): ['fw1.lax'],
('lax', 'lga'): ['fw1.lga', 'fw1.lax'],
('lax', 'ord'): ['fw1.ord', 'fw1.lax'],
('lga', 'lax'): ['fw1.lax', 'fw1.lga'],
('lga', 'lga'): ['fw1.lga'],
('lga', 'ord'): ['fw1.ord', 'fw1.lga'],
('ord', 'lax'): ['fw1.ord', 'fw1.lax'],
('ord', 'lga'): ['fw1.ord', 'fw1.lga'],
('ord', 'ord'): ['fw1.ord'],
}
sources = {
'fw1.ord': ord_rules,
'fw1.lga': lga_rules,
'fw1.lax': lax_rules
}
with no_simplify():
result = process.combine(address_spaces, routes, sources)
eq_(
result,
{
'app': [
# only lga's address space is allowed
Rule(ipset('1.0.0.0/8'), ipset('2.0.0.0/8'), 'app',
'lga+ord'),
]
})
def test_rules_from_to_no_match_dst():
rules = [
Rule(src=ipset('1.0.0.0/8'),
dst=ipset('3.0.0.0/24'),
app='app',
name='r'),
]
eq_(process.rules_from_to(rules, ipset('1.0.0.0/8'), ipset('2.0.0.0/8')),
[])
def test_nonoverlapping_rules():
lga_rules = {
'app': [
Rule(ipset('1.2.5.0/24'), ipset('2.2.5.0/24'), 'app', 'lga'),
]
}
ord_rules = {
'app': [
Rule(ipset('2.7.7.0/24'), ipset('1.7.7.0/24'), 'app', 'ord'),
]
}
address_spaces = {
'ord': ipset('2.0.0.0/8'),
'lga': ipset('1.0.0.0/8'),
}
sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules}
with no_simplify():
result = process.combine(address_spaces, routes, sources)
eq_(result, {})
def make_rules(sgid, local):
sg = security_groups[sgid]
for dir, sgrules in [('in', sg.rules), ('out', sg.rules_egress)]:
for sgrule in sgrules:
if sgrule.app == '*/any':
apps = all_apps | set(['@@other'])
else:
apps = [sgrule.app]
for app in apps:
for grant in sgrule.grants:
if grant.cidr_ip:
remote = IPSet([IP(grant.cidr_ip)])
else:
remote = ips_by_sg.get(grant.group_id, None)
if not remote:
continue
src, dst = (remote, local) if dir == 'in' else (local,
remote)
name = "{}/{}".format(sg.name, dir)
# first make rules involving non-managed space, leaving
# only managed-to-managed
if dir == 'in':
unmanaged_src = src & unmanaged_ip_space
if unmanaged_src:
rules.setdefault(app, []).append(
Rule(src=unmanaged_src,
dst=dst,
app=app,
name=name))
src = src & managed_ip_space
else:
unmanaged_dst = dst & unmanaged_ip_space
if unmanaged_dst:
rules.setdefault(app, []).append(
Rule(src=src,
dst=unmanaged_dst,
app=app,
name=name))
dst = dst & managed_ip_space
if src and dst:
to_intersect.setdefault(app, {}).setdefault(
dir, []).append((src, dst, name))
def rules_from_to(rules, local_sp, remote_sp):
rv = []
for r in rules:
src = r.src & local_sp
if not src:
continue
dst = r.dst & remote_sp
if not dst:
continue
rv.append(Rule(src=src, dst=dst, app=r.app, name=r.name))
return rv
def test_identical_rules():
rules = {
'app': [
Rule(ipset('2.7.7.0/24'), ipset('1.7.7.0/24'), 'app', 'lga-ord'),
]
}
address_spaces = {
'lga': ipset('1.0.0.0/8'),
'ord': ipset('2.0.0.0/8'),
}
sources = {'fw1.ord': rules, 'fw1.lga': rules}
with no_simplify():
result = process.combine(address_spaces, routes, sources)
eq_(result, rules)
def combine(address_spaces, routes, sources):
# get the set of all apps
all_apps = set()
for rules in sources.itervalues():
all_apps = all_apps | set(rules)
# for each address space, add any apps which aren't explicitly specified in that
# address space, but *are* specified in the combined ruleset, as copies of that
# address space's '@@other' app. This ensures that each space has the same set
# of apps.
for rules in sources.itervalues():
other = rules.get('@@other', [])
missing_apps = all_apps - set(rules)
for app in missing_apps:
rules[app] = [
Rule(src=r.src, dst=r.dst, app=app, name=r.name) for r in other
]
combined_rules = {}
for app in all_apps:
logger.info("combining app %s", app)
# The idea here is this: for each pair of address spaces, look at the
# set of rules specified in the routes. Only write combined rules for
# flows for which are allowed by all rulesets.
for local_sp_name, local_sp in address_spaces.iteritems():
for remote_sp_name, remote_sp in address_spaces.iteritems():
source_names = routes[local_sp_name, remote_sp_name]
if not source_names:
continue
logger.debug(" from %s to %s using %s", local_sp_name,
remote_sp_name, ', '.join(source_names))
rulesets = [sources[n][app] for n in source_names]
# if we only have one source, this is pretty easy:
# just limit each rule to the relevant IP spaces; otherwise
# we need to do a recursive intersection
if len(rulesets) == 1:
new_rules = rules_from_to(rulesets[0], local_sp, remote_sp)
else:
new_rules = intersect_rules(rulesets, local_sp, remote_sp)
if new_rules:
combined_rules.setdefault(app, []).extend(new_rules)
rules = simplify_rules(combined_rules)
return rules
def intersect_rules(rulesets, local_sp, remote_sp):
# combine the rulesets into an accumulator seeded with the first
# ruleset
acc = rules_from_to(rulesets.pop(), local_sp, remote_sp)
while rulesets:
rs = rules_from_to(rulesets.pop(), local_sp, remote_sp)
intersected = []
for rl in acc:
for rr in rs:
src = rl.src & rr.src
if not src:
continue
dst = rl.dst & rr.dst
if not dst:
continue
intersected.append(
Rule(src=src,
dst=dst,
app=rl.app,
name=combine_names(rl.name, rr.name)))
acc = intersected
return acc
def test_process_global_priority():
# test the relative priority of "regular" (per-zone) and global policies
policies = [
mkpol(name='ok',
from_zone='pvt',
to_zone='pub',
src_addrs=ipset('192.168.1.2/31'),
dst_addrs=ipset('0.0.0.0/0'),
applications=['junos-ssh'],
sequence=100),
mkpol(name='deny-all-global',
from_zone=None,
to_zone=None,
src_addrs=ipset('0.0.0.0/0'),
dst_addrs=ipset('0.0.0.0/0'),
applications=['junos-ssh'],
sequence=3,
action='deny'),
mkpol(name='ok-global',
from_zone=None,
to_zone=None,
src_addrs=ipset('192.168.1.128/32'),
dst_addrs=ipset('128.135.0.0/16'),
applications=['junos-ssh'],
sequence=1),
]
res = call_process_rules(APP_MAP, policies, ZONE_NETS)
exp = {
'junos-ssh': [
Rule(src=ipset('192.168.1.128/32', '192.168.1.2/31'),
dst=ipset('128.135.0.0/16'),
app='junos-ssh',
name='ok+ok-global'),
],
}
[ruleset.sort() for ruleset in exp.itervalues()]
eq_(res, exp)
]),
('dynamic', '172.16.2.0/24', [
('dynhost', '172.16.2.17', ['admin_ssh']),
('dynhost', '172.16.2.20', ['admin_ssh']),
]),
('perhost', '172.16.3.0/24', [
('server4', '172.16.3.1', ['admin_ssh']),
]),
]),
]),
]
RULES = {
'ssh': [
Rule(src=IPSet([IP('172.16.1.0/24'), IP('172.16.3.0/24')]) - IPSet([IP('172.16.1.1'), IP('172.16.1.2'), IP('172.16.3.1')]),
dst=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('172.16.1.0/24'), IP('172.16.2.0/24'), IP('172.16.3.0/24')]),
app='ssh', name='unoccupied/out'),
Rule(src=IPSet([IP('192.168.10.0/24')]),
dst=IPSet([IP('172.16.1.1'), IP('172.16.1.2'), IP('172.16.2.0/24'), IP('172.16.3.1')]),
app='ssh', name='admin_ssh/in'),
],
'web': [
Rule(src=IPSet([IP('172.16.1.0/24'), IP('172.16.3.0/24')]) - IPSet([IP('172.16.1.1'), IP('172.16.1.2'), IP('172.16.3.1')]),
dst=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('172.16.1.0/24'), IP('172.16.2.0/24'), IP('172.16.3.0/24')])
+ IPSet([IP('172.16.1.1'), IP('172.16.1.2')]),
app='web', name='unoccupied/out+webapp/in'),
Rule(src=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('172.16.1.0/24'), IP('172.16.2.0/24'), IP('172.16.3.0/24')]),
dst=IPSet([IP('172.16.1.1'), IP('172.16.1.2')]),
app='web', name='webapp/in'),
],
}
def test_process_rules_any_app():
# test basic functionality + the "any" app
policies = [
mkpol(name='dbssh',
from_zone='pvt',
to_zone='pvt',
src_addrs=ipset('192.168.1.2/31'),
dst_addrs=ipset('0.0.0.0/0'),
applications=['junos-ssh', 'junos-ping'],
sequence=1),
mkpol(name='admin',
from_zone='pvt',
to_zone='pvt',
src_addrs=ipset('192.168.1.128/32'),
dst_addrs=ipset('0.0.0.0/0'),
applications=['any'],
sequence=2),
mkpol(name='admin',
from_zone='pvt',
to_zone='dmz',
src_addrs=ipset('192.168.1.128/32'),
dst_addrs=ipset('0.0.0.0/0'),
applications=['any']),
mkpol(name='admin',
from_zone='pvt',
to_zone='pub',
src_addrs=ipset('192.168.1.128/32'),
dst_addrs=ipset('0.0.0.0/0'),
applications=['any']),
mkpol(name='http',
from_zone='pub',
to_zone='dmz',
src_addrs=ipset('0.0.0.0/0'),
dst_addrs=ipset('10.1.10.0/24'),
applications=['web']),
mkpol(name='db',
from_zone='dmz',
to_zone='pvt',
src_addrs=ipset('10.1.10.0/24'),
dst_addrs=ipset('192.168.1.2/31'),
applications=['db']),
]
res = call_process_rules(APP_MAP, policies, ZONE_NETS)
exp = {
'junos-ping': [
Rule(src=ipset('192.168.1.128'),
dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'),
app='junos-ping',
name='admin'),
Rule(src=ipset('192.168.1.2/31'),
dst=ipset('192.168.0.0/16'),
app='junos-ping',
name='dbssh'),
],
'web': [
Rule(src=ipset('192.168.1.128'),
dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'),
app='web',
name='admin'),
Rule(src=ipset('128.135.0.0/16'),
dst=ipset('10.1.10.0/24'),
app='web',
name='http'),
],
'db': [
Rule(src=ipset('192.168.1.128'),
dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'),
app='db',
name='admin'),
Rule(src=ipset('10.1.10.0/24'),
dst=ipset('192.168.1.2/31'),
app='db',
name='db'),
],
'junos-ssh': [
Rule(src=ipset('192.168.1.128'),
dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'),
app='junos-ssh',
name='admin'),
Rule(src=ipset('192.168.1.2/31'),
dst=ipset('192.168.0.0/16'),
app='junos-ssh',
name='dbssh'),
],
'@@other': [
Rule(src=ipset('192.168.1.128'),
dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'),
app='@@other',
name='admin'),
],
}
[ruleset.sort() for ruleset in exp.itervalues()]
eq_(res, exp)
def r(src, dst, app='testapp', name='n'):
return Rule(src=_ipset(src), dst=_ipset(dst), app=app, name=name)
def test_other_app():
ord_rules = {
'ordonly': [
Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), 'ordonly', 'ordonly'),
],
'inboth': [
Rule(ipset('1.1.8.8'), ipset('1.1.9.9'), 'inboth', 'inboth_ord'),
],
'@@other': [
Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), '@@other', 'ordother'),
],
}
lga_rules = {
'lgaonly': [
Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), 'lgaonly', 'lgaonly'),
],
'inboth': [
Rule(ipset('65.1.8.8'), ipset('65.1.9.9'), 'inboth', 'inboth_lga'),
],
'@@other': [
Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), '@@other', 'lgaother'),
],
}
address_spaces = {
'ord': ipset('0.0.0.0/2'),
'lga': ipset('64.0.0.0/2'),
}
sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules}
with no_simplify():
result = process.combine(address_spaces, routes, sources)
for apprules in result.itervalues():
apprules.sort()
eq_(
result, {
'ordonly':
sorted([
Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), 'ordonly',
'ordonly'),
Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), 'ordonly',
'lgaother'),
]),
'lgaonly':
sorted([
Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), 'lgaonly',
'lgaonly'),
Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), 'lgaonly',
'ordother'),
]),
'inboth':
sorted([
Rule(ipset('1.1.8.8'), ipset('1.1.9.9'), 'inboth',
'inboth_ord'),
Rule(ipset('65.1.8.8'), ipset('65.1.9.9'), 'inboth',
'inboth_lga'),
]),
'@@other':
sorted([
Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), '@@other',
'ordother'),
Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), '@@other',
'lgaother'),
]),
})
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from nose.tools import eq_
from fwunit.types import Rule
from fwunit.combine import process
from fwunit.test.util import ipset
RULES_10 = {
'http': [
# within this ip space
Rule(src=ipset('10.10.0.0/16'),
dst=ipset('10.20.0.0/16'),
app='http',
name='10->10'),
# from and to "unmanaged" space
Rule(src=ipset('30.10.0.0/16'),
dst=ipset('10.20.0.0/16'),
app='http',
name='30->10'),
Rule(src=ipset('10.10.0.0/16'),
dst=ipset('30.20.0.0/16'),
app='http',
name='10->30'),
# from and to the 20/8 space
Rule(src=ipset('20.20.0.0/16', '20.30.0.0/16'),
dst=ipset('10.20.0.0/16', '10.30.0.0/16'),
app='http',
name='20->10'),
Rule(src=ipset('10.10.0.0/16', '10.20.0.0/16'),
def get_rules(aws, app_map, regions, dynamic_subnets):
if not regions:
logger.info("Getting all regions")
regions = aws.all_regions()
logger.info("collecting subnets")
subnets = []
managed_ip_space = IPSet([])
for id, subnet in aws.get_all_subnets(regions).iteritems():
name = subnet.tags.get('Name', id)
dynamic = name in dynamic_subnets or id in dynamic_subnets
cidr_block = IP(subnet.cidr_block)
subnet = Subnet(cidr_block=cidr_block, name=name, dynamic=dynamic)
subnets.append(subnet)
managed_ip_space = managed_ip_space + IPSet([cidr_block])
unmanaged_ip_space = IPSet([IP('0.0.0.0/0')]) - managed_ip_space
logger.info("collecting dynamic subnet IP ranges")
dynamic_ipsets = {}
per_host_subnet_ips = IPSet()
for subnet in subnets:
if subnet.dynamic:
ipset = dynamic_ipsets.get(subnet.name, IPSet([]))
ipset += IPSet([subnet.cidr_block])
dynamic_ipsets[subnet.name] = ipset
else:
per_host_subnet_ips += IPSet([subnet.cidr_block])
# sort by IP subnet, so we can use a binary search
logger.info("sorting subnets by IP")
subnets.sort(key=lambda s: s.cidr_block)
_subnet_blocks = [s.cidr_block for s in subnets]
def subnet_by_ip(ip):
i = bisect.bisect_right(_subnet_blocks, ip)
if i and ip in _subnet_blocks[i - 1]:
return subnets[i - 1]
logger.info("examining instances")
sgids_by_dynamic_subnet = {} # {subnet name: set of SecurityGroupIds}
sgids_by_instance = {} # {instance_name: [ip, set of SecurityGroupIds]}
all_sgids = set()
ips_by_sg = {} # {group id: IPSet}
for id, instance in aws.get_all_instances(regions).iteritems():
if instance.state == 'terminated' or instance.state == 'shutting-down':
continue # meh, who cares
if not instance.vpc_id:
continue # not in vpc; ignored
if not instance.private_ip_address:
logger.debug(
"ignoring instance with no private_ip_address: %s, tags %r",
instance.id, instance.tags)
continue
ip = IP(instance.private_ip_address)
for g in instance.groups:
ips_by_sg[g.id] = ips_by_sg.get(g.id, IPSet([])) + IPSet([IP(ip)])
subnet = subnet_by_ip(ip)
if not subnet:
logger.debug(
"ignoring instance with no matching subnet for %s: %s, tags %r",
ip, instance.id, instance.tags)
continue
if subnet.dynamic:
sgset = sgids_by_dynamic_subnet.setdefault(subnet.name, set())
else:
inst_name = instance.tags.get('Name', instance.id)
if inst_name in sgids_by_instance:
inst_name = inst_name + ' ({})'.format(instance.id)
sgset = set()
sgids_by_instance[inst_name] = [ip, sgset]
new_sgids = set(
SecurityGroupId(g.id, instance.region.name)
for g in instance.groups)
sgset.update(new_sgids)
all_sgids.update(new_sgids)
logger.info("accumulating security groups")
all_apps = set(app_map.values())
security_groups = {}
for sgid in all_sgids:
sg = security_groups[sgid] = aws.get_security_group(sgid)
assert sg, "no security group with id {}".format(sgid)
# pre-process all of the rules' apps now
for sgrule in itertools.chain(sg.rules, sg.rules_egress):
proto = str(sgrule.ip_protocol)
if proto == '-1':
proto = 'any'
if sgrule.from_port == sgrule.to_port:
if str(sgrule.from_port) in ("None", "-1"):
app = "*/{}".format(proto)
else:
app = '{}/{}'.format(sgrule.from_port, proto)
else:
app = '{}-{}/{}'.format(sgrule.from_port, sgrule.to_port,
proto)
app = app_map[app]
sgrule.app = app
all_apps.add(app)
rules = {}
to_intersect = {}
def make_rules(sgid, local):
sg = security_groups[sgid]
for dir, sgrules in [('in', sg.rules), ('out', sg.rules_egress)]:
for sgrule in sgrules:
if sgrule.app == '*/any':
apps = all_apps | set(['@@other'])
else:
apps = [sgrule.app]
for app in apps:
for grant in sgrule.grants:
if grant.cidr_ip:
remote = IPSet([IP(grant.cidr_ip)])
else:
remote = ips_by_sg.get(grant.group_id, None)
if not remote:
continue
src, dst = (remote, local) if dir == 'in' else (local,
remote)
name = "{}/{}".format(sg.name, dir)
# first make rules involving non-managed space, leaving
# only managed-to-managed
if dir == 'in':
unmanaged_src = src & unmanaged_ip_space
if unmanaged_src:
rules.setdefault(app, []).append(
Rule(src=unmanaged_src,
dst=dst,
app=app,
name=name))
src = src & managed_ip_space
else:
unmanaged_dst = dst & unmanaged_ip_space
if unmanaged_dst:
rules.setdefault(app, []).append(
Rule(src=src,
dst=unmanaged_dst,
app=app,
name=name))
dst = dst & managed_ip_space
if src and dst:
to_intersect.setdefault(app, {}).setdefault(
dir, []).append((src, dst, name))
logger.info("writing rules for dynamic subnets")
for subnet_name, sgids in sgids_by_dynamic_subnet.iteritems():
subnet = dynamic_ipsets[subnet_name]
logger.debug(" subnet %s, %s", subnet_name, subnet)
for sgid in sgids:
make_rules(sgid, subnet)
logger.info("writing rules for instances in per-host subnets")
per_host_host_ips = IPSet()
for inst_name, info in sgids_by_instance.iteritems():
ip, sgids = info
logger.debug(" instance %s at %s", inst_name, ip)
host_ip = IPSet([ip])
per_host_host_ips += host_ip
for sgid in sgids:
make_rules(sgid, host_ip)
logger.info(
"assuming unrestricted outbound access from unoccupied IPs in per-host subnets"
)
unoccupied = per_host_subnet_ips - per_host_host_ips
for app in all_apps:
rules.setdefault(app, []).append(
Rule(src=unoccupied,
dst=unmanaged_ip_space,
app=app,
name='unoccupied/out'))
to_intersect.setdefault(app, {}).setdefault('out', []).append(
(unoccupied, managed_ip_space, 'unoccupied/out'))
# traffic within the manage Ip space is governed both by outbound rules on
# the source and inbound rules on the destination.
logger.info("intersecting inbound and outbound rules")
for app, dirs in to_intersect.iteritems():
in_rules = dirs.get('in', [])
out_rules = dirs.get('out', [])
logger.debug("..for %s", app)
new_rules = []
for inr in in_rules:
for outr in out_rules:
src = inr[0] & outr[0]
if not src:
continue
dst = inr[1] & outr[1]
if not dst:
continue
new_rules.append(
Rule(src=src,
dst=dst,
app=app,
name=combine_names(inr[2], outr[2])))
# simplify now, within this app, to save space and time
new_rules = simplify_rules({app: new_rules})[app]
rules.setdefault(app, []).extend(new_rules)
rules = simplify_rules(rules)
return rules
def test_run_global_policies_and_addrbook():
fake_cfg = {
'firewall': 'fw',
'ssh_username': '******',
'ssh_password': '******',
'application-map': {
'junos-ssh': 'ssh',
'junos-ping': 'ping'
},
}
z = F.add_zone('untrust')
F.add_address(z, 'untrust-host', '9.0.9.1/32')
F.add_interface(z, 'reth0')
z = F.add_zone('trust')
F.add_address(z, 'trust-host', '10.9.1.1/32')
F.add_interface(z, 'reth1')
ab = F.add_addrbook('global')
F.add_address(ab, 'global-host', '10.1.1.1/32')
ab = F.add_addrbook('public')
F.add_address(ab, 'public-host', '2.2.1.1/32')
F.add_attach(ab, 'trust')
F.add_attach(ab, 'untrust')
F.add_policy(('trust', 'untrust'),
dict(sequence=1,
name='ssh',
src='trust-host',
dst='untrust-host',
app='junos-ssh',
action='permit'))
F.add_policy(('untrust', 'trust'),
dict(sequence=1,
name='puppet',
src='public-host',
dst='global-host',
app='puppet',
action='permit'))
F.add_policy(
'global',
dict(sequence=10,
name='ping',
src='any',
dst='any',
app='junos-ping',
action='permit'))
F.add_policy(
'global',
dict(sequence=11,
name='deny',
src='any',
dst='any',
app='any',
action='deny'))
rules = scripts.run(fake_cfg, {})
for r in rules.itervalues():
r.sort()
exp = {
'ping': [
Rule(src=IPSet([IP('0.0.0.0/0')]),
dst=IPSet([IP('0.0.0.0/0')]),
app='ping',
name='ping'),
],
'puppet': [
Rule(src=IPSet([IP('2.2.1.1')]),
dst=IPSet([IP('10.1.1.1')]),
app='puppet',
name='puppet'),
],
'ssh': [
Rule(src=IPSet([IP('10.9.1.1')]),
dst=IPSet([IP('9.0.9.1')]),
app='ssh',
name='ssh'),
],
}
eq_(rules, exp)
def test_multiple_matches():
lga_rules = {
'app': [
Rule(ipset('1.1.1.1'), ipset('2.0.0.0/8'), 'app', 'one'),
Rule(ipset('1.1.1.2'), ipset('2.0.0.0/8'), 'app', 'two'),
Rule(ipset('1.1.1.3'), ipset('2.0.0.0/8'), 'app', 'three'),
]
}
ord_rules = {
'app': [
Rule(ipset('1.0.0.0/8'), ipset('2.7.8.8'), 'app', 'eight'),
Rule(ipset('1.0.0.0/8'), ipset('2.7.8.9'), 'app', 'nine'),
]
}
address_spaces = {
'lga': ipset('1.0.0.0/8'),
'ord': ipset('2.0.0.0/8'),
}
sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules}
with no_simplify():
result = process.combine(address_spaces, routes, sources)
eq_(
sorted(result['app']),
sorted([
# takes the intersection of all rules:
Rule(src=ipset('1.1.1.1'),
dst=ipset('2.7.8.8'),
app='app',
name='eight+one'),
Rule(src=ipset('1.1.1.1'),
dst=ipset('2.7.8.9'),
app='app',
name='nine+one'),
Rule(src=ipset('1.1.1.2'),
dst=ipset('2.7.8.8'),
app='app',
name='eight+two'),
Rule(src=ipset('1.1.1.2'),
dst=ipset('2.7.8.9'),
app='app',
name='nine+two'),
Rule(src=ipset('1.1.1.3'),
dst=ipset('2.7.8.8'),
app='app',
name='eight+three'),
Rule(src=ipset('1.1.1.3'),
dst=ipset('2.7.8.9'),
app='app',
name='nine+three')
]))
def test_run_no_globals():
fake_cfg = {
'firewall': 'fw',
'ssh_username': '******',
'ssh_password': '******',
'application-map': {
'junos-ssh': 'ssh',
'junos-ping': 'ping'
},
}
z = F.add_zone('untrust')
F.add_address(z, 'host1', '9.0.9.1/32')
F.add_address(z, 'host2', '9.0.9.2/32')
F.add_address(z, 'puppet', '9.0.9.2/32')
F.add_address_set(z, 'hosts', 'host1', 'host2')
F.add_interface(z, 'reth0')
z = F.add_zone('trust')
F.add_address(z, 'trustedhost', '10.0.9.2/32')
F.add_address(z, 'dmz', '10.1.0.0/16')
F.add_address(z, 'shadow', '10.1.99.99/32')
F.add_interface(z, 'reth1')
F.add_policy(('trust', 'trust'),
dict(sequence=1,
name='ssh',
src='any',
dst='any',
app='junos-ssh',
action='permit'))
F.add_policy(('trust', 'trust'),
dict(sequence=10,
name='deny',
src='any',
dst='any',
app='any',
action='deny'))
F.add_policy(('trust', 'untrust'),
dict(sequence=1,
name='ssh',
src='any',
dst='any',
app='junos-ssh',
action='permit'))
F.add_policy(('trust', 'untrust'),
dict(sequence=2,
name='puppet',
src='any',
dst='puppet',
app='puppet',
action='permit'))
F.add_policy(('trust', 'untrust'),
dict(sequence=10,
name='deny',
src='any',
dst='any',
app='any',
action='deny'))
F.add_policy(('untrust', 'trust'),
dict(sequence=1,
name='no-shadow-ping',
src='any',
dst='shadow',
app='junos-ping',
action='deny'))
F.add_policy(('untrust', 'trust'),
dict(sequence=2,
name='ping',
src='any',
dst='dmz',
app='junos-ping',
action='permit'))
F.add_policy(('untrust', 'trust'),
dict(sequence=3,
name='admin-puppet',
src='puppet',
dst='trustedhost',
app='junos-ssh',
action='permit'))
F.add_policy(('untrust', 'trust'),
dict(sequence=4,
name='admin',
src='any-ipv6',
dst='trustedhost',
app='junos-ssh',
action='permit'))
F.add_policy(('untrust', 'trust'),
dict(sequence=10,
name='deny',
src='any',
dst='any',
app='any',
action='deny'))
F.add_policy(('untrust', 'untrust'),
dict(sequence=10,
name='deny',
src='any',
dst='any',
app='any',
action='deny'))
rules = scripts.run(fake_cfg, {})
for r in rules.itervalues():
r.sort()
exp = {
'ping': [
Rule(src=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('10.0.0.0/8')]),
dst=IPSet([IP('10.1.0.0/16')]) - IPSet([IP('10.1.99.99')]),
app='ping',
name='ping'),
],
'puppet': [
Rule(src=IPSet([IP('10.0.0.0/8')]),
dst=IPSet([IP('9.0.9.2')]),
app='puppet',
name='puppet'),
],
'ssh': [
Rule(src=IPSet([IP('9.0.9.2')]),
dst=IPSet([IP('10.0.9.2')]),
app='ssh',
name='admin-puppet'),
Rule(src=IPSet([IP('10.0.0.0/8')]),
dst=IPSet([IP('0.0.0.0/0')]),
app='ssh',
name='ssh'),
],
}
eq_(rules, exp)
