def test_intersect_rules(): r1 = [ Rule(src=ipset('1.0.0.0/24'), dst=ipset('0.0.0.0/0'), app='app', name='r1'), ] r2 = [ Rule(src=ipset('1.0.0.10', '1.0.0.14'), dst=ipset('0.0.0.0/0'), app='app', name='r2'), ] r3 = [ Rule(src=ipset('1.0.0.11', '1.0.0.14'), dst=ipset('0.0.0.0/0'), app='app', name='r3'), ] eq_( process.intersect_rules([r1, r2, r3], ipset('0.0.0.0/0'), ipset('2.0.0.0/8')), [ Rule(src=ipset('1.0.0.14'), dst=ipset('2.0.0.0/8'), app='app', name='r1+r2+r3'), ])
def test_overlapping_rules(): lga_rules = { 'app': [ Rule(ipset('1.1.0.0/16'), ipset('2.0.0.0/8'), 'app', 'lga'), ] } ord_rules = { 'app': [ Rule(ipset('1.0.0.0/8'), ipset('2.1.0.0/16'), 'app', 'ord'), ] } address_spaces = { 'lga': ipset('1.0.0.0/8'), 'ord': ipset('2.0.0.0/8'), } sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules} with no_simplify(): result = process.combine(address_spaces, routes, sources) eq_( result, { 'app': [ # takes the intersection of both rules: Rule(ipset('1.1.0.0/16'), ipset('2.1.0.0/16'), 'app', 'lga+ord'), ] })
def test_combine(): address_spaces = { 'ten': ipset('10.0.0.0/8'), 'twenty': ipset('20.0.0.0/8'), 'unmanaged': ipset('0.0.0.0/0') - ipset('10.0.0.0/8') - ipset('20.0.0.0/8'), } routes = { ('ten', 'ten'): ['fw1.ten'], ('ten', 'twenty'): ['fw1.ten', 'fw1.twenty'], ('ten', 'unmanaged'): ['fw1.ten'], ('twenty', 'ten'): ['fw1.ten', 'fw1.twenty'], ('twenty', 'twenty'): ['fw1.twenty'], ('twenty', 'unmanaged'): ['fw1.twenty'], ('unmanaged', 'ten'): ['fw1.ten'], ('unmanaged', 'twenty'): ['fw1.twenty'], ('unmanaged', 'unmanaged'): [], } sources = { 'fw1.ten': RULES_10, 'fw1.twenty': RULES_20, } res = process.combine(address_spaces, routes, sources) res['http'].sort() eq_( res, { 'http': sorted([ Rule(src=ipset('10.10.0.0/16'), dst=ipset('10.20.0.0/16', '30.20.0.0/16'), app='http', name='10->10+10->30'), Rule(src=ipset('20.10.0.0/16'), dst=ipset('20.20.0.0/16', '30.20.0.0/16'), app='http', name='20->20+20->30'), Rule(src=ipset('30.10.0.0/16'), dst=ipset('10.20.0.0/16', '20.20.0.0/16'), app='http', name='30->10+30->20'), # note that only the intersection of these flows makes it through Rule(src=ipset('10.20.0.0/16'), dst=ipset('20.20.0.0/16'), app='http', name='10->20'), Rule(src=ipset('20.20.0.0/16'), dst=ipset('10.20.0.0/16'), app='http', name='20->10'), ]), })
def test_one_address_space(): rules = { 'app': [ Rule(ipset('1.2.3.4'), ipset('1.7.7.7'), 'app', 'p2p'), Rule(ipset('1.2.5.0/24'), ipset('1.7.7.7'), 'app', 'net'), ] } with no_simplify(): result = process.combine({'nyc': ipset('1.0.0.0/8')}, {('nyc', 'nyc'): ['fw1.nyc']}, {'fw1.nyc': rules}) eq_(sorted(result), sorted(rules))
def test_rules_from_to_intersection(): rules = [ Rule(src=ipset('0.0.0.0/7'), dst=ipset('2.0.1.0/24'), app='app', name='r'), ] eq_(process.rules_from_to(rules, ipset('1.0.0.0/8'), ipset('2.0.0.0/8')), [ Rule(src=ipset('1.0.0.0/8'), dst=ipset('2.0.1.0/24'), app='app', name='r'), ])
def process_rules(app_map, policies, zone_nets, policies_by_zone_pair, src_per_policy, dst_per_policy): logger.info("processing rules") # turn policies into a list of Rules (permit only), limited by zone, # that do not overlap. The tricky bit here is processing policies in # order and accounting for denies. We do this once for each # (from_zone, to_zone, app) tuple. The other tricky bit is handling # the application "any", which we treat as including all applications # used anywhere, and also record in a special "@@other" app. rules_by_app = {} all_apps = set(itertools.chain(*[p.applications for p in policies])) all_apps = all_apps | set(app_map.keys()) all_apps.discard('any') global_policies = policies_by_zone_pair.get((None, None), []) for from_zone, to_zone in itertools.product(zone_nets, zone_nets): zpolicies = sorted(policies_by_zone_pair.get((from_zone, to_zone), []), key=lambda p: p.sequence) zpolicies += global_policies logger.debug(" from-zone %s to-zone %s (%d policies)", from_zone, to_zone, len(zpolicies)) rule_count = 0 apps = set(itertools.chain(*[p.applications for p in zpolicies])) if 'any' in apps: apps = all_apps for app in apps | set(['@@other']): mapped_app = app_map[app] # for each app, count down the IP pairs that have not matched a # rule yet, starting with the zones' IP spaces. This simulates sequential # processing of the policies. remaining_pairs = IPPairs( (zone_nets[from_zone], zone_nets[to_zone])) rules = rules_by_app.setdefault(mapped_app, []) for pol in zpolicies + global_policies: if app not in pol.applications and 'any' not in pol.applications: continue src = src_per_policy[pol] dst = dst_per_policy[pol] # if the policy is a "permit", add rules for each # src/destination pair if pol.action == 'permit': for s, d in remaining_pairs: s = s & src d = d & dst if len(s) and len(d): rules.append(Rule(s, d, mapped_app, pol.name)) rule_count += 1 # regardless, consider this src/dst pair matched remaining_pairs = remaining_pairs - IPPairs((src, dst)) # if we've matched everything, we're done if not remaining_pairs: break logger.debug(" from-zone %s to-zone %s => %d rules", from_zone, to_zone, rule_count) # only include @@other if it's used if not rules_by_app['@@other']: del rules_by_app['@@other'] # simplify and return the result return simplify_rules(rules_by_app)
def test_limited_by_space(): lax_rules = {'app': []} lga_rules = { 'app': [ # /7 covers both lax and lga Rule(ipset('0.0.0.0/7'), ipset('2.0.0.0/8'), 'app', 'lga'), ] } ord_rules = { 'app': [ Rule(ipset('0.0.0.0/7'), ipset('2.0.0.0/8'), 'app', 'ord'), ] } address_spaces = { 'lax': ipset('0.0.0.0/8'), 'lga': ipset('1.0.0.0/8'), 'ord': ipset('2.0.0.0/8'), } routes = { ('lax', 'lax'): ['fw1.lax'], ('lax', 'lga'): ['fw1.lga', 'fw1.lax'], ('lax', 'ord'): ['fw1.ord', 'fw1.lax'], ('lga', 'lax'): ['fw1.lax', 'fw1.lga'], ('lga', 'lga'): ['fw1.lga'], ('lga', 'ord'): ['fw1.ord', 'fw1.lga'], ('ord', 'lax'): ['fw1.ord', 'fw1.lax'], ('ord', 'lga'): ['fw1.ord', 'fw1.lga'], ('ord', 'ord'): ['fw1.ord'], } sources = { 'fw1.ord': ord_rules, 'fw1.lga': lga_rules, 'fw1.lax': lax_rules } with no_simplify(): result = process.combine(address_spaces, routes, sources) eq_( result, { 'app': [ # only lga's address space is allowed Rule(ipset('1.0.0.0/8'), ipset('2.0.0.0/8'), 'app', 'lga+ord'), ] })
def test_rules_from_to_no_match_dst(): rules = [ Rule(src=ipset('1.0.0.0/8'), dst=ipset('3.0.0.0/24'), app='app', name='r'), ] eq_(process.rules_from_to(rules, ipset('1.0.0.0/8'), ipset('2.0.0.0/8')), [])
def test_nonoverlapping_rules(): lga_rules = { 'app': [ Rule(ipset('1.2.5.0/24'), ipset('2.2.5.0/24'), 'app', 'lga'), ] } ord_rules = { 'app': [ Rule(ipset('2.7.7.0/24'), ipset('1.7.7.0/24'), 'app', 'ord'), ] } address_spaces = { 'ord': ipset('2.0.0.0/8'), 'lga': ipset('1.0.0.0/8'), } sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules} with no_simplify(): result = process.combine(address_spaces, routes, sources) eq_(result, {})
def make_rules(sgid, local): sg = security_groups[sgid] for dir, sgrules in [('in', sg.rules), ('out', sg.rules_egress)]: for sgrule in sgrules: if sgrule.app == '*/any': apps = all_apps | set(['@@other']) else: apps = [sgrule.app] for app in apps: for grant in sgrule.grants: if grant.cidr_ip: remote = IPSet([IP(grant.cidr_ip)]) else: remote = ips_by_sg.get(grant.group_id, None) if not remote: continue src, dst = (remote, local) if dir == 'in' else (local, remote) name = "{}/{}".format(sg.name, dir) # first make rules involving non-managed space, leaving # only managed-to-managed if dir == 'in': unmanaged_src = src & unmanaged_ip_space if unmanaged_src: rules.setdefault(app, []).append( Rule(src=unmanaged_src, dst=dst, app=app, name=name)) src = src & managed_ip_space else: unmanaged_dst = dst & unmanaged_ip_space if unmanaged_dst: rules.setdefault(app, []).append( Rule(src=src, dst=unmanaged_dst, app=app, name=name)) dst = dst & managed_ip_space if src and dst: to_intersect.setdefault(app, {}).setdefault( dir, []).append((src, dst, name))
def rules_from_to(rules, local_sp, remote_sp): rv = [] for r in rules: src = r.src & local_sp if not src: continue dst = r.dst & remote_sp if not dst: continue rv.append(Rule(src=src, dst=dst, app=r.app, name=r.name)) return rv
def test_identical_rules(): rules = { 'app': [ Rule(ipset('2.7.7.0/24'), ipset('1.7.7.0/24'), 'app', 'lga-ord'), ] } address_spaces = { 'lga': ipset('1.0.0.0/8'), 'ord': ipset('2.0.0.0/8'), } sources = {'fw1.ord': rules, 'fw1.lga': rules} with no_simplify(): result = process.combine(address_spaces, routes, sources) eq_(result, rules)
def combine(address_spaces, routes, sources): # get the set of all apps all_apps = set() for rules in sources.itervalues(): all_apps = all_apps | set(rules) # for each address space, add any apps which aren't explicitly specified in that # address space, but *are* specified in the combined ruleset, as copies of that # address space's '@@other' app. This ensures that each space has the same set # of apps. for rules in sources.itervalues(): other = rules.get('@@other', []) missing_apps = all_apps - set(rules) for app in missing_apps: rules[app] = [ Rule(src=r.src, dst=r.dst, app=app, name=r.name) for r in other ] combined_rules = {} for app in all_apps: logger.info("combining app %s", app) # The idea here is this: for each pair of address spaces, look at the # set of rules specified in the routes. Only write combined rules for # flows for which are allowed by all rulesets. for local_sp_name, local_sp in address_spaces.iteritems(): for remote_sp_name, remote_sp in address_spaces.iteritems(): source_names = routes[local_sp_name, remote_sp_name] if not source_names: continue logger.debug(" from %s to %s using %s", local_sp_name, remote_sp_name, ', '.join(source_names)) rulesets = [sources[n][app] for n in source_names] # if we only have one source, this is pretty easy: # just limit each rule to the relevant IP spaces; otherwise # we need to do a recursive intersection if len(rulesets) == 1: new_rules = rules_from_to(rulesets[0], local_sp, remote_sp) else: new_rules = intersect_rules(rulesets, local_sp, remote_sp) if new_rules: combined_rules.setdefault(app, []).extend(new_rules) rules = simplify_rules(combined_rules) return rules
def intersect_rules(rulesets, local_sp, remote_sp): # combine the rulesets into an accumulator seeded with the first # ruleset acc = rules_from_to(rulesets.pop(), local_sp, remote_sp) while rulesets: rs = rules_from_to(rulesets.pop(), local_sp, remote_sp) intersected = [] for rl in acc: for rr in rs: src = rl.src & rr.src if not src: continue dst = rl.dst & rr.dst if not dst: continue intersected.append( Rule(src=src, dst=dst, app=rl.app, name=combine_names(rl.name, rr.name))) acc = intersected return acc
def test_process_global_priority(): # test the relative priority of "regular" (per-zone) and global policies policies = [ mkpol(name='ok', from_zone='pvt', to_zone='pub', src_addrs=ipset('192.168.1.2/31'), dst_addrs=ipset('0.0.0.0/0'), applications=['junos-ssh'], sequence=100), mkpol(name='deny-all-global', from_zone=None, to_zone=None, src_addrs=ipset('0.0.0.0/0'), dst_addrs=ipset('0.0.0.0/0'), applications=['junos-ssh'], sequence=3, action='deny'), mkpol(name='ok-global', from_zone=None, to_zone=None, src_addrs=ipset('192.168.1.128/32'), dst_addrs=ipset('128.135.0.0/16'), applications=['junos-ssh'], sequence=1), ] res = call_process_rules(APP_MAP, policies, ZONE_NETS) exp = { 'junos-ssh': [ Rule(src=ipset('192.168.1.128/32', '192.168.1.2/31'), dst=ipset('128.135.0.0/16'), app='junos-ssh', name='ok+ok-global'), ], } [ruleset.sort() for ruleset in exp.itervalues()] eq_(res, exp)
]), ('dynamic', '172.16.2.0/24', [ ('dynhost', '172.16.2.17', ['admin_ssh']), ('dynhost', '172.16.2.20', ['admin_ssh']), ]), ('perhost', '172.16.3.0/24', [ ('server4', '172.16.3.1', ['admin_ssh']), ]), ]), ]), ] RULES = { 'ssh': [ Rule(src=IPSet([IP('172.16.1.0/24'), IP('172.16.3.0/24')]) - IPSet([IP('172.16.1.1'), IP('172.16.1.2'), IP('172.16.3.1')]), dst=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('172.16.1.0/24'), IP('172.16.2.0/24'), IP('172.16.3.0/24')]), app='ssh', name='unoccupied/out'), Rule(src=IPSet([IP('192.168.10.0/24')]), dst=IPSet([IP('172.16.1.1'), IP('172.16.1.2'), IP('172.16.2.0/24'), IP('172.16.3.1')]), app='ssh', name='admin_ssh/in'), ], 'web': [ Rule(src=IPSet([IP('172.16.1.0/24'), IP('172.16.3.0/24')]) - IPSet([IP('172.16.1.1'), IP('172.16.1.2'), IP('172.16.3.1')]), dst=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('172.16.1.0/24'), IP('172.16.2.0/24'), IP('172.16.3.0/24')]) + IPSet([IP('172.16.1.1'), IP('172.16.1.2')]), app='web', name='unoccupied/out+webapp/in'), Rule(src=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('172.16.1.0/24'), IP('172.16.2.0/24'), IP('172.16.3.0/24')]), dst=IPSet([IP('172.16.1.1'), IP('172.16.1.2')]), app='web', name='webapp/in'), ], }
def test_process_rules_any_app(): # test basic functionality + the "any" app policies = [ mkpol(name='dbssh', from_zone='pvt', to_zone='pvt', src_addrs=ipset('192.168.1.2/31'), dst_addrs=ipset('0.0.0.0/0'), applications=['junos-ssh', 'junos-ping'], sequence=1), mkpol(name='admin', from_zone='pvt', to_zone='pvt', src_addrs=ipset('192.168.1.128/32'), dst_addrs=ipset('0.0.0.0/0'), applications=['any'], sequence=2), mkpol(name='admin', from_zone='pvt', to_zone='dmz', src_addrs=ipset('192.168.1.128/32'), dst_addrs=ipset('0.0.0.0/0'), applications=['any']), mkpol(name='admin', from_zone='pvt', to_zone='pub', src_addrs=ipset('192.168.1.128/32'), dst_addrs=ipset('0.0.0.0/0'), applications=['any']), mkpol(name='http', from_zone='pub', to_zone='dmz', src_addrs=ipset('0.0.0.0/0'), dst_addrs=ipset('10.1.10.0/24'), applications=['web']), mkpol(name='db', from_zone='dmz', to_zone='pvt', src_addrs=ipset('10.1.10.0/24'), dst_addrs=ipset('192.168.1.2/31'), applications=['db']), ] res = call_process_rules(APP_MAP, policies, ZONE_NETS) exp = { 'junos-ping': [ Rule(src=ipset('192.168.1.128'), dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'), app='junos-ping', name='admin'), Rule(src=ipset('192.168.1.2/31'), dst=ipset('192.168.0.0/16'), app='junos-ping', name='dbssh'), ], 'web': [ Rule(src=ipset('192.168.1.128'), dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'), app='web', name='admin'), Rule(src=ipset('128.135.0.0/16'), dst=ipset('10.1.10.0/24'), app='web', name='http'), ], 'db': [ Rule(src=ipset('192.168.1.128'), dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'), app='db', name='admin'), Rule(src=ipset('10.1.10.0/24'), dst=ipset('192.168.1.2/31'), app='db', name='db'), ], 'junos-ssh': [ Rule(src=ipset('192.168.1.128'), dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'), app='junos-ssh', name='admin'), Rule(src=ipset('192.168.1.2/31'), dst=ipset('192.168.0.0/16'), app='junos-ssh', name='dbssh'), ], '@@other': [ Rule(src=ipset('192.168.1.128'), dst=ipset('10.0.0.0/8', '128.135.0.0/16', '192.168.0.0/16'), app='@@other', name='admin'), ], } [ruleset.sort() for ruleset in exp.itervalues()] eq_(res, exp)
def r(src, dst, app='testapp', name='n'): return Rule(src=_ipset(src), dst=_ipset(dst), app=app, name=name)
def test_other_app(): ord_rules = { 'ordonly': [ Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), 'ordonly', 'ordonly'), ], 'inboth': [ Rule(ipset('1.1.8.8'), ipset('1.1.9.9'), 'inboth', 'inboth_ord'), ], '@@other': [ Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), '@@other', 'ordother'), ], } lga_rules = { 'lgaonly': [ Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), 'lgaonly', 'lgaonly'), ], 'inboth': [ Rule(ipset('65.1.8.8'), ipset('65.1.9.9'), 'inboth', 'inboth_lga'), ], '@@other': [ Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), '@@other', 'lgaother'), ], } address_spaces = { 'ord': ipset('0.0.0.0/2'), 'lga': ipset('64.0.0.0/2'), } sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules} with no_simplify(): result = process.combine(address_spaces, routes, sources) for apprules in result.itervalues(): apprules.sort() eq_( result, { 'ordonly': sorted([ Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), 'ordonly', 'ordonly'), Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), 'ordonly', 'lgaother'), ]), 'lgaonly': sorted([ Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), 'lgaonly', 'lgaonly'), Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), 'lgaonly', 'ordother'), ]), 'inboth': sorted([ Rule(ipset('1.1.8.8'), ipset('1.1.9.9'), 'inboth', 'inboth_ord'), Rule(ipset('65.1.8.8'), ipset('65.1.9.9'), 'inboth', 'inboth_lga'), ]), '@@other': sorted([ Rule(ipset('1.1.0.0'), ipset('1.1.9.9'), '@@other', 'ordother'), Rule(ipset('65.1.0.0'), ipset('65.1.9.9'), '@@other', 'lgaother'), ]), })
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from nose.tools import eq_ from fwunit.types import Rule from fwunit.combine import process from fwunit.test.util import ipset RULES_10 = { 'http': [ # within this ip space Rule(src=ipset('10.10.0.0/16'), dst=ipset('10.20.0.0/16'), app='http', name='10->10'), # from and to "unmanaged" space Rule(src=ipset('30.10.0.0/16'), dst=ipset('10.20.0.0/16'), app='http', name='30->10'), Rule(src=ipset('10.10.0.0/16'), dst=ipset('30.20.0.0/16'), app='http', name='10->30'), # from and to the 20/8 space Rule(src=ipset('20.20.0.0/16', '20.30.0.0/16'), dst=ipset('10.20.0.0/16', '10.30.0.0/16'), app='http', name='20->10'), Rule(src=ipset('10.10.0.0/16', '10.20.0.0/16'),
def get_rules(aws, app_map, regions, dynamic_subnets): if not regions: logger.info("Getting all regions") regions = aws.all_regions() logger.info("collecting subnets") subnets = [] managed_ip_space = IPSet([]) for id, subnet in aws.get_all_subnets(regions).iteritems(): name = subnet.tags.get('Name', id) dynamic = name in dynamic_subnets or id in dynamic_subnets cidr_block = IP(subnet.cidr_block) subnet = Subnet(cidr_block=cidr_block, name=name, dynamic=dynamic) subnets.append(subnet) managed_ip_space = managed_ip_space + IPSet([cidr_block]) unmanaged_ip_space = IPSet([IP('0.0.0.0/0')]) - managed_ip_space logger.info("collecting dynamic subnet IP ranges") dynamic_ipsets = {} per_host_subnet_ips = IPSet() for subnet in subnets: if subnet.dynamic: ipset = dynamic_ipsets.get(subnet.name, IPSet([])) ipset += IPSet([subnet.cidr_block]) dynamic_ipsets[subnet.name] = ipset else: per_host_subnet_ips += IPSet([subnet.cidr_block]) # sort by IP subnet, so we can use a binary search logger.info("sorting subnets by IP") subnets.sort(key=lambda s: s.cidr_block) _subnet_blocks = [s.cidr_block for s in subnets] def subnet_by_ip(ip): i = bisect.bisect_right(_subnet_blocks, ip) if i and ip in _subnet_blocks[i - 1]: return subnets[i - 1] logger.info("examining instances") sgids_by_dynamic_subnet = {} # {subnet name: set of SecurityGroupIds} sgids_by_instance = {} # {instance_name: [ip, set of SecurityGroupIds]} all_sgids = set() ips_by_sg = {} # {group id: IPSet} for id, instance in aws.get_all_instances(regions).iteritems(): if instance.state == 'terminated' or instance.state == 'shutting-down': continue # meh, who cares if not instance.vpc_id: continue # not in vpc; ignored if not instance.private_ip_address: logger.debug( "ignoring instance with no private_ip_address: %s, tags %r", instance.id, instance.tags) continue ip = IP(instance.private_ip_address) for g in instance.groups: ips_by_sg[g.id] = ips_by_sg.get(g.id, IPSet([])) + IPSet([IP(ip)]) subnet = subnet_by_ip(ip) if not subnet: logger.debug( "ignoring instance with no matching subnet for %s: %s, tags %r", ip, instance.id, instance.tags) continue if subnet.dynamic: sgset = sgids_by_dynamic_subnet.setdefault(subnet.name, set()) else: inst_name = instance.tags.get('Name', instance.id) if inst_name in sgids_by_instance: inst_name = inst_name + ' ({})'.format(instance.id) sgset = set() sgids_by_instance[inst_name] = [ip, sgset] new_sgids = set( SecurityGroupId(g.id, instance.region.name) for g in instance.groups) sgset.update(new_sgids) all_sgids.update(new_sgids) logger.info("accumulating security groups") all_apps = set(app_map.values()) security_groups = {} for sgid in all_sgids: sg = security_groups[sgid] = aws.get_security_group(sgid) assert sg, "no security group with id {}".format(sgid) # pre-process all of the rules' apps now for sgrule in itertools.chain(sg.rules, sg.rules_egress): proto = str(sgrule.ip_protocol) if proto == '-1': proto = 'any' if sgrule.from_port == sgrule.to_port: if str(sgrule.from_port) in ("None", "-1"): app = "*/{}".format(proto) else: app = '{}/{}'.format(sgrule.from_port, proto) else: app = '{}-{}/{}'.format(sgrule.from_port, sgrule.to_port, proto) app = app_map[app] sgrule.app = app all_apps.add(app) rules = {} to_intersect = {} def make_rules(sgid, local): sg = security_groups[sgid] for dir, sgrules in [('in', sg.rules), ('out', sg.rules_egress)]: for sgrule in sgrules: if sgrule.app == '*/any': apps = all_apps | set(['@@other']) else: apps = [sgrule.app] for app in apps: for grant in sgrule.grants: if grant.cidr_ip: remote = IPSet([IP(grant.cidr_ip)]) else: remote = ips_by_sg.get(grant.group_id, None) if not remote: continue src, dst = (remote, local) if dir == 'in' else (local, remote) name = "{}/{}".format(sg.name, dir) # first make rules involving non-managed space, leaving # only managed-to-managed if dir == 'in': unmanaged_src = src & unmanaged_ip_space if unmanaged_src: rules.setdefault(app, []).append( Rule(src=unmanaged_src, dst=dst, app=app, name=name)) src = src & managed_ip_space else: unmanaged_dst = dst & unmanaged_ip_space if unmanaged_dst: rules.setdefault(app, []).append( Rule(src=src, dst=unmanaged_dst, app=app, name=name)) dst = dst & managed_ip_space if src and dst: to_intersect.setdefault(app, {}).setdefault( dir, []).append((src, dst, name)) logger.info("writing rules for dynamic subnets") for subnet_name, sgids in sgids_by_dynamic_subnet.iteritems(): subnet = dynamic_ipsets[subnet_name] logger.debug(" subnet %s, %s", subnet_name, subnet) for sgid in sgids: make_rules(sgid, subnet) logger.info("writing rules for instances in per-host subnets") per_host_host_ips = IPSet() for inst_name, info in sgids_by_instance.iteritems(): ip, sgids = info logger.debug(" instance %s at %s", inst_name, ip) host_ip = IPSet([ip]) per_host_host_ips += host_ip for sgid in sgids: make_rules(sgid, host_ip) logger.info( "assuming unrestricted outbound access from unoccupied IPs in per-host subnets" ) unoccupied = per_host_subnet_ips - per_host_host_ips for app in all_apps: rules.setdefault(app, []).append( Rule(src=unoccupied, dst=unmanaged_ip_space, app=app, name='unoccupied/out')) to_intersect.setdefault(app, {}).setdefault('out', []).append( (unoccupied, managed_ip_space, 'unoccupied/out')) # traffic within the manage Ip space is governed both by outbound rules on # the source and inbound rules on the destination. logger.info("intersecting inbound and outbound rules") for app, dirs in to_intersect.iteritems(): in_rules = dirs.get('in', []) out_rules = dirs.get('out', []) logger.debug("..for %s", app) new_rules = [] for inr in in_rules: for outr in out_rules: src = inr[0] & outr[0] if not src: continue dst = inr[1] & outr[1] if not dst: continue new_rules.append( Rule(src=src, dst=dst, app=app, name=combine_names(inr[2], outr[2]))) # simplify now, within this app, to save space and time new_rules = simplify_rules({app: new_rules})[app] rules.setdefault(app, []).extend(new_rules) rules = simplify_rules(rules) return rules
def test_run_global_policies_and_addrbook(): fake_cfg = { 'firewall': 'fw', 'ssh_username': '******', 'ssh_password': '******', 'application-map': { 'junos-ssh': 'ssh', 'junos-ping': 'ping' }, } z = F.add_zone('untrust') F.add_address(z, 'untrust-host', '9.0.9.1/32') F.add_interface(z, 'reth0') z = F.add_zone('trust') F.add_address(z, 'trust-host', '10.9.1.1/32') F.add_interface(z, 'reth1') ab = F.add_addrbook('global') F.add_address(ab, 'global-host', '10.1.1.1/32') ab = F.add_addrbook('public') F.add_address(ab, 'public-host', '2.2.1.1/32') F.add_attach(ab, 'trust') F.add_attach(ab, 'untrust') F.add_policy(('trust', 'untrust'), dict(sequence=1, name='ssh', src='trust-host', dst='untrust-host', app='junos-ssh', action='permit')) F.add_policy(('untrust', 'trust'), dict(sequence=1, name='puppet', src='public-host', dst='global-host', app='puppet', action='permit')) F.add_policy( 'global', dict(sequence=10, name='ping', src='any', dst='any', app='junos-ping', action='permit')) F.add_policy( 'global', dict(sequence=11, name='deny', src='any', dst='any', app='any', action='deny')) rules = scripts.run(fake_cfg, {}) for r in rules.itervalues(): r.sort() exp = { 'ping': [ Rule(src=IPSet([IP('0.0.0.0/0')]), dst=IPSet([IP('0.0.0.0/0')]), app='ping', name='ping'), ], 'puppet': [ Rule(src=IPSet([IP('2.2.1.1')]), dst=IPSet([IP('10.1.1.1')]), app='puppet', name='puppet'), ], 'ssh': [ Rule(src=IPSet([IP('10.9.1.1')]), dst=IPSet([IP('9.0.9.1')]), app='ssh', name='ssh'), ], } eq_(rules, exp)
def test_multiple_matches(): lga_rules = { 'app': [ Rule(ipset('1.1.1.1'), ipset('2.0.0.0/8'), 'app', 'one'), Rule(ipset('1.1.1.2'), ipset('2.0.0.0/8'), 'app', 'two'), Rule(ipset('1.1.1.3'), ipset('2.0.0.0/8'), 'app', 'three'), ] } ord_rules = { 'app': [ Rule(ipset('1.0.0.0/8'), ipset('2.7.8.8'), 'app', 'eight'), Rule(ipset('1.0.0.0/8'), ipset('2.7.8.9'), 'app', 'nine'), ] } address_spaces = { 'lga': ipset('1.0.0.0/8'), 'ord': ipset('2.0.0.0/8'), } sources = {'fw1.ord': ord_rules, 'fw1.lga': lga_rules} with no_simplify(): result = process.combine(address_spaces, routes, sources) eq_( sorted(result['app']), sorted([ # takes the intersection of all rules: Rule(src=ipset('1.1.1.1'), dst=ipset('2.7.8.8'), app='app', name='eight+one'), Rule(src=ipset('1.1.1.1'), dst=ipset('2.7.8.9'), app='app', name='nine+one'), Rule(src=ipset('1.1.1.2'), dst=ipset('2.7.8.8'), app='app', name='eight+two'), Rule(src=ipset('1.1.1.2'), dst=ipset('2.7.8.9'), app='app', name='nine+two'), Rule(src=ipset('1.1.1.3'), dst=ipset('2.7.8.8'), app='app', name='eight+three'), Rule(src=ipset('1.1.1.3'), dst=ipset('2.7.8.9'), app='app', name='nine+three') ]))
def test_run_no_globals(): fake_cfg = { 'firewall': 'fw', 'ssh_username': '******', 'ssh_password': '******', 'application-map': { 'junos-ssh': 'ssh', 'junos-ping': 'ping' }, } z = F.add_zone('untrust') F.add_address(z, 'host1', '9.0.9.1/32') F.add_address(z, 'host2', '9.0.9.2/32') F.add_address(z, 'puppet', '9.0.9.2/32') F.add_address_set(z, 'hosts', 'host1', 'host2') F.add_interface(z, 'reth0') z = F.add_zone('trust') F.add_address(z, 'trustedhost', '10.0.9.2/32') F.add_address(z, 'dmz', '10.1.0.0/16') F.add_address(z, 'shadow', '10.1.99.99/32') F.add_interface(z, 'reth1') F.add_policy(('trust', 'trust'), dict(sequence=1, name='ssh', src='any', dst='any', app='junos-ssh', action='permit')) F.add_policy(('trust', 'trust'), dict(sequence=10, name='deny', src='any', dst='any', app='any', action='deny')) F.add_policy(('trust', 'untrust'), dict(sequence=1, name='ssh', src='any', dst='any', app='junos-ssh', action='permit')) F.add_policy(('trust', 'untrust'), dict(sequence=2, name='puppet', src='any', dst='puppet', app='puppet', action='permit')) F.add_policy(('trust', 'untrust'), dict(sequence=10, name='deny', src='any', dst='any', app='any', action='deny')) F.add_policy(('untrust', 'trust'), dict(sequence=1, name='no-shadow-ping', src='any', dst='shadow', app='junos-ping', action='deny')) F.add_policy(('untrust', 'trust'), dict(sequence=2, name='ping', src='any', dst='dmz', app='junos-ping', action='permit')) F.add_policy(('untrust', 'trust'), dict(sequence=3, name='admin-puppet', src='puppet', dst='trustedhost', app='junos-ssh', action='permit')) F.add_policy(('untrust', 'trust'), dict(sequence=4, name='admin', src='any-ipv6', dst='trustedhost', app='junos-ssh', action='permit')) F.add_policy(('untrust', 'trust'), dict(sequence=10, name='deny', src='any', dst='any', app='any', action='deny')) F.add_policy(('untrust', 'untrust'), dict(sequence=10, name='deny', src='any', dst='any', app='any', action='deny')) rules = scripts.run(fake_cfg, {}) for r in rules.itervalues(): r.sort() exp = { 'ping': [ Rule(src=IPSet([IP('0.0.0.0/0')]) - IPSet([IP('10.0.0.0/8')]), dst=IPSet([IP('10.1.0.0/16')]) - IPSet([IP('10.1.99.99')]), app='ping', name='ping'), ], 'puppet': [ Rule(src=IPSet([IP('10.0.0.0/8')]), dst=IPSet([IP('9.0.9.2')]), app='puppet', name='puppet'), ], 'ssh': [ Rule(src=IPSet([IP('9.0.9.2')]), dst=IPSet([IP('10.0.9.2')]), app='ssh', name='admin-puppet'), Rule(src=IPSet([IP('10.0.0.0/8')]), dst=IPSet([IP('0.0.0.0/0')]), app='ssh', name='ssh'), ], } eq_(rules, exp)