def assignIPs(_upper_ref, _from_key, lower_refs, to_key, ip_start='192.168.0.1', ip_end='192.168.0.254', **_kwargs): '''assign ips to hosts.''' if not ip_start or not ip_end: return {} host_ips = {} unassigned_hosts = [] ips = IPSet(IPRange(ip_start, ip_end)) for lower_key, lower_ref in lower_refs.items(): ip_addr = lower_ref.get(to_key, '') if ip_addr: host_ips[lower_key] = ip_addr ips.remove(ip_addr) else: unassigned_hosts.append(lower_key) for ip_addr in ips: if not unassigned_hosts: break host = unassigned_hosts.pop(0) host_ips[host] = str(ip_addr) logging.debug('assign %s: %s', to_key, host_ips) return host_ips
def main(): parser = ArgumentParser(description='支持从某个CIDR网段排除特定IP!') parser.add_argument('netstr', type=str, metavar='CIDR', help='需要排除IP的网段,如:0.0.0.0/0') parser.add_argument( '-r', type=str, metavar='IP', help='排除的IP,支持同时排除多个,支持网段,例子: -r 192.168.1.128 -r 192.168.2.0/24', action='append') parser.add_argument_group() args = parser.parse_args() try: ipnet = IPNetwork(args.netstr).cidr ips = IPSet([ipnet]) except Exception as e: print('CIDR 输入错误:{}'.format(e)) try: if args.r: for i in args.r: ips.remove(i) except Exception as e: print('-r IP输入错误:{}'.format(e)) print('\n拆分后的网段段为:\n') print(','.join(outip for outip in map(str, ips.iter_cidrs())))
def test_ipset_with_iprange(): s1 = IPSet(['10.0.0.0/25', '10.0.0.128/25']) assert s1.iprange() == IPRange('10.0.0.0', '10.0.0.255') assert s1.iscontiguous() s1.remove('10.0.0.16') assert s1 == IPSet([ '10.0.0.0/28', '10.0.0.17/32', '10.0.0.18/31', '10.0.0.20/30', '10.0.0.24/29', '10.0.0.32/27', '10.0.0.64/26', '10.0.0.128/25', ]) assert not s1.iscontiguous() with pytest.raises(ValueError): s1.iprange() assert list(s1.iter_ipranges()) == [ IPRange('10.0.0.0', '10.0.0.15'), IPRange('10.0.0.17', '10.0.0.255'), ] s2 = IPSet(['0.0.0.0/0']) assert s2.iscontiguous() assert s2.iprange() == IPRange('0.0.0.0', '255.255.255.255') # s3 = IPSet() assert s3.iscontiguous() assert s3.iprange() is None s4 = IPSet(IPRange('10.0.0.0', '10.0.0.8')) assert s4.iscontiguous()
def assign_ips(_upper_ref, _from_key, lower_refs, to_key, ip_start='192.168.0.1', ip_end='192.168.0.254', **_kwargs): """Assign ips to hosts' configurations.""" if not ip_start or not ip_end: return {} host_ips = {} unassigned_hosts = [] ips = IPSet(IPRange(ip_start, ip_end)) for lower_key, lower_ref in lower_refs.items(): ip_addr = lower_ref.get(to_key, '') if ip_addr: host_ips[lower_key] = ip_addr ips.remove(ip_addr) else: unassigned_hosts.append(lower_key) for ip_addr in ips: if not unassigned_hosts: break host = unassigned_hosts.pop(0) host_ips[host] = str(ip_addr) logging.debug('assign %s: %s', to_key, host_ips) return host_ips
def generateIP(self): network = IPSet(IPNetwork(self.cidr)) network.remove(min(network)) network.remove(max(network)) hostlist = IPSet([h.ip for h in self.hosts.all()]) available = network - hostlist return min(available) '''
def subtract_subnet(original_subnet, remove_subnets): """Subtract a list of subnets from original subnet and return leftover subnet(s) in a list""" original = IPSet([original_subnet]) for subnet in remove_subnets: original.remove(subnet) return [str(subnet) for subnet in original.iter_cidrs()]
def generateIP(self): network = IPSet(IPNetwork(self.cidr)) network.remove(min(network)) network.remove(max(network)) hostlist = IPSet([ h.ip for h in self.hosts.all() ]) available = network - hostlist return min(available) '''
def test_ipset_member_insertion_and_deletion(): s1 = IPSet() s1.add('192.0.2.0') assert s1 == IPSet(['192.0.2.0/32']) s1.remove('192.0.2.0') assert s1 == IPSet([]) s1.add(IPRange("10.0.0.0", "10.0.0.255")) assert s1 == IPSet(['10.0.0.0/24']) s1.remove(IPRange("10.0.0.128", "10.10.10.10")) assert s1 == IPSet(['10.0.0.0/25'])
def test_ipset_member_insertion_and_deletion(): s1 = IPSet() s1.add('192.0.2.0') assert s1 == IPSet(['192.0.2.0/32']) s1.remove('192.0.2.0') assert s1 == IPSet([]) s1.add(IPRange("10.0.0.0", "10.0.0.255")) assert s1 == IPSet(['10.0.0.0/24']) s1.remove(IPRange("10.0.0.128", "10.10.10.10")) assert s1 == IPSet(['10.0.0.0/25'])
def choose_ip(routable_cidrs, excluded_cidrs=[], client_addr=''): """Find available IP addresses for both sides of a VPN Tunnel. This method iterates over the settings.ALLOWED_CIDRS list in order to allocate available IP address to both the client and server side of a VPN tunnel. CIDRs that belong to the lists of settings.RESERVED_CIDRS, `routable_cidrs`, and `excluded_cidrs` are excluded from the allocation process. :param routable_cidrs: the CIDRs that are to be routed over a VPN tunnel :param excluded_cidrs: an optional list of CIDRs to be excluded from the address allocation process :param client_addr: the `client_addr` is used to attempt to pick an adjacent IP address for the server side :return: a private IP address """ exc_nets = routable_cidrs + excluded_cidrs + settings.RESERVED_CIDRS # make sure the exc_nets list does not contain any empty strings exc_nets = [exc_net for exc_net in exc_nets if exc_net] # a list of unique, non-overlapping supernets (to be excluded) exc_nets = IPSet(exc_nets).iter_cidrs() for network in settings.ALLOWED_CIDRS: available_cidrs = IPSet(IPNetwork(network)) for exc_net in exc_nets: available_cidrs.remove(exc_net) if not available_cidrs: continue for cidr in available_cidrs.iter_cidrs(): first, last = cidr.first, cidr.last if client_addr: address = IPAddress(client_addr) + 1 else: address = IPAddress(random.randrange(first + 1, last)) for _ in xrange(first + 1, last): if address not in cidr or address == cidr.broadcast: address = cidr.network + 1 try: Tunnel.objects.get(Q(client=str(address)) | Q(server=str(address))) address += 1 except Tunnel.DoesNotExist: return str(address)
def available_ipv4s(self, limit=25): def _gen(ip_set): for ip_net in ip_set.iter_cidrs(): for ip in ip_net.iter_hosts(): if ip.words[-1] not in [0, 255, 0xffff]: yield ip interfaces = Interface.query.filter_by(network_id=self.id).all() static_ips = IPSet([self.static_net]) if self.static_net else IPSet() reserved_ips = IPSet([self.reserved_net]) if self.reserved_net else IPSet() for interface in interfaces: if interface.static_ipv4: static_ips.remove(interface.static_ipv4) if interface.reserved_ipv4: reserved_ips.remove(interface.reserved_ipv4) return (list(itertools.islice(_gen(static_ips), limit)), list(itertools.islice(_gen(reserved_ips), limit)))
def test_ipset_adding_and_removing_members_ip_addresses_as_ints(): s1 = IPSet(['10.0.0.0/25']) s1.add('10.0.0.0/24') assert s1 == IPSet(['10.0.0.0/24']) integer1 = int(IPAddress('10.0.0.1')) integer2 = int(IPAddress('fe80::')) integer3 = int(IPAddress('10.0.0.2')) s2 = IPSet([integer1, integer2]) assert s2 == IPSet(['10.0.0.1/32', 'fe80::/128']) s2.add(integer3) assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128']) s2.remove(integer2) assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32']) s2.update([integer2]) assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
def test_ipset_adding_and_removing_members_ip_addresses_as_ints(): s1 = IPSet(['10.0.0.0/25']) s1.add('10.0.0.0/24') assert s1 == IPSet(['10.0.0.0/24']) integer1 = int(IPAddress('10.0.0.1')) integer2 = int(IPAddress('fe80::')) integer3 = int(IPAddress('10.0.0.2')) s2 = IPSet([integer1, integer2]) assert s2 == IPSet(['10.0.0.1/32', 'fe80::/128']) s2.add(integer3) assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128']) s2.remove(integer2) assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32']) s2.update([integer2]) assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
def __init__(self, whitelist, blacklist): set = IPSet([]) for block in whitelist: set.add(IPNetwork(block)) # Remove invalid broadcast and network addresses if block.broadcast != None: set.remove(IPNetwork(block.broadcast)) if block.size > 2: set.remove(IPNetwork(block.network)) for block in blacklist: set.remove(IPNetwork(block)) for block in set.iter_cidrs(): self.total += block.size self.networks.append({ "network": block, "size": block.size, "start": block[0], "index": 0 }) if self.total < 1: raise Exception( "IPScanManager can not be started with an empty target scope") self.rng = CyclicPRNG(self.total) def blockcomp(b): return b["start"] self.networks.sort(key=blockcomp) start = 1 for i in range(0, len(self.networks)): self.networks[i]["index"] = start start += self.networks[i]["size"]
def test_ipset_with_iprange(): s1 = IPSet(['10.0.0.0/25', '10.0.0.128/25']) assert s1.iprange() == IPRange('10.0.0.0', '10.0.0.255') assert s1.iscontiguous() s1.remove('10.0.0.16') assert s1 == IPSet([ '10.0.0.0/28', '10.0.0.17/32', '10.0.0.18/31', '10.0.0.20/30', '10.0.0.24/29', '10.0.0.32/27', '10.0.0.64/26', '10.0.0.128/25', ]) assert not s1.iscontiguous() with pytest.raises(ValueError): s1.iprange() assert list(s1.iter_ipranges()) == [ IPRange('10.0.0.0', '10.0.0.15'), IPRange('10.0.0.17', '10.0.0.255'), ] s2 = IPSet(['0.0.0.0/0']) assert s2.iscontiguous() assert s2.iprange() == IPRange('0.0.0.0', '255.255.255.255') # s3 = IPSet() assert s3.iscontiguous() assert s3.iprange() is None s4 = IPSet(IPRange('10.0.0.0', '10.0.0.8')) assert s4.iscontiguous()
def _get_free_set(self, network, domain): ip_range_start = self.get_network_ip_range_start(network, domain) ip_range_end = self.get_network_ip_range_end(network, domain) select_range = IPRange(ip_range_start, ip_range_end) netset = IPSet(select_range) if (network == self.get_infra_external_network_name() and domain == self._get_vip_domain()): iterator = netset.__iter__() self.external_vip = str(iterator.next()) netset.remove(self.external_vip) # check for the IP(s) taken by the nodes try: hostsconfig = self.confman.get_hosts_config_handler() hosts = hostsconfig.get_hosts() for host in hosts: try: hostip = self.get_host_ip(host, network) netset.remove(hostip) except configerror.ConfigError: pass except configerror.ConfigError: pass service_profiles_lib = profiles.Profiles() # check for the IP(s) taken as VIPs if network == self.get_infra_internal_network_name( ) and domain == self._get_vip_domain(): vips = self.get_net_vips(network) for _, vip in vips.iteritems(): try: netset.remove(vip) except configerror.ConfigError: pass return netset
def sync_subnets(conn, config): log.debug("loading routing tables") routing_tables = conn.get_all_route_tables() route_tables_by_name = {r.tags.get('Name'): r for r in routing_tables} route_tables_by_subnet_id = {} for r in routing_tables: for a in r.associations: route_tables_by_subnet_id[a.subnet_id] = r # Get list of AZs zones = conn.get_all_zones() for vpc_id in config: # Get a list of all the remote subnets remote_subnets = conn.get_all_subnets(filters={'vpcId': vpc_id}) seen = set() # Go through our config, adjusting or any subnets as appropriate for cidr, block_config in config[vpc_id].items(): cidr_net = IPNetwork(cidr) table_name = block_config.get('routing_table') if table_name and table_name not in route_tables_by_name: log.warn("couldn't find routing table %s for block %s", table_name, cidr) log.warn("skipping rest of %s", cidr) continue my_rt = route_tables_by_name[table_name] ip_set = IPSet(cidr_net) for s in remote_subnets: if IPNetwork(s.cidr_block) in cidr_net: ip_set.remove(s.cidr_block) if s.tags.get('Name') != block_config['name']: log.info("Setting Name of %s to %s", s, block_config['name']) s.add_tag('Name', block_config['name']) if s.id in route_tables_by_subnet_id: remote_rt = route_tables_by_subnet_id[s.id] else: remote_rt = route_tables_by_subnet_id[None] if remote_rt != my_rt: log.info( "Changing routing table for %s (%s) to %s (%s)", s, s.tags.get('Name'), my_rt, my_rt.tags.get('Name')) if raw_input("(y/N) ") == "y": conn.associate_route_table(my_rt.id, s.id) seen.add(s) # Are we missing any subnets? # If so, create them! # TODO: We want to evenly distribute the ip range over the # configured availability zones, without dividing smaller than a # /25 network (128 ips, at least 2 of which are reserved) # For now we'll just split them as small as /24, and then assign # them into the subnets while ip_set: log.info("%s - %s isn't covered by any subnets", cidr, ip_set) my_zones = [ z for z in zones if z.name not in block_config.get('skip_azs', []) ] remaining_cidrs = list(ip_set.iter_cidrs()) remaining_cidrs.sort(key=lambda s: s.size, reverse=True) for s in remaining_cidrs[:]: if s.prefixlen < 24: added = list(s.subnet(24)) remaining_cidrs.remove(s) remaining_cidrs.extend(added) ip_set.remove(s) zg = itertools.cycle(my_zones) while remaining_cidrs: c = remaining_cidrs.pop() z = next(zg) log.info("creating subnet %s in %s/%s", c, z.name, vpc_id) if raw_input("(y/N) ") == "y": log.debug("creating subnet") s = conn.create_subnet(vpc_id, c, z.name) log.debug("adding tag") # TODO: sometimes the subnet isn't actually created by # the time we try and add the tag, so get a 400 error s.add_tag('Name', block_config['name']) log.debug("associating routing") conn.associate_route_table(my_rt.id, s.id) local_missing = set(remote_subnets) - seen for m in local_missing: log.info("%s:%s (name: %s) is unmanaged", m.id, m.cidr_block, m.tags.get('Name'))
#!/usr/bin/python from sys import argv from netaddr import IPSet if len(argv) != 3: print('Usage: netdiff.py3 include.txt exclude.txt') exit() net = IPSet() incfile = open(argv[1]) for line in incfile: net = net | IPSet([line]) exfile = open(argv[2]) for line in exfile: net.remove(line) for cidr in net.iter_cidrs(): print(cidr)
class IPScanManager: networks = [] total = 0 rng = None consistent = None def __init__(self, whitelist, blacklist, consistent: bool): self.networks = [] self.total = 0 self.rng = None self.consistent = consistent self.set_whitelist(whitelist) self.set_blacklist(blacklist) self.initialize_manager() def log_to_db(self, message): log_messages = { "init": "PRNG Starting Up", "restart": "PRNG Cycle Restarted", "default": "Unknown PRNG Event", } db_log = ScopeLog(log_messages.get(message, "default")) db.session.add(db_log) db.session.commit() def set_whitelist(self, whitelist): self.whitelist = IPSet([]) for block in whitelist: self.whitelist.add(IPNetwork(block)) # Remove invalid broadcast and network addresses if block.broadcast is not None: self.whitelist.remove(IPNetwork(block.broadcast)) if block.size > 2: self.whitelist.remove(IPNetwork(block.network)) def set_blacklist(self, blacklist): self.blacklist = IPSet([]) for block in blacklist: self.blacklist.add(IPNetwork(block)) def initialize_manager(self): self.networks = [] self.ipset = self.whitelist - self.blacklist for block in self.ipset.iter_cidrs(): self.total += block.size self.networks.append({ "network": block, "size": block.size, "start": block[0], "index": 0 }) if self.total < 1: raise Exception( "IPScanManager can not be started with an empty target scope") self.rng = CyclicPRNG(self.total, consistent=self.consistent, event_handler=self.log_to_db) def blockcomp(b): return b["start"] self.networks.sort(key=blockcomp) start = 1 for i in range(0, len(self.networks)): self.networks[i]["index"] = start start += self.networks[i]["size"] self.initialized = True def in_whitelist(self, ip): return ip in self.whitelist def in_blacklist(self, ip): return ip in self.blacklist def get_total(self): return self.total def get_ready(self): return self.rng and self.total > 0 and self.initialized def get_next_ip(self): if self.rng: index = self.rng.get_random() return self.get_ip(index) else: return None def get_ip(self, index): def binarysearch(networks, i): middle = int(len(networks) / 2) network = networks[middle] if i < network["index"]: return binarysearch(networks[:middle], i) elif i >= (network["index"] + network["size"]): return binarysearch(networks[middle + 1:], i) else: return network["network"][i - network["index"]] return binarysearch(self.networks, index)
#! /usr/bin/env python # Calculates the disjunction of two sets of IP ranges from sys import argv from netaddr import IPSet if len(argv) != 3: print('Usage: {0} include.txt exclude.txt'.format(argv[0])) exit() net = IPSet() with open(argv[1], 'r') as incfile: for line in incfile: net = net | IPSet([line]) with open(argv[2], 'r') as exfile: for line in exfile: net.remove(line) for cidr in net.iter_cidrs(): print(cidr)
def test_ipset_cidr_fracturing(): s1 = IPSet(['0.0.0.0/0']) s1.remove('255.255.255.255') assert s1 == IPSet([ '0.0.0.0/1', '128.0.0.0/2', '192.0.0.0/3', '224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6', '252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9', '255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12', '255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15', '255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18', '255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21', '255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24', '255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27', '255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30', '255.255.255.252/31', '255.255.255.254/32' ]) cidrs = s1.iter_cidrs() assert len(cidrs) == 32 assert list(cidrs) == [ IPNetwork('0.0.0.0/1'), IPNetwork('128.0.0.0/2'), IPNetwork('192.0.0.0/3'), IPNetwork('224.0.0.0/4'), IPNetwork('240.0.0.0/5'), IPNetwork('248.0.0.0/6'), IPNetwork('252.0.0.0/7'), IPNetwork('254.0.0.0/8'), IPNetwork('255.0.0.0/9'), IPNetwork('255.128.0.0/10'), IPNetwork('255.192.0.0/11'), IPNetwork('255.224.0.0/12'), IPNetwork('255.240.0.0/13'), IPNetwork('255.248.0.0/14'), IPNetwork('255.252.0.0/15'), IPNetwork('255.254.0.0/16'), IPNetwork('255.255.0.0/17'), IPNetwork('255.255.128.0/18'), IPNetwork('255.255.192.0/19'), IPNetwork('255.255.224.0/20'), IPNetwork('255.255.240.0/21'), IPNetwork('255.255.248.0/22'), IPNetwork('255.255.252.0/23'), IPNetwork('255.255.254.0/24'), IPNetwork('255.255.255.0/25'), IPNetwork('255.255.255.128/26'), IPNetwork('255.255.255.192/27'), IPNetwork('255.255.255.224/28'), IPNetwork('255.255.255.240/29'), IPNetwork('255.255.255.248/30'), IPNetwork('255.255.255.252/31'), IPNetwork('255.255.255.254/32') ] assert cidrs == cidr_exclude('0.0.0.0/0', '255.255.255.255') s1.remove('0.0.0.0') assert s1 == IPSet([ '0.0.0.1/32', '0.0.0.2/31', '0.0.0.4/30', '0.0.0.8/29', '0.0.0.16/28', '0.0.0.32/27', '0.0.0.64/26', '0.0.0.128/25', '0.0.1.0/24', '0.0.2.0/23', '0.0.4.0/22', '0.0.8.0/21', '0.0.16.0/20', '0.0.32.0/19', '0.0.64.0/18', '0.0.128.0/17', '0.1.0.0/16', '0.2.0.0/15', '0.4.0.0/14', '0.8.0.0/13', '0.16.0.0/12', '0.32.0.0/11', '0.64.0.0/10', '0.128.0.0/9', '1.0.0.0/8', '2.0.0.0/7', '4.0.0.0/6', '8.0.0.0/5', '16.0.0.0/4', '32.0.0.0/3', '64.0.0.0/2', '128.0.0.0/2', '192.0.0.0/3', '224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6', '252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9', '255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12', '255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15', '255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18', '255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21', '255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24', '255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27', '255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30', '255.255.255.252/31', '255.255.255.254/32', ]) assert len(list(s1.iter_cidrs())) == 62 s1.add('255.255.255.255') s1.add('0.0.0.0') assert s1 == IPSet(['0.0.0.0/0'])
def test_ipset_cidr_fracturing(): s1 = IPSet(['0.0.0.0/0']) s1.remove('255.255.255.255') assert s1 == IPSet([ '0.0.0.0/1', '128.0.0.0/2', '192.0.0.0/3', '224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6', '252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9', '255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12', '255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15', '255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18', '255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21', '255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24', '255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27', '255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30', '255.255.255.252/31', '255.255.255.254/32']) cidrs = s1.iter_cidrs() assert len(cidrs) == 32 assert list(cidrs) == [ IPNetwork('0.0.0.0/1'), IPNetwork('128.0.0.0/2'), IPNetwork('192.0.0.0/3'), IPNetwork('224.0.0.0/4'), IPNetwork('240.0.0.0/5'), IPNetwork('248.0.0.0/6'), IPNetwork('252.0.0.0/7'), IPNetwork('254.0.0.0/8'), IPNetwork('255.0.0.0/9'), IPNetwork('255.128.0.0/10'), IPNetwork('255.192.0.0/11'), IPNetwork('255.224.0.0/12'), IPNetwork('255.240.0.0/13'), IPNetwork('255.248.0.0/14'), IPNetwork('255.252.0.0/15'), IPNetwork('255.254.0.0/16'), IPNetwork('255.255.0.0/17'), IPNetwork('255.255.128.0/18'), IPNetwork('255.255.192.0/19'), IPNetwork('255.255.224.0/20'), IPNetwork('255.255.240.0/21'), IPNetwork('255.255.248.0/22'), IPNetwork('255.255.252.0/23'), IPNetwork('255.255.254.0/24'), IPNetwork('255.255.255.0/25'), IPNetwork('255.255.255.128/26'), IPNetwork('255.255.255.192/27'), IPNetwork('255.255.255.224/28'), IPNetwork('255.255.255.240/29'), IPNetwork('255.255.255.248/30'), IPNetwork('255.255.255.252/31'), IPNetwork('255.255.255.254/32') ] assert cidrs == cidr_exclude('0.0.0.0/0', '255.255.255.255') s1.remove('0.0.0.0') assert s1 == IPSet([ '0.0.0.1/32', '0.0.0.2/31', '0.0.0.4/30', '0.0.0.8/29', '0.0.0.16/28', '0.0.0.32/27', '0.0.0.64/26', '0.0.0.128/25', '0.0.1.0/24', '0.0.2.0/23', '0.0.4.0/22', '0.0.8.0/21', '0.0.16.0/20', '0.0.32.0/19', '0.0.64.0/18', '0.0.128.0/17', '0.1.0.0/16', '0.2.0.0/15', '0.4.0.0/14', '0.8.0.0/13', '0.16.0.0/12', '0.32.0.0/11', '0.64.0.0/10', '0.128.0.0/9', '1.0.0.0/8', '2.0.0.0/7', '4.0.0.0/6', '8.0.0.0/5', '16.0.0.0/4', '32.0.0.0/3', '64.0.0.0/2', '128.0.0.0/2', '192.0.0.0/3', '224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6', '252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9', '255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12', '255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15', '255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18', '255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21', '255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24', '255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27', '255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30', '255.255.255.252/31', '255.255.255.254/32', ]) assert len(list(s1.iter_cidrs())) == 62 s1.add('255.255.255.255') s1.add('0.0.0.0') assert s1 == IPSet(['0.0.0.0/0'])
def sync_subnets(conn, config): log.debug("loading routing tables") routing_tables = conn.get_all_route_tables() route_tables_by_name = {r.tags.get('Name'): r for r in routing_tables} route_tables_by_subnet_id = {} for r in routing_tables: for a in r.associations: route_tables_by_subnet_id[a.subnet_id] = r # Get list of AZs zones = conn.get_all_zones() for vpc_id in config: # Get a list of all the remote subnets remote_subnets = conn.get_all_subnets(filters={'vpcId': vpc_id}) seen = set() # Go through our config, adjusting or any subnets as appropriate for cidr, block_config in config[vpc_id].items(): cidr_net = IPNetwork(cidr) table_name = block_config.get('routing_table') if table_name and table_name not in route_tables_by_name: log.warn("couldn't find routing table %s for block %s", table_name, cidr) log.warn("skipping rest of %s", cidr) continue my_rt = route_tables_by_name[table_name] ip_set = IPSet(cidr_net) for s in remote_subnets: if IPNetwork(s.cidr_block) in cidr_net: ip_set.remove(s.cidr_block) if s.tags.get('Name') != block_config['name']: log.info("Setting Name of %s to %s", s, block_config['name']) s.add_tag('Name', block_config['name']) if s.id in route_tables_by_subnet_id: remote_rt = route_tables_by_subnet_id[s.id] else: remote_rt = route_tables_by_subnet_id[None] if remote_rt != my_rt: log.info( "Changing routing table for %s (%s) to %s (%s)", s, s.tags.get('Name'), my_rt, my_rt.tags.get('Name')) if raw_input("(y/N) ") == "y": conn.associate_route_table(my_rt.id, s.id) seen.add(s) # Are we missing any subnets? # If so, create them! # TODO: We want to evenly distribute the ip range over the # configured availability zones, without dividing smaller than a # /25 network (128 ips, at least 2 of which are reserved) # For now we'll just split them as small as /24, and then assign # them into the subnets while ip_set: log.info("%s - %s isn't covered by any subnets", cidr, ip_set) my_zones = [z for z in zones if z.name not in block_config.get('skip_azs', [])] remaining_cidrs = list(ip_set.iter_cidrs()) remaining_cidrs.sort(key=lambda s: s.size, reverse=True) for s in remaining_cidrs[:]: if s.prefixlen < 24: added = list(s.subnet(24)) remaining_cidrs.remove(s) remaining_cidrs.extend(added) ip_set.remove(s) zg = itertools.cycle(my_zones) while remaining_cidrs: c = remaining_cidrs.pop() z = next(zg) log.info("creating subnet %s in %s/%s", c, z.name, vpc_id) if raw_input("(y/N) ") == "y": log.debug("creating subnet") s = conn.create_subnet(vpc_id, c, z.name) log.debug("adding tag") # TODO: sometimes the subnet isn't actually created by # the time we try and add the tag, so get a 400 error s.add_tag('Name', block_config['name']) log.debug("associating routing") conn.associate_route_table(my_rt.id, s.id) local_missing = set(remote_subnets) - seen for m in local_missing: log.info("%s:%s (name: %s) is unmanaged", m.id, m.cidr_block, m.tags.get('Name'))
class Scope: """Util class to manipulate recon scope. Supports: IP (IPv4 standard only! NO padding, octal, decimal, shortened, etc.) Netblocks (192.168.2.1/24) Netrange (192.168.2.1-25) -> Somewhat limited compared to Nmap funky range selectors FQDN """ def __init__(self, ip_list=[], hostname_list=[], netblock_list=[], netrange_list=[], auto_coalesce_ip=True, auto_resolve_hostnames=False): self._auto_coalesce = auto_coalesce_ip self._auto_resolve_hostnames = auto_resolve_hostnames not_ip = [x for x in ip_list if not Scope.is_ip(x)] if not_ip: raise ValueError('Value(s) ' + str(not_ip) + ' are provided as IP but cannot be casted as such') self._ip_list = IPSet(ip_list) not_hostname = [x for x in hostname_list if not Scope.is_hostname(x)] if not_hostname: raise ValueError('Value(s) ' + str(not_hostname) + ' are provided as hostnames but cannot be casted as such') self._hostname_list = set(hostname_list) not_netblock = [x for x in netblock_list if not Scope.is_netblock(x)] if not_netblock: raise ValueError('Value(s) ' + str(not_netblock) + ' are provided as netblocks but cannot be casted as such') self._netblock_list = set(IPNetwork(x) for x in set(netblock_list)) not_netrange = [x for x in netrange_list if not Scope.is_netrange(x)] if not_netrange: raise ValueError('Value(s) ' + str(not_netrange) + ' are provided as netranges but cannot be casted as such') self._netrange_list = set() for netrange in set(netrange_list): if self._netrange_ip_to_int(netrange): self._netrange_list.add(IPGlob(netrange)) else: (ip1, ip2) = (x.strip() for x in netrange.split('-')) self._netrange_list.add(IPRange(ip1, ip2)) if self._auto_resolve_hostnames: self._resolve_hostnames() if self._auto_coalesce: self._coalesce_ip_list() @property def ip_list(self): return tuple(str(ip) for ip in self._ip_list) @property def hostname_list(self): return tuple(str(hostname) for hostname in self._hostname_list) @property def netblock_list(self): return tuple(str(netblock) for netblock in self._netblock_list) @property def netrange_list(self): return tuple(str(netrange) for netrange in self._netrange_list) def _coalesce_ip_list(self): """Merge IPs in appropriate netblocks/netranges, if possible, and remove them from ip_list""" for ip in self._ip_list: if ip in self._netrange_list: self._ip_list.remove(ip) if ip in self._netblock_list: self._ip_list.remove(ip) def _resolve_hostnames(self): for hostname in self.hostname_list: for result in dns.resolver.query(hostname, 'A'): # Add IP if it's not already in the IP list. # Will be coalesced later if flag enabled if str(result) not in self._ip_list: self._ip_list.add(str(result)) def is_ip_in_scope(self, ip): if ip in self._ip_list: return True for netblock in self._netblock_list: if ip in IPSet(netblock): return True for netrange in self._netrange_list: if ip in IPSet(netrange): return True return False def get_expanded_ip_list(self): ip_netblocks = tuple(str(ip) for nb in self._netblock_list for ip in nb) ip_netrange = tuple(str(ip) for nr in self._netrange_list for ip in nr) ip_list = list(set(self.ip_list + ip_netrange + ip_netblocks)) ip_list.sort(key=lambda s: list(map(int, s.split('.')))) return ip_list @staticmethod def read_scope_from_args(arg_line): """Read scope from comma-separated entry""" ip_list = [] hostname_list = [] netblock_list = [] netrange_list = [] unknown_list = [] for entry in arg_line.split(','): hostname_list, ip_list, netblock_list, netrange_list = Scope._parse_scope_entry(entry, hostname_list, ip_list, netblock_list, netrange_list, unknown_list) return Scope(ip_list=ip_list, hostname_list=hostname_list, netblock_list=netblock_list, netrange_list=netrange_list) @staticmethod def read_scope_from_file(filename): """Read the scope from a file containing what's supported (see class description)""" ip_list = [] hostname_list = [] netblock_list = [] netrange_list = [] unknown_list = [] # Read the scope file, extract the scope with open(filename, 'r') as scope_file: for line in scope_file: hostname_list, ip_list, netblock_list, netrange_list = Scope._parse_scope_entry(line, hostname_list, ip_list, netblock_list, netrange_list, unknown_list) return Scope(ip_list=ip_list, hostname_list=hostname_list, netblock_list=netblock_list, netrange_list=netrange_list) @staticmethod def _parse_scope_entry(scope_entry, hostname_list, ip_list, netblock_list, netrange_list, unknown_list): scope_entry = scope_entry.strip() if Scope.is_ip(scope_entry): ip_list += [scope_entry] elif Scope.is_hostname(scope_entry): hostname_list += [scope_entry] elif Scope.is_netblock(scope_entry): netblock_list += [scope_entry] elif Scope.is_netrange(scope_entry): netrange_list += [scope_entry] else: unknown_list += [scope_entry] return hostname_list, ip_list, netblock_list, netrange_list @staticmethod def is_ip(ip): # inet_aton is lax. If there's space and garbage after, it'll cast the first half as an IP ip_match = re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip.strip()) if ip_match: try: socket.inet_aton(ip) return True except socket.error: return False else: return False @staticmethod def is_hostname(hostname): try: """Will validate if its ip OR fqdn OR domain""" if len(hostname) > 255: return False if hostname[-1] == ".": # strip exactly one dot from the right, if present hostname = hostname[:-1] # Check if there's at least one alpha char for the TLD # Could use a list of TLD... # Otherwise it clashes with ip range if not re.search('[a-zA-Z]', hostname.split('.')[-1]): return False # TODO check what this regex does...comments are nice... # Seems like to only check TLD. allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE) return True if all(allowed.match(x) for x in hostname.split(".")) \ else False except: return False @staticmethod def is_netblock(netblock): try: if '/' in netblock: (ip_addr, cidr) = netblock.split('/') if Scope._is_int(cidr) and Scope.is_ip(ip_addr): return True return False except: return False @staticmethod def _netrange_ip_to_ip(netrange): try: if '-' in netrange: (ip_addr_1, ip_addr_2_or_range) = (x.strip() for x in netrange.split('-')) if Scope.is_ip(ip_addr_2_or_range): if Scope.is_ip(ip_addr_1): return True return False except Exception as e: return False @staticmethod def _netrange_ip_to_int(netrange): try: if '-' in netrange: (ip_addr_1, ip_addr_2_or_range) = (x.strip() for x in netrange.split('-')) if Scope._is_int(ip_addr_2_or_range): if Scope.is_ip(ip_addr_1): return True return False except: return False @staticmethod def is_netrange(netrange): try: if Scope._netrange_ip_to_int(netrange) or Scope._netrange_ip_to_ip(netrange): return True return False except: return False @staticmethod def _is_int(s): try: int(s) return True except ValueError: return False @staticmethod def expand_netblock(netblock): return [str(ip) for ip in IPNetwork(netblock)] @staticmethod def expand_netrange(ip_range): if not Scope._netrange_ip_to_ip(ip_range) and not Scope._netrange_ip_to_int(ip_range): raise ValueError( str(ip_range) + ' is not an ip range (ex: 1.1.1.1-255, 1.1.1.1-2.2.2.2)') (ip_addr, ip_range_end) = (x.strip() for x in ip_range.split('-')) if Scope._netrange_ip_to_int(ip_range): return Scope._get_netrange_from_ip_to_int(ip_addr, ip_range_end) else: return [str(x) for x in IPSet(IPRange(ip_addr, ip_range_end))] @staticmethod def _get_netrange_from_ip_to_int(ip_addr, ip_range_end): ip_bytes = ip_addr.split('.') ip_list = [ip_addr] ip_start = int(ip_bytes[-1]) ip_end = int(ip_range_end) for ip_suffix in range(ip_start + 1, ip_end + 1): ip = str(ip_bytes[0] + '.' + ip_bytes[1] + '.' + ip_bytes[2] + '.' + str(ip_suffix)) ip_list += [ip] return ip_list