def assignIPs(_upper_ref,
_from_key,
lower_refs,
to_key,
ip_start='192.168.0.1',
ip_end='192.168.0.254',
**_kwargs):
'''assign ips to hosts.'''
if not ip_start or not ip_end:
return {}
host_ips = {}
unassigned_hosts = []
ips = IPSet(IPRange(ip_start, ip_end))
for lower_key, lower_ref in lower_refs.items():
ip_addr = lower_ref.get(to_key, '')
if ip_addr:
host_ips[lower_key] = ip_addr
ips.remove(ip_addr)
else:
unassigned_hosts.append(lower_key)
for ip_addr in ips:
if not unassigned_hosts:
break
host = unassigned_hosts.pop(0)
host_ips[host] = str(ip_addr)
logging.debug('assign %s: %s', to_key, host_ips)
return host_ips
def main():
parser = ArgumentParser(description='支持从某个CIDR网段排除特定IP！')
parser.add_argument('netstr',
type=str,
metavar='CIDR',
help='需要排除IP的网段，如：0.0.0.0/0')
parser.add_argument(
'-r',
type=str,
metavar='IP',
help='排除的IP，支持同时排除多个，支持网段，例子: -r 192.168.1.128 -r 192.168.2.0/24',
action='append')
parser.add_argument_group()
args = parser.parse_args()
try:
ipnet = IPNetwork(args.netstr).cidr
ips = IPSet([ipnet])
except Exception as e:
print('CIDR 输入错误：{}'.format(e))
try:
if args.r:
for i in args.r:
ips.remove(i)
except Exception as e:
print('-r IP输入错误：{}'.format(e))
print('\n拆分后的网段段为：\n')
print(','.join(outip for outip in map(str, ips.iter_cidrs())))
def test_ipset_with_iprange():
s1 = IPSet(['10.0.0.0/25', '10.0.0.128/25'])
assert s1.iprange() == IPRange('10.0.0.0', '10.0.0.255')
assert s1.iscontiguous()
s1.remove('10.0.0.16')
assert s1 == IPSet([
'10.0.0.0/28', '10.0.0.17/32', '10.0.0.18/31',
'10.0.0.20/30', '10.0.0.24/29', '10.0.0.32/27',
'10.0.0.64/26', '10.0.0.128/25',
])
assert not s1.iscontiguous()
with pytest.raises(ValueError):
s1.iprange()
assert list(s1.iter_ipranges()) == [
IPRange('10.0.0.0', '10.0.0.15'),
IPRange('10.0.0.17', '10.0.0.255'),
]
s2 = IPSet(['0.0.0.0/0'])
assert s2.iscontiguous()
assert s2.iprange() == IPRange('0.0.0.0', '255.255.255.255')
#
s3 = IPSet()
assert s3.iscontiguous()
assert s3.iprange() is None
s4 = IPSet(IPRange('10.0.0.0', '10.0.0.8'))
assert s4.iscontiguous()
def assign_ips(_upper_ref, _from_key, lower_refs, to_key,
ip_start='192.168.0.1', ip_end='192.168.0.254',
**_kwargs):
"""Assign ips to hosts' configurations."""
if not ip_start or not ip_end:
return {}
host_ips = {}
unassigned_hosts = []
ips = IPSet(IPRange(ip_start, ip_end))
for lower_key, lower_ref in lower_refs.items():
ip_addr = lower_ref.get(to_key, '')
if ip_addr:
host_ips[lower_key] = ip_addr
ips.remove(ip_addr)
else:
unassigned_hosts.append(lower_key)
for ip_addr in ips:
if not unassigned_hosts:
break
host = unassigned_hosts.pop(0)
host_ips[host] = str(ip_addr)
logging.debug('assign %s: %s', to_key, host_ips)
return host_ips
def generateIP(self):
network = IPSet(IPNetwork(self.cidr))
network.remove(min(network))
network.remove(max(network))
hostlist = IPSet([h.ip for h in self.hosts.all()])
available = network - hostlist
return min(available)
'''
def subtract_subnet(original_subnet, remove_subnets):
"""Subtract a list of subnets from original subnet and return leftover
subnet(s) in a list"""
original = IPSet([original_subnet])
for subnet in remove_subnets:
original.remove(subnet)
return [str(subnet) for subnet in original.iter_cidrs()]
def generateIP(self):
network = IPSet(IPNetwork(self.cidr))
network.remove(min(network))
network.remove(max(network))
hostlist = IPSet([ h.ip for h in self.hosts.all() ])
available = network - hostlist
return min(available)
'''
def test_ipset_member_insertion_and_deletion():
s1 = IPSet()
s1.add('192.0.2.0')
assert s1 == IPSet(['192.0.2.0/32'])
s1.remove('192.0.2.0')
assert s1 == IPSet([])
s1.add(IPRange("10.0.0.0", "10.0.0.255"))
assert s1 == IPSet(['10.0.0.0/24'])
s1.remove(IPRange("10.0.0.128", "10.10.10.10"))
assert s1 == IPSet(['10.0.0.0/25'])
def test_ipset_member_insertion_and_deletion():
s1 = IPSet()
s1.add('192.0.2.0')
assert s1 == IPSet(['192.0.2.0/32'])
s1.remove('192.0.2.0')
assert s1 == IPSet([])
s1.add(IPRange("10.0.0.0", "10.0.0.255"))
assert s1 == IPSet(['10.0.0.0/24'])
s1.remove(IPRange("10.0.0.128", "10.10.10.10"))
assert s1 == IPSet(['10.0.0.0/25'])
def choose_ip(routable_cidrs, excluded_cidrs=[], client_addr=''):
"""Find available IP addresses for both sides of a VPN Tunnel.
This method iterates over the settings.ALLOWED_CIDRS list in order to
allocate available IP address to both the client and server side of a
VPN tunnel. CIDRs that belong to the lists of settings.RESERVED_CIDRS,
`routable_cidrs`, and `excluded_cidrs` are excluded from the allocation
process.
:param routable_cidrs: the CIDRs that are to be routed over a VPN tunnel
:param excluded_cidrs: an optional list of CIDRs to be excluded from the
address allocation process
:param client_addr: the `client_addr` is used to attempt to pick an
adjacent IP address for the server side
:return: a private IP address
"""
exc_nets = routable_cidrs + excluded_cidrs + settings.RESERVED_CIDRS
# make sure the exc_nets list does not contain any empty strings
exc_nets = [exc_net for exc_net in exc_nets if exc_net]
# a list of unique, non-overlapping supernets (to be excluded)
exc_nets = IPSet(exc_nets).iter_cidrs()
for network in settings.ALLOWED_CIDRS:
available_cidrs = IPSet(IPNetwork(network))
for exc_net in exc_nets:
available_cidrs.remove(exc_net)
if not available_cidrs:
continue
for cidr in available_cidrs.iter_cidrs():
first, last = cidr.first, cidr.last
if client_addr:
address = IPAddress(client_addr) + 1
else:
address = IPAddress(random.randrange(first + 1, last))
for _ in xrange(first + 1, last):
if address not in cidr or address == cidr.broadcast:
address = cidr.network + 1
try:
Tunnel.objects.get(Q(client=str(address)) |
Q(server=str(address)))
address += 1
except Tunnel.DoesNotExist:
return str(address)
def available_ipv4s(self, limit=25):
def _gen(ip_set):
for ip_net in ip_set.iter_cidrs():
for ip in ip_net.iter_hosts():
if ip.words[-1] not in [0, 255, 0xffff]:
yield ip
interfaces = Interface.query.filter_by(network_id=self.id).all()
static_ips = IPSet([self.static_net]) if self.static_net else IPSet()
reserved_ips = IPSet([self.reserved_net]) if self.reserved_net else IPSet()
for interface in interfaces:
if interface.static_ipv4:
static_ips.remove(interface.static_ipv4)
if interface.reserved_ipv4:
reserved_ips.remove(interface.reserved_ipv4)
return (list(itertools.islice(_gen(static_ips), limit)),
list(itertools.islice(_gen(reserved_ips), limit)))
def test_ipset_adding_and_removing_members_ip_addresses_as_ints():
s1 = IPSet(['10.0.0.0/25'])
s1.add('10.0.0.0/24')
assert s1 == IPSet(['10.0.0.0/24'])
integer1 = int(IPAddress('10.0.0.1'))
integer2 = int(IPAddress('fe80::'))
integer3 = int(IPAddress('10.0.0.2'))
s2 = IPSet([integer1, integer2])
assert s2 == IPSet(['10.0.0.1/32', 'fe80::/128'])
s2.add(integer3)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
s2.remove(integer2)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32'])
s2.update([integer2])
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
def test_ipset_adding_and_removing_members_ip_addresses_as_ints():
s1 = IPSet(['10.0.0.0/25'])
s1.add('10.0.0.0/24')
assert s1 == IPSet(['10.0.0.0/24'])
integer1 = int(IPAddress('10.0.0.1'))
integer2 = int(IPAddress('fe80::'))
integer3 = int(IPAddress('10.0.0.2'))
s2 = IPSet([integer1, integer2])
assert s2 == IPSet(['10.0.0.1/32', 'fe80::/128'])
s2.add(integer3)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
s2.remove(integer2)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32'])
s2.update([integer2])
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
def __init__(self, whitelist, blacklist):
set = IPSet([])
for block in whitelist:
set.add(IPNetwork(block))
# Remove invalid broadcast and network addresses
if block.broadcast != None:
set.remove(IPNetwork(block.broadcast))
if block.size > 2:
set.remove(IPNetwork(block.network))
for block in blacklist:
set.remove(IPNetwork(block))
for block in set.iter_cidrs():
self.total += block.size
self.networks.append({
"network": block,
"size": block.size,
"start": block[0],
"index": 0
})
if self.total < 1:
raise Exception(
"IPScanManager can not be started with an empty target scope")
self.rng = CyclicPRNG(self.total)
def blockcomp(b):
return b["start"]
self.networks.sort(key=blockcomp)
start = 1
for i in range(0, len(self.networks)):
self.networks[i]["index"] = start
start += self.networks[i]["size"]
def test_ipset_with_iprange():
s1 = IPSet(['10.0.0.0/25', '10.0.0.128/25'])
assert s1.iprange() == IPRange('10.0.0.0', '10.0.0.255')
assert s1.iscontiguous()
s1.remove('10.0.0.16')
assert s1 == IPSet([
'10.0.0.0/28',
'10.0.0.17/32',
'10.0.0.18/31',
'10.0.0.20/30',
'10.0.0.24/29',
'10.0.0.32/27',
'10.0.0.64/26',
'10.0.0.128/25',
])
assert not s1.iscontiguous()
with pytest.raises(ValueError):
s1.iprange()
assert list(s1.iter_ipranges()) == [
IPRange('10.0.0.0', '10.0.0.15'),
IPRange('10.0.0.17', '10.0.0.255'),
]
s2 = IPSet(['0.0.0.0/0'])
assert s2.iscontiguous()
assert s2.iprange() == IPRange('0.0.0.0', '255.255.255.255')
#
s3 = IPSet()
assert s3.iscontiguous()
assert s3.iprange() is None
s4 = IPSet(IPRange('10.0.0.0', '10.0.0.8'))
assert s4.iscontiguous()
def _get_free_set(self, network, domain):
ip_range_start = self.get_network_ip_range_start(network, domain)
ip_range_end = self.get_network_ip_range_end(network, domain)
select_range = IPRange(ip_range_start, ip_range_end)
netset = IPSet(select_range)
if (network == self.get_infra_external_network_name()
and domain == self._get_vip_domain()):
iterator = netset.__iter__()
self.external_vip = str(iterator.next())
netset.remove(self.external_vip)
# check for the IP(s) taken by the nodes
try:
hostsconfig = self.confman.get_hosts_config_handler()
hosts = hostsconfig.get_hosts()
for host in hosts:
try:
hostip = self.get_host_ip(host, network)
netset.remove(hostip)
except configerror.ConfigError:
pass
except configerror.ConfigError:
pass
service_profiles_lib = profiles.Profiles()
# check for the IP(s) taken as VIPs
if network == self.get_infra_internal_network_name(
) and domain == self._get_vip_domain():
vips = self.get_net_vips(network)
for _, vip in vips.iteritems():
try:
netset.remove(vip)
except configerror.ConfigError:
pass
return netset
def sync_subnets(conn, config):
log.debug("loading routing tables")
routing_tables = conn.get_all_route_tables()
route_tables_by_name = {r.tags.get('Name'): r for r in routing_tables}
route_tables_by_subnet_id = {}
for r in routing_tables:
for a in r.associations:
route_tables_by_subnet_id[a.subnet_id] = r
# Get list of AZs
zones = conn.get_all_zones()
for vpc_id in config:
# Get a list of all the remote subnets
remote_subnets = conn.get_all_subnets(filters={'vpcId': vpc_id})
seen = set()
# Go through our config, adjusting or any subnets as appropriate
for cidr, block_config in config[vpc_id].items():
cidr_net = IPNetwork(cidr)
table_name = block_config.get('routing_table')
if table_name and table_name not in route_tables_by_name:
log.warn("couldn't find routing table %s for block %s",
table_name, cidr)
log.warn("skipping rest of %s", cidr)
continue
my_rt = route_tables_by_name[table_name]
ip_set = IPSet(cidr_net)
for s in remote_subnets:
if IPNetwork(s.cidr_block) in cidr_net:
ip_set.remove(s.cidr_block)
if s.tags.get('Name') != block_config['name']:
log.info("Setting Name of %s to %s", s,
block_config['name'])
s.add_tag('Name', block_config['name'])
if s.id in route_tables_by_subnet_id:
remote_rt = route_tables_by_subnet_id[s.id]
else:
remote_rt = route_tables_by_subnet_id[None]
if remote_rt != my_rt:
log.info(
"Changing routing table for %s (%s) to %s (%s)",
s, s.tags.get('Name'), my_rt,
my_rt.tags.get('Name'))
if raw_input("(y/N) ") == "y":
conn.associate_route_table(my_rt.id, s.id)
seen.add(s)
# Are we missing any subnets?
# If so, create them!
# TODO: We want to evenly distribute the ip range over the
# configured availability zones, without dividing smaller than a
# /25 network (128 ips, at least 2 of which are reserved)
# For now we'll just split them as small as /24, and then assign
# them into the subnets
while ip_set:
log.info("%s - %s isn't covered by any subnets", cidr, ip_set)
my_zones = [
z for z in zones
if z.name not in block_config.get('skip_azs', [])
]
remaining_cidrs = list(ip_set.iter_cidrs())
remaining_cidrs.sort(key=lambda s: s.size, reverse=True)
for s in remaining_cidrs[:]:
if s.prefixlen < 24:
added = list(s.subnet(24))
remaining_cidrs.remove(s)
remaining_cidrs.extend(added)
ip_set.remove(s)
zg = itertools.cycle(my_zones)
while remaining_cidrs:
c = remaining_cidrs.pop()
z = next(zg)
log.info("creating subnet %s in %s/%s", c, z.name, vpc_id)
if raw_input("(y/N) ") == "y":
log.debug("creating subnet")
s = conn.create_subnet(vpc_id, c, z.name)
log.debug("adding tag")
# TODO: sometimes the subnet isn't actually created by
# the time we try and add the tag, so get a 400 error
s.add_tag('Name', block_config['name'])
log.debug("associating routing")
conn.associate_route_table(my_rt.id, s.id)
local_missing = set(remote_subnets) - seen
for m in local_missing:
log.info("%s:%s (name: %s) is unmanaged", m.id, m.cidr_block,
m.tags.get('Name'))
#!/usr/bin/python
from sys import argv
from netaddr import IPSet
if len(argv) != 3:
print('Usage: netdiff.py3 include.txt exclude.txt')
exit()
net = IPSet()
incfile = open(argv[1])
for line in incfile:
net = net | IPSet([line])
exfile = open(argv[2])
for line in exfile:
net.remove(line)
for cidr in net.iter_cidrs():
print(cidr)
class IPScanManager:
networks = []
total = 0
rng = None
consistent = None
def __init__(self, whitelist, blacklist, consistent: bool):
self.networks = []
self.total = 0
self.rng = None
self.consistent = consistent
self.set_whitelist(whitelist)
self.set_blacklist(blacklist)
self.initialize_manager()
def log_to_db(self, message):
log_messages = {
"init": "PRNG Starting Up",
"restart": "PRNG Cycle Restarted",
"default": "Unknown PRNG Event",
}
db_log = ScopeLog(log_messages.get(message, "default"))
db.session.add(db_log)
db.session.commit()
def set_whitelist(self, whitelist):
self.whitelist = IPSet([])
for block in whitelist:
self.whitelist.add(IPNetwork(block))
# Remove invalid broadcast and network addresses
if block.broadcast is not None:
self.whitelist.remove(IPNetwork(block.broadcast))
if block.size > 2:
self.whitelist.remove(IPNetwork(block.network))
def set_blacklist(self, blacklist):
self.blacklist = IPSet([])
for block in blacklist:
self.blacklist.add(IPNetwork(block))
def initialize_manager(self):
self.networks = []
self.ipset = self.whitelist - self.blacklist
for block in self.ipset.iter_cidrs():
self.total += block.size
self.networks.append({
"network": block,
"size": block.size,
"start": block[0],
"index": 0
})
if self.total < 1:
raise Exception(
"IPScanManager can not be started with an empty target scope")
self.rng = CyclicPRNG(self.total,
consistent=self.consistent,
event_handler=self.log_to_db)
def blockcomp(b):
return b["start"]
self.networks.sort(key=blockcomp)
start = 1
for i in range(0, len(self.networks)):
self.networks[i]["index"] = start
start += self.networks[i]["size"]
self.initialized = True
def in_whitelist(self, ip):
return ip in self.whitelist
def in_blacklist(self, ip):
return ip in self.blacklist
def get_total(self):
return self.total
def get_ready(self):
return self.rng and self.total > 0 and self.initialized
def get_next_ip(self):
if self.rng:
index = self.rng.get_random()
return self.get_ip(index)
else:
return None
def get_ip(self, index):
def binarysearch(networks, i):
middle = int(len(networks) / 2)
network = networks[middle]
if i < network["index"]:
return binarysearch(networks[:middle], i)
elif i >= (network["index"] + network["size"]):
return binarysearch(networks[middle + 1:], i)
else:
return network["network"][i - network["index"]]
return binarysearch(self.networks, index)
#! /usr/bin/env python
# Calculates the disjunction of two sets of IP ranges
from sys import argv
from netaddr import IPSet
if len(argv) != 3:
print('Usage: {0} include.txt exclude.txt'.format(argv[0]))
exit()
net = IPSet()
with open(argv[1], 'r') as incfile:
for line in incfile:
net = net | IPSet([line])
with open(argv[2], 'r') as exfile:
for line in exfile:
net.remove(line)
for cidr in net.iter_cidrs():
print(cidr)
def test_ipset_cidr_fracturing():
s1 = IPSet(['0.0.0.0/0'])
s1.remove('255.255.255.255')
assert s1 == IPSet([
'0.0.0.0/1', '128.0.0.0/2', '192.0.0.0/3', '224.0.0.0/4',
'240.0.0.0/5', '248.0.0.0/6', '252.0.0.0/7', '254.0.0.0/8',
'255.0.0.0/9', '255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15', '255.254.0.0/16',
'255.255.0.0/17', '255.255.128.0/18', '255.255.192.0/19',
'255.255.224.0/20', '255.255.240.0/21', '255.255.248.0/22',
'255.255.252.0/23', '255.255.254.0/24', '255.255.255.0/25',
'255.255.255.128/26', '255.255.255.192/27', '255.255.255.224/28',
'255.255.255.240/29', '255.255.255.248/30', '255.255.255.252/31',
'255.255.255.254/32'
])
cidrs = s1.iter_cidrs()
assert len(cidrs) == 32
assert list(cidrs) == [
IPNetwork('0.0.0.0/1'),
IPNetwork('128.0.0.0/2'),
IPNetwork('192.0.0.0/3'),
IPNetwork('224.0.0.0/4'),
IPNetwork('240.0.0.0/5'),
IPNetwork('248.0.0.0/6'),
IPNetwork('252.0.0.0/7'),
IPNetwork('254.0.0.0/8'),
IPNetwork('255.0.0.0/9'),
IPNetwork('255.128.0.0/10'),
IPNetwork('255.192.0.0/11'),
IPNetwork('255.224.0.0/12'),
IPNetwork('255.240.0.0/13'),
IPNetwork('255.248.0.0/14'),
IPNetwork('255.252.0.0/15'),
IPNetwork('255.254.0.0/16'),
IPNetwork('255.255.0.0/17'),
IPNetwork('255.255.128.0/18'),
IPNetwork('255.255.192.0/19'),
IPNetwork('255.255.224.0/20'),
IPNetwork('255.255.240.0/21'),
IPNetwork('255.255.248.0/22'),
IPNetwork('255.255.252.0/23'),
IPNetwork('255.255.254.0/24'),
IPNetwork('255.255.255.0/25'),
IPNetwork('255.255.255.128/26'),
IPNetwork('255.255.255.192/27'),
IPNetwork('255.255.255.224/28'),
IPNetwork('255.255.255.240/29'),
IPNetwork('255.255.255.248/30'),
IPNetwork('255.255.255.252/31'),
IPNetwork('255.255.255.254/32')
]
assert cidrs == cidr_exclude('0.0.0.0/0', '255.255.255.255')
s1.remove('0.0.0.0')
assert s1 == IPSet([
'0.0.0.1/32',
'0.0.0.2/31',
'0.0.0.4/30',
'0.0.0.8/29',
'0.0.0.16/28',
'0.0.0.32/27',
'0.0.0.64/26',
'0.0.0.128/25',
'0.0.1.0/24',
'0.0.2.0/23',
'0.0.4.0/22',
'0.0.8.0/21',
'0.0.16.0/20',
'0.0.32.0/19',
'0.0.64.0/18',
'0.0.128.0/17',
'0.1.0.0/16',
'0.2.0.0/15',
'0.4.0.0/14',
'0.8.0.0/13',
'0.16.0.0/12',
'0.32.0.0/11',
'0.64.0.0/10',
'0.128.0.0/9',
'1.0.0.0/8',
'2.0.0.0/7',
'4.0.0.0/6',
'8.0.0.0/5',
'16.0.0.0/4',
'32.0.0.0/3',
'64.0.0.0/2',
'128.0.0.0/2',
'192.0.0.0/3',
'224.0.0.0/4',
'240.0.0.0/5',
'248.0.0.0/6',
'252.0.0.0/7',
'254.0.0.0/8',
'255.0.0.0/9',
'255.128.0.0/10',
'255.192.0.0/11',
'255.224.0.0/12',
'255.240.0.0/13',
'255.248.0.0/14',
'255.252.0.0/15',
'255.254.0.0/16',
'255.255.0.0/17',
'255.255.128.0/18',
'255.255.192.0/19',
'255.255.224.0/20',
'255.255.240.0/21',
'255.255.248.0/22',
'255.255.252.0/23',
'255.255.254.0/24',
'255.255.255.0/25',
'255.255.255.128/26',
'255.255.255.192/27',
'255.255.255.224/28',
'255.255.255.240/29',
'255.255.255.248/30',
'255.255.255.252/31',
'255.255.255.254/32',
])
assert len(list(s1.iter_cidrs())) == 62
s1.add('255.255.255.255')
s1.add('0.0.0.0')
assert s1 == IPSet(['0.0.0.0/0'])
def test_ipset_cidr_fracturing():
s1 = IPSet(['0.0.0.0/0'])
s1.remove('255.255.255.255')
assert s1 == IPSet([
'0.0.0.0/1', '128.0.0.0/2', '192.0.0.0/3',
'224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6',
'252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9',
'255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15',
'255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18',
'255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21',
'255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24',
'255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27',
'255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30',
'255.255.255.252/31', '255.255.255.254/32'])
cidrs = s1.iter_cidrs()
assert len(cidrs) == 32
assert list(cidrs) == [
IPNetwork('0.0.0.0/1'), IPNetwork('128.0.0.0/2'), IPNetwork('192.0.0.0/3'),
IPNetwork('224.0.0.0/4'), IPNetwork('240.0.0.0/5'), IPNetwork('248.0.0.0/6'),
IPNetwork('252.0.0.0/7'), IPNetwork('254.0.0.0/8'), IPNetwork('255.0.0.0/9'),
IPNetwork('255.128.0.0/10'), IPNetwork('255.192.0.0/11'), IPNetwork('255.224.0.0/12'),
IPNetwork('255.240.0.0/13'), IPNetwork('255.248.0.0/14'), IPNetwork('255.252.0.0/15'),
IPNetwork('255.254.0.0/16'), IPNetwork('255.255.0.0/17'), IPNetwork('255.255.128.0/18'),
IPNetwork('255.255.192.0/19'), IPNetwork('255.255.224.0/20'), IPNetwork('255.255.240.0/21'),
IPNetwork('255.255.248.0/22'), IPNetwork('255.255.252.0/23'), IPNetwork('255.255.254.0/24'),
IPNetwork('255.255.255.0/25'), IPNetwork('255.255.255.128/26'), IPNetwork('255.255.255.192/27'),
IPNetwork('255.255.255.224/28'), IPNetwork('255.255.255.240/29'), IPNetwork('255.255.255.248/30'),
IPNetwork('255.255.255.252/31'), IPNetwork('255.255.255.254/32')
]
assert cidrs == cidr_exclude('0.0.0.0/0', '255.255.255.255')
s1.remove('0.0.0.0')
assert s1 == IPSet([
'0.0.0.1/32', '0.0.0.2/31', '0.0.0.4/30',
'0.0.0.8/29', '0.0.0.16/28', '0.0.0.32/27',
'0.0.0.64/26', '0.0.0.128/25', '0.0.1.0/24',
'0.0.2.0/23', '0.0.4.0/22', '0.0.8.0/21',
'0.0.16.0/20', '0.0.32.0/19', '0.0.64.0/18',
'0.0.128.0/17', '0.1.0.0/16', '0.2.0.0/15',
'0.4.0.0/14', '0.8.0.0/13', '0.16.0.0/12',
'0.32.0.0/11', '0.64.0.0/10', '0.128.0.0/9',
'1.0.0.0/8', '2.0.0.0/7', '4.0.0.0/6',
'8.0.0.0/5', '16.0.0.0/4', '32.0.0.0/3',
'64.0.0.0/2', '128.0.0.0/2', '192.0.0.0/3',
'224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6',
'252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9',
'255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15',
'255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18',
'255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21',
'255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24',
'255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27',
'255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30',
'255.255.255.252/31', '255.255.255.254/32',
])
assert len(list(s1.iter_cidrs())) == 62
s1.add('255.255.255.255')
s1.add('0.0.0.0')
assert s1 == IPSet(['0.0.0.0/0'])
def sync_subnets(conn, config):
log.debug("loading routing tables")
routing_tables = conn.get_all_route_tables()
route_tables_by_name = {r.tags.get('Name'): r for r in routing_tables}
route_tables_by_subnet_id = {}
for r in routing_tables:
for a in r.associations:
route_tables_by_subnet_id[a.subnet_id] = r
# Get list of AZs
zones = conn.get_all_zones()
for vpc_id in config:
# Get a list of all the remote subnets
remote_subnets = conn.get_all_subnets(filters={'vpcId': vpc_id})
seen = set()
# Go through our config, adjusting or any subnets as appropriate
for cidr, block_config in config[vpc_id].items():
cidr_net = IPNetwork(cidr)
table_name = block_config.get('routing_table')
if table_name and table_name not in route_tables_by_name:
log.warn("couldn't find routing table %s for block %s", table_name, cidr)
log.warn("skipping rest of %s", cidr)
continue
my_rt = route_tables_by_name[table_name]
ip_set = IPSet(cidr_net)
for s in remote_subnets:
if IPNetwork(s.cidr_block) in cidr_net:
ip_set.remove(s.cidr_block)
if s.tags.get('Name') != block_config['name']:
log.info("Setting Name of %s to %s", s, block_config['name'])
s.add_tag('Name', block_config['name'])
if s.id in route_tables_by_subnet_id:
remote_rt = route_tables_by_subnet_id[s.id]
else:
remote_rt = route_tables_by_subnet_id[None]
if remote_rt != my_rt:
log.info(
"Changing routing table for %s (%s) to %s (%s)",
s, s.tags.get('Name'), my_rt,
my_rt.tags.get('Name'))
if raw_input("(y/N) ") == "y":
conn.associate_route_table(my_rt.id, s.id)
seen.add(s)
# Are we missing any subnets?
# If so, create them!
# TODO: We want to evenly distribute the ip range over the
# configured availability zones, without dividing smaller than a
# /25 network (128 ips, at least 2 of which are reserved)
# For now we'll just split them as small as /24, and then assign
# them into the subnets
while ip_set:
log.info("%s - %s isn't covered by any subnets", cidr, ip_set)
my_zones = [z for z in zones if z.name not in block_config.get('skip_azs', [])]
remaining_cidrs = list(ip_set.iter_cidrs())
remaining_cidrs.sort(key=lambda s: s.size, reverse=True)
for s in remaining_cidrs[:]:
if s.prefixlen < 24:
added = list(s.subnet(24))
remaining_cidrs.remove(s)
remaining_cidrs.extend(added)
ip_set.remove(s)
zg = itertools.cycle(my_zones)
while remaining_cidrs:
c = remaining_cidrs.pop()
z = next(zg)
log.info("creating subnet %s in %s/%s", c, z.name, vpc_id)
if raw_input("(y/N) ") == "y":
log.debug("creating subnet")
s = conn.create_subnet(vpc_id, c, z.name)
log.debug("adding tag")
# TODO: sometimes the subnet isn't actually created by
# the time we try and add the tag, so get a 400 error
s.add_tag('Name', block_config['name'])
log.debug("associating routing")
conn.associate_route_table(my_rt.id, s.id)
local_missing = set(remote_subnets) - seen
for m in local_missing:
log.info("%s:%s (name: %s) is unmanaged", m.id, m.cidr_block, m.tags.get('Name'))
class Scope:
"""Util class to manipulate recon scope.
Supports:
IP (IPv4 standard only! NO padding, octal, decimal, shortened, etc.)
Netblocks (192.168.2.1/24)
Netrange (192.168.2.1-25)
-> Somewhat limited compared to Nmap funky range selectors
FQDN
"""
def __init__(self, ip_list=[], hostname_list=[], netblock_list=[], netrange_list=[], auto_coalesce_ip=True,
auto_resolve_hostnames=False):
self._auto_coalesce = auto_coalesce_ip
self._auto_resolve_hostnames = auto_resolve_hostnames
not_ip = [x for x in ip_list if not Scope.is_ip(x)]
if not_ip:
raise ValueError('Value(s) ' + str(not_ip) + ' are provided as IP but cannot be casted as such')
self._ip_list = IPSet(ip_list)
not_hostname = [x for x in hostname_list if not Scope.is_hostname(x)]
if not_hostname:
raise ValueError('Value(s) ' + str(not_hostname) + ' are provided as hostnames but cannot be casted as such')
self._hostname_list = set(hostname_list)
not_netblock = [x for x in netblock_list if not Scope.is_netblock(x)]
if not_netblock:
raise ValueError('Value(s) ' + str(not_netblock) + ' are provided as netblocks but cannot be casted as such')
self._netblock_list = set(IPNetwork(x) for x in set(netblock_list))
not_netrange = [x for x in netrange_list if not Scope.is_netrange(x)]
if not_netrange:
raise ValueError('Value(s) ' + str(not_netrange) + ' are provided as netranges but cannot be casted as such')
self._netrange_list = set()
for netrange in set(netrange_list):
if self._netrange_ip_to_int(netrange):
self._netrange_list.add(IPGlob(netrange))
else:
(ip1, ip2) = (x.strip() for x in netrange.split('-'))
self._netrange_list.add(IPRange(ip1, ip2))
if self._auto_resolve_hostnames:
self._resolve_hostnames()
if self._auto_coalesce:
self._coalesce_ip_list()
@property
def ip_list(self):
return tuple(str(ip) for ip in self._ip_list)
@property
def hostname_list(self):
return tuple(str(hostname) for hostname in self._hostname_list)
@property
def netblock_list(self):
return tuple(str(netblock) for netblock in self._netblock_list)
@property
def netrange_list(self):
return tuple(str(netrange) for netrange in self._netrange_list)
def _coalesce_ip_list(self):
"""Merge IPs in appropriate netblocks/netranges, if possible, and remove them from ip_list"""
for ip in self._ip_list:
if ip in self._netrange_list:
self._ip_list.remove(ip)
if ip in self._netblock_list:
self._ip_list.remove(ip)
def _resolve_hostnames(self):
for hostname in self.hostname_list:
for result in dns.resolver.query(hostname, 'A'):
# Add IP if it's not already in the IP list.
# Will be coalesced later if flag enabled
if str(result) not in self._ip_list:
self._ip_list.add(str(result))
def is_ip_in_scope(self, ip):
if ip in self._ip_list:
return True
for netblock in self._netblock_list:
if ip in IPSet(netblock):
return True
for netrange in self._netrange_list:
if ip in IPSet(netrange):
return True
return False
def get_expanded_ip_list(self):
ip_netblocks = tuple(str(ip) for nb in self._netblock_list for ip in nb)
ip_netrange = tuple(str(ip) for nr in self._netrange_list for ip in nr)
ip_list = list(set(self.ip_list + ip_netrange + ip_netblocks))
ip_list.sort(key=lambda s: list(map(int, s.split('.'))))
return ip_list
@staticmethod
def read_scope_from_args(arg_line):
"""Read scope from comma-separated entry"""
ip_list = []
hostname_list = []
netblock_list = []
netrange_list = []
unknown_list = []
for entry in arg_line.split(','):
hostname_list, ip_list, netblock_list, netrange_list = Scope._parse_scope_entry(entry,
hostname_list,
ip_list,
netblock_list,
netrange_list,
unknown_list)
return Scope(ip_list=ip_list,
hostname_list=hostname_list,
netblock_list=netblock_list,
netrange_list=netrange_list)
@staticmethod
def read_scope_from_file(filename):
"""Read the scope from a file containing what's supported (see class description)"""
ip_list = []
hostname_list = []
netblock_list = []
netrange_list = []
unknown_list = []
# Read the scope file, extract the scope
with open(filename, 'r') as scope_file:
for line in scope_file:
hostname_list, ip_list, netblock_list, netrange_list = Scope._parse_scope_entry(line,
hostname_list,
ip_list,
netblock_list,
netrange_list,
unknown_list)
return Scope(ip_list=ip_list,
hostname_list=hostname_list,
netblock_list=netblock_list,
netrange_list=netrange_list)
@staticmethod
def _parse_scope_entry(scope_entry, hostname_list, ip_list, netblock_list, netrange_list, unknown_list):
scope_entry = scope_entry.strip()
if Scope.is_ip(scope_entry):
ip_list += [scope_entry]
elif Scope.is_hostname(scope_entry):
hostname_list += [scope_entry]
elif Scope.is_netblock(scope_entry):
netblock_list += [scope_entry]
elif Scope.is_netrange(scope_entry):
netrange_list += [scope_entry]
else:
unknown_list += [scope_entry]
return hostname_list, ip_list, netblock_list, netrange_list
@staticmethod
def is_ip(ip):
# inet_aton is lax. If there's space and garbage after, it'll cast the first half as an IP
ip_match = re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip.strip())
if ip_match:
try:
socket.inet_aton(ip)
return True
except socket.error:
return False
else:
return False
@staticmethod
def is_hostname(hostname):
try:
"""Will validate if its ip OR fqdn OR domain"""
if len(hostname) > 255:
return False
if hostname[-1] == ".":
# strip exactly one dot from the right, if present
hostname = hostname[:-1]
# Check if there's at least one alpha char for the TLD
# Could use a list of TLD...
# Otherwise it clashes with ip range
if not re.search('[a-zA-Z]', hostname.split('.')[-1]):
return False
# TODO check what this regex does...comments are nice...
# Seems like to only check TLD.
allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
return True if all(allowed.match(x) for x in hostname.split(".")) \
else False
except:
return False
@staticmethod
def is_netblock(netblock):
try:
if '/' in netblock:
(ip_addr, cidr) = netblock.split('/')
if Scope._is_int(cidr) and Scope.is_ip(ip_addr):
return True
return False
except:
return False
@staticmethod
def _netrange_ip_to_ip(netrange):
try:
if '-' in netrange:
(ip_addr_1, ip_addr_2_or_range) = (x.strip() for x in netrange.split('-'))
if Scope.is_ip(ip_addr_2_or_range):
if Scope.is_ip(ip_addr_1):
return True
return False
except Exception as e:
return False
@staticmethod
def _netrange_ip_to_int(netrange):
try:
if '-' in netrange:
(ip_addr_1, ip_addr_2_or_range) = (x.strip() for x in netrange.split('-'))
if Scope._is_int(ip_addr_2_or_range):
if Scope.is_ip(ip_addr_1):
return True
return False
except:
return False
@staticmethod
def is_netrange(netrange):
try:
if Scope._netrange_ip_to_int(netrange) or Scope._netrange_ip_to_ip(netrange):
return True
return False
except:
return False
@staticmethod
def _is_int(s):
try:
int(s)
return True
except ValueError:
return False
@staticmethod
def expand_netblock(netblock):
return [str(ip) for ip in IPNetwork(netblock)]
@staticmethod
def expand_netrange(ip_range):
if not Scope._netrange_ip_to_ip(ip_range) and not Scope._netrange_ip_to_int(ip_range):
raise ValueError(
str(ip_range) + ' is not an ip range (ex: 1.1.1.1-255, 1.1.1.1-2.2.2.2)')
(ip_addr, ip_range_end) = (x.strip() for x in ip_range.split('-'))
if Scope._netrange_ip_to_int(ip_range):
return Scope._get_netrange_from_ip_to_int(ip_addr, ip_range_end)
else:
return [str(x) for x in IPSet(IPRange(ip_addr, ip_range_end))]
@staticmethod
def _get_netrange_from_ip_to_int(ip_addr, ip_range_end):
ip_bytes = ip_addr.split('.')
ip_list = [ip_addr]
ip_start = int(ip_bytes[-1])
ip_end = int(ip_range_end)
for ip_suffix in range(ip_start + 1, ip_end + 1):
ip = str(ip_bytes[0] +
'.' + ip_bytes[1] +
'.' + ip_bytes[2] +
'.' + str(ip_suffix))
ip_list += [ip]
return ip_list
