def optimize_network_range(ipstr, threshold=0.9, verbose=DEBUG):
"""
Parses the input string and then calculates the subnet usage percentage. If over
the threshold it will return a loose result, otherwise it returns strict.
:param ipstr:
IP string to be parsed.
:param threshold:
The percentage of the network usage required to return a loose result.
:param verbose:
Toggle verbosity.
Example of default behavior using 0.9 (90% usage) threshold:
>>> import cidrize
>>> cidrize.optimize_network_range('10.20.30.40-50', verbose=True)
Subnet usage ratio: 0.34375; Threshold: 0.9
Under threshold, IP Parse Mode: STRICT
[IPNetwork('10.20.30.40/29'), IPNetwork('10.20.30.48/31'), IPNetwork('10.20.30.50/32')]
Example using a 0.3 (30% threshold):
>>> import cidrize
>>> cidrize.optimize_network_range('10.20.30.40-50', threshold=0.3, verbose=True)
Subnet usage ratio: 0.34375; Threshold: 0.3
Over threshold, IP Parse Mode: LOOSE
[IPNetwork('10.20.30.32/27')]
"""
if threshold > 1 or threshold < 0:
raise CidrizeError('Threshold must be from 0.0 to 1.0')
# Can't optimize 0.0.0.0/0!
if ipstr in EVERYTHING:
return cidrize(ipstr)
loose = IPSet(cidrize(ipstr))
strict = IPSet(cidrize(ipstr, strict=True))
ratio = float(len(strict)) / float(len(loose))
if verbose:
print 'Subnet usage ratio: %s; Threshold: %s' % (ratio, threshold)
if ratio >= threshold:
if verbose:
print 'Over threshold, IP Parse Mode: LOOSE'
result = loose.iter_cidrs()
else:
if verbose:
print 'Under threshold, IP Parse Mode: STRICT'
result = strict.iter_cidrs()
return result
def optimize_network_range(ipstr, threshold=0.9, verbose=DEBUG):
"""
Parses the input string and then calculates the subnet usage percentage. If over
the threshold it will return a loose result, otherwise it returns strict.
:param ipstr:
IP string to be parsed.
:param threshold:
The percentage of the network usage required to return a loose result.
:param verbose:
Toggle verbosity.
Example of default behavior using 0.9 (90% usage) threshold:
>>> import cidrize
>>> cidrize.optimize_network_range('10.20.30.40-50', verbose=True)
Subnet usage ratio: 0.34375; Threshold: 0.9
Under threshold, IP Parse Mode: STRICT
[IPNetwork('10.20.30.40/29'), IPNetwork('10.20.30.48/31'), IPNetwork('10.20.30.50/32')]
Example using a 0.3 (30% threshold):
>>> import cidrize
>>> cidrize.optimize_network_range('10.20.30.40-50', threshold=0.3, verbose=True)
Subnet usage ratio: 0.34375; Threshold: 0.3
Over threshold, IP Parse Mode: LOOSE
[IPNetwork('10.20.30.32/27')]
"""
if threshold > 1 or threshold < 0:
raise CidrizeError('Threshold must be from 0.0 to 1.0')
# Can't optimize 0.0.0.0/0!
if ipstr in EVERYTHING:
return cidrize(ipstr)
loose = IPSet(cidrize(ipstr))
strict = IPSet(cidrize(ipstr, strict=True))
ratio = float(len(strict)) / float(len(loose))
if verbose:
print 'Subnet usage ratio: %s; Threshold: %s' % (ratio, threshold)
if ratio >= threshold:
if verbose:
print 'Over threshold, IP Parse Mode: LOOSE'
result = loose.iter_cidrs()
else:
if verbose:
print 'Under threshold, IP Parse Mode: STRICT'
result = strict.iter_cidrs()
return result
def test_ipset_converts_to_cidr_networks_v6():
s1 = IPSet(IPNetwork('fe80::4242/64'))
s1.add(IPNetwork('fe90::4343/64'))
assert list(s1.iter_cidrs()) == [
IPNetwork('fe80::/64'),
IPNetwork('fe90::/64'),
]
def get_ips_list(ranges):
"""Get the IP addresses list from a list of ranges.
:param list ranges: List of ranges.
:returns: List of IP addresses.
:rtype: list of cidr ips
(https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing)
"""
ip_set = IPSet()
for ip_range in ranges:
try:
# It's a glob
if '*' in ip_range or '-' in ip_range:
ip_set.add(IPGlob(ip_range))
# It's a network
elif '/' in ip_range:
ip_set.add(IPNetwork(ip_range))
# Simple IP
else:
ip_set.add(IPAddress(ip_range))
except Exception:
pass
return [str(ip.cidr) for ip in ip_set.iter_cidrs()]
def simplify(ips):
"""
Remove duplicates and entirely overlapping blocks,
and sort by "version"
"""
nets = IPSet(ips)
return sorted(nets.iter_cidrs())
def __init__(self, whitelist, blacklist):
set = IPSet([])
for block in whitelist:
set.add(IPNetwork(block))
# Remove invalid broadcast and network addresses
if block.broadcast != None:
set.remove(IPNetwork(block.broadcast))
if block.size > 2:
set.remove(IPNetwork(block.network))
for block in blacklist:
set.remove(IPNetwork(block))
for block in set.iter_cidrs():
self.total += block.size
self.networks.append({
"network": block,
"size": block.size,
"start": block[0],
"index": 0
})
if self.total < 1:
raise Exception(
"IPScanManager can not be started with an empty target scope")
self.rng = CyclicPRNG(self.total)
def blockcomp(b):
return b["start"]
self.networks.sort(key=blockcomp)
start = 1
for i in range(0, len(self.networks)):
self.networks[i]["index"] = start
start += self.networks[i]["size"]
def test_ipset_converts_to_cidr_networks_v6():
s1 = IPSet(IPNetwork('fe80::4242/64'))
s1.add(IPNetwork('fe90::4343/64'))
assert list(s1.iter_cidrs()) == [
IPNetwork('fe80::/64'),
IPNetwork('fe90::/64'),
]
def test_ipset_converts_to_cidr_networks_v4():
s1 = IPSet(IPNetwork('10.1.2.3/8'))
s1.add(IPNetwork('192.168.1.2/16'))
assert list(s1.iter_cidrs()) == [
IPNetwork('10.0.0.0/8'),
IPNetwork('192.168.0.0/16'),
]
def test_ipset_converts_to_cidr_networks_v4():
s1 = IPSet(IPNetwork('10.1.2.3/8'))
s1.add(IPNetwork('192.168.1.2/16'))
assert list(s1.iter_cidrs()) == [
IPNetwork('10.0.0.0/8'),
IPNetwork('192.168.0.0/16'),
]
def main():
parser = ArgumentParser(description='支持从某个CIDR网段排除特定IP！')
parser.add_argument('netstr',
type=str,
metavar='CIDR',
help='需要排除IP的网段，如：0.0.0.0/0')
parser.add_argument(
'-r',
type=str,
metavar='IP',
help='排除的IP，支持同时排除多个，支持网段，例子: -r 192.168.1.128 -r 192.168.2.0/24',
action='append')
parser.add_argument_group()
args = parser.parse_args()
try:
ipnet = IPNetwork(args.netstr).cidr
ips = IPSet([ipnet])
except Exception as e:
print('CIDR 输入错误：{}'.format(e))
try:
if args.r:
for i in args.r:
ips.remove(i)
except Exception as e:
print('-r IP输入错误：{}'.format(e))
print('\n拆分后的网段段为：\n')
print(','.join(outip for outip in map(str, ips.iter_cidrs())))
def subtract_subnet(original_subnet, remove_subnets):
"""Subtract a list of subnets from original subnet and return leftover
subnet(s) in a list"""
original = IPSet([original_subnet])
for subnet in remove_subnets:
original.remove(subnet)
return [str(subnet) for subnet in original.iter_cidrs()]
def write_file(scope: str, content: IPSet, prefix=''):
if len(prefix) > 0 and not prefix.endswith('-'):
prefix = prefix + '-'
filename = f'output/{prefix}{scope}.txt'
cidrs = content.iter_cidrs()
log.info(f"Writing output file: {filename} with {len(cidrs)} CIDR blocks ...")
with open(filename, 'w') as f:
f.writelines(f'{cidr}\n' for cidr in cidrs)
log.debug(f"Wrote output file: {filename}")
def write_file(scope: str, content: IPSet, prefix=''):
if len(prefix)>0 and not prefix.endswith('-'):
prefix = prefix + '-'
filename = 'output/' + prefix + scope + '.txt'
cidrs = content.iter_cidrs()
log.info(f"Writing output file: {filename}")
log.info(f"There are {len(cidrs)} CIDR blocks in {filename}.")
with open(filename, 'w') as f:
f.writelines(f"{cidr}\n" for cidr in cidrs)
def write_file(scope: str, content: IPSet, prefix=''):
if len(prefix) > 0 and not prefix.endswith('-'):
prefix = prefix + '-'
filename = 'output/' + prefix + scope + '.txt'
cidrs = content.iter_cidrs()
log.info(f"Writing output file: {filename}")
log.info(f"There are {len(cidrs)} CIDR blocks in {filename}.")
with open(filename, 'w') as f:
f.writelines(f"{cidr}\n" for cidr in cidrs)
def allocate_cidr(all_cidrs: List[str], prefix: int) -> str:
cidrs = IPSet(all_cidrs)
for cidr in cidrs.iter_cidrs():
try:
return str(list(cidr.subnet(prefix))[0])
except Exception as err:
# cidr cannot accomodate prefix
print(err)
pass
else:
# Ref: https://docs.python.org/3/reference/simple_stmts.html#raise
raise PrefixLargerThanCidrs({"prefix": prefix, "cidrs": cidrs})
def concat_networks(context, pool_1, pool_2):
if pool_1.is_free and pool_2.is_free:
network_1 = pool_to_network(pool_1)
network_2 = pool_to_network(pool_2)
if network_1.size == network_2.size:
ipset = IPSet([network_1, network_2])
cidr = ipset.iter_cidrs()[0]
pool_1.ip = cidr.first
pool_1.netmask = cidr.netmask.value
count = len(pool_to_network(pool_1))
pool_1.count = count
pool_delete(context, pool_2.pool_id)
concat_pool(context, pool_1)
def concat_networks(context, pool_1, pool_2):
if pool_1.is_free and pool_2.is_free:
network_1 = pool_to_network(pool_1)
network_2 = pool_to_network(pool_2)
if network_1.size == network_2.size:
ipset = IPSet([network_1, network_2])
cidr = ipset.iter_cidrs()[0]
pool_1.ip = cidr.first
pool_1.netmask = cidr.netmask.value
count = len(pool_to_network(pool_1))
pool_1.count = count
pool_delete(context, pool_2.pool_id)
concat_pool(context, pool_1)
def add_available_prefixes(parent, prefix_list):
"""
Create fake Prefix objects for all unallocated space within a prefix.
"""
# Find all unallocated space
available_prefixes = IPSet(parent) ^ IPSet([p.prefix for p in prefix_list])
available_prefixes = [Prefix(prefix=p) for p in available_prefixes.iter_cidrs()]
# Concatenate and sort complete list of children
prefix_list = list(prefix_list) + available_prefixes
prefix_list.sort(key=lambda p: p.prefix)
return prefix_list
def add_available_prefixes(parent, prefix_list):
"""
Create fake Prefix objects for all unallocated space within a prefix.
"""
# Find all unallocated space
available_prefixes = IPSet(parent) ^ IPSet([p.prefix for p in prefix_list])
available_prefixes = [Prefix(prefix=p) for p in available_prefixes.iter_cidrs()]
# Concatenate and sort complete list of children
prefix_list = list(prefix_list) + available_prefixes
prefix_list.sort(key=lambda p: p.prefix)
return prefix_list
def summarizeIPs(inFile, outFile):
netSet = IPSet()
with open(inFile, 'r') as f:
for line in f.readlines():
net = IPSet()
try:
net.add(line.strip())
except AddrFormatError:
continue
else:
netSet = netSet | net
netMin = netSet.iter_cidrs()
with open(outFile, 'w') as f:
for net in netMin:
f.write('{}\n'.format(net))
def choose_ip(routable_cidrs, excluded_cidrs=[], client_addr=''):
"""Find available IP addresses for both sides of a VPN Tunnel.
This method iterates over the settings.ALLOWED_CIDRS list in order to
allocate available IP address to both the client and server side of a
VPN tunnel. CIDRs that belong to the lists of settings.RESERVED_CIDRS,
`routable_cidrs`, and `excluded_cidrs` are excluded from the allocation
process.
:param routable_cidrs: the CIDRs that are to be routed over a VPN tunnel
:param excluded_cidrs: an optional list of CIDRs to be excluded from the
address allocation process
:param client_addr: the `client_addr` is used to attempt to pick an
adjacent IP address for the server side
:return: a private IP address
"""
exc_nets = routable_cidrs + excluded_cidrs + settings.RESERVED_CIDRS
# make sure the exc_nets list does not contain any empty strings
exc_nets = [exc_net for exc_net in exc_nets if exc_net]
# a list of unique, non-overlapping supernets (to be excluded)
exc_nets = IPSet(exc_nets).iter_cidrs()
for network in settings.ALLOWED_CIDRS:
available_cidrs = IPSet(IPNetwork(network))
for exc_net in exc_nets:
available_cidrs.remove(exc_net)
if not available_cidrs:
continue
for cidr in available_cidrs.iter_cidrs():
first, last = cidr.first, cidr.last
if client_addr:
address = IPAddress(client_addr) + 1
else:
address = IPAddress(random.randrange(first + 1, last))
for _ in xrange(first + 1, last):
if address not in cidr or address == cidr.broadcast:
address = cidr.network + 1
try:
Tunnel.objects.get(Q(client=str(address)) |
Q(server=str(address)))
address += 1
except Tunnel.DoesNotExist:
return str(address)
def generate_plain_feed(feed, start, num, desc, value, **kwargs):
zrange = SR.zrange
if desc:
zrange = SR.zrevrange
if num is None:
num = (1 << 32) - 1
translate_ip_ranges = kwargs.pop('translate_ip_ranges', False)
should_aggregate = 'sum' in kwargs
if should_aggregate:
translate_ip_ranges = False
temp_set = set()
cstart = start
while cstart < (start + num):
ilist = zrange(feed, cstart,
cstart - 1 + min(start + num - cstart, FEED_INTERVAL))
if should_aggregate:
for i in ilist:
for n in _extract_cidrs(i):
temp_set.add(n)
else:
if translate_ip_ranges:
ilist = [xi for i in ilist for xi in _translate_ip_ranges(i)]
yield '\n'.join(ilist) + '\n'
if len(ilist) < 100:
break
cstart += 100
if should_aggregate:
ip_set = IPSet(temp_set)
for cidr in ip_set.iter_cidrs():
yield str(cidr) + '\n'
def allocate(self, netmask, net_group_name, stack_id=None, stack_name=None):
context = get_session()
network = IPNetwork('0.0.0.0/%s' % netmask)
pool = db.api.free_pool_find_by_netmask_and_netgroup(context, network.netmask.value, net_group_name)
ip_network = pool_to_network(pool)
if ip_network.size == network.size:
pool.is_free = False
pool.stack_id = stack_id
pool.stack_name = stack_name
pool.save()
allocated_pool = pool
else:
pool_list = list(ip_network.subnet(netmask))
allocated_network = pool_list[0]
pool_list = IPSet(pool_list[1::])
allocated_pool = db.api.used_pool_add(context, {'initial_pool': pool.initial_pool, 'cidr': allocated_network,
'stack_id': stack_id, 'stack_name': stack_name})
for free_pool in pool_list.iter_cidrs():
db.api.free_pool_add(context, {'initial_pool': pool.initial_pool, 'cidr': free_pool})
db.api.pool_delete(context, pool.pool_id)
logger.info('allocate pool id %s %s' % (allocated_pool.pool_id, allocated_network))
return allocated_pool
def insert_network(self, network: IPSet, content: MMDBType):
leaf = SearchTreeLeaf(content)
if not isinstance(network, IPSet):
raise ValueError("network type should be netaddr.IPSet.")
network = network.iter_cidrs()
for cidr in network:
if self.ip_version == 4 and cidr.version == 6:
raise ValueError('You inserted a IPv6 address {} '
'to an IPv4-only database.'.format(cidr))
if self.ip_version == 6 and cidr.version == 4:
if not self.ipv4_compatible:
raise ValueError(
"You inserted a IPv4 address {} to an IPv6 database."
"Please use ipv4_compatible=True option store "
"IPv4 address in IPv6 database as ::/96 format".format(
cidr))
cidr = cidr.ipv6(True)
node = self.tree
bits = list(
bits_rstrip(cidr.value, self._bit_length, cidr.prefixlen))
try:
for i in bits[:-1]:
node = node.get_or_create(i)
if node[bits[-1]] is not None:
logger.warning(
"address %s info is not empty: %s, will override with %s",
cidr, node[bits[-1]], leaf)
except (AttributeError, TypeError) as e:
bits_str = ''.join(map(str, bits))
logger.warning(
"{cidr}({bits_str})[{content}] is subnet of {node}, pass!".
format(cidr=cidr,
bits_str=bits_str,
content=content,
node=node))
continue
node[bits[-1]] = leaf
def allocate(self,
netmask,
net_group_name,
stack_id=None,
stack_name=None):
context = get_session()
network = IPNetwork('0.0.0.0/%s' % netmask)
pool = db.api.free_pool_find_by_netmask_and_netgroup(
context, network.netmask.value, net_group_name)
ip_network = pool_to_network(pool)
if ip_network.size == network.size:
pool.is_free = False
pool.stack_id = stack_id
pool.stack_name = stack_name
pool.save()
allocated_pool = pool
else:
pool_list = list(ip_network.subnet(netmask))
allocated_network = pool_list[0]
pool_list = IPSet(pool_list[1::])
allocated_pool = db.api.used_pool_add(
context, {
'initial_pool': pool.initial_pool,
'cidr': allocated_network,
'stack_id': stack_id,
'stack_name': stack_name
})
for free_pool in pool_list.iter_cidrs():
db.api.free_pool_add(context, {
'initial_pool': pool.initial_pool,
'cidr': free_pool
})
db.api.pool_delete(context, pool.pool_id)
logger.info('allocate pool id %s %s' %
(allocated_pool.pool_id, allocated_network))
return allocated_pool
def parse_commas(ipstr, **kwargs):
"""
This will break up a comma-separated input string of assorted inputs, run them through
cidrize(), flatten the list, and return the list. If any item in the list
fails, it will allow the exception to pass through as if it were parsed
individually. All objects must parse or nothing is returned.
Example:
:param ipstr:
A comma-separated string of IP address patterns.
"""
# Clean whitespace before we process
ipstr = ipstr.replace(' ', '').strip()
items = ipstr.split(',')
# Possibly nested depending on input, so we'll run it thru itertools.chain
# to flatten it. Then we make it a IPSet to optimize adjacencies and finally
# return the list of CIDRs within the IPSet
ipiter = (cidrize(ip, **kwargs) for ip in items)
flatiter = itertools.chain.from_iterable(ipiter)
ipset = IPSet(flatiter)
return ipset.iter_cidrs()
def parse_commas(ipstr, **kwargs):
"""
This will break up a comma-separated input string of assorted inputs, run them through
cidrize(), flatten the list, and return the list. If any item in the list
fails, it will allow the exception to pass through as if it were parsed
individually. All objects must parse or nothing is returned.
Example:
:param ipstr:
A comma-separated string of IP address patterns.
"""
# Clean whitespace before we process
ipstr = ipstr.replace(' ', '').strip()
items = ipstr.split(',')
# Possibly nested depending on input, so we'll run it thru itertools.chain
# to flatten it. Then we make it a IPSet to optimize adjacencies and finally
# return the list of CIDRs within the IPSet
ipiter = (cidrize(ip, **kwargs) for ip in items)
flatiter = itertools.chain.from_iterable(ipiter)
ipset = IPSet(flatiter)
return ipset.iter_cidrs()
def get_labels_expression_size(self, labels_expression: LabelsExpression):
"""
Return the size of the provided labels expression. The labels expression's size is calculated by summing
the following:
1. The amount of cidrs in the union of the subnets in the dynamic criteria of all the labels expression's
label members (labels and labels intersections)
2. The amount of ips of all the assets that match at least one of the expression's members (labels and
labels intersection), provided that those ips are not already contained in the cidrs calculated in (1)
"""
if str(labels_expression) in self.labels_expression_size_cache:
return self.labels_expression_size_cache[str(labels_expression)]
for labels_intersection in labels_expression.labels_intersections:
if str(labels_intersection
) not in self.labels_intersection_size_cache:
_ = self.get_labels_intersection_size(labels_intersection)
if len(labels_expression.labels_intersections) == 1:
self.labels_expression_criteria_cache[str(labels_expression)] = \
self.labels_intersection_criteria_cache[str(labels_expression)]
self.assets_matching_labels_expression_cache[str(labels_expression)] = \
self.assets_matching_labels_intersection_cache[str(
labels_expression)]
self.labels_expression_size_cache[str(labels_expression)] = \
self.labels_intersection_size_cache[str(labels_expression)]
return self.labels_expression_size_cache[str(labels_expression)]
labels_expression_criteria = IPSet()
assets_matching_the_labels_expression = set()
for labels_intersection in labels_expression.labels_intersections:
labels_intersection_criteria = self.labels_intersection_criteria_cache[
str(labels_intersection)]
labels_expression_criteria = labels_expression_criteria | labels_intersection_criteria
assets_matching_the_labels_intersection = \
self.assets_matching_labels_intersection_cache[str(
labels_intersection)]
assets_matching_the_labels_expression = (
assets_matching_the_labels_expression
| assets_matching_the_labels_intersection)
self.assets_matching_labels_expression_cache[str(
labels_expression)] = assets_matching_the_labels_expression
self.labels_expression_criteria_cache[str(
labels_expression)] = labels_expression_criteria
labels_expression_size = len(labels_expression_criteria.iter_cidrs())
for asset_id in assets_matching_the_labels_expression:
if asset_id in self.asset_ips_cache:
asset_ips = self.asset_ips_cache[asset_id]
else:
_ = self.get_asset_size_by_id(asset_id)
asset_ips = self.asset_ips_cache[asset_id]
asset_ips_that_are_not_in_the_labels_expression_criteria = [
ip for ip in asset_ips if ip not in labels_expression_criteria
]
labels_expression_size += len(
asset_ips_that_are_not_in_the_labels_expression_criteria)
self.labels_expression_size_cache[str(
labels_expression)] = labels_expression_size
return labels_expression_size
def scan_fn_ip():
country_code = {}
for line in open('input/country_code', 'r'):
code, name = line.split(" ")
country_code[code] = name.strip().decode("utf-8")
logger.info(code + ' ' + country_code[code])
rtree = ipRadixDB()
ip_area_list = ["input/delegated-arin-latest", "input/delegated-ripencc-latest", "input/delegated-lacnic-latest", "input/delegated-afrinic-latest", "input/delegated-apnic-latest"]
dft = defaultdict(list)
availableIPs = []
for f in ip_area_list:
seed_file = open(f,'r')
for l in seed_file.readlines():
params = l.split('|')
if len(params) >= 4 and params[2] == "ipv4" and params[3] != "*" and params[1] != "CN":
startIP = params[3]
endIP = ip_integer_to_string(ip_integer_from_string(startIP) + int(params[4]) - 1)
logger.info(startIP + ' ' + endIP + ' ' + params[4])
iprange = IPRange(startIP, endIP)
if params[1] == '':
availableIPs += map(str, iprange.cidrs())
else:
dft[params[1]] += map(str, iprange.cidrs())
for key in dft:
prefix = dft[key][-1]
network,masklen = prefix.split('/')
masklen = int(masklen)
ip = generate_random_ip(network,masklen)
ipset = IPSet(dft[key])
for prefix in ipset.iter_cidrs():
network,masklen = str(prefix).split('/')
masklen = int(masklen)
rtree.addPrefix(network,masklen)
data = rtree.rnode.data
country = country_code[key]
logger.info(str(prefix) + ' ' + country)
data['country'] = country #jsonData.get('country','')
data['ip'] = ip
data['ip_amount'] = prefix.size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
for prefix in availableIPs:
network,masklen = prefix.split("/")
masklen = int(masklen)
ip = generate_random_ip(network,masklen)
jsonData = None;
while jsonData == None:
try:
jsonData = query_ip(ip)
except Exception, e:
logger.error(e)
time.sleep(0.5)
rtree.addPrefix(network,masklen)
data = rtree.rnode.data
data['country'] = jsonData.get('country','')
data['ip'] = ip
data['ip_amount'] = IPNetwork(prefix).size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
logger.info(prefix + ' ' + data['country'])
from netaddr import IPSet, IPNetwork
from trace import IPFile
import json
IPSet.__invert__ = lambda s: IPSet(IPNetwork('0.0.0.0/0')) - s
i = IPFile()
i.fromfile('private/REDACTED.ips')
t = json(open('REDACTED', 'r').read())
t['exclude'].clear()
t['exclude'].append('192.168.1.0/24')
compl = IPSet(IPNetwork('0.0.0.0/0'))
for ip in i:
compl = compl & ~IPSet(IPNetwork(ip))
compl.compact()
t['exclude'].extend((str(i) for i in compl.iter_cidrs()))
with open('REDACTED2', 'w') as f:
f.write(json.dumps(t))
# compl should be the complement of all the ips now
unavailable = reserved | private
# Getting a list of public IPs
available = ipv4_addr_space ^ unavailable
# Here I am looking for an open 25 port (SMTP), you can always change this
port_to_check = 25
free_smtp = []
# I will be using this small subset for demo purposes instead of the superset of all valid IPs
test_ip_set = IPSet(['172.17.11.0/24'])
# The below line is commented because I am not going to scan the public IP addresses. But if you think it is legal in
# your state, you can use the "available" object to scan the public IPs. For now, I am only scanning a small subset
# for cidr in available.iter_cidrs():
for cidr in test_ip_set.iter_cidrs():
for each_ip in cidr[1:-1]:
try:
# Core logic of opening a socket for each ip
remote_server_ip = socket.gethostbyname(str(each_ip))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((remote_server_ip, port_to_check))
# If result is 0, then it means that the port is open
if result == 0:
print "IP address with open port 25: %s" % each_ip
free_smtp.append(each_ip)
else:
print "IP not open: %s" % each_ip
sock.close()
except Exception as err:
print "Exception seen - %s" % err
class IPv4():
def __init__(self, clear_cache = None):
self.reserved_ip = 'reserved_ip.txt'
self.cache_apnic = 'cache/apnic.txt'
self.outfile_inwall = 'output/inwall.txt'
self.outfile_outwall = 'output/outwall.txt'
self.clear_cache = clear_cache
self.ipset_reserved = None
self.ipset_inwall = None
self.ipset_outwall = None
self.cidrs_inwall = None
self.cidrs_outwall = None
self.populate_reserved()
def populate_reserved(self):
with open(self.reserved_ip, 'r') as f:
lines = f.readlines()
ip_list = []
for line in lines:
if not line.startswith('#'):
line = line.strip()
if len(line) > 0:
ip_list.append(line)
self.ipset_reserved = IPSet(ip_list)
def download_table(self):
if self.clear_cache:
os.remove(self.cache_apnic)
if (not os.path.exists(self.cache_apnic)) or os.path.getsize(self.cache_apnic) <= 0:
logging.info("Start downloading APNIC stat")
r = requests.get(APNIC_URL, stream=True)
if r.status_code == 200:
with open(self.cache_apnic, 'w') as f:
f.write(r.text)
logging.info("Finished downloading APNIC stat")
def parse_table(self):
logging.info("Start parsing IP table(s)")
with open(self.cache_apnic, 'r') as f:
lines = f.readlines()
ip_list = []
for line in lines:
if line.startswith('apnic|CN|ipv4'):
line = line.rstrip()
apnic, country, v4v6, prefix, count_of_addr, date, status = line.split('|')
if v4v6 == 'ipv4' and country == 'CN':
decimal = 32 - binary_log(int(count_of_addr))
cidr_addr = prefix + '/' + str(decimal)
ip_list.append(cidr_addr)
self.ipset_inwall = IPSet(ip_list)
self.cidrs_inwall = list(self.ipset_inwall.iter_cidrs())
logging.info("Finished parsing in-wall IP table(s). Total: %i CIDR blocks.", len(self.cidrs_inwall), )
def derive_outwall(self):
"""
This would not only inverse the set with the "big one", it would also exclude
See: http://www.tcpipguide.com/free/t_IPReservedPrivateandLoopbackAddresses-3.htm
"""
self.ipset_outwall = IPSet(['0.0.0.0/0']) ^ self.ipset_inwall ^ self.ipset_reserved
self.cidrs_outwall = list(self.ipset_outwall.iter_cidrs())
logging.info("Finished deriving out-wall IP table(s). Total: %i CIDR blocks.", len(self.cidrs_outwall), )
def write_outfiles(self):
logging.info("Writing output file: %s", self.outfile_inwall)
with open(self.outfile_inwall, 'w') as f:
for cidr_block in self.cidrs_inwall:
f.write(str(cidr_block) + '\n')
logging.info("Writing output file: %s", self.outfile_outwall)
with open(self.outfile_outwall, 'w') as f:
for cidr_block in self.cidrs_outwall:
f.write(str(cidr_block) + '\n')
def main_course(self):
self.download_table()
self.parse_table()
self.derive_outwall()
self.write_outfiles()
class IPv4():
def __init__(self, clear_cache = None):
self.reserved_ip = 'reserved_ip.txt'
self.cache_apnic = 'cache/apnic.txt'
self.outfile_inwall = 'output/inwall.txt'
self.outfile_outwall = 'output/outwall.txt'
self.clear_cache = clear_cache
self.ipset_reserved = None
self.ipset_inwall = None
self.ipset_outwall = None
self.cidrs_inwall = None
self.cidrs_outwall = None
self.populate_reserved()
def populate_reserved(self):
with open(self.reserved_ip, 'r') as f:
lines = f.readlines()
ip_list = []
for line in lines:
if not line.startswith('#'):
line = line.strip()
if len(line) > 0:
ip_list.append(line)
self.ipset_reserved = IPSet(ip_list)
def download_table(self):
if self.clear_cache:
os.remove(self.cache_apnic)
if (not os.path.exists(self.cache_apnic)) or os.path.getsize(self.cache_apnic) <= 0:
logging.info("Start downloading APNIC stat")
r = requests.get(APNIC_URL, stream=True)
if r.status_code == 200:
with open(self.cache_apnic, 'w') as f:
f.write(r.text)
logging.info("Finished downloading APNIC stat")
def parse_table(self):
logging.info("Start parsing IP table(s)")
with open(self.cache_apnic, 'r') as f:
lines = f.readlines()
ip_list = []
for line in lines:
if line.startswith('apnic|CN|ipv4'):
line = line.rstrip()
apnic, country, v4v6, prefix, count_of_addr, date, status = line.split('|')
if v4v6 == 'ipv4' and country == 'CN':
decimal = 32 - binary_log(int(count_of_addr))
cidr_addr = prefix + '/' + str(decimal)
ip_list.append(cidr_addr)
self.ipset_inwall = IPSet(ip_list)
self.cidrs_inwall = list(self.ipset_inwall.iter_cidrs())
logging.info("Finished parsing in-wall IP table(s). Total: %i CIDR blocks.", len(self.cidrs_inwall), )
def derive_outwall(self):
"""
This would not only inverse the set with the "big one", it would also exclude
See: http://www.tcpipguide.com/free/t_IPReservedPrivateandLoopbackAddresses-3.htm
"""
self.ipset_outwall = IPSet(['0.0.0.0/0']) - self.ipset_inwall - self.ipset_reserved
self.cidrs_outwall = list(self.ipset_outwall.iter_cidrs())
logging.info("Finished deriving out-wall IP table(s). Total: %i CIDR blocks.", len(self.cidrs_outwall), )
def write_outfiles(self):
logging.info("Writing output file: %s", self.outfile_inwall)
with open(self.outfile_inwall, 'w') as f:
for cidr_block in self.cidrs_inwall:
f.write(str(cidr_block) + '\n')
logging.info("Writing output file: %s", self.outfile_outwall)
with open(self.outfile_outwall, 'w') as f:
for cidr_block in self.cidrs_outwall:
f.write(str(cidr_block) + '\n')
def main_course(self):
self.download_table()
self.parse_table()
self.derive_outwall()
self.write_outfiles()
def check_ip(self, ip):
ip_addr = IPAddress(ip)
logging.info("The IP address to be checked is: %s", ip_addr)
# Populate both IPSets
if self.ipset_inwall is None or self.ipset_outwall is None:
self.main_course()
if ip_addr in self.ipset_inwall:
logging.info("The IP address %s is located in the Wall", ip)
if ip_addr in self.ipset_outwall:
logging.info("The IP address %s is located out the Wall", ip)
def test_ipset_cidr_fracturing():
s1 = IPSet(['0.0.0.0/0'])
s1.remove('255.255.255.255')
assert s1 == IPSet([
'0.0.0.0/1', '128.0.0.0/2', '192.0.0.0/3', '224.0.0.0/4',
'240.0.0.0/5', '248.0.0.0/6', '252.0.0.0/7', '254.0.0.0/8',
'255.0.0.0/9', '255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15', '255.254.0.0/16',
'255.255.0.0/17', '255.255.128.0/18', '255.255.192.0/19',
'255.255.224.0/20', '255.255.240.0/21', '255.255.248.0/22',
'255.255.252.0/23', '255.255.254.0/24', '255.255.255.0/25',
'255.255.255.128/26', '255.255.255.192/27', '255.255.255.224/28',
'255.255.255.240/29', '255.255.255.248/30', '255.255.255.252/31',
'255.255.255.254/32'
])
cidrs = s1.iter_cidrs()
assert len(cidrs) == 32
assert list(cidrs) == [
IPNetwork('0.0.0.0/1'),
IPNetwork('128.0.0.0/2'),
IPNetwork('192.0.0.0/3'),
IPNetwork('224.0.0.0/4'),
IPNetwork('240.0.0.0/5'),
IPNetwork('248.0.0.0/6'),
IPNetwork('252.0.0.0/7'),
IPNetwork('254.0.0.0/8'),
IPNetwork('255.0.0.0/9'),
IPNetwork('255.128.0.0/10'),
IPNetwork('255.192.0.0/11'),
IPNetwork('255.224.0.0/12'),
IPNetwork('255.240.0.0/13'),
IPNetwork('255.248.0.0/14'),
IPNetwork('255.252.0.0/15'),
IPNetwork('255.254.0.0/16'),
IPNetwork('255.255.0.0/17'),
IPNetwork('255.255.128.0/18'),
IPNetwork('255.255.192.0/19'),
IPNetwork('255.255.224.0/20'),
IPNetwork('255.255.240.0/21'),
IPNetwork('255.255.248.0/22'),
IPNetwork('255.255.252.0/23'),
IPNetwork('255.255.254.0/24'),
IPNetwork('255.255.255.0/25'),
IPNetwork('255.255.255.128/26'),
IPNetwork('255.255.255.192/27'),
IPNetwork('255.255.255.224/28'),
IPNetwork('255.255.255.240/29'),
IPNetwork('255.255.255.248/30'),
IPNetwork('255.255.255.252/31'),
IPNetwork('255.255.255.254/32')
]
assert cidrs == cidr_exclude('0.0.0.0/0', '255.255.255.255')
s1.remove('0.0.0.0')
assert s1 == IPSet([
'0.0.0.1/32',
'0.0.0.2/31',
'0.0.0.4/30',
'0.0.0.8/29',
'0.0.0.16/28',
'0.0.0.32/27',
'0.0.0.64/26',
'0.0.0.128/25',
'0.0.1.0/24',
'0.0.2.0/23',
'0.0.4.0/22',
'0.0.8.0/21',
'0.0.16.0/20',
'0.0.32.0/19',
'0.0.64.0/18',
'0.0.128.0/17',
'0.1.0.0/16',
'0.2.0.0/15',
'0.4.0.0/14',
'0.8.0.0/13',
'0.16.0.0/12',
'0.32.0.0/11',
'0.64.0.0/10',
'0.128.0.0/9',
'1.0.0.0/8',
'2.0.0.0/7',
'4.0.0.0/6',
'8.0.0.0/5',
'16.0.0.0/4',
'32.0.0.0/3',
'64.0.0.0/2',
'128.0.0.0/2',
'192.0.0.0/3',
'224.0.0.0/4',
'240.0.0.0/5',
'248.0.0.0/6',
'252.0.0.0/7',
'254.0.0.0/8',
'255.0.0.0/9',
'255.128.0.0/10',
'255.192.0.0/11',
'255.224.0.0/12',
'255.240.0.0/13',
'255.248.0.0/14',
'255.252.0.0/15',
'255.254.0.0/16',
'255.255.0.0/17',
'255.255.128.0/18',
'255.255.192.0/19',
'255.255.224.0/20',
'255.255.240.0/21',
'255.255.248.0/22',
'255.255.252.0/23',
'255.255.254.0/24',
'255.255.255.0/25',
'255.255.255.128/26',
'255.255.255.192/27',
'255.255.255.224/28',
'255.255.255.240/29',
'255.255.255.248/30',
'255.255.255.252/31',
'255.255.255.254/32',
])
assert len(list(s1.iter_cidrs())) == 62
s1.add('255.255.255.255')
s1.add('0.0.0.0')
assert s1 == IPSet(['0.0.0.0/0'])
def scan_fn_ip():
country_code = {}
for line in open('input/country_code', 'r'):
code, name = line.split(" ")
country_code[code] = name.strip().decode("utf-8")
logger.info(code + ' ' + country_code[code])
rtree = ipRadixDB()
ip_area_list = [
"input/delegated-arin-latest", "input/delegated-ripencc-latest",
"input/delegated-lacnic-latest", "input/delegated-afrinic-latest",
"input/delegated-apnic-latest"
]
dft = defaultdict(list)
availableIPs = []
for f in ip_area_list:
seed_file = open(f, 'r')
for l in seed_file.readlines():
params = l.split('|')
if len(params) >= 4 and params[
2] == "ipv4" and params[3] != "*" and params[1] != "CN":
startIP = params[3]
endIP = ip_integer_to_string(
ip_integer_from_string(startIP) + int(params[4]) - 1)
logger.info(startIP + ' ' + endIP + ' ' + params[4])
iprange = IPRange(startIP, endIP)
if params[1] == '':
availableIPs += map(str, iprange.cidrs())
else:
dft[params[1]] += map(str, iprange.cidrs())
for key in dft:
prefix = dft[key][-1]
network, masklen = prefix.split('/')
masklen = int(masklen)
ip = generate_random_ip(network, masklen)
ipset = IPSet(dft[key])
for prefix in ipset.iter_cidrs():
network, masklen = str(prefix).split('/')
masklen = int(masklen)
rtree.addPrefix(network, masklen)
data = rtree.rnode.data
country = country_code[key]
logger.info(str(prefix) + ' ' + country)
data['country'] = country #jsonData.get('country','')
data['ip'] = ip
data['ip_amount'] = prefix.size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
for prefix in availableIPs:
network, masklen = prefix.split("/")
masklen = int(masklen)
ip = generate_random_ip(network, masklen)
jsonData = None
while jsonData == None:
try:
jsonData = query_ip(ip)
except Exception, e:
logger.error(e)
time.sleep(0.5)
rtree.addPrefix(network, masklen)
data = rtree.rnode.data
data['country'] = jsonData.get('country', '')
data['ip'] = ip
data['ip_amount'] = IPNetwork(prefix).size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
logger.info(prefix + ' ' + data['country'])
ipv4_domains['domain'] = ipv4_domains['domain'].str.replace('.inaddr.arpa', '.in-addr.arpa')
ipv6_domains_Series = domainDB_df[domainDB_df['domain'].str.contains('ip6.arpa')]['domain']
ipv6_domains = pd.DataFrame()
ipv6_domains['domain'] = ipv6_domains_Series.copy()
ipv6_domains['prefix'] = ipv6_domains_Series.apply(convertIPv6RevDomainToNetwork)
ipv6_prefixes_list = ipv6_domains['prefix'].tolist()
domains_IPSet = domains_IPSet.union(IPSet(ipv6_prefixes_list))
domains_df = pd.concat([ipv4_domains, ipv6_domains])
prefixes_withoutDomains = IPSet(list(set((delegated_IPSet - domains_IPSet).iter_cidrs()).\
intersection(set(delegated_IPSet.iter_cidrs()))))
prefixes_withDomains = delegated_IPSet - prefixes_withoutDomains
s = Session()
# We compute statistics for those prefixes that have associated domains in
# the domain DB.
for index, alloc_row in delegated_df.iterrows():
prefix = alloc_row['prefix']
if len((IPSet([prefix]).intersection(prefixes_withDomains)).iter_cidrs()) > 0:
if alloc_row['ip_version'] == 'ipv4':
longestPref = 24
else:
longestPref = 64
#! /usr/bin/env python
# Calculates the disjunction of two sets of IP ranges
from sys import argv
from netaddr import IPSet
if len(argv) != 3:
print('Usage: {0} include.txt exclude.txt'.format(argv[0]))
exit()
net = IPSet()
with open(argv[1], 'r') as incfile:
for line in incfile:
net = net | IPSet([line])
with open(argv[2], 'r') as exfile:
for line in exfile:
net.remove(line)
for cidr in net.iter_cidrs():
print(cidr)
for row in aslines:
#we just load the prefixes variable into the radix tree
rt_prefixes.add(row[1])
#load the json exported from module 1
with open("module_out.json", "r") as f:
listas = json.load(f)
#we run the loop for EACH AS
for origin_as_data in listas:
denom = 0.0
coverage = 0.0
bgp_prefixes_comp = IPSet(origin_as_data["prefixes"])
for prefix in bgp_prefixes_comp.iter_cidrs():
prefix_str = str(prefix)
covering_prefixes = rt_prefixes.search_covering(prefix_str)
if len(covering_prefixes) > 0:
#if covering_prefixes > 0 then everything is covered and the fraction
#is 1/1 then we add all the ips both to the denom and the num
coverage += pow(2, 32 - prefix.prefixlen)
denom += pow(2, 32 - prefix.prefixlen)
continue
covered_prefixes = rt_prefixes.search_covered(prefix_str)
if len(covered_prefixes) == 0:
#if covered_prefixes == 0 then no ip is covered
#then the fraction to add is 0/1 and we only add to the denom
coverage += 0
denom += pow(2, 32 - prefix.prefixlen)
def test_ipset_cidr_fracturing():
s1 = IPSet(['0.0.0.0/0'])
s1.remove('255.255.255.255')
assert s1 == IPSet([
'0.0.0.0/1', '128.0.0.0/2', '192.0.0.0/3',
'224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6',
'252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9',
'255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15',
'255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18',
'255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21',
'255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24',
'255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27',
'255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30',
'255.255.255.252/31', '255.255.255.254/32'])
cidrs = s1.iter_cidrs()
assert len(cidrs) == 32
assert list(cidrs) == [
IPNetwork('0.0.0.0/1'), IPNetwork('128.0.0.0/2'), IPNetwork('192.0.0.0/3'),
IPNetwork('224.0.0.0/4'), IPNetwork('240.0.0.0/5'), IPNetwork('248.0.0.0/6'),
IPNetwork('252.0.0.0/7'), IPNetwork('254.0.0.0/8'), IPNetwork('255.0.0.0/9'),
IPNetwork('255.128.0.0/10'), IPNetwork('255.192.0.0/11'), IPNetwork('255.224.0.0/12'),
IPNetwork('255.240.0.0/13'), IPNetwork('255.248.0.0/14'), IPNetwork('255.252.0.0/15'),
IPNetwork('255.254.0.0/16'), IPNetwork('255.255.0.0/17'), IPNetwork('255.255.128.0/18'),
IPNetwork('255.255.192.0/19'), IPNetwork('255.255.224.0/20'), IPNetwork('255.255.240.0/21'),
IPNetwork('255.255.248.0/22'), IPNetwork('255.255.252.0/23'), IPNetwork('255.255.254.0/24'),
IPNetwork('255.255.255.0/25'), IPNetwork('255.255.255.128/26'), IPNetwork('255.255.255.192/27'),
IPNetwork('255.255.255.224/28'), IPNetwork('255.255.255.240/29'), IPNetwork('255.255.255.248/30'),
IPNetwork('255.255.255.252/31'), IPNetwork('255.255.255.254/32')
]
assert cidrs == cidr_exclude('0.0.0.0/0', '255.255.255.255')
s1.remove('0.0.0.0')
assert s1 == IPSet([
'0.0.0.1/32', '0.0.0.2/31', '0.0.0.4/30',
'0.0.0.8/29', '0.0.0.16/28', '0.0.0.32/27',
'0.0.0.64/26', '0.0.0.128/25', '0.0.1.0/24',
'0.0.2.0/23', '0.0.4.0/22', '0.0.8.0/21',
'0.0.16.0/20', '0.0.32.0/19', '0.0.64.0/18',
'0.0.128.0/17', '0.1.0.0/16', '0.2.0.0/15',
'0.4.0.0/14', '0.8.0.0/13', '0.16.0.0/12',
'0.32.0.0/11', '0.64.0.0/10', '0.128.0.0/9',
'1.0.0.0/8', '2.0.0.0/7', '4.0.0.0/6',
'8.0.0.0/5', '16.0.0.0/4', '32.0.0.0/3',
'64.0.0.0/2', '128.0.0.0/2', '192.0.0.0/3',
'224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6',
'252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9',
'255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15',
'255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18',
'255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21',
'255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24',
'255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27',
'255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30',
'255.255.255.252/31', '255.255.255.254/32',
])
assert len(list(s1.iter_cidrs())) == 62
s1.add('255.255.255.255')
s1.add('0.0.0.0')
assert s1 == IPSet(['0.0.0.0/0'])
import ipaddress
from netaddr import IPSet
routes_file = open('routes.txt', 'r')
route_list = []
ipnet_set = IPSet()
for route_line in routes_file.readlines():
route_entry = route_line.split()
ipnet_entry = route_entry[0] + '/' + route_entry[1]
IPNet = ipaddress.ip_network(ipnet_entry)
route_list = route_list + [str(IPNet)]
ipnet_set = ipnet_set | IPSet([str(IPNet)])
routes_file.close()
ipnetwk = []
for cidr in ipnet_set.iter_cidrs():
print(cidr.network, cidr.netmask)
ipnetwk = ipnetwk + [str(cidr)]
print()
print(route_list)
print()
print(ipnetwk)
print()
print('length route list:', len(route_list), 'length reduced list:',
len(ipnetwk))
def sync_subnets(conn, config):
log.debug("loading routing tables")
routing_tables = conn.get_all_route_tables()
route_tables_by_name = {r.tags.get('Name'): r for r in routing_tables}
route_tables_by_subnet_id = {}
for r in routing_tables:
for a in r.associations:
route_tables_by_subnet_id[a.subnet_id] = r
# Get list of AZs
zones = conn.get_all_zones()
for vpc_id in config:
# Get a list of all the remote subnets
remote_subnets = conn.get_all_subnets(filters={'vpcId': vpc_id})
seen = set()
# Go through our config, adjusting or any subnets as appropriate
for cidr, block_config in config[vpc_id].items():
cidr_net = IPNetwork(cidr)
table_name = block_config.get('routing_table')
if table_name and table_name not in route_tables_by_name:
log.warn("couldn't find routing table %s for block %s", table_name, cidr)
log.warn("skipping rest of %s", cidr)
continue
my_rt = route_tables_by_name[table_name]
ip_set = IPSet(cidr_net)
for s in remote_subnets:
if IPNetwork(s.cidr_block) in cidr_net:
ip_set.remove(s.cidr_block)
if s.tags.get('Name') != block_config['name']:
log.info("Setting Name of %s to %s", s, block_config['name'])
s.add_tag('Name', block_config['name'])
if s.id in route_tables_by_subnet_id:
remote_rt = route_tables_by_subnet_id[s.id]
else:
remote_rt = route_tables_by_subnet_id[None]
if remote_rt != my_rt:
log.info(
"Changing routing table for %s (%s) to %s (%s)",
s, s.tags.get('Name'), my_rt,
my_rt.tags.get('Name'))
if raw_input("(y/N) ") == "y":
conn.associate_route_table(my_rt.id, s.id)
seen.add(s)
# Are we missing any subnets?
# If so, create them!
# TODO: We want to evenly distribute the ip range over the
# configured availability zones, without dividing smaller than a
# /25 network (128 ips, at least 2 of which are reserved)
# For now we'll just split them as small as /24, and then assign
# them into the subnets
while ip_set:
log.info("%s - %s isn't covered by any subnets", cidr, ip_set)
my_zones = [z for z in zones if z.name not in block_config.get('skip_azs', [])]
remaining_cidrs = list(ip_set.iter_cidrs())
remaining_cidrs.sort(key=lambda s: s.size, reverse=True)
for s in remaining_cidrs[:]:
if s.prefixlen < 24:
added = list(s.subnet(24))
remaining_cidrs.remove(s)
remaining_cidrs.extend(added)
ip_set.remove(s)
zg = itertools.cycle(my_zones)
while remaining_cidrs:
c = remaining_cidrs.pop()
z = next(zg)
log.info("creating subnet %s in %s/%s", c, z.name, vpc_id)
if raw_input("(y/N) ") == "y":
log.debug("creating subnet")
s = conn.create_subnet(vpc_id, c, z.name)
log.debug("adding tag")
# TODO: sometimes the subnet isn't actually created by
# the time we try and add the tag, so get a 400 error
s.add_tag('Name', block_config['name'])
log.debug("associating routing")
conn.associate_route_table(my_rt.id, s.id)
local_missing = set(remote_subnets) - seen
for m in local_missing:
log.info("%s:%s (name: %s) is unmanaged", m.id, m.cidr_block, m.tags.get('Name'))
#!/usr/bin/python
from sys import argv
from netaddr import IPSet
if len(argv) != 3:
print('Usage: netdiff.py3 include.txt exclude.txt')
exit()
net = IPSet()
incfile = open(argv[1])
for line in incfile:
net = net | IPSet([line])
exfile = open(argv[2])
for line in exfile:
net.remove(line)
for cidr in net.iter_cidrs():
print(cidr)
def sync_subnets(conn, config):
log.debug("loading routing tables")
routing_tables = conn.get_all_route_tables()
route_tables_by_name = {r.tags.get('Name'): r for r in routing_tables}
route_tables_by_subnet_id = {}
for r in routing_tables:
for a in r.associations:
route_tables_by_subnet_id[a.subnet_id] = r
# Get list of AZs
zones = conn.get_all_zones()
for vpc_id in config:
# Get a list of all the remote subnets
remote_subnets = conn.get_all_subnets(filters={'vpcId': vpc_id})
seen = set()
# Go through our config, adjusting or any subnets as appropriate
for cidr, block_config in config[vpc_id].items():
cidr_net = IPNetwork(cidr)
table_name = block_config.get('routing_table')
if table_name and table_name not in route_tables_by_name:
log.warn("couldn't find routing table %s for block %s",
table_name, cidr)
log.warn("skipping rest of %s", cidr)
continue
my_rt = route_tables_by_name[table_name]
ip_set = IPSet(cidr_net)
for s in remote_subnets:
if IPNetwork(s.cidr_block) in cidr_net:
ip_set.remove(s.cidr_block)
if s.tags.get('Name') != block_config['name']:
log.info("Setting Name of %s to %s", s,
block_config['name'])
s.add_tag('Name', block_config['name'])
if s.id in route_tables_by_subnet_id:
remote_rt = route_tables_by_subnet_id[s.id]
else:
remote_rt = route_tables_by_subnet_id[None]
if remote_rt != my_rt:
log.info(
"Changing routing table for %s (%s) to %s (%s)",
s, s.tags.get('Name'), my_rt,
my_rt.tags.get('Name'))
if raw_input("(y/N) ") == "y":
conn.associate_route_table(my_rt.id, s.id)
seen.add(s)
# Are we missing any subnets?
# If so, create them!
# TODO: We want to evenly distribute the ip range over the
# configured availability zones, without dividing smaller than a
# /25 network (128 ips, at least 2 of which are reserved)
# For now we'll just split them as small as /24, and then assign
# them into the subnets
while ip_set:
log.info("%s - %s isn't covered by any subnets", cidr, ip_set)
my_zones = [
z for z in zones
if z.name not in block_config.get('skip_azs', [])
]
remaining_cidrs = list(ip_set.iter_cidrs())
remaining_cidrs.sort(key=lambda s: s.size, reverse=True)
for s in remaining_cidrs[:]:
if s.prefixlen < 24:
added = list(s.subnet(24))
remaining_cidrs.remove(s)
remaining_cidrs.extend(added)
ip_set.remove(s)
zg = itertools.cycle(my_zones)
while remaining_cidrs:
c = remaining_cidrs.pop()
z = next(zg)
log.info("creating subnet %s in %s/%s", c, z.name, vpc_id)
if raw_input("(y/N) ") == "y":
log.debug("creating subnet")
s = conn.create_subnet(vpc_id, c, z.name)
log.debug("adding tag")
# TODO: sometimes the subnet isn't actually created by
# the time we try and add the tag, so get a 400 error
s.add_tag('Name', block_config['name'])
log.debug("associating routing")
conn.associate_route_table(my_rt.id, s.id)
local_missing = set(remote_subnets) - seen
for m in local_missing:
log.info("%s:%s (name: %s) is unmanaged", m.id, m.cidr_block,
m.tags.get('Name'))
