def test_spanning_cidr_handles_strings():
# This that a regression introduced in 0fda41a is fixed. The regression caused an error when str
# addresses were passed to the function.
addresses = [
IPAddress('10.0.0.1'),
IPAddress('10.0.0.2'),
'10.0.0.3',
'10.0.0.4',
]
assert spanning_cidr(addresses) == IPNetwork('10.0.0.0/29')
assert spanning_cidr(reversed(addresses)) == IPNetwork('10.0.0.0/29')
def test_rules_ip_net(self):
prof_n1, prof_n2 = self._get_profiles(self.new_profiles)
n1_ips = [workload.ip for workload in self.n1_workloads]
n2_ips = [workload.ip for workload in self.n2_workloads]
n1_subnet = netaddr.spanning_cidr(n1_ips)
n2_subnet = netaddr.spanning_cidr(n2_ips)
rule = {'action': 'Allow', 'source': {'nets': [str(n1_subnet)]}}
prof_n2['spec']['ingress'].append(rule)
rule = {'action': 'Allow', 'source': {'nets': [str(n2_subnet)]}}
prof_n1['spec']['ingress'].append(rule)
self._apply_new_profile(self.new_profiles, self.host1)
self.assert_connectivity(retries=2,
pass_list=self.n1_workloads +
self.n2_workloads)
def cidr_merge(value, action="merge"):
if not hasattr(value, "__iter__"):
raise errors.AnsibleFilterError("cidr_merge: expected iterable, got " +
repr(value))
if action == "merge":
try:
return [str(ip) for ip in netaddr.cidr_merge(value)]
except Exception as e:
raise errors.AnsibleFilterError(
"cidr_merge: error in netaddr:\n%s" % e)
elif action == "span":
# spanning_cidr needs at least two values
if len(value) == 0:
return None
elif len(value) == 1:
try:
return str(netaddr.IPNetwork(value[0]))
except Exception as e:
raise errors.AnsibleFilterError(
"cidr_merge: error in netaddr:\n%s" % e)
else:
try:
return str(netaddr.spanning_cidr(value))
except Exception as e:
raise errors.AnsibleFilterError(
"cidr_merge: error in netaddr:\n%s" % e)
else:
raise errors.AnsibleFilterError("cidr_merge: invalid action '%s'" %
action)
def cidr_merge(value, action='merge'):
if not hasattr(value, '__iter__'):
raise Exception('cidr_merge: expected iterable, got ' + repr(value))
if action == 'merge':
try:
return [str(ip) for ip in netaddr.cidr_merge(value)]
except Exception as e:
raise Exception('cidr_merge: error in netaddr:\n%s' % e)
elif action == 'span':
# spanning_cidr needs at least two values
if len(value) == 0:
return None
elif len(value) == 1:
try:
return str(netaddr.IPNetwork(value[0]))
except Exception as e:
raise Exception('cidr_merge: error in netaddr:\n%s' % e)
else:
try:
return str(netaddr.spanning_cidr(value))
except Exception as e:
raise Exception('cidr_merge: error in netaddr:\n%s' % e)
else:
raise Exception("cidr_merge: invalid action '%s'" % action)
def iprange(ip_str_list):
ip_str_list = list(set(ip_str_list))
ip_str_list = strip_list(ip_str_list)
#直接获取各个C段
subnet = set()
for ip_str in ip_str_list:
#print ip_str
ip = IPNetwork(ip_str)
ip.prefixlen = 24
subnet.add(ip.cidr)
#print subnet
#尝试根据IP缩小范围
smaller_subnet = set()
for net in subnet:
tmpIPlist = []
for ip_str in ip_str_list:
ip = IPNetwork(ip_str)
ip.prefixlen = 24
if ip.cidr == net: #ip属于net
tmpIPlist.append(ip_str)
#print tmpIPlist
if len(tmpIPlist) == 1:
smaller_subnet.add(tmpIPlist[0])
elif len(tmpIPlist) >= 2:
smaller = netaddr.spanning_cidr(tmpIPlist) #type is IPNetwork
#if smaller != net:
smaller_subnet.add(smaller)
elif len(tmpIPlist) == 0:
print "{0} has no ip".format(net)
result = []
for item in smaller_subnet:
result.append(str(item))
return result
def test_rules_ip_net(self):
prof_n1, prof_n2 = self._get_profiles(self.new_profiles)
n1_ips = [workload.ip for workload in self.n1_workloads]
n2_ips = [workload.ip for workload in self.n2_workloads]
n1_subnet = netaddr.spanning_cidr(n1_ips)
n2_subnet = netaddr.spanning_cidr(n2_ips)
rule = {'action': 'allow',
'source':
{'net': str(n1_subnet)}}
prof_n2['spec']['ingress'].append(rule)
rule = {'action': 'allow',
'source':
{'net': str(n2_subnet)}}
prof_n1['spec']['ingress'].append(rule)
self._apply_new_profile(self.new_profiles, self.host1)
self.assert_connectivity(retries=2,
pass_list=self.n1_workloads + self.n2_workloads)
def output_both(cidr, validate=False):
if validate:
func = validate_cidr
else:
func = output_str
ret = func(cidr)
if len(cidr) > 1:
ret += "\n" + func([spanning_cidr(cidr)])
return ret
def output_both(cidr, validate=False):
if validate:
func = validate_cidr
else:
func = output_str
ret = func(cidr)
if len(cidr) > 1:
ret += '\n' + func([spanning_cidr(cidr)])
return ret
def get_subnet_aggr(subnet_list):
subnet_group = []
subnet_aggr = []
for key, group in groupby(
subnet_list, lambda x: x.split('.')[0] + '.' + x.split('.')[1]):
subnet_group.append(list(group))
for subnets in subnet_group:
if len(subnets) > 1:
cidr = str(netaddr.spanning_cidr(subnets))
subnet_aggr.append(cidr)
else:
subnet_aggr.append(subnets[0])
return subnet_aggr
def get_details_for_ip_range(ip_range):
"""For a given IPRange, return all subnets, a useable prefix and the
reverse DNS suffix calculated from that IP range.
:return: A tuple of:
All subnets of /24 (or smaller if there is no /24 subnet to be
found) in `ip_range`.
A prefix made from the first two octets in the range.
A RDNS suffix calculated from the first two octets in the range.
"""
# Calculate a spanning network for the range above. There are
# 256 /24 networks in a /16, so that's the most /24s we're going
# to have to deal with; this matters later on when we iterate
# through the /24s within this network.
if ip_range.size == 1:
cidr = IPNetwork(IPAddress(ip_range.first))
else:
cidr = spanning_cidr(ip_range)
subnets = cidr.subnet(max(24, cidr.prefixlen))
# Split the spanning network into /24 subnets, then see if they fall
# entirely within the original network range, partially, or not at
# all.
intersecting_subnets = []
for subnet in subnets:
intersect = intersect_iprange(subnet, ip_range)
if intersect is None:
# The subnet does not fall within the original network.
pass
else:
# The subnet falls partially within the original network, so print
# out a $GENERATE expression for a subset of the /24.
intersecting_subnets.append(intersect)
octet_one = (cidr.value & 0xFF000000) >> 24
octet_two = (cidr.value & 0x00FF0000) >> 16
# The first two octets of the network range formatted in the
# usual dotted-quad style. We can precalculate the start of any IP
# address in the range because we're only ever dealing with /16
# networks and smaller.
prefix = "%d.%d" % (octet_one, octet_two)
# Similarly, we can calculate what the reverse DNS suffix is going
# to look like.
rdns_suffix = "%d.%d.in-addr.arpa." % (octet_two, octet_one)
return intersecting_subnets, prefix, rdns_suffix
def dump(cidr):
"""
Dumps a lot of info about a CIDR.
"""
# Copy original cidr for usage later
orig_cidr = cidr[:]
# Flatten it for ops
if len(cidr) == 1:
cidr = cidr[0]
else:
cidr = spanning_cidr(cidr)
# Is this a /32 or /128?
single = (cidr.size == 1)
log.debug('dump(): Single? %r' % single)
log.debug('dump(): Got? %r' % cidr)
ip_first = IPAddress(cidr.first)
ip_firsthost = ip_first if single else cidr.iter_hosts().next()
ip_gateway = IPAddress(cidr.last - 1)
ip_bcast = cidr.broadcast
ip_netmask = cidr.netmask
ip_hostmask = cidr.hostmask
num_hosts = 1 if single else (cidr.last - 1) - cidr.first
out = ''
out += "Information for %s\n\n" % cidr
out += "IP Version:\t\t%s\n" % cidr.version
out += "Spanning CIDR:\t\t%s\n" % cidr
out += "Block Start/Network:\t%s\n" % ip_first
out += "1st host:\t\t%s\n" % ip_firsthost
out += "Gateway:\t\t%s\n" % ip_gateway
out += "Block End/Broadcast:\t%s\n" % ip_bcast
out += "DQ Mask:\t\t%s\n" % ip_netmask
out += "Cisco ACL Mask:\t\t%s\n" % ip_hostmask
out += "# of hosts:\t\t%s\n" % num_hosts
out += "Explicit CIDR blocks:\t%s\n" % output_str(orig_cidr)
return out
def dump(cidr):
"""
Dumps a lot of info about a CIDR.
"""
# Copy original cidr for usage later
orig_cidr = cidr[:]
# Flatten it for ops
if len(cidr) == 1:
cidr = cidr[0]
else:
cidr = spanning_cidr(cidr)
# Is this a /32 or /128?
single = (cidr.size == 1)
log.debug('dump(): Single? %r' % single)
log.debug('dump(): Got? %r' % cidr)
ip_first = IPAddress(cidr.first)
ip_firsthost = ip_first if single else cidr.iter_hosts().next()
ip_gateway = IPAddress(cidr.last - 1)
ip_bcast = cidr.broadcast
ip_netmask = cidr.netmask
ip_hostmask = cidr.hostmask
num_hosts = 1 if single else (cidr.last - 1) - cidr.first
out = ''
out += "Information for %s\n\n" % cidr
out += "IP Version:\t\t%s\n" % cidr.version
out += "Spanning CIDR:\t\t%s\n" % cidr
out += "Block Start/Network:\t%s\n" % ip_first
out += "1st host:\t\t%s\n" % ip_firsthost
out += "Gateway:\t\t%s\n" % ip_gateway
out += "Block End/Broadcast:\t%s\n" % ip_bcast
out += "DQ Mask:\t\t%s\n" % ip_netmask
out += "Cisco ACL Mask:\t\t%s\n" % ip_hostmask
out += "# of hosts:\t\t%s\n" % num_hosts
out += "Explicit CIDR blocks:\t%s\n" % output_str(orig_cidr)
return out
def get_nets_common_cird(radl):
"""
Get a common CIDR in all the RADL nets
"""
nets = []
for num, net in enumerate(radl.networks):
provider_id = net.getValue('provider_id')
if net.getValue('create') == 'yes' and not net.isPublic(
) and not provider_id:
net_cidr = net.getValue('cidr')
if not net_cidr:
net_cidr = CloudConnector.DEFAULT_NET_CIDR
net_cidr_0 = IPNetwork(net_cidr.replace("*", "0"))
if net_cidr_0 not in nets:
nets.append(net_cidr_0)
net_cidr = IPNetwork(net_cidr.replace("*", str(num + 1)))
nets.append(net_cidr)
if len(nets) == 0: # there is no CIDR return the default one
return "10.0.0.0/16"
elif len(nets) == 1: # there is only one, return it
return nets[0]
else: # there are more, get the common CIDR
return str(spanning_cidr(nets))
def consolidate_networks(networks: set, min_prefixlen: int):
'''
>>> from pprint import pprint
>>> test = [IPNetwork('10.47.0.0/17'),
... IPNetwork('10.47.128.0/19'),
... IPNetwork('10.47.168.0/21'),
... IPNetwork('10.47.192.0/18'),
... IPNetwork('10.48.0.0/16'),
... IPNetwork('10.28.0.0/17'),
... IPNetwork('10.28.128.0/19'),
... IPNetwork('172.16.18.205/32'),
... IPNetwork('172.16.48.0/20'),
... IPNetwork('172.16.64.0/18'),
... IPNetwork('172.16.128.0/17'),
... IPNetwork('172.20.0.0/19'),
... IPNetwork('172.20.57.229/32'),
... IPNetwork('172.20.96.0/19'),
... IPNetwork('172.20.128.0/17'),
... IPNetwork('172.21.77.0/16'),
... IPNetwork('173.124.7.8/32'),
... IPNetwork('173.124.34.0/18'),
... IPNetwork('173.93.160.0/19'),
... IPNetwork('173.93.19.241/32'),
... IPNetwork('173.93.25.95/32'),
... IPNetwork('173.154.0.0/17'),
... IPNetwork('173.154.128.0/18'),
... IPNetwork('173.154.192.0/20'),
... IPNetwork('193.99.144.85/32'),
... IPNetwork('54.239.32.138/32'),
... IPNetwork('91.240.34.5/32'),
... IPNetwork('85.183.69.83/32'),
... IPNetwork('95.100.66.202/32')]
>>> pprint(consolidate_networks(test, 16))
[IPNetwork('10.28.0.0/16'),
IPNetwork('10.47.0.0/16'),
IPNetwork('10.48.0.0/16'),
IPNetwork('54.239.32.138/32'),
IPNetwork('85.183.69.83/32'),
IPNetwork('91.240.34.5/32'),
IPNetwork('95.100.66.202/32'),
IPNetwork('172.16.0.0/16'),
IPNetwork('172.20.0.0/15'),
IPNetwork('173.93.16.0/20'),
IPNetwork('173.93.160.0/19'),
IPNetwork('173.124.0.0/18'),
IPNetwork('173.154.0.0/16'),
IPNetwork('193.99.144.85/32')]
'''
networks = sorted([IPNetwork(net) for net in networks])
new_networks = []
for chunk in chunks(networks, 2):
if len(chunk) > 1:
spanning = netaddr.spanning_cidr(chunk)
if spanning.prefixlen >= min_prefixlen:
new_networks.append(spanning)
else:
new_networks.extend(chunk)
else:
new_networks.append(chunk[0])
merged = netaddr.cidr_merge(new_networks)
return merged
def test_multi_host(self, test_type):
"""
Run a mainline multi-host test.
Because multihost tests are slow to setup, this tests most mainline
functionality in a single test.
- Create two hosts
- Create a network using the default IPAM driver, and a workload on
each host assigned to that network.
- Create a network using the Calico IPAM driver, and a workload on
each host assigned to that network.
- Check that hosts on the same network can ping each other.
- Check that hosts on different networks cannot ping each other.
- Modify the profile rules
- Check that connectivity has changed to match the profile we set up
- Re-apply the original profile
- Check that connectivity goes back to what it was originally.
"""
with DockerHost("host1",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False) as host1, \
DockerHost("host2",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False) as host2:
(n1_workloads, n2_workloads, networks) = \
self._setup_workloads(host1, host2)
# Get the original profiles:
output = host1.calicoctl("get profile -o yaml")
original_profiles = yaml.safe_load(output)
# Make a copy of the profiles to mess about with.
new_profiles = copy.deepcopy(original_profiles)
if test_type == "tags":
profile0_tag = new_profiles[0]['metadata']['tags'][0]
profile1_tag = new_profiles[1]['metadata']['tags'][0]
# Make a new profiles dict where the two networks have each
# other in their tags list
new_profiles[0]['metadata']['tags'].append(profile1_tag)
new_profiles[1]['metadata']['tags'].append(profile0_tag)
self._apply_new_profile(new_profiles, host1)
# Check everything can contact everything else now
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.tags":
profile0_tag = new_profiles[0]['metadata']['tags'][0]
profile1_tag = new_profiles[1]['metadata']['tags'][0]
rule0 = {'action': 'allow',
'source':
{'tag': profile1_tag}}
rule1 = {'action': 'allow',
'source':
{'tag': profile0_tag}}
new_profiles[0]['spec']['ingress'].append(rule0)
new_profiles[1]['spec']['ingress'].append(rule1)
self._apply_new_profile(new_profiles, host1)
# Check everything can contact everything else now
self.assert_connectivity(retries=3,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.protocol.icmp":
rule = {'action': 'allow',
'source':
{'protocol': 'icmp'}}
# The copy.deepcopy(rule) is needed to ensure that we don't
# end up with a yaml document with a reference to the same
# rule. While this is probably legal, it isn't main line.
new_profiles[0]['spec']['ingress'].append(rule)
new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule))
self._apply_new_profile(new_profiles, host1)
# Check everything can contact everything else now
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.ip.addr":
prof_n1, prof_n2 = self._get_profiles(new_profiles)
for workload in n1_workloads:
ip = workload.ip
rule = {'action': 'allow',
'source':
{'net': '%s/32' % ip}}
prof_n2['spec']['ingress'].append(rule)
for workload in n2_workloads:
ip = workload.ip
rule = {'action': 'allow',
'source':
{'net': '%s/32' % ip}}
prof_n1['spec']['ingress'].append(rule)
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.ip.net":
prof_n1, prof_n2 = self._get_profiles(new_profiles)
n1_ips = [workload.ip for workload in n1_workloads]
n2_ips = [workload.ip for workload in n2_workloads]
n1_subnet = netaddr.spanning_cidr(n1_ips)
n2_subnet = netaddr.spanning_cidr(n2_ips)
rule = {'action': 'allow',
'source':
{'net': str(n1_subnet)}}
prof_n2['spec']['ingress'].append(rule)
rule = {'action': 'allow',
'source':
{'net': str(n2_subnet)}}
prof_n1['spec']['ingress'].append(rule)
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.selector":
new_profiles[0]['metadata']['labels'] = {'net': 'n1'}
new_profiles[1]['metadata']['labels'] = {'net': 'n2'}
rule = {'action': 'allow',
'source':
{'selector': 'net=="n2"'}}
new_profiles[0]['spec']['ingress'].append(rule)
rule = {'action': 'allow',
'source':
{'selector': "net=='n1'"}}
new_profiles[1]['spec']['ingress'].append(rule)
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.tcp.port":
rule = {'action': 'allow',
'protocol': 'tcp',
'destination':
{'ports': [80]}}
# The copy.deepcopy(rule) is needed to ensure that we don't
# end up with a yaml document with a reference to the same
# rule. While this is probably legal, it isn't main line.
new_profiles[0]['spec']['ingress'].append(rule)
new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule))
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads,
type_list=['tcp'])
self.assert_connectivity(retries=2,
pass_list=n1_workloads,
fail_list=n2_workloads,
type_list=['icmp', 'udp'])
elif test_type == "rules.udp.port":
rule = {'action': 'allow',
'protocol': 'udp',
'destination':
{'ports': [69]}}
# The copy.deepcopy(rule) is needed to ensure that we don't
# end up with a yaml document with a reference to the same
# rule. While this is probably legal, it isn't main line.
new_profiles[0]['spec']['ingress'].append(rule)
new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule))
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads,
type_list=['udp'])
self.assert_connectivity(retries=2,
pass_list=n1_workloads,
fail_list=n2_workloads,
type_list=['icmp', 'tcp'])
else:
print "******************* " \
"ERROR - Unrecognised test type " \
"*******************"
assert False, "Unrecognised test type: %s" % test_type
# Now restore the original profile and check it all works as before
self._apply_new_profile(original_profiles, host1)
host1.calicoctl("get profile -o yaml")
self._check_original_connectivity(n1_workloads, n2_workloads)
# Tidy up
host1.remove_workloads()
host2.remove_workloads()
for network in networks:
network.delete()
def smaller_network(ip_str_list):
x = netaddr.spanning_cidr(
['192.168.0.0', '192.168.2.245', '192.168.2.255'])
print x
def cidrize(ipstr, strict=False, modular=True):
"""
This function tries to determine the best way to parse IP addresses correctly & has
all the logic for trying to do the right thing!
Returns a list of consolidated netaddr objects.
Input can be several formats::
'192.0.2.18'
'192.0.2.64/26'
'192.0.2.80-192.0.2.85'
'192.0.2.170-175'
'192.0.2.8[0-5]'
'192.0.2.[0-29]'
'192.168.4.6[1234]'
'1.2.3.*'
'192.0.2.170-175, 192.0.2.80-192.0.2.85, 192.0.2.64/26'
Hyphenated ranges do not need to form a CIDR block. Netaddr does most of
the heavy lifting for us here.
Input can NOT be::
192.0.2.0 0.0.0.255 (hostmask)
192.0.2.0 255.255.255.0 (netmask)
Does NOT accept network or host mask notation at this time!
Defaults:
* parsing exceptions will raise a CidrizeError (modular=True).
* results will be returned as a spanning CIDR (strict=False).
:param ipstr:
IP string to be parsed.
:param modular:
Set to False to cause exceptions to be stripped & the error text will be
returned as a list. This is intended for use with scripts or APIs
out-of-the box.
Example:
>>> import cidrize as c
>>> c.cidrize('1.2.3.4-1.2.3.1099')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/j/jathan/sandbox/cidrize.py", line 153, in cidrize
raise CidrizeError(err)
cidrize.CidrizeError: base address '1.2.3.1099' is not IPv4
>>> c.cidrize('1.2.3.4-1.2.3.1099', modular=False)
["base address '1.2.3.1099' is not IPv4"]
:param strict:
Set to True to return explicit networks based on start/end addresses.
Example:
>>> import cidrize as c
>>> c.cidrize('1.2.3.4-1.2.3.10')
[IPNetwork('1.2.3.0/28')]
>>> c.cidrize('1.2.3.4-1.2.3.10', strict=True)
[IPNetwork('1.2.3.4/30'), IPNetwork('1.2.3.8/31'), IPNetwork('1.2.3.10/32')]
"""
ip = None
# Short-circuit to parse commas since it calls back here anyway
if ',' in ipstr:
return parse_commas(ipstr, strict=strict, modular=modular)
# Short-circuit for hostnames (we're assuming first char is alpha)
if hostname_re.match(ipstr):
raise CidrizeError('Cannot parse hostnames!')
# Otherwise try everything else
result = None
try:
# Parse "everything" & immediately return; strict/loose doesn't apply
if ipstr in EVERYTHING:
log.debug("Trying everything style...")
return [IPNetwork('0.0.0.0/0')]
# Parse old-fashioned CIDR notation & immediately return; strict/loose doesn't apply
# Now with IPv6!
elif cidr_re.match(ipstr) or is_ipv6(ipstr):
log.debug("Trying CIDR style...")
ip = IPNetwork(ipstr)
return [ip.cidr]
# Parse 1.2.3.118-1.2.3.121 range style
elif range_re.match(ipstr):
log.debug("Trying range style...")
result = parse_range(ipstr)
# Parse 1.2.3.4-70 hyphen style
elif hyphen_re.match(ipstr):
log.debug("Trying hyphen style...")
result = parse_hyphen(ipstr)
# Parse 1.2.3.* glob style
elif glob_re.match(ipstr):
log.debug("Trying glob style...")
ipglob = IPGlob(ipstr)
result = spanning_cidr(ipglob)
# Parse 1.2.3.4[5-9] or 1.2.3.[49] bracket style as a last resort
elif bracket_re.match(ipstr):
log.debug("Trying bracket style...")
result = parse_brackets(ipstr)
# If result still isn't set, let's see if it's IPv6??
elif result is None:
log.debug("Trying bare IPv6 parse...")
result = IPNetwork(ipstr)
# This will probably fail 100% of the time. By design.
else:
raise CidrizeError("Could not determine parse style for '%s'" %
ipstr)
# If it's a single host, just return it wrapped in a list
if result.size == 1:
log.debug("Returning a single host!")
return [result.cidr]
# Logic to honor strict/loose, except IPRange. Doing a spanning_cidr on
# an IPRange can be super slow if the range is large (such as a /8), so
# IPRange objects larger than MAX_RANGE_LEN will always be strict.
if not strict:
if isinstance(result, IPRange) and result.size >= MAX_RANGE_LEN:
log.debug(
'IPRange objects larger than /18 will always be strict.')
return result.cidrs()
return [spanning_cidr(result)]
else:
try:
return result.cidrs() # IPGlob and IPRange have .cidrs()
except AttributeError as err:
return result.cidr # IPNetwork has .cidr
except (AddrFormatError, TypeError, ValueError) as err:
if modular:
raise CidrizeError(err)
return [str(err)]
def consolidate_networks(networks: set, min_prefixlen: int):
"""
>>> from pprint import pprint
>>> test = [IPNetwork('10.47.0.0/17'),
... IPNetwork('10.47.128.0/19'),
... IPNetwork('10.47.168.0/21'),
... IPNetwork('10.47.192.0/18'),
... IPNetwork('10.48.0.0/16'),
... IPNetwork('10.28.0.0/17'),
... IPNetwork('10.28.128.0/19'),
... IPNetwork('172.16.18.205/32'),
... IPNetwork('172.16.48.0/20'),
... IPNetwork('172.16.64.0/18'),
... IPNetwork('172.16.128.0/17'),
... IPNetwork('172.20.0.0/19'),
... IPNetwork('172.20.57.229/32'),
... IPNetwork('172.20.96.0/19'),
... IPNetwork('172.20.128.0/17'),
... IPNetwork('172.21.77.0/16'),
... IPNetwork('173.124.7.8/32'),
... IPNetwork('173.124.34.0/18'),
... IPNetwork('173.93.160.0/19'),
... IPNetwork('173.93.19.241/32'),
... IPNetwork('173.93.25.95/32'),
... IPNetwork('173.154.0.0/17'),
... IPNetwork('173.154.128.0/18'),
... IPNetwork('173.154.192.0/20'),
... IPNetwork('193.99.144.85/32'),
... IPNetwork('54.239.32.138/32'),
... IPNetwork('91.240.34.5/32'),
... IPNetwork('85.183.69.83/32'),
... IPNetwork('95.100.66.202/32')]
>>> pprint(consolidate_networks(test, 16))
[IPNetwork('10.28.0.0/16'),
IPNetwork('10.47.0.0/16'),
IPNetwork('10.48.0.0/16'),
IPNetwork('54.239.32.138/32'),
IPNetwork('85.183.69.83/32'),
IPNetwork('91.240.34.5/32'),
IPNetwork('95.100.66.202/32'),
IPNetwork('172.16.0.0/16'),
IPNetwork('172.20.0.0/15'),
IPNetwork('173.93.16.0/20'),
IPNetwork('173.93.160.0/19'),
IPNetwork('173.124.0.0/18'),
IPNetwork('173.154.0.0/16'),
IPNetwork('193.99.144.85/32')]
"""
networks = sorted([IPNetwork(net) for net in networks])
new_networks = []
for chunk in chunks(networks, 2):
if len(chunk) > 1:
spanning = netaddr.spanning_cidr(chunk)
if spanning.prefixlen >= min_prefixlen:
new_networks.append(spanning)
else:
new_networks.extend(chunk)
else:
new_networks.append(chunk[0])
merged = netaddr.cidr_merge(new_networks)
return merged
def smaller_network():
#list = ['192.168.0.0', '192.168.0.245', '192.168.0.255']
#list = ['192.168.2.245']
x = netaddr.spanning_cidr(list)
print x
def test_multi_host(self, test_type):
"""
Run a mainline multi-host test.
Because multihost tests are slow to setup, this tests most mainline
functionality in a single test.
- Create two hosts
- Create a network using the default IPAM driver, and a workload on
each host assigned to that network.
- Create a network using the Calico IPAM driver, and a workload on
each host assigned to that network.
- Check that hosts on the same network can ping each other.
- Check that hosts on different networks cannot ping each other.
- Modify the profile rules
- Check that connectivity has changed to match the profile we set up
- Re-apply the original profile
- Check that connectivity goes back to what it was originally.
"""
with DockerHost("host1",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False) as host1, \
DockerHost("host2",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False) as host2:
(n1_workloads, n2_workloads, networks) = \
self._setup_workloads(host1, host2)
# Get the original profiles:
output = host1.calicoctl("get profile -o yaml")
original_profiles = yaml.safe_load(output)
# Make a copy of the profiles to mess about with.
new_profiles = copy.deepcopy(original_profiles)
if test_type == "tags":
profile0_tag = new_profiles[0]['metadata']['tags'][0]
profile1_tag = new_profiles[1]['metadata']['tags'][0]
# Make a new profiles dict where the two networks have each
# other in their tags list
new_profiles[0]['metadata']['tags'].append(profile1_tag)
new_profiles[1]['metadata']['tags'].append(profile0_tag)
self._apply_new_profile(new_profiles, host1)
# Check everything can contact everything else now
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.tags":
profile0_tag = new_profiles[0]['metadata']['tags'][0]
profile1_tag = new_profiles[1]['metadata']['tags'][0]
rule0 = {'action': 'allow', 'source': {'tag': profile1_tag}}
rule1 = {'action': 'allow', 'source': {'tag': profile0_tag}}
new_profiles[0]['spec']['ingress'].append(rule0)
new_profiles[1]['spec']['ingress'].append(rule1)
self._apply_new_profile(new_profiles, host1)
# Check everything can contact everything else now
self.assert_connectivity(retries=3,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.protocol.icmp":
rule = {'action': 'allow', 'source': {'protocol': 'icmp'}}
# The copy.deepcopy(rule) is needed to ensure that we don't
# end up with a yaml document with a reference to the same
# rule. While this is probably legal, it isn't main line.
new_profiles[0]['spec']['ingress'].append(rule)
new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule))
self._apply_new_profile(new_profiles, host1)
# Check everything can contact everything else now
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.ip.addr":
prof_n1, prof_n2 = self._get_profiles(new_profiles)
for workload in n1_workloads:
ip = workload.ip
rule = {'action': 'allow', 'source': {'net': '%s/32' % ip}}
prof_n2['spec']['ingress'].append(rule)
for workload in n2_workloads:
ip = workload.ip
rule = {'action': 'allow', 'source': {'net': '%s/32' % ip}}
prof_n1['spec']['ingress'].append(rule)
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.ip.net":
prof_n1, prof_n2 = self._get_profiles(new_profiles)
n1_ips = [workload.ip for workload in n1_workloads]
n2_ips = [workload.ip for workload in n2_workloads]
n1_subnet = netaddr.spanning_cidr(n1_ips)
n2_subnet = netaddr.spanning_cidr(n2_ips)
rule = {'action': 'allow', 'source': {'net': str(n1_subnet)}}
prof_n2['spec']['ingress'].append(rule)
rule = {'action': 'allow', 'source': {'net': str(n2_subnet)}}
prof_n1['spec']['ingress'].append(rule)
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.selector":
new_profiles[0]['metadata']['labels'] = {'net': 'n1'}
new_profiles[1]['metadata']['labels'] = {'net': 'n2'}
rule = {'action': 'allow', 'source': {'selector': 'net=="n2"'}}
new_profiles[0]['spec']['ingress'].append(rule)
rule = {'action': 'allow', 'source': {'selector': "net=='n1'"}}
new_profiles[1]['spec']['ingress'].append(rule)
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads)
elif test_type == "rules.tcp.port":
rule = {
'action': 'allow',
'protocol': 'tcp',
'destination': {
'ports': [80]
}
}
# The copy.deepcopy(rule) is needed to ensure that we don't
# end up with a yaml document with a reference to the same
# rule. While this is probably legal, it isn't main line.
new_profiles[0]['spec']['ingress'].append(rule)
new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule))
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads,
type_list=['tcp'])
self.assert_connectivity(retries=2,
pass_list=n1_workloads,
fail_list=n2_workloads,
type_list=['icmp', 'udp'])
elif test_type == "rules.udp.port":
rule = {
'action': 'allow',
'protocol': 'udp',
'destination': {
'ports': [69]
}
}
# The copy.deepcopy(rule) is needed to ensure that we don't
# end up with a yaml document with a reference to the same
# rule. While this is probably legal, it isn't main line.
new_profiles[0]['spec']['ingress'].append(rule)
new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule))
self._apply_new_profile(new_profiles, host1)
self.assert_connectivity(retries=2,
pass_list=n1_workloads + n2_workloads,
type_list=['udp'])
self.assert_connectivity(retries=2,
pass_list=n1_workloads,
fail_list=n2_workloads,
type_list=['icmp', 'tcp'])
else:
print "******************* " \
"ERROR - Unrecognised test type " \
"*******************"
assert False, "Unrecognised test type: %s" % test_type
# Now restore the original profile and check it all works as before
self._apply_new_profile(original_profiles, host1)
host1.calicoctl("get profile -o yaml")
self._check_original_connectivity(n1_workloads, n2_workloads)
# Tidy up
host1.remove_workloads()
host2.remove_workloads()
for network in networks:
network.delete()
parser.add_argument("input", help="input list of ip addresses / subnets")
parser.add_argument("gw", help="Gateway address to use for routes")
args = parser.parse_args()
input_list = open(args.input).readlines()
nets = {}
for i in input_list:
x = ".".join(i.split(".")[:2])
if x in nets:
nets[x].append(i.strip())
else:
nets[x] = [i.strip()]
ranges = []
for k,v in nets.items():
if len(v) == 1:
x = ".".join(i.split(".")[:3])
x = f"{x}.0/24"
# print(x)
ranges.append(x)
continue
spanning_cidr = netaddr.spanning_cidr(v)
ranges.append(spanning_cidr)
# print(spanning_cidr)
cmd = "add"
if args.d:
cmd = "delete"
for i in ranges:
subprocess.call(["route", cmd, str(i), args.gw])
def cidrize(ipstr, strict=False, modular=True):
"""
This function tries to determine the best way to parse IP addresses correctly & has
all the logic for trying to do the right thing!
Returns a list of consolidated netaddr objects.
Input can be several formats::
'192.0.2.18'
'192.0.2.64/26'
'192.0.2.80-192.0.2.85'
'192.0.2.170-175'
'192.0.2.8[0-5]'
'192.0.2.[0-29]'
'192.168.4.6[1234]'
'1.2.3.*'
'192.0.2.170-175, 192.0.2.80-192.0.2.85, 192.0.2.64/26'
Hyphenated ranges do not need to form a CIDR block. Netaddr does most of
the heavy lifting for us here.
Input can NOT be::
192.0.2.0 0.0.0.255 (hostmask)
192.0.2.0 255.255.255.0 (netmask)
Does NOT accept network or host mask notation at this time!
Defaults:
* parsing exceptions will raise a CidrizeError (modular=True).
* results will be returned as a spanning CIDR (strict=False).
:param ipstr:
IP string to be parsed.
:param modular:
Set to False to cause exceptions to be stripped & the error text will be
returned as a list. This is intended for use with scripts or APIs
out-of-the box.
Example:
>>> import cidrize as c
>>> c.cidrize('1.2.3.4-1.2.3.1099')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/j/jathan/sandbox/cidrize.py", line 153, in cidrize
raise CidrizeError(err)
cidrize.CidrizeError: base address '1.2.3.1099' is not IPv4
>>> c.cidrize('1.2.3.4-1.2.3.1099', modular=False)
["base address '1.2.3.1099' is not IPv4"]
:param strict:
Set to True to return explicit networks based on start/end addresses.
Example:
>>> import cidrize as c
>>> c.cidrize('1.2.3.4-1.2.3.10')
[IPNetwork('1.2.3.0/28')]
>>> c.cidrize('1.2.3.4-1.2.3.10', strict=True)
[IPNetwork('1.2.3.4/30'), IPNetwork('1.2.3.8/31'), IPNetwork('1.2.3.10/32')]
"""
ip = None
# Short-circuit to parse commas since it calls back here anyway
if ',' in ipstr:
return parse_commas(ipstr, strict=strict, modular=modular)
# Short-circuit for hostnames (we're assuming first char is alpha)
if hostname_re.match(ipstr):
raise CidrizeError('Cannot parse hostnames!')
# Otherwise try everything else
result = None
try:
# Parse "everything" & immediately return; strict/loose doesn't apply
if ipstr in EVERYTHING:
log.debug("Trying everything style...")
return [IPNetwork('0.0.0.0/0')]
# Parse old-fashioned CIDR notation & immediately return; strict/loose doesn't apply
# Now with IPv6!
elif cidr_re.match(ipstr) or is_ipv6(ipstr):
log.debug("Trying CIDR style...")
ip = IPNetwork(ipstr)
return [ip.cidr]
# Parse 1.2.3.118-1.2.3.121 range style
elif range_re.match(ipstr):
log.debug("Trying range style...")
result = parse_range(ipstr)
# Parse 1.2.3.4-70 hyphen style
elif hyphen_re.match(ipstr):
log.debug("Trying hyphen style...")
result = parse_hyphen(ipstr)
# Parse 1.2.3.* glob style
elif glob_re.match(ipstr):
log.debug("Trying glob style...")
ipglob = IPGlob(ipstr)
result = spanning_cidr(ipglob)
# Parse 1.2.3.4[5-9] or 1.2.3.[49] bracket style as a last resort
elif bracket_re.match(ipstr):
log.debug("Trying bracket style...")
result = parse_brackets(ipstr)
# If result still isn't set, let's see if it's IPv6??
elif result is None:
log.debug("Trying bare IPv6 parse...")
result = IPNetwork(ipstr)
# This will probably fail 100% of the time. By design.
else:
raise CidrizeError("Could not determine parse style for '%s'" % ipstr)
# If it's a single host, just return it wrapped in a list
if result.size == 1:
log.debug("Returning a single host!")
return [result.cidr]
# Logic to honor strict/loose, except IPRange. Doing a spanning_cidr on
# an IPRange can be super slow if the range is large (such as a /8), so
# IPRange objects larger than MAX_RANGE_LEN will always be strict.
if not strict:
if isinstance(result, IPRange) and result.size >= MAX_RANGE_LEN:
log.debug('IPRange objects larger than /18 will always be strict.')
return result.cidrs()
return [spanning_cidr(result)]
else:
try:
return result.cidrs() # IPGlob and IPRange have .cidrs()
except AttributeError as err:
return result.cidr # IPNetwork has .cidr
except (AddrFormatError, TypeError, ValueError) as err:
if modular:
raise CidrizeError(err)
return [str(err)]
