def test_action_policy(self, name, ip_rules, bypass_rules, append, expected_add,
expected_remove, delete, update):
template = {
'name': 'test-azure-sql-server',
'resource': 'azure.sqlserver',
'filters': [
{'type': 'value',
'key': 'name',
'op': 'glob',
'value_type': 'normalize',
'value': 'cctestsqlserver*'}],
'actions': [
{'type': 'set-firewall-rules',
'append': append}]}
if bypass_rules is not None:
template['actions'][0]['bypass-rules'] = bypass_rules
if ip_rules is not None:
template['actions'][0]['ip-rules'] = ip_rules
p = self.load_policy(template)
resources = p.run()
self.assertEqual(1, len(resources))
# Added IP's
added = IPSet()
for r in [IPRange(args[3], args[4]) for _, args, _ in update.mock_calls]:
added.add(r)
self.assertEqual(IPSet(expected_add), added)
# Removed IP's
self.assertEqual(set(expected_remove), {args[2] for _, args, _ in delete.mock_calls})
def test_ipset_converts_to_cidr_networks_v4():
s1 = IPSet(IPNetwork('10.1.2.3/8'))
s1.add(IPNetwork('192.168.1.2/16'))
assert list(s1.iter_cidrs()) == [
IPNetwork('10.0.0.0/8'),
IPNetwork('192.168.0.0/16'),
]
def test_ipset_converts_to_cidr_networks_v6():
s1 = IPSet(IPNetwork('fe80::4242/64'))
s1.add(IPNetwork('fe90::4343/64'))
assert list(s1.iter_cidrs()) == [
IPNetwork('fe80::/64'),
IPNetwork('fe90::/64'),
]
def auth(event, context):
sns = boto3.client('sns')
body = event['body']
timestamp = event['headers']['X-Slack-Request-Timestamp']
concat_message = ('v0:' + timestamp + ':' + body).encode()
slack_signature = event['headers']['X-Slack-Signature']
key = (os.environ['slack_secret']).encode()
hashed_msg = 'v0=' + \
hmac.new(key, concat_message, hashlib.sha256).hexdigest()
print(hashed_msg)
if (hashed_msg != slack_signature):
return {'statusCode': 404, 'body': json.dumps("Un-Authorized")}
text = "Working on your request..."
command = body.split("text=")[1]
command_send = urllib.parse.unquote(command.split("&")[0])
try:
ip = IPSet()
ip.add(command_send)
except Exception:
return {
'statusCode': 200,
'body': json.dumps("Please enter a valid IP or CIDR")
}
response = body.split("response_url=")[1]
response_url = response.split("&")[0]
decoded = urllib.parse.unquote(response_url)
slack_response = sns.publish(
# Add your aws account id below
TopicArn='arn:aws:sns:us-east-1:{Your AWS Account ID}:processing-topic',
Message=decoded + "-" + command_send,
)
print(slack_response)
return {'statusCode': 200, 'body': json.dumps(text)}
def _parse_amazon_ranges(ranges):
all_amazon = IPSet()
for service_description in ranges["prefixes"]:
if service_description["service"] in AWS_SERVICES:
all_amazon.add(IPNetwork(service_description["ip_prefix"]))
return all_amazon
def __init__(self, whitelist, blacklist):
set = IPSet([])
for block in whitelist:
set.add(IPNetwork(block))
# Remove invalid broadcast and network addresses
if block.broadcast != None:
set.remove(IPNetwork(block.broadcast))
if block.size > 2:
set.remove(IPNetwork(block.network))
for block in blacklist:
set.remove(IPNetwork(block))
for block in set.iter_cidrs():
self.total += block.size
self.networks.append({
"network": block,
"size": block.size,
"start": block[0],
"index": 0
})
if self.total < 1:
raise Exception(
"IPScanManager can not be started with an empty target scope")
self.rng = CyclicPRNG(self.total)
def blockcomp(b):
return b["start"]
self.networks.sort(key=blockcomp)
start = 1
for i in range(0, len(self.networks)):
self.networks[i]["index"] = start
start += self.networks[i]["size"]
def _parse_amazon_ranges(ranges):
all_amazon = IPSet()
for service_description in ranges['prefixes']:
if service_description['service'] in AWS_SERVICES:
all_amazon.add(IPNetwork(service_description['ip_prefix']))
return all_amazon
def pick_ip_in_network(self, network, *, but_not=EMPTY_SET):
excluded_set = IPSet()
for exclusion in but_not:
if isinstance(exclusion, str):
exclusion = IPAddress(exclusion)
excluded_set.add(exclusion)
# Unless the prefix length is very small, make sure we don't select
# a normally-unusable IP address.
if network.version == 6 and network.prefixlen < 127:
# Don't pick the all-zeroes address, since it has special meaning
# in IPv6 as the subnet-router anycast address. IPv6 does not have
# a broadcast address, though.
first, last = network.first + 1, network.last
network_size = network.size - 1
elif network.prefixlen < 31:
# Don't pick broadcast or network addresses.
first, last = network.first + 1, network.last - 1
network_size = network.size - 2
else:
first, last = network.first, network.last
network_size = network.size
if len(but_not) == network_size:
raise ValueError(
"No IP addresses available in network: %s (but_not=%r)" %
(network, but_not))
for _ in range(100):
address = IPAddress(random.randint(first, last))
if address not in excluded_set:
return str(address)
raise TooManyRandomRetries(
"Could not find available IP in network: %s (but_not=%r)" %
(network, but_not))
def test_ipset_converts_to_cidr_networks_v6():
s1 = IPSet(IPNetwork('fe80::4242/64'))
s1.add(IPNetwork('fe90::4343/64'))
assert list(s1.iter_cidrs()) == [
IPNetwork('fe80::/64'),
IPNetwork('fe90::/64'),
]
def test_ipset_converts_to_cidr_networks_v4():
s1 = IPSet(IPNetwork('10.1.2.3/8'))
s1.add(IPNetwork('192.168.1.2/16'))
assert list(s1.iter_cidrs()) == [
IPNetwork('10.0.0.0/8'),
IPNetwork('192.168.0.0/16'),
]
def _query_rules(self, resource):
query = self.client.firewall_rules.list_by_server(
resource['resourceGroup'], resource['name'])
resource_rules = IPSet()
for r in query:
resource_rules.add(IPRange(r.start_ip_address, r.end_ip_address))
return resource_rules
def _query_rules(self, resource):
query = self.client.firewall_rules.list_by_server(
resource['resourceGroup'], resource['name'])
resource_rules = IPSet()
for r in query:
rule = IPRange(r.start_ip_address, r.end_ip_address)
if rule == AZURE_SERVICES:
# Ignore 0.0.0.0 magic value representing Azure Cloud bypass
continue
resource_rules.add(rule)
return resource_rules
def parse(self, data):
mynets = IPSet()
for line in data.split("\n"):
if not line or line[0] == ";":
continue
ip, sbl = line.split(";")
ip = IPNetwork(ip.strip())
mynets.add(ip)
return mynets
def update_blacklist(self):
from app.models import ScopeItem
newBlacklist = []
newBlacklistSet = IPSet()
for item in ScopeItem.getBlacklist():
newItem = ipaddress.ip_network(item.target, False)
newSetItem = IPNetwork(item.target, False)
newBlacklist.append(newItem)
newBlacklistSet.add(newSetItem)
self.blacklist = newBlacklist
self.blacklist_set = newBlacklistSet
self.blacklistSize = len(self.blacklist_set)
def update_scope(self):
from app.models import ScopeItem
newScope = []
newScopeSet = IPSet()
for item in ScopeItem.getScope():
newItem = ipaddress.ip_network(item.target, False)
newSetItem = IPNetwork(item.target, False)
newScope.append(newItem)
newScopeSet.add(newSetItem)
self.scope = newScope
self.scope_set = newScopeSet
self.scopeSize = len(self.scope_set)
def parse(self, data):
mynets = IPSet()
for line in data.split("\n"):
if not line or line[0] == ';':
continue
ip, sbl = line.split(';')
ip = IPNetwork(ip.strip())
mynets.add(ip)
return mynets
def test_ipset_member_insertion_and_deletion():
s1 = IPSet()
s1.add('192.0.2.0')
assert s1 == IPSet(['192.0.2.0/32'])
s1.remove('192.0.2.0')
assert s1 == IPSet([])
s1.add(IPRange("10.0.0.0", "10.0.0.255"))
assert s1 == IPSet(['10.0.0.0/24'])
s1.remove(IPRange("10.0.0.128", "10.10.10.10"))
assert s1 == IPSet(['10.0.0.0/25'])
def test_ipset_member_insertion_and_deletion():
s1 = IPSet()
s1.add('192.0.2.0')
assert s1 == IPSet(['192.0.2.0/32'])
s1.remove('192.0.2.0')
assert s1 == IPSet([])
s1.add(IPRange("10.0.0.0", "10.0.0.255"))
assert s1 == IPSet(['10.0.0.0/24'])
s1.remove(IPRange("10.0.0.128", "10.10.10.10"))
assert s1 == IPSet(['10.0.0.0/25'])
def parse_aws(ip_ranges=None):
if ip_ranges is None:
# Get the current IP Ranges from AWS.
# Fun fact: The minutes in "createDate" in this data file has
# almost 1/2 a chance of being 13, 43, or 59, and the seconds
# has almost 1/2 a chance of being 4, 10, or 11. Well, I
# thought it was a fun fact.
ip_ranges = get("https://ip-ranges.amazonaws.com/ip-ranges.json").json()
if isinstance(ip_ranges, IPSet):
aws = ip_ranges
else:
# Merge everything from AWS into one IPSet.
# This ignores IPv6, but so does everyone else.
aws = IPSet([IPNetwork(x["ip_prefix"]) for x in ip_ranges["prefixes"]])
# These are all of the IPv4 addresses, there are 2^32 of them.
# IPSet used here just because I like the symmetry of it all.
internet = IPSet()
internet.add("0.0.0.0/0")
# There are less than 2^32 IPv4 addresses that AWS could control.
# There are ones not in this private list that they couldn't touch,
# but it's a reasonable start.
private = IPSet([IPNetwork(x) for x in [
"0.0.0.0/8", # RFC 1700 broadcast addresses
"10.0.0.0/8", # RFC 1918 Private address space (aka, your work LAN)
"100.64.0.0/10", # IANA Carrier Grade NAT (not your home NAT, no sirree)
"100.64.0.0/10", # RFC 6598 Carrier graded NAT
"127.0.0.0/8", # Loopback addresses (because you need 16 million IPs for localhost)
"169.254.0.0/16", # RFC 6890 Link Local address (aka, the broken LAN)
"172.16.0.0/12", # RFC 1918 Private address space (aka, Goldilocks' LAN)
"192.0.0.0/24", # RFC 5736 IANA IPv4 Special Purpose Address Registry
"192.0.2.0/24", # RFC 5737 TEST-NET for internal use
"192.168.0.0/16", # RFC 1918 Private address space (aka, your home LAN)
"192.88.99.0/24", # RFC 3068 6to4 anycast relays
"198.18.0.0/15", # RFC 2544 Testing of inter-network communications
"198.51.100.0/24", # RFC 5737 TEST-NET-2 for internal use
"203.0.113.0/24", # RFC 5737 TEST-NET-3 for internal use
"224.0.0.0/4", # RFC 5771 Multicast Addresses
"240.0.0.0/4", # RFC 6890 Reserved for future use (or if the RFC team needs to make a few bucks)
]])
# I hope you had fun on this little journey.
public = internet.size - private.size
internet = internet.size
aws = aws.size
return public, internet, aws
def proc(event, context):
p = 0
sites_list = ['1234', '5678', '2837']
url = 'https://my.incapsula.com/api/prov/v1/sites/status'
for k in sites_list:
params = {
'api_id': os.environ['api_id'],
'api_key': os.environ['api_key'],
'site_id': k
}
incap = requests.post(url, params=params)
inputmsg = event['Records'][0]['Sns']['Message']
webhook_url = inputmsg.split("-")[0]
ip = inputmsg.split("-")[1]
print(webhook_url)
print(ip)
ipset = IPSet()
inputset = IPSet()
incap_response = (json.loads(
incap.text)['security']['acls']['rules'][0])
ip_json = []
ip_json.clear()
for i in incap_response['exceptions']:
ip_json.append(i['values'][0].get("ips"))
ip_json = (str(ip_json).replace('\\n', ' '))
final = re.findall("'(.*?)'", ip_json)
for j in final:
if ("-" in j) == False:
ipset.add(j)
inputset.add(ip)
for j in inputset:
if (j in ipset) == True:
p = 1
else:
p = 0
break
print(p)
if p == 1:
slack_data = '{"text":"IP is Whitelisted!!"}'
final_response = requests.post(
url=webhook_url,
data=slack_data,
headers={'Content-Type': 'application/json'})
else:
slack_data = '{"text":"IP is not Whitelisted!!"}'
final_response = requests.post(
url=webhook_url,
data=slack_data,
headers={'Content-Type': 'application/json'})
print(final_response)
def summarizeIPs(inFile, outFile):
netSet = IPSet()
with open(inFile, 'r') as f:
for line in f.readlines():
net = IPSet()
try:
net.add(line.strip())
except AddrFormatError:
continue
else:
netSet = netSet | net
netMin = netSet.iter_cidrs()
with open(outFile, 'w') as f:
for net in netMin:
f.write('{}\n'.format(net))
def collect_ips(ioc_list: List[str]) -> Tuple[IPSet, set]:
ip_set = IPSet()
non_ip_group = set()
for ioc in ioc_list:
if '-' in ioc:
# handle ip ranges
ip_range = ioc.split('-')
if len(ip_range) == 2 and IP_RE.fullmatch(ip_range[0]) and IP_RE.fullmatch(ip_range[1]):
ip_set.add(IPRange(ip_range[0], ip_range[1]))
else:
non_ip_group.add(ioc)
elif CIDR_RE.findall(ioc) or IP_RE.match(ioc):
ip_set.add(ioc.strip('\n'))
else:
non_ip_group.add(ioc)
return ip_set, non_ip_group
def generate_ips(cidr, ip_start, count=1, pools=None, gateway_ip=None, exclude=None):
"""生成 IP 地址列表
:param cidr: string, 子网 CIDR, eg:'10.1.33.0/24'
:param ip_start: string, 生成 IP 列表的起始 IP, eg:'10.1.33.240'
:param count: interger, 生成 IP 的个数
:param pools: list, IP 池, eg:[{'start': "10.1.33.240", 'end': "10.1.33.250"}]
:param gateway_ip: string, 网关 IP, eg:'10.1.33.1'
:param exclude: Iterable object, 排除的 IP, eg:["10.1.33.242", "10.1.33.246"]
:return: IP 地址列表.返回 None 表示 ip_start 不在 CIDR 网段或者 IP 池内, 或者被排除了
"""
ip = IPNetwork(cidr)
ipset = IPSet()
ipset.add(ip)
ip_start = IPAddress(ip_start)
if isinstance(pools, list):
pool_set = IPSet()
for p in pools:
pool_set.add(IPRange(p['start'], p['end']))
ipset &= pool_set
implicit_exclude = IPSet()
implicit_exclude.add(ip.network)
implicit_exclude.add(ip.broadcast)
if gateway_ip:
implicit_exclude.add(IPAddress(gateway_ip))
if isinstance(exclude, Iterable):
exclude = [IPAddress(i) for i in exclude if i]
exclude = IPSet(exclude)
exclude |= implicit_exclude
else:
exclude = implicit_exclude
if ip_start not in ipset or ip_start in exclude:
return None
else:
ip_list = sorted(list(ipset))
ip_list = ip_list[ip_list.index(ip_start):]
ipset = IPSet(ip_list)
ipset -= exclude
ip_list = sorted(list(ipset))
ip_list = ip_list[:count]
ips = [str(i) for i in ip_list]
return ips
def parse_ip_ranges(data, key):
'''
Parses IP range or CIDR mask.
:param data: Dictionary where to look for the value.
:param key: Key for the value to be parsed.
:return: Set of IP ranges and networks.
'''
if key not in data:
return None
ranges = [[s.strip() for s in r.split('-')] for r in data[key]]
result = IPSet()
for r in ranges:
if len(r) > 2:
raise Exception('Invalid range. Use x.x.x.x-y.y.y.y or x.x.x.x or x.x.x.x/y.')
result.add(IPRange(*r) if len(r) == 2 else IPNetwork(r[0]))
return result
def available_range(vpc_range, used_cidrs_path):
"""Verify if a given range is available in a range."""
click.echo('VPC Range %s' % vpc_range)
click.echo('Used CIDRS file %s' % used_cidrs_path)
click.echo('--------')
vpc = IPSet([vpc_range])
unavailable = IPSet([])
with open(used_cidrs_path) as fp:
for cnt, line in enumerate(fp):
unavailable.add(line.strip())
available = vpc.difference(unavailable)
__print_set_size("vpc", vpc)
__print_set_size("unavailable", unavailable)
__print_set("available", available)
def test_ipset_adding_and_removing_members_ip_addresses_as_ints():
s1 = IPSet(['10.0.0.0/25'])
s1.add('10.0.0.0/24')
assert s1 == IPSet(['10.0.0.0/24'])
integer1 = int(IPAddress('10.0.0.1'))
integer2 = int(IPAddress('fe80::'))
integer3 = int(IPAddress('10.0.0.2'))
s2 = IPSet([integer1, integer2])
assert s2 == IPSet(['10.0.0.1/32', 'fe80::/128'])
s2.add(integer3)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
s2.remove(integer2)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32'])
s2.update([integer2])
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
def test_ipset_adding_and_removing_members_ip_addresses_as_ints():
s1 = IPSet(['10.0.0.0/25'])
s1.add('10.0.0.0/24')
assert s1 == IPSet(['10.0.0.0/24'])
integer1 = int(IPAddress('10.0.0.1'))
integer2 = int(IPAddress('fe80::'))
integer3 = int(IPAddress('10.0.0.2'))
s2 = IPSet([integer1, integer2])
assert s2 == IPSet(['10.0.0.1/32', 'fe80::/128'])
s2.add(integer3)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
s2.remove(integer2)
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32'])
s2.update([integer2])
assert s2 == IPSet(['10.0.0.1/32', '10.0.0.2/32', 'fe80::/128'])
def is_ip_in_list(ip_address, addresses_list):
"""Check if address IP is in list.
:param ip_address: Address IP to check
:param addresses_list: Range of IP, network or simple IP.
:returns: True if given IP is in list.
"""
if not isinstance(addresses_list, list):
raise Exception('Given parameter is not a list.')
ip_set = IPSet()
for ip_range in addresses_list:
try:
# It's a glob
if '*' in ip_range or '-' in ip_range:
ip_set.add(IPGlob(ip_range))
# It's a network
elif '/' in ip_range:
ip_set.add(IPNetwork(ip_range))
# Simple IP
else:
ip_set.add(IPAddress(ip_range))
except Exception:
pass
try:
return ip_address in ip_set
except Exception:
return False
def get_ips_list(ranges):
"""Get the IP addresses list from a list of ranges.
:param list ranges: List of ranges.
:returns: List of IP addresses.
:rtype: list of cidr ips
(https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing)
"""
ip_set = IPSet()
for ip_range in ranges:
try:
# It's a glob
if '*' in ip_range or '-' in ip_range:
ip_set.add(IPGlob(ip_range))
# It's a network
elif '/' in ip_range:
ip_set.add(IPNetwork(ip_range))
# Simple IP
else:
ip_set.add(IPAddress(ip_range))
except Exception:
pass
return [str(ip.cidr) for ip in ip_set.iter_cidrs()]
def extract_router_ipaddress(in_file):
d_router_ipaddress_data = dict()
with bz2.BZ2File(in_file, "r") as asrel_data:
reader = csv.reader(asrel_data, delimiter=' ')
for line in reader:
if "#" not in line[0]:
# if theres something more than just the ipaddress
if len(line) > 1:
for attribute in line:
if attribute == "T":
router_interface_ipaddress = line[0]
if router_interface_ipaddress not in d_router_ipaddress_data:
d_router_ipaddress_data[router_interface_ipaddress] = 1
routers_ips_set = IPSet()
for k, v in d_router_ipaddress_data.iteritems():
routers_ips_set.add(k)
routers_prefixes = cidr_merge(routers_ips_set)
for prefix in routers_prefixes:
print prefix
def ipv6_addresses(lines):
""" Checks if line contains IPv6 Addresses """
# IPv6 - this doesn't handle %interface formats
ip6_regex = ("(([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4})|"
"([0-9a-fA-F]{1,4}:){1,7}:|"
"([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4})|"
"([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|"
"([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|"
"([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|"
"([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|"
"([0-9a-fA-F]{1,4}:){1,1}(:[0-9a-fA-F]{1,4}){1,6}|"
":(:[0-9a-fA-F]{1,4}){1,7})|([0-9a-fA-F]{1,4}:){1,4}:"
r"((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3,3}"
"(25[0-5]|(2[0-4]|1?[0-9])?[0-9])")
ip6_cidr = "(12[0-8]|(1[01]|[0-9])?[0-9])"
ip6_match = "^{}(/{})?$".format(ip6_regex, ip6_cidr)
ip6_range = r"^{}\s*-\s*{}$".format(ip6_regex, ip6_regex)
# Compile regexes increases speed
match_ip6_cidr = re.compile(ip6_match)
match_ip6_range = re.compile(ip6_range)
ip_set = IPSet()
remaining = []
for line in lines:
line = line.strip()
try:
if match_ip6_cidr.match(line):
logging.info('IPv6 match: %s', line)
ip_set.add(line)
elif match_ip6_range.match(line):
logging.info('IPv6 range: %s', line)
start, finish = line.split("-")
start = IPAddress(start.strip())
finish = IPAddress(finish.strip())
if start < finish:
ip_set.add(IPRange(start, finish))
else:
logging.warning(
'IPv4 range: %s (beginning of range '
'larger than end of range', line)
ip_set.add(IPRange(finish, start))
else:
logging.info('Unmatched: %s', line)
remaining.append(line)
except (RuntimeError, TypeError, NameError):
logging.debug('Invalid IPv6 addresses: %s', line)
remaining.append(line)
return ip_set, remaining
def generate_ip_set(
ip_addresses: Optional[Iterable[str]],
extra_addresses: Optional[Iterable[str]] = None,
config_path: Optional[Iterable[str]] = None,
) -> IPSet:
"""
Generate an IPSet from a list of IP addresses or CIDRs.
Additionally, for each IPv4 network in the list of IP addresses, also
includes the corresponding IPv6 networks.
This includes:
* IPv4-Compatible IPv6 Address (see RFC 4291, section 2.5.5.1)
* IPv4-Mapped IPv6 Address (see RFC 4291, section 2.5.5.2)
* 6to4 Address (see RFC 3056, section 2)
Args:
ip_addresses: An iterable of IP addresses or CIDRs.
extra_addresses: An iterable of IP addresses or CIDRs.
config_path: The path in the configuration for error messages.
Returns:
A new IP set.
"""
result = IPSet()
for ip in itertools.chain(ip_addresses or (), extra_addresses or ()):
try:
network = IPNetwork(ip)
except AddrFormatError as e:
raise ConfigError(
"Invalid IP range provided: %s." % (ip,), config_path
) from e
result.add(network)
# It is possible that these already exist in the set, but that's OK.
if ":" not in str(network):
result.add(IPNetwork(network).ipv6(ipv4_compatible=True))
result.add(IPNetwork(network).ipv6(ipv4_compatible=False))
result.add(_6to4(network))
return result
def append_rules_to_rules_ipset(ip_addrs, input_type):
'''
Add rules to IPSet for comparison
:param ip_addrs:
:param input_type:
:return:
'''
rules_set = IPSet()
if input_type == 'address':
rules_set.add(ip_addrs)
elif input_type == 'range':
rules_set.add(IPRange(ip_addrs.split('-')[0], ip_addrs.split('-')[1]))
elif input_type == 'prefix':
rules_set.add(ip_addrs)
logger.info("Rules set: " + str(rules_set))
return rules_set
def ipv4_addresses(lines):
""" Checks if line contains IPv4 Addresses """
# IPv4 - this doesn't handle %interface formats
ip4_regex = (r"((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3,3}"
"(25[0-5]|(2[0-4]|1?[0-9])?[0-9])")
ip4_cidr = "(3[0-2]|[12]?[0-9])"
ip4_match = "^{}(/{})?$".format(ip4_regex, ip4_cidr)
ip4_range = r"^{}\s*-\s*{}$".format(ip4_regex, ip4_regex)
# Compile regexes increases speed
match_ip4_cidr = re.compile(ip4_match)
match_ip4_range = re.compile(ip4_range)
ip_set = IPSet()
remaining = []
for line in lines:
line = line.strip()
try:
if match_ip4_cidr.match(line):
logging.info('IPv4 match: %s', line)
ip_set.add(line)
elif match_ip4_range.match(line):
logging.info('IPv4 range: %s', line)
start, finish = line.split("-")
start = IPAddress(start.strip())
finish = IPAddress(finish.strip())
if start < finish:
ip_set.add(IPRange(start, finish))
else:
logging.warning(
'IPv4 range: %s (beginning of range '
'larger than end of range', line)
ip_set.add(IPRange(finish, start))
else:
remaining.append(line)
except (RuntimeError, TypeError, NameError):
logging.debug('Invalid IPv4 addresses: %s', line)
remaining.append(line)
return ip_set, remaining
def test_ipset_cidr_fracturing():
s1 = IPSet(['0.0.0.0/0'])
s1.remove('255.255.255.255')
assert s1 == IPSet([
'0.0.0.0/1', '128.0.0.0/2', '192.0.0.0/3',
'224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6',
'252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9',
'255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15',
'255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18',
'255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21',
'255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24',
'255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27',
'255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30',
'255.255.255.252/31', '255.255.255.254/32'])
cidrs = s1.iter_cidrs()
assert len(cidrs) == 32
assert list(cidrs) == [
IPNetwork('0.0.0.0/1'), IPNetwork('128.0.0.0/2'), IPNetwork('192.0.0.0/3'),
IPNetwork('224.0.0.0/4'), IPNetwork('240.0.0.0/5'), IPNetwork('248.0.0.0/6'),
IPNetwork('252.0.0.0/7'), IPNetwork('254.0.0.0/8'), IPNetwork('255.0.0.0/9'),
IPNetwork('255.128.0.0/10'), IPNetwork('255.192.0.0/11'), IPNetwork('255.224.0.0/12'),
IPNetwork('255.240.0.0/13'), IPNetwork('255.248.0.0/14'), IPNetwork('255.252.0.0/15'),
IPNetwork('255.254.0.0/16'), IPNetwork('255.255.0.0/17'), IPNetwork('255.255.128.0/18'),
IPNetwork('255.255.192.0/19'), IPNetwork('255.255.224.0/20'), IPNetwork('255.255.240.0/21'),
IPNetwork('255.255.248.0/22'), IPNetwork('255.255.252.0/23'), IPNetwork('255.255.254.0/24'),
IPNetwork('255.255.255.0/25'), IPNetwork('255.255.255.128/26'), IPNetwork('255.255.255.192/27'),
IPNetwork('255.255.255.224/28'), IPNetwork('255.255.255.240/29'), IPNetwork('255.255.255.248/30'),
IPNetwork('255.255.255.252/31'), IPNetwork('255.255.255.254/32')
]
assert cidrs == cidr_exclude('0.0.0.0/0', '255.255.255.255')
s1.remove('0.0.0.0')
assert s1 == IPSet([
'0.0.0.1/32', '0.0.0.2/31', '0.0.0.4/30',
'0.0.0.8/29', '0.0.0.16/28', '0.0.0.32/27',
'0.0.0.64/26', '0.0.0.128/25', '0.0.1.0/24',
'0.0.2.0/23', '0.0.4.0/22', '0.0.8.0/21',
'0.0.16.0/20', '0.0.32.0/19', '0.0.64.0/18',
'0.0.128.0/17', '0.1.0.0/16', '0.2.0.0/15',
'0.4.0.0/14', '0.8.0.0/13', '0.16.0.0/12',
'0.32.0.0/11', '0.64.0.0/10', '0.128.0.0/9',
'1.0.0.0/8', '2.0.0.0/7', '4.0.0.0/6',
'8.0.0.0/5', '16.0.0.0/4', '32.0.0.0/3',
'64.0.0.0/2', '128.0.0.0/2', '192.0.0.0/3',
'224.0.0.0/4', '240.0.0.0/5', '248.0.0.0/6',
'252.0.0.0/7', '254.0.0.0/8', '255.0.0.0/9',
'255.128.0.0/10', '255.192.0.0/11', '255.224.0.0/12',
'255.240.0.0/13', '255.248.0.0/14', '255.252.0.0/15',
'255.254.0.0/16', '255.255.0.0/17', '255.255.128.0/18',
'255.255.192.0/19', '255.255.224.0/20', '255.255.240.0/21',
'255.255.248.0/22', '255.255.252.0/23', '255.255.254.0/24',
'255.255.255.0/25', '255.255.255.128/26', '255.255.255.192/27',
'255.255.255.224/28', '255.255.255.240/29', '255.255.255.248/30',
'255.255.255.252/31', '255.255.255.254/32',
])
assert len(list(s1.iter_cidrs())) == 62
s1.add('255.255.255.255')
s1.add('0.0.0.0')
assert s1 == IPSet(['0.0.0.0/0'])
def _validate_addresses_helper(self, network, net):
# Keep an IPSet of addresses/ranges that can be checked
# against for overlaps
current_set = IPSet()
for address_list in net['addresses']:
address = address_list.split('-')
if len(address) == 1:
try:
ip_addr1 = IPAddress(address[0].strip())
except AddrFormatError:
msg = ("Address %s in network %s is not a valid "
"IP address." % (address[0], net['name']))
self.add_error(msg)
self._valid = False
else:
if ip_addr1 not in network:
msg = ("Address %s in network %s is not within "
"the specified CIDR %s." %
(address[0], net['name'], net['cidr']))
self.add_error(msg)
self._valid = False
elif ip_addr1 not in current_set:
current_set.add(ip_addr1)
else:
try:
ip_addr1 = IPAddress(address[0].strip())
ip_addr2 = IPAddress(address[1].strip())
except AddrFormatError:
msg = ("The address range %s in network %s is not "
"a range of valid IP addresses." %
(address_list, net['name']))
self.add_error(msg)
self._valid = False
else:
if ip_addr1 > ip_addr2:
msg = (
"The address range %s specified in network %s "
"is invalid. The specified first address %s "
"is greater than the specified last address %s." %
(address_list, net['name'], address[0],
address[1]))
self.add_error(msg)
self._valid = False
else:
iprange = IPRange(ip_addr1, ip_addr2)
if iprange not in network:
msg = (
"Address range %s in network %s is not within "
"the specified CIDR %s." %
(address_list, net['name'], net['cidr']))
self.add_error(msg)
self._valid = False
else:
if not current_set.isdisjoint(IPSet(iprange)):
msg = (
"The address range %s in network %s overlaps "
"with another range or address in the network."
% (address_list, net['name']))
self.add_warning(msg)
current_set.add(iprange)
self._ipsets[net['name']] = current_set
def valid(self):
"""
Make sure the input parameters class stick to the form of below example
parameters_tuple(src_zone = 'untrust', src_ip = ['100.1.4.2/32', '100.1.2.0/24'],
dst_zone = 'trust', dst_ip = ['10.1.3.0/30'],
application = {'tcp': {'dst-port': ['80', '20'], 'src-port': ['any', '5-20']}}
)
"""
def ip_validate(ip):
ip_validator = re.compile('^((2[0-4]\d|25[0-5]|[01]?\d\d?)\.){3}(2[0-4]\d|25[0-5]|[01]?\d\d?)/(\d{1,2})$')
if ip_validator.match(ip) != None and int(ip_validator.match(ip).groups()[3]) <= 32:
return True
return False
def application_validate(application):
"""
Unfold the ports from 1-3 to [1,2,3], keep 'any' in it origin form
"""
try:
for tuple in application.values():
if not isinstance(tuple, dict) or not isinstance(tuple['src-port'], list) or not isinstance(
tuple['dst-port'], list):
return False
src_port_list = []
for port in tuple['src-port']:
if not port_validate(port):
return False
src_port_list.extend(port_cal(port))
tuple['src-port'] = src_port_list
dst_port_list = []
for port in tuple['dst-port']:
if not port_validate(port):
return False
dst_port_list.extend(port_cal(port))
tuple['dst-port'] = dst_port_list
except Exception as e:
logging.warn(str(application) + ' is not a valid application input, and the error is "' + str(e) + '"')
return False
else:
return True
def port_validate(port):
port_validator = re.compile(r'(\d+-\d+)|(\d*)|any')
if port_validator.match(port) is None:
return False
return True
src_ip = IPSet([])
for ip in self.src_ip:
if ip == 'any':
ip = '0.0.0.0/0'
if not ip_validate(ip):
return False
src_ip.add(ip)
self.src_ip = src_ip
dst_ip = IPSet([])
for ip in self.dst_ip:
if ip == 'any':
ip = '0.0.0.0/0'
if not ip_validate(ip):
return False
dst_ip.add(ip)
self.dst_ip = dst_ip
if not application_validate(self.application):
return False
return True
def _query_rules(self, resource):
rules = IPSet()
for r in resource['rules']:
rules.add(r)
return rules
help='Seconds to wait before timeout, default is 2')
argparser.add_argument('-s', '--shuffle',
action='store_true',
default=False,
dest='shuffle',
help='Shuffle the target list')
args = argparser.parse_args()
# Check if we are running in a pipe and read from STDIN
if not sys.stdin.isatty():
args.targets = sys.stdin.readlines()
# Add target IPs/Networks to a netaddr-IPSet
targetSet = IPSet()
for t in args.targets:
targetSet.add(t)
# Render IPSets to a list
targetlist = list()
for ip in targetSet:
targetlist.append(str(ip))
# Check for shuffle argument
if args.shuffle:
shuffle(targetlist)
# Split list into [maxThreads] smaller batches
targetlist = split_list(targetlist, args.maxThreads)
# Launch threads
for batch in targetlist:
def targets_to_ip_list(targets):
ipset = IPSet()
for t in targets:
ipset.add(t)
return [str(ip) for ip in ipset]
def parse_ip_set(ipaddrs):
"""Parse a string specification into an IPSet.
This function takes a string representing a set of IP addresses and
parses it into an IPSet object. Acceptable formats for the string
include:
* "all": all possible IPv4 and IPv6 addresses
* "local": all local addresses of the machine
* "A.B.C.D" a single IP address
* "A.B.C.D/N" a network address specification
* "A.B.C.*" a glob matching against all possible numbers
* "A.B.C.D-E" a glob matching against a range of numbers
* a whitespace- or comma-separated string of the above
"""
ipset = IPSet()
ipaddrs = ipaddrs.lower().strip()
if not ipaddrs:
return ipset
for ipspec in _COMMA_OR_WHITESPACE.split(ipaddrs):
# The string "local" maps to all local addresses on the machine.
if ipspec == "local":
ipset.add(IPNetwork("127.0.0.0/8"))
for addr in get_local_ip_addresses():
ipset.add(addr)
# The string "all" maps to app IPv4 and IPv6 addresses.
elif ipspec == "all":
ipset.add(IPNetwork("0.0.0.0/0"))
ipset.add(IPNetwork("::"))
# Strings containing a "/" are assumed to be network specs
elif "/" in ipspec:
ipset.add(IPNetwork(ipspec))
# Strings containing a "*" or "-" are assumed to be glob patterns
elif "*" in ipspec or "-" in ipspec:
for cidr in IPGlob(ipspec).cidrs():
ipset.add(cidr)
# Anything else must be a single address
else:
ipset.add(IPAddress(ipspec))
return ipset
