def test_unique_addresses(self):
input_data = (
'You can have IPs like 74.125.225.229, or 2001:4860:4860::8888'
'Put a port on the end 74.125.225.229:80 or for IPv6: '
'[2001:4860:4860::8888]:443 or even networks like '
'74.125.0.0/16 and 2001:4860::/32.')
expected_result = {
'74.125.225.229': {
'count': 2,
'ports': {
'80': 1
}
},
'2001:4860::/32': {
'count': 1,
'ports': {}
},
'74.125.0.0/16': {
'count': 1,
'ports': {}
},
'2001:4860:4860::8888': {
'count': 2,
'ports': {
'443': 1
}
}
}
self.assertEquals(unique_addresses(input_data), expected_result)
def test_unique_addresses(self):
input_data = (
'You can have IPs like 74.125.225.229, or 2001:4860:4860::8888'
'Put a port on the end 74.125.225.229:80 or for IPv6: '
'[2001:4860:4860::8888]:443 or even networks like '
'74.125.0.0/16 and 2001:4860::/32.'
)
expected_result = {
'74.125.225.229': {'count': 2, 'ports': {'80': 1}},
'2001:4860::/32': {'count': 1, 'ports': {}},
'74.125.0.0/16': {'count': 1, 'ports': {}},
'2001:4860:4860::8888': {'count': 2, 'ports': {'443': 1}}
}
self.assertEquals(unique_addresses(input_data), expected_result)
def test_unique_addresses(self):
self.assertRaises(ValueError, unique_addresses)
input_data = (
'You can have IPs like 74.125.225.229, or 2001:4860:4860::8888'
'Put a port on the end 74.125.225.229:80 or for IPv6: '
'[2001:4860:4860::8888]:443 or even networks like '
'74.125.0.0/16 and 2001:4860::/32.'
)
expected_result = {
'74.125.225.229': {'count': 2, 'ports': {'80': 1}},
'2001:4860::/32': {'count': 1, 'ports': {}},
'74.125.0.0/16': {'count': 1, 'ports': {}},
'2001:4860:4860::8888': {'count': 2, 'ports': {'443': 1}}
}
self.assertEquals(unique_addresses(input_data), expected_result)
data_dir = path.dirname(__file__)
fp = str(data_dir) + '/rdap.json'
# Expected result is different on 2.x vs 3.x, possible issues with
# ipaddr vs ipaddress output. Investigation pending...
if sys.version_info >= (3, 3):
fp_expected_result = {
'74.125.225.0/24': {'count': 1, 'ports': {}},
'62.239.0.0/16': {'count': 1, 'ports': {}},
'2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff':
{'count': 1, 'ports': {}},
'210.0.0.0': {'count': 1, 'ports': {}},
'196.11.240.0/23': {'count': 1, 'ports': {}},
'2001:240:10c:1::ca20:9d1d': {'count': 2, 'ports': {}},
'196.11.240.215': {'count': 2, 'ports': {}},
'62.239.237.0/32': {'count': 1, 'ports': {}},
'210.107.0.0/17': {'count': 6, 'ports': {}},
'2001:4860::/32': {'count': 1, 'ports': {}},
'210.107.73.73': {'count': 2, 'ports': {}},
'210.107.0.0': {'count': 2, 'ports': {}},
'2001:200::/23': {'count': 2, 'ports': {}},
'2001:240:ffff:ffff:ffff:ffff:ffff:ffff':
{'count': 1, 'ports': {}},
'210.255.255.255': {'count': 1, 'ports': {}},
'2001:43f8:7b0::': {'count': 3, 'ports': {}},
'196.255.255.255': {'count': 1, 'ports': {}},
'2001:240::/32': {'count': 6, 'ports': {}},
'196.0.0.0': {'count': 1, 'ports': {}},
'2001:240::': {'count': 1, 'ports': {}},
'196.11.246.255': {'count': 2, 'ports': {}},
'196.11.239.0': {'count': 2, 'ports': {}},
'2001:4200::/23': {'count': 1, 'ports': {}},
'2a00:2380::/25': {'count': 1, 'ports': {}},
'200.57.128.0/20': {'count': 1, 'ports': {}},
'62.239.237.255': {'count': 1, 'ports': {}},
'2001:4860:4860::8888': {'count': 10, 'ports': {}},
'2001:4860::': {'count': 2, 'ports': {}},
'2001:4860:ffff:ffff:ffff:ffff:ffff:ffff':
{'count': 1, 'ports': {}},
'74.125.225.229': {'count': 8, 'ports': {}},
'210.107.127.255': {'count': 2, 'ports': {}},
'200.57.141.161': {'count': 7, 'ports': {}},
'62.239.237.255/32': {'count': 1, 'ports': {}},
'2801:10:c000::': {'count': 7, 'ports': {}},
'2a00:2381:ffff::1': {'count': 4, 'ports': {}},
'62.239.237.0': {'count': 1, 'ports': {}},
'62.239.237.1': {'count': 4, 'ports': {}},
'210.0.0.0/8': {'count': 1, 'ports': {}}
}
self.assertEquals(unique_addresses(file_path=fp),
fp_expected_result)
else:
fp_expected_result = {
'196.11.239.0': {'count': 2, 'ports': {}},
'2a00:2380::/25': {'count': 1, 'ports': {}},
'2a00:2381:ffff::/6': {'count': 1, 'ports': {}},
'2001:4860:4860::8888': {'count': 10, 'ports': {}},
'200.57.128.0/20': {'count': 1, 'ports': {}},
'2001:4860::/32': {'count': 1, 'ports': {}},
'210.107.0.0': {'count': 2, 'ports': {}},
'2001:4200::/23': {'count': 1, 'ports': {}},
'2001:43f8:7b0::/4': {'count': 2, 'ports': {}},
'196.11.240.0/23': {'count': 1, 'ports': {}},
'210.107.73.73': {'count': 2, 'ports': {}},
'2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': {
'count': 1, 'ports': {}},
'210.0.0.0/8': {'count': 1, 'ports': {}},
'2a00:2381:ffff:0:ffff:ffff:ffff:ffff/12': {
'count': 1, 'ports': {}},
'210.107.127.255': {'count': 2, 'ports': {}},
'2a00:2381:ffff::1': {'count': 4, 'ports': {}},
'210.107.0.0/17': {'count': 6, 'ports': {}},
'2a00:2381:ffff::/12': {'count': 1, 'ports': {}},
'2001:240::/32': {'count': 6, 'ports': {}},
'62.239.0.0/16': {'count': 1, 'ports': {}},
'2801:10:c000::': {'count': 7, 'ports': {}},
'2001:43f8:7b0::': {'count': 3, 'ports': {}},
'62.239.237.0': {'count': 1, 'ports': {}},
'62.239.237.1': {'count': 4, 'ports': {}},
'196.11.246.255': {'count': 2, 'ports': {}},
'74.125.225.229': {'count': 8, 'ports': {}},
'196.255.255.255': {'count': 1, 'ports': {}},
'210.0.0.0': {'count': 1, 'ports': {}},
'200.57.141.161': {'count': 7, 'ports': {}},
'210.255.255.255': {'count': 1, 'ports': {}},
'2001:4860::': {'count': 2, 'ports': {}},
'62.239.237.255/32': {'count': 1, 'ports': {}},
'196.0.0.0': {'count': 1, 'ports': {}},
'2001:240:10c:1::ca20:9d1d': {'count': 2, 'ports': {}},
'2001:240::': {'count': 1, 'ports': {}},
'74.125.225.0/24': {'count': 1, 'ports': {}},
'196.11.240.215': {'count': 2, 'ports': {}},
'62.239.237.255': {'count': 1, 'ports': {}},
'2001:200::/23': {'count': 2, 'ports': {}},
'62.239.237.0/32': {'count': 1, 'ports': {}},
'2001:240:ffff:ffff:ffff:ffff:ffff:ffff': {
'count': 1, 'ports': {}},
'2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': {
'count': 1, 'ports': {}}}
self.assertEqual(unique_addresses(file_path=fp),
fp_expected_result)
result = list(unique_everseen(iterable=script_args.unique_everseen[0]))
print(('{0}Unique everseen{1}:\n{2}'.format(
ANSI['green'] if script_args.colorize else '',
ANSI['end'] if script_args.colorize else '', result)))
except Exception as e:
print(('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))))
elif script_args.unique_addresses:
try:
result = unique_addresses(file_path=script_args.unique_addresses[0])
tmp = []
for k, v in sorted(list(result.items()),
key=lambda kv: int(kv[1]['count']),
reverse=True):
tmp.append('{0}{1}{2}: Count: {3}, Ports: {4}'.format(
ANSI['b'] if script_args.colorize else '', k,
ANSI['end'] if script_args.colorize else '', v['count'],
json.dumps(v['ports'])))
print(('{0}Found {1} unique addresses{2}:\n{3}'.format(
ANSI['green'] if script_args.colorize else '', len(result),
ANSI['end'] if script_args.colorize else '', '\n'.join(tmp))))
except Exception as e:
print('{0}Unique everseen{1}:\n{2}'.format(
ANSI['green'] if script_args.colorize else '',
ANSI['end'] if script_args.colorize else '',
result
))
except Exception as e:
print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e)))
elif script_args.unique_addresses:
try:
result = unique_addresses(file_path=script_args.unique_addresses[0])
tmp = []
for k, v in sorted(result.items(), key=lambda kv: int(kv[1]['count']),
reverse=True):
tmp.append('{0}{1}{2}: Count: {3}, Ports: {4}'.format(
ANSI['b'] if script_args.colorize else '',
k,
ANSI['end'] if script_args.colorize else '',
v['count'],
json.dumps(v['ports'])
))
print('{0}Found {1} unique addresses{2}:\n{3}'.format(
ANSI['green'] if script_args.colorize else '',
len(result),
r=requests.get("https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt")
recv_data=r.text
else:
if (self.url != "default" and self.url !=None):
#print "URL provided"
r=requests.get(self.url)
recv_data=r.text
# check if data is recevied from the url listing the bad IPs and query them to 'whois database' if whois=true.
''' This is where the enrichment process takes place'''
if(recv_data !=None):
ips=unique_addresses(data=recv_data,file_path=None)
if self.whois == True:
dict_yield= self.add_kvstore(ips)
for i in range(len(dict_yield)-1):
yield {'sourcetype': "emerging_newthreats",'KVStore':self.KV_Store,'lookup_name': 'emergingthreats' , '_time': time.time(),'_raw':dict_yield[i] ,'event_no': i, 'ASN-Registry': dict_yield[i]['asn_registry'], 'Search Query': dict_yield[i]['query'],'asn_country_code': dict_yield[i]['asn_country_code'], 'asn_cidr': dict_yield[i]['asn_cidr'], 'asn_date': dict_yield[i]['asn_date'],'nets_address':dict_yield[i]['nets'][0]['address'],'nets_cidr':dict_yield[i]['nets'][0]['cidr'],'nets_city':dict_yield[i]['nets'][0]['city'],'nets_country':dict_yield[i]['nets'][0]['country'],'nets_created':dict_yield[i]['nets'][0]['created'],'nets_emails':dict_yield[i]['nets'][0]['emails'],'nets_description':dict_yield[i]['nets'][0]['description'],'nets_handle':dict_yield[i]['nets'][0]['handle'],'nets_name':dict_yield[i]['nets'][0]['name'],'nets_postal_code':dict_yield[i]['nets'][0]['postal_code'],'nets_range':dict_yield[i]['nets'][0]['range'],'nets_state':dict_yield[i]['nets'][0]['state'],'nets_updated':dict_yield[i]['nets'][0]['updated']}
i=i+1
#if whois=false , generate the list of Bad IPs as events in the Splunk Indexer
i=0
if (self.whois == False):
