def __init__(self, api_key, eiq_instance="default"):
self.api_key = api_key
self.api_url = self.api_url.format(eiq_instance)
self.request_session = internet.retry_request_session(
headers={
'Authorization': 'Bearer {}'.format(api_key),
'Content-Type': 'application/json'
})
self.cache = APICache('eclecticiq')
def __init__(self):
self.request_session = internet.retry_request_session()
self.local_db_directory = "./iana"
self.local_db = "./iana/iana_service_names_port_numbers.sqlite"
self.cache = APICache('iana_service_names_port_numbers')
self.iana_csv = '{}/{}'.format(self.local_db_directory,
'service-names-port-numbers.csv')
self.last_lookup = 0
self.data = None
if not os.path.exists(self.local_db_directory):
os.mkdir(self.local_db_directory)
self.__check_for_updates()
class EclecticIQ:
api_url = '"https://{}.eiq-platform.com/private/search-all/"'
def __init__(self, api_key, eiq_instance="default"):
self.api_key = api_key
self.api_url = self.api_url.format(eiq_instance)
self.request_session = internet.retry_request_session(
headers={
'Authorization': 'Bearer {}'.format(api_key),
'Content-Type': 'application/json'
})
self.cache = APICache('eclecticiq')
def __lookup(self, query_payload):
try:
logging.debug(
'Performing EclecticIQ Lookup for: {}'.format(query_payload))
post_query = {"query": {"query_string": {"query": query_payload}}}
response = self.request_session.post(self.api_url, json=post_query)
if response.status_code != 200:
raise Exception(
'Status code {} encountered while looking up IP address. Error: {}'
.format(str(response.status_code), response.content))
return response.json()
except Exception as ex:
logging.exception(ex)
return {'error': str(ex)}
def ip_lookups(self):
return EclecticIQIPAddress(self.request_session, self.cache,
self.api_url, self.__lookup)
def domain_lookups(self):
return EclecticIQDomain(self.request_session, self.cache, self.api_url,
self.__lookup)
# def url_lookups(self):
# return VirusTotalURL(self.request_session, self.cache, self.api_url, self.__lookup)
def search(self, query):
query_payload = 'search?query={}'.format(query)
return self.cache.lookup(self.__lookup, query_payload=query_payload)
def eclecticiq_metadata(self):
query_payload = 'metadata'
return self.cache.lookup(self.__lookup, query_payload=query_payload)
class IPStack:
api_url = 'http://api.ipstack.com/{}?access_key={}&hostname=1&security=1'
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session(
headers={'Key': self.api_key})
self.cache = APICache('ipstack')
def __lookup_ip(self, ip_address):
try:
logging.debug(
'Performing IPStack Lookup for: {}'.format(ip_address))
url = self.api_url.format(ip_address, self.api_key)
response = self.request_session.get(url)
if response.status_code != 200:
raise Exception(
'Status code {} encountered while looking up IP address. Error: {}'
.format(str(response.status_code), response.content))
return response.json()
except Exception as ex:
logging.exception(ex)
return {'error': str(ex)}
def lookup(self, ip_address):
return self.cache.lookup(self.__lookup_ip, ip_address=ip_address)
class VirusTotal:
api_url = 'https://www.virustotal.com/api/v3/{}'
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session(
headers={'x-apikey': api_key})
self.cache = APICache('virus_total')
def __lookup(self, query_url):
try:
logging.debug(
'Performing VirusTotal Lookup for: {}'.format(query_url))
url = self.api_url.format(query_url)
response = self.request_session.get(url)
if response.status_code != 200:
raise Exception(
'Status code {} encountered while looking up IP address. Error: {}'
.format(str(response.status_code), response.content))
return response.json()
except Exception as ex:
logging.exception(ex)
return {'error': str(ex)}
def ip_lookups(self):
return VirusTotalIPAddress(self.request_session, self.cache,
self.api_url, self.__lookup)
def domain_lookups(self):
return VirusTotalDomain(self.request_session, self.cache, self.api_url,
self.__lookup)
# def url_lookups(self):
# return VirusTotalURL(self.request_session, self.cache, self.api_url, self.__lookup)
def search(self, query):
query_url = 'search?query={}'.format(query)
return self.cache.lookup(self.__lookup, query_url=query_url)
def virus_total_metadata(self):
query_url = 'metadata'
return self.cache.lookup(self.__lookup, query_url=query_url)
class IPAbuseDB:
categories = {'1': 'DNS Compromise', '3': 'Fraud Orders', '4': 'DDoS Attack', '5': 'FTP Brute-Force', '6': 'Ping of Death',
'7': 'Phishing', '8': 'Fraud VoIP', '9': 'Open Proxy', '10': 'Web Spam', '11': 'Email Spam',
'12': 'Blog Spam', '13': 'VPN IP', '14': 'Port Scan', '15': 'Hacking', '16': 'SQL Injection',
'17': 'Spoofing', '18': 'Brute-Force', '19': 'Bad Web Bot', '20': 'Exploited Host',
'21': 'Web App Attack', '22': 'SSH' , '23': 'IoT Targeted', '2': 'DNS Poisoning'}
api_url = 'https://api.abuseipdb.com/api/v2/check'
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session(headers={'Key': self.api_key})
self.cache = APICache('abuseipdb')
def __lookup_ip(self, ip_address, max_age_in_days):
try:
logging.debug('Performing AbuseIP Database Lookup for: {}'.format(ip_address))
url = '{}?ipAddress={}&maxAgeInDays={}&verbose=true'.format(self.api_url, ip_address, str(max_age_in_days))
response = self.request_session.get(url)
if response.status_code != 200:
raise Exception('Status code {} encountered while looking up IP address. Error: {}'
.format(str(response.status_code), response.content))
resp = response.json()
abuse_response = {}
if 'errors' not in resp:
abuse_response['ip_address'] = resp['data']['ipAddress']
abuse_response['abuse_total_reports'] = resp['data']['totalReports']
abuse_response['country_name'] = resp['data']['countryName']
abuse_response['usage_type'] = resp['data']['usageType']
abuse_response['isp'] = resp['data']['isp']
abuse_response['original_encoded_response'] = base64.b64encode(json.dumps(resp).encode('UTF-8')) \
.decode('UTF-8')
categories = []
if resp['data']['totalReports'] > 0:
abuse_response['abuse_lastReportedAt'] = resp['data']['lastReportedAt']
for report in resp['data']['reports']:
for cat in report['categories']:
try:
if self.categories[str(cat)] not in categories:
categories.append(self.categories[str(cat)])
except Exception as ex:
logging.exception(ex)
continue
abuse_response['abuse_categories'] = categories
return abuse_response
except Exception as ex:
logging.exception(ex)
return {'error': str(ex)}
def lookup(self, ip_address, max_age_in_days=90):
return self.cache.lookup(self.__lookup_ip, ip_address=ip_address, max_age_in_days=max_age_in_days)
class Shodan:
api_url = 'https://api.shodan.io/{}'
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session()
self.cache = APICache('shodan')
def __lookup(self, query_url):
try:
logging.debug('Performing Shodan Lookup for: {}'.format(query_url))
url = self.api_url.format(query_url) + 'key={}'.format(
self.api_key)
response = self.request_session.get(url)
if response.status_code != 200:
raise Exception(
'Status code {} encountered while looking up IP address. Error: {}'
.format(str(response.status_code), response.content))
try:
return response.json()
except Exception as ex:
logging.exception(response.text)
raise Exception(response.text)
except Exception as ex:
logging.exception(ex)
return {'error': str(ex)}
def lookup_ip(self, ip_address):
query_url = 'shodan/host/{}?'.format(ip_address)
return self.cache.lookup(self.__lookup, query_url=query_url)
def query(self, query):
query_url = 'shodan/host/search?query={}&'.format(query)
return self.cache.lookup(self.__lookup, query_url=query_url)
def api_info(self):
query_url = 'api-info?'
return self.cache.lookup(self.__lookup, query_url=query_url)
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session()
self.local_db_directory = "./maxmind"
self.local_db = "./maxmind/GeoLite2-ASN-Blocks.sqlite"
self.cache = APICache('max_mind')
self.asn_ipv4_db = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN-Blocks-IPv4.csv')
self.asn_ipv6_db = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN-Blocks-IPv6.csv')
self.asn_db_gzip = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN.zip')
self.asn_db_sha256 = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN.sha256')
self.last_lookup = 0
self.data = None
if not os.path.exists(self.local_db_directory):
os.mkdir(self.local_db_directory)
self.__check_for_updates()
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session(
headers={'Key': self.api_key})
self.cache = APICache('ipstack')
class IanaPortServiceNames:
api_url = 'https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv'
refresh_period = 86400000 # 24 hours
def __init__(self):
self.request_session = internet.retry_request_session()
self.local_db_directory = "./iana"
self.local_db = "./iana/iana_service_names_port_numbers.sqlite"
self.cache = APICache('iana_service_names_port_numbers')
self.iana_csv = '{}/{}'.format(self.local_db_directory,
'service-names-port-numbers.csv')
self.last_lookup = 0
self.data = None
if not os.path.exists(self.local_db_directory):
os.mkdir(self.local_db_directory)
self.__check_for_updates()
def __load_data(self, new_data=False):
if self.data is not None:
self.data.close()
if os.path.exists(self.local_db) and new_data:
os.remove(self.local_db)
self.data = _sqlite3.connect(self.local_db, check_same_thread=False)
if new_data:
create_cache_table = """ CREATE TABLE IF NOT EXISTS iana (
id integer PRIMARY KEY,
service text,
port text,
protocol text,
description text,
assignee text,
contact text,
registration_date text,
modification_date text,
reference text,
service_code text,
unauthorized_user_reported text,
notes text
); """
self.data.execute(create_cache_table)
with open(self.iana_csv, 'r') as db_file:
for line in csv.DictReader(db_file):
cursor = self.data.cursor()
cursor.execute(
"INSERT INTO iana(service, port, protocol, description, assignee, contact, registration_date, modification_date, reference, service_code, "
"unauthorized_user_reported, notes)"
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
(line['Service Name'], line['Port Number'],
line['Transport Protocol'], line['Description'],
line['Assignee'], line['Contact'],
line['Registration Date'], line['Modification Date'],
line['Reference'], line['Service Code'],
line['Unauthorized Use Reported'],
line['Assignment Notes']))
self.data.commit()
def __check_for_updates(self):
logging.info('Checking for iana updates')
if self.last_lookup == 0 or (date_utils.convert_to_epoch_mills(
datetime.now()) - self.last_lookup) >= self.refresh_period:
with open(self.iana_csv, 'wb') as iana_csv:
resp = self.request_session.get(self.api_url)
iana_csv.write(resp.content)
logging.info('Applying iana updates')
self.__load_data(new_data=True)
self.last_lookup = date_utils.convert_to_epoch_mills(
datetime.now())
def __build_from_row(self, row):
return {
'service': row[0],
'port': row[1],
'protocol': row[2],
'description': row[3],
'assignee': row[4],
'contact': row[5],
'registration_date': row[6],
'modification_date': row[7],
'reference': row[8],
'service_code': row[9],
'unauthorized_use_reported': row[10],
'notes': row[11]
}
def __lookup_port(self, port):
logging.debug('Performing iana Lookup for: {}'.format(port))
self.__check_for_updates()
cursor = self.data.cursor()
cursor.execute(
"SELECT service, port, protocol, description, assignee, contact, registration_date, modification_date, reference, service_code, "
"unauthorized_user_reported, notes FROM iana WHERE port=" + port)
response = []
for row in cursor.fetchall():
response.append(self.__build_from_row(row))
return response
def lookup_port(self, port):
port = str(port)
return self.cache.lookup(self.__lookup_port, port=port)
def __lookup_service(self, service):
logging.debug('Performing iana Lookup for: {}'.format(service))
self.__check_for_updates()
cursor = self.data.cursor()
cursor.execute(
"SELECT service, port, protocol, description, assignee, contact, registration_date, modification_date, reference, service_code, "
"unauthorized_user_reported, notes FROM iana WHERE service=" +
"'" + service + "'")
response = []
for row in cursor.fetchall():
response.append(self.__build_from_row(row))
return response
def lookup_service(self, service):
return self.cache.lookup(self.__lookup_service, service=service)
def __lookup_protocol(self, protocol):
logging.debug('Performing iana Lookup for: {}'.format(protocol))
self.__check_for_updates()
cursor = self.data.cursor()
cursor.execute(
"SELECT service, port, protocol, description, assignee, contact, registration_date, modification_date, reference, service_code, "
"unauthorized_user_reported, notes FROM iana WHERE protocol=" +
"'" + protocol + "'")
response = []
for row in cursor.fetchall():
response.append(self.__build_from_row(row))
return response
def lookup_protocol(self, protocol):
return self.cache.lookup(self.__lookup_protocol, protocol=protocol)
def __lookup_unauthorised_use(self):
logging.debug(
'Performing iana Lookup for: {}'.format('unauthorised_use'))
self.__check_for_updates()
cursor = self.data.cursor()
cursor.execute(
"SELECT service, port, protocol, description, assignee, contact, registration_date, modification_date, reference, service_code, "
"unauthorized_user_reported, notes FROM iana WHERE unauthorized_user_reported IS NOT ''"
)
response = []
for row in cursor.fetchall():
response.append(self.__build_from_row(row))
return response
def lookup_unauthorised_use(self):
return self.cache.lookup(self.__lookup_unauthorised_use)
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session(
headers={'x-apikey': api_key})
self.cache = APICache('virus_total')
class MaxMind:
api_url = 'https://download.maxmind.com/app/geoip_download?'
refresh_period = 86400000 # 24 hours
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session()
self.local_db_directory = "./maxmind"
self.local_db = "./maxmind/GeoLite2-ASN-Blocks.sqlite"
self.cache = APICache('max_mind')
self.asn_ipv4_db = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN-Blocks-IPv4.csv')
self.asn_ipv6_db = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN-Blocks-IPv6.csv')
self.asn_db_gzip = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN.zip')
self.asn_db_sha256 = '{}/{}'.format(self.local_db_directory,
'GeoLite2-ASN.sha256')
self.last_lookup = 0
self.data = None
if not os.path.exists(self.local_db_directory):
os.mkdir(self.local_db_directory)
self.__check_for_updates()
def __load_data(self, new_data=False):
if self.data is not None:
self.data.close()
if os.path.exists(self.local_db) and new_data:
os.remove(self.local_db)
self.data = _sqlite3.connect(self.local_db, check_same_thread=False)
if new_data:
create_cache_table = """ CREATE TABLE IF NOT EXISTS GeoLite2ASN (
id integer PRIMARY KEY,
network text,
autonomous_system_number text ,
autonomous_system_organization text,
version integer
); """
self.data.execute(create_cache_table)
with open(self.asn_ipv4_db, 'r') as db_file:
for line in csv.DictReader(db_file):
cursor = self.data.cursor()
cursor.execute(
"INSERT INTO GeoLite2ASN(network, autonomous_system_number, autonomous_system_organization, version)"
" VALUES(?,?,?,?)",
(line['network'], line['autonomous_system_number'],
line['autonomous_system_organization'], 4))
with open(self.asn_ipv6_db, 'r') as db_file:
for line in csv.DictReader(db_file):
cursor = self.data.cursor()
cursor.execute(
"INSERT INTO GeoLite2ASN(network, autonomous_system_number, autonomous_system_organization, version)"
" VALUES(?,?,?,?)",
(line['network'], line['autonomous_system_number'],
line['autonomous_system_organization'], 6))
self.data.commit()
def __check_for_updates(self):
logging.info('Checking for maxmind updates')
if self.last_lookup == 0 or (date_utils.convert_to_epoch_mills(
datetime.now()) - self.last_lookup) >= self.refresh_period:
resp = self.request_session.get(
self.api_url +
'edition_id=GeoLite2-ASN-CSV&license_key={}&suffix=zip.sha256'.
format(self.api_key))
sha256 = ""
if os.path.exists(self.asn_db_sha256):
with open(self.asn_db_sha256, 'r') as sha256_file:
sha256 = sha256_file.readline()
if sha256 != resp.text:
logging.info('Applying maxmind updates')
with open(self.asn_db_sha256, 'w') as sha256_file:
sha256_file.write(resp.text)
with open(self.asn_db_gzip, 'wb') as asn_db_gzip:
resp = self.request_session.get(
self.api_url +
'edition_id=GeoLite2-ASN-CSV&license_key={}&suffix=zip'
.format(self.api_key))
asn_db_gzip.write(resp.content)
with zipfile.ZipFile(self.asn_db_gzip) as zip_obj:
zip_obj.extractall(self.local_db_directory + '/temp')
for template_file in glob.glob(self.local_db_directory +
'/temp/**/*.csv'):
if "GeoLite2-ASN-Blocks-IPv4" in template_file:
if os.path.exists(self.asn_ipv4_db):
os.remove(self.asn_ipv4_db)
copyfile(template_file, self.asn_ipv4_db)
if "GeoLite2-ASN-Blocks-IPv6" in template_file:
if os.path.exists(self.asn_ipv6_db):
os.remove(self.asn_ipv6_db)
copyfile(template_file, self.asn_ipv6_db)
shutil.rmtree(self.local_db_directory + '/temp')
os.remove(self.asn_db_gzip)
self.__load_data(new_data=True)
self.last_lookup = date_utils.convert_to_epoch_mills(
datetime.now())
if self.data is None:
self.__load_data()
def __lookup_asn(self, asn):
logging.debug('Performing maxmind Lookup for: {}'.format(asn))
self.__check_for_updates()
cursor = self.data.cursor()
cursor.execute(
"SELECT network, autonomous_system_number, autonomous_system_organization, version FROM GeoLite2ASN WHERE autonomous_system_number="
+ str(asn))
response = []
for row in cursor.fetchall():
response.append({
'network': row[0],
'autonomous_system_number': row[1],
'autonomous_system_organization': row[2],
'version': row[3]
})
return response
def lookup_asn(self, asn):
return self.cache.lookup(self.__lookup_asn, asn=asn)
def __lookup_org(self, org):
logging.debug('Performing maxmind Lookup for: {}'.format(org))
self.__check_for_updates()
cursor = self.data.cursor()
cursor.execute(
"SELECT network, autonomous_system_number, autonomous_system_organization, version FROM GeoLite2ASN WHERE autonomous_system_organization="
+ "'" + org + "'")
response = []
for row in cursor.fetchall():
response.append({
'network': row[0],
'autonomous_system_number': row[1],
'autonomous_system_organization': row[2],
'version': row[3]
})
return response
def lookup_org(self, org):
return self.cache.lookup(self.__lookup_org, org=org)
def __lookup_ip(self, ip_address):
logging.debug('Performing maxmind Lookup for: {}'.format(ip_address))
self.__check_for_updates()
cursor = self.data.cursor()
cursor.execute(
"SELECT network, autonomous_system_number, autonomous_system_organization, version FROM GeoLite2ASN"
)
for row in cursor.fetchall():
if ipaddress.ip_address(ip_address) in ipaddress.ip_network(
row[0]):
return {
'network': row[0],
'autonomous_system_number': row[1],
'autonomous_system_organization': row[2],
'version': row[3]
}
return None
def lookup_ip(self, ip_address):
return self.cache.lookup(self.__lookup_ip, ip_address=ip_address)
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session()
self.cache = APICache('view_dns')
class ViewDNS:
api_url = 'https://api.viewdns.info/{}'
def __init__(self, api_key):
self.api_key = api_key
self.request_session = internet.retry_request_session()
self.cache = APICache('view_dns')
def __lookup(self, query_url):
try:
logging.debug(
'Performing ViewDNS Lookup for: {}'.format(query_url))
url = self.api_url.format(
query_url) + '&apikey={}&output=json'.format(self.api_key)
response = self.request_session.get(url)
if response.status_code != 200:
raise Exception(
'Status code {} encountered while looking up IP address. Error: {}'
.format(str(response.status_code), response.content))
try:
return response.json()
except Exception as ex:
logging.exception(response.text)
raise Exception(response.text)
except Exception as ex:
logging.exception(ex)
return {'error': str(ex)}
def reverseip(self, ip_address):
query_url = 'reverseip/?host={}'.format(ip_address)
return self.cache.lookup(self.__lookup, query_url=query_url)
def whois(self, value):
query_url = 'whois/?domain={}'.format(value)
return self.cache.lookup(self.__lookup, query_url=query_url)
def dnsrecord(self, domain, record_type):
query_url = 'dnsrecord/?domain={}&recordtype={}'.format(
domain, record_type)
return self.cache.lookup(self.__lookup, query_url=query_url)
def maclookup(self, mac_address):
query_url = 'maclookup/?mac={}'.format(mac_address)
return self.cache.lookup(self.__lookup, query_url=query_url)
def reversedns(self, ip_address):
query_url = 'reversedns/?ip={}'.format(ip_address)
return self.cache.lookup(self.__lookup, query_url=query_url)
def reversemx(self, mail_server):
query_url = 'reversemx/?mx={}'.format(mail_server)
return self.cache.lookup(self.__lookup, query_url=query_url)
def reversens(self, name_server):
query_url = 'reversens/?ns={}'.format(name_server)
return self.cache.lookup(self.__lookup, query_url=query_url)
def reversewhois(self, query):
query_url = 'reversewhois/?q={}'.format(query)
return self.cache.lookup(self.__lookup, query_url=query_url)