def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
# search for every IP
ipList = self._get_IPs(searchID)
for ip in ipList:
result = self._get_blacklisting(ip)
self._insert_into_DB(searchID, ip, result)
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
cursor = self._connection.cursor()
insertAge = 'INSERT INTO search_DomainAge (search_ID, age) VALUES (%s, %s)'
age = self._get_age(domain)
cursor.execute(insertAge, (searchID, age))
self._connection.commit()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
cursor = self._connection.cursor()
insertAge = 'INSERT INTO search_DomainAge (search_ID, age) VALUES (%s, %s)'
age = self._get_age(domain)
cursor.execute(insertAge, (searchID, age))
self._connection.commit()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
cursor = self._connection.cursor()
insert = 'INSERT INTO search_GoogleSafeBrowsingAPI (search_ID, state) VALUES (%s, %s)'
state = self._get_state(domain)
cursor.execute(insert, (searchID, state))
self._connection.commit()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
cursor = self._connection.cursor()
insert = 'INSERT INTO search_GoogleSafeBrowsingAPI (search_ID, state) VALUES (%s, %s)'
state = self._get_state(domain)
cursor.execute(insert, (searchID, state))
self._connection.commit()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
searchTypes = {'link:', 'site:', ''}
for addition in searchTypes:
results = self._get_google_results(domain, addition)
self._insert_into_DB(searchID, domain, addition, results)
result = self._get_pagerank(domain)
self._insert_into_DB(searchID, domain, 'pagerank:', result)
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
searchTypes = {'link:', 'site:', ''}
for addition in searchTypes:
results = self._get_google_results(domain, addition)
self._insert_into_DB(searchID, domain, addition, results)
result = self._get_pagerank(domain)
self._insert_into_DB(searchID, domain, 'pagerank:', result)
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
# search for domainname
result = self._get_geoIP(domain)
self._insert_into_DB(searchID, domain, result)
# search for all resolved IPs
ipList = self._get_IPs(searchID)
for ip in ipList:
result = self._get_geoIP(ip)
self._insert_into_DB(searchID, ip, result)
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
# search for domainname
result = self._get_geoIP(domain)
self._insert_into_DB(searchID, domain, result)
# search for all resolved IPs
ipList = self._get_IPs(searchID)
for ip in ipList:
result = self._get_geoIP(ip)
self._insert_into_DB(searchID, ip, result)
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
# search for SOA record with default nameserver
SOANameserver = self._find_nameserver(domain, dns_nameserver)
# lookup for ANY on SOA Nameserver
self._lookup(searchID, domain, SOANameserver)
# reverse lookup for all IPs on default nameserver
self._reverse_lookup(searchID, dns_nameserver)
self._insert_into_DB()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
# search for SOA record with default nameserver
SOANameserver = self._find_nameserver(domain, dns_nameserver)
# lookup for ANY on SOA Nameserver
self._lookup(searchID, domain, SOANameserver)
# reverse lookup for all IPs on default nameserver
self._reverse_lookup(searchID, dns_nameserver)
self._insert_into_DB()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
cursor = self._connection.cursor()
insert = 'INSERT INTO search_RobotsTxt (search_ID, file) VALUES (%s, %s)'
try:
content = self._get_robots(domain)
cursor.execute(insert, (searchID, content))
except socket.gaierror:
msg = 'No webserver on this domain'
cursor.execute(insert, (searchID, msg))
self._connection.commit()
raise RobotsError(msg)
except Exception as e:
cursor.execute(insert, (searchID, (str(e))))
self._connection.commit()
raise RobotsError(e)
self._connection.commit()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
cursor = self._connection.cursor()
insert = 'INSERT INTO search_RobotsTxt (search_ID, file) VALUES (%s, %s)'
try:
content = self._get_robots(domain)
cursor.execute(insert, (searchID, content))
except socket.gaierror:
msg = 'No webserver on this domain'
cursor.execute(insert, (searchID, msg))
self._connection.commit()
raise RobotsError(msg)
except Exception as e:
cursor.execute(insert, (searchID, (str(e))))
self._connection.commit()
raise RobotsError(e)
self._connection.commit()
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
result = self._get_blacklisting(domain)
self._insert_into_DB(searchID, result)
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'mxtoolbox.com'
self._debug = 0
self.query = '''SELECT * FROM search_blacklist WHERE search_ID = %s'''
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
self._get_nmap(searchID, domain)
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'sb-ssl.google.com'
self._debug = 0
self.query = '''SELECT * FROM search_GoogleSafeBrowsingAPI WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self.serverNameGoogle = 'www.google.com'
self.serverNamePagerank = 'www.prapi.net'
self._debug = 0
self.query = '''SELECT * FROM search_GoogleSearch WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self._debug = 0
self.query = '''SELECT * FROM search_RobotsTxt WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self._debug = 0
self.query = '''SELECT search_SpellChecker.ID, search_ID,
def __init__(self):
DatasourceBase.__init__(self)
self._debug = 0
self.query = '''SELECT * FROM search_Traceroute WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'archive.org'
self._debug = 0
self.query = '''SELECT * FROM search_DomainAge WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'archive.org'
self._debug = 0
self.query = '''SELECT * FROM search_DomainAge WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'freegeoip.net'
self._debug = 0
self.query = '''SELECT ID, search_ID,
def __init__(self):
DatasourceBase.__init__(self)
self.serverNameGoogle = 'www.google.com'
self.serverNamePagerank = 'www.prapi.net'
self._debug = 0
self.query = '''SELECT * FROM search_GoogleSearch WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'freegeoip.net'
self._debug = 0
self.query = '''SELECT ID, search_ID,
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
answer = self._query(domain)
self._print(1, answer)
self._insert_into_DB(searchID, answer)
def __init__(self):
DatasourceBase.__init__(self)
self._debug = 0
self.query = '''SELECT * FROM search_DNSresolver WHERE search_ID = %s ORDER BY search_DNSresolver.name'''
self._recursion = 0
self._completeSet = set()
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'sb-ssl.google.com'
self._debug = 0
self.query = '''SELECT * FROM search_GoogleSafeBrowsingAPI WHERE search_ID = %s'''
def do_search(self, searchID, domain):
DatasourceBase.do_search(self, searchID, domain)
numbers = self._calculate_numerical_chars(domain)
words = self._check_dict(domain)
self._insert_into_DB(searchID, domain, numbers, words)
def __init__(self):
DatasourceBase.__init__(self)
self._debug = 0
self.query = '''SELECT * FROM search_DNSresolver WHERE search_ID = %s ORDER BY search_DNSresolver.name'''
self._recursion = 0
self._completeSet = set()
def __init__(self):
DatasourceBase.__init__(self)
self._debug = 0
self.query = '''SELECT * FROM search_RobotsTxt WHERE search_ID = %s'''
def __init__(self):
DatasourceBase.__init__(self)
self.serverName = 'IPVoid.com'
self._debug = 0
self.query = '''SELECT * FROM search_blacklistIPVoid WHERE search_ID = %s'''