def sending(self, url):
r = sender.send_get(self.options, url, cookies=None)
results = []
if r:
response = r.text
if 'class="errormsg"' in response:
utils.print_bad("No entry found for: {0}".format(self.query))
return
soup = utils.soup(response)
result = self.analyze(soup)
if not result:
return False
# checking if we have more than one pages
pages = self.check_pages(soup)
if pages:
utils.print_info("Detect pages {0} for query".format(str(len(pages))))
for page in pages:
page_url = self.baseURL + page
r1 = sender.send_get(self.options, page_url, cookies=None)
if r1:
response1 = r1.text
soup1 = utils.soup(response1)
self.analyze(soup1)
results.extend(result)
return results
def optimize(self, query):
url = 'https://www.shodan.io/search/_summary?query={0}'.format(query)
utils.print_good("Analyze first page for more result")
r = sender.send_get(self.options, url, self.cookies)
if r.status_code == 200:
soup = utils.soup(r.text)
else:
return False
query_by_cities = []
# check if query have country filter or not
if 'country' in query:
links = soup.find_all("a")
country = utils.get_country_code(utils.url_decode(query))
for link in links:
if 'city' in link.get('href'):
item = {
'url': link.get('href'),
'city': link.text,
'country': country
}
utils.print_debug(self.options, item)
query_by_cities.append(item)
else:
links = soup.find_all("a")
countries = []
for link in links:
if 'country' in link.get('href'):
countries.append(utils.get_country_code(utils.url_decode(link.get('href'))))
utils.print_debug(self.options, countries)
for country in countries:
# seding request again to get city
country_query = utils.url_encode(' country:"{0}"'.format(country))
url = 'https://www.shodan.io/search/_summary?query={0}{1}'.format(
query, country_query)
r1 = sender.send_get(self.options, url, self.cookies)
utils.random_sleep(5, 8)
utils.print_info(
"Sleep for couple seconds because Shodan server is really strict")
if r1.status_code == 200:
soup1 = utils.soup(r1.text)
links = soup1.find_all("a")
# countries = []
for link in links:
if 'city' in link.get('href'):
# countries.append(utils.get_city_name(
# utils.url_decode(link.get('href'))))
item = {
'url': link.get('href'),
'city': link.text,
'country': country
}
utils.print_debug(self.options, item)
query_by_cities.append(item)
utils.print_debug(self.options, query_by_cities)
return query_by_cities
def do_login(self):
utils.print_info("Reauthen using credentials from: {0}".format(
self.options.get('config')))
login_url = 'https://censys.io/login'
r = sender.send_get(self.options, login_url, cookies=None)
if r.status_code == 200:
cookies = r.cookies
form = utils.soup(r.text).find_all("form")
if form:
inputs = form[0].findChildren('input')
for tag in inputs:
if tag.get('name') == 'csrf_token':
csrf_token = tag.get('value')
username, password = utils.get_cred(self.options, source='censys')
data = {"csrf_token": csrf_token, "came_from": "/",
"from_censys_owned_external": "False", "login": username, "password": password}
really_login_url = 'https://censys.io/login'
r1 = sender.send_post(
self.options, really_login_url, cookies, data, follow=False)
if r1.status_code == 302:
for item in r1.cookies.items():
if item[1]:
censys_cookies = item[1]
utils.set_session(
self.options, censys_cookies, source='censys')
return censys_cookies
return False
def do_login(self):
utils.print_info("Reauthen using credentials from: {0}".format(
self.options.get('config')))
login_url = 'https://account.shodan.io/login'
r = sender.send_get(self.options, login_url, cookies=None)
if r.status_code == 200:
cookies = r.cookies
form = utils.soup(r.text).find_all("form")
if form:
inputs = form[0].findChildren('input')
for tag in inputs:
if tag.get('name') == 'csrf_token':
csrf_token = tag.get('value')
username, password = utils.get_cred(self.options, source='shodan')
data = {"username": username, "password": password, "grant_type": "password",
"continue": "https://www.shodan.io/", "csrf_token": csrf_token, "login_submit": "Login"}
really_login_url = 'https://account.shodan.io/login'
r1 = sender.send_post(
self.options, really_login_url, cookies, data, follow=False)
if r1.status_code == 302:
for item in r1.cookies.items():
if item.get('polito'):
shodan_cookies = item.get('polito')
utils.set_session(
self.options, shodan_cookies, source='shodan')
return shodan_cookies
return False
def github(self, url):
result = []
r = sender.send_get(self.options, url, cookies=None)
if r.status_code == 200:
response = r.text
# store raw json
raw_file_path = self.options[
'raw'] + '/write_up_github_{0}.html'.format(
self.query.replace(' ', '_'))
if self.options.get('store_content'):
utils.just_write(raw_file_path, response)
utils.print_debug(
self.options,
"Writing raw response to: {0}".format(raw_file_path))
soup = utils.soup(response)
# Custom here
body = soup.find_all('article', 'markdown-body')[0]
links = body.findChildren('a')
for link in links:
if self.query.lower() in link.text.lower():
item = {
'Query': self.query,
'Title': link.text,
'Content': link.text,
'External_url': link.get('href'),
'Source': url,
'Warning': 'Write-Up',
'Raw': raw_file_path
}
utils.print_debug(self.options, item)
result.append(item)
return result
def sending(self, url):
# sending request and return the response
utils.print_debug(self.options, url)
# catch error when session timeout
try:
r = sender.send_get(self.options, url, self.cookies)
except:
r = False
if r:
response = r.text
if self.options['store_content']:
ts = str(int(time.time()))
raw_file = self.options['raw'] + \
"/fofa/{0}_{1}".format(utils.url_encode(
url.replace(self.base_url, '')).replace('/', '_'), ts)
utils.just_write(raw_file, response)
soup = utils.soup(response)
self.analyze(soup)
# checking if there is many pages or not
page_num = self.check_pages(soup)
# if you're log in and have many results
if page_num and self.logged_in and not self.options[
'disable_pages']:
utils.print_info("Continue grab more pages")
self.pages(page_num)
def optimize(self, json_response):
analytics = json_response.get('aggs')
if not analytics:
return False
# get analytics respoonse
url = 'https://www.zoomeye.org/aggs/{0}'.format(analytics)
r = sender.send_get(self.options, url, headers=self.headers)
if r.status_code == 200:
analytics_json = utils.get_json(r.text)
else:
return False
analytics_countries = analytics_json.get('country')
raw_query = self.options['zoomeye_query']
clean_query = self.options['zoomeye_query']
if 'country' in raw_query:
country_code = utils.get_country_code(utils.url_decode(raw_query))
# clean country and subdivisions if it exist
clean_query = raw_query.replace(' +country:', '').replace(
'"{0}"'.format(str(country_code)), '')
for country_item in analytics_countries:
utils.print_info(
"Optimize query by filter with coutry: {0}".format(
country_item.get('name')))
# loop through city
for city in country_item.get('subdivisions'):
if 'country' in raw_query:
real_query = raw_query + ' +subdivisions:"{0}"'.format(
city.get('name'))
else:
real_query = clean_query + \
' +country:"{0}"'.format(country_item.get('name')) + \
' +subdivisions:"{0}"'.format(city.get('name'))
query = utils.url_encode(real_query)
url = 'https://www.zoomeye.org/search?q={0}&t=host'.format(
query)
r = sender.send_get(self.options, url, headers=self.headers)
if r and r.status_code == 200:
json_response = utils.get_json(r.text)
self.analyze(json_response)
def asnlookup(self, company):
utils.print_banner(f"Starting scraping {company} from asnlookup.com")
url = f'http://asnlookup.com/api/lookup?org={company}'
r = sender.send_get(self.options, url, None)
data = r.json()
if not data:
utils.print_bad('No IP found')
else:
content = "\n".join(data)
print(content)
utils.just_write(self.options['output'], content)
utils.just_cleanup(self.options['output'])
def tweet(self, tag):
results = []
query = '#{0} #{1}'.format(self.query, tag)
# @TODO improve by increase the the position
url = 'https://twitter.com/search?vertical=default&q={0}&src=unkn'.format(
utils.url_encode(query))
r = sender.send_get(self.options, url, cookies=None)
if r.status_code == 200:
response = r.text
# store raw json
raw_file_path = self.options['raw'] + '/tweets_{1}_{0}.html'.format(
self.query.replace(' ', '_'), tag)
if self.options.get('store_content'):
utils.just_write(raw_file_path, response)
utils.print_debug(
self.options,
"Writing raw response to: {0}".format(raw_file_path))
soup = utils.soup(response)
# Custom here
divs = soup.find_all('div', 'original-tweet')
for div in divs:
content = div.findChildren('p',
'TweetTextSize')[0].text.strip()
links = [
x.get('data-expanded-url') for x in div.findChildren('a')
if 't.co' in x.get('href')
]
# print(links)
if len(links) == 0:
external_url = 'N/A'
else:
external_url = '|'.join([str(x) for x in links])
item = {
'Query': self.query,
'Title': query,
'Content': content,
'External_url': external_url,
'Source': url,
'Warning': 'Tweet',
'Raw': raw_file_path
}
utils.print_debug(self.options, item)
results.append(item)
return results
def get_num_pages(self, url):
summary_url = 'https://www.shodan.io/search/_summary?{0}'.format(
utils.get_query(url))
r = sender.send_get(self.options, summary_url, self.cookies)
if r.status_code == 200:
soup = utils.soup(r.text)
results_total = soup.find_all('div', 'bignumber')[
0].text.replace(',', '')
page_num = str(int(int(results_total) / 10))
utils.print_good("Detect posible {0} pages per {1} result".format(
page_num, results_total))
return page_num
return False
def get_asn(self):
ip_target = utils.resolve_input(self.target)
if not ip_target:
return False
utils.print_banner(f"Starting scraping detail ASN of {ip_target}")
utils.print_info(f'Get ASN from IP: {ip_target}')
url = f'https://ipinfo.io/{ip_target}/json'
r = sender.send_get(self.options, url, None)
org_info = r.json().get('org')
asn = utils.get_asn(org_info)
if asn:
utils.print_info(f"Detect target running on {asn}")
ips = self.get_asn_ip(asn)
utils.just_write(self.options['output'], "\n".join(ips))
utils.just_cleanup(self.options['output'])
else:
return False
def sending(self, url):
# sending request and return the response
r = sender.send_get(self.options, url, self.cookies)
if r:
response = r.text
if self.options['store_content']:
ts = str(int(time.time()))
raw_file = self.options['raw'] + \
"/shodan/{0}_{1}".format(utils.url_encode(
url.replace(self.base_url, '')).replace('/', '_'), ts)
utils.just_write(raw_file, response)
soup = utils.soup(response)
self.analyze(soup)
# checking if there is many pages or not
if self.logged_in and not self.options['disable_pages']:
utils.print_info("Continue grab more pages")
self.pages(self.get_num_pages(url))
def pages(self, page_num):
for i in range(2, int(page_num) + 1):
utils.print_info("Get more result from page: {0}".format(str(i)))
utils.random_sleep(1, 2)
query = utils.url_encode(self.options['censys_query'])
url = 'https://censys.io/ipv4/_search?q={1}&page={0}'.format(
str(i), query)
r = sender.send_get(self.options, url, self.cookies)
if r.status_code == 200:
response = r.text
if 'class="alert alert-danger"' in response:
utils.print_bad("Reach to the limit at page {0}".format(
str(i)))
return
else:
soup = utils.soup(response)
self.analyze(soup)
def do_login(self):
utils.print_info("Reauthen using credentials from: {0}".format(
self.options.get('config')))
login_url = 'https://i.nosec.org/login?service=http%3A%2F%2Ffofa.so%2Fusers%2Fservice'
r = sender.send_get(self.options, login_url, cookies=None)
if r.status_code == 200:
cookies = r.cookies
form = utils.soup(r.text).find(id="login-form")
inputs = form.findChildren('input')
for tag in inputs:
if tag.get('name') == 'authenticity_token':
authenticity_token = tag.get('value')
if tag.get('name') == 'lt':
lt = tag.get('value')
if tag.get('name') == 'authenticity_token':
authenticity_token = tag.get('value')
username, password = utils.get_cred(self.options, source='fofa')
data = {
"utf8": "\xe2\x9c\x93",
"authenticity_token": authenticity_token,
"lt": lt,
"service": "http://fofa.so/users/service",
"username": username,
"password": password,
"rememberMe": "1",
"button": ''
}
really_login_url = 'https://i.nosec.org/login'
r1 = sender.send_post(self.options, really_login_url, cookies,
data)
if r1.status_code == 200:
fofa_cookie = r1.cookies.get('_fofapro_ars_session')
utils.set_session(self.options, fofa_cookie, source='fofa')
return fofa_cookie
return False
def check_session(self):
utils.print_debug(self.options, "Checking session for Censys")
sess_url = 'https://censys.io/account'
r = sender.send_get(self.options, sess_url, self.cookies)
if r.status_code == 302 or '/login' in r.text:
utils.print_bad("Look like Censys session is invalid.")
new_cookie = self.do_login()
if new_cookie:
utils.print_good("Reauthen success")
self.cookies = {"auth_tkt": new_cookie}
return True
return False
elif r.status_code == 200:
utils.print_good("Getting result as authenticated user")
return True
return False
def pages(self, page_num):
for i in range(2, int(page_num) + 1):
utils.print_info("Get more result from page: {0}".format(str(i)))
query = utils.url_encode(
utils.just_b64_encode(self.options['fofa_query']))
url = 'https://fofa.so/result?page={0}&qbase64={1}'.format(
str(i), query)
utils.print_debug(self.options, url)
r = sender.send_get(self.options, url, self.cookies)
if r.status_code == 200:
response = r.text
if 'class="error"' in response:
utils.print_bad("Reach to the limit at page {0}".format(
str(i)))
return
else:
soup = utils.soup(response)
self.analyze(soup)
def pages(self, page_num):
for i in range(2, int(page_num) + 1):
utils.random_sleep(1, 2)
utils.print_info("Get more result from page: {0}".format(str(i)))
query = utils.url_encode(self.options['zoomeye_query'])
url = 'https://www.zoomeye.org/search?q={0}&t=host&p={1}'.format(
query, str(i))
r = sender.send_get(self.options, url, headers=self.headers)
if r.status_code == 200:
response = r.text
if '"msg": "forbidden"' in response:
utils.print_bad("Reach to the limit at page {0}".format(
str(i)))
return
else:
json_response = utils.get_json(response)
self.analyze(json_response)
self.optimize(json_response)
def check_session(self):
utils.print_debug(self.options, "Checking session for FoFa")
sess_url = 'https://fofa.so/user/users/info'
r = sender.send_get(self.options, sess_url, self.cookies)
if r.status_code == 302 or '/login' in r.text:
utils.print_bad(
"Look like fofa session is invalid. You gonna get very litlle result"
)
new_cookie = self.do_login()
if new_cookie:
utils.print_good("Reauthen success")
self.cookies = {"_fofapro_ars_session": new_cookie}
return False
elif r.status_code == 200:
utils.print_good("Getting result as authenticated user")
return True
return False
def pages(self, page_num):
for i in range(2, int(page_num) + 1):
utils.print_info("Sleep for couple seconds because Shodan server is really strict")
utils.random_sleep(3, 6)
utils.print_info("Get more result from page: {0}".format(str(i)))
query = utils.url_encode(self.options['shodan_query'])
url = 'https://www.shodan.io/search?query={1}&page={0}'.format(
str(i), query)
r = sender.send_get(self.options, url, self.cookies)
if r.status_code == 200:
response = r.text
if 'class="alert alert-error text-center"' in response:
utils.print_bad(
"Reach to the limit at page {0}".format(str(i)))
return
else:
soup = utils.soup(response)
self.analyze(soup)
def initial(self):
product = utils.url_encode(self.query)
url = 'https://www.cvedetails.com/product-search.php?vendor_id=0&search={0}'.format(
product)
# get summary table
products = []
r = sender.send_get(self.options, url, cookies=None)
if r.status_code == 200:
response = r.text
if 'class="errormsg"' in response:
utils.print_bad("No entry found for: {0}".format(self.query))
return
summary_table = utils.soup(response).find_all("table", "listtable")
# <table class = "listtable"
if summary_table:
trs = summary_table[0].findChildren('tr')
if len(trs) <= 1:
utils.print_bad(
"No entry found for: {0}".format(self.query))
return
for tr in trs[1:]:
for td in tr.findChildren('td'):
if td.a:
if 'See all vulnerabilities' in td.a.get('title'):
products.append(td.a.get('href'))
final = []
# if found product and have vulnerabilities, go get it
if products:
for url in products:
results = self.sending(self.baseURL + url)
if results:
final.extend(results)
# self.details(products)
# print(final)
# write final output
self.conclude(final)
def optimize(self, query):
utils.print_good("Analyze metadata page for more result")
raw_query = utils.url_decode(query)
if 'location.country' in raw_query:
country = utils.get_country_code(raw_query, source='censys')
query = raw_query.replace(country, '').replace(
'AND ' + country, '').replace('and ' + country, '')
url = 'https://censys.io/ipv4/metadata?q={0}'.format(query)
r = sender.send_get(self.options, url, self.cookies)
if r.status_code == 200:
soup = utils.soup(r.text)
else:
return False
query_by_countries = []
# check if query have country filter or not
divs = soup.find_all("div", 'left-table')
country_tables = []
for div in divs:
if 'Country Breakdown' in div.h6.text:
country_tables = div.find_all('tr')
for row in country_tables:
item = {
'url': 'N/A',
'country': 'N/A'
}
tds = row.find('td')
for td in tds:
if td.findChildren('a'):
item['url'] = self.base_url + td.a.get('href')
item['country'] = td.a.text
query_by_countries.append(item)
utils.print_debug(self.options, query_by_countries)
return query_by_countries
def optimize(self, query):
utils.print_good("Analyze result by country and city for more result")
# custom headers for stats
custom_headers = {
"User-Agent":
"Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36",
"Accept":
"text/html, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate",
"X-Requested-With": "XMLHttpRequest",
"Connection": "close"
}
url = 'https://fofa.so/search/result_stats?qbase64={0}'.format(query)
r = sender.send_get(self.options,
url,
self.cookies,
headers=custom_headers)
if r.status_code == 200:
html_data = r.text.replace('\/',
'/').replace('\\"',
'"').replace("\'", "'")
soup = utils.soup(html_data)
query_by_cities = []
# custom here
country_divs = soup.find_all("div", "class_sf")
for div in country_divs:
city_links = div.findChildren("a")
for link in city_links:
query_by_cities.append({
'url':
self.base_url + link.get('href'),
'city':
link.text,
})
# utils.print_debug(self.options, query_by_cities)
return query_by_cities
return False
def check_session(self):
utils.print_debug(self.options, "Checking session for ZoomEye")
sess_url = 'https://www.zoomeye.org/user'
# get jwt if it was set
if self.jwt.get('Cube-Authorization'
) and self.jwt.get('Cube-Authorization') != 'None':
self.headers['Cube-Authorization'] = self.jwt.get(
'Cube-Authorization')
r = sender.send_get(self.options, sess_url, headers=self.headers)
if not r or 'login required' in r.text:
utils.print_bad("Look like ZoomEye session is invalid.")
# delete jwt header to prevent getting 401 code
del self.headers['Cube-Authorization']
return False
elif 'uuid' in r.text or 'nickname' in r.text:
utils.print_good("Getting result as authenticated user")
return True
return False
def sending(self, url):
# sending request and return the response
r = sender.send_get(self.options, url, self.cookies)
if r:
response = r.text
if 'ratelimit' in response:
utils.print_bad('Looks like you get block from Censys. Consider using Proxy')
return False
if self.options['store_content']:
ts = str(int(time.time()))
raw_file = self.options['raw'] + \
"/censys/{0}_{1}".format(utils.url_encode(
url.replace(self.base_url, '')).replace('/', '_'), ts)
utils.just_write(raw_file, response)
soup = utils.soup(response)
self.analyze(soup)
# checking if there is many pages or not
if not self.options['disable_pages']:
utils.print_info("Continue grab more pages")
self.pages(self.get_num_pages(soup))
def sending(self, url):
# sending request and return the response
r1 = sender.send_get(self.options, url, headers=self.headers)
if r1:
response = r1.text
if self.options['store_content']:
ts = str(int(time.time()))
raw_file = self.options['raw'] + \
"/zoomeye/{0}_{1}".format(utils.url_encode(
url.replace(self.base_url, '')).replace('/', '_'), ts)
utils.just_write(raw_file, response)
json_response = utils.get_json(response)
self.analyze(json_response)
# loop throuh pages if you're logged in
page_num = self.get_num_pages(json_response)
if self.logged_in and page_num and int(page_num) > 1:
self.pages(page_num)
# get aggs and found more result
self.optimize(json_response)
