def GoogleSearch(query, limit, offset=0):
key = ConfigFileParser().GoogleDeveloperKey()
engine = ConfigFileParser().GoogleEngine()
if not key or not engine:
sys.exit()
try:
service = build("customsearch", "v1", http=_initHttpClient(), developerKey=key)
result_info = service.cse().list(q=query, cx=engine).execute()
msg = 'Max query results: %s' % str(result_info['searchInformation']['totalResults'])
logger.info(msg)
ans = set()
limit += offset
for i in range(int(offset / 10), int((limit + 10 - 1) / 10)):
result = service.cse().list(q=query, cx=engine, num=10, start=i * 10 + 1).execute()
if 'items' in result:
for url in result['items']:
ans.add(url['link'])
return ans
except SocketError:
sys.exit(logger.error('Unable to connect Google, maybe agent/proxy error.'))
except ServerHttpDenied, e:
logger.warning('It seems like Google-Server denied this request.')
sys.exit(logger.error(getSafeExString(e)))
def proxy_regester(args):
# if define proxy
if args.proxy:
proxy = args.proxy
else:
proxy = ConfigFileParser().proxy()
if proxy:
# check proxy format
try:
# check protocol
protocol = proxy.split("://")[0].lower()
if protocol not in ("socks4",'socks5','http'):
raise Exception("proxy protocol format error, please check your proxy (socks4|socks5|http)")
# check ip addr
ip = proxy.split("://")[1].split(":")[0]
compile_ip=re.compile('^(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|[1-9])\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)$')
if not compile_ip.match(ip):
raise Exception("proxy ip format error, please check your proxy")
# check port
port = int(proxy.split("://")[1].split(":")[1])
if not 0 <= port <= 65535:
raise Exception("proxy port format error, please check your proxy")
except Exception as e:
colorprint.red(e)
sys.exit()
msg = "[+] setting proxy: {}://{}:{}".format(protocol, ip, port)
colorprint.green(msg)
conf.proxy = (protocol, ip, port)
else:
conf.proxy = None
def _initHttpClient():
if conf.GOOGLE_PROXY:
proxy_str = conf.GOOGLE_PROXY
elif ConfigFileParser().GoogleProxy():
proxy_str = ConfigFileParser().GoogleProxy()
else:
proxy_str = None
if not proxy_str:
return Http()
msg = 'Proxy: %s' % proxy_str
logger.info(msg)
proxy = proxy_str.strip().split(' ')
if len(proxy) != 3:
msg = 'SyntaxError in GoogleProxy string, Please check your args or config file.'
sys.exit(logger.error(msg))
if proxy[0].lower() == 'http':
type = PROXY_TYPE.HTTP
elif proxy[0].lower() == 'sock5':
type = PROXY_TYPE.SOCKS5
elif proxy[0].lower() == 'sock4':
type = PROXY_TYPE.SOCKS4
else:
msg = 'Invalid proxy-type in GoogleProxy string, Please check your args or config file.'
sys.exit(logger.error(msg))
try:
port = int(proxy[2])
except ValueError:
msg = 'Invalid port in GoogleProxy string, Please check your args or config file.'
sys.exit(logger.error(msg))
else:
http_client = Http(proxy_info=ProxyInfo(type, proxy[1], port))
return http_client
def handle_censys(query, limit, offset):
global UID
global SECRET
UID = ConfigFileParser().censys_UID()
SECRET = ConfigFileParser().censys_SECRET()
if not can_auto_login():
err_msg = '[-] Automatic authorization failed.\n\
[*] Please input your Shodan API Key (https://account.shodan.io/).'
colorprint.cyan(err_msg)
UID = input('[*] UID > ').strip()
SECRET = input('[*] UID > ').strip()
if not can_auto_login(UID, SECRET):
err_msg = "[-] authorization failed"
colorprint.red(err_msg)
else:
pass
else:
page_start = int(offset / 160) + 1
page_stop = page_start + int(limit / 160) + 1
for page in range(page_start, page_stop):
ip_list = get_ip(query, page)
# the last loop dont need sleep
if page < page_stop - 1:
time.sleep(3)
def handle_censys(query, limit, offset):
global UID
global SECRET
UID = ConfigFileParser().censys_UID()
SECRET = ConfigFileParser().censys_SECRET()
msg = '[+] Trying to login with credentials in config file: {}.'.format(
paths.CONFIG_PATH)
colorprint.green(msg)
if not can_auto_login():
err_msg = '[-] Automatic authorization failed.\n[*] Please input your censys API Key (https://censys.io/account/api).'
colorprint.cyan(err_msg)
UID = input('[*] UID > ').strip()
SECRET = input('[*] SECRET > ').strip()
if not can_auto_login():
err_msg = "[-] authorization failed"
colorprint.red(err_msg)
sys.exit()
page_start = int(offset / 100) + 1
page_stop = page_start + int(limit / 100) + 1
for page in range(page_start, page_stop):
get_ip(query, page)
# the last loop dont need sleep
if page < page_stop - 1:
time.sleep(3)
def handle_fofa(query, limit, offset=0):
try:
msg = '[+] Trying to login with credentials in config file: {}.'.format(
paths.CONFIG_PATH)
colorprint.green(msg)
email = ConfigFileParser().fofa_email()
key = ConfigFileParser().fofa_key()
#print(key)
if check(email, key):
pass
else:
raise Exception(
"Automatic authorization failed") # will go to except block
except Exception as e:
logger.debug(e)
msg = '[*] Automatic authorization failed.'
colorprint.cyan(msg)
msg = '[*] Please input your FoFa Email and API Key below.'
colorprint.cyan(msg)
email = input("[*] Fofa Email: ").strip()
key = input('[*] Fofa API Key: ').strip()
if not check(email, key):
msg = '[-] Fofa API authorization failed, Please re-run it and enter a valid key.'
colorprint.red(msg)
sys.exit()
query = base64.b64encode(query.encode('utf-8')).decode('utf-8')
# count how many result to search
size = limit + offset
url = f"https://fofa.so/api/v1/search/all?email={email}&key={key}&qbase64={query}&size={size}"
try:
response = request.get(url).text
resp = json.loads(response)
if not resp["error"]:
for item in resp.get('results')[offset:]:
#print(type(item[0]))
if 'https:' not in item[0]:
try:
requests.get("http://" + item[0],
timeout=5,
verify=False)
conf.target.add("http://" + item[0])
print("http://" + item[0])
except:
pass
else:
try:
requests.get(item[0], timeout=5, verify=False)
conf.target.add(item[0])
print(item[0])
except:
pass
except Exception as e:
colorprint.red(e)
sys.exit()
def auto_login(self):
msg = '[+] Trying to login with credentials in config file: %s.' % paths.CONFIG_PATH
colorprint.green(msg)
try:
self.username = ConfigFileParser().ZoomEyeEmail()
self.password = ConfigFileParser().ZoomEyePassword()
except:
pass
if bool(self.username and self.password):
if self.get_token():
return
msg = '[*] Automatic authorization failed.'
colorprint.cyan(msg)
self.manual_login()
def auto_login(self):
msg = 'Trying to login with credentials in config file: %s.' % paths.CONFIG_PATH
logger.info(msg)
try:
self.username = ConfigFileParser().ZoomEyeEmail()
self.password = ConfigFileParser().ZoomEyePassword()
except:
pass
if bool(self.username and self.password):
if self.get_token():
return
msg = 'Automatic authorization failed.'
logger.warning(msg)
self.manual_login()
def initConfig(args):
"""初始化配置文件toolkit.conf"""
if args.init_config:
cf = ConfigFileParser()
section = args.init_config
option_keys = cf._get_options(section)
update = True
for key in option_keys:
value = raw_input("please input %s:" % key)
res = cf._set_option(section, key, value)
if res == False:
update = False
logger.error("Update Toolkit.conf Fail!")
if update:
logger.info("Update Toolkit.conf Success!")
sys.exit(0)
def BaiduSearch(query, limit=10, offset=0):
urllist = {''}
regex = str(ConfigFileParser().UrlFilter())
try:
while len(urllist) < limit:
url = "http://www.baidu.com/s?{}".format(
urllib.urlencode({
'wd': query,
'pn': str(offset) + '0',
'tn': 'baidurt',
'ie': 'utf-8',
'bsst': '1'
}))
request = urllib2.Request(url)
response = urllib2.urlopen(request)
html = response.read()
soup = BS(html, "lxml")
td = soup.find_all(class_='f')
for t in td:
if regex:
after_url = re.findall(regex, t.h3.a['href'])
if after_url:
urllist.add(after_url[0])
else:
after_url = iterate_path(t.h3.a['href'])
for each_url in after_url:
urllist.add(each_url)
offset = offset + 1
return urllist
except urllib2.URLError, e:
logger.warning('It seems like URL is wrong')
sys.exit(logger.error(getSafeExString(e)))
def handle_fofa(query, limit, offset=0):
try:
msg = '[+] Trying to login with credentials in config file: {}.'.format(
paths.CONFIG_PATH)
colorprint.green(msg)
email = ConfigFileParser().fofa_email()
key = ConfigFileParser().fofa_key()
if check(email, key):
pass
else:
raise Exception(
"Automatic authorization failed") # will go to except block
except Exception as e:
logger.debug(e)
msg = '[*] Automatic authorization failed.'
colorprint.cyan(msg)
msg = '[*] Please input your FoFa Email and API Key below.'
colorprint.cyan(msg)
email = input("[*] Fofa Email: ").strip()
key = input('[*] Fofa API Key: ').strip()
if not check(email, key):
msg = '[-] Fofa API authorization failed, Please re-run it and enter a valid key.'
colorprint.red(msg)
sys.exit()
query = base64.b64encode(query.encode('utf-8')).decode('utf-8')
# count how many result to search
size = limit + offset
url = f"https://fofa.info/api/v1/search/all?email={email}&key={key}&qbase64={query}&size={size}&fields=host,ip,protocol,port"
try:
response = request.get(url).text
resp = json.loads(response)
if not resp["error"]:
for item in resp.get('results')[offset:]:
host = item[0]
protocol = item[2]
# 下面根据host,ip, protocal, port来组装，一般用host就够了，但是对于http/https还需要处理一下
if protocol == "https" or protocol == "http":
if not host.startswith("http"):
host = protocol + "://" + host
conf.target.add(host)
except Exception as e:
colorprint.red(e)
sys.exit()
def _readKey():
msg = 'Trying to auth with credentials in config file: %s.' % paths.CONFIG_PATH
logger.info(msg)
try:
key = ConfigFileParser().ShodanApikey()
except:
key = ''
return key
def login(self):
msg = 'Trying to login with credentials in config file: %s.' % paths.CONFIG_PATH
logger.info(msg)
self.api_key = ConfigFileParser().ShodanApikey()
if not self.api_key:
msg = 'Automatic authorization failed.'
logger.warning(msg)
msg = 'Please input your Shodan API Key (https://account.shodan.io/).'
logger.info(msg)
self.api_key = raw_input('API KEY > ').strip()
def login(self):
msg = '[+] Trying to login with credentials in config file: %s.' % paths.CONFIG_PATH
outputscreen.success(msg)
self.api_key = ConfigFileParser().shodan_apikey()
if not self.api_key:
msg = '[*] Automatic authorization failed.'
outputscreen.warning(msg)
msg = '[*] Please input your Shodan API Key (https://account.shodan.io/).'
outputscreen.warning(msg)
self.api_key = input('[*] API KEY > ').strip()
def handle_fofa(query, limit, offset=0):
try:
msg = '[+] Trying to login with credentials in config file: %s.' % paths.CONFIG_PATH
colorprint.green(msg)
email = ConfigFileParser().fofa_email()
key = ConfigFileParser().fofa_key()
if check(email, key):
pass
else:
raise SystemExit # will go to except block
except Exception as e:
logger.debug(e)
msg = '[*] Automatic authorization failed.'
colorprint.cyan(msg)
msg = '[*] Please input your FoFa Email and API Key below.'
colorprint.cyan(msg)
email = input("[*] Fofa Email: ").strip()
key = input('[*] Fofa API Key: ').strip()
if not check(email, key):
msg = '[-] Fofa API authorization failed, Please re-run it and enter a valid key.'
colorprint.red(msg)
sys.exit()
query = base64.b64encode(query)
request = "https://fofa.so/api/v1/search/all?email={0}&key={1}&qbase64={2}".format(
email, key, query)
try:
response = requests.get(request)
resp = response.readlines()[0]
resp = json.loads(resp)
if resp["error"] is None:
for item in resp.get('results'):
conf.target.append(item[0])
if resp.get('size') >= 100:
colorprint.cyan(
"{0} items found! just 100 returned....".format(
resp.get('size')))
except Exception as e:
colorprint.red(e)
sys.exit()
def handle_google(query, limit, offset=0):
key = ConfigFileParser().google_developer_key()
engine = ConfigFileParser().google_engine()
if not key or not engine:
msg = "[-] Please config your 'developer_key' and 'search_enging' at saucerfram.conf"
outputscreen.error(msg)
sys.exit()
try:
service = build("customsearch",
"v1",
http=_initHttpClient(),
developerKey=key)
result_info = service.cse().list(q=query, cx=engine).execute()
msg = '[+] Max query results: %s' % str(
result_info.get('searchInformation', {}).get('totalResults'))
outputscreen.success(msg)
ans = set()
limit += offset
for i in range(int(offset / 10), int((limit + 10 - 1) / 10)):
result = service.cse().list(q=query,
cx=engine,
num=10,
start=i * 10 + 1).execute()
if 'items' in result:
for url in result.get('items'):
ans.add(url.get('link'))
for t in ans:
conf.target.put(t)
except SocketError:
outputscreen.error(
'[-] Unable to connect Google, maybe agent/proxy error.')
sys.exit()
except ServerHttpDenied as e:
outputscreen.warning(
'[-] It seems like Google-Server denied this request.')
outputscreen.error(e)
sys.exit()
def FofaSearch(query, limit=100, offset=0): # TODO 付费获取结果的功能实现
try:
msg = 'Trying to login with credentials in config file: %s.' % paths.CONFIG_PATH
logger.info(msg)
email = ConfigFileParser().FofaEmail()
key = ConfigFileParser().FofaKey()
if check(email, key):
pass
else:
raise # will go to except block
except:
msg = 'Automatic authorization failed.'
logger.warning(msg)
msg = 'Please input your FoFa Email and API Key below.'
logger.info(msg)
email = raw_input("Fofa Email: ").strip()
key = getpass.getpass(prompt='Fofa API Key: ').strip()
if not check(email, key):
msg = 'Fofa API authorization failed, Please re-run it and enter a valid key.'
sys.exit(logger.error(msg))
query = base64.b64encode(query)
request = "https://fofa.so/api/v1/search/all?email={0}&key={1}&qbase64={2}".format(
email, key, query)
result = []
try:
response = urllib.urlopen(request)
resp = response.readlines()[0]
resp = json.loads(resp)
if resp["error"] is None:
for item in resp.get('results'):
result.append(item[0])
if resp.get('size') >= 100:
logger.info("{0} items found! just 100 returned....".format(
resp.get('size')))
except Exception as e:
sys.exit(logger.error(getSafeExString(e)))
finally:
return result
def loadConf():
"""
加载扫描配置(以后将使用参数，而非从文件加载)
"""
conf.recursive_scan = eval(ConfigFileParser().recursive_scan())
conf.recursive_status_code = eval(
ConfigFileParser().recursive_status_code())
conf.exclude_subdirs = eval(ConfigFileParser().exclude_subdirs())
conf.dict_mode = eval(ConfigFileParser().dict_mode())
conf.dict_mode_load_single_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().dict_mode_load_single_dict()))
conf.dict_mode_load_mult_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().dict_mode_load_mult_dict()))
conf.blast_mode = eval(ConfigFileParser().blast_mode())
conf.blast_mode_min = eval(ConfigFileParser().blast_mode_min())
conf.blast_mode_max = eval(ConfigFileParser().blast_mode_max())
conf.blast_mode_az = eval(ConfigFileParser().blast_mode_az())
conf.blast_mode_num = eval(ConfigFileParser().blast_mode_num())
conf.blast_mode_custom_charset = eval(
ConfigFileParser().blast_mode_custom_charset())
conf.blast_mode_resume_charset = eval(
ConfigFileParser().blast_mode_resume_charset())
conf.crawl_mode = eval(ConfigFileParser().crawl_mode())
conf.crawl_mode_parse_robots = eval(
ConfigFileParser().crawl_mode_parse_robots())
conf.crawl_mode_parse_html = eval(
ConfigFileParser().crawl_mode_parse_html())
conf.crawl_mode_dynamic_fuzz = eval(
ConfigFileParser().crawl_mode_dynamic_fuzz())
conf.fuzz_mode = eval(ConfigFileParser().fuzz_mode())
conf.fuzz_mode_load_single_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().fuzz_mode_load_single_dict()))
conf.fuzz_mode_load_mult_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().fuzz_mode_load_mult_dict()))
conf.fuzz_mode_label = eval(ConfigFileParser().fuzz_mode_label())
conf.request_headers = eval(ConfigFileParser().request_headers())
conf.request_header_ua = eval(ConfigFileParser().request_header_ua())
conf.request_header_cookie = eval(
ConfigFileParser().request_header_cookie())
conf.request_header_401_auth = eval(
ConfigFileParser().request_header_401_auth())
conf.request_timeout = eval(ConfigFileParser().request_timeout())
conf.request_delay = eval(ConfigFileParser().request_delay())
conf.request_limit = eval(ConfigFileParser().request_limit())
conf.request_max_retries = eval(ConfigFileParser().request_max_retries())
conf.request_persistent_connect = eval(
ConfigFileParser().request_persistent_connect())
conf.request_method = eval(ConfigFileParser().request_method())
conf.redirection_302 = eval(ConfigFileParser().redirection_302())
conf.file_extension = eval(ConfigFileParser().file_extension())
conf.response_status_code = eval(ConfigFileParser().response_status_code())
conf.response_header_content_type = eval(
ConfigFileParser().response_header_content_type())
conf.response_size = eval(ConfigFileParser().response_size())
conf.custom_404_page = eval(ConfigFileParser().custom_404_page())
conf.custom_503_page = eval(ConfigFileParser().custom_503_page())
conf.custom_response_page = eval(ConfigFileParser().custom_response_page())
conf.skip_size = eval(ConfigFileParser().skip_size())
conf.proxy_server = eval(ConfigFileParser().proxy_server())
conf.debug = eval(ConfigFileParser().debug())
conf.update = eval(ConfigFileParser().update())
def setUpClass(cls):
cls.cf = ConfigFileParser()
print("CONFIG PATH: %s" % paths.CONFIG_PATH)
return:
{"status": "success", "data": {}}
"""
import random
import requests
import time
from string import ascii_lowercase
import sys, json
sys.path.append('../')
from lib.core.data import logger, paths
# paths.CONFIG_PATH = "../toolkit.conf"
from lib.utils.config import ConfigFileParser
API_KEY = ConfigFileParser()._get_option("dnslog", "api_key")
DNS_DOMAIN = ConfigFileParser()._get_option("dnslog", "dns_domain")
DNS_IP = ConfigFileParser()._get_option("dnslog", "dns_ip")
API_PORT = ConfigFileParser()._get_option("dnslog", "api_port")
class Dnslog:
def __init__(self, custom_preix="vuln"):
self.dns_domain = DNS_DOMAIN
self.dns_ip = DNS_IP
self.api_key = API_KEY
self.api_port = API_PORT
self.custom_preix = custom_preix
self.random = ''.join(
[random.choice(ascii_lowercase) for _ in range(10)])
self.custom_domain = '%s.%s.%s' % (self.random, custom_preix,
def loadConf():
'''
@description: 加载扫描配置(以后将使用参数，而非从文件加载)
@param {}
@return: None
'''
conf.dict_mode = eval(ConfigFileParser().dict_mode())
conf.dict_mode_load_dir_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().dict_mode_load_dir_dict()))
conf.dict_mode_load_mult_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().dict_mode_load_mult_dict()))
conf.fuzz_mode = eval(ConfigFileParser().fuzz_mode())
conf.fuzz_mode_load_main_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().fuzz_mode_load_main_dict()))
conf.fuzz_mode_load_ext_dict = os.path.join(
paths.DATA_PATH, eval(ConfigFileParser().fuzz_mode_load_ext_dict()))
conf.request_headers = eval(ConfigFileParser().request_headers())
conf.request_header_ua = eval(ConfigFileParser().request_header_ua())
conf.request_header_cookie = eval(
ConfigFileParser().request_header_cookie())
conf.request_header_401_auth = eval(
ConfigFileParser().request_header_401_auth())
conf.request_timeout = eval(ConfigFileParser().request_timeout())
conf.request_delay = eval(ConfigFileParser().request_delay())
conf.request_limit = eval(ConfigFileParser().request_limit())
conf.request_persistent_connect = eval(
ConfigFileParser().request_persistent_connect())
conf.request_method = eval(ConfigFileParser().request_method())
conf.redirection_302 = eval(ConfigFileParser().redirection_302())
conf.proxy_server = eval(ConfigFileParser().proxy_server())
except Exception:
pass
print c.verifyDNS(delay=0)
print c.verifyHTTP(delay=0)
print c.getDnsRecord(delay=0)
print c.getHttpRecord(delay=0)
"""
import random
import requests
import time
from string import ascii_lowercase
from lib.utils.config import ConfigFileParser
# load once for all thread
key = ConfigFileParser().CloudEyeApikey()
uniq_domain = ConfigFileParser().ColudEyePersonaldomain().split('.')[0]
class CloudEye:
def __init__(self):
self.unique = uniq_domain
self.random = ''.join(
[random.choice(ascii_lowercase) for _ in range(10)])
def getRandomDomain(self, custom='poc'):
"""
full domain = [random].[custom].[unique].dnslog.info
e.g. fezarvgo.poc.ee8a6f.dnslog.info
"""
self.custom = custom