def fastDir(newurl, target, module):
'''
FastDir scan without portscan
'''
output_file = report_filename(target, module)
newurl = url_handle(newurl)
ip, baidu_status, github_status = '', [], []
print '[*] Scan new_url: ' + newurl
if baidu_engine:
print '[*] Check Baidu site: %s' % urlparse.urlparse(newurl).hostname
baidu_status = baidu_check(newurl)
if github_engine:
print '[*] Check Github status: %s' % urlparse.urlparse(
newurl).hostname
github_status = github_check(newurl)
try:
newtitle, code, lenth, content = '', '', '', ''
try:
newtitle, code, lenth, content = getitle(url=newurl)
except Exception, e:
# print traceback.format_exc()
pass
if code in range(
200, 405) and code != 401: # add Do not scan 401 status_code
try:
print '[+] Get title: %s,status_code: %s,content lenth: %s' % (
newtitle, code, lenth)
except:
pass
alllink, alllinks, emails, ips = [], [], [], []
if title_filter not in newtitle and filter_list(
module=newtitle, filter_list=title_filter_list):
try:
alllink, alllinks, emails, ips = getallink(newurl, content)
except Exception, e:
# print traceback.format_exc()
pass
dirpaths = []
try:
dir_urls = scandir_again(newurl, alllink)
if len(dir_urls) < link_maxnum: # Pass num
for dir_url in dir_urls:
dirpaths += weakfile(dir_url)
except Exception, e:
# print traceback.format_exc()
pass
if len(dirpaths) > dirpaths_maxnum: # Check num
dirpaths = ["more_path"]
baidu_dirs = ''
if baidu_dir_engine:
try:
baidu_dirs = baidu_dir(
command='site:%s' %
urlparse.urlparse(newurl).hostname,
key_domain=urlparse.urlparse(newurl).hostname)
except Exception, e:
# print traceback.format_exc()
pass
def getallink(url,content):
'''
Get response all link
'''
links,emails,ips,check = [],[],[],[]
tags = ['a','A','link','script','area','iframe','form']# img
tos = ['href','src','action']
if url[-1:] == '/':
url = url[:-1]
try:
print '[*] Now regex alllinks,emails,ips'
emails_source = email_regex(str(content))
ips = ip_regex(str(content))
# print '[*] Now regex urls'
# urls = url_regex(content)
for tag in tags:
for to in tos:
link = re.findall(r'<%s.*?%s="(.*?)"' % (tag,to),str(content))
for i in link:
if i not in check and filter_list(module=i,filter_list=links_filter) and i != '':
check.append(i)
if '://' in i:
i = i.replace(' ','')
if str(urlparse.urlparse(i).path) in ['/',''] and str(urlparse.urlparse(i).query) in ['/','']:
link_flag = '<a href="'+i+'" target=_blank />'+urlparse.urlparse(str(i)).netloc+'</a>'
else:
link_flag = '<a href="'+i+'" target=_blank />'+quote(urlparse.urlparse(str(i)).path+urlparse.urlparse(str(i)).query)[:25]+'</a>'
elif '//' in i:
if str(urlparse.urlparse(i).path) in ['/',''] and str(urlparse.urlparse(i).query) in ['/','']:
link_flag = '<a href="http:'+i+'" target=_blank />'+urlparse.urlparse(str(i)).netloc+'</a>'
else:
link_flag = '<a href="http:'+i+'" target=_blank />'+quote(urlparse.urlparse(str(i)).path+urlparse.urlparse(str(i)).query)[:25]+'</a>'
else:
link_flag = '<a href="'+url+'/'+i+'" target=_blank />'+quote(i)[:25]+'</a>'
check.append(url + i)
links.append(link_flag)
except Exception,e:
# print traceback.format_exc()
print '[!] Get regex link error'
pass
def checkDir(url, target, module):
'''
Main requests function with Portscan && Dirscan
'''
output_file = report_filename(target, module)
url = url_handle(url)
try:
if url not in filter_urls and filter_list(module=url,
filter_list=sub_filter_list):
filter_urls.append(url)
ip, open_ports, baidu_status, github_status = url, [], [], []
print '[*] Now scanning: ' + url
if module in ['autoscan', 'dirscan', 'single']: # Handle c_ip scan
if baidu_engine:
print '[*] Check Baidu site: %s' % urlparse.urlparse(
url).hostname
baidu_status = baidu_check(url)
if github_engine:
print '[*] Check Github status: %s' % urlparse.urlparse(
url).hostname
github_status = github_check(url)
try:
ip = url2ip(url)
if not is_internal_ip(ip) and ip not in filter_ips.keys(
) and ip != '': # filter internal_ip
print '[+] Get url2ip: ' + ip
open_ports = portscan(ip)
filter_ips[ip] = open_ports
write_file(
str(ip) + ',' +
str(open_ports).replace('[', '').replace(']', ''),
handle_ext(output_file) + portscan_opens_file)
if len(open_ports) > openports_maxnum:
print '[!] Maybe got port waf'
write_file(
ip,
handle_ext(output_file) + portscan_maxnum_file)
open_ports = []
else:
open_ports = filter_ips[ip]
except Exception, e:
# print traceback.format_exc()
write_file(url, handle_ext(output_file) + url2ip_error_file)
pass
print '[+] Get open ports: ' + str(open_ports)
if open_ports == []: #or 80 not in open_ports
try:
newtitle, code, lenth, content = '', '', '', ''
try:
newtitle, code, lenth, content = getitle(url)
except Exception, e:
# print traceback.format_exc()
pass
if code in range(
200, 405
) and code != 401: # add Do not scan 401 status_code
try:
print '[+] Get title: %s,status_code: %s,content lenth: %s' % (
newtitle, code, lenth)
except:
pass
write_file(
url,
handle_ext(output_file) +
'/%s_alive_urls.txt' % handle_ext_old(target))
if title_filter not in newtitle and filter_list(
module=newtitle,
filter_list=title_filter_list):
alllink, alllinks, emails, ips = [], [], [], []
try:
alllink, alllinks, emails, ips = getallink(
url, content)
except Exception, e:
# print traceback.format_exc()
pass
dirpaths = []
try:
dir_urls = scandir_again(url, alllink)
if len(dir_urls) < link_maxnum:
for dir_url in dir_urls:
dirpaths += weakfile(dir_url)
except Exception, e:
# print traceback.format_exc()
pass
if len(dirpaths) > dirpaths_maxnum:
dirpaths = ["more_path"]
weakuri = []
try:
weakuri = dirscan(url)
except Exception, e:
# print traceback.format_exc()
pass
baidu_dirs = ''
if baidu_dir_engine and module in [
'autoscan', 'dirscan', 'single'
]:
try:
baidu_dirs = baidu_dir(
command='site:%s' %
urlparse.urlparse(url).hostname,
key_domain=urlparse.urlparse(
url).hostname)
except Exception, e:
# print traceback.format_exc()
pass
weakuri = baidu_status + github_status + weakuri
try:
write_file(
'<tr><td><a href="%s" target=_blank />%s</a></td><td>%s</td><td><font color="blue">%s</font></td><td><font color="red">%s</font></td><td>%s b</td><td>%s</td><td><font color="blue">%s%s</font></td><td><ul><li>%s</li><li>%s</li><ul/></td></tr>\n\n'
% (url, url, ip, code, newtitle, lenth, [
dirpath_key
for dirpath_key in set(dirpaths +
weakuri)
], alllinks, baidu_dirs, emails, ips),
output_file)
except Exception, e:
# print traceback.format_exc()
print '[!] output_error'
write_file(
url,
handle_ext(output_file) +
output_error_file)
pass
pass
if code in range(
200, 405
) and code != 401: # add Do not scan 401 status_code
try:
print '[+] Get title: %s,status_code: %s,content lenth: %s' % (
newtitle, code, lenth)
except:
pass
write_file(
newurl,
handle_ext(output_file) +
'/%s_alive_urls.txt' %
handle_ext_old(target))
if title_filter not in newtitle and filter_list(
module=newtitle,
filter_list=title_filter_list):
alllink,alllinks,emails,ips = [],[],[],[]
try:
alllink, alllinks, emails, ips = getallink(
newurl, content)
except Exception, e:
# print traceback.format_exc()
pass
dirpaths = []
try:
dir_urls = scandir_again(
newurl, alllink)
if len(dir_urls
) < link_maxnum: # Pass num
for dir_url in dir_urls:
link_flag = '<a href="'+i+'" target=_blank />'+urlparse.urlparse(str(i)).netloc+'</a>' else: link_flag = '<a href="'+i+'" target=_blank />'+quote(urlparse.urlparse(str(i)).path+urlparse.urlparse(str(i)).query)[:25]+'</a>' elif '//' in i: if str(urlparse.urlparse(i).path) in ['/',''] and str(urlparse.urlparse(i).query) in ['/','']: link_flag = '<a href="http:'+i+'" target=_blank />'+urlparse.urlparse(str(i)).netloc+'</a>' else: link_flag = '<a href="http:'+i+'" target=_blank />'+quote(urlparse.urlparse(str(i)).path+urlparse.urlparse(str(i)).query)[:25]+'</a>' else: link_flag = '<a href="'+url+'/'+i+'" target=_blank />'+quote(i)[:25]+'</a>' check.append(url + i) links.append(link_flag) except Exception,e: # print traceback.format_exc() print '[!] Get regex link error' pass emails_handle = [email[-30:] for email in set(emails_source)]# Filter too lang email for email_check in emails_handle: if filter_list(module=email_check,filter_list=emails_filter):# [email protected]/png emails.append(email_check) ips = [ip for ip in set(ips)] if len(links) > 10: # Click more_links to get detail result mainDiv = ranStr() childDiv = ranStr() return check,u'''<div id="%s" style="color:red" onclick="document.all.%s.style.display=(document.all.%s.style.display =='none')?'':'none'">[more_links]</div><div id="%s" style="display:none">%s</div>'''%(mainDiv,childDiv,childDiv,childDiv,'<br />'.join(links)),'<br />'.join(emails),'<br />'.join(ips) else: return check,links,'<br />'.join(emails),'<br />'.join(ips) if __name__ == '__main__': pass
def checkFast(url,target,module):
'''
Main requests function no Dirscan
'''
output_file = report_filename(target,module)
url = url_handle(url)
try:
if url not in filter_urls and filter_list(module=url,filter_list=sub_filter_list):
filter_urls.append(url)
print '[*] Now scanning: ' + url
ip,open_ports = url,[]
try:
ip = url2ip(url)
if not is_internal_ip(ip) and ip not in filter_ips.keys() and ip != '':# filter internal_ip
print '[+] Get url2ip: ' + ip
open_ports = portscan(ip)
filter_ips[ip] = open_ports
write_file(str(ip)+','+str(open_ports).replace('[','').replace(']',''),handle_ext(output_file)+portscan_opens_file)
if len(open_ports) > openports_maxnum:
print '[!] Maybe got port waf'
write_file(ip,handle_ext(output_file)+portscan_maxnum_file)
open_ports = []
else:
open_ports = filter_ips[ip]
except Exception,e:
# print traceback.format_exc()
write_file(url,handle_ext(output_file)+url2ip_error_file)
pass
print '[+] Get open ports: ' + str(open_ports)
if open_ports == []:#or 80 not in open_ports
try:
newtitle,code,lenth,content = '','','',''
try:
newtitle,code,lenth,content = getitle(url)
except Exception,e:
# print traceback.format_exc()
pass
if code in range(200,405):
try:
print '[+] Get title: %s,status_code: %s,content lenth: %s' % (newtitle,code,lenth)
except:pass
write_file(url,handle_ext(output_file)+'/%s_alive_urls.txt' % handle_ext_old(target)) # save alive `host:port` to dirsearch
alllink,alllinks,emails,ips = [],[],[],[]
if title_filter not in newtitle and filter_list(module=newtitle,filter_list=title_filter_list):
try:
alllink,alllinks,emails,ips = getallink(url,content)
except Exception,e:
# print traceback.format_exc()
pass
try:
write_file('<tr><td><a href="%s" target=_blank />%s</a></td><td>%s</td><td><font color="blue">%s</font></td><td><font color="red">%s</font></td><td>%s b</td><td><font color="blue">%s</font></td><td><ul><li>%s</li><li>%s</li><ul/></td></tr>\n\n' % (url,url,ip,code,newtitle,lenth,alllinks,emails,ips),output_file)
except Exception,e:
# print traceback.format_exc()
print '[!] output_error'
write_file(url,handle_ext(output_file)+output_error_file)
pass
else:
print '[!] Filter title'
write_file(url,handle_ext(output_file)+title_filter_file)
except Exception,e:
# print traceback.format_exc()
pass
def github_site(subdoamin, key_domain):
headers = requests_headers()
proxies = requests_proxies()
if '://' in key_domain:
key_domain = urlparse.urlparse(url).hostname
github_domains = []
session = requests.Session()
headers['Cookie'] = github_cookie
try:
# check_login = '******'
# req_check = session.get(url=check_login,headers=headers,proxies=proxies,timeout=10,verify=False).content
# if github_account in req_check:
# print '[*] Github site:domain login check Success'
headers['Host'] = 'github.com'
headers[
'Referer'] = 'https://github.com/search?utf8=%E2%9C%93&q=*&type=Code'
github_url = 'https://github.com/search?q={}&type=Code&utf8=%E2%9C%93'.format(
subdoamin)
req = session.get(url=github_url,
headers=headers,
proxies=proxies,
timeout=10,
verify=False).content
if 'blankslate' not in req: #if 'code results' in req:
for page in xrange(1, 100):
newurl = 'https://github.com/search?p={}&q={}&type=Code&s=&utf8=%E2%9C%93'.format(
page, subdoamin)
req_new = session.get(url=newurl,
headers=headers,
proxies=proxies,
timeout=10,
verify=False).content
req_new = req_new.replace('</em>',
'').replace('<em>', '').replace(
'</span>', '')
url_regexs = []
url_regex_url,url_regex_host,url_regex_x,url_regex_a,url_regex_b,url_regex_c,url_regex_b_a,url_regex_c_a,url_regex_d = [],[],[],[],[],[],[],[],[]
try:
url_regex_url = re.findall(r'//([\s\S]*?)%s' % key_domain,
req_new)
except:
pass
try:
url_regex_host = re.findall(
r'"([\s\S]*?)%s' % key_domain, req_new)
except:
pass
try:
url_regex_x = re.findall(r''([\s\S]*?)%s' % key_domain,
req_new)
except:
pass
try:
url_regex_a = re.findall(r'/([\s\S]*?)%s' % key_domain,
req_new)
except:
pass
try:
url_regex_b = re.findall(
r'\[<span .*?>([\s\S]*?)%s' % key_domain, req_new)
except:
pass
try:
url_regex_b_a = re.findall(r'\[([\s\S]*?)%s' % key_domain,
req_new)
except:
pass
try:
url_regex_c_a = re.findall(r'\(([\s\S]*?)%s' % key_domain,
req_new)
except:
pass
try:
url_regex_c = re.findall(
r'\(<span .*?>([\s\S]*?)%s' % key_domain, req_new)
except:
pass
try:
url_regex_d = re.findall(
r'<span .*?>([\s\S]*?)%s' % key_domain, req_new)
except:
pass
url_regexs = url_regex_url + url_regex_host + url_regex_x + url_regex_a + url_regex_b + url_regex_c + url_regex_b_a + url_regex_c_a + url_regex_d
for sub in url_regexs:
if sub not in github_domains and sub_filter not in sub and sub != '.' and filter_list(
module=sub, filter_list=github_sub_filter
) and sub[-1:] != '-' and sub[-1:] != '_':
sub.replace(' ', '')
if sub[-1:] == '.':
subs = sub + key_domain
else:
subs = sub + '.' + key_domain
if is_domain(subs) and subs not in github_domainss:
print '[+] Get github site:domain > ' + subs
github_domainss.append(subs)
github_domains.append(subs)
if 'next_page disabled' in req_new:
return github_domains
else:
print '[!] github site:domain no result'
pass
# else:
# print '[!] Github login check Error'
# print '[*] Please try again'
# pass
except Exception, e:
# print traceback.format_exc()
pass