def SenFileScan(domain, url):
"""
敏感文件、目录扫描
字典：dict\SEN_scan.txt
:param domain:
:param
:return:
"""
pools = 20
urlList = []
for i in range(0, redispool.llen("SenScan")):
suffix = redispool.lindex("SenScan", i)
senurl = "{}/{}".format(url, suffix)
urlList.append(senurl)
pool = ThreadPool(pools)
SenFileMessage = pool.map(UrlRequest, urlList)
SenFileMessage2 = ""
pool.close()
pool.join()
url404 = "{}/springbird404page".format(url)
try:
rep404 = requests.get(url404,
headers=core.GetHeaders(),
timeout=3,
verify=False).text
except Exception as e:
print("超时")
rep404 = str(e)
pass
if len(SenFileMessage) != 0:
with app.app_context():
print("Sen file and dir : \n")
for url in SenFileMessage:
try:
if url is None:
continue
rep = requests.get(url,
headers=core.GetHeaders(),
timeout=1,
verify=False)
#添加404界面的判断，避免过多杂乱信息
if not core.is_similar_page(rep404, rep.text, radio=0.85):
print(url)
bug = BugList(oldurl=domain,
bugurl=url,
bugname="SenDir",
buggrade=redispool.hget(
'bugtype', "SenDir"),
payload=url,
bugdetail=rep.text)
SenFileMessage2 += url + "\n"
redispool.pfadd(redispool.hget('bugtype', "SenDir"),
url)
redispool.pfadd("SenDir", url)
db.session.add(bug)
except Exception as e:
# print(e)
pass
db.session.commit()
return SenFileMessage2
def __init__(self, url, redispool):
print("hi!")
self.domain = url
self.redispool = redispool
try:
if not (url.startswith("http://") or url.startswith("https://")):
self.url = "http://" + url
else:
self.url = url
self.rep = requests.get(self.url,
headers=core.GetHeaders(),
timeout=5,
verify=False)
except:
self.rep = None
pass
if self.rep == None:
try:
self.url = "https://" + url
self.rep = requests.get(self.url,
headers=core.GetHeaders(),
timeout=5,
verify=False)
except:
pass
def inputfilter(url):
'''
入口过滤函数
输入源的格式可多变:
127.0.0.1
http://127.0.0.1
www.baidu.com
https://www.baidu.com
等
返回格式为 ： return www.baidu.com,https://www.baidu.com,baidu.rep
:param url:
:return:
'''
rep,rep1,rep2=None,None,None
if url.endswith("/"):
url=url[:-1]
if not url.startswith("http://") and not url.startswith("https://"):
attackurl1="http://"+url
attackurl2="https://"+url
try:
rep1=requests.get(attackurl1, headers=core.GetHeaders(), timeout=4, verify=False)
except Exception as e:
pass
try:
rep2=requests.get(attackurl2, headers=core.GetHeaders(), timeout=4, verify=False)
except Exception as e:
pass
if rep1:
return url,attackurl1,rep1
elif rep2:
return url,attackurl2,rep2
else:
print("None data")
try:
count=redispool.hget('targetscan', 'waitcount')
if 'str' in str(type(count)):
waitcount=int(count)-1
redispool.hset("targetscan", "waitcount", str(waitcount))
else:
redispool.hset("targetscan", "waitcount", "0")
redispool.hdel("targetscan", "nowscan")
except Exception as e:
print(e)
pass
return None,None,None
else:
attackurl=url
try:
rep=requests.get(attackurl, headers=core.GetHeaders(), timeout=4, verify=False)
except:
pass
if rep:
if "http://" in url:
return url.replace("http://",""),attackurl,rep
else:
return url.replace("https://",""),attackurl,rep
else:
print("{}访问超时".format(attackurl))
return None,None,None
def UrlRequest(url):
try:
r = requests.get(url, headers=core.GetHeaders(), timeout=1.0, verify=False)
if r.status_code == 200 or r.status_code==403:
return url
except Exception:
pass
def SenFileScan(domain, redispool):
"""
敏感文件、目录扫描
字典：dict\SEN_scan.txt
:param domain:
:param
:return:
"""
pools = 20
urlList = []
for i in range(0, redispool.llen("SenScan")):
url="http://{}/{}".format(domain, redispool.lindex("SenScan", i))
urlList.append(url)
pool = ThreadPool(pools)
SenFileMessage = pool.map(UrlRequest, urlList)
pool.close()
pool.join()
if len(SenFileMessage)!=0:
with app.app_context():
for url in SenFileMessage:
try:
rep = requests.get(url, headers=core.GetHeaders(), timeout=3, verify=False)
bug = BugList(oldurl=domain, bugurl=url, bugname="SenDir",buggrade=redispool.hget('bugtype', "SenDir"),payload=url, bugdetail=rep.text)
db.session.add(bug)
except Exception as e:
print(e)
pass
db.session.commit()
return "\n".join(list(filter(None, SenFileMessage)))
def getPageLinks(self,url):
'''
获取页面中的所有链接
'''
try:
headers = core.GetHeaders()
content = requests.get(url, timeout=5, headers=headers, verify=False).text.encode('utf-8')
links = []
tags = ['a', 'A', 'link', 'script', 'area', 'iframe', 'form'] # img
tos = ['href', 'src', 'action']
if url[-1:] == '/':
url = url[:-1]
try:
for tag in tags:
for to in tos:
link1 = re.findall(r'<%s.*?%s="(.*?)"' % (tag, to), str(content))
link2 = re.findall(r'<%s.*?%s=\'(.*?)\'' % (tag, to), str(content))
for i in link1:
links.append(i)
for i in link2:
if i not in links:
links.append(i)
except Exception as e:
print(e)
print('[!] Get link error')
pass
return links
except:
return []
def GetSiteStation(ip):
"""
旁站查询
查询网站1：https://www.webscan.cc/search/
查询网站2：http://stool.chinaz.com
查询网站3：http://www.114best.com/ip/114.aspx
:param ip:
:return:
"""
data = {'domain': ip}
url_1 = 'https://www.webscan.cc/search/'
url_2_base = 'http://stool.chinaz.com'
url_2 = 'http://stool.chinaz.com/same?s=' + ip + '&page=1'
text2 = []
try:
rep1 = requests.post(url_1,
data=data,
headers=core.GetHeaders(),
timeout=2.0)
rep1 = etree.HTML(rep1.text)
text1 = rep1.xpath('//a[@class="domain"]/text()')
except:
text1 = []
pass
try:
while 1:
rep2 = requests.get(url_2, headers=core.GetHeaders(), timeout=2.0)
rep2 = etree.HTML(rep2.text)
new_list = rep2.xpath('//div[@class="w30-0 overhid"]/a/text()')
if len(new_list) == 0:
break
text2 += new_list
next_url = "".join(rep2.xpath('//a[@title="下一页"]/@href'))
url_2 = url_2_base + next_url
except:
text2 = []
pass
text = list(set(text1).union(set(text2)))
for i in text:
if "屏蔽的关键字" in i:
text.remove(i)
return "\n".join(text)
def getPageLinks_bak(self,url):
'''
获取页面中的所有链接
'''
try:
headers = core.GetHeaders()
time.sleep(0.5)
pageSource = requests.get(url, timeout=5, headers=headers, verify=False).text.encode('utf-8')
pageLinks = re.findall(r'(?<=href=\").*?(?=\")|(?<=href=\').*?(?=\')', pageSource)
# print pageLinks
except:
# print ('open url error')
return []
return pageLinks
def FindIpAdd(ip):
"""
查找IP真实地址
:param ip:
:return:
"""
str = ""
url = "http://ip.yqie.com/ip.aspx?ip=" + ip
try:
rep = requests.get(url, headers=core.GetHeaders(), timeout=2)
rep = etree.HTML(rep.text)
context = rep.xpath('//input[@id="AddressInfo"]/@value')
str = "\n".join(context)
except:
pass
return str
def GetSubDomain(domain):
chinaz_base_url = 'https://tool.chinaz.com/'
chinaz_url = 'https://tool.chinaz.com/subdomain?domain=' + domain + '&page=1'
attacklist=[]
while 1:
try:
rep = requests.get(chinaz_url, headers=core.GetHeaders(), timeout=2.0)
rep = etree.HTML(rep.text)
data = rep.xpath('//div[@class="w23-0"]/a[@href="javascript:"]/text()')
attacklist.extend(data)
next_url = rep.xpath('//a[@title="下一页"]/@href')[0]
chinaz_url = chinaz_base_url + next_url
except:
break
attacklist[0]="http://"+attacklist[0]
return "\nhttp://".join(attacklist)
def POCScanConsole(attackurl,url):
allpoc=POC.query.all()
with app.app_context():
for poc in allpoc:
try:
rep = requests.get(url+poc.rule, headers=core.GetHeaders(),timeout=2)
if rep.status_code!=404 and poc.expression in rep.text:
bug = BugList(oldurl=attackurl, bugurl=url, bugname=poc.name,buggrade=redispool.hget('bugtype', poc.name), payload=url+poc,
bugdetail=rep.text)
redispool.pfadd(redispool.hget('bugtype', poc.name), url)
redispool.pfadd(poc.name, url)
db.session.add(bug)
except Exception as e:
print(e)
pass
db.session.commit()
def FindDomainAdd(domain):
"""
查找域名真实地址
:param domain:
:return:
"""
str=""
url = "http://ip.yqie.com/ip.aspx?ip=" + domain
try:
rep = requests.get(url, headers=core.GetHeaders(),timeout=4)
rep = etree.HTML(rep.text)
context = rep.xpath('//div[@style="text-align: center; line-height: 30px;"]/text()')
str = "\n".join(context)
except Exception as e:
print(e)
pass
return str.lstrip()
def CScan(ip):
"""
C段扫描
状态码为200有title时返回title
:param ip:
:return:
"""
try:
rep = requests.get("http://" + ip, headers=core.GetHeaders(), timeout=1, verify=False)
if rep.status_code == 200:
title = re.findall(r'<title>(.*?)</title>', rep.text)
if title:
return "[T]" + ip + ' : ' + title[0] + "\n"
else:
return "[H]" + ip + " : have reason\n"
except Exception as e:
pass
def GetWhois(domain):
"""
get_whois函数爬取http://whois.bugscaner.com/网站的英文搜索结果，并以字符串的方式将结果返回
需要传入一个合法的域名domain
爬取使用的requests 和 xpath 库
:param domain:
:return:
"""
whois_url = 'http://whois.bugscaner.com/'
try:
rep = requests.get(whois_url + domain, headers=core.GetHeaders(), timeout=4.0)
rep = etree.HTML(rep.text)
data = rep.xpath('//div[@class="stats_table_91bf7bf"]/b[not(@style)]/text()')[0:19]
str = "\n".join(data)
except:
str = None
pass
return str
def GetBindingIP(domain):
'''
返回域名的历史解析记录字符串
:param domain:
:return:
'''
pattern = re.compile('^\d+\.\d+\.\d+\.\d+$')
ip138_url = 'https://site.ip138.com/' + domain
try:
rep = requests.get(ip138_url, headers=core.GetHeaders(), timeout=1.0)
rep = etree.HTML(rep.text)
if pattern.findall(domain):
context = rep.xpath('//ul[@id="list"]/li/a/text()')
else:
context = rep.xpath('//div[@id="J_ip_history"]//a/text()')
str = "\n".join(context)
except:
pass
return str
def GetRecordInfo(domain):
'''
返回域名的备案信息
:param domain:
:return:
'''
icpurl='https://icp.chinaz.com/'+domain
context=""
try:
rep = requests.get(icpurl, headers=core.GetHeaders(),timeout=4)
rep = etree.HTML(rep.text)
companyname=rep.xpath('//ul[@id="first"]/li/p/text()')[0]
type=rep.xpath('//ul[@id="first"]/li/p/strong/text()')[0]
icpnum=rep.xpath('//ul[@id="first"]/li/p/font/text()')[0]
wwwname=rep.xpath('//ul[@id="first"]/li/p/text()')[1]
wwwurl=rep.xpath('//ul[@id="first"]/li/p/text()')[2]
icpdate=rep.xpath('//ul[@id="first"]/li/p/text()')[3]
context='''主办单位名称:{}\n主办单位性质:{}\n网站备案许可证号:{}\n网站名称:{}\n网站首页地址:{}\n审核时间:{}\n'''.format(companyname,type,icpnum,wwwname,wwwurl,icpdate)
except Exception as e:
pass
return context
def AngelSwordMain(self):
redispool.append("runlog", "正在使用碎遮内置POC进行{}漏洞检测!\n".format(self.url))
print("正在使用碎遮内置POC进行漏洞检测!")
try:
selfpocscan2.AngelSwordMain(self.url)
except Exception as e:
print(e)
pass
if __name__ == '__main__':
# redispool=redis.ConnectionPool(host='127.0.0.1',port=6379, decode_responses=True)
# redispool = redis.Redis(connection_pool=ImportToRedis.redisPool)
try:
rep = requests.get(url="http://127.0.0.1/",
headers=core.GetHeaders(),
timeout=10)
test = GetBaseMessage("127.0.0.1", "http://127.0.0.1", rep)
print(test.GetDate())
# test.AngelSwordMain()
# print(test.GetStatus())
# print(test.GetTitle())
# print(test.GetResponseHeader())
# print(test.GetFinger())
# print(test.PortScan())
# print(test.SenDir())
except Exception as e:
print(e)
print(">>>>>>>>>超时", "cyan")
pass
def analyze(self, webpage):
"""
Return a list of applications that can be detected on the web page.
"""
detected_apps = set()
for app_name, app in self.apps.items():
if self._has_app(app, webpage):
detected_apps.add(app_name)
detected_apps |= self._get_implied_apps(detected_apps)
return detected_apps
def analyze_with_categories(self, webpage):
detected_apps = self.analyze(webpage)
categorised_apps = {}
for app_name in detected_apps:
cat_names = self.get_categories(app_name)
categorised_apps[app_name] = {"categories": cat_names}
return categorised_apps
if __name__ == '__main__':
url = 'https://www.cnblogs.com/'
rep = requests.get(url, headers=core.GetHeaders(), timeout=2)
finger = WebPage(url, rep)
print(finger.info())
pass
def AngelSwordMain(self):
print("正在使用碎遮内置POC进行漏洞检测!")
try:
selfpocscan.AngelSwordMain(self.url)
except Exception as e:
print(e)
pass
if __name__=='__main__':
# redispool=redis.ConnectionPool(host='127.0.0.1',port=6379, decode_responses=True)
# redispool = redis.Redis(connection_pool=ImportToRedis.redisPool)
try:
rep=requests.get(url="https://www.nowcoder.com",headers=core.GetHeaders(),timeout=10)
test=GetBaseMessage("www.nowcoder.com","https://www.nowcoder.com",rep)
# test.AngelSwordMain()
print(test.GetStatus())
print(test.GetTitle())
print(test.GetResponseHeader())
print(test.GetFinger())
print(test.PortScan())
print(test.SenDir())
except Exception as e:
print(e)
pass
