def get_port_info_nmap(self, ip):
try:
nm = nmap.PortScanner()
nm.scan(hosts=ip, arguments=GLOBAL_NMAP_ARGUMENTS)
result = []
"""
>>> nm['127.0.0.1']['tcp']
{8888: {'product': '', 'state': 'open', 'version': '', 'name': 'sun-answerbook', 'conf': '3', 'extrainfo': '', 'reason': 'syn-ack', 'cpe': ''}, 27017: {'product': '', 'state': 'open', 'version': '', 'name': 'unknown', 'conf': '3', 'extrainfo': '', 'reason': 'syn-ack', 'cpe': ''}, 22: {'product': '', 'state': 'open', 'version': '', 'name': 'ssh', 'conf': '10', 'extrainfo': 'protocol 2.0', 'reason': 'syn-ack', 'cpe': ''}, 50000: {'product': '', 'state': 'open', 'version': '', 'name': 'ibm-db2', 'conf': '3', 'extrainfo': '', 'reason': 'syn-ack', 'cpe': ''}}
"""
try:
for v, k in nm[ip].get('tcp').items():
_ = {}
#print k
try:
_['port'] = v
_['name'] = k['name']
_['version'] = k['version']
_['type'] = k['product'].lower()
result.append(_)
except Exception, e:
wrong_log('get_port_info_nmap ' + str(v))
#wrong_log(str(traceback.format_exc()))
except Exception, e:
pass
#print nm
#wrong_log(traceback.format_exc())
return result
def save_to_db(self, service, port_vuls, http_info, wait_scan, tiny_scan):
print 'save_to_db runing', self.target
try:
if len(service) > 0:
print 'store service'
cron = MongoHelper(self.target)
cron.insert_service(service)
if len(port_vuls) > 0:
print 'store port_vuls'
cron = MongoHelper(self.target)
cron.insert_port_vuls(port_vuls)
if len(http_info) > 0:
print 'store http_info'
cron = MongoHelper(self.target)
cron.insert_http_info(http_info)
if len(tiny_scan) > 0:
print 'store tiny_scan'
cron = MongoHelper(self.target)
cron.insert_tiny_scan(tiny_scan)
if len(wait_scan) > 0:
print 'store wait_scan'
cron = MongoHelper(self.target)
cron.insert_wait_scan(wait_scan)
except Exception, e:
wrong_log(traceback.format_exc())
def save_to_db(self,service,port_vuls,http_info,wait_scan,tiny_scan):
print 'save_to_db runing',self.target
try:
if len(service)>0:
print 'store service'
cron = MongoHelper(self.target)
cron.insert_service(service)
if len(port_vuls) > 0:
print 'store port_vuls'
cron = MongoHelper(self.target)
cron.insert_port_vuls(port_vuls)
if len(http_info) > 0:
print 'store http_info'
cron = MongoHelper(self.target)
cron.insert_http_info(http_info)
if len(tiny_scan) > 0:
print 'store tiny_scan'
cron = MongoHelper(self.target)
cron.insert_tiny_scan(tiny_scan)
if len(wait_scan) > 0:
print 'store wait_scan'
cron = MongoHelper(self.target)
cron.insert_wait_scan(wait_scan)
except Exception,e:
wrong_log(traceback.format_exc())
def get_port_info_nmap(self,ip):
try:
nm = nmap.PortScanner()
nm.scan(hosts=ip, arguments=GLOBAL_NMAP_ARGUMENTS)
result=[]
"""
>>> nm['127.0.0.1']['tcp']
{8888: {'product': '', 'state': 'open', 'version': '', 'name': 'sun-answerbook', 'conf': '3', 'extrainfo': '', 'reason': 'syn-ack', 'cpe': ''}, 27017: {'product': '', 'state': 'open', 'version': '', 'name': 'unknown', 'conf': '3', 'extrainfo': '', 'reason': 'syn-ack', 'cpe': ''}, 22: {'product': '', 'state': 'open', 'version': '', 'name': 'ssh', 'conf': '10', 'extrainfo': 'protocol 2.0', 'reason': 'syn-ack', 'cpe': ''}, 50000: {'product': '', 'state': 'open', 'version': '', 'name': 'ibm-db2', 'conf': '3', 'extrainfo': '', 'reason': 'syn-ack', 'cpe': ''}}
"""
try:
for v, k in nm[ip].get('tcp').items():
_ = {}
#print k
try:
_['port'] = v
_['name'] = k['name']
_['version'] = k['version']
_['type'] = k['product'].lower()
result.append(_)
except Exception,e:
wrong_log('get_port_info_nmap ' + str(v))
#wrong_log(str(traceback.format_exc()))
except Exception, e:
pass
#print nm
#wrong_log(traceback.format_exc())
return result
def detect(self, req):
url = str(req[1])
#domain = url.split('/')[2]
# http://140.207.69.83:80/thread-45758770-10-1.html 待解决
#http:fm.qq.com/category/39110_38942
#http://www.iqiyi.com/w_19ruj41o0l.html
try:
if url:
_ = urlparse.urlparse(url)
if req[0]:
# get 伪静态
if _[4]:
pass
elif '_' in url or '-' in url:
pass
else:
return 0
#if self.ip in domain or self.target in domain: # detect domain area
some_control_word = [
'method',
'm',
'action',
'act',
'cmd',
'commmand',
'ac',
]
url = re.sub(r'[\w_-]{28,}', '', url)
url = re.sub(r'[\w_-]{16}', '', url)
url = re.sub(r'[\d]{10,13}', '', url)
_ = urlparse.urlparse(url)
params = urlparse.parse_qs(_[4])
path = re.sub(r'\d+', '', _[2])
path_tmp = path.split('/')
path_new = ''
for i in path_tmp:
count = i.count('_') + i.count('-')
if count:
path_new = path_new + '/' + str(count)
else:
path_new = path_new + '/' + i
tmp = _[0] + _[1] + path_new
for i in some_control_word:
if i in params:
try:
tmp += params[i][0]
except:
pass
#wrong_log(params[i][0])
if tmp in self.detecion:
return 0
else:
self.detecion.append(tmp)
self.req_scan.append(req)
return 1
except Exception, e:
wrong_log(traceback.format_exc() + url)
def get_title(text):
try:
soup = BeautifulSoup(text, "html.parser")
if soup.title:
return soup.title.string
else:
return text[:20]
except Exception, e:
wrong_log(traceback.format_exc())
def get_title(text):
try:
soup = BeautifulSoup(text, "html.parser")
if soup.title:
return soup.title.string
else:
return text[:20]
except Exception,e:
wrong_log(traceback.format_exc())
def scan(self,ipinfo):
try:
service = []
port_vuls = [] # id ip_domain port type version
http_info = [] # id ip url title src tinyscan - ｛ url_domain title src isvul ｝
wait_scan = []
tiny_scan = []
_id = ipinfo['_id']
ip = ipinfo['ip']
print 'scan', ip
if '|sep|' in ipinfo['domain']:
domains = ipinfo['domain'].split('|sep|')[0].split('|')
else:
domains = ipinfo['domain'].split('|')
cron = MongoHelper(self.target)
cron.update_ip_list_status(_id)
if self.new:
portinfos = self.get_port_info_nmap(ip)
else:
portinfos = ipinfo['portinfo']
if portinfos == 0:
print 'nmap error'
return 0
cron = MongoHelper(self.target)
cron.update_ip_list(_id,portinfos)
# portinfo [{port,type,version},]
for portinfo in portinfos:
for domain in domains:
webinfo = self.get_web_info(ip,portinfo['port'],domain)
if webinfo:
#http
ip_domain = ip+'-'+domain
service.append({'ip_domain':ip_domain,'port':portinfo['port'],'type':portinfo['type'],'version':portinfo['version']}) # id ip_domain port type version
# service Redundancy
#info_log('append service' + ip_domain)
if 'https' in webinfo['scheme']:
ssl_info = heratbleed_attack(ip,portinfo['port'])
if ssl_info:
port_vuls.append(ssl_info)
self.http_tiny_scan(webinfo, ip, portinfo['port'], domain,http_info,wait_scan,tiny_scan)
else:
if self.send_to_http_port_plugins(webinfo,ip,portinfo['port'],domain,port_vuls):
self.http_tiny_scan(webinfo,ip,portinfo['port'],domain,http_info,wait_scan,tiny_scan)
self.send_to_normal_plugins(ip, portinfo, domain, port_vuls)
# something wrong strtus2 weblojic
else:
#normal
service.append({'ip_domain': ip, 'port': portinfo['port'], 'type': portinfo['type'],'version': portinfo['version']})
self.send_to_normal_plugins(ip,portinfo,domain,port_vuls)
#info_log('append service'+ip)
break
self.save_to_db(service,port_vuls,http_info,wait_scan,tiny_scan)
except Exception,e:
wrong_log(traceback.format_exc())
def run(self):
while self.reqque.qsize() > 0:
try:
_ = self.reqque.get()
html = self.get_html(_[0])
reqs = self.get_links_forms(html, _[0][1])
if _[1] < self.depth:
new_dep = _[1] + 1
for i in reqs:
self.reqque.put([i, new_dep])
except Exception, e:
wrong_log(traceback.format_exc())
def run(self):
while self.reqque.qsize()>0:
try:
_ = self.reqque.get()
html = self.get_html(_[0])
reqs = self.get_links_forms(html,_[0][1])
if _[1] < self.depth:
new_dep = _[1] + 1
for i in reqs:
self.reqque.put([i,new_dep])
except Exception,e:
wrong_log(traceback.format_exc())
def detect(self,req):
url = str(req[1])
#domain = url.split('/')[2]
# http://140.207.69.83:80/thread-45758770-10-1.html 待解决
#http:fm.qq.com/category/39110_38942
#http://www.iqiyi.com/w_19ruj41o0l.html
try:
if url:
_ = urlparse.urlparse(url)
if req[0]:
# get 伪静态
if _[4]:
pass
elif '_' in url or '-' in url:
pass
else:
return 0
#if self.ip in domain or self.target in domain: # detect domain area
some_control_word = ['method','m','action','act','cmd','commmand','ac',]
url = re.sub(r'[\w_-]{28,}', '', url)
url = re.sub(r'[\w_-]{16}', '', url)
url = re.sub(r'[\d]{10,13}', '', url)
_ = urlparse.urlparse(url)
params = urlparse.parse_qs(_[4])
path = re.sub(r'\d+', '', _[2])
path_tmp = path.split('/')
path_new = ''
for i in path_tmp:
count = i.count('_') + i.count('-')
if count:
path_new = path_new + '/' + str(count)
else:
path_new = path_new + '/' + i
tmp = _[0] + _[1] + path_new
for i in some_control_word:
if i in params:
try:
tmp += params[i][0]
except:
pass
#wrong_log(params[i][0])
if tmp in self.detecion:
return 0
else:
self.detecion.append(tmp)
self.req_scan.append(req)
return 1
except Exception, e:
wrong_log(traceback.format_exc()+url)
def get_links_forms(self, html, url):
reqs = []
if isinstance(html, str):
try:
etreeHtml = etree.HTML(html)
title = etreeHtml.xpath('//title/text()')
if title:
if self.black_detect(title[0]):
return []
links = etreeHtml.xpath('//a[@href]/@href')
# get forms
for i in links:
if i:
if 'javascript' in i or 'mailto' in i:
pass
else:
if '//' not in i:
if i[0] == '/':
i = get_current_host(url) + i
else:
i = get_current_path(url) + i
elif i[:2] == '//':
i = get_scheme(url) + i
if self.detect(['GET', i, '']):
#info_log(i)
reqs.append(['GET', i, ''])
forms = etreeHtml.xpath('//form')
for i in forms:
try:
action = str(i.xpath('@action')[0])
if 'javascript' in i or 'mailto' in action:
continue
else:
if '//' not in action:
if action[0] == '/':
action = get_current_host(url) + action
else:
action = get_current_path(url) + action
elif i[:2] == '//':
action = get_scheme(url) + action
if self.detect(['post', i, '']):
# info_log(i)
reqs.append(['post', i, ''])
inputs = i.xpath('//input')
data = ''
for t in inputs:
name = ''
value = ''
try:
name = t.xpath('@name')[0]
value = t.xpath('@value')[0]
except:
pass
#wrong_log(str(t.xpath('@name')))
#wrong_log(str(t.xpath('@value')))
data = data + name + '=' + value + '&'
if self.detect(['post', action, data[:-1]]):
reqs.append(['post', action, data[:-1]
]) # data[:-1] delet the last &
except Exception, e:
wrong_log(url + traceback.format_exc())
except Exception, e:
wrong_log(traceback.format_exc())
def scan(self, ipinfo):
try:
service = []
port_vuls = [] # id ip_domain port type version
http_info = [
] # id ip url title src tinyscan - ｛ url_domain title src isvul ｝
wait_scan = []
tiny_scan = []
_id = ipinfo['_id']
ip = ipinfo['ip']
print 'scan', ip
if '|sep|' in ipinfo['domain']:
domains = ipinfo['domain'].split('|sep|')[0].split('|')
else:
domains = ipinfo['domain'].split('|')
cron = MongoHelper(self.target)
cron.update_ip_list_status(_id)
if self.new:
portinfos = self.get_port_info_nmap(ip)
else:
portinfos = ipinfo['portinfo']
if portinfos == 0:
print 'nmap error'
return 0
cron = MongoHelper(self.target)
cron.update_ip_list(_id, portinfos)
# portinfo [{port,type,version},]
for portinfo in portinfos:
for domain in domains:
webinfo = self.get_web_info(ip, portinfo['port'], domain)
if webinfo:
#http
ip_domain = ip + '-' + domain
service.append({
'ip_domain': ip_domain,
'port': portinfo['port'],
'type': portinfo['type'],
'version': portinfo['version']
}) # id ip_domain port type version
# service Redundancy
#info_log('append service' + ip_domain)
if 'https' in webinfo['scheme']:
ssl_info = heratbleed_attack(ip, portinfo['port'])
if ssl_info:
port_vuls.append(ssl_info)
self.http_tiny_scan(webinfo, ip, portinfo['port'],
domain, http_info, wait_scan,
tiny_scan)
else:
if self.send_to_http_port_plugins(
webinfo, ip, portinfo['port'], domain,
port_vuls):
self.http_tiny_scan(webinfo, ip,
portinfo['port'], domain,
http_info, wait_scan,
tiny_scan)
self.send_to_normal_plugins(ip, portinfo, domain,
port_vuls)
# something wrong strtus2 weblojic
else:
#normal
service.append({
'ip_domain': ip,
'port': portinfo['port'],
'type': portinfo['type'],
'version': portinfo['version']
})
self.send_to_normal_plugins(ip, portinfo, domain,
port_vuls)
#info_log('append service'+ip)
break
self.save_to_db(service, port_vuls, http_info, wait_scan,
tiny_scan)
except Exception, e:
wrong_log(traceback.format_exc())
def get_links_forms(self,html,url):
reqs = []
if isinstance(html, str):
try:
etreeHtml = etree.HTML(html)
title = etreeHtml.xpath('//title/text()')
if title:
if self.black_detect(title[0]):
return []
links = etreeHtml.xpath('//a[@href]/@href')
# get forms
for i in links:
if i:
if 'javascript' in i or 'mailto' in i:
pass
else:
if '//' not in i:
if i[0] == '/':
i = get_current_host(url) + i
else:
i = get_current_path(url) + i
elif i[:2] == '//':
i = get_scheme(url) + i
if self.detect(['GET',i,'']):
#info_log(i)
reqs.append(['GET',i,''])
forms = etreeHtml.xpath('//form')
for i in forms:
try:
action = str(i.xpath('@action')[0])
if 'javascript' in i or 'mailto' in action:
continue
else:
if '//' not in action:
if action[0] == '/':
action = get_current_host(url) + action
else:
action = get_current_path(url) + action
elif i[:2] == '//':
action = get_scheme(url) + action
if self.detect(['post', i, '']):
# info_log(i)
reqs.append(['post', i, ''])
inputs = i.xpath('//input')
data = ''
for t in inputs:
name = ''
value = ''
try:
name = t.xpath('@name')[0]
value = t.xpath('@value')[0]
except:
pass
#wrong_log(str(t.xpath('@name')))
#wrong_log(str(t.xpath('@value')))
data = data + name + '=' + value + '&'
if self.detect(['post',action,data[:-1]]):
reqs.append(['post',action,data[:-1]]) # data[:-1] delet the last &
except Exception, e:
wrong_log(url+traceback.format_exc())
except Exception, e:
wrong_log(traceback.format_exc())