def robot_txt_parse(self):
url = ''
robot_directory = set()
try:
for open_port in list(self.open_ports):
if open_port.strip('').endswith('80'):
url = "http://" + str(open_port) + '/robot.txt'
dictory = self.robot_parse_process(url)
if dictory == None:
continue
for i in dictory:
robot_directory.add("http://" + str(open_port) + i)
elif open_port.strip().endswith('443'):
url = "https://" + str(open_port) + '/robot.txt'
dictory = self.robot_parse_process(url)
if dictory == None:
continue
for i in dictory:
robot_directory.add("http://" + str(open_port) + i)
else:
url_http = "http://" + str(open_port) + '/robot.txt'
url_https = "https://" + str(open_port) + '/robot.txt'
dictory_http = self.robot_parse_process(url_http)
if dictory_http == None:
continue
for i in dictory_http:
robot_directory.add("http://" + str(open_port) + i)
dictory_https = self.robot_parse_process(url_https)
if dictory_https == None:
continue
for i in dictory_https:
robot_directory.add("https://" + str(open_port) + i)
except Exception, e:
rewrite_logging('HTTPSCAN-ERROR-7', 'the error is %s' % e)
def scan_md5(self):
while self.IPs_md5.qsize() > 0:
cms = self.IPs_md5.get()
url_cms = self.ip + cms['url']
#cms md5 detect
try:
s = requests.Session()
s.mount(
'https:',
Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3
r = s.get(str(url_cms).strip(),
headers=header,
timeout=TimeOut,
verify=False,
allow_redirects=False)
if r.status_code == 200:
a = hashlib.md5(r.content).hexdigest()
b = cms['md5'].lower()
if a == b:
cms_detect(self.ip, cms['name'])
print '%s discovery cms : %s' % (self.ip, cms['name'])
self.IPs_md5.queue.clear()
except Exception, e:
if 'Connection aborted' in str(e):
pass
elif 'HTTPConnectionPool' in str(e):
pass
elif 'HTTPSConnectionPool' in str(e):
pass
else:
rewrite_logging(
'CMSSCAN-ERROR-2',
'ip is %s ,cms url is %s, error is %s' %
(self.ip, cms['url'], e))
def scan_api(self):
while self.IPs_api.qsize() > 0:
url_detect = self.IPs_api.get()
cms_api_url = 'https://00sec.me/api/cms?t=' + url_detect
try:
api_s = requests.Session()
api_s.mount(
'https:',
Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3
api_r = api_s.get(str(cms_api_url).strip(),
headers=header,
timeout=TimeOut,
verify=False,
allow_redirects=False)
res_list = api_r.content.split('\n')
cms_detect = res_list[2].split(':')[1].split('"')[1]
if (cms_detect == 'unknown') or (cms_detect == 'None'):
#if scan_api cannot detect the cms,goto scan_md5
self.ip = url_detect
self.run()
else:
cms_detect_log(url_detect, cms_detect)
print '%s discovery cms : %s' % (url_detect, cms_detect)
print ''
self.IPs_api.queue.clear()
except Exception, e:
rewrite_logging('CMSSCAN-ERROR-1',
'ip is %s , error is %s' % (url_detect, e))
def robot_parse_process(self,ip):
directory_list = set()
try:
s = requests.Session()
s.mount('https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3
r = s.get(str(ip).strip(),headers=header,timeout=TimeOut,verify=False,allow_redirects=False)
reqs_record_all(ip)
res_para = self.get_response_para(ip,response=r)
status = res_para[0]
if status == 200:
disallow_str = re.findall(r'Disallow: \S+',r.content)
for i in disallow_str:
i = i.replace("*","/")
i = i.replace('&','/')
i = i.replace('///','/')
i = i.replace('//','/')
i = i.split(': ')[1]
if i == '/':
continue
if i.endswith('/'):
directory_list.add(i)
else:
return
except Exception,e:
if 'Connection aborted' in str(e):
pass
elif 'HTTPConnectionPool' in str(e):
pass
elif 'HTTPSConnectionPool' in str(e):
pass
else:
message = 'Current IP is %s,the error is %s' % (ip,e)
rewrite_logging('HTTPSCAN-ERROR-8','the current url is %s and the error is %s' % (ip,e))
def get_url(self, ip, response):
try:
page = etree.HTML((response.text.encode('utf-8')).decode('utf-8'))
except Exception, e:
return
rewrite_logging(
'HTTPSCAN-ERROR-5',
'the current ip is %s and the error is %s' % (ip, e))
def request_test(self,url):
s = requests.Session()
s.mount('https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3
r = s.get(str(url).strip(),headers=header,timeout=TimeOut,verify=False,allow_redirects=False)
try:
if r.status_code >= 500:
return False
else:
return True
except Exception,e:
rewrite_logging('HTTPSCAN-ERROR-5','the current url is %s and the error is %s' % (url,e))
return False
def print_log(self,ip,status,banner,title):
#write log to file
message = "|%-46s|%-4s|%-14s|%-30s|" % (ip.strip(),status,banner,title)
rewrite_logging('Result',message)
#return the result to cmsscan
if '?' in ip:
pass
elif ip.endswith('/'):
self.cms_detect_list.add(ip)
else:
pass
def url_processor(self,url): # Get the url domain, protocol, and netloc using urlparse
try:
parsed_url = urlparse.urlparse(url)
path = parsed_url.path
protocol = parsed_url.scheme+'://'
hostname = parsed_url.hostname
netloc = parsed_url.netloc
doc_domain = '.'.join(hostname.split('.')[-2:])
except:
rewrite_logging('HTTPSCAN-ERROR-6','Could not parse url: %s' % url)
return
return (netloc, protocol, doc_domain, path)
def robot_txt_parse(self,ip):
robot_directory = set()
try:
if not ip.endswith('/'):
return
else:
dictory = self.robot_parse_process(ip + 'robots.txt')
if dictory == None:
return
for i in dictory:
robot_directory.add(ip.strip('/')+i)
except Exception,e:
rewrite_logging('HTTPSCAN-ERROR-7','the error is %s' % e)
def cms_file_load(self):
try:
with open('data\cms.txt', 'r') as cms_file:
cms_array = []
for line in cms_file.readlines():
cms_list = {}
cms_str = []
cms_str = line.split(',')
cms_list['md5'] = cms_str[0]
cms_list['url'] = cms_str[1]
cms_list['name'] = cms_str[2]
cms_array.append(cms_list)
return cms_array
except Exception, e:
rewrite_logging('CMSSCAN-ERROR-3', 'the error is %s' % e)
def get_response_para(self,ip,response):
#get the status
status = response.status_code
#get the title
title = re.search(r'<title>(.*)</title>', response.text)
if title:
title = title.group(1).strip()[:30]
else:
title = "No Title Field"
#get the private ip
private_ip = re.search(r'((2[0-4]\d|25[0-5]|[01]?\d\d?)\.){3}(2[0-4]\d|25[0-5]|[01]?\d\d?)', response.text)
if private_ip:
ip_str = private_ip.group()
if int(ip_str.split('.')[0]) >= 10:
rewrite_logging('Result_Find_IP','Get %s from url : %s' % (private_ip.group(),ip))
return (status,title)
def __init__(self, cidr, domain, threads_num, file_source, ports):
self.threads_num = threads_num
self.ports = ports
self.IPs = Queue.Queue()
self.file_source = file_source
self.domain = domain
self.open_ports = set() #ip-port lists
self.nohttp_ports = nohttp_ports
if self.file_source == None:
if domain == None:
try:
self.cidr = IP(cidr)
except Exception, e:
rewrite_logging('PORTSCAN-ERROR-1', e)
for ip in self.cidr:
ip = str(ip)
self.IPs.put(ip)
else:
self.IPs.put(domain)
def redirect_handler_func(self,ip,location):
loc_urlparse = urlparse.urlparse(location)
ip_urlparse = urlparse.urlparse(ip)
if loc_urlparse.netloc.split(':')[0] == ip_urlparse.netloc.split(':')[0]:
if location.strip() not in Deduplicate_list:
self.IPs.put(location.strip())
Deduplicate_list.add(location.strip())
#gc.collect()
rewrite_logging('INFO','rejoin the 302 url: %s' % location)
#rejoin the url_directory of locations
if location != None:
location_url_directory_lists = self.url_parse_func(location)
if location_url_directory_lists != None:
for x in xrange (0,len(location_url_directory_lists)):
url_directory_list = location_url_directory_lists[x]
if url_directory_list not in Deduplicate_list:
self.IPs.put(url_directory_list)
Deduplicate_list.add(url_directory_list)
#gc.collect()
rewrite_logging('INFO','rejoin the url directory from url of 301/302 : %s' % url_directory_list)
def nmapScan(self):
with threading.Lock():
while self.IPs.qsize() > 0:
item = self.IPs.get()
try:
nmScan = nmap.PortScanner()
nmScan.scan(item, arguments=self.ports.read())
for tgthost in nmScan.all_hosts():
for tgtport in nmScan[tgthost]['tcp']:
tgthost = tgthost.strip()
tgtport = int(tgtport)
if nmScan[tgthost]['tcp'][tgtport][
'state'] == 'open':
if self.file_source == None:
if self.domain == None:
open_list = str(tgthost) + ':' + str(
tgtport)
message = 'the target %s has opened port %s' % (
tgthost, tgtport)
if tgtport not in self.nohttp_ports:
self.open_ports.add(open_list)
rewrite_logging('Result', message)
print message + '\n'
else:
open_list = self.domain + ':' + str(
tgtport)
message = 'the target %s has opened port %s' % (
self.domain, tgtport)
if tgtport not in self.nohttp_ports:
self.open_ports.add(open_list)
rewrite_logging('Result', message)
print message + '\n'
else:
open_list = str(
item.strip()) + ':' + str(tgtport)
message = 'the target %s has opened port %s' % (
item.strip(), tgtport)
if tgtport not in self.nohttp_ports:
self.open_ports.add(open_list)
rewrite_logging('Result', message)
print message + '\n'
except Exception, e:
if 'PortScanner' in str(e):
print 'Error:'
print ' You must reinstall module "python-nmap".'
print ' Just do \'pip uninstall python-nmap\'.'
print ' Then \'pip install python-nmap\'.\n'
rewrite_logging('PORTSCAN-ERROR-2', e)
self.IPs.task_done()
def __init__(self, threads_num, open_ports):
self.threads_num = threads_num
self.IPs = Queue.Queue() #build ip queue
self.open_ports = open_ports
self.dict_list_file = 'data\dict.txt' #open the path dictionary
#Process Bar Config
with open(self.dict_list_file, 'r') as dict_lists:
self.dict_num = len(dict_lists.readlines())
self.port_num = len(open_ports)
self.all_num = (self.port_num * self.dict_num) * 2
self.bar = '#'
#parse the robot.txt
robot_directory_list = self.robot_txt_parse()
for url in robot_directory_list:
self.IPs.put(url)
rewrite_logging(
'INFO', 'rejoin the url directory from Robot.txt : %s' % url)
#self.test = test_list
with open(self.dict_list_file, 'r') as dict_lists:
for dict_line in dict_lists.readlines():
dict_line = dict_line.strip()
for open_port in list(self.open_ports):
if open_port.strip().endswith('80'):
self.IPs.put("http://" + str(open_port) +
str(dict_line))
elif open_port.strip().endswith('443'):
self.IPs.put("https://" + str(open_port) +
str(dict_line))
else:
self.IPs.put("http://" + str(open_port) +
str(dict_line))
self.IPs.put("https://" + str(open_port) +
str(dict_line))
self.qsize = self.IPs.qsize()
def cms_scan_process(self, ip):
url_http = 'http://' + ip
url_https = 'https://' + ip
s = requests.Session()
s.mount('https:',
Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3
#http test
try:
r_http = s.get(str(url_http).strip(),
headers=header,
timeout=TimeOut,
verify=False,
allow_redirects=False)
if r_http.status_code >= 500:
pass
else:
self.IPs_api.put(url_http)
except Exception, e:
rewrite_logging(
'CMSSCAN-ERROR-4',
'the current url is %s and the error is %s' % (ip, e))
self.ip = url_http
self.run()
def print_log(self, ip, status, banner, title):
message = "|%-66s|%-6s|%-14s|%-30s|" % (ip.strip(), status, banner,
title)
rewrite_logging('Result', message)
def request(self):
with threading.Lock():
while self.IPs.qsize() > 0:
ip = self.IPs.get()
unfinished_num = self.IPs.unfinished_tasks
#self.progress_bar(unfinished_num) #Wait to do
if ip == None:
continue
ip_original = ip.strip()
ip = self.str_replace(ip)
ip_unparse = self.url_unparse_func(ip)
if (ip not in Deduplicate_list) and (ip_unparse not in Deduplicate_list):
Deduplicate_list.add(ip)
Deduplicate_list.add(ip_original)
Deduplicate_list.add(ip_unparse)
#gc.collect()
#self.test.append(ip.strip())
try:
#parse the robot.txt
robot_directory_list = self.robot_txt_parse(ip)
if robot_directory_list != None:
for url in robot_directory_list:
self.IPs.put(url)
rewrite_logging('INFO','rejoin the url directory from Robot.txt : %s' % url)
#Start to access page
s = requests.Session()
s.mount('https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3
r = s.get(str(ip).strip(),headers=header,timeout=TimeOut,verify=False,allow_redirects=False)
reqs_record_all(ip)
#get_http_parameter
res_para = self.get_response_para(ip,response=r)
status = res_para[0]
title = res_para[1]
try:
if (self.get_url(ip,response=r)) != None:
#In order to further solve the problem of Deduplicate_list
reqs_Deduplicated_set = set()
reqs = self.get_url(ip,response=r)
for x in xrange(0,len(reqs)):
req = reqs[x]
if req == None:
continue
#when the url likes 'https://www.test.com/1/2/3/4.php?id=4'
#put 'https://www.test.com/1/' and 'https://www.test.com/1/2/' and 'https://www.test.com/1/2/3/' to the queue
reqs_Deduplicated_set.add(req)
req_url_directory_lists = self.url_parse_func(req)
if req_url_directory_lists != None:
for x in xrange (0,len(req_url_directory_lists)):
req_directory_list = req_url_directory_lists[x]
reqs_Deduplicated_set.add(req_directory_list)
for x in xrange(0,len(list(reqs_Deduplicated_set))):
req_Deduplicated_list = list(reqs_Deduplicated_set)[x]
if req_Deduplicated_list in Deduplicate_list:
continue
else:
self.IPs.put(req_Deduplicated_list)
Deduplicate_list.add(req_Deduplicated_list)
#gc.collect()
rewrite_logging('INFO','rejoin the url directory from url of get_url_reqs: %s' % req_Deduplicated_list)
except Exception,e:
rewrite_logging('HTTPSCAN-ERROR-1','the current ip is %s and the error is %s' % (ip,e))
if ((status == 301) or (status == 302)) and ('404' not in title):
if 'Location' in r.headers:
try:
location = r.headers['Location']
self.redirect_handler_func(ip,location)
except Exception,e:
rewrite_logging('HTTPSCAN-ERROR-2','the current ip is %s and the error is %s' % (ip,e))
else:
try:
if 'Server' in r.headers:
banner = r.headers['Server'][:20] #get the server banner
else:
banner = 'No Server Field'
self.log_func(ip,ip_original,status,banner,title)
except Exception,e:
message = 'Current IP is %s,the error is %s' % (ip,e)
rewrite_logging('HTTPSCAN-ERROR-3',message)
self.log_func(ip,ip_original,status,banner,title)
except Exception,e:
if 'Connection aborted' in str(e):
pass
elif 'HTTPConnectionPool' in str(e):
pass
elif 'HTTPSConnectionPool' in str(e):
pass
else:
message = 'Current IP is %s,the error is %s' % (ip,e)
rewrite_logging('HTTPSCAN-ERROR-4',message)
allow_redirects=False)
if r_http.status_code >= 500:
pass
else:
self.IPs_api.put(url_http)
except Exception, e:
rewrite_logging(
'CMSSCAN-ERROR-4',
'the current url is %s and the error is %s' % (ip, e))
self.ip = url_http
self.run()
#https test
try:
r_https = s.get(str(url_https).strip(),
headers=header,
timeout=TimeOut,
verify=False,
allow_redirects=False)
if r_https.status_code >= 500:
pass
else:
self.IPs_api.put(url_https)
self.scan_api()
except Exception, e:
rewrite_logging(
'CMSSCAN-ERROR-5',
'the current url is %s and the error is %s' % (ip, e))
self.ip = url_https
self.run()