def robot_txt_parse(self): url = '' robot_directory = set() try: for open_port in list(self.open_ports): if open_port.strip('').endswith('80'): url = "http://" + str(open_port) + '/robot.txt' dictory = self.robot_parse_process(url) if dictory == None: continue for i in dictory: robot_directory.add("http://" + str(open_port) + i) elif open_port.strip().endswith('443'): url = "https://" + str(open_port) + '/robot.txt' dictory = self.robot_parse_process(url) if dictory == None: continue for i in dictory: robot_directory.add("http://" + str(open_port) + i) else: url_http = "http://" + str(open_port) + '/robot.txt' url_https = "https://" + str(open_port) + '/robot.txt' dictory_http = self.robot_parse_process(url_http) if dictory_http == None: continue for i in dictory_http: robot_directory.add("http://" + str(open_port) + i) dictory_https = self.robot_parse_process(url_https) if dictory_https == None: continue for i in dictory_https: robot_directory.add("https://" + str(open_port) + i) except Exception, e: rewrite_logging('HTTPSCAN-ERROR-7', 'the error is %s' % e)
def scan_md5(self): while self.IPs_md5.qsize() > 0: cms = self.IPs_md5.get() url_cms = self.ip + cms['url'] #cms md5 detect try: s = requests.Session() s.mount( 'https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3 r = s.get(str(url_cms).strip(), headers=header, timeout=TimeOut, verify=False, allow_redirects=False) if r.status_code == 200: a = hashlib.md5(r.content).hexdigest() b = cms['md5'].lower() if a == b: cms_detect(self.ip, cms['name']) print '%s discovery cms : %s' % (self.ip, cms['name']) self.IPs_md5.queue.clear() except Exception, e: if 'Connection aborted' in str(e): pass elif 'HTTPConnectionPool' in str(e): pass elif 'HTTPSConnectionPool' in str(e): pass else: rewrite_logging( 'CMSSCAN-ERROR-2', 'ip is %s ,cms url is %s, error is %s' % (self.ip, cms['url'], e))
def scan_api(self): while self.IPs_api.qsize() > 0: url_detect = self.IPs_api.get() cms_api_url = 'https://00sec.me/api/cms?t=' + url_detect try: api_s = requests.Session() api_s.mount( 'https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3 api_r = api_s.get(str(cms_api_url).strip(), headers=header, timeout=TimeOut, verify=False, allow_redirects=False) res_list = api_r.content.split('\n') cms_detect = res_list[2].split(':')[1].split('"')[1] if (cms_detect == 'unknown') or (cms_detect == 'None'): #if scan_api cannot detect the cms,goto scan_md5 self.ip = url_detect self.run() else: cms_detect_log(url_detect, cms_detect) print '%s discovery cms : %s' % (url_detect, cms_detect) print '' self.IPs_api.queue.clear() except Exception, e: rewrite_logging('CMSSCAN-ERROR-1', 'ip is %s , error is %s' % (url_detect, e))
def robot_parse_process(self,ip): directory_list = set() try: s = requests.Session() s.mount('https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3 r = s.get(str(ip).strip(),headers=header,timeout=TimeOut,verify=False,allow_redirects=False) reqs_record_all(ip) res_para = self.get_response_para(ip,response=r) status = res_para[0] if status == 200: disallow_str = re.findall(r'Disallow: \S+',r.content) for i in disallow_str: i = i.replace("*","/") i = i.replace('&','/') i = i.replace('///','/') i = i.replace('//','/') i = i.split(': ')[1] if i == '/': continue if i.endswith('/'): directory_list.add(i) else: return except Exception,e: if 'Connection aborted' in str(e): pass elif 'HTTPConnectionPool' in str(e): pass elif 'HTTPSConnectionPool' in str(e): pass else: message = 'Current IP is %s,the error is %s' % (ip,e) rewrite_logging('HTTPSCAN-ERROR-8','the current url is %s and the error is %s' % (ip,e))
def get_url(self, ip, response): try: page = etree.HTML((response.text.encode('utf-8')).decode('utf-8')) except Exception, e: return rewrite_logging( 'HTTPSCAN-ERROR-5', 'the current ip is %s and the error is %s' % (ip, e))
def request_test(self,url): s = requests.Session() s.mount('https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3 r = s.get(str(url).strip(),headers=header,timeout=TimeOut,verify=False,allow_redirects=False) try: if r.status_code >= 500: return False else: return True except Exception,e: rewrite_logging('HTTPSCAN-ERROR-5','the current url is %s and the error is %s' % (url,e)) return False
def print_log(self,ip,status,banner,title): #write log to file message = "|%-46s|%-4s|%-14s|%-30s|" % (ip.strip(),status,banner,title) rewrite_logging('Result',message) #return the result to cmsscan if '?' in ip: pass elif ip.endswith('/'): self.cms_detect_list.add(ip) else: pass
def url_processor(self,url): # Get the url domain, protocol, and netloc using urlparse try: parsed_url = urlparse.urlparse(url) path = parsed_url.path protocol = parsed_url.scheme+'://' hostname = parsed_url.hostname netloc = parsed_url.netloc doc_domain = '.'.join(hostname.split('.')[-2:]) except: rewrite_logging('HTTPSCAN-ERROR-6','Could not parse url: %s' % url) return return (netloc, protocol, doc_domain, path)
def robot_txt_parse(self,ip): robot_directory = set() try: if not ip.endswith('/'): return else: dictory = self.robot_parse_process(ip + 'robots.txt') if dictory == None: return for i in dictory: robot_directory.add(ip.strip('/')+i) except Exception,e: rewrite_logging('HTTPSCAN-ERROR-7','the error is %s' % e)
def cms_file_load(self): try: with open('data\cms.txt', 'r') as cms_file: cms_array = [] for line in cms_file.readlines(): cms_list = {} cms_str = [] cms_str = line.split(',') cms_list['md5'] = cms_str[0] cms_list['url'] = cms_str[1] cms_list['name'] = cms_str[2] cms_array.append(cms_list) return cms_array except Exception, e: rewrite_logging('CMSSCAN-ERROR-3', 'the error is %s' % e)
def get_response_para(self,ip,response): #get the status status = response.status_code #get the title title = re.search(r'<title>(.*)</title>', response.text) if title: title = title.group(1).strip()[:30] else: title = "No Title Field" #get the private ip private_ip = re.search(r'((2[0-4]\d|25[0-5]|[01]?\d\d?)\.){3}(2[0-4]\d|25[0-5]|[01]?\d\d?)', response.text) if private_ip: ip_str = private_ip.group() if int(ip_str.split('.')[0]) >= 10: rewrite_logging('Result_Find_IP','Get %s from url : %s' % (private_ip.group(),ip)) return (status,title)
def __init__(self, cidr, domain, threads_num, file_source, ports): self.threads_num = threads_num self.ports = ports self.IPs = Queue.Queue() self.file_source = file_source self.domain = domain self.open_ports = set() #ip-port lists self.nohttp_ports = nohttp_ports if self.file_source == None: if domain == None: try: self.cidr = IP(cidr) except Exception, e: rewrite_logging('PORTSCAN-ERROR-1', e) for ip in self.cidr: ip = str(ip) self.IPs.put(ip) else: self.IPs.put(domain)
def redirect_handler_func(self,ip,location): loc_urlparse = urlparse.urlparse(location) ip_urlparse = urlparse.urlparse(ip) if loc_urlparse.netloc.split(':')[0] == ip_urlparse.netloc.split(':')[0]: if location.strip() not in Deduplicate_list: self.IPs.put(location.strip()) Deduplicate_list.add(location.strip()) #gc.collect() rewrite_logging('INFO','rejoin the 302 url: %s' % location) #rejoin the url_directory of locations if location != None: location_url_directory_lists = self.url_parse_func(location) if location_url_directory_lists != None: for x in xrange (0,len(location_url_directory_lists)): url_directory_list = location_url_directory_lists[x] if url_directory_list not in Deduplicate_list: self.IPs.put(url_directory_list) Deduplicate_list.add(url_directory_list) #gc.collect() rewrite_logging('INFO','rejoin the url directory from url of 301/302 : %s' % url_directory_list)
def nmapScan(self): with threading.Lock(): while self.IPs.qsize() > 0: item = self.IPs.get() try: nmScan = nmap.PortScanner() nmScan.scan(item, arguments=self.ports.read()) for tgthost in nmScan.all_hosts(): for tgtport in nmScan[tgthost]['tcp']: tgthost = tgthost.strip() tgtport = int(tgtport) if nmScan[tgthost]['tcp'][tgtport][ 'state'] == 'open': if self.file_source == None: if self.domain == None: open_list = str(tgthost) + ':' + str( tgtport) message = 'the target %s has opened port %s' % ( tgthost, tgtport) if tgtport not in self.nohttp_ports: self.open_ports.add(open_list) rewrite_logging('Result', message) print message + '\n' else: open_list = self.domain + ':' + str( tgtport) message = 'the target %s has opened port %s' % ( self.domain, tgtport) if tgtport not in self.nohttp_ports: self.open_ports.add(open_list) rewrite_logging('Result', message) print message + '\n' else: open_list = str( item.strip()) + ':' + str(tgtport) message = 'the target %s has opened port %s' % ( item.strip(), tgtport) if tgtport not in self.nohttp_ports: self.open_ports.add(open_list) rewrite_logging('Result', message) print message + '\n' except Exception, e: if 'PortScanner' in str(e): print 'Error:' print ' You must reinstall module "python-nmap".' print ' Just do \'pip uninstall python-nmap\'.' print ' Then \'pip install python-nmap\'.\n' rewrite_logging('PORTSCAN-ERROR-2', e) self.IPs.task_done()
def __init__(self, threads_num, open_ports): self.threads_num = threads_num self.IPs = Queue.Queue() #build ip queue self.open_ports = open_ports self.dict_list_file = 'data\dict.txt' #open the path dictionary #Process Bar Config with open(self.dict_list_file, 'r') as dict_lists: self.dict_num = len(dict_lists.readlines()) self.port_num = len(open_ports) self.all_num = (self.port_num * self.dict_num) * 2 self.bar = '#' #parse the robot.txt robot_directory_list = self.robot_txt_parse() for url in robot_directory_list: self.IPs.put(url) rewrite_logging( 'INFO', 'rejoin the url directory from Robot.txt : %s' % url) #self.test = test_list with open(self.dict_list_file, 'r') as dict_lists: for dict_line in dict_lists.readlines(): dict_line = dict_line.strip() for open_port in list(self.open_ports): if open_port.strip().endswith('80'): self.IPs.put("http://" + str(open_port) + str(dict_line)) elif open_port.strip().endswith('443'): self.IPs.put("https://" + str(open_port) + str(dict_line)) else: self.IPs.put("http://" + str(open_port) + str(dict_line)) self.IPs.put("https://" + str(open_port) + str(dict_line)) self.qsize = self.IPs.qsize()
def cms_scan_process(self, ip): url_http = 'http://' + ip url_https = 'https://' + ip s = requests.Session() s.mount('https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3 #http test try: r_http = s.get(str(url_http).strip(), headers=header, timeout=TimeOut, verify=False, allow_redirects=False) if r_http.status_code >= 500: pass else: self.IPs_api.put(url_http) except Exception, e: rewrite_logging( 'CMSSCAN-ERROR-4', 'the current url is %s and the error is %s' % (ip, e)) self.ip = url_http self.run()
def print_log(self, ip, status, banner, title): message = "|%-66s|%-6s|%-14s|%-30s|" % (ip.strip(), status, banner, title) rewrite_logging('Result', message)
def request(self): with threading.Lock(): while self.IPs.qsize() > 0: ip = self.IPs.get() unfinished_num = self.IPs.unfinished_tasks #self.progress_bar(unfinished_num) #Wait to do if ip == None: continue ip_original = ip.strip() ip = self.str_replace(ip) ip_unparse = self.url_unparse_func(ip) if (ip not in Deduplicate_list) and (ip_unparse not in Deduplicate_list): Deduplicate_list.add(ip) Deduplicate_list.add(ip_original) Deduplicate_list.add(ip_unparse) #gc.collect() #self.test.append(ip.strip()) try: #parse the robot.txt robot_directory_list = self.robot_txt_parse(ip) if robot_directory_list != None: for url in robot_directory_list: self.IPs.put(url) rewrite_logging('INFO','rejoin the url directory from Robot.txt : %s' % url) #Start to access page s = requests.Session() s.mount('https:', Ssl3HttpAdapter()) #Mount All Https to ssl.PROTOCOL_SSLV3 r = s.get(str(ip).strip(),headers=header,timeout=TimeOut,verify=False,allow_redirects=False) reqs_record_all(ip) #get_http_parameter res_para = self.get_response_para(ip,response=r) status = res_para[0] title = res_para[1] try: if (self.get_url(ip,response=r)) != None: #In order to further solve the problem of Deduplicate_list reqs_Deduplicated_set = set() reqs = self.get_url(ip,response=r) for x in xrange(0,len(reqs)): req = reqs[x] if req == None: continue #when the url likes 'https://www.test.com/1/2/3/4.php?id=4' #put 'https://www.test.com/1/' and 'https://www.test.com/1/2/' and 'https://www.test.com/1/2/3/' to the queue reqs_Deduplicated_set.add(req) req_url_directory_lists = self.url_parse_func(req) if req_url_directory_lists != None: for x in xrange (0,len(req_url_directory_lists)): req_directory_list = req_url_directory_lists[x] reqs_Deduplicated_set.add(req_directory_list) for x in xrange(0,len(list(reqs_Deduplicated_set))): req_Deduplicated_list = list(reqs_Deduplicated_set)[x] if req_Deduplicated_list in Deduplicate_list: continue else: self.IPs.put(req_Deduplicated_list) Deduplicate_list.add(req_Deduplicated_list) #gc.collect() rewrite_logging('INFO','rejoin the url directory from url of get_url_reqs: %s' % req_Deduplicated_list) except Exception,e: rewrite_logging('HTTPSCAN-ERROR-1','the current ip is %s and the error is %s' % (ip,e)) if ((status == 301) or (status == 302)) and ('404' not in title): if 'Location' in r.headers: try: location = r.headers['Location'] self.redirect_handler_func(ip,location) except Exception,e: rewrite_logging('HTTPSCAN-ERROR-2','the current ip is %s and the error is %s' % (ip,e)) else: try: if 'Server' in r.headers: banner = r.headers['Server'][:20] #get the server banner else: banner = 'No Server Field' self.log_func(ip,ip_original,status,banner,title) except Exception,e: message = 'Current IP is %s,the error is %s' % (ip,e) rewrite_logging('HTTPSCAN-ERROR-3',message) self.log_func(ip,ip_original,status,banner,title) except Exception,e: if 'Connection aborted' in str(e): pass elif 'HTTPConnectionPool' in str(e): pass elif 'HTTPSConnectionPool' in str(e): pass else: message = 'Current IP is %s,the error is %s' % (ip,e) rewrite_logging('HTTPSCAN-ERROR-4',message)
allow_redirects=False) if r_http.status_code >= 500: pass else: self.IPs_api.put(url_http) except Exception, e: rewrite_logging( 'CMSSCAN-ERROR-4', 'the current url is %s and the error is %s' % (ip, e)) self.ip = url_http self.run() #https test try: r_https = s.get(str(url_https).strip(), headers=header, timeout=TimeOut, verify=False, allow_redirects=False) if r_https.status_code >= 500: pass else: self.IPs_api.put(url_https) self.scan_api() except Exception, e: rewrite_logging( 'CMSSCAN-ERROR-5', 'the current url is %s and the error is %s' % (ip, e)) self.ip = url_https self.run()