def verify_https(url): # 验证域名是http或者https的 # 如果域名是302跳转 则获取跳转后的地址 req = Requests() # noinspection PyBroadException if '://' in url: try: r = req.get(url) return url except Exception as e: pass host = parse_host(url) url2 = parse.urlparse(url) if url2.netloc: url = url2.netloc elif url2.path: url = url2.path # noinspection PyBroadException try: r = req.get('https://' + url) getattr(r, 'status_code') console('Verify', host, 'https://' + url + '\n') return 'https://' + url except AttributeError: # noinspection PyBroadException try: req.get('http://' + url) console('Verify', host, 'http://' + url + '\n') return 'http://' + url except Exception: pass except Exception as e: logging.exception(e)
def responseHandler(self, response): ''' @description: 处理响应结果 @param {type} @return: ''' # 结果处理阶段 try: size = self.intToSize(int(response.headers['content-length'])) except (KeyError, ValueError): size = self.intToSize(len(response.content)) # 跳过大小为skip_size的页面 if size == "None": return #与404页面进行匹配 if hashlib.md5( response.content).hexdigest() in self.autodiscriminator_md5: return # 自定义状态码显示 if response.status_code in [ 200, ]: msg = response.url host2 = self.url.replace('http://', '').replace('https://', '').rstrip('/') console('URLS', host2, msg + '\n') # 目标展示 data = { host2: { "rsp_code": str(response.status_code), "contype": response.headers.get('content-type'), "url": msg } } self.outjson.append(data)
def pool(self): out = [] try: # 判断给出的url是www.baiud.com还是www.baidu.com/path这种形式 if (not parse.urlparse(self.ipaddr).path) and (parse.urlparse(self.ipaddr).path != '/'): #没有路径而且 最后也不存在/ self.ipaddr = self.ipaddr.replace('http://', '').replace('https://', '').rstrip('/') #url切换成完整的url www.baidu.com else: self.ipaddr = self.ipaddr.replace('http://', '').replace('https://', '').rstrip('/') #www.baidu.com/programing/virustotal/ self.ipaddr = re.sub(r'/\w+', '', self.ipaddr) #www.baidu.com if re.search(r'\d+\.\d+\.\d+\.\d+', self.ipaddr): #ip匹配 ipaddr = self.ipaddr else: ipaddr = socket.gethostbyname(self.ipaddr) # gethostbyname 返回的是 主机名url 的IPv4 的地址格式 if ':' in ipaddr: ipaddr = re.sub(r':\d+', '', ipaddr) #去掉端口号 self.run(ipaddr) #返回端口信息 except Exception as e: pass self.save(self.ipaddr, self.out) for _ in self.out: out.append('{}:{}'.format(_.get('server'), _.get('port'))) console('PortScan', self.ipaddr, '{}:{}\n'.format(_.get('server'), _.get('port'))) #页面展示信息
def check(self, url): loc = parse.urlparse(url) if getattr(loc, 'netloc'): host = loc.netloc else: host = loc.path try: if not re.search(r'\d+\.\d+\.\d+\.\d+', host): dns.resolver.query(host, 'A') if PING: try: if platform.system() == 'Windows': subprocess.check_output( ['ping', '-n', '2', '-w', '1', host]) else: subprocess.check_output(['ping', '-c 2', '-W 1', host]) self.out.append(url) except subprocess.CalledProcessError: console('PING', host, "is not alive\n") return False except Exception as e: logging.exception(e) else: self.out.append(url) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): console('DnsCheck', host, "No A record\n") except Exception as e: logging.exception(e) return False
def worker(self): current_payload = self.all_task.get() try: console('sub_domain', self.target_domain, current_payload + '\n') web_info(current_payload, flags=2) except: pass
def run(self): scripts = [] try: for _ in glob.glob('script/*.py'): script_name = os.path.basename(_).replace('.py', '') vuln = importlib.import_module('script.%s' % script_name) scripts.append(vuln) # 随机打乱脚本加载顺序 random.shuffle(scripts) with concurrent.futures.ThreadPoolExecutor( max_workers=20) as executor: vulns = { executor.submit(self.vuln, script): script for script in scripts } for future in concurrent.futures.as_completed(vulns, timeout=3): future.result() self.out = list(filter(None, self.out)) for i in self.out: console('Vuln', self.ip, i + '\n') Sqldb(self.dbname).get_vuln(self.ip, self.out) except (EOFError, concurrent.futures._base.TimeoutError): pass except Exception as e: logging.exception(e)
def query_db(self, hosts): result = [] error = False for i in hosts: try: domain = parse_host(i) cursor = self.conn.cursor() sql = "select 1 from webinfo where domain = '{}' limit 1".format(domain) cursor.execute(sql) values = cursor.fetchall() if not values: result.append(i) else: console('CheckDB', i, 'In the db file\n') # sys.stdout.write(Bcolors.OKGREEN + "{} In the db file\n".format(i) + Bcolors.ENDC) except sqlite3.OperationalError: return hosts except Exception as e: error = True logging.exception(e) self.commit() self.close() if error: return hosts else: return result
def virustotal(host): # VT接口,主要用来查询PDNS,绕过CDN pdns = [] history_ip = [] # sys.stdout.write(bcolors.RED + "\nPDNS:\n" + bcolors.ENDC) if VIRUSTOTAL_API: try: vtotal = Virustotal(VIRUSTOTAL_API) if re.search(r'\d+\.\d+\.\d+\.\d+', host): return None resp = vtotal.domain_report(host) if resp.get('status_code') != 403: for i in resp.get('json_resp').get('resolutions'): address = i.get('ip_address') timeout = i.get('last_resolved') if iscdn(address): history_ip.append(address + ' : ' + timeout) pdns = history_ip[10:] except: pass pdns.extend(ipinfo(host)) if pdns: for i in pdns[:10]: console('PDNS', host, i + '\n') else: console('PDNS', host, 'None\n') return pdns
def disable(self): # 求生欲名单 # 禁止扫描所有gov.cn与edu.cn结尾的域名,遵守法律!!! for i in self.out: if re.search(r'gov\.cn|edu\.cn$', i): console('Disable', i, "Do not scan this domain\n") sys.exit(1)
def disable(self): # 禁止扫描所有敏感域名,遵守法律!!! for i in self.out: if re.search( r'\.org\.cn|\.com\.cn|\.cn|gov\.cn|edu\.cn|\.mil|\.aero|\.int|\.go\.\w+$|\.ac\.\w+$', i): console('Disable', i, "Do not scan this domain\n\n") sys.exit(1)
def psqlBruteforce(task): address, username, password = task.split('|') try: conn = psycopg2.connect(host=address, port=5432, user=username, password=password) result = 'IP: ' + address + ' Postgresql User: '******' Pass: '******'BruteForce', address, result) return result except Exception as e: pass
def osdetect(ip): #识别操作系统 nm = nmap.PortScanner() #实例化 try: result = nm.scan(hosts=ip, arguments='-sS -O -vv -n -T4 -p 80,22,443') #参数设置 for k, v in result.get('scan').items(): if v.get('osmatch'): for i in v.get('osmatch'): console('OSdetect', ip, i.get('name') + '\n') #显示 return i.get('name') #返回操作系统名称 else: break except Exception as e: console('OSdetect', ip, 'None\n')
def pool(self): result = self.parse_html(self.host) with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor: executor.map(self.parse_html, result) jslink = JsLeaks().pool(self.js) self.result.extend(jslink) self.links = dedup_link(self.links) self.links = list(map(lambda x: 'Dynamic: ' + x, self.links)) self.result.extend(self.links) self.result = list(set(self.result)) for i in self.result: console('Crawl', self.host, i + '\n') Sqldb('result').get_crawl(self.domain, self.result)
def mysqlBruteforce(task): address, username, password = task.split('|') try: db = pymysql.connect(address, username, password, "mysql", connect_timeout=5) result = 'IP: ' + address + ' Mysql User: '******' Pass: '******'BruteForce', address, result + '\n') return result except Exception as e: pass
def telnetBruteforce(task): address, username, password = task.split('|') try: telnet = telnetlib.Telnet(address, timeout=3) telnet.read_until("login: "******"\n") telnet.read_until("Password: "******"\n") telnet.close() result = 'IP: ' + address + ' Telnet User: '******' Pass: '******'BruteForce', address, result) return result except: pass
def osdetect(ip): # sys.stdout.write(Bcolors.RED + "\nOS:\n" + Bcolors.ENDC) nm = nmap.PortScanner() try: result = nm.scan(hosts=ip, arguments='-sS -O -vv -n -T4 -p 80,22,443') for k, v in result.get('scan').items(): if v.get('osmatch'): for i in v.get('osmatch'): console('OSdetect', ip, i.get('name') + '\n') return i.get('name') else: break except (xml.etree.ElementTree.ParseError, nmap.nmap.PortScannerError): pass except Exception as e: console('OSdetect', ip, 'None\n') logging.exception(e)
def run(self): scripts = [] try: for _ in glob.glob('script/*.py'): script_name = os.path.basename(_).replace('.py', '') vuln = importlib.import_module('script.%s' % script_name) scripts.append(vuln) with concurrent.futures.ThreadPoolExecutor( max_workers=20) as executor: executor.map(self.vuln, scripts) self.out = list(filter(None, self.out)) for i in self.out: console('Vuln', self.ip, i + '\n') brute_result = Crack().pool(self.ip, self.ports) if brute_result: self.out.extend(brute_result) Sqldb('result').get_vuln(self.ip, self.out) except Exception as e: logging.exception(e)
def SSHBruteforce(task): address, username, password = task.split('|') try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=address, port=22, username=username, password=password, compress=True) result = 'IP: ' + address + ' SSH User: '******' Pass: '******'BruteForce', address, result) return result except (paramiko.ssh_exception.AuthenticationException, paramiko.ssh_exception.SSHException, ConnectionResetError, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError, EOFError): pass except Exception as e: pass
def check(self, url): # 检测导入的IP loc = parse.urlparse(url) # urlparse()实现URL的识别和分段 if getattr(loc, 'netloc'): # getattr() 函数用于返回一个对象属性值。 host = loc.netloc # 获得Ip域名 else: host = loc.path # 获得Ip路径 try: # 判断是IP还是域名,域名的话需要检测域名解析 if not re.search(r'\d+\.\d+\.\d+\.\d+', host): # 验证DNS存活并且DNS解析不能是一些特殊IP(DNSIP、内网IP) # resolver = dns.resolver.Resolver() # resolver.nameservers = ['1.1.1.1', '8.8.8.8'] a = dns.resolver.query(host, 'A') # 查询类型为A记录 for i in a.response.answer: # 通过response.answer获取查询回应信息 for j in i.items: if hasattr(j, 'address'): # 用于判断对象是否包含对应的属性。 if re.search( r'1\.1\.1\.1|8\.8\.8\.8|127\.0\.0\.1|114\.114\.114\.114|0\.0\.0\.0', j.address): return False if PING: try: # Windows调用ping判断存活 Linux调用nmap来判断主机存活 # nmap判断存活会先进行ping然后连接80端口,这样不会漏掉 if platform.system() == 'Windows': # 获得系统名称 subprocess.check_output( ['ping', '-n', '2', '-w', '1', host]) # 执行命令 self.out.append(url) # 添加url到列 else: nm = nmap.PortScanner() # 实例化 result = nm.scan(hosts=host, arguments='-sP -n') # 可选参数,要扫描的方式 for k, v in result.get('scan').items(): if not v.get('status').get( 'state') == 'up': # 获得主机信息 console('PING', host, "is not alive\n") # 结果展示 return False else: self.out.append(url) # 添加url到列 except: console('PING', host, "is not alive\n") # 结果展示 return False else: self.out.append(url) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): console('DnsCheck', host, "No A record\n") # 描述失败原因 except dns.exception.Timeout: console('DnsCheck', host, "Timeout\n")
def check(self, url): loc = parse.urlparse(url) if getattr(loc, 'netloc'): host = loc.netloc else: host = loc.path try: if not re.search(r'\d+\.\d+\.\d+\.\d+', host): # 验证DNS存活并且DNS不能是一些特殊IP(DNSIP、内网IP) resolver = dns.resolver.Resolver() resolver.nameservers = ['223.5.5.5', '1.1.1.1', '8.8.8.8'] a = resolver.query(host, 'A') for i in a.response.answer: for j in i.items: if hasattr(j, 'address'): if re.search( r'1\.1\.1\.1|8\.8\.8\.8|127\.0\.0\.1|114\.114\.114\.114', j.address): return False if PING: try: # Windows调用ping判断存活 Linux调用nmap来判断主机存活 # nmap判断存活会先进行ping然后连接80端口,这样不会漏掉 if platform.system() == 'Windows': subprocess.check_output( ['ping', '-n', '2', '-w', '1', host]) else: nm = nmap.PortScanner() result = nm.scan(hosts=host, arguments='-sP -n') for k, v in result.get('scan').items(): if not v.get('status').get('state') == 'up': console('PING', host, "is not alive\n") return False self.out.append(url) except subprocess.CalledProcessError: console('PING', host, "is not alive\n") return False except Exception as e: logging.exception(e) else: self.out.append(url) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): console('DnsCheck', host, "No A record\n") except dns.exception.Timeout: console('DnsCheck', host, "Timeout\n") except Exception as e: logging.exception(e) return False
def pool(self): out = [] try: # 判断给出的url是www.baiud.com还是www.baidu.com/path这种形式 if (not parse.urlparse(self.ipaddr).path) and (parse.urlparse( self.ipaddr).path != '/'): self.ipaddr = self.ipaddr.replace('http://', '').replace('https://', '').rstrip('/') else: self.ipaddr = self.ipaddr.replace('http://', '').replace('https://', '').rstrip('/') self.ipaddr = re.sub(r'/\w+', '', self.ipaddr) if re.search(r'\d+\.\d+\.\d+\.\d+', self.ipaddr): ipaddr = self.ipaddr else: ipaddr = socket.gethostbyname(self.ipaddr) if ':' in ipaddr: ipaddr = re.sub(r':\d+', '', ipaddr) self.run(ipaddr) except Exception as e: pass if self.num == 0: self.save(self.ipaddr, self.out) for _ in self.out: out.append('{}:{}'.format(_.get('server'), _.get('port'))) console('PortScan', self.ipaddr, '{}:{}\n'.format(_.get('server'), _.get('port'))) return out else: self.save(self.ipaddr, [{ "server": 'Portspoof', "port": '0', "banner": '' }]) console('PortScan', self.ipaddr, 'Portspoof:0\n') return ['Portspoof:0']
def pool(self): result = self.parse_html(self.host) try: with concurrent.futures.ThreadPoolExecutor( max_workers=30) as executor: futures = [executor.submit(self.parse_html, i) for i in result] for future in concurrent.futures.as_completed(futures, timeout=3): future.result() except (EOFError, concurrent.futures._base.TimeoutError): pass except Exception as e: logging.exception(e) jslink = JsLeaks().pool(self.js) self.result.extend(jslink) self.result = list(set(self.result)) for i in self.result: console('Crawl', self.host, i + '\n') Sqldb(self.dbname).get_crawl(self.domain, self.result)
def geoip(ipaddr): # 获取IP地理位置 try: reader = geoip2.database.Reader('data/GeoLite2-City.mmdb') response = reader.city(ipaddr) country = response.country.names["zh-CN"] site = response.subdivisions.most_specific.names.get("zh-CN") if not site: site = '' city = response.city.names.get("zh-CN") if not city: city = '' address = '{} {} {}'.format(country, site, city) except FileNotFoundError: address = 'Geoip File Not Found' except (KeyError, geoip2.errors.AddressNotFoundError): address = 'Address Not In Databases' except Exception as e: logging.exception(e) address = 'None' console('GeoIP', ipaddr, 'Address: {}\n'.format(address)) console('GeoIP', ipaddr, 'Ipaddr: {}\n'.format(ipaddr)) return address
def gener(): out = [] for i in [ gen_webinfo(), gen_urls(), gen_ports(), gen_vuln(), gen_crawl(), gen_subdomain() ]: if i: out.append(i) result = {"table": out} result = json.dumps(result) #将dict类型的数据转成str result = re.sub(r'^{|}$', '', result) #格式整理 times = time.strftime("%Y%m%d%H%M%S", time.localtime()) #设定开始时间 name = 'scan_' + times + '.html' with open('report/report.htm', 'r', encoding='utf-8') as f, open(name, 'w') as f1: text = f.read() console('H5_Create', name, '创建......\n') time.sleep(2) f1.write(text.replace("'summary': {}", result)) #H5页面展示
def query_db(self, hosts): #写入到数据库,返回成功的目标 result = [] error = False for i in hosts: try: domain = parse_host(i) #获得主机host头 cursor = self.conn.cursor() #创建数据库游标 sql = "select 1 from webinfo where domain = '{}' limit 1".format( domain) cursor.execute(sql) #执行sql语句 values = cursor.fetchall() #获取结果集 内容形成一个列 if not values: result.append(i) #没有目标添加到result里 else: console('CheckDB', i, 'In the db file\n') #结果展示 except sqlite3.OperationalError: return hosts self.commit() #提交修改请求 self.close() #关闭数据库 if error: return hosts else: return result #返回成功的目标
def geoip(ipaddr): # 获取IP地理位置 #geoip2查询ip的详细地址信息 try: reader = geoip2.database.Reader( 'Information_Scan/data/GeoLite2-City.mmdb') #分析IP颗粒度 读取数据库 response = reader.city(ipaddr) #解析正常的IP物理地址赋值给response country = response.country.names["zh-CN"] #解析成为中文 国家 site = response.subdivisions.most_specific.names["zh-CN"] #省份 if not site: site = '' city = response.city.names["zh-CN"] #城市 if not city: city = '' address = '{} {} {}'.format(country, site, city) #国家 省份 城市 except FileNotFoundError: address = 'Geoip File Not Found' except (KeyError, geoip2.errors.AddressNotFoundError): address = 'Address Not In Databases' except Exception as e: address = 'None' console('GeoIP', ipaddr, 'Address: {}\n'.format(address)) #结果显示 console('GeoIP', ipaddr, 'Ipaddr: {}\n'.format(ipaddr)) return address #返回目标物理地址
def web_info(url, flags=1): #返回H5页面展示信息 host = parse_host(url) #整理地址格式得到host www.baidu.com ipaddr = parse_ip(host) #获得正常的IP,排除DNS服务器 url = url.strip('/') address = geoip(ipaddr) #获取IP地理位置 wafresult = checkwaf(url) #检测waf req = Requests() try: r = req.get(url) #返回session的长连接 coding = chardet.detect(r.content).get('encoding') #获取网站编码格式 r.encoding = coding webinfo = WebPage( r.url, r.text, r.headers).info() #传入url text headers 返回cms信息 网站标题 服务器 except Exception: webinfo = {} if webinfo: console('Webinfo', host, 'Title: {}\n'.format(webinfo.get('title'))) console('Webinfo', host, 'Fingerprint: {}\n'.format(webinfo.get('apps'))) console('Webinfo', host, 'Server: {}\n'.format(webinfo.get('server'))) console('Webinfo', host, 'WAF: {}\n'.format(wafresult)) else: webinfo = {} wafresult = 'None' osname = osdetect(host) #操作系统名称 data = { host: { 'WAF': wafresult, 'Ipaddr': ipaddr, 'Address': address, 'Webinfo': webinfo, 'OS': osname, } } if flags == 1: return data, webinfo.get('apps'), webinfo.get('title') #返回目标信息 数据 标题 else: subdomain_save(data)
def reverse_domain(host): # 查询旁站 sys.stdout.write(Bcolors.RED + "\nReverse IP Domain Check:\n" + Bcolors.ENDC) if iscdn(host): result = [] data = {"remoteAddress": "{0}".format(host), "key": ""} header = get_ua() try: r = requests.post('https://domains.yougetsignal.com/domains.php', headers=header, data=data, timeout=5, verify=False) text = json.loads(r.text) domain = tldextract.extract(host) for i in text.get('domainArray'): url = i[0] if url != host: if tldextract.extract(url).domain == domain.domain: result.append(url) elif re.search(r'\d+\.\d+\.\d+\.\d+', url): result.append(url) except: try: r = requests.get( 'http://api.hackertarget.com/reverseiplookup/?q={}'.format( host), headers=get_ua(), timeout=4, verify=False) if '<html>' not in r.text and 'No DNS A records found for' not in r.text: text = r.text for _ in text.split('\n'): if _: result.append(_) else: result = [] except: pass if len(result) < 20: if result: for i in result: console('reverse_domain', host, i + '\n') else: console('reverse_domain', host, 'None\n') return result else: console('reverse_domain', host, 'The maximum number of domain names exceeded (20)\n') # sys.stdout.write(Bcolors.OKGREEN + 'The maximum number of domain names exceeded (20)\n' + Bcolors.ENDC) return ['The maximum number of domain names exceeded (20)']
def reverse_domain(host): # 查询旁站 result = [] data = {"remoteAddress": "{0}".format(host), "key": ""} header = get_ua() #自定义headers头 try: r = requests.post('https://domains.yougetsignal.com/domains.php', headers=header, data=data, timeout=5, verify=False) text = json.loads(r.text) #返回json格式 domain = tldextract.extract( host ) #将URL分割,获得各个域名 http://forums.news.cnn.com/ subdomain='forums.news', domain='cnn', suffix='com' for i in text.get('domainArray'): url = i[0] if url != host: #看看域名是否一致 if tldextract.extract( url ).domain == domain.domain: #二级域名比较 top.baidu.com m.baidu.com result.append(url) #二级域名添加 elif re.search(r'\d+\.\d+\.\d+\.\d+', url): result.append(url) #IP添加 except: try: r = requests.get( 'http://api.hackertarget.com/reverseiplookup/?q={}'.format( host), headers=get_ua(), #自定义的headers头 timeout=4, verify=False) if '<html>' not in r.text and 'No DNS A records found for' not in r.text: #No DNS A records found for 119.3.60.210 其余情况就是出现了域名 text = r.text for _ in text.split('\n'): if _: result.append(_) #添加域名 else: result = [] except: pass if len(result) < 20: if result: for i in result: console('reverse_domain', host, i + '\n') #旁站进行展示 else: console('reverse_domain', host, 'None\n') return result else: console('reverse_domain', host, 'The maximum number of domain names exceeded (20)\n') return ['The maximum number of reverse_domain names exceeded (20)']
def web_info(url): host = parse_host(url) ipaddr = parse_ip(host) url = url.strip('/') address = geoip(ipaddr) wafresult = checkwaf(url) req = Requests() # noinspection PyBroadException try: r = req.get(url) coding = chardet.detect(r.content).get('encoding') r.encoding = coding webinfo = WebPage(r.url, r.text, r.headers).info() except Exception as e: logging.exception(e) webinfo = {} if webinfo: console('Webinfo', host, 'title: {}\n'.format(webinfo.get('title'))) console('Webinfo', host, 'Fingerprint: {}\n'.format(webinfo.get('apps'))) console('Webinfo', host, 'Server: {}\n'.format(webinfo.get('server'))) console('Webinfo', host, 'WAF: {}\n'.format(wafresult)) else: webinfo = {} wafresult = 'None' if iscdn(host): osname = osdetect(host) else: osname = None data = { host: { 'WAF': wafresult, 'Ipaddr': ipaddr, 'Address': address, 'Webinfo': webinfo, 'OS': osname, } } return data, webinfo.get('apps'), webinfo.get('title')