class ActiveCheck(): def __init__(self, hosts): self.hosts = hosts self.out = [] def check_db(self, hosts): self.out = Sqldb('result').query_db(hosts) def check(self, url): loc = parse.urlparse(url) if getattr(loc, 'netloc'): host = loc.netloc else: host = loc.path try: if not re.search(r'\d+\.\d+\.\d+\.\d+', host): dns.resolver.query(host, 'A') if PING: try: if platform.system() == 'Windows': subprocess.check_output( ['ping', '-n', '2', '-w', '1', host]) else: subprocess.check_output(['ping', '-c 2', '-W 1', host]) self.out.append(url) except subprocess.CalledProcessError: console('PING', host, "is not alive\n") return False except Exception as e: logging.exception(e) else: self.out.append(url) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): console('DnsCheck', host, "No A record\n") except Exception as e: logging.exception(e) return False def disable(self): # 求生欲名单 # 禁止扫描所有gov.cn与edu.cn结尾的域名,遵守法律!!! for i in self.out: if re.search(r'gov\.cn|edu\.cn$', i): console('Disable', i, "Do not scan this domain\n") sys.exit(1) def pool(self): with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: executor.map(self.check, self.hosts) if CHECK_DB: self.check_db(list(set(self.out))) self.disable() return self.out
class ActiveCheck(): def __init__(self, hosts): self.hosts = hosts self.out = [] def check_db(self, hosts): self.out = Sqldb('result').query_db(hosts) def check(self, url): loc = parse.urlparse(url) if getattr(loc, 'netloc'): host = loc.netloc else: host = loc.path try: if not re.search(r'\d+\.\d+\.\d+\.\d+', host): dns.resolver.query(host, 'A') if PING: try: if platform.system() == 'Windows': subprocess.check_output( ['ping', '-n', '2', '-w', '1', host]) else: subprocess.check_output(['ping', '-c 2', '-W 1', host]) self.out.append(url) except Exception as e: sys.stdout.write(bcolors.OKGREEN + "{} is not alive\n".format(host) + bcolors.ENDC) return False else: self.out.append(url) except Exception as e: return False def pool(self): sys.stdout.write(bcolors.RED + "Start Ping ...\n\n" + bcolors.ENDC) with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: executor.map(self.check, self.hosts) if CHECK_DB: self.check_db(list(set(self.out))) return self.out
class ActiveCheck(): def __init__(self, hosts): self.hosts = hosts self.out = [] def check_db(self, hosts): self.out = Sqldb('result').query_db(hosts) def check(self, url): loc = parse.urlparse(url) if getattr(loc, 'netloc'): host = loc.netloc else: host = loc.path try: if not re.search(r'\d+\.\d+\.\d+\.\d+', host): # 验证DNS存活并且DNS不能是一些特殊IP(DNSIP、内网IP) resolver = dns.resolver.Resolver() resolver.nameservers = ['223.5.5.5', '1.1.1.1', '8.8.8.8'] a = resolver.query(host, 'A') for i in a.response.answer: for j in i.items: if hasattr(j, 'address'): if re.search( r'1\.1\.1\.1|8\.8\.8\.8|127\.0\.0\.1|114\.114\.114\.114', j.address): return False if PING: try: # Windows调用ping判断存活 Linux调用nmap来判断主机存活 # nmap判断存活会先进行ping然后连接80端口,这样不会漏掉 if platform.system() == 'Windows': subprocess.check_output( ['ping', '-n', '2', '-w', '1', host]) else: nm = nmap.PortScanner() result = nm.scan(hosts=host, arguments='-sP -n') for k, v in result.get('scan').items(): if not v.get('status').get('state') == 'up': console('PING', host, "is not alive\n") return False self.out.append(url) except subprocess.CalledProcessError: console('PING', host, "is not alive\n") return False except Exception as e: logging.exception(e) else: self.out.append(url) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): console('DnsCheck', host, "No A record\n") except dns.exception.Timeout: console('DnsCheck', host, "Timeout\n") except Exception as e: logging.exception(e) return False def disable(self): # 禁止扫描所有敏感域名,遵守法律!!! for i in self.out: if re.search( r'\.org\.cn|\.com\.cn|\.cn|gov\.cn|edu\.cn|\.mil|\.aero|\.int|\.go\.\w+$|\.ac\.\w+$', i): console('Disable', i, "Do not scan this domain\n\n") sys.exit(1) def pool(self): try: with concurrent.futures.ThreadPoolExecutor( max_workers=20) as executor: result = { executor.submit(self.check, i): i for i in self.hosts } for future in concurrent.futures.as_completed(result, timeout=3): future.result() except (EOFError, concurrent.futures._base.TimeoutError): pass except Exception as e: logging.exception(e) if CHECK_DB: self.check_db(list(set(self.out))) self.disable() return self.out
class ActiveCheck: def __init__(self, hosts): self.hosts = hosts self.out = [] def check_db(self, hosts): self.out = Sqldb('result').query_db(hosts) def check(self, url): loc = parse.urlparse(url) if getattr(loc, 'netloc'): host = loc.netloc else: host = loc.path # 验证国家 if VERIFY_COUNTRY: if verify_country(host): console('Disable', host, "Disable Country\n") return False if re.search(r'\d+\.\d+\.\d+\.\d+', host): if not WhiteIP().checkip(host): console('Disable', host, "China IP\n") return False if re.search( r'\.org\.cn|\.com\.cn|\.cn|gov\.cn|edu\.cn|\.mil|\.aero|\.int|\.go\.\w+$|\.ac\.\w+$', host): console('Disable', host, "Do not scan this domain\n") return False try: # 判断是IP还是域名,域名的话需要检测域名解析 if not re.search(r'\d+\.\d+\.\d+\.\d+', host): # 验证DNS存活并且DNS解析不能是一些特殊IP(DNSIP、内网IP) console('Dnscheck', host, 'query dns a records\n') resolver = dns.resolver.Resolver() resolver.nameservers = ['1.1.1.1', '8.8.8.8'] a = resolver.query(host, 'A') for i in a.response.answer: for j in i.items: if hasattr(j, 'address'): # if re.search(r'\d+\.\d+\.\d+\.\d+', j.address): # if not WhiteIP().checkip(j.address): # console('Disable', j.address, "China IP\n") # return False if re.search( r'1\.1\.1\.1|8\.8\.8\.8|127\.0\.0\.1|114\.114\.114\.114|0\.0\.0\.0', j.address): return False if PING: try: # Windows调用ping判断存活 Linux调用nmap来判断主机存活 # nmap判断存活会先进行ping然后连接80端口,这样不会漏掉 if platform.system() == 'Windows': console('PING', host, 'Ping scan\n') subprocess.check_output( ['ping', '-n', '2', '-w', '1', host]) self.out.append(url) else: console('PING', host, 'Nmap Ping scan\n') nm = nmap.PortScanner() result = nm.scan(hosts=host, arguments='-sP -n') for k, v in result.get('scan').items(): if not v.get('status').get('state') == 'up': console('PING', host, "is not alive\n") return False else: self.out.append(url) except (AttributeError, subprocess.CalledProcessError, xml.etree.ElementTree.ParseError, nmap.nmap.PortScannerError): console('PING', host, "is not alive\n") return False except Exception as e: logging.exception(e) return False else: self.out.append(url) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): console('DnsCheck', host, "No A record\n") except dns.exception.Timeout: console('DnsCheck', host, "Timeout\n") except Exception as e: logging.exception(e) return False def pool(self): try: with concurrent.futures.ThreadPoolExecutor( max_workers=20) as executor: result = { executor.submit(self.check, i): i for i in self.hosts } for future in concurrent.futures.as_completed(result, timeout=3): future.result() except (EOFError, concurrent.futures._base.TimeoutError): pass except Exception as e: logging.exception(e) if CHECK_DB: self.check_db(list(set(self.out))) return self.out