def hand_ip(self, serviceTypes, option='masscan'): ip_list = [] for item in serviceTypes: ip_list.append(item["target"]) ports = MASSCAN_DEFAULT_PORT result2 = {} if option == 'masscan': if MASSCAN_FULL_SCAN: ports = "1-65535" target = os.path.join(PATHS.OUTPUT_PATH, "target_{0}.log".format(time.time())) with open(target, "w+") as fp: fp.write('\n'.join(ip_list)) logger.debug("ip:" + repr(ip_list)) try: result = masscan(target, ports) except Exception as e: logger.error("masscan error msg:{}".format(repr(e))) result = None if result is None: return None # format:{'115.159.39.75': ['80'], '115.159.39.215': ['80', '3306'],} for host, ports in result.items(): ports = list(ports) if host not in result2: result2[host] = [] task_update("running", 1) try: result_nmap = nmapscan(host, ports) except: result_nmap = None task_update("running", -1) if result_nmap is None: for tmp_port in ports: result2[host].append({"port": tmp_port}) continue tmp_r = self.nmap_result_handle(result_nmap, host=host) result2.update(tmp_r) elif option == "nmap": logger.debug("ip:" + repr(ip_list)) for host in ip_list: result_nmap = nmapscan(host, ports.split(",")) tmp_r = self.nmap_result_handle(result_nmap, host=host) if tmp_r: result2.update(tmp_r) data = {} for ip in result2.keys(): # result2[ip] if ip not in data: data[ip] = {} d = ip_location.poc(ip) if d: data[ip]["location"] = d data[ip]["infos"] = result2[ip] collector.add_ips(data) for ip in result2.keys(): collector.send_ok_ip(ip)
def send_ok_ip(self, target): data = self.get_ip(target) data['target'] = target self.del_ip(target) self.cache_ips.put(data) task_update("finished", 1) if self.cache_ips.qsize() > 3: self.submit()
def receive_ip(self): while 1: struct = self.ip_queue.get() serviceType = struct.get("serviceType", 'other') task_update("tasks", self.queue.qsize() + self.ip_queue.qsize()) if serviceType == "ip": flag = False self.lock.acquire() self.cache_ips.append(struct) num = len(self.cache_ips) if num >= NUM_CACHE_IP: flag = True serviceTypes = self.cache_ips self.cache_ips = [] self.lock.release() if not flag: self.ip_queue.task_done() continue task_update("running", 1) try: self.hand_ip(serviceTypes) except Exception as e: logger.error("hand ip error:{}".format(repr(e))) logger.error(repr(sys.exc_info())) task_update("running", -1) self.ip_queue.task_done() task_update("tasks", self.queue.qsize() + self.ip_queue.qsize())
def send_ok(self, domain): ''' 传递ok信号,将域名缓存到缓冲队列,自动检测缓冲队列,大于10个则自动发送到接口 :param domain: :return: ''' data = self.get_domain(domain) data["url"] = domain self.cache_queue.put(data) self.del_domain(domain) task_update("finished", 1) if self.cache_queue.qsize() > 3: self.submit()
def put_target(self, target): # 判断是IP还是域名,加入不同的字段 serviceType = "domain" if is_ip_address_format(target): serviceType = "ip" elif is_url_format(target): serviceType = "domain" target = target.rstrip('/') else: serviceType = "other" tmp = {"target": target, "serviceType": serviceType} self.queue.put(tmp) task_update("tasks", self.queue.qsize())
def run(self): self.queue.join() # 对剩余未处理的域名进行处理 if self.cache_domains: serviceTypes = self.cache_domains # 多线程启动扫描域名 for serviceType in serviceTypes: task_update("running", 1) self.hand_domain(serviceType) task_update("running", -1) self.cache_domains = [] # 对剩余未处理的ip进行处理 if self.cache_ips: serviceTypes = self.cache_ips task_update("running", 1) self.hand_ip(serviceTypes) task_update("running", -1) self.cache_ips = [] # 最后一次提交 collector.submit() task_update("tasks", self.queue.qsize())
def run(self): while 1: # 对剩余未处理的域名进行处理 if self.cache_domains: self.lock.acquire() service_types = self.cache_domains self.cache_domains = [] self.lock.release() # 多线程启动扫描域名 for serviceType in service_types: task_update("running", 1) try: self.hand_domain(serviceType) except Exception as e: logger.error(repr(sys.exc_info())) task_update("running", -1) # 对剩余未处理的ip进行处理 if self.cache_ips: self.lock.acquire() service_types = self.cache_ips self.cache_ips = [] self.lock.release() task_update("running", 1) try: self.hand_ip(service_types) except Exception as e: logger.error(repr(sys.exc_info())) task_update("running", -1) # 最后一次提交 collector.submit() task_update("tasks", self.queue.qsize() + self.ip_queue.qsize()) time.sleep(random.randint(2, 10))
def receive(self): while 1: struct = self.queue.get() task_update("tasks", self.queue.qsize() + self.ip_queue.qsize()) serviceType = struct.get("serviceType", 'other') if serviceType == "other": msg = "not matches target:{}".format(repr(struct)) logger.error(msg) self.queue.task_done() continue elif serviceType == "domain": flag = False self.lock.acquire() self.cache_domains.append(struct) num = len(self.cache_domains) if num >= NUM_CACHE_DOMAIN: flag = True serviceTypes = self.cache_domains self.cache_domains = [] self.lock.release() if not flag: self.queue.task_done() continue # 多线程启动扫描域名 for serviceType in serviceTypes: task_update("running", 1) try: self.hand_domain(serviceType) except Exception as e: logger.error("hand domain error:{}".format(repr(e))) logger.error(repr(sys.exc_info())) task_update("running", -1) self.queue.task_done() task_update("tasks", self.queue.qsize() + self.ip_queue.qsize())