def hand_ip(self, serviceTypes, option='masscan'):
ip_list = []
for item in serviceTypes:
ip_list.append(item["target"])
ports = MASSCAN_DEFAULT_PORT
result2 = {}
if option == 'masscan':
if MASSCAN_FULL_SCAN:
ports = "1-65535"
target = os.path.join(PATHS.OUTPUT_PATH,
"target_{0}.log".format(time.time()))
with open(target, "w+") as fp:
fp.write('\n'.join(ip_list))
logger.debug("ip:" + repr(ip_list))
try:
result = masscan(target, ports)
except Exception as e:
logger.error("masscan error msg:{}".format(repr(e)))
result = None
if result is None:
return None
# format:{'115.159.39.75': ['80'], '115.159.39.215': ['80', '3306'],}
for host, ports in result.items():
ports = list(ports)
if host not in result2:
result2[host] = []
task_update("running", 1)
try:
result_nmap = nmapscan(host, ports)
except:
result_nmap = None
task_update("running", -1)
if result_nmap is None:
for tmp_port in ports:
result2[host].append({"port": tmp_port})
continue
tmp_r = self.nmap_result_handle(result_nmap, host=host)
result2.update(tmp_r)
elif option == "nmap":
logger.debug("ip:" + repr(ip_list))
for host in ip_list:
result_nmap = nmapscan(host, ports.split(","))
tmp_r = self.nmap_result_handle(result_nmap, host=host)
if tmp_r:
result2.update(tmp_r)
data = {}
for ip in result2.keys():
# result2[ip]
if ip not in data:
data[ip] = {}
d = ip_location.poc(ip)
if d:
data[ip]["location"] = d
data[ip]["infos"] = result2[ip]
collector.add_ips(data)
for ip in result2.keys():
collector.send_ok_ip(ip)
def send_ok_ip(self, target):
data = self.get_ip(target)
data['target'] = target
self.del_ip(target)
self.cache_ips.put(data)
task_update("finished", 1)
if self.cache_ips.qsize() > 3:
self.submit()
def receive_ip(self):
while 1:
struct = self.ip_queue.get()
serviceType = struct.get("serviceType", 'other')
task_update("tasks", self.queue.qsize() + self.ip_queue.qsize())
if serviceType == "ip":
flag = False
self.lock.acquire()
self.cache_ips.append(struct)
num = len(self.cache_ips)
if num >= NUM_CACHE_IP:
flag = True
serviceTypes = self.cache_ips
self.cache_ips = []
self.lock.release()
if not flag:
self.ip_queue.task_done()
continue
task_update("running", 1)
try:
self.hand_ip(serviceTypes)
except Exception as e:
logger.error("hand ip error:{}".format(repr(e)))
logger.error(repr(sys.exc_info()))
task_update("running", -1)
self.ip_queue.task_done()
task_update("tasks", self.queue.qsize() + self.ip_queue.qsize())
def send_ok(self, domain):
'''
传递ok信号，将域名缓存到缓冲队列，自动检测缓冲队列，大于10个则自动发送到接口
:param domain:
:return:
'''
data = self.get_domain(domain)
data["url"] = domain
self.cache_queue.put(data)
self.del_domain(domain)
task_update("finished", 1)
if self.cache_queue.qsize() > 3:
self.submit()
def put_target(self, target):
# 判断是IP还是域名，加入不同的字段
serviceType = "domain"
if is_ip_address_format(target):
serviceType = "ip"
elif is_url_format(target):
serviceType = "domain"
target = target.rstrip('/')
else:
serviceType = "other"
tmp = {"target": target, "serviceType": serviceType}
self.queue.put(tmp)
task_update("tasks", self.queue.qsize())
def run(self):
self.queue.join()
# 对剩余未处理的域名进行处理
if self.cache_domains:
serviceTypes = self.cache_domains
# 多线程启动扫描域名
for serviceType in serviceTypes:
task_update("running", 1)
self.hand_domain(serviceType)
task_update("running", -1)
self.cache_domains = []
# 对剩余未处理的ip进行处理
if self.cache_ips:
serviceTypes = self.cache_ips
task_update("running", 1)
self.hand_ip(serviceTypes)
task_update("running", -1)
self.cache_ips = []
# 最后一次提交
collector.submit()
task_update("tasks", self.queue.qsize())
def run(self):
while 1:
# 对剩余未处理的域名进行处理
if self.cache_domains:
self.lock.acquire()
service_types = self.cache_domains
self.cache_domains = []
self.lock.release()
# 多线程启动扫描域名
for serviceType in service_types:
task_update("running", 1)
try:
self.hand_domain(serviceType)
except Exception as e:
logger.error(repr(sys.exc_info()))
task_update("running", -1)
# 对剩余未处理的ip进行处理
if self.cache_ips:
self.lock.acquire()
service_types = self.cache_ips
self.cache_ips = []
self.lock.release()
task_update("running", 1)
try:
self.hand_ip(service_types)
except Exception as e:
logger.error(repr(sys.exc_info()))
task_update("running", -1)
# 最后一次提交
collector.submit()
task_update("tasks", self.queue.qsize() + self.ip_queue.qsize())
time.sleep(random.randint(2, 10))
def receive(self):
while 1:
struct = self.queue.get()
task_update("tasks", self.queue.qsize() + self.ip_queue.qsize())
serviceType = struct.get("serviceType", 'other')
if serviceType == "other":
msg = "not matches target:{}".format(repr(struct))
logger.error(msg)
self.queue.task_done()
continue
elif serviceType == "domain":
flag = False
self.lock.acquire()
self.cache_domains.append(struct)
num = len(self.cache_domains)
if num >= NUM_CACHE_DOMAIN:
flag = True
serviceTypes = self.cache_domains
self.cache_domains = []
self.lock.release()
if not flag:
self.queue.task_done()
continue
# 多线程启动扫描域名
for serviceType in serviceTypes:
task_update("running", 1)
try:
self.hand_domain(serviceType)
except Exception as e:
logger.error("hand domain error:{}".format(repr(e)))
logger.error(repr(sys.exc_info()))
task_update("running", -1)
self.queue.task_done()
task_update("tasks", self.queue.qsize() + self.ip_queue.qsize())