def run(self):
for info in self.domain_info_list:
for ip in info.ip_list:
old_domain = self.ipv4_map.get(ip, set())
old_domain.add(info.domain)
self.ipv4_map[ip] = old_domain
all_ipv4_list = self.ipv4_map.keys()
start_time = time.time()
logger.info("start port_scan {}".format(len(all_ipv4_list)))
ip_port_result = services.port_scan(all_ipv4_list, **self.option)
elapse = time.time() - start_time
logger.info("end port_scan result {}, elapse {}".format(
len(ip_port_result), elapse))
ip_info_obj = []
for result in ip_port_result:
curr_ip = result["ip"]
result["domain"] = list(self.ipv4_map[curr_ip])
port_info_obj_list = []
for port_info in result["port_info"]:
port_info_obj_list.append(modules.PortInfo(**port_info))
result["port_info"] = port_info_obj_list
ip_info_obj.append(modules.IPInfo(**result))
return ip_info_obj
def fofa_search(self):
try:
f = fofaSearch(self.domain_info_list, self.base_domain)
ips = f.run()
for ip in ips:
if ip not in self.ip_set:
self.fofa_ip_set.add(ip)
if self.options.get("port_scan"):
ip_port_result = services.port_scan(self.fofa_ip_set,
**self.scan_port_option)
for ip_info in ip_port_result:
ip_info["domain"] = ["*.{}".format(self.base_domain)]
port_info_obj_list = []
for port_info in ip_info["port_info"]:
port_info_obj_list.append(
modules.PortInfo(**port_info))
ip_info["port_info"] = port_info_obj_list
fake_info_obj = modules.IPInfo(**ip_info)
fake_ip_info = fake_info_obj.dump_json(flag=False)
fake_ip_info["task_id"] = self.task_id
utils.conn_db('ip').insert_one(fake_ip_info)
for ip in self.fofa_ip_set:
self.ipv4_map[ip] = ["*.{}".format(self.base_domain)]
logger.info("fofa search {} {}".format(self.base_domain,
len(self.fofa_ip_set)))
except Exception as e:
logger.exception(e)
logger.warning("fofa search error {}, {}".format(
self.base_domain, e))
def port_scan(self):
scan_port_map = {
"test": ScanPortType.TEST,
"top100": ScanPortType.TOP100,
"top1000": ScanPortType.TOP1000,
"all": ScanPortType.ALL
}
option_scan_port_type = self.options.get("port_scan_type", "test")
scan_port_option = {
"ports": scan_port_map.get(option_scan_port_type,
ScanPortType.TEST),
"service_detect": self.options.get("service_detection", False),
"os_detect": self.options.get("os_detection", False)
}
targets = self.ip_target.split()
ip_port_result = services.port_scan(targets, **scan_port_option)
self.ip_info_list.extend(ip_port_result)
for ip_info in ip_port_result:
self.ip_set.add(ip_info["ip"])
if not utils.not_in_black_ips(ip_info["ip"]):
continue
ip_info["task_id"] = self.task_id
ip_info["ip_type"] = utils.get_ip_type(ip_info["ip"])
ip_info["geo_asn"] = {}
ip_info["geo_city"] = {}
if ip_info["ip_type"] == "PUBLIC":
ip_info["geo_asn"] = utils.get_ip_asn(ip_info["ip"])
ip_info["geo_city"] = utils.get_ip_city(ip_info["ip"])
utils.conn_db('ip').insert_one(ip_info)
def port_scan():
out = services.port_scan(['10.0.86.169', '10.0.83.6', '10.0.83.16'], os_detect=True)
for o in out:
port_info_obj_list = []
for port_info in o["port_info"]:
port_info_obj_list.append(modules.PortInfo(**port_info))
o["port_info"] = port_info_obj_list
print(modules.IPInfo(**o))
def ip_data(self):
if self._ip_data is None:
_ip_data = services.build_domain_info(
["www.baidu.com", "www.qq.com"])
ip_data = []
for x in services.port_scan(["1.1.1.1"]):
x["geo_asn"] = {
"number": 13335,
"organization": "Cloudflare, Inc."
}
ip_data.append(x)
self._ip_data = ip_data
return self._ip_data
def run(self):
for info in self.domain_info_list:
for ip in info.ip_list:
old_domain = self.ipv4_map.get(ip, set())
old_domain.add(info.domain)
self.ipv4_map[ip] = old_domain
if ip not in self.ip_cdn_map:
cdn_name = self.get_cdn_name(ip, info)
self.ip_cdn_map[ip] = cdn_name
if cdn_name:
self.have_cdn_ip_list.append(ip)
all_ipv4_list = self.ipv4_map.keys()
if self.skip_scan_cdn_ip:
all_ipv4_list = list(
set(all_ipv4_list) - set(self.have_cdn_ip_list))
start_time = time.time()
logger.info("start port_scan {}".format(len(all_ipv4_list)))
ip_port_result = []
if all_ipv4_list:
ip_port_result = services.port_scan(all_ipv4_list, **self.option)
elapse = time.time() - start_time
logger.info("end port_scan result {}, elapse {}".format(
len(ip_port_result), elapse))
ip_info_obj = []
for result in ip_port_result:
curr_ip = result["ip"]
result["domain"] = list(self.ipv4_map[curr_ip])
result["cdn_name"] = self.ip_cdn_map.get(curr_ip, "")
port_info_obj_list = []
for port_info in result["port_info"]:
port_info_obj_list.append(modules.PortInfo(**port_info))
result["port_info"] = port_info_obj_list
ip_info_obj.append(modules.IPInfo(**result))
if self.skip_scan_cdn_ip:
fake_cdn_ip_info = self.build_fake_cdn_ip_info()
ip_info_obj.extend(fake_cdn_ip_info)
return ip_info_obj
def port_scan(self):
scan_port_map = {
"test": ScanPortType.TEST,
"top100": ScanPortType.TOP100,
"top1000": ScanPortType.TOP1000,
"all": ScanPortType.ALL
}
option_scan_port_type = self.options.get("port_scan_type", "test")
scan_port_option = {
"ports": scan_port_map.get(option_scan_port_type,
ScanPortType.TEST),
"service_detect": self.options.get("service_detection", False),
"os_detect": self.options.get("os_detection", False)
}
targets = self.ip_target.split()
ip_port_result = services.port_scan(targets, **scan_port_option)
self.ip_info_list.extend(ip_port_result)
if self.task_tag == 'monitor':
self.set_asset_ip()
for ip_info in ip_port_result:
curr_ip = ip_info["ip"]
self.ip_set.add(curr_ip)
if not utils.not_in_black_ips(curr_ip):
continue
ip_info["task_id"] = self.task_id
ip_info["ip_type"] = utils.get_ip_type(curr_ip)
ip_info["geo_asn"] = {}
ip_info["geo_city"] = {}
if ip_info["ip_type"] == "PUBLIC":
ip_info["geo_asn"] = utils.get_ip_asn(curr_ip)
ip_info["geo_city"] = utils.get_ip_city(curr_ip)
# 仅仅资产发现任务将IP全部存储起来
if self.task_tag == 'task':
utils.conn_db('ip').insert_one(ip_info)
# 监控任务同步IP信息
if self.task_tag == 'monitor':
self.async_ip_info()
