def site_data(self):
if self._site_data is None:
sites = ["https://www.baidu.com", "https://www.qq.com/"]
site_data = services.fetch_site(sites, concurrency=2)
self._site_data = site_data
return self._site_data
def fetch_site(self):
'''***站点信息获取***'''
site_info_list = services.fetch_site(self.site_list)
self.site_info_list = site_info_list
for site_info in site_info_list:
curr_site = site_info["site"]
if curr_site not in self.site_list:
self.site_302_list.append(curr_site)
site_path = "/image/" + self.task_id
file_name = '{}/{}.jpg'.format(site_path,
utils.gen_filename(curr_site))
site_info["task_id"] = self.task_id
site_info["screenshot"] = file_name
finger_list = self.web_analyze_map.get(curr_site, [])
site_info["finger"] = finger_list
if self.options.get("site_identify"):
web_app_finger = services.web_app_identify(site_info)
flag = False
if web_app_finger and finger_list:
for finger in finger_list:
if finger["name"].lower(
) == web_app_finger["name"].lower():
flag = True
break
if not flag and web_app_finger:
finger_list.append(web_app_finger)
utils.conn_db('site').insert_one(site_info)
def web_app_identify():
site_list = ["http://10.0.83.6:81/login.php"]
site_info_list = services.fetch_site(site_list)
web_analyze_map = {}
for site_info in site_info_list:
curr_site = site_info["site"]
finger_list = [{
'name': 'xxx',
'confidence': '80',
'version': '',
'icon': 'default.png',
'website': 'https://www.riskivy.com',
'categories': []
}]
site_info["finger"] = finger_list
web_app_finger = services.web_app_identify(site_info)
flag = False
if web_app_finger and finger_list:
for finger in finger_list:
if finger["name"].lower() == web_app_finger["name"].lower():
flag = True
break
if not flag and web_app_finger:
finger_list.append(web_app_finger)
print(site_info["finger"])
def fetch_site(self):
site_info_list = services.fetch_site(self.site_list)
for site_info in site_info_list:
curr_site = site_info["site"]
if curr_site not in self.site_list:
self.site_302_list.append(curr_site)
site_path = "/image/" + self.task_id
file_name = '{}/{}.jpg'.format(site_path,
utils.gen_filename(curr_site))
site_info["task_id"] = self.task_id
site_info["screenshot"] = file_name
# 调用读取站点识别的结果，并且去重
if self.web_analyze_map:
finger_list = self.web_analyze_map.get(curr_site, [])
known_finger_set = set()
for finger_item in site_info["finger"]:
known_finger_set.add(finger_item["name"].lower())
for analyze_finger in finger_list:
analyze_name = analyze_finger["name"].lower()
if analyze_name not in known_finger_set:
site_info["finger"].append(analyze_finger)
utils.conn_db('site').insert_one(site_info)
if self.task_tag == 'monitor':
self.async_site_info(site_info_list)
def test_fetch_site(self):
sites = ["https://www.baidu.com"]
data = services.fetch_site(sites, concurrency=2)
self.assertTrue(len(data) >= 1)
self.assertTrue(len(data[0]["finger"]) >= 3)
self.assertTrue(len(data[0]["favicon"]["data"]) >= 10)
self.assertTrue(data[0]["favicon"]["hash"] == -1588080585)
def fetch_site(self):
site_info_list = services.fetch_site(self.site_list)
for site_info in site_info_list:
curr_site = site_info["site"]
if curr_site not in self.site_list:
self.site_302_list.append(curr_site)
site_path = "/image/" + self.task_id
file_name = '{}/{}.jpg'.format(site_path,
utils.gen_filename(curr_site))
site_info["task_id"] = self.task_id
site_info["screenshot"] = file_name
site_info["finger"] = []
utils.conn_db('site').insert_one(site_info)
def add_site_to_scope(site, scope_id):
fetch_site_data = services.fetch_site([site])
web_analyze_data = services.web_analyze([site])
finger = web_analyze_data.get(site, [])
curr_date = utils.curr_date_obj()
if fetch_site_data:
item = fetch_site_data[0]
item["finger"] = finger
item["screenshot"] = ""
item["scope_id"] = scope_id
item["save_date"] = curr_date
item["update_date"] = curr_date
utils.conn_db('asset_site').insert_one(item)
def fetch_site():
site_info = services.fetch_site(["https://www.baidu.com/"])
print(json.dumps(site_info))
def web_app_identify1():
site_info_list = services.fetch_site(["http://10.0.83.6:7001/"])
for site_info in site_info_list:
print(site_info)
app = services.web_app_identify(site_info)
print(app)
def test_leye_taobao(self):
sites = ["https://leye.taobao.com"]
data = services.fetch_site(sites, concurrency=2)
self.assertTrue(len(data) == 2)
self.assertTrue(len(data[0]["finger"]) >= 1)
