def site_data(self): if self._site_data is None: sites = ["https://www.baidu.com", "https://www.qq.com/"] site_data = services.fetch_site(sites, concurrency=2) self._site_data = site_data return self._site_data
def fetch_site(self): '''***站点信息获取***''' site_info_list = services.fetch_site(self.site_list) self.site_info_list = site_info_list for site_info in site_info_list: curr_site = site_info["site"] if curr_site not in self.site_list: self.site_302_list.append(curr_site) site_path = "/image/" + self.task_id file_name = '{}/{}.jpg'.format(site_path, utils.gen_filename(curr_site)) site_info["task_id"] = self.task_id site_info["screenshot"] = file_name finger_list = self.web_analyze_map.get(curr_site, []) site_info["finger"] = finger_list if self.options.get("site_identify"): web_app_finger = services.web_app_identify(site_info) flag = False if web_app_finger and finger_list: for finger in finger_list: if finger["name"].lower( ) == web_app_finger["name"].lower(): flag = True break if not flag and web_app_finger: finger_list.append(web_app_finger) utils.conn_db('site').insert_one(site_info)
def web_app_identify(): site_list = ["http://10.0.83.6:81/login.php"] site_info_list = services.fetch_site(site_list) web_analyze_map = {} for site_info in site_info_list: curr_site = site_info["site"] finger_list = [{ 'name': 'xxx', 'confidence': '80', 'version': '', 'icon': 'default.png', 'website': 'https://www.riskivy.com', 'categories': [] }] site_info["finger"] = finger_list web_app_finger = services.web_app_identify(site_info) flag = False if web_app_finger and finger_list: for finger in finger_list: if finger["name"].lower() == web_app_finger["name"].lower(): flag = True break if not flag and web_app_finger: finger_list.append(web_app_finger) print(site_info["finger"])
def fetch_site(self): site_info_list = services.fetch_site(self.site_list) for site_info in site_info_list: curr_site = site_info["site"] if curr_site not in self.site_list: self.site_302_list.append(curr_site) site_path = "/image/" + self.task_id file_name = '{}/{}.jpg'.format(site_path, utils.gen_filename(curr_site)) site_info["task_id"] = self.task_id site_info["screenshot"] = file_name # 调用读取站点识别的结果,并且去重 if self.web_analyze_map: finger_list = self.web_analyze_map.get(curr_site, []) known_finger_set = set() for finger_item in site_info["finger"]: known_finger_set.add(finger_item["name"].lower()) for analyze_finger in finger_list: analyze_name = analyze_finger["name"].lower() if analyze_name not in known_finger_set: site_info["finger"].append(analyze_finger) utils.conn_db('site').insert_one(site_info) if self.task_tag == 'monitor': self.async_site_info(site_info_list)
def test_fetch_site(self): sites = ["https://www.baidu.com"] data = services.fetch_site(sites, concurrency=2) self.assertTrue(len(data) >= 1) self.assertTrue(len(data[0]["finger"]) >= 3) self.assertTrue(len(data[0]["favicon"]["data"]) >= 10) self.assertTrue(data[0]["favicon"]["hash"] == -1588080585)
def fetch_site(self): site_info_list = services.fetch_site(self.site_list) for site_info in site_info_list: curr_site = site_info["site"] if curr_site not in self.site_list: self.site_302_list.append(curr_site) site_path = "/image/" + self.task_id file_name = '{}/{}.jpg'.format(site_path, utils.gen_filename(curr_site)) site_info["task_id"] = self.task_id site_info["screenshot"] = file_name site_info["finger"] = [] utils.conn_db('site').insert_one(site_info)
def add_site_to_scope(site, scope_id): fetch_site_data = services.fetch_site([site]) web_analyze_data = services.web_analyze([site]) finger = web_analyze_data.get(site, []) curr_date = utils.curr_date_obj() if fetch_site_data: item = fetch_site_data[0] item["finger"] = finger item["screenshot"] = "" item["scope_id"] = scope_id item["save_date"] = curr_date item["update_date"] = curr_date utils.conn_db('asset_site').insert_one(item)
def fetch_site(): site_info = services.fetch_site(["https://www.baidu.com/"]) print(json.dumps(site_info))
def web_app_identify1(): site_info_list = services.fetch_site(["http://10.0.83.6:7001/"]) for site_info in site_info_list: print(site_info) app = services.web_app_identify(site_info) print(app)
def test_leye_taobao(self): sites = ["https://leye.taobao.com"] data = services.fetch_site(sites, concurrency=2) self.assertTrue(len(data) == 2) self.assertTrue(len(data[0]["finger"]) >= 1)