def get(self, task_id=None): """ 任务停止 """ done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR] task_data = utils.conn_db('task').find_one({'_id': ObjectId(task_id)}) if not task_data: return utils.build_ret(ErrorMsg.NotFoundTask, {"task_id": task_id}) if task_data["status"] in done_status: return utils.build_ret(ErrorMsg.TaskIsRunning, {"task_id": task_id}) celery_id = task_data.get("celery_id") if not celery_id: return utils.build_ret(ErrorMsg.CeleryIdNotFound, {"task_id": task_id}) control = celerytask.celery.control control.revoke(celery_id, signal='SIGTERM', terminate=True) utils.conn_db('task').update_one({'_id': ObjectId(task_id)}, {"$set": { "status": TaskStatus.STOP }}) utils.conn_db('task').update_one( {'_id': ObjectId(task_id)}, {"$set": { "end_time": utils.curr_date() }}) return {"message": "success", "task_id": task_id, "code": 200}
def post(self): """ 添加域名到资产组中 """ args = self.parse_args(add_domain_fields) domain = args.pop("domain") scope_id = args.pop("scope_id") if not utils.is_valid_domain(domain): return utils.build_ret(ErrorMsg.DomainInvalid, {"domain": domain}) scope_data = utils.conn_db('asset_scope').find_one({"_id": ObjectId(scope_id)}) if not scope_data: return utils.build_ret(ErrorMsg.NotFoundScopeID, {"scope_id": scope_id}) if utils.get_fld(domain) not in scope_data["scope"]: return utils.build_ret(ErrorMsg.DomainNotFoundViaScope, {"domain": domain}) domain_data = utils.conn_db("asset_domain").find_one({"domain": domain, "scope_id": scope_id}) if domain_data: return utils.build_ret(ErrorMsg.DomainInScope, {"domain": domain}) options = { "celery_action": CeleryAction.ADD_DOMAIN_TO_SCOPE, "data": { "domain": domain, "scope_id": scope_id } } celerytask.arl_task.delay(options=options) return utils.build_ret(ErrorMsg.Success, options["data"])
def fetch_site(self): site_info_list = services.fetch_site(self.site_list) for site_info in site_info_list: curr_site = site_info["site"] if curr_site not in self.site_list: self.site_302_list.append(curr_site) site_path = "/image/" + self.task_id file_name = '{}/{}.jpg'.format(site_path, utils.gen_filename(curr_site)) site_info["task_id"] = self.task_id site_info["screenshot"] = file_name # 调用读取站点识别的结果,并且去重 if self.web_analyze_map: finger_list = self.web_analyze_map.get(curr_site, []) known_finger_set = set() for finger_item in site_info["finger"]: known_finger_set.add(finger_item["name"].lower()) for analyze_finger in finger_list: analyze_name = analyze_finger["name"].lower() if analyze_name not in known_finger_set: site_info["finger"].append(analyze_finger) utils.conn_db('site').insert_one(site_info) if self.task_tag == 'monitor': self.async_site_info(site_info_list)
def save_service_info(self): self.service_info_list = [] services_list = set() for _data in self.ip_info_list: port_info_lsit = _data.port_info_list for _info in port_info_lsit: if _info.service_name: if _info.service_name not in services_list: _result = {} _result["service_name"] = _info.service_name _result["service_info"] = [] _result["service_info"].append({ 'ip': _data.ip, 'port_id': _info.port_id, 'product': _info.product, 'version': _info.version }) _result["task_id"] = self.task_id self.service_info_list.append(_result) services_list.add(_info.service_name) else: for service_info in self.service_info_list: if service_info.get( "service_name") == _info.service_name: service_info['service_info'].append({ 'ip': _data.ip, 'port_id': _info.port_id, 'product': _info.product, 'version': _info.version }) utils.conn_db('service').insert(self.service_info_list)
def fetch_site(self): '''***站点信息获取***''' site_info_list = services.fetch_site(self.site_list) self.site_info_list = site_info_list for site_info in site_info_list: curr_site = site_info["site"] if curr_site not in self.site_list: self.site_302_list.append(curr_site) site_path = "/image/" + self.task_id file_name = '{}/{}.jpg'.format(site_path, utils.gen_filename(curr_site)) site_info["task_id"] = self.task_id site_info["screenshot"] = file_name finger_list = self.web_analyze_map.get(curr_site, []) site_info["finger"] = finger_list if self.options.get("site_identify"): web_app_finger = services.web_app_identify(site_info) flag = False if web_app_finger and finger_list: for finger in finger_list: if finger["name"].lower( ) == web_app_finger["name"].lower(): flag = True break if not flag and web_app_finger: finger_list.append(web_app_finger) utils.conn_db('site').insert_one(site_info)
def fofa_search(self): try: f = fofaSearch(self.domain_info_list, self.base_domain) ips = f.run() for ip in ips: if ip not in self.ip_set: self.fofa_ip_set.add(ip) if self.options.get("port_scan"): ip_port_result = services.port_scan(self.fofa_ip_set, **self.scan_port_option) for ip_info in ip_port_result: ip_info["domain"] = ["*.{}".format(self.base_domain)] port_info_obj_list = [] for port_info in ip_info["port_info"]: port_info_obj_list.append( modules.PortInfo(**port_info)) ip_info["port_info"] = port_info_obj_list fake_info_obj = modules.IPInfo(**ip_info) fake_ip_info = fake_info_obj.dump_json(flag=False) fake_ip_info["task_id"] = self.task_id utils.conn_db('ip').insert_one(fake_ip_info) for ip in self.fofa_ip_set: self.ipv4_map[ip] = ["*.{}".format(self.base_domain)] logger.info("fofa search {} {}".format(self.base_domain, len(self.fofa_ip_set))) except Exception as e: logger.exception(e) logger.warning("fofa search error {}, {}".format( self.base_domain, e))
def site_spider(self): entry_urls_list = [] for site in self.site_list: entry_urls = [site] entry_urls.extend(self.search_engines_result.get(site, [])) entry_urls_list.append(entry_urls) site_spider_result = services.site_spider_thread(entry_urls_list) for site in site_spider_result: target_urls = site_spider_result[site] new_target_urls = [] for url in target_urls: if url in self.page_url_list: continue new_target_urls.append(url) self.page_url_list.append(url) page_map = services.page_fetch(new_target_urls) for url in page_map: item = { "site": site, "task_id": self.task_id, "source": CollectSource.SITESPIDER } item.update(page_map[url]) domain_parsed = utils.domain_parsed(site) if domain_parsed: item["fld"] = domain_parsed["fld"] utils.conn_db('url').insert_one(item)
def port_scan(self): scan_port_map = { "test": ScanPortType.TEST, "top100": ScanPortType.TOP100, "top1000": ScanPortType.TOP1000, "all": ScanPortType.ALL } option_scan_port_type = self.options.get("port_scan_type", "test") scan_port_option = { "ports": scan_port_map.get(option_scan_port_type, ScanPortType.TEST), "service_detect": self.options.get("service_detection", False), "os_detect": self.options.get("os_detection", False) } targets = self.ip_target.split() ip_port_result = services.port_scan(targets, **scan_port_option) self.ip_info_list.extend(ip_port_result) for ip_info in ip_port_result: self.ip_set.add(ip_info["ip"]) if not utils.not_in_black_ips(ip_info["ip"]): continue ip_info["task_id"] = self.task_id ip_info["ip_type"] = utils.get_ip_type(ip_info["ip"]) ip_info["geo_asn"] = {} ip_info["geo_city"] = {} if ip_info["ip_type"] == "PUBLIC": ip_info["geo_asn"] = utils.get_ip_asn(ip_info["ip"]) ip_info["geo_city"] = utils.get_ip_city(ip_info["ip"]) utils.conn_db('ip').insert_one(ip_info)
def post(self): """ 删除资产站点Tag """ args = self.parse_args(delete_asset_site_tag_fields) site_id = args.pop("_id") tag = args.pop("tag") query = {"_id": ObjectId(site_id)} data = utils.conn_db('asset_site').find_one(query) if not data: return utils.build_ret(ErrorMsg.SiteIdNotFound, {"site_id": site_id}) tag_list = [] old_tag = data.get("tag") if old_tag: if isinstance(old_tag, str): tag_list.append(old_tag) if isinstance(old_tag, list): tag_list.extend(old_tag) if tag not in tag_list: return utils.build_ret(ErrorMsg.SiteTagNotExist, {"tag": tag}) tag_list.remove(tag) utils.conn_db('asset_site').update_one(query, {"$set": { "tag": tag_list }}) return utils.build_ret(ErrorMsg.Success, {"tag": tag})
def run_brute(self): """运行爆破,获取进度""" target = self.site_list + list(self.npoc_service_target_set) plugin_name = self.brute_plugin_name logger.info("start run brute {}*{}".format(len(plugin_name), len(target))) run_total = len(plugin_name) * len(target) npoc_instance = npoc.NPoC(tmp_dir=Config.TMP_PATH, concurrency=10) run_thread = Thread(target=npoc_instance.run_poc, args=(plugin_name, target)) run_thread.start() while run_thread.is_alive(): time.sleep(5) status = "brute {}/{}".format(npoc_instance.runner.runner_cnt, run_total) logger.info("[{}]runner cnt {}/{}".format( self.task_id, npoc_instance.runner.runner_cnt, run_total)) self.update_task_field("status", status) result = npoc_instance.result for item in result: item["task_id"] = self.task_id item["save_date"] = utils.curr_date() utils.conn_db('vuln').insert_one(item)
def post(self): """ 添加站点到资产组中 """ args = self.parse_args(add_site_fields) site = args.pop("site") scope_id = args.pop("scope_id") url = utils.normal_url(site).strip("/") if not url: return utils.build_ret(ErrorMsg.DomainInvalid, {"site": site}) scope_data = utils.conn_db('asset_scope').find_one( {"_id": ObjectId(scope_id)}) if not scope_data: return utils.build_ret(ErrorMsg.NotFoundScopeID, {"scope_id": scope_id}) fld = utils.get_fld(url) if not fld: return utils.build_ret(ErrorMsg.SiteURLNotDomain, {"site": url}) if fld not in scope_data["scope"]: return utils.build_ret(ErrorMsg.DomainNotFoundViaScope, {"site": url}) site_data = utils.conn_db('asset_site').find_one({ "site": url, "scope_id": scope_id }) if site_data: return utils.build_ret(ErrorMsg.SiteInScope, {"site": url}) add_site_to_scope(url, scope_id) return utils.build_ret(ErrorMsg.Success, {"site": url})
def get(self, task_id=None): """ 任务删除 """ done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR] args = self.parser.parse_args() del_task_data_flag = str(args.pop('del_task_data', "false")).lower() task_data = utils.conn_db('task').find_one({'_id': ObjectId(task_id)}) if not task_data: return { "message": "not found task", "task_id": task_id, "code": 103 } if task_data["status"] not in done_status: return { "message": "task is running", "task_id": task_id, "code": 104 } utils.conn_db('task').delete_many({'_id': ObjectId(task_id)}) table_list = [ "cert", "domain", "fileleak", "ip", "service", "site", "url" ] if del_task_data_flag == "true": for name in table_list: utils.conn_db(name).delete_many({'task_id': task_id}) return {"message": "success", "task_id": task_id, "code": 200}
def delete_db(self): for name in self.db_plugin_name_list: if name not in self.plugin_name_list: query = {"plugin_name": name} utils.conn_db('poc').delete_one(query) return True
def save_service_info(self): self.service_info_list = [] services_list = set() for _data in self.ip_info_list: port_info_lsit = _data.get("port_info") for _info in port_info_lsit: if _info.get("service_name"): if _info.get("service_name") not in services_list: _result = {} _result["service_name"] = _info.get("service_name") _result["service_info"] = [] _result["service_info"].append({'ip': _data.get("ip"), 'port_id': _info.get("port_id"), 'product': _info.get("product"), 'version': _info.get("version")}) _result["task_id"] = self.task_id self.service_info_list.append(_result) services_list.add(_info.get("service_name")) else: for service_info in self.service_info_list: if service_info.get("service_name") == _info.get("service_name"): service_info['service_info'].append({'ip': _data.get("ip"), 'port_id': _info.get("port_id"), 'product': _info.get("product"), 'version': _info.get("version")}) if self.service_info_list: utils.conn_db('service').insert(self.service_info_list)
def async_site_info(self, site_info_list): """ 用来同步发现的 site 中的信息,仅仅在监控阶段使用 """ new_site_info_list = [] for site_info in site_info_list: curr_date_obj = utils.curr_date_obj() query = {"site": site_info["site"], "scope_id": self.scope_id} data = utils.conn_db('asset_site').find_one(query) if data: continue new_site_info_list.append(site_info) site_info["save_date"] = curr_date_obj site_info["update_date"] = curr_date_obj site_info["scope_id"] = self.scope_id utils.conn_db('asset_site').insert_one(site_info) new_asset_map = { "site": new_site_info_list[:10], "ip": self.ip_info_list[:10], "task_name": self.task_name } new_asset_counter = { "site": len(new_site_info_list), "ip": len(self.ip_info_list) } if len(self.ip_info_list) > 0: utils.message_push(asset_map=new_asset_map, asset_counter=new_asset_counter)
def post(self): """ 添加指纹信息 """ args = self.parse_args(add_fingerprint_fields) human_rule = args.pop('human_rule') name = args.pop('name') rule_map = parse_human_rule(human_rule) if rule_map is None: return utils.build_ret(ErrorMsg.RuleInvalid, {"rule": human_rule}) data = { "name": name, "rule": rule_map, "human_rule": transform_rule_map(rule_map), "update_date": utils.curr_date_obj() } utils.conn_db('fingerprint').insert_one(data) finger_id = str(data.pop('_id')) data.pop('update_date') return utils.build_ret(ErrorMsg.Success, { "_id": finger_id, "data": data })
def get(self): """ 针对资产组删除范围 """ args = self.parser.parse_args() scope = str(args.pop('scope', "")).lower() scope_id = str(args.pop('scope_id', "")).lower() scope_data = self.get_scope_data(scope_id) if not scope_data: return utils.build_ret(ErrorMsg.NotFoundScopeID, {"scope_id": scope_id}) query = {'_id': ObjectId(scope_id)} if scope not in scope_data.get("scope_array", []): return utils.build_ret(ErrorMsg.NotFoundScope, { "scope_id": scope_id, "scope": scope }) scope_data["scope_array"].remove(scope) scope_data["scope"] = ",".join(scope_data["scope_array"]) utils.conn_db(self._table).find_one_and_replace(query, scope_data) return utils.build_ret(ErrorMsg.Success, { "scope_id": scope_id, "scope": scope })
def add_domain_to_scope(domain, scope_id): task_data = { 'name': '添加域名', 'target': domain, 'start_time': '-', 'status': 'waiting', 'type': 'domain', "task_tag": "task", 'options': { 'domain_brute': True, 'domain_brute_type': 'test', 'port_scan_type': 'test', 'port_scan': True, 'service_detection': False, 'service_brute': False, 'os_detection': False, 'site_identify': True, 'site_capture': False, 'file_leak': False, 'alt_dns': False, 'site_spider': False, 'search_engines': False, 'ssl_cert': False, 'fofa_search': False, 'crtsh_search': True } } utils.conn_db('task').insert_one(task_data) task_id = str(task_data.pop("_id")) domain_task(domain, task_id, task_data["options"]) services.sync_asset(task_id=task_id, scope_id=scope_id)
def post(self): """ 策略编辑 """ args = self.parse_args(edit_policy_fields) policy_id = args.pop('policy_id') policy_data = args.pop('policy_data', {}) query = {'_id': ObjectId(policy_id)} item = utils.conn_db('policy').find_one(query) if not item: return utils.build_ret(ErrorMsg.PolicyIDNotFound, {}) if not policy_data: return utils.build_ret(ErrorMsg.PolicyDataIsEmpty, {}) item = change_dict(item, policy_data) poc_config = item["policy"].pop("poc_config", []) poc_config = _update_plugin_config(poc_config) if isinstance(poc_config, str): return utils.build_ret(poc_config, {}) item["policy"]["poc_config"] = poc_config brute_config = item["policy"].pop("brute_config", []) brute_config = _update_plugin_config(brute_config) if isinstance(brute_config, str): return utils.build_ret(brute_config, {}) item["policy"]["brute_config"] = brute_config item["update_date"] = utils.curr_date() utils.conn_db('policy').find_one_and_replace(query, item) item.pop('_id') return utils.build_ret(ErrorMsg.Success, {"data": item})
def post(self): """ 任务删除 """ done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR] args = self.parse_args(delete_task_fields) task_id_list = args.pop('task_id') del_task_data_flag = args.pop('del_task_data') for task_id in task_id_list: task_data = utils.conn_db('task').find_one( {'_id': ObjectId(task_id)}) if not task_data: return utils.build_ret(ErrorMsg.NotFoundTask, {"task_id": task_id}) if task_data["status"] not in done_status: return utils.build_ret(ErrorMsg.TaskIsRunning, {"task_id": task_id}) for task_id in task_id_list: utils.conn_db('task').delete_many({'_id': ObjectId(task_id)}) table_list = [ "cert", "domain", "fileleak", "ip", "service", "site", "url" ] if del_task_data_flag: for name in table_list: utils.conn_db(name).delete_many({'task_id': task_id}) return utils.build_ret(ErrorMsg.Success, {"task_id": task_id_list})
def insert_task_stat(self): query = {"_id": ObjectId(self.task_id)} stat = utils.arl.task_statistic(self.task_id) update = {"$set": {"statistic": stat}} utils.conn_db('task').update_one(query, update)
def npoc_service_detection(self): logger.info("start npoc_service_detection {}".format( len(self.sniffer_target_set))) result = npoc.run_sniffer(self.sniffer_target_set) for item in result: self.npoc_service_target_set.add(item["target"]) item["task_id"] = self.task_id item["save_date"] = utils.curr_date() utils.conn_db('npoc_service').insert_one(item)
def file_leak(self): for site in self.site_list: pages = services.file_leak([site], utils.load_file(Config.FILE_LEAK_TOP_2k)) for page in pages: item = page.dump_json() item["task_id"] = self.task_id item["site"] = site utils.conn_db('fileleak').insert_one(item)
def save_domain_info_list(self, domain_info_list, source = CollectSource.DOMAIN_BRUTE): for domain_info_obj in domain_info_list: domain_info = domain_info_obj.dump_json(flag=False) domain_info["task_id"] = self.task_id domain_info["source"] = source domain_parsed = utils.domain_parsed(domain_info["domain"]) if domain_parsed: domain_info["fld"] = domain_parsed["fld"] utils.conn_db('domain').insert_one(domain_info)
def post(self): """ 添加域名到资产组中 """ args = self.parse_args(add_domain_fields) raw_domain = args.pop("domain") domain_list = re.split(r",|\s", raw_domain) domain_list = list(set(filter(None, domain_list))) scope_id = args.pop("scope_id") scope_data = utils.conn_db('asset_scope').find_one( {"_id": ObjectId(scope_id)}) if not scope_data: return utils.build_ret(ErrorMsg.NotFoundScopeID, {"scope_id": scope_id}) domain_in_scope_list = [] add_domain_list = [] for domain in domain_list: if not utils.is_valid_domain(domain): return utils.build_ret(ErrorMsg.DomainInvalid, {"domain": domain}) if utils.get_fld(domain) not in scope_data["scope"]: return utils.build_ret(ErrorMsg.DomainNotFoundViaScope, {"domain": domain}) domain_data = utils.conn_db("asset_domain").find_one({ "domain": domain, "scope_id": scope_id }) if domain_data: domain_in_scope_list.append(domain) continue add_domain_list.append(domain) for domain in add_domain_list: options = { "celery_action": CeleryAction.ADD_DOMAIN_TO_SCOPE, "data": { "domain": domain, "scope_id": scope_id } } celerytask.arl_task.delay(options=options) ret_data = { "domain": ",".join(add_domain_list), "scope_id": scope_id, "domain_in_scope": ",".join(domain_in_scope_list) } if len(add_domain_list) == 0: return utils.build_ret(ErrorMsg.DomainNotFoundNotInScope, ret_data) return utils.build_ret(ErrorMsg.Success, ret_data)
def post(self): """ 添加资产范围 """ args = self.parse_args(add_scope_fields) scope = str(args.pop('scope', "")).lower() scope_id = args.pop('scope_id', "") table = 'asset_scope' query = {'_id': ObjectId(scope_id)} scope_data = utils.conn_db(table).find_one(query) if not scope_data: return utils.build_ret(ErrorMsg.NotFoundScopeID, { "scope_id": scope_id, "scope": scope }) scope_type = scope_data.get("scope_type") if scope_type not in [AssetScopeType.IP, AssetScopeType.DOMAIN]: scope_type = AssetScopeType.DOMAIN scope_array = re.split(r",|\s", scope) # 清除空白符 scope_array = list(filter(None, scope_array)) if not scope_array: return utils.build_ret(ErrorMsg.DomainInvalid, {"scope": ""}) for x in scope_array: new_scope = x if scope_type == AssetScopeType.DOMAIN: if not utils.is_valid_domain(x): return utils.build_ret(ErrorMsg.DomainInvalid, {"scope": x}) if scope_type == AssetScopeType.IP: transfer = utils.ip.transfer_ip_scope(x) if transfer is None: return utils.build_ret(ErrorMsg.ScopeTypeIsNotIP, {"scope": x}) new_scope = transfer if new_scope in scope_data.get("scope_array", []): return utils.build_ret(ErrorMsg.ExistScope, { "scope_id": scope_id, "scope": x }) scope_data["scope_array"].append(new_scope) scope_data["scope"] = ",".join(scope_data["scope_array"]) utils.conn_db(table).find_one_and_replace(query, scope_data) return utils.build_ret(ErrorMsg.Success, { "scope_id": scope_id, "scope": scope })
def port_scan(self): ip_info_list = scan_port(self.domain_info_list, self.scan_port_option) for ip_info_obj in ip_info_list: ip_info = ip_info_obj.dump_json(flag=False) ip_info["task_id"] = self.task_id utils.conn_db('ip').insert_one(ip_info) self.ip_info_list.extend(ip_info_list)
def post(self): """ 任务同步 """ done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR] args = self.parse_args(sync_task_fields) task_id = args.pop('task_id') scope_id = args.pop('scope_id') query = {'_id': ObjectId(task_id)} task_data = utils.conn_db('task').find_one(query) if not task_data: return utils.build_ret(ErrorMsg.NotFoundTask, {"task_id": task_id}) asset_scope_data = utils.conn_db('asset_scope').find_one( {'_id': ObjectId(scope_id)}) if not asset_scope_data: return utils.build_ret(ErrorMsg.NotFoundScopeID, {"task_id": task_id}) if task_data.get("type") != "domain": return utils.build_ret(ErrorMsg.TaskTypeIsNotDomain, {"task_id": task_id}) if not utils.is_in_scopes(task_data["target"], asset_scope_data["scope_array"]): return utils.build_ret(ErrorMsg.TaskTargetNotInScope, {"task_id": task_id}) if task_data["status"] not in done_status: return utils.build_ret(ErrorMsg.TaskIsRunning, {"task_id": task_id}) task_sync_status = task_data.get("sync_status", TaskSyncStatus.DEFAULT) if task_sync_status not in [ TaskSyncStatus.DEFAULT, TaskSyncStatus.ERROR ]: return utils.build_ret(ErrorMsg.TaskSyncDealing, {"task_id": task_id}) task_data["sync_status"] = TaskSyncStatus.WAITING options = { "celery_action": CeleryAction.DOMAIN_TASK_SYNC_TASK, "data": { "task_id": task_id, "scope_id": scope_id } } celerytask.arl_task.delay(options=options) conn('task').find_one_and_replace(query, task_data) return utils.build_ret(ErrorMsg.Success, {"task_id": task_id})
def post(self): """ 删除资产组中的IP """ args = self.parse_args(delete_ip_fields) id_list = args.pop('_id', "") for _id in id_list: query = {'_id': ObjectId(_id)} utils.conn_db('asset_ip').delete_one(query) return utils.build_ret(ErrorMsg.Success, {'_id': id_list})
def post(self): """ 删除站点 """ args = self.parse_args(delete_site_fields) id_list = args.pop('_id', []) for _id in id_list: query = {'_id': ObjectId(_id)} utils.conn_db('site').delete_one(query) return utils.build_ret(ErrorMsg.Success, {'_id': id_list})