def run(self): self.update_task_field("start_time", utils.curr_date()) self.domain_fetch() for domain_info in self.domain_info_list: self.domain_set.add(domain_info.domain) self.set_scope_domain() new_domain_set = self.domain_set - self.scope_domain_set self.new_domain_set = new_domain_set self.set_domain_info_list() #仅仅对新增域名保留 self.start_ip_fetch() self.start_site_fetch() self.update_task_field("status", TaskStatus.DONE) self.update_task_field("end_time", utils.curr_date()) ret_new_domain_set = set() for domain_info in self.domain_info_list: ret_new_domain_set.add(domain_info.domain) return ret_new_domain_set
def run(self): try: self.update_task_field("start_time", utils.curr_date()) self.work() self.update_task_field("status", TaskStatus.DONE) except Exception as e: self.update_task_field("status", TaskStatus.ERROR) logger.exception(e) self.update_task_field("end_time", utils.curr_date())
def run(self): self.update_task_field("start_time", utils.curr_date()) self.domain_fetch() self.start_ip_fetch() self.start_site_fetch() self.update_task_field("status", TaskStatus.DONE) self.update_task_field("end_time", utils.curr_date())
def get(self, task_id=None): """ 任务停止 """ done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR] task_data = utils.conn_db('task').find_one({'_id': ObjectId(task_id)}) if not task_data: return utils.build_ret(ErrorMsg.NotFoundTask, {"task_id": task_id}) if task_data["status"] in done_status: return utils.build_ret(ErrorMsg.TaskIsRunning, {"task_id": task_id}) celery_id = task_data.get("celery_id") if not celery_id: return utils.build_ret(ErrorMsg.CeleryIdNotFound, {"task_id": task_id}) control = celerytask.celery.control control.revoke(celery_id, signal='SIGTERM', terminate=True) utils.conn_db('task').update_one({'_id': ObjectId(task_id)}, {"$set": { "status": TaskStatus.STOP }}) utils.conn_db('task').update_one( {'_id': ObjectId(task_id)}, {"$set": { "end_time": utils.curr_date() }}) return {"message": "success", "task_id": task_id, "code": 200}
def run_brute(self): """运行爆破,获取进度""" target = self.site_list + list(self.npoc_service_target_set) plugin_name = self.brute_plugin_name logger.info("start run brute {}*{}".format(len(plugin_name), len(target))) run_total = len(plugin_name) * len(target) npoc_instance = npoc.NPoC(tmp_dir=Config.TMP_PATH, concurrency=10) run_thread = Thread(target=npoc_instance.run_poc, args=(plugin_name, target)) run_thread.start() while run_thread.is_alive(): time.sleep(5) status = "brute {}/{}".format(npoc_instance.runner.runner_cnt, run_total) logger.info("[{}]runner cnt {}/{}".format( self.task_id, npoc_instance.runner.runner_cnt, run_total)) self.update_task_field("status", status) result = npoc_instance.result for item in result: item["task_id"] = self.task_id item["save_date"] = utils.curr_date() utils.conn_db('vuln').insert_one(item)
def post(self): """ 策略编辑 """ args = self.parse_args(edit_policy_fields) policy_id = args.pop('policy_id') policy_data = args.pop('policy_data', {}) query = {'_id': ObjectId(policy_id)} item = utils.conn_db('policy').find_one(query) if not item: return utils.build_ret(ErrorMsg.PolicyIDNotFound, {}) if not policy_data: return utils.build_ret(ErrorMsg.PolicyDataIsEmpty, {}) item = change_dict(item, policy_data) poc_config = item["policy"].pop("poc_config", []) poc_config = _update_plugin_config(poc_config) if isinstance(poc_config, str): return utils.build_ret(poc_config, {}) item["policy"]["poc_config"] = poc_config brute_config = item["policy"].pop("brute_config", []) brute_config = _update_plugin_config(brute_config) if isinstance(brute_config, str): return utils.build_ret(brute_config, {}) item["policy"]["brute_config"] = brute_config item["update_date"] = utils.curr_date() utils.conn_db('policy').find_one_and_replace(query, item) item.pop('_id') return utils.build_ret(ErrorMsg.Success, {"data": item})
def wrap_domain_executors(base_domain=None, job_id=None, scope_id=None, options=None, name=""): celery_id = "" if current_task._get_current_object(): celery_id = current_task.request.id task_data = { 'name': name, 'target': base_domain, 'start_time': '-', 'status': 'waiting', 'type': 'domain', 'task_tag': 'monitor', #标记为监控任务 'options': { 'domain_brute': True, 'domain_brute_type': 'test', 'riskiq_search': False, 'alt_dns': False, 'arl_search': True, 'port_scan_type': 'test', 'port_scan': True, 'service_detection': False, 'service_brute': False, 'os_detection': False, 'site_identify': False, 'site_capture': False, 'file_leak': False, 'site_spider': False, 'search_engines': False, 'ssl_cert': False, 'fofa_search': False, 'scope_id': scope_id }, 'celery_id': celery_id } if options is None: options = {} task_data["options"].update(options) conn('task').insert_one(task_data) task_id = str(task_data.pop("_id")) domain_executor = DomainExecutor(base_domain, task_id, task_data["options"]) try: update_job_run(job_id) new_domain = domain_executor.run() if new_domain: sync_asset(task_id, scope_id, update_flag=True) except Exception as e: logger.exception(e) domain_executor.update_task_field("status", TaskStatus.ERROR) domain_executor.update_task_field("end_time", utils.curr_date()) logger.info("end domain_executors {} {} {}".format(base_domain, scope_id, options))
def domain_task(base_domain, task_id, options): d = DomainTask(base_domain=base_domain, task_id=task_id, options=options) try: d.run() except Exception as e: logger.exception(e) d.update_task_field("status", TaskStatus.ERROR) d.update_task_field("end_time", utils.curr_date())
def npoc_service_detection(self): logger.info("start npoc_service_detection {}".format( len(self.sniffer_target_set))) result = npoc.run_sniffer(self.sniffer_target_set) for item in result: self.npoc_service_target_set.add(item["target"]) item["task_id"] = self.task_id item["save_date"] = utils.curr_date() utils.conn_db('npoc_service').insert_one(item)
def run(self): self.update_task_field("start_time", utils.curr_date()) self.domain_fetch() self.start_ip_fetch() self.start_site_fetch() self.start_poc_run() # cidr ip 结果统计,插入cip 集合中 self.insert_cip_stat() # 任务结果统计 self.insert_task_stat() self.update_task_field("status", TaskStatus.DONE) self.update_task_field("end_time", utils.curr_date())
def sync_to_db(self): for old in self.poc_info_list: new = old.copy() plugin_name = old["plugin_name"] new["update_date"] = utils.curr_date() if plugin_name in self.db_plugin_name_list: continue logger.info("insert {} info to db".format(plugin_name)) utils.conn_db('poc').insert_one(new) return True
def post(self): args = self.parse_args(add_policy_fields) name = args.pop("name") policy = args.pop("policy", {}) if policy is None: return utils.build_ret("Missing policy parameter", {}) domain_config = policy.pop("domain_config", {}) domain_config = self._update_arg(domain_config, domain_config_fields) ip_config = policy.pop("ip_config", {}) ip_config = self._update_arg(ip_config, ip_config_fields) site_config = policy.pop("site_config", {}) site_config = self._update_arg(site_config, site_config_fields) poc_config = policy.pop("poc_config", []) if poc_config is None: poc_config = [] poc_config = _update_plugin_config(poc_config) if isinstance(poc_config, str): return utils.build_ret(poc_config, {}) brute_config = policy.pop("brute_config", []) if brute_config is None: brute_config = [] brute_config = _update_plugin_config(brute_config) if isinstance(brute_config, str): return utils.build_ret(brute_config, {}) file_leak = fields.boolean(policy.pop("file_leak", False)) npoc_service_detection = fields.boolean( policy.pop("npoc_service_detection", False)) desc = args.pop("desc", "") item = { "name": name, "policy": { "domain_config": domain_config, "ip_config": ip_config, "site_config": site_config, "poc_config": poc_config, "brute_config": brute_config, "file_leak": file_leak, "npoc_service_detection": npoc_service_detection }, "desc": desc, "update_date": utils.curr_date() } utils.conn_db("policy").insert_one(item) return utils.build_ret(ErrorMsg.Success, {"policy_id": str(item["_id"])})
def risk_cruising(self): """运行PoC任务""" poc_config = self.options.get("poc_config", []) plugins = [] for info in poc_config: if not info.get("enable"): continue plugins.append(info["plugin_name"]) result = run_risk_cruising(plugins=plugins, targets=self.site_list) for item in result: item["task_id"] = self.task_id item["save_date"] = utils.curr_date() utils.conn_db('vuln').insert_one(item)
def npoc_service_detection(self): targets = [] for ip_info in self.ip_info_list: for port_info in ip_info.port_info_list: if port_info.port_id == 80: continue if port_info.port_id == 443: continue targets.append("{}:{}".format(ip_info.ip, port_info.port_id)) result = run_sniffer(targets) for item in result: self.npoc_service_target_set.add(item["target"]) item["task_id"] = self.task_id item["save_date"] = utils.curr_date() utils.conn_db('npoc_service').insert_one(item)
def brute_config(self): plugins = [] brute_config = self.options.get("brute_config") for x in brute_config: if not x.get("enable"): continue plugins.append(x["plugin_name"]) if not plugins: return targets = self.site_list.copy() targets += list(self.npoc_service_target_set) result = run_risk_cruising(targets=targets, plugins=plugins) for item in result: item["task_id"] = self.task_id item["save_date"] = utils.curr_date() utils.conn_db('vuln').insert_one(item)
def get(self, task_id=None): """ 任务停止 """ done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR] task_data = utils.conn_db('task').find_one({'_id': ObjectId(task_id)}) if not task_data: return { "message": "not found task", "task_id": task_id, "code": 100 } if task_data["status"] in done_status: return { "message": "error, task is done", "task_id": task_id, "code": 101 } celery_id = task_data.get("celery_id") if not celery_id: return { "message": "not found celery_id", "task_id": task_id, "code": 102 } control = celerytask.celery.control #由subprocess启动的进程还存在 /doge control.revoke(celery_id, terminate=True) utils.conn_db('task').update_one({'_id': ObjectId(task_id)}, {"$set": { "status": TaskStatus.STOP }}) utils.conn_db('task').update_one( {'_id': ObjectId(task_id)}, {"$set": { "end_time": utils.curr_date() }}) return {"message": "success", "task_id": task_id, "code": 200}
def run(self): self.update_task_field("start_time", utils.curr_date()) '''***端口扫描开始***''' if self.options.get("port_scan"): self.update_task_field("status", "port_scan") t1 = time.time() self.port_scan() elapse = time.time() - t1 self.update_services("port_scan", elapse) # 存储服务信息 if self.options.get("service_detection"): self.save_service_info() '''***证书获取开始***''' if self.options.get("ssl_cert"): self.update_task_field("status", "ssl_cert") t1 = time.time() self.ssl_cert() elapse = time.time() - t1 self.update_services("ssl_cert", elapse) self.update_task_field("status", "find_site") t1 = time.time() self.find_site() elapse = time.time() - t1 self.update_services("find_site", elapse) '''***站点识别***''' if self.options.get("site_identify"): self.update_task_field("status", "site_identify") t1 = time.time() self.site_identify() elapse = time.time() - t1 self.update_services("site_identify", elapse) self.update_task_field("status", "fetch_site") t1 = time.time() self.fetch_site() elapse = time.time() - t1 self.update_services("fetch_site", elapse) '''***站点截图***''' if self.options.get("site_capture"): self.update_task_field("status", "site_capture") t1 = time.time() self.site_screenshot() elapse = time.time() - t1 self.update_services("site_capture", elapse) '''站点爬虫''' if self.options.get("site_spider"): self.update_task_field("status", "site_spider") t1 = time.time() self.site_spider() elapse = time.time() - t1 self.update_services("site_spider", elapse) '''文件泄露''' if self.options.get("file_leak"): self.update_task_field("status", "file_leak") t1 = time.time() self.file_leak() elapse = time.time() - t1 self.update_services("file_leak", elapse) self.update_task_field("status", TaskStatus.DONE) self.update_task_field("end_time", utils.curr_date())
def run(self): self.update_task_field("start_time", utils.curr_date()) '''****域名爆破开始****''' if self.options.get("domain_brute"): self.update_task_field("status", "domain_brute") t1 = time.time() self.domain_brute() elapse = time.time() - t1 self.update_services("domain_brute", elapse) else: domain_info = self.build_single_domain_info(self.base_domain) if domain_info: self.domain_info_list.append(domain_info) self.save_domain_info_list([domain_info]) '''***RiskIQ查询****''' if services.riskiq_quota() > 0 and self.options.get("riskiq_search"): self.update_task_field("status", "riskiq_search") t1 = time.time() self.riskiq_search() elapse = time.time() - t1 self.update_services("riskiq_search", elapse) if self.options.get("arl_search"): self.update_task_field("status", "arl_search") t1 = time.time() self.arl_search() elapse = time.time() - t1 self.update_services("arl_search", elapse) '''***智能域名生成****''' if self.options.get("alt_dns"): self.update_task_field("status", "alt_dns") t1 = time.time() self.alt_dns() elapse = time.time() - t1 self.update_services("alt_dns", elapse) self.gen_ipv4_map() '''***佛法证书域名关联****''' if self.options.get("fofa_search"): self.update_task_field("status", "fofa_search") t1 = time.time() self.fofa_search() elapse = time.time() - t1 self.update_services("fofa_search", elapse) '''***端口扫描开始***''' if self.options.get("port_scan"): self.update_task_field("status", "port_scan") t1 = time.time() self.port_scan() elapse = time.time() - t1 self.update_services("port_scan", elapse) '''***证书获取***''' if self.options.get("ssl_cert"): self.update_task_field("status", "ssl_cert") t1 = time.time() self.ssl_cert() elapse = time.time() - t1 self.update_services("ssl_cert", elapse) # 服务信息存储 if self.options.get("service_detection"): self.save_service_info() self.save_ip_info() self.update_task_field("status", "find_site") t1 = time.time() self.find_site() elapse = time.time() - t1 self.update_services("find_site", elapse) '''***站点识别***''' if self.options.get("site_identify"): self.update_task_field("status", "site_identify") t1 = time.time() self.site_identify() elapse = time.time() - t1 self.update_services("site_identify", elapse) self.update_task_field("status", "fetch_site") t1 = time.time() self.fetch_site() elapse = time.time() - t1 self.update_services("fetch_site", elapse) '''***站点截图***''' if self.options.get("site_capture"): self.update_task_field("status", "site_capture") t1 = time.time() self.site_screenshot() elapse = time.time() - t1 self.update_services("site_capture", elapse) '''搜索引擎查询''' if self.options.get("search_engines"): self.update_task_field("status", "search_engines") t1 = time.time() self.search_engines() elapse = time.time() - t1 self.update_services("search_engines", elapse) '''站点爬虫''' if self.options.get("site_spider"): self.update_task_field("status", "site_spider") t1 = time.time() self.site_spider() elapse = time.time() - t1 self.update_services("site_spider", elapse) '''文件泄露''' if self.options.get("file_leak"): self.update_task_field("status", "file_leak") t1 = time.time() self.file_leak() elapse = time.time() - t1 self.update_services("file_leak", elapse) self.update_task_field("status", TaskStatus.DONE) self.update_task_field("end_time", utils.curr_date())
def run(self): self.update_task_field("start_time", utils.curr_date()) '''***端口扫描开始***''' if self.options.get("port_scan"): self.update_task_field("status", "port_scan") t1 = time.time() self.port_scan() elapse = time.time() - t1 self.update_services("port_scan", elapse) # 存储服务信息 if self.options.get("service_detection"): self.save_service_info() '''***证书获取开始***''' if self.options.get("ssl_cert"): self.update_task_field("status", "ssl_cert") t1 = time.time() self.ssl_cert() elapse = time.time() - t1 self.update_services("ssl_cert", elapse) self.update_task_field("status", "find_site") t1 = time.time() self.find_site() elapse = time.time() - t1 self.update_services("find_site", elapse) '''***站点识别***''' if self.options.get("site_identify"): self.update_task_field("status", "site_identify") t1 = time.time() self.site_identify() elapse = time.time() - t1 self.update_services("site_identify", elapse) self.update_task_field("status", "fetch_site") t1 = time.time() self.fetch_site() elapse = time.time() - t1 self.update_services("fetch_site", elapse) '''***站点截图***''' if self.options.get("site_capture"): self.update_task_field("status", "site_capture") t1 = time.time() self.site_screenshot() elapse = time.time() - t1 self.update_services("site_capture", elapse) '''站点爬虫''' if self.options.get("site_spider"): self.update_task_field("status", "site_spider") t1 = time.time() self.site_spider() elapse = time.time() - t1 self.update_services("site_spider", elapse) """风险PoC任务""" if self.options.get("poc_config"): self.update_task_field("status", "PoC") t1 = time.time() self.risk_cruising() elapse = time.time() - t1 self.update_services("poc_config", elapse) """服务识别(python)实现""" if self.options.get("npoc_service_detection"): self.update_task_field("status", "npoc_service_detection") t1 = time.time() self.npoc_service_detection() elapse = time.time() - t1 self.update_services("npoc_service_detection", elapse) """弱口令爆破服务""" if self.options.get("brute_config"): self.update_task_field("status", "weak_brute") t1 = time.time() self.brute_config() elapse = time.time() - t1 self.update_services("weak_brute", elapse) '''文件泄露''' if self.options.get("file_leak"): self.update_task_field("status", "file_leak") t1 = time.time() self.file_leak() elapse = time.time() - t1 self.update_services("file_leak", elapse) self.insert_task_stat() self.update_task_field("status", TaskStatus.DONE) self.update_task_field("end_time", utils.curr_date())