def run(self):
self.update_task_field("start_time", utils.curr_date())
self.domain_fetch()
for domain_info in self.domain_info_list:
self.domain_set.add(domain_info.domain)
self.set_scope_domain()
new_domain_set = self.domain_set - self.scope_domain_set
self.new_domain_set = new_domain_set
self.set_domain_info_list()
#仅仅对新增域名保留
self.start_ip_fetch()
self.start_site_fetch()
self.update_task_field("status", TaskStatus.DONE)
self.update_task_field("end_time", utils.curr_date())
ret_new_domain_set = set()
for domain_info in self.domain_info_list:
ret_new_domain_set.add(domain_info.domain)
return ret_new_domain_set
def run(self):
try:
self.update_task_field("start_time", utils.curr_date())
self.work()
self.update_task_field("status", TaskStatus.DONE)
except Exception as e:
self.update_task_field("status", TaskStatus.ERROR)
logger.exception(e)
self.update_task_field("end_time", utils.curr_date())
def run(self):
self.update_task_field("start_time", utils.curr_date())
self.domain_fetch()
self.start_ip_fetch()
self.start_site_fetch()
self.update_task_field("status", TaskStatus.DONE)
self.update_task_field("end_time", utils.curr_date())
def get(self, task_id=None):
"""
任务停止
"""
done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR]
task_data = utils.conn_db('task').find_one({'_id': ObjectId(task_id)})
if not task_data:
return utils.build_ret(ErrorMsg.NotFoundTask, {"task_id": task_id})
if task_data["status"] in done_status:
return utils.build_ret(ErrorMsg.TaskIsRunning,
{"task_id": task_id})
celery_id = task_data.get("celery_id")
if not celery_id:
return utils.build_ret(ErrorMsg.CeleryIdNotFound,
{"task_id": task_id})
control = celerytask.celery.control
control.revoke(celery_id, signal='SIGTERM', terminate=True)
utils.conn_db('task').update_one({'_id': ObjectId(task_id)},
{"$set": {
"status": TaskStatus.STOP
}})
utils.conn_db('task').update_one(
{'_id': ObjectId(task_id)},
{"$set": {
"end_time": utils.curr_date()
}})
return {"message": "success", "task_id": task_id, "code": 200}
def run_brute(self):
"""运行爆破，获取进度"""
target = self.site_list + list(self.npoc_service_target_set)
plugin_name = self.brute_plugin_name
logger.info("start run brute {}*{}".format(len(plugin_name),
len(target)))
run_total = len(plugin_name) * len(target)
npoc_instance = npoc.NPoC(tmp_dir=Config.TMP_PATH, concurrency=10)
run_thread = Thread(target=npoc_instance.run_poc,
args=(plugin_name, target))
run_thread.start()
while run_thread.is_alive():
time.sleep(5)
status = "brute {}/{}".format(npoc_instance.runner.runner_cnt,
run_total)
logger.info("[{}]runner cnt {}/{}".format(
self.task_id, npoc_instance.runner.runner_cnt, run_total))
self.update_task_field("status", status)
result = npoc_instance.result
for item in result:
item["task_id"] = self.task_id
item["save_date"] = utils.curr_date()
utils.conn_db('vuln').insert_one(item)
def post(self):
"""
策略编辑
"""
args = self.parse_args(edit_policy_fields)
policy_id = args.pop('policy_id')
policy_data = args.pop('policy_data', {})
query = {'_id': ObjectId(policy_id)}
item = utils.conn_db('policy').find_one(query)
if not item:
return utils.build_ret(ErrorMsg.PolicyIDNotFound, {})
if not policy_data:
return utils.build_ret(ErrorMsg.PolicyDataIsEmpty, {})
item = change_dict(item, policy_data)
poc_config = item["policy"].pop("poc_config", [])
poc_config = _update_plugin_config(poc_config)
if isinstance(poc_config, str):
return utils.build_ret(poc_config, {})
item["policy"]["poc_config"] = poc_config
brute_config = item["policy"].pop("brute_config", [])
brute_config = _update_plugin_config(brute_config)
if isinstance(brute_config, str):
return utils.build_ret(brute_config, {})
item["policy"]["brute_config"] = brute_config
item["update_date"] = utils.curr_date()
utils.conn_db('policy').find_one_and_replace(query, item)
item.pop('_id')
return utils.build_ret(ErrorMsg.Success, {"data": item})
def wrap_domain_executors(base_domain=None,
job_id=None,
scope_id=None,
options=None,
name=""):
celery_id = ""
if current_task._get_current_object():
celery_id = current_task.request.id
task_data = {
'name': name,
'target': base_domain,
'start_time': '-',
'status': 'waiting',
'type': 'domain',
'task_tag': 'monitor', #标记为监控任务
'options': {
'domain_brute': True,
'domain_brute_type': 'test',
'riskiq_search': False,
'alt_dns': False,
'arl_search': True,
'port_scan_type': 'test',
'port_scan': True,
'service_detection': False,
'service_brute': False,
'os_detection': False,
'site_identify': False,
'site_capture': False,
'file_leak': False,
'site_spider': False,
'search_engines': False,
'ssl_cert': False,
'fofa_search': False,
'scope_id': scope_id
},
'celery_id': celery_id
}
if options is None:
options = {}
task_data["options"].update(options)
conn('task').insert_one(task_data)
task_id = str(task_data.pop("_id"))
domain_executor = DomainExecutor(base_domain, task_id,
task_data["options"])
try:
update_job_run(job_id)
new_domain = domain_executor.run()
if new_domain:
sync_asset(task_id, scope_id, update_flag=True)
except Exception as e:
logger.exception(e)
domain_executor.update_task_field("status", TaskStatus.ERROR)
domain_executor.update_task_field("end_time", utils.curr_date())
logger.info("end domain_executors {} {} {}".format(base_domain, scope_id,
options))
def domain_task(base_domain, task_id, options):
d = DomainTask(base_domain=base_domain, task_id=task_id, options=options)
try:
d.run()
except Exception as e:
logger.exception(e)
d.update_task_field("status", TaskStatus.ERROR)
d.update_task_field("end_time", utils.curr_date())
def npoc_service_detection(self):
logger.info("start npoc_service_detection {}".format(
len(self.sniffer_target_set)))
result = npoc.run_sniffer(self.sniffer_target_set)
for item in result:
self.npoc_service_target_set.add(item["target"])
item["task_id"] = self.task_id
item["save_date"] = utils.curr_date()
utils.conn_db('npoc_service').insert_one(item)
def run(self):
self.update_task_field("start_time", utils.curr_date())
self.domain_fetch()
self.start_ip_fetch()
self.start_site_fetch()
self.start_poc_run()
# cidr ip 结果统计，插入cip 集合中
self.insert_cip_stat()
# 任务结果统计
self.insert_task_stat()
self.update_task_field("status", TaskStatus.DONE)
self.update_task_field("end_time", utils.curr_date())
def sync_to_db(self):
for old in self.poc_info_list:
new = old.copy()
plugin_name = old["plugin_name"]
new["update_date"] = utils.curr_date()
if plugin_name in self.db_plugin_name_list:
continue
logger.info("insert {} info to db".format(plugin_name))
utils.conn_db('poc').insert_one(new)
return True
def post(self):
args = self.parse_args(add_policy_fields)
name = args.pop("name")
policy = args.pop("policy", {})
if policy is None:
return utils.build_ret("Missing policy parameter", {})
domain_config = policy.pop("domain_config", {})
domain_config = self._update_arg(domain_config, domain_config_fields)
ip_config = policy.pop("ip_config", {})
ip_config = self._update_arg(ip_config, ip_config_fields)
site_config = policy.pop("site_config", {})
site_config = self._update_arg(site_config, site_config_fields)
poc_config = policy.pop("poc_config", [])
if poc_config is None:
poc_config = []
poc_config = _update_plugin_config(poc_config)
if isinstance(poc_config, str):
return utils.build_ret(poc_config, {})
brute_config = policy.pop("brute_config", [])
if brute_config is None:
brute_config = []
brute_config = _update_plugin_config(brute_config)
if isinstance(brute_config, str):
return utils.build_ret(brute_config, {})
file_leak = fields.boolean(policy.pop("file_leak", False))
npoc_service_detection = fields.boolean(
policy.pop("npoc_service_detection", False))
desc = args.pop("desc", "")
item = {
"name": name,
"policy": {
"domain_config": domain_config,
"ip_config": ip_config,
"site_config": site_config,
"poc_config": poc_config,
"brute_config": brute_config,
"file_leak": file_leak,
"npoc_service_detection": npoc_service_detection
},
"desc": desc,
"update_date": utils.curr_date()
}
utils.conn_db("policy").insert_one(item)
return utils.build_ret(ErrorMsg.Success,
{"policy_id": str(item["_id"])})
def risk_cruising(self):
"""运行PoC任务"""
poc_config = self.options.get("poc_config", [])
plugins = []
for info in poc_config:
if not info.get("enable"):
continue
plugins.append(info["plugin_name"])
result = run_risk_cruising(plugins=plugins, targets=self.site_list)
for item in result:
item["task_id"] = self.task_id
item["save_date"] = utils.curr_date()
utils.conn_db('vuln').insert_one(item)
def npoc_service_detection(self):
targets = []
for ip_info in self.ip_info_list:
for port_info in ip_info.port_info_list:
if port_info.port_id == 80:
continue
if port_info.port_id == 443:
continue
targets.append("{}:{}".format(ip_info.ip, port_info.port_id))
result = run_sniffer(targets)
for item in result:
self.npoc_service_target_set.add(item["target"])
item["task_id"] = self.task_id
item["save_date"] = utils.curr_date()
utils.conn_db('npoc_service').insert_one(item)
def brute_config(self):
plugins = []
brute_config = self.options.get("brute_config")
for x in brute_config:
if not x.get("enable"):
continue
plugins.append(x["plugin_name"])
if not plugins:
return
targets = self.site_list.copy()
targets += list(self.npoc_service_target_set)
result = run_risk_cruising(targets=targets, plugins=plugins)
for item in result:
item["task_id"] = self.task_id
item["save_date"] = utils.curr_date()
utils.conn_db('vuln').insert_one(item)
def get(self, task_id=None):
"""
任务停止
"""
done_status = [TaskStatus.DONE, TaskStatus.STOP, TaskStatus.ERROR]
task_data = utils.conn_db('task').find_one({'_id': ObjectId(task_id)})
if not task_data:
return {
"message": "not found task",
"task_id": task_id,
"code": 100
}
if task_data["status"] in done_status:
return {
"message": "error, task is done",
"task_id": task_id,
"code": 101
}
celery_id = task_data.get("celery_id")
if not celery_id:
return {
"message": "not found celery_id",
"task_id": task_id,
"code": 102
}
control = celerytask.celery.control
#由subprocess启动的进程还存在 /doge
control.revoke(celery_id, terminate=True)
utils.conn_db('task').update_one({'_id': ObjectId(task_id)},
{"$set": {
"status": TaskStatus.STOP
}})
utils.conn_db('task').update_one(
{'_id': ObjectId(task_id)},
{"$set": {
"end_time": utils.curr_date()
}})
return {"message": "success", "task_id": task_id, "code": 200}
def run(self):
self.update_task_field("start_time", utils.curr_date())
'''***端口扫描开始***'''
if self.options.get("port_scan"):
self.update_task_field("status", "port_scan")
t1 = time.time()
self.port_scan()
elapse = time.time() - t1
self.update_services("port_scan", elapse)
# 存储服务信息
if self.options.get("service_detection"):
self.save_service_info()
'''***证书获取开始***'''
if self.options.get("ssl_cert"):
self.update_task_field("status", "ssl_cert")
t1 = time.time()
self.ssl_cert()
elapse = time.time() - t1
self.update_services("ssl_cert", elapse)
self.update_task_field("status", "find_site")
t1 = time.time()
self.find_site()
elapse = time.time() - t1
self.update_services("find_site", elapse)
'''***站点识别***'''
if self.options.get("site_identify"):
self.update_task_field("status", "site_identify")
t1 = time.time()
self.site_identify()
elapse = time.time() - t1
self.update_services("site_identify", elapse)
self.update_task_field("status", "fetch_site")
t1 = time.time()
self.fetch_site()
elapse = time.time() - t1
self.update_services("fetch_site", elapse)
'''***站点截图***'''
if self.options.get("site_capture"):
self.update_task_field("status", "site_capture")
t1 = time.time()
self.site_screenshot()
elapse = time.time() - t1
self.update_services("site_capture", elapse)
'''站点爬虫'''
if self.options.get("site_spider"):
self.update_task_field("status", "site_spider")
t1 = time.time()
self.site_spider()
elapse = time.time() - t1
self.update_services("site_spider", elapse)
'''文件泄露'''
if self.options.get("file_leak"):
self.update_task_field("status", "file_leak")
t1 = time.time()
self.file_leak()
elapse = time.time() - t1
self.update_services("file_leak", elapse)
self.update_task_field("status", TaskStatus.DONE)
self.update_task_field("end_time", utils.curr_date())
def run(self):
self.update_task_field("start_time", utils.curr_date())
'''****域名爆破开始****'''
if self.options.get("domain_brute"):
self.update_task_field("status", "domain_brute")
t1 = time.time()
self.domain_brute()
elapse = time.time() - t1
self.update_services("domain_brute", elapse)
else:
domain_info = self.build_single_domain_info(self.base_domain)
if domain_info:
self.domain_info_list.append(domain_info)
self.save_domain_info_list([domain_info])
'''***RiskIQ查询****'''
if services.riskiq_quota() > 0 and self.options.get("riskiq_search"):
self.update_task_field("status", "riskiq_search")
t1 = time.time()
self.riskiq_search()
elapse = time.time() - t1
self.update_services("riskiq_search", elapse)
if self.options.get("arl_search"):
self.update_task_field("status", "arl_search")
t1 = time.time()
self.arl_search()
elapse = time.time() - t1
self.update_services("arl_search", elapse)
'''***智能域名生成****'''
if self.options.get("alt_dns"):
self.update_task_field("status", "alt_dns")
t1 = time.time()
self.alt_dns()
elapse = time.time() - t1
self.update_services("alt_dns", elapse)
self.gen_ipv4_map()
'''***佛法证书域名关联****'''
if self.options.get("fofa_search"):
self.update_task_field("status", "fofa_search")
t1 = time.time()
self.fofa_search()
elapse = time.time() - t1
self.update_services("fofa_search", elapse)
'''***端口扫描开始***'''
if self.options.get("port_scan"):
self.update_task_field("status", "port_scan")
t1 = time.time()
self.port_scan()
elapse = time.time() - t1
self.update_services("port_scan", elapse)
'''***证书获取***'''
if self.options.get("ssl_cert"):
self.update_task_field("status", "ssl_cert")
t1 = time.time()
self.ssl_cert()
elapse = time.time() - t1
self.update_services("ssl_cert", elapse)
# 服务信息存储
if self.options.get("service_detection"):
self.save_service_info()
self.save_ip_info()
self.update_task_field("status", "find_site")
t1 = time.time()
self.find_site()
elapse = time.time() - t1
self.update_services("find_site", elapse)
'''***站点识别***'''
if self.options.get("site_identify"):
self.update_task_field("status", "site_identify")
t1 = time.time()
self.site_identify()
elapse = time.time() - t1
self.update_services("site_identify", elapse)
self.update_task_field("status", "fetch_site")
t1 = time.time()
self.fetch_site()
elapse = time.time() - t1
self.update_services("fetch_site", elapse)
'''***站点截图***'''
if self.options.get("site_capture"):
self.update_task_field("status", "site_capture")
t1 = time.time()
self.site_screenshot()
elapse = time.time() - t1
self.update_services("site_capture", elapse)
'''搜索引擎查询'''
if self.options.get("search_engines"):
self.update_task_field("status", "search_engines")
t1 = time.time()
self.search_engines()
elapse = time.time() - t1
self.update_services("search_engines", elapse)
'''站点爬虫'''
if self.options.get("site_spider"):
self.update_task_field("status", "site_spider")
t1 = time.time()
self.site_spider()
elapse = time.time() - t1
self.update_services("site_spider", elapse)
'''文件泄露'''
if self.options.get("file_leak"):
self.update_task_field("status", "file_leak")
t1 = time.time()
self.file_leak()
elapse = time.time() - t1
self.update_services("file_leak", elapse)
self.update_task_field("status", TaskStatus.DONE)
self.update_task_field("end_time", utils.curr_date())
def run(self):
self.update_task_field("start_time", utils.curr_date())
'''***端口扫描开始***'''
if self.options.get("port_scan"):
self.update_task_field("status", "port_scan")
t1 = time.time()
self.port_scan()
elapse = time.time() - t1
self.update_services("port_scan", elapse)
# 存储服务信息
if self.options.get("service_detection"):
self.save_service_info()
'''***证书获取开始***'''
if self.options.get("ssl_cert"):
self.update_task_field("status", "ssl_cert")
t1 = time.time()
self.ssl_cert()
elapse = time.time() - t1
self.update_services("ssl_cert", elapse)
self.update_task_field("status", "find_site")
t1 = time.time()
self.find_site()
elapse = time.time() - t1
self.update_services("find_site", elapse)
'''***站点识别***'''
if self.options.get("site_identify"):
self.update_task_field("status", "site_identify")
t1 = time.time()
self.site_identify()
elapse = time.time() - t1
self.update_services("site_identify", elapse)
self.update_task_field("status", "fetch_site")
t1 = time.time()
self.fetch_site()
elapse = time.time() - t1
self.update_services("fetch_site", elapse)
'''***站点截图***'''
if self.options.get("site_capture"):
self.update_task_field("status", "site_capture")
t1 = time.time()
self.site_screenshot()
elapse = time.time() - t1
self.update_services("site_capture", elapse)
'''站点爬虫'''
if self.options.get("site_spider"):
self.update_task_field("status", "site_spider")
t1 = time.time()
self.site_spider()
elapse = time.time() - t1
self.update_services("site_spider", elapse)
"""风险PoC任务"""
if self.options.get("poc_config"):
self.update_task_field("status", "PoC")
t1 = time.time()
self.risk_cruising()
elapse = time.time() - t1
self.update_services("poc_config", elapse)
"""服务识别（python）实现"""
if self.options.get("npoc_service_detection"):
self.update_task_field("status", "npoc_service_detection")
t1 = time.time()
self.npoc_service_detection()
elapse = time.time() - t1
self.update_services("npoc_service_detection", elapse)
"""弱口令爆破服务"""
if self.options.get("brute_config"):
self.update_task_field("status", "weak_brute")
t1 = time.time()
self.brute_config()
elapse = time.time() - t1
self.update_services("weak_brute", elapse)
'''文件泄露'''
if self.options.get("file_leak"):
self.update_task_field("status", "file_leak")
t1 = time.time()
self.file_leak()
elapse = time.time() - t1
self.update_services("file_leak", elapse)
self.insert_task_stat()
self.update_task_field("status", TaskStatus.DONE)
self.update_task_field("end_time", utils.curr_date())
