def init():
"""
启动的时候初始化，主要是新建一些日志目录之类的
:return:
"""
logger.propagate = False
logger.setLevel(logging.DEBUG)
# logger.setLevel(logging.WARN)
logger.info(celery_logo)
load_default_checkers(True)
modify_default_checkers()
def scan(package, task_id, create_user, status):
"""
:param package:
:param task_id:
:param create_user:
:param status:
:return:
"""
logger.info("hunter task has started")
# 加载插件，只有一个插件
checkers = load_default_checkers()
logger.info('loading package success')
logger.info('loading plugin success')
try:
if checkers["xsseye"].useable == PluginSwith.ON:
xssfork_process = XssForkProcess(package, task_id)
xssfork_process.engine_start()
while not xssfork_process.engine_has_terminated(
) and xssfork_process.process is not None:
logger.info("xsseye program is runing")
time.sleep(5)
xssfork_process.engine_kill()
logger.warn("xsseye program runs to completion")
except KeyboardInterrupt as e:
logger.exception("scan error")
finally:
logger.info("hunter task has done")
def download_checkers():
"""
下载所有的插件，不做鉴权
:return:
"""
checker_name = request.args.get("name")
if not checker_name:
zip_floder_skip(target_file="%stmp/plugins.zip" % PLUGIN_PATH,
origin_folder=PLUGIN_PATH,
is_regular=True,
skip_list=["*DS_Store", "*__pycache__*", "tmp/*"])
return send_file(filename_or_fp="%stmp/plugins.zip" % PLUGIN_PATH,
as_attachment=True,
cache_timeout=0)
checkers = load_default_checkers()
if checker_name not in checkers:
return jsonify(status=400,
message="下载失败",
data={"extra_info": "不存在名字为%s的插件" % checker_name})
return send_file(filename_or_fp=checkers[checker_name].plugin_file_path,
as_attachment=True,
cache_timeout=0)
def modify_plugin():
"""
禁用指定的插件，
服务端:1.从CHECKER_INSTANCE_DICT指定插件设置disable 2.并向MQ中发送一条消息
引擎端:1.等到引擎消费到指令从CHECKER_INSTANCE_DICT，也将该插件的disable设置为True
:return:
"""
from plugins.base.vuln_enum import PluginSwith
from common.plugins_util import modify_default_checkers
post_data = request.get_json(force=True)
param_list = ["name", "switch"]
if has_dict_value_blank(post_data, param_list):
return jsonify(
status=400,
message="更新失败",
data={"extra_info": "请保证%s任一参数值不为空" % ','.join(param_list)})
try:
post_checker_name = post_data.get("name")
post_switch = post_data.get("switch")
plugin_switch = PluginSwith.ON if post_switch else PluginSwith.OFF
checkers = load_default_checkers()
if post_checker_name not in checkers:
return jsonify(
status=400,
message="更新失败",
data={"extra_info": "不存在名为%s的插件" % post_checker_name})
PluginInfoService.update(
fields=({
PluginInfo.useable: plugin_switch
}),
where=(PluginInfo.plugin_name == post_checker_name))
RedisService.modify_plugin_switch(checkers[post_checker_name],
plugin_switch)
system_notice_celery.delay(
broadcast={
"type": BroadCastType.PLUGIN,
"action": BroadCastAction.MODIFY_PLUGIN,
"data": {
"name": post_checker_name,
"switch": plugin_switch
}
})
modify_default_checkers()
return jsonify(status=200,
message="更新成功",
data={
"extra_info":
"{}插件{}".format("启用" if plugin_switch else "禁用",
post_checker_name)
})
except Exception as e:
logger.exception("modify_plugin raise error")
return jsonify(status=500,
message="未知异常",
data={"extra_info": "修改插件信息时出现未知异常，请联系管理员查看异常日志"})
def download_newest_checkers():
"""
第一次启动时同步插件，从服务端下载最新的插件，
:return:
"""
logger.info("download newest checkers when the system starts up")
try:
master_checkers_url = get_system_config()["front_end"]["master_checkers_url"]
download_file(url=master_checkers_url, save_fp=PLUGIN_ZIP_PATH)
unzip_file(origin_file=PLUGIN_ZIP_PATH, target_folder=PLUGIN_PATH)
logger.info("download newest checkers successfully, it still using newest checkers")
except Exception as e:
if isinstance(e, requests.exceptions.ConnectTimeout):
logger.warn("sorry,download newest checkers timeout, it still using old checkers")
else:
logger.warn("sorry,download newest checkers error, it still using old checkers")
finally:
load_default_checkers(True)
modify_default_checkers()
def remove_checker(broadcast):
"""
移除插件，不做物理删除，只从内存中移除
{"type": "plugin", "action": "modify", "data": {"name": checker_name, "switch": PluginSwith.ON}
:param broadcast:
:return:
"""
checker_name = broadcast["data"]["name"]
checkers_dict = load_default_checkers()
if checker_name in checkers_dict:
logger.info('从插件列表中移除插件{}'.format(checker_name))
checkers_dict.pop(checker_name)
def modify_checker(broadcast):
"""
修改本地插件配置信息，只修改本地配置文件
{"type": "plugin", "action": "modify", "data": {"name": checker_name, "switch": PluginSwith.ON}
:param broadcast:
:return:
"""
checker_name = broadcast["data"]["name"]
switch = broadcast["data"]["switch"]
checkers_dict = load_default_checkers()
if checker_name in checkers_dict:
logger.info('接收到修改插件{}状态为{}的请求'.format(checker_name, switch))
LocalFilePluginConfig().modify_plugin_config(checker_name, "useable", switch)
modify_default_checkers()
def parse_plugin_file(plugin_file_path):
"""
warnning: 插件文件路径，这里会存在安全隐患，请按照实际需求考虑是否用imp.load_source
:param plugin_file_path:
:return:
"""
import imp
from exception.hunter_web_exception import BaseHunterException
# 解析插件并且分类
base_checker = imp.load_source('BaseChecker', plugin_file_path)
checker_instance = base_checker.Checker()
checker_instance.check_plugin_info()
# 检测插件是否重名
checker_name = checker_instance.info["name"]
if checker_name in load_default_checkers():
raise BaseHunterException("插件%s已经存在，请重新命名" % checker_name)
return checker_instance
def scan(package, task_id, create_user, status):
"""
:param package:
:param task_id:
:param create_user:
:param status:
:return:
"""
from model.url import UrlService, Url
# logger.setLevel(logging.DEBUG)
classification_url, classification_data = parse_package(package)
# 保存URL
url_task = UrlService.save(status=TaskStatus.WAITING, origin_data=package, task_id=task_id,
classification_url=classification_url, classification_data=classification_data)
result_queue = queue.Queue() # 结果队列
logger.info("hunter task has started")
# 加载插件
checker_list = load_default_checkers()
# checker_list = load_checkers(modle_names)
logger.info('loading package success')
logger.info('loading plugin success')
try:
future_tasks = list()
for checker_name, checker_instance in checker_list.items():
future_tasks.append(asyncio.ensure_future(run_plugin(package, checker_instance, result_queue)))
# 更新URL数据库状态
UrlService.update(fields=({Url.status: TaskStatus.WORKING}), where=(Url.id == url_task.id))
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(future_tasks))
# loop.close()
except KeyboardInterrupt as e:
logger.exception("scan error")
finally:
UrlService.update(fields=({Url.status: TaskStatus.DONE}), where=(Url.id == url_task.id))
# UrlService.update_url_status(TaskStatus.DONE, url_task.id)
logger.info("hunter task has done")
print_result(url_task, result_queue)
def init_plugin_info():
"""
初始化插件信息到数据库中
:return:
"""
# 初始化redis配置信息
RedisService.init_plugin_config()
for checker_name, checker_instance in load_default_checkers().items():
if PluginInfoService.count(
where=(PluginInfo.plugin_name == checker_name)) == 0:
PluginInfoService.save(
author=checker_instance.info["author"],
plugin_name=checker_instance.info["name"],
plugin_tag=checker_instance.info["tag"],
imp_version=checker_instance.info["imp_version"],
description=checker_instance.info["description"],
repair=checker_instance.info["repair"],
type=checker_instance.info["type"]["fullname"],
chinese_type=checker_instance.info["type"]["fullchinesename"],
level=checker_instance.info["type"]["level"],
)
def init_plugin_config():
"""
初始化插件开关
:return:
"""
from common.plugins_util import load_default_checkers
try:
checker_dict = load_default_checkers(True)
for (plugin_name, checker_instance) in checker_dict.items():
plugin_config_info = json.dumps({
"tag":
checker_instance.info["tag"],
"useable":
PluginSwith.ON,
"removed":
False
})
RedisManage.get_redis_client().hset(
RedisService.HUNTER_PLUGIN_SWITCH, plugin_name,
plugin_config_info)
except Exception:
RedisService.logger.exception("RedisService update_plugin error")
def init_plugin_info():
"""
初始化插件信息，2.0 废弃
:return:
"""
from common.plugins_util import load_default_checkers
try:
checker_dict = load_default_checkers()
for (plugin_name, checker_instance) in checker_dict.items():
plugin_instance_info = {
"name": plugin_name,
"tag": checker_instance.info["tag"],
"imp_version": checker_instance.info["imp_version"],
"type": checker_instance.info["type"]["fullchinesename"],
"level": checker_instance.info["type"]["level"],
"description": checker_instance.info["description"],
"useable": PluginSwith.ON
}
RedisManage.get_redis_client().hmset(
RedisService.HUNTER_PLUGIN + plugin_name,
plugin_instance_info)
except Exception:
RedisService.logger.exception(
"RedisService init_plugin_info error")
def insert_plugin():
"""
新增插件，
服务端:1.保存插件到本地，先上传到tmp目录，然后解析插件是否满足格式出tag并移动到tag目录，刷新插件列表 2.向MQ发送一条消息包含插件
引擎端:2.消费到MQ发送消息，并下载最新插件并新增到CHECKER_INSTANCE_DICT
:return:
"""
import shutil
from common.system_util import mkdir
from common.path import PLUGIN_PATH
from common.plugins_util import load_default_checkers
def allowed_file(filename):
"""
检测是否为python文件
:param filename:
:return:
"""
return '.' in filename and filename.rsplit('.', 1)[1] in ["py"]
def parse_plugin_file(plugin_file_path):
"""
warnning: 插件文件路径，这里会存在安全隐患，请按照实际需求考虑是否用imp.load_source
:param plugin_file_path:
:return:
"""
import imp
from exception.hunter_web_exception import BaseHunterException
# 解析插件并且分类
base_checker = imp.load_source('BaseChecker', plugin_file_path)
checker_instance = base_checker.Checker()
checker_instance.check_plugin_info()
# 检测插件是否重名
checker_name = checker_instance.info["name"]
if checker_name in load_default_checkers():
raise BaseHunterException("插件%s已经存在，请重新命名" % checker_name)
return checker_instance
def move_plugin(tmp_plugin_file_path, checker_instance, filename):
"""
:param checker_instance:
:return:
"""
# 移动插件到指定目录
tag = checker_instance.info["tag"]
mkdir(PLUGIN_PATH + tag)
formal_plugin_file_path = os.path.join(PLUGIN_PATH + tag, filename)
shutil.move(tmp_plugin_file_path, formal_plugin_file_path)
def save_plugin_info(checker_instance):
if PluginInfoService.count(where=(
PluginInfo.plugin_name == checker_instance.info["name"])) == 0:
PluginInfoService.save(
author=checker_instance.info["author"],
plugin_name=checker_instance.info["name"],
plugin_tag=checker_instance.info["tag"],
imp_version=checker_instance.info["imp_version"],
description=checker_instance.info["description"],
repair=checker_instance.info["repair"],
type=checker_instance.info["type"]["fullname"],
chinese_type=checker_instance.info["type"]["fullchinesename"],
level=checker_instance.info["type"]["level"],
)
try:
file = request.files['file']
if file and allowed_file(file.filename):
filename = file.filename
tmp_plugin_file_path = os.path.join(PLUGIN_PATH + "tmp/", filename)
file.save(tmp_plugin_file_path)
# 解析插件
checker_instance = parse_plugin_file(tmp_plugin_file_path)
move_plugin(tmp_plugin_file_path, checker_instance, filename)
# 保存到数据
save_plugin_info(checker_instance)
load_default_checkers(True)
RedisService.modify_plugin_switch(checker_instance, PluginSwith.ON)
system_notice_celery.delay(
broadcast={
"type": BroadCastType.PLUGIN,
"action": BroadCastAction.INSERT_PLUGIN,
"data": {
"name": checker_instance.info["name"]
}
})
return jsonify(status=200,
message="上传成功",
data={"extra_info": "您可以刷新网页后使用新插件"})
return jsonify(status=500,
message="上传失败",
data={"extra_info": "您上传的插件不是py文件"})
except Exception as e:
logger.exception("create_plugin raise error")
# 解析插件是否满足格式
return jsonify(status=500, message="上传失败", data={"extra_info": str(e)})
