def __init__(self):
"""
初始化
"""
is_scanner = Communicator().get_module_name().startswith("Scanner")
if is_scanner:
self.logger = Logger().get_scan_plugin_logger(self.plugin_info["name"])
else:
self.logger = Logger()
self._enable = True # 插件是否启用
self._white_reg = None # 扫描url白名单
self._scan_queue = queue.Queue() # 任务队列
self._last_scan_id = 0 # 最近扫描完成的任务在数据库中的id
self._scan_num = 0 # 当前已扫描url数量
self._request_timeout = Config().get_config("scanner.request_timeout")
self._max_concurrent_task = Config().get_config("scanner.max_concurrent_request")
# 共享的report_model 和 failed_task_set 需要在实例化ScanPluginBase类之前设置
try:
self._report_model = Communicator().get_internal_shared("report_model")
self._failed_set = Communicator().get_internal_shared("failed_task_set")
except exceptions.InternalSharedKeyError as e:
Logger().error("Try to init scan_plugin before set internal shared key in Communicator! Check 'error.log' for more information.")
exit(1)
self._request_session = audit_tools.Session()
self._request_data = audit_tools.RequestData
self.mutant_helper = audit_tools.MutantHelper()
self.checker = audit_tools.Checker()
if is_scanner:
self.logger.info("Scanner plugin {} init success!".format(self.plugin_info["name"]))
def _init_plugin(self):
"""
初始化扫描插件
"""
self.plugin_loaded = {}
plugin_import_path = "plugin.scanner"
for plugin_name in self.scan_config["scan_plugin_status"].keys():
try:
plugin_module = __import__(plugin_import_path,
fromlist=[plugin_name])
except Exception as e:
Logger().error("Error in load plugin: {}".format(plugin_name),
exc_info=e)
else:
plugin_instance = getattr(plugin_module,
plugin_name).ScanPlugin()
if isinstance(plugin_instance,
scan_plugin_base.ScanPluginBase):
self.plugin_loaded[plugin_name] = plugin_instance
Logger().debug(
"scanner plugin: {} load success!".format(plugin_name))
else:
Logger().warning(
"scanner plugin {} not inherit class ScanPluginBase!".
format(plugin_name))
if len(self.plugin_loaded) == 0:
Logger().error("No scanner plugin detected, scanner exit!")
raise exceptions.NoPluginError
def upload_report(self):
all_report_model = []
base_report_model = report_model.ReportModel(table_prefix=None,
create_table=False,
multiplexing_conn=True)
tables = base_report_model.get_tables()
for table_name in tables:
if table_name.lower().endswith("_report"):
all_report_model.append(table_name)
for table_name in all_report_model:
table_prefix = table_name[:-7]
try:
model_ins = report_model.ReportModel(table_prefix=table_prefix,
create_table=False,
multiplexing_conn=True)
while True:
data_list = model_ins.get_upload_report(20)
data_count = len(data_list)
if data_count == 0:
break
Logger().info(
"Try to upload {} report to cloud.".format(data_count))
if self._send_report_data(data_list):
model_ins.mark_report(data_count)
else:
time.sleep(5)
except Exception as e:
Logger().warning("Get data from report model error.",
exc_info=e)
def run(self):
"""
启动http server
"""
server = tornado.httpserver.HTTPServer(
self.app,
max_buffer_size=Config().get_config(
"preprocessor.max_buffer_size"))
try:
server.bind(Config().get_config("preprocessor.http_port"))
except OSError as e:
Logger().critical("Preprocessor bind port error!", exc_info=e)
sys.exit(1)
else:
# 这里会创建多个子进程，需要重新初始化Communicator
server.start(Config().get_config("preprocessor.process_num"))
Communicator().init_new_module(type(self).__name__)
# 记录pid
while True:
if Communicator().set_pre_http_pid(os.getpid()):
break
else:
pids = ", ".join(
str(x) for x in Communicator().get_pre_http_pid())
Logger().error(
"Preprocessor HTTP Server set pid failed! Running pids: {}"
.format(pids))
time.sleep(3)
tornado.ioloop.IOLoop.current().start()
async def _fetch_new_scan(self):
"""
获取非扫描请求（新扫描任务），并分发给插件
"""
# 扫描插件任务队列最大值
scan_queue_max = 300
# 已扫描的任务数量
self.scan_num = 0
# 扫描队列数量
self.scan_queue_remaining = 0
# 下次获取任务数量
self.fetch_count = 20
# 待标记的已扫描的最大id
self.mark_id = 0
while True:
try:
await self._fetch_task_from_db()
except exceptions.DatabaseError as e:
Logger().error("Database error occured when fetch scan task.", exc_info=e)
except asyncio.CancelledError as e:
raise e
except Exception as e:
Logger().error("Unexpected error occured when fetch scan task.", exc_info=e)
if self.scan_queue_remaining == 0:
continue
await self._check_scan_progress()
# 调整每次获取的扫描任务数
if self.scan_queue_remaining + self.fetch_count > scan_queue_max:
self.fetch_count = scan_queue_max - self.scan_queue_remaining
elif self.fetch_count < 5:
self.fetch_count = 5
def launch(self):
"""
启动器主函数
"""
self._set_affinity()
Communicator()
Logger().init_module_logger()
ForkProxy()
Logger().info("Launcher init success!")
preprocessor_proc = modules.Process(modules.Preprocessor)
preprocessor_proc.start()
self.preprocessor_pid = preprocessor_proc.pid
Logger().info("Preprocessor fork success!")
monitor_proc = modules.Process(modules.Monitor)
monitor_proc.start()
self.monitor_pid = monitor_proc.pid
Logger().info("Monitor fork success!")
signal.signal(signal.SIGCHLD, self._wait_child)
try:
ForkProxy().listen()
except KeyboardInterrupt:
self.exit = True
def _wait_child(self, signum, frame):
"""
处理进程terminate信号
"""
try:
try:
while True:
cpid, status = os.waitpid(-1, os.WNOHANG)
if cpid == 0:
break
if cpid == self.monitor_pid:
root_proc = psutil.Process(os.getpid())
procs = root_proc.children(recursive=True)
for p in procs:
p.send_signal(psutil.signal.SIGKILL)
if self.exit:
pass
else:
Logger().critical("Detect Monitor down, exit!")
print(
"[!] Detect Monitor down, OpenRASP-IAST exit!")
sys.exit(1)
exitcode = status >> 8
except OSError as e:
if e.errno != errno.ECHILD:
Logger().error(
"Unknow error occurred in method _wait_child!",
exc_info=e)
except KeyboardInterrupt:
pass
async def _fetch_task_from_db(self):
"""
从数据库中获取当前扫描目标的非扫描请求（新扫描任务）
"""
await self.new_scan_model.mark_result(self.mark_id,
list(self.failed_task_set))
self.failed_task_set.clear()
sleep_interval = 1
continuously_sleep = 0
while True:
data_list = await self.new_scan_model.get_new_scan(self.fetch_count
)
data_count = len(data_list)
Logger().debug("Fetch {} task from db.".format(data_count))
if data_count > 0 or self.scan_queue_remaining > 0:
for item in data_list:
for plugin_name in self.plugin_loaded:
# item 格式: {"id": id, "data":rasp_result_json}
self.plugin_loaded[plugin_name].add_task(item)
Logger().debug("Send task with id: {} to plugins.".format(
item["id"]))
self.scan_queue_remaining += data_count
return
else:
Logger().debug("No url need scan, fetch task sleep {}s".format(
sleep_interval * continuously_sleep))
if continuously_sleep < 10:
continuously_sleep += 1
await asyncio.sleep(sleep_interval * continuously_sleep)
def add_result(self, rasp_result):
"""
添加一个RaspResult实例到缓存队列并触发对应的数据到达事件, 同时清空缓存中过期的实例
若RaspResult实例的id未通过register_result方法注册，则直接丢弃
Parameters:
rasp_result - 待添加的RaspResult实例
"""
scan_request_id = rasp_result.get_scan_request_id()
try:
self.rasp_result_collection[scan_request_id][2] = rasp_result
self.rasp_result_collection[scan_request_id][0].set()
except KeyError:
Communicator().increase_value("dropped_rasp_result")
Logger().warning("Drop no registered rasp result data: {}".format(str(rasp_result)))
while True:
try:
key = next(iter(self.rasp_result_collection))
except StopIteration:
break
if self.rasp_result_collection[key][1] < time.time():
if type(self.rasp_result_collection[key][0]) is not dict:
Logger().debug("Rasp result with id: {} timeout, dropped".format(key))
self.rasp_result_collection.popitem(False)
else:
break
def get_param(self, para_type, para_name):
"""
获取HTTP请求的某个变量
Parameters:
para_type - str, 参数类型，可选get, post, cookies, headers, json, files, body
para_name - str, 参数名, 当para_type为json时, para_name应为一个list,包含json path每一级的key
当para_type为files时, para_name应为一个包含两个item的list, 第一个指定要获取的files dict的下标, 第二个指定在dict中获取的key, 同set_param
Returns:
获取目标变量的值
Raises:
exceptions.DataParamError - 参数错误引发此异常
"""
if para_type == "cookies":
return self.http_data["cookies"].get(para_name, None)
elif para_type == "get":
return self.http_data["params"].get(para_name, None)
elif para_type == "post":
return self.http_data["data"].get(para_name, None)
elif para_type == "headers":
return self.http_data["headers"].get(para_name, None)
elif para_type == "json":
json_target = self.http_data["json"]
for name in para_name:
json_target = json_target[name]
return json_target
elif para_type == "files":
return self.http_data["files"][para_name[0]][para_name[1]]
elif para_type == "body":
return self.http_data["body"]
else:
Logger.error("Use an invalid para_type in get_param method!")
raise exceptions.DataTypeNotExist
async def post(self):
"""
处理POST请求
"""
try:
data = self.request.body
headers = self.request.headers
content_type = self.request.headers.get("Content-Type", "None")
if not content_type.startswith("application/json"):
raise exceptions.ContentTypeInvalid
Logger().info("Received request data: " + data.decode('utf-8'))
rasp_result_ins = rasp_result.RaspResult(data)
if rasp_result_ins.is_scan_result():
self.send_data(rasp_result_ins)
else:
await self.dedup_data(rasp_result_ins)
self.write('{"status": 0, "msg":"ok"}\n')
except exceptions.OriExpectedException as e:
self.write('{"status": 1, "msg":"data invalid"}\n')
Communicator().increase_value("invalid_data")
Logger().warning(
"Invalid data: {} posted to http server, rejected!".format(
data))
except Exception as e:
Logger().error(
"Unexpected error occured when process data:{}".format(data),
exc_info=e)
self.send_error(500)
return
async def send_request(self, request_data_ins, proxy_url=None):
"""
异步发送一个http请求, 返回结果
Parameters:
request_data_ins - request_data.RequestData类的实例，包含请求的全部信息
proxy_url - 发送请求使用的代理url, 为None时不使用代理
Returns:
dict, 结构:
{
"status": http响应码,
"headers": http响应头的dict,
"body": http响应body, bytes
}
Raises:
exceptions.ScanRequestFailed - 请求发送失败时引发此异常
"""
http_func = getattr(self.session, request_data_ins.get_method())
request_params_dict = request_data_ins.get_aiohttp_param()
Logger().debug(
"Send scan request data: {}".format(request_params_dict))
retry_times = Config().get_config("scanner.retry_times")
while retry_times >= 0:
try:
async with context.Context():
async with http_func(**request_params_dict,
proxy=proxy_url,
allow_redirects=False,
ssl=False) as response:
response = {
"status": response.status,
"headers": response.headers,
"body": await response.read()
}
break
except (asyncio.TimeoutError,
aiohttp.client_exceptions.ClientError) as e:
Logger().warning(
"Send scan request timeout, request params:{}".format(
request_params_dict))
await asyncio.sleep(1)
retry_times -= 1
except asyncio.CancelledError as e:
raise e
except Exception as e:
Logger().error("Send scan request failed!", exc_info=e)
await asyncio.sleep(1)
retry_times -= 1
if retry_times >= 0:
return response
else:
Logger().warning(
"Scan request timeout many times, skip! request params:{}".
format(request_params_dict))
raise exceptions.ScanRequestFailed
def run(self):
"""
模块主函数，启动协程
"""
try:
asyncio.run(self.async_run())
except RuntimeError:
Logger().info("Scanner process has been killed!")
except Exception as e:
Logger().error("Scanner exit with unknow error!", exc_info=e)
def _run_module(self):
""" module线程主函数 """
try:
Logger().info("Module started!")
module = self.module_cls(**self.module_params)
module.run()
except Exception as e:
Logger().error("Module down with exception:", exc_info=e)
else:
Logger().info("Module stopped!")
def run(self):
"""
模块主函数，启动协程
"""
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(self.async_run())
except RuntimeError:
Logger().info("Scanner process has been killed!")
except Exception as e:
Logger().error("Scanner exit with unknow error!", exc_info=e)
def run(self):
""" 初始化并启动module线程 """
Communicator().init_new_module(self.module_name)
Communicator().set_value("pid", os.getpid())
Logger().init_module_logger()
Logger().debug("Init proc_comm success, current module_name is: " +
Communicator().get_module_name())
try:
self._run_module()
except KeyboardInterrupt:
pass
async def mark_result(self, last_id, failed_list):
"""
将id 小于等于 last_id的result标记为已扫描，更新star_id, 将failed_list中的id标记为失败
Parameters:
last_id - 已扫描的最大id
failed_list - 扫描中出现连接失败的url
Raises:
exceptions.DatabaseError - 数据库错误引发此异常
"""
if last_id > self.start_id:
# 标记失败的扫描记录
query = self.ResultList.update({
self.ResultList.scan_status: 3
}).where((self.ResultList.id <= last_id)
& (self.ResultList.id > self.start_id)
& (self.ResultList.id << failed_list))
try:
await peewee_async.execute(query)
except Exception as e:
Logger().critical("Database error in mark_result method!",
exc_info=e)
raise exceptions.DatabaseError
# 标记已扫描的记录
query = self.ResultList.update({
self.ResultList.scan_status: 1
}).where((self.ResultList.id <= last_id)
& (self.ResultList.id > self.start_id)
& (self.ResultList.scan_status == 2))
try:
await peewee_async.execute(query)
except Exception as e:
Logger().critical("Database error in mark_result method!",
exc_info=e)
raise exceptions.DatabaseError
# 更新start_id
query = self.ResultList.select(peewee.fn.MAX(
self.ResultList.id)).where((self.ResultList.id > self.start_id)
&
(self.ResultList.scan_status == 1))
try:
result = await peewee_async.scalar(query)
except Exception as e:
Logger().critical("Database error in mark_result method!",
exc_info=e)
raise exceptions.DatabaseError
if result is not None:
self.start_id = result
def set_param(self, para_type, para_name, value):
"""
设置HTTP请求的某个变量，覆盖原有值，不存在时创建
Parameters:
para_type - str, 参数类型，可选get, post, cookies, headers, json, files, body
para_name - str/list, 参数名或参数路径,
当para_type为json时, para_name应为一个list,包含json path每一级的key
当para_type为files时, para_name应为一个包含两个item的list, 第一个指定要设置的files dict的下标, 第二个指定dict key， 当设置content时，类型必须为bytes
例如: files: [ {"name":"file", "filename":"name.txt", "content":"xxx"} ...] 设置第一个item的filename -> [0, "filename"]
value - str, 要设置的值
Raises:
exceptions.DataParamError - 参数错误引发此异常
"""
if para_type == "cookies":
self.http_data["cookies"][para_name] = urllib.parse.quote(value)
elif para_type == "get":
self.http_data["params"][para_name] = value
elif para_type == "post":
self.http_data["data"][para_name] = value
elif para_type == "headers":
self.http_data["headers"][para_name] = urllib.parse.quote(value)
elif para_type == "json":
# 如果para_name为空，将root节点为设为value
if len(para_name) == 0:
self.http_data["json"] = value
return
json_target = self.http_data["json"]
for i in range(len(para_name)):
name = para_name[i]
obj = json_target.get(name, None)
if len(para_name) == i + 1:
json_target[name] = value
elif obj is None:
if type(para_name[i]) is int:
json_target[name] = []
else:
json_target[name] = {}
elif para_type == "files":
if para_name[1] == "content" and type(value) is not bytes:
Logger().error(
"RequestData files content must set with bytes type!")
raise exceptions.DataParamError
else:
self.http_data["files"][para_name[0]][para_name[1]] = value
elif para_type == "body":
self.http_data["body"] = value
else:
Logger().error("Use an invalid para_type in set_param method!")
raise exceptions.DataParamError
def _upload_report(self):
try:
self.cloud_api = CloudApi()
except Exception as e:
Logger().critical("CloudApi init failed!", exc_info=e)
Logger().info("Init cloud_api success.")
while True:
time.sleep(10)
try:
self.cloud_api.upload_report()
except Exception as e:
Logger().warning("Upload report error.", exc_info=e)
def __init__(self,
table_prefix=None,
use_async=True,
create_table=True,
multiplexing_conn=False):
"""
初始化
Parameters:
table_prefix - 表名前缀，由扫描目标的 host + "_" + str(port) 组成
use_async - 是否开启数据库连接的异步查询功能，默认为True
create_table - 数据表不存在时是否创建，默认为True
multiplexing_conn - 是否复用连接，为True时，相同的Model的实例会使用同一个连接，默认为False
Raises:
create_table为Fasle且目标数据表不存在时，引发exceptions.TableNotExist
"""
self.use_async = use_async
try:
if multiplexing_conn:
database = BaseModel.mul_database
else:
if self.use_async:
database = peewee_async.MySQLDatabase(**self.connect_para)
else:
database = peewee.MySQLDatabase(**self.connect_para)
database.connect()
# table_prefix 为None则不建立数据表实例，仅用于调用基类方法
if table_prefix is not None:
self._model = self._create_model(database, table_prefix)
if not self._model.table_exists():
if create_table:
try:
database.create_tables([self._model])
Logger().debug("Create table {}_{}".format(
table_prefix, self.__class__.__name__))
if self.__class__.__name__ == "NewRequestModel":
Communicator().update_target_list_status()
except peewee.InternalError:
pass
else:
raise exceptions.TableNotExist
self.database = database
except exceptions.TableNotExist as e:
raise e
except Exception as e:
Logger().critical("Mysql Connection Fail!", exc_info=e)
raise exceptions.DatabaseError
async def _check_scan_progress(self):
"""
监测扫描进度，给出下次获取的任务量
"""
sleep_interval = 1
sleep_count = 0
while True:
await asyncio.sleep(sleep_interval)
sleep_count += 1
scan_num_list = []
scan_id_list = []
for plugin_name in self.plugin_loaded:
plugin_ins = self.plugin_loaded[plugin_name]
plugin_scan_num, plugin_last_id = plugin_ins.get_scan_progress(
)
scan_num_list.append(plugin_scan_num)
scan_id_list.append(plugin_last_id)
plugin_scan_min_num = min(scan_num_list)
plugin_scan_min_id = min(scan_id_list)
finish_count = plugin_scan_min_num - self.scan_num
if sleep_count > 20:
# 20个sleep内未扫描完成，每次最大获取任务量减半
self.scan_queue_remaining -= finish_count
self.scan_num = plugin_scan_min_num
sleep_count = 0
Logger().debug(
"Finish scan num: {}, remain task: {}, max scanned id: {}, decrease task fetch_count."
.format(finish_count, self.scan_queue_remaining,
plugin_scan_min_id))
elif sleep_count > 10:
# 10-20个sleep内完成一半以上，每次最大获取任务量不变
if self.scan_queue_remaining < finish_count * 2:
self.fetch_count = finish_count
break
elif self.scan_queue_remaining == finish_count:
# 10个sleep内完成，每次最大获取任务量加倍
self.fetch_count = self.scan_queue_remaining * 2
break
self.scan_queue_remaining -= finish_count
self.scan_num = plugin_scan_min_num
self.mark_id = plugin_scan_min_id
Logger().debug(
"Finish scan num: {}, remain task: {}, max scanned id: {}".format(
finish_count, self.scan_queue_remaining, plugin_scan_min_id))
def _set_affinity(self):
if Config().get_config("affinity.enable") is True:
try:
core_num = Config().get_config("affinity.core_num")
cpu_count = psutil.cpu_count()
if core_num <= 0 or cpu_count < core_num:
mask = range(1)
Logger().warning(
"Config item affinity.core_num invalid, use defaut (1)"
)
else:
mask = range(core_num)
os.sched_setaffinity(os.getpid(), mask)
except Exception as e:
Logger().error("set affinity error!", exc_info=e)
def _check_alive(self):
"""
判断其他模块是否正常运行
"""
ppid = os.getppid()
if ppid <= 1:
Logger().warning("Detect main process stopped, Monitor exit!")
self.crash_module = "main"
return False
# http server存活检测
# if not self.web_console_thread.isAlive():
# Logger().error("Detect monitor web console stopped, Monitor exit!")
# self.crash_module = "Monitor_web_console"
# return False
if self.cloud_thread is not None and not self.cloud_thread.isAlive():
Logger().error(
"Detect monitor cloud thread stopped, Monitor exit!")
self.crash_module = "cloud_thread"
return False
if self.transaction_thread is not None and not self.transaction_thread.isAlive(
):
Logger().error(
"Detect monitor cloud transaction thread stopped, Monitor exit!"
)
self.crash_module = "transaction_thread"
return False
if self.preprocessor_proc is None:
pid = Communicator().get_value("pid", "Preprocessor")
if pid != 0:
try:
self.preprocessor_proc = psutil.Process(pid)
except Exception:
Logger().error(
"Init Preprocessor proc fail, Monitor exit!")
self.crash_module = "preprocessor"
return False
return True
elif not self.preprocessor_proc.is_running():
Logger().error("Detect preprocessor stopped, Monitor exit!")
self.crash_module = "preprocessor"
return False
return True
def kill_scanner(self, scanner_id):
"""
强制结束一个扫描进程进程
Parameters:
scanner_id - int类型, 要结束的扫描进程的id
Returns:
成功结束返回True，否则返回false
"""
pid = self._scanner_info.get_pid(scanner_id)
if pid is None:
raise exceptions.InvalidScannerId
try:
proc = psutil.Process(pid)
except psutil.NoSuchProcess:
Logger().warning("Try to kill not running scanner!")
raise exceptions.InvalidScannerId
proc.terminate()
try:
proc.wait(timeout=5)
except psutil.TimeoutExpired:
proc.kill()
proc.wait(timeout=5)
if proc.is_running():
return False
else:
self._scanner_info.remove_scanner_info(scanner_id)
module_name = "Scanner_" + str(scanner_id)
Communicator().set_value("pid", 0, module_name)
return True
async def get(self, page=1, perpage=10):
"""
获取数据
Parameters:
page - int, 获取的页码
perpage - int, 每页的数据条数
Returns:
{"total":数据总条数, "data":[ RaspResult组成的list的json字符串, ...]}
Raises:
exceptions.DatabaseError - 数据库错误引发此异常
"""
if page <= 0:
page = 1
if perpage <= 0:
perpage = 1
result = {}
try:
query = self.Report.select().offset((page-1)*perpage).limit(perpage)
data = await peewee_async.execute(query)
result["total"] = len(data)
result["data"] = []
for line in data:
result["data"].append(line.rasp_result_list)
return result
except asyncio.CancelledError as e:
raise e
except Exception as e:
Logger().critical("DB method get_new_scan Fail!", exc_info=e)
raise exceptions.DatabaseError
def monitor_fixture():
from core.components.runtime_info import RuntimeInfo
RuntimeInfo._refresh_system_info = types.MethodType(
_refresh_info_hook, RuntimeInfo)
helper.reset_db()
Communicator()
Logger()
ForkProxy()
module_proc = modules.Process(modules.Monitor)
module_proc.start()
fork_proxy_proc = multiprocessing.Process(target=_fork_proxy)
fork_proxy_proc.start()
yield {"set_system_info": _set_system_info}
root_proc = psutil.Process(module_proc.pid)
fork_proc = psutil.Process(fork_proxy_proc.pid)
procs = root_proc.children(recursive=True)
procs.append(fork_proc)
procs.append(root_proc)
try:
for p in procs:
p.terminate()
p.wait(2)
except Exception:
raise Exception("Module process may not be killed success!")
module_proc.join(3)
fork_proxy_proc.join(3)
helper.reset_db()
Communicator.reset()
def _wait_child(signum, frame):
"""
处理进程terminate信号
"""
try:
while True:
cpid, status = os.waitpid(-1, os.WNOHANG)
if cpid == 0:
break
exitcode = status >> 8
Logger().warning("Module process {} exit with exitcode {}".format(cpid, exitcode))
except OSError as e:
if e.errno == errno.ECHILD:
Logger().warning('Main process has no existing unwaited-for child processes.')
else:
Logger().error("Unknow error occurred in method _wait_child!", exc_info=e)
def get_plugin_info(self, plugin_path, class_prefix):
"""
获取指定类型插件的plugin_info
Parameters:
plugin_path - str, 插件目录
Returns:
list, 每个item为一个plugin_info dict
"""
result = []
plugin_names = []
plugin_import_path = plugin_path.replace(os.sep, ".")
for file_name in os.listdir(plugin_path):
if os.path.isfile(plugin_path + os.sep +
file_name) and file_name.endswith(".py"):
plugin_names.append(file_name[:-3])
for plugin_name in plugin_names:
try:
plugin_module = __import__(plugin_import_path,
fromlist=[plugin_name])
except Exception as e:
Logger().warning(
"Error in import plugin: {}".format(plugin_name),
exc_info=e)
else:
plugin_module = getattr(plugin_module, plugin_name)
plugin_info = getattr(plugin_module,
class_prefix + "Plugin").plugin_info
result.append(plugin_info)
return result
def __init__(self, **kwargs):
"""
初始化
"""
# kwargs 参数初始化
self.target_host = kwargs["host"]
self.target_port = kwargs["port"]
self._init_scan_config()
# 用于记录失败请求并标记
self.failed_task_set = set()
Communicator().set_internal_shared("failed_task_set",
self.failed_task_set)
self.module_id = Communicator().get_module_name().split("_")[-1]
Communicator().set_value("max_concurrent_request", 1)
Communicator().set_value(
"request_interval",
Config().get_config("scanner.min_request_interval"))
self._init_db()
self._init_plugin()
# 更新运行时配置
self._update_scan_config()
Logger().info("Start scanning target host:{}:{}".format(
self.target_host, self.target_port))
def _make_multipart(self):
"""
构造multipart/form-data 类型的aiohttp请求参数
Returns:
aiohttp.MultipartWriter实例
"""
mpwriter = aiohttp.MultipartWriter('form-data')
post_data = self.http_data["data"]
for key in post_data:
part = mpwriter.append(post_data[key])
part.set_content_disposition("form-data", name=key)
part.headers.pop(aiohttp.hdrs.CONTENT_LENGTH, None)
part.headers.pop(aiohttp.hdrs.CONTENT_TYPE, None)
files = self.http_data["files"]
for file_item in files:
content_type = file_item.get("content_type",
'application/octet-stream')
part = mpwriter.append(file_item["content"],
{'CONTENT-TYPE': content_type})
part.set_content_disposition("form-data",
filename=file_item["filename"],
name=file_item["name"])
part.headers.pop(aiohttp.hdrs.CONTENT_LENGTH, None)
Logger().debug("Make multipart data from dict: {}".format(post_data))
return mpwriter
