def get_scan_plugin_logger(self, plugin_name):
"""
配置扫描插件logger
Parameters:
plugin_name - str, 扫描插件名
Returns:
Logger , 生成的logger
"""
log_path = self.module_log_path
handler = logging.handlers.RotatingFileHandler(
log_path + "/plugin_" + plugin_name + ".log",
mode='a',
maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024,
backupCount=Config().get_config("log.rotate_num"))
module_name = Communicator().get_module_name()
logger = logging.getLogger("openrasp_iast.module_name_" + plugin_name)
log_fmt = '[%(asctime)s - %(levelname)s] %(message)s [file: %(pathname)s , line %(lineno)d]'
date_fmt = '%Y-%m-%d %H:%M:%S'
fmt = logging.Formatter(fmt=log_fmt, datefmt=date_fmt)
handler.setFormatter(fmt)
logger.parent = None
logger.propagate = False
logger.handlers = []
logger.addHandler(handler)
logger.setLevel(self._log_level)
return logger
def __init__(self):
"""
初始化
"""
is_scanner = Communicator().get_module_name().startswith("Scanner")
if is_scanner:
self.logger = Logger().get_scan_plugin_logger(self.plugin_info["name"])
else:
self.logger = Logger()
self._enable = True # 插件是否启用
self._white_reg = None # 扫描url白名单
self._scan_queue = queue.Queue() # 任务队列
self._last_scan_id = 0 # 最近扫描完成的任务在数据库中的id
self._scan_num = 0 # 当前已扫描url数量
self._request_timeout = Config().get_config("scanner.request_timeout")
self._max_concurrent_task = Config().get_config("scanner.max_concurrent_request")
# 共享的report_model 和 failed_task_set 需要在实例化ScanPluginBase类之前设置
try:
self._report_model = Communicator().get_internal_shared("report_model")
self._failed_set = Communicator().get_internal_shared("failed_task_set")
except exceptions.InternalSharedKeyError as e:
Logger().error("Try to init scan_plugin before set internal shared key in Communicator! Check 'error.log' for more information.")
exit(1)
self._request_session = audit_tools.Session()
self._request_data = audit_tools.RequestData
self.mutant_helper = audit_tools.MutantHelper()
self.checker = audit_tools.Checker()
if is_scanner:
self.logger.info("Scanner plugin {} init success!".format(self.plugin_info["name"]))
def _set_handler(self, logger, suffix, log_fmt, concurrent=True):
"""
为logger配置Handler
"""
if concurrent is True:
handler = cloghandler.ConcurrentRotatingFileHandler(
self.log_path + "/" + suffix,
mode='a',
maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024,
backupCount=Config().get_config("log.rotate_num"),
debug=False)
else:
handler = logging.handlers.RotatingFileHandler(
self.log_path + "/" + suffix,
mode='a',
maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024,
backupCount=Config().get_config("log.rotate_num"),
)
date_fmt = '%Y-%m-%d %H:%M:%S'
fmt = logging.Formatter(fmt=log_fmt, datefmt=date_fmt)
handler.setFormatter(fmt)
logger.propagate = False
logger.handlers = []
logger.addHandler(handler)
logger.setLevel(self._log_level)
def __init__(self):
# 初始化插件
plugin_path = "plugin.deduplicate"
plugin_name = Config().get_config("preprocessor.plugin_name")
try:
plugin_module = __import__(plugin_path, fromlist=[plugin_name])
self.dedup_plugin = getattr(plugin_module,
plugin_name).DedupPlugin()
assert isinstance(self.dedup_plugin,
dedup_plugin_base.DedupPluginBase)
except Exception as e:
Logger().warning(
"Dedupulicate plugin {} init fail!".format(plugin_name),
exc_info=e)
self.dedup_lru = DedupLru(
Config().get_config("preprocessor.request_lru_size"))
self.new_request_storage = ResultStorage()
self.app = tornado.web.Application([
tornado.web.url(
Config().get_config("preprocessor.api_path"), jsonHandler,
dict(
dedup_lru=self.dedup_lru,
dedup_plugin=self.dedup_plugin,
new_request_storage=self.new_request_storage,
))
])
def _init_error_log(self):
"""
配置统一的error.log
"""
error_logger = logging.getLogger("openrasp_iast.error")
# 前台输出
stream_handler = logging.StreamHandler(sys.stderr)
stream_handler.setLevel(logging.ERROR)
fmt = logging.Formatter('[!] %(message)s')
stream_handler.setFormatter(fmt)
# 文件输出
file_handler = cloghandler.ConcurrentRotatingFileHandler(
self.log_path + "/error.log",
mode='a',
maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024,
backupCount=Config().get_config("log.rotate_num"))
date_fmt = '%Y-%m-%d %H:%M:%S'
log_fmt = '[%(asctime)s - %(levelname)s][%(processName)s] %(message)s [file: %(pathname)s , line %(lineno)d]'
fmt = logging.Formatter(fmt=log_fmt, datefmt=date_fmt)
file_handler.setFormatter(fmt)
error_logger.propagate = False
error_logger.handlers = []
error_logger.addHandler(file_handler)
error_logger.addHandler(stream_handler)
error_logger.setLevel(logging.ERROR)
self.error_logger = error_logger
self.critical = self.error_logger.critical
self.error = self.error_logger.error
def start(args):
"""
启动
"""
Config().load_config(args.config_path)
init_check()
real_log_path = os.path.realpath(Config().get_config("log.path"))
log_level = Config().get_config("log.level").upper()
print("[-] Log file will generate to {}, log level: {}".format(
real_log_path, log_level))
pid, config_path = Config().get_running_info()
if pid != 0:
try:
os.kill(pid, 0)
except OSError:
pass
else:
print("[!] OpenRASP-IAST is already Running!")
return
from core.launcher import Launcher
if not args.foreground:
detach_run()
Config().set_running_info()
Launcher().launch()
def __init__(self):
self.server_url = Config().get_config("cloud_api.backend_url")
self.app_secret = Config().get_config("cloud_api.app_secret")
self.app_id = Config().get_config("cloud_api.app_id")
self.monitor_port = str(Config().config_dict["monitor.console_port"])
self.monitor_url = urlparse(self.server_url).hostname + ":" + self.monitor_port
self.message_bucket = []
def run(self):
"""
启动http server
"""
server = tornado.httpserver.HTTPServer(
self.app,
max_buffer_size=Config().get_config(
"preprocessor.max_buffer_size"))
try:
server.bind(Config().get_config("preprocessor.http_port"))
except OSError as e:
Logger().critical("Preprocessor bind port error!", exc_info=e)
sys.exit(1)
else:
# 这里会创建多个子进程，需要重新初始化Communicator
server.start(Config().get_config("preprocessor.process_num"))
Communicator().init_new_module(type(self).__name__)
# 记录pid
while True:
if Communicator().set_pre_http_pid(os.getpid()):
break
else:
pids = ", ".join(
str(x) for x in Communicator().get_pre_http_pid())
Logger().error(
"Preprocessor HTTP Server set pid failed! Running pids: {}"
.format(pids))
time.sleep(3)
tornado.ioloop.IOLoop.current().start()
def __init__(self):
self.server_url = Config().get_config(
"cloud_api.backend_url") + "/v1/agent/log/attack"
self.app_secret = Config().get_config("cloud_api.app_secret")
self.app_id = Config().get_config("cloud_api.app_id")
self.base_report_model = report_model.ReportModel(
table_prefix=None,
use_async=False,
create_table=False,
multiplexing_conn=True)
def __new__(cls):
if not hasattr(cls, "instance"):
cls.instance = super(Communicator, cls).__new__(cls)
cls.instance.scanner_num = Config().get_config("scanner.max_module_instance")
cls.instance.pre_http_num = Config().get_config("preprocessor.process_num")
cls.instance._init_queues()
cls.instance._init_shared_mem()
cls.instance._init_shared_setting()
cls.instance._init_main_path()
cls.instance.init_new_module("MainProcess")
return cls.instance
def run(self):
remote = self.server_url.replace("https", "wss").replace("http", "ws") + "/v1/iast"
union_header = {
"X-OpenRASP-AppSecret": Config().config_dict["cloud_api.app_secret"],
"X-OpenRASP-AppID": Config().config_dict["cloud_api.app_id"]
}
try:
asyncio.set_event_loop(asyncio.new_event_loop())
asyncio.get_event_loop().run_until_complete(self.start(remote, union_header))
except Exception as e:
Logger().error('Cloud transaction disconnected!', exc_info=e)
def _set_affinity(self):
if Config().get_config("affinity.enable") is True:
try:
core_num = Config().get_config("affinity.core_num")
cpu_count = psutil.cpu_count()
if core_num <= 0 or cpu_count < core_num:
mask = 1
else:
mask = 2**core_num - 1
os.sched_setaffinity(os.getpid(), mask)
except Exception as e:
Logger().error("set affinity error!", exc_info=e)
async def async_init(self):
"""
初始化
"""
cookie_jar = aiohttp.DummyCookieJar()
conn = aiohttp.TCPConnector(
limit=Config().get_config("scanner.max_concurrent_request"))
timeout = aiohttp.ClientTimeout(
total=Config().get_config("scanner.request_timeout"))
self.session = aiohttp.ClientSession(cookie_jar=cookie_jar,
connector=conn,
timeout=timeout)
def __new__(cls):
"""
单例模式初始化
"""
if not hasattr(cls, 'instance'):
cls.instance = super(RaspResultReceiver, cls).__new__(cls)
# 以 request_id 为key ,每个item为一个list，结构为: [获取到result的event, 过期时间, 获取到的结果(未获取前为None)]
# 例如 {scan_request_id_1: [event_1, expire_time1, result_dict_1] , scan_request_id_2:[event_2, expire_time2, None] ...}
cls.instance.rasp_result_collection = collections.OrderedDict()
cls.instance.timeout = Config().get_config("scanner.request_timeout") * \
(Config().get_config("scanner.retry_times") + 1)
return cls.instance
def _set_affinity(self):
if Config().get_config("affinity.enable") is True:
try:
core_num = Config().get_config("affinity.core_num")
cpu_count = psutil.cpu_count()
if core_num <= 0 or cpu_count < core_num:
mask = range(1)
Logger().warning(
"Config item affinity.core_num invalid, use defaut (1)"
)
else:
mask = range(core_num)
os.sched_setaffinity(os.getpid(), mask)
except Exception as e:
Logger().error("set affinity error!", exc_info=e)
def _is_cpu_overused(self):
"""
判断cpu是否负载过高
Returns:
boolean, boolean - cpu是否负载过高，cpu是否空闲
"""
system_info = RuntimeInfo().get_system_info()
if system_info["cpu"] > Config().get_config("monitor.max_cpu"):
Logger().info("CPU percent is higher than limit ({})".format(
system_info["cpu"]))
return True, False
elif system_info["cpu"] < Config().get_config("monitor.min_cpu"):
return False, True
else:
return False, False
def _init_logger(self):
"""
初始化
"""
self._log_level = Config().get_config("log.level").upper()
if self._log_level not in ("DEBUG", "INFO", "WARNING", "ERROR",
"CRITICAL"):
self._log_level = "INFO"
self.log_path = Config().get_config("log.path")
self.module_log_path = self.log_path
if not os.path.exists(self.log_path):
os.makedirs(self.log_path)
self._init_error_log()
self.init_module_logger()
def _creat_conn():
"""
创建mysql连接
"""
if not hasattr(BaseModel, "pymysql_conn") or \
time.time() > BaseModel.pymysql_conn_timeout or \
not BaseModel.pymysql_conn.open:
if hasattr(BaseModel, "pymysql_conn"):
BaseModel.pymysql_conn.close()
BaseModel.pymysql_conn = pymysql.connect(
port=Config().config_dict["database.port"],
host=Config().config_dict["database.host"],
user=Config().config_dict["database.username"],
passwd=Config().config_dict["database.password"],
database=Config().config_dict["database.db_name"])
BaseModel.pymysql_conn_timeout = time.time() + 60
def __init__(self, **kwargs):
"""
初始化
"""
# kwargs 参数初始化
self.target_host = kwargs["host"]
self.target_port = kwargs["port"]
self._init_scan_config()
# 用于记录失败请求并标记
self.failed_task_set = set()
Communicator().set_internal_shared("failed_task_set",
self.failed_task_set)
self.module_id = Communicator().get_module_name().split("_")[-1]
Communicator().set_value("max_concurrent_request", 1)
Communicator().set_value(
"request_interval",
Config().get_config("scanner.min_request_interval"))
self._init_db()
self._init_plugin()
# 更新运行时配置
self._update_scan_config()
Logger().info("Start scanning target host:{}:{}".format(
self.target_host, self.target_port))
def check_start():
"""
检测后台启动是否成功
"""
import requests
port = Config().get_config("preprocessor.http_port")
path = Config().get_config("preprocessor.api_path")
url = "http://127.0.0.1:{}{}".format(port, path)
for i in range(15):
try:
r = requests.get(url=url, timeout=2)
if r.status_code == 405:
return True
break
except Exception:
time.sleep(2)
return False
def __init__(self):
"""
初始化
"""
current_path = os.path.dirname(__file__)
self.static_path = os.path.join(current_path, "../../web")
self.port = Config().get_config("monitor.console_port")
self._init_app()
def test_scheduler(monitor_fixture):
max_cr_last = Communicator().get_value("max_concurrent_request",
"Scanner_0")
for i in range(5):
Communicator().add_value("send_request", "Scanner_0", 30)
time.sleep(Config().get_config("monitor.schedule_interval") * 1.5)
max_cr = Communicator().get_value("max_concurrent_request",
"Scanner_0")
assert max_cr > max_cr_last or max_cr == Config().get_config(
"scanner.max_concurrent_request")
max_cr_last = max_cr
assert max_cr_last == 5
for i in range(20):
Communicator().add_value("send_request", "Scanner_0", 30)
Communicator().add_value("failed_request", "Scanner_0", 1)
time.sleep(Config().get_config("monitor.schedule_interval") * 1.5)
max_cr = Communicator().get_value("max_concurrent_request",
"Scanner_0")
ri = Communicator().get_value("request_interval", "Scanner_0")
assert max_cr < max_cr_last or max_cr == 1 or ri <= 256
max_cr_last = max_cr
assert max_cr_last == 1
for i in range(50):
Communicator().add_value("send_request", "Scanner_0", 30)
time.sleep(Config().get_config("monitor.schedule_interval") * 1.5)
max_cr = Communicator().get_value("max_concurrent_request",
"Scanner_0")
ri = Communicator().get_value("request_interval", "Scanner_0")
assert max_cr == 1
for i in range(50):
Communicator().add_value("send_request", "Scanner_0", 30)
time.sleep(Config().get_config("monitor.schedule_interval") * 1.5)
max_cr = Communicator().get_value("max_concurrent_request",
"Scanner_0")
ri = Communicator().get_value("request_interval", "Scanner_0")
if max_cr > 1:
break
assert max_cr > 1
def __new__(cls,
table_prefix=None,
use_async=True,
create_table=True,
multiplexing_conn=False):
"""
初始化数据库连接，构造peewee model实例
Parameters:
table_prefix - 表名前缀，由扫描目标的 host + "_" + str(port) 组成
use_async - 是否开启数据库连接的异步查询功能，默认为True
create_table - 数据表不存在时是否创建，默认为True
multiplexing_conn - 是否复用连接，为True时，相同的Model的实例会使用同一个连接，默认为False
Raises:
create_table为Fasle且目标数据表不存在时，引发exceptions.TableNotExist
"""
cls.connect_para = {
"database": Config().get_config("database.db_name"),
"host": Config().get_config("database.host"),
"port": Config().get_config("database.port"),
"user": Config().get_config("database.username"),
"password": Config().get_config("database.password")
}
if not hasattr(cls, "db_created"):
conn = pymysql.connect(
host=Config().get_config("database.host"),
port=Config().get_config("database.port"),
user=Config().get_config("database.username"),
passwd=Config().get_config("database.password"))
sql = "CREATE DATABASE IF NOT EXISTS {} default charset utf8mb4 COLLATE utf8mb4_general_ci;".format(
Config().get_config("database.db_name"))
cursor = conn.cursor()
cursor._defer_warnings = True
cursor.execute(sql)
conn.close()
cls.db_created = True
if multiplexing_conn and not hasattr(cls, "mul_database"):
cls.mul_database = peewee_async.MySQLDatabase(**cls.connect_para)
cls.mul_database.connect()
instance = super(BaseModel, cls).__new__(cls)
return instance
async def send_request(self, request_data_ins, proxy_url=None):
"""
异步发送一个http请求, 返回结果
Parameters:
request_data_ins - request_data.RequestData类的实例，包含请求的全部信息
proxy_url - 发送请求使用的代理url, 为None时不使用代理
Returns:
dict, 结构:
{
"status": http响应码,
"headers": http响应头的dict,
"body": http响应body, bytes
}
Raises:
exceptions.ScanRequestFailed - 请求发送失败时引发此异常
"""
http_func = getattr(self.session, request_data_ins.get_method())
request_params_dict = request_data_ins.get_aiohttp_param()
Logger().debug(
"Send scan request data: {}".format(request_params_dict))
retry_times = Config().get_config("scanner.retry_times")
while retry_times >= 0:
try:
async with context.Context():
async with http_func(**request_params_dict,
proxy=proxy_url,
allow_redirects=False,
ssl=False) as response:
response = {
"status": response.status,
"headers": response.headers,
"body": await response.read()
}
break
except (asyncio.TimeoutError,
aiohttp.client_exceptions.ClientError) as e:
Logger().warning(
"Send scan request timeout, request params:{}".format(
request_params_dict))
await asyncio.sleep(1)
retry_times -= 1
except asyncio.CancelledError as e:
raise e
except Exception as e:
Logger().error("Send scan request failed!", exc_info=e)
await asyncio.sleep(1)
retry_times -= 1
if retry_times >= 0:
return response
else:
Logger().warning(
"Scan request timeout many times, skip! request params:{}".
format(request_params_dict))
raise exceptions.ScanRequestFailed
def __init__(self, module_name):
"""
初始化
"""
self.module_name = module_name
self.lock = threading.Lock()
self.cr_maintain_times = 0
self.cr_maintain_times_amplitude = 5
self.last_schedule_decrease = False
self.max_performance = False
self.cr_max = Config().get_config("scanner.max_concurrent_request")
self.ri_max = Config().get_config("scanner.max_request_interval")
self.ri_min = Config().get_config("scanner.min_request_interval")
self.rrt_last = 0
self.fr_last = 0
self.sr_last = 0
async def _send_start_request(self, host, port):
data = {
"host": host,
"port": port,
"config": {}
}
api_port = Config().get_config("monitor.console_port")
url = "http://127.0.0.1:" + str(api_port) + "/api/scanner/new"
async with aiohttp.ClientSession() as session:
await session.post(url, json=data, timeout=5)
def _init_logger(self):
"""
初始化
"""
self._log_level = Config().get_config("log.level").upper()
if self._log_level not in ("DEBUG", "INFO", "WARNING", "ERROR",
"CRITICAL"):
self._log_level = "INFO"
self.log_path = Config().get_config("log.path")
self.module_log_path = self.log_path
if not os.path.exists(self.log_path):
os.makedirs(self.log_path)
self._init_error_log()
self.init_module_logger()
# log目录添加VERSION文件
main_path = Config().get_main_path()
shutil.copyfile(main_path + "/VERSION", self.log_path + "/VERSION")
def init_manager(self, scanner_schedulers):
"""
初始化
Parameters:
scanner_schedulers - 所有扫描任务调度类组成的dict, key为扫描任务的Module_name
"""
self.max_scanner = Config().get_config("scanner.max_module_instance")
self.scanner_schedulers = scanner_schedulers
self.scanner_list = [None] * self.max_scanner
self._init_config()
def stop(args):
"""
停止
"""
pid, config_path = Config().get_running_info()
if pid == 0:
print("[!] OpenRASP-IAST is not Running!")
else:
try:
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
pass
else:
time.sleep(2)
try:
os.kill(pid, 0)
except OSError:
Config().reset_running_info()
print("[-] OpenRASP-IAST stopped!")
else:
print("[!] Stop OpenRASP-IAST failed!")
def restart(args):
"""
重启
"""
# 先获取运行时的配置文件路径
pid, config_path = Config().get_running_info()
stop(args)
args.foreground = False
if config_path != "":
args.config_path = config_path
else:
args.config_path = None
start(args)
