def get_scan_plugin_logger(self, plugin_name): """ 配置扫描插件logger Parameters: plugin_name - str, 扫描插件名 Returns: Logger , 生成的logger """ log_path = self.module_log_path handler = logging.handlers.RotatingFileHandler( log_path + "/plugin_" + plugin_name + ".log", mode='a', maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024, backupCount=Config().get_config("log.rotate_num")) module_name = Communicator().get_module_name() logger = logging.getLogger("openrasp_iast.module_name_" + plugin_name) log_fmt = '[%(asctime)s - %(levelname)s] %(message)s [file: %(pathname)s , line %(lineno)d]' date_fmt = '%Y-%m-%d %H:%M:%S' fmt = logging.Formatter(fmt=log_fmt, datefmt=date_fmt) handler.setFormatter(fmt) logger.parent = None logger.propagate = False logger.handlers = [] logger.addHandler(handler) logger.setLevel(self._log_level) return logger
def __init__(self): """ 初始化 """ is_scanner = Communicator().get_module_name().startswith("Scanner") if is_scanner: self.logger = Logger().get_scan_plugin_logger(self.plugin_info["name"]) else: self.logger = Logger() self._enable = True # 插件是否启用 self._white_reg = None # 扫描url白名单 self._scan_queue = queue.Queue() # 任务队列 self._last_scan_id = 0 # 最近扫描完成的任务在数据库中的id self._scan_num = 0 # 当前已扫描url数量 self._request_timeout = Config().get_config("scanner.request_timeout") self._max_concurrent_task = Config().get_config("scanner.max_concurrent_request") # 共享的report_model 和 failed_task_set 需要在实例化ScanPluginBase类之前设置 try: self._report_model = Communicator().get_internal_shared("report_model") self._failed_set = Communicator().get_internal_shared("failed_task_set") except exceptions.InternalSharedKeyError as e: Logger().error("Try to init scan_plugin before set internal shared key in Communicator! Check 'error.log' for more information.") exit(1) self._request_session = audit_tools.Session() self._request_data = audit_tools.RequestData self.mutant_helper = audit_tools.MutantHelper() self.checker = audit_tools.Checker() if is_scanner: self.logger.info("Scanner plugin {} init success!".format(self.plugin_info["name"]))
def _set_handler(self, logger, suffix, log_fmt, concurrent=True): """ 为logger配置Handler """ if concurrent is True: handler = cloghandler.ConcurrentRotatingFileHandler( self.log_path + "/" + suffix, mode='a', maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024, backupCount=Config().get_config("log.rotate_num"), debug=False) else: handler = logging.handlers.RotatingFileHandler( self.log_path + "/" + suffix, mode='a', maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024, backupCount=Config().get_config("log.rotate_num"), ) date_fmt = '%Y-%m-%d %H:%M:%S' fmt = logging.Formatter(fmt=log_fmt, datefmt=date_fmt) handler.setFormatter(fmt) logger.propagate = False logger.handlers = [] logger.addHandler(handler) logger.setLevel(self._log_level)
def __init__(self): # 初始化插件 plugin_path = "plugin.deduplicate" plugin_name = Config().get_config("preprocessor.plugin_name") try: plugin_module = __import__(plugin_path, fromlist=[plugin_name]) self.dedup_plugin = getattr(plugin_module, plugin_name).DedupPlugin() assert isinstance(self.dedup_plugin, dedup_plugin_base.DedupPluginBase) except Exception as e: Logger().warning( "Dedupulicate plugin {} init fail!".format(plugin_name), exc_info=e) self.dedup_lru = DedupLru( Config().get_config("preprocessor.request_lru_size")) self.new_request_storage = ResultStorage() self.app = tornado.web.Application([ tornado.web.url( Config().get_config("preprocessor.api_path"), jsonHandler, dict( dedup_lru=self.dedup_lru, dedup_plugin=self.dedup_plugin, new_request_storage=self.new_request_storage, )) ])
def _init_error_log(self): """ 配置统一的error.log """ error_logger = logging.getLogger("openrasp_iast.error") # 前台输出 stream_handler = logging.StreamHandler(sys.stderr) stream_handler.setLevel(logging.ERROR) fmt = logging.Formatter('[!] %(message)s') stream_handler.setFormatter(fmt) # 文件输出 file_handler = cloghandler.ConcurrentRotatingFileHandler( self.log_path + "/error.log", mode='a', maxBytes=Config().get_config("log.rotate_size") * 1024 * 1024, backupCount=Config().get_config("log.rotate_num")) date_fmt = '%Y-%m-%d %H:%M:%S' log_fmt = '[%(asctime)s - %(levelname)s][%(processName)s] %(message)s [file: %(pathname)s , line %(lineno)d]' fmt = logging.Formatter(fmt=log_fmt, datefmt=date_fmt) file_handler.setFormatter(fmt) error_logger.propagate = False error_logger.handlers = [] error_logger.addHandler(file_handler) error_logger.addHandler(stream_handler) error_logger.setLevel(logging.ERROR) self.error_logger = error_logger self.critical = self.error_logger.critical self.error = self.error_logger.error
def start(args): """ 启动 """ Config().load_config(args.config_path) init_check() real_log_path = os.path.realpath(Config().get_config("log.path")) log_level = Config().get_config("log.level").upper() print("[-] Log file will generate to {}, log level: {}".format( real_log_path, log_level)) pid, config_path = Config().get_running_info() if pid != 0: try: os.kill(pid, 0) except OSError: pass else: print("[!] OpenRASP-IAST is already Running!") return from core.launcher import Launcher if not args.foreground: detach_run() Config().set_running_info() Launcher().launch()
def __init__(self): self.server_url = Config().get_config("cloud_api.backend_url") self.app_secret = Config().get_config("cloud_api.app_secret") self.app_id = Config().get_config("cloud_api.app_id") self.monitor_port = str(Config().config_dict["monitor.console_port"]) self.monitor_url = urlparse(self.server_url).hostname + ":" + self.monitor_port self.message_bucket = []
def run(self): """ 启动http server """ server = tornado.httpserver.HTTPServer( self.app, max_buffer_size=Config().get_config( "preprocessor.max_buffer_size")) try: server.bind(Config().get_config("preprocessor.http_port")) except OSError as e: Logger().critical("Preprocessor bind port error!", exc_info=e) sys.exit(1) else: # 这里会创建多个子进程,需要重新初始化Communicator server.start(Config().get_config("preprocessor.process_num")) Communicator().init_new_module(type(self).__name__) # 记录pid while True: if Communicator().set_pre_http_pid(os.getpid()): break else: pids = ", ".join( str(x) for x in Communicator().get_pre_http_pid()) Logger().error( "Preprocessor HTTP Server set pid failed! Running pids: {}" .format(pids)) time.sleep(3) tornado.ioloop.IOLoop.current().start()
def __init__(self): self.server_url = Config().get_config( "cloud_api.backend_url") + "/v1/agent/log/attack" self.app_secret = Config().get_config("cloud_api.app_secret") self.app_id = Config().get_config("cloud_api.app_id") self.base_report_model = report_model.ReportModel( table_prefix=None, use_async=False, create_table=False, multiplexing_conn=True)
def __new__(cls): if not hasattr(cls, "instance"): cls.instance = super(Communicator, cls).__new__(cls) cls.instance.scanner_num = Config().get_config("scanner.max_module_instance") cls.instance.pre_http_num = Config().get_config("preprocessor.process_num") cls.instance._init_queues() cls.instance._init_shared_mem() cls.instance._init_shared_setting() cls.instance._init_main_path() cls.instance.init_new_module("MainProcess") return cls.instance
def run(self): remote = self.server_url.replace("https", "wss").replace("http", "ws") + "/v1/iast" union_header = { "X-OpenRASP-AppSecret": Config().config_dict["cloud_api.app_secret"], "X-OpenRASP-AppID": Config().config_dict["cloud_api.app_id"] } try: asyncio.set_event_loop(asyncio.new_event_loop()) asyncio.get_event_loop().run_until_complete(self.start(remote, union_header)) except Exception as e: Logger().error('Cloud transaction disconnected!', exc_info=e)
def _set_affinity(self): if Config().get_config("affinity.enable") is True: try: core_num = Config().get_config("affinity.core_num") cpu_count = psutil.cpu_count() if core_num <= 0 or cpu_count < core_num: mask = 1 else: mask = 2**core_num - 1 os.sched_setaffinity(os.getpid(), mask) except Exception as e: Logger().error("set affinity error!", exc_info=e)
async def async_init(self): """ 初始化 """ cookie_jar = aiohttp.DummyCookieJar() conn = aiohttp.TCPConnector( limit=Config().get_config("scanner.max_concurrent_request")) timeout = aiohttp.ClientTimeout( total=Config().get_config("scanner.request_timeout")) self.session = aiohttp.ClientSession(cookie_jar=cookie_jar, connector=conn, timeout=timeout)
def __new__(cls): """ 单例模式初始化 """ if not hasattr(cls, 'instance'): cls.instance = super(RaspResultReceiver, cls).__new__(cls) # 以 request_id 为key ,每个item为一个list,结构为: [获取到result的event, 过期时间, 获取到的结果(未获取前为None)] # 例如 {scan_request_id_1: [event_1, expire_time1, result_dict_1] , scan_request_id_2:[event_2, expire_time2, None] ...} cls.instance.rasp_result_collection = collections.OrderedDict() cls.instance.timeout = Config().get_config("scanner.request_timeout") * \ (Config().get_config("scanner.retry_times") + 1) return cls.instance
def _set_affinity(self): if Config().get_config("affinity.enable") is True: try: core_num = Config().get_config("affinity.core_num") cpu_count = psutil.cpu_count() if core_num <= 0 or cpu_count < core_num: mask = range(1) Logger().warning( "Config item affinity.core_num invalid, use defaut (1)" ) else: mask = range(core_num) os.sched_setaffinity(os.getpid(), mask) except Exception as e: Logger().error("set affinity error!", exc_info=e)
def _is_cpu_overused(self): """ 判断cpu是否负载过高 Returns: boolean, boolean - cpu是否负载过高,cpu是否空闲 """ system_info = RuntimeInfo().get_system_info() if system_info["cpu"] > Config().get_config("monitor.max_cpu"): Logger().info("CPU percent is higher than limit ({})".format( system_info["cpu"])) return True, False elif system_info["cpu"] < Config().get_config("monitor.min_cpu"): return False, True else: return False, False
def _init_logger(self): """ 初始化 """ self._log_level = Config().get_config("log.level").upper() if self._log_level not in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"): self._log_level = "INFO" self.log_path = Config().get_config("log.path") self.module_log_path = self.log_path if not os.path.exists(self.log_path): os.makedirs(self.log_path) self._init_error_log() self.init_module_logger()
def _creat_conn(): """ 创建mysql连接 """ if not hasattr(BaseModel, "pymysql_conn") or \ time.time() > BaseModel.pymysql_conn_timeout or \ not BaseModel.pymysql_conn.open: if hasattr(BaseModel, "pymysql_conn"): BaseModel.pymysql_conn.close() BaseModel.pymysql_conn = pymysql.connect( port=Config().config_dict["database.port"], host=Config().config_dict["database.host"], user=Config().config_dict["database.username"], passwd=Config().config_dict["database.password"], database=Config().config_dict["database.db_name"]) BaseModel.pymysql_conn_timeout = time.time() + 60
def __init__(self, **kwargs): """ 初始化 """ # kwargs 参数初始化 self.target_host = kwargs["host"] self.target_port = kwargs["port"] self._init_scan_config() # 用于记录失败请求并标记 self.failed_task_set = set() Communicator().set_internal_shared("failed_task_set", self.failed_task_set) self.module_id = Communicator().get_module_name().split("_")[-1] Communicator().set_value("max_concurrent_request", 1) Communicator().set_value( "request_interval", Config().get_config("scanner.min_request_interval")) self._init_db() self._init_plugin() # 更新运行时配置 self._update_scan_config() Logger().info("Start scanning target host:{}:{}".format( self.target_host, self.target_port))
def check_start(): """ 检测后台启动是否成功 """ import requests port = Config().get_config("preprocessor.http_port") path = Config().get_config("preprocessor.api_path") url = "http://127.0.0.1:{}{}".format(port, path) for i in range(15): try: r = requests.get(url=url, timeout=2) if r.status_code == 405: return True break except Exception: time.sleep(2) return False
def __init__(self): """ 初始化 """ current_path = os.path.dirname(__file__) self.static_path = os.path.join(current_path, "../../web") self.port = Config().get_config("monitor.console_port") self._init_app()
def test_scheduler(monitor_fixture): max_cr_last = Communicator().get_value("max_concurrent_request", "Scanner_0") for i in range(5): Communicator().add_value("send_request", "Scanner_0", 30) time.sleep(Config().get_config("monitor.schedule_interval") * 1.5) max_cr = Communicator().get_value("max_concurrent_request", "Scanner_0") assert max_cr > max_cr_last or max_cr == Config().get_config( "scanner.max_concurrent_request") max_cr_last = max_cr assert max_cr_last == 5 for i in range(20): Communicator().add_value("send_request", "Scanner_0", 30) Communicator().add_value("failed_request", "Scanner_0", 1) time.sleep(Config().get_config("monitor.schedule_interval") * 1.5) max_cr = Communicator().get_value("max_concurrent_request", "Scanner_0") ri = Communicator().get_value("request_interval", "Scanner_0") assert max_cr < max_cr_last or max_cr == 1 or ri <= 256 max_cr_last = max_cr assert max_cr_last == 1 for i in range(50): Communicator().add_value("send_request", "Scanner_0", 30) time.sleep(Config().get_config("monitor.schedule_interval") * 1.5) max_cr = Communicator().get_value("max_concurrent_request", "Scanner_0") ri = Communicator().get_value("request_interval", "Scanner_0") assert max_cr == 1 for i in range(50): Communicator().add_value("send_request", "Scanner_0", 30) time.sleep(Config().get_config("monitor.schedule_interval") * 1.5) max_cr = Communicator().get_value("max_concurrent_request", "Scanner_0") ri = Communicator().get_value("request_interval", "Scanner_0") if max_cr > 1: break assert max_cr > 1
def __new__(cls, table_prefix=None, use_async=True, create_table=True, multiplexing_conn=False): """ 初始化数据库连接,构造peewee model实例 Parameters: table_prefix - 表名前缀,由扫描目标的 host + "_" + str(port) 组成 use_async - 是否开启数据库连接的异步查询功能,默认为True create_table - 数据表不存在时是否创建,默认为True multiplexing_conn - 是否复用连接,为True时,相同的Model的实例会使用同一个连接,默认为False Raises: create_table为Fasle且目标数据表不存在时,引发exceptions.TableNotExist """ cls.connect_para = { "database": Config().get_config("database.db_name"), "host": Config().get_config("database.host"), "port": Config().get_config("database.port"), "user": Config().get_config("database.username"), "password": Config().get_config("database.password") } if not hasattr(cls, "db_created"): conn = pymysql.connect( host=Config().get_config("database.host"), port=Config().get_config("database.port"), user=Config().get_config("database.username"), passwd=Config().get_config("database.password")) sql = "CREATE DATABASE IF NOT EXISTS {} default charset utf8mb4 COLLATE utf8mb4_general_ci;".format( Config().get_config("database.db_name")) cursor = conn.cursor() cursor._defer_warnings = True cursor.execute(sql) conn.close() cls.db_created = True if multiplexing_conn and not hasattr(cls, "mul_database"): cls.mul_database = peewee_async.MySQLDatabase(**cls.connect_para) cls.mul_database.connect() instance = super(BaseModel, cls).__new__(cls) return instance
async def send_request(self, request_data_ins, proxy_url=None): """ 异步发送一个http请求, 返回结果 Parameters: request_data_ins - request_data.RequestData类的实例,包含请求的全部信息 proxy_url - 发送请求使用的代理url, 为None时不使用代理 Returns: dict, 结构: { "status": http响应码, "headers": http响应头的dict, "body": http响应body, bytes } Raises: exceptions.ScanRequestFailed - 请求发送失败时引发此异常 """ http_func = getattr(self.session, request_data_ins.get_method()) request_params_dict = request_data_ins.get_aiohttp_param() Logger().debug( "Send scan request data: {}".format(request_params_dict)) retry_times = Config().get_config("scanner.retry_times") while retry_times >= 0: try: async with context.Context(): async with http_func(**request_params_dict, proxy=proxy_url, allow_redirects=False, ssl=False) as response: response = { "status": response.status, "headers": response.headers, "body": await response.read() } break except (asyncio.TimeoutError, aiohttp.client_exceptions.ClientError) as e: Logger().warning( "Send scan request timeout, request params:{}".format( request_params_dict)) await asyncio.sleep(1) retry_times -= 1 except asyncio.CancelledError as e: raise e except Exception as e: Logger().error("Send scan request failed!", exc_info=e) await asyncio.sleep(1) retry_times -= 1 if retry_times >= 0: return response else: Logger().warning( "Scan request timeout many times, skip! request params:{}". format(request_params_dict)) raise exceptions.ScanRequestFailed
def __init__(self, module_name): """ 初始化 """ self.module_name = module_name self.lock = threading.Lock() self.cr_maintain_times = 0 self.cr_maintain_times_amplitude = 5 self.last_schedule_decrease = False self.max_performance = False self.cr_max = Config().get_config("scanner.max_concurrent_request") self.ri_max = Config().get_config("scanner.max_request_interval") self.ri_min = Config().get_config("scanner.min_request_interval") self.rrt_last = 0 self.fr_last = 0 self.sr_last = 0
async def _send_start_request(self, host, port): data = { "host": host, "port": port, "config": {} } api_port = Config().get_config("monitor.console_port") url = "http://127.0.0.1:" + str(api_port) + "/api/scanner/new" async with aiohttp.ClientSession() as session: await session.post(url, json=data, timeout=5)
def _init_logger(self): """ 初始化 """ self._log_level = Config().get_config("log.level").upper() if self._log_level not in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"): self._log_level = "INFO" self.log_path = Config().get_config("log.path") self.module_log_path = self.log_path if not os.path.exists(self.log_path): os.makedirs(self.log_path) self._init_error_log() self.init_module_logger() # log目录添加VERSION文件 main_path = Config().get_main_path() shutil.copyfile(main_path + "/VERSION", self.log_path + "/VERSION")
def init_manager(self, scanner_schedulers): """ 初始化 Parameters: scanner_schedulers - 所有扫描任务调度类组成的dict, key为扫描任务的Module_name """ self.max_scanner = Config().get_config("scanner.max_module_instance") self.scanner_schedulers = scanner_schedulers self.scanner_list = [None] * self.max_scanner self._init_config()
def stop(args): """ 停止 """ pid, config_path = Config().get_running_info() if pid == 0: print("[!] OpenRASP-IAST is not Running!") else: try: os.kill(pid, signal.SIGTERM) except ProcessLookupError: pass else: time.sleep(2) try: os.kill(pid, 0) except OSError: Config().reset_running_info() print("[-] OpenRASP-IAST stopped!") else: print("[!] Stop OpenRASP-IAST failed!")
def restart(args): """ 重启 """ # 先获取运行时的配置文件路径 pid, config_path = Config().get_running_info() stop(args) args.foreground = False if config_path != "": args.config_path = config_path else: args.config_path = None start(args)