def execute_analysis(self, module_type: str, request_json: str) -> str: """Processes the request with the analysis module.""" amt = AnalysisModuleType.from_json(module_type) module = self.module_map[amt.name] # run_until_complete just keeps going until it returns # there's no way to "cancel" it if something gets stuck # the only way out is to kill the process # so we start a thread to monitor the timeout def _module_timeout(): get_logger().critical( f"analysis module {module} timed out analyzing request {request_json}" ) if self.concurrency_mode == CONCURRENCY_MODE_PROCESS: # and then die if we hit it # NOTE that if we're only running threads then there's really no way out other than to log it sys.exit(1) get_logger().debug(f"starting timer for {module.timeout} seconds") t = threading.Timer(module.timeout, _module_timeout) t.start() try: result = self.event_loop.run_until_complete( self.execute_analysis_async(module_type, request_json)) finally: # if we didn't time out make sure we cancel the timer t.cancel() return result
async def upgrade_module_async(self, amt_json: str) -> str: if not self.module_map: raise RuntimeError( "_upgrade_multi_process_module called before _initialize_executor" ) amt = AnalysisModuleType.from_json(amt_json) module = self.module_map[amt.name] await module.upgrade() return module.type.to_json()
async def execute_analysis_async(self, module_type: str, request_json: str) -> str: request = AnalysisRequest.from_json(request_json, self.system) amt = AnalysisModuleType.from_json(module_type) module = self.module_map[amt.name] if not module.type.extended_version_matches(amt): await module.upgrade() if not module.type.extended_version_matches(amt): raise AnalysisModuleTypeExtendedVersionError(amt, module.type) analysis = request.modified_observable.add_analysis( Analysis(type=module.type, details={})) await module.execute_analysis(request.modified_root, request.modified_observable, analysis) return request.to_json()
def test_analysis_module_type_serialization(): amt = AnalysisModuleType( name="test", description="test", observable_types=["test1", "test2"], directives=["test1", "test2"], dependencies=["test1", "test2"], tags=["test1", "test2"], modes=["test1", "test2"], cache_ttl=60, extended_version={"test1": "test2"}, types=["test1", "test2"], ) assert amt == AnalysisModuleType.from_dict(amt.to_dict()) assert amt == AnalysisModuleType.from_json(amt.to_json())
async def upgrade_module(self, module: AnalysisModule) -> bool: """Attempts to upgrade the extended version of the analysis module. Returns True if the upgrade was successful, False otherwise.""" # if the module is async then we attempt to upgrade it here try: if module.is_multi_process: module.type = AnalysisModuleType.from_json( await asyncio.get_event_loop().run_in_executor( self.executor, _cpu_task_executor_upgrade_module, module.type.to_json())) else: await module.upgrade() return True except Exception as e: get_logger().error(f"unable to upgrade module {module.type}: {e}") return False