def execute_analysis(self, module_type: str, request_json: str) -> str:
"""Processes the request with the analysis module."""
amt = AnalysisModuleType.from_json(module_type)
module = self.module_map[amt.name]
# run_until_complete just keeps going until it returns
# there's no way to "cancel" it if something gets stuck
# the only way out is to kill the process
# so we start a thread to monitor the timeout
def _module_timeout():
get_logger().critical(
f"analysis module {module} timed out analyzing request {request_json}"
)
if self.concurrency_mode == CONCURRENCY_MODE_PROCESS:
# and then die if we hit it
# NOTE that if we're only running threads then there's really no way out other than to log it
sys.exit(1)
get_logger().debug(f"starting timer for {module.timeout} seconds")
t = threading.Timer(module.timeout, _module_timeout)
t.start()
try:
result = self.event_loop.run_until_complete(
self.execute_analysis_async(module_type, request_json))
finally:
# if we didn't time out make sure we cancel the timer
t.cancel()
return result
async def upgrade_module_async(self, amt_json: str) -> str:
if not self.module_map:
raise RuntimeError(
"_upgrade_multi_process_module called before _initialize_executor"
)
amt = AnalysisModuleType.from_json(amt_json)
module = self.module_map[amt.name]
await module.upgrade()
return module.type.to_json()
async def execute_analysis_async(self, module_type: str,
request_json: str) -> str:
request = AnalysisRequest.from_json(request_json, self.system)
amt = AnalysisModuleType.from_json(module_type)
module = self.module_map[amt.name]
if not module.type.extended_version_matches(amt):
await module.upgrade()
if not module.type.extended_version_matches(amt):
raise AnalysisModuleTypeExtendedVersionError(amt, module.type)
analysis = request.modified_observable.add_analysis(
Analysis(type=module.type, details={}))
await module.execute_analysis(request.modified_root,
request.modified_observable, analysis)
return request.to_json()
def test_analysis_module_type_serialization():
amt = AnalysisModuleType(
name="test",
description="test",
observable_types=["test1", "test2"],
directives=["test1", "test2"],
dependencies=["test1", "test2"],
tags=["test1", "test2"],
modes=["test1", "test2"],
cache_ttl=60,
extended_version={"test1": "test2"},
types=["test1", "test2"],
)
assert amt == AnalysisModuleType.from_dict(amt.to_dict())
assert amt == AnalysisModuleType.from_json(amt.to_json())
async def upgrade_module(self, module: AnalysisModule) -> bool:
"""Attempts to upgrade the extended version of the analysis module.
Returns True if the upgrade was successful, False otherwise."""
# if the module is async then we attempt to upgrade it here
try:
if module.is_multi_process:
module.type = AnalysisModuleType.from_json(
await asyncio.get_event_loop().run_in_executor(
self.executor, _cpu_task_executor_upgrade_module,
module.type.to_json()))
else:
await module.upgrade()
return True
except Exception as e:
get_logger().error(f"unable to upgrade module {module.type}: {e}")
return False
