def process_exception(
self,
module: AnalysisModule,
request: AnalysisRequest,
e: Exception,
error_message: Optional[str] = None) -> AnalysisRequest:
assert isinstance(module, AnalysisModule)
assert isinstance(request, AnalysisRequest)
assert isinstance(e, Exception)
# use existing analysis if it already exists
analysis = request.modified_observable.get_analysis(module.type)
if analysis is None:
analysis = request.modified_observable.add_analysis(
Analysis(type=module.type))
# set the error message and stack trace details
if not error_message:
analysis.error_message = f"{type(e).__name__}: {e}"
else:
analysis.error_message = error_message
analysis.stack_trace = format_error_report(e)
get_logger().error(error_message)
return request
async def test_apply_merge_analysis_with_observables(system):
amt = AnalysisModuleType("test", "")
root = system.new_root()
observable = root.add_observable("some_type", "some_value")
target_root = system.new_root()
target_observable = target_root.add_observable("some_type", "some_value")
analysis = target_observable.add_analysis(Analysis(type=amt))
extra_observable = analysis.add_observable("other_type", "other_value")
assert not root.get_observable(extra_observable)
observable.apply_merge(target_observable)
assert root.get_observable(extra_observable) == extra_observable
async def test_analysis_completed(system):
await system.register_analysis_module_type(
amt := AnalysisModuleType("test", "test", ["test"]))
root = system.new_root()
observable = root.add_observable("test", "test")
assert not root.analysis_completed(observable, amt)
observable.add_analysis(Analysis(type=amt, details=TEST_DETAILS))
assert root.analysis_completed(observable, amt)
# unknown observable
with pytest.raises(UnknownObservableError):
root.analysis_completed(RootAnalysis().add_observable("test", "blah"),
amt)
def test_add_analysis():
amt = AnalysisModuleType("test", "")
root = RootAnalysis()
observable = root.add_observable("test", "test")
# test adding an Analysis object
analysis = Analysis(details={"hello": "world"}, type=amt)
result = observable.add_analysis(analysis)
assert result == analysis
# test adding just a details, type
root = RootAnalysis()
observable = root.add_observable("test", "test")
analysis = observable.add_analysis(details={"hello": "world"}, type=amt)
assert result == analysis
def test_apply_merge_analysis():
amt = AnalysisModuleType("test", "")
root = RootAnalysis()
observable = root.add_observable("some_type", "some_value")
target_root = RootAnalysis()
target_observable = target_root.add_observable("some_type", "some_value")
target_observable.add_analysis(Analysis(type=amt, details={"test":
"test"}))
assert not observable.analysis
observable.apply_merge(target_observable)
assert observable.analysis
assert observable.get_analysis("test") is not None
assert observable.get_analysis("test")._details == {"test": "test"}
async def execute_analysis_async(self, module_type: str,
request_json: str) -> str:
request = AnalysisRequest.from_json(request_json, self.system)
amt = AnalysisModuleType.from_json(module_type)
module = self.module_map[amt.name]
if not module.type.extended_version_matches(amt):
await module.upgrade()
if not module.type.extended_version_matches(amt):
raise AnalysisModuleTypeExtendedVersionError(amt, module.type)
analysis = request.modified_observable.add_analysis(
Analysis(type=module.type, details={}))
await module.execute_analysis(request.modified_root,
request.modified_observable, analysis)
return request.to_json()
async def test_apply_merge_analysis_with_existing_observables(system):
amt = AnalysisModuleType("test", "")
root = system.new_root()
observable = root.add_observable("some_type", "some_value")
existing_extra_observable = root.add_observable("other_type",
"other_value")
target_root = system.new_root()
target_observable = target_root.add_observable("some_type", "some_value")
analysis = target_observable.add_analysis(Analysis(type=amt))
extra_observable = analysis.add_observable("other_type", "other_value")
# should only have the root as the parent
assert len(existing_extra_observable.parents) == 1
observable.apply_merge(target_observable)
# should now have both the root and the new analysis as parents
assert len(existing_extra_observable.parents) == 2
async def test_apply_merge_error_analysis(system):
amt = AnalysisModuleType("test", "")
root = system.new_root()
observable = root.add_observable("some_type", "some_value")
target_root = system.new_root()
target_observable = target_root.add_observable("some_type", "some_value")
target_observable.add_analysis(
Analysis(type=amt, error_message="test", stack_trace="test"))
assert not observable.analysis
observable.apply_merge(target_observable)
assert observable.analysis
assert observable.get_analysis("test") is not None
assert observable.get_analysis("test")._details is None
assert observable.get_analysis("test").error_message == "test"
assert observable.get_analysis("test").stack_trace == "test"
async def test_analysis_details_deleted_with_root(system):
# any details associated to a root are deleted when the root is deleted
await system.register_analysis_module_type(
amt := AnalysisModuleType("test", ""))
root = system.new_root(details=TEST_DETAILS)
observable = root.add_observable("test", "test")
observable.add_analysis(
analysis := Analysis(root=root, type=amt, details=TEST_DETAILS))
await root.save()
# make sure the details are there
assert await system.get_analysis_details(root.uuid) == TEST_DETAILS
assert await system.get_analysis_details(analysis.uuid) == TEST_DETAILS
# delete the root
assert await system.delete_root_analysis(root.uuid)
assert await system.get_root_analysis(root) is None
# root details should be gone
assert await system.get_analysis_details(root.uuid) is None
# and analysis details should be gone
assert await system.get_analysis_details(analysis.uuid) is None
async def test_delete_analysis_module_type_while_processing_request(system):
amt = await system.register_analysis_module_type(
AnalysisModuleType("test", ""))
root = system.new_root()
observable = root.add_observable("test", "test")
await root.submit()
# go get the request for processing
request = await system.get_next_analysis_request("owner", amt, 0)
# delete the amt while we're processing the request
await system.delete_analysis_module_type(amt)
request.initialize_result()
request.modified_observable.add_analysis(
Analysis(type=amt, details={"test": "test"}))
# should fail since the amt has been removed
with pytest.raises(UnknownAnalysisRequestError):
await request.submit()
# and the request should already be deleted as well
assert await system.get_analysis_request_by_request_id(request.id) is None
async def execute_module(self, module: AnalysisModule, whoami: str,
request: AnalysisRequest) -> AnalysisRequest:
"""Processes the request with the analysis module.
Returns a copy of the original request with the results added"""
assert isinstance(module, AnalysisModule)
assert isinstance(whoami, str) and whoami
assert isinstance(request, AnalysisRequest)
request.initialize_result()
# if this module is going to be analyzing a file observable then we
# want to go ahead and load the content
if request.modified_observable.type == "file":
if not await request.modified_observable.load():
# if we can't load the file we don't bother asking the module to analyze it
get_logger().warning(
f"unable to load file {request.modified_observable} for {request}"
)
request.modified_observable.add_analysis(
Analysis(type=module.type,
details={},
error_message="unknown file"))
return request
if module.is_multi_process:
try:
request_json = request.to_json()
request_result_json = await asyncio.get_event_loop(
).run_in_executor(self.executor,
_cpu_task_executor_execute_analysis,
module.type.to_json(), request_json)
return AnalysisRequest.from_json(request_result_json,
self.system)
except BrokenProcessPool as e:
# when this happens you have to create and start a new one
self.process_exception(
module,
request,
e,
error_message=
f"{module.type} process crashed when analyzing type {request.modified_observable.type} value {request.modified_observable.value}",
)
# we have to start a new executor
self.start_executor()
return request
except Exception as e:
return self.process_exception(
module,
request,
e,
f"{module.type} failed analyzing type {request.modified_observable.type} value {request.modified_observable.value}: {e}",
)
else:
try:
analysis = request.modified_observable.add_analysis(
Analysis(type=module.type, details={}))
await asyncio.wait_for(
module.execute_analysis(request.modified_root,
request.modified_observable,
analysis),
timeout=module.timeout,
)
return request
except asyncio.TimeoutError as e:
return self.process_exception(
module,
request,
e,
f"{module.type} timed out analyzing type {request.modified_observable.type} value {request.modified_observable.value} after {module.timeout} seconds",
)
except Exception as e:
return self.process_exception(
module,
request,
e,
f"{module.type} failed analyzing type {request.modified_observable.type} value {request.modified_observable.value}: {e}",
)
def test_add_analysis_no_amt():
"""An Analysis must have a type before it can be added."""
root = RootAnalysis()
observable = root.add_observable("test", "test")
with pytest.raises(ValueError):
observable.add_analysis(Analysis())