def process_exception( self, module: AnalysisModule, request: AnalysisRequest, e: Exception, error_message: Optional[str] = None) -> AnalysisRequest: assert isinstance(module, AnalysisModule) assert isinstance(request, AnalysisRequest) assert isinstance(e, Exception) # use existing analysis if it already exists analysis = request.modified_observable.get_analysis(module.type) if analysis is None: analysis = request.modified_observable.add_analysis( Analysis(type=module.type)) # set the error message and stack trace details if not error_message: analysis.error_message = f"{type(e).__name__}: {e}" else: analysis.error_message = error_message analysis.stack_trace = format_error_report(e) get_logger().error(error_message) return request
async def test_apply_merge_analysis_with_observables(system): amt = AnalysisModuleType("test", "") root = system.new_root() observable = root.add_observable("some_type", "some_value") target_root = system.new_root() target_observable = target_root.add_observable("some_type", "some_value") analysis = target_observable.add_analysis(Analysis(type=amt)) extra_observable = analysis.add_observable("other_type", "other_value") assert not root.get_observable(extra_observable) observable.apply_merge(target_observable) assert root.get_observable(extra_observable) == extra_observable
async def test_analysis_completed(system): await system.register_analysis_module_type( amt := AnalysisModuleType("test", "test", ["test"])) root = system.new_root() observable = root.add_observable("test", "test") assert not root.analysis_completed(observable, amt) observable.add_analysis(Analysis(type=amt, details=TEST_DETAILS)) assert root.analysis_completed(observable, amt) # unknown observable with pytest.raises(UnknownObservableError): root.analysis_completed(RootAnalysis().add_observable("test", "blah"), amt)
def test_add_analysis(): amt = AnalysisModuleType("test", "") root = RootAnalysis() observable = root.add_observable("test", "test") # test adding an Analysis object analysis = Analysis(details={"hello": "world"}, type=amt) result = observable.add_analysis(analysis) assert result == analysis # test adding just a details, type root = RootAnalysis() observable = root.add_observable("test", "test") analysis = observable.add_analysis(details={"hello": "world"}, type=amt) assert result == analysis
def test_apply_merge_analysis(): amt = AnalysisModuleType("test", "") root = RootAnalysis() observable = root.add_observable("some_type", "some_value") target_root = RootAnalysis() target_observable = target_root.add_observable("some_type", "some_value") target_observable.add_analysis(Analysis(type=amt, details={"test": "test"})) assert not observable.analysis observable.apply_merge(target_observable) assert observable.analysis assert observable.get_analysis("test") is not None assert observable.get_analysis("test")._details == {"test": "test"}
async def execute_analysis_async(self, module_type: str, request_json: str) -> str: request = AnalysisRequest.from_json(request_json, self.system) amt = AnalysisModuleType.from_json(module_type) module = self.module_map[amt.name] if not module.type.extended_version_matches(amt): await module.upgrade() if not module.type.extended_version_matches(amt): raise AnalysisModuleTypeExtendedVersionError(amt, module.type) analysis = request.modified_observable.add_analysis( Analysis(type=module.type, details={})) await module.execute_analysis(request.modified_root, request.modified_observable, analysis) return request.to_json()
async def test_apply_merge_analysis_with_existing_observables(system): amt = AnalysisModuleType("test", "") root = system.new_root() observable = root.add_observable("some_type", "some_value") existing_extra_observable = root.add_observable("other_type", "other_value") target_root = system.new_root() target_observable = target_root.add_observable("some_type", "some_value") analysis = target_observable.add_analysis(Analysis(type=amt)) extra_observable = analysis.add_observable("other_type", "other_value") # should only have the root as the parent assert len(existing_extra_observable.parents) == 1 observable.apply_merge(target_observable) # should now have both the root and the new analysis as parents assert len(existing_extra_observable.parents) == 2
async def test_apply_merge_error_analysis(system): amt = AnalysisModuleType("test", "") root = system.new_root() observable = root.add_observable("some_type", "some_value") target_root = system.new_root() target_observable = target_root.add_observable("some_type", "some_value") target_observable.add_analysis( Analysis(type=amt, error_message="test", stack_trace="test")) assert not observable.analysis observable.apply_merge(target_observable) assert observable.analysis assert observable.get_analysis("test") is not None assert observable.get_analysis("test")._details is None assert observable.get_analysis("test").error_message == "test" assert observable.get_analysis("test").stack_trace == "test"
async def test_analysis_details_deleted_with_root(system): # any details associated to a root are deleted when the root is deleted await system.register_analysis_module_type( amt := AnalysisModuleType("test", "")) root = system.new_root(details=TEST_DETAILS) observable = root.add_observable("test", "test") observable.add_analysis( analysis := Analysis(root=root, type=amt, details=TEST_DETAILS)) await root.save() # make sure the details are there assert await system.get_analysis_details(root.uuid) == TEST_DETAILS assert await system.get_analysis_details(analysis.uuid) == TEST_DETAILS # delete the root assert await system.delete_root_analysis(root.uuid) assert await system.get_root_analysis(root) is None # root details should be gone assert await system.get_analysis_details(root.uuid) is None # and analysis details should be gone assert await system.get_analysis_details(analysis.uuid) is None
async def test_delete_analysis_module_type_while_processing_request(system): amt = await system.register_analysis_module_type( AnalysisModuleType("test", "")) root = system.new_root() observable = root.add_observable("test", "test") await root.submit() # go get the request for processing request = await system.get_next_analysis_request("owner", amt, 0) # delete the amt while we're processing the request await system.delete_analysis_module_type(amt) request.initialize_result() request.modified_observable.add_analysis( Analysis(type=amt, details={"test": "test"})) # should fail since the amt has been removed with pytest.raises(UnknownAnalysisRequestError): await request.submit() # and the request should already be deleted as well assert await system.get_analysis_request_by_request_id(request.id) is None
async def execute_module(self, module: AnalysisModule, whoami: str, request: AnalysisRequest) -> AnalysisRequest: """Processes the request with the analysis module. Returns a copy of the original request with the results added""" assert isinstance(module, AnalysisModule) assert isinstance(whoami, str) and whoami assert isinstance(request, AnalysisRequest) request.initialize_result() # if this module is going to be analyzing a file observable then we # want to go ahead and load the content if request.modified_observable.type == "file": if not await request.modified_observable.load(): # if we can't load the file we don't bother asking the module to analyze it get_logger().warning( f"unable to load file {request.modified_observable} for {request}" ) request.modified_observable.add_analysis( Analysis(type=module.type, details={}, error_message="unknown file")) return request if module.is_multi_process: try: request_json = request.to_json() request_result_json = await asyncio.get_event_loop( ).run_in_executor(self.executor, _cpu_task_executor_execute_analysis, module.type.to_json(), request_json) return AnalysisRequest.from_json(request_result_json, self.system) except BrokenProcessPool as e: # when this happens you have to create and start a new one self.process_exception( module, request, e, error_message= f"{module.type} process crashed when analyzing type {request.modified_observable.type} value {request.modified_observable.value}", ) # we have to start a new executor self.start_executor() return request except Exception as e: return self.process_exception( module, request, e, f"{module.type} failed analyzing type {request.modified_observable.type} value {request.modified_observable.value}: {e}", ) else: try: analysis = request.modified_observable.add_analysis( Analysis(type=module.type, details={})) await asyncio.wait_for( module.execute_analysis(request.modified_root, request.modified_observable, analysis), timeout=module.timeout, ) return request except asyncio.TimeoutError as e: return self.process_exception( module, request, e, f"{module.type} timed out analyzing type {request.modified_observable.type} value {request.modified_observable.value} after {module.timeout} seconds", ) except Exception as e: return self.process_exception( module, request, e, f"{module.type} failed analyzing type {request.modified_observable.type} value {request.modified_observable.value}: {e}", )
def test_add_analysis_no_amt(): """An Analysis must have a type before it can be added.""" root = RootAnalysis() observable = root.add_observable("test", "test") with pytest.raises(ValueError): observable.add_analysis(Analysis())