async def test_known_dependency(system):
amt = AnalysisModuleType(ANALYSIS_TYPE_TEST, "test")
assert await system.register_analysis_module_type(amt) == amt
# amt_dep depends on amt
amt_dep = AnalysisModuleType(ANALYSIS_TYPE_TEST_DEP,
"test",
dependencies=[ANALYSIS_TYPE_TEST])
assert await system.register_analysis_module_type(amt_dep)
root = system.new_root()
test_observable = root.add_observable("test", "test")
root_request = root.create_analysis_request()
await system.process_analysis_request(root_request)
# this should have one entry
assert await system.get_queue_size(amt) == 1
# but not this one (yet) due to the dependency
assert await system.get_queue_size(amt_dep) == 0
# process the amt request
request = await system.get_next_analysis_request(OWNER_UUID, amt, 0)
request.initialize_result()
request.modified_observable.add_analysis(type=amt,
details={"Hello": "World"})
await system.process_analysis_request(request)
# now we should have a request for the dependency
assert await system.get_queue_size(amt_dep) == 1
async def test_process_existing_analysis_merge(system):
# register two different analysis modules
amt_1 = AnalysisModuleType("test_1", "")
await system.register_analysis_module_type(amt_1)
amt_2 = AnalysisModuleType("test_2", "")
await system.register_analysis_module_type(amt_2)
root = system.new_root()
test_observable = root.add_observable("test", "test")
await root.submit()
# act like these two modules are running at the same time
request_1 = await system.get_next_analysis_request(OWNER_UUID, amt_1, 0)
request_2 = await system.get_next_analysis_request(OWNER_UUID, amt_2, 0)
# process the first one
request_1.initialize_result()
request_1.modified_observable.add_tag(
"tag_1") # make a modification to the observable
analysis = request_1.modified_observable.add_analysis(type=amt_1)
await system.process_analysis_request(request_1)
# process the second one
request_2.initialize_result()
analysis = request_2.modified_observable.add_analysis(type=amt_2)
await system.process_analysis_request(request_2)
root = await system.get_root_analysis(root)
test_observable = root.get_observable(test_observable)
assert test_observable.has_tag("tag_1")
assert test_observable.get_analysis(amt_1)
assert test_observable.get_analysis(amt_2)
async def test_EVENT_AMT_MODIFIED(system):
handler = TestEventHandler()
await system.register_event_handler(EVENT_AMT_MODIFIED, handler)
amt = AnalysisModuleType("test", "")
await system.register_analysis_module_type(amt)
assert handler.event is None
handler = TestEventHandler()
await system.register_event_handler(EVENT_AMT_MODIFIED, handler)
# still not modified yet
await system.register_analysis_module_type(amt)
assert handler.event is None
# modify version
amt.version = "1.0.1"
handler = TestEventHandler()
await system.register_event_handler(EVENT_AMT_MODIFIED, handler)
# modified this time
await system.register_analysis_module_type(amt)
await handler.wait()
assert handler.event.name == EVENT_AMT_MODIFIED
assert AnalysisModuleType.from_dict(handler.event.args) == amt
async def test_amt_version_upgrade(system):
# register an analysis module for a specific version of the "intel"
amt = await system.register_analysis_module_type(
AnalysisModuleType("test", "", extended_version={"intel": "v1"}))
# (assume amt goes offline)
# add something to be analyzed
root = system.new_root()
observable = root.add_observable("test", "test")
await root.submit()
# amt comes back on, re-register
amt = await system.register_analysis_module_type(
AnalysisModuleType("test", "", extended_version={"intel": "v1"}))
request = await system.get_next_analysis_request("test", amt, 0)
request.initialize_result()
request.modified_observable.add_analysis(type=amt).add_observable(
"test", "other")
await system.process_analysis_request(request)
# amt gets upgraded from another process
amt_upgraded = await system.register_analysis_module_type(
AnalysisModuleType("test", "", extended_version={"intel": "v2"}))
# but we're still using the old one so this should fail
with pytest.raises(AnalysisModuleTypeExtendedVersionError):
request = await system.get_next_analysis_request("test", amt, 0)
# and the work queue should still have one entry
assert await system.get_queue_size(amt_upgraded) == 1
async def register_analysis_module_type(
self, amt: AnalysisModuleType) -> AnalysisModuleType:
assert isinstance(amt, AnalysisModuleType)
async with self.get_client() as client:
response = await client.post("/amt", json=amt.to_dict())
_raise_exception_on_error(response)
return AnalysisModuleType.from_dict(response.json())
async def test_multiple_amts(system):
"""Test having two different AMTs for the same observable type."""
# define two owners
owner_1 = str(uuid.uuid4())
owner_2 = str(uuid.uuid4())
# register two basic analysis modules
amt_1 = AnalysisModuleType("test_1", "", ["test"])
assert await system.register_analysis_module_type(amt_1)
amt_2 = AnalysisModuleType("test_2", "", ["test"])
assert await system.register_analysis_module_type(amt_2)
# submit an analysis request with a single observable
root = system.new_root()
observable = root.add_observable("test", "test")
await system.process_analysis_request(root.create_analysis_request())
# have both amts receive work items
request_1 = await system.get_next_analysis_request(owner_1, amt_1, 0)
assert isinstance(request_1, AnalysisRequest)
request_2 = await system.get_next_analysis_request(owner_2, amt_2, 0)
assert isinstance(request_2, AnalysisRequest)
analysis_details_1 = {"test_1": "result_1"}
analysis_details_2 = {"test_2": "result_2"}
# "analyze" them
request_1.initialize_result()
request_1.modified_observable.add_analysis(type=amt_1,
details=analysis_details_1)
# submit the result of the analysis
await system.process_analysis_request(request_1)
request_2.initialize_result()
request_2.modified_observable.add_analysis(type=amt_2,
details=analysis_details_2)
# submit the result of the analysis
await system.process_analysis_request(request_2)
# check the results
root = await system.get_root_analysis(root.uuid)
assert isinstance(root, RootAnalysis)
observable = root.get_observable(observable)
assert isinstance(observable, Observable)
analysis = observable.get_analysis(amt_1)
assert isinstance(analysis, Analysis)
assert await analysis.get_details() == analysis_details_1
analysis = observable.get_analysis(amt_2)
assert isinstance(analysis, Analysis)
assert await analysis.get_details() == analysis_details_2
def test_add_excluded_analysis():
observable = RootAnalysis().add_observable("test", "test")
assert not observable.excluded_analysis
assert not observable.is_excluded("test")
observable.exclude_analysis("test")
assert observable.excluded_analysis
assert "test" in observable.excluded_analysis
assert observable.is_excluded("test")
assert observable.is_excluded(AnalysisModuleType("test", ""))
observable.exclude_analysis(AnalysisModuleType("other", ""))
assert "other" in observable.excluded_analysis
async def test_circ_dependency(system):
amt_1 = AnalysisModuleType("test_1", "")
assert await system.register_analysis_module_type(amt_1)
# amt_2 depends on amt_1
amt_2 = AnalysisModuleType("test_2", "", dependencies=["test_1"])
assert await system.register_analysis_module_type(amt_2)
# and now amt_1 depends on amt_2
amt_1 = AnalysisModuleType("test_1", "", dependencies=["test_2"])
with pytest.raises(CircularDependencyError):
assert await system.register_analysis_module_type(amt_1)
async def test_crashing_sync_analysis_module(manager):
if manager.concurrency_mode == CONCURRENCY_MODE_THREADED:
pytest.skip(
f"cannot test in concurrency_mode {manager.concurrency_mode}")
sync = asyncio.Event()
class CustomEventHandler(EventHandler):
async def handle_event(self, event: Event):
sync.set()
async def handle_exception(self, event: str, exception: Exception):
pass
# TODO when events are distributed modify this to use that
await app.state.system.register_event_handler(
EVENT_ANALYSIS_ROOT_COMPLETED, CustomEventHandler())
amt_crashing = AnalysisModuleType("crash_test", "")
amt_ok = AnalysisModuleType("ok", "")
await manager.system.register_analysis_module_type(amt_crashing)
await manager.system.register_analysis_module_type(amt_ok)
# this is only supported in CONCURRENCY_MODE_PROCESS
crashing_module = CrashingAnalysisModule(amt_crashing)
ok_module = SimpleSyncAnalysisModule(amt_ok)
manager.add_module(crashing_module)
manager.add_module(ok_module)
root = manager.system.new_root()
observable = root.add_observable("test", "crash")
await root.submit()
await manager.run_once()
# wait for analysis to complete
assert await sync.wait()
root = await manager.system.get_root_analysis(root)
observable = root.get_observable(observable)
analysis = observable.get_analysis(amt_crashing)
assert analysis.error_message == "crash_testv1.0.0 process crashed when analyzing type test value crash"
assert analysis.stack_trace
observable = root.get_observable(observable)
analysis = observable.get_analysis(amt_ok)
async def test_self_dependency(system):
# depending on amt when amt isn' registered yet
amt_1 = AnalysisModuleType("test_1", "", dependencies=["test_1"])
with pytest.raises(AnalysisModuleTypeDependencyError):
assert await system.register_analysis_module_type(amt_1)
# define it
amt_1 = AnalysisModuleType("test_1", "")
assert await system.register_analysis_module_type(amt_1)
# redefine to depend on yourself
# cannot depend on yourself! (low self esteem error)
amt_1 = AnalysisModuleType("test_1", "", dependencies=["test_1"])
with pytest.raises(CircularDependencyError):
assert await system.register_analysis_module_type(amt_1)
async def test_chained_dependency(system):
amt_1 = AnalysisModuleType("test_1", "")
assert await system.register_analysis_module_type(amt_1)
# amt_2 depends on amt_1
amt_2 = AnalysisModuleType("test_2", "", dependencies=["test_1"])
assert await system.register_analysis_module_type(amt_2)
# amt_3 depends on amt_2
amt_3 = AnalysisModuleType("test_3", "", dependencies=["test_2"])
assert await system.register_analysis_module_type(amt_3)
root = system.new_root()
test_observable = root.add_observable("test", "test")
root_request = root.create_analysis_request()
await system.process_analysis_request(root_request)
# this should have one entry for amt_1
assert await system.get_queue_size(amt_1) == 1
# but not this one the others
assert await system.get_queue_size(amt_2) == 0
assert await system.get_queue_size(amt_3) == 0
# process the amt request
request = await system.get_next_analysis_request(OWNER_UUID, amt_1, 0)
request.initialize_result()
request.modified_observable.add_analysis(type=amt_1,
details={"Hello": "World"})
await system.process_analysis_request(request)
# now amt_2 should be ready but still not amt_3
assert await system.get_queue_size(amt_1) == 0
assert await system.get_queue_size(amt_2) == 1
assert await system.get_queue_size(amt_3) == 0
# process the amt request
request = await system.get_next_analysis_request(OWNER_UUID, amt_2, 0)
request.initialize_result()
request.modified_observable.add_analysis(type=amt_2,
details={"Hello": "World"})
await system.process_analysis_request(request)
# now amt_3 should be ready
assert await system.get_queue_size(amt_1) == 0
assert await system.get_queue_size(amt_2) == 0
assert await system.get_queue_size(amt_3) == 1
async def test_verify_registration(manager):
# registration OK
amt = AnalysisModuleType(
"test",
"",
extended_version={"yara": "6f5902ac237024bdd0c176cb93063dc4"})
assert await manager.system.register_analysis_module_type(amt)
custom_manager = AnalysisModuleManager(manager.system)
custom_manager.add_module(AnalysisModule(amt))
assert await custom_manager.verify_registration()
# missing registration
amt = AnalysisModuleType("missing", "")
custom_manager = AnalysisModuleManager(manager.system)
custom_manager.add_module(AnalysisModule(amt))
assert not await custom_manager.verify_registration()
assert not await custom_manager.run()
# version mismatch
amt = AnalysisModuleType("test", "", version="1.0.1")
custom_manager = AnalysisModuleManager(manager.system)
custom_manager.add_module(AnalysisModule(amt))
assert not await custom_manager.verify_registration()
assert not await custom_manager.run()
# extended version mismatch
amt = AnalysisModuleType(
"test",
"",
extended_version={"yara": "71bec09d78fe6abdb94244a4cc89c740"})
custom_manager = AnalysisModuleManager(manager.system)
custom_manager.add_module(AnalysisModule(amt))
assert not await custom_manager.verify_registration()
# extended version mismatch but upgrade ok
class UpgradableAnalysisModule(AnalysisModule):
async def upgrade(self):
self.type.extended_version = {
"yara": "6f5902ac237024bdd0c176cb93063dc4"
}
# starts out with the wrong set of yara rules but upgrade() fixes that
amt = AnalysisModuleType(
"test",
"",
extended_version={"yara": "71bec09d78fe6abdb94244a4cc89c740"})
custom_manager = AnalysisModuleManager(manager.system)
custom_manager.add_module(UpgradableAnalysisModule(amt))
assert await custom_manager.verify_registration()
async def test_cancel_analysis_from_result(system):
amt = AnalysisModuleType("test", "")
await system.register_analysis_module_type(amt)
root = system.new_root()
observable = root.add_observable("test", "test")
request = root.create_analysis_request()
await system.process_analysis_request(request)
# get the analysis request
request = await system.get_next_analysis_request(OWNER_UUID, amt, 0)
request.initialize_result()
analysis = request.modified_observable.add_analysis(
type=amt, details={"Hello": "World"})
new_observable = analysis.add_observable("new", "new")
request.result.cancel_analysis("test")
await system.process_analysis_request(request)
# root analysis should be cancelled
root = await system.get_root_analysis(root.uuid)
assert root.analysis_cancelled
assert root.analysis_cancelled_reason == "test"
# we should still have the analysis and the new observable we added in that result
observable = root.get_observable(observable)
analysis = observable.get_analysis(amt)
assert analysis is not None
new_observable = root.get_observable(new_observable)
assert new_observable is not None
# we should not have any new work requests since the analysis was cancelled
assert await system.get_queue_size(amt) == 0
async def test_apply_diff_merge_analysis(system):
amt = AnalysisModuleType("test", "")
original_root = system.new_root()
original_observable = original_root.add_observable("test", "test")
modified_root = original_root.copy()
modified_observable = modified_root.get_observable(original_observable)
modified_observable.add_analysis(type=amt)
target_root = original_root.copy()
target_observable = target_root.get_observable(original_observable)
assert not target_observable.analysis
target_observable.apply_diff_merge(original_observable,
modified_observable, amt)
assert target_observable.analysis
assert target_observable.get_analysis("test") is not None
# exists before but not after
original_root = system.new_root()
original_observable = original_root.add_observable("test", "test")
modified_root = original_root.copy()
modified_observable = modified_root.get_observable(original_observable)
target_root = original_root.copy()
target_observable = target_root.get_observable(original_observable)
original_observable.add_analysis(type=amt)
assert not target_observable.analysis
target_observable.apply_diff_merge(original_observable,
modified_observable)
assert not target_observable.analysis
async def test_delete_analysis_module_type_linked_results(system):
amt = AnalysisModuleType("test", "", cache_ttl=300)
await system.register_analysis_module_type(amt)
original_root = system.new_root()
test_observable = original_root.add_observable("test", "test")
original_request = original_root.create_analysis_request()
await system.process_analysis_request(original_request)
# we should have a single work entry in the work queue
assert await system.get_queue_size(amt) == 1
# make another request for the same observable but from a different root analysis
root = system.new_root()
test_observable = root.add_observable("test", "test")
root_request = root.create_analysis_request()
await system.process_analysis_request(root_request)
# there should still only be one outbound request
assert await system.get_queue_size(amt) == 1
# the first analysis request should now be linked to a new analysis request
request = await system.get_next_analysis_request("owner", amt, 0)
linked_requests = await system.get_linked_analysis_requests(request)
# analysis module type gets deleted
await system.delete_analysis_module_type(amt)
# both the original request and the linked request should be gone
assert await system.get_analysis_request_by_request_id(original_request.id
) is None
assert await system.get_analysis_request_by_request_id(request.id) is None
async def test_apply_diff_merge_analysis_with_observables(system):
amt = AnalysisModuleType("test", "")
original_root = system.new_root()
original_observable = original_root.add_observable("test", "test")
modified_root = original_root.copy()
modified_observable = modified_root.get_observable(original_observable)
analysis = modified_observable.add_analysis(type=amt)
new_observable = analysis.add_observable("new", "new")
target_root = original_root.copy()
target_observable = target_root.get_observable(original_observable)
assert not target_root.get_observable(new_observable)
target_observable.apply_diff_merge(original_observable,
modified_observable,
type=amt)
assert target_root.get_observable(new_observable) == new_observable
# exists before but not after
original_root = system.new_root()
original_observable = original_root.add_observable("test", "test")
modified_root = original_root.copy()
modified_observable = modified_root.get_observable(original_observable)
target_root = original_root.copy()
target_observable = target_root.get_observable(original_observable)
analysis = original_observable.add_analysis(type=amt)
new_observable = analysis.add_observable("new", "new")
assert not target_root.get_observable(new_observable)
target_observable.apply_diff_merge(original_observable,
modified_observable)
assert not target_root.get_observable(new_observable)
async def _upgrade():
nonlocal step_1
nonlocal root_2
await step_1.wait()
updated_amt = AnalysisModuleType("test", "", version="1.0.1")
await manager.system.register_analysis_module_type(updated_amt)
await root_2.submit()
async def test_force_stop_stuck_async_task(manager):
control = asyncio.Event()
class CustomAnalysisModule(AnalysisModule):
async def execute_analysis(self, root, observable, analysis):
nonlocal control
control.set()
# get stuck
import sys
await asyncio.sleep(sys.maxsize)
# register the type to the core
amt = AnalysisModuleType("test", "")
await manager.system.register_analysis_module_type(amt)
module = CustomAnalysisModule(amt)
manager.add_module(module)
root = manager.system.new_root()
observable = root.add_observable("test", "test")
await root.submit()
async def _cancel():
nonlocal control
nonlocal manager
await control.wait()
manager.force_stop()
cancel_task = asyncio.get_event_loop().create_task(_cancel())
await manager.run()
await cancel_task
async def test_force_stop_stuck_sync_task(manager):
# there's nothing you can do when concurrency is threaded
if manager.concurrency_mode == CONCURRENCY_MODE_THREADED:
pytest.skip(
f"cannot test in concurrency_mode {manager.concurrency_mode}")
# register the type to the core
amt = AnalysisModuleType("test", "")
await manager.system.register_analysis_module_type(amt)
module = StuckAnalysisModule(amt)
manager.add_module(module)
root = manager.system.new_root()
observable = root.add_observable("test", "test")
await root.submit()
async def _cancel():
nonlocal manager
manager.force_stop()
manager_task = asyncio.get_event_loop().create_task(manager.run())
await asyncio.wait([manager_task], timeout=0.01)
cancel_task = asyncio.get_event_loop().create_task(_cancel())
await manager_task
await cancel_task
async def test_EVENT_CACHE_HIT(system):
handler = TestEventHandler()
await system.register_event_handler(EVENT_CACHE_HIT, handler)
amt = AnalysisModuleType("test", "", cache_ttl=60)
await system.register_analysis_module_type(amt)
root = system.new_root()
observable = root.add_observable("test", "test")
root_request = root.create_analysis_request()
await system.process_analysis_request(root_request)
request = await system.get_next_analysis_request("owner", amt, 0)
request.initialize_result()
request.modified_observable.add_analysis(type=amt,
details={"test": "test"})
await system.process_analysis_request(request)
assert handler.event is None
root = system.new_root()
observable = root.add_observable("test", "test")
root_request = root.create_analysis_request()
await system.process_analysis_request(root_request)
await handler.wait()
assert handler.event.name == EVENT_CACHE_HIT
event_root = RootAnalysis.from_dict(handler.event.args[0], system)
assert event_root.uuid == root.uuid and event_root.version is not None
assert handler.event.args[1]["type"] == observable.type
assert handler.event.args[1]["value"] == observable.value
assert isinstance(AnalysisRequest.from_dict(handler.event.args[2], system),
AnalysisRequest)
def execute_analysis(self, module_type: str, request_json: str) -> str:
"""Processes the request with the analysis module."""
amt = AnalysisModuleType.from_json(module_type)
module = self.module_map[amt.name]
# run_until_complete just keeps going until it returns
# there's no way to "cancel" it if something gets stuck
# the only way out is to kill the process
# so we start a thread to monitor the timeout
def _module_timeout():
get_logger().critical(
f"analysis module {module} timed out analyzing request {request_json}"
)
if self.concurrency_mode == CONCURRENCY_MODE_PROCESS:
# and then die if we hit it
# NOTE that if we're only running threads then there's really no way out other than to log it
sys.exit(1)
get_logger().debug(f"starting timer for {module.timeout} seconds")
t = threading.Timer(module.timeout, _module_timeout)
t.start()
try:
result = self.event_loop.run_until_complete(
self.execute_analysis_async(module_type, request_json))
finally:
# if we didn't time out make sure we cancel the timer
t.cancel()
return result
def test_root_all_detection_points():
amt = AnalysisModuleType("test", "")
root = RootAnalysis()
observable = root.add_observable("test", "test")
analysis = observable.add_analysis(type=amt)
observable_2 = analysis.add_observable("test", "test2")
root.add_detection_point("test")
assert root.all_detection_points == [DetectionPoint("test")]
observable.add_detection_point("test")
assert root.all_detection_points == [
DetectionPoint("test"), DetectionPoint("test")
]
analysis.add_detection_point("test")
assert root.all_detection_points == [
DetectionPoint("test"),
DetectionPoint("test"),
DetectionPoint("test")
]
observable_2.add_detection_point("test")
assert root.all_detection_points == [
DetectionPoint("test"),
DetectionPoint("test"),
DetectionPoint("test"),
DetectionPoint("test"),
]
async def test_EVENT_CACHE_NEW(system):
handler = TestEventHandler()
await system.register_event_handler(EVENT_CACHE_NEW, handler)
amt = AnalysisModuleType(name="test", description="", cache_ttl=600)
root = system.new_root()
observable = root.add_observable("test", "test")
request = observable.create_analysis_request(amt)
request.initialize_result()
analysis = request.modified_observable.add_analysis(type=amt)
assert await system.cache_analysis_result(request) is not None
await handler.wait()
assert handler.event.name == EVENT_CACHE_NEW
assert handler.event.args[0] == generate_cache_key(observable, amt)
assert AnalysisRequest.from_dict(handler.event.args[1], system) == request
# we can potentially see duplicate cache hits
handler = TestEventHandler()
await system.register_event_handler(EVENT_CACHE_NEW, handler)
assert await system.cache_analysis_result(request) is not None
await handler.wait()
assert handler.event.name == EVENT_CACHE_NEW
assert handler.event.args[0] == generate_cache_key(observable, amt)
assert AnalysisRequest.from_dict(handler.event.args[1], system) == request
async def test_track_analysis_request_unknown_amt(system):
unknown_amt = AnalysisModuleType("other", "")
root = system.new_root()
observable = root.add_observable("test", TEST_1)
request = observable.create_analysis_request(unknown_amt)
with pytest.raises(UnknownAnalysisModuleTypeError):
await system.track_analysis_request(request)
async def test_get_next_analysis_request_expired(system):
amt = AnalysisModuleType(
name="test", description="test", version="1.0.0", timeout=0, cache_ttl=600 # immediately expire
)
await system.register_analysis_module_type(amt)
root = system.new_root()
observable = root.add_observable("test", TEST_1)
request = AnalysisRequest(system, root, observable, amt)
await system.queue_analysis_request(request)
next_ar = await system.get_next_analysis_request(TEST_OWNER, amt, 0)
assert next_ar == request
assert next_ar.status == TRACKING_STATUS_ANALYZING
assert next_ar.owner == TEST_OWNER
# this next call should trigger the move of the expired analysis request
# and since it expires right away we should see the same request again
next_ar = await system.get_next_analysis_request(TEST_OWNER, amt, 0)
assert next_ar == request
# execute this manually
await system.process_expired_analysis_requests(amt)
# should be back in the queue
request = await system.get_analysis_request_by_request_id(request.id)
assert request.status == TRACKING_STATUS_QUEUED
assert request.owner is None
# and then we should get it again
next_ar = await system.get_next_analysis_request(TEST_OWNER, amt, 0)
assert next_ar == request
async def test_all_analysis_completed(system):
amt = AnalysisModuleType(name=ANALYSIS_TYPE_TEST,
description="blah",
cache_ttl=60)
assert await system.register_analysis_module_type(amt) == amt
root = system.new_root()
test_observable = root.add_observable("test", "test")
# this is technically correct because the root has not been presented to a core yet
assert root.all_analysis_completed()
await root.submit()
# now this is False because we have an outstanding analysis request
root = await system.get_root_analysis(root)
assert not root.all_analysis_completed()
request = await system.get_next_analysis_request("test", amt, 0)
request.initialize_result()
request.modified_observable.add_analysis(type=amt,
details={"Hello": "World"})
await request.submit()
# and then finally the root is complete once all requested analysis is present
root = await system.get_root_analysis(root)
assert root.all_analysis_completed()
async def test_unknown_dependency(system):
# amt_dep depends on amt which has not registered yet
amt_dep = AnalysisModuleType(ANALYSIS_TYPE_TEST_DEP,
"test",
dependencies=[ANALYSIS_TYPE_TEST])
with pytest.raises(AnalysisModuleTypeDependencyError):
assert await system.register_analysis_module_type(amt_dep)
async def test_process_root_analysis_request(system):
amt = AnalysisModuleType(name=ANALYSIS_TYPE_TEST,
description="blah",
cache_ttl=60)
assert await system.register_analysis_module_type(amt) == amt
assert await system.get_queue_size(amt) == 0
root = system.new_root()
test_observable = root.add_observable("test", "test")
root_request = root.create_analysis_request()
await system.process_analysis_request(root_request)
# the root analysis should be tracked
assert await system.get_root_analysis(root.uuid) is not None
# the test observable should be in the queue
assert await system.get_queue_size(amt) == 1
request = await system.get_next_analysis_request(OWNER_UUID, amt, 0)
assert isinstance(request, AnalysisRequest)
assert request.observable == test_observable
assert request.type == amt
assert request.root.uuid == root.uuid and request.root.version is not None
assert request.status == TRACKING_STATUS_ANALYZING
assert request.owner == OWNER_UUID
assert request.result is None
# the original root analysis request should be deleted
assert await system.get_analysis_request(root_request.id) is None
def from_dict(value: dict,
system: "ace.system.ACESystem") -> "AnalysisRequest":
assert isinstance(value, dict)
data = AnalysisRequestModel(**value)
root = None
if isinstance(data.root, RootAnalysisModel):
root = RootAnalysis.from_dict(data.root.dict(), system=system)
observable = None
if data.observable:
observable = Observable.from_dict(data.observable.dict(), root)
observable = root.get_observable(observable)
type = None
if data.type:
type = AnalysisModuleType.from_dict(data.type.dict())
ar = AnalysisRequest(system, root, observable, type)
ar.id = data.id
# ar.dependency_analysis = json_data["dependency_analysis"]
ar.status = data.status
ar.owner = data.owner
if data.original_root:
ar.original_root = RootAnalysis.from_dict(
data.original_root.dict(), system)
if data.modified_root:
ar.modified_root = RootAnalysis.from_dict(
data.modified_root.dict(), system)
return ar
async def test_root_analysis_expiration_processing_outstanding_root_dependency(
system):
# and then in this case we have one root that depends on the analysis of another root
amt = await system.register_analysis_module_type(
AnalysisModuleType(name="test", description="", cache_ttl=300))
root = system.new_root(expires=True)
observable = root.add_observable("test", "test")
await root.submit()
assert await system.get_root_analysis(root.uuid)
root_2 = system.new_root(expires=True)
observable_2 = root_2.add_observable("test", "test")
await root_2.submit()
# at this point root_2 has piggy-backed on root
assert await system.get_root_analysis(root_2.uuid)
# complete the first root
request = await system.get_next_analysis_request("test", amt, 0)
# analysis requests have still not completed yet...
assert await system.get_root_analysis(root.uuid)
assert await system.get_root_analysis(root_2.uuid)
request.initialize_result()
request.modified_observable.add_analysis(type=amt,
details={"hello": "world"})
await request.submit()
# now they should both be gone because they were both set to expire
assert not await system.get_root_analysis(root.uuid)
assert not await system.get_root_analysis(root_2.uuid)
