async def test_file_expiration_with_multiple_root_reference(tmpdir, system):
"""Tests that a file that expires but still has a root references are not
deleted until all root references are deleted."""
path = str(tmpdir / "test.txt")
with open(path, "w") as fp:
fp.write("Hello, world!")
# add a root with a file that expires right away
root_1 = system.new_root()
file_observable = await root_1.add_file(path, expiration_date=utc_now())
await root_1.save()
await root_1.discard()
# do it again but reference the same file
root_2 = system.new_root()
file_observable = await root_2.add_file(path, expiration_date=utc_now())
await root_2.save()
await root_2.discard()
# the content meta should reference two different roots
meta = await system.get_content_meta(file_observable.value)
assert root_1.uuid in meta.roots
assert root_2.uuid in meta.roots
# this should return 0 since it still has a valid root reference
assert len([_ async for _ in await system.iter_expired_content()]) == 0
# make sure we don't delete anything
assert await system.delete_expired_content() == 0
assert await system.get_content_meta(file_observable.value) is not None
# delete the first root
await system.delete_root_analysis(root_1)
# this should return 0 since we still have a valid root reference
assert len([_ async for _ in await system.iter_expired_content()]) == 0
# make sure we don't delete anything
assert await system.delete_expired_content() == 0
assert await system.get_content_meta(file_observable.value) is not None
# delete the second root
await system.delete_root_analysis(root_2)
# now this should return 1 since the root is gone
assert len([_ async for _ in await system.iter_expired_content()]) == 1
# and now it should clear out
assert await system.delete_expired_content() == 1
# this should return 0 since it still has a valid root reference
assert len([_ async for _ in await system.iter_expired_content()]) == 0
# and the content is gone
assert await system.get_content_meta(file_observable.value) is None
async def test_root_analysis_association(tmp_path, system):
target_path = tmp_path / "test.txt"
target_path.write_text("test")
target_path = str(target_path)
# we store the file with no initial root analysis set to expire now
sha256 = await system.save_file(target_path, expiration_date=utc_now())
assert await system.get_content_meta(sha256)
# submit a root analysis with the given file *after* we upload it
root = system.new_root()
observable = root.add_observable("file", sha256)
await root.submit()
# now attempt to delete all expired content
await system.delete_expired_content()
# we should still have the content
assert await system.get_content_meta(sha256)
# delete the root
await system.delete_root_analysis(root)
# now attempt to delete all expired content
await system.delete_expired_content()
# should be gone
assert await system.get_content_meta(sha256) is None
def test_observable_serialization():
root = RootAnalysis()
o_time = utc_now()
target = root.add_observable("test", "other")
o1 = root.add_observable(
"test",
"test",
time=o_time,
context="text context",
directives=["directive1", "directive2"],
limited_analysis=["limit1", "limit2"],
excluded_analysis=["excluded1", "excluded2"],
requested_analysis=["requested1", "requested2"],
)
o1.add_relationship("test", target)
root = RootAnalysis.from_dict(root.to_model().dict())
o2 = root.get_observable(o1)
# should be two separate instances
assert id(o1) != id(o2)
assert o1.type == o2.type
assert o1.value == o2.value
assert o1.time == o2.time
assert o1.context == o2.context
assert o1.directives == o2.directives
assert o1.limited_analysis == o2.limited_analysis
assert o1.excluded_analysis == o2.excluded_analysis
assert o1.requested_analysis == o2.requested_analysis
assert o1.relationships == o2.relationships
async def test_file_expiration_with_root_reference(tmpdir, system):
"""Tests that a file that expires but still has a root reference does not
get deleted until the root is also deleted."""
path = str(tmpdir / "test.txt")
with open(path, "w") as fp:
fp.write("Hello, world!")
root = system.new_root()
# have the file expire right away
file_observable = await root.add_file(path, expiration_date=utc_now())
await root.save()
await root.discard()
# this should return 0 since it still has a valid root reference
assert len([_ async for _ in await system.iter_expired_content()]) == 0
# make sure we don't delete anything
assert await system.delete_expired_content() == 0
assert await system.get_content_meta(file_observable.value) is not None
# delete the root
await system.delete_root_analysis(root)
# now this should return 1 since the root is gone
assert len([_ async for _ in await system.iter_expired_content()]) == 1
# and now it should clear out
assert await system.delete_expired_content() == 1
# this should return 0 since it still has a valid root reference
assert len([_ async for _ in await system.iter_expired_content()]) == 0
# and the content is gone
assert await system.get_content_meta(file_observable.value) is None
async def i_get_cached_analysis_result(
self, cache_key: str) -> Union[AnalysisRequest, None]:
async with self.get_db() as db:
result = (await db.execute(
select(AnalysisResultCache).where(
AnalysisResultCache.cache_key == cache_key)
)).one_or_none()
if result is None:
return None
result = result[0]
if result.expiration_date is not None and utc_now(
) > result.expiration_date:
return None
return AnalysisRequest.from_json(result.json_data, system=self)
async def test_file_expiration(tmpdir, system):
path = str(tmpdir / "test.txt")
with open(path, "w") as fp:
fp.write("Hello, world!")
# store the file and have it expire right away
sha256 = await system.save_file(path, expiration_date=utc_now())
assert sha256
# we should have a single expired file now
assert len([_ async for _ in await system.iter_expired_content()]) == 1
# clear them out
await system.delete_expired_content()
# now we should have no expired content
assert len([_ async for _ in await system.iter_expired_content()]) == 0
# and the file should be gone
assert await system.get_content_meta(sha256) is None
async def i_cache_analysis_result(self, cache_key: str,
request: AnalysisRequest,
expiration: Optional[int]) -> str:
expiration_date = None
# XXX using system side time
if expiration is not None:
expiration_date = utc_now() + datetime.timedelta(
seconds=expiration)
cache_result = AnalysisResultCache(
cache_key=cache_key,
expiration_date=expiration_date,
analysis_module_type=request.type.name,
json_data=request.to_json(),
)
async with self.get_db() as db:
await db.merge(cache_result)
await db.commit()
return cache_key
def test_observable_eq():
# same type, value no time
assert RootAnalysis().add_observable(
"test", "test") == RootAnalysis().add_observable("test", "test")
# same type, value same time
_now = utc_now()
assert RootAnalysis().add_observable(
"test", "test", time=_now) == RootAnalysis().add_observable("test",
"test",
time=_now)
# same type, value different time
assert RootAnalysis().add_observable(
"test", "test", time=_now) != RootAnalysis().add_observable(
"test", "test", time=_now - datetime.timedelta(seconds=1))
# same type different value
assert RootAnalysis().add_observable(
"test", "test") != RootAnalysis().add_observable("test", "different")
# different type
assert RootAnalysis().add_observable(
"test", "test") != RootAnalysis().add_observable("other", "test")
# wrong object
assert RootAnalysis().add_observable("test", "test") != object()
async def i_register_alert_system(self, name: str) -> bool:
async with self.get_redis_connection() as rc:
return await rc.hsetnx(KEY_ALERT_SYSTEMS, name,
str(utc_now())) == 1
def test_utc_now():
assert utc_now().tzinfo == pytz.utc
async def i_add_work_queue(self, name: str) -> bool:
async with self.get_redis_connection() as rc:
# this has to exist for the queue to exist
return await rc.hsetnx(KEY_WORK_QUEUES, name, str(utc_now())) == 1
def test_observable_str():
assert str(RootAnalysis().add_observable("test", "test"))
assert str(RootAnalysis().add_observable("test", "test", time=utc_now()))
async def i_delete_expired_cached_analysis_results(self):
async with self.get_db() as db:
await db.execute(
delete(AnalysisResultCache).where(
AnalysisResultCache.expiration_date < utc_now()))
await db.commit()