def testExternalFileStoreSubmissionIsTriggeredWhenFileIsSentToFileStore(
self, add_file_mock):
if not data_store.RelationalDBReadEnabled():
self.skipTest(
"Relational filestore has to be enabled for this test.")
client_mock = action_mocks.GetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
flow_test_lib.TestFlowHelper(compatibility.GetName(transfer.GetFile),
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
add_file_mock.assert_called_once()
args = add_file_mock.call_args_list[0][0]
hash_id = list(iterkeys(args[0]))[0]
self.assertIsInstance(hash_id, rdf_objects.SHA256HashID)
self.assertEqual(
args[0][hash_id].client_path,
db.ClientPath.FromPathSpec(self.client_id.Basename(), pathspec))
self.assertNotEmpty(args[0][hash_id].blob_refs)
for blob_ref in args[0][hash_id].blob_refs:
self.assertIsInstance(blob_ref, rdf_objects.BlobReference)
def testGetFilePathCorrection(self):
"""Tests that the pathspec returned is used for the aff4path."""
client_mock = action_mocks.GetFileClientMock()
# Deliberately using the wrong casing.
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "TEST_IMG.dd"))
session_id = flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
results = flow.GRRFlow.ResultCollectionForFID(session_id)
self.assertEqual(len(results), 1)
res_pathspec = results[0].pathspec
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
# Test the AFF4 file that was created.
urn = res_pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
fd2 = open(res_pathspec.path, "rb")
fd2.seek(0, 2)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
def testGetFile(self):
"""Test that the GetFile flow works."""
client_mock = action_mocks.GetFileClientMock()
pathspec = rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(self.base_path, "test_img.dd"))
flow_test_lib.TestFlowHelper(
transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
# Test the AFF4 file that was created.
urn = pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
fd2 = open(pathspec.path, "rb")
fd2.seek(0, 2)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
def testGetFile(self):
"""Test that the GetFile flow works."""
client_mock = action_mocks.GetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
fd2 = open(pathspec.path, "rb")
cp = db.ClientPath.FromPathSpec(self.client_id, pathspec)
fd_rel_db = file_store.OpenFile(cp)
self.CompareFDs(fd2, fd_rel_db)
# Only the sha256 hash of the contents should have been calculated:
# in order to put file contents into the file store.
history = data_store.REL_DB.ReadPathInfoHistory(
cp.client_id, cp.path_type, cp.components)
self.assertEqual(history[-1].hash_entry.sha256,
fd_rel_db.hash_id.AsBytes())
self.assertIsNone(history[-1].hash_entry.sha1)
self.assertIsNone(history[-1].hash_entry.md5)
def AddFileToFileStore(pathspec=None, client_id=None, token=None):
"""Adds file with given pathspec to the hash file store."""
if pathspec is None:
raise ValueError("pathspec can't be None")
if client_id is None:
raise ValueError("client_id can't be None")
urn = pathspec.AFF4Path(client_id)
client_mock = action_mocks.GetFileClientMock()
for _ in flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=token,
client_id=client_id,
pathspec=pathspec):
pass
auth_state = rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED
events.Events.PublishEvent("FileStore.AddFileToStore",
rdf_flows.GrrMessage(payload=urn,
auth_state=auth_state),
token=token)
worker = worker_test_lib.MockWorker(token=token)
worker.Simulate()
return urn
def testGetFileIsDirectory(self):
"""Tests that the flow raises when called on directory."""
client_mock = action_mocks.GetFileClientMock()
with temp.AutoTempDirPath() as temp_dir:
pathspec = rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS, path=temp_dir)
with self.assertRaises(RuntimeError):
flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
def testGetFilePathCorrection(self):
"""Tests that the pathspec returned is used for the aff4path."""
client_mock = action_mocks.GetFileClientMock()
# Deliberately using the wrong casing.
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "TEST_IMG.dd"))
session_id = flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
results = flow_test_lib.GetFlowResults(self.client_id.Basename(),
session_id)
self.assertLen(results, 1)
res_pathspec = results[0].pathspec
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
fd2 = open(res_pathspec.path, "rb")
fd2.seek(0, 2)
if data_store.RelationalDBReadEnabled():
cp = db.ClientPath.FromPathSpec(self.client_id.Basename(),
res_pathspec)
fd_rel_db = file_store.OpenFile(cp)
self.CompareFDs(fd2, fd_rel_db)
# Only the sha256 hash of the contents should have been calculated:
# in order to put file contents into the file store.
history = data_store.REL_DB.ReadPathInfoHistory(
cp.client_id, cp.path_type, cp.components)
self.assertEqual(history[-1].hash_entry.sha256,
fd_rel_db.hash_id.AsBytes())
self.assertIsNone(history[-1].hash_entry.sha1)
self.assertIsNone(history[-1].hash_entry.md5)
else:
# Test the AFF4 file that was created.
urn = res_pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
def testStatEntryToExportedFileConverterWithFetchedAFF4File(self):
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "winexec_img.dd"))
pathspec.Append(path="/Ext2IFS_1_10b.exe",
pathtype=rdf_paths.PathSpec.PathType.TSK)
client_mock = action_mocks.GetFileClientMock()
flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
creator=self.test_username,
client_id=self.client_id,
pathspec=pathspec)
path_info = data_store.REL_DB.ReadPathInfo(
self.client_id,
rdf_objects.PathInfo.PathType.TSK,
components=tuple(pathspec.CollapsePath().lstrip("/").split("/")))
stat = path_info.stat_entry
self.assertTrue(stat)
converter = file.StatEntryToExportedFileConverter()
results = list(converter.Convert(self.metadata, stat))
self.assertLen(results, 1)
self.assertEqual(results[0].basename, "Ext2IFS_1_10b.exe")
urn = "aff4:/%s/fs/tsk%s" % (self.client_id, pathspec.CollapsePath())
self.assertEqual(results[0].urn, urn)
# Check that by default file contents are not exported
self.assertFalse(results[0].content)
self.assertFalse(results[0].content_sha256)
# Convert again, now specifying export_files_contents=True in options.
converter = file.StatEntryToExportedFileConverter(
options=base.ExportOptions(export_files_contents=True))
results = list(converter.Convert(self.metadata, stat))
self.assertTrue(results[0].content)
self.assertEqual(
results[0].content_sha256,
"69264282ca1a3d4e7f9b1f43720f719a4ea47964f0bfd1b2ba88424f1c61395d")
self.assertEqual("", results[0].metadata.annotations)
def AddFileToFileStore(pathspec=None, client_id=None, token=None):
"""Adds file with given pathspec to the hash file store."""
if pathspec is None:
raise ValueError("pathspec can't be None")
if client_id is None:
raise ValueError("client_id can't be None")
client_mock = action_mocks.GetFileClientMock()
flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=token,
client_id=client_id,
pathspec=pathspec)
urn = pathspec.AFF4Path(client_id)
events.Events.PublishEvent("FileStore.AddFileToStore", urn, token=token)
return urn
def _ReadBytesWithGetFile(self,
path,
stat_available=False,
offset=None,
file_size_override=None,
read_length=None):
if stat_available:
client_mock = action_mocks.GetFileClientMock()
else:
client_mock = action_mocks.GetFileWithFailingStatClientMock()
pathspec = rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS,
path=path,
)
if offset is not None:
pathspec.offset = offset
if file_size_override is not None:
pathspec.file_size_override = file_size_override
args = transfer.GetFileArgs(
pathspec=pathspec,
ignore_stat_failure=not stat_available,
)
if read_length is not None:
args.read_length = read_length
flow_id = flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
args=args)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(
results, 1, f"Expected 1 result for offset={offset}, "
f"file_size_override={file_size_override}, "
f"read_length={read_length}, ")
res_pathspec = results[0].pathspec
cp = db.ClientPath.FromPathSpec(self.client_id, res_pathspec)
return file_store.OpenFile(cp).Read()
def testGetFile(self):
"""Test that the GetFile flow works."""
client_mock = action_mocks.GetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
fd2 = open(pathspec.path, "rb")
if data_store.RelationalDBReadEnabled(category="filestore"):
cp = db.ClientPath.FromPathSpec(self.client_id.Basename(),
pathspec)
fd_rel_db = file_store.OpenLatestFileVersion(cp)
self.CompareFDs(fd2, fd_rel_db)
# Only the sha256 hash of the contents should have been calculated:
# in order to put file contents into the file store.
history = data_store.REL_DB.ReadPathInfoHistory(
cp.client_id, cp.path_type, cp.components)
self.assertEqual(history[-1].hash_entry.sha256,
fd_rel_db.hash_id.AsBytes())
self.assertIsNone(history[-1].hash_entry.sha1)
self.assertIsNone(history[-1].hash_entry.md5)
else:
# Test the AFF4 file that was created.
urn = pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
fd2.seek(0, 2)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
def testGetFilePathCorrection(self):
"""Tests that the pathspec returned is used for the aff4path."""
client_mock = action_mocks.GetFileClientMock()
# Deliberately using the wrong casing.
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "TEST_IMG.dd"))
expected_size = os.path.getsize(
os.path.join(self.base_path, "test_img.dd"))
session_id = flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
results = flow_test_lib.GetFlowResults(self.client_id, session_id)
self.assertLen(results, 1)
res_pathspec = results[0].pathspec
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
with open(res_pathspec.path, "rb") as fd2:
fd2.seek(0, 2)
cp = db.ClientPath.FromPathSpec(self.client_id, res_pathspec)
fd_rel_db = file_store.OpenFile(cp)
self.CompareFDs(fd2, fd_rel_db)
# Only the sha256 hash of the contents should have been calculated:
# in order to put file contents into the file store.
history = data_store.REL_DB.ReadPathInfoHistory(
cp.client_id, cp.path_type, cp.components)
self.assertEqual(history[-1].hash_entry.sha256,
fd_rel_db.hash_id.AsBytes())
self.assertEqual(history[-1].hash_entry.num_bytes, expected_size)
self.assertIsNone(history[-1].hash_entry.sha1)
self.assertIsNone(history[-1].hash_entry.md5)
def testExternalFileStoreSubmissionIsTriggeredWhenFileIsSentToFileStore(
self, add_file_mock):
client_mock = action_mocks.GetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
flow_test_lib.TestFlowHelper(compatibility.GetName(transfer.GetFile),
client_mock,
creator=self.test_username,
client_id=self.client_id,
pathspec=pathspec)
add_file_mock.assert_called_once()
args = add_file_mock.call_args_list[0][0]
hash_id = list(args[0].keys())[0]
self.assertIsInstance(hash_id, rdf_objects.SHA256HashID)
self.assertEqual(args[0][hash_id].client_path,
db.ClientPath.FromPathSpec(self.client_id, pathspec))
self.assertNotEmpty(args[0][hash_id].blob_refs)
for blob_ref in args[0][hash_id].blob_refs:
self.assertIsInstance(blob_ref, rdf_objects.BlobReference)
def testStatEntryToExportedFileConverterWithHashedAFF4File(self):
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "winexec_img.dd"))
pathspec.Append(path="/Ext2IFS_1_10b.exe",
pathtype=rdf_paths.PathSpec.PathType.TSK)
client_mock = action_mocks.GetFileClientMock()
flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
creator=self.test_username,
client_id=self.client_id,
pathspec=pathspec)
path_info = rdf_objects.PathInfo.FromPathSpec(pathspec)
path_info = data_store.REL_DB.ReadPathInfo(self.client_id,
path_info.path_type,
tuple(path_info.components))
hash_value = path_info.hash_entry
self.assertTrue(hash_value)
converter = file.StatEntryToExportedFileConverter()
results = list(
converter.Convert(self.metadata,
rdf_client_fs.StatEntry(pathspec=pathspec)))
# Even though the file has a hash, it's not stored in StatEntry and
# doesn't influence the result. Note: this is a change in behavior.
# Previously StatEntry exporter was opening corresponding file objects
# and reading hashes from these objects. This approach was questionable
# at best, since there was no guarantee that hashes actually corresponded
# to files in question.
self.assertFalse(results[0].hash_md5)
self.assertFalse(results[0].hash_sha1)
self.assertFalse(results[0].hash_sha256)