def _RunKBI(self, **kw):
session_id = flow_test_lib.TestFlowHelper(
artifact.KnowledgeBaseInitializationFlow.__name__,
self.client_mock,
client_id=test_lib.TEST_CLIENT_ID,
token=self.token,
**kw)
results = flow_test_lib.GetFlowResults(test_lib.TEST_CLIENT_ID,
session_id)
self.assertLen(results, 1)
return results[0]
def testFindsNothingIfRegexMatchesNothing(self):
value_regex_match = rdf_file_finder.FileFinderContentsRegexMatchCondition(
bytes_before=10, bytes_after=10, regex=b".*CanNotFindMe.*")
client_id = self.SetupClient(0)
session_id = self.RunFlow(client_id, [self.runkey], [
registry.RegistryFinderCondition(
condition_type=registry.RegistryFinderCondition.Type.
VALUE_REGEX_MATCH,
value_regex_match=value_regex_match)
])
self.assertFalse(flow_test_lib.GetFlowResults(client_id, session_id))
def RunFlow(self, client_id, artifact_list=None, use_tsk=False):
if artifact_list is None:
artifact_list = ["WindowsRunKeys"]
session_id = flow_test_lib.TestFlowHelper(
collectors.ArtifactFilesDownloaderFlow.__name__,
client_id=client_id,
artifact_list=artifact_list,
use_tsk=use_tsk,
token=self.token)
return flow_test_lib.GetFlowResults(client_id, session_id)
def _RunCFF(self, paths, action):
session_id = flow_test_lib.TestFlowHelper(
file_finder.ClientFileFinder.__name__,
action_mocks.ClientFileFinderClientMock(),
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
action=rdf_file_finder.FileFinderAction(action_type=action),
process_non_regular_files=True,
token=self.token)
return flow_test_lib.GetFlowResults(self.client_id, session_id)
def testFindsNothingIfNothingMatchesLiteralMatchCondition(self):
vlm = rdf_file_finder.FileFinderContentsLiteralMatchCondition(
bytes_before=10, bytes_after=10, literal=b"CanNotFindMe")
client_id = self.SetupClient(0)
session_id = self.RunFlow(client_id, [self.runkey], [
registry.RegistryFinderCondition(
condition_type=registry.RegistryFinderCondition.Type.
VALUE_LITERAL_MATCH,
value_literal_match=vlm)
])
self.assertFalse(flow_test_lib.GetFlowResults(client_id, session_id))
def RunFlow(self, paths=None, conditions=None, action=None):
self.last_session_id = flow_test_lib.TestFlowHelper(
file_finder.FileFinder.__name__,
self.client_mock,
client_id=self.client_id,
paths=paths or [self.path],
pathtype=rdf_paths.PathSpec.PathType.OS,
action=action,
conditions=conditions,
token=self.token)
return flow_test_lib.GetFlowResults(self.client_id,
self.last_session_id)
def testNestedFlowsHaveTheirResultsSaved(self):
# Run the flow in the simulated way
parent_flow_id = flow_test_lib.StartAndRunFlow(
ParentFlow, client_mock=ClientMock(), client_id=self.client_id)
child_flows = data_store.REL_DB.ReadChildFlowObjects(
self.client_id, parent_flow_id)
self.assertLen(child_flows, 1)
child_flow_results = flow_test_lib.GetFlowResults(
self.client_id, child_flows[0].flow_id)
self.assertNotEmpty(child_flow_results)
def testDumpFlash(self):
"""Dump Flash Image."""
client_mock = DumpFlashImageMock()
flow_id = flow_test_lib.TestFlowHelper(
hardware.DumpFlashImage.__name__,
client_mock,
client_id=self.client_id,
creator=self.test_username)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
def _RunAndCheck(self, chunk_size, download_length):
with utils.Stubber(constants, "CLIENT_MAX_BUFFER_SIZE", chunk_size):
flow_id = flow_test_lib.TestFlowHelper(transfer.GetMBR.__name__,
ClientMock(self.mbr),
token=self.token,
client_id=self.client_id,
length=download_length)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(results[0], self.mbr[:download_length])
def testUseExternalStores(self):
paths = [os.path.join(self.base_path, "test.plist")]
action = rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.DOWNLOAD)
action.download.use_external_stores = False
with mock.patch.object(file_store.EXTERNAL_FILE_STORE, "AddFiles") as efs:
flow_id = flow_test_lib.TestFlowHelper(
compatibility.GetName(file_finder.ClientFileFinder),
action_mocks.ClientFileFinderClientMock(),
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
action=action,
process_non_regular_files=True,
creator=self.test_username)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(efs.call_count, 0)
action.download.use_external_stores = True
with mock.patch.object(file_store.EXTERNAL_FILE_STORE, "AddFiles") as efs:
flow_id = flow_test_lib.TestFlowHelper(
compatibility.GetName(file_finder.ClientFileFinder),
action_mocks.ClientFileFinderClientMock(),
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
action=action,
process_non_regular_files=True,
creator=self.test_username)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(efs.call_count, 1)
def _RunClientActionArtifact(self, client_mock, artifact_list):
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
client.Set(client.Schema.SYSTEM("Linux"))
client.Flush()
self.output_count += 1
session_id = flow_test_lib.TestFlowHelper(
aff4_flows.ArtifactCollectorFlow.__name__,
client_mock,
artifact_list=artifact_list,
token=self.token,
client_id=self.client_id)
return flow_test_lib.GetFlowResults(self.client_id, session_id)
def testCollectSingleFileReturnsFile(self):
flow_id = flow_test_lib.TestFlowHelper(file.CollectSingleFile.__name__,
self.client_mock,
client_id=self.client_id,
path=self.files["bar"].path,
token=self.token)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(results[0].stat.pathspec.path, self.files["bar"].path)
self.assertEqual(results[0].stat.pathspec.pathtype,
rdf_paths.PathSpec.PathType.OS)
self.assertEqual(str(results[0].hash.sha1), self.files["bar"].sha1)
def testPlistValueFilter(self):
client_id = self.SetupClient(0)
session_id = self._RunFlow(client_id,
filetypes.PlistValueFilter.__name__,
context="",
query="")
results = flow_test_lib.GetFlowResults(client_id, session_id)
self.assertLen(results, 1)
result_dict = results[0].dict
self.assertEqual(result_dict["nested1"]["nested11"]["key112"],
"value112")
def _RunCollectBrowserHistory(self, **kwargs):
flow_args = webhistory.CollectBrowserHistoryArgs(**kwargs)
flow_id = flow_test_lib.StartAndRunFlow(
webhistory.CollectBrowserHistory,
creator=self.token.username,
client_mock=action_mocks.ActionMock(),
client_id=self.client_id,
flow_args=flow_args)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id)
return flow_id, results, progress
def testDiskVolumeInfoOSXLinux(self):
client_mock = action_mocks.UnixVolumeClientMock()
session_id = flow_test_lib.TestFlowHelper(
compatibility.GetName(filesystem.DiskVolumeInfo),
client_mock,
client_id=self.client_id,
token=self.token,
path_list=["/usr/local", "/home"])
results = flow_test_lib.GetFlowResults(self.client_id, session_id)
self.assertCountEqual([x.unixvolume.mount_point for x in results],
["/", "/usr"])
def testFindsKeysWithSingleGlobWithoutConditions(self):
client_id = self.SetupClient(0)
session_id = self.RunFlow(client_id, [
"HKEY_USERS/S-1-5-20/Software/Microsoft/"
"Windows/CurrentVersion/Run/*"
])
results = flow_test_lib.GetFlowResults(client_id, session_id)
self.assertLen(results, 2)
# We expect Sidebar and MctAdmin keys here (see
# test_data/client_fixture.py).
basenames = [os.path.basename(r.stat_entry.pathspec.path) for r in results]
self.assertCountEqual(basenames, ["Sidebar", "MctAdmin"])
def testFindsNothingIfModiciationTimeConditionMatchesNothing(self):
modification_time = rdf_file_finder.FileFinderModificationTimeCondition(
min_last_modified_time=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0),
max_last_modified_time=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1))
client_id = self.SetupClient(0)
session_id = self.RunFlow(client_id, [self.runkey], [
registry.RegistryFinderCondition(
condition_type=registry.RegistryFinderCondition.Type
.MODIFICATION_TIME,
modification_time=modification_time)
])
self.assertFalse(flow_test_lib.GetFlowResults(client_id, session_id))
def testFailuresAreLogged(self):
client_id = "C.4815162342abcdef"
now = rdfvalue.RDFDatetime.Now()
data_store.REL_DB.WriteClientMetadata(client_id=client_id,
last_ping=now)
snapshot = rdf_objects.ClientSnapshot(client_id=client_id)
snapshot.knowledge_base.os = "fakeos"
data_store.REL_DB.WriteClientSnapshot(snapshot)
raising_artifact_source = rdf_artifacts.ArtifactSource(
type=rdf_artifacts.ArtifactSource.SourceType.COMMAND,
attributes={
"cmd": "/bin/echo",
"args": ["1"],
})
raising_artifact = rdf_artifacts.Artifact(
name="RaisingArtifact",
doc="Lorem ipsum.",
sources=[raising_artifact_source])
registry = artifact_registry.ArtifactRegistry()
with mock.patch.object(artifact_registry, "REGISTRY", registry):
registry.RegisterArtifact(raising_artifact)
flow_id = flow_test_lib.TestFlowHelper(
collectors.ArtifactCollectorFlow.__name__,
client_mock=action_mocks.ActionMock(standard.ExecuteCommand),
client_id=client_id,
artifact_list=["RaisingArtifact"],
apply_parsers=True,
check_flow_errors=True,
token=self.token)
results = flow_test_lib.GetFlowResults(client_id=client_id,
flow_id=flow_id)
self.assertLen(results, 1)
self.assertEqual(results[0].stdout, "1\n".encode("utf-8"))
logs = data_store.REL_DB.ReadFlowLogEntries(client_id=client_id,
flow_id=flow_id,
offset=0,
count=1024)
# Log should contain two entries. First one about successful execution of
# the command (not interesting), the other one containing the error about
# unsuccessful parsing.
self.assertLen(logs, 2)
self.assertIn("It was bound to happen.", logs[1].message)
def testScanAndDump(self):
client_mock = action_mocks.MultiGetFileClientMock(
memory_actions.YaraProcessScan, memory_actions.YaraProcessDump,
tempfiles.DeleteGRRTempFiles)
procs = [p for p in self.procs if p.pid in [102, 103]]
with mock.patch.object(file_store.EXTERNAL_FILE_STORE,
"AddFiles") as efs:
with utils.MultiStubber(
(psutil, "process_iter", lambda: procs),
(psutil, "Process", functools.partial(self.process, procs)),
(client_utils, "OpenProcessForMemoryAccess",
lambda pid: FakeMemoryProcess(pid=pid, tmp_dir=self._tmp_dir)
)):
session_id = flow_test_lib.TestFlowHelper(
memory.YaraProcessScan.__name__,
client_mock,
yara_signature=_TEST_YARA_SIGNATURE,
client_id=self.client_id,
creator=self.test_username,
include_errors_in_results="ALL_ERRORS",
include_misses_in_results=True,
dump_process_on_match=True)
# Process dumps are not pushed to external file stores.
self.assertEqual(efs.call_count, 0)
results = flow_test_lib.GetFlowResults(self.client_id, session_id)
# 1. Scan result match.
# 2. Scan result miss.
# 3. ProcDump response.
# 4. Stat entry for the dumped file.
self.assertLen(results, 4)
self.assertIsInstance(results[0], rdf_memory.YaraProcessScanMatch)
self.assertIsInstance(results[1], rdf_memory.YaraProcessScanMiss)
self.assertIsInstance(results[2], rdf_memory.YaraProcessDumpResponse)
self.assertIsInstance(results[3], rdf_client_fs.StatEntry)
self.assertLen(results[2].dumped_processes, 1)
self.assertEqual(results[0].process.pid,
results[2].dumped_processes[0].process.pid)
self.assertEmpty(results[2].dumped_processes[0].dump_files)
self.assertLen(results[2].dumped_processes[0].memory_regions, 1)
# TODO: Fix PathSpec.__eq__, then compare PathSpecs here.
self.assertEqual(
results[2].dumped_processes[0].memory_regions[0].file.CollapsePath(
), results[3].pathspec.CollapsePath())
def testCollectRunKeyBinaries(self):
"""Read Run key from the client_fixtures to test parsing and storage."""
client_id = self.SetupClient(0, system="Windows", os_version="6.2")
with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
vfs_test_lib.FakeFullVFSHandler):
client_mock = action_mocks.ActionMock(
file_fingerprint.FingerprintFile,
searching.Find,
standard.GetFileStat,
)
# Get KB initialized
session_id = flow_test_lib.TestFlowHelper(
artifact.KnowledgeBaseInitializationFlow.__name__,
client_mock,
client_id=client_id,
token=self.token)
kb = flow_test_lib.GetFlowResults(client_id, session_id)[0]
if data_store.RelationalDBEnabled():
client = data_store.REL_DB.ReadClientSnapshot(
client_id.Basename())
client.knowledge_base = kb
data_store.REL_DB.WriteClientSnapshot(client)
else:
with aff4.FACTORY.Open(client_id, token=self.token,
mode="rw") as client:
client.Set(client.Schema.KNOWLEDGE_BASE, kb)
if data_store.RelationalDBEnabled():
flow_cls = transfer.MultiGetFile
else:
flow_cls = aff4_flows.MultiGetFile
with test_lib.Instrument(flow_cls, "Start") as getfile_instrument:
# Run the flow in the emulated way.
flow_test_lib.TestFlowHelper(
registry.CollectRunKeyBinaries.__name__,
client_mock,
client_id=client_id,
token=self.token)
# Check MultiGetFile got called for our runkey file
download_requested = False
for pathspec in getfile_instrument.args[0][0].args.pathspecs:
if pathspec.path == u"C:\\Windows\\TEMP\\A.exe":
download_requested = True
self.assertTrue(download_requested)
def _RunRegistryFinder(self, paths=None):
client_mock = action_mocks.GlobClientMock()
client_id = self.SetupClient(0)
session_id = flow_test_lib.TestFlowHelper(
flow_registry.RegistryFinder.__name__,
client_mock,
client_id=client_id,
keys_paths=paths,
conditions=[],
creator=self.test_username)
return flow_test_lib.GetFlowResults(client_id, session_id)
def RunCollectorAndGetCollection(self, artifact_list, client_mock=None, **kw):
"""Helper to handle running the collector flow."""
if client_mock is None:
client_mock = self.MockClient(client_id=self.client_id)
session_id = flow_test_lib.TestFlowHelper(
collectors.ArtifactCollectorFlow.__name__,
client_mock=client_mock,
client_id=self.client_id,
artifact_list=artifact_list,
token=self.token,
**kw)
return flow_test_lib.GetFlowResults(self.client_id, session_id)
def RunFlow(self,
client_id,
artifact_list=None,
use_raw_filesystem_access=False):
if artifact_list is None:
artifact_list = ["WindowsRunKeys"]
session_id = flow_test_lib.TestFlowHelper(
collectors.ArtifactFilesDownloaderFlow.__name__,
client_id=client_id,
artifact_list=artifact_list,
use_raw_filesystem_access=use_raw_filesystem_access,
creator=self.test_username)
return flow_test_lib.GetFlowResults(client_id, session_id)
def testNetstatFilter(self):
client_id = self.SetupClient(0)
session_id = flow_test_lib.TestFlowHelper(network.Netstat.__name__,
ClientMock(),
client_id=client_id,
listening_only=True,
token=self.token)
# Check the results are correct.
conns = flow_test_lib.GetFlowResults(client_id, session_id)
self.assertLen(conns, 1)
self.assertEqual(conns[0].local_address.ip, "192.168.1.1")
self.assertEqual(conns[0].pid, 1)
self.assertEqual(conns[0].remote_address.port, 6667)
self.assertEqual(conns[0].state, "LISTEN")
def testGetMBR(self):
"""Test that the GetMBR flow works."""
flow_id = flow_test_lib.TestFlowHelper(transfer.GetMBR.__name__,
ClientMock(self.mbr),
token=self.token,
client_id=self.client_id)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(results[0], self.mbr)
if data_store.AFF4Enabled():
fd = aff4.FACTORY.Open(self.client_id.Add("mbr"), token=self.token)
self.assertEqual(fd.Read(4096), self.mbr)
def testFingerprintPresence(self):
client_id = self.SetupClient(0)
path = os.path.join(self.base_path, "winexec_img.dd")
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=path)
pathspec.Append(path="/winpmem-amd64.sys",
pathtype=rdf_paths.PathSpec.PathType.TSK)
client_mock = action_mocks.ActionMock(file_fingerprint.FingerprintFile)
session_id = flow_test_lib.TestFlowHelper(
flows_fingerprint.FingerprintFile.__name__,
client_mock,
creator=self.test_username,
client_id=client_id,
pathspec=pathspec)
results = flow_test_lib.GetFlowResults(client_id, session_id)
self.assertLen(results, 1)
for reply in results:
self.assertIsInstance(reply,
flows_fingerprint.FingerprintFileResult)
self.assertTrue(
str(reply.file_urn).endswith(
"test_data/winexec_img.dd/winpmem-amd64.sys"))
self.assertEqual(
str(reply.hash_entry.sha256),
"40ac571d6d85d669a9a19d498d9f926525481430056ff65746f"
"baf36bee8855f")
self.assertEqual(str(reply.hash_entry.sha1),
"6e17df1a1020a152f2bf4445d1004b192ae8e42d")
self.assertEqual(str(reply.hash_entry.md5),
"12be1109aa3d3b46c9398972af2008e1")
path_info = rdf_objects.PathInfo.FromPathSpec(pathspec)
path_info = data_store.REL_DB.ReadPathInfo(client_id,
path_info.path_type,
components=tuple(
path_info.components))
hash_obj = path_info.hash_entry
self.assertEqual(hash_obj.pecoff_sha1,
"1f32fa4eedfba023653c094143d90999f6b9bc4f")
self.assertEqual(hash_obj.signed_data[0].revision, 512)
def testFetchIsRetriedWithTskOnWindows(self):
with mock.patch.object(flow_base.FlowBase, "client_os", "Windows"):
with _PatchVfs():
flow_id = flow_test_lib.TestFlowHelper(
file.CollectSingleFile.__name__,
self.client_mock,
client_id=self.client_id,
path=self.files["bar"].path,
token=self.token)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(results[0].stat.pathspec.path, self.files["bar"].path)
self.assertEqual(results[0].stat.pathspec.pathtype,
rdf_paths.PathSpec.PathType.TSK)
self.assertEqual(str(results[0].hash.sha1), self.files["bar"].sha1)
def testNetstat(self):
"""Test that the Netstat flow works."""
client_id = self.SetupClient(0)
session_id = flow_test_lib.TestFlowHelper(network.Netstat.__name__,
ClientMock(),
client_id=client_id,
token=self.token)
# Check the results are correct.
conns = flow_test_lib.GetFlowResults(client_id, session_id)
self.assertLen(conns, 2)
self.assertEqual(conns[0].local_address.ip, "0.0.0.0")
self.assertEqual(conns[0].local_address.port, 22)
self.assertEqual(conns[1].local_address.ip, "192.168.1.1")
self.assertEqual(conns[1].pid, 1)
self.assertEqual(conns[1].remote_address.port, 6667)
def testSendReplyWorksCorrectlyInIncrementalCallback(self):
flow_id = flow_test_lib.StartAndRunFlow(
FlowWithIncrementalCallback,
client_mock=NotSendingStatusClientMock(),
client_id=self.client_id,
# Set check_flow_errors to False, otherwise test runner will complain
# that the flow has finished in the RUNNING state.
check_flow_errors=False)
flow_obj = flow_test_lib.GetFlowObj(self.client_id, flow_id)
self.assertEqual(flow_obj.flow_state, flow_obj.FlowState.RUNNING)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertListEqual(results, [
rdfvalue.RDFString(f"Hello World {i}") for i in range(
NotSendingStatusClientMock.NUM_INCREMENTAL_RESPONSES)
])
def testGetFilePathCorrection(self):
"""Tests that the pathspec returned is used for the aff4path."""
client_mock = action_mocks.GetFileClientMock()
# Deliberately using the wrong casing.
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "TEST_IMG.dd"))
session_id = flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
results = flow_test_lib.GetFlowResults(self.client_id.Basename(),
session_id)
self.assertLen(results, 1)
res_pathspec = results[0].pathspec
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
fd2 = open(res_pathspec.path, "rb")
fd2.seek(0, 2)
if data_store.RelationalDBReadEnabled():
cp = db.ClientPath.FromPathSpec(self.client_id.Basename(),
res_pathspec)
fd_rel_db = file_store.OpenFile(cp)
self.CompareFDs(fd2, fd_rel_db)
# Only the sha256 hash of the contents should have been calculated:
# in order to put file contents into the file store.
history = data_store.REL_DB.ReadPathInfoHistory(
cp.client_id, cp.path_type, cp.components)
self.assertEqual(history[-1].hash_entry.sha256,
fd_rel_db.hash_id.AsBytes())
self.assertIsNone(history[-1].hash_entry.sha1)
self.assertIsNone(history[-1].hash_entry.md5)
else:
# Test the AFF4 file that was created.
urn = res_pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
