def FetchBinaries(self, responses):
"""Parses the Rekall response and initiates FileFinder flows."""
if not responses.success:
self.Log("Error fetching VAD data: %s", responses.status)
return
self.Log("Found %d binaries", len(responses))
if self.args.filename_regex:
binaries = []
for response in responses:
if self.args.filename_regex.Match(response.CollapsePath()):
binaries.append(response)
self.Log("Applied filename regex. Have %d files after filtering.",
len(binaries))
else:
binaries = responses
if self.args.fetch_binaries:
self.CallFlow(
"FileFinder",
next_state="HandleDownloadedFiles",
paths=[
rdfvalue.GlobExpression(b.CollapsePath()) for b in binaries
],
pathtype=rdfvalue.PathSpec.PathType.OS,
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD))
else:
for b in binaries:
self.SendReply(b)
def StartRequests(self):
"""Generate and send the Find requests."""
client = aff4.FACTORY.Open(self.client_id, token=self.token)
if self.runner.output is not None:
self.runner.output.Set(
self.runner.output.Schema.DESCRIPTION("CacheGrep for {0}".format(
self.args.data_regex)))
usernames = ["%s\\%s" % (u.domain, u.username) for u in self.state.users]
usernames = [u.lstrip("\\") for u in usernames] # Strip \\ if no domain.
condition = rdfvalue.FileFinderCondition(
condition_type=rdfvalue.FileFinderCondition.Type.CONTENTS_REGEX_MATCH,
contents_regex_match=rdfvalue.FileFinderContentsRegexMatchCondition(
regex=self.args.data_regex,
mode=rdfvalue.FileFinderContentsRegexMatchCondition.Mode.FIRST_HIT))
for path in self.state.all_paths:
full_paths = flow_utils.InterpolatePath(path, client, users=usernames)
for full_path in full_paths:
self.CallFlow(
"FileFinder",
paths=[os.path.join(full_path, "**5")],
pathtype=self.state.args.pathtype,
conditions=[condition],
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD),
next_state="HandleResults")
def Start(self):
"""Determine the Chrome directory."""
self.state.Register("hist_count", 0)
# List of paths where history files are located
self.state.Register("history_paths", [])
if self.state.args.history_path:
self.state.history_paths.append(self.state.args.history_path)
if self.runner.output is not None:
self.runner.output = aff4.FACTORY.Create(
self.runner.output.urn, "VFSAnalysisFile", token=self.token)
if not self.state.history_paths:
self.state.history_paths = self.GuessHistoryPaths(
self.state.args.username)
if not self.state.history_paths:
raise flow.FlowError("Could not find valid History paths.")
filenames = ["History"]
if self.state.args.get_archive:
filenames.append("Archived History")
for path in self.state.history_paths:
for fname in filenames:
self.CallFlow(
"FileFinder",
paths=[os.path.join(path, fname)],
pathtype=self.state.args.pathtype,
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD),
next_state="ParseFiles")
def Grep(self, collector, pathtype):
"""Grep files in path_list for any matches to content_regex_list."""
path_list = self.InterpolateList(collector.args.get("path_list", []))
content_regex_list = self.InterpolateList(
collector.args.get("content_regex_list", []))
filters = []
for regex in content_regex_list:
regexfilter = rdfvalue.FileFinderContentsRegexMatchFilter(
regex=regex, bytes_before=0, bytes_after=0)
file_finder_filter = rdfvalue.FileFinderFilter(
filter_type=rdfvalue.FileFinderFilter.Type.
CONTENTS_REGEX_MATCH,
contents_regex_match=regexfilter)
filters.append(file_finder_filter)
self.CallFlow("FileFinder",
paths=path_list,
filters=filters,
action=rdfvalue.FileFinderAction(),
pathtype=pathtype,
request_data={
"artifact_name": self.current_artifact_name,
"collector": collector.ToPrimitiveDict()
},
next_state="ProcessCollected")
def Start(self):
"""Determine the Firefox history directory."""
self.state.Register("hist_count", 0)
self.state.Register("history_paths", [])
if self.args.history_path:
self.state.history_paths.append(self.args.history_path)
else:
self.state.history_paths = self.GuessHistoryPaths(self.args.username)
if not self.state.history_paths:
raise flow.FlowError("Could not find valid History paths.")
if self.runner.output is not None:
self.runner.output = aff4.FACTORY.Create(
self.runner.output.urn, "VFSAnalysisFile", token=self.token)
filename = "places.sqlite"
for path in self.state.history_paths:
self.CallFlow(
"FileFinder",
paths=[os.path.join(path, "**2", filename)],
pathtype=self.state.args.pathtype,
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD),
next_state="ParseFiles")
def testDownloadActionSizeLimit(self):
expected_files = ["dpkg.log", "dpkg_false.log"]
non_expected_files = ["auth.log"]
sizes = [
os.stat(os.path.join(self.base_path, f)).st_size
for f in expected_files
]
action = rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD)
action.download.max_size = max(sizes) + 1
for _ in test_lib.TestFlowHelper(
"FileFinder",
self.client_mock,
client_id=self.client_id,
paths=[self.path],
pathtype=rdfvalue.PathSpec.PathType.OS,
action=action,
token=self.token,
output=self.output_path):
pass
self.CheckFilesDownloaded(expected_files)
self.CheckFilesNotDownloaded(non_expected_files)
# Even though the file is too big to download, we still want the
# hash.
self.CheckFilesHashed(non_expected_files)
def Grep(self, source, pathtype):
"""Grep files in paths for any matches to content_regex_list.
Args:
source: artifact source
pathtype: pathspec path type
When multiple regexes are supplied, combine them into a single regex as an
OR match so that we check all regexes at once.
"""
path_list = self.InterpolateList(source.attributes.get("paths", []))
content_regex_list = self.InterpolateList(
source.attributes.get("content_regex_list", []))
regex_condition = rdfvalue.FileFinderContentsRegexMatchCondition(
regex=self._CombineRegex(content_regex_list),
bytes_before=0,
bytes_after=0)
file_finder_condition = rdfvalue.FileFinderCondition(
condition_type=rdfvalue.FileFinderCondition.Type.
CONTENTS_REGEX_MATCH,
contents_regex_match=regex_condition)
self.CallFlow("FileFinder",
paths=path_list,
conditions=[file_finder_condition],
action=rdfvalue.FileFinderAction(),
pathtype=pathtype,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollected")
def IterateProcesses(self, responses):
"""This stores the processes."""
if not responses.success:
# Check for error, but continue. Errors are common on client.
raise flow.FlowError("Error during process listing %s" %
responses.status)
if self.args.fetch_binaries:
# Filter out processes entries without "exe" attribute and
# deduplicate the list.
paths_to_fetch = set()
for p in responses:
if p.exe and self.args.filename_regex.Match(p.exe):
paths_to_fetch.add(p.exe)
paths_to_fetch = sorted(paths_to_fetch)
self.Log("Got %d processes, fetching binaries for %d...",
len(responses), len(paths_to_fetch))
self.CallFlow(
"FileFinder",
paths=paths_to_fetch,
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD),
next_state="HandleDownloadedFiles")
else:
# Only send the list of processes if we don't fetch the binaries
for response in responses:
self.SendReply(response)
def Start(self):
self.CallFlow("FileFinder",
paths=self.args.keys_paths,
pathtype=rdfvalue.PathSpec.PathType.REGISTRY,
conditions=self.ConditionsToFileFinderConditions(
self.args.conditions),
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.STAT),
next_state="Done")
def GetExtensionName(self, responses):
"""Gets the name of the extension from the manifest."""
if responses.success:
# The pathspec to the manifest file
file_stat = responses.First()
extension_directory = file_stat.pathspec.Dirname()
# Read the manifest file which should be just json - already stored in
fd = aff4.FACTORY.Open(file_stat.aff4path, token=self.token)
try:
manifest_data = fd.read(1000000)
manifest = json.loads(manifest_data)
except ValueError:
self.Log("Unable to parse %s as json. Continuing.", fd.urn)
return
ext_name = manifest.get("name", "")
if ext_name.startswith("__MSG_"):
# Extension has a localized name
if "default_locale" in manifest:
msg_path = extension_directory.Copy().Append(
pathtype=self.args.pathtype,
path="_locales/" + manifest["default_locale"] +
"/messages.json")
request_data = dict(
manifest_data=manifest_data,
extension_directory=extension_directory)
self.CallFlow("GetFile",
next_state="GetLocalizedName",
pathspec=msg_path,
request_data=request_data)
return
else:
logging.error(
"Malformed extension %s, missing default_locale.",
extension_directory.CollapsePath())
# Continue with __MSG_... extension name
self.CreateAnalysisVFile(extension_directory, manifest)
if self.args.download_files:
self.CallFlow(
"FileFinder",
paths=[
os.path.join(extension_directory.CollapsePath(), "**3")
],
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD),
next_state="Done")
def testAppliesLiteralConditionWhenMemoryPathTypeIsUsed(self):
vfs.VFS_HANDLERS[
rdfvalue.PathSpec.PathType.OS] = test_lib.FakeTestDataVFSHandler
vfs.VFS_HANDLERS[rdfvalue.PathSpec.PathType.
MEMORY] = test_lib.FakeTestDataVFSHandler
paths = [
os.path.join(os.path.dirname(self.base_path), "auth.log"),
os.path.join(os.path.dirname(self.base_path), "dpkg.log")
]
literal_condition = rdfvalue.FileFinderCondition(
condition_type=rdfvalue.FileFinderCondition.Type.
CONTENTS_LITERAL_MATCH,
contents_literal_match=rdfvalue.
FileFinderContentsLiteralMatchCondition(
mode=rdfvalue.FileFinderContentsLiteralMatchCondition.Mode.
ALL_HITS,
literal="session opened for user dearjohn"))
# Check this condition with all the actions. This makes sense, as we may
# download memeory or send it to the socket.
for action in sorted(
rdfvalue.FileFinderAction.Action.enum_dict.values()):
for _ in test_lib.TestFlowHelper(
"FileFinder",
self.client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdfvalue.PathSpec.PathType.MEMORY,
conditions=[literal_condition],
action=rdfvalue.FileFinderAction(action_type=action),
token=self.token,
output=self.output_path):
pass
self.CheckFilesInCollection(["auth.log"])
fd = aff4.FACTORY.Open(self.client_id.Add(self.output_path),
aff4_type="RDFValueCollection",
token=self.token)
self.assertEqual(fd[0].stat_entry.pathspec.CollapsePath(),
paths[0])
self.assertEqual(len(fd), 1)
self.assertEqual(len(fd[0].matches), 1)
self.assertEqual(fd[0].matches[0].offset, 350)
self.assertEqual(
fd[0].matches[0].data,
"session): session opened for user dearjohn by (uid=0")
def testDownloadDirectorySub(self):
"""Test a FileFinder flow with depth=5."""
vfs.VFS_HANDLERS[
rdfvalue.PathSpec.PathType.OS] = test_lib.ClientVFSHandlerFixture
# Mock the client actions FileFinder uses.
client_mock = action_mocks.ActionMock("FingerprintFile", "HashBuffer",
"StatFile", "Find",
"TransferBuffer")
for _ in test_lib.TestFlowHelper(
"FileFinder",
client_mock,
client_id=self.client_id,
paths=["/c/Downloads/**5"],
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD),
token=self.token):
pass
# Check if the base path was created
output_path = self.client_id.Add("fs/os/c/Downloads")
output_fd = aff4.FACTORY.Open(output_path, token=self.token)
children = list(output_fd.OpenChildren())
# There should be 6 children:
expected_children = u"a.txt b.txt c.txt d.txt sub1 中国新闻网新闻中.txt"
self.assertEqual(len(children), 6)
self.assertEqual(expected_children.split(),
sorted([child.urn.Basename() for child in children]))
# Find the child named: sub1
for child in children:
if child.urn.Basename() == "sub1":
break
children = list(child.OpenChildren())
# There should be 4 children: a.txt, b.txt, c.txt, d.txt
expected_children = "a.txt b.txt c.txt d.txt"
self.assertEqual(len(children), 4)
self.assertEqual(expected_children.split(),
sorted([child.urn.Basename() for child in children]))
def GetFiles(self, collector, path_type, max_size):
"""Get a set of files."""
new_path_list = []
for path in collector.args["path_list"]:
# Interpolate any attributes from the knowledgebase.
new_path_list.extend(artifact_lib.InterpolateKbAttributes(
path, self.state.knowledge_base))
action = rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD,
download=rdfvalue.FileFinderDownloadActionOptions(max_size=max_size))
self.CallFlow(
"FileFinder", paths=new_path_list, pathtype=path_type, action=action,
request_data={"artifact_name": self.current_artifact_name,
"collector": collector.ToPrimitiveDict()},
next_state="ProcessFileFinderResults")
def RunFlowAndCheckResults(self,
conditions=None,
action=rdfvalue.FileFinderAction.Action.STAT,
expected_files=None,
non_expected_files=None):
conditions = conditions or []
expected_files = expected_files or []
non_expected_files = non_expected_files or []
for fname in expected_files + non_expected_files:
aff4.FACTORY.Delete(self.FileNameToURN(fname), token=self.token)
with test_lib.Instrument(flow.GRRFlow, "SendReply") as send_reply:
for _ in test_lib.TestFlowHelper(
"FileFinder",
self.client_mock,
client_id=self.client_id,
paths=[self.path],
pathtype=rdfvalue.PathSpec.PathType.OS,
action=rdfvalue.FileFinderAction(action_type=action),
conditions=conditions,
token=self.token,
output=self.output_path):
pass
self.CheckReplies(send_reply.args, action, expected_files)
self.CheckFilesInCollection(expected_files)
if action == rdfvalue.FileFinderAction.Action.STAT:
self.CheckFilesNotDownloaded(expected_files + non_expected_files)
self.CheckFilesNotHashed(expected_files + non_expected_files)
elif action == rdfvalue.FileFinderAction.Action.DOWNLOAD:
self.CheckFilesDownloaded(expected_files)
self.CheckFilesNotDownloaded(non_expected_files)
# Downloaded files are hashed to allow for deduping.
elif action == rdfvalue.FileFinderAction.Action.HASH:
self.CheckFilesNotDownloaded(expected_files + non_expected_files)
self.CheckFilesHashed(expected_files)
self.CheckFilesNotHashed(non_expected_files)
def RunFlowAndCheckResults(
self,
filters=None,
action=rdfvalue.FileFinderAction.Action.DO_NOTHING,
expected_files=None,
non_expected_files=None):
filters = filters or []
expected_files = expected_files or []
non_expected_files = non_expected_files or []
for fname in expected_files + non_expected_files:
aff4.FACTORY.Delete(self.FileNameToURN(fname), token=self.token)
for _ in test_lib.TestFlowHelper(
"FileFinder",
self.client_mock,
client_id=self.client_id,
paths=[self.path],
pathtype=rdfvalue.PathSpec.PathType.OS,
action=rdfvalue.FileFinderAction(action_type=action),
filters=filters,
token=self.token,
output=self.output_path):
pass
self.CheckFilesInCollection(expected_files)
if action == rdfvalue.FileFinderAction.Action.DO_NOTHING:
self.CheckFilesNotDownloaded(expected_files + non_expected_files)
self.CheckFilesNotHashed(expected_files + non_expected_files)
elif action == rdfvalue.FileFinderAction.Action.DOWNLOAD:
self.CheckFilesDownloaded(expected_files)
self.CheckFilesNotDownloaded(non_expected_files)
self.CheckFilesNotHashed(expected_files + non_expected_files)
elif action == rdfvalue.FileFinderAction.Action.HASH:
self.CheckFilesNotDownloaded(expected_files + non_expected_files)
self.CheckFilesHashed(expected_files)
self.CheckFilesNotHashed(non_expected_files)
def GetLocalizedName(self, responses):
"""Determines the name of the extension if the extension uses locales."""
if responses.success:
manifest = json.loads(responses.request_data["manifest_data"])
extension_directory = responses.request_data["extension_directory"]
# Parse the locale json.
urn = aff4.AFF4Object.VFSGRRClient.PathspecToURN(
responses.First().pathspec, self.client_id)
fd = aff4.FACTORY.Open(urn, token=self.token)
msg = manifest["name"][6:].rstrip("_")
try:
messages = json.loads(fd.read(1000000))
# Update the extension name from the locale messages
manifest["name"] = messages[msg]["message"]
except (ValueError, KeyError):
pass
else:
logging.error(
"Malformed extension: localization file not found (%s).",
manifest["name"])
self.CreateAnalysisVFile(extension_directory, manifest)
if self.args.download_files:
self.CallFlow(
"FileFinder",
paths=[
os.path.join(extension_directory.CollapsePath(), "**3")
],
action=rdfvalue.FileFinderAction(
action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD),
next_state="Done")
