def FetchBinaries(self, responses): """Parses the Rekall response and initiates FileFinder flows.""" if not responses.success: self.Log("Error fetching VAD data: %s", responses.status) return self.Log("Found %d binaries", len(responses)) if self.args.filename_regex: binaries = [] for response in responses: if self.args.filename_regex.Match(response.CollapsePath()): binaries.append(response) self.Log("Applied filename regex. Have %d files after filtering.", len(binaries)) else: binaries = responses if self.args.fetch_binaries: self.CallFlow( "FileFinder", next_state="HandleDownloadedFiles", paths=[ rdfvalue.GlobExpression(b.CollapsePath()) for b in binaries ], pathtype=rdfvalue.PathSpec.PathType.OS, action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD)) else: for b in binaries: self.SendReply(b)
def StartRequests(self): """Generate and send the Find requests.""" client = aff4.FACTORY.Open(self.client_id, token=self.token) if self.runner.output is not None: self.runner.output.Set( self.runner.output.Schema.DESCRIPTION("CacheGrep for {0}".format( self.args.data_regex))) usernames = ["%s\\%s" % (u.domain, u.username) for u in self.state.users] usernames = [u.lstrip("\\") for u in usernames] # Strip \\ if no domain. condition = rdfvalue.FileFinderCondition( condition_type=rdfvalue.FileFinderCondition.Type.CONTENTS_REGEX_MATCH, contents_regex_match=rdfvalue.FileFinderContentsRegexMatchCondition( regex=self.args.data_regex, mode=rdfvalue.FileFinderContentsRegexMatchCondition.Mode.FIRST_HIT)) for path in self.state.all_paths: full_paths = flow_utils.InterpolatePath(path, client, users=usernames) for full_path in full_paths: self.CallFlow( "FileFinder", paths=[os.path.join(full_path, "**5")], pathtype=self.state.args.pathtype, conditions=[condition], action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD), next_state="HandleResults")
def Start(self): """Determine the Chrome directory.""" self.state.Register("hist_count", 0) # List of paths where history files are located self.state.Register("history_paths", []) if self.state.args.history_path: self.state.history_paths.append(self.state.args.history_path) if self.runner.output is not None: self.runner.output = aff4.FACTORY.Create( self.runner.output.urn, "VFSAnalysisFile", token=self.token) if not self.state.history_paths: self.state.history_paths = self.GuessHistoryPaths( self.state.args.username) if not self.state.history_paths: raise flow.FlowError("Could not find valid History paths.") filenames = ["History"] if self.state.args.get_archive: filenames.append("Archived History") for path in self.state.history_paths: for fname in filenames: self.CallFlow( "FileFinder", paths=[os.path.join(path, fname)], pathtype=self.state.args.pathtype, action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD), next_state="ParseFiles")
def Grep(self, collector, pathtype): """Grep files in path_list for any matches to content_regex_list.""" path_list = self.InterpolateList(collector.args.get("path_list", [])) content_regex_list = self.InterpolateList( collector.args.get("content_regex_list", [])) filters = [] for regex in content_regex_list: regexfilter = rdfvalue.FileFinderContentsRegexMatchFilter( regex=regex, bytes_before=0, bytes_after=0) file_finder_filter = rdfvalue.FileFinderFilter( filter_type=rdfvalue.FileFinderFilter.Type. CONTENTS_REGEX_MATCH, contents_regex_match=regexfilter) filters.append(file_finder_filter) self.CallFlow("FileFinder", paths=path_list, filters=filters, action=rdfvalue.FileFinderAction(), pathtype=pathtype, request_data={ "artifact_name": self.current_artifact_name, "collector": collector.ToPrimitiveDict() }, next_state="ProcessCollected")
def Start(self): """Determine the Firefox history directory.""" self.state.Register("hist_count", 0) self.state.Register("history_paths", []) if self.args.history_path: self.state.history_paths.append(self.args.history_path) else: self.state.history_paths = self.GuessHistoryPaths(self.args.username) if not self.state.history_paths: raise flow.FlowError("Could not find valid History paths.") if self.runner.output is not None: self.runner.output = aff4.FACTORY.Create( self.runner.output.urn, "VFSAnalysisFile", token=self.token) filename = "places.sqlite" for path in self.state.history_paths: self.CallFlow( "FileFinder", paths=[os.path.join(path, "**2", filename)], pathtype=self.state.args.pathtype, action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD), next_state="ParseFiles")
def testDownloadActionSizeLimit(self): expected_files = ["dpkg.log", "dpkg_false.log"] non_expected_files = ["auth.log"] sizes = [ os.stat(os.path.join(self.base_path, f)).st_size for f in expected_files ] action = rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD) action.download.max_size = max(sizes) + 1 for _ in test_lib.TestFlowHelper( "FileFinder", self.client_mock, client_id=self.client_id, paths=[self.path], pathtype=rdfvalue.PathSpec.PathType.OS, action=action, token=self.token, output=self.output_path): pass self.CheckFilesDownloaded(expected_files) self.CheckFilesNotDownloaded(non_expected_files) # Even though the file is too big to download, we still want the # hash. self.CheckFilesHashed(non_expected_files)
def Grep(self, source, pathtype): """Grep files in paths for any matches to content_regex_list. Args: source: artifact source pathtype: pathspec path type When multiple regexes are supplied, combine them into a single regex as an OR match so that we check all regexes at once. """ path_list = self.InterpolateList(source.attributes.get("paths", [])) content_regex_list = self.InterpolateList( source.attributes.get("content_regex_list", [])) regex_condition = rdfvalue.FileFinderContentsRegexMatchCondition( regex=self._CombineRegex(content_regex_list), bytes_before=0, bytes_after=0) file_finder_condition = rdfvalue.FileFinderCondition( condition_type=rdfvalue.FileFinderCondition.Type. CONTENTS_REGEX_MATCH, contents_regex_match=regex_condition) self.CallFlow("FileFinder", paths=path_list, conditions=[file_finder_condition], action=rdfvalue.FileFinderAction(), pathtype=pathtype, request_data={ "artifact_name": self.current_artifact_name, "source": source.ToPrimitiveDict() }, next_state="ProcessCollected")
def IterateProcesses(self, responses): """This stores the processes.""" if not responses.success: # Check for error, but continue. Errors are common on client. raise flow.FlowError("Error during process listing %s" % responses.status) if self.args.fetch_binaries: # Filter out processes entries without "exe" attribute and # deduplicate the list. paths_to_fetch = set() for p in responses: if p.exe and self.args.filename_regex.Match(p.exe): paths_to_fetch.add(p.exe) paths_to_fetch = sorted(paths_to_fetch) self.Log("Got %d processes, fetching binaries for %d...", len(responses), len(paths_to_fetch)) self.CallFlow( "FileFinder", paths=paths_to_fetch, action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD), next_state="HandleDownloadedFiles") else: # Only send the list of processes if we don't fetch the binaries for response in responses: self.SendReply(response)
def Start(self): self.CallFlow("FileFinder", paths=self.args.keys_paths, pathtype=rdfvalue.PathSpec.PathType.REGISTRY, conditions=self.ConditionsToFileFinderConditions( self.args.conditions), action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.STAT), next_state="Done")
def GetExtensionName(self, responses): """Gets the name of the extension from the manifest.""" if responses.success: # The pathspec to the manifest file file_stat = responses.First() extension_directory = file_stat.pathspec.Dirname() # Read the manifest file which should be just json - already stored in fd = aff4.FACTORY.Open(file_stat.aff4path, token=self.token) try: manifest_data = fd.read(1000000) manifest = json.loads(manifest_data) except ValueError: self.Log("Unable to parse %s as json. Continuing.", fd.urn) return ext_name = manifest.get("name", "") if ext_name.startswith("__MSG_"): # Extension has a localized name if "default_locale" in manifest: msg_path = extension_directory.Copy().Append( pathtype=self.args.pathtype, path="_locales/" + manifest["default_locale"] + "/messages.json") request_data = dict( manifest_data=manifest_data, extension_directory=extension_directory) self.CallFlow("GetFile", next_state="GetLocalizedName", pathspec=msg_path, request_data=request_data) return else: logging.error( "Malformed extension %s, missing default_locale.", extension_directory.CollapsePath()) # Continue with __MSG_... extension name self.CreateAnalysisVFile(extension_directory, manifest) if self.args.download_files: self.CallFlow( "FileFinder", paths=[ os.path.join(extension_directory.CollapsePath(), "**3") ], action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD), next_state="Done")
def testAppliesLiteralConditionWhenMemoryPathTypeIsUsed(self): vfs.VFS_HANDLERS[ rdfvalue.PathSpec.PathType.OS] = test_lib.FakeTestDataVFSHandler vfs.VFS_HANDLERS[rdfvalue.PathSpec.PathType. MEMORY] = test_lib.FakeTestDataVFSHandler paths = [ os.path.join(os.path.dirname(self.base_path), "auth.log"), os.path.join(os.path.dirname(self.base_path), "dpkg.log") ] literal_condition = rdfvalue.FileFinderCondition( condition_type=rdfvalue.FileFinderCondition.Type. CONTENTS_LITERAL_MATCH, contents_literal_match=rdfvalue. FileFinderContentsLiteralMatchCondition( mode=rdfvalue.FileFinderContentsLiteralMatchCondition.Mode. ALL_HITS, literal="session opened for user dearjohn")) # Check this condition with all the actions. This makes sense, as we may # download memeory or send it to the socket. for action in sorted( rdfvalue.FileFinderAction.Action.enum_dict.values()): for _ in test_lib.TestFlowHelper( "FileFinder", self.client_mock, client_id=self.client_id, paths=paths, pathtype=rdfvalue.PathSpec.PathType.MEMORY, conditions=[literal_condition], action=rdfvalue.FileFinderAction(action_type=action), token=self.token, output=self.output_path): pass self.CheckFilesInCollection(["auth.log"]) fd = aff4.FACTORY.Open(self.client_id.Add(self.output_path), aff4_type="RDFValueCollection", token=self.token) self.assertEqual(fd[0].stat_entry.pathspec.CollapsePath(), paths[0]) self.assertEqual(len(fd), 1) self.assertEqual(len(fd[0].matches), 1) self.assertEqual(fd[0].matches[0].offset, 350) self.assertEqual( fd[0].matches[0].data, "session): session opened for user dearjohn by (uid=0")
def testDownloadDirectorySub(self): """Test a FileFinder flow with depth=5.""" vfs.VFS_HANDLERS[ rdfvalue.PathSpec.PathType.OS] = test_lib.ClientVFSHandlerFixture # Mock the client actions FileFinder uses. client_mock = action_mocks.ActionMock("FingerprintFile", "HashBuffer", "StatFile", "Find", "TransferBuffer") for _ in test_lib.TestFlowHelper( "FileFinder", client_mock, client_id=self.client_id, paths=["/c/Downloads/**5"], action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD), token=self.token): pass # Check if the base path was created output_path = self.client_id.Add("fs/os/c/Downloads") output_fd = aff4.FACTORY.Open(output_path, token=self.token) children = list(output_fd.OpenChildren()) # There should be 6 children: expected_children = u"a.txt b.txt c.txt d.txt sub1 中国新闻网新闻中.txt" self.assertEqual(len(children), 6) self.assertEqual(expected_children.split(), sorted([child.urn.Basename() for child in children])) # Find the child named: sub1 for child in children: if child.urn.Basename() == "sub1": break children = list(child.OpenChildren()) # There should be 4 children: a.txt, b.txt, c.txt, d.txt expected_children = "a.txt b.txt c.txt d.txt" self.assertEqual(len(children), 4) self.assertEqual(expected_children.split(), sorted([child.urn.Basename() for child in children]))
def GetFiles(self, collector, path_type, max_size): """Get a set of files.""" new_path_list = [] for path in collector.args["path_list"]: # Interpolate any attributes from the knowledgebase. new_path_list.extend(artifact_lib.InterpolateKbAttributes( path, self.state.knowledge_base)) action = rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD, download=rdfvalue.FileFinderDownloadActionOptions(max_size=max_size)) self.CallFlow( "FileFinder", paths=new_path_list, pathtype=path_type, action=action, request_data={"artifact_name": self.current_artifact_name, "collector": collector.ToPrimitiveDict()}, next_state="ProcessFileFinderResults")
def RunFlowAndCheckResults(self, conditions=None, action=rdfvalue.FileFinderAction.Action.STAT, expected_files=None, non_expected_files=None): conditions = conditions or [] expected_files = expected_files or [] non_expected_files = non_expected_files or [] for fname in expected_files + non_expected_files: aff4.FACTORY.Delete(self.FileNameToURN(fname), token=self.token) with test_lib.Instrument(flow.GRRFlow, "SendReply") as send_reply: for _ in test_lib.TestFlowHelper( "FileFinder", self.client_mock, client_id=self.client_id, paths=[self.path], pathtype=rdfvalue.PathSpec.PathType.OS, action=rdfvalue.FileFinderAction(action_type=action), conditions=conditions, token=self.token, output=self.output_path): pass self.CheckReplies(send_reply.args, action, expected_files) self.CheckFilesInCollection(expected_files) if action == rdfvalue.FileFinderAction.Action.STAT: self.CheckFilesNotDownloaded(expected_files + non_expected_files) self.CheckFilesNotHashed(expected_files + non_expected_files) elif action == rdfvalue.FileFinderAction.Action.DOWNLOAD: self.CheckFilesDownloaded(expected_files) self.CheckFilesNotDownloaded(non_expected_files) # Downloaded files are hashed to allow for deduping. elif action == rdfvalue.FileFinderAction.Action.HASH: self.CheckFilesNotDownloaded(expected_files + non_expected_files) self.CheckFilesHashed(expected_files) self.CheckFilesNotHashed(non_expected_files)
def RunFlowAndCheckResults( self, filters=None, action=rdfvalue.FileFinderAction.Action.DO_NOTHING, expected_files=None, non_expected_files=None): filters = filters or [] expected_files = expected_files or [] non_expected_files = non_expected_files or [] for fname in expected_files + non_expected_files: aff4.FACTORY.Delete(self.FileNameToURN(fname), token=self.token) for _ in test_lib.TestFlowHelper( "FileFinder", self.client_mock, client_id=self.client_id, paths=[self.path], pathtype=rdfvalue.PathSpec.PathType.OS, action=rdfvalue.FileFinderAction(action_type=action), filters=filters, token=self.token, output=self.output_path): pass self.CheckFilesInCollection(expected_files) if action == rdfvalue.FileFinderAction.Action.DO_NOTHING: self.CheckFilesNotDownloaded(expected_files + non_expected_files) self.CheckFilesNotHashed(expected_files + non_expected_files) elif action == rdfvalue.FileFinderAction.Action.DOWNLOAD: self.CheckFilesDownloaded(expected_files) self.CheckFilesNotDownloaded(non_expected_files) self.CheckFilesNotHashed(expected_files + non_expected_files) elif action == rdfvalue.FileFinderAction.Action.HASH: self.CheckFilesNotDownloaded(expected_files + non_expected_files) self.CheckFilesHashed(expected_files) self.CheckFilesNotHashed(non_expected_files)
def GetLocalizedName(self, responses): """Determines the name of the extension if the extension uses locales.""" if responses.success: manifest = json.loads(responses.request_data["manifest_data"]) extension_directory = responses.request_data["extension_directory"] # Parse the locale json. urn = aff4.AFF4Object.VFSGRRClient.PathspecToURN( responses.First().pathspec, self.client_id) fd = aff4.FACTORY.Open(urn, token=self.token) msg = manifest["name"][6:].rstrip("_") try: messages = json.loads(fd.read(1000000)) # Update the extension name from the locale messages manifest["name"] = messages[msg]["message"] except (ValueError, KeyError): pass else: logging.error( "Malformed extension: localization file not found (%s).", manifest["name"]) self.CreateAnalysisVFile(extension_directory, manifest) if self.args.download_files: self.CallFlow( "FileFinder", paths=[ os.path.join(extension_directory.CollapsePath(), "**3") ], action=rdfvalue.FileFinderAction( action_type=rdfvalue.FileFinderAction.Action.DOWNLOAD), next_state="Done")