def testKnowledgeBaseRetrievalLinuxNoUsers(self): """Cause a users.username dependency failure.""" self.ClearKB() with test_lib.ConfigOverrider({ "Artifacts.knowledge_base": [ "NetgroupConfiguration", "NssCacheLinuxPasswdHomedirs", "LinuxRelease" ], "Artifacts.netgroup_filter_regexes": ["^doesntexist$"] }): with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, vfs_test_lib.FakeTestDataVFSHandler): for _ in flow_test_lib.TestFlowHelper( artifact.KnowledgeBaseInitializationFlow.__name__, self.client_mock, require_complete=False, client_id=self.client_id, token=self.token): pass client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.os_major_version, 14) self.assertEqual(kb.os_minor_version, 4) self.assertItemsEqual([x.username for x in kb.users], [])
def testKnowledgeBaseRetrievalLinuxNoUsers(self): """Cause a users.username dependency failure.""" test_lib.ClientFixture(self.client_id, token=self.token) self.SetLinuxClient() config_lib.CONFIG.Set("Artifacts.knowledge_base", [ "NetgroupConfiguration", "NssCacheLinuxPasswdHomedirs", "LinuxRelease" ]) config_lib.CONFIG.Set("Artifacts.netgroup_filter_regexes", ["^doesntexist$"]) vfs.VFS_HANDLERS[ rdfvalue.PathSpec.PathType.OS] = test_lib.FakeTestDataVFSHandler client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile", "Find", "HashBuffer", "ListDirectory", "FingerprintFile") for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow", client_mock, require_complete=False, client_id=self.client_id, token=self.token): pass client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.os_major_version, 14) self.assertEqual(kb.os_minor_version, 4) self.assertItemsEqual([x.username for x in kb.users], [])
def testKnowledgeBaseRetrievalDarwin(self): """Check we can retrieve a Darwin kb.""" test_lib.ClientFixture(self.client_id, token=self.token) self.SetDarwinClient() vfs.VFS_HANDLERS[ rdfvalue.PathSpec.PathType.OS] = test_lib.ClientVFSHandlerFixture client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile", "Find", "HashBuffer", "ListDirectory", "FingerprintFile") for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow", client_mock, client_id=self.client_id, token=self.token): pass client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.os_major_version, 10) self.assertEqual(kb.os_minor_version, 9) # scalzi from /Users dir listing. # Bert and Ernie not present (Users fixture overriden by kb). self.assertItemsEqual([x.username for x in kb.users], ["scalzi"]) user = kb.GetUser(username="******") self.assertEqual(user.homedir, "/Users/scalzi")
def testKnowledgeBaseRetrievalWindows(self): """Check we can retrieve a knowledge base from a client.""" self.SetupWindowsMocks() client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile", "Find", "HashBuffer", "ListDirectory", "FingerprintFile") for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow", client_mock, client_id=self.client_id, token=self.token): pass # The client should now be populated with the data we care about. client = aff4.FACTORY.Open(self.client_id, token=self.token) kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.environ_systemroot, "C:\\Windows") self.assertEqual(kb.time_zone, "US/Alaska") self.assertEqual(kb.code_page, "cp_1252") self.assertEqual(kb.environ_windir, "C:\\Windows") self.assertEqual(kb.environ_allusersprofile, "C:\\Users\\All Users") self.assertEqual(kb.environ_allusersappdata, "C:\\ProgramData") self.assertEqual(kb.environ_temp, "C:\\Windows\\TEMP") self.assertEqual(kb.environ_systemdrive, "C:") self.assertItemsEqual([x.username for x in kb.users], ["jim", "kovacs"]) user = kb.GetUser(username="******") self.assertEqual(user.username, "jim") self.assertEqual(user.sid, "S-1-5-21-702227068-2140022151-3110739409-1000")
def ParseRunKeys(self, responses): """Get filenames from the RunKeys and download the files.""" filenames = [] client = aff4.FACTORY.Open(self.client_id, mode="r", token=self.token) kb = artifact.GetArtifactKnowledgeBase(client) for response in responses: runkey = response.registry_data.string environ_vars = artifact_utils.GetWindowsEnvironmentVariablesMap(kb) path_guesses = path_detection_windows.DetectExecutablePaths( [runkey], environ_vars) if not path_guesses: self.Log("Couldn't guess path for %s", runkey) for path in path_guesses: filenames.append( rdf_paths.PathSpec( path=path, pathtype=rdf_paths.PathSpec.PathType.TSK)) if filenames: self.CallFlow("MultiGetFile", pathspecs=filenames, next_state="Done")
def testKnowledgeBaseRetrievalLinux(self): """Check we can retrieve a Linux kb.""" test_lib.ClientFixture(self.client_id, token=self.token) self.SetLinuxClient() config_lib.CONFIG.Set("Artifacts.knowledge_base", [ "LinuxWtmp", "NetgroupConfiguration", "LinuxPasswdHomedirs", "LinuxRelease" ]) config_lib.CONFIG.Set("Artifacts.netgroup_filter_regexes", ["^login$"]) config_lib.CONFIG.Set("Artifacts.netgroup_user_blacklist", ["isaac"]) vfs.VFS_HANDLERS[ rdfvalue.PathSpec.PathType.OS] = test_lib.FakeTestDataVFSHandler client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile", "Find", "HashBuffer", "ListDirectory", "FingerprintFile", "Grep") for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow", client_mock, client_id=self.client_id, token=self.token): pass client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.os_major_version, 14) self.assertEqual(kb.os_minor_version, 4) # user 1,2,3 from wtmp. yagharek from netgroup. # Bert and Ernie not present (Users fixture overriden by kb). self.assertItemsEqual([x.username for x in kb.users], ["user1", "user2", "user3", "yagharek"]) user = kb.GetUser(username="******") self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552099) self.assertEqual(user.homedir, "/home/user1")
def testKnowledgeBaseRetrievalLinuxPasswd(self): """Check we can retrieve a Linux kb.""" self.ClearKB() with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, test_lib.FakeTestDataVFSHandler): with test_lib.ConfigOverrider({ "Artifacts.knowledge_base": ["LinuxWtmp", "LinuxPasswdHomedirs", "LinuxRelease"], "Artifacts.knowledge_base_additions": [], "Artifacts.knowledge_base_skip": []}): for _ in test_lib.TestFlowHelper( "KnowledgeBaseInitializationFlow", self.client_mock, client_id=self.client_id, token=self.token): pass client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.os_major_version, 14) self.assertEqual(kb.os_minor_version, 4) # user 1,2,3 from wtmp. # Bert and Ernie not present (Users fixture overriden by kb). self.assertItemsEqual([x.username for x in kb.users], ["user1", "user2", "user3"]) user = kb.GetUser(username="******") self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552099) self.assertEqual(user.homedir, "/home/user1") user = kb.GetUser(username="******") self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552102) self.assertEqual(user.homedir, "/home/user2") self.assertFalse(kb.GetUser(username="******"))
def Start(self): """For each artifact, create subflows for each collector.""" self.client = aff4.FACTORY.Open(self.client_id, token=self.token) self.state.Register("artifacts_skipped_due_to_condition", []) self.state.Register("failed_count", 0) self.state.Register("artifacts_failed", []) self.state.Register("bootstrap_complete", False) self.state.Register("knowledge_base", self.args.knowledge_base) self.state.Register("client_anomaly_store", None) if self.args.use_tsk: self.state.Register("path_type", rdfvalue.PathSpec.PathType.TSK) else: self.state.Register("path_type", rdfvalue.PathSpec.PathType.OS) if not self.state.knowledge_base: # If not provided, get a knowledge base from the client. try: self.state.knowledge_base = artifact.GetArtifactKnowledgeBase( self.client) except artifact_lib.KnowledgeBaseUninitializedError: # If no-one has ever initialized the knowledge base, we should do so # now. if not self._AreArtifactsKnowledgeBaseArtifacts(): self.CallFlow("KnowledgeBaseInitializationFlow", next_state="StartCollection") return # In all other cases start the collection state. self.CallState(next_state="StartCollection")
def testKnowledgeBaseRetrievalDarwin(self): """Check we can retrieve a Darwin kb.""" self.ClearKB() with test_lib.ConfigOverrider( {"Artifacts.knowledge_base": ["OSXUsers"]}): with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, test_lib.ClientVFSHandlerFixture): for _ in test_lib.TestFlowHelper( "KnowledgeBaseInitializationFlow", self.client_mock, client_id=self.client_id, token=self.token): pass client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.os_major_version, 10) self.assertEqual(kb.os_minor_version, 9) # scalzi from /Users dir listing. self.assertItemsEqual([x.username for x in kb.users], ["scalzi"]) user = kb.GetUser(username="******") self.assertEqual(user.homedir, "/Users/scalzi")
def testKnowledgeBaseRetrievalLinux(self): """Check we can retrieve a Linux kb.""" self.ClearKB() with test_lib.ConfigOverrider({ "Artifacts.knowledge_base": [ "LinuxWtmp", "NetgroupConfiguration", "LinuxPasswdHomedirs", "LinuxRelease" ], "Artifacts.netgroup_filter_regexes": ["^login$"], "Artifacts.netgroup_user_blacklist": ["isaac"] }): with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, vfs_test_lib.FakeTestDataVFSHandler): for _ in flow_test_lib.TestFlowHelper( artifact.KnowledgeBaseInitializationFlow.__name__, self.client_mock, client_id=self.client_id, token=self.token): pass client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.os_major_version, 14) self.assertEqual(kb.os_minor_version, 4) # user 1,2,3 from wtmp. yagharek from netgroup. self.assertItemsEqual([x.username for x in kb.users], ["user1", "user2", "user3", "yagharek"]) user = kb.GetUser(username="******") self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552099) self.assertEqual(user.homedir, "/home/user1")
def ParseRunKeys(self, responses): """Get filenames from the RunKeys and download the files.""" filenames = [] client = aff4.FACTORY.Open(self.client_id, mode="r", token=self.token) kb = artifact.GetArtifactKnowledgeBase(client) for response in responses: runkey = response.registry_data.string path_guesses = utils.GuessWindowsFileNameFromString(runkey) path_guesses = filter(self._IsExecutableExtension, path_guesses) if not path_guesses: self.Log("Couldn't guess path for %s", runkey) for path in path_guesses: full_path = artifact_lib.ExpandWindowsEnvironmentVariables( path, kb) filenames.append( rdfvalue.PathSpec(path=full_path, pathtype=rdfvalue.PathSpec.PathType.TSK)) if filenames: self.CallFlow("MultiGetFile", pathspecs=filenames, next_state="Done")
def testKnowledgeBaseMultiProvides(self): """Check we can handle multi-provides.""" self.SetupWindowsMocks() # Replace some artifacts with test one that will run the MultiProvideParser. self.LoadTestArtifacts() artifacts = config_lib.CONFIG["Artifacts.knowledge_base"] artifacts.append("DepsProvidesMultiple") # Our test artifact. artifacts.remove("WinPathEnvironmentVariable") artifacts.remove("TempEnvironmentVariable") config_lib.CONFIG.Set("Artifacts.knowledge_base", artifacts) client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile", "Find", "HashBuffer", "ListDirectory", "FingerprintFile") for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow", client_mock, client_id=self.client_id, token=self.token): pass # The client should now be populated with the data we care about. client = aff4.FACTORY.Open(self.client_id, token=self.token) kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.environ_temp, "tempvalue") self.assertEqual(kb.environ_path, "pathvalue")
def ProcessKnowledgeBase(self, responses): """Update the SUMMARY from the knowledgebase data.""" if not responses.success: raise flow.FlowError("Error collecting artifacts: %s" % responses.status) knowledge_base = artifact.GetArtifactKnowledgeBase(self.client) for kbuser in knowledge_base.users: self.state.summary.users.Append( rdfvalue.User().FromKnowledgeBaseUser(kbuser))
def Start(self): """For each artifact, create subflows for each collector.""" self.client = aff4.FACTORY.Open(self.client_id, token=self.token) self.state.Register("artifacts_skipped_due_to_condition", []) self.state.Register("response_count", 0) self.state.Register("failed_count", 0) self.state.Register("artifacts_failed", []) self.state.Register("knowledge_base", self.args.knowledge_base) self.state.Register("called_fallbacks", set()) self.state.Register("client_anomaly_store", None) if self.args.use_tsk: self.state.Register("path_type", paths.PathSpec.PathType.TSK) else: self.state.Register("path_type", paths.PathSpec.PathType.OS) if (self.args.dependencies == artifact_utils.ArtifactCollectorFlowArgs.Dependency.FETCH_NOW): # Don't retrieve a full knowledgebase, just get the dependencies we # need. CollectArtifactDependencies calls back to this flow to retrieve # the necessary dependencies. We avoid a loop because # dependencies defaults to USE_CACHED set and a knowledgebase is # provided. self.CallFlow("CollectArtifactDependencies", artifact_list=self.args.artifact_list, next_state="StartCollection") return elif (self.args.dependencies == artifact_utils.ArtifactCollectorFlowArgs.Dependency.USE_CACHED) and ( not self.state.knowledge_base): # If not provided, get a knowledge base from the client. try: self.state.knowledge_base = artifact.GetArtifactKnowledgeBase( self.client) except artifact_utils.KnowledgeBaseUninitializedError: # If no-one has ever initialized the knowledge base, we should do so # now. if not self._AreArtifactsKnowledgeBaseArtifacts(): self.CallFlow("KnowledgeBaseInitializationFlow", next_state="StartCollection") return # In all other cases start the collection state. self.CallState(next_state="StartCollection")
def testKnowledgeBaseMultiProvides(self): """Check we can handle multi-provides.""" self.ClearKB() # Replace some artifacts with test one that will run the MultiProvideParser. self.LoadTestArtifacts() with test_lib.ConfigOverrider({"Artifacts.knowledge_base": ["DepsProvidesMultiple"]}): for _ in test_lib.TestFlowHelper( "KnowledgeBaseInitializationFlow", self.client_mock, client_id=self.client_id, token=self.token): pass # The client should now be populated with the data we care about. client = aff4.FACTORY.Open(self.client_id, token=self.token) kb = artifact.GetArtifactKnowledgeBase(client) self.assertEqual(kb.environ_temp, "tempvalue") self.assertEqual(kb.environ_path, "pathvalue")
def Start(self): """For each artifact, create subflows for each collector.""" self.client = aff4.FACTORY.Open(self.client_id, token=self.token) self.state.artifacts_failed = [] self.state.artifacts_skipped_due_to_condition = [] self.state.called_fallbacks = set() self.state.client_anomalies = [] self.state.failed_count = 0 self.state.knowledge_base = self.args.knowledge_base self.state.response_count = 0 if (self.args.dependencies == artifact_utils.ArtifactCollectorFlowArgs.Dependency.FETCH_NOW): # Don't retrieve a full knowledgebase, just get the dependencies we # need. CollectArtifactDependencies calls back to this flow to retrieve # the necessary dependencies. We avoid a loop because # dependencies defaults to USE_CACHED set and a knowledgebase is # provided. self.CallFlow(artifact.CollectArtifactDependencies.__name__, artifact_list=self.args.artifact_list, next_state="StartCollection") return elif (self.args.dependencies == artifact_utils.ArtifactCollectorFlowArgs.Dependency.USE_CACHED ) and (not self.state.knowledge_base): # If not provided, get a knowledge base from the client. try: self.state.knowledge_base = artifact.GetArtifactKnowledgeBase( self.client) except artifact_utils.KnowledgeBaseUninitializedError: # If no-one has ever initialized the knowledge base, we should do so # now. if not self._AreArtifactsKnowledgeBaseArtifacts(): self.CallFlow( artifact.KnowledgeBaseInitializationFlow.__name__, next_state="StartCollection") return # In all other cases start the collection state. self.CallState(next_state="StartCollection")
def StartCollection(self, responses): """Start collecting.""" if not responses.success: raise artifact_utils.KnowledgeBaseUninitializedError( "Attempt to initialize Knowledge Base failed.") if not self.state.knowledge_base: self.client = aff4.FACTORY.Open(self.client_id, token=self.token) # If we are processing the knowledge base, it still won't exist yet. self.state.knowledge_base = artifact.GetArtifactKnowledgeBase( self.client, allow_uninitialized=True) for artifact_name in self.args.artifact_list: artifact_obj = self._GetArtifactFromName(artifact_name) # Ensure artifact has been written sanely. Note that this could be # removed if it turns out to be expensive. Artifact tests should catch # these. artifact_obj.Validate() self.Collect(artifact_obj)
def FindMatchingPathspecs(self, response): # If we're dealing with plain file StatEntry, just # return it's pathspec - there's nothing to parse # and guess. if (isinstance(response, rdf_client.StatEntry) and response.pathspec.pathtype in [paths.PathSpec.PathType.TSK, paths.PathSpec.PathType.OS]): return [response.pathspec] client = aff4.FACTORY.Open(self.client_id, token=self.token) knowledge_base = artifact.GetArtifactKnowledgeBase(client) if self.args.use_tsk: path_type = paths.PathSpec.PathType.TSK else: path_type = paths.PathSpec.PathType.OS parser = windows_persistence.WindowsPersistenceMechanismsParser() parsed_items = list(parser.Parse(response, knowledge_base, path_type)) return [item.pathspec for item in parsed_items]
def ProcessKnowledgeBase(self, responses): """Update the SUMMARY from the knowledgebase data.""" if not responses.success: raise flow.FlowError("Error collecting artifacts: %s" % responses.status) knowledge_base = artifact.GetArtifactKnowledgeBase(self.client) for kbuser in knowledge_base.users: self.state.summary.users.Append( rdfvalue.User().FromKnowledgeBaseUser(kbuser)) # If the knowledge base came back with values for these collected via # artifacts, we trust those more than the values returned via the # GetPlatformInfo client action so we override them here. if knowledge_base.os_release: os_release = knowledge_base.os_release self.client.Set(self.client.Schema.OS_RELEASE(os_release)) # Override OS version field too. # TODO(user): this actually results in incorrect versions for things # like Ubuntu (14.4 instead of 14.04). I don't think zero- # padding is always correct, however. os_version = "%d.%d" % (knowledge_base.os_major_version, knowledge_base.os_minor_version) self.client.Set(self.client.Schema.OS_VERSION(os_version)) # Update client summary accordingly. self.state.summary.system_info.release = os_release self.state.summary.system_info.version = os_version # Collect any non-knowledgebase artifacts that will be stored in aff4. artifact_list = self._GetExtraArtifactsForCollection() if artifact_list: self.CallFlow("ArtifactCollectorFlow", artifact_list=artifact_list, next_state="ProcessArtifactResponses", store_results_in_aff4=True)