def testKnowledgeBaseRetrievalLinuxNoUsers(self):
"""Cause a users.username dependency failure."""
self.ClearKB()
with test_lib.ConfigOverrider({
"Artifacts.knowledge_base": [
"NetgroupConfiguration", "NssCacheLinuxPasswdHomedirs",
"LinuxRelease"
],
"Artifacts.netgroup_filter_regexes": ["^doesntexist$"]
}):
with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
vfs_test_lib.FakeTestDataVFSHandler):
for _ in flow_test_lib.TestFlowHelper(
artifact.KnowledgeBaseInitializationFlow.__name__,
self.client_mock,
require_complete=False,
client_id=self.client_id,
token=self.token):
pass
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.os_major_version, 14)
self.assertEqual(kb.os_minor_version, 4)
self.assertItemsEqual([x.username for x in kb.users], [])
def testKnowledgeBaseRetrievalLinuxNoUsers(self):
"""Cause a users.username dependency failure."""
test_lib.ClientFixture(self.client_id, token=self.token)
self.SetLinuxClient()
config_lib.CONFIG.Set("Artifacts.knowledge_base", [
"NetgroupConfiguration", "NssCacheLinuxPasswdHomedirs",
"LinuxRelease"
])
config_lib.CONFIG.Set("Artifacts.netgroup_filter_regexes",
["^doesntexist$"])
vfs.VFS_HANDLERS[
rdfvalue.PathSpec.PathType.OS] = test_lib.FakeTestDataVFSHandler
client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile",
"Find", "HashBuffer",
"ListDirectory",
"FingerprintFile")
for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow",
client_mock,
require_complete=False,
client_id=self.client_id,
token=self.token):
pass
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.os_major_version, 14)
self.assertEqual(kb.os_minor_version, 4)
self.assertItemsEqual([x.username for x in kb.users], [])
def testKnowledgeBaseRetrievalDarwin(self):
"""Check we can retrieve a Darwin kb."""
test_lib.ClientFixture(self.client_id, token=self.token)
self.SetDarwinClient()
vfs.VFS_HANDLERS[
rdfvalue.PathSpec.PathType.OS] = test_lib.ClientVFSHandlerFixture
client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile",
"Find", "HashBuffer",
"ListDirectory",
"FingerprintFile")
for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow",
client_mock,
client_id=self.client_id,
token=self.token):
pass
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.os_major_version, 10)
self.assertEqual(kb.os_minor_version, 9)
# scalzi from /Users dir listing.
# Bert and Ernie not present (Users fixture overriden by kb).
self.assertItemsEqual([x.username for x in kb.users], ["scalzi"])
user = kb.GetUser(username="******")
self.assertEqual(user.homedir, "/Users/scalzi")
def testKnowledgeBaseRetrievalWindows(self):
"""Check we can retrieve a knowledge base from a client."""
self.SetupWindowsMocks()
client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile",
"Find", "HashBuffer",
"ListDirectory",
"FingerprintFile")
for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow",
client_mock,
client_id=self.client_id,
token=self.token):
pass
# The client should now be populated with the data we care about.
client = aff4.FACTORY.Open(self.client_id, token=self.token)
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.environ_systemroot, "C:\\Windows")
self.assertEqual(kb.time_zone, "US/Alaska")
self.assertEqual(kb.code_page, "cp_1252")
self.assertEqual(kb.environ_windir, "C:\\Windows")
self.assertEqual(kb.environ_allusersprofile, "C:\\Users\\All Users")
self.assertEqual(kb.environ_allusersappdata, "C:\\ProgramData")
self.assertEqual(kb.environ_temp, "C:\\Windows\\TEMP")
self.assertEqual(kb.environ_systemdrive, "C:")
self.assertItemsEqual([x.username for x in kb.users],
["jim", "kovacs"])
user = kb.GetUser(username="******")
self.assertEqual(user.username, "jim")
self.assertEqual(user.sid,
"S-1-5-21-702227068-2140022151-3110739409-1000")
def ParseRunKeys(self, responses):
"""Get filenames from the RunKeys and download the files."""
filenames = []
client = aff4.FACTORY.Open(self.client_id, mode="r", token=self.token)
kb = artifact.GetArtifactKnowledgeBase(client)
for response in responses:
runkey = response.registry_data.string
environ_vars = artifact_utils.GetWindowsEnvironmentVariablesMap(kb)
path_guesses = path_detection_windows.DetectExecutablePaths(
[runkey], environ_vars)
if not path_guesses:
self.Log("Couldn't guess path for %s", runkey)
for path in path_guesses:
filenames.append(
rdf_paths.PathSpec(
path=path, pathtype=rdf_paths.PathSpec.PathType.TSK))
if filenames:
self.CallFlow("MultiGetFile",
pathspecs=filenames,
next_state="Done")
def testKnowledgeBaseRetrievalLinux(self):
"""Check we can retrieve a Linux kb."""
test_lib.ClientFixture(self.client_id, token=self.token)
self.SetLinuxClient()
config_lib.CONFIG.Set("Artifacts.knowledge_base", [
"LinuxWtmp", "NetgroupConfiguration", "LinuxPasswdHomedirs",
"LinuxRelease"
])
config_lib.CONFIG.Set("Artifacts.netgroup_filter_regexes", ["^login$"])
config_lib.CONFIG.Set("Artifacts.netgroup_user_blacklist", ["isaac"])
vfs.VFS_HANDLERS[
rdfvalue.PathSpec.PathType.OS] = test_lib.FakeTestDataVFSHandler
client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile",
"Find", "HashBuffer",
"ListDirectory",
"FingerprintFile", "Grep")
for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow",
client_mock,
client_id=self.client_id,
token=self.token):
pass
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.os_major_version, 14)
self.assertEqual(kb.os_minor_version, 4)
# user 1,2,3 from wtmp. yagharek from netgroup.
# Bert and Ernie not present (Users fixture overriden by kb).
self.assertItemsEqual([x.username for x in kb.users],
["user1", "user2", "user3", "yagharek"])
user = kb.GetUser(username="******")
self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552099)
self.assertEqual(user.homedir, "/home/user1")
def testKnowledgeBaseRetrievalLinuxPasswd(self):
"""Check we can retrieve a Linux kb."""
self.ClearKB()
with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
test_lib.FakeTestDataVFSHandler):
with test_lib.ConfigOverrider({
"Artifacts.knowledge_base": ["LinuxWtmp",
"LinuxPasswdHomedirs",
"LinuxRelease"],
"Artifacts.knowledge_base_additions": [],
"Artifacts.knowledge_base_skip": []}):
for _ in test_lib.TestFlowHelper(
"KnowledgeBaseInitializationFlow", self.client_mock,
client_id=self.client_id, token=self.token):
pass
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.os_major_version, 14)
self.assertEqual(kb.os_minor_version, 4)
# user 1,2,3 from wtmp.
# Bert and Ernie not present (Users fixture overriden by kb).
self.assertItemsEqual([x.username for x in kb.users], ["user1", "user2",
"user3"])
user = kb.GetUser(username="******")
self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552099)
self.assertEqual(user.homedir, "/home/user1")
user = kb.GetUser(username="******")
self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552102)
self.assertEqual(user.homedir, "/home/user2")
self.assertFalse(kb.GetUser(username="******"))
def Start(self):
"""For each artifact, create subflows for each collector."""
self.client = aff4.FACTORY.Open(self.client_id, token=self.token)
self.state.Register("artifacts_skipped_due_to_condition", [])
self.state.Register("failed_count", 0)
self.state.Register("artifacts_failed", [])
self.state.Register("bootstrap_complete", False)
self.state.Register("knowledge_base", self.args.knowledge_base)
self.state.Register("client_anomaly_store", None)
if self.args.use_tsk:
self.state.Register("path_type", rdfvalue.PathSpec.PathType.TSK)
else:
self.state.Register("path_type", rdfvalue.PathSpec.PathType.OS)
if not self.state.knowledge_base:
# If not provided, get a knowledge base from the client.
try:
self.state.knowledge_base = artifact.GetArtifactKnowledgeBase(
self.client)
except artifact_lib.KnowledgeBaseUninitializedError:
# If no-one has ever initialized the knowledge base, we should do so
# now.
if not self._AreArtifactsKnowledgeBaseArtifacts():
self.CallFlow("KnowledgeBaseInitializationFlow",
next_state="StartCollection")
return
# In all other cases start the collection state.
self.CallState(next_state="StartCollection")
def testKnowledgeBaseRetrievalDarwin(self):
"""Check we can retrieve a Darwin kb."""
self.ClearKB()
with test_lib.ConfigOverrider(
{"Artifacts.knowledge_base": ["OSXUsers"]}):
with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
test_lib.ClientVFSHandlerFixture):
for _ in test_lib.TestFlowHelper(
"KnowledgeBaseInitializationFlow",
self.client_mock,
client_id=self.client_id,
token=self.token):
pass
client = aff4.FACTORY.Open(self.client_id,
token=self.token,
mode="rw")
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.os_major_version, 10)
self.assertEqual(kb.os_minor_version, 9)
# scalzi from /Users dir listing.
self.assertItemsEqual([x.username for x in kb.users],
["scalzi"])
user = kb.GetUser(username="******")
self.assertEqual(user.homedir, "/Users/scalzi")
def testKnowledgeBaseRetrievalLinux(self):
"""Check we can retrieve a Linux kb."""
self.ClearKB()
with test_lib.ConfigOverrider({
"Artifacts.knowledge_base": [
"LinuxWtmp", "NetgroupConfiguration", "LinuxPasswdHomedirs",
"LinuxRelease"
],
"Artifacts.netgroup_filter_regexes": ["^login$"],
"Artifacts.netgroup_user_blacklist": ["isaac"]
}):
with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
vfs_test_lib.FakeTestDataVFSHandler):
for _ in flow_test_lib.TestFlowHelper(
artifact.KnowledgeBaseInitializationFlow.__name__,
self.client_mock,
client_id=self.client_id,
token=self.token):
pass
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.os_major_version, 14)
self.assertEqual(kb.os_minor_version, 4)
# user 1,2,3 from wtmp. yagharek from netgroup.
self.assertItemsEqual([x.username for x in kb.users],
["user1", "user2", "user3", "yagharek"])
user = kb.GetUser(username="******")
self.assertEqual(user.last_logon.AsSecondsFromEpoch(), 1296552099)
self.assertEqual(user.homedir, "/home/user1")
def ParseRunKeys(self, responses):
"""Get filenames from the RunKeys and download the files."""
filenames = []
client = aff4.FACTORY.Open(self.client_id, mode="r", token=self.token)
kb = artifact.GetArtifactKnowledgeBase(client)
for response in responses:
runkey = response.registry_data.string
path_guesses = utils.GuessWindowsFileNameFromString(runkey)
path_guesses = filter(self._IsExecutableExtension, path_guesses)
if not path_guesses:
self.Log("Couldn't guess path for %s", runkey)
for path in path_guesses:
full_path = artifact_lib.ExpandWindowsEnvironmentVariables(
path, kb)
filenames.append(
rdfvalue.PathSpec(path=full_path,
pathtype=rdfvalue.PathSpec.PathType.TSK))
if filenames:
self.CallFlow("MultiGetFile",
pathspecs=filenames,
next_state="Done")
def testKnowledgeBaseMultiProvides(self):
"""Check we can handle multi-provides."""
self.SetupWindowsMocks()
# Replace some artifacts with test one that will run the MultiProvideParser.
self.LoadTestArtifacts()
artifacts = config_lib.CONFIG["Artifacts.knowledge_base"]
artifacts.append("DepsProvidesMultiple") # Our test artifact.
artifacts.remove("WinPathEnvironmentVariable")
artifacts.remove("TempEnvironmentVariable")
config_lib.CONFIG.Set("Artifacts.knowledge_base", artifacts)
client_mock = action_mocks.ActionMock("TransferBuffer", "StatFile",
"Find", "HashBuffer",
"ListDirectory",
"FingerprintFile")
for _ in test_lib.TestFlowHelper("KnowledgeBaseInitializationFlow",
client_mock,
client_id=self.client_id,
token=self.token):
pass
# The client should now be populated with the data we care about.
client = aff4.FACTORY.Open(self.client_id, token=self.token)
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.environ_temp, "tempvalue")
self.assertEqual(kb.environ_path, "pathvalue")
def ProcessKnowledgeBase(self, responses):
"""Update the SUMMARY from the knowledgebase data."""
if not responses.success:
raise flow.FlowError("Error collecting artifacts: %s" % responses.status)
knowledge_base = artifact.GetArtifactKnowledgeBase(self.client)
for kbuser in knowledge_base.users:
self.state.summary.users.Append(
rdfvalue.User().FromKnowledgeBaseUser(kbuser))
def Start(self):
"""For each artifact, create subflows for each collector."""
self.client = aff4.FACTORY.Open(self.client_id, token=self.token)
self.state.Register("artifacts_skipped_due_to_condition", [])
self.state.Register("response_count", 0)
self.state.Register("failed_count", 0)
self.state.Register("artifacts_failed", [])
self.state.Register("knowledge_base", self.args.knowledge_base)
self.state.Register("called_fallbacks", set())
self.state.Register("client_anomaly_store", None)
if self.args.use_tsk:
self.state.Register("path_type", paths.PathSpec.PathType.TSK)
else:
self.state.Register("path_type", paths.PathSpec.PathType.OS)
if (self.args.dependencies ==
artifact_utils.ArtifactCollectorFlowArgs.Dependency.FETCH_NOW):
# Don't retrieve a full knowledgebase, just get the dependencies we
# need. CollectArtifactDependencies calls back to this flow to retrieve
# the necessary dependencies. We avoid a loop because
# dependencies defaults to USE_CACHED set and a knowledgebase is
# provided.
self.CallFlow("CollectArtifactDependencies",
artifact_list=self.args.artifact_list,
next_state="StartCollection")
return
elif (self.args.dependencies ==
artifact_utils.ArtifactCollectorFlowArgs.Dependency.USE_CACHED) and (
not self.state.knowledge_base):
# If not provided, get a knowledge base from the client.
try:
self.state.knowledge_base = artifact.GetArtifactKnowledgeBase(
self.client)
except artifact_utils.KnowledgeBaseUninitializedError:
# If no-one has ever initialized the knowledge base, we should do so
# now.
if not self._AreArtifactsKnowledgeBaseArtifacts():
self.CallFlow("KnowledgeBaseInitializationFlow",
next_state="StartCollection")
return
# In all other cases start the collection state.
self.CallState(next_state="StartCollection")
def testKnowledgeBaseMultiProvides(self):
"""Check we can handle multi-provides."""
self.ClearKB()
# Replace some artifacts with test one that will run the MultiProvideParser.
self.LoadTestArtifacts()
with test_lib.ConfigOverrider({"Artifacts.knowledge_base":
["DepsProvidesMultiple"]}):
for _ in test_lib.TestFlowHelper(
"KnowledgeBaseInitializationFlow", self.client_mock,
client_id=self.client_id, token=self.token):
pass
# The client should now be populated with the data we care about.
client = aff4.FACTORY.Open(self.client_id, token=self.token)
kb = artifact.GetArtifactKnowledgeBase(client)
self.assertEqual(kb.environ_temp, "tempvalue")
self.assertEqual(kb.environ_path, "pathvalue")
def Start(self):
"""For each artifact, create subflows for each collector."""
self.client = aff4.FACTORY.Open(self.client_id, token=self.token)
self.state.artifacts_failed = []
self.state.artifacts_skipped_due_to_condition = []
self.state.called_fallbacks = set()
self.state.client_anomalies = []
self.state.failed_count = 0
self.state.knowledge_base = self.args.knowledge_base
self.state.response_count = 0
if (self.args.dependencies ==
artifact_utils.ArtifactCollectorFlowArgs.Dependency.FETCH_NOW):
# Don't retrieve a full knowledgebase, just get the dependencies we
# need. CollectArtifactDependencies calls back to this flow to retrieve
# the necessary dependencies. We avoid a loop because
# dependencies defaults to USE_CACHED set and a knowledgebase is
# provided.
self.CallFlow(artifact.CollectArtifactDependencies.__name__,
artifact_list=self.args.artifact_list,
next_state="StartCollection")
return
elif (self.args.dependencies
== artifact_utils.ArtifactCollectorFlowArgs.Dependency.USE_CACHED
) and (not self.state.knowledge_base):
# If not provided, get a knowledge base from the client.
try:
self.state.knowledge_base = artifact.GetArtifactKnowledgeBase(
self.client)
except artifact_utils.KnowledgeBaseUninitializedError:
# If no-one has ever initialized the knowledge base, we should do so
# now.
if not self._AreArtifactsKnowledgeBaseArtifacts():
self.CallFlow(
artifact.KnowledgeBaseInitializationFlow.__name__,
next_state="StartCollection")
return
# In all other cases start the collection state.
self.CallState(next_state="StartCollection")
def StartCollection(self, responses):
"""Start collecting."""
if not responses.success:
raise artifact_utils.KnowledgeBaseUninitializedError(
"Attempt to initialize Knowledge Base failed.")
if not self.state.knowledge_base:
self.client = aff4.FACTORY.Open(self.client_id, token=self.token)
# If we are processing the knowledge base, it still won't exist yet.
self.state.knowledge_base = artifact.GetArtifactKnowledgeBase(
self.client, allow_uninitialized=True)
for artifact_name in self.args.artifact_list:
artifact_obj = self._GetArtifactFromName(artifact_name)
# Ensure artifact has been written sanely. Note that this could be
# removed if it turns out to be expensive. Artifact tests should catch
# these.
artifact_obj.Validate()
self.Collect(artifact_obj)
def FindMatchingPathspecs(self, response):
# If we're dealing with plain file StatEntry, just
# return it's pathspec - there's nothing to parse
# and guess.
if (isinstance(response, rdf_client.StatEntry)
and response.pathspec.pathtype
in [paths.PathSpec.PathType.TSK, paths.PathSpec.PathType.OS]):
return [response.pathspec]
client = aff4.FACTORY.Open(self.client_id, token=self.token)
knowledge_base = artifact.GetArtifactKnowledgeBase(client)
if self.args.use_tsk:
path_type = paths.PathSpec.PathType.TSK
else:
path_type = paths.PathSpec.PathType.OS
parser = windows_persistence.WindowsPersistenceMechanismsParser()
parsed_items = list(parser.Parse(response, knowledge_base, path_type))
return [item.pathspec for item in parsed_items]
def ProcessKnowledgeBase(self, responses):
"""Update the SUMMARY from the knowledgebase data."""
if not responses.success:
raise flow.FlowError("Error collecting artifacts: %s" %
responses.status)
knowledge_base = artifact.GetArtifactKnowledgeBase(self.client)
for kbuser in knowledge_base.users:
self.state.summary.users.Append(
rdfvalue.User().FromKnowledgeBaseUser(kbuser))
# If the knowledge base came back with values for these collected via
# artifacts, we trust those more than the values returned via the
# GetPlatformInfo client action so we override them here.
if knowledge_base.os_release:
os_release = knowledge_base.os_release
self.client.Set(self.client.Schema.OS_RELEASE(os_release))
# Override OS version field too.
# TODO(user): this actually results in incorrect versions for things
# like Ubuntu (14.4 instead of 14.04). I don't think zero-
# padding is always correct, however.
os_version = "%d.%d" % (knowledge_base.os_major_version,
knowledge_base.os_minor_version)
self.client.Set(self.client.Schema.OS_VERSION(os_version))
# Update client summary accordingly.
self.state.summary.system_info.release = os_release
self.state.summary.system_info.version = os_version
# Collect any non-knowledgebase artifacts that will be stored in aff4.
artifact_list = self._GetExtraArtifactsForCollection()
if artifact_list:
self.CallFlow("ArtifactCollectorFlow",
artifact_list=artifact_list,
next_state="ProcessArtifactResponses",
store_results_in_aff4=True)
