def testUnknownAttributeScopeNotARealScope(self):
kb = rdf_client.KnowledgeBase(os="Linux")
with self.assertRaisesUnknownAttrs() as context:
artifact_utils.InterpolateKbAttributes("%%os.version%%", kb)
self.assertEqual(context.exception.attrs, ["os"])
def GetRegistryValue(self, source):
"""Retrieve directly specified registry values, returning Stat objects."""
new_paths = set()
has_glob = False
for kvdict in source.attributes["key_value_pairs"]:
if "*" in kvdict["key"] or rdf_paths.GROUPING_PATTERN.search(
kvdict["key"]):
has_glob = True
if kvdict["value"]:
# This currently only supports key value pairs specified using forward
# slash.
path = "\\".join((kvdict["key"], kvdict["value"]))
else:
# If value is not set, we want to get the default value. In
# GRR this is done by specifying the key only, so this is what
# we do here.
path = kvdict["key"]
expanded_paths = artifact_utils.InterpolateKbAttributes(
path,
self.state.knowledge_base,
ignore_errors=self.args.ignore_interpolation_errors)
new_paths.update(expanded_paths)
if has_glob:
self.CallFlow(
filesystem.Glob.__name__,
paths=new_paths,
pathtype=rdf_paths.PathSpec.PathType.REGISTRY,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollected")
else:
# We call statfile directly for keys that don't include globs because it
# is faster and some artifacts rely on getting an IOError to trigger
# fallback processing.
for new_path in new_paths:
pathspec = rdf_paths.PathSpec(
path=new_path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY)
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client.GetFileStatRequest(pathspec=pathspec)
else:
stub = server_stubs.StatFile
request = rdf_client.ListDirRequest(pathspec=pathspec)
self.CallClient(
stub,
request,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollectedRegistryStatEntry")
def testUnknownAttributeScopeVar(self):
kb = rdf_client.KnowledgeBase()
with self.assertRaisesUnknownAttrs() as context:
artifact_utils.InterpolateKbAttributes("%%users.nonexistent%%", kb)
self.assertEqual(context.exception.attrs, ["users.nonexistent"])
def testEmptyAttribute(self):
kb = rdf_client.KnowledgeBase(environ_allusersprofile="")
with self.assertRaisesMissingAttrs() as context:
artifact_utils.InterpolateKbAttributes("%%environ_allusersprofile%%", kb)
self.assertEqual(context.exception.attrs, ["environ_allusersprofile"])
def testMissingAttributeScope(self):
kb = rdf_client.KnowledgeBase()
with self.assertRaisesMissingAttrs() as context:
artifact_utils.InterpolateKbAttributes("test%%users.username%%test", kb)
self.assertEqual(context.exception.attrs, ["users"])
def testMissingAttributeScopeVar(self):
kb = rdf_client.KnowledgeBase()
kb.users.Append(rdf_client.User(username="******", uid=42))
with self.assertRaisesMissingAttrs() as context:
artifact_utils.InterpolateKbAttributes("test%%users.temp%%test", kb)
self.assertEqual(context.exception.attrs, ["users.temp"])
def GetFiles(self, source, path_type, implementation_type, max_size):
"""Get a set of files."""
new_path_list = []
for path in source.attributes["paths"]:
try:
interpolated = artifact_utils.InterpolateKbAttributes(
path, self.state.knowledge_base)
except artifact_utils.KbInterpolationMissingAttributesError as error:
logging.error(str(error))
if not self.args.ignore_interpolation_errors:
raise
else:
interpolated = []
new_path_list.extend(interpolated)
action = rdf_file_finder.FileFinderAction.Download(max_size=max_size)
self.CallFlow(
file_finder.FileFinder.__name__,
paths=new_path_list,
pathtype=path_type,
implementation_type=implementation_type,
action=action,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state=compatibility.GetName(self.ProcessFileFinderResults))
def _Interpolate(self,
pattern: Text,
knowledgebase: Optional[rdf_client.KnowledgeBase] = None)\
-> Sequence[Text]:
"""Performs a knowledgebase interpolation.
Args:
pattern: A pattern to interpolate.
knowledgebase: Knowledgebase to use for interpolation. If no knowledgebase
is provided, the one provided as a flow argument is used.
Returns:
A list of possible interpolation results.
"""
if knowledgebase is None:
knowledgebase = self.state.knowledge_base
try:
return artifact_utils.InterpolateKbAttributes(pattern, knowledgebase)
except artifact_utils.KbInterpolationMissingAttributesError as error:
if self.args.old_client_snapshot_fallback:
return []
if self.args.ignore_interpolation_errors:
logging.error(str(error))
return []
raise
def ExplainComponents(
self, example_count: int,
knowledge_base) -> Sequence[GlobComponentExplanation]:
"""Returns a list of GlobComponentExplanations with examples."""
parts = _COMPONENT_SPLIT_PATTERN.split(self._value)
components = []
for glob_part in parts:
if not glob_part:
continue
component = GlobComponentExplanation(glob_expression=glob_part)
if GROUPING_PATTERN.match(glob_part):
examples = self.InterpolateGrouping(glob_part)
elif _VAR_PATTERN.match(glob_part):
# Examples for variable substitutions might not be 100 % accurate,
# because the scope is not shared between two variables. Thus,
# if a GlobExpression uses %%users.a%% and %%users.b%%, the underlying
# user might be different for a and b. For the sake of explaining
# possible values, this should still be enough.
examples = artifact_utils.InterpolateKbAttributes(
glob_part, knowledge_base)
else:
examples = []
component.examples = list(itertools.islice(examples,
example_count))
components.append(component)
return components
def _ProcessRegistryValueSource(self, args):
new_paths = set()
has_glob = False
for kvdict in args.base_source.attributes["key_value_pairs"]:
if "*" in kvdict["key"] or rdf_paths.GROUPING_PATTERN.search(
kvdict["key"]):
has_glob = True
if kvdict["value"]:
path = "\\".join((kvdict["key"], kvdict["value"]))
else:
path = kvdict["key"]
expanded_paths = artifact_utils.InterpolateKbAttributes(
path,
self.knowledge_base,
ignore_errors=self.ignore_interpolation_errors)
new_paths.update(expanded_paths)
if has_glob:
# TODO(user): If a path has a wildcard we need to glob the filesystem
# for patterns to collect matching files. The corresponding flow is
# filesystem.Glob.
pass
else:
action = standard.GetFileStat
for new_path in new_paths:
pathspec = rdf_paths.PathSpec(
path=new_path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY)
request = rdf_client.GetFileStatRequest(pathspec=pathspec)
yield action, request
def _InterpolatePaths(self, globs):
client = aff4.FACTORY.Open(self.client_id, token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
for glob in globs:
param_path = glob.SerializeToString()
for path in artifact_utils.InterpolateKbAttributes(param_path, kb):
yield path
def _InterpolatePaths(self, globs):
kb = self.client_knowledge_base
for glob in globs:
param_path = str(glob)
for path in artifact_utils.InterpolateKbAttributes(param_path, kb):
yield path
def testSimpleVariable(self):
kb = rdf_client.KnowledgeBase()
kb.environ_allusersprofile = "c:\\programdata"
paths = artifact_utils.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb)
self.assertEqual(list(paths), ["c:\\programdata\\a"])
def testSingleUserHasValueOthersDoNot(self):
kb = rdf_client.KnowledgeBase()
kb.users.Append(rdf_client.User(username="******", uid=1, temp="C:\\Temp"))
kb.users.Append(rdf_client.User(username="******", uid=2))
kb.users.Append(rdf_client.User(username="******", uid=3))
paths = artifact_utils.InterpolateKbAttributes(r"%%users.temp%%\abcd", kb)
self.assertCountEqual(paths, ["C:\\Temp\\abcd"])
def testMultipleUsersWithOneFullyApplicableValue(self):
kb = rdf_client.KnowledgeBase()
kb.users.Append(rdf_client.User(username="******", uid=1, temp="C:\\Temp"))
kb.users.Append(rdf_client.User(username="******", uid=2))
kb.users.Append(rdf_client.User(username="******", uid=3))
paths = artifact_utils.InterpolateKbAttributes(
"%%users.temp%%\\%%users.username%%.txt", kb)
self.assertCountEqual(paths, ["C:\\Temp\\foo.txt"])
def testMultipleUsersHaveValues(self):
kb = rdf_client.KnowledgeBase()
kb.users.Append(rdf_client.User(username="******", uid=1, sid="sid1"))
kb.users.Append(rdf_client.User(username="******", uid=2, sid="sid2"))
kb.Set("environ_allusersprofile", "c:\\programdata")
paths = artifact_utils.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\%%users.sid%%\\%%users.username%%", kb)
self.assertCountEqual(
paths, ["c:\\programdata\\sid1\\joe", "c:\\programdata\\sid2\\jim"])
def InterpolateList(self, input_list):
"""Interpolate all items from a given source array.
Args:
input_list: list of values to interpolate
Returns:
original list of values extended with strings interpolated
"""
# We want to ignore errors either when it was explicitly requested by the
# user or when the fallback mode is turned on (because one of the older
# snapshot might successfully interpolate without errors).
ignore_interpolation_errors = (
self.args.ignore_interpolation_errors or
self.args.old_client_snapshot_fallback)
new_args = []
for value in input_list:
if isinstance(value, string_types):
results = list(
artifact_utils.InterpolateKbAttributes(
value,
self.state.knowledge_base,
ignore_errors=ignore_interpolation_errors))
if not results and self.args.old_client_snapshot_fallback:
client_id = self.client_id
snapshots = data_store.REL_DB.ReadClientSnapshotHistory(client_id)
for snapshot in snapshots:
results = list(
artifact_utils.InterpolateKbAttributes(
value,
knowledge_base=snapshot.knowledge_base,
ignore_errors=True))
if results:
break
new_args.extend(results)
else:
new_args.extend(value)
return new_args
def _Interpolate(self, pattern: Text) -> Sequence[Text]:
try:
return artifact_utils.InterpolateKbAttributes(pattern,
self.knowledge_base)
except artifact_utils.KbInterpolationMissingAttributesError as error:
if self.ignore_interpolation_errors:
logging.error(str(error))
return []
else:
raise
def testMultipleUsers(self):
kb = rdf_client.KnowledgeBase()
kb.users.Append(rdf_client.User(username="******", uid=1))
kb.users.Append(rdf_client.User(username="******", uid=2))
paths = artifact_utils.InterpolateKbAttributes(
"test%%users.username%%test", kb)
paths = list(paths)
self.assertLen(paths, 2)
self.assertCountEqual(paths, ["testjoetest", "testjimtest"])
def Interpolate(self, knowledge_base=None):
kb = knowledge_base
patterns = artifact_utils.InterpolateKbAttributes(self._value, kb)
for pattern in patterns:
# Normalize the component path (this allows us to resolve ../
# sequences).
pattern = utils.NormalizePath(pattern.replace("\\", "/"))
for p in self.InterpolateGrouping(pattern):
yield p
def Interpolate(self, client=None):
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
patterns = artifact_utils.InterpolateKbAttributes(self._value, kb)
for pattern in patterns:
# Normalize the component path (this allows us to resolve ../
# sequences).
pattern = utils.NormalizePath(pattern.replace("\\", "/"))
for pattern in self.InterpolateGrouping(pattern):
yield pattern
def _GetSingleExpansion(self, value):
results = list(
artifact_utils.InterpolateKbAttributes(
value,
self.state.knowledge_base,
ignore_errors=self.args.ignore_interpolation_errors))
if len(results) > 1:
raise ValueError(
"Interpolation generated multiple results, use a"
" list for multi-value expansions. %s yielded: %s" % (value, results))
return results[0]
def _ProcessWmiSource(self, args):
# pylint: disable= g-import-not-at-top
from grr_response_client.client_actions.windows import windows
# pylint: enable=g-import-not-at-top
action = windows.WmiQuery
query = args.base_source.attributes["query"]
queries = artifact_utils.InterpolateKbAttributes(
query, self.knowledge_base, self.ignore_interpolation_errors)
base_object = args.base_source.attributes.get("base_object")
for query in queries:
request = rdf_client.WMIRequest(query=query, base_object=base_object)
yield action, request
def testInterpolation(self):
"""Check we can interpolate values from the knowledge base."""
kb = rdf_client.KnowledgeBase()
# No users yet, this should raise
self.assertRaises(
artifact_utils.KnowledgeBaseInterpolationError, list,
artifact_utils.InterpolateKbAttributes(
"test%%users.username%%test", kb))
# Now we have two users
kb.users.Append(rdf_client.User(username="******", uid=1))
kb.users.Append(rdf_client.User(username="******", uid=2))
kb.Set("environ_allusersprofile", "c:\\programdata")
paths = artifact_utils.InterpolateKbAttributes(
"test%%users.username%%test", kb)
paths = list(paths)
self.assertLen(paths, 2)
self.assertCountEqual(paths, ["testjoetest", "testjimtest"])
paths = artifact_utils.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb)
self.assertEqual(list(paths), ["c:\\programdata\\a"])
# Check a bad attribute raises
self.assertRaises(
artifact_utils.KnowledgeBaseInterpolationError, list,
artifact_utils.InterpolateKbAttributes("%%nonexistent%%\\a", kb))
# Empty values should also raise
kb.Set("environ_allusersprofile", "")
self.assertRaises(
artifact_utils.KnowledgeBaseInterpolationError, list,
artifact_utils.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb))
# No users have temp defined, so this should raise
self.assertRaises(
artifact_utils.KnowledgeBaseInterpolationError, list,
artifact_utils.InterpolateKbAttributes("%%users.temp%%\\a", kb))
# One user has users.temp defined, the others do not. This is common on
# windows where users have been created but have never logged in. We should
# get just one value back.
kb.users.Append(
rdf_client.User(username="******",
uid=1,
temp="C:\\Users\\jason\\AppData\\Local\\Temp"))
paths = artifact_utils.InterpolateKbAttributes(r"%%users.temp%%\abcd",
kb)
self.assertCountEqual(paths,
["C:\\Users\\jason\\AppData\\Local\\Temp\\abcd"])
def testMultipleUserInterpolations(self):
kb = rdf_client.KnowledgeBase()
kb.users.Append(rdf_client.User(username="******", uid=1, sid="sid1"))
kb.users.Append(rdf_client.User(username="******", uid=2, sid="sid2"))
kb.Set("environ_allusersprofile", "c:\\programdata")
paths = artifact_utils.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\%%users.sid%%\\%%users.username%%", kb)
self.assertCountEqual(
paths, ["c:\\programdata\\sid1\\joe", "c:\\programdata\\sid2\\jim"])
# Only one user has a temp dir set. If we request all usernames and temp
# dirs, we should only get one entry.
kb.users.Append(
rdf_client.User(
username="******",
uid=1,
temp="C:\\Users\\jason\\AppData\\Local\\Temp"))
paths = artifact_utils.InterpolateKbAttributes(
"%%users.temp%%\\%%users.username%%.txt", kb)
self.assertCountEqual(paths,
["C:\\Users\\jason\\AppData\\Local\\Temp\\jason.txt"])
def WMIQuery(self, source):
"""Run a Windows WMI Query."""
query = source.attributes["query"]
queries = artifact_utils.InterpolateKbAttributes(
query,
self.state.knowledge_base,
ignore_errors=self.args.ignore_interpolation_errors)
base_object = source.attributes.get("base_object")
for query in queries:
self.CallClient(server_stubs.WmiQuery,
query=query,
base_object=base_object,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollected")
def _ProcessFileSource(self, source):
"""Prepare a request for the client file finder."""
if source.path_type != rdf_paths.PathSpec.PathType.OS:
raise ValueError("Only supported path type is OS.")
paths = []
for path in source.base_source.attributes["paths"]:
paths.extend(
artifact_utils.InterpolateKbAttributes(
path, self.knowledge_base, self.ignore_interpolation_errors))
file_finder_action = rdf_file_finder.FileFinderAction.Stat()
request = rdf_file_finder.FileFinderArgs(
paths=paths, pathtype=source.path_type, action=file_finder_action)
action = file_finder.FileFinderOSFromClient
yield action, request
def InterpolateList(self, input_list):
"""Interpolate all items from a given source array.
Args:
input_list: list of values to interpolate
Returns:
original list of values extended with strings interpolated
"""
new_args = []
for value in input_list:
if isinstance(value, basestring):
results = list(
artifact_utils.InterpolateKbAttributes(
value,
self.state.knowledge_base,
ignore_errors=self.args.ignore_interpolation_errors))
new_args.extend(results)
else:
new_args.extend(value)
return new_args
def GetFiles(self, source, path_type, max_size):
"""Get a set of files."""
new_path_list = []
for path in source.attributes["paths"]:
# Interpolate any attributes from the knowledgebase.
new_path_list.extend(
artifact_utils.InterpolateKbAttributes(
path,
self.state.knowledge_base,
ignore_errors=self.args.ignore_interpolation_errors))
action = rdf_file_finder.FileFinderAction.Download(max_size=max_size)
self.CallFlow(file_finder.FileFinder.__name__,
paths=new_path_list,
pathtype=path_type,
action=action,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessFileFinderResults")
