def _ProcessRegistryValueSource(self, args):
new_paths = set()
has_glob = False
for kvdict in args.base_source.attributes["key_value_pairs"]:
if "*" in kvdict["key"] or rdf_paths.GROUPING_PATTERN.search(
kvdict["key"]):
has_glob = True
if kvdict["value"]:
path = "\\".join((kvdict["key"], kvdict["value"]))
else:
path = kvdict["key"]
expanded_paths = artifact_utils.InterpolateKbAttributes(
path,
self.knowledge_base,
ignore_errors=self.ignore_interpolation_errors)
new_paths.update(expanded_paths)
if has_glob:
# TODO(user): If a path has a wildcard we need to glob the filesystem
# for patterns to collect matching files. The corresponding flow is
# filesystem.Glob.
pass
else:
action = standard.GetFileStat
for new_path in new_paths:
pathspec = rdf_paths.PathSpec(
path=new_path,
pathtype=rdf_paths.PathSpec.PathType.REGISTRY)
request = rdf_client_action.GetFileStatRequest(
pathspec=pathspec)
yield action, request
def _ScheduleStatFile(self, index: int, pathspec: rdf_paths.PathSpec) -> None:
"""Schedules the appropriate Stat File Client Action.
Args:
index: Index of the current file to get Stat for.
pathspec: Pathspec of the current file to get Stat for.
"""
# Add the file tracker to the pending stats list where it waits until the
# stat comes back.
self.state.pending_stats[index] = {"index": index}
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if not self.client_version or self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(pathspec=pathspec)
request.follow_symlink = True
request_name = "GetFileStat"
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(pathspec=pathspec)
request_name = "StatFile"
self.CallClient(
stub,
request,
next_state=compatibility.GetName(self._ReceiveFileStat),
request_data=dict(index=index, request_name=request_name))
def GetRegistryValue(self, source):
"""Retrieve directly specified registry values, returning Stat objects."""
new_paths = set()
has_glob = False
for kvdict in source.attributes["key_value_pairs"]:
if "*" in kvdict["key"] or rdf_paths.GROUPING_PATTERN.search(
kvdict["key"]):
has_glob = True
if kvdict["value"]:
# This currently only supports key value pairs specified using forward
# slash.
path = "\\".join((kvdict["key"], kvdict["value"]))
else:
# If value is not set, we want to get the default value. In
# GRR this is done by specifying the key only, so this is what
# we do here.
path = kvdict["key"]
expanded_paths = artifact_utils.InterpolateKbAttributes(
path,
self.state.knowledge_base,
ignore_errors=self.args.ignore_interpolation_errors)
new_paths.update(expanded_paths)
if has_glob:
self.CallFlow(
filesystem.Glob.__name__,
paths=new_paths,
pathtype=rdf_paths.PathSpec.PathType.REGISTRY,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollected")
else:
# We call statfile directly for keys that don't include globs because it
# is faster and some artifacts rely on getting an IOError to trigger
# fallback processing.
for new_path in new_paths:
pathspec = rdf_paths.PathSpec(
path=new_path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY)
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(pathspec=pathspec)
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(pathspec=pathspec)
self.CallClient(
stub,
request,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollectedRegistryStatEntry")
def Start(self):
"""Get information about the file from the client."""
self.state.max_chunk_number = max(
2, self.args.read_length // self.CHUNK_SIZE)
self.state.current_chunk_number = 0
self.state.file_size = 0
self.state.blobs = []
self.state.stat_entry = None
self.state.num_bytes_collected = 0
self.state.target_pathspec = self.args.pathspec.Copy()
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if not self.client_version or self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(
pathspec=self.state.target_pathspec, follow_symlink=True)
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(
pathspec=self.state.target_pathspec)
self.CallClient(stub,
request,
next_state=compatibility.GetName(self.Stat))
def _TryToStartNextPathspec(self):
"""Try to schedule the next pathspec if there is enough capacity."""
# Nothing to do here.
if self.state.maximum_pending_files <= len(self.state.pending_files):
return
if self.state.maximum_pending_files <= len(self.state.pending_hashes):
return
try:
index = self.state.next_pathspec_to_start
pathspec = self.state.indexed_pathspecs[index]
self.state.next_pathspec_to_start = index + 1
except IndexError:
# We did all the pathspecs, nothing left to do here.
return
# Add the file tracker to the pending hashes list where it waits until the
# hash comes back.
self.state.pending_hashes[index] = {"index": index}
# First state the file, then hash the file.
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if not self.client_version or self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(pathspec=pathspec)
request.follow_symlink = True
request_name = "GetFileStat"
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(pathspec=pathspec)
request_name = "StatFile"
self.CallClient(stub,
request,
next_state=compatibility.GetName(self._StoreStat),
request_data=dict(index=index,
request_name=request_name))
request = rdf_client_action.FingerprintRequest(
pathspec=pathspec, max_filesize=self.state.file_size)
request.AddRequest(
fp_type=rdf_client_action.FingerprintTuple.Type.FPT_GENERIC,
hashers=[
rdf_client_action.FingerprintTuple.HashType.MD5,
rdf_client_action.FingerprintTuple.HashType.SHA1,
rdf_client_action.FingerprintTuple.HashType.SHA256
])
self.CallClient(server_stubs.HashFile,
request,
next_state=compatibility.GetName(
self._ReceiveFileHash),
request_data=dict(index=index))
def Start(self):
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(
pathspec=self.args.pathspec)
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(pathspec=self.args.pathspec)
self.CallClient(stub, request, next_state="ProcessStat")
def testStatSize(self):
with temp.AutoTempFilePath() as temp_filepath:
with io.open(temp_filepath, "wb") as temp_file:
temp_file.write(b"123456")
pathspec = rdf_paths.PathSpec(
path=temp_filepath, pathtype=rdf_paths.PathSpec.PathType.OS)
request = rdf_client_action.GetFileStatRequest(pathspec=pathspec)
results = self.RunAction(standard.GetFileStat, request)
self.assertLen(results, 1)
self.assertEqual(results[0].st_size, 6)
def testStatExtAttrsDisabled(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.SetExtAttr(temp_filepath, name="user.foo", value="bar")
pathspec = rdf_paths.PathSpec(
path=temp_filepath, pathtype=rdf_paths.PathSpec.PathType.OS)
request = rdf_client_action.GetFileStatRequest(
pathspec=pathspec, collect_ext_attrs=False)
results = self.RunAction(standard.GetFileStat, request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].ext_attrs), 0)
def _FileCollectionFromColumns(
self,
responses: Iterable[rdf_osquery.OsqueryResult],
) -> None:
pathspecs = self._GetPathSpecsToCollect(responses)
stub = server_stubs.GetFileStat
for pathspec in pathspecs:
request = rdf_client_action.GetFileStatRequest(pathspec=pathspec)
self.CallClient(stub,
request,
next_state=compatibility.GetName(
self._StatForFileArrived))
def Start(self):
"""Issue a request to list the directory."""
self.state.urn = None
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(
pathspec=self.args.pathspec)
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(
pathspec=self.args.pathspec)
self.CallClient(stub, request, next_state="Stat")
# We use data to pass the path to the callback:
self.CallClient(server_stubs.ListDirectory,
pathspec=self.args.pathspec,
next_state="List")
def Start(self):
"""Get information about the file from the client."""
self.state.max_chunk_number = max(2,
self.args.read_length // self.CHUNK_SIZE)
self.state.current_chunk_number = 0
self.state.file_size = 0
self.state.blobs = []
self.state.stat_entry = None
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(
pathspec=self.args.pathspec)
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(pathspec=self.args.pathspec)
self.CallClient(stub, request, next_state="Stat")
def testFollowSymlinkEnabled(self):
data = b"quux" * 1024
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
target_filepath = os.path.join(temp_dirpath, "target")
symlink_filepath = os.path.join(temp_dirpath, "symlink")
filesystem_test_lib.CreateFile(target_filepath, data)
os.symlink(target_filepath, symlink_filepath)
request = rdf_client_action.GetFileStatRequest()
request.pathspec = rdf_paths.PathSpec.OS(path=symlink_filepath)
request.follow_symlink = True
results = self.RunAction(standard.GetFileStat, request)
self.assertLen(results, 1)
self.assertFalse(stat.S_ISLNK(int(results[0].st_mode)))
self.assertEqual(results[0].st_size, len(data))
# TODO: Required to clean-up the temp directory.
files.FlushHandleCache()
def _ProcessResponse(self, response, component_paths, base_wildcard=False):
for component_path in component_paths:
regexes_to_get = []
recursions_to_get = {}
node = self.FindNode(component_path)
if not node:
# Node is empty representing a leaf node - we found a hit - report it.
self.GlobReportMatch(response)
return
# There are further components in the tree - iterate over them.
for component_str, next_node in iteritems(node):
component = rdf_paths.PathSpec.FromSerializedString(
component_str)
next_component = component_path + [component_str]
# If we reach this point, we are instructed to go deeper into the
# directory structure. We only want to actually do this if
# - the last response was a proper directory,
# - or it was a file (an image) that was explicitly given meaning
# no wildcards or groupings,
# - or process_non_regular_files was set.
#
# This reduces the number of TSK opens on the client that may
# sometimes lead to instabilities due to bugs in the library.
if response and (not (stat.S_ISDIR(response.st_mode)
or not base_wildcard or
self.state.process_non_regular_files)):
continue
if component.path_options == component.Options.RECURSIVE:
recursions_to_get.setdefault(component.recursion_depth,
[]).append(component)
elif component.path_options == component.Options.REGEX:
regexes_to_get.append(component)
elif component.path_options == component.Options.CASE_INSENSITIVE:
# Here we need to create the next pathspec by appending the current
# component to what we already have. If we don't have anything yet, we
# fall back to the root path. If there is no root path either, the
# current component becomes the new base.
base_pathspec = self._GetBasePathspec(response)
if base_pathspec:
pathspec = base_pathspec.Append(component)
else:
pathspec = component
if not next_node:
# Check for the existence of the last node.
if (response is None
or (response and
(response.st_mode == 0
or not stat.S_ISREG(response.st_mode)))):
# If next node is empty, this node is a leaf node, we therefore
# must stat it to check that it is there. There is a special case
# here where this pathspec points to a file/directory in the root
# directory. In this case, response will be None but we still need
# to stat it.
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client_action.GetFileStatRequest(
pathspec=pathspec,
collect_ext_attrs=self.state.
collect_ext_attrs)
else:
stub = server_stubs.StatFile
request = rdf_client_action.ListDirRequest(
pathspec=pathspec)
self.CallClient(stub,
request,
next_state="ProcessEntry",
request_data=dict(
component_path=next_component))
else:
# There is no need to go back to the client for intermediate
# paths in the prefix tree, just emulate this by recursively
# calling this state inline.
self.CallStateInline(
[rdf_client_fs.StatEntry(pathspec=pathspec)],
next_state="ProcessEntry",
request_data=dict(component_path=next_component))
if recursions_to_get or regexes_to_get:
# Recursions or regexes need a base pathspec to operate on. If we
# have neither a response or a root path, we send a default pathspec
# that opens the root with pathtype set to the pathtype expected by the
# user.
base_pathspec = self._GetBasePathspec(response)
if not base_pathspec:
base_pathspec = rdf_paths.PathSpec(
path="/", pathtype=self.state.pathtype)
for depth, recursions in iteritems(recursions_to_get):
path_regex = "(?i)^" + "$|^".join(
set([c.path for c in recursions])) + "$"
findspec = rdf_client_fs.FindSpec(pathspec=base_pathspec,
cross_devs=True,
max_depth=depth,
path_regex=path_regex)
findspec.iterator.number = self.FILE_MAX_PER_DIR
self.CallClient(
server_stubs.Find,
findspec,
next_state="ProcessEntry",
request_data=dict(base_path=component_path))
if regexes_to_get:
path_regex = "(?i)^" + "$|^".join(
set([c.path for c in regexes_to_get])) + "$"
findspec = rdf_client_fs.FindSpec(pathspec=base_pathspec,
max_depth=1,
path_regex=path_regex)
findspec.iterator.number = self.FILE_MAX_PER_DIR
self.CallClient(
server_stubs.Find,
findspec,
next_state="ProcessEntry",
request_data=dict(base_path=component_path))
def ProcessAction(self, response):
"""Applies action specified by user to responses."""
action = self.args.action.action_type
if action == rdf_file_finder.FileFinderAction.Action.STAT:
# If we are dealing with the operating system file api, the stat action
# might need to collect extended attributes or gather information about
# links instead of their targets. In those cases, we need to issue more
# GetFileStatRequest client requests. In all other cases, we already have
# all the data we need to send the response.
s = self.args.action.stat
if (self.args.pathtype != rdf_paths.PathSpec.PathType.OS
or (s.resolve_links and not s.collect_ext_attrs)):
self.state.files_found += 1
self.SendReply(response)
else:
if self.client_version < 3221:
self.Error(
"Client is too old to get requested stat information.")
request = rdf_client_action.GetFileStatRequest(
pathspec=response.stat_entry.pathspec,
collect_ext_attrs=s.collect_ext_attrs,
follow_symlink=s.resolve_links)
self.CallClient(server_stubs.GetFileStat,
request,
next_state=compatibility.GetName(
self.ReceiveFileStat),
request_data=dict(original_result=response))
elif (self.args.process_non_regular_files
or stat.S_ISREG(int(response.stat_entry.st_mode))):
# Hashing and downloading are only safe for regular files. User has to
# explicitly set args.process_non_regular_files to True to make
# FileFinder look into non-regular files.
# In both cases (regular and non-regular files) max_size limit is applied
# (either action.hash.max_size or action.download.max_size, depending on
# the action type).
# Reply is sent only when we get file's hash.
self.state.files_found += 1
if action == rdf_file_finder.FileFinderAction.Action.HASH:
hash_file = False
file_size = response.stat_entry.st_size
if file_size > self.args.action.hash.max_size:
policy = self.args.action.hash.oversized_file_policy
options = rdf_file_finder.FileFinderHashActionOptions
if policy == options.OversizedFilePolicy.SKIP:
self.Log(
"%s too large to hash, skipping according to SKIP "
"policy. Size=%d",
response.stat_entry.pathspec.CollapsePath(),
file_size)
elif policy == options.OversizedFilePolicy.HASH_TRUNCATED:
self.Log(
"%s too large to hash, hashing its first %d bytes "
"according to HASH_TRUNCATED policy. Size=%d",
response.stat_entry.pathspec.CollapsePath(),
self.args.action.download.max_size, file_size)
hash_file = True
else:
hash_file = True
if hash_file:
self.FingerprintFile(
response.stat_entry.pathspec,
max_filesize=self.args.action.hash.max_size,
request_data=dict(original_result=response))
elif action == rdf_file_finder.FileFinderAction.Action.DOWNLOAD:
fetch_file = False
# If the binary is too large we don't download it, but take a
# fingerprint instead.
file_size = response.stat_entry.st_size
if file_size > self.args.action.download.max_size:
policy = self.args.action.download.oversized_file_policy
options = rdf_file_finder.FileFinderDownloadActionOptions
if policy == options.OversizedFilePolicy.SKIP:
self.Log(
"%s too large to fetch, skipping according to SKIP "
"policy. Size=%d",
response.stat_entry.pathspec.CollapsePath(),
file_size)
elif policy == options.OversizedFilePolicy.HASH_TRUNCATED:
self.Log(
"%s too large to fetch, hashing its first %d bytes "
"according to HASH_TRUNCATED policy. Size=%d",
response.stat_entry.pathspec.CollapsePath(),
self.args.action.download.max_size, file_size)
self.FingerprintFile(
response.stat_entry.pathspec,
max_filesize=self.args.action.download.max_size,
request_data=dict(original_result=response))
elif policy == "DOWNLOAD_TRUNCATED":
fetch_file = True
else:
fetch_file = True
if fetch_file:
self.StartFileFetch(
response.stat_entry.pathspec,
request_data=dict(original_result=response))
