def testIteratedListDirectory(self): """Tests iterated listing of directories.""" p = rdf_paths.PathSpec( path=self.base_path, pathtype=rdf_paths.PathSpec.PathType.OS) non_iterated_results = self.RunAction( standard.ListDirectory, rdf_client_action.ListDirRequest(pathspec=p)) # Make sure we get some results. l = len(non_iterated_results) self.assertTrue(l > 0) iterated_results = [] request = rdf_client_action.ListDirRequest(pathspec=p) request.iterator.number = 2 while True: responses = self.RunAction(standard.IteratedListDirectory, request) results = responses[:-1] if not results: break for result in results: iterated_results.append(result) for x, y in zip(non_iterated_results, iterated_results): # Reset the st_atime in the results to avoid potential flakiness. x.st_atime = y.st_atime = 0 self.assertRDFValuesEqual(x, y)
def _ScheduleStatFile(self, index: int, pathspec: rdf_paths.PathSpec) -> None: """Schedules the appropriate Stat File Client Action. Args: index: Index of the current file to get Stat for. pathspec: Pathspec of the current file to get Stat for. """ # Add the file tracker to the pending stats list where it waits until the # stat comes back. self.state.pending_stats[index] = {"index": index} # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if not self.client_version or self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest(pathspec=pathspec) request.follow_symlink = True request_name = "GetFileStat" else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest(pathspec=pathspec) request_name = "StatFile" self.CallClient( stub, request, next_state=compatibility.GetName(self._ReceiveFileStat), request_data=dict(index=index, request_name=request_name))
def GetRegistryValue(self, source): """Retrieve directly specified registry values, returning Stat objects.""" new_paths = set() has_glob = False for kvdict in source.attributes["key_value_pairs"]: if "*" in kvdict["key"] or rdf_paths.GROUPING_PATTERN.search( kvdict["key"]): has_glob = True if kvdict["value"]: # This currently only supports key value pairs specified using forward # slash. path = "\\".join((kvdict["key"], kvdict["value"])) else: # If value is not set, we want to get the default value. In # GRR this is done by specifying the key only, so this is what # we do here. path = kvdict["key"] expanded_paths = artifact_utils.InterpolateKbAttributes( path, self.state.knowledge_base, ignore_errors=self.args.ignore_interpolation_errors) new_paths.update(expanded_paths) if has_glob: self.CallFlow( filesystem.Glob.__name__, paths=new_paths, pathtype=rdf_paths.PathSpec.PathType.REGISTRY, request_data={ "artifact_name": self.current_artifact_name, "source": source.ToPrimitiveDict() }, next_state="ProcessCollected") else: # We call statfile directly for keys that don't include globs because it # is faster and some artifacts rely on getting an IOError to trigger # fallback processing. for new_path in new_paths: pathspec = rdf_paths.PathSpec( path=new_path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY) # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest(pathspec=pathspec) else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest(pathspec=pathspec) self.CallClient( stub, request, request_data={ "artifact_name": self.current_artifact_name, "source": source.ToPrimitiveDict() }, next_state="ProcessCollectedRegistryStatEntry")
def Start(self): """Get information about the file from the client.""" self.state.max_chunk_number = max( 2, self.args.read_length // self.CHUNK_SIZE) self.state.current_chunk_number = 0 self.state.file_size = 0 self.state.blobs = [] self.state.stat_entry = None self.state.num_bytes_collected = 0 self.state.target_pathspec = self.args.pathspec.Copy() # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if not self.client_version or self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest( pathspec=self.state.target_pathspec, follow_symlink=True) else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest( pathspec=self.state.target_pathspec) self.CallClient(stub, request, next_state=compatibility.GetName(self.Stat))
def _TryToStartNextPathspec(self): """Try to schedule the next pathspec if there is enough capacity.""" # Nothing to do here. if self.state.maximum_pending_files <= len(self.state.pending_files): return if self.state.maximum_pending_files <= len(self.state.pending_hashes): return try: index = self.state.next_pathspec_to_start pathspec = self.state.indexed_pathspecs[index] self.state.next_pathspec_to_start = index + 1 except IndexError: # We did all the pathspecs, nothing left to do here. return # Add the file tracker to the pending hashes list where it waits until the # hash comes back. self.state.pending_hashes[index] = {"index": index} # First state the file, then hash the file. # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if not self.client_version or self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest(pathspec=pathspec) request.follow_symlink = True request_name = "GetFileStat" else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest(pathspec=pathspec) request_name = "StatFile" self.CallClient(stub, request, next_state=compatibility.GetName(self._StoreStat), request_data=dict(index=index, request_name=request_name)) request = rdf_client_action.FingerprintRequest( pathspec=pathspec, max_filesize=self.state.file_size) request.AddRequest( fp_type=rdf_client_action.FingerprintTuple.Type.FPT_GENERIC, hashers=[ rdf_client_action.FingerprintTuple.HashType.MD5, rdf_client_action.FingerprintTuple.HashType.SHA1, rdf_client_action.FingerprintTuple.HashType.SHA256 ]) self.CallClient(server_stubs.HashFile, request, next_state=compatibility.GetName( self._ReceiveFileHash), request_data=dict(index=index))
def Start(self): # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest( pathspec=self.args.pathspec) else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest(pathspec=self.args.pathspec) self.CallClient(stub, request, next_state="ProcessStat")
def testListDirectory(self): """Tests listing directories.""" p = rdf_paths.PathSpec(path=self.base_path, pathtype=0) results = self.RunAction(standard.ListDirectory, rdf_client_action.ListDirRequest(pathspec=p)) # Find the number.txt file result = None for result in results: if os.path.basename(result.pathspec.path) == "morenumbers.txt": break self.assertTrue(result) self.assertEqual(result.__class__, rdf_client_fs.StatEntry) self.assertEqual(result.pathspec.Basename(), "morenumbers.txt") self.assertEqual(result.st_size, 3893) self.assertTrue(stat.S_ISREG(int(result.st_mode)))
def Start(self): """Issue a request to list the directory.""" self.state.responses = [] self.state.urn = None # We use data to pass the path to the callback: self.state.request = rdf_client_action.ListDirRequest( pathspec=self.args.pathspec) # For this example we will use a really small number to force many round # trips with the client. This is a performance killer. self.state.request.iterator.number = 50 self.CallClient( server_stubs.IteratedListDirectory, self.state.request, next_state="List")
def Start(self): """Issue a request to list the directory.""" self.state.urn = None # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest( pathspec=self.args.pathspec) else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest( pathspec=self.args.pathspec) self.CallClient(stub, request, next_state="Stat") # We use data to pass the path to the callback: self.CallClient(server_stubs.ListDirectory, pathspec=self.args.pathspec, next_state="List")
def Start(self): """Get information about the file from the client.""" self.state.max_chunk_number = max(2, self.args.read_length // self.CHUNK_SIZE) self.state.current_chunk_number = 0 self.state.file_size = 0 self.state.blobs = [] self.state.stat_entry = None # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest( pathspec=self.args.pathspec) else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest(pathspec=self.args.pathspec) self.CallClient(stub, request, next_state="Stat")
def _ProcessResponse(self, response, component_paths, base_wildcard=False): for component_path in component_paths: regexes_to_get = [] recursions_to_get = {} node = self.FindNode(component_path) if not node: # Node is empty representing a leaf node - we found a hit - report it. self.GlobReportMatch(response) return # There are further components in the tree - iterate over them. for component_str, next_node in iteritems(node): component = rdf_paths.PathSpec.FromSerializedString( component_str) next_component = component_path + [component_str] # If we reach this point, we are instructed to go deeper into the # directory structure. We only want to actually do this if # - the last response was a proper directory, # - or it was a file (an image) that was explicitly given meaning # no wildcards or groupings, # - or process_non_regular_files was set. # # This reduces the number of TSK opens on the client that may # sometimes lead to instabilities due to bugs in the library. if response and (not (stat.S_ISDIR(response.st_mode) or not base_wildcard or self.state.process_non_regular_files)): continue if component.path_options == component.Options.RECURSIVE: recursions_to_get.setdefault(component.recursion_depth, []).append(component) elif component.path_options == component.Options.REGEX: regexes_to_get.append(component) elif component.path_options == component.Options.CASE_INSENSITIVE: # Here we need to create the next pathspec by appending the current # component to what we already have. If we don't have anything yet, we # fall back to the root path. If there is no root path either, the # current component becomes the new base. base_pathspec = self._GetBasePathspec(response) if base_pathspec: pathspec = base_pathspec.Append(component) else: pathspec = component if not next_node: # Check for the existence of the last node. if (response is None or (response and (response.st_mode == 0 or not stat.S_ISREG(response.st_mode)))): # If next node is empty, this node is a leaf node, we therefore # must stat it to check that it is there. There is a special case # here where this pathspec points to a file/directory in the root # directory. In this case, response will be None but we still need # to stat it. # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client_action.GetFileStatRequest( pathspec=pathspec, collect_ext_attrs=self.state. collect_ext_attrs) else: stub = server_stubs.StatFile request = rdf_client_action.ListDirRequest( pathspec=pathspec) self.CallClient(stub, request, next_state="ProcessEntry", request_data=dict( component_path=next_component)) else: # There is no need to go back to the client for intermediate # paths in the prefix tree, just emulate this by recursively # calling this state inline. self.CallStateInline( [rdf_client_fs.StatEntry(pathspec=pathspec)], next_state="ProcessEntry", request_data=dict(component_path=next_component)) if recursions_to_get or regexes_to_get: # Recursions or regexes need a base pathspec to operate on. If we # have neither a response or a root path, we send a default pathspec # that opens the root with pathtype set to the pathtype expected by the # user. base_pathspec = self._GetBasePathspec(response) if not base_pathspec: base_pathspec = rdf_paths.PathSpec( path="/", pathtype=self.state.pathtype) for depth, recursions in iteritems(recursions_to_get): path_regex = "(?i)^" + "$|^".join( set([c.path for c in recursions])) + "$" findspec = rdf_client_fs.FindSpec(pathspec=base_pathspec, cross_devs=True, max_depth=depth, path_regex=path_regex) findspec.iterator.number = self.FILE_MAX_PER_DIR self.CallClient( server_stubs.Find, findspec, next_state="ProcessEntry", request_data=dict(base_path=component_path)) if regexes_to_get: path_regex = "(?i)^" + "$|^".join( set([c.path for c in regexes_to_get])) + "$" findspec = rdf_client_fs.FindSpec(pathspec=base_pathspec, max_depth=1, path_regex=path_regex) findspec.iterator.number = self.FILE_MAX_PER_DIR self.CallClient( server_stubs.Find, findspec, next_state="ProcessEntry", request_data=dict(base_path=component_path))