def testIteratedListDirectory(self): """Tests iterated listing of directories.""" p = rdf_paths.PathSpec(path=self.base_path, pathtype=rdf_paths.PathSpec.PathType.OS) non_iterated_results = self.RunAction( "ListDirectory", rdf_client.ListDirRequest(pathspec=p)) # Make sure we get some results. l = len(non_iterated_results) self.assertTrue(l > 0) iterated_results = [] request = rdf_client.ListDirRequest(pathspec=p) request.iterator.number = 2 while True: responses = self.RunAction("IteratedListDirectory", request) results = responses[:-1] if not results: break for result in results: iterated_results.append(result) for x, y in zip(non_iterated_results, iterated_results): # Reset the st_atime in the results to avoid potential flakiness. x.st_atime = y.st_atime = 0 self.assertRDFValueEqual(x, y)
def testSuspendableListDirectory(self): request = rdf_client.ListDirRequest() request.pathspec.path = self.base_path request.pathspec.pathtype = "OS" request.iterator.number = 2 results = [] grr_worker = worker_mocks.FakeClientWorker() while request.iterator.state != request.iterator.State.FINISHED: responses = self.RunAction("SuspendableListDirectory", request, grr_worker=grr_worker) results.extend(responses) for response in responses: if isinstance(response, rdf_client.Iterator): request.iterator = response filenames = [ os.path.basename(r.pathspec.path) for r in results if isinstance(r, rdf_client.StatEntry) ] self.assertItemsEqual(filenames, os.listdir(self.base_path)) iterators = [r for r in results if isinstance(r, rdf_client.Iterator)] # One for two files plus one extra with the FINISHED status. nr_files = len(os.listdir(self.base_path)) expected_iterators = (nr_files / 2) + 1 if nr_files % 2: expected_iterators += 1 self.assertEqual(len(iterators), expected_iterators) # Make sure the thread has been deleted. self.assertEqual(grr_worker.suspended_actions, {})
def StatFile(self, args): """StatFile action mock.""" req = rdf_client.ListDirRequest(args) response = rdf_client.StatEntry(pathspec=req.pathspec, st_mode=33184, st_ino=1063090, st_dev=64512L, st_nlink=1, st_uid=139592, st_gid=5000, st_size=len(self.data), st_atime=1336469177, st_mtime=1336129892, st_ctime=1336129892) self.responses += 1 self.count += 1 # Create status message to report sample resource usage status = rdf_flows.GrrStatus( status=rdf_flows.GrrStatus.ReturnedStatus.OK) status.cpu_time_used.user_cpu_time = self.responses status.cpu_time_used.system_cpu_time = self.responses * 2 status.network_bytes_sent = self.responses * 3 # Every "failrate" client does not have this file. if self.count == self.failrate: self.count = 0 return [status] return [response, status]
def GetRegistryValue(self, source): """Retrieve directly specified registry values, returning Stat objects.""" new_paths = set() has_glob = False for kvdict in source.attributes["key_value_pairs"]: if "*" in kvdict["key"] or paths.GROUPING_PATTERN.search( kvdict["key"]): has_glob = True if kvdict["value"]: # This currently only supports key value pairs specified using forward # slash. path = "\\".join((kvdict["key"], kvdict["value"])) else: # If value is not set, we want to get the default value. In # GRR this is done by specifying the key only, so this is what # we do here. path = kvdict["key"] expanded_paths = artifact_utils.InterpolateKbAttributes( path, self.state.knowledge_base, ignore_errors=self.args.ignore_interpolation_errors) new_paths.update(expanded_paths) if has_glob: self.CallFlow(filesystem.Glob.__name__, paths=new_paths, pathtype=paths.PathSpec.PathType.REGISTRY, request_data={ "artifact_name": self.current_artifact_name, "source": source.ToPrimitiveDict() }, next_state="ProcessCollected") else: # We call statfile directly for keys that don't include globs because it # is faster and some artifacts rely on getting an IOError to trigger # fallback processing. for new_path in new_paths: pathspec = paths.PathSpec( path=new_path, pathtype=paths.PathSpec.PathType.REGISTRY) # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client.GetFileStatRequest(pathspec=pathspec) else: stub = server_stubs.StatFile request = rdf_client.ListDirRequest(pathspec=pathspec) self.CallClient(stub, request, request_data={ "artifact_name": self.current_artifact_name, "source": source.ToPrimitiveDict() }, next_state="ProcessCollectedRegistryStatEntry")
def Start(self): # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client.GetFileStatRequest(pathspec=self.args.pathspec) else: stub = server_stubs.StatFile request = rdf_client.ListDirRequest(pathspec=self.args.pathspec) self.CallClient(stub, request, next_state="ProcessStat")
def Start(self): """Get information about the file from the client.""" self.state.Register("max_chunk_number", max(2, self.args.read_length / self.CHUNK_SIZE)) self.state.Register("current_chunk_number", 0) self.state.Register("file_size", 0) self.state.Register("fd", None) self.state.Register("stat", None) self.CallClient("StatFile", rdf_client.ListDirRequest( pathspec=self.args.pathspec), next_state="Stat")
def Start(self): """Get information about the file from the client.""" self.state.max_chunk_number = max( 2, self.args.read_length / self.CHUNK_SIZE) self.state.current_chunk_number = 0 self.state.file_size = 0 self.state.blobs = [] self.state.stat_entry = None self.CallClient(server_stubs.StatFile, rdf_client.ListDirRequest(pathspec=self.args.pathspec), next_state="Stat")
def _TryToStartNextPathspec(self): """Try to schedule the next pathspec if there is enough capacity.""" # Nothing to do here. if self.state.maximum_pending_files <= len(self.state.pending_files): return if self.state.maximum_pending_files <= len(self.state.pending_hashes): return try: index = self.state.next_pathspec_to_start pathspec = self.state.indexed_pathspecs[index] self.state.next_pathspec_to_start = index + 1 except IndexError: # We did all the pathspecs, nothing left to do here. return # Add the file tracker to the pending hashes list where it waits until the # hash comes back. self.state.pending_hashes[index] = {"index": index} # First state the file, then hash the file. # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client.GetFileStatRequest(pathspec=pathspec) else: stub = server_stubs.StatFile request = rdf_client.ListDirRequest(pathspec=pathspec) self.CallClient(stub, request, next_state="StoreStat", request_data=dict(index=index)) request = rdf_client.FingerprintRequest( pathspec=pathspec, max_filesize=self.state.file_size) request.AddRequest( fp_type=rdf_client.FingerprintTuple.Type.FPT_GENERIC, hashers=[ rdf_client.FingerprintTuple.HashType.MD5, rdf_client.FingerprintTuple.HashType.SHA1, rdf_client.FingerprintTuple.HashType.SHA256 ]) self.CallClient(server_stubs.HashFile, request, next_state="ReceiveFileHash", request_data=dict(index=index))
def Start(self): """Issue a request to list the directory.""" self.state.responses = [] self.state.urn = None # We use data to pass the path to the callback: self.state.request = rdf_client.ListDirRequest(pathspec=self.args.pathspec) # For this example we will use a really small number to force many round # trips with the client. This is a performance killer. self.state.request.iterator.number = 50 self.CallClient( "IteratedListDirectory", self.state.request, next_state="List")
def testListDirectory(self): """Tests listing directories.""" p = rdf_paths.PathSpec(path=self.base_path, pathtype=0) results = self.RunAction("ListDirectory", rdf_client.ListDirRequest(pathspec=p)) # Find the number.txt file result = None for result in results: if os.path.basename(result.pathspec.path) == "morenumbers.txt": break self.assert_(result) self.assertEqual(result.__class__, rdf_client.StatEntry) self.assertEqual(result.pathspec.Basename(), "morenumbers.txt") self.assertEqual(result.st_size, 3893) self.assert_(stat.S_ISREG(result.st_mode))
def testClientAction(self): client_mock = action_mocks.ActionMock(standard.ListDirectory) pathspec = rdf_paths.PathSpec(path=os.path.join( self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) request = rdf_client.ListDirRequest(pathspec=pathspec) for _ in test_lib.TestFlowHelper("ClientAction", client_mock, client_id=self.client_id, action="ListDirectory", break_pdb=False, action_args=request, token=self.token): pass
def testSuspendableActionException(self): class testActionWorker(actions.ClientActionWorker): def run(self): try: return super(testActionWorker, self).run() except Exception as e: # pylint: disable=broad-except logging.info("Expected exception: %s", e) class RaisingListDirectory(standard.SuspendableListDirectory): iterations = 3 def Suspend(self): RaisingListDirectory.iterations -= 1 if not RaisingListDirectory.iterations: raise IOError("Ran out of iterations.") return super(RaisingListDirectory, self).Suspend() p = rdf_paths.PathSpec(path=self.base_path, pathtype=rdf_paths.PathSpec.PathType.OS) request = rdf_client.ListDirRequest(pathspec=p) request.iterator.number = 2 results = [] grr_worker = worker_mocks.FakeClientWorker() while request.iterator.state != request.iterator.State.FINISHED: responses = self.ExecuteAction("RaisingListDirectory", request, grr_worker=grr_worker, action_worker_cls=testActionWorker) results.extend(responses) for response in responses: if isinstance(response, rdf_client.Iterator): request.iterator = response status = responses[-1] self.assertTrue(isinstance(status, rdf_flows.GrrStatus)) if status.status != rdf_flows.GrrStatus.ReturnedStatus.OK: break if len(results) > 100: self.fail("Endless loop detected.") self.assertIn("Ran out of iterations", status.error_message) self.assertEqual(grr_worker.suspended_actions, {})
def Start(self): """Issue a request to list the directory.""" self.state.urn = None # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client.GetFileStatRequest(pathspec=self.args.pathspec) else: stub = server_stubs.StatFile request = rdf_client.ListDirRequest(pathspec=self.args.pathspec) self.CallClient(stub, request, next_state="Stat") # We use data to pass the path to the callback: self.CallClient( server_stubs.ListDirectory, pathspec=self.args.pathspec, next_state="List")
def Start(self): """Get information about the file from the client.""" self.state.max_chunk_number = max( 2, self.args.read_length / self.CHUNK_SIZE) self.state.current_chunk_number = 0 self.state.file_size = 0 self.state.blobs = [] self.state.stat_entry = None # TODO(hanuszczak): Support for old clients ends on 2021-01-01. # This conditional should be removed after that date. if self.client_version >= 3221: stub = server_stubs.GetFileStat request = rdf_client.GetFileStatRequest( pathspec=self.args.pathspec) else: stub = server_stubs.StatFile request = rdf_client.ListDirRequest(pathspec=self.args.pathspec) self.CallClient(stub, request, next_state="Stat")
def _ProcessResponse(self, response, component_paths, base_wildcard=False): for component_path in component_paths: regexes_to_get = [] recursions_to_get = {} node = self.FindNode(component_path) if not node: # Node is empty representing a leaf node - we found a hit - report it. self.GlobReportMatch(response) return # There are further components in the tree - iterate over them. for component_str, next_node in node.items(): component = rdf_paths.PathSpec.FromSerializedString( component_str) next_component = component_path + [component_str] # If we reach this point, we are instructed to go deeper into the # directory structure. We only want to actually do this if # - the last response was a proper directory, # - or it was a file (an image) that was explicitly given meaning # no wildcards or groupings, # - or process_non_regular_files was set. # # This reduces the number of TSK opens on the client that may # sometimes lead to instabilities due to bugs in the library. if response and (not (stat.S_ISDIR(response.st_mode) or not base_wildcard or self.state.process_non_regular_files)): continue if component.path_options == component.Options.RECURSIVE: recursions_to_get.setdefault(component.recursion_depth, []).append(component) elif component.path_options == component.Options.REGEX: regexes_to_get.append(component) elif component.path_options == component.Options.CASE_INSENSITIVE: # Here we need to create the next pathspec by appending the current # component to what we already have. If we don't have anything yet, we # fall back to the root path. If there is no root path either, the # current component becomes the new base. base_pathspec = self._GetBasePathspec(response) if base_pathspec: pathspec = base_pathspec.Append(component) else: pathspec = component if not next_node: # Check for the existence of the last node. request = rdf_client.ListDirRequest(pathspec=pathspec) if (response is None or (response and (response.st_mode == 0 or not stat.S_ISREG(response.st_mode)))): # If next node is empty, this node is a leaf node, we therefore # must stat it to check that it is there. There is a special case # here where this pathspec points to a file/directory in the root # directory. In this case, response will be None but we still need # to stat it. self.CallClient(server_stubs.StatFile, request, next_state="ProcessEntry", request_data=dict( component_path=next_component)) else: # There is no need to go back to the client for intermediate # paths in the prefix tree, just emulate this by recursively # calling this state inline. self.CallStateInline( [rdf_client.StatEntry(pathspec=pathspec)], next_state="ProcessEntry", request_data=dict(component_path=next_component)) if recursions_to_get or regexes_to_get: # Recursions or regexes need a base pathspec to operate on. If we # have neither a response or a root path, we send a default pathspec # that opens the root with pathtype "OS". base_pathspec = self._GetBasePathspec(response) if not base_pathspec: base_pathspec = rdf_paths.PathSpec(path="/", pathtype="OS") for depth, recursions in recursions_to_get.iteritems(): path_regex = "(?i)^" + "$|^".join( set([c.path for c in recursions])) + "$" findspec = rdf_client.FindSpec(pathspec=base_pathspec, cross_devs=True, max_depth=depth, path_regex=path_regex) findspec.iterator.number = self.FILE_MAX_PER_DIR self.CallClient( server_stubs.Find, findspec, next_state="ProcessEntry", request_data=dict(base_path=component_path)) if regexes_to_get: path_regex = "(?i)^" + "$|^".join( set([c.path for c in regexes_to_get])) + "$" findspec = rdf_client.FindSpec(pathspec=base_pathspec, max_depth=1, path_regex=path_regex) findspec.iterator.number = self.FILE_MAX_PER_DIR self.CallClient( server_stubs.Find, findspec, next_state="ProcessEntry", request_data=dict(base_path=component_path))