def RunHunt(self, plugin_name, plugin_args): with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs(pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS)), regex_rules=[ rdfvalue.ForemanAttributeRegex(attribute_name="GRR client", attribute_regex="GRR") ], output_plugins=[ rdfvalue.OutputPlugin(plugin_name=plugin_name, plugin_args=plugin_args) ], client_rate=0, token=self.token) as hunt: hunt.Run() hunt.StartClients(hunt.session_id, self.client_ids) # Run the hunt. client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, self.client_ids, False, self.token) # Stop the hunt now. hunt.GetRunner().Stop() # Run cron flow that executes actual output plugins for _ in test_lib.TestFlowHelper("ProcessHuntResultsCronFlow", token=self.token): pass return hunt.urn
def CreateSampleHunt(self, stopped=False): self.client_ids = self.SetupClients(10) with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs( flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs( pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS, ) ), regex_rules=[rdfvalue.ForemanAttributeRegex( attribute_name="GRR client", attribute_regex="GRR")], output_plugins=[], client_rate=0, token=self.token) as hunt: if not stopped: hunt.Run() with aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token) as foreman: for client_id in self.client_ids: foreman.AssignTasksToClient(client_id) self.hunt_urn = hunt.urn return aff4.FACTORY.Open(hunt.urn, mode="rw", token=self.token, age=aff4.ALL_TIMES)
def testCreatorPropagation(self): self.CreateAdminUser("adminuser") admin_token = access_control.ACLToken(username="******", reason="testing") # Start a flow that requires admin privileges in the hunt. The # parameters are not valid so the flow will error out but it's # enough to check if the flow was actually run (i.e., it passed # the label test). with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="UpdateClient"), flow_args=rdfvalue.UpdateClientArgs(), regex_rules=[ rdfvalue.ForemanAttributeRegex(attribute_name="GRR client", attribute_regex="GRR"), ], client_rate=0, token=admin_token) as hunt: hunt.Run() self.CreateUser("nonadmin") nonadmin_token = access_control.ACLToken(username="******", reason="testing") self.AssignTasksToClients() client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, self.client_ids, False, nonadmin_token) errors = list(hunt.GetClientsErrors()) # Make sure there are errors... self.assertTrue(errors) # but they are not UnauthorizedAccess. for e in errors: self.assertTrue("UnauthorizedAccess" not in e.backtrace)
def testStoppingHuntMarksAllStartedFlowsAsPendingForTermination(self): with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="InfiniteFlow"), regex_rules=[ rdfvalue.ForemanAttributeRegex(attribute_name="GRR client", attribute_regex="GRR"), ], client_rate=0, token=self.token) as hunt: hunt.Run() self.AssignTasksToClients() # Run long enough for InfiniteFlows to start. self.RunHunt(iteration_limit=len(self.client_ids) * 2) self.StopHunt(hunt.urn) # All flows should be marked for termination now. RunHunt should raise. # If something is wrong with GRRFlow.MarkForTermination mechanism, then # this will run forever. self.RunHunt() for client_id in self.client_ids: flows_root = aff4.FACTORY.Open(client_id.Add("flows"), token=self.token) flows_list = list(flows_root.ListChildren()) # Only one flow (issued by the hunt) is expected. self.assertEqual(len(flows_list), 1) flow_obj = aff4.FACTORY.Open(flows_list[0], aff4_type="InfiniteFlow", token=self.token) self.assertEqual(flow_obj.state.context.state, "ERROR") self.assertEqual(flow_obj.state.context.backtrace, "Parent hunt stopped.")
def InitializeContext(self, args): """Initializes the context of this flow.""" if args is None: args = rdfvalue.FlowRunnerArgs() context = flows.DataObject( args=args, backtrace=None, client_resources=rdfvalue.ClientResources(), create_time=rdfvalue.RDFDatetime().Now(), creator=self.token.username, current_state="Start", network_bytes_sent=0, next_outbound_id=1, next_processed_request=1, next_states=set(), output=self._CreateOutputCollection(args), outstanding_requests=0, remaining_cpu_quota=args.cpu_limit, state=rdfvalue.Flow.State.RUNNING, user=self.token.username, # Have we sent a notification to the user. user_notified=False, ) # Store the context in the flow_obj for next time. self.flow_obj.state.Register("context", context) return context
def _AppendFlowRequest(self, flows, client_id, file_id): flows.Append( client_ids=["C.1%015d" % client_id], runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), args=rdfvalue.GetFileArgs( pathspec=rdfvalue.PathSpec( path="/tmp/evil%s.txt" % file_id, pathtype=rdfvalue.PathSpec.PathType.OS), ) )
def testHuntExpiration(self): """This tests that hunts with a client limit terminate correctly.""" with test_lib.FakeTime(1000): with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs( flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs(pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS)), regex_rules=[ rdfvalue.ForemanAttributeRegex( attribute_name="GRR client", attribute_regex="GRR") ], client_limit=5, expiry_time=rdfvalue.Duration("1000s"), token=self.token) as hunt: hunt.Run() # Pretend to be the foreman now and dish out hunting jobs to all the # clients (Note we have 10 clients here). foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token) for client_id in self.client_ids: foreman.AssignTasksToClient(client_id) hunt_obj = aff4.FACTORY.Open(hunt.session_id, age=aff4.ALL_TIMES, token=self.token) self.assertEqual(hunt_obj.Get(hunt_obj.Schema.STATE), "STARTED") # Now advance the time such that the hunt expires. time.time = lambda: 5000 # Run the hunt. client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, self.client_ids, check_flow_errors=False, token=self.token) # No client should be processed since the hunt is expired. started, finished, errors = hunt_obj.GetClientsCounts() self.assertEqual(started, 0) self.assertEqual(finished, 0) self.assertEqual(errors, 0) hunt_obj = aff4.FACTORY.Open(hunt.session_id, age=aff4.ALL_TIMES, token=self.token) # Hunts are automatically stopped when they expire. self.assertEqual(hunt_obj.Get(hunt_obj.Schema.STATE), "STOPPED")
def CreateSampleHunt(description, token=None): return hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", description=description, flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs(pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS, )), client_rate=0, token=token)
def _CreateHunt(self, token): return hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs(pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS)), regex_rules=[ rdfvalue.ForemanAttributeRegex(attribute_name="GRR client", attribute_regex="GRR"), ], client_rate=0, token=token)
def InitializeContext(self, args): """Initializes the context of this flow.""" if args is None: args = rdfvalue.FlowRunnerArgs() output_collection = self._CreateOutputCollection(args) # Output collection is nullified when flow is terminated, so we're # keeping the urn separately for further reference. output_urn = (output_collection is not None) and output_collection.urn output_plugins_states = [] for plugin_descriptor in args.output_plugins: plugin_class = plugin_descriptor.GetPluginClass() plugin = plugin_class(output_urn, args=plugin_descriptor.plugin_args, token=self.token) try: plugin.Initialize() output_plugins_states.append((plugin_descriptor, plugin.state)) except Exception as e: # pylint: disable=broad-except self.Log("Plugin %s failed to initialize (%s), ignoring it." % (plugin, e)) context = utils.DataObject( args=args, backtrace=None, client_resources=rdfvalue.ClientResources(), create_time=rdfvalue.RDFDatetime().Now(), creator=args.creator or self.token.username, current_state="Start", # If not None, kill-stuck-flow notification is scheduled at the given # time. kill_timestamp=None, network_bytes_sent=0, next_outbound_id=1, next_processed_request=1, next_states=set(), output=output_collection, output_plugins_states=output_plugins_states, output_urn=output_urn, outstanding_requests=0, remaining_cpu_quota=args.cpu_limit, state=rdfvalue.Flow.State.RUNNING, # Have we sent a notification to the user. user_notified=False, ) # Store the context in the flow_obj for next time. self.flow_obj.state.Register("context", context) return context
def CreateSampleHunt(self, description): hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", description=description, flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs(pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS, )), regex_rules=[], output_plugins=[], client_rate=0, token=self.token)
def Start(self): self.state.Register("hunt_id", None) self.state.Register("client_ids", set()) self.state.Register("client_ids_failures", set()) self.state.Register("client_ids_result_reported", set()) self.state.client_ids = base.GetClientTestTargets(token=self.token) if not self.state.client_ids: self.Log("No clients to test on, define them in " "Test.end_to_end_client_ids") return token = access_control.ACLToken(username="******", reason="Running endtoend tests.").SetUID() runner_args = rdfvalue.FlowRunnerArgs(flow_name="EndToEndTestFlow") flow_request = rdfvalue.FlowRequest( client_ids=self.state.client_ids, args=rdfvalue.EndToEndTestFlowArgs(), runner_args=runner_args) bogus_rule = rdfvalue.ForemanAttributeRegex( attribute_name="System", attribute_regex="Does not match anything") hunt_args = rdfvalue.VariableGenericHuntArgs(flows=[flow_request]) hunt_args.output_plugins = self.GetOutputPlugins() with hunts.GRRHunt.StartHunt( hunt_name="VariableGenericHunt", args=hunt_args, regex_rules=[bogus_rule], client_rate=0, expiry_time="1d", token=token) as hunt: self.state.hunt_id = hunt.session_id hunt.SetDescription("EndToEnd tests run by cron") hunt.Run() hunt.ManuallyScheduleClients(token=token) # Set a callback to check the results after 50 minutes. This should be # plenty of time for the clients to receive the hunt and run the tests, but # not so long that the flow lease will expire. wait_duration = rdfvalue.Duration( config_lib.CONFIG.Get("Test.end_to_end_result_check_wait")) completed_time = rdfvalue.RDFDatetime().Now() + wait_duration self.CallState(next_state="CheckResults", start_time=completed_time)
def Start(self): with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="Interrogate"), flow_args=rdfvalue.InterrogateArgs(lightweight=False), regex_rules=[], output_plugins=self.GetOutputPlugins(), token=self.token) as hunt: runner = hunt.GetRunner() runner.args.client_rate = 50 runner.args.expiry_time = "1w" runner.args.description = ("Interrogate run by cron to keep host" "info fresh.") runner.Start()
def testHuntTermination(self): """This tests that hunts with a client limit terminate correctly.""" with test_lib.FakeTime(1000): with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs( pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS) ), regex_rules=[rdfvalue.ForemanAttributeRegex( attribute_name="GRR client", attribute_regex="GRR")], client_limit=5, client_rate=0, expiry_time=rdfvalue.Duration("1000s"), token=self.token) as hunt: hunt.Run() # Pretend to be the foreman now and dish out hunting jobs to all the # clients (Note we have 10 clients here). foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token) for client_id in self.client_ids: foreman.AssignTasksToClient(client_id) # Run the hunt. client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, self.client_ids, check_flow_errors=False, token=self.token) hunt_obj = aff4.FACTORY.Open(hunt.session_id, age=aff4.ALL_TIMES, token=self.token) started = hunt_obj.GetValuesForAttribute(hunt_obj.Schema.CLIENTS) finished = hunt_obj.GetValuesForAttribute(hunt_obj.Schema.FINISHED) errors = hunt_obj.GetValuesForAttribute(hunt_obj.Schema.ERRORS) self.assertEqual(len(set(started)), 5) self.assertEqual(len(set(finished)), 5) self.assertEqual(len(set(errors)), 2) hunt_obj = aff4.FACTORY.Open(hunt.session_id, age=aff4.ALL_TIMES, token=self.token) # Hunts are automatically paused when they reach the client limit. self.assertEqual(hunt_obj.Get(hunt_obj.Schema.STATE), "PAUSED")
def RunHunt(self, plugin_args=None, responses=None, process_responses_separately=False): if responses is None: responses = [] with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs(pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS)), regex_rules=[ rdfvalue.ForemanAttributeRegex(attribute_name="GRR client", attribute_regex="GRR"), ], client_rate=0, token=self.token) as hunt: hunt_urn = hunt.urn plugin_def = rdfvalue.OutputPlugin(plugin_name="CSVOutputPlugin", plugin_args=plugin_args) plugin = plugin_def.GetPluginForHunt(hunt) # We don't want to test the whole output plugins subsystem as it's # tested in its own tests. We only want to test logic specific to # ColumnIOHuntOutputPlugin. messages = [] for response in responses: messages.append( rdfvalue.GrrMessage(source=self.client_id, payload=response)) if process_responses_separately: for message in messages: plugin.ProcessResponses([message]) else: plugin.ProcessResponses(messages) plugin.Flush() return (hunt_urn, plugin)
def testHuntCollectionLogging(self): """This tests running the hunt on some clients.""" with hunts.GRRHunt.StartHunt(hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs( flow_name="DummyLogFlow"), client_rate=0, token=self.token) as hunt: hunt.Run() hunt.Log("Log from the hunt itself") hunt_urn = hunt.urn self.AssignTasksToClients() self.RunHunt() self.StopHunt(hunt_urn) # Check logs were written to the hunt collection with aff4.FACTORY.Open(hunt_urn.Add("Logs"), token=self.token, age=aff4.ALL_TIMES) as hunt_logs: # Can't use len with PackedVersionCollection count = 0 for log in hunt_logs: if log.client_id: self.assertTrue(log.client_id in self.client_ids) self.assertTrue(log.log_message in [ "First", "Second", "Third", "Fourth", "Uno", "Dos", "Tres", "Cuatro" ]) self.assertTrue( log.flow_name in ["DummyLogFlow", "DummyLogFlowChild"]) self.assertTrue(str(hunt_urn) in str(log.urn)) else: self.assertEqual(log.log_message, "Log from the hunt itself") self.assertEqual(log.flow_name, "GenericHunt") self.assertEqual(log.urn, hunt_urn) count += 1 # 4 logs for each flow, 2 flow run. One hunt-level log. self.assertEqual(count, 8 * len(self.client_ids) + 1)
class TestFileFinderOSHomedir(TestFileFinderOSLinux): """List files in homedir with FileFinder. Exercise globbing and interpolation. """ platforms = ["Linux", "Darwin", "Windows"] test_output_path = "/analysis/test/homedirs" action = file_finder.FileFinderAction( action_type=file_finder.FileFinderAction.Action.STAT) args = { "paths": ["%%users.homedir%%/*"], "action": action, "runner_args": rdfvalue.FlowRunnerArgs(output=test_output_path) } def CheckFlow(self): results = aff4.FACTORY.Open(self.client_id.Add(self.test_output_path), token=self.token) self.assertEqual(type(results), aff4.RDFValueCollection) self.assertTrue(len(results) > 1)
def InitializeContext(self, args): """Initializes the context of this flow.""" if args is None: args = rdfvalue.FlowRunnerArgs() output_collection = self._CreateOutputCollection(args) context = utils.DataObject( args=args, backtrace=None, client_resources=rdfvalue.ClientResources(), create_time=rdfvalue.RDFDatetime().Now(), creator=args.creator or self.token.username, current_state="Start", # If not None, kill-stuck-flow notification is scheduled at the given # time. kill_timestamp=None, network_bytes_sent=0, next_outbound_id=1, next_processed_request=1, next_states=set(), output=output_collection, # Output collection is nullified when flow is terminated, so we're # keeping the urn separately for further reference. output_urn=(output_collection is not None) and output_collection.urn, outstanding_requests=0, remaining_cpu_quota=args.cpu_limit, state=rdfvalue.Flow.State.RUNNING, # Have we sent a notification to the user. user_notified=False, ) # Store the context in the flow_obj for next time. self.flow_obj.state.Register("context", context) return context
def testResourceUsageStats(self): client_ids = self.SetupClients(10) with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs( flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs( pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS, ) ), regex_rules=[rdfvalue.ForemanAttributeRegex( attribute_name="GRR client", attribute_regex="GRR")], output_plugins=[], client_rate=0, token=self.token) as hunt: hunt.Run() with aff4.FACTORY.Open( "aff4:/foreman", mode="rw", token=self.token) as foreman: for client_id in client_ids: foreman.AssignTasksToClient(client_id) client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, client_ids, False, self.token) hunt = aff4.FACTORY.Open(hunt.urn, aff4_type="GenericHunt", token=self.token) # This is called once for each state method. Each flow above runs the # Start and the StoreResults methods. usage_stats = hunt.state.context.usage_stats self.assertEqual(usage_stats.user_cpu_stats.num, 10) self.assertTrue(math.fabs(usage_stats.user_cpu_stats.mean - 5.5) < 1e-7) self.assertTrue(math.fabs(usage_stats.user_cpu_stats.std - 2.8722813) < 1e-7) self.assertEqual(usage_stats.system_cpu_stats.num, 10) self.assertTrue(math.fabs(usage_stats.system_cpu_stats.mean - 11) < 1e-7) self.assertTrue(math.fabs(usage_stats.system_cpu_stats.std - 5.7445626) < 1e-7) self.assertEqual(usage_stats.network_bytes_sent_stats.num, 10) self.assertTrue(math.fabs(usage_stats.network_bytes_sent_stats.mean - 16.5) < 1e-7) self.assertTrue(math.fabs(usage_stats.network_bytes_sent_stats.std - 8.61684396) < 1e-7) # NOTE: Not checking histograms here. RunningStatsTest tests that mean, # standard deviation and histograms are calculated correctly. Therefore # if mean/stdev values are correct histograms should be ok as well. self.assertEqual(len(usage_stats.worst_performers), 10) prev = usage_stats.worst_performers[0] for p in usage_stats.worst_performers[1:]: self.assertTrue(prev.cpu_usage.user_cpu_time + prev.cpu_usage.system_cpu_time > p.cpu_usage.user_cpu_time + p.cpu_usage.system_cpu_time) prev = p
def testHuntModificationWorksCorrectly(self): """This tests running the hunt on some clients.""" with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdfvalue.FlowRunnerArgs(flow_name="GetFile"), flow_args=rdfvalue.GetFileArgs( pathspec=rdfvalue.PathSpec( path="/tmp/evil.txt", pathtype=rdfvalue.PathSpec.PathType.OS), ), regex_rules=[rdfvalue.ForemanAttributeRegex( attribute_name="GRR client", attribute_regex="GRR")], client_limit=1, client_rate=0, token=self.token) as hunt: hunt.Run() # Forget about hunt object, we'll use AFF4 for everything. hunt_session_id = hunt.session_id hunt = None # Pretend to be the foreman now and dish out hunting jobs to all the # client.. with aff4.FACTORY.Open( "aff4:/foreman", mode="rw", token=self.token) as foreman: for client_id in self.client_ids: foreman.AssignTasksToClient(client_id) # Run the hunt. client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, self.client_ids, False, self.token) # Re-open the hunt to get fresh data. hunt_obj = aff4.FACTORY.Open(hunt_session_id, age=aff4.ALL_TIMES, ignore_cache=True, token=self.token) started = hunt_obj.GetValuesForAttribute(hunt_obj.Schema.CLIENTS) # There should be only one client, due to the limit self.assertEqual(len(set(started)), 1) # Check the hunt is paused. self.assertEqual(hunt_obj.Get(hunt_obj.Schema.STATE), "PAUSED") with aff4.FACTORY.Open( hunt_session_id, mode="rw", token=self.token) as hunt_obj: with hunt_obj.GetRunner() as runner: runner.args.client_limit = 10 runner.Start() # Pretend to be the foreman now and dish out hunting jobs to all the # clients. with aff4.FACTORY.Open( "aff4:/foreman", mode="rw", token=self.token) as foreman: for client_id in self.client_ids: foreman.AssignTasksToClient(client_id) test_lib.TestHuntHelper(client_mock, self.client_ids, False, self.token) hunt_obj = aff4.FACTORY.Open(hunt_session_id, age=aff4.ALL_TIMES, token=self.token) started = hunt_obj.GetValuesForAttribute(hunt_obj.Schema.CLIENTS) # There should be only one client, due to the limit self.assertEqual(len(set(started)), 10)