def _StartHunt(self): with hunts.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=test_lib.FlowWithOneNestedFlow.__name__), client_rate=0, token=self.token) as hunt: hunt.Run() self.AssignTasksToClients(client_ids=[self.client_urn]) self.RunHunt(client_ids=[self.client_urn])
def testRunnerArgsBaseSessionIdDoesNotAffectCreatedFlow(self): """When multiple clients match, check we run on the latest one.""" flow_runner_args = rdf_flows.FlowRunnerArgs( base_session_id="aff4:/foo") args = flow_plugin.ApiCreateFlowArgs( client_id=self.client_id.Basename(), flow=flow_plugin.ApiFlow(name=processes.ListProcesses.__name__, runner_args=flow_runner_args)) result = self.handler.Handle(args, token=self.token) self.assertFalse(utils.SmartStr(result.urn).startswith("aff4:/foo"))
def _CreateHunt(self, description): output_plugins = [ output_plugin.OutputPluginDescriptor(plugin_name="TestOutputPlugin") ] with hunts.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=rdf_flows.FlowRunnerArgs(flow_name="GetFile"), output_plugins=output_plugins, description=description, client_rate=0, token=self.token) as hunt: return hunt
def InitializeContext(self, args): """Initializes the context of this flow.""" if args is None: args = rdf_flows.FlowRunnerArgs() output_plugins_states = [] for plugin_descriptor in args.output_plugins: if not args.client_id: self.Log( "Not initializing output plugin %s as flow does not run on " "the client.", plugin_descriptor.plugin_name) continue output_base_urn = self.session_id.Add(OUTPUT_PLUGIN_BASE_SUFFIX) plugin_class = plugin_descriptor.GetPluginClass() plugin = plugin_class(self.flow_obj.output_urn, args=plugin_descriptor.plugin_args, output_base_urn=output_base_urn, token=self.token) try: plugin.InitializeState() # TODO(user): Those do not need to be inside the state, they # could be part of the plugin descriptor. plugin.state["logs"] = [] plugin.state["errors"] = [] output_plugins_states.append( rdf_flows.OutputPluginState( plugin_state=plugin.state, plugin_descriptor=plugin_descriptor)) except Exception as e: # pylint: disable=broad-except logging.info( "Plugin %s failed to initialize (%s), ignoring it.", plugin, e) parent_creator = None if self.parent_runner: parent_creator = self.parent_runner.context.creator context = rdf_flows.FlowContext( create_time=rdfvalue.RDFDatetime.Now(), creator=parent_creator or self.token.username, current_state="Start", output_plugins_states=output_plugins_states, remaining_cpu_quota=args.cpu_limit, state=rdf_flows.FlowContext.State.RUNNING, # Have we sent a notification to the user. user_notified=False, ) return context
def Start(self): with hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", client_limit=0, flow_runner_args=rdf_flows.FlowRunnerArgs(flow_name="Interrogate"), flow_args=flows_discovery.InterrogateArgs(lightweight=False), output_plugins=self.GetOutputPlugins(), token=self.token) as hunt: runner = hunt.GetRunner() runner.runner_args.client_rate = 50 runner.runner_args.expiry_time = "1w" runner.runner_args.description = ("Interrogate run by cron to keep host" "info fresh.") runner.Start()
class TestFileFinderOSHomedir(base.AutomatedTest): """List files in homedir with FileFinder. Exercise globbing and interpolation. """ platforms = ["Linux", "Darwin", "Windows"] flow = file_finder.FileFinder.__name__ action = rdf_file_finder.FileFinderAction.Stat() args = { "paths": ["%%users.homedir%%/*"], "action": action, "runner_args": rdf_flows.FlowRunnerArgs() } def CheckFlow(self): self.CheckResultCollectionNotEmptyWithRetry(self.session_id)
class TestFileFinderOSHomedir(base.AutomatedTest): """List files in homedir with FileFinder. Exercise globbing and interpolation. """ platforms = ["Linux", "Darwin", "Windows"] flow = "FileFinder" action = file_finder.FileFinderAction( action_type=file_finder.FileFinderAction.Action.STAT) args = {"paths": ["%%users.homedir%%/*"], "action": action, "runner_args": rdf_flows.FlowRunnerArgs()} def CheckFlow(self): self.CheckCollectionNotEmptyWithRetry( self.session_id.Add(flow_runner.RESULTS_SUFFIX), self.token)
def setUp(self): super(ApiGetExportedHuntResultsHandlerTest, self).setUp() self.handler = hunt_plugin.ApiGetExportedHuntResultsHandler() self.hunt = hunts.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=DummyFlowWithSingleReply.__name__), client_rate=0, token=self.token) self.hunt.Run() client_ids = self.SetupClients(5) self.AssignTasksToClients(client_ids=client_ids) client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, client_ids, token=self.token)
def CreateSampleHunt(self, path=None, stopped=False, output_plugins=None, client_limit=0, client_count=10, token=None): token = token or self.token self.client_ids = self.SetupClients(client_count) client_rule_set = rdf_foreman.ForemanClientRuleSet(rules=[ rdf_foreman.ForemanClientRule( rule_type=rdf_foreman.ForemanClientRule.Type.REGEX, regex=rdf_foreman.ForemanRegexClientRule( attribute_name="GRR client", attribute_regex="GRR")) ]) with implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=transfer.GetFile.__name__), flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec( path=path or "/tmp/evil.txt", pathtype=rdf_paths.PathSpec.PathType.OS, )), client_rule_set=client_rule_set, output_plugins=output_plugins or [], client_rate=0, client_limit=client_limit, token=token) as hunt: if not stopped: hunt.Run() with aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=token) as foreman: for client_id in self.client_ids: foreman.AssignTasksToClient(client_id) self.hunt_urn = hunt.urn return aff4.FACTORY.Open(hunt.urn, mode="rw", token=token, age=aff4.ALL_TIMES)
def CreateSampleHunt(self, description, token=None): implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, description=description, flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=transfer.GetFile.__name__), flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec( path="/tmp/evil.txt", pathtype=rdf_paths.PathSpec.PathType.TSK, )), client_rule_set=self._CreateForemanClientRuleSet(), output_plugins=[ output_plugin.OutputPluginDescriptor( plugin_name="DummyOutputPlugin", plugin_args=gui_test_lib.DummyOutputPlugin.args_type( filename_regex="blah!", fetch_binaries=True)) ], client_rate=60, token=token)
def Run(self): def ReplaceFlowId(): flows_dir_fd = aff4.FACTORY.Open(self.client_id.Add("flows"), token=self.token) flow_urn = list(flows_dir_fd.ListChildren())[0] return {flow_urn.Basename(): "W:ABCDEF"} with test_lib.FakeTime(42): self.Check("CreateFlow", args=flow_plugin.ApiCreateFlowArgs( client_id=self.client_id.Basename(), flow=flow_plugin.ApiFlow( name=processes.ListProcesses.__name__, args=processes.ListProcessesArgs( filename_regex=".", fetch_binaries=True), runner_args=rdf_flows.FlowRunnerArgs( output_plugins=[], priority="HIGH_PRIORITY", notify_to_user=False))), replace=ReplaceFlowId)
def testGetPluginArgsHandlesMissingPluginsCorrectly(self): descriptor = output_plugin.OutputPluginDescriptor( plugin_name="TestOutputPluginWithArgs", plugin_args=rdf_flows.FlowRunnerArgs(flow_name="GetFile")) serialized = descriptor.SerializeToString() deserialized = output_plugin.OutputPluginDescriptor() deserialized.ParseFromString(serialized) self.assertEqual(deserialized, descriptor) self.assertEqual(deserialized.GetPluginClass(), TestOutputPluginWithArgs) with utils.Stubber(output_plugin.OutputPlugin, "classes", {}): deserialized = output_plugin.OutputPluginDescriptor() deserialized.ParseFromString(serialized) self.assertTrue(deserialized.GetPluginClass(), output_plugin.UnknownOutputPlugin) # UnknownOutputPlugin should just return serialized arguments as bytes. self.assertEqual(deserialized.plugin_args, descriptor.plugin_args.SerializeToString())
def setUp(self): super(ApiGetHuntFilesArchiveHandlerTest, self).setUp() self.handler = hunt_plugin.ApiGetHuntFilesArchiveHandler() self.hunt = hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=file_finder.FileFinder.__name__), flow_args=file_finder.FileFinderArgs( paths=[os.path.join(self.base_path, "test.plist")], action=file_finder.FileFinderAction(action_type="DOWNLOAD"),), client_rate=0, token=self.token) self.hunt.Run() client_ids = self.SetupClients(10) self.AssignTasksToClients(client_ids=client_ids) action_mock = action_mocks.FileFinderClientMock() test_lib.TestHuntHelper(action_mock, client_ids, token=self.token)
def Layout(self, request, response): """Render the form for creating the flow args.""" self.flow_name = self._GetFlowName(request) self.flow_cls = flow.GRRFlow.classes.get(self.flow_name) if aff4.issubclass(self.flow_cls, flow.GRRFlow): self.flow_found = True self.form = forms.SemanticProtoFormRenderer( self.flow_cls.GetDefaultArgs(token=request.token), prefix="args").RawHTML(request) self.runner_form = forms.SemanticProtoFormRenderer( rdf_flows.FlowRunnerArgs(flow_name=self.flow_name), prefix="runner").RawHTML(request) response = super(SemanticProtoFlowForm, self).Layout(request, response) return self.CallJavascript( response, "SemanticProtoFlowForm.Layout", renderer=self.__class__.__name__)
def ParseFlowArgs(self): """Parse the flow and flow_runner args.""" if self.flow_runner_args is not None: return self.flow_runner_args, self.flow_args flow_path = self.request.REQ.get("flow_path", "") flow_name = os.path.basename(flow_path) if not flow_name: raise ValueError("No flow specified. Please select a flow.") flow_cls = flow.GRRFlow.GetPlugin(flow_name) self.flow_args = forms.SemanticProtoFormRenderer( flow_cls.args_type(), prefix="args").ParseArgs(self.request) self.flow_runner_args = forms.SemanticProtoFormRenderer( rdf_flows.FlowRunnerArgs(), prefix="runner").ParseArgs(self.request) self.flow_runner_args.flow_name = flow_name return self.flow_runner_args, self.flow_args
def setUp(self): super(ApiGetExportedHuntResultsHandlerTest, self).setUp() self.handler = hunt_plugin.ApiGetExportedHuntResultsHandler() self.hunt = implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=flow_test_lib.DummyFlowWithSingleReply.__name__), client_rate=0, token=self.token) self.hunt.Run() self.client_ids = self.SetupClients(5) # Ensure that clients are processed sequentially - this way the test won't # depend on the order of results in the collection (which is normally # random). for cid in self.client_ids: self.AssignTasksToClients(client_ids=[cid]) client_mock = hunt_test_lib.SampleHuntMock() hunt_test_lib.TestHuntHelper(client_mock, [cid], token=self.token)
def RenderAjax(self, request, response): """Parse the flow args from the form and launch the flow.""" self.flow_name = self._GetFlowName(request) self.client_id = request.REQ.get("client_id", None) self.dom_node = request.REQ.get("dom_node") flow_cls = flow.GRRFlow.classes.get(self.flow_name) if flow_cls is not None: self.args = forms.SemanticProtoFormRenderer( flow_cls.args_type(), prefix="args").ParseArgs(request) try: self.args.Validate() except ValueError as e: return self.CallJavascript( response, "SemanticProtoFlowForm.RenderAjaxError", error=str(e)) self.runner_args = forms.SemanticProtoFormRenderer( rdf_flows.FlowRunnerArgs(), prefix="runner_").ParseArgs(request) self.runner_args.Validate() self.flow_id = flow.GRRFlow.StartFlow( client_id=self.client_id, flow_name=self.flow_name, token=request.token, args=self.args, runner_args=self.runner_args) self.args_html = semantic.FindRendererForObject(self.args).RawHTML(request) self.runner_args_html = semantic.FindRendererForObject( self.runner_args).RawHTML(request) response = renderers.TemplateRenderer.Layout( self, request, response, apply_template=self.ajax_template) return self.CallJavascript( response, "SemanticProtoFlowForm.RenderAjax", renderer=self.__class__.__name__, dom_node=self.dom_node)
def testCopyHuntPreservesRuleType(self): implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, description="model hunt", flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=transfer.GetFile.__name__), flow_args=transfer.GetFileArgs( pathspec=rdf_paths.PathSpec( path="/tmp/evil.txt", pathtype=rdf_paths.PathSpec.PathType.TSK, )), client_rule_set=foreman_rules.ForemanClientRuleSet(rules=[ foreman_rules.ForemanClientRule( rule_type=foreman_rules.ForemanClientRule.Type.OS, os=foreman_rules.ForemanOsClientRule(os_darwin=True)) ]), token=self.token) self.Open("/#main=ManageHunts") self.Click("css=tr:contains('model hunt')") self.Click("css=button[name=CopyHunt]:not([disabled])") # Wait until dialog appears. self.WaitUntil(self.IsElementPresent, "css=grr-wizard-form:contains('What to run?')") # Click on "Next" button self.Click("css=grr-new-hunt-wizard-form button.Next") self.WaitUntil(self.IsElementPresent, "css=grr-wizard-form:contains('Hunt parameters')") # Click on "Next" button. self.Click("css=grr-new-hunt-wizard-form button.Next") self.WaitUntil(self.IsElementPresent, "css=grr-wizard-form:contains('How to process results')") # Click on "Next" button self.Click("css=grr-new-hunt-wizard-form button.Next") self.WaitUntil(self.IsElementPresent, "css=grr-wizard-form:contains('Where to run?')") self.WaitUntil( self.IsElementPresent, "css=grr-new-hunt-wizard-form " "label:contains('Os darwin') ~ * input:checked")
def CreateSampleHunt(description, token=None): hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", description=description, flow_runner_args=rdf_flows.FlowRunnerArgs(flow_name="GetFile"), flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec( path="/tmp/evil.txt", pathtype=rdf_paths.PathSpec.PathType.TSK, )), client_rule_set=rdf_foreman.ForemanClientRuleSet(rules=[ rdf_foreman.ForemanClientRule( rule_type=rdf_foreman.ForemanClientRule.Type.REGEX, regex=rdf_foreman.ForemanRegexClientRule( attribute_name="GRR client", attribute_regex="GRR")) ]), output_plugins=[ output_plugin.OutputPluginDescriptor( plugin_name="DummyOutputPlugin", plugin_args=DummyOutputPlugin.args_type( filename_regex="blah!", fetch_binaries=True)) ], client_rate=60, token=token)
def testListHuntClients(self): hunt = implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=file_finder.FileFinder.__name__), flow_args=rdf_file_finder.FileFinderArgs( paths=[os.path.join(self.base_path, "test.plist")], action=rdf_file_finder.FileFinderAction( action_type="DOWNLOAD"), ), client_rate=0, token=self.token) hunt.Run() client_ids = self.SetupClients(5) self.AssignTasksToClients(client_ids=client_ids) action_mock = action_mocks.FileFinderClientMock() hunt_test_lib.TestHuntHelper(action_mock, client_ids, iteration_limit=10, token=self.token) result = self.handler.Handle( hunt_plugin.ApiListHuntClientsArgs(hunt_id=hunt.urn.Basename()), token=self.token) # TODO(user): This still uses data store internals and will fail on some # data stores. # This is not super deterministic, we start processing some # clients, run the hunt for a bit but there is no order to all # this. We should have some clients half finished though (i.e., # with pending requests) and five clients in total. self.assertEqual(result.total_count, 5) clients = list(result.items) pending_requests = [client.pending_requests for client in clients] self.assertTrue(any(r.next_state) for r in pending_requests)
def Handle(self, args, token=None): client_urn = self.GetClientTarget(args, token=token) size_condition = rdf_file_finder.FileFinderCondition( condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE, size=rdf_file_finder.FileFinderSizeCondition( max_file_size=args.max_file_size)) file_finder_args = rdf_file_finder.FileFinderArgs( paths=args.paths, action=rdf_file_finder.FileFinderAction(action_type=args.action), conditions=[size_condition]) # Check our flow throttling limits, will raise if there are problems. throttler = throttle.FlowThrottler( daily_req_limit=config_lib.CONFIG.Get("API.DailyFlowRequestLimit"), dup_interval=config_lib.CONFIG.Get("API.FlowDuplicateInterval")) throttler.EnforceLimits(client_urn, token.username, file_finder.FileFinder.__name__, file_finder_args, token=token) # Limit the whole flow to 200MB so if a glob matches lots of small files we # still don't have too much impact. runner_args = rdf_flows.FlowRunnerArgs( client_id=client_urn, flow_name=file_finder.FileFinder.__name__, network_bytes_limit=200 * 1000 * 1000) flow_id = flow.GRRFlow.StartFlow(runner_args=runner_args, token=token, args=file_finder_args) return ApiStartRobotGetFilesOperationResult( operation_id=utils.SmartUnicode(flow_id))
def setUp(self): super(ApiGetHuntFileHandlerTest, self).setUp() self.handler = hunt_plugin.ApiGetHuntFileHandler() self.file_path = os.path.join(self.base_path, "test.plist") self.hunt = hunts.GRRHunt.StartHunt( hunt_name="GenericHunt", flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=file_finder.FileFinder.__name__), flow_args=file_finder.FileFinderArgs( paths=[self.file_path], action=file_finder.FileFinderAction(action_type="DOWNLOAD"),), client_rate=0, token=self.token) self.hunt.Run() self.results_urn = self.hunt.results_collection_urn self.aff4_file_path = "fs/os/%s" % self.file_path self.client_id = self.SetupClients(1)[0] self.AssignTasksToClients(client_ids=[self.client_id]) action_mock = action_mocks.FileFinderClientMock() test_lib.TestHuntHelper(action_mock, [self.client_id], token=self.token)
def testCPULimitForHunts(self): worker_obj = worker.GRRWorker(token=self.token) client_ids = ["C.%016X" % i for i in xrange(10, 20)] result = {} client_mocks = [] for client_id in client_ids: client_mock = action_mocks.CPULimitClientMock(result) client_mock = flow_test_lib.MockClient( rdf_client.ClientURN(client_id), client_mock, token=self.token) client_mock.EnableResourceUsage(user_cpu_usage=[10], system_cpu_usage=[10], network_usage=[1000]) client_mocks.append(client_mock) flow_runner_args = rdf_flows.FlowRunnerArgs( flow_name=flow_test_lib.CPULimitFlow.__name__) with implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=flow_runner_args, cpu_limit=5000, per_client_cpu_limit=10000, network_bytes_limit=1000000, client_rate=0, token=self.token) as hunt: hunt.GetRunner().Start() implementation.GRRHunt.StartClients(hunt.session_id, client_ids[:1]) self._Process(client_mocks, worker_obj) implementation.GRRHunt.StartClients(hunt.session_id, client_ids[1:2]) self._Process(client_mocks, worker_obj) implementation.GRRHunt.StartClients(hunt.session_id, client_ids[2:3]) self._Process(client_mocks, worker_obj) # The limiting factor here is the overall hunt limit of 5000 cpu # seconds. Clients that finish should decrease the remaining quota # and the following clients should get the reduced quota. self.assertEqual(result["cpulimit"], [ 5000.0, 4980.0, 4960.0, 4940.0, 4920.0, 4900.0, 4880.0, 4860.0, 4840.0 ]) self.assertEqual(result["networklimit"], [ 1000000L, 999000L, 998000L, 997000L, 996000L, 995000L, 994000L, 993000L, 992000L ]) result.clear() with implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=flow_runner_args, per_client_cpu_limit=3000, per_client_network_limit_bytes=3000000, client_rate=0, token=self.token) as hunt: hunt.GetRunner().Start() implementation.GRRHunt.StartClients(hunt.session_id, client_ids[:1]) self._Process(client_mocks, worker_obj) implementation.GRRHunt.StartClients(hunt.session_id, client_ids[1:2]) self._Process(client_mocks, worker_obj) implementation.GRRHunt.StartClients(hunt.session_id, client_ids[2:3]) self._Process(client_mocks, worker_obj) # This time, the per client limit is 3000s / 3000000 bytes. Every # client should get the same limit. self.assertEqual(result["cpulimit"], [ 3000.0, 2980.0, 2960.0, 3000.0, 2980.0, 2960.0, 3000.0, 2980.0, 2960.0 ]) self.assertEqual(result["networklimit"], [ 3000000, 2999000, 2998000, 3000000, 2999000, 2998000, 3000000, 2999000, 2998000 ]) result.clear() for client_mock in client_mocks: client_mock.EnableResourceUsage(user_cpu_usage=[500], system_cpu_usage=[500], network_usage=[1000000]) with implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, flow_runner_args=flow_runner_args, per_client_cpu_limit=3000, cpu_limit=5000, per_client_network_limit_bytes=3000000, network_bytes_limit=5000000, client_rate=0, token=self.token) as hunt: hunt.GetRunner().Start() implementation.GRRHunt.StartClients(hunt.session_id, client_ids[:1]) self._Process(client_mocks, worker_obj) implementation.GRRHunt.StartClients(hunt.session_id, client_ids[1:2]) self._Process(client_mocks, worker_obj) implementation.GRRHunt.StartClients(hunt.session_id, client_ids[2:3]) self._Process(client_mocks, worker_obj) # The first client gets the full per client limit of 3000s, and # uses all of it. The hunt has a limit of just 5000 total so the # second client gets started with a limit of 2000. It can only run # two of three states, the last client will not be started at all # due to out of quota. self.assertEqual(result["cpulimit"], [3000.0, 2000.0, 1000.0, 2000.0, 1000.0]) self.assertEqual(result["networklimit"], [3000000, 2000000, 1000000, 2000000, 1000000]) errors = list(hunt.GetClientsErrors()) self.assertEqual(len(errors), 2) # Client side out of cpu. self.assertIn("CPU limit exceeded", errors[0].log_message) # Server side out of cpu. self.assertIn("Out of CPU quota", errors[1].backtrace)
def StartFlow(cls, args=None, runner_args=None, parent_flow=None, sync=True, token=None, **kwargs): """The main factory function for Creating and executing a new flow. Args: args: An arg protocol buffer which is an instance of the required flow's args_type class attribute. runner_args: an instance of FlowRunnerArgs() protocol buffer which is used to initialize the runner for this flow. parent_flow: A parent flow or None if this is a top level flow. sync: If True, the Start method of this flow will be called inline. Otherwise we schedule the starting of this flow on another worker. token: Security credentials token identifying the user. **kwargs: If args or runner_args are not specified, we construct these protobufs from these keywords. Returns: the session id of the flow. Raises: RuntimeError: Unknown or invalid parameters were provided. """ # Build the runner args from the keywords. if runner_args is None: runner_args = rdf_flows.FlowRunnerArgs() cls.FilterArgsFromSemanticProtobuf(runner_args, kwargs) # When asked to run a flow in the future this implied it will run # asynchronously. if runner_args.start_time: sync = False # Is the required flow a known flow? if runner_args.flow_name not in GRRFlow.classes: stats.STATS.IncrementCounter("grr_flow_invalid_flow_count") raise RuntimeError("Unable to locate flow %s" % runner_args.flow_name) # If no token is specified, raise. if not token: raise access_control.UnauthorizedAccess( "A token must be specified.") # For the flow itself we use a supervisor token. token = token.SetUID() # Extend the expiry time of this token indefinitely. Python on Windows only # supports dates up to the year 3000. token.expiry = rdfvalue.RDFDatetime.FromHumanReadable("2997-01-01") flow_cls = aff4.AFF4Object.classes.get(runner_args.flow_name) if flow_cls.category and not runner_args.client_id: raise RuntimeError( "Flow with category (user-visible flow) has to be " "started on a client, but runner_args.client_id " "is missing.") # We create an anonymous AFF4 object first, The runner will then generate # the appropriate URN. flow_obj = aff4.FACTORY.Create(None, flow_cls, token=token) # Now parse the flow args into the new object from the keywords. if args is None: args = flow_obj.args_type() cls.FilterArgsFromSemanticProtobuf(args, kwargs) # Check that the flow args are valid. args.Validate() # Store the flow args. flow_obj.args = args flow_obj.runner_args = runner_args # At this point we should exhaust all the keyword args. If any are left # over, we do not know what to do with them so raise. if kwargs: raise type_info.UnknownArg("Unknown parameters to StartFlow: %s" % kwargs) # Create a flow runner to run this flow with. if parent_flow: parent_runner = parent_flow.runner else: parent_runner = None runner = flow_obj.CreateRunner(parent_runner=parent_runner, runner_args=runner_args) logging.info(u"Scheduling %s(%s) on %s", flow_obj.urn, runner_args.flow_name, runner_args.client_id) if sync: # Just run the first state inline. NOTE: Running synchronously means # that this runs on the thread that starts the flow. The advantage is # that that Start method can raise any errors immediately. flow_obj.Start() else: # Running Asynchronously: Schedule the start method on another worker. runner.CallState(next_state="Start", start_time=runner_args.start_time) # The flow does not need to actually remain running. if not runner.OutstandingRequests(): flow_obj.Terminate() flow_obj.Close() # Publish an audit event, only for top level flows. if parent_flow is None: events.Events.PublishEvent("Audit", events.AuditEvent( user=token.username, action="RUN_FLOW", flow_name=runner_args.flow_name, urn=flow_obj.urn, client=runner_args.client_id), token=token) return flow_obj.urn
def testCopyHuntHandlesLiteralExpressionCorrectly(self): """Literals are raw bytes. Testing that raw bytes are processed right.""" literal_match = rdf_file_finder.FileFinderContentsLiteralMatchCondition( literal="foo\x0d\xc8bar") implementation.GRRHunt.StartHunt( hunt_name=standard.GenericHunt.__name__, description="model hunt", flow_runner_args=rdf_flows.FlowRunnerArgs( flow_name=file_finder.FileFinder.__name__), flow_args=rdf_file_finder.FileFinderArgs( conditions=[ rdf_file_finder.FileFinderCondition( condition_type="CONTENTS_LITERAL_MATCH", contents_literal_match=literal_match) ], paths=["/tmp/evil.txt"]), token=self.token) self.Open("/#main=ManageHunts") self.Click("css=tr:contains('model hunt')") self.Click("css=button[name=CopyHunt]:not([disabled])") # Wait until dialog appears. self.WaitUntil(self.IsTextPresent, "What to run?") # Check that non-default values of sample hunt are prefilled. self.WaitUntilEqual("foo\\x0d\\xc8bar", self.GetValue, "css=grr-new-hunt-wizard-form " "label:contains('Literal') ~ * input:text") # Click on "Next" button. self.Click("css=grr-new-hunt-wizard-form button.Next") self.WaitUntil(self.IsTextPresent, "Output Processing") # Click on "Next" button self.Click("css=grr-new-hunt-wizard-form button.Next") self.WaitUntil(self.IsTextPresent, "Where to run?") # Click on "Next" button self.Click("css=grr-new-hunt-wizard-form button.Next") self.WaitUntil(self.IsTextPresent, "Review") # Check that the arguments summary is present. self.WaitUntil(self.IsTextPresent, file_finder.FileFinder.__name__) self.WaitUntil(self.IsTextPresent, "foo\\x0d\\xc8bar") # Click on "Run" button self.Click("css=grr-new-hunt-wizard-form button.Next") self.WaitUntil(self.IsTextPresent, "Created Hunt") # Close the window and check that the hunt was created. self.Click("css=button.Next") hunts_root = aff4.FACTORY.Open("aff4:/hunts", token=self.token) hunts_list = sorted(list(hunts_root.ListChildren()), key=lambda x: x.age) self.assertEqual(len(hunts_list), 2) last_hunt = aff4.FACTORY.Open(hunts_list[-1], token=self.token) # Check that the hunt was created with a correct literal value. self.assertEqual(last_hunt.args.flow_runner_args.flow_name, file_finder.FileFinder.__name__) self.assertEqual(last_hunt.args.flow_args.conditions[0] .contents_literal_match.literal, "foo\x0d\xc8bar")
def StartFlow(cls, args=None, runner_args=None, # pylint: disable=g-bad-name parent_flow=None, sync=True, **kwargs): """The main factory function for Creating and executing a new flow. Args: args: An arg protocol buffer which is an instance of the required flow's args_type class attribute. runner_args: an instance of FlowRunnerArgs() protocol buffer which is used to initialize the runner for this flow. parent_flow: A parent flow or None if this is a top level flow. sync: If True, the Start method of this flow will be called inline. Otherwise we schedule the starting of this flow on another worker. **kwargs: If args or runner_args are not specified, we construct these protobufs from these keywords. Returns: the session id of the flow. Raises: RuntimeError: Unknown or invalid parameters were provided. """ # Build the runner args from the keywords. if runner_args is None: runner_args = rdf_flows.FlowRunnerArgs() cls.FilterArgsFromSemanticProtobuf(runner_args, kwargs) # When asked to run a flow in the future this implied it will run # asynchronously. if runner_args.start_time: sync = False # Is the required flow a known flow? if runner_args.flow_name not in GRRFlow.classes: stats.STATS.IncrementCounter("grr_flow_invalid_flow_count") raise RuntimeError("Unable to locate flow %s" % runner_args.flow_name) # If no token is specified, use the default token. if not runner_args.HasField("token"): if data_store.default_token is None: raise access_control.UnauthorizedAccess("A token must be specified.") runner_args.token = data_store.default_token.Copy() # Make sure we are allowed to run this flow. If not, we raise here. We # respect SUID (supervisor) if it is already set. SUID cannot be set by the # user since it isn't part of the ACLToken proto. data_store.DB.security_manager.CheckIfCanStartFlow( runner_args.token, runner_args.flow_name, with_client_id=runner_args.client_id) flow_cls = GRRFlow.GetPlugin(runner_args.flow_name) # If client id was specified and flow doesn't have exemption from ACL # checking policy, then check that the user has access to the client # where the flow is going to run. if flow_cls.ACL_ENFORCED and runner_args.client_id: data_store.DB.security_manager.CheckClientAccess(runner_args.token, runner_args.client_id) # For the flow itself we use a supervisor token. token = runner_args.token.SetUID() # Extend the expiry time of this token indefinitely. Python on Windows only # supports dates up to the year 3000, this number corresponds to July, 2997. token.expiry = 32427003069 * rdfvalue.RDFDatetime.converter # We create an anonymous AFF4 object first, The runner will then generate # the appropriate URN. flow_obj = aff4.FACTORY.Create( None, aff4.AFF4Object.classes.get(runner_args.flow_name), token=token) # Now parse the flow args into the new object from the keywords. if args is None: args = flow_obj.args_type() cls.FilterArgsFromSemanticProtobuf(args, kwargs) # Check that the flow args are valid. args.Validate() # Store the flow args. flow_obj.args = args flow_obj.runner_args = runner_args # At this point we should exhaust all the keyword args. If any are left # over, we do not know what to do with them so raise. if kwargs: raise type_info.UnknownArg("Unknown parameters to StartFlow: %s" % kwargs) # Create a flow runner to run this flow with. if parent_flow: parent_runner = parent_flow.runner else: parent_runner = None runner = flow_obj.CreateRunner( parent_runner=parent_runner, runner_args=runner_args) logging.info(u"Scheduling %s(%s) on %s", flow_obj.urn, runner_args.flow_name, runner_args.client_id) if sync: # Just run the first state inline. NOTE: Running synchronously means # that this runs on the thread that starts the flow. The advantage is # that that Start method can raise any errors immediately. flow_obj.Start() else: # Running Asynchronously: Schedule the start method on another worker. runner.CallState(next_state="Start", start_time=runner_args.start_time) # The flow does not need to actually remain running. if not runner.OutstandingRequests(): flow_obj.Terminate() flow_obj.Close() # Publish an audit event, only for top level flows. if parent_flow is None: events.Events.PublishEvent( "Audit", events.AuditEvent( user=token.username, action="RUN_FLOW", flow_name=runner_args.flow_name, urn=flow_obj.urn, client=runner_args.client_id), token=token) return flow_obj.urn
def Start(self): self.state.hunt_id = None self.state.client_ids_failures = set() self.state.client_ids_result_reported = set() # TODO(user): Figure out if this can just be a set. Add a tap test to # meaningfully exercise this code. self.state.client_ids = list( base.GetClientTestTargets(token=self.token)) if not self.state.client_ids: self.Log("No clients to test on, define them in " "Test.end_to_end_client_ids") return # SetUID is required to run a hunt on the configured end-to-end client # targets without an approval. token = access_control.ACLToken( username="******", reason="Running endtoend tests.").SetUID() runner_args = rdf_flows.FlowRunnerArgs( flow_name=flows_endtoend.EndToEndTestFlow.__name__) flow_request = hunts_standard.FlowRequest( client_ids=self.state.client_ids, args=flows_endtoend.EndToEndTestFlowArgs(), runner_args=runner_args) bogus_rule = rdf_foreman.ForemanRegexClientRule( attribute_name="System", attribute_regex="Does not match anything") client_rule_set = rdf_foreman.ForemanClientRuleSet(rules=[ rdf_foreman.ForemanClientRule( rule_type=rdf_foreman.ForemanClientRule.Type.REGEX, regex=bogus_rule) ]) hunt_args = hunts_standard.VariableGenericHuntArgs( flows=[flow_request]) hunt_args.output_plugins = self.GetOutputPlugins() with hunts_implementation.GRRHunt.StartHunt( hunt_name=hunts_standard.VariableGenericHunt.__name__, args=hunt_args, client_rule_set=client_rule_set, client_rate=0, expiry_time="1d", token=token) as hunt: self.state.hunt_id = hunt.session_id hunt.SetDescription("EndToEnd tests run by cron") hunt.Run() hunt.ManuallyScheduleClients(token=token) # Set a callback to check the results after 50 minutes. This should be # plenty of time for the clients to receive the hunt and run the tests, but # not so long that the flow lease will expire. wait_duration = rdfvalue.Duration( config.CONFIG.Get("Test.end_to_end_result_check_wait")) completed_time = rdfvalue.RDFDatetime.Now() + wait_duration self.CallState(next_state="CheckResults", start_time=completed_time)