def _StartHunt(self):
with hunts.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=test_lib.FlowWithOneNestedFlow.__name__),
client_rate=0,
token=self.token) as hunt:
hunt.Run()
self.AssignTasksToClients(client_ids=[self.client_urn])
self.RunHunt(client_ids=[self.client_urn])
def testRunnerArgsBaseSessionIdDoesNotAffectCreatedFlow(self):
"""When multiple clients match, check we run on the latest one."""
flow_runner_args = rdf_flows.FlowRunnerArgs(
base_session_id="aff4:/foo")
args = flow_plugin.ApiCreateFlowArgs(
client_id=self.client_id.Basename(),
flow=flow_plugin.ApiFlow(name=processes.ListProcesses.__name__,
runner_args=flow_runner_args))
result = self.handler.Handle(args, token=self.token)
self.assertFalse(utils.SmartStr(result.urn).startswith("aff4:/foo"))
def _CreateHunt(self, description):
output_plugins = [
output_plugin.OutputPluginDescriptor(plugin_name="TestOutputPlugin")
]
with hunts.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=rdf_flows.FlowRunnerArgs(flow_name="GetFile"),
output_plugins=output_plugins,
description=description,
client_rate=0,
token=self.token) as hunt:
return hunt
def InitializeContext(self, args):
"""Initializes the context of this flow."""
if args is None:
args = rdf_flows.FlowRunnerArgs()
output_plugins_states = []
for plugin_descriptor in args.output_plugins:
if not args.client_id:
self.Log(
"Not initializing output plugin %s as flow does not run on "
"the client.", plugin_descriptor.plugin_name)
continue
output_base_urn = self.session_id.Add(OUTPUT_PLUGIN_BASE_SUFFIX)
plugin_class = plugin_descriptor.GetPluginClass()
plugin = plugin_class(self.flow_obj.output_urn,
args=plugin_descriptor.plugin_args,
output_base_urn=output_base_urn,
token=self.token)
try:
plugin.InitializeState()
# TODO(user): Those do not need to be inside the state, they
# could be part of the plugin descriptor.
plugin.state["logs"] = []
plugin.state["errors"] = []
output_plugins_states.append(
rdf_flows.OutputPluginState(
plugin_state=plugin.state,
plugin_descriptor=plugin_descriptor))
except Exception as e: # pylint: disable=broad-except
logging.info(
"Plugin %s failed to initialize (%s), ignoring it.",
plugin, e)
parent_creator = None
if self.parent_runner:
parent_creator = self.parent_runner.context.creator
context = rdf_flows.FlowContext(
create_time=rdfvalue.RDFDatetime.Now(),
creator=parent_creator or self.token.username,
current_state="Start",
output_plugins_states=output_plugins_states,
remaining_cpu_quota=args.cpu_limit,
state=rdf_flows.FlowContext.State.RUNNING,
# Have we sent a notification to the user.
user_notified=False,
)
return context
def Start(self):
with hunts.GRRHunt.StartHunt(
hunt_name="GenericHunt",
client_limit=0,
flow_runner_args=rdf_flows.FlowRunnerArgs(flow_name="Interrogate"),
flow_args=flows_discovery.InterrogateArgs(lightweight=False),
output_plugins=self.GetOutputPlugins(),
token=self.token) as hunt:
runner = hunt.GetRunner()
runner.runner_args.client_rate = 50
runner.runner_args.expiry_time = "1w"
runner.runner_args.description = ("Interrogate run by cron to keep host"
"info fresh.")
runner.Start()
class TestFileFinderOSHomedir(base.AutomatedTest):
"""List files in homedir with FileFinder.
Exercise globbing and interpolation.
"""
platforms = ["Linux", "Darwin", "Windows"]
flow = file_finder.FileFinder.__name__
action = rdf_file_finder.FileFinderAction.Stat()
args = {
"paths": ["%%users.homedir%%/*"],
"action": action,
"runner_args": rdf_flows.FlowRunnerArgs()
}
def CheckFlow(self):
self.CheckResultCollectionNotEmptyWithRetry(self.session_id)
class TestFileFinderOSHomedir(base.AutomatedTest):
"""List files in homedir with FileFinder.
Exercise globbing and interpolation.
"""
platforms = ["Linux", "Darwin", "Windows"]
flow = "FileFinder"
action = file_finder.FileFinderAction(
action_type=file_finder.FileFinderAction.Action.STAT)
args = {"paths": ["%%users.homedir%%/*"],
"action": action,
"runner_args": rdf_flows.FlowRunnerArgs()}
def CheckFlow(self):
self.CheckCollectionNotEmptyWithRetry(
self.session_id.Add(flow_runner.RESULTS_SUFFIX), self.token)
def setUp(self):
super(ApiGetExportedHuntResultsHandlerTest, self).setUp()
self.handler = hunt_plugin.ApiGetExportedHuntResultsHandler()
self.hunt = hunts.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=DummyFlowWithSingleReply.__name__),
client_rate=0,
token=self.token)
self.hunt.Run()
client_ids = self.SetupClients(5)
self.AssignTasksToClients(client_ids=client_ids)
client_mock = test_lib.SampleHuntMock()
test_lib.TestHuntHelper(client_mock, client_ids, token=self.token)
def CreateSampleHunt(self,
path=None,
stopped=False,
output_plugins=None,
client_limit=0,
client_count=10,
token=None):
token = token or self.token
self.client_ids = self.SetupClients(client_count)
client_rule_set = rdf_foreman.ForemanClientRuleSet(rules=[
rdf_foreman.ForemanClientRule(
rule_type=rdf_foreman.ForemanClientRule.Type.REGEX,
regex=rdf_foreman.ForemanRegexClientRule(
attribute_name="GRR client", attribute_regex="GRR"))
])
with implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=transfer.GetFile.__name__),
flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec(
path=path or "/tmp/evil.txt",
pathtype=rdf_paths.PathSpec.PathType.OS,
)),
client_rule_set=client_rule_set,
output_plugins=output_plugins or [],
client_rate=0,
client_limit=client_limit,
token=token) as hunt:
if not stopped:
hunt.Run()
with aff4.FACTORY.Open("aff4:/foreman", mode="rw",
token=token) as foreman:
for client_id in self.client_ids:
foreman.AssignTasksToClient(client_id)
self.hunt_urn = hunt.urn
return aff4.FACTORY.Open(hunt.urn,
mode="rw",
token=token,
age=aff4.ALL_TIMES)
def CreateSampleHunt(self, description, token=None):
implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
description=description,
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=transfer.GetFile.__name__),
flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec(
path="/tmp/evil.txt",
pathtype=rdf_paths.PathSpec.PathType.TSK,
)),
client_rule_set=self._CreateForemanClientRuleSet(),
output_plugins=[
output_plugin.OutputPluginDescriptor(
plugin_name="DummyOutputPlugin",
plugin_args=gui_test_lib.DummyOutputPlugin.args_type(
filename_regex="blah!", fetch_binaries=True))
],
client_rate=60,
token=token)
def Run(self):
def ReplaceFlowId():
flows_dir_fd = aff4.FACTORY.Open(self.client_id.Add("flows"),
token=self.token)
flow_urn = list(flows_dir_fd.ListChildren())[0]
return {flow_urn.Basename(): "W:ABCDEF"}
with test_lib.FakeTime(42):
self.Check("CreateFlow",
args=flow_plugin.ApiCreateFlowArgs(
client_id=self.client_id.Basename(),
flow=flow_plugin.ApiFlow(
name=processes.ListProcesses.__name__,
args=processes.ListProcessesArgs(
filename_regex=".", fetch_binaries=True),
runner_args=rdf_flows.FlowRunnerArgs(
output_plugins=[],
priority="HIGH_PRIORITY",
notify_to_user=False))),
replace=ReplaceFlowId)
def testGetPluginArgsHandlesMissingPluginsCorrectly(self):
descriptor = output_plugin.OutputPluginDescriptor(
plugin_name="TestOutputPluginWithArgs",
plugin_args=rdf_flows.FlowRunnerArgs(flow_name="GetFile"))
serialized = descriptor.SerializeToString()
deserialized = output_plugin.OutputPluginDescriptor()
deserialized.ParseFromString(serialized)
self.assertEqual(deserialized, descriptor)
self.assertEqual(deserialized.GetPluginClass(), TestOutputPluginWithArgs)
with utils.Stubber(output_plugin.OutputPlugin, "classes", {}):
deserialized = output_plugin.OutputPluginDescriptor()
deserialized.ParseFromString(serialized)
self.assertTrue(deserialized.GetPluginClass(),
output_plugin.UnknownOutputPlugin)
# UnknownOutputPlugin should just return serialized arguments as bytes.
self.assertEqual(deserialized.plugin_args,
descriptor.plugin_args.SerializeToString())
def setUp(self):
super(ApiGetHuntFilesArchiveHandlerTest, self).setUp()
self.handler = hunt_plugin.ApiGetHuntFilesArchiveHandler()
self.hunt = hunts.GRRHunt.StartHunt(
hunt_name="GenericHunt",
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__),
flow_args=file_finder.FileFinderArgs(
paths=[os.path.join(self.base_path, "test.plist")],
action=file_finder.FileFinderAction(action_type="DOWNLOAD"),),
client_rate=0,
token=self.token)
self.hunt.Run()
client_ids = self.SetupClients(10)
self.AssignTasksToClients(client_ids=client_ids)
action_mock = action_mocks.FileFinderClientMock()
test_lib.TestHuntHelper(action_mock, client_ids, token=self.token)
def Layout(self, request, response):
"""Render the form for creating the flow args."""
self.flow_name = self._GetFlowName(request)
self.flow_cls = flow.GRRFlow.classes.get(self.flow_name)
if aff4.issubclass(self.flow_cls, flow.GRRFlow):
self.flow_found = True
self.form = forms.SemanticProtoFormRenderer(
self.flow_cls.GetDefaultArgs(token=request.token),
prefix="args").RawHTML(request)
self.runner_form = forms.SemanticProtoFormRenderer(
rdf_flows.FlowRunnerArgs(flow_name=self.flow_name),
prefix="runner").RawHTML(request)
response = super(SemanticProtoFlowForm, self).Layout(request, response)
return self.CallJavascript(
response,
"SemanticProtoFlowForm.Layout",
renderer=self.__class__.__name__)
def ParseFlowArgs(self):
"""Parse the flow and flow_runner args."""
if self.flow_runner_args is not None:
return self.flow_runner_args, self.flow_args
flow_path = self.request.REQ.get("flow_path", "")
flow_name = os.path.basename(flow_path)
if not flow_name:
raise ValueError("No flow specified. Please select a flow.")
flow_cls = flow.GRRFlow.GetPlugin(flow_name)
self.flow_args = forms.SemanticProtoFormRenderer(
flow_cls.args_type(), prefix="args").ParseArgs(self.request)
self.flow_runner_args = forms.SemanticProtoFormRenderer(
rdf_flows.FlowRunnerArgs(),
prefix="runner").ParseArgs(self.request)
self.flow_runner_args.flow_name = flow_name
return self.flow_runner_args, self.flow_args
def setUp(self):
super(ApiGetExportedHuntResultsHandlerTest, self).setUp()
self.handler = hunt_plugin.ApiGetExportedHuntResultsHandler()
self.hunt = implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=flow_test_lib.DummyFlowWithSingleReply.__name__),
client_rate=0,
token=self.token)
self.hunt.Run()
self.client_ids = self.SetupClients(5)
# Ensure that clients are processed sequentially - this way the test won't
# depend on the order of results in the collection (which is normally
# random).
for cid in self.client_ids:
self.AssignTasksToClients(client_ids=[cid])
client_mock = hunt_test_lib.SampleHuntMock()
hunt_test_lib.TestHuntHelper(client_mock, [cid], token=self.token)
def RenderAjax(self, request, response):
"""Parse the flow args from the form and launch the flow."""
self.flow_name = self._GetFlowName(request)
self.client_id = request.REQ.get("client_id", None)
self.dom_node = request.REQ.get("dom_node")
flow_cls = flow.GRRFlow.classes.get(self.flow_name)
if flow_cls is not None:
self.args = forms.SemanticProtoFormRenderer(
flow_cls.args_type(), prefix="args").ParseArgs(request)
try:
self.args.Validate()
except ValueError as e:
return self.CallJavascript(
response, "SemanticProtoFlowForm.RenderAjaxError", error=str(e))
self.runner_args = forms.SemanticProtoFormRenderer(
rdf_flows.FlowRunnerArgs(), prefix="runner_").ParseArgs(request)
self.runner_args.Validate()
self.flow_id = flow.GRRFlow.StartFlow(
client_id=self.client_id,
flow_name=self.flow_name,
token=request.token,
args=self.args,
runner_args=self.runner_args)
self.args_html = semantic.FindRendererForObject(self.args).RawHTML(request)
self.runner_args_html = semantic.FindRendererForObject(
self.runner_args).RawHTML(request)
response = renderers.TemplateRenderer.Layout(
self, request, response, apply_template=self.ajax_template)
return self.CallJavascript(
response,
"SemanticProtoFlowForm.RenderAjax",
renderer=self.__class__.__name__,
dom_node=self.dom_node)
def testCopyHuntPreservesRuleType(self):
implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
description="model hunt",
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=transfer.GetFile.__name__),
flow_args=transfer.GetFileArgs(
pathspec=rdf_paths.PathSpec(
path="/tmp/evil.txt",
pathtype=rdf_paths.PathSpec.PathType.TSK,
)),
client_rule_set=foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.OS,
os=foreman_rules.ForemanOsClientRule(os_darwin=True))
]),
token=self.token)
self.Open("/#main=ManageHunts")
self.Click("css=tr:contains('model hunt')")
self.Click("css=button[name=CopyHunt]:not([disabled])")
# Wait until dialog appears.
self.WaitUntil(self.IsElementPresent,
"css=grr-wizard-form:contains('What to run?')")
# Click on "Next" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsElementPresent,
"css=grr-wizard-form:contains('Hunt parameters')")
# Click on "Next" button.
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsElementPresent,
"css=grr-wizard-form:contains('How to process results')")
# Click on "Next" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsElementPresent,
"css=grr-wizard-form:contains('Where to run?')")
self.WaitUntil(
self.IsElementPresent, "css=grr-new-hunt-wizard-form "
"label:contains('Os darwin') ~ * input:checked")
def CreateSampleHunt(description, token=None):
hunts.GRRHunt.StartHunt(
hunt_name="GenericHunt",
description=description,
flow_runner_args=rdf_flows.FlowRunnerArgs(flow_name="GetFile"),
flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec(
path="/tmp/evil.txt",
pathtype=rdf_paths.PathSpec.PathType.TSK,
)),
client_rule_set=rdf_foreman.ForemanClientRuleSet(rules=[
rdf_foreman.ForemanClientRule(
rule_type=rdf_foreman.ForemanClientRule.Type.REGEX,
regex=rdf_foreman.ForemanRegexClientRule(
attribute_name="GRR client", attribute_regex="GRR"))
]),
output_plugins=[
output_plugin.OutputPluginDescriptor(
plugin_name="DummyOutputPlugin",
plugin_args=DummyOutputPlugin.args_type(
filename_regex="blah!", fetch_binaries=True))
],
client_rate=60,
token=token)
def testListHuntClients(self):
hunt = implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__),
flow_args=rdf_file_finder.FileFinderArgs(
paths=[os.path.join(self.base_path, "test.plist")],
action=rdf_file_finder.FileFinderAction(
action_type="DOWNLOAD"),
),
client_rate=0,
token=self.token)
hunt.Run()
client_ids = self.SetupClients(5)
self.AssignTasksToClients(client_ids=client_ids)
action_mock = action_mocks.FileFinderClientMock()
hunt_test_lib.TestHuntHelper(action_mock,
client_ids,
iteration_limit=10,
token=self.token)
result = self.handler.Handle(
hunt_plugin.ApiListHuntClientsArgs(hunt_id=hunt.urn.Basename()),
token=self.token)
# TODO(user): This still uses data store internals and will fail on some
# data stores.
# This is not super deterministic, we start processing some
# clients, run the hunt for a bit but there is no order to all
# this. We should have some clients half finished though (i.e.,
# with pending requests) and five clients in total.
self.assertEqual(result.total_count, 5)
clients = list(result.items)
pending_requests = [client.pending_requests for client in clients]
self.assertTrue(any(r.next_state) for r in pending_requests)
def Handle(self, args, token=None):
client_urn = self.GetClientTarget(args, token=token)
size_condition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE,
size=rdf_file_finder.FileFinderSizeCondition(
max_file_size=args.max_file_size))
file_finder_args = rdf_file_finder.FileFinderArgs(
paths=args.paths,
action=rdf_file_finder.FileFinderAction(action_type=args.action),
conditions=[size_condition])
# Check our flow throttling limits, will raise if there are problems.
throttler = throttle.FlowThrottler(
daily_req_limit=config_lib.CONFIG.Get("API.DailyFlowRequestLimit"),
dup_interval=config_lib.CONFIG.Get("API.FlowDuplicateInterval"))
throttler.EnforceLimits(client_urn,
token.username,
file_finder.FileFinder.__name__,
file_finder_args,
token=token)
# Limit the whole flow to 200MB so if a glob matches lots of small files we
# still don't have too much impact.
runner_args = rdf_flows.FlowRunnerArgs(
client_id=client_urn,
flow_name=file_finder.FileFinder.__name__,
network_bytes_limit=200 * 1000 * 1000)
flow_id = flow.GRRFlow.StartFlow(runner_args=runner_args,
token=token,
args=file_finder_args)
return ApiStartRobotGetFilesOperationResult(
operation_id=utils.SmartUnicode(flow_id))
def setUp(self):
super(ApiGetHuntFileHandlerTest, self).setUp()
self.handler = hunt_plugin.ApiGetHuntFileHandler()
self.file_path = os.path.join(self.base_path, "test.plist")
self.hunt = hunts.GRRHunt.StartHunt(
hunt_name="GenericHunt",
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__),
flow_args=file_finder.FileFinderArgs(
paths=[self.file_path],
action=file_finder.FileFinderAction(action_type="DOWNLOAD"),),
client_rate=0,
token=self.token)
self.hunt.Run()
self.results_urn = self.hunt.results_collection_urn
self.aff4_file_path = "fs/os/%s" % self.file_path
self.client_id = self.SetupClients(1)[0]
self.AssignTasksToClients(client_ids=[self.client_id])
action_mock = action_mocks.FileFinderClientMock()
test_lib.TestHuntHelper(action_mock, [self.client_id], token=self.token)
def testCPULimitForHunts(self):
worker_obj = worker.GRRWorker(token=self.token)
client_ids = ["C.%016X" % i for i in xrange(10, 20)]
result = {}
client_mocks = []
for client_id in client_ids:
client_mock = action_mocks.CPULimitClientMock(result)
client_mock = flow_test_lib.MockClient(
rdf_client.ClientURN(client_id), client_mock, token=self.token)
client_mock.EnableResourceUsage(user_cpu_usage=[10],
system_cpu_usage=[10],
network_usage=[1000])
client_mocks.append(client_mock)
flow_runner_args = rdf_flows.FlowRunnerArgs(
flow_name=flow_test_lib.CPULimitFlow.__name__)
with implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=flow_runner_args,
cpu_limit=5000,
per_client_cpu_limit=10000,
network_bytes_limit=1000000,
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[:1])
self._Process(client_mocks, worker_obj)
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[1:2])
self._Process(client_mocks, worker_obj)
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[2:3])
self._Process(client_mocks, worker_obj)
# The limiting factor here is the overall hunt limit of 5000 cpu
# seconds. Clients that finish should decrease the remaining quota
# and the following clients should get the reduced quota.
self.assertEqual(result["cpulimit"], [
5000.0, 4980.0, 4960.0, 4940.0, 4920.0, 4900.0, 4880.0, 4860.0,
4840.0
])
self.assertEqual(result["networklimit"], [
1000000L, 999000L, 998000L, 997000L, 996000L, 995000L, 994000L,
993000L, 992000L
])
result.clear()
with implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=flow_runner_args,
per_client_cpu_limit=3000,
per_client_network_limit_bytes=3000000,
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[:1])
self._Process(client_mocks, worker_obj)
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[1:2])
self._Process(client_mocks, worker_obj)
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[2:3])
self._Process(client_mocks, worker_obj)
# This time, the per client limit is 3000s / 3000000 bytes. Every
# client should get the same limit.
self.assertEqual(result["cpulimit"], [
3000.0, 2980.0, 2960.0, 3000.0, 2980.0, 2960.0, 3000.0, 2980.0,
2960.0
])
self.assertEqual(result["networklimit"], [
3000000, 2999000, 2998000, 3000000, 2999000, 2998000, 3000000,
2999000, 2998000
])
result.clear()
for client_mock in client_mocks:
client_mock.EnableResourceUsage(user_cpu_usage=[500],
system_cpu_usage=[500],
network_usage=[1000000])
with implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=flow_runner_args,
per_client_cpu_limit=3000,
cpu_limit=5000,
per_client_network_limit_bytes=3000000,
network_bytes_limit=5000000,
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[:1])
self._Process(client_mocks, worker_obj)
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[1:2])
self._Process(client_mocks, worker_obj)
implementation.GRRHunt.StartClients(hunt.session_id, client_ids[2:3])
self._Process(client_mocks, worker_obj)
# The first client gets the full per client limit of 3000s, and
# uses all of it. The hunt has a limit of just 5000 total so the
# second client gets started with a limit of 2000. It can only run
# two of three states, the last client will not be started at all
# due to out of quota.
self.assertEqual(result["cpulimit"],
[3000.0, 2000.0, 1000.0, 2000.0, 1000.0])
self.assertEqual(result["networklimit"],
[3000000, 2000000, 1000000, 2000000, 1000000])
errors = list(hunt.GetClientsErrors())
self.assertEqual(len(errors), 2)
# Client side out of cpu.
self.assertIn("CPU limit exceeded", errors[0].log_message)
# Server side out of cpu.
self.assertIn("Out of CPU quota", errors[1].backtrace)
def StartFlow(cls,
args=None,
runner_args=None,
parent_flow=None,
sync=True,
token=None,
**kwargs):
"""The main factory function for Creating and executing a new flow.
Args:
args: An arg protocol buffer which is an instance of the required flow's
args_type class attribute.
runner_args: an instance of FlowRunnerArgs() protocol buffer which is used
to initialize the runner for this flow.
parent_flow: A parent flow or None if this is a top level flow.
sync: If True, the Start method of this flow will be called
inline. Otherwise we schedule the starting of this flow on another
worker.
token: Security credentials token identifying the user.
**kwargs: If args or runner_args are not specified, we construct these
protobufs from these keywords.
Returns:
the session id of the flow.
Raises:
RuntimeError: Unknown or invalid parameters were provided.
"""
# Build the runner args from the keywords.
if runner_args is None:
runner_args = rdf_flows.FlowRunnerArgs()
cls.FilterArgsFromSemanticProtobuf(runner_args, kwargs)
# When asked to run a flow in the future this implied it will run
# asynchronously.
if runner_args.start_time:
sync = False
# Is the required flow a known flow?
if runner_args.flow_name not in GRRFlow.classes:
stats.STATS.IncrementCounter("grr_flow_invalid_flow_count")
raise RuntimeError("Unable to locate flow %s" %
runner_args.flow_name)
# If no token is specified, raise.
if not token:
raise access_control.UnauthorizedAccess(
"A token must be specified.")
# For the flow itself we use a supervisor token.
token = token.SetUID()
# Extend the expiry time of this token indefinitely. Python on Windows only
# supports dates up to the year 3000.
token.expiry = rdfvalue.RDFDatetime.FromHumanReadable("2997-01-01")
flow_cls = aff4.AFF4Object.classes.get(runner_args.flow_name)
if flow_cls.category and not runner_args.client_id:
raise RuntimeError(
"Flow with category (user-visible flow) has to be "
"started on a client, but runner_args.client_id "
"is missing.")
# We create an anonymous AFF4 object first, The runner will then generate
# the appropriate URN.
flow_obj = aff4.FACTORY.Create(None, flow_cls, token=token)
# Now parse the flow args into the new object from the keywords.
if args is None:
args = flow_obj.args_type()
cls.FilterArgsFromSemanticProtobuf(args, kwargs)
# Check that the flow args are valid.
args.Validate()
# Store the flow args.
flow_obj.args = args
flow_obj.runner_args = runner_args
# At this point we should exhaust all the keyword args. If any are left
# over, we do not know what to do with them so raise.
if kwargs:
raise type_info.UnknownArg("Unknown parameters to StartFlow: %s" %
kwargs)
# Create a flow runner to run this flow with.
if parent_flow:
parent_runner = parent_flow.runner
else:
parent_runner = None
runner = flow_obj.CreateRunner(parent_runner=parent_runner,
runner_args=runner_args)
logging.info(u"Scheduling %s(%s) on %s", flow_obj.urn,
runner_args.flow_name, runner_args.client_id)
if sync:
# Just run the first state inline. NOTE: Running synchronously means
# that this runs on the thread that starts the flow. The advantage is
# that that Start method can raise any errors immediately.
flow_obj.Start()
else:
# Running Asynchronously: Schedule the start method on another worker.
runner.CallState(next_state="Start",
start_time=runner_args.start_time)
# The flow does not need to actually remain running.
if not runner.OutstandingRequests():
flow_obj.Terminate()
flow_obj.Close()
# Publish an audit event, only for top level flows.
if parent_flow is None:
events.Events.PublishEvent("Audit",
events.AuditEvent(
user=token.username,
action="RUN_FLOW",
flow_name=runner_args.flow_name,
urn=flow_obj.urn,
client=runner_args.client_id),
token=token)
return flow_obj.urn
def testCopyHuntHandlesLiteralExpressionCorrectly(self):
"""Literals are raw bytes. Testing that raw bytes are processed right."""
literal_match = rdf_file_finder.FileFinderContentsLiteralMatchCondition(
literal="foo\x0d\xc8bar")
implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
description="model hunt",
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__),
flow_args=rdf_file_finder.FileFinderArgs(
conditions=[
rdf_file_finder.FileFinderCondition(
condition_type="CONTENTS_LITERAL_MATCH",
contents_literal_match=literal_match)
],
paths=["/tmp/evil.txt"]),
token=self.token)
self.Open("/#main=ManageHunts")
self.Click("css=tr:contains('model hunt')")
self.Click("css=button[name=CopyHunt]:not([disabled])")
# Wait until dialog appears.
self.WaitUntil(self.IsTextPresent, "What to run?")
# Check that non-default values of sample hunt are prefilled.
self.WaitUntilEqual("foo\\x0d\\xc8bar", self.GetValue,
"css=grr-new-hunt-wizard-form "
"label:contains('Literal') ~ * input:text")
# Click on "Next" button.
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Output Processing")
# Click on "Next" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Where to run?")
# Click on "Next" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Review")
# Check that the arguments summary is present.
self.WaitUntil(self.IsTextPresent, file_finder.FileFinder.__name__)
self.WaitUntil(self.IsTextPresent, "foo\\x0d\\xc8bar")
# Click on "Run" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Created Hunt")
# Close the window and check that the hunt was created.
self.Click("css=button.Next")
hunts_root = aff4.FACTORY.Open("aff4:/hunts", token=self.token)
hunts_list = sorted(list(hunts_root.ListChildren()), key=lambda x: x.age)
self.assertEqual(len(hunts_list), 2)
last_hunt = aff4.FACTORY.Open(hunts_list[-1], token=self.token)
# Check that the hunt was created with a correct literal value.
self.assertEqual(last_hunt.args.flow_runner_args.flow_name,
file_finder.FileFinder.__name__)
self.assertEqual(last_hunt.args.flow_args.conditions[0]
.contents_literal_match.literal, "foo\x0d\xc8bar")
def StartFlow(cls, args=None, runner_args=None, # pylint: disable=g-bad-name
parent_flow=None, sync=True, **kwargs):
"""The main factory function for Creating and executing a new flow.
Args:
args: An arg protocol buffer which is an instance of the required flow's
args_type class attribute.
runner_args: an instance of FlowRunnerArgs() protocol buffer which is used
to initialize the runner for this flow.
parent_flow: A parent flow or None if this is a top level flow.
sync: If True, the Start method of this flow will be called
inline. Otherwise we schedule the starting of this flow on another
worker.
**kwargs: If args or runner_args are not specified, we construct these
protobufs from these keywords.
Returns:
the session id of the flow.
Raises:
RuntimeError: Unknown or invalid parameters were provided.
"""
# Build the runner args from the keywords.
if runner_args is None:
runner_args = rdf_flows.FlowRunnerArgs()
cls.FilterArgsFromSemanticProtobuf(runner_args, kwargs)
# When asked to run a flow in the future this implied it will run
# asynchronously.
if runner_args.start_time:
sync = False
# Is the required flow a known flow?
if runner_args.flow_name not in GRRFlow.classes:
stats.STATS.IncrementCounter("grr_flow_invalid_flow_count")
raise RuntimeError("Unable to locate flow %s" % runner_args.flow_name)
# If no token is specified, use the default token.
if not runner_args.HasField("token"):
if data_store.default_token is None:
raise access_control.UnauthorizedAccess("A token must be specified.")
runner_args.token = data_store.default_token.Copy()
# Make sure we are allowed to run this flow. If not, we raise here. We
# respect SUID (supervisor) if it is already set. SUID cannot be set by the
# user since it isn't part of the ACLToken proto.
data_store.DB.security_manager.CheckIfCanStartFlow(
runner_args.token,
runner_args.flow_name,
with_client_id=runner_args.client_id)
flow_cls = GRRFlow.GetPlugin(runner_args.flow_name)
# If client id was specified and flow doesn't have exemption from ACL
# checking policy, then check that the user has access to the client
# where the flow is going to run.
if flow_cls.ACL_ENFORCED and runner_args.client_id:
data_store.DB.security_manager.CheckClientAccess(runner_args.token,
runner_args.client_id)
# For the flow itself we use a supervisor token.
token = runner_args.token.SetUID()
# Extend the expiry time of this token indefinitely. Python on Windows only
# supports dates up to the year 3000, this number corresponds to July, 2997.
token.expiry = 32427003069 * rdfvalue.RDFDatetime.converter
# We create an anonymous AFF4 object first, The runner will then generate
# the appropriate URN.
flow_obj = aff4.FACTORY.Create(
None, aff4.AFF4Object.classes.get(runner_args.flow_name), token=token)
# Now parse the flow args into the new object from the keywords.
if args is None:
args = flow_obj.args_type()
cls.FilterArgsFromSemanticProtobuf(args, kwargs)
# Check that the flow args are valid.
args.Validate()
# Store the flow args.
flow_obj.args = args
flow_obj.runner_args = runner_args
# At this point we should exhaust all the keyword args. If any are left
# over, we do not know what to do with them so raise.
if kwargs:
raise type_info.UnknownArg("Unknown parameters to StartFlow: %s" % kwargs)
# Create a flow runner to run this flow with.
if parent_flow:
parent_runner = parent_flow.runner
else:
parent_runner = None
runner = flow_obj.CreateRunner(
parent_runner=parent_runner, runner_args=runner_args)
logging.info(u"Scheduling %s(%s) on %s", flow_obj.urn,
runner_args.flow_name, runner_args.client_id)
if sync:
# Just run the first state inline. NOTE: Running synchronously means
# that this runs on the thread that starts the flow. The advantage is
# that that Start method can raise any errors immediately.
flow_obj.Start()
else:
# Running Asynchronously: Schedule the start method on another worker.
runner.CallState(next_state="Start", start_time=runner_args.start_time)
# The flow does not need to actually remain running.
if not runner.OutstandingRequests():
flow_obj.Terminate()
flow_obj.Close()
# Publish an audit event, only for top level flows.
if parent_flow is None:
events.Events.PublishEvent(
"Audit",
events.AuditEvent(
user=token.username,
action="RUN_FLOW",
flow_name=runner_args.flow_name,
urn=flow_obj.urn,
client=runner_args.client_id),
token=token)
return flow_obj.urn
def Start(self):
self.state.hunt_id = None
self.state.client_ids_failures = set()
self.state.client_ids_result_reported = set()
# TODO(user): Figure out if this can just be a set. Add a tap test to
# meaningfully exercise this code.
self.state.client_ids = list(
base.GetClientTestTargets(token=self.token))
if not self.state.client_ids:
self.Log("No clients to test on, define them in "
"Test.end_to_end_client_ids")
return
# SetUID is required to run a hunt on the configured end-to-end client
# targets without an approval.
token = access_control.ACLToken(
username="******", reason="Running endtoend tests.").SetUID()
runner_args = rdf_flows.FlowRunnerArgs(
flow_name=flows_endtoend.EndToEndTestFlow.__name__)
flow_request = hunts_standard.FlowRequest(
client_ids=self.state.client_ids,
args=flows_endtoend.EndToEndTestFlowArgs(),
runner_args=runner_args)
bogus_rule = rdf_foreman.ForemanRegexClientRule(
attribute_name="System", attribute_regex="Does not match anything")
client_rule_set = rdf_foreman.ForemanClientRuleSet(rules=[
rdf_foreman.ForemanClientRule(
rule_type=rdf_foreman.ForemanClientRule.Type.REGEX,
regex=bogus_rule)
])
hunt_args = hunts_standard.VariableGenericHuntArgs(
flows=[flow_request])
hunt_args.output_plugins = self.GetOutputPlugins()
with hunts_implementation.GRRHunt.StartHunt(
hunt_name=hunts_standard.VariableGenericHunt.__name__,
args=hunt_args,
client_rule_set=client_rule_set,
client_rate=0,
expiry_time="1d",
token=token) as hunt:
self.state.hunt_id = hunt.session_id
hunt.SetDescription("EndToEnd tests run by cron")
hunt.Run()
hunt.ManuallyScheduleClients(token=token)
# Set a callback to check the results after 50 minutes. This should be
# plenty of time for the clients to receive the hunt and run the tests, but
# not so long that the flow lease will expire.
wait_duration = rdfvalue.Duration(
config.CONFIG.Get("Test.end_to_end_result_check_wait"))
completed_time = rdfvalue.RDFDatetime.Now() + wait_duration
self.CallState(next_state="CheckResults", start_time=completed_time)
