def ParseOption(self, option, request):
"""Parse the form that is selected by option."""
if option == "Windows":
return implementation.GRRHunt.MATCH_WINDOWS
elif option == "Linux":
return implementation.GRRHunt.MATCH_LINUX
elif option == "OSX":
return implementation.GRRHunt.MATCH_DARWIN
elif option == "Label":
label_name = ClientLabelNameFormRenderer(
descriptor=type_info.TypeInfoObject(),
default="",
prefix=self.prefix).ParseArgs(request)
regex = aff4_rdfvalues.AFF4ObjectLabelsList.RegexForStringifiedValueMatch(
label_name)
return rdf_foreman.ForemanAttributeRegex(attribute_name="Labels",
attribute_regex=regex)
elif option == "Regex":
return forms.SemanticProtoFormRenderer(
rdf_foreman.ForemanAttributeRegex(),
prefix=self.prefix).ParseArgs(request)
elif option == "Integer":
return forms.SemanticProtoFormRenderer(
rdf_foreman.ForemanAttributeInteger(),
prefix=self.prefix).ParseArgs(request)
def testCallback(self, client_limit=None):
"""Checks that the foreman uses the callback specified in the action."""
with hunts.GRRHunt.StartHunt(hunt_name="SampleHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")
],
client_limit=client_limit,
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
# Create a client that matches our regex.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
info = client.Schema.CLIENT_INFO()
info.client_name = "GRR Monitor"
client.Set(client.Schema.CLIENT_INFO, info)
client.Close()
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
with utils.Stubber(hunts.SampleHunt, "StartClients", self.Callback):
self.called = []
foreman.AssignTasksToClient(client.urn)
self.assertEqual(len(self.called), 1)
self.assertEqual(self.called[0][1], [client.urn])
def testHuntNotifications(self):
"""This tests the Hunt notification event."""
TestHuntListener.received_events = []
# Set up 10 clients.
client_ids = self.SetupClients(10)
with hunts.GRRHunt.StartHunt(hunt_name="BrokenSampleHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")
],
client_rate=0,
notification_event="TestHuntDone",
token=self.token) as hunt:
hunt.GetRunner().Start()
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
for client_id in client_ids:
foreman.AssignTasksToClient(client_id)
# Run the hunt.
client_mock = test_lib.SampleHuntMock()
test_lib.TestHuntHelper(client_mock,
client_ids,
check_flow_errors=False,
token=self.token)
self.assertEqual(len(TestHuntListener.received_events), 5)
def testBrokenHunt(self):
"""This tests the behavior when a hunt raises an exception."""
# Set up 10 clients.
client_ids = self.SetupClients(10)
with hunts.GRRHunt.StartHunt(
hunt_name="BrokenSampleHunt",
regex_rules=[rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")],
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
for client_id in client_ids:
foreman.AssignTasksToClient(client_id)
# Run the hunt.
client_mock = test_lib.SampleHuntMock()
test_lib.TestHuntHelper(client_mock, client_ids, False, self.token)
hunt_obj = aff4.FACTORY.Open(hunt.session_id, mode="rw",
age=aff4.ALL_TIMES, token=self.token)
started, finished, errors = hunt_obj.GetClientsCounts()
self.assertEqual(started, 10)
# There should be errors for the five clients where the hunt raised.
self.assertEqual(errors, 5)
# All of the clients that have the file should still finish eventually.
self.assertEqual(finished, 5)
def _RunRateLimitedHunt(self, client_ids, start_time):
with hunts.GRRHunt.StartHunt(
hunt_name="DummyHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(attribute_name="GRR client",
attribute_regex="GRR"),
],
client_rate=1, token=self.token) as hunt:
hunt.Run()
# Pretend to be the foreman now and dish out hunting jobs to all the
# clients..
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
for client_id in client_ids:
foreman.AssignTasksToClient(client_id)
self.assertEqual(len(DummyHunt.client_ids), 0)
# Run the hunt.
worker_mock = test_lib.MockWorker(check_flow_errors=True,
queues=queues.HUNTS,
token=self.token)
# One client is scheduled in the first minute.
with test_lib.FakeTime(start_time + 2):
worker_mock.Simulate()
self.assertEqual(len(DummyHunt.client_ids), 1)
# No further clients will be scheduled until the end of the first minute.
with test_lib.FakeTime(start_time + 59):
worker_mock.Simulate()
self.assertEqual(len(DummyHunt.client_ids), 1)
return worker_mock, hunt.urn
def testProcessing(self):
"""This tests running the hunt on some clients."""
# Set up 10 clients.
client_ids = self.SetupClients(10)
with hunts.GRRHunt.StartHunt(
hunt_name="SampleHunt",
regex_rules=[rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")],
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
for client_id in client_ids:
foreman.AssignTasksToClient(client_id)
# Run the hunt.
client_mock = test_lib.SampleHuntMock()
test_lib.TestHuntHelper(client_mock, client_ids, False, self.token)
hunt_obj = aff4.FACTORY.Open(
hunt.session_id, mode="r", age=aff4.ALL_TIMES,
aff4_type=hunts.SampleHunt, token=self.token)
started, finished, _ = hunt_obj.GetClientsCounts()
self.assertEqual(started, 10)
self.assertEqual(finished, 10)
def testHangingClients(self):
"""This tests if the hunt completes when some clients hang or raise."""
# Set up 10 clients.
client_ids = self.SetupClients(10)
with hunts.GRRHunt.StartHunt(
hunt_name="SampleHunt",
regex_rules=[rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")],
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
for client_id in client_ids:
foreman.AssignTasksToClient(client_id)
client_mock = test_lib.SampleHuntMock()
# Just pass 8 clients to run, the other two went offline.
test_lib.TestHuntHelper(client_mock, client_ids[1:9], False, self.token)
hunt_obj = aff4.FACTORY.Open(hunt.session_id, mode="rw",
age=aff4.ALL_TIMES, token=self.token)
started, finished, _ = hunt_obj.GetClientsCounts()
# We started the hunt on 10 clients.
self.assertEqual(started, 10)
# But only 8 should have finished.
self.assertEqual(finished, 8)
def CreateGenericHuntWithCollection(self, values=None):
self.client_ids = self.SetupClients(10)
if values is None:
values = [
rdfvalue.RDFURN("aff4:/sample/1"),
rdfvalue.RDFURN("aff4:/C.0000000000000001/fs/os/c/bin/bash"),
rdfvalue.RDFURN("aff4:/sample/3")
]
with hunts.GRRHunt.StartHunt(hunt_name="GenericHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")
],
output_plugins=[],
token=self.token) as hunt:
runner = hunt.GetRunner()
runner.Start()
with aff4.FACTORY.Open(runner.context.results_collection_urn,
aff4_type="HuntResultCollection",
mode="w",
token=self.token) as collection:
for value in values:
collection.Add(rdf_flows.GrrMessage(payload=value))
return hunt.urn
def testClientLimit(self):
"""This tests that we can limit hunts to a number of clients."""
# Set up 10 clients.
client_ids = self.SetupClients(10)
with hunts.GRRHunt.StartHunt(hunt_name="SampleHunt",
client_limit=5,
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")
],
client_rate=0,
token=self.token) as hunt:
hunt.Run()
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
for client_id in client_ids:
foreman.AssignTasksToClient(client_id)
# Run the hunt.
client_mock = test_lib.SampleHuntMock()
test_lib.TestHuntHelper(client_mock, client_ids, False, self.token)
hunt_obj = aff4.FACTORY.Open(hunt.urn,
mode="rw",
age=aff4.ALL_TIMES,
token=self.token)
started, finished, _ = hunt_obj.GetClientsCounts()
# We limited here to 5 clients.
self.assertEqual(started, 5)
self.assertEqual(finished, 5)
def SetUpCrashedFlowInHunt(self):
client_ids = [
rdf_client.ClientURN("C.%016X" % i) for i in range(0, 10)
]
client_mocks = dict([(client_id,
test_lib.CrashClientMock(client_id, self.token))
for client_id in client_ids])
with hunts.GRRHunt.StartHunt(hunt_name="SampleHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")
],
client_rate=0,
token=self.token) as hunt:
hunt.Run()
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
for client_id in client_ids:
foreman.AssignTasksToClient(client_id)
test_lib.TestHuntHelperWithMultipleMocks(client_mocks, False,
self.token)
return client_ids
def CreateSampleHunt(self, token=None):
with hunts.GRRHunt.StartHunt(hunt_name="SampleHunt",
regex_rules=[rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")],
token=token or self.token) as hunt:
return hunt.session_id
def testRuleAdding(self):
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
rules = foreman.Get(foreman.Schema.RULES)
# Make sure there are no rules yet in the foreman.
self.assertEqual(len(rules), 0)
hunt = hunts.GRRHunt.StartHunt(
hunt_name="SampleHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="HUNT")
],
integer_rules=[
rdf_foreman.ForemanAttributeInteger(
attribute_name="Clock",
operator=
rdf_foreman.ForemanAttributeInteger.Operator.GREATER_THAN,
value=1336650631137737)
],
client_rate=0,
token=self.token)
# Push the rules to the foreman.
with hunt:
hunt.GetRunner().Start()
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
rules = foreman.Get(foreman.Schema.RULES)
# Make sure they were written correctly.
self.assertEqual(len(rules), 1)
rule = rules[0]
self.assertEqual(len(rule.regex_rules), 1)
self.assertEqual(rule.regex_rules[0].attribute_name, "GRR client")
self.assertEqual(rule.regex_rules[0].attribute_regex, "HUNT")
self.assertEqual(len(rule.integer_rules), 1)
self.assertEqual(rule.integer_rules[0].attribute_name, "Clock")
self.assertEqual(rule.integer_rules[0].operator,
rdf_foreman.ForemanAttributeInteger.Operator.GREATER_THAN)
self.assertEqual(rule.integer_rules[0].value, 1336650631137737)
self.assertEqual(len(rule.actions), 1)
self.assertEqual(rule.actions[0].hunt_name, "SampleHunt")
# Running a second time should not change the rules any more.
with hunt:
hunt.GetRunner().Start()
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
rules = foreman.Get(foreman.Schema.RULES)
# Still just one rule.
self.assertEqual(len(rules), 1)
def Start(self):
self.state.Register("hunt_id", None)
self.state.Register("client_ids", set())
self.state.Register("client_ids_failures", set())
self.state.Register("client_ids_result_reported", set())
self.state.client_ids = base.GetClientTestTargets(token=self.token)
if not self.state.client_ids:
self.Log("No clients to test on, define them in "
"Test.end_to_end_client_ids")
return
# SetUID is required to run a hunt on the configured end-to-end client
# targets without an approval.
token = access_control.ACLToken(
username="******", reason="Running endtoend tests.").SetUID()
runner_args = flow_runner.FlowRunnerArgs(flow_name="EndToEndTestFlow")
flow_request = hunts_standard.FlowRequest(
client_ids=self.state.client_ids,
args=flows_endtoend.EndToEndTestFlowArgs(),
runner_args=runner_args)
bogus_rule = rdf_foreman.ForemanAttributeRegex(
attribute_name="System", attribute_regex="Does not match anything")
hunt_args = hunts_standard.VariableGenericHuntArgs(
flows=[flow_request])
hunt_args.output_plugins = self.GetOutputPlugins()
with hunts.GRRHunt.StartHunt(hunt_name="VariableGenericHunt",
args=hunt_args,
regex_rules=[bogus_rule],
client_rate=0,
expiry_time="1d",
token=token) as hunt:
self.state.hunt_id = hunt.session_id
hunt.SetDescription("EndToEnd tests run by cron")
hunt.Run()
hunt.ManuallyScheduleClients(token=token)
# Set a callback to check the results after 50 minutes. This should be
# plenty of time for the clients to receive the hunt and run the tests, but
# not so long that the flow lease will expire.
wait_duration = rdfvalue.Duration(
config_lib.CONFIG.Get("Test.end_to_end_result_check_wait"))
completed_time = rdfvalue.RDFDatetime().Now() + wait_duration
self.CallState(next_state="CheckResults", start_time=completed_time)
def testInvalidRules(self):
"""Tests the behavior when a wrong attribute name is passed in a rule."""
with hunts.GRRHunt.StartHunt(
hunt_name="BrokenSampleHunt",
regex_rules=[rdf_foreman.ForemanAttributeRegex(
attribute_name="no such attribute",
attribute_regex="HUNT")],
client_rate=0,
token=self.token) as hunt:
runner = hunt.GetRunner()
self.assertRaises(ValueError, runner.Start)
def testPausingAndRestartingDoesNotStartHuntTwiceOnTheSameClient(self):
"""This tests if the hunt completes when some clients hang or raise."""
client_ids = self.SetupClients(10)
with hunts.GRRHunt.StartHunt(hunt_name="SampleHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")
],
client_rate=0,
token=self.token) as hunt:
hunt.GetRunner().Start()
hunt_id = hunt.urn
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
for client_id in client_ids:
num_tasks = foreman.AssignTasksToClient(client_id)
self.assertEqual(num_tasks, 1)
client_mock = test_lib.SampleHuntMock()
test_lib.TestHuntHelper(client_mock, client_ids, False, self.token)
# Pausing and running hunt: this leads to the fresh rules being written
# to Foreman.RULES.
with aff4.FACTORY.Open(hunt_id, mode="rw", token=self.token) as hunt:
runner = hunt.GetRunner()
runner.Pause()
runner.Start()
# Recreating the foreman so that it updates list of rules.
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
for client_id in client_ids:
num_tasks = foreman.AssignTasksToClient(client_id)
# No tasks should be assigned as this hunt ran on all the clients
# before.
self.assertEqual(num_tasks, 0)
def RenderOption(self, option, request, response):
if option == "Windows":
return self.RenderFromTemplate(self.match_system_template,
response,
system="Windows")
elif option == "Linux":
return self.RenderFromTemplate(self.match_system_template,
response,
system="Linux")
elif option == "OSX":
return self.RenderFromTemplate(self.match_system_template,
response,
system="OSX")
elif option == "Label":
return self.RenderFromTemplate(
self.form_template,
response,
form=ClientLabelNameFormRenderer(
descriptor=type_info.TypeInfoObject(friendly_name="Label"),
default="",
prefix=self.prefix).RawHTML(request))
elif option == "Regex":
return self.RenderFromTemplate(
self.form_template,
response,
form=forms.SemanticProtoFormRenderer(
rdf_foreman.ForemanAttributeRegex(),
prefix=self.prefix).RawHTML(request))
elif option == "Integer":
return self.RenderFromTemplate(
self.form_template,
response,
form=forms.SemanticProtoFormRenderer(
rdf_foreman.ForemanAttributeInteger(),
prefix=self.prefix).RawHTML(request))
def CreateSampleHunt(self,
path=None,
stopped=False,
output_plugins=None,
client_limit=0,
client_count=10,
token=None):
token = token or self.token
self.client_ids = self.SetupClients(client_count)
with hunts.GRRHunt.StartHunt(
hunt_name="GenericHunt",
flow_runner_args=flow_runner.FlowRunnerArgs(
flow_name="GetFile"),
flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec(
path=path or "/tmp/evil.txt",
pathtype=rdf_paths.PathSpec.PathType.OS,
)),
regex_rules=[
rdf_foreman.ForemanAttributeRegex(
attribute_name="GRR client", attribute_regex="GRR")
],
output_plugins=output_plugins or [],
client_rate=0,
client_limit=client_limit,
token=token) as hunt:
if not stopped:
hunt.Run()
with aff4.FACTORY.Open("aff4:/foreman", mode="rw",
token=token) as foreman:
for client_id in self.client_ids:
foreman.AssignTasksToClient(client_id)
self.hunt_urn = hunt.urn
return aff4.FACTORY.Open(hunt.urn,
mode="rw",
token=token,
age=aff4.ALL_TIMES)
class GRRHunt(flow.GRRFlow):
"""The GRR Hunt class."""
# Some common rules.
MATCH_WINDOWS = rdf_foreman.ForemanAttributeRegex(
attribute_name="System", attribute_regex="Windows")
MATCH_LINUX = rdf_foreman.ForemanAttributeRegex(attribute_name="System",
attribute_regex="Linux")
MATCH_DARWIN = rdf_foreman.ForemanAttributeRegex(attribute_name="System",
attribute_regex="Darwin")
class SchemaCls(flow.GRRFlow.SchemaCls):
"""The schema for hunts.
This object stores the persistent information for the hunt.
"""
CLIENT_COUNT = aff4.Attribute("aff4:client_count",
rdfvalue.RDFInteger,
"The total number of clients scheduled.",
versioned=False,
creates_new_object_version=False)
# This needs to be kept out the args semantic value since must be updated
# without taking a lock on the hunt object.
STATE = aff4.Attribute(
"aff4:hunt_state",
rdfvalue.RDFString, "The state of a hunt can be "
"'STARTED': running, "
"'STOPPED': stopped by the user, "
"'PAUSED': paused due to client limit, "
"'COMPLETED': hunt has met its expiry time. New hunts are created in"
" the PAUSED state.",
versioned=False,
lock_protected=False,
default="PAUSED")
args_type = None
runner_cls = HuntRunner
def Initialize(self):
super(GRRHunt, self).Initialize()
# Hunts run in multiple threads so we need to protect access.
self.lock = threading.RLock()
self.processed_responses = False
if "r" in self.mode:
self.client_count = self.Get(self.Schema.CLIENT_COUNT)
@property
def logs_collection_urn(self):
return self.urn.Add("Logs")
@property
def all_clients_collection_urn(self):
return self.urn.Add("AllClients")
@property
def completed_clients_collection_urn(self):
return self.urn.Add("CompletedClients")
@property
def clients_errors_collection_urn(self):
return self.urn.Add("ErrorClients")
@property
def clients_with_results_collection_urn(self):
return self.urn.Add("ClientsWithResults")
@property
def output_plugins_status_collection_urn(self):
return self.urn.Add("OutputPluginsStatus")
@property
def output_plugins_errors_collection_urn(self):
return self.urn.Add("OutputPluginsErrors")
@property
def creator(self):
return self.state.context.creator
def _AddURNToCollection(self, urn, collection_urn):
# TODO(user): Change to use StaticAdd once all active hunts are
# migrated.
try:
aff4.FACTORY.Open(collection_urn,
"UrnCollection",
mode="rw",
token=self.token).Add(urn)
except IOError:
aff4_collections.PackedVersionedCollection.AddToCollection(
collection_urn, [urn], sync=False, token=self.token)
def _AddHuntErrorToCollection(self, error, collection_urn):
# TODO(user) Change to use StaticAdd once all active hunts are
# migrated.
try:
aff4.FACTORY.Open(collection_urn,
"HuntErrorCollection",
mode="rw",
token=self.token).Add(error)
except IOError:
aff4_collections.PackedVersionedCollection.AddToCollection(
collection_urn, [error], sync=False, token=self.token)
def _GetCollectionItems(self, collection_urn):
collection = aff4.FACTORY.Open(collection_urn,
mode="r",
token=self.token)
return collection.GenerateItems()
def _ClientSymlinkUrn(self, client_id):
return client_id.Add("flows").Add("%s:hunt" % (self.urn.Basename()))
def RegisterClient(self, client_urn):
self._AddURNToCollection(client_urn, self.all_clients_collection_urn)
def RegisterCompletedClient(self, client_urn):
self._AddURNToCollection(client_urn,
self.completed_clients_collection_urn)
def RegisterClientWithResults(self, client_urn):
self._AddURNToCollection(client_urn,
self.clients_with_results_collection_urn)
def RegisterClientError(self, client_id, log_message=None, backtrace=None):
error = rdf_flows.HuntError(client_id=client_id, backtrace=backtrace)
if log_message:
error.log_message = utils.SmartUnicode(log_message)
self._AddHuntErrorToCollection(error,
self.clients_errors_collection_urn)
def OnDelete(self, deletion_pool=None):
super(GRRHunt, self).OnDelete(deletion_pool=deletion_pool)
# Delete all the symlinks in the clients namespace that point to the flows
# initiated by this hunt.
children_urns = deletion_pool.ListChildren(self.urn)
clients_ids = []
for urn in children_urns:
try:
clients_ids.append(rdf_client.ClientURN(urn.Basename()))
except type_info.TypeValueError:
# Ignore children that are not valid clients ids.
continue
symlinks_urns = [
self._ClientSymlinkUrn(client_id) for client_id in clients_ids
]
deletion_pool.MultiMarkForDeletion(symlinks_urns)
@flow.StateHandler()
def RunClient(self, client_id):
"""This method runs the hunt on a specific client.
Note that this method holds a lock on the hunt object and runs in the main
thread. It is safe to access any hunt parameters from here.
Args:
client_id: The new client assigned to this hunt.
"""
@classmethod
def StartHunt(cls, args=None, runner_args=None, **kwargs):
"""This class method creates new hunts."""
# Build the runner args from the keywords.
if runner_args is None:
runner_args = HuntRunnerArgs()
cls.FilterArgsFromSemanticProtobuf(runner_args, kwargs)
# Is the required flow a known flow?
if (runner_args.hunt_name not in cls.classes or not aff4.issubclass(
cls.classes[runner_args.hunt_name], GRRHunt)):
raise RuntimeError("Unable to locate hunt %s" %
runner_args.hunt_name)
# Make a new hunt object and initialize its runner.
hunt_obj = aff4.FACTORY.Create(None,
runner_args.hunt_name,
mode="w",
token=runner_args.token)
# Hunt is called using keyword args. We construct an args proto from the
# kwargs..
if hunt_obj.args_type and args is None:
args = hunt_obj.args_type()
cls.FilterArgsFromSemanticProtobuf(args, kwargs)
if hunt_obj.args_type and not isinstance(args, hunt_obj.args_type):
raise RuntimeError("Hunt args must be instance of %s" %
hunt_obj.args_type)
if kwargs:
raise type_info.UnknownArg("Unknown parameters to StartHunt: %s" %
kwargs)
# Store the hunt args in the state.
hunt_obj.state.Register("args", args)
# Hunts are always created in the paused state. The runner method Start
# should be called to start them.
hunt_obj.Set(hunt_obj.Schema.STATE("PAUSED"))
runner = hunt_obj.CreateRunner(runner_args=runner_args)
# Allow the hunt to do its own initialization.
runner.RunStateMethod("Start")
hunt_obj.Flush()
try:
flow_name = args.flow_runner_args.flow_name
except AttributeError:
flow_name = ""
event = flow.AuditEvent(user=runner_args.token.username,
action="HUNT_CREATED",
urn=hunt_obj.urn,
flow_name=flow_name,
description=runner_args.description)
flow.Events.PublishEvent("Audit", event, token=runner_args.token)
return hunt_obj
@classmethod
def StartClients(cls, hunt_id, client_ids, token=None):
"""This method is called by the foreman for each client it discovers.
Note that this function is performance sensitive since it is called by the
foreman for every client which needs to be scheduled.
Args:
hunt_id: The hunt to schedule.
client_ids: List of clients that should be added to the hunt.
token: An optional access token to use.
"""
token = token or access_control.ACLToken(username="******",
reason="hunting")
with queue_manager.QueueManager(token=token) as flow_manager:
for client_id in client_ids:
# Now we construct a special response which will be sent to the hunt
# flow. Randomize the request_id so we do not overwrite other messages
# in the queue.
state = rdf_flows.RequestState(id=utils.PRNG.GetULong(),
session_id=hunt_id,
client_id=client_id,
next_state="AddClient")
# Queue the new request.
flow_manager.QueueRequest(hunt_id, state)
# Send a response.
msg = rdf_flows.GrrMessage(
session_id=hunt_id,
request_id=state.id,
response_id=1,
auth_state=rdf_flows.GrrMessage.AuthorizationState.
AUTHENTICATED,
type=rdf_flows.GrrMessage.Type.STATUS,
payload=rdf_flows.GrrStatus())
flow_manager.QueueResponse(hunt_id, msg)
# And notify the worker about it.
flow_manager.QueueNotification(session_id=hunt_id)
def Run(self):
"""A shortcut method for starting the hunt."""
self.GetRunner().Start()
def Pause(self):
"""A shortcut method for pausing the hunt."""
self.GetRunner().Pause()
def Stop(self):
"""A shortcut method for stopping the hunt."""
self.GetRunner().Stop()
def AddResultsToCollection(self, responses, client_id):
if responses.success:
with self.lock:
self.processed_responses = True
msgs = [
rdf_flows.GrrMessage(payload=response, source=client_id)
for response in responses
]
try:
with aff4.FACTORY.Open(
self.state.context.results_collection_urn,
hunts_results.HuntResultCollection.__name__,
mode="rw",
token=self.token) as collection:
for msg in msgs:
collection.Add(msg)
except IOError:
aff4.ResultsOutputCollection.AddToCollection(
self.state.context.results_collection_urn,
msgs,
sync=True,
token=self.token)
if responses:
self.RegisterClientWithResults(client_id)
# Update stats.
stats.STATS.IncrementCounter("hunt_results_added",
delta=len(msgs))
else:
self.LogClientError(client_id,
log_message=utils.SmartStr(responses.status))
def CallFlow(self,
flow_name=None,
next_state=None,
request_data=None,
client_id=None,
**kwargs):
"""Create a new child flow from a hunt."""
base_session_id = None
if client_id:
# The flow is stored in the hunt namespace,
base_session_id = self.urn.Add(client_id.Basename())
# Actually start the new flow.
# We need to pass the logs_collection_urn here rather than in __init__ to
# wait for the hunt urn to be created.
child_urn = self.runner.CallFlow(
flow_name=flow_name,
next_state=next_state,
base_session_id=base_session_id,
client_id=client_id,
request_data=request_data,
logs_collection_urn=self.logs_collection_urn,
**kwargs)
if client_id:
# But we also create a symlink to it from the client's namespace.
hunt_link_urn = client_id.Add("flows").Add("%s:hunt" %
(self.urn.Basename()))
hunt_link = aff4.FACTORY.Create(hunt_link_urn,
"AFF4Symlink",
token=self.token)
hunt_link.Set(hunt_link.Schema.SYMLINK_TARGET(child_urn))
hunt_link.Close()
return child_urn
def Name(self):
return self.state.context.args.hunt_name
def SetDescription(self, description=None):
if description:
self.state.context.args.description = description
else:
try:
flow_name = self.state.args.flow_runner_args.flow_name
except AttributeError:
flow_name = ""
self.state.context.args.description = flow_name
@flow.StateHandler()
def Start(self):
"""Initializes this hunt from arguments."""
self.state.context.Register("results_metadata_urn",
self.urn.Add("ResultsMetadata"))
self.state.context.Register("results_collection_urn",
self.urn.Add("Results"))
self.state.context.Register("output_plugins_base_urn",
self.urn.Add("Results"))
with aff4.FACTORY.Create(self.state.context.results_metadata_urn,
"HuntResultsMetadata",
mode="rw",
token=self.token) as results_metadata:
state = rdf_flows.FlowState()
try:
plugins_descriptors = self.state.args.output_plugins
except AttributeError:
plugins_descriptors = []
for index, plugin_descriptor in enumerate(plugins_descriptors):
output_base_urn = self.state.context.output_plugins_base_urn.Add(
plugin_descriptor.plugin_name)
plugin_class = plugin_descriptor.GetPluginClass()
plugin_obj = plugin_class(
self.state.context.results_collection_urn,
output_base_urn=output_base_urn,
args=plugin_descriptor.plugin_args,
token=self.token)
state.Register(
"%s_%d" % (plugin_descriptor.plugin_name, index),
(plugin_descriptor, plugin_obj.state))
results_metadata.Set(results_metadata.Schema.OUTPUT_PLUGINS(state))
# Create the collection for results.
with aff4.FACTORY.Create(self.state.context.results_collection_urn,
"HuntResultCollection",
mode="w",
token=self.token):
pass
# Create the collection for logs.
with aff4.FACTORY.Create(self.logs_collection_urn,
flow_runner.FlowLogCollection.__name__,
mode="w",
token=self.token):
pass
# Create the collections for urns.
for urn in [
self.all_clients_collection_urn,
self.completed_clients_collection_urn,
self.clients_with_results_collection_urn
]:
with aff4.FACTORY.Create(urn,
"UrnCollection",
mode="w",
token=self.token):
pass
# Create the collection for errors.
with aff4.FACTORY.Create(self.clients_errors_collection_urn,
"HuntErrorCollection",
mode="w",
token=self.token):
pass
# Create the collections for PluginStatus messages.
for urn in [
self.output_plugins_status_collection_urn,
self.output_plugins_errors_collection_urn
]:
with aff4.FACTORY.Create(urn,
"PluginStatusCollection",
mode="w",
token=self.token):
pass
if not self.state.context.args.description:
self.SetDescription()
@flow.StateHandler()
def End(self):
"""Final state."""
def MarkClientDone(self, client_id):
"""Adds a client_id to the list of completed tasks."""
self.RegisterCompletedClient(client_id)
if self.state.context.args.notification_event:
status = hunts.HuntNotification(session_id=self.session_id,
client_id=client_id)
self.Publish(self.state.context.args.notification_event, status)
def LogClientError(self, client_id, log_message=None, backtrace=None):
"""Logs an error for a client."""
self.RegisterClientError(client_id,
log_message=log_message,
backtrace=backtrace)
def ProcessClientResourcesStats(self, client_id, status):
"""Process status message from a client and update the stats.
This method may be implemented in the subclasses. It's called
once *per every hunt's state per every client*.
Args:
client_id: Client id.
status: Status returned from the client.
"""
def GetClientsCounts(self):
collections = aff4.FACTORY.MultiOpen([
self.all_clients_collection_urn,
self.completed_clients_collection_urn,
self.clients_errors_collection_urn
],
mode="r",
token=self.token)
collections_dict = dict((coll.urn, coll) for coll in collections)
def CollectionLen(collection_urn):
if collection_urn in collections_dict:
return collections_dict[collection_urn].CalculateLength()
else:
return 0
all_clients_count = CollectionLen(self.all_clients_collection_urn)
completed_clients_count = CollectionLen(
self.completed_clients_collection_urn)
clients_errors_count = CollectionLen(
self.clients_errors_collection_urn)
return all_clients_count, completed_clients_count, clients_errors_count
def GetClientsErrors(self, client_id=None):
errors = self._GetCollectionItems(self.clients_errors_collection_urn)
if not client_id:
return errors
else:
return [error for error in errors if error.client_id == client_id]
def GetClients(self):
return set(self._GetCollectionItems(self.all_clients_collection_urn))
def GetClientsByStatus(self):
"""Get all the clients in a dict of {status: [client_list]}."""
started = set(self._GetCollectionItems(
self.all_clients_collection_urn))
completed = set(
self._GetCollectionItems(self.completed_clients_collection_urn))
outstanding = started - completed
return {
"STARTED": sorted(started),
"COMPLETED": sorted(completed),
"OUTSTANDING": sorted(outstanding)
}
def GetClientStates(self, client_list, client_chunk=50):
"""Take in a client list and return dicts with their age and hostname."""
for client_group in utils.Grouper(client_list, client_chunk):
for fd in aff4.FACTORY.MultiOpen(client_group,
mode="r",
aff4_type="VFSGRRClient",
token=self.token):
result = {}
result["age"] = fd.Get(fd.Schema.PING)
result["hostname"] = fd.Get(fd.Schema.HOSTNAME)
yield (fd.urn, result)
def GetLog(self, client_id=None):
log_vals = aff4.FACTORY.Open(self.logs_collection_urn,
mode="r",
token=self.token)
if not client_id:
return log_vals
else:
return [val for val in log_vals if val.client_id == client_id]
def Save(self):
super(GRRHunt, self).Save()
runner = self.GetRunner()
if not runner.IsCompleted():
runner.CheckExpiry()
def GetSystemForemanRule(os_string):
return rdf_foreman.ForemanAttributeRegex(attribute_name="System",
attribute_regex=os_string)
def testStopping(self):
"""Tests if we can stop a hunt."""
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
rules = foreman.Get(foreman.Schema.RULES)
# Make sure there are no rules yet.
self.assertEqual(len(rules), 0)
now = rdfvalue.RDFDatetime().Now()
expires = rdfvalue.Duration("1h").Expiry()
# Add some rules.
rules = [
rdf_foreman.ForemanRule(created=now,
expires=expires,
description="Test rule1"),
rdf_foreman.ForemanRule(created=now,
expires=expires,
description="Test rule2")
]
self.AddForemanRules(rules)
hunt = hunts.GRRHunt.StartHunt(
hunt_name="SampleHunt",
regex_rules=[
rdf_foreman.ForemanAttributeRegex(attribute_name="GRR client",
attribute_regex="HUNT")
],
integer_rules=[
rdf_foreman.ForemanAttributeInteger(
attribute_name="Clock",
operator=rdf_foreman.ForemanAttributeInteger.Operator.
GREATER_THAN,
value=1336650631137737)
],
client_rate=0,
token=self.token)
with hunt:
runner = hunt.GetRunner()
runner.Start()
# Add some more rules.
rules = [
rdf_foreman.ForemanRule(created=now,
expires=expires,
description="Test rule3"),
rdf_foreman.ForemanRule(created=now,
expires=expires,
description="Test rule4")
]
self.AddForemanRules(rules)
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
rules = foreman.Get(foreman.Schema.RULES)
self.assertEqual(len(rules), 5)
# It should be running.
self.assertTrue(runner.IsHuntStarted())
# Now we stop the hunt.
hunt.Stop()
foreman = aff4.FACTORY.Open("aff4:/foreman",
mode="rw",
token=self.token)
rules = foreman.Get(foreman.Schema.RULES)
# The rule for this hunt should be deleted but the rest should be there.
self.assertEqual(len(rules), 4)
# And the hunt should report no outstanding requests any more.
with hunt:
self.assertFalse(hunt.GetRunner().IsHuntStarted())
