def testListCrashes(self):
hunt_urn = self.StartHunt()
client_ids = self.SetupClients(2)
client_mocks = dict([
(client_id, flow_test_lib.CrashClientMock(client_id, self.token))
for client_id in client_ids
])
self.AssignTasksToClients(client_ids)
hunt_test_lib.TestHuntHelperWithMultipleMocks(client_mocks, False,
self.token)
crashes = list(self.api.Hunt(hunt_urn.Basename()).ListCrashes())
self.assertLen(crashes, 2)
self.assertEqual(set(x.client.client_id for x in crashes),
set(x.Basename() for x in client_ids))
for c in crashes:
self.assertEqual(c.crash_message,
"Client killed during transaction")
def Run(self):
if data_store.RelationalDBReadEnabled():
client = self.SetupTestClientObject(0)
client_id = client.client_id
client_ids = [rdf_client.ClientURN(client_id)]
else:
client_ids = self.SetupClients(1)
client_id = client_ids[0].Basename()
client_mock = flow_test_lib.CrashClientMock(
rdf_client.ClientURN(client_id), self.token)
with test_lib.FakeTime(42):
with self.CreateHunt(description="the hunt") as hunt_obj:
hunt_obj.Run()
with test_lib.FakeTime(45):
self.AssignTasksToClients(client_ids)
hunt_test_lib.TestHuntHelperWithMultipleMocks(
{client_id: client_mock}, False, self.token)
crashes = aff4_grr.VFSGRRClient.CrashCollectionForCID(
rdf_client.ClientURN(client_id))
crash = list(crashes)[0]
session_id = crash.session_id.Basename()
replace = {
hunt_obj.urn.Basename(): "H:123456",
session_id: "H:11223344"
}
self.Check(
"ListClientCrashes",
args=client_plugin.ApiListClientCrashesArgs(client_id=client_id),
replace=replace)
self.Check("ListClientCrashes",
args=client_plugin.ApiListClientCrashesArgs(
client_id=client_id, count=1),
replace=replace)
self.Check("ListClientCrashes",
args=client_plugin.ApiListClientCrashesArgs(
client_id=client_id, offset=1, count=1),
replace=replace)
def Run(self):
if data_store.RelationalDBReadEnabled():
client_obj = self.SetupTestClientObject(0)
client_id = client_obj.client_id
else:
client_id = self.SetupClient(0).Basename()
client_mocks = {
client_id: flow_test_lib.CrashClientMock(client_id, self.token)
}
with test_lib.FakeTime(42):
with self.CreateHunt(description="the hunt") as hunt_obj:
hunt_obj.Run()
with test_lib.FakeTime(45):
self.AssignTasksToClients([client_id])
hunt_test_lib.TestHuntHelperWithMultipleMocks(
client_mocks, False, self.token)
crashes = implementation.GRRHunt.CrashCollectionForHID(hunt_obj.urn)
crash = list(crashes)[0]
session_id = crash.session_id.Basename()
replace = {
hunt_obj.urn.Basename(): "H:123456",
session_id: "H:11223344"
}
self.Check("ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(
hunt_id=hunt_obj.urn.Basename()),
replace=replace)
self.Check("ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(
hunt_id=hunt_obj.urn.Basename(), count=1),
replace=replace)
self.Check("ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(
hunt_id=hunt_obj.urn.Basename(), offset=1, count=1),
replace=replace)
def Run(self):
client_id = self.SetupClient(0)
client_mocks = {client_id: flow_test_lib.CrashClientMock(client_id)}
hunt_id = self.CreateHunt(
description="the hunt", creator=self.test_username)
hunt.StartHunt(hunt_id)
with test_lib.FakeTime(45):
self.AssignTasksToClients([client_id])
hunt_test_lib.TestHuntHelperWithMultipleMocks(client_mocks)
crash = data_store.REL_DB.ReadHuntFlows(
hunt_id,
0,
1,
filter_condition=db.HuntFlowsCondition.CRASHED_FLOWS_ONLY,
)[0].client_crash_info
replace = {
hunt_id:
"H:123456",
str(crash.session_id):
"aff4:/hunts/H:123456/C.1000000000000000/H:11223344"
}
self.Check(
"ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(hunt_id=hunt_id),
replace=replace)
self.Check(
"ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(hunt_id=hunt_id, count=1),
replace=replace)
self.Check(
"ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(
hunt_id=hunt_id, offset=1, count=1),
replace=replace)
def testHuntIsStoppedIfCrashNumberOverThreshold(self):
client_ids = self.SetupClients(4)
hunt_id = self._CreateHunt(
client_rule_set=foreman_rules.ForemanClientRuleSet(),
client_rate=0,
crash_limit=3,
args=self.GetFileHuntArgs())
client_mock = flow_test_lib.CrashClientMock()
self._RunHunt(client_ids[:2], client_mock=client_mock)
hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
self.assertEqual(hunt_obj.hunt_state,
rdf_hunt_objects.Hunt.HuntState.STARTED)
self._RunHunt(client_ids[2:], client_mock=client_mock)
hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
self.assertEqual(hunt_obj.hunt_state,
rdf_hunt_objects.Hunt.HuntState.STOPPED)
self._CheckHuntStoppedNotification("reached the crashes limit")
def testHuntCreatorIsNotifiedWhenHuntIsStoppedDueToCrashes(self):
with self.CreateHunt(crash_limit=3, token=self.token) as hunt:
hunt.Run()
# Run the hunt on 3 clients, one by one. Crash detection check happens
# when client is scheduled, so it's important to schedule the clients
# one by one in the test.
for client_id in self.SetupClients(3):
self.AssignTasksToClients([client_id])
client_mock = flow_test_lib.CrashClientMock(client_id,
token=self.token)
hunt_test_lib.TestHuntHelper(client_mock, [client_id],
check_flow_errors=False,
token=self.token)
self.Open("/")
# Wait until the notification is there and show the notifications list.
self.WaitUntilEqual("1", self.GetText,
"css=button[id=notification_button]")
self.Click("css=button[id=notification_button]")
# Click on the "hunt [id] reached the crashes limit" notificaiton.
self.Click("css=td:contains(Hunt %s reached the crashes limit)" %
hunt.urn.Basename())
# Clicking on notification should shown the hunt's overview page.
self.WaitUntil(self.IsTextPresent, "/tmp/evil.txt")
# Go to the logs and check that a reason for hunt's stopping is the
# hunts logs.
# Click the Log Tab.
self.Click("css=li[heading=Log]")
self.WaitUntil(
self.IsTextPresent,
"Hunt %s reached the crashes limit of 3 and was stopped." %
hunt.urn.Basename())
def Run(self):
client_ids = self.SetupClients(1)
client_mocks = dict([
(client_id, flow_test_lib.CrashClientMock(client_id, self.token))
for client_id in client_ids
])
with test_lib.FakeTime(42):
with self.CreateHunt(description="the hunt") as hunt_obj:
hunt_obj.Run()
with test_lib.FakeTime(45):
self.AssignTasksToClients(client_ids)
hunt_test_lib.TestHuntHelperWithMultipleMocks(
client_mocks, False, self.token)
crashes = implementation.GRRHunt.CrashCollectionForHID(
hunt_obj.urn, token=self.token)
crash = list(crashes)[0]
session_id = crash.session_id.Basename()
replace = {
hunt_obj.urn.Basename(): "H:123456",
session_id: "H:11223344"
}
self.Check("ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(
hunt_id=hunt_obj.urn.Basename()),
replace=replace)
self.Check("ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(
hunt_id=hunt_obj.urn.Basename(), count=1),
replace=replace)
self.Check("ListHuntCrashes",
args=hunt_plugin.ApiListHuntCrashesArgs(
hunt_id=hunt_obj.urn.Basename(), offset=1, count=1),
replace=replace)
def SetUpCrashedFlowInHunt(self):
client_ids = self.SetupClients(10)
client_mocks = dict([(client_id,
flow_test_lib.CrashClientMock(client_id, self.token))
for client_id in client_ids])
client_rule_set = self._CreateForemanClientRuleSet()
# Make this not match anything.
client_rule_set.rules[0].regex.attribute_regex = ""
with implementation.GRRHunt.StartHunt(
hunt_name=standard.SampleHunt.__name__,
client_rule_set=client_rule_set,
client_rate=0,
token=self.token) as hunt:
hunt.Run()
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
for client_id in client_ids:
self.assertTrue(foreman.AssignTasksToClient(client_id.Basename()))
hunt_test_lib.TestHuntHelperWithMultipleMocks(client_mocks, False,
self.token)
return client_ids
def testAlertEmailIsSentWhenClientKilled(self):
"""Test that client killed messages are handled correctly."""
client_id = self.SetupClient(0)
self.SetupTestClientObject(0)
self.email_messages = []
def SendEmail(address, sender, title, message, **_):
self.email_messages.append(
dict(address=address,
sender=sender,
title=title,
message=message))
with utils.Stubber(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmail):
client = flow_test_lib.CrashClientMock(client_id, self.token)
flow_test_lib.TestFlowHelper(
flow_test_lib.FlowWithOneClientRequest.__name__,
client,
client_id=client_id,
token=self.token,
check_flow_errors=False)
self.assertEqual(len(self.email_messages), 1)
email_message = self.email_messages[0]
# We expect the email to be sent.
self.assertEqual(email_message.get("address", ""),
config.CONFIG["Monitoring.alert_email"])
self.assertTrue(str(client_id) in email_message["title"])
# Make sure the flow state is included in the email message.
for s in [
"flow_name", flow_test_lib.FlowWithOneClientRequest.__name__,
"current_state"
]:
self.assertTrue(s in email_message["message"])
flow_obj = aff4.FACTORY.Open(client.flow_id,
age=aff4.ALL_TIMES,
token=self.token)
self.assertEqual(flow_obj.context.state,
rdf_flow_runner.FlowContext.State.ERROR)
# Make sure client object is updated with the last crash.
# AFF4.
client_obj = aff4.FACTORY.Open(client_id, token=self.token)
crash = client_obj.Get(client_obj.Schema.LAST_CRASH)
self.CheckCrash(crash, flow_obj.session_id, client_id)
# Relational db.
crash = data_store.REL_DB.ReadClientCrashInfo(client_id.Basename())
self.CheckCrash(crash, flow_obj.session_id, client_id)
# Make sure crashes collections are created and written
# into proper locations. First check the per-client crashes collection.
client_crashes = sorted(list(
aff4_grr.VFSGRRClient.CrashCollectionForCID(client_id)),
key=lambda x: x.timestamp)
self.assertTrue(len(client_crashes) >= 1)
crash = list(client_crashes)[0]
self.CheckCrash(crash, flow_obj.session_id, client_id)
# Check per-flow crash collection. Check that crash written there is
# equal to per-client crash.
flow_crashes = sorted(list(
flow_obj.GetValuesForAttribute(flow_obj.Schema.CLIENT_CRASH)),
key=lambda x: x.timestamp)
self.assertEqual(len(flow_crashes), len(client_crashes))
for a, b in zip(flow_crashes, client_crashes):
self.assertEqual(a, b)
def CrashClient(client_id):
self.AssignTasksToClients([client_id])
client_mock = flow_test_lib.CrashClientMock(client_id, token=self.token)
hunt_test_lib.TestHuntHelper(
client_mock, [client_id], check_flow_errors=False, token=self.token)
