def testCrashHistory(self):
d = self.db
client_id = self.InitializeClient()
ci = rdf_client.ClientCrash(timestamp=12345, crash_message="Crash #1")
d.WriteClientCrashInfo(client_id, ci)
ci.crash_message = "Crash #2"
d.WriteClientCrashInfo(client_id, ci)
ci.crash_message = "Crash #3"
d.WriteClientCrashInfo(client_id, ci)
last_is = d.ReadClientCrashInfo(client_id)
self.assertIsInstance(last_is, rdf_client.ClientCrash)
self.assertEqual(last_is.crash_message, "Crash #3")
self.assertIsInstance(last_is.timestamp, rdfvalue.RDFDatetime)
hist = d.ReadClientCrashInfoHistory(client_id)
self.assertEqual(len(hist), 3)
self.assertEqual([ci.crash_message for ci in hist],
["Crash #3", "Crash #2", "Crash #1"])
self.assertIsInstance(hist[0].timestamp, rdfvalue.RDFDatetime)
self.assertGreater(hist[0].timestamp, hist[1].timestamp)
self.assertGreater(hist[1].timestamp, hist[2].timestamp)
md = self.db.ReadClientMetadata(client_id)
self.assertEqual(md.last_crash_timestamp, hist[0].timestamp)
self.assertIsNone(d.ReadClientCrashInfo("C.0000000000000000"))
self.assertEqual(d.ReadClientCrashInfoHistory("C.0000000000000000"),
[])
def testCrashInfoUpdate(self):
client_id, flow_id = self._SetupClientAndFlow()
crash_info = rdf_client.ClientCrash(crash_message="oh no")
self.db.UpdateFlow(client_id, flow_id, client_crash_info=crash_info)
read_flow = self.db.ReadFlowObject(client_id, flow_id)
self.assertEqual(read_flow.client_crash_info, crash_info)
def _BuildFilterConditionExpectations(self, hunt_obj):
_, running_flow_id = self._SetupHuntClientAndFlow(
flow_state=rdf_flow_objects.Flow.FlowState.RUNNING,
hunt_id=hunt_obj.hunt_id)
_, succeeded_flow_id = self._SetupHuntClientAndFlow(
flow_state=rdf_flow_objects.Flow.FlowState.FINISHED,
hunt_id=hunt_obj.hunt_id)
_, failed_flow_id = self._SetupHuntClientAndFlow(
flow_state=rdf_flow_objects.Flow.FlowState.ERROR,
hunt_id=hunt_obj.hunt_id)
_, crashed_flow_id = self._SetupHuntClientAndFlow(
client_crash_info=rdf_client.ClientCrash(),
hunt_id=hunt_obj.hunt_id)
client_id, flow_with_results_id = self._SetupHuntClientAndFlow(
hunt_id=hunt_obj.hunt_id)
sample_results = self._SampleSingleTypeHuntResults()
self._WriteHuntResults({
(client_id, flow_with_results_id):
sample_results
})
return {
db.HuntFlowsCondition.UNSET: [
running_flow_id, succeeded_flow_id, failed_flow_id,
crashed_flow_id, flow_with_results_id
],
db.HuntFlowsCondition.FAILED_FLOWS_ONLY: [failed_flow_id],
db.HuntFlowsCondition.SUCCEEDED_FLOWS_ONLY: [succeeded_flow_id],
db.HuntFlowsCondition.COMPLETED_FLOWS_ONLY:
[failed_flow_id, succeeded_flow_id],
db.HuntFlowsCondition.FLOWS_IN_PROGRESS_ONLY: [running_flow_id],
db.HuntFlowsCondition.CRASHED_FLOWS_ONLY: [crashed_flow_id],
db.HuntFlowsCondition.FLOWS_WITH_RESULTS_ONLY:
[flow_with_results_id],
}
def _SampleTwoTypeHuntResults(self,
client_id=None,
flow_id=None,
hunt_id=None,
count_per_type=5,
timestamp_start=10):
self.assertIsNotNone(client_id)
self.assertIsNotNone(flow_id)
self.assertIsNotNone(hunt_id)
return [
rdf_flow_objects.FlowResult(
client_id=client_id,
flow_id=flow_id,
hunt_id=hunt_id,
tag="tag_%d" % i,
payload=rdf_client.ClientSummary(
client_id=client_id,
system_manufacturer="manufacturer_%d" % i,
install_date=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(
timestamp_start + i))) for i in range(count_per_type)
] + [
rdf_flow_objects.FlowResult(
client_id=client_id,
flow_id=flow_id,
hunt_id=hunt_id,
tag="tag_%d" % i,
payload=rdf_client.ClientCrash(
client_id=client_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(
timestamp_start + i))) for i in range(count_per_type)
]
def HandleMessage(self, message):
"""Handle client messages."""
crash_details = rdf_client.ClientCrash(
client_id=self.client_id,
session_id=message.session_id,
crash_message="Client killed during transaction",
timestamp=rdfvalue.RDFDatetime.Now())
self.flow_id = message.session_id
# This is normally done by the FrontEnd when a CLIENT_KILLED message is
# received.
events.Events.PublishEvent("ClientCrash", crash_details, token=self.token)
def ProcessMessage(self, message=None):
"""Processes this event."""
client_id = message.source
message = message.payload.string
logging.info(self.logline, client_id, message)
# Write crash data.
if data_store.RelationalDBReadEnabled():
client = data_store.REL_DB.ReadClientSnapshot(client_id)
client_info = client.startup_info.client_info
else:
client = aff4.FACTORY.Open(client_id, token=self.token)
client_info = client.Get(client.Schema.CLIENT_INFO)
crash_details = rdf_client.ClientCrash(client_id=client_id,
client_info=client_info,
crash_message=message,
timestamp=int(time.time() *
1e6),
crash_type="Nanny Message")
WriteAllCrashDetails(client_id, crash_details, token=self.token)
# Also send email.
if config.CONFIG["Monitoring.alert_email"]:
client = aff4.FACTORY.Open(client_id, token=self.token)
hostname = client.Get(client.Schema.HOSTNAME)
url = "/clients/%s" % client_id.Basename()
body = self.__class__.mail_template.render(
client_id=client_id,
admin_ui=config.CONFIG["AdminUI.url"],
hostname=utils.SmartUnicode(hostname),
signature=config.CONFIG["Email.signature"],
url=url,
message=utils.SmartUnicode(message))
email_alerts.EMAIL_ALERTER.SendEmail(
config.CONFIG["Monitoring.alert_email"],
"GRR server",
self.subject % client_id,
utils.SmartStr(body),
is_html=True)
def SendEmail(self, client_id, message):
"""Processes this event."""
logging.info(self.logline, client_id, message)
client_info = None
hostname = None
# Write crash data.
client = data_store.REL_DB.ReadClientSnapshot(client_id)
if client is not None:
client_info = client.startup_info.client_info
hostname = client.knowledge_base.fqdn
crash_details = rdf_client.ClientCrash(client_id=client_id,
client_info=client_info,
crash_message=message,
timestamp=int(time.time() *
1e6),
crash_type="Nanny Message")
WriteAllCrashDetails(client_id, crash_details)
# Also send email.
if config.CONFIG["Monitoring.alert_email"]:
url = "/clients/%s" % client_id
body = self.__class__.mail_template.render(
client_id=client_id,
admin_ui=config.CONFIG["AdminUI.url"],
hostname=utils.SmartUnicode(hostname),
signature=config.CONFIG["Email.signature"],
url=url,
message=utils.SmartUnicode(message))
email_alerts.EMAIL_ALERTER.SendEmail(
config.CONFIG["Monitoring.alert_email"],
"GRR server",
self.subject % client_id,
body,
is_html=True)
def ReceiveMessages(self, client_id: str,
messages: Iterable[rdf_flows.GrrMessage]):
"""Receives and processes the messages.
For each message we update the request object, and place the
response in that request's queue. If the request is complete, we
send a message to the worker.
Args:
client_id: The client which sent the messages.
messages: A list of GrrMessage RDFValues.
"""
now = time.time()
unprocessed_msgs = []
worker_message_handler_requests = []
frontend_message_handler_requests = []
dropped_count = 0
msgs_by_session_id = collection.Group(messages, lambda m: m.session_id)
for session_id, msgs in msgs_by_session_id.items():
for msg in msgs:
if (msg.auth_state != msg.AuthorizationState.AUTHENTICATED
and msg.session_id != self.unauth_allowed_session_id):
dropped_count += 1
continue
session_id_str = str(session_id)
if session_id_str in message_handlers.session_id_map:
request = rdf_objects.MessageHandlerRequest(
client_id=msg.source.Basename(),
handler_name=message_handlers.
session_id_map[session_id],
request_id=msg.response_id or random.UInt32(),
request=msg.payload)
if request.handler_name in self._SHORTCUT_HANDLERS:
frontend_message_handler_requests.append(request)
else:
worker_message_handler_requests.append(request)
elif session_id_str in self.legacy_well_known_session_ids:
logging.debug(
"Dropping message for legacy well known session id %s",
session_id)
else:
unprocessed_msgs.append(msg)
if dropped_count:
logging.info("Dropped %d unauthenticated messages for %s",
dropped_count, client_id)
if unprocessed_msgs:
flow_responses = []
for message in unprocessed_msgs:
try:
flow_responses.append(
rdf_flow_objects.FlowResponseForLegacyResponse(
message))
except ValueError as e:
logging.warning(
"Failed to parse legacy FlowResponse:\n%s\n%s", e,
message)
data_store.REL_DB.WriteFlowResponses(flow_responses)
for msg in unprocessed_msgs:
if msg.type == rdf_flows.GrrMessage.Type.STATUS:
stat = rdf_flows.GrrStatus(msg.payload)
if stat.status == rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:
# A client crashed while performing an action, fire an event.
crash_details = rdf_client.ClientCrash(
client_id=client_id,
session_id=msg.session_id,
backtrace=stat.backtrace,
crash_message=stat.error_message,
nanny_status=stat.nanny_status,
timestamp=rdfvalue.RDFDatetime.Now())
events.Events.PublishEvent("ClientCrash",
crash_details,
token=self.token)
if worker_message_handler_requests:
data_store.REL_DB.WriteMessageHandlerRequests(
worker_message_handler_requests)
if frontend_message_handler_requests:
worker_lib.ProcessMessageHandlerRequests(
frontend_message_handler_requests)
logging.debug("Received %s messages from %s in %s sec", len(messages),
client_id,
time.time() - now)
def ReceiveMessages(self, client_id, messages):
"""Receives and processes the messages from the source.
For each message we update the request object, and place the
response in that request's queue. If the request is complete, we
send a message to the worker.
Args:
client_id: The client which sent the messages.
messages: A list of GrrMessage RDFValues.
"""
if data_store.RelationalDBEnabled():
return self.ReceiveMessagesRelationalFlows(client_id, messages)
now = time.time()
with queue_manager.QueueManager(token=self.token) as manager:
for session_id, msgs in iteritems(
collection.Group(messages,
operator.attrgetter("session_id"))):
# Remove and handle messages to WellKnownFlows
leftover_msgs = self.HandleWellKnownFlows(msgs)
unprocessed_msgs = []
for msg in leftover_msgs:
if (msg.auth_state == msg.AuthorizationState.AUTHENTICATED
or msg.session_id
== self.unauth_allowed_session_id):
unprocessed_msgs.append(msg)
if len(unprocessed_msgs) < len(leftover_msgs):
logging.info("Dropped %d unauthenticated messages for %s",
len(leftover_msgs) - len(unprocessed_msgs),
client_id)
if not unprocessed_msgs:
continue
for msg in unprocessed_msgs:
manager.QueueResponse(msg)
for msg in unprocessed_msgs:
# Messages for well known flows should notify even though they don't
# have a status.
if msg.request_id == 0:
manager.QueueNotification(session_id=msg.session_id)
# Those messages are all the same, one notification is enough.
break
elif msg.type == rdf_flows.GrrMessage.Type.STATUS:
# If we receive a status message from the client it means the client
# has finished processing this request. We therefore can de-queue it
# from the client queue. msg.task_id will raise if the task id is
# not set (message originated at the client, there was no request on
# the server), so we have to check .HasTaskID() first.
if msg.HasTaskID():
manager.DeQueueClientRequest(msg)
manager.QueueNotification(session_id=msg.session_id,
last_status=msg.request_id)
stat = rdf_flows.GrrStatus(msg.payload)
if stat.status == rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:
# A client crashed while performing an action, fire an event.
crash_details = rdf_client.ClientCrash(
client_id=client_id,
session_id=session_id,
backtrace=stat.backtrace,
crash_message=stat.error_message,
nanny_status=stat.nanny_status,
timestamp=rdfvalue.RDFDatetime.Now())
events.Events.PublishEvent("ClientCrash",
crash_details,
token=self.token)
logging.debug("Received %s messages from %s in %s sec", len(messages),
client_id,
time.time() - now)
def ReceiveMessagesRelationalFlows(self, client_id, messages):
"""Receives and processes messages for flows stored in the relational db.
Args:
client_id: The client which sent the messages.
messages: A list of GrrMessage RDFValues.
"""
now = time.time()
unprocessed_msgs = []
message_handler_requests = []
dropped_count = 0
for session_id, msgs in iteritems(
collection.Group(messages, operator.attrgetter("session_id"))):
# Remove and handle messages to WellKnownFlows
leftover_msgs = self.HandleWellKnownFlows(msgs)
for msg in leftover_msgs:
if (msg.auth_state != msg.AuthorizationState.AUTHENTICATED
and msg.session_id != self.unauth_allowed_session_id):
dropped_count += 1
continue
if session_id in queue_manager.session_id_map:
message_handler_requests.append(
rdf_objects.MessageHandlerRequest(
client_id=msg.source.Basename(),
handler_name=queue_manager.
session_id_map[session_id],
request_id=msg.response_id,
request=msg.payload))
elif session_id in self.legacy_well_known_session_ids:
logging.debug(
"Dropping message for legacy well known session id %s",
session_id)
else:
unprocessed_msgs.append(msg)
if dropped_count:
logging.info("Dropped %d unauthenticated messages for %s",
dropped_count, client_id)
if unprocessed_msgs:
flow_responses = []
for message in unprocessed_msgs:
flow_responses.append(
rdf_flow_objects.FlowResponseForLegacyResponse(message))
data_store.REL_DB.WriteFlowResponses(flow_responses)
for msg in unprocessed_msgs:
if msg.type == rdf_flows.GrrMessage.Type.STATUS:
stat = rdf_flows.GrrStatus(msg.payload)
if stat.status == rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:
# A client crashed while performing an action, fire an event.
crash_details = rdf_client.ClientCrash(
client_id=client_id,
session_id=msg.session_id,
backtrace=stat.backtrace,
crash_message=stat.error_message,
nanny_status=stat.nanny_status,
timestamp=rdfvalue.RDFDatetime.Now())
events.Events.PublishEvent("ClientCrash",
crash_details,
token=self.token)
if message_handler_requests:
data_store.REL_DB.WriteMessageHandlerRequests(
message_handler_requests)
logging.debug("Received %s messages from %s in %s sec", len(messages),
client_id,
time.time() - now)
