def testClientMetadataCrash(self):
d = self.db
client_id_1 = "C.fc413187fefa1dcf"
self._InitializeClient(client_id_1)
# Typical update on client crash.
d.WriteClientMetadata(
client_id_1,
last_crash=rdf_client.ClientCrash(crash_message="Et tu, Brute?"))
res = d.ReadClientMetadatas([client_id_1])
self.assertEqual(len(res), 1)
m1 = res[client_id_1]
self.assertEqual(m1.last_crash,
rdf_client.ClientCrash(crash_message="Et tu, Brute?"))
def testCrashHistory(self):
d = self.db
client_id = self.InitializeClient()
ci = rdf_client.ClientCrash(timestamp=12345, crash_message="Crash #1")
d.WriteClientCrashInfo(client_id, ci)
ci.crash_message = "Crash #2"
d.WriteClientCrashInfo(client_id, ci)
ci.crash_message = "Crash #3"
d.WriteClientCrashInfo(client_id, ci)
last_is = d.ReadClientCrashInfo(client_id)
self.assertIsInstance(last_is, rdf_client.ClientCrash)
self.assertEqual(last_is.crash_message, "Crash #3")
self.assertIsInstance(last_is.timestamp, rdfvalue.RDFDatetime)
hist = d.ReadClientCrashInfoHistory(client_id)
self.assertEqual(len(hist), 3)
self.assertEqual([ci.crash_message for ci in hist],
["Crash #3", "Crash #2", "Crash #1"])
self.assertIsInstance(hist[0].timestamp, rdfvalue.RDFDatetime)
self.assertGreater(hist[0].timestamp, hist[1].timestamp)
self.assertGreater(hist[1].timestamp, hist[2].timestamp)
md = self.db.ReadClientMetadata(client_id)
self.assertEqual(md.last_crash_timestamp, hist[0].timestamp)
self.assertIsNone(d.ReadClientCrashInfo("C.0000000000000000"))
self.assertEqual(d.ReadClientCrashInfoHistory("C.0000000000000000"),
[])
def testClientMetadataCrash(self):
d = self.CreateDatabase()
client_id_1 = "C.fc413187fefa1dcf"
# Typical initial FS enabled write
d.WriteClientMetadata(client_id_1, fleetspeak_enabled=True)
# Typical update on client crash.
d.WriteClientMetadata(
client_id_1,
last_crash=rdf_client.ClientCrash(crash_message="Et tu, Brute?"))
res = d.ReadClientMetadatas([client_id_1])
self.assertEqual(len(res), 1)
m1 = res[client_id_1]
self.assertEqual(m1.last_crash,
rdf_client.ClientCrash(crash_message="Et tu, Brute?"))
def testCrashHistory(self):
d = self.db
client_id = "C.0000000050000001"
ci = rdf_client.ClientCrash(timestamp=12345, crash_message="Crash #1")
with self.assertRaises(db.UnknownClientError):
d.WriteClientCrashInfo(client_id, ci)
self._InitializeClient(client_id)
d.WriteClientCrashInfo(client_id, ci)
ci.crash_message = "Crash #2"
d.WriteClientCrashInfo(client_id, ci)
ci.crash_message = "Crash #3"
d.WriteClientCrashInfo(client_id, ci)
last_is = d.ReadClientCrashInfo(client_id)
self.assertIsInstance(last_is, rdf_client.ClientCrash)
self.assertEqual(last_is.crash_message, "Crash #3")
self.assertIsInstance(last_is.timestamp, rdfvalue.RDFDatetime)
hist = d.ReadClientCrashInfoHistory(client_id)
self.assertEqual(len(hist), 3)
self.assertEqual([ci.crash_message for ci in hist],
["Crash #3", "Crash #2", "Crash #1"])
self.assertIsInstance(hist[0].timestamp, rdfvalue.RDFDatetime)
self.assertGreater(hist[0].timestamp, hist[1].timestamp)
self.assertGreater(hist[1].timestamp, hist[2].timestamp)
self.assertIsNone(d.ReadClientCrashInfo("C.0000000000000000"))
self.assertEqual(d.ReadClientCrashInfoHistory("C.0000000000000000"),
[])
def HandleMessage(self, message):
"""Handle client messages."""
crash_details = rdf_client.ClientCrash(
client_id=self.client_id,
session_id=message.session_id,
crash_message="Client killed during transaction",
timestamp=rdfvalue.RDFDatetime.Now())
self.flow_id = message.session_id
# This is normally done by the FrontEnd when a CLIENT_KILLED message is
# received.
events.Events.PublishEvent("ClientCrash", crash_details, token=self.token)
def ProcessMessage(self, message=None, event=None):
"""Processes this event."""
_ = event
client_id = message.source
message = message.payload.string
logging.info(self.logline, client_id, message)
# Write crash data.
if data_store.RelationalDBReadEnabled():
client = data_store.REL_DB.ReadClientSnapshot(client_id)
client_info = client.startup_info.client_info
else:
client = aff4.FACTORY.Open(client_id, token=self.token)
client_info = client.Get(client.Schema.CLIENT_INFO)
crash_details = rdf_client.ClientCrash(
client_id=client_id,
client_info=client_info,
crash_message=message,
timestamp=long(time.time() * 1e6),
crash_type=self.well_known_session_id)
self.WriteAllCrashDetails(client_id, crash_details)
# Also send email.
if config.CONFIG["Monitoring.alert_email"]:
client = aff4.FACTORY.Open(client_id, token=self.token)
hostname = client.Get(client.Schema.HOSTNAME)
url = "/clients/%s" % client_id.Basename()
body = self.__class__.mail_template.render(
client_id=client_id,
admin_ui=config.CONFIG["AdminUI.url"],
hostname=utils.SmartUnicode(hostname),
signature=config.CONFIG["Email.signature"],
url=url,
message=utils.SmartUnicode(message))
email_alerts.EMAIL_ALERTER.SendEmail(
config.CONFIG["Monitoring.alert_email"],
"GRR server",
self.subject % client_id,
utils.SmartStr(body),
is_html=True)
def HandleMessage(self, message):
"""Handle client messages."""
crash_details = rdf_client.ClientCrash(
client_id=self.client_id,
session_id=message.session_id,
crash_message="Client killed during transaction",
timestamp=rdfvalue.RDFDatetime.Now())
msg = rdf_flows.GrrMessage(
payload=crash_details,
source=self.client_id,
auth_state=rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED)
self.flow_id = message.session_id
# This is normally done by the FrontEnd when a CLIENT_KILLED message is
# received.
events.Events.PublishEvent("ClientCrash", msg, token=self.token)
def ProcessMessage(self, message=None, event=None):
"""Processes this event."""
_ = event
client_id = message.source
message = message.payload.string
logging.info(self.logline, client_id, message)
# Write crash data to AFF4.
client = aff4.FACTORY.Open(client_id, token=self.token)
client_info = client.Get(client.Schema.CLIENT_INFO)
crash_details = rdf_client.ClientCrash(
client_id=client_id,
client_info=client_info,
crash_message=message,
timestamp=long(time.time() * 1e6),
crash_type=self.well_known_session_id)
self.WriteAllCrashDetails(client_id, crash_details)
# Also send email.
if config_lib.CONFIG["Monitoring.alert_email"]:
client = aff4.FACTORY.Open(client_id, token=self.token)
hostname = client.Get(client.Schema.HOSTNAME)
url = urllib.urlencode(
(("c", client_id), ("main", "HostInformation")))
email_alerts.EMAIL_ALERTER.SendEmail(
config_lib.CONFIG["Monitoring.alert_email"],
"GRR server",
self.subject % client_id,
self.mail_template %
dict(client_id=client_id,
admin_ui=config_lib.CONFIG["AdminUI.url"],
hostname=hostname,
signature=config_lib.CONFIG["Email.signature"],
urn=url,
message=message),
is_html=True)
def ProcessMessage(self, message=None, event=None):
"""Processes this event."""
_ = event
client_id = message.source
nanny_msg = ""
flow_obj = aff4.FACTORY.Open(message.session_id, token=self.token)
# Log.
logging.info("Client crash reported, client %s.", client_id)
# Only kill the flow it is does not handle its own crashes. Some flows
# restart the client and therefore expect to get a crash notification.
if flow_obj.handles_crashes:
return
# Export.
stats.STATS.IncrementCounter("grr_client_crashes")
# Write crash data to AFF4.
client = aff4.FACTORY.Open(client_id, token=self.token)
client_info = client.Get(client.Schema.CLIENT_INFO)
status = rdf_flows.GrrStatus(message.payload)
crash_details = rdf_client.ClientCrash(
client_id=client_id,
session_id=message.session_id,
client_info=client_info,
crash_message=status.error_message,
timestamp=rdfvalue.RDFDatetime().Now(),
crash_type=self.well_known_session_id)
self.WriteAllCrashDetails(client_id,
crash_details,
flow_session_id=message.session_id)
# Also send email.
to_send = []
try:
hunt_session_id = self._ExtractHuntId(message.session_id)
if hunt_session_id and hunt_session_id != message.session_id:
hunt_obj = aff4.FACTORY.Open(hunt_session_id,
aff4_type=implementation.GRRHunt,
token=self.token)
email = hunt_obj.GetRunner().args.crash_alert_email
if email:
to_send.append(email)
except aff4.InstantiationError:
logging.error("Failed to open hunt %s.", hunt_session_id)
email = config_lib.CONFIG["Monitoring.alert_email"]
if email:
to_send.append(email)
for email_address in to_send:
if status.nanny_status:
nanny_msg = "Nanny status: %s" % status.nanny_status
client = aff4.FACTORY.Open(client_id, token=self.token)
hostname = client.Get(client.Schema.HOSTNAME)
url = urllib.urlencode(
(("c", client_id), ("main", "HostInformation")))
renderer = rendering.FindRendererForObject(flow_obj.state)
email_alerts.EMAIL_ALERTER.SendEmail(
email_address,
"GRR server",
"Client %s reported a crash." % client_id,
self.mail_template %
dict(client_id=client_id,
admin_ui=config_lib.CONFIG["AdminUI.url"],
hostname=hostname,
state=renderer.RawHTML(),
urn=url,
nanny_msg=nanny_msg,
signature=config_lib.CONFIG["Email.signature"]),
is_html=True)
if nanny_msg:
msg = "Client crashed, " + nanny_msg
else:
msg = "Client crashed."
# Now terminate the flow.
flow.GRRFlow.TerminateFlow(message.session_id,
reason=msg,
token=self.token,
force=True)
def ReceiveMessages(self, client_id, messages):
"""Receives and processes the messages from the source.
For each message we update the request object, and place the
response in that request's queue. If the request is complete, we
send a message to the worker.
Args:
client_id: The client which sent the messages.
messages: A list of GrrMessage RDFValues.
"""
now = time.time()
with queue_manager.QueueManager(token=self.token,
store=self.data_store) as manager:
for session_id, msgs in utils.GroupBy(
messages, operator.attrgetter("session_id")).iteritems():
# Remove and handle messages to WellKnownFlows
unprocessed_msgs = self.HandleWellKnownFlows(msgs)
if not unprocessed_msgs:
continue
for msg in unprocessed_msgs:
manager.QueueResponse(msg)
for msg in unprocessed_msgs:
# Messages for well known flows should notify even though they don't
# have a status.
if msg.request_id == 0:
manager.QueueNotification(session_id=msg.session_id,
priority=msg.priority)
# Those messages are all the same, one notification is enough.
break
elif msg.type == rdf_flows.GrrMessage.Type.STATUS:
# If we receive a status message from the client it means the client
# has finished processing this request. We therefore can de-queue it
# from the client queue. msg.task_id will raise if the task id is
# not set (message originated at the client, there was no request on
# the server), so we have to check .HasTaskID() first.
if msg.HasTaskID():
manager.DeQueueClientRequest(
client_id, msg.task_id)
manager.QueueNotification(session_id=msg.session_id,
priority=msg.priority,
last_status=msg.request_id)
stat = rdf_flows.GrrStatus(msg.payload)
if stat.status == rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:
# A client crashed while performing an action, fire an event.
crash_details = rdf_client.ClientCrash(
client_id=client_id,
session_id=session_id,
backtrace=stat.backtrace,
crash_message=stat.error_message,
nanny_status=stat.nanny_status,
timestamp=rdfvalue.RDFDatetime.Now())
msg = rdf_flows.GrrMessage(
source=client_id,
payload=crash_details,
auth_state=(rdf_flows.GrrMessage.
AuthorizationState.AUTHENTICATED))
events.Events.PublishEvent("ClientCrash",
msg,
token=self.token)
logging.debug("Received %s messages from %s in %s sec", len(messages),
client_id,
time.time() - now)
def ProcessMessage(self, message=None, event=None):
"""Processes this event."""
_ = event
client_id = message.source
nanny_msg = ""
flow_obj = aff4.FACTORY.Open(message.session_id, token=self.token)
# Log.
logging.info("Client crash reported, client %s.", client_id)
# Export.
stats.STATS.IncrementCounter("grr_client_crashes")
# Write crash data to AFF4.
client = aff4.FACTORY.Open(client_id, token=self.token)
client_info = client.Get(client.Schema.CLIENT_INFO)
status = rdf_flows.GrrStatus(message.payload)
crash_details = rdf_client.ClientCrash(
client_id=client_id,
session_id=message.session_id,
client_info=client_info,
crash_message=status.error_message,
timestamp=rdfvalue.RDFDatetime().Now(),
crash_type=self.well_known_session_id)
self.WriteAllCrashDetails(client_id,
crash_details,
flow_session_id=message.session_id)
# Also send email.
if config_lib.CONFIG["Monitoring.alert_email"]:
if status.nanny_status:
nanny_msg = "Nanny status: %s" % status.nanny_status
client = aff4.FACTORY.Open(client_id, token=self.token)
hostname = client.Get(client.Schema.HOSTNAME)
url = urllib.urlencode(
(("c", client_id), ("main", "HostInformation")))
renderer = rendering.FindRendererForObject(flow_obj.state)
email_alerts.EMAIL_ALERTER.SendEmail(
config_lib.CONFIG["Monitoring.alert_email"],
"GRR server",
"Client %s reported a crash." % client_id,
self.mail_template %
dict(client_id=client_id,
admin_ui=config_lib.CONFIG["AdminUI.url"],
hostname=hostname,
state=renderer.RawHTML(),
urn=url,
nanny_msg=nanny_msg,
signature=config_lib.CONFIG["Email.signature"]),
is_html=True)
if nanny_msg:
msg = "Client crashed, " + nanny_msg
else:
msg = "Client crashed."
# Now terminate the flow.
flow.GRRFlow.TerminateFlow(message.session_id,
reason=msg,
token=self.token,
force=True)