def FromDict(self, dictionary):
# First clear and then set the dictionary.
self.dat = None
for key, value in dictionary.iteritems():
self.dat.Append(k=rdfvalue.DataBlob().SetValue(key),
v=rdfvalue.DataBlob().SetValue(value))
return self
def testRdfFormatterHandlesKeyValuePair(self):
"""rdfvalue.KeyValue items need special handling to expand k and v."""
key = rdfvalue.DataBlob().SetValue("skynet")
value = rdfvalue.DataBlob().SetValue([1997])
rdf = rdfvalue.KeyValue(k=key, v=value)
template = "{k}: {v}"
hinter = hints.Hinter(template=template)
expected = "skynet: 1997"
result = hinter.Render(rdf)
self.assertEqual(expected, result)
def ProcessMessage(self, message):
"""Write the blob into the AFF4 blob storage area."""
# Check that the message is authenticated
if (message.auth_state !=
rdfvalue.GrrMessage.AuthorizationState.AUTHENTICATED):
logging.error("TransferStore request from %s is not authenticated.",
message.source)
return
read_buffer = rdfvalue.DataBlob(message.payload)
# Only store non empty buffers
if read_buffer.data:
data = read_buffer.data
if (read_buffer.compression ==
rdfvalue.DataBlob.CompressionType.ZCOMPRESSION):
cdata = data
data = zlib.decompress(cdata)
elif (read_buffer.compression ==
rdfvalue.DataBlob.CompressionType.UNCOMPRESSED):
cdata = zlib.compress(data)
else:
raise RuntimeError("Unsupported compression")
# The hash is done on the uncompressed data
digest = hashlib.sha256(data).digest()
urn = rdfvalue.RDFURN("aff4:/blobs").Add(digest.encode("hex"))
fd = aff4.FACTORY.Create(urn, "AFF4MemoryStream", mode="w",
token=self.token)
fd.OverwriteAndClose(cdata, len(data), sync=True)
logging.debug("Got blob %s (length %s)", digest.encode("hex"),
len(cdata))
def SendResponse(self, session_id, data, client_id=None, well_known=False):
if not isinstance(data, rdfvalue.RDFValue):
data = rdfvalue.DataBlob(string=data)
if well_known:
request_id, response_id = 0, 12345
else:
request_id, response_id = 1, 1
with queue_manager.QueueManager(token=self.token) as flow_manager:
flow_manager.QueueResponse(
session_id,
rdfvalue.GrrMessage(source=client_id,
session_id=session_id,
payload=data,
request_id=request_id,
response_id=response_id))
if not well_known:
# For normal flows we have to send a status as well.
flow_manager.QueueResponse(
session_id,
rdfvalue.GrrMessage(
source=client_id,
session_id=session_id,
payload=rdfvalue.GrrStatus(
status=rdfvalue.GrrStatus.ReturnedStatus.OK),
request_id=request_id,
response_id=response_id + 1,
type=rdfvalue.GrrMessage.Type.STATUS))
# Signal on the worker queue that this flow is ready.
data_store.DB.Set(worker.DEFAULT_WORKER_QUEUE,
"task:%s" % session_id,
"X",
token=self.token)
def SendResponse(self, session_id, data, client_id=None, well_known=False):
if not isinstance(data, rdfvalue.RDFValue):
data = rdfvalue.DataBlob(string=data)
if well_known:
request_id, response_id = 0, 12345
else:
request_id, response_id = 1, 1
with queue_manager.QueueManager(token=self.token) as flow_manager:
flow_manager.QueueResponse(
session_id,
rdfvalue.GrrMessage(source=client_id,
session_id=session_id,
payload=data,
request_id=request_id,
response_id=response_id))
if not well_known:
# For normal flows we have to send a status as well.
flow_manager.QueueResponse(
session_id,
rdfvalue.GrrMessage(
source=client_id,
session_id=session_id,
payload=rdfvalue.GrrStatus(
status=rdfvalue.GrrStatus.ReturnedStatus.OK),
request_id=request_id,
response_id=response_id + 1,
type=rdfvalue.GrrMessage.Type.STATUS))
flow_manager.QueueNotification(session_id=session_id)
timestamp = flow_manager.frozen_timestamp
return timestamp
def testNannyMessage(self):
nanny_message = "Oh no!"
self.email_message = {}
def SendEmail(address, sender, title, message, **_):
self.email_message.update(
dict(address=address,
sender=sender,
title=title,
message=message))
with utils.Stubber(email_alerts, "SendEmail", SendEmail):
msg = rdfvalue.GrrMessage(
session_id=rdfvalue.SessionID(flow_name="NannyMessage"),
payload=rdfvalue.DataBlob(string=nanny_message),
source=self.client_id,
auth_state=rdfvalue.GrrMessage.AuthorizationState.AUTHENTICATED
)
# This is normally done by the FrontEnd when a CLIENT_KILLED message is
# received.
flow.Events.PublishEvent("NannyMessage", msg, token=self.token)
# Now emulate a worker to process the event.
worker = test_lib.MockWorker(token=self.token)
while worker.Next():
pass
worker.pool.Join()
# We expect the email to be sent.
self.assertEqual(self.email_message.get("address"),
config_lib.CONFIG["Monitoring.alert_email"])
self.assertTrue(str(self.client_id) in self.email_message["title"])
# Make sure the message is included in the email message.
self.assertTrue(nanny_message in self.email_message["message"])
# Make sure crashes RDFValueCollections are created and written
# into proper locations. First check the per-client crashes collection.
client_crashes = list(
aff4.FACTORY.Open(self.client_id.Add("crashes"),
aff4_type="PackedVersionedCollection",
token=self.token))
self.assertEqual(len(client_crashes), 1)
crash = client_crashes[0]
self.assertEqual(crash.client_id, self.client_id)
self.assertEqual(crash.client_info.client_name, "GRR Monitor")
self.assertEqual(
crash.crash_type,
"aff4:/flows/" + queues.FLOWS.Basename() + ":NannyMessage")
self.assertEqual(crash.crash_message, nanny_message)
# Check global crash collection. Check that crash written there is
# equal to per-client crash.
global_crashes = list(
aff4.FACTORY.Open(aff4.ROOT_URN.Add("crashes"),
aff4_type="PackedVersionedCollection",
token=self.token))
self.assertEqual(len(global_crashes), 1)
self.assertEqual(global_crashes[0], crash)
def Run(self, args):
"""Reads a buffer on the client and sends it to the server."""
# Make sure we limit the size of our output
if args.length > MAX_BUFFER_SIZE:
raise RuntimeError("Can not read buffers this large.")
data = vfs.ReadVFS(args.pathspec,
args.offset,
args.length,
progress_callback=self.Progress)
result = rdfvalue.DataBlob(
data=zlib.compress(data),
compression=rdfvalue.DataBlob.CompressionType.ZCOMPRESSION)
digest = hashlib.sha256(data).digest()
# Ensure that the buffer is counted against this response. Check network
# send limit.
self.ChargeBytesToSession(len(data))
# Now return the data to the server into the special TransferStore well
# known flow.
self.grr_worker.SendReply(
result, session_id=rdfvalue.SessionID(flow_name="TransferStore"))
# Now report the hash of this blob to our flow as well as the offset and
# length.
self.SendReply(offset=args.offset, length=len(data), data=digest)
def testStatEntryToExportedRegistryKeyConverter(self):
stat = rdfvalue.StatEntry(
aff4path=rdfvalue.RDFURN(
"aff4:/C.0000000000000000/registry/HKEY_USERS/S-1-5-20/Software/"
"Microsoft/Windows/CurrentVersion/Run/Sidebar"),
st_mode=32768,
st_size=51,
st_mtime=1247546054,
registry_type=rdfvalue.StatEntry.RegistryType.REG_EXPAND_SZ,
pathspec=rdfvalue.PathSpec(
path="/HKEY_USERS/S-1-5-20/Software/Microsoft/Windows/"
"CurrentVersion/Run/Sidebar",
pathtype=rdfvalue.PathSpec.PathType.REGISTRY),
registry_data=rdfvalue.DataBlob(string="Sidebar.exe"))
converter = export.StatEntryToExportedRegistryKeyConverter()
results = list(converter.Convert(rdfvalue.ExportedMetadata(), stat,
token=self.token))
self.assertEqual(len(results), 1)
self.assertEqual(results[0].urn, rdfvalue.RDFURN(
"aff4:/C.0000000000000000/registry/HKEY_USERS/S-1-5-20/Software/"
"Microsoft/Windows/CurrentVersion/Run/Sidebar"))
self.assertEqual(results[0].last_modified,
rdfvalue.RDFDatetimeSeconds(1247546054))
self.assertEqual(results[0].type,
rdfvalue.StatEntry.RegistryType.REG_EXPAND_SZ)
self.assertEqual(results[0].data, "Sidebar.exe")
def SendNannyMessage(self):
msg = self.nanny_controller.GetNannyMessage()
if msg:
self.SendReply(rdfvalue.DataBlob(string=msg),
session_id="W:NannyMessage",
priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY,
require_fastpoll=False)
self.nanny_controller.ClearNannyMessage()
def _MakeRegStat(self, path, value, registry_type):
options = rdfvalue.PathSpec.Options.CASE_LITERAL
pathspec = rdfvalue.PathSpec(
path=path,
path_options=options,
pathtype=rdfvalue.PathSpec.PathType.REGISTRY)
if registry_type == rdfvalue.StatEntry.RegistryType.REG_MULTI_SZ:
reg_data = rdfvalue.DataBlob(list=rdfvalue.BlobArray(
content=rdfvalue.DataBlob(string=value)))
else:
reg_data = rdfvalue.DataBlob().SetValue(value)
return rdfvalue.StatEntry(
aff4path=self.client_id.Add("registry").Add(path),
pathspec=pathspec,
registry_data=reg_data,
registry_type=registry_type)
def UnlockSubject(self, subject, transid, token):
"""Unlocks subject using transaction id."""
request = rdfvalue.DataStoreRequest(subject=[subject])
if token:
request.token = token
blob = rdfvalue.DataBlob(string=transid)
value = rdfvalue.DataStoreValue(value=blob)
request.values.Append(value)
# We do not care about the server response.
typ = rdfvalue.DataStoreCommand.Command.UNLOCK_SUBJECT
self._MakeSyncRequest(request, typ)
return transid
def _ScheduleResponseAndStatus(self, client_id, flow_id):
with queue_manager.QueueManager(token=self.token) as flow_manager:
# Schedule a response.
flow_manager.QueueResponse(flow_id, rdfvalue.GrrMessage(
source=client_id,
session_id=flow_id,
payload=rdfvalue.DataBlob(string="Helllo"),
request_id=1,
response_id=1))
# And a STATUS message.
flow_manager.QueueResponse(flow_id, rdfvalue.GrrMessage(
source=client_id,
session_id=flow_id,
payload=rdfvalue.GrrStatus(
status=rdfvalue.GrrStatus.ReturnedStatus.OK),
request_id=1, response_id=2,
type=rdfvalue.GrrMessage.Type.STATUS))
def ProcessFileStats(self, responses):
"""Extract DataBlob from Stat response."""
if not responses.success:
return
system_root_paths = ["Windows", "WinNT", "WINNT35", "WTSRV", "WINDOWS"]
for response in responses:
if response.pathspec.path[4:] in system_root_paths:
systemdrive = response.pathspec.path[1:3]
systemroot = "%s\\%s" % (systemdrive,
response.pathspec.path[4:])
# Put the data back into the original format expected for the artifact
data = rdfvalue.DataBlob().SetValue(systemroot)
self.SendReply(rdfvalue.StatEntry(registry_data=data))
self.state.success = True
break
def testParse(self):
parser = windows_persistence.WindowsPersistenceMechanismsParser()
path = (r"HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion"
r"\Run\test")
pathspec = rdfvalue.PathSpec(
path=path, pathtype=rdfvalue.PathSpec.PathType.REGISTRY)
reg_data = "C:\\blah\\some.exe /v"
reg_type = rdfvalue.StatEntry.RegistryType.REG_SZ
stat = rdfvalue.StatEntry(
aff4path="aff4:/asdfasdf/",
pathspec=pathspec,
registry_type=reg_type,
registry_data=rdfvalue.DataBlob(string=reg_data))
persistence = [stat]
image_paths = [
"system32\\drivers\\ACPI.sys",
"%systemroot%\\system32\\svchost.exe -k netsvcs",
"\\SystemRoot\\system32\\drivers\\acpipmi.sys"
]
reg_key = rdfvalue.RDFURN("aff4:/C.1000000000000000/registry"
"/HKEY_LOCAL_MACHINE/SYSTEM/ControlSet001"
"/services/AcpiPmi")
for path in image_paths:
serv_info = rdfvalue.WindowsServiceInformation(
name="blah",
display_name="GRRservice",
image_path=path,
registry_key=reg_key)
persistence.append(serv_info)
knowledge_base = rdfvalue.KnowledgeBase()
knowledge_base.environ_systemroot = "C:\\Windows"
expected = [
"C:\\blah\\some.exe", "C:\\Windows\\system32\\drivers\\ACPI.sys",
"C:\\Windows\\system32\\svchost.exe",
"C:\\Windows\\system32\\drivers\\acpipmi.sys"
]
for index, item in enumerate(persistence):
results = list(
parser.Parse(item, knowledge_base,
rdfvalue.PathSpec.PathType.OS))
self.assertEqual(results[0].pathspec.path, expected[index])
self.assertEqual(len(results), 1)
def ProcessMessage(self, message=None, event=None):
"""Processes this event."""
_ = event
client_id = message.source
message = rdfvalue.DataBlob(message.args).string
logging.info(self.logline, client_id, message)
# Write crash data to AFF4.
client = aff4.FACTORY.Open(client_id, token=self.token)
client_info = client.Get(client.Schema.CLIENT_INFO)
crash_details = rdfvalue.ClientCrash(
client_id=client_id,
client_info=client_info,
crash_message=message,
timestamp=long(time.time() * 1e6),
crash_type=self.well_known_session_id)
self.WriteAllCrashDetails(client_id, crash_details)
# Also send email.
if config_lib.CONFIG["Monitoring.alert_email"]:
client = aff4.FACTORY.Open(client_id, token=self.token)
hostname = client.Get(client.Schema.HOSTNAME)
url = urllib.urlencode(
(("c", client_id), ("main", "HostInformation")))
email_alerts.SendEmail(
config_lib.CONFIG["Monitoring.alert_email"],
"GRR server",
self.subject % client_id,
self.mail_template %
dict(client_id=client_id,
admin_ui=config_lib.CONFIG["AdminUI.url"],
hostname=hostname,
signature=config_lib.CONFIG["Email.signature"],
urn=url,
message=message),
is_html=True)
def ExtendSubjectLock(self, subject, transid, lease_time, token):
"""Extends lock of subject."""
request = rdfvalue.DataStoreRequest(subject=[subject])
specific = rdfvalue.TimestampSpec.Type.SPECIFIC_TIME
request.timestamp = rdfvalue.TimestampSpec(start=lease_time, type=specific)
if token:
request.token = token
blob = rdfvalue.DataBlob(string=transid)
value = rdfvalue.DataStoreValue(value=blob)
request.values.Append(value)
typ = rdfvalue.DataStoreCommand.Command.EXTEND_SUBJECT
response = self._MakeSyncRequest(request, typ)
if not response.results:
return None
result = response.results[0]
if not result.values:
return None
value = result.values[0].value.string
return transid if transid == value else None
def _Stat(self, name, value, value_type):
response = rdfvalue.StatEntry()
response_pathspec = self.pathspec.Copy()
# No matter how we got here, there is no need to do case folding from now on
# since this is the exact filename casing.
response_pathspec.path_options = rdfvalue.PathSpec.Options.CASE_LITERAL
response_pathspec.last.path = utils.JoinPath(
response_pathspec.last.path, name)
response.pathspec = response_pathspec
if self.IsDirectory():
response.st_mode = stat.S_IFDIR
else:
response.st_mode = stat.S_IFREG
response.st_mtime = self.last_modified
response.st_size = len(utils.SmartStr(value))
response.registry_type = self.registry_map.get(value_type, 0)
response.registry_data = rdfvalue.DataBlob().SetValue(value)
return response
def Run(self):
"""A Generator which makes a single request to the GRR server.
Callers should generate this when they wish to make a connection
to the server. It is up to the caller to sleep between calls in
order to enforce the required network and CPU utilization
policies.
Raises:
RuntimeError: Too many connection errors have been encountered.
Yields:
A Status() object indicating how the last POST went.
"""
while True:
self.consecutive_connection_errors = 0
while self.active_server_url is None:
if self.EstablishConnection():
# Everything went as expected - we don't need to return to
# the main loop (which would mean sleeping for a poll_time).
break
else:
# If we can't reconnect to the server for a long time, we restart
# to reset our state. In some very rare cases, the urrlib can get
# confused and we need to reset it before we can start talking to
# the server again.
self.consecutive_connection_errors += 1
limit = config_lib.CONFIG["Client.connection_error_limit"]
if self.consecutive_connection_errors > limit:
raise RuntimeError(
"Too many connection errors, exiting.")
# Constantly retrying will not work, we back off a bit.
time.sleep(60)
yield Status()
# Check if there is a message from the nanny to be sent.
self.client_worker.SendNannyMessage()
now = time.time()
# Check with the foreman if we need to
if (now > self.last_foreman_check +
config_lib.CONFIG["Client.foreman_check_frequency"]):
# We must not queue messages from the comms thread with blocking=True
# or we might deadlock. If the output queue is full, we can't accept
# more work from the foreman anyways so it's ok to drop the message.
try:
self.client_worker.SendReply(
rdfvalue.DataBlob(),
session_id=rdfvalue.FlowSessionID(flow_name="Foreman"),
priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY,
require_fastpoll=False,
blocking=False)
self.last_foreman_check = now
except Queue.Full:
pass
status = self.RunOnce()
# We suicide if our memory is exceeded, and there is no more work to do
# right now. Our death should not result in loss of messages since we are
# not holding any requests in our input queues.
if (self.client_worker.MemoryExceeded()
and not self.client_worker.IsActive()
and self.client_worker.InQueueSize() == 0
and self.client_worker.OutQueueSize() == 0):
logging.warning("Memory exceeded - exiting.")
self.client_worker.SendClientAlert(
"Memory limit exceeded, exiting.")
# Make sure this will return True so we don't get more work.
# pylint: disable=g-bad-name
self.client_worker.MemoryExceeded = lambda: True
# pylint: enable=g-bad-name
# Now send back the client message.
self.RunOnce()
# And done for now.
sys.exit(-1)
self.Wait(status)
yield status
def Store(self, data): self.storage.append(self.in_rdfvalue(data).string) return [rdfvalue.DataBlob(string="Hello World")]
def ReturnBlob(self, unused_args): return [rdfvalue.DataBlob(integer=100)]
def _AddTransactionId(self, response, subject, transid):
blob = rdfvalue.DataBlob(string=transid)
value = rdfvalue.DataStoreValue(value=blob)
response.results.Append(subject=subject, values=[value])
def GetInstallDate(self, _):
return [rdfvalue.DataBlob(integer=100)]
def GetInstallDate(self, _):
self.response_count += 1
return [rdfvalue.DataBlob(integer=100)]
def testNoValidStatusRaceIsResolved(self):
# This tests for the regression of a long standing race condition we saw
# where notifications would trigger the reading of another request that
# arrives later but wasn't completely written to the database yet.
# Timestamp based notification handling should eliminate this bug.
# We need a random flow object for this test.
session_id = flow.GRRFlow.StartFlow(client_id=self.client_id,
flow_name="WorkerSendingTestFlow",
token=self.token)
worker_obj = worker.GRRWorker(worker.DEFAULT_WORKER_QUEUE,
token=self.token)
manager = queue_manager.QueueManager(token=self.token)
manager.DeleteNotification(session_id)
manager.Flush()
# We have a first request that is complete (request_id 1, response_id 1).
self.SendResponse(session_id, "Response 1")
# However, we also have request #2 already coming in. The race is that
# the queue manager might write the status notification to
# session_id/state as "status:00000002" but not the status response
# itself yet under session_id/state/request:00000002
request_id = 2
response_id = 1
flow_manager = queue_manager.QueueManager(token=self.token)
flow_manager.FreezeTimestamp()
flow_manager.QueueResponse(
session_id,
rdfvalue.GrrMessage(source=self.client_id,
session_id=session_id,
payload=rdfvalue.DataBlob(string="Response 2"),
request_id=request_id,
response_id=response_id))
status = rdfvalue.GrrMessage(
source=self.client_id,
session_id=session_id,
payload=rdfvalue.GrrStatus(
status=rdfvalue.GrrStatus.ReturnedStatus.OK),
request_id=request_id,
response_id=response_id + 1,
type=rdfvalue.GrrMessage.Type.STATUS)
# Now we write half the status information.
subject = session_id.Add("state")
queue = flow_manager.to_write.setdefault(subject, {})
queue.setdefault(flow_manager.FLOW_STATUS_TEMPLATE % request_id,
[]).append((status.SerializeToString(), None))
flow_manager.Flush()
# We make the race even a bit harder by saying the new notification gets
# written right before the old one gets deleted. If we are not careful here,
# we delete the new notification as well and the flow becomes stuck.
def WriteNotification(self, arg_session_id, start=None, end=None):
if arg_session_id == session_id:
flow_manager.QueueNotification(session_id=arg_session_id)
flow_manager.Flush()
self.DeleteNotification.old_target(self,
arg_session_id,
start=start,
end=end)
with utils.Stubber(queue_manager.QueueManager, "DeleteNotification",
WriteNotification):
# This should process request 1 but not touch request 2.
worker_obj.RunOnce()
worker_obj.thread_pool.Join()
flow_obj = aff4.FACTORY.Open(session_id, token=self.token)
self.assertFalse(flow_obj.state.context.backtrace)
self.assertNotEqual(flow_obj.state.context.state,
rdfvalue.Flow.State.ERROR)
request2_data = data_store.DB.ResolveRegex(session_id.Add("state"),
".*:00000002",
token=self.token)
# Make sure the status field and the original request are still there.
self.assertEqual(len(request2_data), 2)
request1_data = data_store.DB.ResolveRegex(session_id.Add("state"),
".*:00000001",
token=self.token)
# Everything from request 1 should have been deleted.
self.assertEqual(len(request1_data), 0)
# The notification for request 2 should have survived.
with queue_manager.QueueManager(token=self.token) as manager:
notifications = manager.GetNotifications(
worker.DEFAULT_WORKER_QUEUE)
self.assertEqual(len(notifications), 1)
notification = notifications[0]
self.assertEqual(notification.session_id, session_id)
self.assertEqual(notification.timestamp,
flow_manager.frozen_timestamp)
self.assertEqual(RESULTS, ["Response 1"])
# The last missing piece of request 2 is the actual status message.
flow_manager.QueueResponse(session_id, status)
flow_manager.Flush()
# Now make sure request 2 runs as expected.
worker_obj.RunOnce()
worker_obj.thread_pool.Join()
self.assertEqual(RESULTS, ["Response 1", "Response 2"])
def Start(self):
for i in range(10):
self.CallClient("Test",
rdfvalue.DataBlob(string="test%s" % i),
data=str(i),
next_state="Incoming")
def SendForemanRequest(self):
self.client_worker.SendReply(
rdfvalue.DataBlob(),
session_id=rdfvalue.FlowSessionID(flow_name="Foreman"),
priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY,
require_fastpoll=False)
def SendClientAlert(self, msg):
self.SendReply(
rdfvalue.DataBlob(string=msg),
session_id=rdfvalue.FlowSessionID(flow_name="ClientAlert"),
priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY,
require_fastpoll=False)
