def FromDict(self, dictionary): # First clear and then set the dictionary. self.dat = None for key, value in dictionary.iteritems(): self.dat.Append(k=rdfvalue.DataBlob().SetValue(key), v=rdfvalue.DataBlob().SetValue(value)) return self
def testRdfFormatterHandlesKeyValuePair(self): """rdfvalue.KeyValue items need special handling to expand k and v.""" key = rdfvalue.DataBlob().SetValue("skynet") value = rdfvalue.DataBlob().SetValue([1997]) rdf = rdfvalue.KeyValue(k=key, v=value) template = "{k}: {v}" hinter = hints.Hinter(template=template) expected = "skynet: 1997" result = hinter.Render(rdf) self.assertEqual(expected, result)
def ProcessMessage(self, message): """Write the blob into the AFF4 blob storage area.""" # Check that the message is authenticated if (message.auth_state != rdfvalue.GrrMessage.AuthorizationState.AUTHENTICATED): logging.error("TransferStore request from %s is not authenticated.", message.source) return read_buffer = rdfvalue.DataBlob(message.payload) # Only store non empty buffers if read_buffer.data: data = read_buffer.data if (read_buffer.compression == rdfvalue.DataBlob.CompressionType.ZCOMPRESSION): cdata = data data = zlib.decompress(cdata) elif (read_buffer.compression == rdfvalue.DataBlob.CompressionType.UNCOMPRESSED): cdata = zlib.compress(data) else: raise RuntimeError("Unsupported compression") # The hash is done on the uncompressed data digest = hashlib.sha256(data).digest() urn = rdfvalue.RDFURN("aff4:/blobs").Add(digest.encode("hex")) fd = aff4.FACTORY.Create(urn, "AFF4MemoryStream", mode="w", token=self.token) fd.OverwriteAndClose(cdata, len(data), sync=True) logging.debug("Got blob %s (length %s)", digest.encode("hex"), len(cdata))
def SendResponse(self, session_id, data, client_id=None, well_known=False): if not isinstance(data, rdfvalue.RDFValue): data = rdfvalue.DataBlob(string=data) if well_known: request_id, response_id = 0, 12345 else: request_id, response_id = 1, 1 with queue_manager.QueueManager(token=self.token) as flow_manager: flow_manager.QueueResponse( session_id, rdfvalue.GrrMessage(source=client_id, session_id=session_id, payload=data, request_id=request_id, response_id=response_id)) if not well_known: # For normal flows we have to send a status as well. flow_manager.QueueResponse( session_id, rdfvalue.GrrMessage( source=client_id, session_id=session_id, payload=rdfvalue.GrrStatus( status=rdfvalue.GrrStatus.ReturnedStatus.OK), request_id=request_id, response_id=response_id + 1, type=rdfvalue.GrrMessage.Type.STATUS)) # Signal on the worker queue that this flow is ready. data_store.DB.Set(worker.DEFAULT_WORKER_QUEUE, "task:%s" % session_id, "X", token=self.token)
def SendResponse(self, session_id, data, client_id=None, well_known=False): if not isinstance(data, rdfvalue.RDFValue): data = rdfvalue.DataBlob(string=data) if well_known: request_id, response_id = 0, 12345 else: request_id, response_id = 1, 1 with queue_manager.QueueManager(token=self.token) as flow_manager: flow_manager.QueueResponse( session_id, rdfvalue.GrrMessage(source=client_id, session_id=session_id, payload=data, request_id=request_id, response_id=response_id)) if not well_known: # For normal flows we have to send a status as well. flow_manager.QueueResponse( session_id, rdfvalue.GrrMessage( source=client_id, session_id=session_id, payload=rdfvalue.GrrStatus( status=rdfvalue.GrrStatus.ReturnedStatus.OK), request_id=request_id, response_id=response_id + 1, type=rdfvalue.GrrMessage.Type.STATUS)) flow_manager.QueueNotification(session_id=session_id) timestamp = flow_manager.frozen_timestamp return timestamp
def testNannyMessage(self): nanny_message = "Oh no!" self.email_message = {} def SendEmail(address, sender, title, message, **_): self.email_message.update( dict(address=address, sender=sender, title=title, message=message)) with utils.Stubber(email_alerts, "SendEmail", SendEmail): msg = rdfvalue.GrrMessage( session_id=rdfvalue.SessionID(flow_name="NannyMessage"), payload=rdfvalue.DataBlob(string=nanny_message), source=self.client_id, auth_state=rdfvalue.GrrMessage.AuthorizationState.AUTHENTICATED ) # This is normally done by the FrontEnd when a CLIENT_KILLED message is # received. flow.Events.PublishEvent("NannyMessage", msg, token=self.token) # Now emulate a worker to process the event. worker = test_lib.MockWorker(token=self.token) while worker.Next(): pass worker.pool.Join() # We expect the email to be sent. self.assertEqual(self.email_message.get("address"), config_lib.CONFIG["Monitoring.alert_email"]) self.assertTrue(str(self.client_id) in self.email_message["title"]) # Make sure the message is included in the email message. self.assertTrue(nanny_message in self.email_message["message"]) # Make sure crashes RDFValueCollections are created and written # into proper locations. First check the per-client crashes collection. client_crashes = list( aff4.FACTORY.Open(self.client_id.Add("crashes"), aff4_type="PackedVersionedCollection", token=self.token)) self.assertEqual(len(client_crashes), 1) crash = client_crashes[0] self.assertEqual(crash.client_id, self.client_id) self.assertEqual(crash.client_info.client_name, "GRR Monitor") self.assertEqual( crash.crash_type, "aff4:/flows/" + queues.FLOWS.Basename() + ":NannyMessage") self.assertEqual(crash.crash_message, nanny_message) # Check global crash collection. Check that crash written there is # equal to per-client crash. global_crashes = list( aff4.FACTORY.Open(aff4.ROOT_URN.Add("crashes"), aff4_type="PackedVersionedCollection", token=self.token)) self.assertEqual(len(global_crashes), 1) self.assertEqual(global_crashes[0], crash)
def Run(self, args): """Reads a buffer on the client and sends it to the server.""" # Make sure we limit the size of our output if args.length > MAX_BUFFER_SIZE: raise RuntimeError("Can not read buffers this large.") data = vfs.ReadVFS(args.pathspec, args.offset, args.length, progress_callback=self.Progress) result = rdfvalue.DataBlob( data=zlib.compress(data), compression=rdfvalue.DataBlob.CompressionType.ZCOMPRESSION) digest = hashlib.sha256(data).digest() # Ensure that the buffer is counted against this response. Check network # send limit. self.ChargeBytesToSession(len(data)) # Now return the data to the server into the special TransferStore well # known flow. self.grr_worker.SendReply( result, session_id=rdfvalue.SessionID(flow_name="TransferStore")) # Now report the hash of this blob to our flow as well as the offset and # length. self.SendReply(offset=args.offset, length=len(data), data=digest)
def testStatEntryToExportedRegistryKeyConverter(self): stat = rdfvalue.StatEntry( aff4path=rdfvalue.RDFURN( "aff4:/C.0000000000000000/registry/HKEY_USERS/S-1-5-20/Software/" "Microsoft/Windows/CurrentVersion/Run/Sidebar"), st_mode=32768, st_size=51, st_mtime=1247546054, registry_type=rdfvalue.StatEntry.RegistryType.REG_EXPAND_SZ, pathspec=rdfvalue.PathSpec( path="/HKEY_USERS/S-1-5-20/Software/Microsoft/Windows/" "CurrentVersion/Run/Sidebar", pathtype=rdfvalue.PathSpec.PathType.REGISTRY), registry_data=rdfvalue.DataBlob(string="Sidebar.exe")) converter = export.StatEntryToExportedRegistryKeyConverter() results = list(converter.Convert(rdfvalue.ExportedMetadata(), stat, token=self.token)) self.assertEqual(len(results), 1) self.assertEqual(results[0].urn, rdfvalue.RDFURN( "aff4:/C.0000000000000000/registry/HKEY_USERS/S-1-5-20/Software/" "Microsoft/Windows/CurrentVersion/Run/Sidebar")) self.assertEqual(results[0].last_modified, rdfvalue.RDFDatetimeSeconds(1247546054)) self.assertEqual(results[0].type, rdfvalue.StatEntry.RegistryType.REG_EXPAND_SZ) self.assertEqual(results[0].data, "Sidebar.exe")
def SendNannyMessage(self): msg = self.nanny_controller.GetNannyMessage() if msg: self.SendReply(rdfvalue.DataBlob(string=msg), session_id="W:NannyMessage", priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY, require_fastpoll=False) self.nanny_controller.ClearNannyMessage()
def _MakeRegStat(self, path, value, registry_type): options = rdfvalue.PathSpec.Options.CASE_LITERAL pathspec = rdfvalue.PathSpec( path=path, path_options=options, pathtype=rdfvalue.PathSpec.PathType.REGISTRY) if registry_type == rdfvalue.StatEntry.RegistryType.REG_MULTI_SZ: reg_data = rdfvalue.DataBlob(list=rdfvalue.BlobArray( content=rdfvalue.DataBlob(string=value))) else: reg_data = rdfvalue.DataBlob().SetValue(value) return rdfvalue.StatEntry( aff4path=self.client_id.Add("registry").Add(path), pathspec=pathspec, registry_data=reg_data, registry_type=registry_type)
def UnlockSubject(self, subject, transid, token): """Unlocks subject using transaction id.""" request = rdfvalue.DataStoreRequest(subject=[subject]) if token: request.token = token blob = rdfvalue.DataBlob(string=transid) value = rdfvalue.DataStoreValue(value=blob) request.values.Append(value) # We do not care about the server response. typ = rdfvalue.DataStoreCommand.Command.UNLOCK_SUBJECT self._MakeSyncRequest(request, typ) return transid
def _ScheduleResponseAndStatus(self, client_id, flow_id): with queue_manager.QueueManager(token=self.token) as flow_manager: # Schedule a response. flow_manager.QueueResponse(flow_id, rdfvalue.GrrMessage( source=client_id, session_id=flow_id, payload=rdfvalue.DataBlob(string="Helllo"), request_id=1, response_id=1)) # And a STATUS message. flow_manager.QueueResponse(flow_id, rdfvalue.GrrMessage( source=client_id, session_id=flow_id, payload=rdfvalue.GrrStatus( status=rdfvalue.GrrStatus.ReturnedStatus.OK), request_id=1, response_id=2, type=rdfvalue.GrrMessage.Type.STATUS))
def ProcessFileStats(self, responses): """Extract DataBlob from Stat response.""" if not responses.success: return system_root_paths = ["Windows", "WinNT", "WINNT35", "WTSRV", "WINDOWS"] for response in responses: if response.pathspec.path[4:] in system_root_paths: systemdrive = response.pathspec.path[1:3] systemroot = "%s\\%s" % (systemdrive, response.pathspec.path[4:]) # Put the data back into the original format expected for the artifact data = rdfvalue.DataBlob().SetValue(systemroot) self.SendReply(rdfvalue.StatEntry(registry_data=data)) self.state.success = True break
def testParse(self): parser = windows_persistence.WindowsPersistenceMechanismsParser() path = (r"HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion" r"\Run\test") pathspec = rdfvalue.PathSpec( path=path, pathtype=rdfvalue.PathSpec.PathType.REGISTRY) reg_data = "C:\\blah\\some.exe /v" reg_type = rdfvalue.StatEntry.RegistryType.REG_SZ stat = rdfvalue.StatEntry( aff4path="aff4:/asdfasdf/", pathspec=pathspec, registry_type=reg_type, registry_data=rdfvalue.DataBlob(string=reg_data)) persistence = [stat] image_paths = [ "system32\\drivers\\ACPI.sys", "%systemroot%\\system32\\svchost.exe -k netsvcs", "\\SystemRoot\\system32\\drivers\\acpipmi.sys" ] reg_key = rdfvalue.RDFURN("aff4:/C.1000000000000000/registry" "/HKEY_LOCAL_MACHINE/SYSTEM/ControlSet001" "/services/AcpiPmi") for path in image_paths: serv_info = rdfvalue.WindowsServiceInformation( name="blah", display_name="GRRservice", image_path=path, registry_key=reg_key) persistence.append(serv_info) knowledge_base = rdfvalue.KnowledgeBase() knowledge_base.environ_systemroot = "C:\\Windows" expected = [ "C:\\blah\\some.exe", "C:\\Windows\\system32\\drivers\\ACPI.sys", "C:\\Windows\\system32\\svchost.exe", "C:\\Windows\\system32\\drivers\\acpipmi.sys" ] for index, item in enumerate(persistence): results = list( parser.Parse(item, knowledge_base, rdfvalue.PathSpec.PathType.OS)) self.assertEqual(results[0].pathspec.path, expected[index]) self.assertEqual(len(results), 1)
def ProcessMessage(self, message=None, event=None): """Processes this event.""" _ = event client_id = message.source message = rdfvalue.DataBlob(message.args).string logging.info(self.logline, client_id, message) # Write crash data to AFF4. client = aff4.FACTORY.Open(client_id, token=self.token) client_info = client.Get(client.Schema.CLIENT_INFO) crash_details = rdfvalue.ClientCrash( client_id=client_id, client_info=client_info, crash_message=message, timestamp=long(time.time() * 1e6), crash_type=self.well_known_session_id) self.WriteAllCrashDetails(client_id, crash_details) # Also send email. if config_lib.CONFIG["Monitoring.alert_email"]: client = aff4.FACTORY.Open(client_id, token=self.token) hostname = client.Get(client.Schema.HOSTNAME) url = urllib.urlencode( (("c", client_id), ("main", "HostInformation"))) email_alerts.SendEmail( config_lib.CONFIG["Monitoring.alert_email"], "GRR server", self.subject % client_id, self.mail_template % dict(client_id=client_id, admin_ui=config_lib.CONFIG["AdminUI.url"], hostname=hostname, signature=config_lib.CONFIG["Email.signature"], urn=url, message=message), is_html=True)
def ExtendSubjectLock(self, subject, transid, lease_time, token): """Extends lock of subject.""" request = rdfvalue.DataStoreRequest(subject=[subject]) specific = rdfvalue.TimestampSpec.Type.SPECIFIC_TIME request.timestamp = rdfvalue.TimestampSpec(start=lease_time, type=specific) if token: request.token = token blob = rdfvalue.DataBlob(string=transid) value = rdfvalue.DataStoreValue(value=blob) request.values.Append(value) typ = rdfvalue.DataStoreCommand.Command.EXTEND_SUBJECT response = self._MakeSyncRequest(request, typ) if not response.results: return None result = response.results[0] if not result.values: return None value = result.values[0].value.string return transid if transid == value else None
def _Stat(self, name, value, value_type): response = rdfvalue.StatEntry() response_pathspec = self.pathspec.Copy() # No matter how we got here, there is no need to do case folding from now on # since this is the exact filename casing. response_pathspec.path_options = rdfvalue.PathSpec.Options.CASE_LITERAL response_pathspec.last.path = utils.JoinPath( response_pathspec.last.path, name) response.pathspec = response_pathspec if self.IsDirectory(): response.st_mode = stat.S_IFDIR else: response.st_mode = stat.S_IFREG response.st_mtime = self.last_modified response.st_size = len(utils.SmartStr(value)) response.registry_type = self.registry_map.get(value_type, 0) response.registry_data = rdfvalue.DataBlob().SetValue(value) return response
def Run(self): """A Generator which makes a single request to the GRR server. Callers should generate this when they wish to make a connection to the server. It is up to the caller to sleep between calls in order to enforce the required network and CPU utilization policies. Raises: RuntimeError: Too many connection errors have been encountered. Yields: A Status() object indicating how the last POST went. """ while True: self.consecutive_connection_errors = 0 while self.active_server_url is None: if self.EstablishConnection(): # Everything went as expected - we don't need to return to # the main loop (which would mean sleeping for a poll_time). break else: # If we can't reconnect to the server for a long time, we restart # to reset our state. In some very rare cases, the urrlib can get # confused and we need to reset it before we can start talking to # the server again. self.consecutive_connection_errors += 1 limit = config_lib.CONFIG["Client.connection_error_limit"] if self.consecutive_connection_errors > limit: raise RuntimeError( "Too many connection errors, exiting.") # Constantly retrying will not work, we back off a bit. time.sleep(60) yield Status() # Check if there is a message from the nanny to be sent. self.client_worker.SendNannyMessage() now = time.time() # Check with the foreman if we need to if (now > self.last_foreman_check + config_lib.CONFIG["Client.foreman_check_frequency"]): # We must not queue messages from the comms thread with blocking=True # or we might deadlock. If the output queue is full, we can't accept # more work from the foreman anyways so it's ok to drop the message. try: self.client_worker.SendReply( rdfvalue.DataBlob(), session_id=rdfvalue.FlowSessionID(flow_name="Foreman"), priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY, require_fastpoll=False, blocking=False) self.last_foreman_check = now except Queue.Full: pass status = self.RunOnce() # We suicide if our memory is exceeded, and there is no more work to do # right now. Our death should not result in loss of messages since we are # not holding any requests in our input queues. if (self.client_worker.MemoryExceeded() and not self.client_worker.IsActive() and self.client_worker.InQueueSize() == 0 and self.client_worker.OutQueueSize() == 0): logging.warning("Memory exceeded - exiting.") self.client_worker.SendClientAlert( "Memory limit exceeded, exiting.") # Make sure this will return True so we don't get more work. # pylint: disable=g-bad-name self.client_worker.MemoryExceeded = lambda: True # pylint: enable=g-bad-name # Now send back the client message. self.RunOnce() # And done for now. sys.exit(-1) self.Wait(status) yield status
def Store(self, data): self.storage.append(self.in_rdfvalue(data).string) return [rdfvalue.DataBlob(string="Hello World")]
def ReturnBlob(self, unused_args): return [rdfvalue.DataBlob(integer=100)]
def _AddTransactionId(self, response, subject, transid): blob = rdfvalue.DataBlob(string=transid) value = rdfvalue.DataStoreValue(value=blob) response.results.Append(subject=subject, values=[value])
def GetInstallDate(self, _): return [rdfvalue.DataBlob(integer=100)]
def GetInstallDate(self, _): self.response_count += 1 return [rdfvalue.DataBlob(integer=100)]
def testNoValidStatusRaceIsResolved(self): # This tests for the regression of a long standing race condition we saw # where notifications would trigger the reading of another request that # arrives later but wasn't completely written to the database yet. # Timestamp based notification handling should eliminate this bug. # We need a random flow object for this test. session_id = flow.GRRFlow.StartFlow(client_id=self.client_id, flow_name="WorkerSendingTestFlow", token=self.token) worker_obj = worker.GRRWorker(worker.DEFAULT_WORKER_QUEUE, token=self.token) manager = queue_manager.QueueManager(token=self.token) manager.DeleteNotification(session_id) manager.Flush() # We have a first request that is complete (request_id 1, response_id 1). self.SendResponse(session_id, "Response 1") # However, we also have request #2 already coming in. The race is that # the queue manager might write the status notification to # session_id/state as "status:00000002" but not the status response # itself yet under session_id/state/request:00000002 request_id = 2 response_id = 1 flow_manager = queue_manager.QueueManager(token=self.token) flow_manager.FreezeTimestamp() flow_manager.QueueResponse( session_id, rdfvalue.GrrMessage(source=self.client_id, session_id=session_id, payload=rdfvalue.DataBlob(string="Response 2"), request_id=request_id, response_id=response_id)) status = rdfvalue.GrrMessage( source=self.client_id, session_id=session_id, payload=rdfvalue.GrrStatus( status=rdfvalue.GrrStatus.ReturnedStatus.OK), request_id=request_id, response_id=response_id + 1, type=rdfvalue.GrrMessage.Type.STATUS) # Now we write half the status information. subject = session_id.Add("state") queue = flow_manager.to_write.setdefault(subject, {}) queue.setdefault(flow_manager.FLOW_STATUS_TEMPLATE % request_id, []).append((status.SerializeToString(), None)) flow_manager.Flush() # We make the race even a bit harder by saying the new notification gets # written right before the old one gets deleted. If we are not careful here, # we delete the new notification as well and the flow becomes stuck. def WriteNotification(self, arg_session_id, start=None, end=None): if arg_session_id == session_id: flow_manager.QueueNotification(session_id=arg_session_id) flow_manager.Flush() self.DeleteNotification.old_target(self, arg_session_id, start=start, end=end) with utils.Stubber(queue_manager.QueueManager, "DeleteNotification", WriteNotification): # This should process request 1 but not touch request 2. worker_obj.RunOnce() worker_obj.thread_pool.Join() flow_obj = aff4.FACTORY.Open(session_id, token=self.token) self.assertFalse(flow_obj.state.context.backtrace) self.assertNotEqual(flow_obj.state.context.state, rdfvalue.Flow.State.ERROR) request2_data = data_store.DB.ResolveRegex(session_id.Add("state"), ".*:00000002", token=self.token) # Make sure the status field and the original request are still there. self.assertEqual(len(request2_data), 2) request1_data = data_store.DB.ResolveRegex(session_id.Add("state"), ".*:00000001", token=self.token) # Everything from request 1 should have been deleted. self.assertEqual(len(request1_data), 0) # The notification for request 2 should have survived. with queue_manager.QueueManager(token=self.token) as manager: notifications = manager.GetNotifications( worker.DEFAULT_WORKER_QUEUE) self.assertEqual(len(notifications), 1) notification = notifications[0] self.assertEqual(notification.session_id, session_id) self.assertEqual(notification.timestamp, flow_manager.frozen_timestamp) self.assertEqual(RESULTS, ["Response 1"]) # The last missing piece of request 2 is the actual status message. flow_manager.QueueResponse(session_id, status) flow_manager.Flush() # Now make sure request 2 runs as expected. worker_obj.RunOnce() worker_obj.thread_pool.Join() self.assertEqual(RESULTS, ["Response 1", "Response 2"])
def Start(self): for i in range(10): self.CallClient("Test", rdfvalue.DataBlob(string="test%s" % i), data=str(i), next_state="Incoming")
def SendForemanRequest(self): self.client_worker.SendReply( rdfvalue.DataBlob(), session_id=rdfvalue.FlowSessionID(flow_name="Foreman"), priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY, require_fastpoll=False)
def SendClientAlert(self, msg): self.SendReply( rdfvalue.DataBlob(string=msg), session_id=rdfvalue.FlowSessionID(flow_name="ClientAlert"), priority=rdfvalue.GrrMessage.Priority.LOW_PRIORITY, require_fastpoll=False)