def WriteAllCrashDetails(client_id, crash_details, flow_session_id=None, hunt_session_id=None, token=None): """Updates the last crash attribute of the client.""" # AFF4. if data_store.AFF4Enabled(): with aff4.FACTORY.Create( client_id, aff4_grr.VFSGRRClient, token=token) as client_obj: client_obj.Set(client_obj.Schema.LAST_CRASH(crash_details)) # Duplicate the crash information in a number of places so we can find it # easily. client_urn = rdf_client.ClientURN(client_id) client_crashes = aff4_grr.VFSGRRClient.CrashCollectionURNForCID(client_urn) with data_store.DB.GetMutationPool() as pool: grr_collections.CrashCollection.StaticAdd( client_crashes, crash_details, mutation_pool=pool) # Relational db. if data_store.RelationalDBEnabled(): try: data_store.REL_DB.WriteClientCrashInfo(client_id, crash_details) except db.UnknownClientError: pass if not flow_session_id: return if data_store.RelationalDBEnabled(): flow_id = flow_session_id.Basename() data_store.REL_DB.UpdateFlow( client_id, flow_id, client_crash_info=crash_details) flow_obj = data_store.REL_DB.ReadFlowObject(client_id, flow_id) if (flow_obj.parent_hunt_id and not hunt.IsLegacyHunt(flow_obj.parent_hunt_id)): hunt.StopHuntIfCrashLimitExceeded(flow_obj.parent_hunt_id) else: with aff4.FACTORY.Open( flow_session_id, flow.GRRFlow, mode="rw", age=aff4.NEWEST_TIME, token=token) as aff4_flow: aff4_flow.Set(aff4_flow.Schema.CLIENT_CRASH(crash_details)) hunt_session_id = ExtractHuntId(flow_session_id) if hunt_session_id and hunt_session_id != flow_session_id: hunt_obj = aff4.FACTORY.Open( hunt_session_id, aff4_type=implementation.GRRHunt, mode="rw", token=token) hunt_obj.RegisterCrash(crash_details)
def LookupClients(self, keywords): """Returns a list of client URNs associated with keywords. Args: keywords: The list of keywords to search by. Returns: A list of client URNs. Raises: ValueError: A string (single keyword) was passed instead of an iterable. """ if isinstance(keywords, basestring): raise ValueError( "Keywords should be an iterable, not a string (got %s)." % keywords) start_time, end_time, filtered_keywords, unversioned_keywords = ( self._AnalyzeKeywords(keywords)) last_seen_map = None if unversioned_keywords: last_seen_map = {} # TODO(user): Make keyword index datetime aware so that # AsMicrosecondsSinceEpoch is unnecessary. raw_results = self.Lookup( list(map(self._NormalizeKeyword, filtered_keywords)), start_time=start_time.AsMicrosecondsSinceEpoch(), end_time=end_time.AsMicrosecondsSinceEpoch(), last_seen_map=last_seen_map) if not raw_results: return [] if unversioned_keywords: universal_last_seen_raw = {} self.ReadPostingLists( list(map(self._NormalizeKeyword, raw_results)), start_time=start_time.AsMicrosecondsSinceEpoch(), end_time=end_time.AsMicrosecondsSinceEpoch(), last_seen_map=universal_last_seen_raw) universal_last_seen = {} for (_, client_id), ts in iteritems(universal_last_seen_raw): universal_last_seen[client_id] = ts old_results = set() for keyword in unversioned_keywords: for result in raw_results: if last_seen_map[(keyword, result)] < universal_last_seen[result]: old_results.add(result) raw_results -= old_results return [rdf_client.ClientURN(result) for result in raw_results]
def testClientApprovalMultiLabel(self): """Multi-label client approval test. This client requires one legal and two prod admin approvals. The requester must also be in the prod admin group. """ self.TouchFile(rdf_client.ClientURN(self.client_prod_id), "fs/os/foo") self.token.username = u"prod1" webauth.WEBAUTH_MANAGER.SetUserName(self.token.username) # No approvals yet, this should fail. self.assertRaises( grr_api_errors.AccessForbiddenError, self.api.Client(self.client_prod_id).File("fs/os/foo").Get) approval_id = self.RequestAndGrantClientApproval( self.client_prod_id, requestor=self.token.username) # This approval from "approver" isn't enough. self.assertRaises( grr_api_errors.AccessForbiddenError, self.api.Client(self.client_prod_id).File("fs/os/foo").Get) # Grant an approval from a user in the legal_approval list in # approvers.yaml self.GrantClientApproval(self.client_prod_id, requestor=self.token.username, approval_id=approval_id, approver=u"legal1") # We have "approver", "legal1": not enough. self.assertRaises( grr_api_errors.AccessForbiddenError, self.api.Client(self.client_prod_id).File("fs/os/foo").Get) # Grant an approval from a user in the prod_admin_approval list in # approvers.yaml self.GrantClientApproval(self.client_prod_id, requestor=self.token.username, approval_id=approval_id, approver=u"prod2") # We have "approver", "legal1", "prod2": not enough. self.assertRaises( grr_api_errors.AccessForbiddenError, self.api.Client(self.client_prod_id).File("fs/os/foo").Get) self.GrantClientApproval(self.client_prod_id, requestor=self.token.username, approval_id=approval_id, approver=u"prod3") # We have "approver", "legal1", "prod2", "prod3": we should have # access. self.api.Client(self.client_prod_id).File("fs/os/foo").Get()
def testNoClientActionIsDisplayedWhenFlowIsStarted(self): self.RequestAndGrantClientApproval(self.client_id) self.Open("/#/clients/%s/load-stats" % self.client_id) self.WaitUntil(self.IsTextPresent, "No actions currently in progress.") flow.StartAFF4Flow( client_id=rdf_client.ClientURN(self.client_id), flow_name=processes.ListProcesses.__name__, token=self.token)
def _testProcessMessagesWellKnown(self): worker_obj = self._TestWorker() # Send a message to a WellKnownFlow - ClientStatsAuto. session_id = administrative.GetClientStatsAuto.well_known_session_id client_id = rdf_client.ClientURN("C.1100110011001100") if data_store.RelationalDBReadEnabled(category="message_handlers"): done = threading.Event() def handle(l): worker_obj._ProcessMessageHandlerRequests(l) done.set() data_store.REL_DB.RegisterMessageHandler( handle, worker_obj.well_known_flow_lease_time, limit=1000) self.SendResponse(session_id, data=rdf_client_stats.ClientStats(RSS_size=1234), client_id=client_id, well_known=True) self.assertTrue(done.wait(10)) else: self.SendResponse(session_id, data=rdf_client_stats.ClientStats(RSS_size=1234), client_id=client_id, well_known=True) # Process all messages worker_obj.RunOnce() worker_obj.thread_pool.Join() if data_store.RelationalDBReadEnabled("client_stats"): results = data_store.REL_DB.ReadClientStats( client_id=client_id.Basename(), min_timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0), max_timestamp=rdfvalue.RDFDatetime.Now()) self.assertLen(results, 1) stats = results[0] else: client = aff4.FACTORY.Open(client_id.Add("stats"), token=self.token) stats = client.Get(client.Schema.STATS) self.assertEqual(stats.RSS_size, 1234) # Make sure no notifications have been sent. user = aff4.FACTORY.Open("aff4:/users/%s" % self.token.username, token=self.token) notifications = user.Get(user.Schema.PENDING_NOTIFICATIONS) self.assertIsNone(notifications) if data_store.RelationalDBReadEnabled(category="message_handlers"): data_store.REL_DB.UnregisterMessageHandler(timeout=60)
def ApprovalRequest(client_id, token=None, approver="approver", reason="testing"): token = token or GetToken() approval_reason = reason or token.reason security.ClientApprovalRequestor( reason=approval_reason, subject_urn=rdf_client.ClientURN(client_id), approver=approver, token=token).Request()
def testUnknownNotificationIsParsedCorrectly(self): urn = rdf_client.ClientURN(self.client_id).Add("foo/bar") n = user_plugin.ApiNotification().InitFromNotification( rdf_flows.Notification(type="ViewObject", subject=urn)) self.assertEqual(n.reference.type, "UNKNOWN") self.assertEqual(n.reference.unknown.subject_urn, urn) n = user_plugin.ApiNotification().InitFromNotification( rdf_flows.Notification(type="FlowStatus", subject="foo/bar")) self.assertEqual(n.reference.type, "UNKNOWN") self.assertEqual(n.reference.unknown.subject_urn, "foo/bar")
def testClickingOnTreeNodeRefreshesChildrenFoldersList(self): self.Open("/#/clients/C.0000000000000001/vfs/fs/os/c/") self.WaitUntilNot(self.IsElementPresent, "link=foo") gui_test_lib.CreateFolder(rdf_client.ClientURN("C.0000000000000001"), "fs/os/c/foo", timestamp=gui_test_lib.TIME_0, token=self.token) self.Click("link=c") self.WaitUntil(self.IsElementPresent, "link=foo")
def setUp(self): super(TestFileView, self).setUp() # Prepare our fixture. self.client_id, self.unapproved_client_id = [ u.Basename() for u in self.SetupClients(2) ] fixture_test_lib.ClientFixture(self.client_id, self.token) gui_test_lib.CreateFileVersions(rdf_client.ClientURN(self.client_id), self.token) self.RequestAndGrantClientApproval(self.client_id)
def testListErrors(self): client_urn_1 = rdf_client.ClientURN("C.0000111122223333") with test_lib.FakeTime(52): self.hunt_obj.LogClientError(client_urn_1, "Error foo.") client_urn_2 = rdf_client.ClientURN("C.1111222233334444") with test_lib.FakeTime(55): self.hunt_obj.LogClientError(client_urn_2, "Error bar.", "<some backtrace>") errors = list(self.api.Hunt(self.hunt_obj.urn.Basename()).ListErrors()) self.assertEqual(len(errors), 2) self.assertEqual(errors[0].log_message, "Error foo.") self.assertEqual(errors[0].client.client_id, client_urn_1.Basename()) self.assertEqual(errors[0].backtrace, "") self.assertEqual(errors[1].log_message, "Error bar.") self.assertEqual(errors[1].client.client_id, client_urn_2.Basename()) self.assertEqual(errors[1].backtrace, "<some backtrace>")
def Run(self): if data_store.RelationalDBReadEnabled(): client = self.SetupTestClientObject(0) client_id = client.client_id client_ids = [rdf_client.ClientURN(client_id)] else: client_ids = self.SetupClients(1) client_id = client_ids[0].Basename() client_mock = flow_test_lib.CrashClientMock( rdf_client.ClientURN(client_id), self.token) with test_lib.FakeTime(42): with self.CreateHunt(description="the hunt") as hunt_obj: hunt_obj.Run() with test_lib.FakeTime(45): self.AssignTasksToClients(client_ids) hunt_test_lib.TestHuntHelperWithMultipleMocks( {client_id: client_mock}, False, self.token) crashes = aff4_grr.VFSGRRClient.CrashCollectionForCID( rdf_client.ClientURN(client_id)) crash = list(crashes)[0] session_id = crash.session_id.Basename() replace = { hunt_obj.urn.Basename(): "H:123456", session_id: "H:11223344" } self.Check( "ListClientCrashes", args=client_plugin.ApiListClientCrashesArgs(client_id=client_id), replace=replace) self.Check("ListClientCrashes", args=client_plugin.ApiListClientCrashesArgs( client_id=client_id, count=1), replace=replace) self.Check("ListClientCrashes", args=client_plugin.ApiListClientCrashesArgs( client_id=client_id, offset=1, count=1), replace=replace)
def EnrolFleetspeakClient(self, client_id): """Enrols a Fleetspeak-enabled client for use with GRR. Args: client_id: GRR client-id for the client. Returns: True if the client is new, and actually got enrolled. This method is a no-op if the client already exists (in which case False is returned). """ client_urn = rdf_client.ClientURN(client_id) # If already enrolled, return. if data_store.RelationalDBReadEnabled(): try: data_store.REL_DB.ReadClientMetadata(client_id) return False except db.UnknownClientError: pass else: if aff4.FACTORY.ExistsWithType( client_urn, aff4_type=aff4_grr.VFSGRRClient, token=self.token): return False logging.info("Enrolling a new Fleetspeak client: %r", client_id) if data_store.RelationalDBWriteEnabled(): now = rdfvalue.RDFDatetime.Now() data_store.REL_DB.WriteClientMetadata( client_id, first_seen=now, fleetspeak_enabled=True, last_ping=now) # TODO(fleetspeak-team,grr-team): If aff4 isn't reliable enough, we can # catch exceptions from it and forward them to Fleetspeak by failing its # gRPC call. Fleetspeak will then retry with a random, perhaps healthier, # instance of the GRR frontend. with aff4.FACTORY.Create( client_urn, aff4_type=aff4_grr.VFSGRRClient, mode="rw", token=self.token) as client: client.Set(client.Schema.FLEETSPEAK_ENABLED, rdfvalue.RDFBool(True)) index = client_index.CreateClientIndex(token=self.token) index.AddClient(client) if data_store.RelationalDBWriteEnabled(): index = client_index.ClientIndex() index.AddClient(data_migration.ConvertVFSGRRClient(client)) # Publish the client enrollment message. events.Events.PublishEvent("ClientEnrollment", client_urn, token=self.token) return True
def ProcessResponse(self, client_id, response): """Actually processes the contents of the response.""" urn = rdf_client.ClientURN(client_id).Add("stats") downsampled = rdf_client_stats.ClientStats.Downsampled(response) with aff4.FACTORY.Create(urn, aff4_stats.ClientStats, token=self.token, mode="w") as stats_fd: # Only keep the average of all values that fall within one minute. stats_fd.AddAttribute(stats_fd.Schema.STATS, downsampled) return downsampled
def _GetClientUrns(): """Returns a set of client URNs available in the data store.""" result = set() for urn in aff4.FACTORY.ListChildren("aff4:/"): try: client_urn = rdf_client.ClientURN(urn) except type_info.TypeValueError: continue result.add(client_urn) return result
def _KillFlow(self, client_id, flow_id): if data_store.RelationalDBFlowsEnabled(): rdf_flow = data_store.REL_DB.ReadFlowForProcessing( client_id, flow_id, rdfvalue.Duration("5m")) flow_cls = registry.FlowRegistry.FlowClassByName(rdf_flow.flow_class_name) flow_obj = flow_cls(rdf_flow) flow_obj.Error("Fake error") data_store.REL_DB.ReturnProcessedFlow(rdf_flow) else: flow_urn = rdf_client.ClientURN(client_id).Add("flows").Add(flow_id) with aff4.FACTORY.Open( flow_urn, aff4_type=flow.GRRFlow, mode="rw", token=self.token) as flow_obj: flow_obj.GetRunner().Error("Fake error")
def _testAFF4Path_mountPointResolution( self, pathtype: rdf_paths.PathSpec.PathType) -> None: path = rdf_paths.PathSpec(path="\\\\.\\Volume{1234}\\", pathtype=rdf_paths.PathSpec.PathType.OS, mount_point="/c:/", nested_path=rdf_paths.PathSpec( path="/windows/", pathtype=pathtype, )) prefix = rdf_paths.PathSpec.AFF4_PREFIXES[pathtype] self.assertEqual( str(path.AFF4Path(rdf_client.ClientURN("C.0000000000000001"))), f"aff4:/C.0000000000000001{prefix}/\\\\.\\Volume{{1234}}\\/windows" )
def testClientNoLabels(self): self.TouchFile(rdf_client.ClientURN(self.client_nolabel_id), "fs/os/foo") self.assertRaises( grr_api_errors.AccessForbiddenError, self.api.Client(self.client_nolabel_id).File("fs/os/foo").Get) # approvers.yaml rules don't get checked because this client has no # labels. Regular approvals still required. self.RequestAndGrantClientApproval( self.client_nolabel_id, requestor=self.token.username) # Check we now have access self.api.Client(self.client_nolabel_id).File("fs/os/foo").Get()
def setUp(self): super(TestFlowCopy, self).setUp() # Prepare our fixture. self.client_id = rdf_client.ClientURN("C.0000000000000001") # This attribute is used by StandardHuntTestMixin. self.client_ids = [self.client_id] fixture_test_lib.ClientFixture(self.client_id, self.token) self.RequestAndGrantClientApproval("C.0000000000000001") self.email_descriptor = rdf_output_plugin.OutputPluginDescriptor( plugin_name=email_plugin.EmailOutputPlugin.__name__, plugin_args=email_plugin.EmailOutputPluginArgs( email_address="test@localhost", emails_limit=42))
def testRefreshButtonGetsReenabledWhenUpdateEnds(self): self.Open("/#/clients/C.0000000000000001/vfs/fs/os/c/") self.Click("css=button[id=refresh-dir]:not([disabled])") # Check that the button got disabled. self.WaitUntil(self.IsElementPresent, "css=button[id=refresh-dir][disabled]") client_id = rdf_client.ClientURN("C.0000000000000001") self._RunUpdateFlow(client_id) # Check that the button got re-enabled. self.WaitUntil(self.IsElementPresent, "css=button[id=refresh-dir]:not([disabled])")
def _SetupMinimalClient(self): client_id = "C.0000000000000000" client_urn = rdf_client.ClientURN("aff4:/C.0000000000000000") if data_store.AFF4Enabled(): with aff4.FACTORY.Create( client_urn, aff4_type=aff4_grr.VFSGRRClient, mode="w", token=self.token): pass if data_store.RelationalDBWriteEnabled(): data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) return client_id, client_urn
def testUsesDefaultClientURNIfGrrMessageHasNoSource(self): with self.pool: self.collection.Add(rdf_flows.GrrMessage( payload=rdfvalue.RDFString("foo"), source=None), mutation_pool=self.pool) chunks = self.ProcessPlugin( source_urn=rdf_client.ClientURN("C.1111222233334444")) self.assertListEqual(chunks, [ "Start: aff4:/foo/bar", "Values of type: RDFString", "First pass: foo (source=aff4:/C.1111222233334444)", "Second pass: foo (source=aff4:/C.1111222233334444)", "Finish: aff4:/foo/bar" ]) # pyformat: disable
def testRefreshButtonGetsDisabledWhileUpdateIsRunning(self): self.Open("/#/clients/C.0000000000000001/vfs/fs/os/c/") self.Click("css=button[id=refresh-dir]:not([disabled])") # Check that the button got disabled. self.WaitUntil(self.IsElementPresent, "css=button[id=refresh-dir][disabled]") client_id = rdf_client.ClientURN("C.0000000000000001") self._RunUpdateFlow(client_id) # Ensure that refresh button is enabled again. # self.WaitUntilNot(self.IsElementPresent, "css=button[id=refresh-dir][disabled]")
def setUp(self): super(TestFileView, self).setUp() # Prepare our fixture. self.client_id, self.unapproved_client_id = [ u.Basename() for u in self.SetupClients(2) ] fixture_test_lib.ClientFixture(self.client_id, self.token) self.content_1, self.content_2 = gui_test_lib.CreateFileVersions( rdf_client.ClientURN(self.client_id), self.token) self.content_1_hash = rdf_objects.SHA256HashID.FromData( self.content_1).AsBytes() self.content_2_hash = rdf_objects.SHA256HashID.FromData( self.content_2).AsBytes() self.RequestAndGrantClientApproval(self.client_id)
def RequestAndGrantClientApproval(client_id, token=None, approver="approver", reason="testing"): token = token or GetToken() ApprovalRequest(client_id, token=token, approver=approver, reason=reason) user = aff4.FACTORY.Create("aff4:/users/%s" % approver, users.GRRUser, token=token.SetUID()) user.Flush() approver_token = access_control.ACLToken(username=approver) security.ClientApprovalGrantor(reason=reason, delegate=token.username, subject_urn=rdf_client.ClientURN(client_id), token=approver_token).Grant()
def GRRMessageFromClientActionRequest(request): stub = action_registry.ACTION_STUB_BY_ID[request.action_identifier] name = compatibility.GetName(stub) return rdf_flows.GrrMessage( session_id="%s/%s" % (request.client_id, request.flow_id), name=name, request_id=request.request_id, queue=rdf_client.ClientURN(request.client_id), payload=request.action_args, cpu_limit=request.cpu_limit_ms / 1000.0, network_bytes_limit=request.network_bytes_limit, # Legacy clients will fail if the task id is not set. # TODO(amoser): Remove task ids after April 2021. generate_task_id=True)
def IsFleetspeakEnabledClient(grr_id, token): if grr_id is None: return False if data_store.RelationalDBReadEnabled(): md = data_store.REL_DB.ReadClientMetadata(grr_id) if not md: return False return md.fleetspeak_enabled else: with aff4.FACTORY.Create(rdf_client.ClientURN(grr_id), aff4.AFF4Object.classes["VFSGRRClient"], mode="r", token=token) as client: return bool(client.Get(client.Schema.FLEETSPEAK_ENABLED))
def testTreeAndFileListRefreshedWhenRefreshCompletes(self): self.Open("/#/clients/C.0000000000000001/vfs/fs/os/c/") self.Click("css=button[id=refresh-dir]:not([disabled])") self.WaitUntil(self.IsElementPresent, "css=button[id=refresh-dir][disabled]") client_id = rdf_client.ClientURN("C.0000000000000001") self._RunUpdateFlow(client_id) # The flow should be finished now, and file/tree lists update should # be triggered. # Ensure that the tree got updated as well as files list. self.WaitUntil(self.IsElementPresent, "css=tr:contains('TestFolder')") self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-TestFolder i.jstree-icon")
def __init__(self, client_id, client_mock, token=None): if not isinstance(client_id, rdf_client.ClientURN): client_id = rdf_client.ClientURN(client_id) if client_mock is None: client_mock = action_mocks.InvalidActionMock() else: utils.AssertType(client_mock, action_mocks.ActionMock) self._mock_task_queue = getattr(client_mock, "mock_task_queue", []) self.client_id = client_id self.client_mock = client_mock self.token = token # Well known flows are run on the front end. self.well_known_flows = flow.WellKnownFlow.GetAllWellKnownFlows(token=token)
def ProcessHuntFlowDone(flow_obj, status_msg=None): """Notifis hunt about a given hunt-induced flow completion.""" if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id): resources = _FlowStatusToClientResources(flow_obj, status_msg) def UpdateFn(hunt_obj): hunt_obj.num_successful_clients += 1 if flow_obj.num_replies_sent: hunt_obj.num_clients_with_results += 1 hunt_obj.client_resources_stats.RegisterResources(resources) return hunt_obj hunt_obj = data_store.REL_DB.UpdateHuntObject(flow_obj.parent_hunt_id, UpdateFn) hunt_obj = hunt.StopHuntIfAverageLimitsExceeded(hunt_obj) hunt.CompleteHuntIfExpirationTimeReached(hunt_obj) return hunt_urn = rdfvalue.RDFURN("hunts").Add(flow_obj.parent_hunt_id) client_urn = rdf_client.ClientURN(flow_obj.client_id) # Update the counter metrics separately from collections to minimize # contention. with aff4.FACTORY.Open(hunt_urn, mode="rw") as fd: # Legacy AFF4 code expects token to be set. fd.token = access_control.ACLToken(username=fd.creator) if flow_obj.num_replies_sent: fd.context.clients_with_results_count += 1 fd.context.completed_clients_count += 1 fd.context.results_count += flow_obj.num_replies_sent fd.GetRunner().SaveResourceUsage(flow_obj.client_id, status_msg) with aff4.FACTORY.Open(hunt_urn, mode="rw") as fd: # Legacy AFF4 code expects token to be set. fd.token = access_control.ACLToken(username=fd.creator) fd.RegisterCompletedClient(client_urn) if flow_obj.num_replies_sent: fd.RegisterClientWithResults(client_urn) fd.StopHuntIfAverageLimitsExceeded()
def testRealPathspec(self): client_id = rdf_client.ClientURN("C.%016X" % 1234) for path in ["a/b", "a/b/c/d"]: d = aff4.FACTORY.Create(client_id.Add("fs/os").Add(path), aff4_type=aff4_standard.VFSDirectory, token=self.token) pathspec = rdf_paths.PathSpec( path=path, pathtype=rdf_paths.PathSpec.PathType.OS) d.Set(d.Schema.PATHSPEC, pathspec) d.Close() d = aff4.FACTORY.Create(client_id.Add("fs/os").Add("a/b/c"), aff4_type=aff4_standard.VFSDirectory, mode="rw", token=self.token) self.assertEqual(d.real_pathspec.CollapsePath(), "a/b/c")