def testGrep(self): class MockCallFlow(object): def CallFlow(self, *args, **kwargs): self.args = args self.kwargs = kwargs mock_call_flow = MockCallFlow() with utils.Stubber(collectors.ArtifactCollectorFlow, "CallFlow", mock_call_flow.CallFlow): collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token) collect_flow.args = mock.Mock() collect_flow.args.ignore_interpolation_errors = False collect_flow.state.Register("knowledge_base", rdf_client.KnowledgeBase()) collect_flow.current_artifact_name = "blah" collect_flow.state.knowledge_base.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******")) collect_flow.state.knowledge_base.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******")) collector = artifact_registry.ArtifactSource( type=artifact_registry.ArtifactSource.SourceType.GREP, attributes={ "paths": ["/etc/passwd"], "content_regex_list": [r"^a%%users.username%%b$"] }) collect_flow.Grep(collector, rdf_paths.PathSpec.PathType.TSK) conditions = mock_call_flow.kwargs["conditions"] self.assertEqual(len(conditions), 1) regexes = conditions[0].contents_regex_match.regex.SerializeToString() self.assertItemsEqual(regexes.split("|"), ["(^atest1b$)", "(^atest2b$)"]) self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testClientStatsCollectionHappensEveryMinuteWhenClientIsBusy(self): """Tests that client stats are collected more often when client is busy.""" now = 1000000 # Pretend we have already sent stats. self.client_communicator.client_worker.last_stats_sent_time = ( rdfvalue.RDFDatetime().FromSecondsFromEpoch(now)) self.client_communicator.client_worker._is_active = True with test_lib.FakeTime(now): self.client_communicator.client_worker.CheckStats() runs = [] with utils.Stubber(admin.GetClientStatsAuto, "Run", lambda cls, _: runs.append(1)): # No stats collection after 30 seconds. with test_lib.FakeTime(now + 30): self.client_communicator.client_worker.CheckStats() self.assertEqual(len(runs), 0) # Let 61 seconds pass. with test_lib.FakeTime(now + 61): self.client_communicator.client_worker.CheckStats() # This time the client should collect stats. self.assertEqual(len(runs), 1) # No stats collection within one minute from the last time. with test_lib.FakeTime(now + 61 + 59): self.client_communicator.client_worker.CheckStats() self.assertEqual(len(runs), 1) # Stats collection happens as more than one minute has passed since the # last one. with test_lib.FakeTime(now + 61 + 61): self.client_communicator.client_worker.CheckStats() self.assertEqual(len(runs), 2)
def Run(self): stats_collector = stats.StatsCollector() stats_collector.RegisterCounterMetric( "sample_counter", docstring="Sample counter metric.") stats_collector.RegisterGaugeMetric( "sample_gauge_value", float, docstring="Sample gauge metric.") stats_collector.RegisterEventMetric( "sample_event", docstring="Sample event metric.") with utils.Stubber(stats, "STATS", stats_collector): for i in range(10): with test_lib.FakeTime(42 + i * 60): stats_collector.IncrementCounter("sample_counter") stats_collector.SetGaugeValue("sample_gauge_value", i * 0.5) stats_collector.RecordEvent("sample_event", 0.42 + 0.5 * i) with aff4.FACTORY.Create( None, "StatsStore", mode="w", token=self.token) as stats_store: stats_store.WriteStats(process_id="worker_1", sync=True) self.Check("GET", "/api/stats/store/WORKER/metrics/sample_counter?" "start=42000000&end=3600000000") self.Check("GET", "/api/stats/store/WORKER/metrics/sample_counter?" "start=42000000&end=3600000000&rate=1m") self.Check("GET", "/api/stats/store/WORKER/metrics/sample_gauge_value?" "start=42000000&end=3600000000") self.Check("GET", "/api/stats/store/WORKER/metrics/sample_event?" "start=42000000&end=3600000000") self.Check("GET", "/api/stats/store/WORKER/metrics/sample_event?" "start=42000000&end=3600000000&" "distribution_handling_mode=DH_COUNT")
def testBreakGlass(self): """Test the breakglass mechanism.""" client_id = rdfvalue.ClientURN("C.%016X" % 0) urn = client_id.Add("/fs/os/c") self.assertRaises(access_control.UnauthorizedAccess, aff4.FACTORY.Open, urn, token=self.token) # We expect to receive an email about this email = {} def SendEmail(to, from_user, subject, message, **_): email["to"] = to email["from_user"] = from_user email["subject"] = subject email["message"] = message with utils.Stubber(email_alerts, "SendEmail", SendEmail): flow.GRRFlow.StartFlow( client_id=client_id, flow_name="BreakGlassGrantClientApprovalFlow", token=self.token, reason=self.token.reason) # Reset the emergency state of the token. self.token.is_emergency = False # This access is using the emergency_access granted, so we expect the # token to be tagged as such. aff4.FACTORY.Open(urn, token=self.token) self.assertEqual(email["to"], config_lib.CONFIG["Monitoring.emergency_access_email"]) self.assert_(self.token.username in email["message"]) self.assertEqual(email["from_user"], self.token.username) # Make sure the token is tagged as an emergency token: self.assertEqual(self.token.is_emergency, True)
def testProcessHuntResultsCronFlowDoesNotAbortsIfRunningInTime(self): self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 0) test = [0] def TimeStub(): test[0] += 1e-6 return test[0] with utils.Stubber(time, "time", TimeStub): self.StartHunt(output_plugins=[ output_plugin.OutputPluginDescriptor( plugin_name="LongRunningDummyHuntOutputPlugin") ]) self.AssignTasksToClients() self.RunHunt(failrate=-1) # LongRunningDummyHuntOutputPlugin will set the time to 100s on the first # run, which will effectively mean that it's running in time. self.ProcessHuntOutputPlugins( batch_size=1, max_running_time=rdfvalue.Duration("101s")) # In normal conditions, there should be 10 results generated. self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 10)
def testEmailCronJobApprovalRequestLinkLeadsToACorrectPage(self): with self.ACLChecksDisabled(): cronjobs.ScheduleSystemCronFlows( names=[cron_system.OSBreakDown.__name__], token=self.token) cronjobs.CRON_MANAGER.DisableJob( rdfvalue.RDFURN("aff4:/cron/OSBreakDown")) messages_sent = [] def SendEmailStub(unused_from_user, unused_to_user, unused_subject, message, **unused_kwargs): messages_sent.append(message) # Request client approval, it will trigger an email message. with utils.Stubber(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmailStub): flow.GRRFlow.StartFlow( flow_name="RequestCronJobApprovalFlow", reason="Please please let me", subject_urn="aff4:/cron/OSBreakDown", approver=self.token.username, token=access_control.ACLToken(username="******", reason="test")) self.assertEqual(len(messages_sent), 1) # Extract link from the message text and open it. m = re.search(r"href='(.+?)'", messages_sent[0], re.MULTILINE) link = urlparse.urlparse(m.group(1)) self.Open(link.path + "?" + link.query + "#" + link.fragment) # Check that requestor's username and reason are correctly displayed. self.WaitUntil(self.IsTextPresent, "iwantapproval") self.WaitUntil(self.IsTextPresent, "Please please let me") # Check that host information is displayed. self.WaitUntil(self.IsTextPresent, "OSBreakDown") self.WaitUntil(self.IsTextPresent, "Periodicity")
def testWrongOwnerGetsFixed(self): def mystat(filename): stat_info = os.lstat.old_target(filename) stat_list = list(stat_info) # Adjust the UID. stat_list[4] += 1 return posix.stat_result(stat_list) # Place a malicious file in the temp dir. This needs to be deleted # before we can use the temp dir. fd = tempfiles.CreateGRRTempFile(filename="maliciousfile", mode="wb") fd.close() self.assertTrue(os.path.exists(fd.name)) with utils.Stubber(os, "lstat", mystat): fd2 = tempfiles.CreateGRRTempFile(filename="temptemp", mode="wb") fd2.close() # Old file is gone. self.assertFalse(os.path.exists(fd.name)) # Cleanup. tempfiles.DeleteGRRTempFile(fd2.name)
def testIndexIsUsedWhenRandomAccessIsUsed(self): with utils.MultiStubber( (aff4.PackedVersionedCollection, "COMPACTION_BATCH_SIZE", 100), (aff4.PackedVersionedCollection, "INDEX_INTERVAL", 1)): with aff4.FACTORY.Create(self.collection_urn, "PackedVersionedCollection", mode="w", token=self.token): pass for i in range(20): with aff4.FACTORY.Open( self.collection_urn, "PackedVersionedCollection", mode="w", token=self.token) as fd: fd.Add(rdf_flows.GrrMessage(request_id=i)) with aff4.FACTORY.OpenWithLock( self.collection_urn, "PackedVersionedCollection", token=self.token) as fd: fd.Compact() collection = aff4.FACTORY.Open(self.collection_urn, token=self.token) item_size = collection.fd.size / len(collection) # There's no seek expected for the first element for i in range(1, 20): seek_ops = [] old_seek = collection.fd.Seek def SeekStub(offset): seek_ops.append(offset) # pylint: disable=cell-var-from-loop old_seek(offset) # pylint: disable=cell-var-from-loop # Check that the stream is seeked to a correct byte offset on every # GenerateItems() call with an offset specified. with utils.Stubber(collection.fd, "Seek", SeekStub): _ = list(collection.GenerateItems(offset=i)) self.assertListEqual([item_size * i], seek_ops)
def Run(self): config_obj = config_lib.GrrConfigManager() config_obj.DEFINE_string("SectionFoo.sample_string_option", "", "Sample string option.") config_obj.DEFINE_string("Mysql.database_password", "", "Secret password.") # This has to be defined as http_api.HttpRequestHandler.HandleRequest # depends on it and regression data won't get rendered without # this config option defined. config_obj.DEFINE_string("AdminUI.debug_impersonate_user", None, "") config = """ SectionBar.sample_string_option: "%(sAmPlE|lower)" Mysql.database_password: "******" """ config_lib.LoadConfig( config_obj, config_fd=StringIO.StringIO(config), parser=config_lib.YamlParser) with utils.Stubber(config_lib, "CONFIG", config_obj): self.Check("GET", "/api/config/SectionFoo.sample_string_option") self.Check("GET", "/api/config/Mysql.database_password") self.Check("GET", "/api/config/NonExistingOption")
def testVersionDropDownChangesFileContentAndDownloads(self): """Test the fileview interface.""" # Set up multiple version for an attribute on the client for tests. with self.ACLChecksDisabled(): for fake_time, hostname in [(gui_test_lib.TIME_0, "HostnameV1"), (gui_test_lib.TIME_1, "HostnameV2"), (gui_test_lib.TIME_2, "HostnameV3")]: with test_lib.FakeTime(fake_time): client = aff4.FACTORY.Open(u"C.0000000000000001", mode="rw", token=self.token) client.Set(client.Schema.HOSTNAME(hostname)) client.Close() self.Open("/") self.Type("client_query", "C.0000000000000001") self.Click("client_query_submit") self.WaitUntilEqual(u"C.0000000000000001", self.GetText, "css=span[type=subject]") # Choose client 1. self.Click("css=td:contains('0001')") # Go to Browse VFS. self.Click("css=a[grrtarget='client.vfs']") self.Click("css=#_fs i.jstree-icon") self.Click("css=#_fs-os i.jstree-icon") self.Click("css=#_fs-os-c i.jstree-icon") # Test file versioning. self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads") self.Click("link=Downloads") # Verify that we have the latest version in the table by default. self.assertTrue( gui_test_lib.DateString(gui_test_lib.TIME_2) in self.GetText( "css=tr:contains(\"a.txt\")")) # Click on the row. self.Click("css=tr:contains(\"a.txt\")") self.WaitUntilContains("a.txt", self.GetText, "css=div#main_bottomPane h1") self.WaitUntilContains("HEAD", self.GetText, "css=.version-dropdown > option[selected]") self.WaitUntilContains(gui_test_lib.DateString(gui_test_lib.TIME_2), self.GetText, "css=.version-dropdown > option:nth(1)") # Check the data in this file. self.Click("css=li[heading=TextView]") self.WaitUntilContains("Goodbye World", self.GetText, "css=div.monospace pre") downloaded_files = [] def FakeDownloadHandle(unused_self, args, token=None): _ = token # Avoid unused variable linter warnings. aff4_path = args.client_id.ToClientURN().Add(args.file_path) age = args.timestamp or aff4.NEWEST_TIME downloaded_files.append((aff4_path, age)) return api_call_handler_base.ApiBinaryStream( filename=aff4_path.Basename(), content_generator=xrange(42)) with utils.Stubber(api_vfs.ApiGetFileBlobHandler, "Handle", FakeDownloadHandle): # Try to download the file. self.Click("css=li[heading=Download]") self.WaitUntilContains( gui_test_lib.DateTimeString(gui_test_lib.TIME_2), self.GetText, "css=grr-file-download-view") self.Click("css=button:contains(\"Download\")") # Select the previous version. self.Click( "css=select.version-dropdown > option:contains(\"%s\")" % gui_test_lib.DateString(gui_test_lib.TIME_1)) # Now we should have a different time. self.WaitUntilContains( gui_test_lib.DateTimeString(gui_test_lib.TIME_1), self.GetText, "css=grr-file-download-view") self.Click("css=button:contains(\"Download\")") self.WaitUntil(self.IsElementPresent, "css=li[heading=TextView]") # the FakeDownloadHandle method was actually called four times, since # a file download first sends a HEAD request to check user access. self.WaitUntil(lambda: len(downloaded_files) == 4) # Both files should be the same... self.assertEqual(downloaded_files[0][0], u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt") self.assertEqual(downloaded_files[2][0], u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt") # But from different times. The downloaded file timestamp is only accurate # to the nearest second. Also, the HEAD version of the file is downloaded # with age=NEWEST_TIME. self.assertEqual(downloaded_files[0][1], aff4.NEWEST_TIME) self.assertAlmostEqual(downloaded_files[2][1], gui_test_lib.TIME_1, delta=rdfvalue.Duration("1s")) self.Click("css=li[heading=TextView]") # Make sure the file content has changed. This version has "Hello World" in # it. self.WaitUntilContains("Hello World", self.GetText, "css=div.monospace pre")
def NoAuthorizationChecks(self): return utils.Stubber(api_auth_manager, "API_AUTH_MGR", NullAPIAuthorizationManager())
def testEmailPlugin(self): def SendEmail(address, sender, title, message, **_): self.email_messages.append( dict(address=address, sender=sender, title=title, message=message)) with utils.Stubber(email_alerts, "SendEmail", SendEmail): self.email_messages = [] email_alerts.SendEmail = SendEmail email_address = "notify@%s" % config_lib.CONFIG["Logging.domain"] hunt_urn = self.RunHunt( "EmailPlugin", rdfvalue.EmailPluginArgs(email=email_address, email_limit=10)) hunt_obj = aff4.FACTORY.Open(hunt_urn, age=aff4.ALL_TIMES, mode="rw", token=self.token) self.client_ids = self.SetupClients(40) hunt_obj.StartClients(hunt_obj.session_id, self.client_ids) # Run the hunt. client_mock = test_lib.SampleHuntMock() test_lib.TestHuntHelper(client_mock, self.client_ids, False, self.token) # Run cron flow that executes actual output plugins for _ in test_lib.TestFlowHelper("ProcessHuntResultsCronFlow", token=self.token): pass # Stop the hunt now. hunt_obj.GetRunner().Stop() hunt_obj = aff4.FACTORY.Open(hunt_urn, age=aff4.ALL_TIMES, token=self.token) started, finished, errors = hunt_obj.GetClientsCounts() self.assertEqual(started, 40) self.assertEqual(finished, 40) self.assertEqual(errors, 20) collection = aff4.FACTORY.Open(hunt_urn.Add("Results"), mode="r", token=self.token) self.assertEqual(len(collection), 20) # Due to the limit there should only by 10 messages. self.assertEqual(len(self.email_messages), 10) for msg in self.email_messages: self.assertEqual(msg["address"], email_address) self.assertTrue( "%s got a new result" % hunt_obj.session_id.Add("Results") in msg["title"]) self.assertTrue("fs/os/tmp/evil.txt" in msg["message"]) self.assertTrue("sending of emails will be disabled now" in self.email_messages[-1]["message"])
def testIntegerComparisons(self): """Tests that we can use integer matching rules on the foreman.""" base_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch( 1336480583.077736) boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch( 1336300000.000000) self.SetupTestClientObject(0x11, system="Windows XP", install_time=base_time) self.SetupTestClientObject(0x12, system="Windows 7", install_time=base_time) # This one was installed one week earlier. one_week_ago = base_time - rdfvalue.Duration("1w") self.SetupTestClientObject(0x13, system="Windows 7", install_time=one_week_ago) self.SetupTestClientObject(0x14, system="Windows 7", last_boot_time=boot_time) with utils.Stubber(implementation.GRRHunt, "StartClients", self.StartClients): now = rdfvalue.RDFDatetime.Now() expiration_time = now + rdfvalue.Duration("1h") # Make a new rule rule = foreman_rules.ForemanCondition( creation_time=now, expiration_time=expiration_time, description="Test rule(old)", hunt_name=standard.GenericHunt.__name__, hunt_id="H:111111") # Matches the old client one_hour_ago = base_time - rdfvalue.Duration("1h") rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[ foreman_rules.ForemanClientRule( rule_type=foreman_rules.ForemanClientRule.Type.INTEGER, integer=foreman_rules.ForemanIntegerClientRule( field="INSTALL_TIME", operator=foreman_rules.ForemanIntegerClientRule. Operator.LESS_THAN, value=one_hour_ago.AsSecondsSinceEpoch())) ]) data_store.REL_DB.WriteForemanRule(rule) # Make a new rule rule = foreman_rules.ForemanCondition( creation_time=now, expiration_time=expiration_time, description="Test rule(new)", hunt_name=standard.GenericHunt.__name__, hunt_id="H:222222") # Matches the newer clients rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[ foreman_rules.ForemanClientRule( rule_type=foreman_rules.ForemanClientRule.Type.INTEGER, integer=foreman_rules.ForemanIntegerClientRule( field="INSTALL_TIME", operator=foreman_rules.ForemanIntegerClientRule. Operator.GREATER_THAN, value=one_hour_ago.AsSecondsSinceEpoch())) ]) data_store.REL_DB.WriteForemanRule(rule) # Make a new rule rule = foreman_rules.ForemanCondition( creation_time=now, expiration_time=expiration_time, description="Test rule(eq)", hunt_name=standard.GenericHunt.__name__, hunt_id="H:333333") # Note that this also tests the handling of nonexistent attributes. rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[ foreman_rules.ForemanClientRule( rule_type=foreman_rules.ForemanClientRule.Type.INTEGER, integer=foreman_rules.ForemanIntegerClientRule( field="LAST_BOOT_TIME", operator="EQUAL", value=boot_time.AsSecondsSinceEpoch())) ]) data_store.REL_DB.WriteForemanRule(rule) foreman_obj = foreman.GetForeman() self.clients_started = [] foreman_obj.AssignTasksToClient("C.1000000000000011") foreman_obj.AssignTasksToClient("C.1000000000000012") foreman_obj.AssignTasksToClient("C.1000000000000013") foreman_obj.AssignTasksToClient("C.1000000000000014") # Make sure that the clients ran the correct flows. self.assertEqual(len(self.clients_started), 4) self.assertEqual(self.clients_started[0][1], "C.1000000000000011") self.assertEqual("H:222222", self.clients_started[0][0].Basename()) self.assertEqual(self.clients_started[1][1], "C.1000000000000012") self.assertEqual("H:222222", self.clients_started[1][0].Basename()) self.assertEqual(self.clients_started[2][1], "C.1000000000000013") self.assertEqual("H:111111", self.clients_started[2][0].Basename()) self.assertEqual(self.clients_started[3][1], "C.1000000000000014") self.assertEqual("H:333333", self.clients_started[3][0].Basename())
def testGetClientSummary(self): hostname = "test" system = "Linux" os_release = "12.02" kernel = "3.15-rc2" fqdn = "test.test.com" arch = "amd64" install_time = rdfvalue.RDFDatetime.Now() user = "******" userobj = rdf_client.User(username=user) interface = rdf_client.Interface(ifname="eth0") google_cloud_instance = cloud.GoogleCloudInstance( instance_id="1771384456894610289", zone="projects/123456789733/zones/us-central1-a", project_id="myproject", unique_id="us-central1-a/myproject/1771384456894610289") cloud_instance = cloud.CloudInstance(cloud_type="GOOGLE", google=google_cloud_instance) timestamp = 1 with utils.Stubber(time, "time", lambda: timestamp): with aff4.FACTORY.Create("C.0000000000000000", aff4_grr.VFSGRRClient, mode="rw", token=self.token) as fd: kb = rdf_client.KnowledgeBase() kb.users.Append(userobj) empty_summary = fd.GetSummary() self.assertEqual(empty_summary.client_id, "C.0000000000000000") self.assertFalse(empty_summary.system_info.version) self.assertEqual(empty_summary.timestamp.AsSecondsFromEpoch(), 1) # This will cause TYPE to be written with current time = 101 when the # object is closed timestamp += 100 fd.Set(fd.Schema.HOSTNAME(hostname)) fd.Set(fd.Schema.SYSTEM(system)) fd.Set(fd.Schema.OS_RELEASE(os_release)) fd.Set(fd.Schema.KERNEL(kernel)) fd.Set(fd.Schema.FQDN(fqdn)) fd.Set(fd.Schema.ARCH(arch)) fd.Set(fd.Schema.INSTALL_DATE(install_time)) fd.Set(fd.Schema.KNOWLEDGE_BASE(kb)) fd.Set(fd.Schema.USERNAMES([user])) fd.Set(fd.Schema.INTERFACES([interface])) fd.Set(fd.Schema.CLOUD_INSTANCE(cloud_instance)) with aff4.FACTORY.Open("C.0000000000000000", aff4_grr.VFSGRRClient, mode="rw", token=self.token) as fd: summary = fd.GetSummary() self.assertEqual(summary.system_info.node, hostname) self.assertEqual(summary.system_info.system, system) self.assertEqual(summary.system_info.release, os_release) self.assertEqual(summary.system_info.kernel, kernel) self.assertEqual(summary.system_info.fqdn, fqdn) self.assertEqual(summary.system_info.machine, arch) self.assertEqual(summary.system_info.install_date, install_time) self.assertItemsEqual(summary.users, [userobj]) self.assertItemsEqual(summary.interfaces, [interface]) self.assertFalse(summary.client_info) self.assertEqual(summary.timestamp.AsSecondsFromEpoch(), 101) self.assertEqual(summary.cloud_type, "GOOGLE") self.assertEqual( summary.cloud_instance_id, "us-central1-a/myproject/1771384456894610289")
def Run(self): stats_collector = stats.StatsCollector() stats_collector.RegisterCounterMetric( "sample_counter", docstring="Sample counter metric.") stats_collector.RegisterGaugeMetric( "sample_gauge_value", float, docstring="Sample gauge metric.") stats_collector.RegisterEventMetric( "sample_event", docstring="Sample event metric.") with utils.Stubber(stats, "STATS", stats_collector): for i in range(10): with test_lib.FakeTime(42 + i * 60): stats_collector.IncrementCounter("sample_counter") stats_collector.SetGaugeValue("sample_gauge_value", i * 0.5) stats_collector.RecordEvent("sample_event", 0.42 + 0.5 * i) with aff4.FACTORY.Create( None, aff4_stats_store.StatsStore, mode="w", token=self.token) as stats_store: stats_store.WriteStats(process_id="worker_1") self.Check( "GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name="sample_counter", start=42000000, end=3600000000)) self.Check( "GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name="sample_counter", start=42000000, end=3600000000, rate="1m")) self.Check( "GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name="sample_gauge_value", start=42000000, end=3600000000)) self.Check( "GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name="sample_event", start=42000000, end=3600000000)) self.Check( "GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name="sample_event", start=42000000, end=3600000000, distribution_handling_mode="DH_COUNT"))
def testAlertEmailIsSentWhenClientKilled(self): """Test that client killed messages are handled correctly.""" self.email_messages = [] def SendEmail(address, sender, title, message, **_): self.email_messages.append(dict(address=address, sender=sender, title=title, message=message)) with utils.Stubber(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmail): client = test_lib.CrashClientMock(self.client_id, self.token) for _ in test_lib.TestFlowHelper("FlowWithOneClientRequest", client, client_id=self.client_id, token=self.token, check_flow_errors=False): pass self.assertEqual(len(self.email_messages), 1) email_message = self.email_messages[0] # We expect the email to be sent. self.assertEqual( email_message.get("address", ""), config_lib.CONFIG["Monitoring.alert_email"]) self.assertTrue(str(self.client_id) in email_message["title"]) # Make sure the flow state is included in the email message. for s in ["Flow name", "FlowWithOneClientRequest", "current_state"]: self.assertTrue(s in email_message["message"]) flow_obj = aff4.FACTORY.Open(client.flow_id, age=aff4.ALL_TIMES, token=self.token) self.assertEqual(flow_obj.state.context.state, rdf_flows.Flow.State.ERROR) # Make sure client object is updated with the last crash. client_obj = aff4.FACTORY.Open(self.client_id, token=self.token) crash = client_obj.Get(client_obj.Schema.LAST_CRASH) self.CheckCrash(crash, flow_obj.session_id) # Make sure crashes RDFValueCollections are created and written # into proper locations. First check the per-client crashes collection. client_crashes = sorted( list(aff4.FACTORY.Open( self.client_id.Add("crashes"), aff4_type=collects.PackedVersionedCollection, token=self.token)), key=lambda x: x.timestamp) self.assertTrue(len(client_crashes) >= 1) crash = list(client_crashes)[0] self.CheckCrash(crash, flow_obj.session_id) # Check per-flow crash collection. Check that crash written there is # equal to per-client crash. flow_crashes = sorted( list(flow_obj.GetValuesForAttribute(flow_obj.Schema.CLIENT_CRASH)), key=lambda x: x.timestamp) self.assertEqual(len(flow_crashes), len(client_crashes)) for a, b in zip(flow_crashes, client_crashes): self.assertEqual(a, b) # Check global crash collection. Check that crash written there is # equal to per-client crash. global_crashes = sorted( aff4.FACTORY.Open( aff4.ROOT_URN.Add("crashes"), aff4_type=collects.PackedVersionedCollection, token=self.token), key=lambda x: x.timestamp) self.assertEqual(len(global_crashes), len(client_crashes)) for a, b in zip(global_crashes, client_crashes): self.assertEqual(a, b)
def testNewArtifactLoaded(self): """Simulate a new artifact being loaded into the store via the UI.""" cmd_artifact = """name: "TestCmdArtifact" doc: "Test command artifact for dpkg." sources: - type: "COMMAND" attributes: cmd: "/usr/bin/dpkg" args: ["--list"] labels: [ "Software" ] supported_os: [ "Linux" ] """ no_datastore_artifact = """name: "NotInDatastore" doc: "Test command artifact for dpkg." sources: - type: "COMMAND" attributes: cmd: "/usr/bin/dpkg" args: ["--list"] labels: [ "Software" ] supported_os: [ "Linux" ] """ test_registry = artifact_registry.ArtifactRegistry() test_registry.ClearRegistry() test_registry._dirty = False with utils.Stubber(artifact_registry, "REGISTRY", test_registry): collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token) with self.assertRaises( artifact_registry.ArtifactNotRegisteredError): artifact_registry.REGISTRY.GetArtifact("TestCmdArtifact") with self.assertRaises( artifact_registry.ArtifactNotRegisteredError): artifact_registry.REGISTRY.GetArtifact("NotInDatastore") # Add artifact to datastore but not registry artifact_coll = artifact_registry.ArtifactCollection( rdfvalue.RDFURN("aff4:/artifact_store"), token=self.token) with data_store.DB.GetMutationPool(token=self.token) as pool: for artifact_val in artifact_registry.REGISTRY.ArtifactsFromYaml( cmd_artifact): artifact_coll.Add(artifact_val, mutation_pool=pool) # Add artifact to registry but not datastore for artifact_val in artifact_registry.REGISTRY.ArtifactsFromYaml( no_datastore_artifact): artifact_registry.REGISTRY.RegisterArtifact( artifact_val, source="datastore", overwrite_if_exists=False) # This should succeeded because the artifacts will be reloaded from the # datastore. self.assertTrue( collect_flow._GetArtifactFromName("TestCmdArtifact")) # We registered this artifact with datastore source but didn't # write it into aff4. This simulates an artifact that was # uploaded in the UI then later deleted. We expect it to get # cleared when the artifacts are reloaded from the datastore. with self.assertRaises( artifact_registry.ArtifactNotRegisteredError): artifact_registry.REGISTRY.GetArtifact("NotInDatastore")
def NoAuthorizationChecks(self): return utils.Stubber(api_call_handlers, "API_AUTH_MGR", NullAPIAuthorizationManager())
def testRegister(self): """Create master and register other servers.""" m = master.DataMaster(self.ports[0], self.mock_service) self.assertNotEqual(m, None) self.assertFalse(m.AllRegistered()) servers = [None] for (i, port) in enumerate(self.ports): if i == 0: # Skip master server. continue self.assertFalse(m.AllRegistered()) server = m.RegisterServer(self.host, port) servers.append(server) self.assertNotEqual(server, None) self.assertEqual(server.Address(), self.host) self.assertEqual(server.Port(), port) self.assertEqual(server.Index(), i) self.assertTrue(m.AllRegistered()) # Try to register something that does not exist. self.assertFalse(m.RegisterServer(self.host, 7004)) # Deregister a server. m.DeregisterServer(servers[1]) self.assertFalse(m.AllRegistered()) # Register again. m.RegisterServer(servers[1].Address(), servers[1].Port()) self.assertTrue(m.AllRegistered()) for port in self.ports: for response_sequence in [ [constants.RESPONSE_OK, constants.RESPONSE_SERVER_NOT_AUTHORIZED], [constants.RESPONSE_OK, constants.RESPONSE_SERVER_NOT_ALLOWED], [constants.RESPONSE_OK, constants.RESPONSE_NOT_MASTER_SERVER] ]: response_mocks = [] for response_status in response_sequence: response_mocks.append(MockResponse(response_status)) pool_class = GetMockHTTPConnectionPoolClass(response_mocks) with libutils.Stubber(urllib3.connectionpool, "HTTPConnectionPool", pool_class): m = data_server.StandardDataServer(port, data_server.DataServerHandler) m.handler_cls.NONCE_STORE = auth.NonceStore() self.assertRaises(errors.DataServerError, m._DoRegister) # Ensure two requests have been made. self.assertEqual(len(pool_class.requests), 2) # Ensure the register body is non-empty. self.assertTrue(pool_class.requests[1]["body"]) # Ensure that the register body is a valid rdfvalue. rdf_data_server.DataStoreRegistrationRequest.FromSerializedString( pool_class.requests[1]["body"]) # Ensure the requests are POST requests. self.assertEqual(pool_class.requests[0]["method"], "POST") self.assertEqual(pool_class.requests[1]["method"], "POST") # Ensure the correct URLs are hit according to the API. self.assertEqual(pool_class.requests[0]["url"], "/server/handshake") self.assertEqual(pool_class.requests[1]["url"], "/server/register")
def testFileView(self): """Test the fileview interface.""" # This is ugly :( Django gets confused when you import in the wrong order # though and fileview imports the Django http module so we have to delay # import until the Django server is properly set up. # pylint: disable=g-import-not-at-top from grr.gui.plugins import fileview # pylint: enable=g-import-not-at-top # Set up multiple version for an attribute on the client for tests. with self.ACLChecksDisabled(): for fake_time, hostname in [(1333788833, "HostnameV1"), (1333888833, "HostnameV2"), (1333988833, "HostnameV3")]: with test_lib.FakeTime(fake_time): client = aff4.FACTORY.Open(u"C.0000000000000001", mode="rw", token=self.token) client.Set(client.Schema.HOSTNAME(hostname)) client.Close() self.Open("/") self.Type("client_query", "0001") self.Click("client_query_submit") self.WaitUntilEqual(u"C.0000000000000001", self.GetText, "css=span[type=subject]") # Choose client 1 self.Click("css=td:contains('0001')") # Go to Browse VFS self.Click("css=a:contains('Browse Virtual Filesystem')") # Test the historical view for AFF4 elements. self.Click("css=*[attribute=HOSTNAME] > ins") self.WaitUntil(self.AllTextsPresent, ["HostnameV1", "HostnameV2", "HostnameV3"]) self.Click("css=*[attribute=HOSTNAME] > ins") self.WaitUntilNot(self.IsTextPresent, "HostnameV1") self.WaitUntilNot(self.IsTextPresent, "HostnameV2") self.Click("css=#_fs ins.jstree-icon") self.Click("css=#_fs-os ins.jstree-icon") self.Click("css=#_fs-os-c ins.jstree-icon") # Test file versioning. self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads") self.Click("link=Downloads") # Verify that we have the latest version in the table by default self.assertTrue("2012-04-09 16:27:13" in self.GetText( "css=tr:contains(\"a.txt\")")) # Click on the row. self.Click("css=tr:contains(\"a.txt\")") self.WaitUntilContains("a.txt @ 2012-04-09", self.GetText, "css=div#main_rightBottomPane h3") # Check the data in this file. self.Click("css=#TextView") self.WaitUntilContains("Goodbye World", self.GetText, "css=div#text_viewer_data_content") downloaded_files = [] def FakeDownload(unused_self, request, _): aff4_path = request.REQ.get("aff4_path") age = rdfvalue.RDFDatetime( request.REQ.get("age")) or aff4.NEWEST_TIME downloaded_files.append((aff4_path, age)) return fileview.http.HttpResponse( content="<script>window.close()</script>") with utils.Stubber(fileview.DownloadView, "Download", FakeDownload): # Try to download the file. self.Click("css=#Download") self.WaitUntil(self.IsTextPresent, "As downloaded on 2012-04-09 16:27:13") self.Click("css=button:contains(\"Download\")") # Click on the version selector. self.Click("css=tr:contains(\"a.txt\") img.version-selector") self.WaitUntilContains("Versions of", self.GetText, "css=.version-selector-dialog h4") # Select the previous version. self.Click("css=td:contains(\"2012-04-07\")") # Now we should have a different time. self.WaitUntil(self.IsTextPresent, "As downloaded on 2012-04-07 08:53:53") self.Click("css=button:contains(\"Download\")") self.WaitUntil(self.IsElementPresent, "css=#TextView") self.WaitUntil(lambda: len(downloaded_files) == 2) # Both files should be the same... self.assertEqual(downloaded_files[0][0], u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt") self.assertEqual(downloaded_files[1][0], u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt") # But from different times. self.assertEqual(downloaded_files[0][1], 1333988833000000) self.assertEqual(downloaded_files[1][1], 1333788833000000) self.Click("css=#TextView") # Make sure the file content has changed. This version has "Hello World" in # it. self.WaitUntilContains("Hello World", self.GetText, "css=div#text_viewer_data_content") # Some more unicode testing. self.Click(u"css=tr:contains(\"中.txt\")") self.Click("css=#Download") self.WaitUntil(self.IsTextPresent, u"fs/os/c/Downloads/中国新闻网新闻中.txt") # Test the hex viewer. self.Click("css=#_fs-os-proc ins.jstree-icon") self.Click("css=#_fs-os-proc-10 a") self.Click("css=span[type=subject]:contains(\"cmdline\")") target_aff4_path = "aff4:/C.0000000000000001/fs/os/proc/10/cmdline" self.Click("css=[state-aff4_path='%s'] > li > #HexView" % target_aff4_path) for i, value in enumerate( "6c 73 00 68 65 6c 6c 6f 20 77 6f 72 6c 64 27 00 2d 6c".split( " ")): self.WaitUntilEqual(value, self.GetText, "css=#hex_area tr:first td:nth(%d)" % i) for i, value in enumerate( "l s . h e l l o w o r l d ' . - l".split(" ")): self.WaitUntilEqual(value, self.GetText, "css=#data_area tr:first td:nth(%d)" % i) self.Click("css=a[renderer=\"AFF4Stats\"]") # Navigate to the bin C.0000000000000001 directory self.Click("link=bin C.0000000000000001") # Filter the table for bash (should match both bash and rbash) self.WaitUntil(self.IsElementPresent, "css=td:contains('bash')") self.Click("css=th:contains('Name') img") self.Type("css=.sort-dialog input[type=text]", "bash", end_with_enter=True) self.WaitUntilEqual("rbash", self.GetText, "css=tr:nth(2) span") self.assertEqual( 2, self.GetCssCount("css=#main_rightTopPane tbody > tr")) self.assertEqual("bash", self.GetText("css=tr:nth(1) span")) self.assertEqual("rbash", self.GetText("css=tr:nth(2) span")) # Check that the previous search test is still available in the form. self.Click("css=th:contains('Name') img") self.assertEqual("bash", self.GetValue("css=.sort-dialog input")) # If we anchor cat at the start should only filter one. self.Type("css=.sort-dialog input[type=text]", "^cat", end_with_enter=True) self.WaitUntilEqual("cat", self.GetText, "css=tr:nth(1) span") self.assertEqual( 1, self.GetCssCount("css=#main_rightTopPane tbody > tr")) self.Click("css=tr:nth(1)") self.WaitUntilContains( "aff4:/C.0000000000000001/fs/os/c/bin C.0000000000000001/cat", self.GetText, "css=.tab-content h3") self.WaitUntil(self.IsTextPresent, "1026267") ## st_inode. # Lets download it. self.Click("Download") self.Click("css=button:contains(\"Get a new Version\")") self.Click("path_0") self.WaitUntilEqual("fs", self.GetText, "css=tr td span:contains(fs)") self.Click("Stats") self.WaitUntilContains("aff4:/C.0000000000000001", self.GetText, "css=.tab-content h3") # Grab the root directory again - should produce an Interrogate flow. self.Click("css=button[id^=refresh]") # Go to the flow management screen. self.Click("css=a:contains('Manage launched flows')") # For the client update, 2 flows have to be issued: UpdateVFSFile and # Interrogate. UpdateVFSFile triggers VFSGRRClient.Update() method which # triggers Interrogate. self.WaitUntilEqual("Interrogate", self.GetText, "//table/tbody/tr[1]/td[3]") self.WaitUntilEqual("UpdateVFSFile", self.GetText, "//table/tbody/tr[2]/td[3]") self.Click("//table/tbody/tr[2]/td[3]") self.WaitUntilEqual( "aff4:/C.0000000000000001", self.GetText, "css=table > tbody td.proto_key:contains(\"Vfs file urn\") " "~ td.proto_value") # Check that UpdateVFSFile is called for the cat file. # During the test this file is VFSMemoryFile, so its' Update method does # nothing, therefore UpdateVFSFile won't issue any other flows. self.WaitUntilEqual("UpdateVFSFile", self.GetText, "//table/tbody/tr[3]/td[3]") self.Click("//table/tbody/tr[3]/td[3]") self.WaitUntilContains( "cat", self.GetText, "css=table > tbody td.proto_key:contains(\"Vfs file urn\") " "~ td.proto_value")
def testDeleteHuntFlow(self): # We'll need two users for this test. self.CreateUser("user1") token1 = access_control.ACLToken(username="******", reason="testing") self.CreateUser("user2") token2 = access_control.ACLToken(username="******", reason="testing") manager = user_managers.FullAccessControlManager() with utils.Stubber(data_store.DB, "security_manager", manager): # Let user1 create a hunt and delete it, this should work. hunt = self._CreateHunt(token1.SetUID()) aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1) flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow", token=token1, hunt_urn=hunt.urn) self._CheckHuntIsDeleted(hunt.urn) # Let user1 create a hunt and user2 delete it, this should fail. hunt = self._CreateHunt(token1.SetUID()) aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1) with self.assertRaises(access_control.UnauthorizedAccess): flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow", token=token2, hunt_urn=hunt.urn) # Hunt is still there. aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1) # If user2 gets an approval, deletion is ok though. self.GrantHuntApproval(hunt.urn, token=token2) flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow", token=token2, hunt_urn=hunt.urn) self._CheckHuntIsDeleted(hunt.urn) # Let user1 create a hunt and run it. We are not allowed to delete # running hunts. hunt = self._CreateHunt(token1.SetUID()) hunt.Run() hunt.Flush() aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1) with self.assertRaises(RuntimeError): flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow", token=token1, hunt_urn=hunt.urn) # The same is true if the hunt was scheduled on at least one client. hunt = self._CreateHunt(token1.SetUID()) hunt.Set(hunt.Schema.CLIENT_COUNT(1)) hunt.Flush() aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1) with self.assertRaises(RuntimeError): flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow", token=token1, hunt_urn=hunt.urn)
def testCheckPermissionsReturnsTrueIfGroupWasAuthorized(self): self.auth_manager.DenyAll("subject-bar") with utils.Stubber(self.group_access_manager, "MemberOfAuthorizedGroup", lambda *args: True): self.assertTrue( self.auth_manager.CheckPermissions("user-bar", "subject-bar"))
def testIntegerComparisons(self): """Tests that we can use integer matching rules on the foreman.""" base_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch( 1336480583.077736) boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch( 1336300000.000000) self.SetupClient(0x11, system="Windows XP", install_time=base_time) self.SetupClient(0x12, system="Windows 7", install_time=base_time) # This one was installed one week earlier. one_week_ago = base_time - rdfvalue.Duration("1w") self.SetupClient(0x13, system="Windows 7", install_time=one_week_ago) self.SetupClient(0x14, system="Windows 7", last_boot_time=boot_time) with utils.Stubber(flow.GRRFlow, "StartFlow", self.StartFlow): # Now setup the filters now = rdfvalue.RDFDatetime.Now() expires = now + rdfvalue.Duration("1h") foreman_obj = foreman.GetForeman(token=self.token) # Make a new rule rule = foreman_rules.ForemanRule(created=now, expires=expires, description="Test rule(old)") # Matches the old client one_hour_ago = base_time - rdfvalue.Duration("1h") rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[ foreman_rules.ForemanClientRule( rule_type=foreman_rules.ForemanClientRule.Type.INTEGER, integer=foreman_rules.ForemanIntegerClientRule( field="INSTALL_TIME", operator=foreman_rules.ForemanIntegerClientRule. Operator.LESS_THAN, value=one_hour_ago.AsSecondsSinceEpoch())) ]) old_flow = "Test flow for old clients" # Will run Test Flow rule.actions.Append(flow_name=old_flow, argv=rdf_protodict.Dict(dict(foo="bar"))) # Clear the rule set and add the new rule to it. rule_set = foreman_obj.Schema.RULES() rule_set.Append(rule) # Make a new rule rule = foreman_rules.ForemanRule(created=now, expires=expires, description="Test rule(new)") # Matches the newer clients rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[ foreman_rules.ForemanClientRule( rule_type=foreman_rules.ForemanClientRule.Type.INTEGER, integer=foreman_rules.ForemanIntegerClientRule( field="INSTALL_TIME", operator=foreman_rules.ForemanIntegerClientRule. Operator.GREATER_THAN, value=one_hour_ago.AsSecondsSinceEpoch())) ]) new_flow = "Test flow for newer clients" # Will run Test Flow rule.actions.Append(flow_name=new_flow, argv=rdf_protodict.Dict(dict(foo="bar"))) rule_set.Append(rule) # Make a new rule rule = foreman_rules.ForemanRule(created=now, expires=expires, description="Test rule(eq)") # Note that this also tests the handling of nonexistent attributes. rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[ foreman_rules.ForemanClientRule( rule_type=foreman_rules.ForemanClientRule.Type.INTEGER, integer=foreman_rules.ForemanIntegerClientRule( field="LAST_BOOT_TIME", operator="EQUAL", value=boot_time.AsSecondsSinceEpoch())) ]) eq_flow = "Test flow for LAST_BOOT_TIME" rule.actions.Append(flow_name=eq_flow, argv=rdf_protodict.Dict(dict(foo="bar"))) rule_set.Append(rule) # Assign it to the foreman foreman_obj.Set(foreman_obj.Schema.RULES, rule_set) foreman_obj.Close() self.clients_launched = [] foreman_obj.AssignTasksToClient("C.1000000000000011") foreman_obj.AssignTasksToClient("C.1000000000000012") foreman_obj.AssignTasksToClient("C.1000000000000013") foreman_obj.AssignTasksToClient("C.1000000000000014") # Make sure that the clients ran the correct flows. self.assertEqual(len(self.clients_launched), 4) self.assertEqual(self.clients_launched[0][0], rdf_client.ClientURN("C.1000000000000011")) self.assertEqual(self.clients_launched[0][1], new_flow) self.assertEqual(self.clients_launched[1][0], rdf_client.ClientURN("C.1000000000000012")) self.assertEqual(self.clients_launched[1][1], new_flow) self.assertEqual(self.clients_launched[2][0], rdf_client.ClientURN("C.1000000000000013")) self.assertEqual(self.clients_launched[2][1], old_flow) self.assertEqual(self.clients_launched[3][0], rdf_client.ClientURN("C.1000000000000014")) self.assertEqual(self.clients_launched[3][1], eq_flow)
def __init__(self): self.stubber = utils.Stubber(report_plugins.REGISTRY, "plugins", { "FooReportPlugin": FooReportPlugin, "BarReportPlugin": BarReportPlugin })
def testCorruption(self): """Simulate corruption of the http payload.""" self.corruptor_field = None def Corruptor(url="", data=None, **kwargs): """Futz with some of the fields.""" comm_cls = rdf_flows.ClientCommunication if data is not None: self.client_communication = comm_cls.FromSerializedString(data) else: self.client_communication = comm_cls(None) if self.corruptor_field and "server.pem" not in url: orig_str_repr = self.client_communication.SerializeToString() field_data = getattr(self.client_communication, self.corruptor_field) if hasattr(field_data, "SerializeToString"): # This converts encryption keys to a string so we can corrupt them. field_data = field_data.SerializeToString() modified_data = array.array("c", field_data) offset = len(field_data) / 2 modified_data[offset] = chr((ord(field_data[offset]) % 250) + 1) setattr(self.client_communication, self.corruptor_field, modified_data.tostring()) # Make sure we actually changed the data. self.assertNotEqual(field_data, modified_data) mod_str_repr = self.client_communication.SerializeToString() self.assertEqual(len(orig_str_repr), len(mod_str_repr)) differences = [ True for x, y in zip(orig_str_repr, mod_str_repr) if x != y ] self.assertEqual(len(differences), 1) data = self.client_communication.SerializeToString() return self.UrlMock(url=url, data=data, **kwargs) with utils.Stubber(requests, "request", Corruptor): self.SendToServer() status = self.client_communicator.RunOnce() self.assertEqual(status.code, 200) for field in ["packet_iv", "encrypted"]: # Corrupting each field should result in HMAC verification errors. self.corruptor_field = field self.SendToServer() status = self.client_communicator.RunOnce() self.assertEqual(status.code, 500) self.assertTrue( "HMAC verification failed" in str(self.last_urlmock_error)) # Corruption of these fields will likely result in RSA errors, since we do # the RSA operations before the HMAC verification (in order to recover the # hmac key): for field in ["encrypted_cipher", "encrypted_cipher_metadata"]: # Corrupting each field should result in HMAC verification errors. self.corruptor_field = field self.SendToServer() status = self.client_communicator.RunOnce() self.assertEqual(status.code, 500)
def Run(self): with utils.Stubber(output_plugin.OutputPlugin, "classes", { "EmailOutputPlugin": email_plugin.EmailOutputPlugin, "CSVOutputPlugin": csv_plugin.CSVOutputPlugin }): self.Check("GET", "/api/output-plugins/all")
def testGlob(self): """Test that glob works properly.""" # Add some usernames we can interpolate later. client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) kb = client.Get(client.Schema.KNOWLEDGE_BASE) kb.MergeOrAddUser(rdf_client.User(username="******")) kb.MergeOrAddUser(rdf_client.User(username="******")) client.Set(kb) client.Close() client_mock = action_mocks.ActionMock("Find", "StatFile") # This glob selects all files which start with the username on this system. paths = [ os.path.join(self.base_path, "%%Users.username%%*"), os.path.join(self.base_path, "VFSFixture/var/*/wtmp") ] # Set iterator really low to force iteration. with utils.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2): for _ in test_lib.TestFlowHelper( "Glob", client_mock, client_id=self.client_id, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token, sync=False, check_flow_errors=False): pass output_path = self.client_id.Add("fs/os").Add( self.base_path.replace("\\", "/")) children = [] fd = aff4.FACTORY.Open(output_path, token=self.token) for child in fd.ListChildren(): filename = child.Basename() if filename != "VFSFixture": children.append(filename) expected = [ filename for filename in os.listdir(self.base_path) if filename.startswith("test") or filename.startswith("syslog") ] self.assertTrue([x for x in expected if x.startswith("test")], "Need a file starting with 'test'" " in test_data for this test!") self.assertTrue([x for x in expected if x.startswith("syslog")], "Need a file starting with 'syslog'" " in test_data for this test!") self.assertItemsEqual(expected, children) children = [] fd = aff4.FACTORY.Open(output_path.Add("VFSFixture/var/log"), token=self.token) for child in fd.ListChildren(): children.append(child.Basename()) self.assertItemsEqual(children, ["wtmp"])
def testBlockingTasks(self): # The pool starts off with the minimum number of threads. self.assertEqual(len(self.test_pool), self.NUMBER_OF_THREADS) done_event = threading.Event() self.lock = threading.Lock() res = [] def Block(done): done.wait() def Insert(list_obj, element): with self.lock: list_obj.append(element) # Ensure that the thread pool is able to add the correct number of threads # deterministically. with utils.Stubber(self.test_pool, "CPUUsage", lambda: 0): # Schedule the maximum number of threads of blocking tasks and the same of # insert tasks. The threads are now all blocked, and the inserts are # waiting in the queue. for _ in range(self.MAXIMUM_THREADS): self.test_pool.AddTask(Block, (done_event, ), "Blocking") # Wait until the threadpool picks up the task. self.WaitUntil( lambda: self.test_pool.busy_threads == self.NUMBER_OF_THREADS) # Now there are minimum number of threads active and the rest are sitting # on the queue. self.assertEqual(self.test_pool.pending_tasks, self.MAXIMUM_THREADS - self.NUMBER_OF_THREADS) # Now as we push these tasks on the queue, new threads will be created and # they will receive the blocking tasks from the queue. for i in range(self.MAXIMUM_THREADS): self.test_pool.AddTask(Insert, (res, i), "Insert", blocking=True, inline=False) # There should be 20 workers created and they should consume all the # blocking tasks. self.WaitUntil( lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS) # No Insert tasks are running yet. self.assertEqual(res, []) # There are 20 tasks waiting on the queue. self.assertEqual(self.test_pool.pending_tasks, self.MAXIMUM_THREADS) # Inserting more tasks than the queue can hold should lead to processing # the tasks inline. This effectively causes these tasks to skip over the # tasks which are waiting in the queue. for i in range(10, 20): self.test_pool.AddTask(Insert, (res, i), "Insert", inline=True) res.sort() self.assertEqual(res, range(10, 20)) # This should release all the busy tasks. It will also cause the workers # to process all the Insert tasks in the queue. done_event.set() self.test_pool.Join() # Now the rest of the tasks should have been processed as well. self.assertEqual(sorted(res[10:]), range(20))
def testNoValidStatusRaceIsResolved(self): # This tests for the regression of a long standing race condition we saw # where notifications would trigger the reading of another request that # arrives later but wasn't completely written to the database yet. # Timestamp based notification handling should eliminate this bug. # We need a random flow object for this test. session_id = flow.GRRFlow.StartFlow(client_id=self.client_id, flow_name="WorkerSendingTestFlow", token=self.token) worker_obj = worker.GRRWorker(token=self.token) manager = queue_manager.QueueManager(token=self.token) manager.DeleteNotification(session_id) manager.Flush() # We have a first request that is complete (request_id 1, response_id 1). self.SendResponse(session_id, "Response 1") # However, we also have request #2 already coming in. The race is that # the queue manager might write the status notification to # session_id/state as "status:00000002" but not the status response # itself yet under session_id/state/request:00000002 request_id = 2 response_id = 1 flow_manager = queue_manager.QueueManager(token=self.token) flow_manager.FreezeTimestamp() flow_manager.QueueResponse( rdf_flows.GrrMessage( source=self.client_id, session_id=session_id, payload=rdf_protodict.DataBlob(string="Response 2"), request_id=request_id, response_id=response_id)) status = rdf_flows.GrrMessage( source=self.client_id, session_id=session_id, payload=rdf_flows.GrrStatus( status=rdf_flows.GrrStatus.ReturnedStatus.OK), request_id=request_id, response_id=response_id + 1, type=rdf_flows.GrrMessage.Type.STATUS) # Now we write half the status information. data_store.DB.StoreRequestsAndResponses(new_responses=[(status, None)]) # We make the race even a bit harder by saying the new notification gets # written right before the old one gets deleted. If we are not careful here, # we delete the new notification as well and the flow becomes stuck. def WriteNotification(self, arg_session_id, start=None, end=None): if arg_session_id == session_id: flow_manager.QueueNotification(session_id=arg_session_id) flow_manager.Flush() self.DeleteNotification.old_target(self, arg_session_id, start=start, end=end) with utils.Stubber(queue_manager.QueueManager, "DeleteNotification", WriteNotification): # This should process request 1 but not touch request 2. worker_obj.RunOnce() worker_obj.thread_pool.Join() flow_obj = aff4.FACTORY.Open(session_id, token=self.token) self.assertFalse(flow_obj.context.backtrace) self.assertNotEqual(flow_obj.context.state, rdf_flows.FlowContext.State.ERROR) request_data = data_store.DB.ReadResponsesForRequestId(session_id, 2) request_data.sort(key=lambda msg: msg.response_id) self.assertEqual(len(request_data), 2) # Make sure the status and the original request are still there. self.assertEqual(request_data[0].args_rdf_name, "DataBlob") self.assertEqual(request_data[1].args_rdf_name, "GrrStatus") # But there is nothing for request 1. request_data = data_store.DB.ReadResponsesForRequestId(session_id, 1) self.assertEqual(request_data, []) # The notification for request 2 should have survived. with queue_manager.QueueManager(token=self.token) as manager: notifications = manager.GetNotifications(queues.FLOWS) self.assertEqual(len(notifications), 1) notification = notifications[0] self.assertEqual(notification.session_id, session_id) self.assertEqual(notification.timestamp, flow_manager.frozen_timestamp) self.assertEqual(RESULTS, ["Response 1"]) # The last missing piece of request 2 is the actual status message. flow_manager.QueueResponse(status) flow_manager.Flush() # Now make sure request 2 runs as expected. worker_obj.RunOnce() worker_obj.thread_pool.Join() self.assertEqual(RESULTS, ["Response 1", "Response 2"])
def testPartialRead(self): with utils.Stubber(memory, "win32file", Win32FileMock()): super(TestWindowsMemory, self).testPartialRead()