def testGrep(self):
class MockCallFlow(object):
def CallFlow(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
mock_call_flow = MockCallFlow()
with utils.Stubber(collectors.ArtifactCollectorFlow, "CallFlow",
mock_call_flow.CallFlow):
collect_flow = collectors.ArtifactCollectorFlow(None,
token=self.token)
collect_flow.args = mock.Mock()
collect_flow.args.ignore_interpolation_errors = False
collect_flow.state.Register("knowledge_base",
rdf_client.KnowledgeBase())
collect_flow.current_artifact_name = "blah"
collect_flow.state.knowledge_base.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******"))
collect_flow.state.knowledge_base.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******"))
collector = artifact_registry.ArtifactSource(
type=artifact_registry.ArtifactSource.SourceType.GREP,
attributes={
"paths": ["/etc/passwd"],
"content_regex_list": [r"^a%%users.username%%b$"]
})
collect_flow.Grep(collector, rdf_paths.PathSpec.PathType.TSK)
conditions = mock_call_flow.kwargs["conditions"]
self.assertEqual(len(conditions), 1)
regexes = conditions[0].contents_regex_match.regex.SerializeToString()
self.assertItemsEqual(regexes.split("|"),
["(^atest1b$)", "(^atest2b$)"])
self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testClientStatsCollectionHappensEveryMinuteWhenClientIsBusy(self):
"""Tests that client stats are collected more often when client is busy."""
now = 1000000
# Pretend we have already sent stats.
self.client_communicator.client_worker.last_stats_sent_time = (
rdfvalue.RDFDatetime().FromSecondsFromEpoch(now))
self.client_communicator.client_worker._is_active = True
with test_lib.FakeTime(now):
self.client_communicator.client_worker.CheckStats()
runs = []
with utils.Stubber(admin.GetClientStatsAuto, "Run",
lambda cls, _: runs.append(1)):
# No stats collection after 30 seconds.
with test_lib.FakeTime(now + 30):
self.client_communicator.client_worker.CheckStats()
self.assertEqual(len(runs), 0)
# Let 61 seconds pass.
with test_lib.FakeTime(now + 61):
self.client_communicator.client_worker.CheckStats()
# This time the client should collect stats.
self.assertEqual(len(runs), 1)
# No stats collection within one minute from the last time.
with test_lib.FakeTime(now + 61 + 59):
self.client_communicator.client_worker.CheckStats()
self.assertEqual(len(runs), 1)
# Stats collection happens as more than one minute has passed since the
# last one.
with test_lib.FakeTime(now + 61 + 61):
self.client_communicator.client_worker.CheckStats()
self.assertEqual(len(runs), 2)
def Run(self):
stats_collector = stats.StatsCollector()
stats_collector.RegisterCounterMetric(
"sample_counter", docstring="Sample counter metric.")
stats_collector.RegisterGaugeMetric(
"sample_gauge_value", float, docstring="Sample gauge metric.")
stats_collector.RegisterEventMetric(
"sample_event", docstring="Sample event metric.")
with utils.Stubber(stats, "STATS", stats_collector):
for i in range(10):
with test_lib.FakeTime(42 + i * 60):
stats_collector.IncrementCounter("sample_counter")
stats_collector.SetGaugeValue("sample_gauge_value", i * 0.5)
stats_collector.RecordEvent("sample_event", 0.42 + 0.5 * i)
with aff4.FACTORY.Create(
None, "StatsStore", mode="w", token=self.token) as stats_store:
stats_store.WriteStats(process_id="worker_1", sync=True)
self.Check("GET", "/api/stats/store/WORKER/metrics/sample_counter?"
"start=42000000&end=3600000000")
self.Check("GET", "/api/stats/store/WORKER/metrics/sample_counter?"
"start=42000000&end=3600000000&rate=1m")
self.Check("GET", "/api/stats/store/WORKER/metrics/sample_gauge_value?"
"start=42000000&end=3600000000")
self.Check("GET", "/api/stats/store/WORKER/metrics/sample_event?"
"start=42000000&end=3600000000")
self.Check("GET", "/api/stats/store/WORKER/metrics/sample_event?"
"start=42000000&end=3600000000&"
"distribution_handling_mode=DH_COUNT")
def testBreakGlass(self):
"""Test the breakglass mechanism."""
client_id = rdfvalue.ClientURN("C.%016X" % 0)
urn = client_id.Add("/fs/os/c")
self.assertRaises(access_control.UnauthorizedAccess, aff4.FACTORY.Open, urn,
token=self.token)
# We expect to receive an email about this
email = {}
def SendEmail(to, from_user, subject, message, **_):
email["to"] = to
email["from_user"] = from_user
email["subject"] = subject
email["message"] = message
with utils.Stubber(email_alerts, "SendEmail", SendEmail):
flow.GRRFlow.StartFlow(
client_id=client_id, flow_name="BreakGlassGrantClientApprovalFlow",
token=self.token, reason=self.token.reason)
# Reset the emergency state of the token.
self.token.is_emergency = False
# This access is using the emergency_access granted, so we expect the
# token to be tagged as such.
aff4.FACTORY.Open(urn, token=self.token)
self.assertEqual(email["to"],
config_lib.CONFIG["Monitoring.emergency_access_email"])
self.assert_(self.token.username in email["message"])
self.assertEqual(email["from_user"], self.token.username)
# Make sure the token is tagged as an emergency token:
self.assertEqual(self.token.is_emergency, True)
def testProcessHuntResultsCronFlowDoesNotAbortsIfRunningInTime(self):
self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 0)
test = [0]
def TimeStub():
test[0] += 1e-6
return test[0]
with utils.Stubber(time, "time", TimeStub):
self.StartHunt(output_plugins=[
output_plugin.OutputPluginDescriptor(
plugin_name="LongRunningDummyHuntOutputPlugin")
])
self.AssignTasksToClients()
self.RunHunt(failrate=-1)
# LongRunningDummyHuntOutputPlugin will set the time to 100s on the first
# run, which will effectively mean that it's running in time.
self.ProcessHuntOutputPlugins(
batch_size=1, max_running_time=rdfvalue.Duration("101s"))
# In normal conditions, there should be 10 results generated.
self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 10)
def testEmailCronJobApprovalRequestLinkLeadsToACorrectPage(self):
with self.ACLChecksDisabled():
cronjobs.ScheduleSystemCronFlows(
names=[cron_system.OSBreakDown.__name__], token=self.token)
cronjobs.CRON_MANAGER.DisableJob(
rdfvalue.RDFURN("aff4:/cron/OSBreakDown"))
messages_sent = []
def SendEmailStub(unused_from_user, unused_to_user, unused_subject,
message, **unused_kwargs):
messages_sent.append(message)
# Request client approval, it will trigger an email message.
with utils.Stubber(email_alerts.EMAIL_ALERTER, "SendEmail",
SendEmailStub):
flow.GRRFlow.StartFlow(
flow_name="RequestCronJobApprovalFlow",
reason="Please please let me",
subject_urn="aff4:/cron/OSBreakDown",
approver=self.token.username,
token=access_control.ACLToken(username="******",
reason="test"))
self.assertEqual(len(messages_sent), 1)
# Extract link from the message text and open it.
m = re.search(r"href='(.+?)'", messages_sent[0], re.MULTILINE)
link = urlparse.urlparse(m.group(1))
self.Open(link.path + "?" + link.query + "#" + link.fragment)
# Check that requestor's username and reason are correctly displayed.
self.WaitUntil(self.IsTextPresent, "iwantapproval")
self.WaitUntil(self.IsTextPresent, "Please please let me")
# Check that host information is displayed.
self.WaitUntil(self.IsTextPresent, "OSBreakDown")
self.WaitUntil(self.IsTextPresent, "Periodicity")
def testWrongOwnerGetsFixed(self):
def mystat(filename):
stat_info = os.lstat.old_target(filename)
stat_list = list(stat_info)
# Adjust the UID.
stat_list[4] += 1
return posix.stat_result(stat_list)
# Place a malicious file in the temp dir. This needs to be deleted
# before we can use the temp dir.
fd = tempfiles.CreateGRRTempFile(filename="maliciousfile", mode="wb")
fd.close()
self.assertTrue(os.path.exists(fd.name))
with utils.Stubber(os, "lstat", mystat):
fd2 = tempfiles.CreateGRRTempFile(filename="temptemp", mode="wb")
fd2.close()
# Old file is gone.
self.assertFalse(os.path.exists(fd.name))
# Cleanup.
tempfiles.DeleteGRRTempFile(fd2.name)
def testIndexIsUsedWhenRandomAccessIsUsed(self):
with utils.MultiStubber(
(aff4.PackedVersionedCollection, "COMPACTION_BATCH_SIZE", 100),
(aff4.PackedVersionedCollection, "INDEX_INTERVAL", 1)):
with aff4.FACTORY.Create(self.collection_urn, "PackedVersionedCollection",
mode="w", token=self.token):
pass
for i in range(20):
with aff4.FACTORY.Open(
self.collection_urn, "PackedVersionedCollection",
mode="w", token=self.token) as fd:
fd.Add(rdf_flows.GrrMessage(request_id=i))
with aff4.FACTORY.OpenWithLock(
self.collection_urn, "PackedVersionedCollection",
token=self.token) as fd:
fd.Compact()
collection = aff4.FACTORY.Open(self.collection_urn, token=self.token)
item_size = collection.fd.size / len(collection)
# There's no seek expected for the first element
for i in range(1, 20):
seek_ops = []
old_seek = collection.fd.Seek
def SeekStub(offset):
seek_ops.append(offset) # pylint: disable=cell-var-from-loop
old_seek(offset) # pylint: disable=cell-var-from-loop
# Check that the stream is seeked to a correct byte offset on every
# GenerateItems() call with an offset specified.
with utils.Stubber(collection.fd, "Seek", SeekStub):
_ = list(collection.GenerateItems(offset=i))
self.assertListEqual([item_size * i], seek_ops)
def Run(self):
config_obj = config_lib.GrrConfigManager()
config_obj.DEFINE_string("SectionFoo.sample_string_option", "",
"Sample string option.")
config_obj.DEFINE_string("Mysql.database_password", "", "Secret password.")
# This has to be defined as http_api.HttpRequestHandler.HandleRequest
# depends on it and regression data won't get rendered without
# this config option defined.
config_obj.DEFINE_string("AdminUI.debug_impersonate_user", None, "")
config = """
SectionBar.sample_string_option: "%(sAmPlE|lower)"
Mysql.database_password: "******"
"""
config_lib.LoadConfig(
config_obj,
config_fd=StringIO.StringIO(config),
parser=config_lib.YamlParser)
with utils.Stubber(config_lib, "CONFIG", config_obj):
self.Check("GET", "/api/config/SectionFoo.sample_string_option")
self.Check("GET", "/api/config/Mysql.database_password")
self.Check("GET", "/api/config/NonExistingOption")
def testVersionDropDownChangesFileContentAndDownloads(self):
"""Test the fileview interface."""
# Set up multiple version for an attribute on the client for tests.
with self.ACLChecksDisabled():
for fake_time, hostname in [(gui_test_lib.TIME_0, "HostnameV1"),
(gui_test_lib.TIME_1, "HostnameV2"),
(gui_test_lib.TIME_2, "HostnameV3")]:
with test_lib.FakeTime(fake_time):
client = aff4.FACTORY.Open(u"C.0000000000000001",
mode="rw",
token=self.token)
client.Set(client.Schema.HOSTNAME(hostname))
client.Close()
self.Open("/")
self.Type("client_query", "C.0000000000000001")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000001", self.GetText,
"css=span[type=subject]")
# Choose client 1.
self.Click("css=td:contains('0001')")
# Go to Browse VFS.
self.Click("css=a[grrtarget='client.vfs']")
self.Click("css=#_fs i.jstree-icon")
self.Click("css=#_fs-os i.jstree-icon")
self.Click("css=#_fs-os-c i.jstree-icon")
# Test file versioning.
self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads")
self.Click("link=Downloads")
# Verify that we have the latest version in the table by default.
self.assertTrue(
gui_test_lib.DateString(gui_test_lib.TIME_2) in self.GetText(
"css=tr:contains(\"a.txt\")"))
# Click on the row.
self.Click("css=tr:contains(\"a.txt\")")
self.WaitUntilContains("a.txt", self.GetText,
"css=div#main_bottomPane h1")
self.WaitUntilContains("HEAD", self.GetText,
"css=.version-dropdown > option[selected]")
self.WaitUntilContains(gui_test_lib.DateString(gui_test_lib.TIME_2),
self.GetText,
"css=.version-dropdown > option:nth(1)")
# Check the data in this file.
self.Click("css=li[heading=TextView]")
self.WaitUntilContains("Goodbye World", self.GetText,
"css=div.monospace pre")
downloaded_files = []
def FakeDownloadHandle(unused_self, args, token=None):
_ = token # Avoid unused variable linter warnings.
aff4_path = args.client_id.ToClientURN().Add(args.file_path)
age = args.timestamp or aff4.NEWEST_TIME
downloaded_files.append((aff4_path, age))
return api_call_handler_base.ApiBinaryStream(
filename=aff4_path.Basename(), content_generator=xrange(42))
with utils.Stubber(api_vfs.ApiGetFileBlobHandler, "Handle",
FakeDownloadHandle):
# Try to download the file.
self.Click("css=li[heading=Download]")
self.WaitUntilContains(
gui_test_lib.DateTimeString(gui_test_lib.TIME_2), self.GetText,
"css=grr-file-download-view")
self.Click("css=button:contains(\"Download\")")
# Select the previous version.
self.Click(
"css=select.version-dropdown > option:contains(\"%s\")" %
gui_test_lib.DateString(gui_test_lib.TIME_1))
# Now we should have a different time.
self.WaitUntilContains(
gui_test_lib.DateTimeString(gui_test_lib.TIME_1), self.GetText,
"css=grr-file-download-view")
self.Click("css=button:contains(\"Download\")")
self.WaitUntil(self.IsElementPresent, "css=li[heading=TextView]")
# the FakeDownloadHandle method was actually called four times, since
# a file download first sends a HEAD request to check user access.
self.WaitUntil(lambda: len(downloaded_files) == 4)
# Both files should be the same...
self.assertEqual(downloaded_files[0][0],
u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt")
self.assertEqual(downloaded_files[2][0],
u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt")
# But from different times. The downloaded file timestamp is only accurate
# to the nearest second. Also, the HEAD version of the file is downloaded
# with age=NEWEST_TIME.
self.assertEqual(downloaded_files[0][1], aff4.NEWEST_TIME)
self.assertAlmostEqual(downloaded_files[2][1],
gui_test_lib.TIME_1,
delta=rdfvalue.Duration("1s"))
self.Click("css=li[heading=TextView]")
# Make sure the file content has changed. This version has "Hello World" in
# it.
self.WaitUntilContains("Hello World", self.GetText,
"css=div.monospace pre")
def NoAuthorizationChecks(self):
return utils.Stubber(api_auth_manager, "API_AUTH_MGR",
NullAPIAuthorizationManager())
def testEmailPlugin(self):
def SendEmail(address, sender, title, message, **_):
self.email_messages.append(
dict(address=address,
sender=sender,
title=title,
message=message))
with utils.Stubber(email_alerts, "SendEmail", SendEmail):
self.email_messages = []
email_alerts.SendEmail = SendEmail
email_address = "notify@%s" % config_lib.CONFIG["Logging.domain"]
hunt_urn = self.RunHunt(
"EmailPlugin",
rdfvalue.EmailPluginArgs(email=email_address, email_limit=10))
hunt_obj = aff4.FACTORY.Open(hunt_urn,
age=aff4.ALL_TIMES,
mode="rw",
token=self.token)
self.client_ids = self.SetupClients(40)
hunt_obj.StartClients(hunt_obj.session_id, self.client_ids)
# Run the hunt.
client_mock = test_lib.SampleHuntMock()
test_lib.TestHuntHelper(client_mock, self.client_ids, False,
self.token)
# Run cron flow that executes actual output plugins
for _ in test_lib.TestFlowHelper("ProcessHuntResultsCronFlow",
token=self.token):
pass
# Stop the hunt now.
hunt_obj.GetRunner().Stop()
hunt_obj = aff4.FACTORY.Open(hunt_urn,
age=aff4.ALL_TIMES,
token=self.token)
started, finished, errors = hunt_obj.GetClientsCounts()
self.assertEqual(started, 40)
self.assertEqual(finished, 40)
self.assertEqual(errors, 20)
collection = aff4.FACTORY.Open(hunt_urn.Add("Results"),
mode="r",
token=self.token)
self.assertEqual(len(collection), 20)
# Due to the limit there should only by 10 messages.
self.assertEqual(len(self.email_messages), 10)
for msg in self.email_messages:
self.assertEqual(msg["address"], email_address)
self.assertTrue(
"%s got a new result" %
hunt_obj.session_id.Add("Results") in msg["title"])
self.assertTrue("fs/os/tmp/evil.txt" in msg["message"])
self.assertTrue("sending of emails will be disabled now" in
self.email_messages[-1]["message"])
def testIntegerComparisons(self):
"""Tests that we can use integer matching rules on the foreman."""
base_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(
1336480583.077736)
boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(
1336300000.000000)
self.SetupTestClientObject(0x11,
system="Windows XP",
install_time=base_time)
self.SetupTestClientObject(0x12,
system="Windows 7",
install_time=base_time)
# This one was installed one week earlier.
one_week_ago = base_time - rdfvalue.Duration("1w")
self.SetupTestClientObject(0x13,
system="Windows 7",
install_time=one_week_ago)
self.SetupTestClientObject(0x14,
system="Windows 7",
last_boot_time=boot_time)
with utils.Stubber(implementation.GRRHunt, "StartClients",
self.StartClients):
now = rdfvalue.RDFDatetime.Now()
expiration_time = now + rdfvalue.Duration("1h")
# Make a new rule
rule = foreman_rules.ForemanCondition(
creation_time=now,
expiration_time=expiration_time,
description="Test rule(old)",
hunt_name=standard.GenericHunt.__name__,
hunt_id="H:111111")
# Matches the old client
one_hour_ago = base_time - rdfvalue.Duration("1h")
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.
Operator.LESS_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
data_store.REL_DB.WriteForemanRule(rule)
# Make a new rule
rule = foreman_rules.ForemanCondition(
creation_time=now,
expiration_time=expiration_time,
description="Test rule(new)",
hunt_name=standard.GenericHunt.__name__,
hunt_id="H:222222")
# Matches the newer clients
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.
Operator.GREATER_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
data_store.REL_DB.WriteForemanRule(rule)
# Make a new rule
rule = foreman_rules.ForemanCondition(
creation_time=now,
expiration_time=expiration_time,
description="Test rule(eq)",
hunt_name=standard.GenericHunt.__name__,
hunt_id="H:333333")
# Note that this also tests the handling of nonexistent attributes.
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="LAST_BOOT_TIME",
operator="EQUAL",
value=boot_time.AsSecondsSinceEpoch()))
])
data_store.REL_DB.WriteForemanRule(rule)
foreman_obj = foreman.GetForeman()
self.clients_started = []
foreman_obj.AssignTasksToClient("C.1000000000000011")
foreman_obj.AssignTasksToClient("C.1000000000000012")
foreman_obj.AssignTasksToClient("C.1000000000000013")
foreman_obj.AssignTasksToClient("C.1000000000000014")
# Make sure that the clients ran the correct flows.
self.assertEqual(len(self.clients_started), 4)
self.assertEqual(self.clients_started[0][1], "C.1000000000000011")
self.assertEqual("H:222222", self.clients_started[0][0].Basename())
self.assertEqual(self.clients_started[1][1], "C.1000000000000012")
self.assertEqual("H:222222", self.clients_started[1][0].Basename())
self.assertEqual(self.clients_started[2][1], "C.1000000000000013")
self.assertEqual("H:111111", self.clients_started[2][0].Basename())
self.assertEqual(self.clients_started[3][1], "C.1000000000000014")
self.assertEqual("H:333333", self.clients_started[3][0].Basename())
def testGetClientSummary(self):
hostname = "test"
system = "Linux"
os_release = "12.02"
kernel = "3.15-rc2"
fqdn = "test.test.com"
arch = "amd64"
install_time = rdfvalue.RDFDatetime.Now()
user = "******"
userobj = rdf_client.User(username=user)
interface = rdf_client.Interface(ifname="eth0")
google_cloud_instance = cloud.GoogleCloudInstance(
instance_id="1771384456894610289",
zone="projects/123456789733/zones/us-central1-a",
project_id="myproject",
unique_id="us-central1-a/myproject/1771384456894610289")
cloud_instance = cloud.CloudInstance(cloud_type="GOOGLE",
google=google_cloud_instance)
timestamp = 1
with utils.Stubber(time, "time", lambda: timestamp):
with aff4.FACTORY.Create("C.0000000000000000",
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
kb = rdf_client.KnowledgeBase()
kb.users.Append(userobj)
empty_summary = fd.GetSummary()
self.assertEqual(empty_summary.client_id, "C.0000000000000000")
self.assertFalse(empty_summary.system_info.version)
self.assertEqual(empty_summary.timestamp.AsSecondsFromEpoch(),
1)
# This will cause TYPE to be written with current time = 101 when the
# object is closed
timestamp += 100
fd.Set(fd.Schema.HOSTNAME(hostname))
fd.Set(fd.Schema.SYSTEM(system))
fd.Set(fd.Schema.OS_RELEASE(os_release))
fd.Set(fd.Schema.KERNEL(kernel))
fd.Set(fd.Schema.FQDN(fqdn))
fd.Set(fd.Schema.ARCH(arch))
fd.Set(fd.Schema.INSTALL_DATE(install_time))
fd.Set(fd.Schema.KNOWLEDGE_BASE(kb))
fd.Set(fd.Schema.USERNAMES([user]))
fd.Set(fd.Schema.INTERFACES([interface]))
fd.Set(fd.Schema.CLOUD_INSTANCE(cloud_instance))
with aff4.FACTORY.Open("C.0000000000000000",
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
summary = fd.GetSummary()
self.assertEqual(summary.system_info.node, hostname)
self.assertEqual(summary.system_info.system, system)
self.assertEqual(summary.system_info.release, os_release)
self.assertEqual(summary.system_info.kernel, kernel)
self.assertEqual(summary.system_info.fqdn, fqdn)
self.assertEqual(summary.system_info.machine, arch)
self.assertEqual(summary.system_info.install_date,
install_time)
self.assertItemsEqual(summary.users, [userobj])
self.assertItemsEqual(summary.interfaces, [interface])
self.assertFalse(summary.client_info)
self.assertEqual(summary.timestamp.AsSecondsFromEpoch(), 101)
self.assertEqual(summary.cloud_type, "GOOGLE")
self.assertEqual(
summary.cloud_instance_id,
"us-central1-a/myproject/1771384456894610289")
def Run(self):
stats_collector = stats.StatsCollector()
stats_collector.RegisterCounterMetric(
"sample_counter", docstring="Sample counter metric.")
stats_collector.RegisterGaugeMetric(
"sample_gauge_value", float, docstring="Sample gauge metric.")
stats_collector.RegisterEventMetric(
"sample_event", docstring="Sample event metric.")
with utils.Stubber(stats, "STATS", stats_collector):
for i in range(10):
with test_lib.FakeTime(42 + i * 60):
stats_collector.IncrementCounter("sample_counter")
stats_collector.SetGaugeValue("sample_gauge_value", i * 0.5)
stats_collector.RecordEvent("sample_event", 0.42 + 0.5 * i)
with aff4.FACTORY.Create(
None, aff4_stats_store.StatsStore, mode="w",
token=self.token) as stats_store:
stats_store.WriteStats(process_id="worker_1")
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name="sample_counter",
start=42000000,
end=3600000000))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name="sample_counter",
start=42000000,
end=3600000000,
rate="1m"))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name="sample_gauge_value",
start=42000000,
end=3600000000))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name="sample_event",
start=42000000,
end=3600000000))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name="sample_event",
start=42000000,
end=3600000000,
distribution_handling_mode="DH_COUNT"))
def testAlertEmailIsSentWhenClientKilled(self):
"""Test that client killed messages are handled correctly."""
self.email_messages = []
def SendEmail(address, sender, title, message, **_):
self.email_messages.append(dict(address=address,
sender=sender,
title=title,
message=message))
with utils.Stubber(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmail):
client = test_lib.CrashClientMock(self.client_id, self.token)
for _ in test_lib.TestFlowHelper("FlowWithOneClientRequest",
client,
client_id=self.client_id,
token=self.token,
check_flow_errors=False):
pass
self.assertEqual(len(self.email_messages), 1)
email_message = self.email_messages[0]
# We expect the email to be sent.
self.assertEqual(
email_message.get("address", ""),
config_lib.CONFIG["Monitoring.alert_email"])
self.assertTrue(str(self.client_id) in email_message["title"])
# Make sure the flow state is included in the email message.
for s in ["Flow name", "FlowWithOneClientRequest", "current_state"]:
self.assertTrue(s in email_message["message"])
flow_obj = aff4.FACTORY.Open(client.flow_id,
age=aff4.ALL_TIMES,
token=self.token)
self.assertEqual(flow_obj.state.context.state, rdf_flows.Flow.State.ERROR)
# Make sure client object is updated with the last crash.
client_obj = aff4.FACTORY.Open(self.client_id, token=self.token)
crash = client_obj.Get(client_obj.Schema.LAST_CRASH)
self.CheckCrash(crash, flow_obj.session_id)
# Make sure crashes RDFValueCollections are created and written
# into proper locations. First check the per-client crashes collection.
client_crashes = sorted(
list(aff4.FACTORY.Open(
self.client_id.Add("crashes"),
aff4_type=collects.PackedVersionedCollection,
token=self.token)),
key=lambda x: x.timestamp)
self.assertTrue(len(client_crashes) >= 1)
crash = list(client_crashes)[0]
self.CheckCrash(crash, flow_obj.session_id)
# Check per-flow crash collection. Check that crash written there is
# equal to per-client crash.
flow_crashes = sorted(
list(flow_obj.GetValuesForAttribute(flow_obj.Schema.CLIENT_CRASH)),
key=lambda x: x.timestamp)
self.assertEqual(len(flow_crashes), len(client_crashes))
for a, b in zip(flow_crashes, client_crashes):
self.assertEqual(a, b)
# Check global crash collection. Check that crash written there is
# equal to per-client crash.
global_crashes = sorted(
aff4.FACTORY.Open(
aff4.ROOT_URN.Add("crashes"),
aff4_type=collects.PackedVersionedCollection,
token=self.token),
key=lambda x: x.timestamp)
self.assertEqual(len(global_crashes), len(client_crashes))
for a, b in zip(global_crashes, client_crashes):
self.assertEqual(a, b)
def testNewArtifactLoaded(self):
"""Simulate a new artifact being loaded into the store via the UI."""
cmd_artifact = """name: "TestCmdArtifact"
doc: "Test command artifact for dpkg."
sources:
- type: "COMMAND"
attributes:
cmd: "/usr/bin/dpkg"
args: ["--list"]
labels: [ "Software" ]
supported_os: [ "Linux" ]
"""
no_datastore_artifact = """name: "NotInDatastore"
doc: "Test command artifact for dpkg."
sources:
- type: "COMMAND"
attributes:
cmd: "/usr/bin/dpkg"
args: ["--list"]
labels: [ "Software" ]
supported_os: [ "Linux" ]
"""
test_registry = artifact_registry.ArtifactRegistry()
test_registry.ClearRegistry()
test_registry._dirty = False
with utils.Stubber(artifact_registry, "REGISTRY", test_registry):
collect_flow = collectors.ArtifactCollectorFlow(None,
token=self.token)
with self.assertRaises(
artifact_registry.ArtifactNotRegisteredError):
artifact_registry.REGISTRY.GetArtifact("TestCmdArtifact")
with self.assertRaises(
artifact_registry.ArtifactNotRegisteredError):
artifact_registry.REGISTRY.GetArtifact("NotInDatastore")
# Add artifact to datastore but not registry
artifact_coll = artifact_registry.ArtifactCollection(
rdfvalue.RDFURN("aff4:/artifact_store"), token=self.token)
with data_store.DB.GetMutationPool(token=self.token) as pool:
for artifact_val in artifact_registry.REGISTRY.ArtifactsFromYaml(
cmd_artifact):
artifact_coll.Add(artifact_val, mutation_pool=pool)
# Add artifact to registry but not datastore
for artifact_val in artifact_registry.REGISTRY.ArtifactsFromYaml(
no_datastore_artifact):
artifact_registry.REGISTRY.RegisterArtifact(
artifact_val,
source="datastore",
overwrite_if_exists=False)
# This should succeeded because the artifacts will be reloaded from the
# datastore.
self.assertTrue(
collect_flow._GetArtifactFromName("TestCmdArtifact"))
# We registered this artifact with datastore source but didn't
# write it into aff4. This simulates an artifact that was
# uploaded in the UI then later deleted. We expect it to get
# cleared when the artifacts are reloaded from the datastore.
with self.assertRaises(
artifact_registry.ArtifactNotRegisteredError):
artifact_registry.REGISTRY.GetArtifact("NotInDatastore")
def NoAuthorizationChecks(self):
return utils.Stubber(api_call_handlers, "API_AUTH_MGR",
NullAPIAuthorizationManager())
def testRegister(self):
"""Create master and register other servers."""
m = master.DataMaster(self.ports[0], self.mock_service)
self.assertNotEqual(m, None)
self.assertFalse(m.AllRegistered())
servers = [None]
for (i, port) in enumerate(self.ports):
if i == 0:
# Skip master server.
continue
self.assertFalse(m.AllRegistered())
server = m.RegisterServer(self.host, port)
servers.append(server)
self.assertNotEqual(server, None)
self.assertEqual(server.Address(), self.host)
self.assertEqual(server.Port(), port)
self.assertEqual(server.Index(), i)
self.assertTrue(m.AllRegistered())
# Try to register something that does not exist.
self.assertFalse(m.RegisterServer(self.host, 7004))
# Deregister a server.
m.DeregisterServer(servers[1])
self.assertFalse(m.AllRegistered())
# Register again.
m.RegisterServer(servers[1].Address(), servers[1].Port())
self.assertTrue(m.AllRegistered())
for port in self.ports:
for response_sequence in [
[constants.RESPONSE_OK, constants.RESPONSE_SERVER_NOT_AUTHORIZED],
[constants.RESPONSE_OK, constants.RESPONSE_SERVER_NOT_ALLOWED],
[constants.RESPONSE_OK, constants.RESPONSE_NOT_MASTER_SERVER]
]:
response_mocks = []
for response_status in response_sequence:
response_mocks.append(MockResponse(response_status))
pool_class = GetMockHTTPConnectionPoolClass(response_mocks)
with libutils.Stubber(urllib3.connectionpool, "HTTPConnectionPool",
pool_class):
m = data_server.StandardDataServer(port,
data_server.DataServerHandler)
m.handler_cls.NONCE_STORE = auth.NonceStore()
self.assertRaises(errors.DataServerError, m._DoRegister)
# Ensure two requests have been made.
self.assertEqual(len(pool_class.requests), 2)
# Ensure the register body is non-empty.
self.assertTrue(pool_class.requests[1]["body"])
# Ensure that the register body is a valid rdfvalue.
rdf_data_server.DataStoreRegistrationRequest.FromSerializedString(
pool_class.requests[1]["body"])
# Ensure the requests are POST requests.
self.assertEqual(pool_class.requests[0]["method"], "POST")
self.assertEqual(pool_class.requests[1]["method"], "POST")
# Ensure the correct URLs are hit according to the API.
self.assertEqual(pool_class.requests[0]["url"], "/server/handshake")
self.assertEqual(pool_class.requests[1]["url"], "/server/register")
def testFileView(self):
"""Test the fileview interface."""
# This is ugly :( Django gets confused when you import in the wrong order
# though and fileview imports the Django http module so we have to delay
# import until the Django server is properly set up.
# pylint: disable=g-import-not-at-top
from grr.gui.plugins import fileview
# pylint: enable=g-import-not-at-top
# Set up multiple version for an attribute on the client for tests.
with self.ACLChecksDisabled():
for fake_time, hostname in [(1333788833, "HostnameV1"),
(1333888833, "HostnameV2"),
(1333988833, "HostnameV3")]:
with test_lib.FakeTime(fake_time):
client = aff4.FACTORY.Open(u"C.0000000000000001",
mode="rw",
token=self.token)
client.Set(client.Schema.HOSTNAME(hostname))
client.Close()
self.Open("/")
self.Type("client_query", "0001")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000001", self.GetText,
"css=span[type=subject]")
# Choose client 1
self.Click("css=td:contains('0001')")
# Go to Browse VFS
self.Click("css=a:contains('Browse Virtual Filesystem')")
# Test the historical view for AFF4 elements.
self.Click("css=*[attribute=HOSTNAME] > ins")
self.WaitUntil(self.AllTextsPresent,
["HostnameV1", "HostnameV2", "HostnameV3"])
self.Click("css=*[attribute=HOSTNAME] > ins")
self.WaitUntilNot(self.IsTextPresent, "HostnameV1")
self.WaitUntilNot(self.IsTextPresent, "HostnameV2")
self.Click("css=#_fs ins.jstree-icon")
self.Click("css=#_fs-os ins.jstree-icon")
self.Click("css=#_fs-os-c ins.jstree-icon")
# Test file versioning.
self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads")
self.Click("link=Downloads")
# Verify that we have the latest version in the table by default
self.assertTrue("2012-04-09 16:27:13" in self.GetText(
"css=tr:contains(\"a.txt\")"))
# Click on the row.
self.Click("css=tr:contains(\"a.txt\")")
self.WaitUntilContains("a.txt @ 2012-04-09", self.GetText,
"css=div#main_rightBottomPane h3")
# Check the data in this file.
self.Click("css=#TextView")
self.WaitUntilContains("Goodbye World", self.GetText,
"css=div#text_viewer_data_content")
downloaded_files = []
def FakeDownload(unused_self, request, _):
aff4_path = request.REQ.get("aff4_path")
age = rdfvalue.RDFDatetime(
request.REQ.get("age")) or aff4.NEWEST_TIME
downloaded_files.append((aff4_path, age))
return fileview.http.HttpResponse(
content="<script>window.close()</script>")
with utils.Stubber(fileview.DownloadView, "Download", FakeDownload):
# Try to download the file.
self.Click("css=#Download")
self.WaitUntil(self.IsTextPresent,
"As downloaded on 2012-04-09 16:27:13")
self.Click("css=button:contains(\"Download\")")
# Click on the version selector.
self.Click("css=tr:contains(\"a.txt\") img.version-selector")
self.WaitUntilContains("Versions of", self.GetText,
"css=.version-selector-dialog h4")
# Select the previous version.
self.Click("css=td:contains(\"2012-04-07\")")
# Now we should have a different time.
self.WaitUntil(self.IsTextPresent,
"As downloaded on 2012-04-07 08:53:53")
self.Click("css=button:contains(\"Download\")")
self.WaitUntil(self.IsElementPresent, "css=#TextView")
self.WaitUntil(lambda: len(downloaded_files) == 2)
# Both files should be the same...
self.assertEqual(downloaded_files[0][0],
u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt")
self.assertEqual(downloaded_files[1][0],
u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt")
# But from different times.
self.assertEqual(downloaded_files[0][1], 1333988833000000)
self.assertEqual(downloaded_files[1][1], 1333788833000000)
self.Click("css=#TextView")
# Make sure the file content has changed. This version has "Hello World" in
# it.
self.WaitUntilContains("Hello World", self.GetText,
"css=div#text_viewer_data_content")
# Some more unicode testing.
self.Click(u"css=tr:contains(\"中.txt\")")
self.Click("css=#Download")
self.WaitUntil(self.IsTextPresent, u"fs/os/c/Downloads/中国新闻网新闻中.txt")
# Test the hex viewer.
self.Click("css=#_fs-os-proc ins.jstree-icon")
self.Click("css=#_fs-os-proc-10 a")
self.Click("css=span[type=subject]:contains(\"cmdline\")")
target_aff4_path = "aff4:/C.0000000000000001/fs/os/proc/10/cmdline"
self.Click("css=[state-aff4_path='%s'] > li > #HexView" %
target_aff4_path)
for i, value in enumerate(
"6c 73 00 68 65 6c 6c 6f 20 77 6f 72 6c 64 27 00 2d 6c".split(
" ")):
self.WaitUntilEqual(value, self.GetText,
"css=#hex_area tr:first td:nth(%d)" % i)
for i, value in enumerate(
"l s . h e l l o w o r l d ' . - l".split(" ")):
self.WaitUntilEqual(value, self.GetText,
"css=#data_area tr:first td:nth(%d)" % i)
self.Click("css=a[renderer=\"AFF4Stats\"]")
# Navigate to the bin C.0000000000000001 directory
self.Click("link=bin C.0000000000000001")
# Filter the table for bash (should match both bash and rbash)
self.WaitUntil(self.IsElementPresent, "css=td:contains('bash')")
self.Click("css=th:contains('Name') img")
self.Type("css=.sort-dialog input[type=text]",
"bash",
end_with_enter=True)
self.WaitUntilEqual("rbash", self.GetText, "css=tr:nth(2) span")
self.assertEqual(
2, self.GetCssCount("css=#main_rightTopPane tbody > tr"))
self.assertEqual("bash", self.GetText("css=tr:nth(1) span"))
self.assertEqual("rbash", self.GetText("css=tr:nth(2) span"))
# Check that the previous search test is still available in the form.
self.Click("css=th:contains('Name') img")
self.assertEqual("bash", self.GetValue("css=.sort-dialog input"))
# If we anchor cat at the start should only filter one.
self.Type("css=.sort-dialog input[type=text]",
"^cat",
end_with_enter=True)
self.WaitUntilEqual("cat", self.GetText, "css=tr:nth(1) span")
self.assertEqual(
1, self.GetCssCount("css=#main_rightTopPane tbody > tr"))
self.Click("css=tr:nth(1)")
self.WaitUntilContains(
"aff4:/C.0000000000000001/fs/os/c/bin C.0000000000000001/cat",
self.GetText, "css=.tab-content h3")
self.WaitUntil(self.IsTextPresent, "1026267") ## st_inode.
# Lets download it.
self.Click("Download")
self.Click("css=button:contains(\"Get a new Version\")")
self.Click("path_0")
self.WaitUntilEqual("fs", self.GetText, "css=tr td span:contains(fs)")
self.Click("Stats")
self.WaitUntilContains("aff4:/C.0000000000000001", self.GetText,
"css=.tab-content h3")
# Grab the root directory again - should produce an Interrogate flow.
self.Click("css=button[id^=refresh]")
# Go to the flow management screen.
self.Click("css=a:contains('Manage launched flows')")
# For the client update, 2 flows have to be issued: UpdateVFSFile and
# Interrogate. UpdateVFSFile triggers VFSGRRClient.Update() method which
# triggers Interrogate.
self.WaitUntilEqual("Interrogate", self.GetText,
"//table/tbody/tr[1]/td[3]")
self.WaitUntilEqual("UpdateVFSFile", self.GetText,
"//table/tbody/tr[2]/td[3]")
self.Click("//table/tbody/tr[2]/td[3]")
self.WaitUntilEqual(
"aff4:/C.0000000000000001", self.GetText,
"css=table > tbody td.proto_key:contains(\"Vfs file urn\") "
"~ td.proto_value")
# Check that UpdateVFSFile is called for the cat file.
# During the test this file is VFSMemoryFile, so its' Update method does
# nothing, therefore UpdateVFSFile won't issue any other flows.
self.WaitUntilEqual("UpdateVFSFile", self.GetText,
"//table/tbody/tr[3]/td[3]")
self.Click("//table/tbody/tr[3]/td[3]")
self.WaitUntilContains(
"cat", self.GetText,
"css=table > tbody td.proto_key:contains(\"Vfs file urn\") "
"~ td.proto_value")
def testDeleteHuntFlow(self):
# We'll need two users for this test.
self.CreateUser("user1")
token1 = access_control.ACLToken(username="******", reason="testing")
self.CreateUser("user2")
token2 = access_control.ACLToken(username="******", reason="testing")
manager = user_managers.FullAccessControlManager()
with utils.Stubber(data_store.DB, "security_manager", manager):
# Let user1 create a hunt and delete it, this should work.
hunt = self._CreateHunt(token1.SetUID())
aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1)
flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow",
token=token1,
hunt_urn=hunt.urn)
self._CheckHuntIsDeleted(hunt.urn)
# Let user1 create a hunt and user2 delete it, this should fail.
hunt = self._CreateHunt(token1.SetUID())
aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1)
with self.assertRaises(access_control.UnauthorizedAccess):
flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow",
token=token2,
hunt_urn=hunt.urn)
# Hunt is still there.
aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1)
# If user2 gets an approval, deletion is ok though.
self.GrantHuntApproval(hunt.urn, token=token2)
flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow",
token=token2,
hunt_urn=hunt.urn)
self._CheckHuntIsDeleted(hunt.urn)
# Let user1 create a hunt and run it. We are not allowed to delete
# running hunts.
hunt = self._CreateHunt(token1.SetUID())
hunt.Run()
hunt.Flush()
aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1)
with self.assertRaises(RuntimeError):
flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow",
token=token1,
hunt_urn=hunt.urn)
# The same is true if the hunt was scheduled on at least one client.
hunt = self._CreateHunt(token1.SetUID())
hunt.Set(hunt.Schema.CLIENT_COUNT(1))
hunt.Flush()
aff4.FACTORY.Open(hunt.urn, aff4_type="GRRHunt", token=token1)
with self.assertRaises(RuntimeError):
flow.GRRFlow.StartFlow(flow_name="DeleteHuntFlow",
token=token1,
hunt_urn=hunt.urn)
def testCheckPermissionsReturnsTrueIfGroupWasAuthorized(self):
self.auth_manager.DenyAll("subject-bar")
with utils.Stubber(self.group_access_manager,
"MemberOfAuthorizedGroup", lambda *args: True):
self.assertTrue(
self.auth_manager.CheckPermissions("user-bar", "subject-bar"))
def testIntegerComparisons(self):
"""Tests that we can use integer matching rules on the foreman."""
base_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(
1336480583.077736)
boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(
1336300000.000000)
self.SetupClient(0x11, system="Windows XP", install_time=base_time)
self.SetupClient(0x12, system="Windows 7", install_time=base_time)
# This one was installed one week earlier.
one_week_ago = base_time - rdfvalue.Duration("1w")
self.SetupClient(0x13, system="Windows 7", install_time=one_week_ago)
self.SetupClient(0x14, system="Windows 7", last_boot_time=boot_time)
with utils.Stubber(flow.GRRFlow, "StartFlow", self.StartFlow):
# Now setup the filters
now = rdfvalue.RDFDatetime.Now()
expires = now + rdfvalue.Duration("1h")
foreman_obj = foreman.GetForeman(token=self.token)
# Make a new rule
rule = foreman_rules.ForemanRule(created=now,
expires=expires,
description="Test rule(old)")
# Matches the old client
one_hour_ago = base_time - rdfvalue.Duration("1h")
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.
Operator.LESS_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
old_flow = "Test flow for old clients"
# Will run Test Flow
rule.actions.Append(flow_name=old_flow,
argv=rdf_protodict.Dict(dict(foo="bar")))
# Clear the rule set and add the new rule to it.
rule_set = foreman_obj.Schema.RULES()
rule_set.Append(rule)
# Make a new rule
rule = foreman_rules.ForemanRule(created=now,
expires=expires,
description="Test rule(new)")
# Matches the newer clients
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.
Operator.GREATER_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
new_flow = "Test flow for newer clients"
# Will run Test Flow
rule.actions.Append(flow_name=new_flow,
argv=rdf_protodict.Dict(dict(foo="bar")))
rule_set.Append(rule)
# Make a new rule
rule = foreman_rules.ForemanRule(created=now,
expires=expires,
description="Test rule(eq)")
# Note that this also tests the handling of nonexistent attributes.
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="LAST_BOOT_TIME",
operator="EQUAL",
value=boot_time.AsSecondsSinceEpoch()))
])
eq_flow = "Test flow for LAST_BOOT_TIME"
rule.actions.Append(flow_name=eq_flow,
argv=rdf_protodict.Dict(dict(foo="bar")))
rule_set.Append(rule)
# Assign it to the foreman
foreman_obj.Set(foreman_obj.Schema.RULES, rule_set)
foreman_obj.Close()
self.clients_launched = []
foreman_obj.AssignTasksToClient("C.1000000000000011")
foreman_obj.AssignTasksToClient("C.1000000000000012")
foreman_obj.AssignTasksToClient("C.1000000000000013")
foreman_obj.AssignTasksToClient("C.1000000000000014")
# Make sure that the clients ran the correct flows.
self.assertEqual(len(self.clients_launched), 4)
self.assertEqual(self.clients_launched[0][0],
rdf_client.ClientURN("C.1000000000000011"))
self.assertEqual(self.clients_launched[0][1], new_flow)
self.assertEqual(self.clients_launched[1][0],
rdf_client.ClientURN("C.1000000000000012"))
self.assertEqual(self.clients_launched[1][1], new_flow)
self.assertEqual(self.clients_launched[2][0],
rdf_client.ClientURN("C.1000000000000013"))
self.assertEqual(self.clients_launched[2][1], old_flow)
self.assertEqual(self.clients_launched[3][0],
rdf_client.ClientURN("C.1000000000000014"))
self.assertEqual(self.clients_launched[3][1], eq_flow)
def __init__(self):
self.stubber = utils.Stubber(report_plugins.REGISTRY, "plugins", {
"FooReportPlugin": FooReportPlugin,
"BarReportPlugin": BarReportPlugin
})
def testCorruption(self):
"""Simulate corruption of the http payload."""
self.corruptor_field = None
def Corruptor(url="", data=None, **kwargs):
"""Futz with some of the fields."""
comm_cls = rdf_flows.ClientCommunication
if data is not None:
self.client_communication = comm_cls.FromSerializedString(data)
else:
self.client_communication = comm_cls(None)
if self.corruptor_field and "server.pem" not in url:
orig_str_repr = self.client_communication.SerializeToString()
field_data = getattr(self.client_communication, self.corruptor_field)
if hasattr(field_data, "SerializeToString"):
# This converts encryption keys to a string so we can corrupt them.
field_data = field_data.SerializeToString()
modified_data = array.array("c", field_data)
offset = len(field_data) / 2
modified_data[offset] = chr((ord(field_data[offset]) % 250) + 1)
setattr(self.client_communication, self.corruptor_field,
modified_data.tostring())
# Make sure we actually changed the data.
self.assertNotEqual(field_data, modified_data)
mod_str_repr = self.client_communication.SerializeToString()
self.assertEqual(len(orig_str_repr), len(mod_str_repr))
differences = [
True for x, y in zip(orig_str_repr, mod_str_repr) if x != y
]
self.assertEqual(len(differences), 1)
data = self.client_communication.SerializeToString()
return self.UrlMock(url=url, data=data, **kwargs)
with utils.Stubber(requests, "request", Corruptor):
self.SendToServer()
status = self.client_communicator.RunOnce()
self.assertEqual(status.code, 200)
for field in ["packet_iv", "encrypted"]:
# Corrupting each field should result in HMAC verification errors.
self.corruptor_field = field
self.SendToServer()
status = self.client_communicator.RunOnce()
self.assertEqual(status.code, 500)
self.assertTrue(
"HMAC verification failed" in str(self.last_urlmock_error))
# Corruption of these fields will likely result in RSA errors, since we do
# the RSA operations before the HMAC verification (in order to recover the
# hmac key):
for field in ["encrypted_cipher", "encrypted_cipher_metadata"]:
# Corrupting each field should result in HMAC verification errors.
self.corruptor_field = field
self.SendToServer()
status = self.client_communicator.RunOnce()
self.assertEqual(status.code, 500)
def Run(self):
with utils.Stubber(output_plugin.OutputPlugin, "classes", {
"EmailOutputPlugin": email_plugin.EmailOutputPlugin,
"CSVOutputPlugin": csv_plugin.CSVOutputPlugin
}):
self.Check("GET", "/api/output-plugins/all")
def testGlob(self):
"""Test that glob works properly."""
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(rdf_client.User(username="******"))
kb.MergeOrAddUser(rdf_client.User(username="******"))
client.Set(kb)
client.Close()
client_mock = action_mocks.ActionMock("Find", "StatFile")
# This glob selects all files which start with the username on this system.
paths = [
os.path.join(self.base_path, "%%Users.username%%*"),
os.path.join(self.base_path, "VFSFixture/var/*/wtmp")
]
# Set iterator really low to force iteration.
with utils.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2):
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token,
sync=False,
check_flow_errors=False):
pass
output_path = self.client_id.Add("fs/os").Add(
self.base_path.replace("\\", "/"))
children = []
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
filename = child.Basename()
if filename != "VFSFixture":
children.append(filename)
expected = [
filename for filename in os.listdir(self.base_path)
if filename.startswith("test") or filename.startswith("syslog")
]
self.assertTrue([x for x in expected if x.startswith("test")],
"Need a file starting with 'test'"
" in test_data for this test!")
self.assertTrue([x for x in expected if x.startswith("syslog")],
"Need a file starting with 'syslog'"
" in test_data for this test!")
self.assertItemsEqual(expected, children)
children = []
fd = aff4.FACTORY.Open(output_path.Add("VFSFixture/var/log"),
token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
self.assertItemsEqual(children, ["wtmp"])
def testBlockingTasks(self):
# The pool starts off with the minimum number of threads.
self.assertEqual(len(self.test_pool), self.NUMBER_OF_THREADS)
done_event = threading.Event()
self.lock = threading.Lock()
res = []
def Block(done):
done.wait()
def Insert(list_obj, element):
with self.lock:
list_obj.append(element)
# Ensure that the thread pool is able to add the correct number of threads
# deterministically.
with utils.Stubber(self.test_pool, "CPUUsage", lambda: 0):
# Schedule the maximum number of threads of blocking tasks and the same of
# insert tasks. The threads are now all blocked, and the inserts are
# waiting in the queue.
for _ in range(self.MAXIMUM_THREADS):
self.test_pool.AddTask(Block, (done_event, ), "Blocking")
# Wait until the threadpool picks up the task.
self.WaitUntil(
lambda: self.test_pool.busy_threads == self.NUMBER_OF_THREADS)
# Now there are minimum number of threads active and the rest are sitting
# on the queue.
self.assertEqual(self.test_pool.pending_tasks,
self.MAXIMUM_THREADS - self.NUMBER_OF_THREADS)
# Now as we push these tasks on the queue, new threads will be created and
# they will receive the blocking tasks from the queue.
for i in range(self.MAXIMUM_THREADS):
self.test_pool.AddTask(Insert, (res, i),
"Insert",
blocking=True,
inline=False)
# There should be 20 workers created and they should consume all the
# blocking tasks.
self.WaitUntil(
lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS)
# No Insert tasks are running yet.
self.assertEqual(res, [])
# There are 20 tasks waiting on the queue.
self.assertEqual(self.test_pool.pending_tasks,
self.MAXIMUM_THREADS)
# Inserting more tasks than the queue can hold should lead to processing
# the tasks inline. This effectively causes these tasks to skip over the
# tasks which are waiting in the queue.
for i in range(10, 20):
self.test_pool.AddTask(Insert, (res, i), "Insert", inline=True)
res.sort()
self.assertEqual(res, range(10, 20))
# This should release all the busy tasks. It will also cause the workers
# to process all the Insert tasks in the queue.
done_event.set()
self.test_pool.Join()
# Now the rest of the tasks should have been processed as well.
self.assertEqual(sorted(res[10:]), range(20))
def testNoValidStatusRaceIsResolved(self):
# This tests for the regression of a long standing race condition we saw
# where notifications would trigger the reading of another request that
# arrives later but wasn't completely written to the database yet.
# Timestamp based notification handling should eliminate this bug.
# We need a random flow object for this test.
session_id = flow.GRRFlow.StartFlow(client_id=self.client_id,
flow_name="WorkerSendingTestFlow",
token=self.token)
worker_obj = worker.GRRWorker(token=self.token)
manager = queue_manager.QueueManager(token=self.token)
manager.DeleteNotification(session_id)
manager.Flush()
# We have a first request that is complete (request_id 1, response_id 1).
self.SendResponse(session_id, "Response 1")
# However, we also have request #2 already coming in. The race is that
# the queue manager might write the status notification to
# session_id/state as "status:00000002" but not the status response
# itself yet under session_id/state/request:00000002
request_id = 2
response_id = 1
flow_manager = queue_manager.QueueManager(token=self.token)
flow_manager.FreezeTimestamp()
flow_manager.QueueResponse(
rdf_flows.GrrMessage(
source=self.client_id,
session_id=session_id,
payload=rdf_protodict.DataBlob(string="Response 2"),
request_id=request_id,
response_id=response_id))
status = rdf_flows.GrrMessage(
source=self.client_id,
session_id=session_id,
payload=rdf_flows.GrrStatus(
status=rdf_flows.GrrStatus.ReturnedStatus.OK),
request_id=request_id,
response_id=response_id + 1,
type=rdf_flows.GrrMessage.Type.STATUS)
# Now we write half the status information.
data_store.DB.StoreRequestsAndResponses(new_responses=[(status, None)])
# We make the race even a bit harder by saying the new notification gets
# written right before the old one gets deleted. If we are not careful here,
# we delete the new notification as well and the flow becomes stuck.
def WriteNotification(self, arg_session_id, start=None, end=None):
if arg_session_id == session_id:
flow_manager.QueueNotification(session_id=arg_session_id)
flow_manager.Flush()
self.DeleteNotification.old_target(self,
arg_session_id,
start=start,
end=end)
with utils.Stubber(queue_manager.QueueManager, "DeleteNotification",
WriteNotification):
# This should process request 1 but not touch request 2.
worker_obj.RunOnce()
worker_obj.thread_pool.Join()
flow_obj = aff4.FACTORY.Open(session_id, token=self.token)
self.assertFalse(flow_obj.context.backtrace)
self.assertNotEqual(flow_obj.context.state,
rdf_flows.FlowContext.State.ERROR)
request_data = data_store.DB.ReadResponsesForRequestId(session_id, 2)
request_data.sort(key=lambda msg: msg.response_id)
self.assertEqual(len(request_data), 2)
# Make sure the status and the original request are still there.
self.assertEqual(request_data[0].args_rdf_name, "DataBlob")
self.assertEqual(request_data[1].args_rdf_name, "GrrStatus")
# But there is nothing for request 1.
request_data = data_store.DB.ReadResponsesForRequestId(session_id, 1)
self.assertEqual(request_data, [])
# The notification for request 2 should have survived.
with queue_manager.QueueManager(token=self.token) as manager:
notifications = manager.GetNotifications(queues.FLOWS)
self.assertEqual(len(notifications), 1)
notification = notifications[0]
self.assertEqual(notification.session_id, session_id)
self.assertEqual(notification.timestamp,
flow_manager.frozen_timestamp)
self.assertEqual(RESULTS, ["Response 1"])
# The last missing piece of request 2 is the actual status message.
flow_manager.QueueResponse(status)
flow_manager.Flush()
# Now make sure request 2 runs as expected.
worker_obj.RunOnce()
worker_obj.thread_pool.Join()
self.assertEqual(RESULTS, ["Response 1", "Response 2"])
def testPartialRead(self):
with utils.Stubber(memory, "win32file", Win32FileMock()):
super(TestWindowsMemory, self).testPartialRead()
