def testNewArtifactLoaded(self):
"""Simulate a new artifact being loaded into the store via the UI."""
cmd_artifact = """name: "TestCmdArtifact"
doc: "Test command artifact for dpkg."
sources:
- type: "COMMAND"
attributes:
cmd: "/usr/bin/dpkg"
args: ["--list"]
labels: [ "Software" ]
supported_os: [ "Linux" ]
"""
no_datastore_artifact = """name: "NotInDatastore"
doc: "Test command artifact for dpkg."
sources:
- type: "COMMAND"
attributes:
cmd: "/usr/bin/dpkg"
args: ["--list"]
labels: [ "Software" ]
supported_os: [ "Linux" ]
"""
test_registry = artifact_registry.ArtifactRegistry()
test_registry.ClearRegistry()
test_registry.AddDatastoreSource(
rdfvalue.RDFURN("aff4:/artifact_store"))
test_registry._dirty = False
with utils.Stubber(artifact_registry, "REGISTRY", test_registry):
with self.assertRaises(rdf_artifacts.ArtifactNotRegisteredError):
artifact_registry.REGISTRY.GetArtifact("TestCmdArtifact")
with self.assertRaises(rdf_artifacts.ArtifactNotRegisteredError):
artifact_registry.REGISTRY.GetArtifact("NotInDatastore")
# Add artifact to datastore but not registry
artifact_coll = artifact_registry.ArtifactCollection(
rdfvalue.RDFURN("aff4:/artifact_store"))
with data_store.DB.GetMutationPool() as pool:
for artifact_val in artifact_registry.REGISTRY.ArtifactsFromYaml(
cmd_artifact):
artifact_coll.Add(artifact_val, mutation_pool=pool)
# Add artifact to registry but not datastore
for artifact_val in artifact_registry.REGISTRY.ArtifactsFromYaml(
no_datastore_artifact):
artifact_registry.REGISTRY.RegisterArtifact(
artifact_val,
source="datastore",
overwrite_if_exists=False)
# This should succeeded because the artifacts will be reloaded from the
# datastore.
self.assertTrue(
artifact_registry.REGISTRY.GetArtifact("TestCmdArtifact"))
# We registered this artifact with datastore source but didn't
# write it into aff4. This simulates an artifact that was
# uploaded in the UI then later deleted. We expect it to get
# cleared when the artifacts are reloaded from the datastore.
with self.assertRaises(rdf_artifacts.ArtifactNotRegisteredError):
artifact_registry.REGISTRY.GetArtifact("NotInDatastore")
def testCheckPermissionsReturnsTrueIfGroupWasAuthorized(self):
self.auth_manager.DenyAll("subject-bar")
with utils.Stubber(self.group_access_manager, "MemberOfAuthorizedGroup",
lambda *args: True):
self.assertTrue(
self.auth_manager.CheckPermissions("user-bar", "subject-bar"))
def testStartupHandler(self):
client_id = test_lib.TEST_CLIENT_ID
rel_client_id = client_id.Basename()
data_store.REL_DB.WriteClientMetadata(rel_client_id,
fleetspeak_enabled=True)
self._RunSendStartupInfo(client_id)
# AFF4 client.
# Check the client's boot time and info.
fd = aff4.FACTORY.Open(client_id, token=self.token)
client_info = fd.Get(fd.Schema.CLIENT_INFO)
boot_time = fd.Get(fd.Schema.LAST_BOOT_TIME)
self.assertEqual(client_info.client_name, config.CONFIG["Client.name"])
self.assertEqual(client_info.client_description,
config.CONFIG["Client.description"])
# Check that the boot time is accurate.
self.assertAlmostEqual(psutil.boot_time(),
boot_time.AsSecondsSinceEpoch())
# objects.ClientSnapshot.
si = data_store.REL_DB.ReadClientStartupInfo(rel_client_id)
self.assertIsNotNone(si)
self.assertEqual(si.client_info.client_name,
config.CONFIG["Client.name"])
self.assertEqual(si.client_info.client_description,
config.CONFIG["Client.description"])
# Run it again - this should not update any record.
self._RunSendStartupInfo(client_id)
# AFF4 client.
fd = aff4.FACTORY.Open(client_id, token=self.token)
self.assertEqual(boot_time.age, fd.Get(fd.Schema.LAST_BOOT_TIME).age)
self.assertEqual(client_info.age, fd.Get(fd.Schema.CLIENT_INFO).age)
# objects.ClientSnapshot.
new_si = data_store.REL_DB.ReadClientStartupInfo(rel_client_id)
self.assertEqual(new_si, si)
# Simulate a reboot.
current_boot_time = psutil.boot_time()
with utils.Stubber(psutil, "boot_time",
lambda: current_boot_time + 600):
# Run it again - this should now update the boot time.
self._RunSendStartupInfo(client_id)
# AFF4 client.
# Ensure only this attribute is updated.
fd = aff4.FACTORY.Open(client_id, token=self.token)
self.assertNotEqual(int(boot_time.age),
int(fd.Get(fd.Schema.LAST_BOOT_TIME).age))
self.assertEqual(int(client_info.age),
int(fd.Get(fd.Schema.CLIENT_INFO).age))
# objects.ClientSnapshot.
new_si = data_store.REL_DB.ReadClientStartupInfo(rel_client_id)
self.assertIsNotNone(new_si)
self.assertNotEqual(new_si.boot_time, si.boot_time)
# Now set a new client build time.
with test_lib.ConfigOverrider({"Client.build_time": time.ctime()}):
# Run it again - this should now update the client info.
self._RunSendStartupInfo(client_id)
# AFF4 client.
# Ensure the client info attribute is updated.
fd = aff4.FACTORY.Open(client_id, token=self.token)
self.assertNotEqual(int(client_info.age),
int(fd.Get(fd.Schema.CLIENT_INFO).age))
# objects.ClientSnapshot.
new_si = data_store.REL_DB.ReadClientStartupInfo(rel_client_id)
self.assertIsNotNone(new_si)
self.assertNotEqual(new_si.client_info, si.client_info)
def testBlockingTasks(self):
# The pool starts off with the minimum number of threads.
self.assertEqual(len(self.test_pool), self.NUMBER_OF_THREADS)
done_event = threading.Event()
self.lock = threading.Lock()
res = []
def Block(done):
done.wait()
def Insert(list_obj, element):
with self.lock:
list_obj.append(element)
# Ensure that the thread pool is able to add the correct number of threads
# deterministically.
with utils.Stubber(self.test_pool, "CPUUsage", lambda: 0):
# Schedule the maximum number of threads of blocking tasks and the same of
# insert tasks. The threads are now all blocked, and the inserts are
# waiting in the queue.
for _ in range(self.MAXIMUM_THREADS):
self.test_pool.AddTask(Block, (done_event, ), "Blocking")
# Wait until the threadpool picks up the task.
self.WaitUntil(
lambda: self.test_pool.busy_threads == self.NUMBER_OF_THREADS)
# Now there are minimum number of threads active and the rest are sitting
# on the queue.
self.assertEqual(self.test_pool.pending_tasks,
self.MAXIMUM_THREADS - self.NUMBER_OF_THREADS)
# Now as we push these tasks on the queue, new threads will be created and
# they will receive the blocking tasks from the queue.
for i in range(self.MAXIMUM_THREADS):
self.test_pool.AddTask(Insert, (res, i),
"Insert",
blocking=True,
inline=False)
# There should be 20 workers created and they should consume all the
# blocking tasks.
self.WaitUntil(
lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS)
# No Insert tasks are running yet.
self.assertEqual(res, [])
# There are 20 tasks waiting on the queue.
self.assertEqual(self.test_pool.pending_tasks,
self.MAXIMUM_THREADS)
# Inserting more tasks than the queue can hold should lead to processing
# the tasks inline. This effectively causes these tasks to skip over the
# tasks which are waiting in the queue.
for i in range(10, 20):
self.test_pool.AddTask(Insert, (res, i), "Insert", inline=True)
res.sort()
self.assertEqual(res, range(10, 20))
# This should release all the busy tasks. It will also cause the workers
# to process all the Insert tasks in the queue.
done_event.set()
self.test_pool.Join()
# Now the rest of the tasks should have been processed as well.
self.assertEqual(sorted(res[10:]), range(20))
def testNoValidStatusRaceIsResolved(self):
# This tests for the regression of a long standing race condition we saw
# where notifications would trigger the reading of another request that
# arrives later but wasn't completely written to the database yet.
# Timestamp based notification handling should eliminate this bug.
# We need a random flow object for this test.
session_id = flow.StartFlow(client_id=self.client_id,
flow_name="WorkerSendingTestFlow",
token=self.token)
worker_obj = worker_lib.GRRWorker(token=self.token)
manager = queue_manager.QueueManager(token=self.token)
manager.DeleteNotification(session_id)
manager.Flush()
# We have a first request that is complete (request_id 1, response_id 1).
self.SendResponse(session_id, "Response 1")
# However, we also have request #2 already coming in. The race is that
# the queue manager might write the status notification to
# session_id/state as "status:00000002" but not the status response
# itself yet under session_id/state/request:00000002
request_id = 2
response_id = 1
flow_manager = queue_manager.QueueManager(token=self.token)
flow_manager.FreezeTimestamp()
flow_manager.QueueResponse(
rdf_flows.GrrMessage(
source=self.client_id,
session_id=session_id,
payload=rdf_protodict.DataBlob(string="Response 2"),
request_id=request_id,
auth_state="AUTHENTICATED",
response_id=response_id))
status = rdf_flows.GrrMessage(
source=self.client_id,
session_id=session_id,
payload=rdf_flows.GrrStatus(
status=rdf_flows.GrrStatus.ReturnedStatus.OK),
request_id=request_id,
response_id=response_id + 1,
auth_state="AUTHENTICATED",
type=rdf_flows.GrrMessage.Type.STATUS)
# Now we write half the status information.
data_store.DB.StoreRequestsAndResponses(new_responses=[(status, None)])
# We make the race even a bit harder by saying the new notification gets
# written right before the old one gets deleted. If we are not careful here,
# we delete the new notification as well and the flow becomes stuck.
# pylint: disable=invalid-name
def WriteNotification(self, arg_session_id, start=None, end=None):
if arg_session_id == session_id:
flow_manager.QueueNotification(session_id=arg_session_id)
flow_manager.Flush()
self.DeleteNotification.old_target(self,
arg_session_id,
start=start,
end=end)
# pylint: enable=invalid-name
with utils.Stubber(queue_manager.QueueManager, "DeleteNotification",
WriteNotification):
# This should process request 1 but not touch request 2.
worker_obj.RunOnce()
worker_obj.thread_pool.Join()
flow_obj = aff4.FACTORY.Open(session_id, token=self.token)
self.assertFalse(flow_obj.context.backtrace)
self.assertNotEqual(flow_obj.context.state,
rdf_flow_runner.FlowContext.State.ERROR)
request_data = data_store.DB.ReadResponsesForRequestId(session_id, 2)
request_data.sort(key=lambda msg: msg.response_id)
self.assertEqual(len(request_data), 2)
# Make sure the status and the original request are still there.
self.assertEqual(request_data[0].args_rdf_name, "DataBlob")
self.assertEqual(request_data[1].args_rdf_name, "GrrStatus")
# But there is nothing for request 1.
request_data = data_store.DB.ReadResponsesForRequestId(session_id, 1)
self.assertEqual(request_data, [])
# The notification for request 2 should have survived.
with queue_manager.QueueManager(token=self.token) as manager:
notifications = manager.GetNotifications(queues.FLOWS)
self.assertEqual(len(notifications), 1)
notification = notifications[0]
self.assertEqual(notification.session_id, session_id)
self.assertEqual(notification.timestamp,
flow_manager.frozen_timestamp)
self.assertEqual(RESULTS, ["Response 1"])
# The last missing piece of request 2 is the actual status message.
flow_manager.QueueResponse(status)
flow_manager.Flush()
# Now make sure request 2 runs as expected.
worker_obj.RunOnce()
worker_obj.thread_pool.Join()
self.assertEqual(RESULTS, ["Response 1", "Response 2"])
def testAlertEmailIsSentWhenClientKilled(self):
"""Test that client killed messages are handled correctly."""
client_id = self.SetupClient(0)
self.SetupTestClientObject(0)
self.email_messages = []
def SendEmail(address, sender, title, message, **_):
self.email_messages.append(
dict(address=address,
sender=sender,
title=title,
message=message))
with utils.Stubber(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmail):
client = flow_test_lib.CrashClientMock(client_id, self.token)
flow_test_lib.TestFlowHelper(
flow_test_lib.FlowWithOneClientRequest.__name__,
client,
client_id=client_id,
token=self.token,
check_flow_errors=False)
self.assertEqual(len(self.email_messages), 1)
email_message = self.email_messages[0]
# We expect the email to be sent.
self.assertEqual(email_message.get("address", ""),
config.CONFIG["Monitoring.alert_email"])
self.assertTrue(str(client_id) in email_message["title"])
# Make sure the flow state is included in the email message.
for s in [
"flow_name", flow_test_lib.FlowWithOneClientRequest.__name__,
"current_state"
]:
self.assertTrue(s in email_message["message"])
flow_obj = aff4.FACTORY.Open(client.flow_id,
age=aff4.ALL_TIMES,
token=self.token)
self.assertEqual(flow_obj.context.state,
rdf_flow_runner.FlowContext.State.ERROR)
# Make sure client object is updated with the last crash.
# AFF4.
client_obj = aff4.FACTORY.Open(client_id, token=self.token)
crash = client_obj.Get(client_obj.Schema.LAST_CRASH)
self.CheckCrash(crash, flow_obj.session_id, client_id)
# Relational db.
crash = data_store.REL_DB.ReadClientCrashInfo(client_id.Basename())
self.CheckCrash(crash, flow_obj.session_id, client_id)
# Make sure crashes collections are created and written
# into proper locations. First check the per-client crashes collection.
client_crashes = sorted(list(
aff4_grr.VFSGRRClient.CrashCollectionForCID(client_id)),
key=lambda x: x.timestamp)
self.assertTrue(len(client_crashes) >= 1)
crash = list(client_crashes)[0]
self.CheckCrash(crash, flow_obj.session_id, client_id)
# Check per-flow crash collection. Check that crash written there is
# equal to per-client crash.
flow_crashes = sorted(list(
flow_obj.GetValuesForAttribute(flow_obj.Schema.CLIENT_CRASH)),
key=lambda x: x.timestamp)
self.assertEqual(len(flow_crashes), len(client_crashes))
for a, b in zip(flow_crashes, client_crashes):
self.assertEqual(a, b)
def testIntegerComparisons(self):
"""Tests that we can use integer matching rules on the foreman."""
base_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1336480583.077736)
boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1336300000.000000)
self.SetupClient(0x11, system="Windows XP", install_time=base_time)
self.SetupClient(0x12, system="Windows 7", install_time=base_time)
# This one was installed one week earlier.
one_week_ago = base_time - rdfvalue.Duration("1w")
self.SetupClient(0x13, system="Windows 7", install_time=one_week_ago)
self.SetupClient(0x14, system="Windows 7", last_boot_time=boot_time)
with utils.Stubber(flow, "StartFlow", self.StartFlow):
# Now setup the filters
now = rdfvalue.RDFDatetime.Now()
expires = now + rdfvalue.Duration("1h")
foreman_obj = foreman.GetForeman(token=self.token)
# Make a new rule
rule = foreman_rules.ForemanRule(
created=now, expires=expires, description="Test rule(old)")
# Matches the old client
one_hour_ago = base_time - rdfvalue.Duration("1h")
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.Operator.
LESS_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
old_flow = "Test flow for old clients"
# Will run Test Flow
rule.actions.Append(
flow_name=old_flow, argv=rdf_protodict.Dict(dict(foo="bar")))
# Clear the rule set and add the new rule to it.
rule_set = foreman_obj.Schema.RULES()
rule_set.Append(rule)
# Make a new rule
rule = foreman_rules.ForemanRule(
created=now, expires=expires, description="Test rule(new)")
# Matches the newer clients
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.Operator.
GREATER_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
new_flow = "Test flow for newer clients"
# Will run Test Flow
rule.actions.Append(
flow_name=new_flow, argv=rdf_protodict.Dict(dict(foo="bar")))
rule_set.Append(rule)
# Make a new rule
rule = foreman_rules.ForemanRule(
created=now, expires=expires, description="Test rule(eq)")
# Note that this also tests the handling of nonexistent attributes.
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="LAST_BOOT_TIME",
operator="EQUAL",
value=boot_time.AsSecondsSinceEpoch()))
])
eq_flow = "Test flow for LAST_BOOT_TIME"
rule.actions.Append(
flow_name=eq_flow, argv=rdf_protodict.Dict(dict(foo="bar")))
rule_set.Append(rule)
# Assign it to the foreman
foreman_obj.Set(foreman_obj.Schema.RULES, rule_set)
foreman_obj.Close()
self.clients_launched = []
foreman_obj.AssignTasksToClient("C.1000000000000011")
foreman_obj.AssignTasksToClient("C.1000000000000012")
foreman_obj.AssignTasksToClient("C.1000000000000013")
foreman_obj.AssignTasksToClient("C.1000000000000014")
# Make sure that the clients ran the correct flows.
self.assertEqual(len(self.clients_launched), 4)
self.assertEqual(self.clients_launched[0][0],
rdf_client.ClientURN("C.1000000000000011"))
self.assertEqual(self.clients_launched[0][1], new_flow)
self.assertEqual(self.clients_launched[1][0],
rdf_client.ClientURN("C.1000000000000012"))
self.assertEqual(self.clients_launched[1][1], new_flow)
self.assertEqual(self.clients_launched[2][0],
rdf_client.ClientURN("C.1000000000000013"))
self.assertEqual(self.clients_launched[2][1], old_flow)
self.assertEqual(self.clients_launched[3][0],
rdf_client.ClientURN("C.1000000000000014"))
self.assertEqual(self.clients_launched[3][1], eq_flow)
def testVersionDropDownChangesFileContentAndDownloads(self):
"""Test the fileview interface."""
self.Open("/#/clients/%s" % self.client_id)
# Go to Browse VFS.
self.Click("css=a[grrtarget='client.vfs']")
self.Click("css=#_fs i.jstree-icon")
self.Click("css=#_fs-os i.jstree-icon")
self.Click("css=#_fs-os-c i.jstree-icon")
# Test file versioning.
self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads")
self.Click("link=Downloads")
# Verify that we have the latest version in the table by default.
self.assertTrue(
gui_test_lib.DateString(gui_test_lib.TIME_2) in self.GetText(
"css=tr:contains(\"a.txt\")"))
# Click on the row.
self.Click("css=tr:contains(\"a.txt\")")
self.WaitUntilContains("a.txt", self.GetText,
"css=div#main_bottomPane h1")
self.WaitUntilContains("HEAD", self.GetText,
"css=.version-dropdown > option[selected]")
self.WaitUntilContains(gui_test_lib.DateString(gui_test_lib.TIME_2),
self.GetText,
"css=.version-dropdown > option:nth(1)")
# Check the data in this file.
self.Click("css=li[heading=TextView]")
self.WaitUntilContains("Goodbye World", self.GetText,
"css=div.monospace pre")
downloaded_files = []
def FakeDownloadHandle(unused_self, args, token=None):
_ = token # Avoid unused variable linter warnings.
aff4_path = args.client_id.ToClientURN().Add(args.file_path)
age = args.timestamp or aff4.NEWEST_TIME
downloaded_files.append((aff4_path, age))
return api_call_handler_base.ApiBinaryStream(
filename=aff4_path.Basename(), content_generator=xrange(42))
with utils.Stubber(api_vfs.ApiGetFileBlobHandler, "Handle",
FakeDownloadHandle):
# Try to download the file.
self.Click("css=li[heading=Download]")
self.WaitUntilContains(
gui_test_lib.DateTimeString(gui_test_lib.TIME_2), self.GetText,
"css=grr-file-download-view")
self.Click("css=button:contains(\"Download\")")
# Select the previous version.
self.Click(
"css=select.version-dropdown > option:contains(\"%s\")" %
gui_test_lib.DateString(gui_test_lib.TIME_1))
# Now we should have a different time.
self.WaitUntilContains(
gui_test_lib.DateTimeString(gui_test_lib.TIME_1), self.GetText,
"css=grr-file-download-view")
self.Click("css=button:contains(\"Download\")")
self.WaitUntil(self.IsElementPresent, "css=li[heading=TextView]")
# the FakeDownloadHandle method was actually called four times, since
# a file download first sends a HEAD request to check user access.
self.WaitUntil(lambda: len(downloaded_files) == 4)
# Both files should be the same...
self.assertEqual(downloaded_files[0][0],
u"aff4:/%s/fs/os/c/Downloads/a.txt" % self.client_id)
self.assertEqual(downloaded_files[2][0],
u"aff4:/%s/fs/os/c/Downloads/a.txt" % self.client_id)
# But from different times. The downloaded file timestamp is only accurate
# to the nearest second. Also, the HEAD version of the file is downloaded
# with age=NEWEST_TIME.
self.assertEqual(downloaded_files[0][1], aff4.NEWEST_TIME)
self.assertAlmostEqual(downloaded_files[2][1],
gui_test_lib.TIME_1,
delta=rdfvalue.Duration("1s"))
self.Click("css=li[heading=TextView]")
# Make sure the file content has changed. This version has "Hello World" in
# it.
self.WaitUntilContains("Hello World", self.GetText,
"css=div.monospace pre")
def testIntegerComparisons(self):
"""Tests that we can use integer matching rules on the foreman."""
base_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1336480583.077736)
boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1336300000.000000)
self.SetupTestClientObject(
0x11, system="Windows XP", install_time=base_time)
self.SetupTestClientObject(0x12, system="Windows 7", install_time=base_time)
# This one was installed one week earlier.
one_week_ago = base_time - rdfvalue.Duration("1w")
self.SetupTestClientObject(
0x13, system="Windows 7", install_time=one_week_ago)
self.SetupTestClientObject(
0x14, system="Windows 7", last_boot_time=boot_time)
with utils.Stubber(implementation.GRRHunt, "StartClients",
self.StartClients):
now = rdfvalue.RDFDatetime.Now()
expiration_time = now + rdfvalue.Duration("1h")
# Make a new rule
rule = foreman_rules.ForemanCondition(
creation_time=now,
expiration_time=expiration_time,
description="Test rule(old)",
hunt_name=standard.GenericHunt.__name__,
hunt_id="H:111111")
# Matches the old client
one_hour_ago = base_time - rdfvalue.Duration("1h")
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.Operator.
LESS_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
data_store.REL_DB.WriteForemanRule(rule)
# Make a new rule
rule = foreman_rules.ForemanCondition(
creation_time=now,
expiration_time=expiration_time,
description="Test rule(new)",
hunt_name=standard.GenericHunt.__name__,
hunt_id="H:222222")
# Matches the newer clients
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="INSTALL_TIME",
operator=foreman_rules.ForemanIntegerClientRule.Operator.
GREATER_THAN,
value=one_hour_ago.AsSecondsSinceEpoch()))
])
data_store.REL_DB.WriteForemanRule(rule)
# Make a new rule
rule = foreman_rules.ForemanCondition(
creation_time=now,
expiration_time=expiration_time,
description="Test rule(eq)",
hunt_name=standard.GenericHunt.__name__,
hunt_id="H:333333")
# Note that this also tests the handling of nonexistent attributes.
rule.client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
foreman_rules.ForemanClientRule(
rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
integer=foreman_rules.ForemanIntegerClientRule(
field="LAST_BOOT_TIME",
operator="EQUAL",
value=boot_time.AsSecondsSinceEpoch()))
])
data_store.REL_DB.WriteForemanRule(rule)
foreman_obj = foreman.GetForeman()
self.clients_started = []
foreman_obj.AssignTasksToClient("C.1000000000000011")
foreman_obj.AssignTasksToClient("C.1000000000000012")
foreman_obj.AssignTasksToClient("C.1000000000000013")
foreman_obj.AssignTasksToClient("C.1000000000000014")
# Make sure that the clients ran the correct flows.
self.assertEqual(len(self.clients_started), 4)
self.assertEqual(self.clients_started[0][1], "C.1000000000000011")
self.assertEqual("H:222222", self.clients_started[0][0].Basename())
self.assertEqual(self.clients_started[1][1], "C.1000000000000012")
self.assertEqual("H:222222", self.clients_started[1][0].Basename())
self.assertEqual(self.clients_started[2][1], "C.1000000000000013")
self.assertEqual("H:111111", self.clients_started[2][0].Basename())
self.assertEqual(self.clients_started[3][1], "C.1000000000000014")
self.assertEqual("H:333333", self.clients_started[3][0].Basename())
def __init__(self):
self.stubber = utils.Stubber(report_plugins.REGISTRY, "plugins", {
"FooReportPlugin": FooReportPlugin,
"BarReportPlugin": BarReportPlugin
})
def _testUpdateSparseImageChunks(self):
"""Make sure the right chunks get updated when we read a sparse file."""
filename = "bigfile.txt"
path = os.path.join(self.temp_dir, filename)
chunksize = aff4_standard.AFF4SparseImage.chunksize
# 8 chunks of data.
contents = "bigdata!" * chunksize
# We want to start reading in the middle of a chunk.
start_point = int(2.5 * chunksize)
read_len = int(2.5 * chunksize)
client_path = self.ClientPathToAFF4Path(path)
with open(path, "wb") as f:
f.seek(start_point)
f.write(contents)
# Update the directory listing so we can see the file.
self.ListDirectoryOnClient(self.temp_dir)
# Make sure refreshing is allowed.
with utils.Stubber(self.grr_fuse, "max_age_before_refresh",
datetime.timedelta(seconds=0)):
# Read 3 chunks, from #2 to #4.
data = self.grr_fuse.Read(client_path,
length=read_len,
offset=start_point)
self.assertEqual(data,
contents[start_point:start_point + read_len],
"Fuse contents don't match.")
# Make sure it's an AFF4SparseImage
fd = aff4.FACTORY.Open(client_path, mode="rw", token=self.token)
self.assertIsInstance(fd, aff4_standard.AFF4SparseImage)
missing_chunks = self.grr_fuse.GetMissingChunks(fd,
length=10 * chunksize,
offset=0)
# 10 chunks but not #2 - #4 that we already got.
self.assertEqual(missing_chunks, [0, 1, 5, 6, 7, 8, 9])
# Make sure refreshing is allowed.
with utils.Stubber(self.grr_fuse, "max_age_before_refresh",
datetime.timedelta(seconds=0)):
# Now we read and make sure the contents are as we expect.
fuse_contents = self.grr_fuse.Read(client_path,
length=8 * chunksize,
offset=0)
expected_contents = ("\x00" * start_point + contents)[:8 *
chunksize]
self.assertEqual(fuse_contents, expected_contents,
"Fuse contents don't match.")
expected_contents = ("Y" * start_point + contents)[:8 * chunksize]
# Now, we'll write to the file in those previously missing chunks.
with open(path, "wb+") as f:
f.seek(0)
f.write(expected_contents)
# Enable refresh.
with utils.Stubber(self.grr_fuse, "max_age_before_refresh",
datetime.timedelta(seconds=0)):
fuse_contents = self.grr_fuse.Read(client_path,
length=len(contents),
offset=0)
self.assertEqual(fuse_contents, expected_contents,
"Fuse contents don't match.")
# Put a cache time back on, all chunks should be not missing.
self.grr_fuse.max_age_before_refresh = datetime.timedelta(seconds=30)
missing_chunks = self.grr_fuse.GetMissingChunks(fd,
length=8 * chunksize,
offset=0)
self.assertEqual(missing_chunks, [])
def testGlob(self):
"""Test that glob works properly."""
client_id = self.SetupClient(0)
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(rdf_client.User(username="******"))
kb.MergeOrAddUser(rdf_client.User(username="******"))
client.Set(kb)
client.Close()
client_mock = action_mocks.GlobClientMock()
# This glob selects all files which start with the username on this system.
paths = [
os.path.join(self.base_path, "%%Users.username%%*"),
os.path.join(self.base_path, "VFSFixture/var/*/wtmp")
]
# Set iterator really low to force iteration.
with utils.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2):
flow_test_lib.TestFlowHelper(
filesystem.Glob.__name__,
client_mock,
client_id=client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token,
sync=False,
check_flow_errors=False)
output_path = client_id.Add("fs/os").Add(
self.base_path.replace("\\", "/"))
children = []
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
filename = child.Basename()
if filename != "VFSFixture":
children.append(filename)
expected = [
filename for filename in os.listdir(self.base_path)
if filename.startswith("test") or filename.startswith("syslog")
]
self.assertTrue([x for x in expected if x.startswith("test")],
"Need a file starting with 'test'"
" in test_data for this test!")
self.assertTrue([x for x in expected if x.startswith("syslog")],
"Need a file starting with 'syslog'"
" in test_data for this test!")
self.assertItemsEqual(expected, children)
children = []
fd = aff4.FACTORY.Open(output_path.Add("VFSFixture/var/log"),
token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
self.assertItemsEqual(children, ["wtmp"])
def testGetClientSummary(self):
hostname = "test"
system = "Linux"
os_release = "12.02"
kernel = "3.15-rc2"
fqdn = "test.test.com"
arch = "amd64"
install_time = rdfvalue.RDFDatetime.Now()
user = "******"
userobj = rdf_client.User(username=user)
interface = rdf_client.Interface(ifname="eth0")
google_cloud_instance = rdf_cloud.GoogleCloudInstance(
instance_id="1771384456894610289",
zone="projects/123456789733/zones/us-central1-a",
project_id="myproject",
unique_id="us-central1-a/myproject/1771384456894610289")
cloud_instance = rdf_cloud.CloudInstance(
cloud_type="GOOGLE", google=google_cloud_instance)
serial_number = "DSD33679FZ"
system_manufacturer = "Foobar Inc."
system_uuid = "C31292AD-6Z4F-55D8-28AC-EC1100E42222"
hwinfo = rdf_client.HardwareInfo(
serial_number=serial_number,
system_manufacturer=system_manufacturer,
system_uuid=system_uuid)
timestamp = 1
with utils.Stubber(time, "time", lambda: timestamp):
with aff4.FACTORY.Create(
"C.0000000000000000",
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
kb = rdf_client.KnowledgeBase()
kb.users.Append(userobj)
empty_summary = fd.GetSummary()
self.assertEqual(empty_summary.client_id, "C.0000000000000000")
self.assertFalse(empty_summary.system_info.version)
self.assertEqual(empty_summary.timestamp.AsSecondsSinceEpoch(), 1)
# This will cause TYPE to be written with current time = 101 when the
# object is closed
timestamp += 100
fd.Set(fd.Schema.HOSTNAME(hostname))
fd.Set(fd.Schema.SYSTEM(system))
fd.Set(fd.Schema.OS_RELEASE(os_release))
fd.Set(fd.Schema.KERNEL(kernel))
fd.Set(fd.Schema.FQDN(fqdn))
fd.Set(fd.Schema.ARCH(arch))
fd.Set(fd.Schema.INSTALL_DATE(install_time))
fd.Set(fd.Schema.KNOWLEDGE_BASE(kb))
fd.Set(fd.Schema.USERNAMES([user]))
fd.Set(fd.Schema.HARDWARE_INFO(hwinfo))
fd.Set(fd.Schema.INTERFACES([interface]))
fd.Set(fd.Schema.CLOUD_INSTANCE(cloud_instance))
with aff4.FACTORY.Open(
"C.0000000000000000",
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
summary = fd.GetSummary()
self.assertEqual(summary.system_info.system, system)
self.assertEqual(summary.system_info.release, os_release)
self.assertEqual(summary.system_info.kernel, kernel)
self.assertEqual(summary.system_info.fqdn, fqdn)
self.assertEqual(summary.system_info.machine, arch)
self.assertEqual(summary.system_info.install_date, install_time)
self.assertItemsEqual(summary.users, [userobj])
self.assertItemsEqual(summary.interfaces, [interface])
self.assertFalse(summary.client_info)
self.assertEqual(summary.timestamp.AsSecondsSinceEpoch(), 101)
self.assertEqual(summary.cloud_type, "GOOGLE")
self.assertEqual(summary.cloud_instance_id,
"us-central1-a/myproject/1771384456894610289")
self.assertEqual(summary.serial_number, serial_number)
self.assertEqual(summary.system_manufacturer, system_manufacturer)
self.assertEqual(summary.system_uuid, system_uuid)
