def testApprovalExpiry(self):
"""Tests that approvals expire after the correct time."""
client_id = "C.%016X" % 0
urn = rdfvalue.ClientURN(client_id).Add("/fs/os/c")
token = access_control.ACLToken(username="******", reason="For testing")
self.assertRaises(access_control.UnauthorizedAccess, aff4.FACTORY.Open,
urn, None, "rw", token)
with test_lib.FakeTime(100.0):
self.GrantClientApproval(client_id, token)
# This should work now.
aff4.FACTORY.Open(urn, mode="rw", token=token)
# 3 weeks later.
with test_lib.FakeTime(100.0 + 3 * 7 * 24 * 60 * 60):
# This should still work.
aff4.FACTORY.Open(urn, mode="rw", token=token)
# Getting close.
with test_lib.Stubber(time, "time",
lambda: 100.0 + 4 * 7 * 24 * 60 * 60 - 100.0):
# This should still work.
aff4.FACTORY.Open(urn, mode="rw", token=token)
# Over 4 weeks now.
with test_lib.Stubber(time, "time",
lambda: 100.0 + 4 * 7 * 24 * 60 * 60 + 100.0):
self.assertRaises(access_control.UnauthorizedAccess,
aff4.FACTORY.Open, urn, None, "rw", token)
def testGrep(self):
class MockCallFlow(object):
def CallFlow(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
mock_call_flow = MockCallFlow()
with test_lib.Stubber(collectors.ArtifactCollectorFlow, "CallFlow",
mock_call_flow.CallFlow):
collect_flow = collectors.ArtifactCollectorFlow(None,
token=self.token)
collect_flow.state.Register("knowledge_base",
rdfvalue.KnowledgeBase())
collect_flow.current_artifact_name = "blah"
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
collector = rdfvalue.Collector(action="Grep",
args={
"path_list": ["/etc/passwd"],
"content_regex_list":
[r"^a%%users.username%%b$"]
})
collect_flow.Grep(collector, rdfvalue.PathSpec.PathType.TSK)
filters = mock_call_flow.kwargs["filters"]
regexes = [
f.contents_regex_match.regex.SerializeToString() for f in filters
]
self.assertItemsEqual(regexes, [r"^atest1b$", r"^atest2b$"])
self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testSearchDependencies(self):
with test_lib.Stubber(artifact_lib.ArtifactRegistry, "artifacts", {}):
# Just use the test artifacts to verify dependency correctness so we
# aren't subject to changing dependencies in the whole set
test_artifacts_file = os.path.join(
config_lib.CONFIG["Test.data_dir"], "test_artifacts.json")
artifact_lib.LoadArtifactsFromFiles([test_artifacts_file])
names, expansions = artifact_lib.ArtifactRegistry.SearchDependencies(
"Windows", [u"TestAggregationArtifactDeps", u"DepsParent"])
# This list contains all artifacts that can provide the dependency, e.g.
# DepsHomedir and DepsHomedir2 both provide
# users.homedir.
self.assertItemsEqual(names, [
u"DepsHomedir", u"DepsHomedir2", u"DepsDesktop", u"DepsParent",
u"DepsWindir", u"DepsWindirRegex", u"DepsControlSet",
u"TestAggregationArtifactDeps"
])
self.assertItemsEqual(expansions, [
"current_control_set", "users.homedir", "users.desktop",
"environ_windir", "users.username"
])
# None of these match the OS, so we should get an empty list.
names, expansions = artifact_lib.ArtifactRegistry.SearchDependencies(
"Darwin", [u"TestCmdArtifact", u"TestFileArtifact"])
self.assertItemsEqual(names, [])
def testGetByPriority(self):
priority1 = aff4.FACTORY.Create("aff4:/files/1",
"FileStore",
mode="rw",
token=self.token)
priority1.PRIORITY = 1
priority1.Set(priority1.Schema.ACTIVE(False))
priority2 = aff4.FACTORY.Create("aff4:/files/2",
"FileStore",
mode="rw",
token=self.token)
priority2.PRIORITY = 2
priority3 = aff4.FACTORY.Create("aff4:/files/3",
"FileStore",
mode="rw",
token=self.token)
priority3.PRIORITY = 3
fs = aff4.FACTORY.Open(filestore.FileStore.PATH,
"FileStore",
token=self.token)
with test_lib.Stubber(fs, "OpenChildren",
lambda: [priority3, priority1, priority2]):
child_list = list(fs.GetChildrenByPriority())
self.assertEqual(child_list[0].PRIORITY, 2)
self.assertEqual(child_list[1].PRIORITY, 3)
child_list = list(fs.GetChildrenByPriority(allow_external=False))
self.assertEqual(child_list[0].PRIORITY, 2)
def testFileAdd(self):
fs = aff4.FACTORY.Open(filestore.FileStore.PATH,
"FileStore",
token=self.token)
fake_store1 = FakeStore("aff4:/files/temp1", self.token)
fake_store2 = FakeStore("aff4:/files/temp2", self.token)
with test_lib.Stubber(fs, "OpenChildren",
lambda: [fake_store1, fake_store2]):
src_fd = aff4.FACTORY.Create(aff4.ROOT_URN.Add("temp").Add("src"),
"VFSBlobImage",
token=self.token,
mode="rw")
src_fd.SetChunksize(filestore.FileStore.CHUNK_SIZE)
src_data = "ABC" * filestore.FileStore.CHUNK_SIZE
src_data_fd = StringIO.StringIO(src_data)
src_fd.AppendContent(src_data_fd)
fs.AddFile(src_fd)
# Reset file pointers
src_fd.Seek(0)
fake_store1.dest_file.Seek(0)
fake_store2.dest_file.Seek(0)
# Check file content got written to both data stores.
self.assertEqual(src_data, fake_store1.dest_file.Read(-1))
self.assertEqual(src_data, fake_store2.dest_file.Read(-1))
def testGetHitsForHashesWithAge(self):
with test_lib.Stubber(time, "time", lambda: 42):
self.AddFile("/Ext2IFS_1_10b.exe")
self.AddFile("/idea.dll")
hash1 = rdfvalue.FileStoreHash(
fingerprint_type="generic",
hash_type="md5",
hash_value="bb0a15eefe63fd41f8dc9dee01c5cf9a")
hash2 = rdfvalue.FileStoreHash(
fingerprint_type="generic",
hash_type="sha1",
hash_value="e1f7e62b3909263f3a2518bbae6a9ee36d5b502b")
hits = dict(
aff4.HashFileStore.GetHitsForHashes([hash1, hash2],
age=41e6,
token=self.token))
self.assertEqual(len(hits), 0)
hits = dict(
aff4.HashFileStore.GetHitsForHashes([hash1, hash2],
age=43e6,
token=self.token))
self.assertEqual(len(hits), 2)
hits = dict(
aff4.HashFileStore.GetHitsForHashes([hash1, hash2],
token=self.token))
self.assertEqual(len(hits), 2)
def testGetHitsForHashWithAge(self):
with test_lib.Stubber(time, "time", lambda: 42):
self.AddFile("/Ext2IFS_1_10b.exe")
self.AddFile("/idea.dll")
hits = list(
aff4.HashFileStore.GetHitsForHash(rdfvalue.FileStoreHash(
fingerprint_type="generic",
hash_type="md5",
hash_value="bb0a15eefe63fd41f8dc9dee01c5cf9a"),
age=41e6,
token=self.token))
self.assertEqual(len(hits), 0)
hits = list(
aff4.HashFileStore.GetHitsForHash(rdfvalue.FileStoreHash(
fingerprint_type="generic",
hash_type="md5",
hash_value="bb0a15eefe63fd41f8dc9dee01c5cf9a"),
age=43e6,
token=self.token))
self.assertEqual(len(hits), 1)
hits = list(
aff4.HashFileStore.GetHitsForHash(rdfvalue.FileStoreHash(
fingerprint_type="generic",
hash_type="md5",
hash_value="bb0a15eefe63fd41f8dc9dee01c5cf9a"),
token=self.token))
self.assertEqual(len(hits), 1)
def testCallback(self, client_limit=None):
"""Checks that the foreman uses the callback specified in the action."""
with hunts.GRRHunt.StartHunt(
hunt_name="SampleHunt",
regex_rules=[rdfvalue.ForemanAttributeRegex(
attribute_name="GRR client",
attribute_regex="GRR")],
client_limit=client_limit,
client_rate=0,
token=self.token) as hunt:
with hunt.GetRunner() as runner:
runner.Start()
# Create a client that matches our regex.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
info = client.Schema.CLIENT_INFO()
info.client_name = "GRR Monitor"
client.Set(client.Schema.CLIENT_INFO, info)
client.Close()
foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
with test_lib.Stubber(hunts.SampleHunt, "StartClients", self.Callback):
self.called = []
foreman.AssignTasksToClient(client.urn)
self.assertEqual(len(self.called), 1)
self.assertEqual(self.called[0][1], [client.urn])
def testHuntResultsArrivingWhileOldResultsAreProcessedAreHandled(self):
self.StartHunt(output_plugins=[rdfvalue.OutputPlugin(
plugin_name="DummyHuntOutputPlugin")])
# Process hunt results.
self.ProcessHuntOutputPlugins()
# Check that nothing has happened because hunt hasn't reported any
# results yet.
self.assertEqual(DummyHuntOutputPlugin.num_calls, 0)
self.assertEqual(DummyHuntOutputPlugin.num_responses, 0)
# Generate new results while the plugin is working.
def ProcessResponsesStub(_, responses):
self.assertEqual(len(responses), 5)
self.AssignTasksToClients(self.client_ids[5:])
self.RunHunt(failrate=-1)
with test_lib.Stubber(DummyHuntOutputPlugin, "ProcessResponses",
ProcessResponsesStub):
# Process first 5 clients.
self.AssignTasksToClients(self.client_ids[:5])
self.RunHunt(failrate=-1)
self.ProcessHuntOutputPlugins()
self.ProcessHuntOutputPlugins()
# New results should get processed, even though they were added to the
# collection while plugin was processing previous results.
self.assertEqual(DummyHuntOutputPlugin.num_calls, 1)
self.assertEqual(DummyHuntOutputPlugin.num_responses, 5)
def testGetDependencies(self):
"""Test that dependencies are calculated correctly."""
self.SetupMocks()
with test_lib.Stubber(artifact_lib.ArtifactRegistry, "artifacts", {}):
test_artifacts_file = os.path.join(
config_lib.CONFIG["Test.data_dir"], "test_artifacts.json")
artifact_lib.LoadArtifactsFromFiles([test_artifacts_file])
config_lib.CONFIG.Set("Artifacts.knowledge_base", ["DepsParent",
"DepsDesktop",
"DepsHomedir",
"DepsWindir",
"DepsWindirRegex",
"DepsControlSet"])
config_lib.CONFIG.Set("Artifacts.knowledge_base_additions",
["DepsHomedir2"])
config_lib.CONFIG.Set("Artifacts.knowledge_base_skip", ["DepsWindir"])
kb_init = artifact.KnowledgeBaseInitializationFlow(None, token=self.token)
kb_init.state.Register("all_deps", set())
kb_init.state.Register("awaiting_deps_artifacts", [])
kb_init.state.Register("knowledge_base",
rdfvalue.KnowledgeBase(os="Windows"))
no_deps, all_deps, waiting = kb_init._GetDependencies()
self.assertItemsEqual(no_deps, ["DepsControlSet", "DepsHomedir2"])
self.assertItemsEqual(all_deps, ["users.homedir", "users.desktop",
"users.username", "environ_windir",
"current_control_set"])
self.assertItemsEqual(waiting, ["DepsParent", "DepsDesktop",
"DepsHomedir", "DepsWindirRegex"])
def testCmdArtifact(self):
"""Check we can run command based artifacts and get anomalies."""
class Popen(object):
"""A mock object for subprocess.Popen."""
def __init__(self, run, stdout, stderr, stdin):
Popen.running_args = run
Popen.stdout = stdout
Popen.stderr = stderr
Popen.stdin = stdin
Popen.returncode = 0
def communicate(self): # pylint: disable=g-bad-name
return "stdout here", "stderr here"
client_mock = self.MockClient("ExecuteCommand")
with test_lib.Stubber(subprocess, "Popen", Popen):
for _ in test_lib.TestFlowHelper(
"ArtifactCollectorFlow", client_mock, client_id=self.client_id,
store_results_in_aff4=True, use_tsk=False,
artifact_list=["TestCmdArtifact"], token=self.token):
pass
urn = self.client_id.Add("info/software")
fd = aff4.FACTORY.Open(urn, token=self.token)
packages = fd.Get(fd.Schema.INSTALLED_PACKAGES)
self.assertEquals(len(packages), 2)
self.assertEquals(packages[0].__class__.__name__, "SoftwarePackage")
with aff4.FACTORY.Open(self.client_id.Add("anomalies"),
token=self.token) as anomaly_coll:
self.assertEquals(len(anomaly_coll), 1)
self.assertTrue("gremlin" in anomaly_coll[0].symptom)
def testRunRaisingTask(self):
"""Tests the behavior of the pool if a task throws an exception."""
self.lock = threading.Lock()
def IRaise(some_obj):
"""This method just raises an exception."""
with self.lock:
# This simulates an error by calling a non-existent function.
some_obj.process()
self.exception_args = []
def MockException(*args):
self.exception_args = args
with test_lib.Stubber(logging, "exception", MockException):
self.test_pool.AddTask(IRaise, (None,), "Raising")
self.test_pool.AddTask(IRaise, (None,), "Raising")
self.test_pool.Join()
# Check that an exception is raised.
self.assertTrue(self.exception_args[0],
"exception in worker thread")
self.assertTrue(self.exception_args[1], "Raising")
# Make sure that both exceptions have been counted.
self.assertEqual(
stats.STATS.GetMetricValue(self.test_pool.name + "_task_exceptions"),
2)
def testRetryWrapper(self):
subject = "aff4:/subject"
data_store.DB.DeleteSubject(subject, token=self.token)
call_count = stats.STATS.GetMetricValue("datastore_retries")
def MockSleep(_):
pass
def Callback(unused_transaction):
# Now that we have a transaction, lets try to get another one on the same
# subject. Since it is locked this should retry.
try:
data_store.DB.RetryWrapper(subject,
lambda _: None,
token=self.token)
self.fail("Transaction error not raised.")
except data_store.TransactionError as e:
self.assertEqual("Retry number exceeded.", str(e))
self.assertEqual(
stats.STATS.GetMetricValue("datastore_retries") -
call_count, 10)
# By mocking out sleep we can ensure all retries are exhausted.
with test_lib.Stubber(time, "sleep", MockSleep):
data_store.DB.RetryWrapper(subject, Callback, token=self.token)
def testBadPickle(self):
"""Test that we can recover some of the bad pickle."""
state = rdfvalue.FlowState()
# Store an instance of a RDFURN here.
state.Register("urn", rdfvalue.RDFURN("aff4:/"))
serialized = state.SerializeToString()
# Substitute the class with something entirely different.
with test_lib.Stubber(rdfvalue, "RDFURN", None):
# We now should not be able to restore the state normally since we can not
# find the RDFURN instance.
result = rdfvalue.FlowState(serialized)
# The pickle error should be available here.
self.assertTrue(isinstance(result.errors, TypeError))
# The bad field should be replaced with an UnknownObject instance.
self.assertTrue(isinstance(result.urn, flows.UnknownObject))
# Missing attribute is a different kind of error, but this is still
# trapped.
del rdfvalue.RDFURN
result = rdfvalue.FlowState(serialized)
self.assertTrue(isinstance(result.errors, AttributeError))
self.assertTrue(isinstance(result.urn, flows.UnknownObject))
def testFailToCreateThread(self):
"""Test that we handle thread creation problems ok."""
# The pool starts off with the minimum number of threads.
self.assertEqual(len(self.test_pool), self.NUMBER_OF_THREADS)
done_event = threading.Event()
def Block(done):
done.wait()
def RaisingStart(_):
raise threading.ThreadError()
# Now simulate failure of creating threads.
with test_lib.Stubber(threadpool._WorkerThread, "start", RaisingStart):
# Fill all the existing threads and wait for them to become busy.
self.test_pool.AddTask(Block, (done_event,))
self.WaitUntil(
lambda: self.test_pool.busy_threads == self.NUMBER_OF_THREADS)
# Now fill the queue completely..
for _ in range(self.MAXIMUM_THREADS):
self.test_pool.AddTask(Block, (done_event,))
# Trying to push this task will overflow the queue, and would normally
# cause a new thread to start. We use non blocking mode to receive the
# exception.
self.assertRaises(
threadpool.Full,
self.test_pool.AddTask, Block, (done_event,), blocking=False,
inline=False)
# Release the blocking tasks.
done_event.set()
self.test_pool.Join()
def testExecuteBinariesWithArgs(self):
client_mock = test_lib.ActionMock("ExecuteBinaryCommand")
code = "I am a binary file"
upload_path = config_lib.CONFIG["Executables.aff4_path"].Add(
"test.exe")
maintenance_utils.UploadSignedConfigBlob(code,
aff4_path=upload_path,
token=self.token)
class Popen(object):
"""A mock object for subprocess.Popen."""
def __init__(self, run, stdout, stderr, stdin):
Popen.running_args = run
Popen.stdout = stdout
Popen.stderr = stderr
Popen.stdin = stdin
Popen.returncode = 0
# Store the content of the executable file.
Popen.binary = open(run[0]).read()
def communicate(self): # pylint: disable=g-bad-name
return "stdout here", "stderr here"
# This flow has an acl, the user needs to be admin.
user = aff4.FACTORY.Create("aff4:/users/%s" % self.token.username,
mode="rw",
aff4_type="GRRUser",
token=self.token)
user.SetLabels("admin")
user.Close()
with test_lib.Stubber(subprocess, "Popen", Popen):
for _ in test_lib.TestFlowHelper("LaunchBinary",
client_mock,
client_id=self.client_id,
binary=upload_path,
command_line="--value 356",
token=self.token):
pass
# Check that the executable file contains the code string.
self.assertEqual(Popen.binary, code)
# At this point, the actual binary should have been cleaned up by the
# client action so it should not exist.
self.assertRaises(IOError, open, Popen.running_args[0])
# Check the binary was run with the correct command line.
self.assertEqual(Popen.running_args[1], "--value")
self.assertEqual(Popen.running_args[2], "356")
# Check the command was in the tmp file.
self.assertTrue(Popen.running_args[0].startswith(
config_lib.CONFIG["Client.tempdir"]))
def _RunGlob(self, paths):
self.flow_replies = []
client_mock = test_lib.ActionMock("Find", "StatFile")
with test_lib.Stubber(flow.GRRFlow, "SendReply", self._MockSendReply):
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdfvalue.PathSpec.PathType.OS,
token=self.token):
pass
def testHashAgeUpdatedWhenNewHitAddedAfterAFF4IndexCacheAge(self):
# Check that there are no hashes.
hashes = list(
aff4.HashFileStore.ListHashes(token=self.token, age=(41e6, 1e10)))
self.assertEqual(len(hashes), 0)
with test_lib.Stubber(time, "time", lambda: 42):
self.AddFileToFileStore(rdfvalue.PathSpec(
pathtype=rdfvalue.PathSpec.PathType.OS,
path=os.path.join(self.base_path, "empty_file")),
client_id=self.client_id,
token=self.token)
hashes = list(
aff4.HashFileStore.ListHashes(token=self.token, age=(41e6, 1e10)))
self.assertTrue(hashes)
hits = list(
aff4.HashFileStore.GetHitsForHash(hashes[0], token=self.token))
self.assertEqual(len(hits), 1)
latest_time = 42 + config_lib.CONFIG["AFF4.intermediate_cache_age"] + 1
with test_lib.Stubber(time, "time", lambda: latest_time):
self.AddFileToFileStore(rdfvalue.PathSpec(
pathtype=rdfvalue.PathSpec.PathType.OS,
path=os.path.join(self.base_path, "a", "b", "c",
"helloc.txt")),
client_id=self.client_id,
token=self.token)
# Check that now we have two hits for the previosly added hash.
hits = list(
aff4.HashFileStore.GetHitsForHash(hashes[0], token=self.token))
self.assertEqual(len(hits), 2)
# Check that new hit affects hash age.
hashes = list(
aff4.HashFileStore.ListHashes(token=self.token, age=(43e6, 1e10)))
self.assertTrue(hashes)
def testListHashesWithAge(self):
with test_lib.Stubber(time, "time", lambda: 42):
self.AddFile("/Ext2IFS_1_10b.exe")
hashes = list(aff4.HashFileStore.ListHashes(token=self.token,
age=41e6))
self.assertEqual(len(hashes), 0)
hashes = list(aff4.HashFileStore.ListHashes(token=self.token,
age=43e6))
self.assertEqual(len(hashes), 5)
hashes = list(aff4.HashFileStore.ListHashes(token=self.token))
self.assertEqual(len(hashes), 5)
def testGlob(self):
"""Test that glob works properly."""
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
users = client.Schema.USER()
users.Append(username="******")
users.Append(username="******")
client.Set(users)
client.Close()
client_mock = test_lib.ActionMock("Find", "StatFile")
# This glob selects all files which start with the username on this system.
paths = [
os.path.join(self.base_path, "%%Users.username%%*"),
os.path.join(self.base_path, "wtmp")
]
# Set iterator really low to force iteration.
with test_lib.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2):
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdfvalue.PathSpec.PathType.OS,
token=self.token,
sync=False,
check_flow_errors=False):
pass
output_path = self.client_id.Add("fs/os").Add(
self.base_path.replace("\\", "/"))
children = []
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
# We should find some files.
self.assertEqual(
sorted(children),
sorted([
"syslog", "syslog_compress.gz", "syslog_false.gz",
"test_artifacts.json", "test_artifact.json", "test_img.dd",
"test.plist", "tests", "tests_long", "wtmp"
]))
def GetVFSHandler(self):
def FakeOpen(filename, mode="r"):
"""The linux driver just opens the device."""
# It uses /proc/iomem to find the protected areas.
if filename == "/proc/iomem":
return StringIO.StringIO(self.IOMEM)
self.assertEqual(filename, "/dev/pmem")
self.assertEqual(mode, "rb")
return FakeFile()
with test_lib.Stubber(memory, "open", FakeOpen):
result = memory.LinuxMemory(
None,
pathspec=rdfvalue.PathSpec(
path="/dev/pmem",
pathtype=rdfvalue.PathSpec.PathType.MEMORY))
self.assertEqual(result.size, 0x82fffffff)
return result
def testBreakGlass(self):
"""Test the breakglass mechanism."""
client_id = rdfvalue.ClientURN("C.%016X" % 0)
urn = client_id.Add("/fs/os/c")
self.assertRaises(access_control.UnauthorizedAccess,
aff4.FACTORY.Open,
urn,
token=self.token)
# We expect to receive an email about this
email = {}
def SendEmail(to, from_user, subject, message, **_):
email["to"] = to
email["from_user"] = from_user
email["subject"] = subject
email["message"] = message
with test_lib.Stubber(email_alerts, "SendEmail", SendEmail):
flow.GRRFlow.StartFlow(
client_id=client_id,
flow_name="BreakGlassGrantClientApprovalFlow",
token=self.token,
reason=self.token.reason)
# Reset the emergency state of the token.
self.token.is_emergency = False
# This access is using the emergency_access granted, so we expect the
# token to be tagged as such.
aff4.FACTORY.Open(urn, token=self.token)
self.assertEqual(
email["to"],
config_lib.CONFIG["Monitoring.emergency_access_email"])
self.assert_(self.token.username in email["message"])
self.assertEqual(email["from_user"], self.token.username)
# Make sure the token is tagged as an emergency token:
self.assertEqual(self.token.is_emergency, True)
def testProcessHuntResultsCronFlowDoesNotAbortsIfRunningInTime(self):
self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 0)
test = [0]
def TimeStub():
test[0] += 1e-6
return test[0]
with test_lib.Stubber(time, "time", TimeStub):
self.StartHunt(output_plugins=[rdfvalue.OutputPlugin(
plugin_name="LongRunningDummyHuntOutputPlugin")])
self.AssignTasksToClients()
self.RunHunt(failrate=-1)
# LongRunningDummyHuntOutputPlugin will set the time to 100s on the first
# run, which will effectively mean that it's running in time.
self.ProcessHuntOutputPlugins(batch_size=1,
max_running_time=rdfvalue.Duration("101s"))
# In normal conditions, there should be 10 results generated.
self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 10)
def testAllConfigs(self):
"""Go through all our config files looking for errors."""
configs = []
# Test the current loaded configuration.
configs.append(config_lib.CONFIG)
test_filter_map = config_lib.ConfigFilter.classes_by_name
for filter_name in self.disabled_filters:
test_filter_map[filter_name] = config_lib.ConfigFilter
with test_lib.Stubber(config_lib.ConfigFilter, "classes_by_name",
test_filter_map):
for config_file in configs:
errors = ValidateConfig(config_file)
for exception in self.exceptions:
errors.pop(exception, None)
if errors:
self.fail("Validation of %s returned errors: %s" %
(config_file, errors))
def testExportCollectionWithEmailPlugin(self):
# Create a collection with URNs to some files.
fd = aff4.FACTORY.Create("aff4:/testcoll",
"RDFValueCollection",
token=self.token)
fd.Add(
rdfvalue.GrrMessage(
payload=rdfvalue.StatEntry(aff4path=self.out.Add("testfile")),
source=self.client_id))
fd.Close()
plugin = collection_plugin.CollectionExportPlugin()
parser = argparse.ArgumentParser()
plugin.ConfigureArgParser(parser)
def SendEmail(address, sender, title, message, **_):
self.email_messages.append(
dict(address=address,
sender=sender,
title=title,
message=message))
email_address = "notify@%s" % config_lib.CONFIG["Logging.domain"]
with test_lib.Stubber(email_alerts, "SendEmail", SendEmail):
self.email_messages = []
plugin.Run(
parser.parse_args(args=[
"--path", "aff4:/testcoll", "email", "--email",
email_address, "--email_limit", "100"
]))
self.assertEqual(len(self.email_messages), 1)
for msg in self.email_messages:
self.assertEqual(msg["address"], email_address)
self.assertEqual(
"GRR Hunt results collection aff4:/testcoll got a new "
"result.", msg["title"])
self.assertTrue("testfile" in msg["message"])
def testProcessHuntResultsCronFlowAbortsIfRunningTooLong(self):
self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 0)
test = [0]
def TimeStub():
test[0] += 1e-6
return test[0]
with test_lib.Stubber(time, "time", TimeStub):
self.StartHunt(output_plugins=[rdfvalue.OutputPlugin(
plugin_name="LongRunningDummyHuntOutputPlugin")])
self.AssignTasksToClients()
self.RunHunt(failrate=-1)
# LongRunningDummyHuntOutputPlugin will set the time to 100s on the first
# run, which will effectively mean that it's running for too long.
self.ProcessHuntOutputPlugins(batch_size=1,
max_running_time=rdfvalue.Duration("99s"))
# In normal conditions, there should be 10 results generated.
# With batch size of 1 this should result in 10 calls to output plugin.
# But as we were using TimeStub, the flow should have aborted after 1
# call.
self.assertEqual(LongRunningDummyHuntOutputPlugin.num_calls, 1)
def testClientKilled(self):
"""Test that client killed messages are handled correctly."""
self.email_message = {}
def SendEmail(address, sender, title, message, **_):
self.email_message.update(
dict(address=address,
sender=sender,
title=title,
message=message))
with test_lib.Stubber(email_alerts, "SendEmail", SendEmail):
client = test_lib.CrashClientMock(self.client_id, self.token)
for _ in test_lib.TestFlowHelper(
"ListDirectory",
client,
client_id=self.client_id,
pathspec=rdfvalue.PathSpec(path="/"),
token=self.token,
check_flow_errors=False):
pass
# We expect the email to be sent.
self.assertEqual(self.email_message.get("address", ""),
config_lib.CONFIG["Monitoring.alert_email"])
self.assertTrue(str(self.client_id) in self.email_message["title"])
# Make sure the flow state is included in the email message.
for s in ["Flow name", "ListDirectory", "current_state"]:
self.assertTrue(s in self.email_message["message"])
flow_obj = aff4.FACTORY.Open(client.flow_id,
age=aff4.ALL_TIMES,
token=self.token)
self.assertEqual(flow_obj.state.context.state,
rdfvalue.Flow.State.ERROR)
# Make sure crashes RDFValueCollections are created and written
# into proper locations. First check the per-client crashes collection.
client_crashes = sorted(list(
aff4.FACTORY.Open(self.client_id.Add("crashes"),
aff4_type="PackedVersionedCollection",
token=self.token)),
key=lambda x: x.timestamp)
self.assertTrue(len(client_crashes) >= 1)
crash = list(client_crashes)[0]
self.assertEqual(crash.client_id, self.client_id)
self.assertEqual(crash.session_id, flow_obj.session_id)
self.assertEqual(crash.client_info.client_name, "GRR Monitor")
self.assertEqual(crash.crash_type, "aff4:/flows/W:CrashHandler")
self.assertEqual(crash.crash_message,
"Client killed during transaction")
# Check per-flow crash collection. Check that crash written there is
# equal to per-client crash.
flow_crashes = sorted(list(
flow_obj.GetValuesForAttribute(flow_obj.Schema.CLIENT_CRASH)),
key=lambda x: x.timestamp)
self.assertEqual(len(flow_crashes), len(client_crashes))
for a, b in zip(flow_crashes, client_crashes):
self.assertEqual(a, b)
# Check global crash collection. Check that crash written there is
# equal to per-client crash.
global_crashes = sorted(aff4.FACTORY.Open(
aff4.ROOT_URN.Add("crashes"),
aff4_type="PackedVersionedCollection",
token=self.token),
key=lambda x: x.timestamp)
self.assertEqual(len(global_crashes), len(client_crashes))
for a, b in zip(global_crashes, client_crashes):
self.assertEqual(a, b)
def testFileView(self):
"""Test the fileview interface."""
# This is ugly :( Django gets confused when you import in the wrong order
# though and fileview imports the Django http module so we have to delay
# import until the Django server is properly set up.
# pylint: disable=g-import-not-at-top
from grr.gui.plugins import fileview
# pylint: enable=g-import-not-at-top
# Set up multiple version for an attribute on the client for tests.
with self.ACLChecksDisabled():
for fake_time, hostname in [(1333788833, "HostnameV1"),
(1333888833, "HostnameV2"),
(1333988833, "HostnameV3")]:
with test_lib.FakeTime(fake_time):
client = aff4.FACTORY.Open(u"C.0000000000000001",
mode="rw",
token=self.token)
client.Set(client.Schema.HOSTNAME(hostname))
client.Close()
self.Open("/")
self.Type("client_query", "0001")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000001", self.GetText,
"css=span[type=subject]")
# Choose client 1
self.Click("css=td:contains('0001')")
# Go to Browse VFS
self.Click("css=a:contains('Browse Virtual Filesystem')")
# Test the historical view for AFF4 elements.
self.Click("css=#[attribute=HOSTNAME] > ins")
self.WaitUntil(self.AllTextsPresent,
["HostnameV1", "HostnameV2", "HostnameV3"])
self.Click("css=#[attribute=HOSTNAME] > ins")
self.WaitUntilNot(self.IsTextPresent, "HostnameV1")
self.WaitUntilNot(self.IsTextPresent, "HostnameV2")
self.Click("css=#_fs ins.jstree-icon")
self.Click("css=#_fs-os ins.jstree-icon")
self.Click("css=#_fs-os-c ins.jstree-icon")
# Test file versioning.
self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads")
self.Click("link=Downloads")
# Verify that we have the latest version in the table by default
self.assertTrue("2012-04-09 16:27:13" in self.GetText(
"css=tr:contains(\"a.txt\")"))
# Click on the row.
self.Click("css=tr:contains(\"a.txt\")")
self.WaitUntilContains("a.txt @ 2012-04-09", self.GetText,
"css=div#main_rightBottomPane h3")
# Check the data in this file.
self.Click("css=#TextView")
self.WaitUntilContains("Goodbye World", self.GetText,
"css=div#text_viewer_data_content")
downloaded_files = []
def FakeDownload(unused_self, request, _):
aff4_path = request.REQ.get("aff4_path")
age = rdfvalue.RDFDatetime(
request.REQ.get("age")) or aff4.NEWEST_TIME
downloaded_files.append((aff4_path, age))
return fileview.http.HttpResponse(
content="<script>window.close()</script>")
with test_lib.Stubber(fileview.DownloadView, "Download", FakeDownload):
# Try to download the file.
self.Click("css=#Download")
self.WaitUntil(self.IsTextPresent,
"As downloaded on 2012-04-09 16:27:13")
self.Click("css=button:contains(\"Download\")")
# Click on the version selector.
self.Click("css=tr:contains(\"a.txt\") img.version-selector")
self.WaitUntilContains("Versions of", self.GetText,
"css=.version-selector-dialog h4")
# Select the previous version.
self.Click("css=td:contains(\"2012-04-07\")")
# Now we should have a different time.
self.WaitUntil(self.IsTextPresent,
"As downloaded on 2012-04-07 08:53:53")
self.Click("css=button:contains(\"Download\")")
self.WaitUntil(self.IsElementPresent, "css=#TextView")
self.WaitUntil(lambda: len(downloaded_files) == 2)
# Both files should be the same...
self.assertEqual(downloaded_files[0][0],
u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt")
self.assertEqual(downloaded_files[1][0],
u"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt")
# But from different times.
self.assertEqual(downloaded_files[0][1], 1333988833000000)
self.assertEqual(downloaded_files[1][1], 1333788833000000)
self.Click("css=#TextView")
# Make sure the file content has changed. This version has "Hello World" in
# it.
self.WaitUntilContains("Hello World", self.GetText,
"css=div#text_viewer_data_content")
# Test the hex viewer.
self.Click("css=#_fs-os-proc ins.jstree-icon")
self.Click("css=#_fs-os-proc-10 a")
self.Click("css=span[type=subject]:contains(\"cmdline\")")
target_aff4_path = "aff4:/C.0000000000000001/fs/os/proc/10/cmdline"
self.Click("css=[state-aff4_path='%s'] > li > #HexView" %
target_aff4_path)
for i, value in enumerate(
"6c 73 00 68 65 6c 6c 6f 20 77 6f 72 6c 64 27 00 2d 6c".split(
" ")):
self.WaitUntilEqual(value, self.GetText,
"css=#hex_area tr:first td:nth(%d)" % i)
for i, value in enumerate(
"l s . h e l l o w o r l d ' . - l".split(" ")):
self.WaitUntilEqual(value, self.GetText,
"css=#data_area tr:first td:nth(%d)" % i)
self.Click("css=a[renderer=\"AFF4Stats\"]")
# Navigate to the bin C.0000000000000001 directory
self.Click("link=bin C.0000000000000001")
# Filter the table for bash (should match both bash and rbash)
self.WaitUntil(self.IsElementPresent, "css=td:contains('bash')")
self.Click("css=th:contains('Name') img")
self.Type("css=.sort-dialog input[type=text]",
"bash",
end_with_enter=True)
self.WaitUntilEqual("rbash", self.GetText, "css=tr:nth(2) span")
self.assertEqual(
2, self.GetCssCount("css=#main_rightTopPane tbody > tr"))
self.assertEqual("bash", self.GetText("css=tr:nth(1) span"))
self.assertEqual("rbash", self.GetText("css=tr:nth(2) span"))
# Check that the previous search test is still available in the form.
self.Click("css=th:contains('Name') img")
self.assertEqual("bash", self.GetValue("css=.sort-dialog input"))
# If we anchor cat at the start should only filter one.
self.Type("css=.sort-dialog input[type=text]",
"^cat",
end_with_enter=True)
self.WaitUntilEqual("cat", self.GetText, "css=tr:nth(1) span")
self.assertEqual(
1, self.GetCssCount("css=#main_rightTopPane tbody > tr"))
self.Click("css=tr:nth(1)")
self.WaitUntilContains(
"aff4:/C.0000000000000001/fs/os/c/bin C.0000000000000001/cat",
self.GetText, "css=.tab-content h3")
self.WaitUntil(self.IsTextPresent, "1026267") ## st_inode.
# Lets download it.
self.Click("Download")
self.Click("css=button:contains(\"Get a new Version\")")
self.Click("path_0")
self.WaitUntilEqual("fs", self.GetText, "css=tr td span:contains(fs)")
self.Click("Stats")
self.WaitUntilContains("aff4:/C.0000000000000001", self.GetText,
"css=.tab-content h3")
# Grab the root directory again - should produce an Interrogate flow.
self.Click("css=button[id^=refresh]")
# Go to the flow management screen.
self.Click("css=a:contains('Manage launched flows')")
# For the client update, 2 flows have to be issued: UpdateVFSFile and
# Interrogate. UpdateVFSFile triggers VFSGRRClient.Update() method which
# triggers Interrogate.
self.WaitUntilEqual("Interrogate", self.GetText,
"//table/tbody/tr[1]/td[3]")
self.WaitUntilEqual("UpdateVFSFile", self.GetText,
"//table/tbody/tr[2]/td[3]")
self.Click("//table/tbody/tr[2]/td[3]")
self.WaitUntilEqual(
"aff4:/C.0000000000000001", self.GetText,
"css=table > tbody td.proto_key:contains(\"Vfs file urn\") "
"~ td.proto_value")
# Check that UpdateVFSFile is called for the cat file.
# During the test this file is VFSMemoryFile, so its' Update method does
# nothing, therefore UpdateVFSFile won't issue any other flows.
self.WaitUntilEqual("UpdateVFSFile", self.GetText,
"//table/tbody/tr[3]/td[3]")
self.Click("//table/tbody/tr[3]/td[3]")
self.WaitUntilContains(
"cat", self.GetText,
"css=table > tbody td.proto_key:contains(\"Vfs file urn\") "
"~ td.proto_value")
def testPartialRead(self):
with test_lib.Stubber(memory, "win32file", Win32FileMock()):
super(TestWindowsMemory, self).testPartialRead()
def testBlockingTasks(self):
# The pool starts off with the minimum number of threads.
self.assertEqual(len(self.test_pool), self.NUMBER_OF_THREADS)
done_event = threading.Event()
self.lock = threading.Lock()
res = []
def Block(done):
done.wait()
def Insert(list_obj, element):
with self.lock:
list_obj.append(element)
# Ensure that the thread pool is able to add the correct number of threads
# deterministically.
with test_lib.Stubber(self.test_pool, "CPUUsage", lambda: 0):
# Schedule the maximum number of threads of blocking tasks and the same of
# insert tasks. The threads are now all blocked, and the inserts are
# waiting in the queue.
for _ in range(self.MAXIMUM_THREADS):
self.test_pool.AddTask(Block, (done_event,), "Blocking")
# Wait until the threadpool picks up the task.
self.WaitUntil(
lambda: self.test_pool.busy_threads == self.NUMBER_OF_THREADS)
# Now there are minimum number of threads active and the rest are sitting
# on the queue.
self.assertEqual(self.test_pool.pending_tasks,
self.MAXIMUM_THREADS - self.NUMBER_OF_THREADS)
# Now as we push these tasks on the queue, new threads will be created and
# they will receive the blocking tasks from the queue.
for i in range(self.MAXIMUM_THREADS):
self.test_pool.AddTask(Insert, (res, i), "Insert", blocking=True,
inline=False)
# There should be 20 workers created and they should consume all the
# blocking tasks.
self.WaitUntil(
lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS)
# No Insert tasks are running yet.
self.assertEqual(res, [])
# There are 20 tasks waiting on the queue.
self.assertEqual(self.test_pool.pending_tasks, self.MAXIMUM_THREADS)
# Inserting more tasks than the queue can hold should lead to processing
# the tasks inline. This effectively causes these tasks to skip over the
# tasks which are waiting in the queue.
for i in range(10, 20):
self.test_pool.AddTask(Insert, (res, i), "Insert", inline=True)
res.sort()
self.assertEqual(res, range(10, 20))
# This should release all the busy tasks. It will also cause the workers
# to process all the Insert tasks in the queue.
done_event.set()
self.test_pool.Join()
# Now the rest of the tasks should have been processed as well.
self.assertEqual(sorted(res[10:]), range(20))
