def testUpdateSparseImageChunks(self):
"""Make sure the right chunks get updated when we read a sparse file."""
with utils.MultiStubber((self.grr_fuse, "force_sparse_image", True),
(self.grr_fuse, "max_age_before_refresh",
datetime.timedelta(seconds=30)),
(self.grr_fuse, "size_threshold", 0)):
self._testUpdateSparseImageChunks()
def testRegions(self):
def MockedOpen64(requested_path, mode="rb"):
del mode
if "proc/100/mem" in requested_path._obj.value:
return True
raise OSError("Error in open64.")
def MockedOpen(requested_path, mode="rb"):
if "proc/100/maps" in requested_path:
return open.old_target(os.path.join(self.base_path, "maps"),
mode=mode)
raise OSError("Error in open.")
with utils.MultiStubber((__builtin__, "open", MockedOpen),
(process, "open64", MockedOpen64)):
with process.Process(pid=100) as proc:
self.assertEqual(len(list(proc.Regions())), 32)
self.assertEqual(
len(list(proc.Regions(skip_mapped_files=True))), 10)
self.assertEqual(
len(list(proc.Regions(skip_shared_regions=True))), 31)
self.assertEqual(
len(list(proc.Regions(skip_executable_regions=True))), 27)
self.assertEqual(
len(list(proc.Regions(skip_readonly_regions=True))), 10)
self.assertEqual(
len(
list(
proc.Regions(skip_executable_regions=True,
skip_shared_regions=True))), 26)
def testEnumerateUsersLinux(self):
"""Enumerate users from the wtmp file."""
def MockedOpen(requested_path, mode="rb"):
try:
fixture_path = os.path.join(self.base_path, "VFSFixture",
requested_path.lstrip("/"))
return __builtin__.open.old_target(fixture_path, mode)
except IOError:
return __builtin__.open.old_target(requested_path, mode)
with utils.MultiStubber((__builtin__, "open", MockedOpen),
(glob, "glob", lambda x: ["/var/log/wtmp"])):
results = self.RunAction(linux.EnumerateUsers)
found = 0
for result in results:
if result.username == "user1":
found += 1
self.assertEqual(result.last_logon, 1296552099 * 1000000)
elif result.username == "user2":
found += 1
self.assertEqual(result.last_logon, 1296552102 * 1000000)
elif result.username == "user3":
found += 1
self.assertEqual(result.last_logon, 1296569997 * 1000000)
elif result.username == "utuser":
self.assertEqual(result.last_logon, 1510318881 * 1000000)
else:
self.fail("Unexpected user found: %s" % result.username)
self.assertEqual(found, 3)
def testCallClientChildFlowRace(self):
close_call_times = []
def Close(self):
if isinstance(self, CallClientParentFlow):
close_call_times.append(time.time())
return aff4.AFF4Object.Close.old_target(self)
qst_call_times = []
def QueueScheduleTasks(self, *args):
qst_call_times.append(time.time())
return data_store.DB.mutation_pool_cls.QueueScheduleTasks.old_target(
self, *args)
with utils.MultiStubber(
(aff4.AFF4Object, "Close", Close),
# This is a pool method and will only get written once the
# pool is flushed. The flush will happen after the queuing and
# doesn't change the test.
(data_store.DB.mutation_pool_cls, "QueueScheduleTasks",
QueueScheduleTasks)):
flow.StartFlow(client_id=self.client_id,
flow_name="CallClientParentFlow",
token=self.token)
self.assertEqual(len(close_call_times), 1)
self.assertEqual(len(qst_call_times), 1)
# Check that the client request was written after the flow was created.
self.assertLess(
close_call_times[0], qst_call_times[0],
"The client request was issued before "
"the flow was created.")
def testAutoIndexing(self):
indexing_done = threading.Event()
def UpdateIndex(_):
indexing_done.set()
# To reduce the time for the test to run, reduce the delays, so that
# indexing should happen instantaneously.
isq = sequential_collection.IndexedSequentialCollection
biu = sequential_collection.BACKGROUND_INDEX_UPDATER
with utils.MultiStubber(
(biu, "INDEX_DELAY", 0),
(isq, "INDEX_WRITE_DELAY", rdfvalue.Duration("0s")),
(isq, "INDEX_SPACING", 8), (isq, "UpdateIndex", UpdateIndex)):
urn = "aff4:/sequential_collection/testAutoIndexing"
collection = self._TestCollection(urn)
# TODO(amoser): Without using a mutation pool, this test is really
# slow on MySQL data store.
with data_store.DB.GetMutationPool() as pool:
for i in range(2048):
collection.StaticAdd(rdfvalue.RDFURN(urn),
rdfvalue.RDFInteger(i),
mutation_pool=pool)
# Wait for the updater thread to finish the indexing.
if not indexing_done.wait(timeout=10):
self.fail("Indexing did not finish in time.")
def testThreadsReaped(self):
"""Check that threads are reaped when too old."""
self.now = 0
with utils.MultiStubber((time, "time", lambda: self.now),
(threading, "_time", lambda: self.now),
(Queue, "_time", lambda: self.now),
(self.test_pool, "CPUUsage", lambda: 0)):
done_event = threading.Event()
res = []
def Block(done, count):
done.wait()
res.append(count)
for i in range(2 * self.MAXIMUM_THREADS):
self.test_pool.AddTask(Block, (done_event, i),
"Blocking",
inline=False)
self.assertEqual(len(self.test_pool), self.MAXIMUM_THREADS)
# Release the threads. All threads are now idle.
done_event.set()
# Fast forward the time
self.now = 1000
# Threads will now kill themselves off and the threadpool will be reduced
# to the minimum number of threads..
self.WaitUntil(
lambda: len(self.test_pool) == self.NUMBER_OF_THREADS)
# Ensure we have the minimum number of threads left now.
self.assertEqual(len(self.test_pool), self.NUMBER_OF_THREADS)
def Run(self):
with utils.MultiStubber(
(output_plugin.OutputPlugin, "classes", {
"EmailOutputPlugin": email_plugin.EmailOutputPlugin,
}), (instant_output_plugin.InstantOutputPlugin, "classes", {
"CSVInstantOutputPlugin": csv_plugin.CSVInstantOutputPlugin
})):
self.Check("ListOutputPluginDescriptors")
def MakeRequest(self,
instrumentor,
manager,
path,
verify_cb=lambda x: True):
with utils.MultiStubber((requests, "request", instrumentor.request),
(time, "sleep", instrumentor.sleep)):
return manager.OpenServerEndpoint(path, verify_cb=verify_cb)
def instrument(self):
"""Install the mocks required.
Returns:
A context manager that when exits restores the mocks.
"""
self.actions = []
return utils.MultiStubber((requests, "request", self.request),
(time, "sleep", self.sleep))
def setUp(self):
super(GRRBaseTest, self).setUp()
self.temp_dir = TempDirPath()
config.CONFIG.SetWriteBack(os.path.join(self.temp_dir, "writeback.yaml"))
logging.info("Starting test: %s.%s", self.__class__.__name__,
self._testMethodName)
self.last_start_time = time.time()
data_store.DB.ClearTestDB()
# Each datastore is wrapped with DatabaseValidationWrapper, so we have
# to access the delegate directly (assuming it's an InMemoryDB
# implementation).
data_store.REL_DB.delegate.ClearTestDB()
aff4.FACTORY.Flush()
# Create a Foreman and Filestores, they are used in many tests.
aff4_grr.GRRAFF4Init().Run()
filestore.FileStoreInit().Run()
hunts_results.ResultQueueInitHook().Run()
email_alerts.EmailAlerterInit().RunOnce()
audit.AuditEventListener.created_logs.clear()
# Stub out the email function
self.emails_sent = []
def SendEmailStub(to_user, from_user, subject, message, **unused_kwargs):
self.emails_sent.append((to_user, from_user, subject, message))
self.mail_stubber = utils.MultiStubber(
(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmailStub),
(email.utils, "make_msgid", lambda: "<message id stub>"))
self.mail_stubber.Start()
# We don't want to send actual email in our tests
self.smtp_patcher = mock.patch("smtplib.SMTP")
self.mock_smtp = self.smtp_patcher.start()
def DisabledSet(*unused_args, **unused_kw):
raise NotImplementedError(
"Usage of Set() is disabled, please use a configoverrider in tests.")
self.config_set_disable = utils.Stubber(config.CONFIG, "Set", DisabledSet)
self.config_set_disable.Start()
if self.use_relational_reads:
self.relational_read_stubber = utils.Stubber(
data_store, "RelationalDBReadEnabled", lambda: True)
self.relational_read_stubber.Start()
def testStatFS(self):
f_bsize = 4096
# Simulate pre-2.6 kernel
f_frsize = 0
f_blocks = 9743394
f_bfree = 5690052
f_bavail = 5201809
f_files = 2441216
f_ffree = 2074221
f_favail = 2074221
f_flag = 4096
f_namemax = 255
def MockStatFS(unused_path):
return posix.statvfs_result(
(f_bsize, f_frsize, f_blocks, f_bfree, f_bavail, f_files, f_ffree,
f_favail, f_flag, f_namemax))
def MockIsMount(path):
"""Only return True for the root path."""
return path == "/"
with utils.MultiStubber((os, "statvfs", MockStatFS),
(os.path, "ismount", MockIsMount)):
# This test assumes "/" is the mount point for /usr/bin
results = self.RunAction(
standard.StatFS,
rdf_client.StatFSRequest(path_list=["/usr/bin", "/"]))
self.assertEqual(len(results), 2)
# Both results should have mount_point as "/"
self.assertEqual(results[0].unixvolume.mount_point,
results[1].unixvolume.mount_point)
result = results[0]
self.assertEqual(result.bytes_per_sector, f_bsize)
self.assertEqual(result.sectors_per_allocation_unit, 1)
self.assertEqual(result.total_allocation_units, f_blocks)
self.assertEqual(result.actual_available_allocation_units, f_bavail)
self.assertAlmostEqual(result.FreeSpacePercent(), 53.388, delta=0.001)
self.assertEqual(result.unixvolume.mount_point, "/")
self.assertEqual(result.Name(), "/")
# Test we get a result even if one path is bad
results = self.RunAction(
standard.StatFS,
rdf_client.StatFSRequest(path_list=["/does/not/exist", "/"]))
self.assertEqual(len(results), 1)
self.assertEqual(result.Name(), "/")
def setUp(self):
super(ApiE2ETest, self).setUp()
api_auth_manager.APIACLInit.InitApiAuthManager()
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
self.port = ApiE2ETest.server_port
self.endpoint = "http://localhost:%s" % self.port
self.api = grr_api.InitHttp(api_endpoint=self.endpoint)
self.poll_stubber = utils.MultiStubber(
(grr_api_utils, "DEFAULT_POLL_INTERVAL", 0.1),
(grr_api_utils, "DEFAULT_POLL_TIMEOUT", 10))
self.poll_stubber.Start()
def ValidateConfig(self, config_file=None):
"""Iterate over all the sections in the config file and validate them."""
logging.debug("Processing %s", config_file)
if isinstance(config_file, config_lib.GrrConfigManager):
conf_obj = config_file
else:
conf_obj = config.CONFIG.MakeNewConfig()
conf_obj.Initialize(filename=config_file, reset=True)
with utils.MultiStubber((config, "CONFIG", conf_obj),
(config_lib, "_CONFIG", conf_obj)):
all_sections = conf_obj.GetSections()
errors = conf_obj.Validate(sections=all_sections)
return errors
def setUp(self): # pylint: disable=invalid-name
"""Set up test method."""
super(ApiRegressionTest, self).setUp()
if not self.__class__.api_method:
raise ValueError("%s.api_method has to be set." %
self.__class__.__name__)
if not self.__class__.handler:
raise ValueError("%s.handler has to be set." %
self.__class__.__name__)
self.checks = []
p = psutil.Process(os.getpid())
self.syscalls_stubber = utils.MultiStubber(
(socket, "gethostname", lambda: "test.host"),
(os, "getpid", lambda: 42), (psutil, "Process", lambda _=None: p))
self.syscalls_stubber.Start()
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
# Force creation of new APIAuthorizationManager.
api_auth_manager.APIACLInit.InitApiAuthManager()
self.config_overrider = test_lib.ConfigOverrider({
# For regression tests we want to use a fixed version number instead of
# current one. Otherwise regression data would have to be re-generated
# each time GRR version is increased.
"Source.version_major":
1,
"Source.version_minor":
2,
"Source.version_revision":
3,
"Source.version_release":
4,
"Source.version_string":
"1.2.3.4",
"Source.version_numeric":
1234,
})
self.config_overrider.Start()
def testCPULimit(self):
received_messages = []
class MockWorker(object):
def Heartbeat(self):
pass
def SendClientAlert(self, msg):
received_messages.append(msg)
class FakeProcess(object):
times = [(1, 0), (2, 0), (3, 0), (10000, 0), (10001, 0)]
def __init__(self, unused_pid=None):
self.pcputimes = collections.namedtuple("pcputimes", ["user", "system"])
def cpu_times(self): # pylint: disable=g-bad-name
return self.pcputimes(*self.times.pop(0))
results = []
def MockSendReply(unused_self, reply=None, **kwargs):
results.append(reply or rdf_client.LogMessage(**kwargs))
message = rdf_flows.GrrMessage(name="ProgressAction", cpu_limit=3600)
action_cls = actions.ActionPlugin.classes[message.name]
with utils.MultiStubber((psutil, "Process", FakeProcess),
(action_cls, "SendReply", MockSendReply)):
action_cls._authentication_required = False
action = action_cls(grr_worker=MockWorker())
action.Execute(message)
self.assertTrue("Action exceeded cpu limit." in results[0].error_message)
self.assertTrue("CPUExceededError" in results[0].error_message)
self.assertEqual(len(received_messages), 1)
self.assertEqual(received_messages[0], "Cpu limit exceeded.")
def _RunYaraProcessScan(self,
procs,
ignore_grr_process=False,
include_errors_in_results=False,
include_misses_in_results=False,
max_results_per_process=0,
**kw):
client_mock = action_mocks.ActionMock(yara_actions.YaraProcessScan)
with utils.MultiStubber(
(psutil, "process_iter", lambda: procs),
(psutil, "Process", functools.partial(self.process, procs)),
(client_utils, "OpenProcessForMemoryAccess",
lambda pid: FakeMemoryProcess(pid=pid))):
session_id = flow_test_lib.TestFlowHelper(
yara_flows.YaraProcessScan.__name__,
client_mock,
yara_signature=test_yara_signature,
client_id=self.client_id,
ignore_grr_process=ignore_grr_process,
include_errors_in_results=include_errors_in_results,
include_misses_in_results=include_misses_in_results,
max_results_per_process=max_results_per_process,
token=self.token,
**kw)
flow_obj = aff4.FACTORY.Open(session_id)
results = flow_obj.TypedResultCollection()
matches = [
x[1].payload
for x in results.ScanByType(rdf_yara.YaraProcessScanMatch.__name__)
]
errors = [
x[1].payload
for x in results.ScanByType(rdf_yara.YaraProcessError.__name__)
]
misses = [
x[1].payload
for x in results.ScanByType(rdf_yara.YaraProcessScanMiss.__name__)
]
return (matches, errors, misses)
def _RunProcessDump(self, pids=None, size_limit=None, chunk_size=None):
procs = self.procs
with utils.MultiStubber(
(psutil, "process_iter", lambda: procs),
(psutil, "Process", functools.partial(self.process, procs)),
(client_utils, "OpenProcessForMemoryAccess",
lambda pid: FakeMemoryProcess(pid=pid))):
client_mock = action_mocks.MultiGetFileClientMock(
yara_actions.YaraProcessDump, tempfiles.DeleteGRRTempFiles)
session_id = flow_test_lib.TestFlowHelper(
yara_flows.YaraDumpProcessMemory.__name__,
client_mock,
pids=pids or [105],
size_limit=size_limit,
chunk_size=chunk_size,
client_id=self.client_id,
ignore_grr_process=True,
token=self.token)
flow_obj = aff4.FACTORY.Open(session_id, flow.GRRFlow)
return flow_obj.ResultCollection()
def testScanAndDump(self):
client_mock = action_mocks.MultiGetFileClientMock(
yara_actions.YaraProcessScan, yara_actions.YaraProcessDump,
tempfiles.DeleteGRRTempFiles)
procs = [p for p in self.procs if p.pid in [102, 103]]
with utils.MultiStubber(
(psutil, "process_iter", lambda: procs),
(psutil, "Process", functools.partial(self.process, procs)),
(client_utils, "OpenProcessForMemoryAccess",
lambda pid: FakeMemoryProcess(pid=pid))):
session_id = flow_test_lib.TestFlowHelper(
yara_flows.YaraProcessScan.__name__,
client_mock,
yara_signature=test_yara_signature,
client_id=self.client_id,
token=self.token,
include_errors_in_results=True,
include_misses_in_results=True,
dump_process_on_match=True)
flow_obj = aff4.FACTORY.Open(session_id)
results = list(flow_obj.ResultCollection())
# 1. Scan result match.
# 2. Scan result miss.
# 3. ProcDump response.
# 3. Stat entry for the dumped file.
self.assertEqual(len(results), 4)
self.assertIsInstance(results[0], rdf_yara.YaraProcessScanMatch)
self.assertIsInstance(results[1], rdf_yara.YaraProcessScanMiss)
self.assertIsInstance(results[2], rdf_yara.YaraProcessDumpResponse)
self.assertIsInstance(results[3], rdf_client.StatEntry)
self.assertEqual(len(results[2].dumped_processes), 1)
self.assertEqual(results[0].process.pid,
results[2].dumped_processes[0].process.pid)
self.assertIn(str(results[2].dumped_processes[0].process.pid),
results[3].pathspec.path)
def Start(self):
"""Install the stubs."""
modules = {
"_winreg": mock.MagicMock(),
"ctypes": mock.MagicMock(),
"ctypes.wintypes": mock.MagicMock(),
# Requires mocking because exceptions.WindowsError does not exist
"exceptions": mock.MagicMock(),
}
self.module_patcher = mock.patch.dict("sys.modules", modules)
self.module_patcher.start()
# pylint: disable= g-import-not-at-top
from grr_response_client.vfs_handlers import registry
import exceptions
import _winreg
# pylint: enable=g-import-not-at-top
fixture = RegistryFake()
self.stubber = utils.MultiStubber(
(registry, "KeyHandle", RegistryFake.FakeKeyHandle),
(registry, "OpenKey", fixture.OpenKey),
(registry, "QueryValueEx", fixture.QueryValueEx),
(registry, "QueryInfoKey", fixture.QueryInfoKey),
(registry, "EnumValue", fixture.EnumValue),
(registry, "EnumKey", fixture.EnumKey))
self.stubber.Start()
# Add the Registry handler to the vfs.
vfs.VFSInit().Run()
_winreg.HKEY_USERS = "HKEY_USERS"
_winreg.HKEY_LOCAL_MACHINE = "HKEY_LOCAL_MACHINE"
exceptions.WindowsError = IOError
def testConfigFileInclusionWindowsPaths(self):
one = r"""
Config.includes:
- 2.yaml
Section1.int: 1
"""
two = r"""
Section1.int: 2
SecondaryFileIncluded: true
"""
config_path = "C:\\Windows\\System32\\GRR"
def MockedWindowsOpen(filename, _=None):
basename = ntpath.basename(filename)
dirname = ntpath.dirname(filename)
# Make sure we only try to open files from this directory.
if dirname != config_path:
raise IOError("Tried to open wrong file %s" % filename)
if basename == "1.yaml":
return StringIO.StringIO(one)
if basename == "2.yaml":
return StringIO.StringIO(two)
raise IOError("File not found %s" % filename)
# We need to also use the nt path manipulation modules.
with utils.MultiStubber((__builtin__, "open", MockedWindowsOpen),
(os, "path", ntpath)):
conf = self._GetNewConf()
conf.Initialize(filename=ntpath.join(config_path, "1.yaml"))
self.assertEqual(conf["Section1.int"], 2)
self.assertEqual(conf["SecondaryFileIncluded"], True)
def _ConfigStub(self, sections=None):
mock = GetConfigMockClass(sections)
return utils.MultiStubber(
(config.CONFIG, "GetRaw", mock["GetRaw"]),
(config.CONFIG, "Get", mock["Get"]),
(config.CONFIG, "type_infos", mock["type_infos"]))
