def TestInit(): """Only used in tests and will rerun all the hooks to create a clean state.""" global INIT_RAN stats_collector = prometheus_stats_collector.PrometheusStatsCollector() stats_collector_instance.Set(stats_collector) # Tests use both the server template grr_server.yaml as a primary config file # (this file does not contain all required options, e.g. private keys), and # additional configuration in test_data/grr_test.yaml which contains typical # values for a complete installation. flags.FLAGS.config = package.ResourcePath("grr-response-core", "install_data/etc/grr-server.yaml") flags.FLAGS.secondary_configs.append( package.ResourcePath("grr-response-test", "grr_response_test/test_data/grr_test.yaml")) # This config contains non-public settings that should be applied during # tests. extra_test_config = config.CONFIG["Test.additional_test_config"] if os.path.exists(extra_test_config): flags.FLAGS.secondary_configs.append(extra_test_config) # Prevent using the default writeback location since it may clash with local # setup. writeback_filepath = temp.TempFilePath(prefix="grr_writeback", suffix=".yaml") config.CONFIG.global_override["Config.writeback"] = writeback_filepath # Tests additionally add a test configuration file. config_lib.SetPlatformArchContext() config_lib.ParseConfigCommandLine() # We are running a test so let the config system know that. config.CONFIG.AddContext(contexts.TEST_CONTEXT, "Context applied when we run tests.") if not INIT_RAN: server_logging.ServerLoggingStartupInit() server_logging.SetTestVerbosity() blob_store_test_lib.UseTestBlobStore() data_store.InitializeDataStore() artifact.LoadArtifactsOnce() checks.LoadChecksFromFilesystemOnce() client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce() email_alerts.InitializeEmailAlerterOnce() http_api.InitializeHttpRequestHandlerOnce() ip_resolver.IPResolverInitOnce() stats_server.InitializeStatsServerOnce() webauth.InitializeWebAuthOnce() if not utils.TimeBasedCache.house_keeper_thread: utils.TimeBasedCache() utils.TimeBasedCache.house_keeper_thread.exit = True utils.TimeBasedCache.house_keeper_thread.join() INIT_RAN = True
def testWeakRefSet(self): c1 = utils.TimeBasedCache() c2 = utils.TimeBasedCache() self.assertIn(c1, utils.TimeBasedCache.active_caches) self.assertIn(c2, utils.TimeBasedCache.active_caches) l = len(utils.TimeBasedCache.active_caches) del c1 # This should work even though the weak ref to c1 should be gone. utils.TimeBasedCache.house_keeper_thread.target() # Make sure it's actually gone. self.assertLess(len(utils.TimeBasedCache.active_caches), l)
def __init__(self, queues=queues_config.WORKER_LIST, threadpool_prefix="grr_threadpool", threadpool_size=None, token=None): """Constructor. Args: queues: The queues we use to fetch new messages from. threadpool_prefix: A name for the thread pool used by this worker. threadpool_size: The number of workers to start in this thread pool. token: The token to use for the worker. Raises: RuntimeError: If the token is not provided. """ logging.info("started worker with queues: %s", str(queues)) self.queues = queues # self.queued_flows is a timed cache of locked flows. If this worker # encounters a lock failure on a flow, it will not attempt to grab this flow # until the timeout. self.queued_flows = utils.TimeBasedCache(max_size=10, max_age=60) if token is None: raise RuntimeError("A valid ACLToken is required.") # Make the thread pool a global so it can be reused for all workers. if self.__class__.thread_pool is None: if threadpool_size is None: threadpool_size = config.CONFIG["Threadpool.size"] self.__class__.thread_pool = threadpool.ThreadPool.Factory( threadpool_prefix, min_threads=2, max_threads=threadpool_size) self.__class__.thread_pool.Start() self.token = token self.last_active = 0 self.last_mh_lease_attempt = rdfvalue.RDFDatetime.FromSecondsSinceEpoch( 0) # Well known flows are just instantiated. self.well_known_flows = flow.WellKnownFlow.GetAllWellKnownFlows( token=token)
def test05TimeBasedCache(self): key = "key" tested_cache = utils.TimeBasedCache(max_age=50) with test_lib.FakeTime(100): # Stop the housekeeper thread - we test it explicitely here tested_cache.exit = True tested_cache.Put(key, "hello") self.assertEqual(tested_cache.Get(key), "hello") with test_lib.FakeTime(160): # Force the housekeeper to run tested_cache.house_keeper_thread.target() # This should now be expired self.assertRaises(KeyError, tested_cache.Get, key)
def testTimeBasedCacheSingleThread(self): utils.TimeBasedCache() num_threads = threading.active_count() utils.TimeBasedCache() self.assertEqual(threading.active_count(), num_threads)
import stat from typing import Text import pytsk3 from grr_response_client import client_utils from grr_response_client.vfs_handlers import base as vfs_base from grr_response_core.lib import utils from grr_response_core.lib.rdfvalues import client_fs as rdf_client_fs from grr_response_core.lib.rdfvalues import paths as rdf_paths from grr_response_core.lib.util import compatibility from grr_response_core.lib.util import precondition # A central Cache for vfs handlers. This can be used to keep objects alive # for a limited time. DEVICE_CACHE = utils.TimeBasedCache() def _DecodeUTF8WithWarning(string): try: return string.decode("utf-8") except UnicodeDecodeError as e: result = string.decode("utf-8", "replace") logging.warning("%s. Decoded %r to %r", e, string, result) return result class CachedFilesystem(object): """A container for the filesystem and image.""" def __init__(self, fs, img): self.fs = fs
import threading from grr_response_client import client_utils from grr_response_client.vfs_handlers import base as vfs_base from grr_response_core.lib import utils from grr_response_core.lib.rdfvalues import paths as rdf_paths from grr_response_core.lib.util import filesystem # File handles are cached here. After expiration, the file handle is garbage # collected, which implicitly unlocks the file on Windows. Although a cache is # not optimal, it is currently the best solution: TSK as example mounts Windows # drives using the OS handler and performs frequent reads to read the file # tables. Since VFSHandlers have no de-facto support for context managers, it is # hard to determine when the file can be freed again, thus this caching is hard # to remove. FILE_HANDLE_CACHE = utils.TimeBasedCache(max_age=30) class LockedFileHandle(object): """An object which encapsulates access to a file.""" def __init__(self, filename, mode="rb"): self.lock = threading.RLock() self.fd = open(filename, mode) self.filename = filename def Seek(self, offset, whence=0): self.fd.seek(offset, whence) def Read(self, length): return self.fd.read(length)
"""Virtual filesystem module based on pyfsntfs.""" import stat from typing import Any, Callable, Dict, Iterable, Optional, Text, Type import pyfsntfs from grr_response_client import client_utils from grr_response_client.vfs_handlers import base as vfs_base from grr_response_core.lib import rdfvalue from grr_response_core.lib import utils from grr_response_core.lib.rdfvalues import client_fs as rdf_client_fs from grr_response_core.lib.rdfvalues import paths as rdf_paths # Caches pyfsntfs.volume instances. MOUNT_CACHE = utils.TimeBasedCache() # See # https://github.com/libyal/libfsntfs/blob/master/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc#file_attribute_flags FILE_ATTRIBUTE_READONLY = 0x00000001 FILE_ATTRIBUTE_HIDDEN = 0x00000002 def _GetAlternateDataStreamCaseInsensitive( fd: pyfsntfs.file_entry, name: Text) -> Optional[pyfsntfs.data_stream]: name = name.lower() for data_stream in fd.alternate_data_streams: if data_stream.name.lower() == name: return data_stream
def TestInit(): """Only used in tests and will rerun all the hooks to create a clean state.""" global INIT_RAN metric_metadata = server_metrics.GetMetadata() metric_metadata.extend(client_metrics.GetMetadata()) metric_metadata.extend(communicator.GetMetricMetadata()) stats_collector = prometheus_stats_collector.PrometheusStatsCollector( metric_metadata) stats_collector_instance.Set(stats_collector) # Tests use both the server template grr_server.yaml as a primary config file # (this file does not contain all required options, e.g. private keys), and # additional configuration in test_data/grr_test.yaml which contains typical # values for a complete installation. flags.FLAGS.config = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") flags.FLAGS.secondary_configs.append( package.ResourcePath("grr-response-test", "grr_response_test/test_data/grr_test.yaml")) # This config contains non-public settings that should be applied during # tests. extra_test_config = config.CONFIG["Test.additional_test_config"] if os.path.exists(extra_test_config): flags.FLAGS.secondary_configs.append(extra_test_config) # Tests additionally add a test configuration file. config_lib.SetPlatformArchContext() config_lib.ParseConfigCommandLine() # We are running a test so let the config system know that. config.CONFIG.AddContext(contexts.TEST_CONTEXT, "Context applied when we run tests.") test_ds = flags.FLAGS.test_data_store if test_ds is None: test_ds = compatibility.GetName(fake_data_store.FakeDataStore) config.CONFIG.Set("Datastore.implementation", test_ds) if not INIT_RAN: server_logging.ServerLoggingStartupInit() server_logging.SetTestVerbosity() blob_store_test_lib.UseTestBlobStore() data_store.InitializeDataStore() if data_store.AFF4Enabled(): aff4.AFF4Init() # Requires data_store.InitializeDataStore. aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init. filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init. results.ResultQueueInit() # Requires aff4.AFF4Init. sequential_collection.StartUpdaterOnce() artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init. checks.LoadChecksFromFilesystemOnce() client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce() cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init. email_alerts.InitializeEmailAlerterOnce() http_api.InitializeHttpRequestHandlerOnce() ip_resolver.IPResolverInitOnce() stats_server.InitializeStatsServerOnce() webauth.InitializeWebAuthOnce() db = data_store.DB.SetupTestDB() if db: data_store.DB = db data_store.DB.Initialize() if not utils.TimeBasedCache.house_keeper_thread: utils.TimeBasedCache() utils.TimeBasedCache.house_keeper_thread.exit = True utils.TimeBasedCache.house_keeper_thread.join() INIT_RAN = True
def TestInit(): """Only used in tests and will rerun all the hooks to create a clean state.""" global INIT_RAN metric_metadata = server_metrics.GetMetadata() metric_metadata.extend(client_metrics.GetMetadata()) metric_metadata.extend(communicator.GetMetricMetadata()) stats_collector = prometheus_stats_collector.PrometheusStatsCollector( metric_metadata) stats_collector_instance.Set(stats_collector) # Tests use both the server template grr_server.yaml as a primary config file # (this file does not contain all required options, e.g. private keys), and # additional configuration in test_data/grr_test.yaml which contains typical # values for a complete installation. flags.FLAGS.config = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") flags.FLAGS.secondary_configs.append( package.ResourcePath("grr-response-test", "grr_response_test/test_data/grr_test.yaml")) # This config contains non-public settings that should be applied during # tests. extra_test_config = config.CONFIG["Test.additional_test_config"] if os.path.exists(extra_test_config): flags.FLAGS.secondary_configs.append(extra_test_config) # Tests additionally add a test configuration file. config_lib.SetPlatformArchContext() config_lib.ParseConfigCommandLine() # We are running a test so let the config system know that. config.CONFIG.AddContext(contexts.TEST_CONTEXT, "Context applied when we run tests.") test_ds = flags.FLAGS.test_data_store if test_ds is None: test_ds = compatibility.GetName(fake_data_store.FakeDataStore) config.CONFIG.Set("Datastore.implementation", test_ds) if not INIT_RAN: server_logging.ServerLoggingStartupInit() server_logging.SetTestVerbosity() blob_store_test_lib.UseTestBlobStore() registry.TestInit() db = data_store.DB.SetupTestDB() if db: data_store.DB = db data_store.DB.Initialize() aff4.AFF4InitHook().Run() if not utils.TimeBasedCache.house_keeper_thread: utils.TimeBasedCache() utils.TimeBasedCache.house_keeper_thread.exit = True utils.TimeBasedCache.house_keeper_thread.join() INIT_RAN = True