def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
stats_collector = prometheus_stats_collector.PrometheusStatsCollector()
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath("grr-response-core",
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Prevent using the default writeback location since it may clash with local
# setup.
writeback_filepath = temp.TempFilePath(prefix="grr_writeback", suffix=".yaml")
config.CONFIG.global_override["Config.writeback"] = writeback_filepath
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
artifact.LoadArtifactsOnce()
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
def testWeakRefSet(self):
c1 = utils.TimeBasedCache()
c2 = utils.TimeBasedCache()
self.assertIn(c1, utils.TimeBasedCache.active_caches)
self.assertIn(c2, utils.TimeBasedCache.active_caches)
l = len(utils.TimeBasedCache.active_caches)
del c1
# This should work even though the weak ref to c1 should be gone.
utils.TimeBasedCache.house_keeper_thread.target()
# Make sure it's actually gone.
self.assertLess(len(utils.TimeBasedCache.active_caches), l)
def __init__(self,
queues=queues_config.WORKER_LIST,
threadpool_prefix="grr_threadpool",
threadpool_size=None,
token=None):
"""Constructor.
Args:
queues: The queues we use to fetch new messages from.
threadpool_prefix: A name for the thread pool used by this worker.
threadpool_size: The number of workers to start in this thread pool.
token: The token to use for the worker.
Raises:
RuntimeError: If the token is not provided.
"""
logging.info("started worker with queues: %s", str(queues))
self.queues = queues
# self.queued_flows is a timed cache of locked flows. If this worker
# encounters a lock failure on a flow, it will not attempt to grab this flow
# until the timeout.
self.queued_flows = utils.TimeBasedCache(max_size=10, max_age=60)
if token is None:
raise RuntimeError("A valid ACLToken is required.")
# Make the thread pool a global so it can be reused for all workers.
if self.__class__.thread_pool is None:
if threadpool_size is None:
threadpool_size = config.CONFIG["Threadpool.size"]
self.__class__.thread_pool = threadpool.ThreadPool.Factory(
threadpool_prefix, min_threads=2, max_threads=threadpool_size)
self.__class__.thread_pool.Start()
self.token = token
self.last_active = 0
self.last_mh_lease_attempt = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(
0)
# Well known flows are just instantiated.
self.well_known_flows = flow.WellKnownFlow.GetAllWellKnownFlows(
token=token)
def test05TimeBasedCache(self):
key = "key"
tested_cache = utils.TimeBasedCache(max_age=50)
with test_lib.FakeTime(100):
# Stop the housekeeper thread - we test it explicitely here
tested_cache.exit = True
tested_cache.Put(key, "hello")
self.assertEqual(tested_cache.Get(key), "hello")
with test_lib.FakeTime(160):
# Force the housekeeper to run
tested_cache.house_keeper_thread.target()
# This should now be expired
self.assertRaises(KeyError, tested_cache.Get, key)
def testTimeBasedCacheSingleThread(self):
utils.TimeBasedCache()
num_threads = threading.active_count()
utils.TimeBasedCache()
self.assertEqual(threading.active_count(), num_threads)
import stat
from typing import Text
import pytsk3
from grr_response_client import client_utils
from grr_response_client.vfs_handlers import base as vfs_base
from grr_response_core.lib import utils
from grr_response_core.lib.rdfvalues import client_fs as rdf_client_fs
from grr_response_core.lib.rdfvalues import paths as rdf_paths
from grr_response_core.lib.util import compatibility
from grr_response_core.lib.util import precondition
# A central Cache for vfs handlers. This can be used to keep objects alive
# for a limited time.
DEVICE_CACHE = utils.TimeBasedCache()
def _DecodeUTF8WithWarning(string):
try:
return string.decode("utf-8")
except UnicodeDecodeError as e:
result = string.decode("utf-8", "replace")
logging.warning("%s. Decoded %r to %r", e, string, result)
return result
class CachedFilesystem(object):
"""A container for the filesystem and image."""
def __init__(self, fs, img):
self.fs = fs
import threading
from grr_response_client import client_utils
from grr_response_client.vfs_handlers import base as vfs_base
from grr_response_core.lib import utils
from grr_response_core.lib.rdfvalues import paths as rdf_paths
from grr_response_core.lib.util import filesystem
# File handles are cached here. After expiration, the file handle is garbage
# collected, which implicitly unlocks the file on Windows. Although a cache is
# not optimal, it is currently the best solution: TSK as example mounts Windows
# drives using the OS handler and performs frequent reads to read the file
# tables. Since VFSHandlers have no de-facto support for context managers, it is
# hard to determine when the file can be freed again, thus this caching is hard
# to remove.
FILE_HANDLE_CACHE = utils.TimeBasedCache(max_age=30)
class LockedFileHandle(object):
"""An object which encapsulates access to a file."""
def __init__(self, filename, mode="rb"):
self.lock = threading.RLock()
self.fd = open(filename, mode)
self.filename = filename
def Seek(self, offset, whence=0):
self.fd.seek(offset, whence)
def Read(self, length):
return self.fd.read(length)
"""Virtual filesystem module based on pyfsntfs."""
import stat
from typing import Any, Callable, Dict, Iterable, Optional, Text, Type
import pyfsntfs
from grr_response_client import client_utils
from grr_response_client.vfs_handlers import base as vfs_base
from grr_response_core.lib import rdfvalue
from grr_response_core.lib import utils
from grr_response_core.lib.rdfvalues import client_fs as rdf_client_fs
from grr_response_core.lib.rdfvalues import paths as rdf_paths
# Caches pyfsntfs.volume instances.
MOUNT_CACHE = utils.TimeBasedCache()
# See
# https://github.com/libyal/libfsntfs/blob/master/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc#file_attribute_flags
FILE_ATTRIBUTE_READONLY = 0x00000001
FILE_ATTRIBUTE_HIDDEN = 0x00000002
def _GetAlternateDataStreamCaseInsensitive(
fd: pyfsntfs.file_entry, name: Text) -> Optional[pyfsntfs.data_stream]:
name = name.lower()
for data_stream in fd.alternate_data_streams:
if data_stream.name.lower() == name:
return data_stream
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(client_metrics.GetMetadata())
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
metric_metadata)
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = compatibility.GetName(fake_data_store.FakeDataStore)
config.CONFIG.Set("Datastore.implementation", test_ds)
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
if data_store.AFF4Enabled():
aff4.AFF4Init() # Requires data_store.InitializeDataStore.
aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init.
filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init.
results.ResultQueueInit() # Requires aff4.AFF4Init.
sequential_collection.StartUpdaterOnce()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init.
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(client_metrics.GetMetadata())
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
metric_metadata)
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = compatibility.GetName(fake_data_store.FakeDataStore)
config.CONFIG.Set("Datastore.implementation", test_ds)
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
registry.TestInit()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
aff4.AFF4InitHook().Run()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
