def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
stats_collector = prometheus_stats_collector.PrometheusStatsCollector()
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath("grr-response-core",
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Prevent using the default writeback location since it may clash with local
# setup.
writeback_filepath = temp.TempFilePath(prefix="grr_writeback", suffix=".yaml")
config.CONFIG.global_override["Config.writeback"] = writeback_filepath
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
artifact.LoadArtifactsOnce()
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
def setUp(self):
super(GRRBaseTest, self).setUp()
test_user = u"test"
system_users_patcher = mock.patch.object(
access_control, "SYSTEM_USERS",
frozenset(itertools.chain(access_control.SYSTEM_USERS, [test_user])))
system_users_patcher.start()
self.addCleanup(system_users_patcher.stop)
self.token = access_control.ACLToken(
username=test_user, reason="Running tests")
self.temp_dir = temp.TempDirPath()
config.CONFIG.SetWriteBack(os.path.join(self.temp_dir, "writeback.yaml"))
self.addCleanup(lambda: shutil.rmtree(self.temp_dir, ignore_errors=True))
# Each datastore is wrapped with DatabaseValidationWrapper, so we have
# to access the delegate directly (assuming it's an InMemoryDB
# implementation).
data_store.REL_DB.delegate.ClearTestDB()
email_alerts.InitializeEmailAlerterOnce()
# Stub out the email function
self.emails_sent = []
def SendEmailStub(to_user, from_user, subject, message, **unused_kwargs):
self.emails_sent.append((to_user, from_user, subject, message))
self.mail_stubber = utils.MultiStubber(
(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmailStub),
(email.utils, "make_msgid", lambda: "<message id stub>"))
self.mail_stubber.Start()
self.addCleanup(self.mail_stubber.Stop)
# We don't want to send actual email in our tests
self.smtp_patcher = mock.patch("smtplib.SMTP")
self.mock_smtp = self.smtp_patcher.start()
self.addCleanup(self.smtp_patcher.stop)
def DisabledSet(*unused_args, **unused_kw):
raise NotImplementedError(
"Usage of Set() is disabled, please use a configoverrider in tests.")
self.config_set_disable = utils.Stubber(config.CONFIG, "Set", DisabledSet)
self.config_set_disable.Start()
self.addCleanup(self.config_set_disable.Stop)
self._SetupFakeStatsContext()
# Turn off WithLimitedCallFrequency-based caching in tests. Tests that need
# to test caching behavior explicitly, should turn it on explicitly.
with_limited_call_frequency_stubber = utils.Stubber(
cache, "WITH_LIMITED_CALL_FREQUENCY_PASS_THROUGH", True)
with_limited_call_frequency_stubber.Start()
self.addCleanup(with_limited_call_frequency_stubber.Stop)
def Init():
"""Run all required startup routines and initialization hooks."""
# Set up a temporary syslog handler so we have somewhere to log problems
# with ConfigInit() which needs to happen before we can start our create our
# proper logging setup.
syslog_logger = logging.getLogger("TempLogger")
if os.path.exists("/dev/log"):
handler = logging.handlers.SysLogHandler(address="/dev/log")
else:
handler = logging.handlers.SysLogHandler()
syslog_logger.addHandler(handler)
# The default behavior of server components is to raise errors when
# encountering unknown config options.
flags.FLAGS.disallow_missing_config_definitions = True
try:
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine(rename_invalid_writeback=False)
except config_lib.Error:
syslog_logger.exception("Died during config initialization")
raise
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
registry=prometheus_client.REGISTRY)
stats_collector_instance.Set(stats_collector)
server_logging.ServerLoggingStartupInit()
bs_registry_init.RegisterBlobStores()
all_decoders.Register()
all_parsers.Register()
ec_registry_init.RegisterExportConverters()
gui_api_registry_init.RegisterApiCallRouters()
data_store.InitializeDataStore()
if contexts.ADMIN_UI_CONTEXT in config.CONFIG.context:
api_auth_manager.InitializeApiAuthManager()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce()
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
# Exempt config updater from this check because it is the one responsible for
# setting the variable.
if not config.CONFIG.ContextApplied("ConfigUpdater Context"):
if not config.CONFIG.Get("Server.initialized"):
raise RuntimeError(
"Config not initialized, run \"grr_config_updater"
" initialize\". If the server is already configured,"
" add \"Server.initialized: True\" to your config.")
def Init():
"""Run all required startup routines and initialization hooks."""
# Set up a temporary syslog handler so we have somewhere to log problems
# with ConfigInit() which needs to happen before we can start our create our
# proper logging setup.
syslog_logger = logging.getLogger("TempLogger")
if os.path.exists("/dev/log"):
handler = logging.handlers.SysLogHandler(address="/dev/log")
else:
handler = logging.handlers.SysLogHandler()
syslog_logger.addHandler(handler)
try:
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
except config_lib.Error:
syslog_logger.exception("Died during config initialization")
raise
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
metric_metadata, registry=prometheus_client.REGISTRY)
stats_collector_instance.Set(stats_collector)
server_logging.ServerLoggingStartupInit()
bs_registry_init.RegisterBlobStores()
all_decoders.Register()
all_parsers.Register()
data_store.InitializeDataStore()
if data_store.AFF4Enabled():
aff4.AFF4Init() # Requires data_store.InitializeDataStore.
aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init.
filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init.
results.ResultQueueInit() # Requires aff4.AFF4Init.
sequential_collection.StartUpdaterOnce()
if contexts.ADMIN_UI_CONTEXT in config.CONFIG.context:
api_auth_manager.InitializeApiAuthManager()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init.
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
# Exempt config updater from this check because it is the one responsible for
# setting the variable.
if not config.CONFIG.ContextApplied("ConfigUpdater Context"):
if not config.CONFIG.Get("Server.initialized"):
raise RuntimeError(
"Config not initialized, run \"grr_config_updater"
" initialize\". If the server is already configured,"
" add \"Server.initialized: True\" to your config.")
def setUp(self):
super(GRRBaseTest, self).setUp()
self.temp_dir = temp.TempDirPath()
config.CONFIG.SetWriteBack(
os.path.join(self.temp_dir, "writeback.yaml"))
self.addCleanup(
lambda: shutil.rmtree(self.temp_dir, ignore_errors=True))
data_store.DB.ClearTestDB()
# Each datastore is wrapped with DatabaseValidationWrapper, so we have
# to access the delegate directly (assuming it's an InMemoryDB
# implementation).
data_store.REL_DB.delegate.ClearTestDB()
if aff4.FACTORY is not None:
aff4.FACTORY.Flush()
# Create a Foreman, it's used in many tests.
aff4_grr.GRRAFF4Init()
hunts_results.ResultQueueInit()
email_alerts.InitializeEmailAlerterOnce()
audit.AuditEventListener._created_logs.clear()
# Stub out the email function
self.emails_sent = []
def SendEmailStub(to_user, from_user, subject, message,
**unused_kwargs):
self.emails_sent.append((to_user, from_user, subject, message))
self.mail_stubber = utils.MultiStubber(
(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmailStub),
(email.utils, "make_msgid", lambda: "<message id stub>"))
self.mail_stubber.Start()
self.addCleanup(self.mail_stubber.Stop)
# We don't want to send actual email in our tests
self.smtp_patcher = mock.patch("smtplib.SMTP")
self.mock_smtp = self.smtp_patcher.start()
self.addCleanup(self.smtp_patcher.stop)
def DisabledSet(*unused_args, **unused_kw):
raise NotImplementedError(
"Usage of Set() is disabled, please use a configoverrider in tests."
)
self.config_set_disable = utils.Stubber(config.CONFIG, "Set",
DisabledSet)
self.config_set_disable.Start()
self.addCleanup(self.config_set_disable.Stop)
if self.use_relational_reads:
self.relational_read_stubber = utils.Stubber(
data_store, "RelationalDBEnabled", lambda: True)
self.relational_read_stubber.Start()
self.addCleanup(self.relational_read_stubber.Stop)
self._SetupFakeStatsContext()
# Turn off WithLimitedCallFrequency-based caching in tests. Tests that need
# to test caching behavior explicitly, should turn it on explicitly.
with_limited_call_frequency_stubber = utils.Stubber(
cache, "WITH_LIMITED_CALL_FREQUENCY_PASS_THROUGH", True)
with_limited_call_frequency_stubber.Start()
self.addCleanup(with_limited_call_frequency_stubber.Stop)
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(client_metrics.GetMetadata())
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
metric_metadata)
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = compatibility.GetName(fake_data_store.FakeDataStore)
config.CONFIG.Set("Datastore.implementation", test_ds)
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
if data_store.AFF4Enabled():
aff4.AFF4Init() # Requires data_store.InitializeDataStore.
aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init.
filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init.
results.ResultQueueInit() # Requires aff4.AFF4Init.
sequential_collection.StartUpdaterOnce()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init.
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
