def testUploadArtifactYamlFileAndDumpToYaml(self):
artifact_registry.REGISTRY.ClearRegistry()
artifact_registry.REGISTRY.ClearSources()
artifact_registry.REGISTRY._CheckDirty()
try:
test_artifacts_file = os.path.join(config.CONFIG["Test.data_dir"],
"artifacts",
"test_artifacts.json")
with io.open(test_artifacts_file, mode="r",
encoding="utf-8") as filedesc:
artifact.UploadArtifactYamlFile(filedesc.read())
loaded_artifacts = artifact_registry.REGISTRY.GetArtifacts()
self.assertGreaterEqual(len(loaded_artifacts), 20)
self.assertIn("DepsWindirRegex",
[a.name for a in loaded_artifacts])
# Now dump back to YAML.
yaml_data = artifact_registry.REGISTRY.DumpArtifactsToYaml()
for snippet in [
"name: TestFilesArtifact",
"urls:\\s*- https://msdn.microsoft.com/en-us/library/",
"returned_types:\\s*- SoftwarePackage",
"args:\\s*- --list",
"cmd: /usr/bin/dpkg",
]:
self.assertRegexpMatches(yaml_data, snippet)
finally:
artifact.LoadArtifactsOnce()
def testGetKBDependencies(self):
"""Test that KB dependencies are calculated correctly."""
artifact_registry.REGISTRY.ClearSources()
try:
test_artifacts_file = os.path.join(config.CONFIG["Test.data_dir"],
"artifacts", "test_artifacts.json")
artifact_registry.REGISTRY.AddFileSource(test_artifacts_file)
with test_lib.ConfigOverrider({
"Artifacts.knowledge_base": [
"DepsParent", "DepsDesktop", "DepsHomedir", "DepsWindir",
"DepsWindirRegex", "DepsControlSet", "FakeArtifact"
],
"Artifacts.knowledge_base_additions": ["DepsHomedir2"],
"Artifacts.knowledge_base_skip": ["DepsWindir"],
"Artifacts.knowledge_base_heavyweight": ["FakeArtifact"]
}):
args = artifact.KnowledgeBaseInitializationArgs(lightweight=True)
kb_init = artifact.KnowledgeBaseInitializationFlow(
rdf_flow_objects.Flow(args=args))
kb_init.state["all_deps"] = set()
kb_init.state["awaiting_deps_artifacts"] = []
kb_init.state["knowledge_base"] = rdf_client.KnowledgeBase(os="Windows")
no_deps = kb_init.GetFirstFlowsForCollection()
self.assertCountEqual(no_deps, ["DepsControlSet", "DepsHomedir2"])
self.assertCountEqual(kb_init.state.all_deps, [
"users.homedir", "users.desktop", "users.username",
"environ_windir", "current_control_set"
])
self.assertCountEqual(
kb_init.state.awaiting_deps_artifacts,
["DepsParent", "DepsDesktop", "DepsHomedir", "DepsWindirRegex"])
finally:
artifact.LoadArtifactsOnce()
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
stats_collector = prometheus_stats_collector.PrometheusStatsCollector()
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath("grr-response-core",
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Prevent using the default writeback location since it may clash with local
# setup.
writeback_filepath = temp.TempFilePath(prefix="grr_writeback", suffix=".yaml")
config.CONFIG.global_override["Config.writeback"] = writeback_filepath
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
artifact.LoadArtifactsOnce()
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
def Init():
"""Run all required startup routines and initialization hooks."""
# Set up a temporary syslog handler so we have somewhere to log problems
# with ConfigInit() which needs to happen before we can start our create our
# proper logging setup.
syslog_logger = logging.getLogger("TempLogger")
if os.path.exists("/dev/log"):
handler = logging.handlers.SysLogHandler(address="/dev/log")
else:
handler = logging.handlers.SysLogHandler()
syslog_logger.addHandler(handler)
# The default behavior of server components is to raise errors when
# encountering unknown config options.
flags.FLAGS.disallow_missing_config_definitions = True
try:
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine(rename_invalid_writeback=False)
except config_lib.Error:
syslog_logger.exception("Died during config initialization")
raise
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
registry=prometheus_client.REGISTRY)
stats_collector_instance.Set(stats_collector)
server_logging.ServerLoggingStartupInit()
bs_registry_init.RegisterBlobStores()
all_decoders.Register()
all_parsers.Register()
ec_registry_init.RegisterExportConverters()
gui_api_registry_init.RegisterApiCallRouters()
data_store.InitializeDataStore()
if contexts.ADMIN_UI_CONTEXT in config.CONFIG.context:
api_auth_manager.InitializeApiAuthManager()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce()
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
# Exempt config updater from this check because it is the one responsible for
# setting the variable.
if not config.CONFIG.ContextApplied("ConfigUpdater Context"):
if not config.CONFIG.Get("Server.initialized"):
raise RuntimeError(
"Config not initialized, run \"grr_config_updater"
" initialize\". If the server is already configured,"
" add \"Server.initialized: True\" to your config.")
def Init():
"""Run all required startup routines and initialization hooks."""
# Set up a temporary syslog handler so we have somewhere to log problems
# with ConfigInit() which needs to happen before we can start our create our
# proper logging setup.
syslog_logger = logging.getLogger("TempLogger")
if os.path.exists("/dev/log"):
handler = logging.handlers.SysLogHandler(address="/dev/log")
else:
handler = logging.handlers.SysLogHandler()
syslog_logger.addHandler(handler)
try:
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
except config_lib.Error:
syslog_logger.exception("Died during config initialization")
raise
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
metric_metadata, registry=prometheus_client.REGISTRY)
stats_collector_instance.Set(stats_collector)
server_logging.ServerLoggingStartupInit()
bs_registry_init.RegisterBlobStores()
all_decoders.Register()
all_parsers.Register()
data_store.InitializeDataStore()
if data_store.AFF4Enabled():
aff4.AFF4Init() # Requires data_store.InitializeDataStore.
aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init.
filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init.
results.ResultQueueInit() # Requires aff4.AFF4Init.
sequential_collection.StartUpdaterOnce()
if contexts.ADMIN_UI_CONTEXT in config.CONFIG.context:
api_auth_manager.InitializeApiAuthManager()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init.
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
# Exempt config updater from this check because it is the one responsible for
# setting the variable.
if not config.CONFIG.ContextApplied("ConfigUpdater Context"):
if not config.CONFIG.Get("Server.initialized"):
raise RuntimeError(
"Config not initialized, run \"grr_config_updater"
" initialize\". If the server is already configured,"
" add \"Server.initialized: True\" to your config.")
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(client_metrics.GetMetadata())
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
metric_metadata)
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = compatibility.GetName(fake_data_store.FakeDataStore)
config.CONFIG.Set("Datastore.implementation", test_ds)
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
if data_store.AFF4Enabled():
aff4.AFF4Init() # Requires data_store.InitializeDataStore.
aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init.
filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init.
results.ResultQueueInit() # Requires aff4.AFF4Init.
sequential_collection.StartUpdaterOnce()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init.
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
