def InterpolateFiles(self):
if self.fleetspeak_enabled:
shutil.copy(flags.FLAGS.fleetspeak_service_config,
self.pkg_fleetspeak_service_dir)
build_files_dir = package.ResourcePath(
"grr-response-core", "install_data/macosx/client/fleetspeak")
else:
build_files_dir = package.ResourcePath(
"grr-response-core", "install_data/macosx/client")
self.GenerateFile(
input_filename=os.path.join(build_files_dir, "grr.plist.in"),
output_filename=os.path.join(self.pkg_root,
"Library/LaunchDaemons",
self.plist_name))
# We pass in scripts separately with --scripts so they don't go in pkg_root
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"preinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"preinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"postinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"postinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"Distribution.xml.in"),
output_filename=os.path.join(self.build_dir,
"Distribution.xml"))
def GetServerComponentArgs(config_path):
"""Returns a set of command line arguments for server components.
Args:
config_path: Path to a config path generated by
self_contained_config_writer.
Returns:
An iterable with command line arguments to use.
"""
primary_config_path = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
secondary_config_path = package.ResourcePath(
"grr-response-test", "grr_response_test/test_data/grr_test.yaml")
return [
"--config",
primary_config_path,
"--secondary_configs",
",".join([secondary_config_path, config_path]),
"-p",
"Monitoring.http_port=%d" % portpicker.pick_unused_port(),
"-p",
"AdminUI.webauth_manager=NullWebAuthManager",
]
def InterpolateFiles(self):
if self.fleetspeak_enabled:
fleetspeak_template = config.CONFIG.Get(
"ClientBuilder.fleetspeak_config_path", context=self.context)
dest_fleetspeak_config = os.path.join(
self.pkg_fleetspeak_service_dir,
config.CONFIG.Get("Client.fleetspeak_unsigned_config_fname",
context=self.context))
self.GenerateFile(input_filename=fleetspeak_template,
output_filename=dest_fleetspeak_config)
build_files_dir = package.ResourcePath(
"grr-response-core", "install_data/macosx/client/fleetspeak")
else:
build_files_dir = package.ResourcePath(
"grr-response-core", "install_data/macosx/client")
self.GenerateFile(
input_filename=os.path.join(build_files_dir, "grr.plist.in"),
output_filename=os.path.join(self.pkg_root,
"Library/LaunchDaemons",
self.plist_name))
# We pass in scripts separately with --scripts so they don't go in pkg_root
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"preinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"preinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"postinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"postinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"Distribution.xml.in"),
output_filename=os.path.join(self.build_dir,
"Distribution.xml"))
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
stats_collector = prometheus_stats_collector.PrometheusStatsCollector()
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath("grr-response-core",
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Prevent using the default writeback location since it may clash with local
# setup.
writeback_filepath = temp.TempFilePath(prefix="grr_writeback", suffix=".yaml")
config.CONFIG.global_override["Config.writeback"] = writeback_filepath
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
artifact.LoadArtifactsOnce()
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(client_metrics.GetMetadata())
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = default_stats_collector.DefaultStatsCollector(
metric_metadata)
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = compatibility.GetName(fake_data_store.FakeDataStore)
config.CONFIG.Set("Datastore.implementation", test_ds)
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
registry.TestInit()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
aff4.AFF4InitHook().Run()
INIT_RAN = True
def VersionPath():
"""Returns a path to version.ini."""
# Try to get a version.ini. It should be in the resources if the code
# was packed with "pip sdist". It will be 2 levels up from grr_response_core
# if the code was installed via "pip install -e".
version_ini = (package.ResourcePath("grr-response-core", "version.ini")
or package.ResourcePath("grr-response-core",
"../../version.ini"))
if not os.path.exists(version_ini):
raise RuntimeError("Can't find version.ini at %s" % version_ini)
return version_ini
def _CreateClientConfig(tmp_dir: str) -> str:
"""Creates and returns the path to a fleetspeak client config."""
def TmpPath(*args):
return os.path.join(tmp_dir, *args)
server_config_dir = package.ResourcePath(
"fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server")
if not os.path.exists(server_config_dir):
raise Error(
f"Fleetspeak server config dir not found: {server_config_dir}. "
"Please make sure `grr_config_updater initialize` has been run.")
client_config_name = {
"Linux": "linux_client.config",
"Windows": "windows_client.config",
"Darwin": "darwin_client.config",
}
client_config_path = os.path.join(server_config_dir,
client_config_name[platform.system()])
with open(client_config_path, "r") as f:
client_config = text_format.Parse(f.read(), fs_cli_config_pb2.Config())
if client_config.HasField("filesystem_handler"):
client_config.filesystem_handler.configuration_directory = TmpPath()
# We store the client state file in `Logging.path`.
# 1) We need a writable path, where writing a file doesn't surprise the
# user (as opposed to other paths in the source tree).
# 2) We need the state file to be somewhat persistent, i.e. kept after
# re-runs of this command. Otherwise the client ID of the client would
# change at each re-run.
client_config.filesystem_handler.state_file = os.path.join(
config.CONFIG["Logging.path"], "fleetspeak-client.state")
with open(TmpPath("config"), "w") as f:
f.write(text_format.MessageToString(client_config))
return TmpPath("config")
def CopyFiles(self):
"""This sets up the template directory."""
# Copy the nanny binary.
shutil.copy(
package.ResourcePath("grr-response-core",
"install_data/debian/dpkg_client/nanny.sh.in"),
self.output_dir)
dpkg_dir = config.CONFIG.Get("PyInstaller.dpkg_root", context=self.context)
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter("install_data/debian/dpkg_client/debian"),
os.path.join(dpkg_dir, "debian/debian.in"))
# Copy upstart files
outdir = os.path.join(dpkg_dir, "debian/upstart.in")
utils.EnsureDirExists(outdir)
shutil.copy(config_lib.Resource().Filter(
"install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir)
# Copy init files
outdir = os.path.join(dpkg_dir, "debian/initd.in")
utils.EnsureDirExists(outdir)
shutil.copy(config_lib.Resource().Filter(
"install_data/debian/dpkg_client/initd/grr-client"), outdir)
# Copy systemd unit file
outdir = os.path.join(dpkg_dir, "debian/systemd.in")
utils.EnsureDirExists(outdir)
shutil.copy(config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"), outdir)
def _CopyNonFleetspeakDpkgFiles(dist_dir, package_dir):
"""Copies non-Fleetspeak-enabled DPKG files to template directory."""
# Copy the nanny binary.
shutil.copy(
package.ResourcePath("grr-response-core",
"install_data/debian/dpkg_client/nanny.sh.in"),
dist_dir)
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter("install_data/debian/dpkg_client/debian"),
os.path.join(package_dir, "debian/legacy-debian.in"))
# Copy upstart files
outdir = os.path.join(package_dir, "debian/upstart.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir)
# Copy init files
outdir = os.path.join(package_dir, "debian/initd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/initd/grr-client"), outdir)
# Copy systemd unit file
outdir = os.path.join(package_dir, "debian/systemd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"), outdir)
def setUpModule():
# client_vfs needs a config to be loaded to work.
flags.FLAGS.config = package.ResourcePath(
"grr-response-test", "grr_response_test/test_data/dummyconfig.yaml")
config_lib.ParseConfigCommandLine()
client_vfs.Init()
def _UploadBinary(self, bin_name, server_path):
"""Uploads a binary from the GRR installation dir to the datastore."""
# TODO(user): Upload binaries via the GRR API.
logging.info("Uploading %s binary to server.", server_path)
package_dir = package.ResourcePath("grr-response-test", "grr_response_test")
with open(os.path.join(package_dir, "test_data", bin_name), "rb") as f:
maintenance_utils.UploadSignedConfigBlob(
f.read(), "aff4:/config/executables/%s" % server_path)
def __init__(self):
self.use_fleetspeak: bool = False
self.external_hostname: str = None
self.admin_port = 4444
self.grr_port = 11111
self.https_port = 4443
self.mysql_username: str = None
self.mysql_password: str = None
self.mysql_host: str = None
self.mysql_port = 3306
self.mysql_database: str = None
self.mysql_unix_socket: str = None
self.config_dir = package.ResourcePath(
"fleetspeak-server-bin",
"fleetspeak-server-bin/etc/fleetspeak-server")
self._fleetspeak_config_command_path = package.ResourcePath(
"fleetspeak-server-bin",
"fleetspeak-server-bin/usr/bin/fleetspeak-config")
def testConfigOptionsDefined(self):
"""Test that all config options in use are defined."""
# We need to use the actual config.CONFIG variable since that is where
# all the variables are already defined.
conf = config.CONFIG.MakeNewConfig()
# Check our actual config validates
configpath = package.ResourcePath("grr-response-core",
"install_data/etc/grr-server.yaml")
conf.Initialize(filename=configpath)
def main(argv):
del argv # Unused.
if not flags.FLAGS.dest_server_config_path:
raise ValueError("dest_server_config_path flag has to be provided.")
if not flags.FLAGS.dest_client_config_path:
raise ValueError("dest_client_config_path flag has to be provided.")
admin_ui_port = portpicker.pick_unused_port()
frontend_port = portpicker.pick_unused_port()
source_server_config_path = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
config_lib.LoadConfig(config.CONFIG, source_server_config_path)
config.CONFIG.SetWriteBack(flags.FLAGS.dest_server_config_path)
# TODO(user): remove when AFF4 is gone.
config.CONFIG.Set("Database.enabled", True)
config.CONFIG.Set("Blobstore.implementation", "DbBlobStore")
config.CONFIG.Set("Database.implementation", "MysqlDB")
config.CONFIG.Set("Mysql.database", flags.FLAGS.config_mysql_database)
if flags.FLAGS.config_mysql_username is not None:
config.CONFIG.Set("Mysql.username", flags.FLAGS.config_mysql_username)
if flags.FLAGS.config_mysql_password is not None:
config.CONFIG.Set("Mysql.password", flags.FLAGS.config_mysql_password)
config.CONFIG.Set("AdminUI.port", admin_ui_port)
config.CONFIG.Set("AdminUI.headless", True)
config.CONFIG.Set("Frontend.bind_address", "127.0.0.1")
config.CONFIG.Set("Frontend.bind_port", frontend_port)
config.CONFIG.Set("Server.initialized", True)
config.CONFIG.Set("Cron.active", False)
config.CONFIG.Set("Client.poll_max", 1)
config.CONFIG.Set("Client.server_urls",
["http://localhost:%d/" % frontend_port])
if flags.FLAGS.config_logging_path is not None:
config.CONFIG.Set("Logging.path", flags.FLAGS.config_logging_path)
config.CONFIG.Set("Logging.verbose", False)
if flags.FLAGS.config_osquery_path:
config.CONFIG.Set("Osquery.path", flags.FLAGS.config_osquery_path)
config_updater_keys_util.GenerateKeys(config.CONFIG)
config.CONFIG.Write()
config_lib.SetPlatformArchContext()
context = list(config.CONFIG.context)
context.append("Client Context")
config_data = build_helpers.GetClientConfig(context,
validate=False,
deploy_timestamp=False)
with io.open(flags.FLAGS.dest_client_config_path, "w") as fd:
fd.write(config_data)
def testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = package.ResourcePath("grr-response-core",
"executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1)
# Validate the config appended to the OS X package.
zf = zipfile.ZipFile(glob.glob(
os.path.join(new_dir, "installers/*.pkg")).pop(),
mode="r")
fd = zf.open("config.yaml")
# We can't load the included build.yaml because the package hasn't been
# installed.
loaded = yaml.safe_load(fd)
loaded.pop("Config.includes")
packaged_config = config.CONFIG.MakeNewConfig()
data = yaml.safe_dump(loaded)
packaged_config.Initialize(parser=config_lib.YamlParser,
data=data.decode("utf-8"))
packaged_config.Validate(
sections=build.ClientRepacker.CONFIG_SECTIONS)
repacker = build.ClientRepacker()
repacker.ValidateEndConfig(packaged_config)
def testSampleDataParsedCorrectly(self):
for fname, expected in self.SAMPLE_LIST.items():
path = package.ResourcePath(
'grr-response-test',
os.path.join('grr_response_test', 'test_data', 'fingerprint', fname))
with io.open(path, 'rb') as f:
fp = fingerprint.Fingerprinter(f)
fp.EvalGeneric()
fp.EvalPecoff()
result = fp.HashIt()
self.assertCountEqual(result, expected,
'Hashing results for %s do not match.' % fname)
def testNotEmpty(self):
parser = ie_history.IEHistoryParser()
index_dat = os.path.join("grr_response_test", "test_data", "index.dat")
index_dat_path = package.ResourcePath("grr-response-test", index_dat)
knowledge_base = rdf_client.KnowledgeBase()
pathspec = rdf_paths.PathSpec.OS(path=index_dat_path)
with io.open(index_dat_path, mode="rb") as filedesc:
results = list(parser.ParseFile(knowledge_base, pathspec, filedesc))
self.assertNotEmpty(results)
def main(argv):
config_dir = package.ResourcePath(
"fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server")
if not os.path.exists(config_dir):
raise Error(
f"Configuration directory not found: {config_dir}. "
"Please make sure `grr_config_updater initialize` has been run.")
fleetspeak_server = package.ResourcePath(
"fleetspeak-server-bin",
"fleetspeak-server-bin/usr/bin/fleetspeak-server")
if not os.path.exists(fleetspeak_server):
raise Error(
f"Fleetspeak server binary not found: {fleetspeak_server}. "
"Please make sure that the package `fleetspeak-server-bin` has been "
"installed.")
command = [
fleetspeak_server,
"--logtostderr",
"--services_config",
os.path.join(config_dir, "server.services.config"),
"--components_config",
os.path.join(config_dir, "server.components.config"),
]
subprocess.check_call(command)
def _CopyCommonRpmFiles(package_dir, dist_dir):
"""Copies common (fleetspeak and legacy) files into the template folder."""
# Copy the wrapper script.
shutil.copy(
package.ResourcePath("grr-response-core",
"install_data/wrapper.sh.in"), dist_dir)
shutil.move(os.path.join(package_dir, "debian"),
os.path.join(package_dir, "rpmbuild"))
# Copy prelink blacklist file. Without this file, prelink will mangle
# the GRR binary.
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/prelink_blacklist.conf.in"),
os.path.join(package_dir, "rpmbuild/prelink_blacklist.conf.in"))
def main(argv):
del argv # Unused.
if not flags.FLAGS.dest_server_config_path:
raise ValueError("dest_server_config_path flag has to be provided.")
if not flags.FLAGS.dest_client_config_path:
raise ValueError("dest_client_config_path flag has to be provided.")
admin_ui_port = portpicker.pick_unused_port()
frontend_port = portpicker.pick_unused_port()
datastore_port = portpicker.pick_unused_port()
source_server_config_path = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
config_lib.LoadConfig(config.CONFIG, source_server_config_path)
config.CONFIG.SetWriteBack(flags.FLAGS.dest_server_config_path)
# TODO(user): remove when AFF4 is gone.
config.CONFIG.Set("Database.aff4_enabled", False)
config.CONFIG.Set("Database.enabled", True)
config.CONFIG.Set("Blobstore.implementation", "DbBlobStore")
config.CONFIG.Set("Database.implementation", "SharedMemoryDB")
config.CONFIG.Set("SharedMemoryDB.port", datastore_port)
config.CONFIG.Set("AdminUI.port", admin_ui_port)
config.CONFIG.Set("AdminUI.headless", True)
config.CONFIG.Set("Frontend.bind_address", "127.0.0.1")
config.CONFIG.Set("Frontend.bind_port", frontend_port)
config.CONFIG.Set("Server.initialized", True)
config.CONFIG.Set("Client.poll_max", 1)
config.CONFIG.Set("Client.server_urls",
["http://localhost:%d/" % frontend_port])
config_updater_keys_util.GenerateKeys(config.CONFIG)
config.CONFIG.Write()
config_lib.SetPlatformArchContext()
context = list(config.CONFIG.context)
context.append("Client Context")
deployer = build.ClientRepacker()
config_data = deployer.GetClientConfig(context,
validate=False,
deploy_timestamp=False)
with io.open(flags.FLAGS.dest_client_config_path, "w") as fd:
fd.write(config_data)
def CopyFiles(self):
"""This sets up the template directory."""
if self.fleetspeak_enabled:
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/fleetspeak-debian"),
os.path.join(self.package_dir, "debian/debian.in"))
return
# Copy the nanny binary.
shutil.copy(
package.ResourcePath(
"grr-response-core",
"install_data/debian/dpkg_client/nanny.sh.in"),
self.output_dir)
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/debian"),
os.path.join(self.package_dir, "debian/debian.in"))
# Copy upstart files
outdir = os.path.join(self.package_dir, "debian/upstart.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/upstart/grr-client.conf"),
outdir)
# Copy init files
outdir = os.path.join(self.package_dir, "debian/initd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/initd/grr-client"), outdir)
# Copy systemd unit file
outdir = os.path.join(self.package_dir, "debian/systemd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"), outdir)
def _RunClient(tmp_dir: str) -> None:
"""Runs the fleetspeak client."""
config_path = _CreateClientConfig(tmp_dir)
_CreateServiceConfig(tmp_dir)
fleetspeak_client = package.ResourcePath(
"fleetspeak-client-bin",
"fleetspeak-client-bin/usr/bin/fleetspeak-client")
if not fleetspeak_client or not os.path.exists(fleetspeak_client):
raise Error(
f"Fleetspeak client binary not found: {fleetspeak_client}."
"Please make sure that the package `fleetspeak-client-bin` has been "
"installed.")
command = [
fleetspeak_client,
"--logtostderr",
"-config",
config_path,
]
subprocess.check_call(command)
def setUpClass(cls):
super(GRRSeleniumTest, cls).setUpClass()
with GRRSeleniumTest._selenium_set_up_lock:
if not GRRSeleniumTest._selenium_set_up_done:
port = portpicker.pick_unused_port()
logging.info("Picked free AdminUI port %d.", port)
# Start up a server in another thread
GRRSeleniumTest._server_trd = wsgiapp_testlib.ServerThread(
port, name="SeleniumServerThread")
GRRSeleniumTest._server_trd.StartAndWaitUntilServing()
GRRSeleniumTest._SetUpSelenium(port)
jquery_path = package.ResourcePath(
"grr-response-test",
"grr_response_test/test_data/jquery_3.1.0.min.js")
with open(jquery_path, mode="r", encoding="utf-8") as fd:
GRRSeleniumTest._jquery_source = fd.read()
GRRSeleniumTest._selenium_set_up_done = True
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
metric_metadata = server_metrics.GetMetadata()
metric_metadata.extend(client_metrics.GetMetadata())
metric_metadata.extend(communicator.GetMetricMetadata())
stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
metric_metadata)
stats_collector_instance.Set(stats_collector)
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(
package.ResourcePath("grr-response-test",
"grr_response_test/test_data/grr_test.yaml"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = compatibility.GetName(fake_data_store.FakeDataStore)
config.CONFIG.Set("Datastore.implementation", test_ds)
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
blob_store_test_lib.UseTestBlobStore()
data_store.InitializeDataStore()
if data_store.AFF4Enabled():
aff4.AFF4Init() # Requires data_store.InitializeDataStore.
aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init.
filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init.
results.ResultQueueInit() # Requires aff4.AFF4Init.
sequential_collection.StartUpdaterOnce()
artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init.
checks.LoadChecksFromFilesystemOnce()
client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce()
cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init.
email_alerts.InitializeEmailAlerterOnce()
http_api.InitializeHttpRequestHandlerOnce()
ip_resolver.IPResolverInitOnce()
stats_server.InitializeStatsServerOnce()
webauth.InitializeWebAuthOnce()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
if not utils.TimeBasedCache.house_keeper_thread:
utils.TimeBasedCache()
utils.TimeBasedCache.house_keeper_thread.exit = True
utils.TimeBasedCache.house_keeper_thread.join()
INIT_RAN = True
def _CopyCommonRpmFiles(dist_dir):
# Copy the wrapper script.
shutil.copy(
package.ResourcePath("grr-response-core",
"install_data/wrapper.sh.in"), dist_dir)
def _MakeMsi(input_dir: str, output_path: str) -> None:
"""Packages the PyInstaller files as MSI."""
wxs_file = package.ResourcePath("grr-response-core",
"install_data/windows/grr.wxs")
fleetspeak_wxs_lib = os.path.join(
config.CONFIG["ClientBuilder.fleetspeak_install_dir"],
"fleetspeak_lib.wxs")
# Don't automatically harvest these files using heat.exe, since they
# are treated specially in grr.wxs.
exclude_files = [
"grr-client.exe",
"dbg_grr-client.exe",
"GRRservice.exe",
"dbg_GRRservice.exe",
"fleetspeak-client.exe",
"grr-client.exe.manifest",
]
def Run(args: List[str]):
logging.info("Running: %s.", args)
subprocess.check_call(args)
with utils.TempDirectory() as temp_dir:
for exclude_file in exclude_files:
shutil.move(os.path.join(input_dir, exclude_file),
os.path.join(temp_dir, exclude_file))
Run([
os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin",
"heat.exe"),
"dir",
input_dir,
"-sfrag",
"-srd",
"-cg",
"CompGrrAutoFiles",
"-ag",
"-var",
"var.InputDir",
"-dr",
"INSTALLDIR",
"-out",
os.path.join(temp_dir, "heat.wxs"),
])
for exclude_file in exclude_files:
shutil.move(os.path.join(temp_dir, exclude_file),
os.path.join(input_dir, exclude_file))
# Generate padded placeholder files for these config files.
# The reason for the padding is that at repacking time, we have to replace
# the config files with files of the same size. So we make them large
# enough to fit any expected config file.
for placeholder_file in [
"grr-config.yaml", "fleetspeak-client.config",
"fleetspeak-service-config.txt"
]:
with open(os.path.join(input_dir, placeholder_file),
"w",
newline="\n") as f:
f.write("__BEGIN__\n")
f.write("\n" * (100 * 1024))
f.write("__END__")
object_files = []
for source_file in (wxs_file, fleetspeak_wxs_lib,
os.path.join(temp_dir, "heat.wxs")):
object_file = os.path.join(
temp_dir,
os.path.basename(source_file) + ".wxobj")
Run([
os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"],
"bin", "candle.exe"),
source_file,
"-arch",
"x64",
"-ext",
"WixUtilExtension",
"-dFLEETSPEAK_EXECUTABLE=" +
os.path.join(input_dir, "fleetspeak-client.exe"),
"-dVERSION=" + config.CONFIG["Source.version_string"],
"-sw1150",
f"-dInputDir={input_dir}",
"-out",
object_file,
])
object_files.append(object_file)
Run([
os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin",
"light.exe"),
] + object_files + [
"-ext",
"WixUtilExtension",
"-sw1076",
"-out",
os.path.join(temp_dir, "installer.msi"),
])
with zipfile.ZipFile(output_path, "w") as zip_output:
zip_output.write(os.path.join(temp_dir, "installer.msi"),
"installer.msi")
zip_output.write(os.path.join(input_dir, "build.yaml"),
"build.yaml")
def _MakeMsi(input_dir: str, output_path: str) -> None:
"""Packages the PyInstaller files as MSI."""
wxs_file = package.ResourcePath("grr-response-core",
"install_data/windows/grr.wxs")
fleetspeak_wxs_lib = os.path.join(
config.CONFIG["ClientBuilder.fleetspeak_install_dir"],
"fleetspeak_lib.wxs")
# Don't automatically harvest these files using heat.exe, since they
# are treated specially in grr.wxs.
exclude_files = [
"grr-client.exe",
"dbg_grr-client.exe",
"GRRservice.exe",
"dbg_GRRservice.exe",
"fleetspeak-client.exe",
"grr-client.exe.manifest",
]
def Run(args: List[str]):
logging.info("Running: %s.", args)
subprocess.check_call(args)
with utils.TempDirectory() as temp_dir:
for exclude_file in exclude_files:
shutil.move(os.path.join(input_dir, exclude_file),
os.path.join(temp_dir, exclude_file))
Run([
os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin",
"heat.exe"),
"dir",
input_dir,
"-sfrag",
"-srd",
"-cg",
"CompGrrAutoFiles",
"-ag",
"-var",
"var.InputDir",
"-dr",
"INSTALLDIR",
"-out",
os.path.join(temp_dir, "heat.wxs"),
])
for exclude_file in exclude_files:
shutil.move(os.path.join(temp_dir, exclude_file),
os.path.join(input_dir, exclude_file))
for placeholder_file in [
"grr-config.yaml", "fleetspeak-client.config",
"fleetspeak-service-config.txt"
]:
with open(os.path.join(input_dir, placeholder_file),
"w",
newline="\n"):
pass
# Due to a limitation in the olefile library, at repacking time, the CAB
# in the MSI file needs to be repacked to a CAB file of same size.
# To do so, we add 3 MB of random (uncompressable) data into a padding
# file.
# At repacking time, the padding file will get truncated to make space for
# other files (config files, signed EXE and DLL files) to grow.
with open(os.path.join(input_dir, "padding-file.bin"), "wb") as f:
for _ in range(3):
f.write(os.urandom(1024 * 1024))
object_files = []
for source_file in (wxs_file, fleetspeak_wxs_lib,
os.path.join(temp_dir, "heat.wxs")):
object_file = os.path.join(
temp_dir,
os.path.basename(source_file) + ".wxobj")
Run([
os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"],
"bin", "candle.exe"),
source_file,
"-arch",
"x64",
"-ext",
"WixUtilExtension",
"-dFLEETSPEAK_EXECUTABLE=" +
os.path.join(input_dir, "fleetspeak-client.exe"),
"-dVERSION=" + config.CONFIG["Source.version_string"],
"-sw1150",
f"-dInputDir={input_dir}",
"-out",
object_file,
])
object_files.append(object_file)
Run([
os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin",
"light.exe"),
] + object_files + [
"-ext",
"WixUtilExtension",
"-sw1076",
"-out",
os.path.join(temp_dir, "installer.msi"),
])
with zipfile.ZipFile(output_path, "w") as zip_output:
zip_output.write(os.path.join(temp_dir, "installer.msi"),
"installer.msi")
zip_output.write(os.path.join(input_dir, "build.yaml"),
"build.yaml")
