def InterpolateFiles(self): if self.fleetspeak_enabled: shutil.copy(flags.FLAGS.fleetspeak_service_config, self.pkg_fleetspeak_service_dir) build_files_dir = package.ResourcePath( "grr-response-core", "install_data/macosx/client/fleetspeak") else: build_files_dir = package.ResourcePath( "grr-response-core", "install_data/macosx/client") self.GenerateFile( input_filename=os.path.join(build_files_dir, "grr.plist.in"), output_filename=os.path.join(self.pkg_root, "Library/LaunchDaemons", self.plist_name)) # We pass in scripts separately with --scripts so they don't go in pkg_root self.GenerateFile(input_filename=os.path.join(build_files_dir, "preinstall.sh.in"), output_filename=os.path.join(self.script_dir, "preinstall")) self.GenerateFile(input_filename=os.path.join(build_files_dir, "postinstall.sh.in"), output_filename=os.path.join(self.script_dir, "postinstall")) self.GenerateFile(input_filename=os.path.join(build_files_dir, "Distribution.xml.in"), output_filename=os.path.join(self.build_dir, "Distribution.xml"))
def GetServerComponentArgs(config_path): """Returns a set of command line arguments for server components. Args: config_path: Path to a config path generated by self_contained_config_writer. Returns: An iterable with command line arguments to use. """ primary_config_path = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") secondary_config_path = package.ResourcePath( "grr-response-test", "grr_response_test/test_data/grr_test.yaml") return [ "--config", primary_config_path, "--secondary_configs", ",".join([secondary_config_path, config_path]), "-p", "Monitoring.http_port=%d" % portpicker.pick_unused_port(), "-p", "AdminUI.webauth_manager=NullWebAuthManager", ]
def InterpolateFiles(self): if self.fleetspeak_enabled: fleetspeak_template = config.CONFIG.Get( "ClientBuilder.fleetspeak_config_path", context=self.context) dest_fleetspeak_config = os.path.join( self.pkg_fleetspeak_service_dir, config.CONFIG.Get("Client.fleetspeak_unsigned_config_fname", context=self.context)) self.GenerateFile(input_filename=fleetspeak_template, output_filename=dest_fleetspeak_config) build_files_dir = package.ResourcePath( "grr-response-core", "install_data/macosx/client/fleetspeak") else: build_files_dir = package.ResourcePath( "grr-response-core", "install_data/macosx/client") self.GenerateFile( input_filename=os.path.join(build_files_dir, "grr.plist.in"), output_filename=os.path.join(self.pkg_root, "Library/LaunchDaemons", self.plist_name)) # We pass in scripts separately with --scripts so they don't go in pkg_root self.GenerateFile(input_filename=os.path.join(build_files_dir, "preinstall.sh.in"), output_filename=os.path.join(self.script_dir, "preinstall")) self.GenerateFile(input_filename=os.path.join(build_files_dir, "postinstall.sh.in"), output_filename=os.path.join(self.script_dir, "postinstall")) self.GenerateFile(input_filename=os.path.join(build_files_dir, "Distribution.xml.in"), output_filename=os.path.join(self.build_dir, "Distribution.xml"))
def TestInit(): """Only used in tests and will rerun all the hooks to create a clean state.""" global INIT_RAN stats_collector = prometheus_stats_collector.PrometheusStatsCollector() stats_collector_instance.Set(stats_collector) # Tests use both the server template grr_server.yaml as a primary config file # (this file does not contain all required options, e.g. private keys), and # additional configuration in test_data/grr_test.yaml which contains typical # values for a complete installation. flags.FLAGS.config = package.ResourcePath("grr-response-core", "install_data/etc/grr-server.yaml") flags.FLAGS.secondary_configs.append( package.ResourcePath("grr-response-test", "grr_response_test/test_data/grr_test.yaml")) # This config contains non-public settings that should be applied during # tests. extra_test_config = config.CONFIG["Test.additional_test_config"] if os.path.exists(extra_test_config): flags.FLAGS.secondary_configs.append(extra_test_config) # Prevent using the default writeback location since it may clash with local # setup. writeback_filepath = temp.TempFilePath(prefix="grr_writeback", suffix=".yaml") config.CONFIG.global_override["Config.writeback"] = writeback_filepath # Tests additionally add a test configuration file. config_lib.SetPlatformArchContext() config_lib.ParseConfigCommandLine() # We are running a test so let the config system know that. config.CONFIG.AddContext(contexts.TEST_CONTEXT, "Context applied when we run tests.") if not INIT_RAN: server_logging.ServerLoggingStartupInit() server_logging.SetTestVerbosity() blob_store_test_lib.UseTestBlobStore() data_store.InitializeDataStore() artifact.LoadArtifactsOnce() checks.LoadChecksFromFilesystemOnce() client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce() email_alerts.InitializeEmailAlerterOnce() http_api.InitializeHttpRequestHandlerOnce() ip_resolver.IPResolverInitOnce() stats_server.InitializeStatsServerOnce() webauth.InitializeWebAuthOnce() if not utils.TimeBasedCache.house_keeper_thread: utils.TimeBasedCache() utils.TimeBasedCache.house_keeper_thread.exit = True utils.TimeBasedCache.house_keeper_thread.join() INIT_RAN = True
def TestInit(): """Only used in tests and will rerun all the hooks to create a clean state.""" global INIT_RAN metric_metadata = server_metrics.GetMetadata() metric_metadata.extend(client_metrics.GetMetadata()) metric_metadata.extend(communicator.GetMetricMetadata()) stats_collector = default_stats_collector.DefaultStatsCollector( metric_metadata) stats_collector_instance.Set(stats_collector) # Tests use both the server template grr_server.yaml as a primary config file # (this file does not contain all required options, e.g. private keys), and # additional configuration in test_data/grr_test.yaml which contains typical # values for a complete installation. flags.FLAGS.config = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") flags.FLAGS.secondary_configs.append( package.ResourcePath("grr-response-test", "grr_response_test/test_data/grr_test.yaml")) # This config contains non-public settings that should be applied during # tests. extra_test_config = config.CONFIG["Test.additional_test_config"] if os.path.exists(extra_test_config): flags.FLAGS.secondary_configs.append(extra_test_config) # Tests additionally add a test configuration file. config_lib.SetPlatformArchContext() config_lib.ParseConfigCommandLine() # We are running a test so let the config system know that. config.CONFIG.AddContext(contexts.TEST_CONTEXT, "Context applied when we run tests.") test_ds = flags.FLAGS.test_data_store if test_ds is None: test_ds = compatibility.GetName(fake_data_store.FakeDataStore) config.CONFIG.Set("Datastore.implementation", test_ds) if not INIT_RAN: server_logging.ServerLoggingStartupInit() server_logging.SetTestVerbosity() blob_store_test_lib.UseTestBlobStore() registry.TestInit() db = data_store.DB.SetupTestDB() if db: data_store.DB = db data_store.DB.Initialize() aff4.AFF4InitHook().Run() INIT_RAN = True
def VersionPath(): """Returns a path to version.ini.""" # Try to get a version.ini. It should be in the resources if the code # was packed with "pip sdist". It will be 2 levels up from grr_response_core # if the code was installed via "pip install -e". version_ini = (package.ResourcePath("grr-response-core", "version.ini") or package.ResourcePath("grr-response-core", "../../version.ini")) if not os.path.exists(version_ini): raise RuntimeError("Can't find version.ini at %s" % version_ini) return version_ini
def _CreateClientConfig(tmp_dir: str) -> str: """Creates and returns the path to a fleetspeak client config.""" def TmpPath(*args): return os.path.join(tmp_dir, *args) server_config_dir = package.ResourcePath( "fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server") if not os.path.exists(server_config_dir): raise Error( f"Fleetspeak server config dir not found: {server_config_dir}. " "Please make sure `grr_config_updater initialize` has been run.") client_config_name = { "Linux": "linux_client.config", "Windows": "windows_client.config", "Darwin": "darwin_client.config", } client_config_path = os.path.join(server_config_dir, client_config_name[platform.system()]) with open(client_config_path, "r") as f: client_config = text_format.Parse(f.read(), fs_cli_config_pb2.Config()) if client_config.HasField("filesystem_handler"): client_config.filesystem_handler.configuration_directory = TmpPath() # We store the client state file in `Logging.path`. # 1) We need a writable path, where writing a file doesn't surprise the # user (as opposed to other paths in the source tree). # 2) We need the state file to be somewhat persistent, i.e. kept after # re-runs of this command. Otherwise the client ID of the client would # change at each re-run. client_config.filesystem_handler.state_file = os.path.join( config.CONFIG["Logging.path"], "fleetspeak-client.state") with open(TmpPath("config"), "w") as f: f.write(text_format.MessageToString(client_config)) return TmpPath("config")
def CopyFiles(self): """This sets up the template directory.""" # Copy the nanny binary. shutil.copy( package.ResourcePath("grr-response-core", "install_data/debian/dpkg_client/nanny.sh.in"), self.output_dir) dpkg_dir = config.CONFIG.Get("PyInstaller.dpkg_root", context=self.context) # Copy files needed for dpkg-buildpackage. shutil.copytree( config_lib.Resource().Filter("install_data/debian/dpkg_client/debian"), os.path.join(dpkg_dir, "debian/debian.in")) # Copy upstart files outdir = os.path.join(dpkg_dir, "debian/upstart.in") utils.EnsureDirExists(outdir) shutil.copy(config_lib.Resource().Filter( "install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir) # Copy init files outdir = os.path.join(dpkg_dir, "debian/initd.in") utils.EnsureDirExists(outdir) shutil.copy(config_lib.Resource().Filter( "install_data/debian/dpkg_client/initd/grr-client"), outdir) # Copy systemd unit file outdir = os.path.join(dpkg_dir, "debian/systemd.in") utils.EnsureDirExists(outdir) shutil.copy(config_lib.Resource().Filter( "install_data/systemd/client/grr-client.service"), outdir)
def _CopyNonFleetspeakDpkgFiles(dist_dir, package_dir): """Copies non-Fleetspeak-enabled DPKG files to template directory.""" # Copy the nanny binary. shutil.copy( package.ResourcePath("grr-response-core", "install_data/debian/dpkg_client/nanny.sh.in"), dist_dir) # Copy files needed for dpkg-buildpackage. shutil.copytree( config_lib.Resource().Filter("install_data/debian/dpkg_client/debian"), os.path.join(package_dir, "debian/legacy-debian.in")) # Copy upstart files outdir = os.path.join(package_dir, "debian/upstart.in") utils.EnsureDirExists(outdir) shutil.copy( config_lib.Resource().Filter( "install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir) # Copy init files outdir = os.path.join(package_dir, "debian/initd.in") utils.EnsureDirExists(outdir) shutil.copy( config_lib.Resource().Filter( "install_data/debian/dpkg_client/initd/grr-client"), outdir) # Copy systemd unit file outdir = os.path.join(package_dir, "debian/systemd.in") utils.EnsureDirExists(outdir) shutil.copy( config_lib.Resource().Filter( "install_data/systemd/client/grr-client.service"), outdir)
def setUpModule(): # client_vfs needs a config to be loaded to work. flags.FLAGS.config = package.ResourcePath( "grr-response-test", "grr_response_test/test_data/dummyconfig.yaml") config_lib.ParseConfigCommandLine() client_vfs.Init()
def _UploadBinary(self, bin_name, server_path): """Uploads a binary from the GRR installation dir to the datastore.""" # TODO(user): Upload binaries via the GRR API. logging.info("Uploading %s binary to server.", server_path) package_dir = package.ResourcePath("grr-response-test", "grr_response_test") with open(os.path.join(package_dir, "test_data", bin_name), "rb") as f: maintenance_utils.UploadSignedConfigBlob( f.read(), "aff4:/config/executables/%s" % server_path)
def __init__(self): self.use_fleetspeak: bool = False self.external_hostname: str = None self.admin_port = 4444 self.grr_port = 11111 self.https_port = 4443 self.mysql_username: str = None self.mysql_password: str = None self.mysql_host: str = None self.mysql_port = 3306 self.mysql_database: str = None self.mysql_unix_socket: str = None self.config_dir = package.ResourcePath( "fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server") self._fleetspeak_config_command_path = package.ResourcePath( "fleetspeak-server-bin", "fleetspeak-server-bin/usr/bin/fleetspeak-config")
def testConfigOptionsDefined(self): """Test that all config options in use are defined.""" # We need to use the actual config.CONFIG variable since that is where # all the variables are already defined. conf = config.CONFIG.MakeNewConfig() # Check our actual config validates configpath = package.ResourcePath("grr-response-core", "install_data/etc/grr-server.yaml") conf.Initialize(filename=configpath)
def main(argv): del argv # Unused. if not flags.FLAGS.dest_server_config_path: raise ValueError("dest_server_config_path flag has to be provided.") if not flags.FLAGS.dest_client_config_path: raise ValueError("dest_client_config_path flag has to be provided.") admin_ui_port = portpicker.pick_unused_port() frontend_port = portpicker.pick_unused_port() source_server_config_path = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") config_lib.LoadConfig(config.CONFIG, source_server_config_path) config.CONFIG.SetWriteBack(flags.FLAGS.dest_server_config_path) # TODO(user): remove when AFF4 is gone. config.CONFIG.Set("Database.enabled", True) config.CONFIG.Set("Blobstore.implementation", "DbBlobStore") config.CONFIG.Set("Database.implementation", "MysqlDB") config.CONFIG.Set("Mysql.database", flags.FLAGS.config_mysql_database) if flags.FLAGS.config_mysql_username is not None: config.CONFIG.Set("Mysql.username", flags.FLAGS.config_mysql_username) if flags.FLAGS.config_mysql_password is not None: config.CONFIG.Set("Mysql.password", flags.FLAGS.config_mysql_password) config.CONFIG.Set("AdminUI.port", admin_ui_port) config.CONFIG.Set("AdminUI.headless", True) config.CONFIG.Set("Frontend.bind_address", "127.0.0.1") config.CONFIG.Set("Frontend.bind_port", frontend_port) config.CONFIG.Set("Server.initialized", True) config.CONFIG.Set("Cron.active", False) config.CONFIG.Set("Client.poll_max", 1) config.CONFIG.Set("Client.server_urls", ["http://localhost:%d/" % frontend_port]) if flags.FLAGS.config_logging_path is not None: config.CONFIG.Set("Logging.path", flags.FLAGS.config_logging_path) config.CONFIG.Set("Logging.verbose", False) if flags.FLAGS.config_osquery_path: config.CONFIG.Set("Osquery.path", flags.FLAGS.config_osquery_path) config_updater_keys_util.GenerateKeys(config.CONFIG) config.CONFIG.Write() config_lib.SetPlatformArchContext() context = list(config.CONFIG.context) context.append("Client Context") config_data = build_helpers.GetClientConfig(context, validate=False, deploy_timestamp=False) with io.open(flags.FLAGS.dest_client_config_path, "w") as fd: fd.write(config_data)
def testRepackAll(self): """Test repacking all binaries.""" self.executables_dir = package.ResourcePath("grr-response-core", "executables") with utils.TempDirectory() as tmp_dir: new_dir = os.path.join(tmp_dir, "grr", "executables") os.makedirs(new_dir) # Copy unzipsfx so it can be used in repacking/ shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-i386.exe"), new_dir) shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-amd64.exe"), new_dir) with test_lib.ConfigOverrider({ "ClientBuilder.executables_dir": new_dir, "ClientBuilder.unzipsfx_stub_dir": new_dir }): repacking.TemplateRepacker().RepackAllTemplates() self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1) # Validate the config appended to the OS X package. zf = zipfile.ZipFile(glob.glob( os.path.join(new_dir, "installers/*.pkg")).pop(), mode="r") fd = zf.open("config.yaml") # We can't load the included build.yaml because the package hasn't been # installed. loaded = yaml.safe_load(fd) loaded.pop("Config.includes") packaged_config = config.CONFIG.MakeNewConfig() data = yaml.safe_dump(loaded) packaged_config.Initialize(parser=config_lib.YamlParser, data=data.decode("utf-8")) packaged_config.Validate( sections=build.ClientRepacker.CONFIG_SECTIONS) repacker = build.ClientRepacker() repacker.ValidateEndConfig(packaged_config)
def testSampleDataParsedCorrectly(self): for fname, expected in self.SAMPLE_LIST.items(): path = package.ResourcePath( 'grr-response-test', os.path.join('grr_response_test', 'test_data', 'fingerprint', fname)) with io.open(path, 'rb') as f: fp = fingerprint.Fingerprinter(f) fp.EvalGeneric() fp.EvalPecoff() result = fp.HashIt() self.assertCountEqual(result, expected, 'Hashing results for %s do not match.' % fname)
def testNotEmpty(self): parser = ie_history.IEHistoryParser() index_dat = os.path.join("grr_response_test", "test_data", "index.dat") index_dat_path = package.ResourcePath("grr-response-test", index_dat) knowledge_base = rdf_client.KnowledgeBase() pathspec = rdf_paths.PathSpec.OS(path=index_dat_path) with io.open(index_dat_path, mode="rb") as filedesc: results = list(parser.ParseFile(knowledge_base, pathspec, filedesc)) self.assertNotEmpty(results)
def main(argv): config_dir = package.ResourcePath( "fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server") if not os.path.exists(config_dir): raise Error( f"Configuration directory not found: {config_dir}. " "Please make sure `grr_config_updater initialize` has been run.") fleetspeak_server = package.ResourcePath( "fleetspeak-server-bin", "fleetspeak-server-bin/usr/bin/fleetspeak-server") if not os.path.exists(fleetspeak_server): raise Error( f"Fleetspeak server binary not found: {fleetspeak_server}. " "Please make sure that the package `fleetspeak-server-bin` has been " "installed.") command = [ fleetspeak_server, "--logtostderr", "--services_config", os.path.join(config_dir, "server.services.config"), "--components_config", os.path.join(config_dir, "server.components.config"), ] subprocess.check_call(command)
def _CopyCommonRpmFiles(package_dir, dist_dir): """Copies common (fleetspeak and legacy) files into the template folder.""" # Copy the wrapper script. shutil.copy( package.ResourcePath("grr-response-core", "install_data/wrapper.sh.in"), dist_dir) shutil.move(os.path.join(package_dir, "debian"), os.path.join(package_dir, "rpmbuild")) # Copy prelink blacklist file. Without this file, prelink will mangle # the GRR binary. shutil.copy( config_lib.Resource().Filter( "install_data/centos/prelink_blacklist.conf.in"), os.path.join(package_dir, "rpmbuild/prelink_blacklist.conf.in"))
def main(argv): del argv # Unused. if not flags.FLAGS.dest_server_config_path: raise ValueError("dest_server_config_path flag has to be provided.") if not flags.FLAGS.dest_client_config_path: raise ValueError("dest_client_config_path flag has to be provided.") admin_ui_port = portpicker.pick_unused_port() frontend_port = portpicker.pick_unused_port() datastore_port = portpicker.pick_unused_port() source_server_config_path = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") config_lib.LoadConfig(config.CONFIG, source_server_config_path) config.CONFIG.SetWriteBack(flags.FLAGS.dest_server_config_path) # TODO(user): remove when AFF4 is gone. config.CONFIG.Set("Database.aff4_enabled", False) config.CONFIG.Set("Database.enabled", True) config.CONFIG.Set("Blobstore.implementation", "DbBlobStore") config.CONFIG.Set("Database.implementation", "SharedMemoryDB") config.CONFIG.Set("SharedMemoryDB.port", datastore_port) config.CONFIG.Set("AdminUI.port", admin_ui_port) config.CONFIG.Set("AdminUI.headless", True) config.CONFIG.Set("Frontend.bind_address", "127.0.0.1") config.CONFIG.Set("Frontend.bind_port", frontend_port) config.CONFIG.Set("Server.initialized", True) config.CONFIG.Set("Client.poll_max", 1) config.CONFIG.Set("Client.server_urls", ["http://localhost:%d/" % frontend_port]) config_updater_keys_util.GenerateKeys(config.CONFIG) config.CONFIG.Write() config_lib.SetPlatformArchContext() context = list(config.CONFIG.context) context.append("Client Context") deployer = build.ClientRepacker() config_data = deployer.GetClientConfig(context, validate=False, deploy_timestamp=False) with io.open(flags.FLAGS.dest_client_config_path, "w") as fd: fd.write(config_data)
def CopyFiles(self): """This sets up the template directory.""" if self.fleetspeak_enabled: # Copy files needed for dpkg-buildpackage. shutil.copytree( config_lib.Resource().Filter( "install_data/debian/dpkg_client/fleetspeak-debian"), os.path.join(self.package_dir, "debian/debian.in")) return # Copy the nanny binary. shutil.copy( package.ResourcePath( "grr-response-core", "install_data/debian/dpkg_client/nanny.sh.in"), self.output_dir) # Copy files needed for dpkg-buildpackage. shutil.copytree( config_lib.Resource().Filter( "install_data/debian/dpkg_client/debian"), os.path.join(self.package_dir, "debian/debian.in")) # Copy upstart files outdir = os.path.join(self.package_dir, "debian/upstart.in") utils.EnsureDirExists(outdir) shutil.copy( config_lib.Resource().Filter( "install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir) # Copy init files outdir = os.path.join(self.package_dir, "debian/initd.in") utils.EnsureDirExists(outdir) shutil.copy( config_lib.Resource().Filter( "install_data/debian/dpkg_client/initd/grr-client"), outdir) # Copy systemd unit file outdir = os.path.join(self.package_dir, "debian/systemd.in") utils.EnsureDirExists(outdir) shutil.copy( config_lib.Resource().Filter( "install_data/systemd/client/grr-client.service"), outdir)
def _RunClient(tmp_dir: str) -> None: """Runs the fleetspeak client.""" config_path = _CreateClientConfig(tmp_dir) _CreateServiceConfig(tmp_dir) fleetspeak_client = package.ResourcePath( "fleetspeak-client-bin", "fleetspeak-client-bin/usr/bin/fleetspeak-client") if not fleetspeak_client or not os.path.exists(fleetspeak_client): raise Error( f"Fleetspeak client binary not found: {fleetspeak_client}." "Please make sure that the package `fleetspeak-client-bin` has been " "installed.") command = [ fleetspeak_client, "--logtostderr", "-config", config_path, ] subprocess.check_call(command)
def setUpClass(cls): super(GRRSeleniumTest, cls).setUpClass() with GRRSeleniumTest._selenium_set_up_lock: if not GRRSeleniumTest._selenium_set_up_done: port = portpicker.pick_unused_port() logging.info("Picked free AdminUI port %d.", port) # Start up a server in another thread GRRSeleniumTest._server_trd = wsgiapp_testlib.ServerThread( port, name="SeleniumServerThread") GRRSeleniumTest._server_trd.StartAndWaitUntilServing() GRRSeleniumTest._SetUpSelenium(port) jquery_path = package.ResourcePath( "grr-response-test", "grr_response_test/test_data/jquery_3.1.0.min.js") with open(jquery_path, mode="r", encoding="utf-8") as fd: GRRSeleniumTest._jquery_source = fd.read() GRRSeleniumTest._selenium_set_up_done = True
def TestInit(): """Only used in tests and will rerun all the hooks to create a clean state.""" global INIT_RAN metric_metadata = server_metrics.GetMetadata() metric_metadata.extend(client_metrics.GetMetadata()) metric_metadata.extend(communicator.GetMetricMetadata()) stats_collector = prometheus_stats_collector.PrometheusStatsCollector( metric_metadata) stats_collector_instance.Set(stats_collector) # Tests use both the server template grr_server.yaml as a primary config file # (this file does not contain all required options, e.g. private keys), and # additional configuration in test_data/grr_test.yaml which contains typical # values for a complete installation. flags.FLAGS.config = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") flags.FLAGS.secondary_configs.append( package.ResourcePath("grr-response-test", "grr_response_test/test_data/grr_test.yaml")) # This config contains non-public settings that should be applied during # tests. extra_test_config = config.CONFIG["Test.additional_test_config"] if os.path.exists(extra_test_config): flags.FLAGS.secondary_configs.append(extra_test_config) # Tests additionally add a test configuration file. config_lib.SetPlatformArchContext() config_lib.ParseConfigCommandLine() # We are running a test so let the config system know that. config.CONFIG.AddContext(contexts.TEST_CONTEXT, "Context applied when we run tests.") test_ds = flags.FLAGS.test_data_store if test_ds is None: test_ds = compatibility.GetName(fake_data_store.FakeDataStore) config.CONFIG.Set("Datastore.implementation", test_ds) if not INIT_RAN: server_logging.ServerLoggingStartupInit() server_logging.SetTestVerbosity() blob_store_test_lib.UseTestBlobStore() data_store.InitializeDataStore() if data_store.AFF4Enabled(): aff4.AFF4Init() # Requires data_store.InitializeDataStore. aff4_grr.GRRAFF4Init() # Requires aff4.AFF4Init. filestore.FileStoreInit() # Requires aff4_grr.GRRAFF4Init. results.ResultQueueInit() # Requires aff4.AFF4Init. sequential_collection.StartUpdaterOnce() artifact.LoadArtifactsOnce() # Requires aff4.AFF4Init. checks.LoadChecksFromFilesystemOnce() client_approval_auth.InitializeClientApprovalAuthorizationManagerOnce() cronjobs.InitializeCronWorkerOnce() # Requires aff4.AFF4Init. email_alerts.InitializeEmailAlerterOnce() http_api.InitializeHttpRequestHandlerOnce() ip_resolver.IPResolverInitOnce() stats_server.InitializeStatsServerOnce() webauth.InitializeWebAuthOnce() db = data_store.DB.SetupTestDB() if db: data_store.DB = db data_store.DB.Initialize() if not utils.TimeBasedCache.house_keeper_thread: utils.TimeBasedCache() utils.TimeBasedCache.house_keeper_thread.exit = True utils.TimeBasedCache.house_keeper_thread.join() INIT_RAN = True
def _CopyCommonRpmFiles(dist_dir): # Copy the wrapper script. shutil.copy( package.ResourcePath("grr-response-core", "install_data/wrapper.sh.in"), dist_dir)
def _MakeMsi(input_dir: str, output_path: str) -> None: """Packages the PyInstaller files as MSI.""" wxs_file = package.ResourcePath("grr-response-core", "install_data/windows/grr.wxs") fleetspeak_wxs_lib = os.path.join( config.CONFIG["ClientBuilder.fleetspeak_install_dir"], "fleetspeak_lib.wxs") # Don't automatically harvest these files using heat.exe, since they # are treated specially in grr.wxs. exclude_files = [ "grr-client.exe", "dbg_grr-client.exe", "GRRservice.exe", "dbg_GRRservice.exe", "fleetspeak-client.exe", "grr-client.exe.manifest", ] def Run(args: List[str]): logging.info("Running: %s.", args) subprocess.check_call(args) with utils.TempDirectory() as temp_dir: for exclude_file in exclude_files: shutil.move(os.path.join(input_dir, exclude_file), os.path.join(temp_dir, exclude_file)) Run([ os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin", "heat.exe"), "dir", input_dir, "-sfrag", "-srd", "-cg", "CompGrrAutoFiles", "-ag", "-var", "var.InputDir", "-dr", "INSTALLDIR", "-out", os.path.join(temp_dir, "heat.wxs"), ]) for exclude_file in exclude_files: shutil.move(os.path.join(temp_dir, exclude_file), os.path.join(input_dir, exclude_file)) # Generate padded placeholder files for these config files. # The reason for the padding is that at repacking time, we have to replace # the config files with files of the same size. So we make them large # enough to fit any expected config file. for placeholder_file in [ "grr-config.yaml", "fleetspeak-client.config", "fleetspeak-service-config.txt" ]: with open(os.path.join(input_dir, placeholder_file), "w", newline="\n") as f: f.write("__BEGIN__\n") f.write("\n" * (100 * 1024)) f.write("__END__") object_files = [] for source_file in (wxs_file, fleetspeak_wxs_lib, os.path.join(temp_dir, "heat.wxs")): object_file = os.path.join( temp_dir, os.path.basename(source_file) + ".wxobj") Run([ os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin", "candle.exe"), source_file, "-arch", "x64", "-ext", "WixUtilExtension", "-dFLEETSPEAK_EXECUTABLE=" + os.path.join(input_dir, "fleetspeak-client.exe"), "-dVERSION=" + config.CONFIG["Source.version_string"], "-sw1150", f"-dInputDir={input_dir}", "-out", object_file, ]) object_files.append(object_file) Run([ os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin", "light.exe"), ] + object_files + [ "-ext", "WixUtilExtension", "-sw1076", "-out", os.path.join(temp_dir, "installer.msi"), ]) with zipfile.ZipFile(output_path, "w") as zip_output: zip_output.write(os.path.join(temp_dir, "installer.msi"), "installer.msi") zip_output.write(os.path.join(input_dir, "build.yaml"), "build.yaml")
def _MakeMsi(input_dir: str, output_path: str) -> None: """Packages the PyInstaller files as MSI.""" wxs_file = package.ResourcePath("grr-response-core", "install_data/windows/grr.wxs") fleetspeak_wxs_lib = os.path.join( config.CONFIG["ClientBuilder.fleetspeak_install_dir"], "fleetspeak_lib.wxs") # Don't automatically harvest these files using heat.exe, since they # are treated specially in grr.wxs. exclude_files = [ "grr-client.exe", "dbg_grr-client.exe", "GRRservice.exe", "dbg_GRRservice.exe", "fleetspeak-client.exe", "grr-client.exe.manifest", ] def Run(args: List[str]): logging.info("Running: %s.", args) subprocess.check_call(args) with utils.TempDirectory() as temp_dir: for exclude_file in exclude_files: shutil.move(os.path.join(input_dir, exclude_file), os.path.join(temp_dir, exclude_file)) Run([ os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin", "heat.exe"), "dir", input_dir, "-sfrag", "-srd", "-cg", "CompGrrAutoFiles", "-ag", "-var", "var.InputDir", "-dr", "INSTALLDIR", "-out", os.path.join(temp_dir, "heat.wxs"), ]) for exclude_file in exclude_files: shutil.move(os.path.join(temp_dir, exclude_file), os.path.join(input_dir, exclude_file)) for placeholder_file in [ "grr-config.yaml", "fleetspeak-client.config", "fleetspeak-service-config.txt" ]: with open(os.path.join(input_dir, placeholder_file), "w", newline="\n"): pass # Due to a limitation in the olefile library, at repacking time, the CAB # in the MSI file needs to be repacked to a CAB file of same size. # To do so, we add 3 MB of random (uncompressable) data into a padding # file. # At repacking time, the padding file will get truncated to make space for # other files (config files, signed EXE and DLL files) to grow. with open(os.path.join(input_dir, "padding-file.bin"), "wb") as f: for _ in range(3): f.write(os.urandom(1024 * 1024)) object_files = [] for source_file in (wxs_file, fleetspeak_wxs_lib, os.path.join(temp_dir, "heat.wxs")): object_file = os.path.join( temp_dir, os.path.basename(source_file) + ".wxobj") Run([ os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin", "candle.exe"), source_file, "-arch", "x64", "-ext", "WixUtilExtension", "-dFLEETSPEAK_EXECUTABLE=" + os.path.join(input_dir, "fleetspeak-client.exe"), "-dVERSION=" + config.CONFIG["Source.version_string"], "-sw1150", f"-dInputDir={input_dir}", "-out", object_file, ]) object_files.append(object_file) Run([ os.path.join(config.CONFIG["ClientBuilder.wix_tools_path"], "bin", "light.exe"), ] + object_files + [ "-ext", "WixUtilExtension", "-sw1076", "-out", os.path.join(temp_dir, "installer.msi"), ]) with zipfile.ZipFile(output_path, "w") as zip_output: zip_output.write(os.path.join(temp_dir, "installer.msi"), "installer.msi") zip_output.write(os.path.join(input_dir, "build.yaml"), "build.yaml")