def testGenClientConfig_ignoreBuilderContext(self): with test_lib.PreserveConfig(): # Define a secondary config with special values for the ClientBuilder # context. str_override = """ Test Context: Client.labels: [label0, label1] ClientBuilder Context: Client.labels: [build-label0, build-label1] """ parser = config_parser.YamlConfigFileParser("") override = parser.RawDataFromBytes(str_override.encode("utf-8")) config.CONFIG.MergeData(override) # Sanity-check that the secondary config was merged into the global # config. self.assertEqual(config.CONFIG["Client.labels"], ["label0", "label1"]) context = [ "Test Context", "ClientBuilder Context", "Client Context" ] str_client_config = build_helpers.GetClientConfig(context) client_config = parser.RawDataFromBytes( str_client_config.encode("utf-8")) # Settings particular to the ClientBuilder context should not carry over # into the generated client config. self.assertEqual(client_config["Client.labels"], ["label0", "label1"])
def main(argv): del argv # Unused. if not flags.FLAGS.dest_server_config_path: raise ValueError("dest_server_config_path flag has to be provided.") if not flags.FLAGS.dest_client_config_path: raise ValueError("dest_client_config_path flag has to be provided.") admin_ui_port = portpicker.pick_unused_port() frontend_port = portpicker.pick_unused_port() source_server_config_path = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") config_lib.LoadConfig(config.CONFIG, source_server_config_path) config.CONFIG.SetWriteBack(flags.FLAGS.dest_server_config_path) # TODO(user): remove when AFF4 is gone. config.CONFIG.Set("Database.enabled", True) config.CONFIG.Set("Blobstore.implementation", "DbBlobStore") config.CONFIG.Set("Database.implementation", "MysqlDB") config.CONFIG.Set("Mysql.database", flags.FLAGS.config_mysql_database) if flags.FLAGS.config_mysql_username is not None: config.CONFIG.Set("Mysql.username", flags.FLAGS.config_mysql_username) if flags.FLAGS.config_mysql_password is not None: config.CONFIG.Set("Mysql.password", flags.FLAGS.config_mysql_password) config.CONFIG.Set("AdminUI.port", admin_ui_port) config.CONFIG.Set("AdminUI.headless", True) config.CONFIG.Set("Frontend.bind_address", "127.0.0.1") config.CONFIG.Set("Frontend.bind_port", frontend_port) config.CONFIG.Set("Server.initialized", True) config.CONFIG.Set("Cron.active", False) config.CONFIG.Set("Client.poll_max", 1) config.CONFIG.Set("Client.server_urls", ["http://localhost:%d/" % frontend_port]) if flags.FLAGS.config_logging_path is not None: config.CONFIG.Set("Logging.path", flags.FLAGS.config_logging_path) config.CONFIG.Set("Logging.verbose", False) if flags.FLAGS.config_osquery_path: config.CONFIG.Set("Osquery.path", flags.FLAGS.config_osquery_path) config_updater_keys_util.GenerateKeys(config.CONFIG) config.CONFIG.Write() config_lib.SetPlatformArchContext() context = list(config.CONFIG.context) context.append("Client Context") config_data = build_helpers.GetClientConfig(context, validate=False, deploy_timestamp=False) with io.open(flags.FLAGS.dest_client_config_path, "w") as fd: fd.write(config_data)
def testGenClientConfig(self): with test_lib.ConfigOverrider({"Client.build_environment": "test_env"}): data = build_helpers.GetClientConfig(["Client Context"], validate=True) parser = config_parser.YamlConfigFileParser("") raw_data = parser.RawDataFromBytes(data.encode("utf-8")) self.assertIn("Client.deploy_time", raw_data)
def testGenClientConfig(self): with test_lib.ConfigOverrider({"Client.build_environment": "test_env"}): data = build_helpers.GetClientConfig(["Client Context"], validate=True) parser = config_lib.YamlParser(data=data) raw_data = parser.RawData() self.assertIn("Client.deploy_time", raw_data)
def _WriteClientConfig(self): # Generate a config file. with io.open( os.path.join( self.target_binary_dir, config.CONFIG.Get("ClientBuilder.config_filename", context=self.context)), "w") as fd: fd.write( build_helpers.GetClientConfig(["Client Context"] + self.context, validate=False))
def GetClientConfig(filename): """Write client config to filename.""" config_lib.SetPlatformArchContext() config_lib.ParseConfigCommandLine() context = list(grr_config.CONFIG.context) context.append("Client Context") # Disable timestamping so we can get a reproducible and cacheable config file. config_data = build_helpers.GetClientConfig( context, validate=True, deploy_timestamp=False) with open(filename, "w") as fd: fd.write(config_data) build_helpers.WriteBuildYaml(fd, build_timestamp=False, context=context)
def MakeDeployableBinary(self, template_path, output_path): """This will add the config to the client template.""" context = self.context + ["Client Context"] utils.EnsureDirExists(os.path.dirname(output_path)) client_config_data = build_helpers.GetClientConfig(context) shutil.copyfile(template_path, output_path) zip_file = zipfile.ZipFile(output_path, mode="a") zip_info = zipfile.ZipInfo(filename="config.yaml") zip_file.writestr(zip_info, client_config_data) zip_file.close() return output_path
def _MakeDeployableBinaryV2(self, template_path, output_path): context = self.context + ["Client Context"] utils.EnsureDirExists(os.path.dirname(output_path)) fleetspeak_enabled = config.CONFIG.Get("Client.fleetspeak_enabled", context=self.context) fleetspeak_bundled = config.CONFIG.Get( "ClientBuilder.fleetspeak_bundled", context=self.context) with contextlib.ExitStack() as stack: tmp_dir = stack.enter_context(utils.TempDirectory()) shutil.unpack_archive(template_path, tmp_dir, format="zip") if fleetspeak_bundled: variant = "fleetspeak-bundled" elif fleetspeak_enabled: variant = "fleetspeak-enabled" else: variant = "legacy" pkg_utils.JoinPkg(os.path.join(tmp_dir, variant), os.path.join(tmp_dir, "blocks"), output_path) zf = stack.enter_context(zipfile.ZipFile(output_path, mode="a")) with open(os.path.join(tmp_dir, "build.yaml"), "r") as build_yaml_file: zf.writestr("build.yaml", build_yaml_file.read()) client_config_data = build_helpers.GetClientConfig(context) zf.writestr("config.yaml", client_config_data) if fleetspeak_bundled: fleetspeak_client_config = config.CONFIG.Get( "ClientBuilder.fleetspeak_client_config", context=self.context) with open(fleetspeak_client_config, "r") as fleetspeak_client_config_file: zf.writestr("client.config", fleetspeak_client_config_file.read()) return output_path
def _MakeSelfExtractingZip(self, payload_data, output_path): """Repack the installer into the payload. Args: payload_data: data payload for zip file output_path: filename for the zip output Raises: RuntimeError: if the ClientBuilder.unzipsfx_stub doesn't require admin. Returns: output_path: filename string of zip output file """ context = self.context + ["Client Context"] src_zip = zipfile.ZipFile(io.BytesIO(payload_data), mode="r") zip_data = io.BytesIO() output_zip = zipfile.ZipFile(zip_data, mode="w", compression=zipfile.ZIP_DEFLATED) config_file_name = config.CONFIG.Get("ClientBuilder.config_filename", context=context) # Copy the rest of the files from the package to the new zip. for template_file in src_zip.namelist(): if template_file != config_file_name: # Avoid writing the config file twice if we're repacking a binary that # has already been run through deployment. We write it in the next step, # so no need to copy over from the original here. _CopyFileInZip(src_zip, template_file, output_zip) client_config_content = build_helpers.GetClientConfig(context) self._ValidateEndConfig(client_config_content) output_zip.writestr( config_file_name, client_config_content.encode("utf-8"), # pytype: disable=attribute-error compress_type=zipfile.ZIP_STORED) # The zip file comment is used by the self extractor to run the installation # script. Comment has to be `bytes` object because `zipfile` module is not # smart enough to properly handle `unicode` objects. output_zip.comment = b"$AUTORUN$>%s" % config.CONFIG.Get( "ClientBuilder.autorun_command_line", context=context).encode("utf-8") output_zip.close() utils.EnsureDirExists(os.path.dirname(output_path)) with io.open(output_path, "wb") as fd: # First write the installer stub stub_data = io.BytesIO() unzipsfx_stub = config.CONFIG.Get("ClientBuilder.unzipsfx_stub", context=context) stub_raw = io.open(unzipsfx_stub, "rb").read() # Check stub has been compiled with the requireAdministrator manifest. if b"level=\"requireAdministrator" not in stub_raw: raise RuntimeError( "Bad unzip binary in use. Not compiled with the" "requireAdministrator manifest option.") stub_data.write(stub_raw) # If in verbose mode, modify the unzip bins PE header to run in console # mode for easier debugging. build_helpers.SetPeSubsystem(stub_data, console=config.CONFIG.Get( "ClientBuilder.console", context=context)) # Now patch up the .rsrc section to contain the payload. end_of_file = zip_data.tell() + stub_data.tell() # This is the IMAGE_SECTION_HEADER.Name which is also the start of # IMAGE_SECTION_HEADER. offset_to_rsrc = stub_data.getvalue().find(b".rsrc") # IMAGE_SECTION_HEADER.PointerToRawData is a 32 bit int. stub_data.seek(offset_to_rsrc + 20) start_of_rsrc_section = struct.unpack("<I", stub_data.read(4))[0] # Adjust IMAGE_SECTION_HEADER.SizeOfRawData to span from the old start to # the end of file. stub_data.seek(offset_to_rsrc + 16) stub_data.write( struct.pack("<I", end_of_file - start_of_rsrc_section)) # Concatenate stub and zip file. out_data = io.BytesIO() out_data.write(stub_data.getvalue()) out_data.write(zip_data.getvalue()) # Then write the actual output file. fd.write(out_data.getvalue()) if self.signer: self.signer.SignFile(output_path) logging.info("Deployable binary generated at %s", output_path) return output_path
def MakeDeployableBinary(self, template_path: str, output_path: str) -> str: context = self.context + ["Client Context"] utils.EnsureDirExists(os.path.dirname(output_path)) def GetConfig(name: str) -> Any: return config.CONFIG.Get(name, context=self.context) fleetspeak_enabled = GetConfig("Client.fleetspeak_enabled") fleetspeak_bundled = GetConfig("ClientBuilder.fleetspeak_bundled") legacy = not (fleetspeak_enabled or fleetspeak_bundled) with contextlib.ExitStack() as stack: tmp_dir = stack.enter_context(utils.TempDirectory()) shutil.unpack_archive(template_path, tmp_dir, format="zip") msi_file = MsiFile(os.path.join(tmp_dir, "installer.msi")) def EnableFeature(name: str) -> None: msi_file.EnableFeature(name.encode("utf-8")) def ReplaceString(src: str, dst: str) -> None: msi_file.ReplaceString(src.encode("utf-8"), dst.encode("utf-8")) def RenameFile(src: str, dst: str) -> None: msi_file.RenameFile(src.encode("utf-8"), dst.encode("utf-8")) def ReplaceStringConfig(src: str, dst: str) -> None: ReplaceString(src, GetConfig(dst)) def RenameFileConfig(src: str, dst: str) -> None: RenameFile(src, GetConfig(dst)) # Set product information ReplaceStringConfig("__ProductName", "Client.name") ReplaceStringConfig("__ProductManufacturer", "Client.company_name") # Enable features if GetConfig("ClientBuilder.console"): EnableFeature("DbgGrrExe") else: EnableFeature("GrrExe") if legacy: if GetConfig("ClientBuilder.console"): EnableFeature("DbgNanny") else: EnableFeature("Nanny") if fleetspeak_bundled: EnableFeature("FleetspeakClient") if fleetspeak_enabled or fleetspeak_bundled: EnableFeature("FleetspeakServiceRegistryEntry") # Rename directories RenameFileConfig("__GrrDirectory", "Client.name") RenameFileConfig("__GrrVersion", "Source.version_string") # Rename files if GetConfig("ClientBuilder.console"): RenameFileConfig("__dbg_grr-client.exe", "Client.binary_name") RenameFileConfig("__dbg_GRRService.exe", "Nanny.service_binary_name") else: RenameFileConfig("__grr-client.exe", "Client.binary_name") RenameFileConfig("__GRRService.exe", "Nanny.service_binary_name") # Write Configs if fleetspeak_bundled: with open(GetConfig("ClientBuilder.fleetspeak_client_config"), "rb") as f: msi_file.SetFleetspeakConfig(f.read()) RenameFileConfig("grr-config.yaml", "ClientBuilder.config_filename") msi_file.SetGrrConfig( build_helpers.GetClientConfig(context).encode("utf-8")) # Write Fleetspeak service registry data if fleetspeak_enabled or fleetspeak_bundled: key_name = GetConfig( "Client.fleetspeak_unsigned_services_regkey") key_name = key_name.replace("HKEY_LOCAL_MACHINE\\", "") ReplaceString("__FleetspeakServiceRegistryKey", key_name) ReplaceStringConfig("__FleetspeakServiceRegistryName", "Client.name") ReplaceString( "__FleetspeakServiceRegistryValue", f"[INSTALLDIR]{GetConfig('Client.fleetspeak_unsigned_config_fname')}" ) if fleetspeak_bundled: ReplaceStringConfig("FleetspeakClientService", "Client.fleetspeak_service_name") # Write Fleetspeak service config # If we don't need to re-write the file after installation, just run # a dummy command. gen_fleespeak_service_file_cmd = "cmd.exe /c exit" if fleetspeak_enabled or fleetspeak_bundled: path = GetConfig("ClientBuilder.fleetspeak_config_path") with open(path, "rb") as f: msi_file.SetFleetspeakServiceConfig(f.read()) RenameFileConfig("fleetspeak-service-config.txt", "Client.fleetspeak_unsigned_config_fname") if path.endswith(".in"): args = [ "[INSTALLDIR]" + GetConfig("Client.binary_name"), "--config", "[INSTALLDIR]" + GetConfig("ClientBuilder.config_filename"), "-p", "Client.install_path=[INSTALLDIR]", "--install", "--interpolate_fleetspeak_service_config", "[INSTALLDIR]" + GetConfig("Client.fleetspeak_unsigned_config_fname"), ] gen_fleespeak_service_file_cmd = subprocess.list2cmdline( args) ReplaceString("__GenFleetspeakServiceFileCmd", gen_fleespeak_service_file_cmd) # Configure nanny service if legacy: nanny_args = ["--service_key", GetConfig("Client.config_key")] ReplaceString("__NannyArguments", subprocess.list2cmdline(nanny_args)) ReplaceStringConfig("__NannyServiceDescription", "Nanny.service_description") if GetConfig("ClientBuilder.console"): ReplaceStringConfig("__DbgNannyRegistryKey", "Client.config_key") ReplaceStringConfig("__DbgNannyServiceName", "Nanny.service_name") else: ReplaceStringConfig("__NannyRegistryKey", "Client.config_key") ReplaceStringConfig("__NannyServiceName", "Nanny.service_name") grr_binary = GetConfig("Client.binary_name") grr_config = GetConfig("ClientBuilder.config_filename") ReplaceString("__NannyChildBinary", f"[INSTALLDIR]{grr_binary}") child_args = [ f"[INSTALLDIR]{grr_binary}", "--config", f"[INSTALLDIR]{grr_config}" ] ReplaceString("__NannyChildCommandLine", subprocess.list2cmdline(child_args)) msi_file.Write() msi_file.Close() if os.path.exists(output_path): os.remove(output_path) shutil.move(os.path.join(tmp_dir, "installer.msi"), output_path) return output_path
def MakeDeployableBinary(self, template_path, output_path): """This will add the config to the client template and create a .rpm.""" rpmbuild_binary = "/usr/bin/rpmbuild" if not os.path.exists(rpmbuild_binary): logging.error("rpmbuild not found, unable to repack client.") return None with utils.TempDirectory() as tmp_dir: template_dir = os.path.join(tmp_dir, "dist") utils.EnsureDirExists(template_dir) zf = zipfile.ZipFile(template_path) for name in zf.namelist(): dirname = os.path.dirname(name) utils.EnsureDirExists(os.path.join(template_dir, dirname)) with io.open(os.path.join(template_dir, name), "wb") as fd: fd.write(zf.read(name)) self._ProcessUniversalTemplate(template_dir) # Set up a RPM building environment. rpm_root_dir = os.path.join(tmp_dir, "rpmbuild") rpm_build_dir = os.path.join(rpm_root_dir, "BUILD") utils.EnsureDirExists(rpm_build_dir) rpm_buildroot_dir = os.path.join(rpm_root_dir, "BUILDROOT") utils.EnsureDirExists(rpm_buildroot_dir) rpm_rpms_dir = os.path.join(rpm_root_dir, "RPMS") utils.EnsureDirExists(rpm_rpms_dir) rpm_specs_dir = os.path.join(rpm_root_dir, "SPECS") utils.EnsureDirExists(rpm_specs_dir) template_binary_dir = os.path.join(tmp_dir, "dist/rpmbuild/grr-client") target_binary_dir = "%s%s" % (rpm_build_dir, config.CONFIG.Get( "ClientBuilder.target_dir", context=self.context)) utils.EnsureDirExists(os.path.dirname(target_binary_dir)) try: shutil.rmtree(target_binary_dir) except OSError: pass # TODO(user):pytype: incorrect move() definition in typeshed. # pytype: disable=wrong-arg-types shutil.move(template_binary_dir, target_binary_dir) # pytype: enable=wrong-arg-types client_name = config.CONFIG.Get("Client.name", context=self.context) client_binary_name = config.CONFIG.Get("Client.binary_name", context=self.context) if client_binary_name != "grr-client": # TODO(user):pytype: incorrect move() definition in typeshed. # pytype: disable=wrong-arg-types shutil.move( os.path.join(target_binary_dir, "grr-client"), os.path.join(target_binary_dir, client_binary_name)) # pytype: enable=wrong-arg-types if config.CONFIG.Get("Client.fleetspeak_enabled", context=self.context): self._GenerateFleetspeakConfig(template_dir, rpm_build_dir) if not config.CONFIG.Get("Client.fleetspeak_service_name", context=self.context): # The Fleetspeak service name is required when generating the RPM # spec file. raise build.BuildError( "Client.fleetspeak_service_name is not set.") if config.CONFIG.Get("ClientBuilder.fleetspeak_bundled", context=self.context): self._GenerateBundledFleetspeakFiles( os.path.join(template_dir, "bundled-fleetspeak"), rpm_build_dir) shutil.copy( config.CONFIG.Get( "ClientBuilder.fleetspeak_client_config", context=self.context), os.path.join(rpm_build_dir, "etc/fleetspeak-client/client.config")) else: self._GenerateInitConfigs(template_dir, rpm_build_dir) # Generate spec spec_filename = os.path.join(rpm_specs_dir, "%s.spec" % client_name) build_helpers.GenerateFile(os.path.join( tmp_dir, "dist/rpmbuild/grr.spec.in"), spec_filename, context=self.context) # Generate prelinking blacklist file prelink_target_filename = os.path.join(rpm_build_dir, "etc/prelink.conf.d", "%s.conf" % client_name) utils.EnsureDirExists(os.path.dirname(prelink_target_filename)) build_helpers.GenerateFile(os.path.join( tmp_dir, "dist/rpmbuild/prelink_blacklist.conf.in"), prelink_target_filename, context=self.context) # Create a client config. client_context = ["Client Context"] + self.context client_config_content = build_helpers.GetClientConfig( client_context) with io.open(os.path.join( target_binary_dir, config.CONFIG.Get("ClientBuilder.config_filename", context=self.context)), "w", encoding="utf-8") as fd: fd.write(client_config_content) # Set the daemon to executable. os.chmod(os.path.join(target_binary_dir, client_binary_name), 0o755) client_arch = config.CONFIG.Get("Template.arch", context=self.context) if client_arch == "amd64": client_arch = "x86_64" # Create wrapper script if os.path.exists(os.path.join(target_binary_dir, "wrapper.sh.in")): build_helpers.GenerateFile( os.path.join(target_binary_dir, "wrapper.sh.in"), os.path.join(target_binary_dir, "wrapper.sh"), context=self.context) os.chmod(os.path.join(target_binary_dir, "wrapper.sh"), 0o755) command = [ rpmbuild_binary, "--define", "_topdir " + rpm_root_dir, "--target", client_arch, "--buildroot", rpm_buildroot_dir, "-bb", spec_filename ] try: subprocess.check_output(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: logging.error("Error calling %s.", command) logging.error(e.output) raise client_version = config.CONFIG.Get("Template.version_string", context=self.context) rpm_filename = os.path.join( rpm_rpms_dir, client_arch, "%s-%s-1.%s.rpm" % (client_name, client_version, client_arch)) utils.EnsureDirExists(os.path.dirname(output_path)) shutil.move(rpm_filename, output_path) logging.info("Created package %s", output_path) self._Sign(output_path) return output_path
def MakeDeployableBinary(self, template_path, output_path): """This will add the config to the client template and create a .deb.""" buildpackage_binary = "/usr/bin/dpkg-buildpackage" if not os.path.exists(buildpackage_binary): logging.error( "dpkg-buildpackage not found, unable to repack client.") return None with utils.TempDirectory() as tmp_dir: template_dir = os.path.join(tmp_dir, "dist") utils.EnsureDirExists(template_dir) zf = zipfile.ZipFile(template_path) for name in zf.namelist(): dirname = os.path.dirname(name) utils.EnsureDirExists(os.path.join(template_dir, dirname)) with io.open(os.path.join(template_dir, name), "wb") as fd: fd.write(zf.read(name)) # Generate the dpkg files. self._GenerateDPKGFiles(tmp_dir) # Create a client config. client_context = ["Client Context"] + self.context client_config_content = build_helpers.GetClientConfig( client_context) # We need to strip leading /'s or .join will ignore everything that comes # before it. target_dir = config.CONFIG.Get("ClientBuilder.target_dir", context=self.context).lstrip("/") agent_dir = os.path.join( template_dir, "debian", config.CONFIG.Get("ClientBuilder.package_name", context=self.context), target_dir) with io.open(os.path.join( agent_dir, config.CONFIG.Get("ClientBuilder.config_filename", context=self.context)), "w", encoding="utf-8") as fd: fd.write(client_config_content) # Set the daemon to executable. os.chmod( os.path.join( agent_dir, config.CONFIG.Get("Client.binary_name", context=self.context)), 0o755) arch = config.CONFIG.Get("Template.arch", context=self.context) try: old_working_dir = os.getcwd() except OSError: old_working_dir = os.environ.get("HOME", "/tmp") try: os.chdir(template_dir) command = [ buildpackage_binary, "-uc", "-d", "-b", "-a%s" % arch ] try: subprocess.check_output(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: if b"Failed to sign" not in e.output: logging.error("Error calling %s.", command) logging.error(e.output) raise filename_base = config.CONFIG.Get( "ClientBuilder.debian_package_base", context=self.context) output_base = config.CONFIG.Get( "ClientRepacker.output_basename", context=self.context) finally: try: os.chdir(old_working_dir) except OSError: pass utils.EnsureDirExists(os.path.dirname(output_path)) for extension in [ ".changes", config.CONFIG.Get("ClientBuilder.output_extension", context=self.context) ]: input_name = "%s%s" % (filename_base, extension) output_name = "%s%s" % (output_base, extension) # TODO(user):pytype: incorrect move() definition in typeshed. # pytype: disable=wrong-arg-types shutil.move( os.path.join(tmp_dir, input_name), os.path.join(os.path.dirname(output_path), output_name)) # pytype: enable=wrong-arg-types logging.info("Created package %s", output_path) return output_path