def main(_): """Launch the appropriate builder.""" if flags.FLAGS.subparser_name == "generate_client_config": # We don't need a full init to just build a config. GetClientConfig(flags.FLAGS.client_config_output) return # We deliberately use flags.FLAGS.context because client_startup.py pollutes # grr_config.CONFIG.context with the running system context. context = flags.FLAGS.context context.append("ClientBuilder Context") client_startup.ClientInit() # Use basic console output logging so we can see what is happening. logger = logging.getLogger() handler = logging.StreamHandler() handler.setLevel(logging.INFO) logger.handlers = [handler] if args.subparser_name == "build": TemplateBuilder().BuildTemplate(context=context, output=flags.FLAGS.output) elif args.subparser_name == "repack": if args.debug_build: context.append("DebugClientBuild Context") result_path = repacking.TemplateRepacker().RepackTemplate( args.template, args.output_dir, context=context, sign=args.sign, signed_template=args.signed_template) if not result_path: raise ErrorDuringRepacking(" ".join(sys.argv[:])) elif args.subparser_name == "repack_multiple": # Resolve globs manually on Windows. templates = args.templates if templates and "*" in templates[0]: templates = glob.glob(templates[0]) repack_configs = args.repack_configs if repack_configs and "*" in repack_configs[0]: repack_configs = glob.glob(repack_configs[0]) MultiTemplateRepacker().RepackTemplates( repack_configs, templates, args.output_dir, config=args.config, sign=args.sign, signed_template=args.signed_template) elif args.subparser_name == "sign_template": repacking.TemplateRepacker().SignTemplate(args.template, args.output_file, context=context) if not os.path.exists(args.output_file): raise RuntimeError("Signing failed: output not written")
def main(_): """Launch the appropriate builder.""" # We deliberately use flags.FLAGS.context because startup.py pollutes # config_lib.CONFIG.context with the running system context. context = flags.FLAGS.context context.append("ClientBuilder Context") startup.ClientInit() # Use basic console output logging so we can see what is happening. logger = logging.getLogger() handler = logging.StreamHandler() handler.setLevel(logging.INFO) logger.handlers = [handler] if args.subparser_name == "build": TemplateBuilder().BuildTemplate(context=context, output=flags.FLAGS.output) elif args.subparser_name == "repack": if args.debug_build: context.append("DebugClientBuild Context") result_path = repacking.TemplateRepacker().RepackTemplate( args.template, args.output_dir, context=context, sign=args.sign, signed_template=args.signed_template) if not result_path: raise ErrorDuringRepacking(" ".join(sys.argv[:])) elif args.subparser_name == "repack_multiple": MultiTemplateRepacker().RepackTemplates( args.repack_configs, args.templates, args.output_dir, config=args.config, sign=args.sign, signed_template=args.signed_template) elif args.subparser_name == "build_components": component.BuildComponents(output_dir=flags.FLAGS.output) elif args.subparser_name == "build_component": component.BuildComponent(flags.FLAGS.setup_file, output_dir=flags.FLAGS.output) elif args.subparser_name == "sign_template": repacking.TemplateRepacker().SignTemplate(args.template, args.output_file, context=context) if not os.path.exists(args.output_file): raise RuntimeError("Signing failed: output not written")
def ManageBinaries(config=None, token=None): """Repack templates into installers.""" print "\nStep 4: Repackaging clients with new configuration." redownload_templates = False repack_templates = False if flags.FLAGS.noprompt: redownload_templates = flags.FLAGS.redownload_templates repack_templates = not flags.FLAGS.norepack_templates else: redownload_templates = RetryBoolQuestion( "Server debs include client templates. Re-download templates?", False) repack_templates = RetryBoolQuestion("Repack client templates?", True) if redownload_templates: InstallTemplatePackage() # Build debug binaries, then build release binaries. if repack_templates: repacking.TemplateRepacker().RepackAllTemplates(upload=True, token=token) print "\nStep 5: Signing and uploading client components." maintenance_utils.SignAllComponents(token=token) print "\nInitialization complete, writing configuration." config.Write() print "Please restart the service for it to take effect.\n\n"
def ManageBinaries(config=None, token=None): """Repack templates into installers.""" print("\nStep 4: Installing template package and repackaging clients with" " new configuration.") pack_templates = False download_templates = False if flags.FLAGS.noprompt: if not flags.FLAGS.no_templates_download: download_templates = pack_templates = True else: if ((raw_input("Download client templates? You can skip " "this if templates are already installed." "[Yn]: ").upper() or "Y") == "Y"): download_templates = True if (raw_input("Repack client templates?" "[Yn]: ").upper() or "Y") == "Y": pack_templates = True if download_templates: InstallTemplatePackage() # Build debug binaries, then build release binaries. if pack_templates: repacking.TemplateRepacker().RepackAllTemplates(upload=True, token=token) print "\nStep 5: Signing and uploading client components." maintenance_utils.SignAllComponents(token=token) print "\nInitialization complete, writing configuration." config.Write() print "Please restart the service for it to take effect.\n\n"
def Repack(context, signer=None): """Reconfigure a client template to match config. Args: context: config_lib context signer: lib.builders.signing.CodeSigner object """ repacking.TemplateRepacker().RepackTemplate(args.template, args.outputdir, context=context, signer=signer)
def testRepackAll(self): """Test repacking all binaries.""" self.executables_dir = config_lib.Resource().Filter("executables") with utils.TempDirectory() as tmp_dir: new_dir = os.path.join(tmp_dir, "grr", "executables") os.makedirs(new_dir) # Copy unzipsfx so it can be used in repacking/ shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-i386.exe"), new_dir) shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-amd64.exe"), new_dir) with test_lib.ConfigOverrider({ "ClientBuilder.executables_dir": new_dir, "ClientBuilder.unzipsfx_stub_dir": new_dir }): repacking.TemplateRepacker().RepackAllTemplates() self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1) # Validate the config appended to the OS X package. zf = zipfile.ZipFile(glob.glob( os.path.join(new_dir, "installers/*.pkg")).pop(), mode="r") fd = zf.open("config.yaml") # We can't load the included build.yaml because the package hasn't been # installed. loaded = yaml.safe_load(fd) loaded.pop("Config.includes") packaged_config = config.CONFIG.MakeNewConfig() packaged_config.Initialize(parser=config_lib.YamlParser, data=yaml.safe_dump(loaded)) packaged_config.Validate( sections=build.ClientRepacker.CONFIG_SECTIONS) repacker = build.ClientRepacker() repacker.ValidateEndConfig(packaged_config)
def disabled_testRepackAll(self): """Test repacking all binaries.""" self.executables_dir = config_lib.Resource().Filter("executables") with utils.TempDirectory() as tmp_dir: new_dir = os.path.join(tmp_dir, "grr", "executables") os.makedirs(new_dir) # Copy unzipsfx so it can be used in repacking/ shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-i386.exe"), new_dir) shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-amd64.exe"), new_dir) with test_lib.ConfigOverrider({ "ClientBuilder.executables_dir": new_dir, "ClientBuilder.unzipsfx_stub_dir": new_dir }): repacking.TemplateRepacker().RepackAllTemplates() self.assertEqual( len(glob.glob(os.path.join(new_dir, "linux/installers/*.deb"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "linux/installers/*.rpm"))), 2) self.assertEqual( len( glob.glob(os.path.join(new_dir, "windows/installers/*.exe"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "darwin/installers/*.pkg"))), 1)
except AttributeError: raise RuntimeError("No valid config specified.") if flags.FLAGS.subparser_name == "generate_keys": try: GenerateKeys(config_lib.CONFIG, overwrite_keys=flags.FLAGS.overwrite_keys) except RuntimeError, e: # GenerateKeys will raise if keys exist and overwrite_keys is not set. print "ERROR: %s" % e sys.exit(1) config_lib.CONFIG.Write() elif flags.FLAGS.subparser_name == "repack_clients": upload = not flags.FLAGS.noupload repacking.TemplateRepacker().RepackAllTemplates(upload=upload, token=token) elif flags.FLAGS.subparser_name == "show_user": maintenance_utils.ShowUser(flags.FLAGS.username, token=token) elif flags.FLAGS.subparser_name == "update_user": try: maintenance_utils.UpdateUser(flags.FLAGS.username, flags.FLAGS.password, flags.FLAGS.add_labels, flags.FLAGS.delete_labels, token=token) except maintenance_utils.UserError as e: print e elif flags.FLAGS.subparser_name == "delete_user":
print "Using configuration %s" % config_lib.CONFIG except AttributeError: raise RuntimeError("No valid config specified.") if flags.FLAGS.subparser_name == "generate_keys": try: GenerateKeys(config_lib.CONFIG, overwrite_keys=flags.FLAGS.overwrite_keys) except RuntimeError, e: # GenerateKeys will raise if keys exist and overwrite_keys is not set. print "ERROR: %s" % e sys.exit(1) config_lib.CONFIG.Write() elif flags.FLAGS.subparser_name == "repack_clients": repacking.TemplateRepacker().RepackAllTemplates( upload=flags.FLAGS.upload, token=token) elif flags.FLAGS.subparser_name == "show_user": ShowUser(flags.FLAGS.username, token=token) elif flags.FLAGS.subparser_name == "update_user": try: UpdateUser(flags.FLAGS.username, flags.FLAGS.password, flags.FLAGS.add_labels, flags.FLAGS.delete_labels, token=token) except UserError as e: print e elif flags.FLAGS.subparser_name == "delete_user":
def BuildAndRepack(context, signer=None, timestamp=None): """Run build and repack to create installers.""" # ISO 8601 date timestamp = timestamp or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) # Output directory like: 2015-02-13T21:48:47-0800/linux_amd64_deb/ spec = "_".join((args.platform, args.arch, args.package_format)) output_dir = os.path.join( config_lib.CONFIG.Get("ClientBuilder.executables_dir", context=context), timestamp, spec) # If we weren't passed a template, build one if args.templatedir: template_path = TemplateInputFilename(context) else: component.BuildComponents(output_dir=output_dir) template_path = os.path.join( output_dir, config_lib.CONFIG.Get("PyInstaller.template_filename", context=context)) builder_obj = GetBuilder(context) builder_obj.MakeExecutableTemplate(output_file=template_path) # Get the list of contexts which we should be building. context_list = config_lib.CONFIG.Get("ClientBuilder.BuildTargets") logging.info("Building installers for: %s", context_list) repacked_list = [] for repackcontext in context_list: # Add the settings for this context for newcontext in repackcontext.split(","): context.append(newcontext) try: # If the ClientBuilder.target_platforms doesn't match our environment, # skip. if not config_lib.CONFIG.MatchBuildContext( args.platform, args.arch, args.package_format, context=context): continue # Make a nicer filename out of the context string. context_filename = repackcontext.replace("AllPlatforms Context,", "").replace(",", "_").replace( " ", "_") logging.info( "Repacking %s as %s with labels: %s", repackcontext, config_lib.CONFIG.Get("Client.name", context=context), config_lib.CONFIG.Get("Client.labels", context=context)) repacking.TemplateRepacker().RepackTemplate(template_path, os.path.join( output_dir, context_filename), signer=signer, context=context) repacked_list.append(context_filename) finally: # Remove the custom settings for the next repack for newcontext in repackcontext.split(","): context.remove(newcontext) logging.info("Complete, installers for %s are in %s", repacked_list, output_dir)
def main(_): """Launch the appropriate builder.""" grr_config.CONFIG.AddContext(contexts.CLIENT_BUILD_CONTEXT) args = flags.FLAGS if args.subparser_name == "generate_client_config": # We don't need a full init to just build a config. GetClientConfig(args.client_config_output) return # We deliberately use args.context because client_startup.py pollutes # grr_config.CONFIG.context with the running system context. context = args.context context.append(contexts.CLIENT_BUILD_CONTEXT) client_startup.ClientInit() # Use basic console output logging so we can see what is happening. logger = logging.getLogger() handler = logging.StreamHandler() handler.setLevel(logging.INFO) logger.handlers = [handler] if args.subparser_name == "build": if grr_config.CONFIG.Get("ClientBuilder.fleetspeak_enabled"): if "Target:Darwin" in context: if not args.fleetspeak_service_config: raise RuntimeError( "--fleetspeak_service_config must be provided.") if not grr_config.CONFIG.Get("ClientBuilder.install_dir"): raise RuntimeError( "ClientBuilder.install_dir must be set.") if not grr_config.CONFIG.Get( "ClientBuilder.fleetspeak_plist_path"): raise RuntimeError( "ClientBuilder.fleetspeak_plist_path must be set.") grr_config.CONFIG.Set("ClientBuilder.client_path", "grr_response_client.grr_fs_client") TemplateBuilder().BuildTemplate(context=context, output=args.output) elif args.subparser_name == "repack": if args.debug_build: context.append("DebugClientBuild Context") result_path = repacking.TemplateRepacker().RepackTemplate( args.template, args.output_dir, context=context, sign=args.sign, signed_template=args.signed_template) if not result_path: raise ErrorDuringRepacking(" ".join(sys.argv[:])) elif args.subparser_name == "repack_multiple": # Resolve globs manually on Windows. templates = [] for template in args.templates: if "*" in template: templates.extend(glob.glob(template)) else: # This could go through glob but then we'd swallow errors for # non existing files. templates.append(template) repack_configs = [] for repack_config in args.repack_configs: if "*" in repack_config: repack_configs.extend(glob.glob(repack_config)) else: # This could go through glob but then we'd swallow errors for # non existing files. repack_configs.append(repack_config) MultiTemplateRepacker().RepackTemplates( repack_configs, templates, args.output_dir, config=args.config, sign=args.sign, signed_template=args.signed_template) elif args.subparser_name == "sign_template": repacking.TemplateRepacker().SignTemplate(args.template, args.output_file, context=context) if not os.path.exists(args.output_file): raise RuntimeError("Signing failed: output not written")
def RepackTemplates(self, repack_configs, templates, output_dir, config=None, sign=False, signed_template=False): """Call repacker in a subprocess.""" pool = multiprocessing.Pool(processes=10) results = [] bulk_sign_installers = False for repack_config in repack_configs: for template in templates: repack_args = ["grr_client_build"] if config: repack_args.extend(["--config", config]) repack_args.extend([ "--secondary_configs", repack_config, "repack", "--template", template, "--output_dir", self.GetOutputDir(output_dir, repack_config) ]) # We only sign exes and rpms at the moment. The others will raise if we # try to ask for signing. passwd = None if sign: if template.endswith(".exe.zip"): # This is for osslsigncode only. if platform.system() != "Windows": passwd = self.GetWindowsPassphrase() repack_args.append("--sign") else: bulk_sign_installers = True if signed_template: repack_args.append("--signed_template") elif template.endswith(".rpm.zip"): bulk_sign_installers = True print "Calling %s" % " ".join(repack_args) results.append( pool.apply_async(SpawnProcess, (repack_args, ), dict(passwd=passwd))) # Also build debug if it's windows. if template.endswith(".exe.zip"): debug_args = [] debug_args.extend(repack_args) debug_args.append("--debug_build") print "Calling %s" % " ".join(debug_args) results.append( pool.apply_async(SpawnProcess, (debug_args, ), dict(passwd=passwd))) try: pool.close() # Workaround to handle keyboard kills # http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool # get will raise if the child raises. for result_obj in results: result_obj.get(9999) pool.join() except KeyboardInterrupt: print "parent received control-c" pool.terminate() except ErrorDuringRepacking: pool.terminate() raise if bulk_sign_installers: to_sign = {} for root, _, files in os.walk(output_dir): for f in files: if f.endswith(".exe"): to_sign.setdefault("windows", []).append(os.path.join(root, f)) elif f.endswith(".rpm"): to_sign.setdefault("rpm", []).append(os.path.join(root, f)) if to_sign.get("windows"): signer = repacking.TemplateRepacker().GetSigner([ "ClientBuilder Context", "Platform:%s" % platform.system(), "Target:Windows" ]) signer.SignFiles(to_sign.get("windows")) if to_sign.get("rpm"): signer = repacking.TemplateRepacker().GetSigner([ "ClientBuilder Context", "Platform:%s" % platform.system(), "Target:Linux", "Target:LinuxRpm" ]) signer.AddSignatureToRPMs(to_sign.get("rpm"))