def main(_):
"""Launch the appropriate builder."""
if flags.FLAGS.subparser_name == "generate_client_config":
# We don't need a full init to just build a config.
GetClientConfig(flags.FLAGS.client_config_output)
return
# We deliberately use flags.FLAGS.context because client_startup.py pollutes
# grr_config.CONFIG.context with the running system context.
context = flags.FLAGS.context
context.append("ClientBuilder Context")
client_startup.ClientInit()
# Use basic console output logging so we can see what is happening.
logger = logging.getLogger()
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
logger.handlers = [handler]
if args.subparser_name == "build":
TemplateBuilder().BuildTemplate(context=context,
output=flags.FLAGS.output)
elif args.subparser_name == "repack":
if args.debug_build:
context.append("DebugClientBuild Context")
result_path = repacking.TemplateRepacker().RepackTemplate(
args.template,
args.output_dir,
context=context,
sign=args.sign,
signed_template=args.signed_template)
if not result_path:
raise ErrorDuringRepacking(" ".join(sys.argv[:]))
elif args.subparser_name == "repack_multiple":
# Resolve globs manually on Windows.
templates = args.templates
if templates and "*" in templates[0]:
templates = glob.glob(templates[0])
repack_configs = args.repack_configs
if repack_configs and "*" in repack_configs[0]:
repack_configs = glob.glob(repack_configs[0])
MultiTemplateRepacker().RepackTemplates(
repack_configs,
templates,
args.output_dir,
config=args.config,
sign=args.sign,
signed_template=args.signed_template)
elif args.subparser_name == "sign_template":
repacking.TemplateRepacker().SignTemplate(args.template,
args.output_file,
context=context)
if not os.path.exists(args.output_file):
raise RuntimeError("Signing failed: output not written")
def main(_):
"""Launch the appropriate builder."""
# We deliberately use flags.FLAGS.context because startup.py pollutes
# config_lib.CONFIG.context with the running system context.
context = flags.FLAGS.context
context.append("ClientBuilder Context")
startup.ClientInit()
# Use basic console output logging so we can see what is happening.
logger = logging.getLogger()
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
logger.handlers = [handler]
if args.subparser_name == "build":
TemplateBuilder().BuildTemplate(context=context,
output=flags.FLAGS.output)
elif args.subparser_name == "repack":
if args.debug_build:
context.append("DebugClientBuild Context")
result_path = repacking.TemplateRepacker().RepackTemplate(
args.template,
args.output_dir,
context=context,
sign=args.sign,
signed_template=args.signed_template)
if not result_path:
raise ErrorDuringRepacking(" ".join(sys.argv[:]))
elif args.subparser_name == "repack_multiple":
MultiTemplateRepacker().RepackTemplates(
args.repack_configs,
args.templates,
args.output_dir,
config=args.config,
sign=args.sign,
signed_template=args.signed_template)
elif args.subparser_name == "build_components":
component.BuildComponents(output_dir=flags.FLAGS.output)
elif args.subparser_name == "build_component":
component.BuildComponent(flags.FLAGS.setup_file,
output_dir=flags.FLAGS.output)
elif args.subparser_name == "sign_template":
repacking.TemplateRepacker().SignTemplate(args.template,
args.output_file,
context=context)
if not os.path.exists(args.output_file):
raise RuntimeError("Signing failed: output not written")
def ManageBinaries(config=None, token=None):
"""Repack templates into installers."""
print "\nStep 4: Repackaging clients with new configuration."
redownload_templates = False
repack_templates = False
if flags.FLAGS.noprompt:
redownload_templates = flags.FLAGS.redownload_templates
repack_templates = not flags.FLAGS.norepack_templates
else:
redownload_templates = RetryBoolQuestion(
"Server debs include client templates. Re-download templates?",
False)
repack_templates = RetryBoolQuestion("Repack client templates?", True)
if redownload_templates:
InstallTemplatePackage()
# Build debug binaries, then build release binaries.
if repack_templates:
repacking.TemplateRepacker().RepackAllTemplates(upload=True,
token=token)
print "\nStep 5: Signing and uploading client components."
maintenance_utils.SignAllComponents(token=token)
print "\nInitialization complete, writing configuration."
config.Write()
print "Please restart the service for it to take effect.\n\n"
def ManageBinaries(config=None, token=None):
"""Repack templates into installers."""
print("\nStep 4: Installing template package and repackaging clients with"
" new configuration.")
pack_templates = False
download_templates = False
if flags.FLAGS.noprompt:
if not flags.FLAGS.no_templates_download:
download_templates = pack_templates = True
else:
if ((raw_input("Download client templates? You can skip "
"this if templates are already installed."
"[Yn]: ").upper() or "Y") == "Y"):
download_templates = True
if (raw_input("Repack client templates?" "[Yn]: ").upper() or "Y") == "Y":
pack_templates = True
if download_templates:
InstallTemplatePackage()
# Build debug binaries, then build release binaries.
if pack_templates:
repacking.TemplateRepacker().RepackAllTemplates(upload=True, token=token)
print "\nStep 5: Signing and uploading client components."
maintenance_utils.SignAllComponents(token=token)
print "\nInitialization complete, writing configuration."
config.Write()
print "Please restart the service for it to take effect.\n\n"
def Repack(context, signer=None):
"""Reconfigure a client template to match config.
Args:
context: config_lib context
signer: lib.builders.signing.CodeSigner object
"""
repacking.TemplateRepacker().RepackTemplate(args.template,
args.outputdir,
context=context,
signer=signer)
def testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = config_lib.Resource().Filter("executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1)
# Validate the config appended to the OS X package.
zf = zipfile.ZipFile(glob.glob(
os.path.join(new_dir, "installers/*.pkg")).pop(),
mode="r")
fd = zf.open("config.yaml")
# We can't load the included build.yaml because the package hasn't been
# installed.
loaded = yaml.safe_load(fd)
loaded.pop("Config.includes")
packaged_config = config.CONFIG.MakeNewConfig()
packaged_config.Initialize(parser=config_lib.YamlParser,
data=yaml.safe_dump(loaded))
packaged_config.Validate(
sections=build.ClientRepacker.CONFIG_SECTIONS)
repacker = build.ClientRepacker()
repacker.ValidateEndConfig(packaged_config)
def disabled_testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = config_lib.Resource().Filter("executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir,
"linux/installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir,
"linux/installers/*.rpm"))), 2)
self.assertEqual(
len(
glob.glob(os.path.join(new_dir,
"windows/installers/*.exe"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir,
"darwin/installers/*.pkg"))), 1)
except AttributeError:
raise RuntimeError("No valid config specified.")
if flags.FLAGS.subparser_name == "generate_keys":
try:
GenerateKeys(config_lib.CONFIG,
overwrite_keys=flags.FLAGS.overwrite_keys)
except RuntimeError, e:
# GenerateKeys will raise if keys exist and overwrite_keys is not set.
print "ERROR: %s" % e
sys.exit(1)
config_lib.CONFIG.Write()
elif flags.FLAGS.subparser_name == "repack_clients":
upload = not flags.FLAGS.noupload
repacking.TemplateRepacker().RepackAllTemplates(upload=upload,
token=token)
elif flags.FLAGS.subparser_name == "show_user":
maintenance_utils.ShowUser(flags.FLAGS.username, token=token)
elif flags.FLAGS.subparser_name == "update_user":
try:
maintenance_utils.UpdateUser(flags.FLAGS.username,
flags.FLAGS.password,
flags.FLAGS.add_labels,
flags.FLAGS.delete_labels,
token=token)
except maintenance_utils.UserError as e:
print e
elif flags.FLAGS.subparser_name == "delete_user":
print "Using configuration %s" % config_lib.CONFIG
except AttributeError:
raise RuntimeError("No valid config specified.")
if flags.FLAGS.subparser_name == "generate_keys":
try:
GenerateKeys(config_lib.CONFIG,
overwrite_keys=flags.FLAGS.overwrite_keys)
except RuntimeError, e:
# GenerateKeys will raise if keys exist and overwrite_keys is not set.
print "ERROR: %s" % e
sys.exit(1)
config_lib.CONFIG.Write()
elif flags.FLAGS.subparser_name == "repack_clients":
repacking.TemplateRepacker().RepackAllTemplates(
upload=flags.FLAGS.upload, token=token)
elif flags.FLAGS.subparser_name == "show_user":
ShowUser(flags.FLAGS.username, token=token)
elif flags.FLAGS.subparser_name == "update_user":
try:
UpdateUser(flags.FLAGS.username,
flags.FLAGS.password,
flags.FLAGS.add_labels,
flags.FLAGS.delete_labels,
token=token)
except UserError as e:
print e
elif flags.FLAGS.subparser_name == "delete_user":
def BuildAndRepack(context, signer=None, timestamp=None):
"""Run build and repack to create installers."""
# ISO 8601 date
timestamp = timestamp or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
# Output directory like: 2015-02-13T21:48:47-0800/linux_amd64_deb/
spec = "_".join((args.platform, args.arch, args.package_format))
output_dir = os.path.join(
config_lib.CONFIG.Get("ClientBuilder.executables_dir",
context=context), timestamp, spec)
# If we weren't passed a template, build one
if args.templatedir:
template_path = TemplateInputFilename(context)
else:
component.BuildComponents(output_dir=output_dir)
template_path = os.path.join(
output_dir,
config_lib.CONFIG.Get("PyInstaller.template_filename",
context=context))
builder_obj = GetBuilder(context)
builder_obj.MakeExecutableTemplate(output_file=template_path)
# Get the list of contexts which we should be building.
context_list = config_lib.CONFIG.Get("ClientBuilder.BuildTargets")
logging.info("Building installers for: %s", context_list)
repacked_list = []
for repackcontext in context_list:
# Add the settings for this context
for newcontext in repackcontext.split(","):
context.append(newcontext)
try:
# If the ClientBuilder.target_platforms doesn't match our environment,
# skip.
if not config_lib.CONFIG.MatchBuildContext(
args.platform, args.arch, args.package_format,
context=context):
continue
# Make a nicer filename out of the context string.
context_filename = repackcontext.replace("AllPlatforms Context,",
"").replace(",",
"_").replace(
" ", "_")
logging.info(
"Repacking %s as %s with labels: %s", repackcontext,
config_lib.CONFIG.Get("Client.name", context=context),
config_lib.CONFIG.Get("Client.labels", context=context))
repacking.TemplateRepacker().RepackTemplate(template_path,
os.path.join(
output_dir,
context_filename),
signer=signer,
context=context)
repacked_list.append(context_filename)
finally:
# Remove the custom settings for the next repack
for newcontext in repackcontext.split(","):
context.remove(newcontext)
logging.info("Complete, installers for %s are in %s", repacked_list,
output_dir)
def main(_):
"""Launch the appropriate builder."""
grr_config.CONFIG.AddContext(contexts.CLIENT_BUILD_CONTEXT)
args = flags.FLAGS
if args.subparser_name == "generate_client_config":
# We don't need a full init to just build a config.
GetClientConfig(args.client_config_output)
return
# We deliberately use args.context because client_startup.py pollutes
# grr_config.CONFIG.context with the running system context.
context = args.context
context.append(contexts.CLIENT_BUILD_CONTEXT)
client_startup.ClientInit()
# Use basic console output logging so we can see what is happening.
logger = logging.getLogger()
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
logger.handlers = [handler]
if args.subparser_name == "build":
if grr_config.CONFIG.Get("ClientBuilder.fleetspeak_enabled"):
if "Target:Darwin" in context:
if not args.fleetspeak_service_config:
raise RuntimeError(
"--fleetspeak_service_config must be provided.")
if not grr_config.CONFIG.Get("ClientBuilder.install_dir"):
raise RuntimeError(
"ClientBuilder.install_dir must be set.")
if not grr_config.CONFIG.Get(
"ClientBuilder.fleetspeak_plist_path"):
raise RuntimeError(
"ClientBuilder.fleetspeak_plist_path must be set.")
grr_config.CONFIG.Set("ClientBuilder.client_path",
"grr_response_client.grr_fs_client")
TemplateBuilder().BuildTemplate(context=context, output=args.output)
elif args.subparser_name == "repack":
if args.debug_build:
context.append("DebugClientBuild Context")
result_path = repacking.TemplateRepacker().RepackTemplate(
args.template,
args.output_dir,
context=context,
sign=args.sign,
signed_template=args.signed_template)
if not result_path:
raise ErrorDuringRepacking(" ".join(sys.argv[:]))
elif args.subparser_name == "repack_multiple":
# Resolve globs manually on Windows.
templates = []
for template in args.templates:
if "*" in template:
templates.extend(glob.glob(template))
else:
# This could go through glob but then we'd swallow errors for
# non existing files.
templates.append(template)
repack_configs = []
for repack_config in args.repack_configs:
if "*" in repack_config:
repack_configs.extend(glob.glob(repack_config))
else:
# This could go through glob but then we'd swallow errors for
# non existing files.
repack_configs.append(repack_config)
MultiTemplateRepacker().RepackTemplates(
repack_configs,
templates,
args.output_dir,
config=args.config,
sign=args.sign,
signed_template=args.signed_template)
elif args.subparser_name == "sign_template":
repacking.TemplateRepacker().SignTemplate(args.template,
args.output_file,
context=context)
if not os.path.exists(args.output_file):
raise RuntimeError("Signing failed: output not written")
def RepackTemplates(self,
repack_configs,
templates,
output_dir,
config=None,
sign=False,
signed_template=False):
"""Call repacker in a subprocess."""
pool = multiprocessing.Pool(processes=10)
results = []
bulk_sign_installers = False
for repack_config in repack_configs:
for template in templates:
repack_args = ["grr_client_build"]
if config:
repack_args.extend(["--config", config])
repack_args.extend([
"--secondary_configs", repack_config, "repack",
"--template", template, "--output_dir",
self.GetOutputDir(output_dir, repack_config)
])
# We only sign exes and rpms at the moment. The others will raise if we
# try to ask for signing.
passwd = None
if sign:
if template.endswith(".exe.zip"):
# This is for osslsigncode only.
if platform.system() != "Windows":
passwd = self.GetWindowsPassphrase()
repack_args.append("--sign")
else:
bulk_sign_installers = True
if signed_template:
repack_args.append("--signed_template")
elif template.endswith(".rpm.zip"):
bulk_sign_installers = True
print "Calling %s" % " ".join(repack_args)
results.append(
pool.apply_async(SpawnProcess, (repack_args, ),
dict(passwd=passwd)))
# Also build debug if it's windows.
if template.endswith(".exe.zip"):
debug_args = []
debug_args.extend(repack_args)
debug_args.append("--debug_build")
print "Calling %s" % " ".join(debug_args)
results.append(
pool.apply_async(SpawnProcess, (debug_args, ),
dict(passwd=passwd)))
try:
pool.close()
# Workaround to handle keyboard kills
# http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool
# get will raise if the child raises.
for result_obj in results:
result_obj.get(9999)
pool.join()
except KeyboardInterrupt:
print "parent received control-c"
pool.terminate()
except ErrorDuringRepacking:
pool.terminate()
raise
if bulk_sign_installers:
to_sign = {}
for root, _, files in os.walk(output_dir):
for f in files:
if f.endswith(".exe"):
to_sign.setdefault("windows",
[]).append(os.path.join(root, f))
elif f.endswith(".rpm"):
to_sign.setdefault("rpm",
[]).append(os.path.join(root, f))
if to_sign.get("windows"):
signer = repacking.TemplateRepacker().GetSigner([
"ClientBuilder Context",
"Platform:%s" % platform.system(), "Target:Windows"
])
signer.SignFiles(to_sign.get("windows"))
if to_sign.get("rpm"):
signer = repacking.TemplateRepacker().GetSigner([
"ClientBuilder Context",
"Platform:%s" % platform.system(), "Target:Linux",
"Target:LinuxRpm"
])
signer.AddSignatureToRPMs(to_sign.get("rpm"))