def _CopyFleetspeakRpmFiles(package_dir, context=None):
"""Copies Fleetspeak-enabled RPM files into the template folder."""
if context is None:
raise ValueError("context can't be None")
shutil.move(os.path.join(package_dir, "debian"),
os.path.join(package_dir, "rpmbuild"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/fleetspeak.grr.spec.in"),
os.path.join(package_dir, "rpmbuild/grr.spec.in"))
# Copy prelink blacklist file. Without this file, prelink will mangle
# the GRR binary.
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/prelink_blacklist.conf.in"),
os.path.join(package_dir, "rpmbuild/prelink_blacklist.conf.in"))
# Include the Fleetspeak service config in the template.
fleetspeak_dir = os.path.join(package_dir, "fleetspeak")
utils.EnsureDirExists(fleetspeak_dir)
shutil.copy(
config.CONFIG.Get("ClientBuilder.fleetspeak_config_path",
context=context), fleetspeak_dir)
def CopyFiles(self):
"""This sets up the template directory."""
shutil.move(os.path.join(self.package_dir, "debian"),
os.path.join(self.package_dir, "rpmbuild"))
if self.fleetspeak_enabled:
rpm_spec_filename = "fleetspeak.grr.spec.in"
else:
rpm_spec_filename = "grr.spec.in"
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/grr-client.initd.in"),
os.path.join(self.package_dir, "rpmbuild/grr-client.initd.in"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"),
os.path.join(self.package_dir,
"rpmbuild/grr-client.service.in"))
shutil.copy(
config_lib.Resource().Filter("install_data/centos/" +
rpm_spec_filename),
os.path.join(self.package_dir, "rpmbuild/grr.spec.in"))
# Copy prelink blacklist file. Without this file, prelink will mangle
# the GRR binary.
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/prelink_blacklist.conf.in"),
os.path.join(self.package_dir,
"rpmbuild/prelink_blacklist.conf.in"))
def InterpolateFiles(self):
if self.fleetspeak_enabled:
shutil.copy(flags.FLAGS.fleetspeak_service_config,
self.pkg_fleetspeak_service_dir)
build_files_dir = config_lib.Resource().Filter(
"install_data/macosx/client/fleetspeak")
else:
build_files_dir = config_lib.Resource().Filter(
"install_data/macosx/client")
self.GenerateFile(
input_filename=os.path.join(build_files_dir, "grr.plist.in"),
output_filename=os.path.join(self.pkg_root,
"Library/LaunchDaemons",
self.plist_name))
# We pass in scripts separately with --scripts so they don't go in pkg_root
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"preinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"preinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"postinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"postinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"Distribution.xml.in"),
output_filename=os.path.join(self.build_dir,
"Distribution.xml"))
def CopyFiles(self):
"""This sets up the template directory."""
# Copy the nanny binary.
shutil.copy(
package.ResourcePath("grr-response-core",
"install_data/debian/dpkg_client/nanny.sh.in"),
self.output_dir)
dpkg_dir = config.CONFIG.Get("PyInstaller.dpkg_root", context=self.context)
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter("install_data/debian/dpkg_client/debian"),
os.path.join(dpkg_dir, "debian/debian.in"))
# Copy upstart files
outdir = os.path.join(dpkg_dir, "debian/upstart.in")
utils.EnsureDirExists(outdir)
shutil.copy(config_lib.Resource().Filter(
"install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir)
# Copy init files
outdir = os.path.join(dpkg_dir, "debian/initd.in")
utils.EnsureDirExists(outdir)
shutil.copy(config_lib.Resource().Filter(
"install_data/debian/dpkg_client/initd/grr-client"), outdir)
# Copy systemd unit file
outdir = os.path.join(dpkg_dir, "debian/systemd.in")
utils.EnsureDirExists(outdir)
shutil.copy(config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"), outdir)
def _CopyNonFleetspeakDpkgFiles(dist_dir, package_dir):
"""Copies non-Fleetspeak-enabled DPKG files to template directory."""
# Copy the nanny binary.
shutil.copy(
package.ResourcePath("grr-response-core",
"install_data/debian/dpkg_client/nanny.sh.in"),
dist_dir)
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter("install_data/debian/dpkg_client/debian"),
os.path.join(package_dir, "debian/legacy-debian.in"))
# Copy upstart files
outdir = os.path.join(package_dir, "debian/upstart.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir)
# Copy init files
outdir = os.path.join(package_dir, "debian/initd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/initd/grr-client"), outdir)
# Copy systemd unit file
outdir = os.path.join(package_dir, "debian/systemd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"), outdir)
def CopyFiles(self):
"""This sets up the template directory."""
build_dir = config.CONFIG.Get("PyInstaller.dpkg_root",
context=self.context)
# Copy files needed for rpmbuild.
shutil.move(os.path.join(build_dir, "debian"),
os.path.join(build_dir, "rpmbuild"))
shutil.copy(
config_lib.Resource().Filter("install_data/centos/grr.spec.in"),
os.path.join(build_dir, "rpmbuild/grr.spec.in"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/grr-client.initd.in"),
os.path.join(build_dir, "rpmbuild/grr-client.initd.in"))
# Copy systemd unit file
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"),
os.path.join(build_dir, "rpmbuild/grr-client.service.in"))
# Copy prelink blacklist file
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/prelink_blacklist.conf.in"),
os.path.join(build_dir, "rpmbuild/prelink_blacklist.conf.in"))
def _CopyNonFleetspeakRpmFiles(package_dir):
"""Copies non-Fleetspeak-enabled RPM files into the template folder."""
shutil.move(os.path.join(package_dir, "debian"),
os.path.join(package_dir, "rpmbuild"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/grr-client.initd.in"),
os.path.join(package_dir, "rpmbuild/grr-client.initd.in"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"),
os.path.join(package_dir, "rpmbuild/grr-client.service.in"))
shutil.copy(
config_lib.Resource().Filter("install_data/centos/grr.spec.in"),
os.path.join(package_dir, "rpmbuild/grr.spec.in"))
# Copy prelink blacklist file. Without this file, prelink will mangle
# the GRR binary.
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/prelink_blacklist.conf.in"),
os.path.join(package_dir, "rpmbuild/prelink_blacklist.conf.in"))
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
if stats.STATS is None:
stats.STATS = stats.StatsCollector()
threadpool.InitializeMetrics()
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml@grr-response-core")
flags.FLAGS.secondary_configs.append(config_lib.Resource().Filter(
"grr_response_test/test_data/grr_test.yaml@grr-response-test"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = utils.GetName(fake_data_store.FakeDataStore)
config.CONFIG.Set("Datastore.implementation", test_ds)
config.CONFIG.Set("Blobstore.implementation",
utils.GetName(db_blob_store.DbBlobstore))
if not INIT_RAN:
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
registry.TestInit()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
aff4.AFF4InitHook().Run()
INIT_RAN = True
def _CopyNonFleetspeakRpmFiles(package_dir):
"""Copies non-Fleetspeak-enabled RPM files into the template folder."""
utils.EnsureDirExists(os.path.join(package_dir, "legacy/rpmbuild"))
shutil.copy(
config_lib.Resource().Filter("install_data/centos/grr-client.initd.in"),
os.path.join(package_dir, "legacy/rpmbuild/grr-client.initd.in"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"),
os.path.join(package_dir, "legacy/rpmbuild/grr-client.service.in"))
shutil.copy(config_lib.Resource().Filter("install_data/centos/grr.spec.in"),
os.path.join(package_dir, "legacy/rpmbuild/grr.spec.in"))
def VersionPath():
"""Returns a path to version.ini."""
# Try to get a version.ini. It should be in the resources if the code
# was packed with "pip sdist". It will be 2 levels up from grr_response_core
# if the code was installed via "pip install -e".
try:
version_ini = config_lib.Resource().Filter("version.ini")
except config_lib.FilterError:
version_ini = config_lib.Resource().Filter("../../version.ini")
if not os.path.exists(version_ini):
raise RuntimeError("Can't find version.ini at %s" % version_ini)
return version_ini
def SetConfigOptions():
"""Set location of configuration flags.
All GRR tools must use the same configuration files so they could all work
together. This needs to happen even before the configuration subsystem is
loaded so it must be bootstrapped by this code (all other options are
tweakable via the configuration system).
There are two main parts for the config system:
1) The main config file is shipped with the package and controls general
parameters. Note that this file is highly dependent on the exact version of
the grr package which is using it because it might have options which are
not understood by another version. We typically always use the config file
from package resources because in most cases this is the right thing to do
as this file matches exactly the running version. If you really have a good
reason you can override with the --config flag.
2) The writeback location. If any GRR component updates the configuration,
changes will be written back to a different locally modified config
file. This file specifies overrides of the main configuration file. The
main reason is that typically the same locally written config file may be
used with multiple versions of the GRR server because it specifies a very
small and rarely changing set of options.
"""
# The writeback config file is searched in the following order:
# 1. Specified on the command line with the "--config XXXX" option.
# 2. Use the default config file from the grr-response package.
flags.PARSER.set_defaults(config=config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml@grr-response-core"))
def _UploadBinary(self, bin_name, server_path):
"""Uploads a binary from the GRR installation dir to the datastore."""
# TODO(user): Upload binaries via the GRR API.
logging.info("Uploading %s binary to server.", server_path)
package_dir = config_lib.Resource().Filter(
"grr_response_test@grr-response-test")
with open(os.path.join(package_dir, "test_data", bin_name), "rb") as f:
maintenance_utils.UploadSignedConfigBlob(
f.read(), "aff4:/config/executables/%s" % server_path)
def CopyFiles(self):
"""This sets up the template directory."""
if self.fleetspeak_enabled:
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/fleetspeak-debian"),
os.path.join(self.package_dir, "debian/debian.in"))
return
# Copy the nanny binary.
shutil.copy(
package.ResourcePath(
"grr-response-core",
"install_data/debian/dpkg_client/nanny.sh.in"),
self.output_dir)
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/debian"),
os.path.join(self.package_dir, "debian/debian.in"))
# Copy upstart files
outdir = os.path.join(self.package_dir, "debian/upstart.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/upstart/grr-client.conf"),
outdir)
# Copy init files
outdir = os.path.join(self.package_dir, "debian/initd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/initd/grr-client"), outdir)
# Copy systemd unit file
outdir = os.path.join(self.package_dir, "debian/systemd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"), outdir)
def testConfigOptionsDefined(self):
"""Test that all config options in use are defined."""
# We need to use the actual config.CONFIG variable since that is where
# all the variables are already defined.
conf = config.CONFIG.MakeNewConfig()
# Check our actual config validates
configpath = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml")
conf.Initialize(filename=configpath)
def testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = config_lib.Resource().Filter("executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1)
# Validate the config appended to the OS X package.
zf = zipfile.ZipFile(glob.glob(
os.path.join(new_dir, "installers/*.pkg")).pop(),
mode="r")
fd = zf.open("config.yaml")
# We can't load the included build.yaml because the package hasn't been
# installed.
loaded = yaml.safe_load(fd)
loaded.pop("Config.includes")
packaged_config = config.CONFIG.MakeNewConfig()
packaged_config.Initialize(parser=config_lib.YamlParser,
data=yaml.safe_dump(loaded))
packaged_config.Validate(
sections=build.ClientRepacker.CONFIG_SECTIONS)
repacker = build.ClientRepacker()
repacker.ValidateEndConfig(packaged_config)
def _CopyCommonRpmFiles(package_dir, dist_dir):
"""Copies common (fleetspeak and legacy) files into the template folder."""
# Copy the wrapper script.
shutil.copy(
package.ResourcePath("grr-response-core",
"install_data/wrapper.sh.in"), dist_dir)
shutil.move(os.path.join(package_dir, "debian"),
os.path.join(package_dir, "rpmbuild"))
# Copy prelink blacklist file. Without this file, prelink will mangle
# the GRR binary.
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/prelink_blacklist.conf.in"),
os.path.join(package_dir, "rpmbuild/prelink_blacklist.conf.in"))
def _CopyFleetspeakDpkgFiles(package_dir, context=None):
"""Copies Fleetspeak-enabled DPKG files to template directory."""
if context is None:
raise ValueError("context can't be None")
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/fleetspeak-debian"),
os.path.join(package_dir, "debian/fleetspeak-debian.in"))
# Include the Fleetspeak service config in the template.
fleetspeak_dir = os.path.join(package_dir, "fleetspeak")
utils.EnsureDirExists(fleetspeak_dir)
shutil.copy(
config.CONFIG.Get("ClientBuilder.fleetspeak_config_path",
context=context), fleetspeak_dir)
def _CopyFleetspeakRpmFiles(package_dir, context=None):
"""Copies Fleetspeak-enabled RPM files into the template folder."""
if context is None:
raise ValueError("context can't be None")
utils.EnsureDirExists(os.path.join(package_dir, "fleetspeak/rpmbuild"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/fleetspeak.grr.spec.in"),
os.path.join(package_dir, "fleetspeak/rpmbuild/grr.spec.in"))
# Include the Fleetspeak service config in the template.
fleetspeak_dir = os.path.join(package_dir, "fleetspeak/fleetspeak")
utils.EnsureDirExists(fleetspeak_dir)
shutil.copy(
config.CONFIG.Get("ClientBuilder.fleetspeak_config_path",
context=context), fleetspeak_dir)
def SetConfigOptions():
"""Sets the default value for the config flag."""
flags.PARSER.set_defaults(config=config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml@grr-response-core"))
def SetConfigOptions():
"""Set location of configuration flags."""
flags.PARSER.set_defaults(
config=config_lib.Resource().Filter("install_data/etc/grr-server.yaml"))
