def CopyFiles(self):
"""This sets up the template directory."""
build_dir = config.CONFIG.Get("PyInstaller.dpkg_root",
context=self.context)
# Copy files needed for rpmbuild.
shutil.move(os.path.join(build_dir, "debian"),
os.path.join(build_dir, "rpmbuild"))
shutil.copy(
config_lib.Resource().Filter("install_data/centos/grr.spec.in"),
os.path.join(build_dir, "rpmbuild/grr.spec.in"))
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/grr-client.initd.in"),
os.path.join(build_dir, "rpmbuild/grr-client.initd.in"))
# Copy systemd unit file
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"),
os.path.join(build_dir, "rpmbuild/grr-client.service.in"))
# Copy prelink blacklist file
shutil.copy(
config_lib.Resource().Filter(
"install_data/centos/prelink_blacklist.conf.in"),
os.path.join(build_dir, "rpmbuild/prelink_blacklist.conf.in"))
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
if stats.STATS is None:
stats.STATS = stats.StatsCollector()
flags.FLAGS.config = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs = [
config_lib.Resource().Filter(
"test_data/grr_test.yaml@grr-response-test")
]
# We are running a test so let the config system know that.
config_lib.CONFIG.AddContext("Test Context",
"Context applied when we run tests.")
AddConfigContext()
ConfigInit()
# Tests additionally add a test configuration file.
ServerLoggingStartupInit()
registry.TestInit()
def CopyFiles(self):
"""This sets up the template directory."""
dpkg_dir = config_lib.CONFIG.Get("PyInstaller.dpkg_root",
context=self.context)
# Copy files needed for dpkg-buildpackage.
shutil.copytree(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/debian"),
os.path.join(dpkg_dir, "debian/debian.in"))
# Copy upstart files
outdir = os.path.join(dpkg_dir, "debian/upstart.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/upstart/grr-client.conf"),
outdir)
# Copy init files
outdir = os.path.join(dpkg_dir, "debian/initd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/debian/dpkg_client/initd/grr-client"), outdir)
# Copy systemd files
outdir = os.path.join(dpkg_dir, "debian/systemd.in")
utils.EnsureDirExists(outdir)
shutil.copy(
config_lib.Resource().Filter(
"install_data/systemd/client/grr-client.service"), outdir)
def InterpolateFiles(self):
if config.CONFIG["ClientBuilder.fleetspeak_enabled"]:
build_files_dir = config_lib.Resource().Filter(
"install_data/macosx/client/fleetspeak")
else:
build_files_dir = config_lib.Resource().Filter(
"install_data/macosx/client")
self.GenerateFile(
input_filename=os.path.join(build_files_dir, "grr.plist.in"),
output_filename=os.path.join(self.pkg_root,
"Library/LaunchDaemons",
self.plist_name))
# We pass in scripts separately with --scripts so they don't go in pkg_root
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"preinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"preinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"postinstall.sh.in"),
output_filename=os.path.join(self.script_dir,
"postinstall"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"Distribution.xml.in"),
output_filename=os.path.join(self.build_dir,
"Distribution.xml"))
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
if stats.STATS is None:
stats.STATS = stats.StatsCollector()
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs.append(config_lib.Resource().Filter(
"test_data/grr_test.yaml@grr-response-test"))
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
# We are running a test so let the config system know that.
config.CONFIG.AddContext(contexts.TEST_CONTEXT,
"Context applied when we run tests.")
test_ds = flags.FLAGS.test_data_store
if test_ds is None:
test_ds = fake_data_store.FakeDataStore.__name__
if not INIT_RAN:
config.CONFIG.Set("Datastore.implementation", test_ds)
config.CONFIG.Set("Blobstore.implementation",
memory_stream_bs.MemoryStreamBlobstore.__name__)
server_logging.ServerLoggingStartupInit()
server_logging.SetTestVerbosity()
registry.TestInit()
db = data_store.DB.SetupTestDB()
if db:
data_store.DB = db
data_store.DB.Initialize()
aff4.AFF4InitHook().Run()
INIT_RAN = True
def SetConfigOptions():
"""Set location of configuration flags.
All GRR tools must use the same configuration files so they could all work
together. This needs to happen even before the configuration subsystem is
loaded so it must be bootstrapped by this code (all other options are
tweakable via the configuration system).
There are two main parts for the config system:
1) The main config file is shipped with the package and controls general
parameters. Note that this file is highly dependent on the exact version of
the grr package which is using it because it might have options which are
not understood by another version. We typically always use the config file
from package resources because in most cases this is the right thing to do
as this file matches exactly the running version. If you really have a good
reason you can override with the --config flag.
2) The writeback location. If any GRR component updates the configuration,
changes will be written back to a different locally modified config
file. This file specifies overrides of the main configuration file. The
main reason is that typically the same locally written config file may be
used with multiple versions of the GRR server because it specifies a very
small and rarely changing set of options.
"""
# The writeback config file is searched in the following order:
# 1. Specified on the command line with the "--config XXXX" option.
# 2. Use the default config file from the grr-response package.
flags.PARSER.set_defaults(config=config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml"))
def BuildWithPyInstaller(self):
"""Use pyinstaller to build a client package."""
self.CleanDirectory(
config.CONFIG.Get("PyInstaller.distpath", context=self.context))
logging.info("Copying pyinstaller support files")
self.spec_file = os.path.join(self.build_dir, "grr.spec")
with open(self.spec_file, "wb") as fd:
fd.write(
config.CONFIG.Get("PyInstaller.spec", context=self.context))
with open(os.path.join(self.build_dir, "version.txt"), "wb") as fd:
fd.write(
config.CONFIG.Get("PyInstaller.version", context=self.context))
with open(os.path.join(self.build_dir, "grr.ico"), "wb") as fd:
fd.write(
config.CONFIG.Get("PyInstaller.icon", context=self.context))
# We expect the onedir output at this location.
self.output_dir = os.path.join(
config.CONFIG.Get("PyInstaller.distpath", context=self.context),
"grr-client")
# Pyinstaller doesn't handle unicode strings.
args = [
"--distpath",
str(config.CONFIG.Get("PyInstaller.distpath",
context=self.context)), "--workpath",
str(
config.CONFIG.Get("PyInstaller.workpath_dir",
context=self.context)),
str(self.spec_file)
]
logging.info("Running pyinstaller: %s", args)
PyInstallerMain.run(pyi_args=[utils.SmartStr(x) for x in args])
# Clear out some crud that pyinstaller includes.
for path in ["tcl", "tk", "pytz"]:
dir_path = os.path.join(self.output_dir, path)
try:
shutil.rmtree(dir_path)
os.mkdir(dir_path)
# Create an empty file so the directories get put in the installers.
with open(os.path.join(dir_path, path), "wb"):
pass
except OSError:
pass
version_ini = config_lib.Resource().Filter("version.ini")
if not os.path.exists(version_ini):
raise RuntimeError(
"Couldn't find version_ini in virtual env root: %s" %
version_ini)
shutil.copy(version_ini, os.path.join(self.output_dir, "version.ini"))
with open(os.path.join(self.output_dir, "build.yaml"), "wb") as fd:
self.WriteBuildYaml(fd)
def main(argv):
"""Sets up all the component in their own threads."""
config_lib.CONFIG.AddContext(
"Demo Context",
"The demo runs all functions in a single process using the "
"in memory data store.")
config_lib.CONFIG.AddContext("Test Context",
"Context applied when we run tests.")
flags.FLAGS.config = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs = [
config_lib.Resource().Filter(
"test_data/grr_test.yaml@grr-response-test")
]
startup.Init()
# pylint: disable=unused-import,unused-variable,g-import-not-at-top
from grr.gui import gui_plugins
# pylint: enable=unused-import,unused-variable,g-import-not-at-top
# This is the worker thread.
worker_thread = threading.Thread(target=worker.main,
args=[argv],
name="Worker")
worker_thread.daemon = True
worker_thread.start()
# This is the http server Frontend that clients communicate with.
http_thread = threading.Thread(target=http_server.main,
args=[argv],
name="HTTP Server")
http_thread.daemon = True
http_thread.start()
client_thread = threading.Thread(target=client.main,
args=[argv],
name="Client")
client_thread.daemon = True
client_thread.start()
# The UI is running in the main thread.
runtests.main(argv)
def UploadBinaryIfAbsent(server_paths, bin_name, server_path):
if server_path in server_paths:
return
logging.info("Binary %s not uploaded yet. Will upload.", server_path)
package_dir = config_lib.Resource().Filter(
"grr_response_test@grr-response-test")
with open(os.path.join(package_dir, "test_data", bin_name), "rb") as f:
maintenance_utils.UploadSignedConfigBlob(
f.read(), "aff4:/config/executables/%s" % server_path)
def _UploadBinary(self, bin_name, server_path):
"""Uploads a binary from the GRR installation dir to the datastore."""
# TODO(user): Upload binaries via the GRR API.
logging.info("Uploading %s binary to server.", server_path)
package_dir = config_lib.Resource().Filter(
"grr_response_test@grr-response-test")
with open(os.path.join(package_dir, "test_data", bin_name), "rb") as f:
maintenance_utils.UploadSignedConfigBlob(
f.read(), "aff4:/config/executables/%s" % server_path)
def testConfigOptionsDefined(self):
"""Test that all config options in use are defined."""
# We need to use the actual config.CONFIG variable since that is where
# all the variables are already defined.
conf = config.CONFIG.MakeNewConfig()
# Check our actual config validates
configpath = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml")
conf.Initialize(filename=configpath)
def testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = config_lib.Resource().Filter("executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1)
# Validate the config appended to the OS X package.
zf = zipfile.ZipFile(glob.glob(
os.path.join(new_dir, "installers/*.pkg")).pop(),
mode="r")
fd = zf.open("config.yaml")
# We can't load the included build.yaml because the package hasn't been
# installed.
loaded = yaml.safe_load(fd)
loaded.pop("Config.includes")
packaged_config = config.CONFIG.MakeNewConfig()
packaged_config.Initialize(parser=config_lib.YamlParser,
data=yaml.safe_dump(loaded))
packaged_config.Validate(
sections=build.ClientRepacker.CONFIG_SECTIONS)
repacker = build.ClientRepacker()
repacker.ValidateEndConfig(packaged_config)
def SetConfigOptions():
"""Set location of configuration flags.
All GRR tools must use the same configuration files so they could all work
together. This needs to happen even before the configuration subsystem is
loaded so it must be bootstrapped by this code (all other options are
tweakable via the configuration system).
There are two main parts for the config system:
1) The main config file is shipped with the package and controls general
parameters. Note that this file is highly dependent on the exact version of
the grr package which is using it because it might have options which are
not understood by another version. We typically always use the config file
from package resources because in most cases this is the right thing to do
as this file matches exactly the running version. If you really have a good
reason you can override with the --config flag.
2) The writeback location. If any GRR component updates the configuration,
changes will be written back to a different locally modified config
file. This file specifies overrides of the main configuration file. The
main reason is that typically the same locally written config file may be
used with multiple versions of the GRR server because it specifies a very
small and rarely changing set of options.
"""
config_opts = {}
flag_defaults = {}
# Allow the installer to override the platform defaults for the location of
# the writeback config file. The writeback config file is therefore searched
# in the following order:
# 1. Specified on the command line with the "--config XXXX" option.
# 2. Specified in the environment variable GRR_CONFIG_FILE.
# 3. Specified during installation with:
# "python setup.py install --config-file=XXXX"
# 4. Use the default config file from the grr-response package.
if os.environ.get("GRR_CONFIG_FILE"):
flag_defaults["config"] = os.environ.get("GRR_CONFIG_FILE")
elif defaults.CONFIG_FILE:
flag_defaults["config"] = defaults.CONFIG_FILE
else:
flag_defaults["config"] = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml")
for option, value in config_opts.items():
config_lib.CONFIG.Set(option, value)
flags.PARSER.set_defaults(**flag_defaults)
def version():
# Delay imports until we have the config system to find the version.ini file.
# pylint: disable=g-import-not-at-top
import ConfigParser
from grr.lib import config_lib
config = ConfigParser.SafeConfigParser()
config.read(config_lib.Resource().Filter("version.ini"))
return dict(packageversion=config.get("Version", "packageversion"),
major=config.getint("Version", "major"),
minor=config.getint("Version", "minor"),
revision=config.getint("Version", "revision"),
release=config.getint("Version", "release"))
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
if stats.STATS is None:
stats.STATS = stats.StatsCollector()
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml")
flags.FLAGS.secondary_configs = [
config_lib.Resource().Filter("test_data/grr_test.yaml@grr-response-test")
]
# This config contains non-public settings that should be applied during
# tests.
extra_test_config = config_lib.CONFIG["Test.additional_test_config"]
if os.path.exists(extra_test_config):
flags.FLAGS.secondary_configs.append(extra_test_config)
# We are running a test so let the config system know that.
config_lib.CONFIG.AddContext("Test Context",
"Context applied when we run tests.")
# Tests additionally add a test configuration file.
config_lib.SetPlatformArchContext()
config_lib.ParseConfigCommandLine()
if not INIT_RAN:
log.ServerLoggingStartupInit()
registry.TestInit()
INIT_RAN = True
def version():
"""Return a dict with GRR version information."""
# Delay import until we have the config system to find the version.ini file.
# pylint: disable=g-import-not-at-top
from grr.lib import config_lib
version_ini = config_lib.Resource().Filter("version.ini")
if not os.path.exists(version_ini):
raise RuntimeError("Can't find version.ini at %s" % version_ini)
config = ConfigParser.SafeConfigParser()
config.read(version_ini)
return dict(packageversion=config.get("Version", "packageversion"),
major=config.getint("Version", "major"),
minor=config.getint("Version", "minor"),
revision=config.getint("Version", "revision"),
release=config.getint("Version", "release"))
def disabled_testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = config_lib.Resource().Filter("executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir,
"linux/installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir,
"linux/installers/*.rpm"))), 2)
self.assertEqual(
len(
glob.glob(os.path.join(new_dir,
"windows/installers/*.exe"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir,
"darwin/installers/*.pkg"))), 1)
def SetConfigOptions():
"""Set location of configuration flags."""
flags.PARSER.set_defaults(
config=config_lib.Resource().Filter("install_data/etc/grr-server.yaml"))
def SetConfigOptions():
"""Sets the default value for the config flag."""
flags.PARSER.set_defaults(config=config_lib.Resource().Filter(
"install_data/etc/grr-server.yaml"))
def setUp(self):
super(BuildTests, self).setUp()
self.executables_dir = config_lib.Resource().Filter("executables")
def BuildInstallerPkg(self, output_file):
"""Builds a package (.pkg) using PackageMaker."""
build_files_dir = config_lib.Resource().Filter(
"install_data/macosx/client")
pmdoc_dir = os.path.join(build_files_dir, "grr.pmdoc")
client_name = config_lib.CONFIG.Get("Client.name",
context=self.context)
plist_name = config_lib.CONFIG.Get("Client.plist_filename",
context=self.context)
out_build_files_dir = build_files_dir.replace(
config_lib.Resource().Filter("grr"), self.build_dir)
out_pmdoc_dir = os.path.join(self.build_dir, "%s.pmdoc" % client_name)
utils.EnsureDirExists(out_build_files_dir)
utils.EnsureDirExists(out_pmdoc_dir)
utils.EnsureDirExists(
config_lib.CONFIG.Get("ClientBuilder.package_dir",
context=self.context))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"grr.plist.in"),
output_filename=os.path.join(self.build_dir,
plist_name))
self.GenerateFile(input_filename=os.path.join(pmdoc_dir,
"index.xml.in"),
output_filename=os.path.join(out_pmdoc_dir,
"index.xml"))
self.GenerateFile(input_filename=os.path.join(pmdoc_dir,
"01grr.xml.in"),
output_filename=os.path.join(
out_pmdoc_dir, "01%s.xml" % client_name))
self.GenerateFile(input_filename=os.path.join(pmdoc_dir,
"01grr-contents.xml"),
output_filename=os.path.join(
out_pmdoc_dir,
"01%s-contents.xml" % client_name))
self.GenerateFile(input_filename=os.path.join(pmdoc_dir,
"02com.xml.in"),
output_filename=os.path.join(out_pmdoc_dir,
"02com.xml"))
self.GenerateFile(input_filename=os.path.join(pmdoc_dir,
"02com-contents.xml"),
output_filename=os.path.join(out_pmdoc_dir,
"02com-contents.xml"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"preinstall.sh.in"),
output_filename=os.path.join(self.build_dir,
"preinstall.sh"))
self.GenerateFile(input_filename=os.path.join(build_files_dir,
"postinstall.sh.in"),
output_filename=os.path.join(self.build_dir,
"postinstall.sh"))
output_basename = config_lib.CONFIG.Get(
"ClientBuilder.output_basename", context=self.context)
# Rename the generated binaries to the correct name.
template_binary_dir = os.path.join(
config_lib.CONFIG.Get("PyInstaller.distpath",
context=self.context), "grr-client")
target_binary_dir = os.path.join(self.build_dir,
"%s" % output_basename)
if template_binary_dir != target_binary_dir:
shutil.move(template_binary_dir, target_binary_dir)
shutil.move(
os.path.join(target_binary_dir, "grr-client"),
os.path.join(
target_binary_dir,
config_lib.CONFIG.Get("Client.binary_name",
context=self.context)))
deployer = build.ClientDeployer(context=self.context)
deployer.context = self.context
# Generate a config file.
with open(
os.path.join(
target_binary_dir,
config_lib.CONFIG.Get("ClientBuilder.config_filename",
context=self.context)), "wb") as fd:
fd.write(
deployer.GetClientConfig(["Client Context"] + self.context,
validate=False))
print "Fixing file ownership and permissions"
command = [
"sudo", "/usr/sbin/chown", "-R", "root:wheel", self.build_dir
]
# Change the owner, group and permissions of the binaries
print "Running: %s" % " ".join(command)
subprocess.call(command)
command = ["sudo", "/bin/chmod", "-R", "755", self.build_dir]
print "Running: %s" % " ".join(command)
subprocess.call(command)
print "Building a package with PackageMaker"
pkg = "%s-%s.pkg" % (config_lib.CONFIG.Get("Client.name",
context=self.context),
config_lib.CONFIG.Get("Client.version_string",
context=self.context))
output_pkg_path = os.path.join(self.pkg_dir, pkg)
command = [
config_lib.CONFIG.Get("ClientBuilder.package_maker_path",
context=self.context), "--doc",
out_pmdoc_dir, "--out", output_pkg_path
]
print "Running: %s " % " ".join(command)
ret = subprocess.call(command)
if ret != 0:
msg = "PackageMaker returned an error (%d)." % ret
print msg
raise RuntimeError(msg)
print "Copying output to templates location: %s -> %s" % (
output_pkg_path, output_file)
utils.EnsureDirExists(os.path.dirname(output_file))
shutil.copyfile(output_pkg_path, output_file)
# Change the owner, group and permissions of the binaries back.
command = [
"sudo", "/usr/sbin/chown", "-R",
"%s:staff" % getpass.getuser(), self.build_dir
]
print "Running: %s" % " ".join(command)
subprocess.call(command)