def testDownloadCollectionWithFoldersEntries(self):
"""Check we can download a collection that also references folders."""
fd = sequential_collection.GeneralIndexedCollection(self.collection_urn)
with data_store.DB.GetMutationPool() as pool:
fd.Add(
rdf_file_finder.FileFinderResult(
stat_entry=rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(path="testfile5",
pathtype="OS"))),
mutation_pool=pool)
fd.Add(
rdf_file_finder.FileFinderResult(
stat_entry=rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(path="testdir1", pathtype="OS"),
st_mode=stat.S_IFDIR)),
mutation_pool=pool)
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(
self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertIn("testfile5", os.listdir(expected_outdir))
self.assertIn("testdir1", os.listdir(expected_outdir))
def GetClientConfig(self, context, validate=True, deploy_timestamp=True):
"""Generates the client config file for inclusion in deployable binaries."""
with utils.TempDirectory() as tmp_dir:
# Make sure we write the file in yaml format.
filename = os.path.join(
tmp_dir,
config.CONFIG.Get("ClientBuilder.config_filename",
context=context))
new_config = config.CONFIG.MakeNewConfig()
new_config.Initialize(reset=True, data="")
new_config.SetWriteBack(filename)
# Only copy certain sections to the client. We enumerate all
# defined options and then resolve those from the config in the
# client's context. The result is the raw option as if the
# client read our config file.
client_context = context[:]
while contexts.CLIENT_BUILD_CONTEXT in client_context:
client_context.remove(contexts.CLIENT_BUILD_CONTEXT)
for descriptor in sorted(config.CONFIG.type_infos,
key=lambda x: x.name):
if descriptor.name in self.SKIP_OPTION_LIST:
continue
if descriptor.section in self.CONFIG_SECTIONS:
value = config.CONFIG.GetRaw(descriptor.name,
context=client_context,
default=None)
if value is not None:
logging.debug("Copying config option to client: %s",
descriptor.name)
new_config.SetRaw(descriptor.name, value)
if deploy_timestamp:
deploy_time_string = str(rdfvalue.RDFDatetime.Now())
new_config.Set("Client.deploy_time", deploy_time_string)
new_config.Write()
if validate:
self.ValidateEndConfig(new_config)
private_validator = config.CONFIG.Get(
"ClientBuilder.private_config_validator_class",
context=context)
if private_validator:
try:
validator = config_validator_base.PrivateConfigValidator.classes[
private_validator]()
except KeyError:
logging.error(
"Couldn't find config validator class %s, "
"you probably need to copy it into lib/local",
private_validator)
raise
validator.ValidateEndConfig(new_config, self.context)
return io.open(filename, "r").read()
def SignGRRPyinstallerBinaries(self):
cert_name = config.CONFIG.Get("ClientBuilder.signing_cert_name",
context=self.context)
keychain_file = config.CONFIG.Get(
"ClientBuilder.signing_keychain_file", context=self.context)
if not keychain_file:
print("No keychain file specified in the config, skipping "
"binaries signing...")
return
print("Signing binaries with keychain: %s" % keychain_file)
with utils.TempDirectory() as temp_dir:
# codesign needs the directory name to adhere to a particular
# naming format.
bundle_dir = os.path.join(
temp_dir, "%s_%s" % (self.client_name, self.version))
shutil.move(self.target_binary_dir, bundle_dir)
temp_binary_path = os.path.join(
bundle_dir,
config.CONFIG.Get("Client.binary_name", context=self.context))
subprocess.check_call([
"codesign", "--verbose", "--deep", "--force", "--sign",
cert_name, "--keychain", keychain_file, temp_binary_path
])
shutil.move(bundle_dir, self.target_binary_dir)
def testConfigFileInclusionCanBeTurnedOff(self):
one = r"""
Config.includes:
- 2.yaml
Section1.int: 1
"""
two = r"""
SecondaryFileIncluded: true
Section1.int: 2
"""
with utils.TempDirectory() as temp_dir:
configone = os.path.join(temp_dir, "1.yaml")
configtwo = os.path.join(temp_dir, "2.yaml")
with io.open(configone, "w") as fd:
fd.write(one)
with io.open(configtwo, "w") as fd:
fd.write(two)
# Using filename
conf = self._GetNewConf()
conf.Initialize(parser=config_lib.YamlParser,
filename=configone,
process_includes=False)
self.assertFalse(conf.Get("SecondaryFileIncluded"))
self.assertEqual(conf.Get("Section1.int"), 1)
def testGeneratesTarGzArchive(self):
result = self.handler.Handle(hunt_plugin.ApiGetHuntFilesArchiveArgs(
hunt_id=self.hunt_id, archive_format="TAR_GZ"),
context=self.context)
with utils.TempDirectory() as temp_dir:
tar_path = os.path.join(temp_dir, "archive.tar.gz")
with open(tar_path, "wb") as fd:
for chunk in result.GenerateContent():
fd.write(chunk)
with tarfile.open(tar_path) as tar_fd:
tar_fd.extractall(path=temp_dir)
manifest_file_path = None
for parent, _, files in os.walk(temp_dir):
if "MANIFEST" in files:
manifest_file_path = os.path.join(parent, "MANIFEST")
break
self.assertTrue(manifest_file_path)
with open(manifest_file_path, "rb") as fd:
manifest = yaml.safe_load(fd.read())
self.assertDictContainsSubset(
{
"archived_files": 10,
"failed_files": 0,
"processed_files": 10,
"ignored_files": 0,
}, manifest)
def testGeneratesTarGzArchive(self):
result = self.handler.Handle(flow_plugin.ApiGetFlowFilesArchiveArgs(
client_id=self.client_id,
flow_id=self.flow_id,
archive_format="TAR_GZ"),
token=self.token)
with utils.TempDirectory() as temp_dir:
tar_path = os.path.join(temp_dir, "archive.tar.gz")
with open(tar_path, "wb") as fd:
for chunk in result.GenerateContent():
fd.write(chunk)
with tarfile.open(tar_path) as tar_fd:
tar_fd.extractall(path=temp_dir)
manifest_file_path = None
for parent, _, files in os.walk(temp_dir):
if "MANIFEST" in files:
manifest_file_path = os.path.join(parent, "MANIFEST")
break
self.assertTrue(manifest_file_path)
with open(manifest_file_path, "rb") as fd:
manifest = yaml.safe_load(fd.read())
self.assertEqual(manifest["archived_files"], 1)
self.assertEqual(manifest["failed_files"], 0)
self.assertEqual(manifest["processed_files"], 1)
self.assertEqual(manifest["ignored_files"], 0)
def setUp(self):
super().setUp()
stack = contextlib.ExitStack()
self.addCleanup(stack.close)
self._tmp_dir = stack.enter_context(utils.TempDirectory())
self.client_id = self.SetupClient(0)
self.procs = [
client_test_lib.MockWindowsProcess(pid=101, name="proc101.exe"),
client_test_lib.MockWindowsProcess(pid=102,
name="proc102.exe",
ppid=101),
client_test_lib.MockWindowsProcess(pid=103,
name="proc103.exe",
ppid=1),
client_test_lib.MockWindowsProcess(pid=104,
name="proc104.exe",
ppid=103),
client_test_lib.MockWindowsProcess(pid=105,
name="proc105.exe",
ppid=1),
client_test_lib.MockWindowsProcess(pid=106,
name="proc106.exe",
ppid=104),
client_test_lib.MockWindowsProcess(pid=108, name="proc108.exe"),
client_test_lib.MockWindowsProcess(pid=109, name="proc109.exe"),
client_test_lib.MockWindowsProcess(pid=110, name="proc110.exe"),
]
def setUp(self):
super(WindowsMsiTest, self).setUp()
stack = contextlib.ExitStack()
self.addCleanup(stack.close)
self._tmp_dir = stack.enter_context(utils.TempDirectory())
def testExportFile(self):
"""Check we can export a file without errors."""
with utils.TempDirectory() as tmpdir:
export_utils.CopyAFF4ToLocal(
self.out.Add("testfile1"), tmpdir, overwrite=True, token=self.token)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
self.assertIn("testfile1", os.listdir(expected_outdir))
def testRecursiveDownload(self):
"""Check we can export a file without errors."""
with utils.TempDirectory() as tmpdir:
export_utils.RecursiveDownload(
aff4.FACTORY.Open(self.out, token=self.token), tmpdir, overwrite=True)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
self.assertIn("testfile1", os.listdir(expected_outdir))
full_outdir = os.path.join(expected_outdir, "testdir1", "testdir2")
self.assertIn("testfile4", os.listdir(full_outdir))
def testConfigFileInclusion(self):
one = r"""
Config.includes:
- 2.yaml
Section1.int: 1
"""
two = r"""
SecondaryFileIncluded: true
Section1.int: 2
Config.includes:
- subdir/3.yaml
"""
three = r"""
TertiaryFileIncluded: true
Section1.int: 3
"""
with utils.TempDirectory() as temp_dir:
configone = os.path.join(temp_dir, "1.yaml")
configtwo = os.path.join(temp_dir, "2.yaml")
subdir = os.path.join(temp_dir, "subdir")
os.makedirs(subdir)
configthree = os.path.join(subdir, "3.yaml")
with open(configone, "wb") as fd:
fd.write(one)
with open(configtwo, "wb") as fd:
fd.write(two)
with open(configthree, "wb") as fd:
fd.write(three)
# Using filename
conf = self._GetNewConf()
conf.Initialize(parser=config_lib.YamlParser, filename=configone)
self._CheckConf(conf)
# Using fd with no fd.name should raise because there is no way to resolve
# the relative path.
conf = self._GetNewConf()
fd = io.StringIO(one)
self.assertRaises(
config_lib.ConfigFileNotFound,
conf.Initialize,
parser=config_lib.YamlParser,
fd=fd)
# Using data
conf = self._GetNewConf()
self.assertRaises(
config_lib.ConfigFileNotFound,
conf.Initialize,
parser=config_lib.YamlParser,
data=one)
def testFileCasing(self):
"""Test our ability to read the correct casing from filesystem."""
try:
os.lstat(os.path.join(self.base_path, "nUmBeRs.txt"))
os.lstat(os.path.join(self.base_path, "nuMbErs.txt"))
# If we reached this point we are on a case insensitive file system
# and the tests below do not make any sense.
logging.warning(
"Case insensitive file system detected. Skipping test.")
return
except (IOError, OSError):
pass
# Create 2 files with names that differ only in casing.
with utils.TempDirectory() as temp_dir:
path1 = os.path.join(temp_dir, "numbers.txt")
shutil.copy(os.path.join(self.base_path, "numbers.txt"), path1)
path2 = os.path.join(temp_dir, "numbers.TXT")
shutil.copy(os.path.join(self.base_path, "numbers.txt.ver2"),
path2)
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=path1,
pathtype=rdf_paths.PathSpec.PathType.OS))
self.assertEqual(fd.pathspec.Basename(), "numbers.txt")
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=path2,
pathtype=rdf_paths.PathSpec.PathType.OS))
self.assertEqual(fd.pathspec.Basename(), "numbers.TXT")
path = os.path.join(self.base_path, "Numbers.txt")
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS))
read_path = fd.pathspec.Basename()
# The exact file now is non deterministic but should be either of the two:
if read_path != "numbers.txt" and read_path != "numbers.TXT":
raise RuntimeError("read path is %s" % read_path)
# Ensure that the produced pathspec specified no case folding:
s = fd.Stat()
self.assertEqual(s.pathspec.path_options,
rdf_paths.PathSpec.Options.CASE_LITERAL)
# Case folding will only occur when requested - this should raise because
# we have the CASE_LITERAL option:
pathspec = rdf_paths.PathSpec(
path=path,
pathtype=rdf_paths.PathSpec.PathType.OS,
path_options=rdf_paths.PathSpec.Options.CASE_LITERAL)
self.assertRaises(IOError, vfs.VFSOpen, pathspec)
def testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = package.ResourcePath("grr-response-core",
"executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1)
# Validate the config appended to the OS X package.
zf = zipfile.ZipFile(glob.glob(
os.path.join(new_dir, "installers/*.pkg")).pop(),
mode="r")
fd = zf.open("config.yaml")
# We can't load the included build.yaml because the package hasn't been
# installed.
loaded = yaml.safe_load(fd)
loaded.pop("Config.includes")
packaged_config = config.CONFIG.MakeNewConfig()
data = yaml.safe_dump(loaded)
packaged_config.Initialize(parser=config_lib.YamlParser,
data=data.decode("utf-8"))
packaged_config.Validate(
sections=build.ClientRepacker.CONFIG_SECTIONS)
repacker = build.ClientRepacker()
repacker.ValidateEndConfig(packaged_config)
def setUp(self):
super().setUp()
stack = contextlib.ExitStack()
self.addCleanup(stack.close)
self._tmp_dir = stack.enter_context(utils.TempDirectory())
# This file can't be located in self._tmp_dir, since self._tmp_dir is
# created with secure permissions and unaccessible to the fake fleetspeak
# service.
self._fake_fleetspeak_service_log_file = os.path.join(
tempfile.gettempdir(), "fake_fleetspeak_service_log_file.txt")
stack.callback(os.unlink, self._fake_fleetspeak_service_log_file)
def testMergeDirectories(self):
stack = contextlib.ExitStack()
self.addCleanup(stack.close)
src_dir = stack.enter_context(utils.TempDirectory())
dst_dir = stack.enter_context(utils.TempDirectory())
def SrcPath(*components):
return os.path.join(src_dir, *components)
def DstPath(*components):
return os.path.join(dst_dir, *components)
def WriteFile(path, contents):
utils.EnsureDirExists(os.path.dirname(path))
with open(path, "w") as f:
f.write(contents)
def ReadFile(path):
with open(path) as f:
return f.read()
WriteFile(SrcPath("a", "b", "c", "file1.txt"), "file1")
WriteFile(SrcPath("a", "b", "c", "file2.txt"), "file2")
WriteFile(SrcPath("file3.txt"), "file3")
WriteFile(DstPath("a", "file4.txt"), "file4")
WriteFile(DstPath("file5.txt"), "file5")
utils.MergeDirectories(src_dir, dst_dir)
self.assertEqual(ReadFile(DstPath("a", "b", "c", "file1.txt")),
"file1")
self.assertEqual(ReadFile(DstPath("a", "b", "c", "file2.txt")),
"file2")
self.assertEqual(ReadFile(DstPath("file3.txt")), "file3")
self.assertEqual(ReadFile(DstPath("a", "file4.txt")), "file4")
self.assertEqual(ReadFile(DstPath("file5.txt")), "file5")
def testUploadOverlyLargeSignedBinary(self):
with mock.patch.object(config_updater_util, "_MAX_SIGNED_BINARY_BYTES", 5):
with utils.TempDirectory() as dir_path:
executable_path = os.path.join(dir_path, "foo.exe")
with open(executable_path, "wb") as f:
f.write(b"\xaa\xbb\xcc\xdd\xee\xff")
expected_message = (
"File [%s] is of size 6 (bytes), which exceeds the allowed maximum "
"of 5 bytes." % executable_path)
with self.assertRaisesWithLiteralMatch(
config_updater_util.BinaryTooLargeError, expected_message):
config_updater_util.UploadSignedBinary(
executable_path, config_pb2.ApiGrrBinary.Type.EXECUTABLE,
"windows")
def _GetTarGzManifest(self, result):
with utils.TempDirectory() as temp_dir:
tar_path = os.path.join(temp_dir, "archive.tar.gz")
with open(tar_path, "wb") as fd:
for chunk in result.GenerateContent():
fd.write(chunk)
with tarfile.open(tar_path) as tar_fd:
tar_fd.extractall(path=temp_dir)
for parent, _, files in os.walk(temp_dir):
if "MANIFEST" in files:
with open(os.path.join(parent, "MANIFEST"), "rb") as fd:
return yaml.safe_load(fd.read())
raise RuntimeError("MANIFEST not found")
def testDownloadCollectionIgnoresArtifactResultsWithoutFiles(self):
# Create a collection with URNs to some files.
fd = sequential_collection.GeneralIndexedCollection(self.collection_urn)
with data_store.DB.GetMutationPool() as pool:
fd.Add(collectors.ArtifactFilesDownloaderResult(), mutation_pool=pool)
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(
self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
self.assertFalse(os.path.exists(expected_outdir))
def testUploadPythonHack(self):
with utils.TempDirectory() as dir_path:
python_hack_path = os.path.join(dir_path, "hello_world.py")
with open(python_hack_path, "wb") as f:
f.write(b"print('Hello, world!')")
config_updater_util.UploadSignedBinary(
python_hack_path,
objects_pb2.SignedBinaryID.BinaryType.PYTHON_HACK,
"linux",
upload_subdirectory="test")
python_hack_urn = rdfvalue.RDFURN(
"aff4:/config/python_hacks/linux/test/hello_world.py")
blob_iterator, _ = signed_binary_utils.FetchBlobsForSignedBinaryByURN(
python_hack_urn)
uploaded_blobs = list(
signed_binary_utils.StreamSignedBinaryContents(blob_iterator))
uploaded_content = b"".join(uploaded_blobs)
self.assertEqual(uploaded_content, b"print('Hello, world!')")
def _VerifyDownload(self):
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile1" in os.listdir(expected_outdir))
self.assertTrue("testfile2" in os.listdir(expected_outdir))
self.assertTrue("testfile5" in os.listdir(expected_outdir))
self.assertTrue("testfile6" in os.listdir(expected_outdir))
# Check we dumped a YAML file to the root of the client.
expected_rootdir = os.path.join(tmpdir, self.client_id.Basename())
self.assertTrue("client_info.yaml" in os.listdir(expected_rootdir))
def testUploadExecutable(self):
with utils.TempDirectory() as dir_path:
executable_path = os.path.join(dir_path, "foo.exe")
with open(executable_path, "wb") as f:
f.write(b"\xaa\xbb\xcc\xdd")
config_updater_util.UploadSignedBinary(
executable_path,
objects_pb2.SignedBinaryID.BinaryType.EXECUTABLE,
"windows",
upload_subdirectory="anti-malware/registry-tools")
executable_urn = rdfvalue.RDFURN(
"aff4:/config/executables/windows/anti-malware/registry-tools/"
"foo.exe")
blob_iterator, _ = signed_binary_utils.FetchBlobsForSignedBinaryByURN(
executable_urn)
uploaded_blobs = list(
signed_binary_utils.StreamSignedBinaryContents(blob_iterator))
uploaded_content = b"".join(uploaded_blobs)
self.assertEqual(uploaded_content, b"\xaa\xbb\xcc\xdd")
def testConfigFileIncludeAbsolutePaths(self):
one = r"""
Section1.int: 1
"""
with utils.TempDirectory() as temp_dir:
configone = os.path.join(temp_dir, "1.yaml")
with io.open(configone, "w") as fd:
fd.write(one)
absolute_include = (r"""
Config.includes:
- %s
Section1.int: 2
""" % configone)
conf = self._GetNewConf()
conf.Initialize(parser=config_parser.YamlConfigFileParser,
data=absolute_include)
self.assertEqual(conf["Section1.int"], 1)
relative_include = r"""
Config.includes:
- 1.yaml
Section1.int: 2
"""
conf = self._GetNewConf()
# Can not include a relative path from config without a filename.
self.assertRaises(config_lib.ConfigFileNotFound,
conf.Initialize,
parser=config_parser.YamlConfigFileParser,
data=relative_include)
# If we write it to a file it should work though.
configtwo = os.path.join(temp_dir, "2.yaml")
with io.open(configtwo, "w") as fd:
fd.write(relative_include)
conf.Initialize(parser=config_parser.YamlConfigFileParser,
filename=configtwo)
self.assertEqual(conf["Section1.int"], 1)
def _MakeDeployableBinaryV2(self, template_path, output_path):
context = self.context + ["Client Context"]
utils.EnsureDirExists(os.path.dirname(output_path))
fleetspeak_enabled = config.CONFIG.Get("Client.fleetspeak_enabled",
context=self.context)
fleetspeak_bundled = config.CONFIG.Get(
"ClientBuilder.fleetspeak_bundled", context=self.context)
with contextlib.ExitStack() as stack:
tmp_dir = stack.enter_context(utils.TempDirectory())
shutil.unpack_archive(template_path, tmp_dir, format="zip")
if fleetspeak_bundled:
variant = "fleetspeak-bundled"
elif fleetspeak_enabled:
variant = "fleetspeak-enabled"
else:
variant = "legacy"
pkg_utils.JoinPkg(os.path.join(tmp_dir, variant),
os.path.join(tmp_dir, "blocks"), output_path)
zf = stack.enter_context(zipfile.ZipFile(output_path, mode="a"))
with open(os.path.join(tmp_dir, "build.yaml"),
"r") as build_yaml_file:
zf.writestr("build.yaml", build_yaml_file.read())
client_config_data = build_helpers.GetClientConfig(context)
zf.writestr("config.yaml", client_config_data)
if fleetspeak_bundled:
fleetspeak_client_config = config.CONFIG.Get(
"ClientBuilder.fleetspeak_client_config",
context=self.context)
with open(fleetspeak_client_config,
"r") as fleetspeak_client_config_file:
zf.writestr("client.config",
fleetspeak_client_config_file.read())
return output_path
def testDownloadCollectionWithFlattenOption(self):
"""Check we can download files references in a collection."""
# Create a collection with URNs to some files.
fd = sequential_collection.GeneralIndexedCollection(self.collection_urn)
with data_store.DB.GetMutationPool() as pool:
fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1")), mutation_pool=pool)
fd.Add(
rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(path="testfile2", pathtype="OS")),
mutation_pool=pool)
fd.Add(
rdf_file_finder.FileFinderResult(
stat_entry=rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(path="testfile5",
pathtype="OS"))),
mutation_pool=pool)
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(
self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
flatten=True,
token=self.token,
max_threads=2)
# Check that "files" folder is filled with symlinks to downloaded files.
symlinks = os.listdir(os.path.join(tmpdir, "files"))
self.assertLen(symlinks, 3)
self.assertListEqual(
sorted(symlinks), [
"C.1000000000000000_fs_os_testfile1",
"C.1000000000000000_fs_os_testfile2",
"C.1000000000000000_fs_os_testfile5"
])
self.assertEqual(
os.readlink(
os.path.join(tmpdir, "files",
"C.1000000000000000_fs_os_testfile1")),
os.path.join(tmpdir, "C.1000000000000000", "fs", "os", "testfile1"))
def _GenerateFleetspeakServiceConfig(self, zip_file):
orig_fs_config_path = config.CONFIG["ClientBuilder.fleetspeak_config_path"]
final_fs_config_fname = config.CONFIG[
"Client.fleetspeak_unsigned_config_fname"]
if orig_fs_config_path.endswith(".in"):
logging.info("Interpolating %s", orig_fs_config_path)
logging.warning("Backslashes will be naively re-escaped after "
"interpolation. If this is not desired, use a Fleetspeak "
"config file without the '.in' extension.")
with utils.TempDirectory() as temp_dir:
temp_fs_config_path = os.path.join(temp_dir, final_fs_config_fname)
with io.open(orig_fs_config_path, "r") as source:
with io.open(temp_fs_config_path, "w") as dest:
interpolated = config.CONFIG.InterpolateValue(
source.read(), context=self.context)
dest.write(re.sub(r"\\", r"\\\\", interpolated))
self._ValidateFleetspeakServiceConfig(temp_fs_config_path)
zip_file.write(temp_fs_config_path, final_fs_config_fname)
else:
self._ValidateFleetspeakServiceConfig(orig_fs_config_path)
zip_file.write(orig_fs_config_path, final_fs_config_fname)
def testConfigFileInclusionWithContext(self):
one = r"""
Client Context:
Config.includes:
- 2.yaml
Section1.int: 1
"""
two = r"""
Section1.int: 2
SecondaryFileIncluded: true
"""
with utils.TempDirectory() as temp_dir:
configone = os.path.join(temp_dir, "1.yaml")
configtwo = os.path.join(temp_dir, "2.yaml")
with io.open(configone, "w") as fd:
fd.write(one)
with io.open(configtwo, "w") as fd:
fd.write(two)
# Without specifying the context the includes are not processed.
conf = self._GetNewConf()
conf.Initialize(parser=config_parser.YamlConfigFileParser,
filename=configone)
self.assertEqual(conf["Section1.int"], 1)
# Only one config is loaded.
self.assertEqual(conf.files, [configone])
# Now we specify the context.
conf = self._GetNewConf()
conf.AddContext("Client Context")
conf.Initialize(parser=config_parser.YamlConfigFileParser,
filename=configone)
# Both config files were loaded. Note that load order is important and
# well defined.
self.assertEqual(conf.files, [configone, configtwo])
self.assertEqual(conf["Section1.int"], 2)
def testZipFileWithSymlink(self):
"""Test that symlinks are preserved when unpacking generated zips."""
compressions = [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED]
for compression in compressions:
outfd = io.BytesIO()
infd1 = io.BytesIO(b"this is a test string")
infd2 = io.BytesIO(b"this is another test string")
with utils.StreamingZipWriter(outfd,
compression=compression) as writer:
writer.WriteFromFD(infd1, "test1.txt")
writer.WriteFromFD(infd2, "subdir/test2.txt")
writer.WriteSymlink("test1.txt", "test1.txt.link")
writer.WriteSymlink("subdir/test2.txt", "test2.txt.link")
with utils.TempDirectory() as temp_dir:
zip_path = os.path.join(temp_dir, "archive.zip")
with open(zip_path, "wb") as fd:
fd.write(outfd.getvalue())
zip_fd = zipfile.ZipFile(outfd, "r")
link_info = zip_fd.getinfo("test1.txt.link")
self.assertEqual(link_info.external_attr,
(0o644 | 0o120000) << 16)
self.assertEqual(link_info.create_system, 3)
link_contents = zip_fd.read("test1.txt.link")
self.assertEqual(link_contents, "test1.txt")
link_info = zip_fd.getinfo("test2.txt.link")
self.assertEqual(link_info.external_attr,
(0o644 | 0o120000) << 16)
self.assertEqual(link_info.create_system, 3)
link_contents = zip_fd.read("test2.txt.link")
self.assertEqual(link_contents, "subdir/test2.txt")
def __init__(self, path: str):
self._olefile = olefile.OleFileIO(path, write_mode=True)
self._stack = contextlib.ExitStack()
self._tmp_dir = self._stack.enter_context(utils.TempDirectory())
def ReadStream(name):
with self._olefile.openstream(name) as stream:
return stream.read(self._olefile.get_size(name))
string_pool_raw = ReadStream(STRING_POOL_STREAM_NAME)
string_data_raw = ReadStream(STRING_DATA_STREAM_NAME)
self._string_pool = StringPool(string_pool_raw, string_data_raw)
feature_raw = ReadStream(FEATURE_STREAM_NAME)
self._feature_table = FeatureTable(feature_raw, self._string_pool)
cab_path = os.path.join(self._tmp_dir, "input.cab")
cab_tmp_path = os.path.join(self._tmp_dir, "cab_tmp_dir")
with open(cab_path, "wb") as f:
f.write(ReadStream(GRR_CAB_STREAM_NAME))
self._cab = cab_utils.Cab(cab_path, cab_tmp_path)
self._cab.ExtractFiles()
self._cab.WriteFile("PaddingFile", b"")
def MakeDeployableBinary(self, template_path: str,
output_path: str) -> str:
context = self.context + ["Client Context"]
utils.EnsureDirExists(os.path.dirname(output_path))
def GetConfig(name: str) -> Any:
return config.CONFIG.Get(name, context=self.context)
fleetspeak_enabled = GetConfig("Client.fleetspeak_enabled")
fleetspeak_bundled = GetConfig("ClientBuilder.fleetspeak_bundled")
legacy = not (fleetspeak_enabled or fleetspeak_bundled)
with contextlib.ExitStack() as stack:
tmp_dir = stack.enter_context(utils.TempDirectory())
shutil.unpack_archive(template_path, tmp_dir, format="zip")
msi_file = MsiFile(os.path.join(tmp_dir, "installer.msi"))
def EnableFeature(name: str) -> None:
msi_file.EnableFeature(name.encode("utf-8"))
def ReplaceString(src: str, dst: str) -> None:
msi_file.ReplaceString(src.encode("utf-8"),
dst.encode("utf-8"))
def RenameFile(src: str, dst: str) -> None:
msi_file.RenameFile(src.encode("utf-8"), dst.encode("utf-8"))
def ReplaceStringConfig(src: str, dst: str) -> None:
ReplaceString(src, GetConfig(dst))
def RenameFileConfig(src: str, dst: str) -> None:
RenameFile(src, GetConfig(dst))
# Set product information
ReplaceStringConfig("__ProductName", "Client.name")
ReplaceStringConfig("__ProductManufacturer", "Client.company_name")
# Enable features
if GetConfig("ClientBuilder.console"):
EnableFeature("DbgGrrExe")
else:
EnableFeature("GrrExe")
if legacy:
if GetConfig("ClientBuilder.console"):
EnableFeature("DbgNanny")
else:
EnableFeature("Nanny")
if fleetspeak_bundled:
EnableFeature("FleetspeakClient")
if fleetspeak_enabled or fleetspeak_bundled:
EnableFeature("FleetspeakServiceRegistryEntry")
# Rename directories
RenameFileConfig("__GrrDirectory", "Client.name")
RenameFileConfig("__GrrVersion", "Source.version_string")
# Rename files
if GetConfig("ClientBuilder.console"):
RenameFileConfig("__dbg_grr-client.exe", "Client.binary_name")
RenameFileConfig("__dbg_GRRService.exe",
"Nanny.service_binary_name")
else:
RenameFileConfig("__grr-client.exe", "Client.binary_name")
RenameFileConfig("__GRRService.exe",
"Nanny.service_binary_name")
# Write Configs
if fleetspeak_bundled:
with open(GetConfig("ClientBuilder.fleetspeak_client_config"),
"rb") as f:
msi_file.SetFleetspeakConfig(f.read())
RenameFileConfig("grr-config.yaml",
"ClientBuilder.config_filename")
msi_file.SetGrrConfig(
build_helpers.GetClientConfig(context).encode("utf-8"))
# Write Fleetspeak service registry data
if fleetspeak_enabled or fleetspeak_bundled:
key_name = GetConfig(
"Client.fleetspeak_unsigned_services_regkey")
key_name = key_name.replace("HKEY_LOCAL_MACHINE\\", "")
ReplaceString("__FleetspeakServiceRegistryKey", key_name)
ReplaceStringConfig("__FleetspeakServiceRegistryName",
"Client.name")
ReplaceString(
"__FleetspeakServiceRegistryValue",
f"[INSTALLDIR]{GetConfig('Client.fleetspeak_unsigned_config_fname')}"
)
if fleetspeak_bundled:
ReplaceStringConfig("FleetspeakClientService",
"Client.fleetspeak_service_name")
# Write Fleetspeak service config
# If we don't need to re-write the file after installation, just run
# a dummy command.
gen_fleespeak_service_file_cmd = "cmd.exe /c exit"
if fleetspeak_enabled or fleetspeak_bundled:
path = GetConfig("ClientBuilder.fleetspeak_config_path")
with open(path, "rb") as f:
msi_file.SetFleetspeakServiceConfig(f.read())
RenameFileConfig("fleetspeak-service-config.txt",
"Client.fleetspeak_unsigned_config_fname")
if path.endswith(".in"):
args = [
"[INSTALLDIR]" + GetConfig("Client.binary_name"),
"--config",
"[INSTALLDIR]" +
GetConfig("ClientBuilder.config_filename"),
"-p",
"Client.install_path=[INSTALLDIR]",
"--install",
"--interpolate_fleetspeak_service_config",
"[INSTALLDIR]" +
GetConfig("Client.fleetspeak_unsigned_config_fname"),
]
gen_fleespeak_service_file_cmd = subprocess.list2cmdline(
args)
ReplaceString("__GenFleetspeakServiceFileCmd",
gen_fleespeak_service_file_cmd)
# Configure nanny service
if legacy:
nanny_args = ["--service_key", GetConfig("Client.config_key")]
ReplaceString("__NannyArguments",
subprocess.list2cmdline(nanny_args))
ReplaceStringConfig("__NannyServiceDescription",
"Nanny.service_description")
if GetConfig("ClientBuilder.console"):
ReplaceStringConfig("__DbgNannyRegistryKey",
"Client.config_key")
ReplaceStringConfig("__DbgNannyServiceName",
"Nanny.service_name")
else:
ReplaceStringConfig("__NannyRegistryKey",
"Client.config_key")
ReplaceStringConfig("__NannyServiceName",
"Nanny.service_name")
grr_binary = GetConfig("Client.binary_name")
grr_config = GetConfig("ClientBuilder.config_filename")
ReplaceString("__NannyChildBinary",
f"[INSTALLDIR]{grr_binary}")
child_args = [
f"[INSTALLDIR]{grr_binary}", "--config",
f"[INSTALLDIR]{grr_config}"
]
ReplaceString("__NannyChildCommandLine",
subprocess.list2cmdline(child_args))
msi_file.Write()
msi_file.Close()
if os.path.exists(output_path):
os.remove(output_path)
shutil.move(os.path.join(tmp_dir, "installer.msi"), output_path)
return output_path
def MakeDeployableBinary(self, template_path, output_path):
"""This will add the config to the client template and create a .rpm."""
rpmbuild_binary = "/usr/bin/rpmbuild"
if not os.path.exists(rpmbuild_binary):
logging.error("rpmbuild not found, unable to repack client.")
return
with utils.TempDirectory() as tmp_dir:
template_dir = os.path.join(tmp_dir, "dist")
utils.EnsureDirExists(template_dir)
zf = zipfile.ZipFile(template_path)
for name in zf.namelist():
dirname = os.path.dirname(name)
utils.EnsureDirExists(os.path.join(template_dir, dirname))
with open(os.path.join(template_dir, name), "wb") as fd:
fd.write(zf.read(name))
# Set up a RPM building environment.
rpm_root_dir = os.path.join(tmp_dir, "rpmbuild")
rpm_build_dir = os.path.join(rpm_root_dir, "BUILD")
utils.EnsureDirExists(rpm_build_dir)
rpm_buildroot_dir = os.path.join(rpm_root_dir, "BUILDROOT")
utils.EnsureDirExists(rpm_buildroot_dir)
rpm_rpms_dir = os.path.join(rpm_root_dir, "RPMS")
utils.EnsureDirExists(rpm_rpms_dir)
rpm_specs_dir = os.path.join(rpm_root_dir, "SPECS")
utils.EnsureDirExists(rpm_specs_dir)
template_binary_dir = os.path.join(tmp_dir,
"dist/rpmbuild/grr-client")
target_binary_dir = "%s%s" % (rpm_build_dir,
config.CONFIG.Get(
"ClientBuilder.target_dir",
context=self.context))
utils.EnsureDirExists(os.path.dirname(target_binary_dir))
try:
shutil.rmtree(target_binary_dir)
except OSError:
pass
shutil.move(template_binary_dir, target_binary_dir)
client_name = config.CONFIG.Get("Client.name",
context=self.context)
client_binary_name = config.CONFIG.Get("Client.binary_name",
context=self.context)
if client_binary_name != "grr-client":
shutil.move(
os.path.join(target_binary_dir, "grr-client"),
os.path.join(target_binary_dir, client_binary_name))
# Generate spec
spec_filename = os.path.join(rpm_specs_dir,
"%s.spec" % client_name)
self.GenerateFile(
os.path.join(tmp_dir, "dist/rpmbuild/grr.spec.in"),
spec_filename)
initd_target_filename = os.path.join(rpm_build_dir, "etc/init.d",
client_name)
# Generate init.d
utils.EnsureDirExists(os.path.dirname(initd_target_filename))
self.GenerateFile(
os.path.join(tmp_dir, "dist/rpmbuild/grr-client.initd.in"),
initd_target_filename)
# Generate systemd unit
if config.CONFIG["Template.version_numeric"] >= 3125:
systemd_target_filename = os.path.join(
rpm_build_dir, "usr/lib/systemd/system/",
"%s.service" % client_name)
utils.EnsureDirExists(os.path.dirname(systemd_target_filename))
self.GenerateFile(
os.path.join(tmp_dir,
"dist/rpmbuild/grr-client.service.in"),
systemd_target_filename)
# Generate prelinking blacklist file
prelink_target_filename = os.path.join(rpm_build_dir,
"etc/prelink.conf.d",
"%s.conf" % client_name)
utils.EnsureDirExists(os.path.dirname(prelink_target_filename))
self.GenerateFile(
os.path.join(tmp_dir,
"dist/rpmbuild/prelink_blacklist.conf.in"),
prelink_target_filename)
# Create a client config.
client_context = ["Client Context"] + self.context
client_config_content = self.GetClientConfig(client_context)
with open(
os.path.join(
target_binary_dir,
config.CONFIG.Get("ClientBuilder.config_filename",
context=self.context)), "wb") as fd:
fd.write(client_config_content)
# Set the daemon to executable.
os.chmod(os.path.join(target_binary_dir, client_binary_name),
0o755)
client_arch = config.CONFIG.Get("Template.arch",
context=self.context)
if client_arch == "amd64":
client_arch = "x86_64"
command = [
rpmbuild_binary, "--define", "_topdir " + rpm_root_dir,
"--target", client_arch, "--buildroot", rpm_buildroot_dir,
"-bb", spec_filename
]
try:
subprocess.check_output(command, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logging.error("Error calling %s.", command)
logging.error(e.output)
raise
client_version = config.CONFIG.Get("Template.version_string",
context=self.context)
rpm_filename = os.path.join(
rpm_rpms_dir, client_arch,
"%s-%s-1.%s.rpm" % (client_name, client_version, client_arch))
utils.EnsureDirExists(os.path.dirname(output_path))
shutil.move(rpm_filename, output_path)
logging.info("Created package %s", output_path)
self.Sign(output_path)
return output_path
