def _GenerateDescription(self):
"""Generates description into a MANIFEST file in the archive."""
manifest = {
"description": self.description,
"processed_files": len(self.processed_files),
"archived_files": len(self.archived_files),
"ignored_files": len(self.ignored_files),
"failed_files": len(self.failed_files)
}
if self.ignored_files:
manifest["ignored_files_list"] = [
_ClientPathToString(cp, prefix="aff4:") for cp in self.ignored_files
]
if self.failed_files:
manifest["failed_files_list"] = [
_ClientPathToString(cp, prefix="aff4:") for cp in self.failed_files
]
manifest_fd = io.BytesIO()
if self.total_files != len(self.archived_files):
manifest_fd.write(self.FILES_SKIPPED_WARNING)
manifest_fd.write(yaml.Dump(manifest).encode("utf-8"))
manifest_fd.seek(0)
st = os.stat_result(
(0o644, 0, 0, 0, 0, 0, len(manifest_fd.getvalue()), 0, 0, 0))
for chunk in self.archive_generator.WriteFromFD(
manifest_fd, os.path.join(self.prefix, "MANIFEST"), st=st):
yield chunk
def WriteBuildYaml(self, fd, build_timestamp=True):
"""Write build spec to fd."""
output = {
"Client.build_environment":
rdf_client.Uname.FromCurrentSystem().signature(),
"Template.build_type":
config.CONFIG.Get("ClientBuilder.build_type",
context=self.context),
"Template.version_major":
config.CONFIG.Get("Source.version_major", context=self.context),
"Template.version_minor":
config.CONFIG.Get("Source.version_minor", context=self.context),
"Template.version_revision":
config.CONFIG.Get("Source.version_revision", context=self.context),
"Template.version_release":
config.CONFIG.Get("Source.version_release", context=self.context),
"Template.arch":
config.CONFIG.Get("Client.arch", context=self.context)
}
if build_timestamp:
output["Client.build_time"] = rdfvalue.RDFDatetime.Now()
else:
self.REQUIRED_BUILD_YAML_KEYS.remove("Client.build_time")
for key, value in iteritems(output):
output[key] = str(value)
output["Template.build_context"] = self.context
output_keys = set(iterkeys(output))
if output_keys != self.REQUIRED_BUILD_YAML_KEYS:
raise RuntimeError("Bad build.yaml: expected %s, got %s" %
(self.REQUIRED_BUILD_YAML_KEYS, output_keys))
fd.write(yaml.Dump(output).encode("utf-8"))
def testComplexDict(self):
dumped = yaml.Dump({
"foo.bar": {
"quux": [4, 8, 15, 16, 23, 42],
"thud": ["blargh", "norf"],
},
"foo.baz": [3.14, 1.62],
})
expected = """\
foo.bar:
quux:
- 4
- 8
- 15
- 16
- 23
- 42
thud:
- blargh
- norf
foo.baz:
- 3.14
- 1.62
"""
self.assertEqual(dumped, expected)
def Finish(self):
manifest = {"export_stats": self.export_counts}
manifest_bytes = yaml.Dump(manifest).encode("utf-8")
yield self.archive_generator.WriteFileHeader(self.path_prefix + "/MANIFEST")
yield self.archive_generator.WriteFileChunk(manifest_bytes)
yield self.archive_generator.WriteFileFooter()
yield self.archive_generator.Close()
def _SerializeToYaml(value):
preserialized = []
if isinstance(value, rdf_structs.RDFProtoStruct):
preserialized.append(value.ToPrimitiveDict(stringify_leaf_fields=True))
else:
preserialized.append(str(value))
# Produce a YAML list entry in block format.
# Note that the order of the fields is not guaranteed to correspond to that of
# other output formats.
return yaml.Dump(preserialized)
def _GenerateClientInfo(self, client_id, client_fd):
"""Yields chucks of archive information for given client."""
summary_dict = client_fd.ToPrimitiveDict(stringify_leaf_fields=True)
summary = yaml.Dump(summary_dict).encode("utf-8")
client_info_path = os.path.join(self.prefix, client_id, "client_info.yaml")
st = os.stat_result((0o644, 0, 0, 0, 0, 0, len(summary), 0, 0, 0))
yield self.archive_generator.WriteFileHeader(client_info_path, st=st)
yield self.archive_generator.WriteFileChunk(summary)
yield self.archive_generator.WriteFileFooter()
def testSimpleDict(self):
dumped = yaml.Dump({
"foo": "bar",
"quux": 42,
})
expected = """\
foo: bar
quux: 42
"""
self.assertEqual(dumped, expected)
def testRepackAll(self):
"""Test repacking all binaries."""
self.executables_dir = package.ResourcePath("grr-response-core",
"executables")
with utils.TempDirectory() as tmp_dir:
new_dir = os.path.join(tmp_dir, "grr", "executables")
os.makedirs(new_dir)
# Copy unzipsfx so it can be used in repacking/
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-i386.exe"),
new_dir)
shutil.copy(
os.path.join(self.executables_dir,
"windows/templates/unzipsfx/unzipsfx-amd64.exe"),
new_dir)
with test_lib.ConfigOverrider({
"ClientBuilder.executables_dir":
new_dir,
"ClientBuilder.unzipsfx_stub_dir":
new_dir
}):
repacking.TemplateRepacker().RepackAllTemplates()
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4)
self.assertEqual(
len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1)
# Validate the config appended to the OS X package.
zf = zipfile.ZipFile(glob.glob(
os.path.join(new_dir, "installers/*.pkg")).pop(),
mode="r")
fd = zf.open("config.yaml")
# We can't load the included build.yaml because the package hasn't been
# installed.
loaded = yaml.Parse(fd.read().decode("utf-8"))
loaded.pop("Config.includes")
packaged_config = config.CONFIG.MakeNewConfig()
data = yaml.Dump(loaded)
packaged_config.Initialize(parser=config_lib.YamlParser, data=data)
packaged_config.Validate(
sections=build.ClientRepacker.CONFIG_SECTIONS)
repacker = build.ClientRepacker()
repacker.ValidateEndConfig(packaged_config)
def testUnicode(self):
data = collections.OrderedDict()
data["gęsi"] = ["zbożowa", "krótkodzioba", "białoczelna"]
data["grzebiące"] = ["jarząbek", "głuszec", "bażant"]
dumped = yaml.Dump(data)
expected = """\
gęsi:
- zbożowa
- krótkodzioba
- białoczelna
grzebiące:
- jarząbek
- głuszec
- bażant
"""
self.assertEqual(dumped, expected)
def WriteBuildYaml(fd, build_timestamp=True, context=None):
"""Write build spec to fd."""
if context is None:
raise ValueError("context has to be specified")
output = {
"Client.build_environment":
rdf_client.Uname.FromCurrentSystem().signature(),
"Template.build_type":
config.CONFIG.Get("ClientBuilder.build_type", context=context),
"Template.version_major":
config.CONFIG.Get("Source.version_major", context=context),
"Template.version_minor":
config.CONFIG.Get("Source.version_minor", context=context),
"Template.version_revision":
config.CONFIG.Get("Source.version_revision", context=context),
"Template.version_release":
config.CONFIG.Get("Source.version_release", context=context),
"Template.arch":
config.CONFIG.Get("Client.arch", context=context)
}
yaml_keys = set(build.REQUIRED_BUILD_YAML_KEYS)
if build_timestamp:
output["Client.build_time"] = rdfvalue.RDFDatetime.Now()
else:
yaml_keys.remove("Client.build_time")
for key, value in output.items():
output[key] = str(value)
output["Template.build_context"] = context
output_keys = set(output.keys())
if output_keys != yaml_keys:
raise RuntimeError("Bad build.yaml: expected %s, got %s" %
(yaml_keys, output_keys))
for k, v in output.items():
if v is None:
raise RuntimeError("Bad build.yaml: expected %s to be not None" %
k)
fd.write(yaml.Dump(output))
def ToYaml(self):
artifact_dict = self.ToPrimitiveDict()
# Remove redundant empty defaults.
def ReduceDict(in_dict):
return dict((k, v) for (k, v) in in_dict.items() if v)
artifact_dict = ReduceDict(artifact_dict)
sources_dict = artifact_dict.get("sources")
if sources_dict:
artifact_dict["sources"] = [ReduceDict(c) for c in sources_dict]
# Do some clunky stuff to put the name and doc first in the YAML.
# Unfortunately PYYaml makes doing this difficult in other ways.
ordered_artifact_dict = collections.OrderedDict()
ordered_artifact_dict["name"] = artifact_dict.pop("name")
ordered_artifact_dict["doc"] = artifact_dict.pop("doc")
ordered_artifact_dict.update(artifact_dict)
return yaml.Dump(ordered_artifact_dict)
def _GenerateDescription(self, processed_files: Dict[str, str],
missing_files: Iterable[str]) -> Iterable[bytes]:
"""Generates a MANIFEST file in the archive."""
manifest = {
"processed_files": processed_files,
"missing_files": missing_files,
"client_id": self.flow.client_id,
"flow_id": self.flow.flow_id,
}
manifest_fd = io.BytesIO()
manifest_fd.write(yaml.Dump(manifest).encode("utf-8"))
manifest_fd.seek(0)
st = os.stat_result(
(0o644, 0, 0, 0, 0, 0, len(manifest_fd.getvalue()), 0, 0, 0))
for chunk in self.archive_generator.WriteFromFD(
manifest_fd, os.path.join(self.prefix, "MANIFEST"), st=st):
yield chunk
def YamlDumper(aff4object):
"""Dumps the given aff4object into a yaml representation."""
aff4object.Flush()
result = {}
for attribute, values in iteritems(aff4object.synced_attributes):
result[attribute.predicate] = []
for value in values:
# This value is really a LazyDecoder() instance. We need to get at the
# real data here.
value = value.ToRDFValue()
result[attribute.predicate].append([
value.__class__.__name__,
value.SerializeToString(),
str(value.age)
])
return yaml.Dump({
"aff4_class": compatibility.GetName(aff4object),
"_urn": aff4object.urn.SerializeToString(),
"attributes": result,
"age_policy": aff4object.age_policy,
})
def RawDataToBytes(self, raw_data: Dict[str, Any]) -> bytes:
return yaml.Dump(raw_data).encode("utf-8")
