def _GenerateDescription(self): """Generates description into a MANIFEST file in the archive.""" manifest = { "description": self.description, "processed_files": len(self.processed_files), "archived_files": len(self.archived_files), "ignored_files": len(self.ignored_files), "failed_files": len(self.failed_files) } if self.ignored_files: manifest["ignored_files_list"] = [ _ClientPathToString(cp, prefix="aff4:") for cp in self.ignored_files ] if self.failed_files: manifest["failed_files_list"] = [ _ClientPathToString(cp, prefix="aff4:") for cp in self.failed_files ] manifest_fd = io.BytesIO() if self.total_files != len(self.archived_files): manifest_fd.write(self.FILES_SKIPPED_WARNING) manifest_fd.write(yaml.Dump(manifest).encode("utf-8")) manifest_fd.seek(0) st = os.stat_result( (0o644, 0, 0, 0, 0, 0, len(manifest_fd.getvalue()), 0, 0, 0)) for chunk in self.archive_generator.WriteFromFD( manifest_fd, os.path.join(self.prefix, "MANIFEST"), st=st): yield chunk
def WriteBuildYaml(self, fd, build_timestamp=True): """Write build spec to fd.""" output = { "Client.build_environment": rdf_client.Uname.FromCurrentSystem().signature(), "Template.build_type": config.CONFIG.Get("ClientBuilder.build_type", context=self.context), "Template.version_major": config.CONFIG.Get("Source.version_major", context=self.context), "Template.version_minor": config.CONFIG.Get("Source.version_minor", context=self.context), "Template.version_revision": config.CONFIG.Get("Source.version_revision", context=self.context), "Template.version_release": config.CONFIG.Get("Source.version_release", context=self.context), "Template.arch": config.CONFIG.Get("Client.arch", context=self.context) } if build_timestamp: output["Client.build_time"] = rdfvalue.RDFDatetime.Now() else: self.REQUIRED_BUILD_YAML_KEYS.remove("Client.build_time") for key, value in iteritems(output): output[key] = str(value) output["Template.build_context"] = self.context output_keys = set(iterkeys(output)) if output_keys != self.REQUIRED_BUILD_YAML_KEYS: raise RuntimeError("Bad build.yaml: expected %s, got %s" % (self.REQUIRED_BUILD_YAML_KEYS, output_keys)) fd.write(yaml.Dump(output).encode("utf-8"))
def testComplexDict(self): dumped = yaml.Dump({ "foo.bar": { "quux": [4, 8, 15, 16, 23, 42], "thud": ["blargh", "norf"], }, "foo.baz": [3.14, 1.62], }) expected = """\ foo.bar: quux: - 4 - 8 - 15 - 16 - 23 - 42 thud: - blargh - norf foo.baz: - 3.14 - 1.62 """ self.assertEqual(dumped, expected)
def Finish(self): manifest = {"export_stats": self.export_counts} manifest_bytes = yaml.Dump(manifest).encode("utf-8") yield self.archive_generator.WriteFileHeader(self.path_prefix + "/MANIFEST") yield self.archive_generator.WriteFileChunk(manifest_bytes) yield self.archive_generator.WriteFileFooter() yield self.archive_generator.Close()
def _SerializeToYaml(value): preserialized = [] if isinstance(value, rdf_structs.RDFProtoStruct): preserialized.append(value.ToPrimitiveDict(stringify_leaf_fields=True)) else: preserialized.append(str(value)) # Produce a YAML list entry in block format. # Note that the order of the fields is not guaranteed to correspond to that of # other output formats. return yaml.Dump(preserialized)
def _GenerateClientInfo(self, client_id, client_fd): """Yields chucks of archive information for given client.""" summary_dict = client_fd.ToPrimitiveDict(stringify_leaf_fields=True) summary = yaml.Dump(summary_dict).encode("utf-8") client_info_path = os.path.join(self.prefix, client_id, "client_info.yaml") st = os.stat_result((0o644, 0, 0, 0, 0, 0, len(summary), 0, 0, 0)) yield self.archive_generator.WriteFileHeader(client_info_path, st=st) yield self.archive_generator.WriteFileChunk(summary) yield self.archive_generator.WriteFileFooter()
def testSimpleDict(self): dumped = yaml.Dump({ "foo": "bar", "quux": 42, }) expected = """\ foo: bar quux: 42 """ self.assertEqual(dumped, expected)
def testRepackAll(self): """Test repacking all binaries.""" self.executables_dir = package.ResourcePath("grr-response-core", "executables") with utils.TempDirectory() as tmp_dir: new_dir = os.path.join(tmp_dir, "grr", "executables") os.makedirs(new_dir) # Copy unzipsfx so it can be used in repacking/ shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-i386.exe"), new_dir) shutil.copy( os.path.join(self.executables_dir, "windows/templates/unzipsfx/unzipsfx-amd64.exe"), new_dir) with test_lib.ConfigOverrider({ "ClientBuilder.executables_dir": new_dir, "ClientBuilder.unzipsfx_stub_dir": new_dir }): repacking.TemplateRepacker().RepackAllTemplates() self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.deb"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.rpm"))), 2) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.exe"))), 4) self.assertEqual( len(glob.glob(os.path.join(new_dir, "installers/*.pkg"))), 1) # Validate the config appended to the OS X package. zf = zipfile.ZipFile(glob.glob( os.path.join(new_dir, "installers/*.pkg")).pop(), mode="r") fd = zf.open("config.yaml") # We can't load the included build.yaml because the package hasn't been # installed. loaded = yaml.Parse(fd.read().decode("utf-8")) loaded.pop("Config.includes") packaged_config = config.CONFIG.MakeNewConfig() data = yaml.Dump(loaded) packaged_config.Initialize(parser=config_lib.YamlParser, data=data) packaged_config.Validate( sections=build.ClientRepacker.CONFIG_SECTIONS) repacker = build.ClientRepacker() repacker.ValidateEndConfig(packaged_config)
def testUnicode(self): data = collections.OrderedDict() data["gęsi"] = ["zbożowa", "krótkodzioba", "białoczelna"] data["grzebiące"] = ["jarząbek", "głuszec", "bażant"] dumped = yaml.Dump(data) expected = """\ gęsi: - zbożowa - krótkodzioba - białoczelna grzebiące: - jarząbek - głuszec - bażant """ self.assertEqual(dumped, expected)
def WriteBuildYaml(fd, build_timestamp=True, context=None): """Write build spec to fd.""" if context is None: raise ValueError("context has to be specified") output = { "Client.build_environment": rdf_client.Uname.FromCurrentSystem().signature(), "Template.build_type": config.CONFIG.Get("ClientBuilder.build_type", context=context), "Template.version_major": config.CONFIG.Get("Source.version_major", context=context), "Template.version_minor": config.CONFIG.Get("Source.version_minor", context=context), "Template.version_revision": config.CONFIG.Get("Source.version_revision", context=context), "Template.version_release": config.CONFIG.Get("Source.version_release", context=context), "Template.arch": config.CONFIG.Get("Client.arch", context=context) } yaml_keys = set(build.REQUIRED_BUILD_YAML_KEYS) if build_timestamp: output["Client.build_time"] = rdfvalue.RDFDatetime.Now() else: yaml_keys.remove("Client.build_time") for key, value in output.items(): output[key] = str(value) output["Template.build_context"] = context output_keys = set(output.keys()) if output_keys != yaml_keys: raise RuntimeError("Bad build.yaml: expected %s, got %s" % (yaml_keys, output_keys)) for k, v in output.items(): if v is None: raise RuntimeError("Bad build.yaml: expected %s to be not None" % k) fd.write(yaml.Dump(output))
def ToYaml(self): artifact_dict = self.ToPrimitiveDict() # Remove redundant empty defaults. def ReduceDict(in_dict): return dict((k, v) for (k, v) in in_dict.items() if v) artifact_dict = ReduceDict(artifact_dict) sources_dict = artifact_dict.get("sources") if sources_dict: artifact_dict["sources"] = [ReduceDict(c) for c in sources_dict] # Do some clunky stuff to put the name and doc first in the YAML. # Unfortunately PYYaml makes doing this difficult in other ways. ordered_artifact_dict = collections.OrderedDict() ordered_artifact_dict["name"] = artifact_dict.pop("name") ordered_artifact_dict["doc"] = artifact_dict.pop("doc") ordered_artifact_dict.update(artifact_dict) return yaml.Dump(ordered_artifact_dict)
def _GenerateDescription(self, processed_files: Dict[str, str], missing_files: Iterable[str]) -> Iterable[bytes]: """Generates a MANIFEST file in the archive.""" manifest = { "processed_files": processed_files, "missing_files": missing_files, "client_id": self.flow.client_id, "flow_id": self.flow.flow_id, } manifest_fd = io.BytesIO() manifest_fd.write(yaml.Dump(manifest).encode("utf-8")) manifest_fd.seek(0) st = os.stat_result( (0o644, 0, 0, 0, 0, 0, len(manifest_fd.getvalue()), 0, 0, 0)) for chunk in self.archive_generator.WriteFromFD( manifest_fd, os.path.join(self.prefix, "MANIFEST"), st=st): yield chunk
def YamlDumper(aff4object): """Dumps the given aff4object into a yaml representation.""" aff4object.Flush() result = {} for attribute, values in iteritems(aff4object.synced_attributes): result[attribute.predicate] = [] for value in values: # This value is really a LazyDecoder() instance. We need to get at the # real data here. value = value.ToRDFValue() result[attribute.predicate].append([ value.__class__.__name__, value.SerializeToString(), str(value.age) ]) return yaml.Dump({ "aff4_class": compatibility.GetName(aff4object), "_urn": aff4object.urn.SerializeToString(), "attributes": result, "age_policy": aff4object.age_policy, })
def RawDataToBytes(self, raw_data: Dict[str, Any]) -> bytes: return yaml.Dump(raw_data).encode("utf-8")