class Images(productmd.common.MetadataBase):
def __init__(self):
super(Images, self).__init__()
self.header = Header(self)
self.compose = Compose(self)
self.images = {}
def __getitem__(self, variant):
return self.images[variant]
def __delitem__(self, variant):
del self.images[variant]
def serialize(self, parser):
data = parser
self.header.serialize(data)
data["payload"] = {}
data["payload"]["images"] = {}
self.compose.serialize(data["payload"])
for variant in self.images:
for arch in self.images[variant]:
for image_obj in self.images[variant][arch]:
images = data["payload"]["images"].setdefault(
variant, {}).setdefault(arch, [])
image_obj.serialize(images)
images.sort(key=lambda x: x["path"])
return data
def deserialize(self, data):
self.header.deserialize(data)
self.compose.deserialize(data["payload"])
for variant in data["payload"]["images"]:
for arch in data["payload"]["images"][variant]:
for image in data["payload"]["images"][variant][arch]:
image_obj = Image(self)
image_obj.deserialize(image)
self.add(variant, arch, image_obj)
self.header.set_current_version()
def add(self, variant, arch, image):
"""
Assign an :class:`.Image` object to variant and arch.
:param variant: compose variant UID
:type variant: str
:param arch: compose architecture
:type arch: str
:param image: image
:type image: :class:`.Image`
"""
if arch not in productmd.common.RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
self.images.setdefault(variant, {}).setdefault(arch, set()).add(image)
class Images(productmd.common.MetadataBase):
def __init__(self):
super(Images, self).__init__()
self.header = Header(self)
self.compose = Compose(self)
self.images = {}
def __getitem__(self, variant):
return self.images[variant]
def __delitem__(self, variant):
del self.images[variant]
def serialize(self, parser):
data = parser
self.header.serialize(data)
data["payload"] = {}
data["payload"]["images"] = {}
self.compose.serialize(data["payload"])
for variant in self.images:
for arch in self.images[variant]:
for image_obj in self.images[variant][arch]:
images = data["payload"]["images"].setdefault(variant, {}).setdefault(arch, [])
image_obj.serialize(images)
images.sort(key=lambda x: x["path"])
return data
def deserialize(self, data):
self.header.deserialize(data)
self.compose.deserialize(data["payload"])
for variant in data["payload"]["images"]:
for arch in data["payload"]["images"][variant]:
for image in data["payload"]["images"][variant][arch]:
image_obj = Image(self)
image_obj.deserialize(image)
self.add(variant, arch, image_obj)
self.header.set_current_version()
def add(self, variant, arch, image):
"""
Assign an :class:`.Image` object to variant and arch.
:param variant: compose variant UID
:type variant: str
:param arch: compose architecture
:type arch: str
:param image: image
:type image: :class:`.Image`
"""
if arch not in productmd.common.RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
self.images.setdefault(variant, {}).setdefault(arch, set()).add(image)
class Images(productmd.common.MetadataBase):
def __init__(self):
super(Images, self).__init__()
self.header = Header(self, "productmd.images")
self.compose = Compose(self)
self.images = {}
def __getitem__(self, variant):
return self.images[variant]
def __delitem__(self, variant):
del self.images[variant]
def serialize(self, parser):
data = parser
self.header.serialize(data)
data["payload"] = {}
data["payload"]["images"] = {}
self.compose.serialize(data["payload"])
for variant in self.images:
for arch in self.images[variant]:
for image_obj in self.images[variant][arch]:
images = data["payload"]["images"].setdefault(
variant, {}).setdefault(arch, [])
image_obj.serialize(images)
images.sort(key=lambda x: x["path"])
return data
def deserialize(self, data):
self.header.deserialize(data)
self.compose.deserialize(data["payload"])
for variant in data["payload"]["images"]:
for arch in data["payload"]["images"][variant]:
for image in data["payload"]["images"][variant][arch]:
image_obj = Image(self)
image_obj.deserialize(image)
if self.header.version_tuple <= (1, 1):
self._add_1_1(data, variant, arch, image_obj)
else:
self.add(variant, arch, image_obj)
self.header.set_current_version()
def _add_1_1(self, data, variant, arch, image):
if arch == "src":
# move src under binary arches
for variant_arch in data["payload"]["images"][variant]:
if variant_arch == "src":
continue
self.add(variant, variant_arch, image)
else:
self.add(variant, arch, image)
def add(self, variant, arch, image):
"""
Assign an :class:`.Image` object to variant and arch.
:param variant: compose variant UID
:type variant: str
:param arch: compose architecture
:type arch: str
:param image: image
:type image: :class:`.Image`
"""
if arch not in productmd.common.RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
if arch in ["src", "nosrc"]:
raise ValueError(
"Source arch is not allowed. Map source files under binary arches."
)
if self.header.version_tuple >= (1, 1):
# disallow adding a different image with same 'unique'
# attributes. can't do this pre-1.1 as we couldn't truly
# identify images before subvariant
for checkvar in self.images:
for checkarch in self.images[checkvar]:
for curimg in self.images[checkvar][checkarch]:
if identify_image(curimg) == identify_image(
image) and curimg.checksums != image.checksums:
raise ValueError(
"Image {0} shares all UNIQUE_IMAGE_ATTRIBUTES with "
"image {1}! This is forbidden.".format(
image, curimg))
self.images.setdefault(variant, {}).setdefault(arch, set()).add(image)
class Modules(productmd.common.MetadataBase):
def __init__(self):
super(Modules, self).__init__()
self.header = Header(self, "productmd.modules")
self.compose = Compose(self)
self.modules = {}
def __getitem__(self, variant):
return self.modules[variant]
def __delitem__(self, variant):
del self.modules[variant]
@staticmethod
def parse_uid(uid):
if not isinstance(uid, six.string_types):
raise ValueError("Uid has to be string: %s" % uid)
# pattern to parse uid MODULE_NAME:STREAM[:VERSION[:CONTEXT]]
UID_RE = re.compile(
r"^(.*/)?(?P<module_name>[^:]+):(?P<stream>[^:]+)(:(?P<version>[^:]+))?(:(?P<context>[^:]+))?$"
)
matched = UID_RE.match(uid)
if matched:
uid_dict = matched.groupdict()
else:
raise ValueError("Invalid uid: %s" % uid)
if uid_dict["version"] is None:
uid_dict["version"] = ""
if uid_dict["context"] is None:
uid_dict["context"] = ""
return uid_dict
def _check_uid(self, uid):
if not isinstance(uid, six.string_types):
raise ValueError("Uid has to be string: %s" % uid)
if ":" not in uid:
raise ValueError("Missing stream in uid: %s" % uid)
try:
uid_dict = self.parse_uid(uid)
except ValueError:
raise ValueError("Invalid uid format: %s" % uid)
uid = "%(module_name)s:%(stream)s" % uid_dict
uid += ":%s" % uid_dict['version'] if uid_dict['version'] else ""
uid += ":%s" % uid_dict['context'] if uid_dict['context'] else ""
return uid, uid_dict
def serialize(self, parser):
self.validate()
data = parser
self.header.serialize(data)
data["payload"] = {}
self.compose.serialize(data["payload"])
data["payload"]["modules"] = self.modules
return data
def deserialize(self, data):
self.header.deserialize(data)
self.compose.deserialize(data["payload"])
self.modules = data["payload"]["modules"]
self.validate()
def add(self, variant, arch, uid, koji_tag, modulemd_path, category, rpms):
if not variant:
raise ValueError("Non-empty variant is expected")
if arch not in RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
if category not in SUPPORTED_CATEGORIES:
raise ValueError("Invalid category value: %s" % category)
uid, uid_dict = self._check_uid(uid)
name = uid_dict["module_name"]
stream = uid_dict["stream"]
version = uid_dict["version"]
context = uid_dict["context"]
if modulemd_path.startswith("/"):
raise ValueError("Relative path expected: %s" % modulemd_path)
if not koji_tag:
raise ValueError("Non-empty 'koji_tag' is expected")
for param_name, param in {
"variant": variant,
"koji_tag": koji_tag,
"modulemd_path": modulemd_path
}.items():
if not param:
raise ValueError("Non-empty '%s' is expected" % param_name)
if not isinstance(rpms, (list, tuple)):
raise ValueError("Wrong type of 'rpms'")
arches = self.modules.setdefault(variant, {})
uids = arches.setdefault(arch, {})
metadata = uids.setdefault(uid, {})
metadata["metadata"] = {
"uid": uid,
"name": name,
"stream": stream,
"version": version,
"context": context,
"koji_tag": koji_tag,
}
metadata.setdefault("modulemd_path", {})[category] = modulemd_path
metadata.setdefault("rpms", []).extend(list(rpms))
class Rpms(productmd.common.MetadataBase):
def __init__(self):
super(Rpms, self).__init__()
self.header = Header(self, "productmd.rpms")
self.compose = Compose(self)
self.rpms = {}
def __getitem__(self, variant):
return self.rpms[variant]
def __delitem__(self, variant):
del self.rpms[variant]
def _check_nevra(self, nevra):
if ":" not in nevra:
raise ValueError("Missing epoch in N-E:V-R.A: %s" % nevra)
try:
nevra_dict = productmd.common.parse_nvra(nevra)
except ValueError:
raise ValueError("Invalid N-E:V-R.A: %s" % nevra)
nevra_dict["epoch"] = nevra_dict["epoch"] or 0
nevra = "%(name)s-%(epoch)s:%(version)s-%(release)s.%(arch)s" % nevra_dict
return nevra, nevra_dict
def serialize(self, parser):
data = parser
self.header.serialize(data)
data["payload"] = {}
data["payload"]["rpms"] = {}
self.compose.serialize(data["payload"])
data["payload"]["rpms"] = self.rpms
return data
def deserialize(self, data):
self.header.deserialize(data)
if self.header.version_tuple <= (0, 3):
self.deserialize_0_3(data)
else:
self.deserialize_1_0(data)
self.validate()
self.header.set_current_version()
def deserialize_0_3(self, data):
self.compose.deserialize(data["payload"])
payload = data["payload"]["manifest"]
self.rpms = {}
for variant in payload:
for arch in payload[variant]:
if arch == "src":
continue
for srpm_nevra, rpms in payload[variant][arch].items():
srpm_data = payload[variant].get("src",
{}).get(srpm_nevra, None)
for rpm_nevra, rpm_data in rpms.items():
category = rpm_data["type"]
if category == "package":
category = "binary"
self.add(variant, arch, rpm_nevra, rpm_data["path"],
rpm_data["sigkey"], category, srpm_nevra)
if srpm_data is not None:
self.add(variant, arch, srpm_nevra,
srpm_data["path"], srpm_data["sigkey"],
"source")
def deserialize_1_0(self, data):
self.compose.deserialize(data["payload"])
self.rpms = data["payload"]["rpms"]
def add(self,
variant,
arch,
nevra,
path,
sigkey,
category,
srpm_nevra=None):
"""
Map RPM to to variant and arch.
:param variant: compose variant UID
:type variant: str
:param arch: compose architecture
:type arch: str
:param nevra: name-epoch:version-release.arch
:type nevra: str
:param sigkey: sigkey hash
:type sigkey: str or None
:param category: RPM category, one of binary, debug, source
:type category: str
:param srpm_nevra: name-epoch:version-release.arch of RPM's SRPM
:type srpm_nevra: str
"""
if arch not in productmd.common.RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
if arch in ["src", "nosrc"]:
raise ValueError(
"Source arch is not allowed. Map source files under binary arches."
)
if category not in SUPPORTED_CATEGORIES:
raise ValueError("Invalid category value: %s" % category)
if path.startswith("/"):
raise ValueError("Relative path expected: %s" % path)
nevra, nevra_dict = self._check_nevra(nevra)
if category == "source" and srpm_nevra is not None:
raise ValueError(
"Expected blank srpm_nevra for source package: %s" % nevra)
if category != "source" and srpm_nevra is None:
raise ValueError("Missing srpm_nevra for package: %s" % nevra)
if (category == "source") != (nevra_dict["arch"] in ("src", "nosrc")):
raise ValueError("Invalid category/arch combination: %s/%s" %
(category, nevra))
if sigkey is not None:
sigkey = sigkey.lower()
if srpm_nevra:
srpm_nevra, _ = self._check_nevra(srpm_nevra)
else:
srpm_nevra = nevra
arches = self.rpms.setdefault(variant, {})
srpms = arches.setdefault(arch, {})
rpms = srpms.setdefault(srpm_nevra, {})
rpms[nevra] = {"sigkey": sigkey, "path": path, "category": category}
class Images(productmd.common.MetadataBase):
def __init__(self):
super(Images, self).__init__()
self.header = Header(self, "productmd.images")
self.compose = Compose(self)
self.images = {}
def __getitem__(self, variant):
return self.images[variant]
def __delitem__(self, variant):
del self.images[variant]
def serialize(self, parser):
data = parser
self.header.serialize(data)
data["payload"] = {}
data["payload"]["images"] = {}
self.compose.serialize(data["payload"])
for variant in self.images:
for arch in self.images[variant]:
for image_obj in self.images[variant][arch]:
images = data["payload"]["images"].setdefault(variant, {}).setdefault(arch, [])
image_obj.serialize(images)
images.sort(key=lambda x: x["path"])
return data
def deserialize(self, data):
self.header.deserialize(data)
self.compose.deserialize(data["payload"])
for variant in data["payload"]["images"]:
for arch in data["payload"]["images"][variant]:
for image in data["payload"]["images"][variant][arch]:
image_obj = Image(self)
image_obj.deserialize(image)
if self.header.version_tuple <= (1, 1):
self._add_1_1(data, variant, arch, image_obj)
else:
self.add(variant, arch, image_obj)
self.header.set_current_version()
def _add_1_1(self, data, variant, arch, image):
if arch == "src":
# move src under binary arches
for variant_arch in data["payload"]["images"][variant]:
if variant_arch == "src":
continue
self.add(variant, variant_arch, image)
else:
self.add(variant, arch, image)
def add(self, variant, arch, image):
"""
Assign an :class:`.Image` object to variant and arch.
:param variant: compose variant UID
:type variant: str
:param arch: compose architecture
:type arch: str
:param image: image
:type image: :class:`.Image`
"""
if arch not in productmd.common.RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
if arch in ["src", "nosrc"]:
raise ValueError("Source arch is not allowed. Map source files under binary arches.")
self.images.setdefault(variant, {}).setdefault(arch, set()).add(image)
class Rpms(productmd.common.MetadataBase):
def __init__(self):
super(Rpms, self).__init__()
self.header = Header(self)
self.compose = Compose(self)
self.rpms = {}
def __getitem__(self, variant):
return self.rpms[variant]
def __delitem__(self, variant):
del self.rpms[variant]
def _check_nevra(self, nevra):
if ":" not in nevra:
raise ValueError("Missing epoch in N-E:V-R.A: %s" % nevra)
try:
nevra_dict = productmd.common.parse_nvra(nevra)
except ValueError:
raise ValueError("Invalid N-E:V-R.A: %s" % nevra)
nevra_dict["epoch"] = nevra_dict["epoch"] or 0
nevra = "%(name)s-%(epoch)s:%(version)s-%(release)s.%(arch)s" % nevra_dict
return nevra, nevra_dict
def serialize(self, parser):
data = parser
self.header.serialize(data)
data["payload"] = {}
data["payload"]["rpms"] = {}
self.compose.serialize(data["payload"])
data["payload"]["rpms"] = self.rpms
return data
def deserialize(self, data):
self.header.deserialize(data)
if self.header.version_tuple <= (0, 3):
self.deserialize_0_3(data)
else:
self.deserialize_1_0(data)
self.validate()
self.header.set_current_version()
def deserialize_0_3(self, data):
self.compose.deserialize(data["payload"])
payload = data["payload"]["manifest"]
self.rpms = {}
for variant in payload:
for arch in payload[variant]:
if arch == "src":
continue
for srpm_nevra, rpms in payload[variant][arch].items():
srpm_data = payload[variant].get("src", {}).get(srpm_nevra, None)
for rpm_nevra, rpm_data in rpms.items():
category = rpm_data["type"]
if category == "package":
category = "binary"
self.add(variant, arch, rpm_nevra, rpm_data["path"], rpm_data["sigkey"], category, srpm_nevra)
if srpm_data is not None:
self.add(variant, arch, srpm_nevra, srpm_data["path"], srpm_data["sigkey"], "source")
def deserialize_1_0(self, data):
self.compose.deserialize(data["payload"])
self.rpms = data["payload"]["rpms"]
def add(self, variant, arch, nevra, path, sigkey, category, srpm_nevra=None):
"""
Map RPM to to variant and arch.
:param variant: compose variant UID
:type variant: str
:param arch: compose architecture
:type arch: str
:param nevra: name-epoch:version-release.arch
:type nevra: str
:param sigkey: sigkey hash
:type sigkey: str or None
:param category: RPM category, one of binary, debug, source
:type category: str
:param srpm_nevra: name-epoch:version-release.arch of RPM's SRPM
:type srpm_nevra: str
"""
if arch not in productmd.common.RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
if category not in SUPPORTED_CATEGORIES:
raise ValueError("Invalid category value: %s" % category)
if path.startswith("/"):
raise ValueError("Relative path expected: %s" % path)
nevra, nevra_dict = self._check_nevra(nevra)
if category == "source" and srpm_nevra is not None:
raise ValueError("Expected blank srpm_nevra for source package: %s" % nevra)
if category != "source" and srpm_nevra is None:
raise ValueError("Missing srpm_nevra for package: %s" % nevra)
if (category == "source") != (nevra_dict["arch"] in ("src", "nosrc")):
raise ValueError("Invalid category/arch combination: %s/%s" % (category, nevra))
if sigkey is not None:
sigkey = sigkey.lower()
if srpm_nevra:
srpm_nevra, _ = self._check_nevra(srpm_nevra)
else:
srpm_nevra = nevra
arches = self.rpms.setdefault(variant, {})
srpms = arches.setdefault(arch, {})
rpms = srpms.setdefault(srpm_nevra, {})
rpms[nevra] = {"sigkey": sigkey, "path": path, "category": category}
class ExtraFiles(productmd.common.MetadataBase):
def __init__(self):
super(ExtraFiles, self).__init__()
self.header = Header(self, "productmd.extra_files")
self.compose = Compose(self)
self.extra_files = {}
def __getitem__(self, variant):
return self.extra_files[variant]
def __delitem__(self, variant):
del self.extra_files[variant]
def serialize(self, parser):
self.validate()
data = parser
self.header.serialize(data)
data["payload"] = {}
self.compose.serialize(data["payload"])
data["payload"]["extra_files"] = self.extra_files
return data
def deserialize(self, data):
self.header.deserialize(data)
self.compose.deserialize(data["payload"])
self.extra_files = data["payload"]["extra_files"]
self.validate()
def add(self, variant, arch, path, size, checksums):
if not variant:
raise ValueError("Non-empty variant is expected")
if arch not in RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
if not path:
raise ValueError("Path can not be empty.")
if path.startswith("/"):
raise ValueError("Relative path expected: %s" % path)
if not isinstance(checksums, dict):
raise TypeError("Checksums must be a dict.")
metadata = self.extra_files.setdefault(variant,
{}).setdefault(arch, [])
metadata.append({"file": path, "size": size, "checksums": checksums})
def dump_for_tree(self, output, variant, arch, basepath):
"""Dump the serialized metadata for given tree. The basepath is
stripped from all paths.
"""
metadata = {"header": {"version": "1.0"}, "data": []}
for item in self.extra_files[variant][arch]:
metadata["data"].append({
"file":
_relative_to(item["file"], basepath),
"size":
item["size"],
"checksums":
item["checksums"],
})
json.dump(metadata,
output,
sort_keys=True,
indent=4,
separators=(",", ": "))
class Images(productmd.common.MetadataBase):
def __init__(self):
super(Images, self).__init__()
self.header = Header(self, "productmd.images")
self.compose = Compose(self)
self.images = {}
def __getitem__(self, variant):
return self.images[variant]
def __delitem__(self, variant):
del self.images[variant]
def serialize(self, parser):
data = parser
self.header.serialize(data)
data["payload"] = {}
data["payload"]["images"] = {}
self.compose.serialize(data["payload"])
for variant in self.images:
for arch in self.images[variant]:
for image_obj in self.images[variant][arch]:
images = data["payload"]["images"].setdefault(variant, {}).setdefault(arch, [])
image_obj.serialize(images)
images.sort(key=lambda x: x["path"])
return data
def deserialize(self, data):
self.header.deserialize(data)
self.compose.deserialize(data["payload"])
for variant in data["payload"]["images"]:
for arch in data["payload"]["images"][variant]:
for image in data["payload"]["images"][variant][arch]:
image_obj = Image(self)
image_obj.deserialize(image)
if self.header.version_tuple <= (1, 1):
self._add_1_1(data, variant, arch, image_obj)
else:
self.add(variant, arch, image_obj)
self.header.set_current_version()
def _add_1_1(self, data, variant, arch, image):
if arch == "src":
# move src under binary arches
for variant_arch in data["payload"]["images"][variant]:
if variant_arch == "src":
continue
self.add(variant, variant_arch, image)
else:
self.add(variant, arch, image)
def add(self, variant, arch, image):
"""
Assign an :class:`.Image` object to variant and arch.
:param variant: compose variant UID
:type variant: str
:param arch: compose architecture
:type arch: str
:param image: image
:type image: :class:`.Image`
"""
if arch not in productmd.common.RPM_ARCHES:
raise ValueError("Arch not found in RPM_ARCHES: %s" % arch)
if arch in ["src", "nosrc"]:
raise ValueError("Source arch is not allowed. Map source files under binary arches.")
if self.header.version_tuple >= (1, 1):
# disallow adding a different image with same 'unique'
# attributes. can't do this pre-1.1 as we couldn't truly
# identify images before subvariant
for checkvar in self.images:
for checkarch in self.images[checkvar]:
for curimg in self.images[checkvar][checkarch]:
if identify_image(curimg) == identify_image(image) and curimg.checksums != image.checksums:
raise ValueError("Image {0} shares all UNIQUE_IMAGE_ATTRIBUTES with "
"image {1}! This is forbidden.".format(image, curimg))
self.images.setdefault(variant, {}).setdefault(arch, set()).add(image)