def digest_specs_converter(specs):
# Do the generic handling
out = specs_converter(specs, ContainerImageDigestPullSpec)
# Then also sort them by preferred media types
out = sorted(out, key=lambda spec: -MEDIA_TYPE_ORDER.get(spec.media_type, 0))
return frozenlist(out)
def test_cannot_hash_with_mutable_elements():
frozen = frozenlist(["a", "b", "c", {"d": "e"}])
try:
hash(frozen)
raise AssertionError("Was expected to raise!")
except TypeError:
# expected behavior
pass
def sloppylist(value, elem_converter=None):
"""Accept real lists or comma-separated values, and output a frozen list.
Optionally use elem_converter to convert each list element.
"""
if isinstance(value, six.string_types):
value = value.split(",")
if elem_converter:
value = [elem_converter(elem) for elem in value]
return frozenlist(value)
def freeze(node):
"""
Converts dict to frozendict and list to frozenlist.
"""
if isinstance(node, list):
# Iterating using index instead of enumeration
# so we can replace the list items in place
for index in range(len(node)): # pylint: disable=consider-using-enumerate
node[index] = freeze(node[index])
return frozenlist(node)
if isinstance(node, dict):
for key, value in node.items():
node[key] = freeze(value)
return frozendict(node)
return node
def specs_converter(specs, expected_class):
# a converter for pull specs, ensures every spec is an
# instance of the expected class while de-duplicating
out = []
out_strs = set()
for spec in specs:
if not isinstance(spec, expected_class):
raise TypeError("Expected %s, got: %s" % (expected_class, repr(spec)))
# de-duplicate specs
if str(spec) not in out_strs:
out_strs.add(str(spec))
out.append(spec)
return frozenlist(out)
def test_no_remove():
x = frozenlist([1, 2, 3])
with raises(NotImplementedError):
x.remove(0)
def test_no_append():
x = frozenlist()
with raises(NotImplementedError):
x.append(1)
def test_no_pop():
x = frozenlist([1, 2, 3])
with raises(NotImplementedError):
x.pop()
def test_no_extend():
x = frozenlist()
with raises(NotImplementedError):
x.extend([1, 2, 3])
def test_no_sort():
x = frozenlist([7, 5, 3])
with raises(NotImplementedError):
x.sort()
def test_no_insert():
x = frozenlist()
with raises(NotImplementedError):
x.insert(0, 123)
def test_no_iadd():
x = frozenlist()
with raises(NotImplementedError):
x += [1]
def test_can_hash():
frozen = frozenlist(["a", "b", "c"])
assert hash(frozen)
class Task(PulpObject):
"""Represents a Pulp task."""
_SCHEMA = load_schema("task")
id = pulp_attrib(type=str, pulp_field="task_id")
"""ID of this task (str)."""
completed = pulp_attrib(default=None, type=bool)
"""True if this task has completed, successfully or otherwise.
May be `None` if the state of this task is unknown.
"""
succeeded = pulp_attrib(default=None, type=bool)
"""True if this task has completed successfully.
May be `None` if the state of this task is unknown.
"""
error_summary = pulp_attrib(default=None, type=str)
"""A summary of the reason for this task's failure (if any).
This is a short string, generally a single line, suitable for display to users.
The string includes the ID of the failed task.
"""
error_details = pulp_attrib(default=None, type=str)
"""Detailed information for this task's failure (if any).
This may be a multi-line string and may include technical information such as
a Python backtrace generated by Pulp.
``error_details`` is a superset of the information available via ``error_summary``,
so it is not necessary to display both.
"""
tags = pulp_attrib(
default=attr.Factory(frozenlist),
type=list,
converter=frozenlist,
pulp_field="tags",
)
"""The tags for this task.
Typically includes info on the task's associated action and
repo, such as:
.. code-block:: python
["pulp:repository:rhel-7-server-rpms__7Server_x86_64",
"pulp:action:publish"]
"""
# TODO: is it a bug that this only allows a single repo ID??
# Some tasks, like copy, involve multiple repos. We'll only include
# the first repo ID from tags here... seems arbitrary.
repo_id = pulp_attrib(type=str)
"""The ID of the repository associated with this task, otherwise None."""
units = pulp_attrib(
default=attr.Factory(frozenlist),
type=list,
pulp_field="result.units_successful",
converter=frozenlist,
pulp_py_converter=lambda raw: frozenlist(
[Unit._from_task_data(x) for x in raw]),
)
"""Info on the units which were processed as part of this task
(e.g. associated or unassociated).
This is an iterable. Each element is an instance of
:class:`~pubtools.pulplib.Unit` containing information on a processed
unit.
.. versionadded:: 1.5.0
"""
units_data = pulp_attrib(
default=attr.Factory(frozenlist),
type=list,
converter=frozenlist,
pulp_field="result.units_successful",
)
"""Info on the units which were processed as part of this task
(e.g. associated or unassociated).
This is a list. The list elements are the raw dicts as returned
by Pulp. These should at least contain a 'type_id' and a 'unit_key'.
.. deprecated:: 1.5.0
Use :meth:`~pubtools.pulplib.Task.units` instead.
"""
@repo_id.default
def _repo_id_default(self):
prefix = "pulp:repository:"
for tag in self.tags or []:
if tag.startswith(prefix):
return tag[len(prefix):]
return None
@succeeded.validator
def _check_succeeded(self, _, value):
if value and not self.completed:
raise ValueError(
"Cannot have task with completed=False, succeeded=True")
@classmethod
def _data_to_init_args(cls, data):
out = super(Task, cls)._data_to_init_args(data)
state = data["state"]
out["completed"] = state in ("finished", "error", "canceled",
"skipped")
out["succeeded"] = state in ("finished", "skipped")
if state == "canceled":
out["error_summary"] = "Pulp task [%s] was canceled" % data[
"task_id"]
out["error_details"] = out["error_summary"]
elif state == "error":
out["error_summary"] = cls._error_summary(data)
out["error_details"] = cls._error_details(data)
return out
@classmethod
def _error_summary(cls, data):
prefix = "Pulp task [%s] failed" % data["task_id"]
error = data.get("error")
if not error:
return "%s: <unknown error>" % prefix
return "%s: %s: %s" % (prefix, error["code"], error["description"])
@classmethod
def _error_details(cls, data):
out = cls._error_summary(data)
error = data.get("error")
if not error:
return out
# Error looks like this:
#
# {
# 'code': u'PLP0001',
# 'data': {
# 'message': 'a message'
# },
# 'description': 'A general pulp exception occurred',
# 'sub_errors': []
# }
#
# See: https://docs.pulpproject.org/en/2.9/dev-guide/conventions/exceptions.html#error-details
#
# data can contain anything, or nothing.
# It's only a convention that it often contains a message.
#
# sub_errors is currently ignored because I've never seen a non-empty
# sub_errors yet.
error_data = error.get("data") or {}
messages = []
# Message in a general exception
if error_data.get("message"):
messages.append(error_data["message"])
# Some exceptions stash additional strings under details.errors
if (error_data.get("details") or {}).get("errors"):
error_messages = error_data["details"]["errors"]
if isinstance(error_messages, list):
messages.extend(error_messages)
# Pulp docs refer to this as deprecated, but actually it's still
# used and no alternative is provided.
if data.get("traceback"):
messages.append(data["traceback"])
message = "\n".join(messages)
if message:
# message can have CRLF line endings in rare cases.
message = message.replace("\r\n", "\n").strip()
out = "%s:\n%s" % (out, _indent(message))
return out
class FileRepository(Repository):
"""A :class:`~pubtools.pulplib.Repository` for generic file distribution."""
# this class only overrides some defaults for attributes defined in super
type = pulp_attrib(default="iso-repo",
type=str,
pulp_field="notes._repo-type")
is_sigstore = attr.ib(
default=attr.Factory(lambda self: self.id == "redhat-sigstore",
takes_self=True),
type=bool,
validator=validators.instance_of(bool),
)
mutable_urls = attr.ib(
default=attr.Factory(lambda: frozenlist(["PULP_MANIFEST"])),
type=list,
converter=frozenlist,
)
def upload_file(self, file_obj, relative_url=None, **kwargs):
"""Upload a file to this repository.
Args:
file_obj (str, file object)
If it's a string, then it's the path of a file to upload.
Otherwise, it should be a
`file-like object <https://docs.python.org/3/glossary.html#term-file-object>`_
pointing at the bytes to upload. The client takes ownership
of this file object; it should not be modified elsewhere,
and will be closed when upload completes.
relative_url (str)
Path that should be used in remote repository, can either
be a path to a directory or a path to a file, e.g:
- if relative_url is 'foo/bar/' and file_obj has name 'f.txt',
the resulting remote path wll be 'foo/bar/f.txt'.
- if relative_url is 'foo/bar/f.txt', no matter what the
name of file_obj is, the remote path is 'foo/bar/f.txt'.
If omitted, the local name of the file will be used. Or,
if file_obj is a file object without a `name` attribute,
passing `relative_url` is mandatory.
kwargs
Additional field values to set on the uploaded unit.
Any :class:`~pubtools.pulplib.FileUnit` fields documented as
*mutable* may be included here (for example, ``cdn_path``).
An error will occur if attempting to set other fields.
Returns:
Future[list of :class:`~pubtools.pulplib.Task`]
A future which is resolved after content has been imported
to this repo.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 1.2.0
.. versionadded:: 2.20.0
Added ability to set mutable fields on upload.
"""
relative_url = self._get_relative_url(file_obj, relative_url)
unit_key_fn = lambda upload: {
"name": relative_url,
"checksum": upload[0],
"size": upload[1],
}
unit_metadata_fn = None
usermeta = FileUnit._usermeta_from_kwargs(**kwargs)
if usermeta:
unit_metadata_fn = lambda _: usermeta
return self._upload_then_import(file_obj, relative_url, "iso",
unit_key_fn, unit_metadata_fn)
def _get_relative_url(self, file_obj, relative_url):
is_file_object = "close" in dir(file_obj)
if not is_file_object:
name = os.path.basename(file_obj)
if not relative_url:
relative_url = name
elif relative_url.endswith("/"):
relative_url = os.path.join(relative_url, name)
elif is_file_object and (not relative_url
or relative_url.endswith("/")):
msg = "%s is missing a name attribute and relative_url was not provided"
raise ValueError(msg % file_obj)
return relative_url
class YumRepository(Repository):
"""A :class:`~pubtools.pulplib.Repository` for RPMs, errata and related content."""
# this class only overrides some defaults for attributes defined in super
type = pulp_attrib(default="rpm-repo",
type=str,
pulp_field="notes._repo-type")
population_sources = pulp_attrib(
default=attr.Factory(frozenlist),
type=list,
converter=frozenlist,
pulp_field="notes.population_sources",
)
"""List of repository IDs used to populate this repository
"""
ubi_population = pulp_attrib(default=False,
type=bool,
pulp_field="notes.ubi_population")
"""Flag indicating whether repo should be populated from population_sources for the purposes of UBI
"""
mutable_urls = attr.ib(
default=attr.Factory(lambda: frozenlist(["repodata/repomd.xml"])),
type=list,
converter=frozenlist,
)
ubi_config_version = pulp_attrib(default=None,
type=str,
pulp_field="notes.ubi_config_version")
"""Version of UBI config that should be used for population of this repository."""
def get_binary_repository(self):
"""Find and return the binary repository relating to this repository.
Yum repositories usually come in triplets of
(binary RPMs, debuginfo RPMs, source RPMs). For example:
.. list-table::
:widths: 75 25
* - ``rhel-7-server-rpms__7Server__x86_64``
- binary
* - ``rhel-7-server-debug-rpms__7Server__x86_64``
- debug
* - ``rhel-7-server-source-rpms__7Server__x86_64``
- source
This method along with :meth:`get_debug_repository` and :meth:`get_source_repository` allow locating other repositories
from within this group.
Returns:
``Future[YumRepository]``
Binary repository relating to this repository.
``Future[None]``
If there is no related repository.
"""
return self._get_related_repository(repo_t="binary")
def get_debug_repository(self):
"""Find and return the debug repository relating to this repository.
Returns:
``Future[YumRepository]``
Debug repository relating to this repository.
``Future[None]``
If there is no related repository.
"""
return self._get_related_repository(repo_t="debug")
def get_source_repository(self):
"""Find and return the source repository relating to this repository.
Returns:
``Future[YumRepository]``
Source repository relating to this repository.
``Future[None]``
If there is no related repository.
"""
return self._get_related_repository(repo_t="source")
def _get_related_repository(self, repo_t):
if not self._client:
raise DetachedException()
suffixes_mapping = {
"binary": "/os",
"debug": "/debug",
"source": "/source/SRPMS",
}
regex = r"(/os|/source/SRPMS|/debug)$"
def unpack_page(page):
if len(page.data) != 1:
return None
return page.data[0]
suffix = suffixes_mapping[repo_t]
if str(self.relative_url).endswith(suffix):
return f_proxy(f_return(self))
base_url = re.sub(regex, "", self.relative_url)
relative_url = base_url + suffix
criteria = Criteria.with_field("notes.relative_url", relative_url)
page_f = self._client.search_repository(criteria)
repo_f = f_map(page_f, unpack_page)
return f_proxy(repo_f)
def upload_rpm(self, file_obj, **kwargs):
"""Upload an RPM to this repository.
.. warning::
For RPMs belonging to a module, it's strongly advised to upload
the module metadata first (using :meth:`upload_modules`) and only
proceed with uploading RPMs once module upload has completed.
This reduces the risk of accidentally publishing a repository with
modular RPMs without the corresponding metadata (which has a much
worse impact than publishing metadata without the corresponding RPMs).
Args:
file_obj (str, file object)
If it's a string, then it's the path of an RPM to upload.
Otherwise, it should be a
`file-like object <https://docs.python.org/3/glossary.html#term-file-object>`_
pointing at the bytes to upload.
The client takes ownership of this file object; it should
not be modified elsewhere, and will be closed when upload
completes.
kwargs
Additional field values to set on the uploaded unit.
Any :class:`~pubtools.pulplib.RpmUnit` fields documented as
*mutable* may be included here (for example, ``cdn_path``).
An error will occur if attempting to set other fields.
Returns:
Future[list of :class:`~pubtools.pulplib.Task`]
A future which is resolved after content has been imported
to this repo.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 2.16.0
.. versionadded:: 2.20.0
Added ability to set mutable fields on upload.
"""
# We want some name of what we're uploading for logging purposes, but the
# input could be a plain string, or a file object with 'name' attribute, or
# a file object without 'name' ... make sure we do something reasonable in
# all cases.
if isinstance(file_obj, six.string_types):
name = file_obj
else:
# If we don't know what we're uploading we just say it's "an RPM"...
name = getattr(file_obj, "name", "an RPM")
unit_metadata_fn = None
usermeta = RpmUnit._usermeta_from_kwargs(**kwargs)
if usermeta:
unit_metadata_fn = lambda _: usermeta
return self._upload_then_import(file_obj,
name,
"rpm",
unit_metadata_fn=unit_metadata_fn)
def upload_metadata(self, file_obj, metadata_type):
"""Upload a metadata file to this repository.
A metadata file is any additional file which will be published alongside,
and referenced from, the repodata ``.xml`` and ``.sqlite`` files when this
repo is published.
Args:
file_obj (str, file object)
If it's a string, then it's the path of a file to upload.
Otherwise, it should be a
`file-like object <https://docs.python.org/3/glossary.html#term-file-object>`_
pointing at the bytes to upload.
The client takes ownership of this file object; it should
not be modified elsewhere, and will be closed when upload
completes.
metadata_type (str)
Identifies the type of metadata being uploaded.
This is an arbitrary string which will be reproduced in the yum
repo metadata on publish. The appropriate value depends on the
type of data being uploaded. For example, ``"productid"`` should
be used when uploading an RHSM-style product certificate.
A repository may only contain a single metadata file of each type.
If a file of this type is already present in the repo, it will be
overwritten by the upload.
Returns:
Future[list of :class:`~pubtools.pulplib.Task`]
A future which is resolved after content has been imported
to this repo.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 2.17.0
"""
if isinstance(file_obj, six.string_types):
name = "%s (%s)" % (file_obj, metadata_type)
else:
# If we don't know what we're uploading we just say "<type> metadata"...
name = getattr(file_obj, "name", "%s metadata" % metadata_type)
return self._upload_then_import(
file_obj,
name,
"yum_repo_metadata_file",
# Requirements around unit key and metadata can be found at:
# https://github.com/pulp/pulp_rpm/blob/5c5a7dcc058b29d89b3a913d29cfcab41db96686/plugins/pulp_rpm/plugins/importers/yum/upload.py#L246
unit_key_fn=lambda _: {
"data_type": metadata_type,
"repo_id": self.id
},
unit_metadata_fn=lambda upload: {
"checksum": upload[0],
"checksum_type": "sha256",
},
)
def upload_modules(self, file_obj):
"""Upload a modulemd stream to this repository.
All supported documents in the given stream will be imported to this
repository. On current versions of Pulp 2.x, this means only:
* `modulemd v2 <https://github.com/fedora-modularity/libmodulemd/blob/main/yaml_specs/modulemd_stream_v2.yaml>`_
* `modulemd-defaults v1 <https://github.com/fedora-modularity/libmodulemd/blob/main/yaml_specs/modulemd_defaults_v1.yaml>`_
Attempting to use other document types may result in an error.
Args:
file_obj (str, file object)
If it's a string, then it's the path of a modulemd YAML
file to upload.
Otherwise, it should be a
`file-like object <https://docs.python.org/3/glossary.html#term-file-object>`_
pointing at the text to upload.
The client takes ownership of this file object; it should
not be modified elsewhere, and will be closed when upload
completes.
Returns:
Future[list of :class:`~pubtools.pulplib.Task`]
A future which is resolved after content has been imported
to this repo.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 2.17.0
"""
if isinstance(file_obj, six.string_types):
name = file_obj
else:
name = getattr(file_obj, "name", "modulemds")
return self._upload_then_import(file_obj, name, "modulemd")
def upload_comps_xml(self, file_obj):
"""Upload a comps.xml file to this repository.
.. warning::
Beware of the following quirks with respect to the upload of comps.xml:
* Pulp does not directly store the uploaded XML. Instead, this library
parses the XML and uses the content to store various units. The comps
XML rendered as a yum repository is published is therefore not
guaranteed to be bytewise-identical to the uploaded content.
* The uploaded XML must contain all comps data for the repo, as
any existing comps data will be removed from the repo.
* The XML parser is not secure against maliciously constructed data.
* The process of parsing the XML and storing units consists of multiple
steps which cannot be executed atomically. That means *if this
operation is interrupted, the repository may be left with incomplete
data*. It's recommended to avoid publishing a repository in this state.
Args:
file_obj (str, file object)
If it's a string, then it's the path of a comps XML
file to upload.
Otherwise, it should be a
`file-like object <https://docs.python.org/3/glossary.html#term-file-object>`_
pointing at the bytes of a valid comps.xml file.
The client takes ownership of this file object; it should
not be modified elsewhere, and will be closed when upload
completes.
Returns:
Future[list of :class:`~pubtools.pulplib.Task`]
A future which is resolved after content has been imported
to this repo.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 2.17.0
"""
if isinstance(file_obj, six.string_types):
file_name = file_obj
file_obj = open(file_obj, "rb")
else:
file_name = getattr(file_obj, "name", "comps.xml")
# Parse the provided XML. We will crash here if the given XML is not
# valid.
with file_obj:
unit_dicts = comps.units_for_xml(file_obj)
# Every comps-related unit type has a repo_id which should reference the repo
# we're uploading to.
for unit in unit_dicts:
unit["repo_id"] = self.id
comps_type_ids = [
"package_group",
"package_category",
"package_environment",
"package_langpacks",
]
# Remove former units of comps-related types so that the end result is only
# those units included in the current XML.
out = self.remove_content(type_ids=comps_type_ids)
# Once removal is done we can upload each unit.
upload_f = []
for unit_dict in unit_dicts:
type_id = unit_dict["_content_type_id"]
# For one comps.xml we are doing multiple upload operations, each of
# which would be logged independently. Come up with some reasonable name
# for each unit to put into the logs.
#
# Example: if uploading my-comps.xml and processing a package_group
# with id kde-desktop-environment, the name for logging purposes would
# be: "my-comps.xml [group.kde-desktop-environment]".
#
unit_name = type_id.replace("package_", "")
if unit_dict.get("id"):
unit_name = "%s.%s" % (unit_name, unit_dict["id"])
unit_name = "%s [%s]" % (file_name, unit_name)
upload_f.append(
f_flat_map(
out,
self._comps_unit_uploader(unit_name, type_id, unit_dict)))
# If there were no units to upload then just return the removal.
if not upload_f:
return out
# There were uploads, then we'll wait for all of them to complete and
# return the tasks for all.
out = f_zip(*upload_f)
out = f_map(out, lambda uploads: sum(uploads, []))
return out
def _comps_unit_uploader(self, name, type_id, metadata):
# A helper used from upload_comps_xml.
#
# This helper only exists to eagerly bind arguments for a single
# unit upload, due to confusing behavior around variable scope
# when combining loops and lambdas.
def upload(_unused):
return self._upload_then_import(
file_obj=None,
name=name,
type_id=type_id,
unit_metadata_fn=lambda _: metadata,
)
return upload
def upload_erratum(self, erratum):
"""Upload an erratum/advisory object to this repository.
.. warning::
There are many quirks with respect to advisory upload. Please be aware
of the following before using this API:
* Only one advisory with a given ``id`` may exist in the system.
* When uploading an advisory with an ``id`` equal to one already in the
system, the upload will generally be ignored (i.e. complete successfully
but have no effect), unless either the ``version`` or ``updated`` fields
have a value larger than the existing advisory.
This implies that, if you want to ensure an existing advisory is updated,
you must first search for the existing object and mutate one of these
fields before uploading a modified object. *The library will not take
care of this for you.*
* When overwriting an existing advisory, all fields will be overwritten.
The sole exception is the ``pkglist`` field which will be merged with
existing data when applicable.
* If an advisory with the same ``id`` is present in multiple published yum
repositories with inconsistent fields, yum/dnf client errors or warnings
may occur. It's therefore recommended that, whenever an existing
advisory is modified, every repository containing that advisory should
be republished. *The library will not take care of this for you.*
* The ``repository_memberships`` field on the provided object has no effect
(it cannot be used to upload an advisory to multiple repos at once).
Args:
erratum (:class:`~pubtools.pulplib.ErratumUnit`)
An erratum object.
Unlike most other uploaded content, errata are not backed by any
file; any arbitrarily constructed ErratumUnit may be uploaded.
Returns:
Future[list of :class:`~pubtools.pulplib.Task`]
A future which is resolved after content has been imported
to this repo.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 2.17.0
"""
# Convert from ErratumUnit to a raw Pulp-style dict, recursively.
erratum_dict = erratum._to_data()
# Drop this one field because the _content_type_id, though embedded
# in unit dicts on read, is passed as a separate parameter on write.
type_id = erratum_dict.pop("_content_type_id")
# Drop this because the caller cannot influence the _id (unit id)
# for uploaded units.
del erratum_dict["_id"]
# And drop this one because repository_memberships is synthesized when
# Pulp renders units, and can't be set during import.
del erratum_dict["repository_memberships"]
return self._upload_then_import(
file_obj=None,
name=erratum_dict["id"],
type_id=type_id,
unit_key_fn=lambda _: {"id": erratum_dict["id"]},
unit_metadata_fn=lambda _: erratum_dict,
)
def _default_media_types(self):
return frozenlist(
[spec.media_type for spec in self.digest_specs if spec.media_type]
)
def test_equals_list():
frozen = frozenlist(["a", "b", "c"])
assert frozen == ["a", "b", "c"]
assert frozen != ["a", "b", "d"]
def test_repr_list():
frozen = frozenlist(["a", "b", "c"])
assert repr(frozen) == repr(["a", "b", "c"])
class Repository(PulpObject, Deletable):
"""Represents a Pulp repository."""
_SCHEMA = load_schema("repository")
# The distributors (by ID) which should be activated when publishing this repo.
# Order matters. Distributors which don't exist will be ignored.
_PUBLISH_DISTRIBUTORS = [
"iso_distributor",
"yum_distributor",
"cdn_distributor",
"cdn_distributor_unprotected",
"docker_web_distributor_name_cli",
]
id = pulp_attrib(type=str, pulp_field="id")
"""ID of this repository (str)."""
type = pulp_attrib(default=None, type=str, pulp_field="notes._repo-type")
"""Type of this repository (str).
This is a brief string denoting the content / Pulp plugin type used with
this repository, e.g. ``rpm-repo``.
"""
created = pulp_attrib(default=None,
type=datetime.datetime,
pulp_field="notes.created")
""":class:`~datetime.datetime` in UTC at which this repository was created,
or None if this information is unavailable.
"""
distributors = pulp_attrib(
default=attr.Factory(frozenlist),
type=list,
pulp_field="distributors",
converter=frozenlist,
pulp_py_converter=lambda ds: frozenlist(
[Distributor.from_data(d) for d in ds]),
# It's too noisy to let repr descend into sub-objects
repr=False,
)
"""list of :class:`~pubtools.pulplib.Distributor` objects belonging to this
repository.
"""
eng_product_id = pulp_attrib(
default=None,
type=int,
pulp_field="notes.eng_product",
pulp_py_converter=int,
py_pulp_converter=str,
)
"""ID of the product to which this repository belongs (if any)."""
relative_url = pulp_attrib(default=None, type=str)
"""Default publish URL for this repository, relative to the Pulp content root."""
mutable_urls = pulp_attrib(default=attr.Factory(frozenlist),
type=list,
converter=frozenlist)
"""A list of URLs relative to repository publish root which are expected
to change at every publish (if any content of repo changed)."""
is_sigstore = pulp_attrib(default=False, type=bool)
"""True if this is a sigstore repository, used for container image manifest
signatures.
.. deprecated:: 2.24.0
The signatures are not stored in a Pulp repository any more.
"""
is_temporary = pulp_attrib(
default=False,
type=bool,
validator=validators.instance_of(bool),
pulp_field="notes.pub_temp_repo",
)
"""True if this is a temporary repository.
A temporary repository is a repository created by release-engineering tools
for temporary use during certain workflows. Such repos are not expected to
be published externally and generally should have a lifetime of a few days
or less.
.. versionadded:: 1.3.0
"""
signing_keys = pulp_attrib(
default=attr.Factory(frozenlist),
type=list,
pulp_field="notes.signatures",
pulp_py_converter=lambda sigs: sigs.split(","),
py_pulp_converter=",".join,
converter=lambda keys: frozenlist([k.strip() for k in keys]),
)
"""A list of GPG signing key IDs used to sign content in this repository."""
skip_rsync_repodata = pulp_attrib(default=False, type=bool)
"""True if this repository is explicitly configured such that a publish of
this repository will not publish repository metadata to remote hosts.
"""
content_set = pulp_attrib(default=None,
type=str,
pulp_field="notes.content_set")
"""Name of content set that is associated with this repository."""
@distributors.validator
def _check_repo_id(self, _, value):
# checks if distributor's repository id is same as the repository it
# is attached to
for distributor in value:
if not distributor.repo_id:
return
if distributor.repo_id == self.id:
return
raise ValueError(
"repo_id doesn't match for %s. repo_id: %s, distributor.repo_id: %s"
% (distributor.id, self.id, distributor.repo_id))
@property
def _distributors_by_id(self):
out = {}
for dist in self.distributors:
out[dist.id] = dist
return out
def distributor(self, distributor_id):
"""Look up a distributor by ID.
Returns:
:class:`~pubtools.pulplib.Distributor`
The distributor belonging to this repository with the given ID.
None
If this repository has no distributor with the given ID.
"""
return self._distributors_by_id.get(distributor_id)
@property
def file_content(self):
"""A list of file units stored in this repository.
Returns:
list[:class:`~pubtools.pulplib.FileUnit`]
.. versionadded:: 2.4.0
"""
return list(
self.search_content(Criteria.with_field("content_type_id", "iso")))
@property
def rpm_content(self):
"""A list of rpm units stored in this repository.
Returns:
list[:class:`~pubtools.pulplib.RpmUnit`]
.. versionadded:: 2.4.0
"""
return list(
self.search_content(Criteria.with_field("content_type_id", "rpm")))
@property
def srpm_content(self):
"""A list of srpm units stored in this repository.
Returns:
list[:class:`~pubtools.pulplib.Unit`]
.. versionadded:: 2.4.0
"""
return list(
self.search_content(Criteria.with_field("content_type_id",
"srpm")))
@property
def modulemd_content(self):
"""A list of modulemd units stored in this repository.
Returns:
list[:class:`~pubtools.pulplib.ModulemdUnit`]
.. versionadded:: 2.4.0
"""
return list(
self.search_content(
Criteria.with_field("content_type_id", "modulemd")))
@property
def modulemd_defaults_content(self):
"""A list of modulemd_defaults units stored in this repository.
Returns:
list[:class:`~pubtools.pulplib.ModulemdDefaultsUnit`]
.. versionadded:: 2.4.0
"""
return list(
self.search_content(
Criteria.with_field("content_type_id", "modulemd_defaults")))
def search_content(self, criteria=None):
"""Search this repository for content matching the given criteria.
Args:
criteria (:class:`~pubtools.pulplib.Criteria`)
A criteria object used for this search.
Returns:
Future[:class:`~pubtools.pulplib.Page`]
A future representing the first page of results.
Each page will contain a collection of
:class:`~pubtools.pulplib.Unit` objects.
.. versionadded:: 2.4.0
"""
if not self._client:
raise DetachedException()
return self._client._search_repo_units(self.id, criteria)
def delete(self):
"""Delete this repository from Pulp.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when the repository deletion has completed.
The future contains a list of zero or more tasks triggered and awaited
during the delete operation.
This object also becomes detached from the client; no further updates
are possible.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
"""
return self._delete("repositories", self.id)
def publish(self, options=PublishOptions()):
"""Publish this repository.
The specific operations triggered on Pulp in order to publish a repo are not defined,
but in Pulp 2.x, generally consists of triggering one or more distributors in sequence.
Args:
options (PublishOptions)
Options used to customize the behavior of this publish.
If omitted, the Pulp server's defaults apply.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when publish succeeds.
The future contains a list of zero or more tasks triggered and awaited
during the publish operation.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
"""
if not self._client:
raise DetachedException()
# Before adding distributors and publishing, we'll activate this hook
# to allow subscribing implementers the opportunity to adjust options.
hook_rets = pm.hook.pulp_repository_pre_publish(repository=self,
options=options)
# Use the first non-None hook return value to replace options.
hook_rets = [ret for ret in hook_rets if ret is not None]
options = hook_rets[0] if hook_rets else options
# All distributor IDs we're willing to invoke. Anything else is ignored.
# They'll be invoked in the order listed here.
candidate_distributor_ids = self._PUBLISH_DISTRIBUTORS
to_publish = []
for candidate in candidate_distributor_ids:
distributor = self._distributors_by_id.get(candidate)
if not distributor:
# nothing to be done
continue
if (distributor.id == "docker_web_distributor_name_cli"
and options.origin_only):
continue
config = self._config_for_distributor(distributor, options)
to_publish.append((distributor, config))
out = self._client._publish_repository(self, to_publish)
def do_published_hook(tasks):
# Whenever we've published successfully, we'll activate this hook
# before returning.
pm.hook.pulp_repository_published(repository=self, options=options)
return tasks
out = f_map(out, do_published_hook)
return f_proxy(out)
def sync(self, options=None):
"""Sync repository with feed.
Args:
options (SyncOptions)
Options used to customize the behavior of sync process.
If omitted, the Pulp server's defaults apply.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when sync succeeds.
The future contains a list of zero or more tasks triggered and awaited
during the sync operation.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 2.5.0
"""
options = options or SyncOptions(feed="")
if not self._client:
raise DetachedException()
return f_proxy(
self._client._do_sync(
self.id,
asdict(options, filter=lambda name, val: val is not None)))
def remove_content(self, **kwargs):
"""Remove all content of requested types from this repository.
Args:
type_ids (list[str])
IDs of content type(s) to be removed.
See :meth:`~pubtools.pulplib.Client.get_content_type_ids`.
If omitted, content of all types will be removed.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when content has been removed.
The future contains a list of zero or more tasks triggered and awaited
during the removal.
To obtain information on the removed content, use
:meth:`~pubtools.pulplib.Task.units`.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 1.5.0
"""
if not self._client:
raise DetachedException()
# Note: we use dynamic kwargs because it's very likely that a future
# version of this method will support some "criteria". Let's not fix the
# argument order at least until then.
# start down the path of using Criteria per this issue:
# https://github.com/release-engineering/pubtools-pulplib/issues/62
# by using Criteria internally.
# there seems to be pretty complex handling of Criteria
# for serialization in the search API, and it is unclear which parts
# might also be necessary to use for content removal.
# If any or all of the same handling is needed, it would be beneficial
# to encapsulate the preparation of a criteria JSON object in some
# (more generically named) functions or a class to avoid duplicating code.
# for reference see search_content, _impl.client.Client._search_repo_units,
# _impl.client.Client._search, and _impl.client.search.search_for_criteria
criteria = None
type_ids = kwargs.get("type_ids")
# use _content_type_id field name to coerce
# search_for_criteria to fill out the PulpSearch#type_ids field.
# passing a criteria with an empty type_ids list rather than
# None results in failing tests due to the implementation of
# FakeClient#_do_unassociate
if type_ids is not None:
criteria = Criteria.with_field(
"_content_type_id",
Matcher.in_(type_ids), # Criteria.with_field_in is deprecated
)
return f_proxy(self._client._do_unassociate(self.id,
criteria=criteria))
@classmethod
def from_data(cls, data):
# delegate to concrete subclass as needed
if cls is Repository:
notes = data.get("notes") or {}
for notes_type, klass in REPO_CLASSES.items():
if notes.get("_repo-type") == notes_type:
return klass.from_data(data)
return super(Repository, cls).from_data(data)
@classmethod
def _data_to_init_args(cls, data):
out = super(Repository, cls)._data_to_init_args(data)
for dist in data.get("distributors") or []:
if dist["distributor_type_id"] in ("yum_distributor",
"iso_distributor"):
out["relative_url"] = (dist.get("config")
or {}).get("relative_url")
if dist["id"] == "cdn_distributor":
skip_repodata = (dist.get("config") or {}).get("skip_repodata")
if skip_repodata is not None:
out["skip_rsync_repodata"] = skip_repodata
return out
@classmethod
def _config_for_distributor(cls, distributor, options):
out = {}
if distributor.is_rsync:
if options.clean is not None:
out["delete"] = options.clean
if options.origin_only is not None:
out["content_units_only"] = options.origin_only
if options.rsync_extra_args is not None:
out["rsync_extra_args"] = options.rsync_extra_args
if options.force is not None:
out["force_full"] = options.force
return out
def _set_client(self, client):
super(Repository, self)._set_client(client)
# distributors use the same client as owning repository
for distributor in self.distributors or []:
distributor._set_client(client)
def _upload_then_import(self,
file_obj,
name,
type_id,
unit_key_fn=None,
unit_metadata_fn=None):
"""Private helper to upload and import a piece of content into this repo.
To be called by the type-specific subclasses (e.g. YumRepository,
FileRepository...)
Args:
file_obj (str, file-like object, None):
file object or path (as documented in public methods), or None
if this unit type has no associated file
name (str):
a brief user-meaningful name for the content being uploaded
(appears in logs)
type_id (str):
pulp unit type ID
unit_key_fn (callable):
a callable which will be invoked with the return value of
_do_upload_file (or None if file_obj is None).
It should return the unit key for this piece of
content. If omitted, an empty unit key is used, which means Pulp
is wholly responsible for calculating the unit key.
unit_metadata_fn (callable):
a callable which will be invoked with the return value of
_do_upload_file (or None if file_obj is None). It should return
the unit metadata for this piece of
content. If omitted, metadata is not included in the import call to
Pulp.
"""
if not self._client:
raise DetachedException()
unit_key_fn = unit_key_fn or (lambda _: {})
unit_metadata_fn = unit_metadata_fn or (lambda _: None)
upload_id_f = f_map(self._client._request_upload(name),
lambda upload: upload["upload_id"])
f_map(
upload_id_f,
lambda upload_id: LOG.info("Uploading %s to %s [%s]", name, self.
id, upload_id),
)
if file_obj is None:
# If there is no file for this kind of unit (e.g. erratum),
# we still have to use the request_upload and import APIs; we just
# never upload any bytes. That means the upload is 'complete' as
# soon as the upload ID is known. A real upload returns a (size, checksum)
# tuple; we force a no-content upload to return None.
upload_complete_f = f_map(upload_id_f, lambda _: None)
else:
upload_complete_f = f_flat_map(
upload_id_f,
lambda upload_id: self._client._do_upload_file(
upload_id, file_obj, name),
)
import_complete_f = f_flat_map(
upload_complete_f,
lambda upload: self._client._do_import(
self.id,
upload_id_f.result(),
type_id,
unit_key_fn(upload),
unit_metadata_fn(upload),
),
)
f_map(
import_complete_f,
lambda _: self._client._delete_upload_request(
upload_id_f.result(), name),
)
return f_proxy(import_complete_f)
def test_no_setitem():
x = frozenlist([0])
with raises(NotImplementedError):
x[0] = 1234
def frozenlist_or_none_converter(obj, map_fn=(lambda x: x)):
if obj is not None:
return frozenlist(map_fn(obj))
return None
def test_no_delitem():
x = frozenlist([0])
with raises(NotImplementedError):
del x[0]
