def ensure_product_versions_uptodate(self, ctx):
# Ensures that the product_versions field contains all the product
# versions from this cert, in all repos containing this productid as well
# as any repos sharing a certain relationship.
# First we need to figure out the repos to handle.
# We start from the repos we're contained in.
repo_ids = self.in_pulp_repos
repo_fs = [ctx.client.get_repository(repo_id) for repo_id in repo_ids]
# Then start looking up any 'related repos' for them.
# This finder class manages the searches and avoids duplicate searches.
finder = RepoFinder(client=ctx.client)
find_related_fs = [
f_map(repo_f, finder.find_related) for repo_f in repo_fs
]
# Once all the find_related searches have been set up, we can get the
# iterable over all repos.
repo_iter_f = f_map(f_sequence(find_related_fs),
lambda _: finder.all_results)
return f_flat_map(
repo_iter_f,
lambda repos: self.ensure_product_versions_uptodate_in_repos(
ctx, repos),
)
def clear_content(self, repos):
out = []
for repo in repos:
f = repo.remove_content(type_ids=self.content_type)
f = f_map(f, partial(ClearedRepo, repo=repo))
f = f_map(f, self.log_remove)
out.append(f)
return out
def test_map_error():
map_in = f_return(0)
# This one should fail...
mapped = f_map(map_in, div10)
# Now map it through an error handler
mapped = f_map(mapped, error_fn=str)
result = mapped.result()
assert "division" in result
def get_raw_f(self, advisory_id):
"""Returns Future[ErrataRaw] holding all ET responses for a particular advisory."""
all_responses = f_zip(
self._executor.submit(self._get_advisory_cdn_metadata,
advisory_id),
self._executor.submit(self._get_advisory_cdn_file_list,
advisory_id),
self._executor.submit(self._get_advisory_cdn_docker_file_list,
advisory_id),
self._executor.submit(self._get_ftp_paths, advisory_id),
)
all_responses = f_map(
all_responses,
partial(self._log_queried_et, advisory_id=advisory_id))
return f_map(all_responses, lambda tup: ErrataRaw(*tup))
def empty_future(value):
if "add_done_callback" in dir(value):
# It's a future, map it to None
return f_map(value, lambda _: None)
# It's not a future => operation has already completed,
# return empty future to denote success
return f_return()
def _get_related_repository(self, repo_t):
if not self._client:
raise DetachedException()
suffixes_mapping = {
"binary": "/os",
"debug": "/debug",
"source": "/source/SRPMS",
}
regex = r"(/os|/source/SRPMS|/debug)$"
def unpack_page(page):
if len(page.data) != 1:
return None
return page.data[0]
suffix = suffixes_mapping[repo_t]
if str(self.relative_url).endswith(suffix):
return f_proxy(f_return(self))
base_url = re.sub(regex, "", self.relative_url)
relative_url = base_url + suffix
criteria = Criteria.with_field("notes.relative_url", relative_url)
page_f = self._client.search_repository(criteria)
repo_f = f_map(page_f, unpack_page)
return f_proxy(repo_f)
def __iter__(self):
# Get file list of all advisories first.
#
# We get the file list first because it contains koji build NVRs,
# so by getting these first we can issue koji build queries and advisory
# metadata queries concurrently.
file_list_fs = [
self._executor.submit(self._get_advisory_file_list, advisory_id)
for advisory_id in self._advisory_ids
]
# Then get metadata.
metadata_fs = [
self._executor.submit(self._get_advisory_metadata, advisory_id)
for advisory_id in self._advisory_ids
]
# Zip up the metadata and file list for each advisory, since both are
# needed together in order to make push items.
advisory_fs = [
f_zip(metadata_f, file_list_f)
for (metadata_f, file_list_f) in zip(metadata_fs, file_list_fs)
]
# Convert them to lists of push items
advisory_push_items_fs = [
f_map(f, self._push_items_from_raw) for f in advisory_fs
]
completed_fs = futures.as_completed(advisory_push_items_fs,
timeout=self._timeout)
for f in completed_fs:
# If an exception occurred, this is where it will be raised.
for pushitem in f.result():
yield pushitem
def update_repository(self, repository):
"""Update mutable fields on an existing repository.
Args:
repository (:class:`~pubtools.pulplib.Repository`)
A repository to be updated.
Only those fields documented as *mutable* will have any effect during
the update (e.g. ``product_versions``). Other fields cannot be updated
using this library.
Returns:
Future
A future which is resolved with a value of ``None`` once the
repository has been updated.
.. versionadded:: 2.29.0
"""
url = os.path.join(self._url,
"pulp/api/v2/repositories/%s/" % repository.id)
update = {"delta": {"notes": repository._mutable_notes}}
out = self._request_executor.submit(self._do_request,
method="PUT",
url=url,
json=update)
# The Pulp API may actually return an updated version of the repository,
# but for consistency with update_content we won't return it.
# The caller can re-fetch if desired.
out = f_map(out, lambda _: None)
return out
def _handle_page(self, url, object_class, search, raw_data):
LOG.debug("Got pulp response for %s, %s elems", search, len(raw_data))
# Extract metadata from Pulp units
if object_class is Unit and url.endswith("units/"):
raw_data = [elem["metadata"] for elem in raw_data]
page_data = [object_class.from_data(elem) for elem in raw_data]
for obj in page_data:
# set_client is only applicable for repository and distributor objects
if hasattr(obj, "_set_client"):
obj._set_client(self)
# Do we need a next page?
next_page = None
limit = search["criteria"]["limit"]
if len(raw_data) == limit:
# Yeah, we might...
search = search.copy()
search["criteria"] = search["criteria"].copy()
search["criteria"]["skip"] = search["criteria"]["skip"] + limit
response_f = self._do_search(url, search)
next_page = f_proxy(
f_map(
response_f,
lambda data: self._handle_page(url, object_class, search,
data),
))
return Page(data=page_data, next=next_page)
def get_content_type_ids(self):
"""Get the content types supported by this Pulp server.
Returns:
Future[list[str]]
A future holding the IDs of supported content types.
The returned values will depend on the plugins installed
on the connected Pulp server.
"modulemd", "rpm", "srpm" and "erratum" are some examples
of common return values.
.. versionadded:: 1.4.0
"""
url = os.path.join(self._url, "pulp/api/v2/plugins/types/")
out = self._request_executor.submit(self._do_request,
method="GET",
url=url)
# The pulp API returns an object per supported type.
# We only support returning the ID at this time.
return f_proxy(
f_map(out, lambda types: sorted([t["id"] for t in types])))
def delete(self):
"""Delete this repository from Pulp.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when the repository deletion has completed.
The future contains a list of zero or more tasks triggered and awaited
during the delete operation.
This object also becomes detached from the client; no further updates
are possible.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
"""
if not self._client:
raise DetachedException()
delete_f = self._client._delete_resource("repositories", self.id)
def detach(tasks):
LOG.debug("Detaching %s after successful delete", self)
self.__dict__["_client"] = None
return tasks
return f_map(delete_f, detach)
def _search(self,
return_type,
resource_type,
criteria=None,
search_options=None):
pulp_crit = {
"skip": 0,
"limit": self._PAGE_SIZE,
"filters": filters_for_criteria(criteria, return_type),
}
type_ids = search_options.pop("type_ids",
None) if search_options else None
if type_ids:
pulp_crit["type_ids"] = type_ids
search = {"criteria": pulp_crit}
search.update(search_options or {})
url = os.path.join(self._url, "pulp/api/v2/%s/search/" % resource_type)
if return_type is Unit and search["criteria"]["type_ids"]:
url = os.path.join(url, "units/")
response_f = self._do_search(url, search)
# When this request is resolved, we'll have the first page of data.
# We'll need to convert that into a page and also keep going with
# the search if there's more to be done.
return f_proxy(
f_map(
response_f,
lambda data: self._handle_page(url, return_type, search, data),
))
def search_repository(self, criteria=None):
"""Search for repositories matching the given criteria.
Args:
criteria (:class:`~pubtools.pulplib.Criteria`)
A criteria object used for this search.
If None, search for all repositories.
Returns:
Future[:class:`~pubtools.pulplib.Page`]
A future representing the first page of results.
Each page will contain a collection of
:class:`~pubtools.pulplib.Repository` objects.
"""
pulp_crit = {
"skip": 0,
"limit": self._PAGE_SIZE,
"filters": filters_for_criteria(criteria, Repository),
}
search = {"criteria": pulp_crit, "distributors": True}
response_f = self._do_search("repositories", search)
# When this request is resolved, we'll have the first page of data.
# We'll need to convert that into a page and also keep going with
# the search if there's more to be done.
return f_map(response_f,
lambda data: self._handle_page(Repository, search, data))
def compile_ud_mappings(repo, do_request):
"""Perform the UD mappings note compilation & update process for a given repo.
Arguments:
repo (~pulplib.FileRepository)
A repository.
do_request (callable)
A function which can be invoked to perform an HTTP request to Pulp.
Returns:
A Future, resolved when the update completes successfully.
"""
LOG.debug("%s: compiling %s", repo.id, UD_MAPPINGS_NOTE)
# 1. Get current mappings.
#
# Requires a fresh retrieval of the repo since we don't store
# these mappings on our model.
#
repo_url = "pulp/api/v2/repositories/%s/" % repo.id
repo_raw_f = do_request(repo_url, method="GET")
mappings_f = f_map(
repo_raw_f, lambda data:
(data.get("notes") or {}).get(UD_MAPPINGS_NOTE) or "{}")
# Mappings are stored as JSON, so decode them
mappings_f = f_map(mappings_f, json.loads)
# Wrap them in our helper for keeping track of changes
mappings_f = f_map(mappings_f, MappingsHelper)
# 2. Iterate over all files in the repo
files_f = repo.search_content(Criteria.with_unit_type(FileUnit))
# 3. Mutate the mappings as needed for each file
updated_mappings_f = f_flat_map(
f_zip(mappings_f, files_f),
lambda tup: update_mappings_for_files(tup[0], tup[1]),
)
# 4. Upload them back if any changes
handle_changes = functools.partial(upload_changed_mappings,
repo=repo,
repo_url=repo_url,
do_request=do_request)
return f_flat_map(updated_mappings_f, handle_changes)
def _delete(self, resource_type, resource_id):
client = self._client
if not client:
raise DetachedException()
delete_f = client._delete_resource(resource_type, resource_id)
delete_f = f_map(delete_f, self.__detach)
return f_proxy(delete_f)
def do_next_publish(accumulated_tasks, distributor, config):
distributor_tasks_f = self._publish_distributor(
repo.id, distributor.id, config)
return f_map(
distributor_tasks_f,
lambda distributor_tasks: accumulated_tasks +
distributor_tasks,
)
def delete_rpms(self, repos, rpms_info, signing_keys=None):
# get rpms from Pulp
rpms_f = self.get_rpms(rpms_info, signing_keys)
# map rpms to repos
f = f_map(rpms_f, partial(self.map_to_repo, repos=repos, unit_attr="filename"))
repo_map_f = f_map(f, self.log_units)
# remove rpms from repos
cleared_repos_f = f_map(repo_map_f, self.remove_rpms)
# collect items
f = f_map(cleared_repos_f, self.record_clears)
self.to_await.extend(f.result())
# return affected repos
return cleared_repos_f
def delete_files(self, repos, file_names):
# get files from Pulp
files_f = self.get_files(file_names)
# map files to repos
f = f_map(files_f, partial(self.map_to_repo, repos=repos, unit_attr="path"))
repo_map_f = f_map(f, self.log_units)
# remove from repos
cleared_repos_f = f_map(repo_map_f, self.remove_files)
# collect items
f = f_map(cleared_repos_f, self.record_clears)
self.to_await.extend(f.result())
# return affected repos
return cleared_repos_f
def _rpm_futures(self):
# Get info from each requested RPM.
rpm_fs = [(self._executor.submit(self._get_rpm, rpm), rpm) for rpm in self._rpm]
# Convert them to lists of push items
return [
f_map(f, partial(self._push_items_from_rpm_meta, rpm)) for f, rpm in rpm_fs
]
def delete_modules(self, repos, module_names, signing_keys=None):
# get modules from Pulp
mods_f = self.get_modules(module_names)
# map modules to repos
f = f_map(mods_f, partial(self.map_to_repo, repos=repos, unit_attr="nsvca"))
repo_map_f = f_map(f, self.log_units)
# remove packages from modules
unit_map_f = f_map(f, lambda map: map[1])
artifact_f = f_map(
unit_map_f, partial(self.remove_mod_artifacts, signing_keys=signing_keys)
)
# hold for rpms in the artifacts to be removed before removing modules
# wait for rpms_fs to resolve for f_zip to resolve and return repo_map_f
# to remove_modules in next step
repo_map_f = f_map(
f_zip(repo_map_f, f_sequence(artifact_f.result())), lambda t: t[0]
)
# remove modules
cleared_repos_f = f_map(repo_map_f, self.remove_modules)
# collect items
f = f_map(cleared_repos_f, self.record_clears)
self.to_await.extend(f.result())
# return affected repos
return cleared_repos_f
def delete_content(self, unit_type, repo_map, criteria_fn):
if not repo_map:
LOG.warning("Nothing mapped for removal")
return []
out = []
# get repos
repos = self.search_repo(repo_map.keys())
# request removal
for repo in sorted(repos):
units = repo_map.get(repo.id)
criteria = self.unit_criteria(unit_type, criteria_fn(units))
f = repo.remove_content(criteria=criteria)
f = f_map(f, partial(ClearedRepo, repo=repo, content=units))
f = f_map(f, self.log_remove)
out.append(f)
return out
def _container_futures(self):
# Get info from each requested container build.
build_fs = [(self._executor.submit(self._get_build, build), build)
for build in self._container_build]
# Convert them to lists of push items
return [
f_map(f, partial(self._push_items_from_container_build, build))
for f, build in build_fs
]
def purge_repo(repo):
to_flush = []
for url in repo.mutable_urls:
flush_url = os.path.join(self.fastpurge_root_url,
repo.relative_url, url)
to_flush.append(flush_url)
LOG.debug("Flush: %s", to_flush)
flush = self.fastpurge_client.purge_by_url(to_flush)
return f_map(flush, lambda _: repo)
def publish(self, options=PublishOptions()):
"""Publish this repository.
The specific operations triggered on Pulp in order to publish a repo are not defined,
but in Pulp 2.x, generally consists of triggering one or more distributors in sequence.
Args:
options (PublishOptions)
Options used to customize the behavior of this publish.
If omitted, the Pulp server's defaults apply.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when publish succeeds.
The future contains a list of zero or more tasks triggered and awaited
during the publish operation.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
"""
if not self._client:
raise DetachedException()
# all distributor IDs we're willing to invoke. Anything else is ignored.
# They'll be invoked in the order listed here.
candidate_distributor_ids = self._PUBLISH_DISTRIBUTORS
to_publish = []
for candidate in candidate_distributor_ids:
distributor = self._distributors_by_id.get(candidate)
if not distributor:
# nothing to be done
continue
if (distributor.id == "docker_web_distributor_name_cli"
and options.origin_only):
continue
config = self._config_for_distributor(distributor, options)
to_publish.append((distributor, config))
out = self._client._publish_repository(self, to_publish)
def do_published_hook(tasks):
# Whenever we've published successfully, we'll activate this hook
# before returning.
pm.hook.pulp_repository_published(repository=self, options=options)
return tasks
out = f_map(out, do_published_hook)
return f_proxy(out)
def _modulemd_futures(self):
# Get info from each requested module build.
module_build_fs = [(self._executor.submit(self._get_build,
build), build)
for build in self._module_build]
# Convert them to lists of push items
return [
f_map(f, partial(self._push_items_from_module_build, build))
for f, build in module_build_fs
]
def _search(
self,
return_type,
resource_types,
search_type="search",
search_options=None,
criteria=None,
): # pylint:disable = too-many-arguments
if not isinstance(resource_types, (list, tuple)):
resource_types = [resource_types]
responses = []
searches = []
urls = []
for resource_type in resource_types:
url = os.path.join(
self._url, "pulp/api/v2/%s/%s/" % (resource_type, search_type))
urls.append(url)
prepared_search = search_for_criteria(criteria, return_type)
search = {
"criteria": {
"skip": 0,
"limit": self._PAGE_SIZE,
"filters": prepared_search.filters,
}
}
search.update(search_options or {})
if search_type == "search/units":
# Unit searches need a little special handling:
# - serialization might have extracted some type_ids
# - filters should be wrapped under 'unit'
# (we do not support searching on associations right now)
if prepared_search.type_ids:
search["criteria"]["type_ids"] = prepared_search.type_ids
search["criteria"]["filters"] = {
"unit": search["criteria"]["filters"]
}
searches.append(search)
responses.append(self._do_search(url, search))
# When this request is resolved, we'll have the first page of data.
# We'll need to convert that into a page and also keep going with
# the search if there's more to be done.
return f_proxy(
f_map(
f_sequence(responses),
lambda data: self._handle_page(urls, return_type, searches,
data),
))
def put_future_outputs(self, values, task_done=True):
"""Like put_future_output, but works with a list of values rather
than a single value.
It is more efficient to call this function with a list of size N
than to call put_future_output N times.
"""
f = f_map(
values,
partial(self.__future_output_done, task_done=task_done),
error_fn=self.__future_output_failed,
)
self.__add_future_puts(f)
def get_files(self, file_names):
criteria = self.unit_criteria(FileUnit, self._file_search_criteria(file_names))
files_f = self.search_content(criteria)
files_f = f_map(
files_f,
partial(
self.log_missing_units,
unit_names=file_names,
unit_type="file",
unit_attr="path",
),
)
return files_f
def get_modules(self, module_names):
criteria = self.unit_criteria(
ModulemdUnit, self._module_search_criteria(module_names)
)
mods_f = self.search_content(criteria)
mods_f = f_map(
mods_f,
partial(
self.log_missing_units,
unit_names=module_names,
unit_type="module",
unit_attr="nsvca",
),
)
return mods_f
def _flush_object(self, object_type, object_id):
url = os.path.join(self._url, "internal/rcm/flush-cache", object_type,
str(object_id))
LOG.info("Invalidating %s %s", object_type, object_id)
# This is pretty odd, but yes, an HTTP *GET* here is used to flush cache.
out = self._executor.submit(self._get, url)
# Wrap with logging on failure
out = f_map(
out,
error_fn=lambda ex: self._on_failure(object_type, object_id, ex))
return out
