def test_num_types():
xint = f_proxy(f_return(123))
xfloat = f_proxy(f_return(1.23))
assert float(xint) == 123.0
assert complex(xint) == complex(123)
assert int(xfloat) == 1
def _get_related_repository(self, repo_t):
if not self._client:
raise DetachedException()
suffixes_mapping = {
"binary": "/os",
"debug": "/debug",
"source": "/source/SRPMS",
}
regex = r"(/os|/source/SRPMS|/debug)$"
def unpack_page(page):
if len(page.data) != 1:
return None
return page.data[0]
suffix = suffixes_mapping[repo_t]
if str(self.relative_url).endswith(suffix):
return f_proxy(f_return(self))
base_url = re.sub(regex, "", self.relative_url)
relative_url = base_url + suffix
criteria = Criteria.with_field("notes.relative_url", relative_url)
page_f = self._client.search_repository(criteria)
repo_f = f_map(page_f, unpack_page)
return f_proxy(repo_f)
def fixture_ubi_repo_set_no_debug():
ubi_binary = get_test_repo(id="ubi-foo-rpms", ubi_config_version="7")
ubi_source = get_test_repo(id="ubi-foo-source", ubi_config_version="7")
ubi_binary.get_source_repository = MagicMock()
ubi_binary.get_source_repository.return_value = ubi_source
ubi_binary.get_debug_repository = MagicMock()
ubi_binary.get_debug_repository.return_value = f_proxy(f_return())
rhel_binary = get_test_repo(id="foo-rpms", ubi_config_version="7")
rhel_source = get_test_repo(id="foo-source", ubi_config_version="7")
rhel_binary.get_source_repository = MagicMock()
rhel_binary.get_source_repository.return_value = rhel_source
rhel_binary.get_debug_repository = MagicMock()
rhel_binary.get_debug_repository.return_value = f_proxy(f_return())
yield UbiRepoSet(
RepoSet(
[rhel_binary],
[rhel_source],
f_proxy(f_return()),
),
RepoSet(
ubi_binary,
ubi_source,
f_proxy(f_return()),
),
)
def test_publish_out_repos(mock_ubipop_runner):
dt = datetime(2019, 9, 12, 0, 0, 0)
d1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo",
last_publish=dt,
relative_url="content/unit/2/client",
)
repo = YumRepository(
id="repo",
eng_product_id=102,
distributors=[d1],
relative_url="content/unit/2/client",
)
fake_pulp = FakeController()
repo.__dict__["_client"] = fake_pulp.client
fake_pulp.insert_repository(repo)
# Setup output repos, leave only binary repo as the actual one
mock_ubipop_runner.repos.out_repos = RepoSet(
f_proxy(f_return(repo)), f_proxy(f_return()), f_proxy(f_return())
)
fts = mock_ubipop_runner._publish_out_repos()
# we should publish only one repository with one distributor
assert len(fts) == 1
assert [hist.repository.id for hist in fake_pulp.publish_history] == ["repo"]
def test_get_modular_srpms_criteria(ubi_config):
"""Testing creation of criteria for srpms query"""
matcher = ModularMatcher(None, ubi_config.modules)
unit_1 = UbiUnit(
RpmUnit(
name="test",
version="1.0",
release="1",
arch="x86_64",
sourcerpm="test.x86_64.src.rpm",
),
None,
)
unit_2 = UbiUnit(
RpmUnit(
name="test-debug",
version="1.0",
release="1",
arch="i386",
sourcerpm="test-debug.i386.src.rpm",
),
None,
)
# we need to set up binary and debug rpms
# the criteria are based on sourcerpm attr of those units
matcher.binary_rpms = f_proxy(f_return(set([unit_1])))
matcher.debug_rpms = f_proxy(f_return(set([unit_2])))
criteria = matcher._get_srpms_criteria()
# there should be 1 criteria created
assert len(criteria) == 2
# it should be instance of Criteria
for crit in criteria:
assert isinstance(crit, Criteria)
def test_bool():
xtrue = f_proxy(f_return(True))
xfalse = f_proxy(f_return(False))
xempty = f_proxy(f_return([]))
# All of these count as truthy, as we want to allow testing if
# a future was returned without blocking
assert xtrue
assert xfalse
assert xempty
def fixture_mock_current_content():
rpm = get_rpm_unit(
name="rpm_current",
filename="rpm_current.rpm",
version="1",
release="0",
arch="x86_64",
src_repo_id="ubi-foo-rpms",
)
srpm = get_srpm_unit(
name="srpm_current",
filename="srpm_current.src.rpm",
version="1",
release="0",
arch="x86_64",
src_repo_id="ubi-foo-source",
)
debug_rpm = get_rpm_unit(
name="debug_rpm_current",
filename="debug_rpm_current.rpm",
version="1",
release="0",
arch="x86_64",
src_repo_id="ubi-foo-debug",
)
modulemd_unit = get_modulemd_unit(
name="md_current",
stream="foo",
version=1,
context="bar",
arch="x86_64",
src_repo_id="ubi-foo-rpms",
)
modulemd_defaults_unit = get_modulemd_defaults_unit(
name="mdd_current",
stream="rhel",
profiles={"2.5": ["common"]},
repo_id="ubi-foo-rpms",
src_repo_id="ubi-foo-rpms",
)
binary_rpms = f_proxy(f_return([rpm]))
debug_rpms = f_proxy(f_return([srpm]))
source_rpms = f_proxy(f_return([debug_rpm]))
modulemds = f_proxy(f_return([modulemd_unit]))
modulemd_defaults = f_proxy(f_return([modulemd_defaults_unit]))
repo_content = RepoContent(
binary_rpms, debug_rpms, source_rpms, modulemds, modulemd_defaults
)
yield repo_content
def test_sign():
xint = f_proxy(f_return(123))
xnegint = f_proxy(f_return(-123))
assert -xint == -123
assert +xint == 123
assert abs(xnegint) == 123
assert ~xint == -124
assert ~xnegint == 122
assert ~~xint == 123
assert ~~xnegint == -123
def test_len():
xlist = f_proxy(f_return([1, 2, 3]))
xint = f_proxy(f_return(42))
xstr = f_proxy(f_return("hello"))
assert len(xlist) == 3
assert len(xstr) == 5
with pytest.raises(TypeError) as exc:
len(xint)
assert "has no len" in str(exc.value)
def get_maintenance_report(self):
if self._maintenance_report:
report = MaintenanceReport._from_data(
json.loads(self._maintenance_report))
else:
report = MaintenanceReport()
return f_proxy(f_return(report))
def remove_content(self, **kwargs):
"""Remove all content of requested types from this repository.
Args:
type_ids (list[str])
IDs of content type(s) to be removed.
See :meth:`~pubtools.pulplib.Client.get_content_type_ids`.
If omitted, content of all types will be removed.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when content has been removed.
The future contains a list of zero or more tasks triggered and awaited
during the removal.
To obtain information on the removed content, use
:meth:`~pubtools.pulplib.Task.units`.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 1.5.0
"""
if not self._client:
raise DetachedException()
# Note: we use dynamic kwargs because it's very likely that a future
# version of this method will support some "criteria". Let's not fix the
# argument order at least until then.
type_ids = kwargs.get("type_ids")
return f_proxy(self._client._do_unassociate(self.id, type_ids))
def _get_ubi_repo_sets(self, ubi_binary_cs):
"""
Searches for ubi repository triplet (binary rpm, srpm, debug) for
one ubi config item and tries to determine their population sources
(input repositories). Returns list UbiRepoSet objects that provides
input and output repositories that are used for population process.
"""
rpm_repos = self.pulp_client.search_repository(
Criteria.and_(
Criteria.with_field("notes.content_set", ubi_binary_cs),
Criteria.with_field("ubi_population", True),
))
ubi_repo_sets = []
for out_rpm_repo in rpm_repos:
out_source_repo = out_rpm_repo.get_source_repository()
out_debug_repo = out_rpm_repo.get_debug_repository()
in_rpm_repos = self._get_population_sources(out_rpm_repo)
in_source_repos = self._get_population_sources(out_source_repo)
in_debug_repos = self._get_population_sources(out_debug_repo)
# we need to apply f_proxy(f_return()) for out_rpm_repo for keeping consistency
# that all objects in out|in_repos are futures
out_repos = (
f_proxy(f_return(out_rpm_repo)),
out_source_repo,
out_debug_repo,
)
in_repos = (in_rpm_repos, in_source_repos, in_debug_repos)
ubi_repo_sets.append(
UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos)))
return ubi_repo_sets
def test_get_modular_rpms_criteria(ubi_config):
"""Test creation of criteria for rpms query"""
matcher = ModularMatcher(None, ubi_config.modules)
unit = UbiUnit(
ModulemdUnit(
name="test",
stream="10",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src",
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
],
),
None,
)
matcher.modules = f_proxy(f_return(set([unit])))
criteria = matcher._get_modular_rpms_criteria()
# there should be 1 criterium created - srpm is skipped
assert len(criteria) == 1
# it should be instance of Criteria
for crit in criteria:
assert isinstance(crit, Criteria)
def test_modular_rpms_filenames_per_profiles_missing_profile(ubi_config):
"""Test getting filename from module artifacts, request for non-existing profile in modulemd"""
matcher = ModularMatcher(None, ubi_config.modules)
unit = UbiUnit(
ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src",
"perl-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
"bash-7:10.5-el6.x86_64",
"bash-devel-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
],
profiles={"another": ["bash"]},
),
None,
)
modules = f_proxy(f_return(set([unit])))
filenames = matcher._modular_rpms_filenames(modules)
# all non-src pkgs are in result
# this result is driven by ubi_config that force to use only profile called "test"
# but we don't have this profile in the testing modulemd, so we take all non-src artifacts
assert len(filenames) == 3
assert filenames == set([
"bash-10.5-el6.x86_64.rpm",
"bash-devel-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm",
"perl-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm",
])
def test_modular_rpms_filenames_per_profiles(ubi_config):
"""Test getting filename from module artifacts, limited by profiles"""
matcher = ModularMatcher(None, ubi_config.modules)
unit = UbiUnit(
ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src",
"perl-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
"bash-7:10.5-el6.x86_64",
"bash-devel-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
],
profiles={
"test": ["perl", "bash"],
"another": ["bash"]
},
),
None,
)
modules = f_proxy(f_return(set([unit])))
filenames = matcher._modular_rpms_filenames(modules)
# only pkgs from test profile perl+bash should be in result
# this result is driven also by ubi_config that force to use only profile called "test"
assert len(filenames) == 2
assert filenames == set([
"bash-10.5-el6.x86_64.rpm",
"perl-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm",
])
def _handle_page(self, url, object_class, search, raw_data):
LOG.debug("Got pulp response for %s, %s elems", search, len(raw_data))
# Extract metadata from Pulp units
if object_class is Unit and url.endswith("units/"):
raw_data = [elem["metadata"] for elem in raw_data]
page_data = [object_class.from_data(elem) for elem in raw_data]
for obj in page_data:
# set_client is only applicable for repository and distributor objects
if hasattr(obj, "_set_client"):
obj._set_client(self)
# Do we need a next page?
next_page = None
limit = search["criteria"]["limit"]
if len(raw_data) == limit:
# Yeah, we might...
search = search.copy()
search["criteria"] = search["criteria"].copy()
search["criteria"]["skip"] = search["criteria"]["skip"] + limit
response_f = self._do_search(url, search)
next_page = f_proxy(
f_map(
response_f,
lambda data: self._handle_page(url, object_class, search,
data),
))
return Page(data=page_data, next=next_page)
def _search(self,
return_type,
resource_type,
criteria=None,
search_options=None):
pulp_crit = {
"skip": 0,
"limit": self._PAGE_SIZE,
"filters": filters_for_criteria(criteria, return_type),
}
type_ids = search_options.pop("type_ids",
None) if search_options else None
if type_ids:
pulp_crit["type_ids"] = type_ids
search = {"criteria": pulp_crit}
search.update(search_options or {})
url = os.path.join(self._url, "pulp/api/v2/%s/search/" % resource_type)
if return_type is Unit and search["criteria"]["type_ids"]:
url = os.path.join(url, "units/")
response_f = self._do_search(url, search)
# When this request is resolved, we'll have the first page of data.
# We'll need to convert that into a page and also keep going with
# the search if there's more to be done.
return f_proxy(
f_map(
response_f,
lambda data: self._handle_page(url, return_type, search, data),
))
def test_modular_rpms_filenames(ubi_config):
"""Test getting filename from module artifacts, srpms are skipped."""
matcher = ModularMatcher(None, ubi_config.modules)
unit = UbiUnit(
ModulemdUnit(
name="test",
stream="10",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src",
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
],
),
None,
)
modules = f_proxy(f_return(set([unit])))
filenames = matcher._modular_rpms_filenames(modules)
# there should be only 1 filename because srpms are skipped
assert len(filenames) == 1
assert (filenames.pop() ==
"perl-version-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm")
def get_content_type_ids(self):
"""Get the content types supported by this Pulp server.
Returns:
Future[list[str]]
A future holding the IDs of supported content types.
The returned values will depend on the plugins installed
on the connected Pulp server.
"modulemd", "rpm", "srpm" and "erratum" are some examples
of common return values.
.. versionadded:: 1.4.0
"""
url = os.path.join(self._url, "pulp/api/v2/plugins/types/")
out = self._request_executor.submit(self._do_request,
method="GET",
url=url)
# The pulp API returns an object per supported type.
# We only support returning the ID at this time.
return f_proxy(
f_map(out, lambda types: sorted([t["id"] for t in types])))
def sync(self, options=None):
"""Sync repository with feed.
Args:
options (SyncOptions)
Options used to customize the behavior of sync process.
If omitted, the Pulp server's defaults apply.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when sync succeeds.
The future contains a list of zero or more tasks triggered and awaited
during the sync operation.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 2.5.0
"""
options = options or SyncOptions(feed="")
if not self._client:
raise DetachedException()
return f_proxy(
self._client._do_sync(
self.id,
asdict(options, filter=lambda name, val: val is not None)))
def search_content(self, criteria=None):
"""Search for units across all repositories.
Args:
criteria (:class:`~pubtools.pulplib.Criteria`)
A criteria object used for this search.
If None, search for all units.
Returns:
Future[:class:`~pubtools.pulplib.Page`]
A future representing the first page of results.
Each page will contain a collection of
:class:`~pubtools.pulplib.Unit` subclasses objects.
.. versionadded:: 2.6.0
"""
# Criteria will be serialized into a Pulp search at the time we
# actually do the query, but validate eagerly as well so we raise
# ASAP on invalid input.
search_for_criteria(criteria, Unit, None)
if self._server_type_ids is None:
# We'll be using this in a moment either to set default IDs or to
# reject searches for invalid types.
# Note: no locking, so if we're called from multiple threads we in
# theory might waste some time querying the types more than once.
self._server_type_ids = self.get_content_type_ids()
return f_proxy(
f_flat_map(
self._server_type_ids,
lambda ids: self._search_content_with_server_type_ids(
criteria, ids),
))
def copy_content(self, from_repository, to_repository, criteria=None):
"""Copy content from one repository to another.
Args:
from_repository (:class:`~pubtools.pulplib.Repository`)
A repository used as the source of content.
to_repository (:class:`~pubtools.pulplib.Repository`)
A repository to receive copied content.
criteria (:class:`~pubtools.pulplib.Criteria`)
A criteria object used to find units for copy.
If None, all units in the source repo will be copied.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when the copy completes.
The :meth:`~pubtools.pulplib.Task.units` attribute may be inspected
to determine which unit(s) were copied. Note that the returned units
typically will have only a subset of available fields.
.. versionadded:: 2.17.0
"""
return f_proxy(
self._do_associate(from_repository.id, to_repository.id, criteria))
def remove_content(self, **kwargs):
"""Remove all content of requested types from this repository.
Args:
type_ids (list[str])
IDs of content type(s) to be removed.
See :meth:`~pubtools.pulplib.Client.get_content_type_ids`.
If omitted, content of all types will be removed.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when content has been removed.
The future contains a list of zero or more tasks triggered and awaited
during the removal.
To obtain information on the removed content, use
:meth:`~pubtools.pulplib.Task.units`.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 1.5.0
"""
if not self._client:
raise DetachedException()
# Note: we use dynamic kwargs because it's very likely that a future
# version of this method will support some "criteria". Let's not fix the
# argument order at least until then.
# start down the path of using Criteria per this issue:
# https://github.com/release-engineering/pubtools-pulplib/issues/62
# by using Criteria internally.
# there seems to be pretty complex handling of Criteria
# for serialization in the search API, and it is unclear which parts
# might also be necessary to use for content removal.
# If any or all of the same handling is needed, it would be beneficial
# to encapsulate the preparation of a criteria JSON object in some
# (more generically named) functions or a class to avoid duplicating code.
# for reference see search_content, _impl.client.Client._search_repo_units,
# _impl.client.Client._search, and _impl.client.search.search_for_criteria
criteria = None
type_ids = kwargs.get("type_ids")
# use _content_type_id field name to coerce
# search_for_criteria to fill out the PulpSearch#type_ids field.
# passing a criteria with an empty type_ids list rather than
# None results in failing tests due to the implementation of
# FakeClient#_do_unassociate
if type_ids is not None:
criteria = Criteria.with_field(
"_content_type_id",
Matcher.in_(type_ids), # Criteria.with_field_in is deprecated
)
return f_proxy(self._client._do_unassociate(self.id,
criteria=criteria))
def test_math_basic():
xstr = f_proxy(f_return("xyz"))
xint = f_proxy(f_return(12))
xfloat = f_proxy(f_return(5.0))
assert xstr * 2 == "xyzxyz"
assert xstr + "abc" == "xyzabc"
assert xint + 2 == 14
assert xint - 2 == 10
assert xint % 10 == 2
assert xint / 2 == 6
assert xint**2 == 144
assert xfloat // 2.0 == 2.0
assert divmod(xint, 5) == (2, 2)
def test_attrs():
xdict = f_proxy(f_return({"foo": "bar", "baz": "quux"}))
assert sorted(xdict.keys()) == ["baz", "foo"]
assert sorted(xdict.values()) == ["bar", "quux"]
with pytest.raises(AttributeError):
xdict.no_such_attr()
def _delete(self, resource_type, resource_id):
client = self._client
if not client:
raise DetachedException()
delete_f = client._delete_resource(resource_type, resource_id)
delete_f = f_map(delete_f, self.__detach)
return f_proxy(delete_f)
def test_bits():
xbin = f_proxy(f_return(0b00110))
assert xbin << 1 == 0b01100
assert xbin >> 1 == 0b00011
assert xbin & 0b10010 == 0b00010
assert xbin | 0b10010 == 0b10110
assert xbin ^ 0b10010 == 0b10100
def remove_content(self, criteria=None, **kwargs):
"""Remove all content of requested types from this repository.
Args:
criteria (:class:`~pubtools.pulplib.Criteria`)
A criteria object used to filter the contents for removal.
Type IDs must be included in the criteria with any other filters.
If omitted, filter criteria will be ignored and all the content will
be removed.
If criteria is omitted, all the content will be removed.
Returns:
Future[list[:class:`~pubtools.pulplib.Task`]]
A future which is resolved when content has been removed.
The future contains a list of zero or more tasks triggered and awaited
during the removal.
To obtain information on the removed content, use
:meth:`~pubtools.pulplib.Task.units`.
Raises:
DetachedException
If this instance is not attached to a Pulp client.
.. versionadded:: 1.5.0
"""
if not self._client:
raise DetachedException()
# Type IDs are must for the criteria filter to be effective. It must be included
# in the criteria. Type IDs provided as type_id kwargs will be ignored for the
# criteria and will remove all the content in the repo.
# Refer to https://bugzilla.redhat.com/show_bug.cgi?id=1021579 and Pulp
# documentation for more details.
# Note: type_ids is deprecated. Criteria.with_unit_type should be used to
# filter on type_ids. This is kept for backward compatibility and will be
# removed in future versions.
if not criteria:
type_ids = kwargs.get("type_ids")
# use content_type_id field name to coerce
# search_for_criteria to fill out the PulpSearch#type_ids field.
# passing a criteria with an empty type_ids list rather than
# None results in failing tests due to the implementation of
# FakeClient#_do_unassociate
if type_ids is not None:
warnings.warn(
"type_ids is deprecated, use criteria instead", DeprecationWarning
)
criteria = Criteria.with_field(
"content_type_id",
Matcher.in_(type_ids), # Criteria.with_field_in is deprecated
)
return f_proxy(self._client._do_unassociate(self.id, criteria=criteria))
def copy_content(
self, from_repository, to_repository, criteria=None, options=CopyOptions()
):
self._ensure_alive()
from_id = from_repository.id
to_id = to_repository.id
found = list(from_repository.search_content(criteria).result())
# RPM signature filter: if signatures are required, unsigned RPMs are not
# included in the copy.
# Because we don't model this flag on distributor objects and because in
# practice it's set to True, we default to True.
if options.require_signed_rpms is not False:
found = [u for u in found if not isinstance(u, RpmUnit) or u.signing_key]
# Units are being copied to this repo, so that value obviously must appear
# in repository_memberships from now on.
found = [attr.evolve(unit, repository_memberships=[to_id]) for unit in found]
with self._state.lock:
# Now put the found units into the destination repo.
# Any kind of merging or replacing of units is handled within this step.
self._state.insert_repo_units(to_id, found)
# Arbitrarily limit the number of units included per task. The point is
# to enforce that the caller doesn't expect any specific number of tasks.
tasks = []
while found:
next_batch = found[:5]
found = found[5:]
tasks.append(
Task(
id=self._state.next_task_id(),
repo_id=from_id,
completed=True,
succeeded=True,
units=units.with_key_only(next_batch),
)
)
if not tasks:
# This indicates that nothing was found at all.
# That's fine, just return a task with empty units.
tasks.append(
Task(
id=self._state.next_task_id(),
repo_id=from_id,
completed=True,
succeeded=True,
units=[],
)
)
return f_proxy(f_return(tasks))
def test_math_round():
xstr = f_proxy(f_return("xyz"))
xint = f_proxy(f_return(12))
xfloat = f_proxy(f_return(5.5))
assert round(xint) == 12
assert round(xfloat) == 6.0
assert floor(xint) == 12
assert floor(xfloat) == 5.0
assert ceil(xint) == 12
assert ceil(xfloat) == 6.0
assert trunc(xint) == 12
assert trunc(xfloat) == 5.0
with pytest.raises(Exception):
assert round(xstr) == "foo"