def test_field_or_criteria():
"""or is translated to a mongo fragment as expected."""
c1 = Criteria.with_field("f1", "v1")
c2 = Criteria.with_field("f2", "v2")
assert filters_for_criteria(Criteria.or_(c1, c2)) == {
"$or": [{
"f1": {
"$eq": "v1"
}
}, {
"f2": {
"$eq": "v2"
}
}]
}
def test_search_content_type_id_in_or(client, requests_mocker):
"""Searching with a content_type_id within $or fails as unsupported"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
crit = Criteria.or_(
Criteria.with_field("name", "hello.txt"),
Criteria.with_field_in("content_type_id", ["rpm", "iso"]),
)
with pytest.raises(ValueError) as e:
repo.search_content(crit).result()
assert "Can't serialize criteria for Pulp query; too complicated" in str(
e.value)
def test_mixed_search(client, requests_mocker):
"""Searching with a criteria mixing several fields works correctly"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/",
json=[{
"metadata": {
"_content_type_id": "iso",
"name": "hello.txt",
"size": 23,
"checksum": "a" * 64,
}
}],
)
crit = Criteria.and_(
Criteria.with_field_in("content_type_id", ["rpm", "iso"]),
Criteria.with_field("name", "hello.txt"),
)
files = list(repo.search_content(crit))
assert files == [FileUnit(path="hello.txt", size=23, sha256sum="a" * 64)]
history = requests_mocker.request_history
# There should have been just one request
assert len(history) == 1
request = history[0]
body = request.json()
# This should have been the request body.
assert body == {
"criteria": {
"type_ids": ["rpm", "iso"],
"skip": 0,
"limit": 2000,
"filters": {
"unit": {
"name": {
"$eq": "hello.txt"
}
}
},
}
}
def test_search_mapped_field_less_than():
controller = FakeController()
dist1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1",
last_publish=datetime.datetime(2019, 8, 23, 2, 5, 0, tzinfo=None),
)
dist2 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1",
last_publish=datetime.datetime(2019, 8, 27, 2, 5, 0, tzinfo=None),
)
repo1 = Repository(id="repo1", distributors=(dist1, dist2))
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.with_field(
"last_publish",
Matcher.less_than(datetime.datetime(2019, 8, 24, 0, 0, 0)))
found = client.search_distributor(crit).result().data
assert found == [dist1]
def test_search_content_invalid_content_type(populated_units, controller):
"""search_content_by_type with invalid content type"""
with pytest.raises(PulpException):
for x in controller.client.search_content(
Criteria.with_field("content_type_id", "invalid")
).result():
pass
def _get_current_content(self):
"""
Gather current content of output repos
"""
criteria = [Criteria.true()]
current_modulemds = f_proxy(
self._executor.submit(Matcher.search_modulemds, criteria,
[self.repos.out_repos.rpm]))
current_modulemd_defaults = f_proxy(
self._executor.submit(Matcher.search_modulemd_defaults, criteria,
[self.repos.out_repos.rpm]))
current_rpms = f_proxy(
self._executor.submit(Matcher.search_rpms, criteria,
[self.repos.out_repos.rpm]))
current_srpms = f_proxy(
self._executor.submit(Matcher.search_srpms, criteria,
[self.repos.out_repos.source]))
if self.repos.out_repos.debug.result():
current_debug_rpms = f_proxy(
self._executor.submit(Matcher.search_rpms, criteria,
[self.repos.out_repos.debug]))
else:
current_debug_rpms = f_proxy(f_return([]))
current_content = RepoContent(
current_rpms,
current_srpms,
current_debug_rpms,
current_modulemds,
current_modulemd_defaults,
)
return current_content
def test_search_content_unsupported_operator(populated_repo):
"""search_content using unsupported operators on content_type_id raises"""
with pytest.raises(ValueError) as e:
populated_repo.search_content(
Criteria.with_field("content_type_id", Matcher.regex("foobar")))
assert "unsupported expression for content_type_id" in str(e.value)
def test_no_update_on_same_cves(command_tester):
with FakeFixCves() as fake_fix_cves:
fake_pulp = fake_fix_cves.pulp_client_controller
client = fake_pulp.client
_setup_controller(fake_pulp)
command_tester.test(
fake_fix_cves.main,
[
"test-fix-cves",
"--pulp-url",
"https://pulp.example.com",
"--advisory",
"RHSA-1234:56",
"--cves",
"CVE-123",
],
)
erratum = list(
client.search_content(Criteria.with_field("id", "RHSA-1234:56")))
assert len(erratum) == 1
erratum = erratum[0]
assert erratum.id == "RHSA-1234:56"
# no updates. erratum version not bumped.
assert erratum.version == "2"
cves = sorted([ref for ref in erratum.references if ref.type == "cve"])
# same CVE exists
assert len(cves) == 1
assert cves[0].id == "CVE-123"
def test_fix_cves(command_tester):
with FakeFixCves() as fake_fix_cves:
fake_pulp = fake_fix_cves.pulp_client_controller
client = fake_pulp.client
_setup_controller(fake_pulp)
command_tester.test(
fake_fix_cves.main,
[
"test-fix-cves",
"--pulp-url",
"https://pulp.example.com",
"--advisory",
"RHSA-1234:56",
"--cves",
"CVE-987,CVE-456",
],
)
updated_erratum = list(
client.search_content(Criteria.with_field("id", "RHSA-1234:56")))
# only single erratum exists on Pulp
assert len(updated_erratum) == 1
updated_erratum = updated_erratum[0]
assert updated_erratum.id == "RHSA-1234:56"
# erratum version bumped
assert updated_erratum.version == "3"
cves = sorted(
[ref for ref in updated_erratum.references if ref.type == "cve"])
# only new CVEs in the erratum
assert len(cves) == 2
assert cves[0].id == "CVE-456"
assert cves[1].id == "CVE-987"
def test_search_content_by_unit_type(populated_repo):
"""search_content on unit_type returns only units of that type"""
crit = Criteria.with_unit_type(ModulemdUnit)
units = list(populated_repo.search_content(crit))
assert sorted(units) == [
ModulemdUnit(
unit_id="23a7711a-8133-2876-37eb-dcd9e87a1613",
name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64",
repository_memberships=["repo1"],
),
ModulemdUnit(
unit_id="e6f4590b-9a16-4106-cf6a-659eb4862b21",
name="module2",
stream="s2",
version=1234,
context="a1b2",
arch="x86_64",
repository_memberships=["repo1"],
),
]
def search_content(self, criteria=None):
self._ensure_alive()
criteria = criteria or Criteria.true()
out = []
# Pass the criteria through the code used by the real client to build
# up the Pulp query. We don't actually *use* the resulting query since
# we're not accessing a real Pulp server. The point is to ensure the
# same validation and error behavior as used by the real client also
# applies to the fake.
prepared_search = search_for_criteria(criteria, Unit)
available_type_ids = set(self._type_ids)
missing_type_ids = set(prepared_search.type_ids
or []) - available_type_ids
if missing_type_ids:
return f_return_error(
PulpException(
"following type ids are not supported on the server: %s" %
",".join(missing_type_ids)))
for unit in self._all_units:
if (prepared_search.type_ids
and unit.content_type_id not in prepared_search.type_ids):
continue
if match_object(criteria, unit):
out.append(unit)
# callers should not make any assumption about the order of returned
# values. Encourage that by returning output in unpredictable order
random.shuffle(out)
return self._prepare_pages(out)
def test_search_paginates():
controller = FakeController()
repos = []
for i in range(0, 1000):
repo = Repository(id="repo-%s" % i)
repos.append(repo)
controller.insert_repository(repo)
client = controller.client
crit = Criteria.true()
page = client.search_repository(crit)
found_repos = list(page)
page_count = 1
while page.next:
page_count += 1
page = page.next.result()
# There should have been several pages (it is not defined exactly
# what page size the fake client uses, but it should be relatively
# small to enforce that clients think about pagination)
assert page_count >= 10
# All repos should have been found
assert sorted(found_repos) == sorted(repos)
def _search_repo_units(self, repo_id, criteria):
criteria = criteria or Criteria.true()
# Pass the criteria through the same handling as used by the real client
# for serialization, to ensure we reject criteria also rejected by real client
# and also accumulate unit_fields.
prepared_search = search_for_criteria(criteria, Unit)
repo_f = self.get_repository(repo_id)
if repo_f.exception():
return repo_f
with self._state.lock:
repo_units = self._state.repo_units(repo_id)
out = []
try:
for unit in repo_units:
if match_object(criteria, unit):
unit = units.with_filtered_fields(unit, prepared_search.unit_fields)
out.append(unit)
except Exception as ex: # pylint: disable=broad-except
return f_return_error(ex)
random.shuffle(out)
return self._prepare_pages(out)
def check_repos(self):
self._sanitize_repo_ids_args()
repo_ids = self.args.repo_ids
found_repo_ids = []
out = []
# apply the filters and get repo_ids
repo_ids = self._filter_repos(repo_ids)
# get repo objects for the repo_ids
searched_repos = self.pulp_client.search_repository(Criteria.with_id(repo_ids))
for repo in searched_repos:
out.append(repo)
found_repo_ids.append(repo.id)
# Bail out if user requested repos which don't exist
# or there are no repos returned to publish
missing = set(repo_ids) - set(found_repo_ids)
missing = sorted(list(missing))
if missing:
self.fail("Requested repo(s) don't exist: %s", ", ".join(missing))
if not out:
self.fail("No repo(s) found to publish")
return out
def adjust_maintenance_report(self, report):
"""Remove entries from maintenance report by repo ids or repo url regex or both"""
to_remove = []
existed_repo_ids = [entry.repo_id for entry in report.entries]
if self.args.repo_url_regex:
# search all repos with id existed in the report
existed_repos = self.pulp_client.search_repository(
Criteria.with_id(existed_repo_ids)).result()
for repo in existed_repos:
if repo.relative_url and re.search(self.args.repo_url_regex,
repo.relative_url):
to_remove.append(repo.id)
if self.args.repo_ids:
for repo_id in self.args.repo_ids:
if repo_id in existed_repo_ids:
to_remove.append(repo_id)
else:
LOG.warning("Repository %s is not in maintenance mode",
repo_id)
if to_remove:
LOG.info(
"Following repositories will be removed from maintenance mode:"
)
for repo_id in to_remove:
LOG.info(" - %s", repo_id)
report = report.remove(to_remove, owner=self.args.owner)
return report
def test_field_less_than_criteria():
"""with_field with less_than is translated as expected for
date and non-date types
"""
publish_date = datetime.datetime(2019, 8, 27, 0, 0, 0)
c1 = Criteria.with_field("num_field", Matcher.less_than(5))
c2 = Criteria.with_field("date_field", Matcher.less_than(publish_date))
assert filters_for_criteria(c1) == {"num_field": {"$lt": 5}}
assert filters_for_criteria(c2) == {
"date_field": {
"$lt": {
"$date": "2019-08-27T00:00:00Z"
}
}
}
def test_search_distributor_with_relative_url():
controller = FakeController()
dist1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1",
relative_url="relative/path",
)
dist2 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1",
relative_url="relative/path",
)
repo1 = Repository(id="repo1", distributors=(dist1, dist2))
dist3 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo2",
relative_url="another/path",
)
repo2 = Repository(id="repo2", distributors=(dist3, ))
controller.insert_repository(repo1)
controller.insert_repository(repo2)
client = controller.client
crit = Criteria.with_field("relative_url", Matcher.regex("relative/path"))
found = client.search_distributor(crit).result().data
assert sorted(found) == [dist2, dist1]
def test_search_and():
controller = FakeController()
repo1 = Repository(id="repo1")
repo2 = Repository(id="repo2", created=datetime.datetime.utcnow())
repo3 = Repository(id="repo3", created=datetime.datetime.utcnow())
controller.insert_repository(repo1)
controller.insert_repository(repo2)
controller.insert_repository(repo3)
client = controller.client
crit = Criteria.and_(Criteria.with_field("notes.created", Criteria.exists),
Criteria.with_id("repo2"))
found = client.search_repository(crit).data
assert sorted(found) == [repo2]
def _filtered_repo_distributors(self):
published_before = self.args.published_before
url_regex = self.args.repo_url_regex
# define the criteria on available filters
crit = [Criteria.true()]
if published_before:
crit.append(
Criteria.with_field("last_publish", Matcher.less_than(published_before))
)
if url_regex:
crit.append(
Criteria.with_field("relative_url", Matcher.regex(url_regex.pattern))
)
crit = Criteria.and_(*crit)
return self.pulp_client.search_distributor(crit)
def test_eng_product_in():
"""eng_product is mapped correctly"""
crit = Criteria.with_field_in("eng_product_id", [12, 34, 56])
assert filters_for_criteria(crit, Repository) == {
"notes.eng_product": {
"$in": ["12", "34", "56"]
}
}
def test_signing_keys():
"""signing_keys are mapped correctly"""
crit = Criteria.with_field("signing_keys", ["abc", "def", "123"])
assert filters_for_criteria(crit, Repository) == {
"notes.signatures": {
"$eq": "abc,def,123"
}
}
def test_type():
"""type is mapped correctly"""
crit = Criteria.with_field("type", Matcher.regex("foobar"))
assert filters_for_criteria(crit, Repository) == {
"notes._repo-type": {
"$regex": "foobar"
}
}
def test_search_or():
controller = FakeController()
repo1 = Repository(id="repo1")
repo2 = Repository(id="repo2")
repo3 = Repository(id="repo3")
controller.insert_repository(repo1)
controller.insert_repository(repo2)
controller.insert_repository(repo3)
client = controller.client
crit = Criteria.or_(Criteria.with_id("repo3"),
Criteria.with_field("id", Matcher.equals("repo1")))
found = client.search_repository(crit).data
assert sorted(found) == [repo1, repo3]
def test_field_exists_criteria():
"""with_field using 'exists' is translated to a mongo fragment as expected."""
assert filters_for_criteria(
Criteria.with_field("some.field", Criteria.exists)) == {
"some.field": {
"$exists": True
}
}
def test_field_eq_criteria():
"""with_field is translated to a mongo fragment as expected."""
assert filters_for_criteria(Criteria.with_field("some.field",
"someval")) == {
"some.field": {
"$eq": "someval"
}
}
def test_field_in_criteria():
"""with_field_in is translated to a mongo fragment as expected."""
assert filters_for_criteria(
Criteria.with_field_in("some.field", ["val1", "val2"])) == {
"some.field": {
"$in": ["val1", "val2"]
}
}
def test_is_temporary():
"""is_temporary is mapped correctly"""
crit = Criteria.with_field("is_temporary", True)
assert filters_for_criteria(crit, Repository) == {
"notes.pub_temp_repo": {
"$eq": True
}
}
def run_with_client(self, client):
# At the time we run, it is the case that all items exist with the desired
# state, in the desired repos. Now we need to publish affected repos.
#
# This is also a synchronization point. The idea is that publishing repo
# makes content available, and there may be dependencies between the bits
# of content we've handled, so we should ensure *all* of them are in correct
# repos before we start publish of *any* repos to increase the chance that
# all of them land at once.
#
# TODO: once exodus is live, consider refactoring this to not be a
# synchronization point (or make it optional?) as the above motivation goes
# away - the CDN origin supports near-atomic update.
all_repo_ids = set()
set_cdn_published = set()
all_items = []
for item in self.iter_input():
all_repo_ids.update(item.publish_pulp_repos)
# any unit which supports cdn_published but hasn't had it set yet should
# have it set once the publish completes.
unit = item.pulp_unit
if hasattr(unit, "cdn_published") and unit.cdn_published is None:
set_cdn_published.add(unit)
all_items.append(item)
# Locate all the repos for publish.
repo_fs = client.search_repository(
Criteria.with_id(sorted(all_repo_ids)))
# Start publishing them, including cache flushes.
publish_fs = self.publish_with_cache_flush(repo_fs, set_cdn_published,
client)
# Then wait for publishes to finish.
for f in publish_fs:
f.result()
# At this stage we consider all items to be fully "pushed".
pushed_items = [
attr.evolve(item,
pushsource_item=attr.evolve(item.pushsource_item,
state="PUSHED"))
for item in all_items
]
self.update_push_items(pushed_items)
# Mark as done for accurate progress logs.
# Note we don't keep track of exactly which items got published through each
# repo, so this will simply show that everything moved from in progress to done
# at once.
for _ in pushed_items:
self.in_queue.task_done()
# And we know nothing more happens to the push items, so we can tell
# collector that we're finished.
self.update_push_items([self.FINISHED])
def test_search_content_by_type(populated_repo):
"""search_content for particular type returns matching content"""
crit = Criteria.with_field("content_type_id", "rpm")
units = list(populated_repo.search_content(crit))
assert sorted(units) == [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
def test_search_content_mixed_fields(populated_repo):
"""search_content crossing multiple fields and types returns matching units"""
crit = Criteria.and_(
Criteria.with_field_in("content_type_id", ["rpm", "modulemd"]),
Criteria.with_field_in("name", ["bash", "module1"]),
)
units = list(populated_repo.search_content(crit))
# Note: sorting different types not natively supported, hence sorting by repr
assert sorted(units, key=repr) == [
ModulemdUnit(name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64"),
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
]
