def _get_ubi_repo_sets(self, ubi_binary_cs):
"""
Searches for ubi repository triplet (binary rpm, srpm, debug) for
one ubi config item and tries to determine their population sources
(input repositories). Returns list UbiRepoSet objects that provides
input and output repositories that are used for population process.
"""
rpm_repos = self.pulp_client.search_repository(
Criteria.and_(
Criteria.with_field("notes.content_set", ubi_binary_cs),
Criteria.with_field("ubi_population", True),
))
ubi_repo_sets = []
for out_rpm_repo in rpm_repos:
out_source_repo = out_rpm_repo.get_source_repository()
out_debug_repo = out_rpm_repo.get_debug_repository()
in_rpm_repos = self._get_population_sources(out_rpm_repo)
in_source_repos = self._get_population_sources(out_source_repo)
in_debug_repos = self._get_population_sources(out_debug_repo)
# we need to apply f_proxy(f_return()) for out_rpm_repo for keeping consistency
# that all objects in out|in_repos are futures
out_repos = (
f_proxy(f_return(out_rpm_repo)),
out_source_repo,
out_debug_repo,
)
in_repos = (in_rpm_repos, in_source_repos, in_debug_repos)
ubi_repo_sets.append(
UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos)))
return ubi_repo_sets
def run(self):
LOG.debug("Garbage collection begins")
criteria = Criteria.and_(
Criteria.with_field("notes.created", Matcher.exists()),
Criteria.with_field("notes.pub_temp_repo", True),
)
# fetch repos for the criteria
repos = self.pulp_client.search_repository(criteria).result()
LOG.debug("repos fetched")
gc_threshold = self.args.gc_threshold
deleted_repos = []
# initiate deletion task for the repos
for repo in repos.as_iter():
repo_age = datetime.utcnow() - repo.created
if repo_age > timedelta(days=gc_threshold):
LOG.info("Deleting %s (created on %s)", repo.id, repo.created)
deleted_repos.append(repo.delete())
if not deleted_repos:
LOG.info("No repo(s) found older than %s day(s)", gc_threshold)
# log for error during deletion
for task in deleted_repos:
out = task.result()[0]
if out.error_details or out.error_summary:
LOG.error(out.error_details or out.error_summary)
LOG.info("Temporary repo(s) deletion completed")
def test_field_or_criteria():
"""or is translated to a mongo fragment as expected."""
c1 = Criteria.with_field("f1", "v1")
c2 = Criteria.with_field("f2", "v2")
assert filters_for_criteria(Criteria.or_(c1, c2)) == {
"$or": [{"f1": {"$eq": "v1"}}, {"f2": {"$eq": "v2"}}]
}
def test_search_content_all(populated_units, controller):
"""search_content_by_type with no criteria specified"""
units1 = [
u
for u in controller.client.search_content(
Criteria.with_field("content_type_id", "rpm")
).result()
]
units2 = [
u
for u in controller.client.search_content(
Criteria.with_field("content_type_id", "srpm")
).result()
]
units3 = [u for u in controller.client.search_content().result()]
assert len(units1) == 3
assert len([u for u in units1 if u.content_type_id == "rpm"]) == 3
assert len(units2) == 2
assert len([u for u in units2 if u.content_type_id == "srpm"]) == 2
# + two modulemd, one modulemd_defaults
assert len(units3) == 8
assert set(sum([u.repository_memberships for u in units1], [])) == set(
["repo1", "repo2"]
)
def clean_all_rpm_content(self):
# Clear out old all-rpm-content
LOG.info("Start old all-rpm-content deletion")
arc_threshold = self.args.arc_threshold
criteria = Criteria.and_(
Criteria.with_unit_type(RpmUnit),
Criteria.with_field(
"cdn_published",
Matcher.less_than(datetime.utcnow() -
timedelta(days=arc_threshold)),
),
)
clean_repos = list(
self.pulp_client.search_repository(
Criteria.with_field("id", "all-rpm-content")))
if not clean_repos:
LOG.info("No repos found for cleaning.")
return
arc_repo = clean_repos[0]
deleted_arc = list(arc_repo.remove_content(criteria=criteria))
deleted_content = []
for task in deleted_arc:
if task.repo_id == "all-rpm-content":
for unit in task.units:
LOG.info("Old all-rpm-content deleted: %s", unit.name)
deleted_content.append(unit)
if not deleted_content:
LOG.info("No all-rpm-content found older than %s", arc_threshold)
def test_remove_fail_without_type_id(fast_poller, client):
"""Remove fails when a critria is provided without unit type"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
criteria = Criteria.and_(
Criteria.with_field("filename", "some.rpm"),
Criteria.with_field("signing_key", Matcher.in_(["key123"])),
)
with pytest.raises(ValueError):
repo.remove_content(criteria=criteria)
def test_field_less_than_criteria():
"""with_field with less_than is translated as expected for
date and non-date types
"""
publish_date = datetime.datetime(2019, 8, 27, 0, 0, 0)
c1 = Criteria.with_field("num_field", Matcher.less_than(5))
c2 = Criteria.with_field("date_field", Matcher.less_than(publish_date))
assert filters_for_criteria(c1) == {"num_field": {"$lt": 5}}
assert filters_for_criteria(c2) == {
"date_field": {"$lt": {"$date": "2019-08-27T00:00:00Z"}}
}
def _rpm_criteria(self, filename, signing_keys=None, sha256sum=None):
if signing_keys:
return Criteria.and_(
Criteria.with_field("filename", filename),
Criteria.with_field("signing_key", Matcher.in_(signing_keys)),
)
if sha256sum:
return Criteria.and_(
Criteria.with_field("filename", filename),
Criteria.with_field("sha256sum", sha256sum),
)
return Criteria.with_field("filename", filename)
def _search_units(
cls, repo, criteria_list, content_type_id, batch_size_override=None
):
"""
Search for units of one content type associated with given repository by criteria.
"""
units = set()
batch_size = batch_size_override or BATCH_SIZE
def handle_results(page):
for unit in page.data:
unit = UbiUnit(unit, repo.id)
units.add(unit)
if page.next:
return f_flat_map(page.next, handle_results)
return f_return(units)
criteria_split = []
for start in range(0, len(criteria_list), batch_size):
criteria_split.append(criteria_list[start : start + batch_size])
fts = []
for criteria_batch in criteria_split:
_criteria = Criteria.and_(
Criteria.with_field("content_type_id", content_type_id),
Criteria.or_(*criteria_batch),
)
page_f = repo.search_content(_criteria)
handled_f = f_flat_map(page_f, handle_results)
fts.append(handled_f)
return f_flat_map(f_sequence(fts), flatten_list_of_sets)
def test_search_mapped_field_less_than():
controller = FakeController()
dist1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1",
last_publish=datetime.datetime(2019, 8, 23, 2, 5, 0, tzinfo=None),
)
dist2 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1",
last_publish=datetime.datetime(2019, 8, 27, 2, 5, 0, tzinfo=None),
)
repo1 = Repository(id="repo1", distributors=(dist1, dist2))
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.with_field(
"last_publish",
Matcher.less_than(datetime.datetime(2019, 8, 24, 0, 0, 0)))
found = client.search_distributor(crit).result().data
assert found == [dist1]
def test_search_distributor_with_relative_url():
controller = FakeController()
dist1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1",
relative_url="relative/path",
)
dist2 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1",
relative_url="relative/path",
)
repo1 = Repository(id="repo1", distributors=(dist1, dist2))
dist3 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo2",
relative_url="another/path",
)
repo2 = Repository(id="repo2", distributors=(dist3, ))
controller.insert_repository(repo1)
controller.insert_repository(repo2)
client = controller.client
crit = Criteria.with_field("relative_url", Matcher.regex("relative/path"))
found = client.search_distributor(crit).result().data
assert sorted(found) == [dist2, dist1]
def test_search_content_invalid_content_type(populated_units, controller):
"""search_content_by_type with invalid content type"""
with pytest.raises(PulpException):
for x in controller.client.search_content(
Criteria.with_field("content_type_id", "invalid")
).result():
pass
def test_search_content_unsupported_operator(populated_repo):
"""search_content using unsupported operators on content_type_id raises"""
with pytest.raises(ValueError) as e:
populated_repo.search_content(
Criteria.with_field("content_type_id", Matcher.regex("foobar")))
assert "unsupported expression for content_type_id" in str(e.value)
def test_no_update_on_same_cves(command_tester):
with FakeFixCves() as fake_fix_cves:
fake_pulp = fake_fix_cves.pulp_client_controller
client = fake_pulp.client
_setup_controller(fake_pulp)
command_tester.test(
fake_fix_cves.main,
[
"test-fix-cves",
"--pulp-url",
"https://pulp.example.com",
"--advisory",
"RHSA-1234:56",
"--cves",
"CVE-123",
],
)
erratum = list(
client.search_content(Criteria.with_field("id", "RHSA-1234:56")))
assert len(erratum) == 1
erratum = erratum[0]
assert erratum.id == "RHSA-1234:56"
# no updates. erratum version not bumped.
assert erratum.version == "2"
cves = sorted([ref for ref in erratum.references if ref.type == "cve"])
# same CVE exists
assert len(cves) == 1
assert cves[0].id == "CVE-123"
def test_fix_cves(command_tester):
with FakeFixCves() as fake_fix_cves:
fake_pulp = fake_fix_cves.pulp_client_controller
client = fake_pulp.client
_setup_controller(fake_pulp)
command_tester.test(
fake_fix_cves.main,
[
"test-fix-cves",
"--pulp-url",
"https://pulp.example.com",
"--advisory",
"RHSA-1234:56",
"--cves",
"CVE-987,CVE-456",
],
)
updated_erratum = list(
client.search_content(Criteria.with_field("id", "RHSA-1234:56")))
# only single erratum exists on Pulp
assert len(updated_erratum) == 1
updated_erratum = updated_erratum[0]
assert updated_erratum.id == "RHSA-1234:56"
# erratum version bumped
assert updated_erratum.version == "3"
cves = sorted(
[ref for ref in updated_erratum.references if ref.type == "cve"])
# only new CVEs in the erratum
assert len(cves) == 2
assert cves[0].id == "CVE-456"
assert cves[1].id == "CVE-987"
def test_stringify_complex_criteria():
crit = Criteria.and_(
Criteria.with_field("must-exist", Matcher.exists()),
Criteria.with_field("foo", Matcher.equals("bar")),
Criteria.true(),
Criteria.or_(
Criteria.with_field("foo", Matcher.regex("quux")),
Criteria.with_field("other", Matcher.in_(["x", "y", "z"])),
Criteria.with_field("num", Matcher.less_than(9000)),
),
Criteria.with_unit_type(FileUnit),
)
assert (str(crit) == "((must-exist EXISTS) AND foo=='bar' AND TRUE "
"AND (foo=~/quux/ OR (other IN ['x', 'y', 'z']) OR num<9000) "
"AND (content_type_id IN ['iso']))")
def test_field_eq_criteria():
"""with_field is translated to a mongo fragment as expected."""
assert filters_for_criteria(Criteria.with_field("some.field",
"someval")) == {
"some.field": {
"$eq": "someval"
}
}
def test_is_temporary():
"""is_temporary is mapped correctly"""
crit = Criteria.with_field("is_temporary", True)
assert filters_for_criteria(crit, Repository) == {
"notes.pub_temp_repo": {
"$eq": True
}
}
def test_type():
"""type is mapped correctly"""
crit = Criteria.with_field("type", Matcher.regex("foobar"))
assert filters_for_criteria(crit, Repository) == {
"notes._repo-type": {
"$regex": "foobar"
}
}
def _filtered_repo_distributors(self):
published_before = self.args.published_before
url_regex = self.args.repo_url_regex
# define the criteria on available filters
crit = [Criteria.true()]
if published_before:
crit.append(
Criteria.with_field("last_publish", Matcher.less_than(published_before))
)
if url_regex:
crit.append(
Criteria.with_field("relative_url", Matcher.regex(url_regex.pattern))
)
crit = Criteria.and_(*crit)
return self.pulp_client.search_distributor(crit)
def test_field_exists_criteria():
"""with_field using 'exists' is translated to a mongo fragment as expected."""
assert filters_for_criteria(
Criteria.with_field("some.field", Criteria.exists)) == {
"some.field": {
"$exists": True
}
}
def test_signing_keys():
"""signing_keys are mapped correctly"""
crit = Criteria.with_field("signing_keys", ["abc", "def", "123"])
assert filters_for_criteria(crit, Repository) == {
"notes.signatures": {
"$eq": "abc,def,123"
}
}
def test_search_content_by_type(populated_repo):
"""search_content for particular type returns matching content"""
crit = Criteria.with_field("content_type_id", "rpm")
units = list(populated_repo.search_content(crit))
assert sorted(units) == [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
def test_field_regex_criteria():
"""with_field with regex is translated to a mongo fragment as expected."""
assert filters_for_criteria(
Criteria.with_field("some.field", Matcher.regex("abc"))) == {
"some.field": {
"$regex": "abc"
}
}
def test_complex_type_ids(client):
"""content searches raise if using criteria with unsupported operators on content_type_id"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
with pytest.raises(ValueError) as e:
repo.search_content(
Criteria.with_field("content_type_id", Matcher.regex("foobar")))
assert "unsupported expression for content_type_id" in str(e.value)
def test_search_content_criteria(populated_units, controller):
"""search_content_by_type with criteria"""
units1 = [
u
for u in controller.client.search_content(
Criteria.with_field("name", "bash")
).result()
]
# match both srpm and rpm
assert len(units1) == 2
assert sorted([u.arch for u in units1]) == ["src", "x86_64"]
def test_dict_matcher_value():
"""criteria using a dict as matcher value"""
crit = Criteria.with_field(
"created",
Matcher.less_than({"created_date": datetime.datetime(2019, 9, 4, 0, 0, 0)}),
)
assert filters_for_criteria(crit) == {
"created": {"$lt": {"created_date": {"$date": "2019-09-04T00:00:00Z"}}}
}
def test_search_mapped_field_eq():
"""Can do equality search with fields subject to Python<=>Pulp conversion."""
controller = FakeController()
repo1 = Repository(id="repo1", eng_product_id=888)
repo2 = Repository(id="repo2", signing_keys=["foo", "bar"])
repo3 = Repository(id="repo3", eng_product_id=123)
controller.insert_repository(repo1)
controller.insert_repository(repo2)
controller.insert_repository(repo3)
client = controller.client
keys_crit = Criteria.with_field("signing_keys", ["foo", "bar"])
product_crit = Criteria.with_field("eng_product_id", 123)
found_by_keys = client.search_repository(keys_crit).data
found_by_product = client.search_repository(product_crit).data
assert found_by_keys == [repo2]
assert found_by_product == [repo3]
def test_copy_with_criteria(fast_poller, requests_mocker, client):
"""Copy with criteria succeeds, and serializes criteria correctly."""
src = Repository(id="src-repo")
dest = Repository(id="dest-repo")
src.__dict__["_client"] = client
dest.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/dest-repo/actions/associate/",
[{"json": {"spawned_tasks": [{"task_id": "task1"}, {"task_id": "task2"}]}}],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{
"json": [
{"task_id": "task1", "state": "finished"},
{"task_id": "task2", "state": "skipped"},
]
}
],
)
crit = Criteria.and_(
Criteria.with_unit_type(RpmUnit),
Criteria.with_field("name", Matcher.in_(["bash", "glibc"])),
)
# Copy should succeed, and return the tasks (in this case with no matches)
assert sorted(client.copy_content(src, dest, crit), key=lambda t: t.id) == [
Task(id="task1", completed=True, succeeded=True),
Task(id="task2", completed=True, succeeded=True),
]
hist = requests_mocker.request_history
# First request should have been the associate.
assert (
hist[0].url
== "https://pulp.example.com/pulp/api/v2/repositories/dest-repo/actions/associate/"
)
# It should have encoded our criteria object as needed by the Pulp API.
assert hist[0].json() == {
"criteria": {
"filters": {"unit": {"name": {"$in": ["bash", "glibc"]}}},
"type_ids": ["rpm", "srpm"],
},
"source_repo_id": "src-repo",
}
def test_search_content_by_unit_field(populated_repo):
"""search_content on regular field returns matching content"""
crit = Criteria.with_field("name", "bash")
units = list(populated_repo.search_content(crit))
assert sorted(units) == [
RpmUnit(content_type_id="srpm",
name="bash",
version="4.0",
release="1",
arch="src"),
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
]
