def _rpm_criteria(self, filename, signing_keys=None, sha256sum=None):
if signing_keys:
return Criteria.and_(
Criteria.with_field("filename", filename),
Criteria.with_field("signing_key", Matcher.in_(signing_keys)),
)
if sha256sum:
return Criteria.and_(
Criteria.with_field("filename", filename),
Criteria.with_field("sha256sum", sha256sum),
)
return Criteria.with_field("filename", filename)
def clean_all_rpm_content(self):
# Clear out old all-rpm-content
LOG.info("Start old all-rpm-content deletion")
arc_threshold = self.args.arc_threshold
criteria = Criteria.and_(
Criteria.with_unit_type(RpmUnit),
Criteria.with_field(
"cdn_published",
Matcher.less_than(datetime.utcnow() -
timedelta(days=arc_threshold)),
),
)
clean_repos = list(
self.pulp_client.search_repository(
Criteria.with_field("id", "all-rpm-content")))
if not clean_repos:
LOG.info("No repos found for cleaning.")
return
arc_repo = clean_repos[0]
deleted_arc = list(arc_repo.remove_content(criteria=criteria))
deleted_content = []
for task in deleted_arc:
if task.repo_id == "all-rpm-content":
for unit in task.units:
LOG.info("Old all-rpm-content deleted: %s", unit.name)
deleted_content.append(unit)
if not deleted_content:
LOG.info("No all-rpm-content found older than %s", arc_threshold)
def test_field_and_criteria():
"""and is translated to a mongo fragment as expected."""
c1 = Criteria.with_field("f1", "v1")
c2 = Criteria.with_field("f2", "v2")
assert filters_for_criteria(Criteria.and_(c1, c2)) == {
"$and": [{"f1": {"$eq": "v1"}}, {"f2": {"$eq": "v2"}}]
}
def _get_ubi_repo_sets(self, ubi_binary_cs):
"""
Searches for ubi repository triplet (binary rpm, srpm, debug) for
one ubi config item and tries to determine their population sources
(input repositories). Returns list UbiRepoSet objects that provides
input and output repositories that are used for population process.
"""
rpm_repos = self.pulp_client.search_repository(
Criteria.and_(
Criteria.with_field("notes.content_set", ubi_binary_cs),
Criteria.with_field("ubi_population", True),
))
ubi_repo_sets = []
for out_rpm_repo in rpm_repos:
out_source_repo = out_rpm_repo.get_source_repository()
out_debug_repo = out_rpm_repo.get_debug_repository()
in_rpm_repos = self._get_population_sources(out_rpm_repo)
in_source_repos = self._get_population_sources(out_source_repo)
in_debug_repos = self._get_population_sources(out_debug_repo)
# we need to apply f_proxy(f_return()) for out_rpm_repo for keeping consistency
# that all objects in out|in_repos are futures
out_repos = (
f_proxy(f_return(out_rpm_repo)),
out_source_repo,
out_debug_repo,
)
in_repos = (in_rpm_repos, in_source_repos, in_debug_repos)
ubi_repo_sets.append(
UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos)))
return ubi_repo_sets
def run(self):
LOG.debug("Garbage collection begins")
criteria = Criteria.and_(
Criteria.with_field("notes.created", Matcher.exists()),
Criteria.with_field("notes.pub_temp_repo", True),
)
# fetch repos for the criteria
repos = self.pulp_client.search_repository(criteria).result()
LOG.debug("repos fetched")
gc_threshold = self.args.gc_threshold
deleted_repos = []
# initiate deletion task for the repos
for repo in repos.as_iter():
repo_age = datetime.utcnow() - repo.created
if repo_age > timedelta(days=gc_threshold):
LOG.info("Deleting %s (created on %s)", repo.id, repo.created)
deleted_repos.append(repo.delete())
if not deleted_repos:
LOG.info("No repo(s) found older than %s day(s)", gc_threshold)
# log for error during deletion
for task in deleted_repos:
out = task.result()[0]
if out.error_details or out.error_summary:
LOG.error(out.error_details or out.error_summary)
LOG.info("Temporary repo(s) deletion completed")
def _search_units(
cls, repo, criteria_list, content_type_id, batch_size_override=None
):
"""
Search for units of one content type associated with given repository by criteria.
"""
units = set()
batch_size = batch_size_override or BATCH_SIZE
def handle_results(page):
for unit in page.data:
unit = UbiUnit(unit, repo.id)
units.add(unit)
if page.next:
return f_flat_map(page.next, handle_results)
return f_return(units)
criteria_split = []
for start in range(0, len(criteria_list), batch_size):
criteria_split.append(criteria_list[start : start + batch_size])
fts = []
for criteria_batch in criteria_split:
_criteria = Criteria.and_(
Criteria.with_field("content_type_id", content_type_id),
Criteria.or_(*criteria_batch),
)
page_f = repo.search_content(_criteria)
handled_f = f_flat_map(page_f, handle_results)
fts.append(handled_f)
return f_flat_map(f_sequence(fts), flatten_list_of_sets)
def test_search_content_mixed_fields(populated_repo):
"""search_content crossing multiple fields and types returns matching units"""
crit = Criteria.and_(
Criteria.with_field_in("content_type_id", ["rpm", "modulemd"]),
Criteria.with_field_in("name", ["bash", "module1"]),
)
units = list(populated_repo.search_content(crit))
# Note: sorting different types not natively supported, hence sorting by repr
assert sorted(units, key=repr) == [
ModulemdUnit(
unit_id="23a7711a-8133-2876-37eb-dcd9e87a1613",
name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64",
repository_memberships=["repo1"],
),
RpmUnit(
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
name="bash",
version="4.0",
release="1",
arch="x86_64",
repository_memberships=["repo1"],
),
]
def get_advisories(self, advisory_ids):
criteria = criteria = Criteria.and_(
Criteria.with_unit_type(ErratumUnit), Criteria.with_id(advisory_ids)
)
adv_f = self.search_content(criteria)
advisories = [a for a in adv_f.result()]
if not advisories:
self.fail("Advisory(ies) not found: %s", ", ".join(sorted(advisory_ids)))
return advisories
def test_remove_fail_without_type_id(fast_poller, client):
"""Remove fails when a critria is provided without unit type"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
criteria = Criteria.and_(
Criteria.with_field("filename", "some.rpm"),
Criteria.with_field("signing_key", Matcher.in_(["key123"])),
)
with pytest.raises(ValueError):
repo.remove_content(criteria=criteria)
def test_search_null_and():
"""Search with an empty AND gives an error."""
controller = FakeController()
repo1 = Repository(id="repo1")
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.and_()
assert "Invalid AND in search query" in str(
client.search_repository(crit).exception())
def test_copy_with_criteria(fast_poller, requests_mocker, client):
"""Copy with criteria succeeds, and serializes criteria correctly."""
src = Repository(id="src-repo")
dest = Repository(id="dest-repo")
src.__dict__["_client"] = client
dest.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/dest-repo/actions/associate/",
[{"json": {"spawned_tasks": [{"task_id": "task1"}, {"task_id": "task2"}]}}],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{
"json": [
{"task_id": "task1", "state": "finished"},
{"task_id": "task2", "state": "skipped"},
]
}
],
)
crit = Criteria.and_(
Criteria.with_unit_type(RpmUnit),
Criteria.with_field("name", Matcher.in_(["bash", "glibc"])),
)
# Copy should succeed, and return the tasks (in this case with no matches)
assert sorted(client.copy_content(src, dest, crit), key=lambda t: t.id) == [
Task(id="task1", completed=True, succeeded=True),
Task(id="task2", completed=True, succeeded=True),
]
hist = requests_mocker.request_history
# First request should have been the associate.
assert (
hist[0].url
== "https://pulp.example.com/pulp/api/v2/repositories/dest-repo/actions/associate/"
)
# It should have encoded our criteria object as needed by the Pulp API.
assert hist[0].json() == {
"criteria": {
"filters": {"unit": {"name": {"$in": ["bash", "glibc"]}}},
"type_ids": ["rpm", "srpm"],
},
"source_repo_id": "src-repo",
}
def test_mixed_search(client, requests_mocker):
"""Searching with a criteria mixing several fields works correctly"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/",
json=[{
"metadata": {
"_content_type_id": "iso",
"name": "hello.txt",
"size": 23,
"checksum": "a" * 64,
}
}],
)
crit = Criteria.and_(
Criteria.with_field_in("content_type_id", ["rpm", "iso"]),
Criteria.with_field("name", "hello.txt"),
)
files = list(repo.search_content(crit))
assert files == [FileUnit(path="hello.txt", size=23, sha256sum="a" * 64)]
history = requests_mocker.request_history
# There should have been just one request
assert len(history) == 1
request = history[0]
body = request.json()
# This should have been the request body.
assert body == {
"criteria": {
"type_ids": ["rpm", "iso"],
"skip": 0,
"limit": 2000,
"filters": {
"unit": {
"name": {
"$eq": "hello.txt"
}
}
},
}
}
def test_stringify_complex_criteria():
crit = Criteria.and_(
Criteria.with_field("must-exist", Matcher.exists()),
Criteria.with_field("foo", Matcher.equals("bar")),
Criteria.true(),
Criteria.or_(
Criteria.with_field("foo", Matcher.regex("quux")),
Criteria.with_field("other", Matcher.in_(["x", "y", "z"])),
Criteria.with_field("num", Matcher.less_than(9000)),
),
Criteria.with_unit_type(FileUnit),
)
assert (str(crit) == "((must-exist EXISTS) AND foo=='bar' AND TRUE "
"AND (foo=~/quux/ OR (other IN ['x', 'y', 'z']) OR num<9000) "
"AND (content_type_id IN ['iso']))")
def test_search_null_and():
"""Search with an empty AND gives an error."""
controller = FakeController()
dist1 = Distributor(id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1")
repo1 = Repository(id="repo1", distributors=[dist1])
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.and_()
assert "Invalid AND in search query" in str(
client.search_repository(crit).exception())
assert "Invalid AND in search query" in str(
client.search_distributor(crit).exception())
def _filtered_repo_distributors(self):
published_before = self.args.published_before
url_regex = self.args.repo_url_regex
# define the criteria on available filters
crit = [Criteria.true()]
if published_before:
crit.append(
Criteria.with_field("last_publish", Matcher.less_than(published_before))
)
if url_regex:
crit.append(
Criteria.with_field("relative_url", Matcher.regex(url_regex.pattern))
)
crit = Criteria.and_(*crit)
return self.pulp_client.search_distributor(crit)
def test_search_and():
controller = FakeController()
repo1 = Repository(id="repo1")
repo2 = Repository(id="repo2", created=datetime.datetime.utcnow())
repo3 = Repository(id="repo3", created=datetime.datetime.utcnow())
controller.insert_repository(repo1)
controller.insert_repository(repo2)
controller.insert_repository(repo3)
client = controller.client
crit = Criteria.and_(Criteria.with_field("notes.created", Criteria.exists),
Criteria.with_id("repo2"))
found = client.search_repository(crit).data
assert sorted(found) == [repo2]
def test_remove_fails_without_type_id():
"""repo.remove_content() fails when a criteria is provided without unit type"""
controller = FakeController()
client = controller.client
rpm_units = [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
repo = YumRepository(id="repo1")
controller.insert_repository(repo)
controller.insert_units(repo, rpm_units)
criteria = Criteria.and_(Criteria.with_field("name", "bash"))
with pytest.raises(ValueError):
client.get_repository("repo1").remove_content(criteria=criteria)
def test_search_content_mixed_fields(populated_repo):
"""search_content crossing multiple fields and types returns matching units"""
crit = Criteria.and_(
Criteria.with_field_in("content_type_id", ["rpm", "modulemd"]),
Criteria.with_field_in("name", ["bash", "module1"]),
)
units = list(populated_repo.search_content(crit))
# Note: sorting different types not natively supported, hence sorting by repr
assert sorted(units, key=repr) == [
ModulemdUnit(name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64"),
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
]
def _create_or_criteria(self, fields, values):
# fields - list/tuple of fields [field1, field2]
# values - list of tuples [(field1 value, field2 value), ...]
# creates criteria for pulp query in a following way
# one tuple in values uses AND logic
# each criteria for one tuple are agregated by to or_criteria list
or_criteria = []
for val_tuple in values:
inner_and_criteria = []
if len(val_tuple) != len(fields):
raise ValueError
for index, field in enumerate(fields):
inner_and_criteria.append(Criteria.with_field(field, val_tuple[index]))
or_criteria.append(Criteria.and_(*inner_and_criteria))
return or_criteria
def test_search_fields(client, requests_mocker):
"""Searching with limited fields works correctly"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/",
json=[],
)
crit = Criteria.and_(
Criteria.with_unit_type(FileUnit, unit_fields=["sha256sum"]),
Criteria.with_field("name", "hello.txt"),
)
repo.search_content(crit).result()
history = requests_mocker.request_history
# There should have been just one request
assert len(history) == 1
request = history[0]
body = request.json()
# This should have been the request body.
assert body == {
"criteria": {
"type_ids": ["iso"],
"skip": 0,
"limit": 2000,
"filters": {
"unit": {
"name": {
"$eq": "hello.txt"
}
}
},
"fields": {
"unit": ["checksum", "name", "size"]
},
}
}
def find_related(self, repo):
"""Start searching for the repos which should have product_versions updated
after a productid has been uploaded to 'repo'.
Returns None, but influences the return value of a later call to all_results.
"""
arch = repo.arch
eng_product_id = repo.eng_product_id
platform_full_version = repo.platform_full_version
search_key = (arch, eng_product_id, platform_full_version)
with self.lock:
self.repos.append(repo)
if not self.searches.get(search_key):
search_f = self.client.search_repository(
Criteria.and_(
Criteria.with_field("arch", arch),
Criteria.with_field("eng_product_id", eng_product_id),
Criteria.with_field("platform_full_version",
platform_full_version),
))
self.searches[search_key] = search_f
def test_copy_content_with_criteria(controller):
"""copy_content can filter copied units by field values"""
src = YumRepository(id="src-repo")
dest = YumRepository(id="dest-repo")
controller.insert_repository(src)
controller.insert_repository(dest)
src_units = [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="bash", version="4.0", release="2", arch="x86_64"),
RpmUnit(name="bash", version="4.1", release="3", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
controller.insert_units(src, src_units)
client = controller.client
# Repos are initially detached, re-fetch them via client
src = client.get_repository(src.id).result()
dest = client.get_repository(dest.id).result()
# This is what we want to copy...
crit = Criteria.and_(
Criteria.with_field("name", "bash"),
Criteria.with_field("release", Matcher.in_(["1", "3"])),
)
# Copy should succeed
copy_tasks = list(
client.copy_content(src,
dest,
crit,
options=CopyOptions(require_signed_rpms=False)))
# It should have copied only those units matching the criteria
units = sum([t.units for t in copy_tasks], [])
assert sorted(units, key=repr) == [
RpmUnit(name="bash",
version="4.0",
release="1",
arch="x86_64",
epoch="0"),
RpmUnit(name="bash",
version="4.1",
release="3",
arch="x86_64",
epoch="0"),
]
# The copy should also impact subsequent content searches.
dest_units = list(dest.search_content())
assert sorted(dest_units, key=repr) == [
RpmUnit(
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
name="bash",
version="4.0",
release="1",
arch="x86_64",
epoch="0",
repository_memberships=["src-repo", "dest-repo"],
),
RpmUnit(
unit_id="d4713d60-c8a7-0639-eb11-67b367a9c378",
name="bash",
version="4.1",
release="3",
arch="x86_64",
epoch="0",
repository_memberships=["src-repo", "dest-repo"],
),
]
def criteria(self):
return Criteria.and_(
Criteria.with_field("sha256sum", self.pushsource_item.sha256sum),
Criteria.with_field("path", self.pushsource_item.name),
)
def test_remove_filtered_content():
"""repo.remove_content() succeeds and removes expected units inserted via controller."""
controller = FakeController()
client = controller.client
rpm_units = [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
modulemd_units = [
ModulemdUnit(name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64"),
ModulemdUnit(name="module1",
stream="s1",
version=1235,
context="a1b2",
arch="x86_64"),
]
units = rpm_units + modulemd_units
repo = YumRepository(id="repo1")
controller.insert_repository(repo)
controller.insert_units(repo, units)
criteria = Criteria.and_(Criteria.with_unit_type(RpmUnit),
Criteria.with_field("name", "bash"))
remove_rpms = client.get_repository("repo1").remove_content(
criteria=criteria)
assert len(remove_rpms) == 1
task = remove_rpms[0]
# It should have completed successfully
assert task.completed
assert task.succeeded
# It should have removed the specific rpm
assert len(task.units) == 1
assert task.units[0] == sorted(rpm_units)[0]
# Try removing a module with incorrect type_ids
criteria = Criteria.and_(Criteria.with_unit_type(RpmUnit),
Criteria.with_field("name", "module1"))
remove_rpms = client.get_repository("repo1").remove_content(
criteria=criteria)
assert len(remove_rpms) == 1
task = remove_rpms[0]
# It should have completed successfully
assert task.completed
assert task.succeeded
# Nothing's removed as criteria isn't fulfilled
assert len(task.units) == 0
# Removing module with correct type_ids
criteria = Criteria.and_(Criteria.with_unit_type(ModulemdUnit),
Criteria.with_field("name", "module1"))
remove_rpms = client.get_repository("repo1").remove_content(
criteria=criteria)
assert len(remove_rpms) == 1
task = remove_rpms[0]
# It should have completed successfully
assert task.completed
assert task.succeeded
# It should have removed both the modules as they
# match the criteria
assert len(task.units) == 2
assert sorted(task.units) == sorted(modulemd_units)
def get_erratum_from_pulp(self, advisory_id):
crit = Criteria.and_(Criteria.with_unit_type(ErratumUnit),
Criteria.with_id(advisory_id))
return self.pulp_client.search_content(criteria=crit).result()
def test_remove_with_criteria(fast_poller, requests_mocker, client):
"""Remove succeeds when given a critria/filter for removal"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/unassociate/",
[
{
"json": {
"spawned_tasks": [{
"task_id": "task1"
}]
}
},
{
"json": {
"spawned_tasks": [{
"task_id": "task2"
}]
}
},
],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{
"json": [{
"task_id": "task1",
"state": "finished"
}]
},
{
"json": [{
"task_id": "task2",
"state": "finished"
}]
},
],
)
criteria = Criteria.and_(
Criteria.with_unit_type(RpmUnit),
Criteria.with_field("filename", "some.rpm"),
Criteria.with_field("signing_key", Matcher.in_(["key123"])),
)
assert repo.remove_content(criteria=criteria).result() == [
Task(id="task1", completed=True, succeeded=True)
]
# It should have passed the criteria to Pulp
req = requests_mocker.request_history
assert (
req[0].url ==
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/unassociate/"
)
assert req[0].json() == {
"criteria": {
"filters": {
"unit": {
"$and": [
{
"filename": {
"$eq": "some.rpm"
}
},
{
"signing_key": {
"$in": ["key123"]
}
},
]
}
},
"type_ids": ["rpm", "srpm"],
}
}
# Providing both criteria and type_ids
assert repo.remove_content(criteria=criteria,
type_ids=["type1", "type2"]).result() == [
Task(id="task2",
completed=True,
succeeded=True)
]
# It should have passed only the critera to Pulp and ignore type_ids as kwarg
req = requests_mocker.request_history
assert (
req[0].url ==
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/unassociate/"
)
assert req[0].json() == {
"criteria": {
"filters": {
"unit": {
"$and": [
{
"filename": {
"$eq": "some.rpm"
}
},
{
"signing_key": {
"$in": ["key123"]
}
},
]
}
},
"type_ids": ["rpm", "srpm"],
}
}
def test_search_null_and(populated_repo):
"""Search with an empty AND gives an error."""
crit = Criteria.and_()
assert "Invalid AND in search query" in str(
populated_repo.search_content(crit).exception())
def _module_criteria(self, module_name):
crit = []
mod_nsvca_dict = self._get_nsvca_dict(module_name)
for m_part, value in mod_nsvca_dict.items():
crit.append(Criteria.with_field(m_part, value))
return Criteria.and_(*crit)
def test_stringify_simplify_and():
assert str(Criteria.and_(Criteria.with_field("x", 123))) == "x==123"
def unit_criteria(self, unit_type, partial_crit):
criteria = Criteria.and_(
Criteria.with_unit_type(unit_type), Criteria.or_(*partial_crit)
)
return criteria
