def _search_units(
cls, repo, criteria_list, content_type_id, batch_size_override=None
):
"""
Search for units of one content type associated with given repository by criteria.
"""
units = set()
batch_size = batch_size_override or BATCH_SIZE
def handle_results(page):
for unit in page.data:
unit = UbiUnit(unit, repo.id)
units.add(unit)
if page.next:
return f_flat_map(page.next, handle_results)
return f_return(units)
criteria_split = []
for start in range(0, len(criteria_list), batch_size):
criteria_split.append(criteria_list[start : start + batch_size])
fts = []
for criteria_batch in criteria_split:
_criteria = Criteria.and_(
Criteria.with_field("content_type_id", content_type_id),
Criteria.or_(*criteria_batch),
)
page_f = repo.search_content(_criteria)
handled_f = f_flat_map(page_f, handle_results)
fts.append(handled_f)
return f_flat_map(f_sequence(fts), flatten_list_of_sets)
def test_field_or_criteria():
"""or is translated to a mongo fragment as expected."""
c1 = Criteria.with_field("f1", "v1")
c2 = Criteria.with_field("f2", "v2")
assert filters_for_criteria(Criteria.or_(c1, c2)) == {
"$or": [{"f1": {"$eq": "v1"}}, {"f2": {"$eq": "v2"}}]
}
def test_search_null_or():
"""Search with an empty OR gives an error."""
controller = FakeController()
repo1 = Repository(id="repo1")
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.or_()
assert "Invalid OR in search query" in str(
client.search_repository(crit).exception())
def test_search_content_type_id_in_or(client, requests_mocker):
"""Searching with a content_type_id within $or fails as unsupported"""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
crit = Criteria.or_(
Criteria.with_field("name", "hello.txt"),
Criteria.with_field_in("content_type_id", ["rpm", "iso"]),
)
with pytest.raises(ValueError) as e:
repo.search_content(crit).result()
assert "Can't serialize criteria for Pulp query; too complicated" in str(
e.value)
def test_stringify_complex_criteria():
crit = Criteria.and_(
Criteria.with_field("must-exist", Matcher.exists()),
Criteria.with_field("foo", Matcher.equals("bar")),
Criteria.true(),
Criteria.or_(
Criteria.with_field("foo", Matcher.regex("quux")),
Criteria.with_field("other", Matcher.in_(["x", "y", "z"])),
Criteria.with_field("num", Matcher.less_than(9000)),
),
Criteria.with_unit_type(FileUnit),
)
assert (str(crit) == "((must-exist EXISTS) AND foo=='bar' AND TRUE "
"AND (foo=~/quux/ OR (other IN ['x', 'y', 'z']) OR num<9000) "
"AND (content_type_id IN ['iso']))")
def test_search_or():
controller = FakeController()
repo1 = Repository(id="repo1")
repo2 = Repository(id="repo2")
repo3 = Repository(id="repo3")
controller.insert_repository(repo1)
controller.insert_repository(repo2)
controller.insert_repository(repo3)
client = controller.client
crit = Criteria.or_(Criteria.with_id("repo3"),
Criteria.with_field("id", Matcher.equals("repo1")))
found = client.search_repository(crit).data
assert sorted(found) == [repo1, repo3]
def unit_criteria(self, unit_type, partial_crit):
criteria = Criteria.and_(
Criteria.with_unit_type(unit_type), Criteria.or_(*partial_crit)
)
return criteria
def test_delete_rpms(command_tester, fake_collector, monkeypatch):
"""Deleting RPMs from repos succeeds"""
repo1 = YumRepository(
id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
)
repo2 = YumRepository(
id="other-yumrepo",
relative_url="other/publish/url",
mutable_urls=["repomd.xml"],
)
files1 = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file1_rpm1",
),
RpmUnit(
name="dash",
version="2.25",
release="1.test8",
arch="x86_64",
filename="dash-2.25-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file1_rpm2",
),
ModulemdUnit(
name="mymod",
stream="s1",
version=123,
context="a1c2",
arch="s390x",
unit_id="file1_mod1",
),
]
files2 = [
RpmUnit(
name="crash",
version="3.30",
release="1.test8",
arch="s390x",
filename="crash-3.30-1.test8_s390x.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file2_rpm1",
)
]
files3 = [
RpmUnit(
name="rash",
version="1.30",
release="1.test8",
arch="noarch",
filename="rash-1.30-1.test8_noarch.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file3_rpm1",
)
]
undeleted = [
RpmUnit(
name="exist",
version="1.34",
release="1.test8",
arch="noarch",
filename="exist-1.34-1.test8_noarch.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="undeleted_rpm1",
)
]
files1.extend(files3)
files1.extend(undeleted)
files2.extend(files3)
with FakeDeletePackages() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo1)
task_instance.pulp_client_controller.insert_repository(repo2)
task_instance.pulp_client_controller.insert_units(repo1, files1)
task_instance.pulp_client_controller.insert_units(repo2, files2)
# Let's try setting the cache flush root via env.
monkeypatch.setenv("FASTPURGE_ROOT_URL", "https://cdn.example2.com/")
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--fastpurge-host",
"fakehost-xxx.example.net",
"--fastpurge-client-secret",
"abcdef",
"--fastpurge-client-token",
"efg",
"--fastpurge-access-token",
"tok",
"--repo",
"some-yumrepo,other-yumrepo",
"--repo",
"some-other-repo",
"--file",
"bash-1.23-1.test8_x86_64.rpm",
"--file",
"dash-2.25-1.test8_x86_64.rpm,crash-3.30-1.test8_s390x.rpm",
"--file",
"trash-1.0-1.test8_noarch.rpm,rash-1.30-1.test8_noarch.rpm",
"--signing-key",
"aabbcc",
],
)
# It should record that it removed these push items:
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "other-yumrepo",
"signing_key": None,
"filename": "crash-3.30-1.test8.s390x.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "dash-2.25-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "other-yumrepo",
"signing_key": None,
"filename": "rash-1.30-1.test8.noarch.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "rash-1.30-1.test8.noarch.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
]
# verify whether files were deleted on Pulp
client = task_instance.pulp_client
# get the repo where the files were deleted
repos = sorted(
list(
client.search_repository(
Criteria.with_id(["some-yumrepo", "other-yumrepo"])
).result()
),
key=lambda r: r.id,
)
assert len(repos) == 2
r2, r1 = repos
assert r1.id == repo1.id
assert r2.id == repo2.id
# criteria with the unit_ids
# critera1 for files1 in repo1
unit_ids = []
for f in files1:
unit_ids.append(f.unit_id)
criteria1 = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# critera2 for files2 in repo2
unit_ids = []
for f in files2:
unit_ids.append(f.unit_id)
criteria2 = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# files are not in the repo1 except undeleted rpm and module
result1 = sorted(
list(r1.search_content(criteria1).result()), key=lambda v: v.unit_id
)
assert len(result1) == 2
# modulemd in files1
assert result1[0].unit_id == files1[2].unit_id
# undeleted file
assert result1[1].unit_id == undeleted[0].unit_id
# files are not in repo2
result2 = list(r2.search_content(criteria1).result())
assert len(result2) == 0
# All the files exist on Pulp
files_search = list(
client.search_content(Criteria.or_(criteria1, criteria2)).result()
)
assert len(files_search) == 6
def test_stringify_simplify_or():
assert str(Criteria.or_(Criteria.with_field("x", 123))) == "x==123"
def test_stringify_noop_or():
assert str(Criteria.or_()) == "<empty OR>"
