def test_can_upload_units(data_path, use_file_object):
"""repo.upload_rpm() succeeds with fake client and populates units."""
rpm_path = os.path.join(data_path, "rpms/walrus-5.21-1.noarch.rpm")
controller = FakeController()
controller.insert_repository(YumRepository(id="repo1"))
client = controller.client
repo1 = client.get_repository("repo1").result()
to_upload = rpm_path
if use_file_object:
to_upload = open(rpm_path, "rb")
upload_f = repo1.upload_rpm(to_upload)
# Upload should complete successfully.
tasks = upload_f.result()
# At least one task.
assert tasks
# Every task should have succeeded.
for t in tasks:
assert t.succeeded
# If I now search for content in that repo, or content across all repos...
units_in_repo = sorted(repo1.search_content().result(),
key=lambda u: u.sha256sum)
units_all = sorted(client.search_content().result(),
key=lambda u: u.sha256sum)
# They should be equal
assert units_all == units_in_repo
# And they should be this
assert units_in_repo == [
RpmUnit(
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
name="walrus",
version="5.21",
release="1",
arch="noarch",
epoch="0",
signing_key="f78fb195",
filename="walrus-5.21-1.noarch.rpm",
sourcerpm="walrus-5.21-1.src.rpm",
md5sum="6a3eec6d45e0ea80eab05870bf7a8d4b",
sha1sum="8dea2b64fc52062d79d5f96ba6415bffae4d2153",
sha256sum=
"e837a635cc99f967a70f34b268baa52e0f412c1502e08e924ff5b09f1f9573f2",
content_type_id="rpm",
repository_memberships=["repo1"],
requires=[
RpmDependency(
name="rpmlib(CompressedFileNames)",
version="3.0.4",
release="1",
flags="LE",
epoch="0",
),
RpmDependency(
name="rpmlib(PayloadFilesHavePrefix)",
version="4.0",
release="1",
flags="LE",
epoch="0",
),
],
provides=[
RpmDependency(name="walrus",
version="5.21",
release="1",
flags="EQ",
epoch="0")
],
)
]
def test_can_upload_with_versionless_deps(data_path):
"""repo.upload_rpm() succeeds on RPM having versionless deps."""
rpm_path = os.path.join(data_path,
"rpms/crash-trace-command-1.0-4.el6.src.rpm")
controller = FakeController()
controller.insert_repository(YumRepository(id="repo1"))
client = controller.client
repo1 = client.get_repository("repo1").result()
upload_f = repo1.upload_rpm(rpm_path)
# Upload should complete successfully.
tasks = upload_f.result()
# At least one task.
assert tasks
# Every task should have succeeded.
for t in tasks:
assert t.succeeded
# If I now search for content in that repo, or content across all repos...
units_in_repo = sorted(repo1.search_content().result(),
key=lambda u: u.sha256sum)
units_all = sorted(client.search_content().result(),
key=lambda u: u.sha256sum)
# They should be equal
assert units_all == units_in_repo
# And they should be this
assert units_in_repo == [
RpmUnit(
name="crash-trace-command",
version="1.0",
release="4.el6",
arch="src",
signing_key="fd431d51",
filename="crash-trace-command-1.0-4.el6.src.rpm",
md5sum="e06137db87b9b690fc36e8bac9071771",
sha1sum="bacbb2ee0572d39b1ac8543cfe57c3e584ed986a",
sha256sum=
"633e316eb23603ffc9474f926fcff152b2a61487653be97602fae06fcb46814d",
repository_memberships=["repo1"],
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
requires=[
RpmDependency(name="crash-devel"),
RpmDependency(name="zlib-devel"),
RpmDependency(
name="rpmlib(FileDigests)",
version="4.6.0",
release="1",
epoch="0",
flags="LE",
),
RpmDependency(
name="rpmlib(CompressedFileNames)",
version="3.0.4",
release="1",
epoch="0",
flags="LE",
),
],
provides=[],
)
]
def test_copy_content_all(controller):
"""copy_content with no criteria copies all units from a repo"""
src = YumRepository(id="src-repo")
dest = YumRepository(id="dest-repo")
other = YumRepository(id="other-repo")
controller.insert_repository(src)
controller.insert_repository(dest)
controller.insert_repository(other)
# Set up that both 'src' and 'other' contain some units.
# The units in 'other' are there to ensure that copying only
# happens from the given source repo and not other repos unexpectedly.
src_units = [
ErratumUnit(id="RHSA-1111:22", summary="Fixes bad things"),
ModulemdUnit(
name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64",
repository_memberships=["repoA", "repoB"],
),
RpmUnit(
name="bash",
version="4.0",
release="1",
arch="x86_64",
# Note: the next two fields aren't part of the unit key on RPM units.
# We put these here specifically to verify they'll be filtered out
# on the copy response (as a real Pulp server does).
filename="bash-4.0-1.x86_64.rpm",
signing_key="a1b2c3",
),
]
controller.insert_units(src, src_units)
controller.insert_units(
other,
[
RpmUnit(
name="glibc",
version="5.0",
release="1",
arch="x86_64",
sourcerpm="glibc-5.0-1.el5_11.1.src.rpm",
)
],
)
client = controller.client
# Repos are initially detached, re-fetch them via client
src = client.get_repository(src.id).result()
dest = client.get_repository(dest.id).result()
# It should succeed
copy_tasks = list(client.copy_content(src, dest))
# Gather all units apparently copied
units = sum([t.units for t in copy_tasks], [])
# It should copy just the units we expect, from src.
# Note that these are incomplete views of the units, as (just like real pulp)
# the fake will only return fields present in the unit_key after a copy.
assert sorted(units, key=repr) == [
ErratumUnit(id="RHSA-1111:22"),
ModulemdUnit(name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64"),
RpmUnit(name="bash",
version="4.0",
release="1",
arch="x86_64",
epoch="0"),
]
# The copy should also impact subsequent content searches.
dest_units = list(dest.search_content())
# The units we get from the search are not *precisely* the same as src_units,
# because repository_memberships has been updated.
assert sorted(dest_units, key=repr) == [
ErratumUnit(
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
id="RHSA-1111:22",
summary="Fixes bad things",
content_type_id="erratum",
repository_memberships=["src-repo", "dest-repo"],
),
ModulemdUnit(
unit_id="82e2e662-f728-b4fa-4248-5e3a0a5d2f34",
name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64",
content_type_id="modulemd",
repository_memberships=["src-repo", "dest-repo", "repoA", "repoB"],
),
RpmUnit(
unit_id="d4713d60-c8a7-0639-eb11-67b367a9c378",
name="bash",
version="4.0",
release="1",
arch="x86_64",
epoch="0",
signing_key="a1b2c3",
filename="bash-4.0-1.x86_64.rpm",
content_type_id="rpm",
repository_memberships=["src-repo", "dest-repo"],
),
]
def test_modular_matcher_run(pulp, ubi_config):
"""Test run() method which asynchronously creates criteria for queries to pulp
and exectues those query. Finally it sets up public attrs of the ModularMatcher
object that can be used in ubipop"""
repo_1 = YumRepository(id="binary_repo", )
repo_1.__dict__["_client"] = pulp.client
repo_2 = YumRepository(id="debug_repo", )
repo_2.__dict__["_client"] = pulp.client
repo_3 = YumRepository(id="source_repo", )
repo_3.__dict__["_client"] = pulp.client
# binary
unit_1 = RpmUnit(
name="test",
version="1.0",
release="1",
arch="x86_64",
filename="test-1.0-1.x86_64.x86_64.rpm",
sourcerpm="test-1.0-1.x86_64.src.rpm",
)
# debug
unit_2 = RpmUnit(
name="test-debug",
version="1.0",
release="1",
arch="x86_64",
filename="test-debug-1.0-1.x86_64.rpm",
)
# source
unit_3 = RpmUnit(
name="test-src",
version="1.0",
release="1",
arch="src",
filename="test-1.0-1.x86_64.src.rpm",
content_type_id="srpm",
)
modulemd = ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"test-7:1.0-1.x86_64.x86_64",
"test-debug-7:1.0-1.x86_64",
"test-7:1.0-1.x86_64.src",
],
)
modulemd_defaults = ModulemdDefaultsUnit(
name="fake_name",
stream="fake_stream",
repo_id="binary_repo",
)
pulp.insert_repository(repo_1)
pulp.insert_repository(repo_2)
pulp.insert_repository(repo_3)
pulp.insert_units(repo_1, [unit_1, modulemd, modulemd_defaults])
pulp.insert_units(repo_2, [unit_2])
pulp.insert_units(repo_3, [unit_3])
repos_set = RepoSet(rpm=[repo_1], debug=[repo_2], source=[repo_3])
matcher = ModularMatcher(repos_set, ubi_config.modules)
matcher.run()
# each public attribute is properly set with one unit
assert len(matcher.modules) == 1
assert len(matcher.modulemd_defaults) == 1
assert len(matcher.binary_rpms) == 1
assert len(matcher.debug_rpms) == 1
assert len(matcher.source_rpms) == 1
# each unit is properly queried
output_module = matcher.modules.pop()
assert output_module.nsvca == "fake_name:fake_stream:100:abcd:x86_64"
assert output_module.associate_source_repo_id == "binary_repo"
output_modulemd_defaults = matcher.modulemd_defaults.pop()
assert output_modulemd_defaults.name == "fake_name"
assert output_modulemd_defaults.stream == "fake_stream"
rpm = matcher.binary_rpms.pop()
assert rpm.filename == "test-1.0-1.x86_64.x86_64.rpm"
assert rpm.associate_source_repo_id == "binary_repo"
rpm = matcher.debug_rpms.pop()
assert rpm.filename == "test-debug-1.0-1.x86_64.rpm"
assert rpm.associate_source_repo_id == "debug_repo"
rpm = matcher.source_rpms.pop()
assert rpm.filename == "test-1.0-1.x86_64.src.rpm"
assert rpm.associate_source_repo_id == "source_repo"
def test_delete_unsigned_rpms(command_tester, fake_collector, monkeypatch):
"""Deleting unsigned RPMs from repos succeeds"""
repo = YumRepository(
id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
)
files = [
RpmUnit(
name="signed",
version="1.23",
release="1.test8",
arch="x86_64",
filename="signed-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="signed_rpm",
),
RpmUnit(
name="unsigned",
version="2.25",
release="1.test8",
arch="x86_64",
filename="unsigned-2.25-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key=None,
unit_id="unsigned_rpm",
),
]
with FakeDeletePackages() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo)
task_instance.pulp_client_controller.insert_units(repo, files)
# Let's try setting the cache flush root via env.
monkeypatch.setenv("FASTPURGE_ROOT_URL", "https://cdn.example2.com/")
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--fastpurge-host",
"fakehost-xxx.example.net",
"--fastpurge-client-secret",
"abcdef",
"--fastpurge-client-token",
"efg",
"--fastpurge-access-token",
"tok",
"--repo",
"some-yumrepo",
"--file",
"unsigned-2.25-1.test8_x86_64.rpm,signed-1.23-1.test8_x86_64.rpm",
"--allow-unsigned",
],
)
# It should record that it removed these push items:
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "unsigned-2.25-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
}
]
# verify whether files were deleted on Pulp
client = task_instance.pulp_client
# get the repo where the files were deleted
repos = list(
client.search_repository(Criteria.with_id("some-yumrepo")).result()
)
assert len(repos) == 1
repo = repos[0]
# criteria with the unit_ids
unit_ids = []
for f in files:
unit_ids.append(f.unit_id)
criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# unsigned RPM is deleted, only signed RPM left in the repo
result_files = list(repo.search_content(criteria).result())
assert len(result_files) == 1
assert files[0].filename == "signed-1.23-1.test8_x86_64.rpm"
def populated_units(controller):
units1 = [
RpmUnit(
name="bash",
version="4.0",
release="1",
arch="x86_64",
repository_memberships=["repo1"],
),
RpmUnit(
content_type_id="srpm",
name="bash",
version="4.0",
release="1",
arch="src",
repository_memberships=["repo1"],
),
RpmUnit(
name="glibc",
version="5.0",
release="1",
arch="x86_64",
repository_memberships=["repo1"],
),
ModulemdUnit(
name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64",
repository_memberships=["repo1"],
),
ModulemdUnit(
name="module2",
stream="s2",
version=1234,
context="a1b2",
arch="x86_64",
repository_memberships=["repo1"],
),
ModulemdDefaultsUnit(
name="module1", repo_id="repo1", repository_memberships=["repo1"]
),
]
units2 = [
RpmUnit(
name="glic",
version="2.3.4",
release="1",
arch="x86_64",
repository_memberships=["repo2"],
),
RpmUnit(
content_type_id="srpm",
name="gnu-efi",
version="3.0c",
release="1",
arch="src",
repository_memberships=["repo2"],
),
]
repo1 = YumRepository(id="repo1")
repo2 = YumRepository(id="repo2")
controller.insert_repository(repo1)
controller.insert_units(repo1, units1)
controller.insert_units(repo2, units2)
def test_populate_ubi_repos(get_debug_repository, get_source_repository, requests_mock):
"""Test run of populate_ubi_repos that check correct number of repo publication. It's simplified to
contain only actions on RPM packages."""
dt = datetime(2019, 9, 12, 0, 0, 0)
d1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="ubi_binary",
last_publish=dt,
relative_url="content/unit/2/client",
)
d2 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="ubi_source",
last_publish=dt,
relative_url="content/unit/3/client",
)
d3 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="ubi_debug",
last_publish=dt,
relative_url="content/unit/4/client",
)
output_binary_repo = YumRepository(
id="ubi_binary",
content_set="ubi-8-for-x86_64-appstream-rpms",
population_sources=["input_binary"],
ubi_population=True,
ubi_config_version="8",
eng_product_id=102,
distributors=[d1],
relative_url="content/unit/2/client",
)
input_binary_repo = YumRepository(id="input_binary")
input_source_repo = YumRepository(id="input_source")
input_debug_repo = YumRepository(id="input_debug")
output_source_repo = YumRepository(
id="ubi_source",
population_sources=["input_source"],
eng_product_id=102,
distributors=[d2],
relative_url="content/unit/2/client",
)
output_debug_repo = YumRepository(
id="ubi_debug",
population_sources=["input_debug"],
eng_product_id=102,
distributors=[d3],
relative_url="content/unit/2/client",
)
ubi_populate = FakeUbiPopulate(
"foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR
)
fake_pulp = ubi_populate.pulp_client_controller
fake_pulp.insert_repository(input_binary_repo)
fake_pulp.insert_repository(input_source_repo)
fake_pulp.insert_repository(input_debug_repo)
fake_pulp.insert_repository(output_binary_repo)
fake_pulp.insert_repository(output_source_repo)
fake_pulp.insert_repository(output_debug_repo)
get_debug_repository.return_value = fake_pulp.client.get_repository("ubi_debug")
get_source_repository.return_value = fake_pulp.client.get_repository("ubi_source")
old_rpm = RpmUnit(
name="golang",
version="1",
release="a",
arch="x86_64",
filename="golang-1.a.x86_64.rpm",
sourcerpm="golang-1.a.x86_64.src.rpm",
)
new_rpm = RpmUnit(
name="golang",
version="2",
release="a",
arch="x86_64",
filename="golang-2.a.x86_64.rpm",
sourcerpm="golang-2.a.x86_64.src.rpm",
)
fake_pulp.insert_units(output_binary_repo, [old_rpm])
fake_pulp.insert_units(input_binary_repo, [new_rpm])
url = "/pulp/api/v2/repositories/{dst_repo}/actions/associate/".format(
dst_repo="ubi_binary"
)
requests_mock.register_uri(
"POST", url, json={"spawned_tasks": [{"task_id": "foo_task_id"}]}
)
url = "/pulp/api/v2/repositories/{dst_repo}/actions/unassociate/".format(
dst_repo="ubi_binary"
)
requests_mock.register_uri(
"POST", url, json={"spawned_tasks": [{"task_id": "foo_task_id"}]}
)
url = "/pulp/api/v2/tasks/{task_id}/".format(task_id="foo_task_id")
requests_mock.register_uri(
"GET", url, json={"state": "finished", "task_id": "foo_task_id"}
)
# let's run actual population
ubi_populate.populate_ubi_repos()
history = fake_pulp.publish_history
# there should be 3 repositories succesfully published
assert len(history) == 3
expected_published_repo_ids = set(["ubi_binary", "ubi_debug", "ubi_source"])
repo_ids_published = set()
for publish in history:
assert publish.repository.id in expected_published_repo_ids
repo_ids_published.add(publish.repository.id)
assert len(publish.tasks) == 1
assert publish.tasks[0].completed
assert publish.tasks[0].succeeded
assert repo_ids_published == expected_published_repo_ids
def test_clear_yum_repo(command_tester, fake_collector, monkeypatch):
"""Clearing a repo with yum content succeeds."""
task_instance = FakeClearRepo()
repo = YumRepository(id="some-yumrepo",
relative_url="some/publish/url",
mutable_urls=["repomd.xml"])
files = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
),
ModulemdUnit(name="mymod",
stream="s1",
version=123,
context="a1c2",
arch="s390x"),
]
task_instance.pulp_client_controller.insert_repository(repo)
task_instance.pulp_client_controller.insert_units(repo, files)
# Let's try setting the cache flush root via env.
monkeypatch.setenv("FASTPURGE_ROOT_URL", "https://cdn.example2.com/")
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-clear-repo",
"--pulp-url",
"https://pulp.example.com/",
"--fastpurge-host",
"fakehost-xxx.example.net",
"--fastpurge-client-secret",
"abcdef",
"--fastpurge-client-token",
"efg",
"--fastpurge-access-token",
"tok",
"some-yumrepo",
],
)
# It should record that it removed these push items:
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"state": "DELETED",
"origin": "pulp",
"src": None,
"dest": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"checksums": {
"sha256": "a" * 64,
"md5": "b" * 32
},
"signing_key": "AABBCC",
"build": None,
},
{
"state": "DELETED",
"origin": "pulp",
"src": None,
"dest": None,
"filename": "mymod:s1:123:a1c2:s390x",
"checksums": None,
"signing_key": None,
"build": None,
},
]
# It should have flushed these URLs
assert task_instance.fastpurge_client.purged_urls == [
"https://cdn.example2.com/some/publish/url/repomd.xml"
]
def test_delete_advisory(command_tester, fake_collector, monkeypatch):
"""Deletion of packages and modules in advisories from provided repos succeeds"""
repo1 = YumRepository(id="some-yumrepo",
relative_url="some/publish/url",
mutable_urls=["repomd.xml"])
repo2 = YumRepository(
id="some-other-repo",
relative_url="other/publish/url",
mutable_urls=["repomd.xml"],
)
pkglist = [
ErratumPackageCollection(
name="colection-0",
packages=None,
short="",
module=ErratumModule(name="mymod",
stream="s1",
version="123",
context="a1c2",
arch="s390x"),
),
ErratumPackageCollection(
name="collection-1",
packages=[
ErratumPackage(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
),
ErratumPackage(
name="dash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="dash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
),
],
short="",
module=None,
),
]
files1 = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="files1_rpm1",
),
RpmUnit(
name="dash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="dash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="files1_rpm2",
),
RpmUnit(
name="crash",
version="1.23",
release="1.test8",
arch="x86_64",
filename=
"crash-1.23-1.test8.module+el8.0.0+3049+59fd2bba.x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="files1_rpm3",
),
ModulemdUnit(
name="mymod",
stream="s1",
version=123,
context="a1c2",
arch="s390x",
artifacts=[
"crash-0:1.23-1.test8.module+el8.0.0+3049+59fd2bba.x86_64"
],
unit_id="files1_mod1",
),
ErratumUnit(
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
id="RHSA-1111:22",
summary="Dummy erratum",
content_type_id="erratum",
repository_memberships=["some-yumrepo"],
pkglist=pkglist,
),
]
files2 = [
ErratumUnit(
unit_id="x4e73262-e239-44ac-629f-6fbed82c07cd",
id="RHBA-1001:22",
summary="Other erratum",
content_type_id="erratum",
repository_memberships=["some-other-repo"],
pkglist=[],
),
]
with FakeDeleteAdvisory() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo1)
task_instance.pulp_client_controller.insert_repository(repo2)
task_instance.pulp_client_controller.insert_units(repo1, files1)
task_instance.pulp_client_controller.insert_units(repo2, files2)
# Let's try setting the cache flush root via env.
monkeypatch.setenv("FASTPURGE_ROOT_URL", "https://cdn.example2.com/")
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--fastpurge-host",
"fakehost-xxx.example.net",
"--fastpurge-client-secret",
"abcdef",
"--fastpurge-client-token",
"efg",
"--fastpurge-access-token",
"tok",
"--repo",
"some-yumrepo,other-yumrepo",
"--advisory",
"RHSA-1111:22",
"--advisory",
"RHBA-1001:22",
],
)
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {
"sha256": "a" * 64
},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "crash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {
"sha256": "a" * 64
},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "dash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {
"sha256": "a" * 64
},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "mymod:s1:123:a1c2:s390x",
"state": "DELETED",
"build": None,
"checksums": None,
},
]
# verify whether the rpms and modules were deleted from the repo on Pulp
client = task_instance.pulp_client
# effectively only some-yumrepo(repo1) was modified
repos = list(
client.search_repository(
Criteria.with_id("some-yumrepo")).result())
assert len(repos) == 1
repo = repos[0]
# list the removed unit's unit_id
# RPMs from the erratum package list
unit_ids = ["files1_rpm1", "files1_rpm2"]
# module from the erratum package list
unit_ids.append("files1_mod1")
# package in the above module
unit_ids.append("files1_rpm3")
criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# deleted files are not in the repo
files = list(repo.search_content(criteria).result())
assert len(files) == 0
# same files exist on Pulp as orphans
files_search = list(client.search_content(criteria).result())
assert len(files_search) == 4
def test_get_current_content(mock_ubipop_runner, pulp, skip_debug_repo):
"""Tests getting current content from ubi repos, using Fake Client from pubtools-pulplib"""
rpm_repo = YumRepository(
id="rpm_repo",
)
rpm_repo.__dict__["_client"] = pulp.client
debug_repo = YumRepository(
id="debug_repo",
)
debug_repo.__dict__["_client"] = pulp.client
source_repo = YumRepository(
id="source_repo",
)
source_repo.__dict__["_client"] = pulp.client
binary_rpm = RpmUnit(name="test", version="1.0", release="1", arch="x86_64")
modulemd = ModulemdUnit(
name="test_module_md",
stream="fake-stream",
version=1,
context="fake-context",
arch="x86_64",
)
modulemd_defaults = ModulemdDefaultsUnit(
name="test_modulemd_defaults", stream="rhel", repo_id="rpm_repo"
)
debug_rpm = RpmUnit(
name="test-debuginfo", version="1.0", release="1", arch="x86_64"
)
source_rpm = RpmUnit(
name="test-srpm", version="1.0", release="1", arch="src", content_type_id="srpm"
)
pulp.insert_repository(rpm_repo)
pulp.insert_units(rpm_repo, [binary_rpm, modulemd, modulemd_defaults])
if not skip_debug_repo:
pulp.insert_repository(debug_repo)
pulp.insert_units(debug_repo, [debug_rpm])
pulp.insert_repository(source_repo)
pulp.insert_units(source_repo, [source_rpm])
if skip_debug_repo:
debug_repo = f_return(None)
else:
debug_repo = f_proxy(f_return(debug_repo))
# overwrite out_repos with the testing ones
mock_ubipop_runner.repos.out_repos = RepoSet(
f_proxy(f_return(rpm_repo)), f_proxy(f_return(source_repo)), debug_repo
)
content = mock_ubipop_runner._get_current_content()
binary_rpms = list(content.binary_rpms)
assert len(binary_rpms) == 1
assert binary_rpms[0].name == "test"
modules = list(content.modules)
assert len(modules) == 1
assert modules[0].name == "test_module_md"
modulemd_defaults = list(content.modulemd_defaults)
assert len(modulemd_defaults) == 1
assert modulemd_defaults[0].name == "test_modulemd_defaults"
debug_rpms = list(content.debug_rpms)
if skip_debug_repo:
assert len(debug_rpms) == 0
else:
assert len(debug_rpms) == 1
assert debug_rpms[0].name == "test-debuginfo"
source_rpms = list(content.source_rpms)
assert len(source_rpms) == 1
assert source_rpms[0].name == "test-srpm"
def test_delete_advisory_no_repos_provided(command_tester, fake_collector,
monkeypatch):
"""Deletion of packages succeeds in all the repos when the same advisory is
present in multiple repos and repos are not provided in the request"""
repo1 = YumRepository(id="some-yumrepo",
relative_url="some/publish/url",
mutable_urls=["repomd.xml"])
repo2 = YumRepository(
id="other-yumrepo",
relative_url="other/publish/url",
mutable_urls=["repomd.xml"],
)
pkglist = [
ErratumPackageCollection(
name="collection-1",
packages=[
ErratumPackage(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
),
ErratumPackage(
name="dash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="dash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
),
],
short="",
module=None,
),
]
files = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="files1_rpm1",
),
RpmUnit(
name="dash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="dash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="files1_rpm2",
),
RpmUnit(
name="crash",
version="1.23",
release="1.test8",
arch="x86_64",
filename=
"crash-1.23-1.test8.module+el8.0.0+3049+59fd2bba.x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="files1_rpm3",
),
ErratumUnit(
unit_id="x4e73262-e239-44ac-629f-6fbed82c07cd",
id="RHBA-1001:22",
summary="Other erratum",
content_type_id="erratum",
repository_memberships=["some-yumrepo", "other-yumrepo"],
pkglist=pkglist,
),
]
with FakeDeleteAdvisory() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo1)
task_instance.pulp_client_controller.insert_repository(repo2)
task_instance.pulp_client_controller.insert_units(repo1, files)
task_instance.pulp_client_controller.insert_units(repo2, files)
# Let's try setting the cache flush root via env.
monkeypatch.setenv("FASTPURGE_ROOT_URL", "https://cdn.example2.com/")
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--fastpurge-host",
"fakehost-xxx.example.net",
"--fastpurge-client-secret",
"abcdef",
"--fastpurge-client-token",
"efg",
"--fastpurge-access-token",
"tok",
"--advisory",
"RHBA-1001:22",
],
)
assert sorted(fake_collector.items,
key=lambda pi: (pi["filename"], pi["dest"])) == [
{
"build": None,
"checksums": {
"sha256": "a" * 64
},
"dest": "other-yumrepo",
"filename": "bash-1.23-1.test8.x86_64.rpm",
"origin": "pulp",
"signing_key": None,
"src": None,
"state": "DELETED",
},
{
"build": None,
"checksums": {
"sha256": "a" * 64
},
"dest": "some-yumrepo",
"filename": "bash-1.23-1.test8.x86_64.rpm",
"origin": "pulp",
"signing_key": None,
"src": None,
"state": "DELETED",
},
{
"build": None,
"checksums": {
"sha256": "a" * 64
},
"dest": "other-yumrepo",
"filename": "dash-1.23-1.test8.x86_64.rpm",
"origin": "pulp",
"signing_key": None,
"src": None,
"state": "DELETED",
},
{
"build": None,
"checksums": {
"sha256": "a" * 64
},
"dest": "some-yumrepo",
"filename": "dash-1.23-1.test8.x86_64.rpm",
"origin": "pulp",
"signing_key": None,
"src": None,
"state": "DELETED",
},
]
# verify whether the rpms were deleted from the repo on Pulp
client = task_instance.pulp_client
# get all the repos
repos = list(
client.search_repository(
Criteria.with_id("some-yumrepo")).result())
assert len(repos) == 1
repo1 = repos[0]
repos = list(
client.search_repository(
Criteria.with_id("other-yumrepo")).result())
assert len(repos) == 1
repo2 = repos[0]
# list the removed unit's unit_id
# RPMs from the erratum package list
unit_ids = ["files1_rpm1", "files1_rpm2"]
criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# deleted packages from the advisory are not in both the repos
files = list(repo1.search_content(criteria).result())
assert len(files) == 0
files = list(repo2.search_content(criteria).result())
assert len(files) == 0
# same files exist on Pulp as orphans
files_search = list(client.search_content(criteria).result())
assert len(files_search) == 2
def test_get_rpm_output_set_include_modular_and_all_versions(ubi_config):
"""tests getting rpm output set from RpmMatcher, modular packages included
as well as all version of given package, only blacklist is applied"""
matcher = RpmMatcher(None, ubi_config)
# This unit will be in the output set - we allow all versions
unit_1 = UbiUnit(
RpmUnit(
name="test",
version="10",
release="20",
arch="x86_64",
),
None,
)
# this one will be excluded using blacklist
unit_2 = UbiUnit(
RpmUnit(
name="excluded_with_globbing123456789",
version="11",
release="20",
arch="x86_64",
),
None,
)
# this one will be excluded using blacklist
unit_3 = UbiUnit(
RpmUnit(
name="excluded_package",
version="10",
release="20",
arch="x86_64",
),
None,
)
# this is another unit that will be in the output set
unit_4 = UbiUnit(
RpmUnit(
name="test",
version="11",
release="20",
arch="x86_64",
),
None,
)
# this one is included, we don't exclude modular packages this time
unit_5 = UbiUnit(
RpmUnit(
name="modular_package",
version="12",
release="20",
arch="x86_64",
filename="modular_package.rpm",
),
None,
)
rpms = [unit_1, unit_2, unit_3, unit_4, unit_5]
output = matcher._get_rpm_output_set(rpms,
modular_rpm_filenames=None,
keep_all_versions=True)
# in the output set there should be three units
assert len(output) == 3
# let's sort the output, order is not guranteed
output.sort(key=lambda x: x.version)
assert output[0].name == "test"
assert output[0].version == "10"
assert output[1].name == "test"
assert output[1].version == "11"
assert output[2].name == "modular_package"
assert output[2].version == "12"
def test_get_rpm_output_set(ubi_config):
"""tests getting rpm output set from RpmMatcher"""
matcher = RpmMatcher(None, ubi_config)
# this unit won't be in output set, there is a newver one - unit_4 with higher version
unit_1 = UbiUnit(
RpmUnit(
name="test",
version="10",
release="20",
arch="x86_64",
),
None,
)
# this one will be excluded using blacklist
unit_2 = UbiUnit(
RpmUnit(
name="excluded_with_globbing123456789",
version="11",
release="20",
arch="x86_64",
),
None,
)
# this one will be excluded using blacklist
unit_3 = UbiUnit(
RpmUnit(
name="excluded_package",
version="10",
release="20",
arch="x86_64",
),
None,
)
# this it the only one that will in output set
unit_4 = UbiUnit(
RpmUnit(
name="test",
version="11",
release="20",
arch="x86_64",
),
None,
)
# this one is excluded because it's a modular package
unit_5 = UbiUnit(
RpmUnit(
name="modular_package",
version="10",
release="20",
arch="x86_64",
filename="modular_package.rpm",
),
None,
)
modular_pkgs_filenames = f_proxy(f_return(set(["modular_package.rpm"])))
rpms = [unit_1, unit_2, unit_3, unit_4, unit_5]
output = matcher._get_rpm_output_set(rpms, modular_pkgs_filenames)
# in the output set there is only one unit, according to the rules
assert len(output) == 1
assert output[0].name == "test"
assert output[0].version == "11"
def test_clear_repo_multiple_content_types(command_tester, fake_collector,
monkeypatch):
"""Test clearing a Yum repo given multiple content type values."""
repo = YumRepository(id="some-yumrepo",
relative_url="some/publish/url",
mutable_urls=["repomd.xml"])
files = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
),
ModulemdUnit(name="mymod",
stream="s1",
version=123,
context="a1c2",
arch="s390x"),
]
with FakeClearRepo() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo)
task_instance.pulp_client_controller.insert_units(repo, files)
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-clear-repo",
"--pulp-url",
"https://pulp.example.com/",
"--content-type",
"rpm",
"--content-type",
"modulemd",
"--content-type",
"iso",
"some-yumrepo",
],
)
# test the new ClearRepo argument handling for --content-type
# produces the expected output
assert task_instance.args.content_type == ["rpm", "modulemd", "iso"]
# It should record that it removed these push items:
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"state": "DELETED",
"origin": "pulp",
"src": None,
"dest": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"checksums": {
"sha256": "a" * 64
},
"signing_key": None,
"build": None,
},
{
"state": "DELETED",
"origin": "pulp",
"src": None,
"dest": None,
"filename": "mymod:s1:123:a1c2:s390x",
"checksums": None,
"signing_key": None,
"build": None,
},
]
def test_can_search_content(client, requests_mocker):
"""search_content basic call"""
requests_mocker.get(
"https://pulp.example.com/pulp/api/v2/plugins/types/",
json=[{
"id": "rpm"
}, {
"id": "srpm"
}],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/content/units/rpm/search/",
json=RPM_TEST_UNITS,
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/content/units/srpm/search/",
json=SRPM_TEST_UNITS,
)
units = client.search_content()
# It should have returned the repos as objects
assert sorted(units) == [
RpmUnit(
content_type_id="srpm",
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="src",
epoch="0",
name="glibc",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm=None,
version="2.3.4",
),
RpmUnit(
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="ia64",
epoch="0",
name="glibc-headers",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm="glibc-2.3.4-2.57.el4.1.src.rpm",
version="2.3.4",
),
RpmUnit(
sha1sum="ca995eb1a635c97393466f67aaec8e9e753b8ed5",
sha256sum=
"1c4baac658fd56e6ec9cca37f440a4bd8c9c0b02a21f41b30b8ea17b402a1907",
arch="i386",
epoch="0",
name="gnu-efi-debuginfo",
release="1.1",
repository_memberships=[
"fake-repository-id-1", "fake-repository-id-2"
],
sourcerpm="gnu-efi-3.0c-1.1.src.rpm",
version="3.0c",
),
]
# 3 requests, 1 for server type_ids, 1 for rpm, 1 for srpm
assert requests_mocker.call_count == 3
def test_remove_loads_units(fast_poller, requests_mocker, client):
"""Remove returns unit info loaded from units_successful."""
repo = Repository(id="some-repo")
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/unassociate/",
[{"json": {"spawned_tasks": [{"task_id": "task1"}]}}],
)
unit_data = [
{
"type_id": "iso",
"unit_key": {"name": "hello.txt", "size": 23, "checksum": "a" * 64},
},
{
"type_id": "rpm",
"unit_key": {
"name": "bash",
"epoch": "0",
"version": "4.0",
"release": "1",
"arch": "x86_64",
},
},
{
"type_id": "modulemd",
"unit_key": {
"name": "module",
"stream": "s1",
"version": 1234,
"context": "a1b2c3",
"arch": "s390x",
},
},
{
"type_id": "yum_repo_metadata_file",
"unit_key": {"data_type": "productid", "repo_id": "some-repo"},
},
{"type_id": "bizarre_type", "unit_key": {"whatever": "data"}},
]
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{
"json": [
{
"task_id": "task1",
"state": "finished",
"result": {"units_successful": unit_data},
}
]
}
],
)
tasks = repo.remove_content().result()
# It should return one task
assert len(tasks) == 1
task = tasks[0]
# It should be the expected successful task
assert task.id == "task1"
assert task.completed
assert task.succeeded
# It should have loaded expected units from the units_successful dict
assert set(task.units) == set(
[
FileUnit(path="hello.txt", size=23, sha256sum="a" * 64),
RpmUnit(
name="bash",
epoch="0",
version="4.0",
release="1",
arch="x86_64",
sourcerpm=None,
),
ModulemdUnit(
name="module", stream="s1", version=1234, context="a1b2c3", arch="s390x"
),
YumRepoMetadataFileUnit(
data_type="productid", content_type_id="yum_repo_metadata_file"
),
Unit(content_type_id="bizarre_type"),
]
)
def test_clients_share_state(controller):
"""Multiple clients created by the same controller share state"""
src = YumRepository(id="src-repo")
dest = YumRepository(id="dest-repo")
controller.insert_repository(src)
controller.insert_repository(dest)
src_units = [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="bash", version="4.0", release="2", arch="x86_64"),
RpmUnit(name="bash", version="4.1", release="3", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
controller.insert_units(src, src_units)
client1 = controller.new_client()
client2 = controller.new_client()
# The two clients should not be the same object
assert client1 is not client2
# Repos are initially detached, re-fetch them via client
src = client1.get_repository(src.id).result()
dest = client1.get_repository(dest.id).result()
# Do a copy via client1
client1.copy_content(
src, dest, options=CopyOptions(require_signed_rpms=False)).result()
# Then search for content via client2
found_units = list(client2.search_content())
# It should be able to see the outcome of the copy done via client1;
# i.e. repository_memberships contains both repos after the copy
assert sorted(found_units, key=repr) == [
RpmUnit(
name="bash",
version="4.0",
release="1",
arch="x86_64",
repository_memberships=["src-repo", "dest-repo"],
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
),
RpmUnit(
name="bash",
version="4.0",
release="2",
arch="x86_64",
repository_memberships=["src-repo", "dest-repo"],
unit_id="82e2e662-f728-b4fa-4248-5e3a0a5d2f34",
),
RpmUnit(
name="bash",
version="4.1",
release="3",
arch="x86_64",
repository_memberships=["src-repo", "dest-repo"],
unit_id="d4713d60-c8a7-0639-eb11-67b367a9c378",
),
RpmUnit(
name="glibc",
version="5.0",
release="1",
arch="x86_64",
repository_memberships=["src-repo", "dest-repo"],
unit_id="23a7711a-8133-2876-37eb-dcd9e87a1613",
),
]
def test_delete_rpms(command_tester, fake_collector, monkeypatch):
"""Deleting RPMs from repos succeeds"""
repo1 = YumRepository(
id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
)
repo2 = YumRepository(
id="other-yumrepo",
relative_url="other/publish/url",
mutable_urls=["repomd.xml"],
)
files1 = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file1_rpm1",
),
RpmUnit(
name="dash",
version="2.25",
release="1.test8",
arch="x86_64",
filename="dash-2.25-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file1_rpm2",
),
ModulemdUnit(
name="mymod",
stream="s1",
version=123,
context="a1c2",
arch="s390x",
unit_id="file1_mod1",
),
]
files2 = [
RpmUnit(
name="crash",
version="3.30",
release="1.test8",
arch="s390x",
filename="crash-3.30-1.test8_s390x.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file2_rpm1",
)
]
files3 = [
RpmUnit(
name="rash",
version="1.30",
release="1.test8",
arch="noarch",
filename="rash-1.30-1.test8_noarch.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file3_rpm1",
)
]
undeleted = [
RpmUnit(
name="exist",
version="1.34",
release="1.test8",
arch="noarch",
filename="exist-1.34-1.test8_noarch.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="undeleted_rpm1",
)
]
files1.extend(files3)
files1.extend(undeleted)
files2.extend(files3)
with FakeDeletePackages() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo1)
task_instance.pulp_client_controller.insert_repository(repo2)
task_instance.pulp_client_controller.insert_units(repo1, files1)
task_instance.pulp_client_controller.insert_units(repo2, files2)
# Let's try setting the cache flush root via env.
monkeypatch.setenv("FASTPURGE_ROOT_URL", "https://cdn.example2.com/")
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--fastpurge-host",
"fakehost-xxx.example.net",
"--fastpurge-client-secret",
"abcdef",
"--fastpurge-client-token",
"efg",
"--fastpurge-access-token",
"tok",
"--repo",
"some-yumrepo,other-yumrepo",
"--repo",
"some-other-repo",
"--file",
"bash-1.23-1.test8_x86_64.rpm",
"--file",
"dash-2.25-1.test8_x86_64.rpm,crash-3.30-1.test8_s390x.rpm",
"--file",
"trash-1.0-1.test8_noarch.rpm,rash-1.30-1.test8_noarch.rpm",
"--signing-key",
"aabbcc",
],
)
# It should record that it removed these push items:
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "other-yumrepo",
"signing_key": None,
"filename": "crash-3.30-1.test8.s390x.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "dash-2.25-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "other-yumrepo",
"signing_key": None,
"filename": "rash-1.30-1.test8.noarch.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "rash-1.30-1.test8.noarch.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
]
# verify whether files were deleted on Pulp
client = task_instance.pulp_client
# get the repo where the files were deleted
repos = sorted(
list(
client.search_repository(
Criteria.with_id(["some-yumrepo", "other-yumrepo"])
).result()
),
key=lambda r: r.id,
)
assert len(repos) == 2
r2, r1 = repos
assert r1.id == repo1.id
assert r2.id == repo2.id
# criteria with the unit_ids
# critera1 for files1 in repo1
unit_ids = []
for f in files1:
unit_ids.append(f.unit_id)
criteria1 = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# critera2 for files2 in repo2
unit_ids = []
for f in files2:
unit_ids.append(f.unit_id)
criteria2 = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# files are not in the repo1 except undeleted rpm and module
result1 = sorted(
list(r1.search_content(criteria1).result()), key=lambda v: v.unit_id
)
assert len(result1) == 2
# modulemd in files1
assert result1[0].unit_id == files1[2].unit_id
# undeleted file
assert result1[1].unit_id == undeleted[0].unit_id
# files are not in repo2
result2 = list(r2.search_content(criteria1).result())
assert len(result2) == 0
# All the files exist on Pulp
files_search = list(
client.search_content(Criteria.or_(criteria1, criteria2)).result()
)
assert len(files_search) == 6
def test_can_search_content(client, requests_mocker):
"""search_content basic call"""
requests_mocker.get(
"https://pulp.example.com/pulp/api/v2/plugins/types/",
json=[{
"id": "rpm"
}, {
"id": "srpm"
}],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/content/units/rpm/search/",
json=RPM_TEST_UNITS,
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/content/units/srpm/search/",
json=SRPM_TEST_UNITS,
)
units = client.search_content()
# It should have returned the repos as objects
assert sorted(units) == [
RpmUnit(
content_type_id="srpm",
unit_id="bd2e0321-48f6-4997-a5dc-e73c771bc17d",
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="src",
epoch="0",
name="glibc",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm=None,
version="2.3.4",
),
RpmUnit(
unit_id="bd2e0321-48f6-4997-a5dc-e73c771bc17d",
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="ia64",
epoch="0",
name="glibc-headers",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm="glibc-2.3.4-2.57.el4.1.src.rpm",
version="2.3.4",
),
RpmUnit(
unit_id="d4633746-1ccc-4d85-9733-0007c87e0724",
sha1sum="ca995eb1a635c97393466f67aaec8e9e753b8ed5",
sha256sum=
"1c4baac658fd56e6ec9cca37f440a4bd8c9c0b02a21f41b30b8ea17b402a1907",
arch="i386",
epoch="0",
name="gnu-efi-debuginfo",
release="1.1",
repository_memberships=[
"fake-repository-id-1", "fake-repository-id-2"
],
sourcerpm="gnu-efi-3.0c-1.1.src.rpm",
version="3.0c",
),
]
# 3 requests, 1 for server type_ids, 1 for rpm, 1 for srpm
assert requests_mocker.call_count == 3
# Request body should look like this: limit is always applied,
# repos are always requested
assert requests_mocker.request_history[-1].json() == {
"criteria": {
"skip": 0,
"limit": 2000,
"filters": {}
},
"include_repos": True,
}
def test_delete_modules(command_tester, fake_collector, monkeypatch):
"""Deleting modules and it's artifacts from repos succeeds"""
repo = YumRepository(
id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
)
files = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
provides=[],
requires=[],
unit_id="rpm1",
),
RpmUnit(
name="dash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="dash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
provides=[],
requires=[],
unit_id="rpm2",
),
ModulemdUnit(
name="mymod",
stream="s1",
version=123,
context="a1c2",
arch="s390x",
artifacts=["bash-0:1.23-1.test8_x86_64", "dash-0:1.23-1.test8_x86_64"],
unit_id="module1",
),
]
with FakeDeletePackages() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo)
task_instance.pulp_client_controller.insert_units(repo, files)
# Let's try setting the cache flush root via env.
monkeypatch.setenv("FASTPURGE_ROOT_URL", "https://cdn.example2.com/")
# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--fastpurge-host",
"fakehost-xxx.example.net",
"--fastpurge-client-secret",
"abcdef",
"--fastpurge-client-token",
"efg",
"--fastpurge-access-token",
"tok",
"--repo",
"some-yumrepo",
"--file",
"mymod:s1:123:a1c2:s390x",
"--signing-key",
"aabbcc",
],
)
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"origin": "pulp",
"src": None,
"state": "DELETED",
"build": None,
"dest": "some-yumrepo",
"checksums": {"sha256": "a" * 64},
"signing_key": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
},
{
"origin": "pulp",
"src": None,
"state": "DELETED",
"build": None,
"dest": "some-yumrepo",
"checksums": {"sha256": "a" * 64},
"signing_key": None,
"filename": "dash-1.23-1.test8.x86_64.rpm",
},
{
"origin": "pulp",
"src": None,
"state": "DELETED",
"build": None,
"dest": "some-yumrepo",
"checksums": None,
"signing_key": None,
"filename": "mymod:s1:123:a1c2:s390x",
},
]
# verify whether files were deleted on Pulp
client = task_instance.pulp_client
# get the repo where the files were deleted
repos = list(
client.search_repository(Criteria.with_id("some-yumrepo")).result()
)
assert len(repos) == 1
repo = repos[0]
# criteria with the unit_ids
unit_ids = []
for f in files:
unit_ids.append(f.unit_id)
criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
# deleted files are not in the repo
files = list(repo.search_content(criteria).result())
assert len(files) == 0
# same files exist on Pulp as orphans
files_search = list(client.search_content(criteria).result())
assert len(files_search) == 3
def test_can_search_content_pagination(client, requests_mocker):
"""search_content pagination should search only for types that returns non-empty"""
requests_mocker.get(
"https://pulp.example.com/pulp/api/v2/plugins/types/",
json=[{
"id": "rpm"
}, {
"id": "srpm"
}],
)
requests_mocker.register_uri(
"POST",
"https://pulp.example.com/pulp/api/v2/content/units/rpm/search/",
[{
"json": [RPM_TEST_UNITS[0]]
}, {
"json": [RPM_TEST_UNITS[1]]
}, {
"json": []
}],
)
requests_mocker.register_uri(
"POST",
"https://pulp.example.com/pulp/api/v2/content/units/srpm/search/",
[{
"json": SRPM_TEST_UNITS
}, {
"json": []
}],
)
client._PAGE_SIZE = 1
units = client.search_content()
# It should have returned the repos as objects
assert sorted(units) == [
RpmUnit(
unit_id="bd2e0321-48f6-4997-a5dc-e73c771bc17d",
content_type_id="srpm",
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="src",
epoch="0",
name="glibc",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm=None,
version="2.3.4",
),
RpmUnit(
unit_id="bd2e0321-48f6-4997-a5dc-e73c771bc17d",
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="ia64",
epoch="0",
name="glibc-headers",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm="glibc-2.3.4-2.57.el4.1.src.rpm",
version="2.3.4",
),
RpmUnit(
unit_id="d4633746-1ccc-4d85-9733-0007c87e0724",
sha1sum="ca995eb1a635c97393466f67aaec8e9e753b8ed5",
sha256sum=
"1c4baac658fd56e6ec9cca37f440a4bd8c9c0b02a21f41b30b8ea17b402a1907",
arch="i386",
epoch="0",
name="gnu-efi-debuginfo",
release="1.1",
repository_memberships=[
"fake-repository-id-1", "fake-repository-id-2"
],
sourcerpm="gnu-efi-3.0c-1.1.src.rpm",
version="3.0c",
),
]
# 3 requests, 1 for server type_ids, 1 for rpm, 1 for srpm
assert [h.url for h in requests_mocker.request_history].count(
"https://pulp.example.com/pulp/api/v2/content/units/rpm/search/") == 3
assert [h.url for h in requests_mocker.request_history].count(
"https://pulp.example.com/pulp/api/v2/content/units/srpm/search/") == 2
def test_copy_content_with_criteria(controller):
"""copy_content can filter copied units by field values"""
src = YumRepository(id="src-repo")
dest = YumRepository(id="dest-repo")
controller.insert_repository(src)
controller.insert_repository(dest)
src_units = [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="bash", version="4.0", release="2", arch="x86_64"),
RpmUnit(name="bash", version="4.1", release="3", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
controller.insert_units(src, src_units)
client = controller.client
# Repos are initially detached, re-fetch them via client
src = client.get_repository(src.id).result()
dest = client.get_repository(dest.id).result()
# This is what we want to copy...
crit = Criteria.and_(
Criteria.with_field("name", "bash"),
Criteria.with_field("release", Matcher.in_(["1", "3"])),
)
# Copy should succeed
copy_tasks = list(
client.copy_content(src,
dest,
crit,
options=CopyOptions(require_signed_rpms=False)))
# It should have copied only those units matching the criteria
units = sum([t.units for t in copy_tasks], [])
assert sorted(units, key=repr) == [
RpmUnit(name="bash",
version="4.0",
release="1",
arch="x86_64",
epoch="0"),
RpmUnit(name="bash",
version="4.1",
release="3",
arch="x86_64",
epoch="0"),
]
# The copy should also impact subsequent content searches.
dest_units = list(dest.search_content())
assert sorted(dest_units, key=repr) == [
RpmUnit(
unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd",
name="bash",
version="4.0",
release="1",
arch="x86_64",
epoch="0",
repository_memberships=["src-repo", "dest-repo"],
),
RpmUnit(
unit_id="d4713d60-c8a7-0639-eb11-67b367a9c378",
name="bash",
version="4.1",
release="3",
arch="x86_64",
epoch="0",
repository_memberships=["src-repo", "dest-repo"],
),
]
def test_can_search_content_by_type(client, requests_mocker):
"""search_content can search for a specified unit type."""
requests_mocker.get(
"https://pulp.example.com/pulp/api/v2/plugins/types/",
json=[{
"id": "rpm"
}, {
"id": "srpm"
}, {
"id": "iso"
}],
)
# Note although "iso" is supported, we don't mock the search URL for
# that content type, thus proving that we don't query it if we search
# for a specific unit_type
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/content/units/rpm/search/",
json=RPM_TEST_UNITS,
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/content/units/srpm/search/",
json=SRPM_TEST_UNITS,
)
units = client.search_content(Criteria.with_unit_type(RpmUnit))
# It should have returned the expected units
assert sorted(units) == [
RpmUnit(
unit_id="bd2e0321-48f6-4997-a5dc-e73c771bc17d",
content_type_id="srpm",
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="src",
epoch="0",
name="glibc",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm=None,
version="2.3.4",
),
RpmUnit(
unit_id="bd2e0321-48f6-4997-a5dc-e73c771bc17d",
sha256sum=
"4f5a3a0da6f404f6d9988987cd75f13982bd655a0a4f692406611afbbc597679",
arch="ia64",
epoch="0",
name="glibc-headers",
release="2.57.el4.1",
repository_memberships=["fake-repository-id-3"],
sourcerpm="glibc-2.3.4-2.57.el4.1.src.rpm",
version="2.3.4",
),
RpmUnit(
unit_id="d4633746-1ccc-4d85-9733-0007c87e0724",
sha1sum="ca995eb1a635c97393466f67aaec8e9e753b8ed5",
sha256sum=
"1c4baac658fd56e6ec9cca37f440a4bd8c9c0b02a21f41b30b8ea17b402a1907",
arch="i386",
epoch="0",
name="gnu-efi-debuginfo",
release="1.1",
repository_memberships=[
"fake-repository-id-1", "fake-repository-id-2"
],
sourcerpm="gnu-efi-3.0c-1.1.src.rpm",
version="3.0c",
),
]
def test_remove_filtered_content():
"""repo.remove_content() succeeds and removes expected units inserted via controller."""
controller = FakeController()
client = controller.client
rpm_units = [
RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"),
RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"),
]
modulemd_units = [
ModulemdUnit(name="module1",
stream="s1",
version=1234,
context="a1b2",
arch="x86_64"),
ModulemdUnit(name="module1",
stream="s1",
version=1235,
context="a1b2",
arch="x86_64"),
]
units = rpm_units + modulemd_units
repo = YumRepository(id="repo1")
controller.insert_repository(repo)
controller.insert_units(repo, units)
criteria = Criteria.and_(Criteria.with_unit_type(RpmUnit),
Criteria.with_field("name", "bash"))
remove_rpms = client.get_repository("repo1").remove_content(
criteria=criteria)
assert len(remove_rpms) == 1
task = remove_rpms[0]
# It should have completed successfully
assert task.completed
assert task.succeeded
# It should have removed the specific rpm
assert len(task.units) == 1
assert task.units[0] == sorted(rpm_units)[0]
# Try removing a module with incorrect type_ids
criteria = Criteria.and_(Criteria.with_unit_type(RpmUnit),
Criteria.with_field("name", "module1"))
remove_rpms = client.get_repository("repo1").remove_content(
criteria=criteria)
assert len(remove_rpms) == 1
task = remove_rpms[0]
# It should have completed successfully
assert task.completed
assert task.succeeded
# Nothing's removed as criteria isn't fulfilled
assert len(task.units) == 0
# Removing module with correct type_ids
criteria = Criteria.and_(Criteria.with_unit_type(ModulemdUnit),
Criteria.with_field("name", "module1"))
remove_rpms = client.get_repository("repo1").remove_content(
criteria=criteria)
assert len(remove_rpms) == 1
task = remove_rpms[0]
# It should have completed successfully
assert task.completed
assert task.succeeded
# It should have removed both the modules as they
# match the criteria
assert len(task.units) == 2
assert sorted(task.units) == sorted(modulemd_units)
def test_rpm_matcher_run(pulp, ubi_config):
"""Test run() method which asynchronously creates criteria for queries to pulp
and exectues those query. Finally it sets up public attrs of the RpmMatcher
object that can be used in ubipop"""
repo_1 = YumRepository(id="binary_repo", )
repo_1.__dict__["_client"] = pulp.client
repo_2 = YumRepository(id="debug_repo", )
repo_2.__dict__["_client"] = pulp.client
repo_3 = YumRepository(id="source_repo", )
repo_3.__dict__["_client"] = pulp.client
# binary - will be in output set
unit_1 = RpmUnit(
name="test",
version="1.0",
release="1",
arch="x86_64",
filename="test-1.0-1.x86_64.rpm",
sourcerpm="test-1.0-1.src.rpm",
)
# debug - will be in output set
unit_2 = RpmUnit(
name="test-debug",
version="1.0",
release="1",
arch="x86_64",
filename="test-debug-1.0-1.x86_64.rpm",
sourcerpm="test-1.0-1.src.rpm",
)
# source - will be in output set
unit_3 = RpmUnit(
name="test",
version="1.0",
release="1",
arch="src",
filename="test-1.0-1.src.rpm",
content_type_id="srpm",
)
# modular - will be skipped
unit_4 = RpmUnit(
name="test",
version="100.0",
release="1",
arch="x86_64",
filename="test-100.0-1.x86_64.rpm",
sourcerpm="test-100.0-1.src.rpm",
)
# blacklisted - will be also skipped
unit_5 = RpmUnit(
name="excluded_package",
version="1.0",
release="1",
arch="x86_64",
filename="excluded_package-1.0-1.x86_64.rpm",
sourcerpm="excluded_package-1.0-1.src.rpm",
)
# non-matching - not in output
unit_6 = RpmUnit(
name="no_match",
version="1.0",
release="1",
arch="x86_64",
filename="no_match-1.0-1.x86_64.rpm",
sourcerpm="no_match-1.0-1.src.rpm",
)
modulemd = ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"test-0:100.0-1.x86_64",
],
)
pulp.insert_repository(repo_1)
pulp.insert_repository(repo_2)
pulp.insert_repository(repo_3)
pulp.insert_units(repo_1, [unit_1, modulemd, unit_4, unit_5, unit_6])
pulp.insert_units(repo_2, [unit_2])
pulp.insert_units(repo_3, [unit_3])
repos_set = RepoSet(rpm=[repo_1], debug=[repo_2], source=[repo_3])
matcher = RpmMatcher(repos_set, ubi_config)
matcher.run()
# each public attribute is properly set with one unit
assert len(matcher.binary_rpms) == 1
assert len(matcher.debug_rpms) == 1
assert len(matcher.source_rpms) == 1
# each unit is properly queried
rpm = matcher.binary_rpms.pop()
assert rpm.filename == "test-1.0-1.x86_64.rpm"
assert rpm.associate_source_repo_id == "binary_repo"
rpm = matcher.debug_rpms.pop()
assert rpm.filename == "test-debug-1.0-1.x86_64.rpm"
assert rpm.associate_source_repo_id == "debug_repo"
rpm = matcher.source_rpms.pop()
assert rpm.filename == "test-1.0-1.src.rpm"
assert rpm.associate_source_repo_id == "source_repo"