def test_content_remote_delete(self):
"""Assert that an HTTP error is raised when remote is deleted.
Also verify that the content can be downloaded from Pulp once the
remote is recreated and another sync is triggered.
"""
cfg = config.get_config()
delete_orphans()
client = api.Client(cfg, api.page_handler)
repo = client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["pulp_href"])
body = gen_file_remote(policy=choice(ON_DEMAND_DOWNLOAD_POLICIES))
remote = client.post(FILE_REMOTE_PATH, body)
# Sync the repository using a lazy download policy.
sync(cfg, remote, repo)
repo = client.get(repo["pulp_href"])
publication = create_file_publication(cfg, repo)
self.addCleanup(client.delete, publication["pulp_href"])
# Delete the remote.
client.delete(remote["pulp_href"])
body = gen_distribution()
body["publication"] = publication["pulp_href"]
distribution = client.using_handler(api.task_handler).post(
FILE_DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution["pulp_href"])
unit_path = choice([
content_unit["relative_path"]
for content_unit in get_content(repo)[FILE_CONTENT_NAME]
])
# Assert that an HTTP error is raised when one to fetch content from
# the distribution once the remote was removed.
with self.assertRaises(HTTPError) as ctx:
download_content_unit(cfg, distribution, unit_path)
for key in ("not", "found"):
self.assertIn(key, ctx.exception.response.reason.lower())
# Recreating a remote and re-triggering a sync will cause these broken
# units to recover again.
body = gen_file_remote(policy=choice(ON_DEMAND_DOWNLOAD_POLICIES))
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote["pulp_href"])
sync(cfg, remote, repo)
repo = client.get(repo["pulp_href"])
content = download_content_unit(cfg, distribution, unit_path)
pulp_hash = hashlib.sha256(content).hexdigest()
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(FILE_FIXTURE_URL, unit_path))).hexdigest()
self.assertEqual(pulp_hash, fixtures_hash)
def sync_and_download_check(self, policy):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available. For the cookbook plugin
this is below a live API at ``pulp_cookbook/market/`` e.g.
``http://example.com/pulp_cookbook/market/foo/`` and
``http://example.com/pulp_cookbook/market/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
delete_orphans()
client = api.Client(cfg, api.json_handler)
repo = self.create_and_sync_repo(cfg, client, policy)
distribution = self.create_distribution(cfg, client, repo)
self.download_check(cfg, client, repo, distribution, policy)
def do_test(self, policy):
"""Access lazy synced content on using content endpoint."""
# delete orphans to assure that no content units are present on the
# file system
delete_orphans()
content_api = ContentFilesApi(self.client)
repo_api = RepositoriesFileApi(self.client)
remote_api = RemotesFileApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote(**{"policy": policy})
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
# Assert that no HTTP error was raised.
# Assert that the number of units present is according to the synced
# feed.
content = content_api.list().to_dict()["results"]
self.assertEqual(len(content), FILE_FIXTURE_COUNT, content)
def setUpClass(cls) -> None:
"""Set up the class."""
cls.client = gen_rpm_client()
cls.repo_api = RepositoriesRpmApi(cls.client)
cls.publications_api = PublicationsRpmApi(cls.client)
cls.distributions_api = DistributionsRpmApi(cls.client)
delete_orphans()
def setUpClass(cls):
"""Create class-wide variable."""
delete_orphans()
cls.client = gen_rpm_client()
cls.cfg = Configuration(cls.client)
cls.repo_api = RepositoriesRpmApi(cls.client)
cls.remote_api = RemotesRpmApi(cls.client)
def tearDownClass(cls):
"""Destroys the distros and repos used for the tests."""
monitor_task(cls.repo_api.delete(cls.repo.pulp_href).task)
monitor_task(cls.distributions_api.delete(cls.distro1.pulp_href).task)
monitor_task(cls.distributions_api.delete(cls.distro2.pulp_href).task)
delete_signing_service(cls.signing_service.name)
delete_orphans()
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
delete_orphans()
populate_pulp(cls.cfg, url=FILE_LARGE_FIXTURE_MANIFEST_URL)
cls.client = api.Client(cls.cfg, api.page_handler)
cls.content = cls.client.get(FILE_CONTENT_PATH)
def test_on_demand_policy_mirror_complete(self):
"""Verify that content synced with on_demand policy mode can be consumed."""
delete_orphans()
self.do_test("on_demand", sync_policy="mirror_complete")
new_artifact_count = self.artifacts_api.list().count
self.assertGreater(new_artifact_count,
self.before_consumption_artifact_count)
def tearDownClass(cls):
"""Clean up after ourselves."""
for remote in cls.remotes:
cls.remote_api.delete(remote.pulp_href)
for repo in cls.repos:
cls.repo_api.delete(repo.pulp_href)
delete_orphans()
def test_immediate_policy_mirror_complete(self):
"""Verify that content synced with immediate policy mode can be consumed."""
delete_orphans()
self.do_test("immediate", sync_policy="mirror_complete")
new_artifact_count = self.artifacts_api.list().count
self.assertEqual(new_artifact_count,
self.before_consumption_artifact_count)
def setUpClass(cls):
"""Clean out Pulp before testing."""
delete_orphans()
client = gen_file_client()
cls.cont_api = ContentFilesApi(client)
cls.repo_api = RepositoriesFileApi(client)
cls.remote_api = RemotesFileApi(client)
def setUp(self):
"""Initialize Pulp with some content for our repair tests.
1. Create and sync a repo.
2. Select two content units from the repo, delete one artifact and corrupt another.
"""
# STEP 1
delete_orphans()
repo = self.api_client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(self.api_client.delete, repo["pulp_href"])
body = gen_file_remote()
remote = self.api_client.post(FILE_REMOTE_PATH, body)
self.addCleanup(self.api_client.delete, remote["pulp_href"])
sync(self.cfg, remote, repo)
repo = self.api_client.get(repo["pulp_href"])
# STEP 2
media_root = utils.get_pulp_setting(self.cli_client, "MEDIA_ROOT")
content1, content2 = sample(get_content(repo)[FILE_CONTENT_NAME], 2)
# Muddify one artifact on disk.
artifact1_path = os.path.join(
media_root,
self.api_client.get(content1["artifact"])["file"])
cmd1 = ("sed", "-i", "-e", r"$a bit rot", artifact1_path)
self.cli_client.run(cmd1, sudo=True)
# Delete another one from disk.
artifact2_path = os.path.join(
media_root,
self.api_client.get(content2["artifact"])["file"])
cmd2 = ("rm", artifact2_path)
self.cli_client.run(cmd2, sudo=True)
self.repo = repo
def setUpClass(cls):
"""Create class-wide variables."""
delete_orphans()
cls.client = gen_ansible_client()
cls.repo_api = RepositoriesAnsibleApi(cls.client)
cls.remote_collection_api = RemotesCollectionApi(cls.client)
cls.distributions_api = DistributionsAnsibleApi(cls.client)
def test_streamed_policy_mirror_content_only(self):
"""Verify that content synced with streamed policy mode can be consumed."""
delete_orphans()
self.do_test("streamed", sync_policy="mirror_content_only")
new_artifact_count = self.artifacts_api.list().count
self.assertEqual(new_artifact_count,
self.before_consumption_artifact_count)
def do_test(self, policy):
"""Access lazy synced content on using content endpoint."""
# delete orphans to assure that no content units are present on the
# file system
delete_orphans()
repo_api = deb_repository_api
remote_api = deb_remote_api
packages_api = deb_package_api
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_deb_remote(policy=policy)
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/1/")
# Assert that no HTTP error was raised.
# Assert that the number of units present is according to the synced
# feed.
content = packages_api.list()
self.assertEqual(content.count, DEB_FIXTURE_PACKAGE_COUNT, content)
def setUpClass(cls):
"""Create class-wide variables."""
cls.client = gen_rpm_client()
cls.repo_api = RepositoriesRpmApi(cls.client)
cls.remote_api = RemotesRpmApi(cls.client)
cls.advisory_api = ContentAdvisoriesApi(cls.client)
delete_orphans()
def _sync(url=None):
repo = cls.repo_api.create(gen_repo())
remote = cls.remote_api.create(gen_rpm_remote(url))
repository_sync_data = RpmRepositorySyncURL(remote=remote.pulp_href)
sync_response = cls.repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
return cls.repo_api.read(repo.pulp_href)
# sync repos to get two conflicting advisories to use later in tests
cls.repo_rpm_unsigned = _sync(url=RPM_UNSIGNED_FIXTURE_URL)
cls.repo_rpm_advisory_diffpkgs = _sync(url=RPM_ADVISORY_DIFFERENT_PKGLIST_URL)
cls.advisory_rpm_unsigned_href = (
cls.advisory_api.list(
repository_version=cls.repo_rpm_unsigned.latest_version_href,
id=RPM_ADVISORY_TEST_ID,
)
.results[0]
.pulp_href
)
cls.advisory_rpm_advisory_diffpkgs_href = (
cls.advisory_api.list(
repository_version=cls.repo_rpm_advisory_diffpkgs.latest_version_href,
id=RPM_ADVISORY_TEST_ID,
)
.results[0]
.pulp_href
)
def tearDownClass(cls):
"""Clean up after ourselves."""
for repo in cls.export_repos:
cls.repo_api.delete(repo.pulp_href)
cls.remote_collection_api.delete(cls.remotes[0].pulp_href)
cls.remote_role_api.delete(cls.remotes[1].pulp_href)
delete_orphans()
def test_sync_with_retention_and_modules(self):
"""Verify functionality with sync.
Do the following:
1. Create a repository, and a remote.
2. Sync the remote.
3. Assert that the correct number of units were added and are present in the repo.
4. Change the "retain_package_versions" on the repository to 1 (retain the latest
version only).
5. Sync the remote one more time.
6. Assert that repository version is the same as the previous one, because the older
versions are part of modules, and they should be ignored by the retention policy.
"""
delete_orphans()
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
remote = self.remote_api.create(
gen_rpm_remote(
url=RPM_MODULES_STATIC_CONTEXT_FIXTURE_URL,
policy="on_demand",
))
self.addCleanup(self.remote_api.delete, remote.pulp_href)
task = self.sync(repository=repo, remote=remote, optimize=False)
repo = self.repo_api.read(repo.pulp_href)
self.addCleanup(delete_orphans) # TODO: #2587
# Test that, by default, everything is retained / nothing is tossed out.
self.assertDictEqual(get_content_summary(repo.to_dict()),
RPM_MODULAR_STATIC_FIXTURE_SUMMARY)
self.assertDictEqual(get_added_content_summary(repo.to_dict()),
RPM_MODULAR_STATIC_FIXTURE_SUMMARY)
# Test that the # of packages processed is correct
reports = self.get_progress_reports_by_code(task)
self.assertEqual(reports["sync.parsing.packages"].total,
RPM_MODULAR_PACKAGE_COUNT)
self.assertEqual(reports["sync.skipped.packages"].total, 0)
# Set the retention policy to retain only 1 version of each package
repo_data = repo.to_dict()
repo_data.update({"retain_package_versions": 1})
self.repo_api.update(repo.pulp_href, repo_data)
repo = self.repo_api.read(repo.pulp_href)
task = self.sync(repository=repo, remote=remote, optimize=False)
repo = self.repo_api.read(repo.pulp_href)
# Test that no RPMs were removed (and no advisories etc. touched)
# it should be the same because the older version are covered by modules)
self.assertDictEqual(get_removed_content_summary(repo.to_dict()), {})
# Test that the number of packages processed is correct
reports = self.get_progress_reports_by_code(task)
self.assertEqual(reports["sync.parsing.packages"].total,
RPM_MODULAR_PACKAGE_COUNT)
self.assertEqual(reports["sync.skipped.packages"].total, 0)
def setUpClass(cls):
"""Delete orphans and create class-wide variables."""
cfg = config.get_config()
delete_orphans()
cls.client = api.Client(cfg, api.json_handler)
cls.file = {"file": utils.http_get(FILE_URL)}
cls.file_sha256 = hashlib.sha256(cls.file["file"]).hexdigest()
cls.file_size = len(cls.file["file"])
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = gen_rpm_client()
cls.dist_tree_api = ContentDistributionTreesApi(cls.client)
cls.repo_api = RepositoriesRpmApi(cls.client)
cls.remote_api = RemotesRpmApi(cls.client)
delete_orphans()
def setUpClass(cls):
"""Create class-wide variable."""
delete_orphans()
rpm_client = gen_rpm_client()
cls.tasks_api = TasksApi(core_client)
cls.content_api = ContentPackagesApi(rpm_client)
cls.file_to_use = os.path.join(RPM_UNSIGNED_FIXTURE_URL,
RPM_PACKAGE_FILENAME)
def tearDownClass(cls):
"""Delete api users and things created in setUpclass."""
namespace = cls.namespace_api.list(name="test_push_repo").results[0]
cls.namespace_api.delete(namespace.pulp_href)
delete_orphans()
del_user(cls.user_creator)
del_user(cls.user_reader)
del_user(cls.user_helpless)
def tearDownClass(cls):
"""Delete created users and a distribution that was created in the first stage."""
monitor_task(cls.distributions_api.delete(cls.distribution.pulp_href).task)
delete_orphans()
del_user(cls.user_pull)
del_user(cls.user_push)
del_user(cls.user_anon)
def setUpClass(cls):
"""Create class-wide variable."""
delete_orphans()
cls.content_unit = {}
# FIXME: Instantiate APIs for all content types.
cls.container_content_api = ContentManifestsApi(gen_container_client())
cls.artifact = gen_artifact()
def setUpClass(cls):
"""Create class-wide variable."""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg)
delete_orphans()
cls.rpm_client = gen_rpm_client()
cls.tasks_api = TasksApi(core_client)
cls.content_api = ContentAdvisoriesApi(cls.rpm_client)
cls.bad_file_to_use = os.path.join(RPM_UNSIGNED_FIXTURE_URL, RPM_PACKAGE_FILENAME)
def test_all(self):
"""Test whether a particular repository version can be published.
1. Create a repository with at least 2 repository versions.
2. Create a publication without supplying a repository_version (i.e.
take the latest ``repository_version``).
3. Assert that the publication ``repository_version`` attribute points
to the latest repository version.
4. Create a publication by supplying the non-latest
``repository_version``.
5. Assert that the publication ``repository_version`` attribute points
to the supplied repository version.
6. Assert that an exception is raised when providing two different
repository versions to be published at same time.
"""
cfg = config.get_config()
delete_orphans()
client = api.Client(cfg, api.json_handler)
body = gen_remote(fixture_u1.url, cookbooks={fixture_u1.example1_name: ""})
remote = client.post(COOKBOOK_REMOTE_PATH, body)
self.addCleanup(client.delete, remote["pulp_href"])
repo = client.post(COOKBOOK_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["pulp_href"])
sync(cfg, remote, repo, mirror=True)
repo = client.get(repo["pulp_href"])
repo_content = get_cookbook_content(repo)
self.assertTrue(repo_content)
# Step 1
repo = client.post(COOKBOOK_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["pulp_href"])
for cookbook in repo_content:
modify_repo(cfg, repo, add_units=[cookbook])
version_hrefs = tuple(ver["pulp_href"] for ver in get_versions(repo))
non_latest = choice(version_hrefs[:-1])
# Step 2
publication = create_publication(cfg, repo)
# Step 3
self.assertEqual(publication["repository_version"], version_hrefs[-1])
# Step 4
publication = create_publication(cfg, repo, version_href=non_latest)
# Step 5
self.assertEqual(publication["repository_version"], non_latest)
# Step 6
with self.assertRaises(HTTPError):
body = {"repository": repo["pulp_href"], "repository_version": non_latest}
client.post(COOKBOOK_PUBLICATION_PATH, body)
def tearDownClass(cls):
"""Clean up."""
for repo in cls.import_repos:
cls.repo_api.delete(repo.pulp_href)
delete_orphans()
cli_client = cli.Client(cls.cfg)
cmd = ("rm", "-rf", cls.exporter.path)
cli_client.run(cmd, sudo=True)
def test_standard_publish(self):
"""Test the "complex" fixture that covers more of the metadata cases.
The standard fixtures have no changelogs and don't cover "ghost" files. The repo
with the "complex-package" does, and also does a better job of covering rich deps
and other atypical metadata.
"""
delete_orphans()
self.do_test(False)
def tearDownClass(cls):
"""Delete api users."""
del_user(cls.user_creator)
del_user(cls.user_dist_collaborator)
del_user(cls.user_dist_consumer)
del_user(cls.user_namespace_collaborator)
del_user(cls.user_reader)
del_user(cls.user_helpless)
delete_orphans()
def setUpClass(cls):
"""Create class-wide variables."""
delete_orphans()
cls.cfg = config.get_config()
cls.client = gen_rpm_client()
cls.repo_api = RepositoriesRpmApi(cls.client)
cls.remote_api = RemotesRpmApi(cls.client)
cls.publications = PublicationsRpmApi(cls.client)
cls.distributions = DistributionsRpmApi(cls.client)