def test_clean_orphan_content_unit(self):
"""Test whether orphaned content units can be cleaned up."""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
content = choice(get_content(repo.to_dict())[FILE_CONTENT_NAME])
# Create an orphan content unit.
repo_api.modify(repo.pulp_href, dict(remove_content_units=[content["pulp_href"]]))
artifacts_api = ArtifactsApi(core_client)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact is present on disk.
relative_path = artifacts_api.read(content["artifact"]).file
artifact_path = os.path.join(self.media_root, relative_path)
cmd = ("ls", artifact_path)
self.cli_client.run(cmd, sudo=True)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content["pulp_href"], content_units_href)
content_before_cleanup = file_contents_api.list().count
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 10})
monitor_task(orphans_response.task)
# assert content was not removed
content_after_cleanup = file_contents_api.list().count
self.assertEqual(content_after_cleanup, content_before_cleanup)
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 0})
monitor_task(orphans_response.task)
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content["pulp_href"], content_units_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def test_clean_specific_orphans(self):
"""Test whether the `content_hrefs` param removes specific orphans but not others"""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
# Create two orphaned content units.
content_a = get_content(
repo.to_dict())[FILE_CONTENT_NAME][0]["pulp_href"]
content_b = get_content(
repo.to_dict())[FILE_CONTENT_NAME][1]["pulp_href"]
content_to_remove = dict(remove_content_units=[content_a, content_b])
repo_api.modify(repo.pulp_href, content_to_remove)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content_a, content_units_href)
self.assertIn(content_b, content_units_href)
content_hrefs_dict = {"content_hrefs": [content_a]}
orphans_response = self.orphans_cleanup_api.cleanup(content_hrefs_dict)
monitor_task(orphans_response.task)
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content_a, content_units_href)
self.assertIn(content_b, content_units_href)
def test_second_unit_raises_error(self):
"""Create a duplicate content unit with different ``artifacts`` and same ``relative_path``.
Artifacts are unique by ``relative_path`` and ``file``.
"""
delete_orphans()
content_api = ContentFilesApi(self.client)
repo_api = RepositoriesFileApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
artifact = gen_artifact()
# create first content unit.
content_attrs = gen_file_content_attrs(artifact)
response = content_api.create(**content_attrs)
monitor_task(response.task)
artifact = gen_artifact(file=__file__)
# create second content unit.
second_content_attrs = gen_file_content_attrs(artifact)
second_content_attrs["relative_path"] = content_attrs["relative_path"]
response = content_api.create(**second_content_attrs)
monitor_task(response.task)
data = {
"add_content_units":
[c.pulp_href for c in content_api.list().results]
}
response = repo_api.modify(repo.pulp_href, data)
with self.assertRaises(PulpTaskError) as cm:
monitor_task(response.task)
task = cm.exception.task.to_dict()
error_message = ("Cannot create repository version. "
"More than one file.file content with "
"the duplicate values for relative_path.")
self.assertEqual(task["error"]["description"], error_message)
def test_clean_orphan_content_unit(self):
"""Test whether orphan content units can be clean up.
Do the following:
1. Create, and sync a repo.
2. Remove a content unit from the repo. This will create a second
repository version, and create an orphan content unit.
3. Assert that content unit that was removed from the repo and its
artifact are present on disk.
4. Delete orphans.
5. Assert that the orphan content unit was cleaned up, and its artifact
is not present on disk.
"""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
content = choice(get_content(repo.to_dict())[FILE_CONTENT_NAME])
# Create an orphan content unit.
repo_api.modify(repo.pulp_href,
dict(remove_content_units=[content["pulp_href"]]))
artifacts_api = ArtifactsApi(core_client)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact is present on disk.
artifact_path = os.path.join(
MEDIA_PATH,
artifacts_api.read(content["artifact"]).file)
cmd = ("ls", artifact_path)
self.cli_client.run(cmd, sudo=True)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content["pulp_href"], content_units_href)
delete_orphans()
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content["pulp_href"], content_units_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
repository_version_1 = filerepoversions.read(created_resources[0])
pprint(repository_version_1)
# Create an artifact from a local file
file_path = os.path.join(os.environ["GITHUB_WORKSPACE"], ".ci/assets/bindings/test_bindings.py")
artifact = artifacts.create(file=file_path)
pprint(artifact)
# Create a FileContent from the artifact
filecontent_response = filecontent.create(relative_path="foo.tar.gz", artifact=artifact.pulp_href)
created_resources = monitor_task(filecontent_response.task)
# Add the new FileContent to a repository version
repo_version_data = {"add_content_units": [created_resources[0]]}
repo_version_response = filerepositories.modify(repository.pulp_href, repo_version_data)
# Monitor the repo version creation task
created_resources = monitor_task(repo_version_response.task)
repository_version_2 = filerepoversions.read(created_resources[0])
pprint(repository_version_2)
# List all the repository versions
filerepoversions.list(repository.pulp_href)
# Create a publication from the latest version of the repository
publish_data = FileFilePublication(repository=repository.pulp_href)
publish_response = filepublications.create(publish_data)
# Monitor the publish task
class RepoVersionRetentionTestCase(unittest.TestCase):
"""Test retain_repo_versions for repositories
This test targets the following issues:
* `Pulp #8368 <https:://pulp.plan.io/issues/8368>`_
"""
@classmethod
def setUp(self):
"""Add content to Pulp."""
self.cfg = config.get_config()
self.client = api.Client(self.cfg, api.json_handler)
self.core_client = CoreApiClient(configuration=self.cfg.get_bindings_config())
self.file_client = gen_file_client()
self.content_api = ContentFilesApi(self.file_client)
self.repo_api = RepositoriesFileApi(self.file_client)
self.version_api = RepositoriesFileVersionsApi(self.file_client)
self.distro_api = DistributionsFileApi(self.file_client)
self.publication_api = PublicationsFileApi(self.file_client)
delete_orphans()
populate_pulp(self.cfg, url=FILE_LARGE_FIXTURE_MANIFEST_URL)
self.content = sample(self.content_api.list().results, 3)
self.publications = []
def _create_repo_versions(self, repo_attributes={}):
self.repo = self.repo_api.create(gen_repo(**repo_attributes))
self.addCleanup(self.repo_api.delete, self.repo.pulp_href)
if "autopublish" in repo_attributes and repo_attributes["autopublish"]:
self.distro = create_distribution(repository_href=self.repo.pulp_href)
self.addCleanup(self.distro_api.delete, self.distro.pulp_href)
for content in self.content:
result = self.repo_api.modify(
self.repo.pulp_href, {"add_content_units": [content.pulp_href]}
)
monitor_task(result.task)
self.repo = self.repo_api.read(self.repo.pulp_href)
self.publications += self.publication_api.list(
repository_version=self.repo.latest_version_href
).results
def test_retain_repo_versions(self):
"""Test repo version retention."""
self._create_repo_versions({"retain_repo_versions": 1})
versions = self.version_api.list(file_file_repository_href=self.repo.pulp_href).results
self.assertEqual(len(versions), 1)
latest_version = self.version_api.read(
file_file_repository_version_href=self.repo.latest_version_href
)
self.assertEqual(latest_version.number, 3)
self.assertEqual(latest_version.content_summary.present["file.file"]["count"], 3)
self.assertEqual(latest_version.content_summary.added["file.file"]["count"], 3)
def test_retain_repo_versions_on_update(self):
"""Test repo version retention when retain_repo_versions is set."""
self._create_repo_versions()
versions = self.version_api.list(file_file_repository_href=self.repo.pulp_href).results
self.assertEqual(len(versions), 4)
# update retain_repo_versions to 2
result = self.repo_api.partial_update(self.repo.pulp_href, {"retain_repo_versions": 2})
monitor_task(result.task)
versions = self.version_api.list(file_file_repository_href=self.repo.pulp_href).results
self.assertEqual(len(versions), 2)
latest_version = self.version_api.read(
file_file_repository_version_href=self.repo.latest_version_href
)
self.assertEqual(latest_version.number, 3)
self.assertEqual(latest_version.content_summary.present["file.file"]["count"], 3)
self.assertEqual(latest_version.content_summary.added["file.file"]["count"], 1)
def test_autodistribute(self):
"""Test repo version retention with autopublish/autodistribute."""
self._create_repo_versions({"retain_repo_versions": 1, "autopublish": True})
# all but the last publication should be gone
for publication in self.publications[:-1]:
with self.assertRaises(ApiException) as ae:
self.publication_api.read(publication.pulp_href)
self.assertEqual(404, ae.exception.status)
# check that the last publication is distributed
manifest = download_content_unit(self.cfg, self.distro.to_dict(), "PULP_MANIFEST")
self.assertEqual(manifest.decode("utf-8").count("\n"), len(self.content))
