def gen_artifact(url=CONTAINER_IMAGE_URL):
"""Create an artifact."""
response = requests.get(url)
with NamedTemporaryFile() as temp_file:
temp_file.write(response.content)
artifact = ArtifactsApi(core_client).create(file=temp_file.name)
return artifact.to_dict()
def gen_artifact(url=PYTHON_URL):
"""Creates an artifact."""
with NamedTemporaryFile() as temp_file:
temp_file.write(http_get(url))
temp_file.flush()
artifact = ArtifactsApi(core_client).create(file=temp_file.name)
return artifact.to_dict()
def test_clean_orphan_content_unit(self):
"""Test whether orphaned content units can be cleaned up."""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
content = choice(get_content(repo.to_dict())[FILE_CONTENT_NAME])
# Create an orphan content unit.
repo_api.modify(repo.pulp_href, dict(remove_content_units=[content["pulp_href"]]))
artifacts_api = ArtifactsApi(core_client)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact is present on disk.
relative_path = artifacts_api.read(content["artifact"]).file
artifact_path = os.path.join(self.media_root, relative_path)
cmd = ("ls", artifact_path)
self.cli_client.run(cmd, sudo=True)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content["pulp_href"], content_units_href)
content_before_cleanup = file_contents_api.list().count
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 10})
monitor_task(orphans_response.task)
# assert content was not removed
content_after_cleanup = file_contents_api.list().count
self.assertEqual(content_after_cleanup, content_before_cleanup)
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 0})
monitor_task(orphans_response.task)
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content["pulp_href"], content_units_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def gen_artifact(url=RPM_SIGNED_URL):
"""Creates an artifact."""
response = requests.get(url)
with NamedTemporaryFile() as temp_file:
temp_file.write(response.content)
temp_file.flush()
artifact = ArtifactsApi(core_client).create(file=temp_file.name)
return artifact.to_dict()
def gen_artifact(url):
"""Creates an artifact."""
response = utils.http_get(url)
with NamedTemporaryFile() as temp_file:
temp_file.write(response)
temp_file.flush()
artifact = ArtifactsApi(core_client).create(file=temp_file.name)
return artifact.to_dict()
def gen_artifact(url=FILE_URL, file=None):
"""Creates an artifact."""
if not file:
response = requests.get(url)
with NamedTemporaryFile() as temp_file:
temp_file.write(response.content)
return ArtifactsApi(core_client).create(
file=temp_file.name).to_dict()
return ArtifactsApi(core_client).create(file=file).to_dict()
def test_clean_orphan_artifact(self):
"""Test whether orphan artifacts units can be clean up."""
repo_api = RepositoriesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
artifacts_api = ArtifactsApi(core_client)
artifact = artifacts_api.create(file=__file__)
if self.storage == "pulpcore.app.models.storage.FileSystem":
cmd = ("ls", os.path.join(MEDIA_PATH, artifact.file))
self.cli_client.run(cmd, sudo=True)
delete_orphans()
with self.assertRaises(ApiException):
artifacts_api.read(artifact.pulp_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def test_clean_orphan_artifact(self):
"""Test whether orphan artifacts units can be clean up."""
repo_api = RepositoriesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
artifacts_api = ArtifactsApi(core_client)
artifact = artifacts_api.create(file=__file__)
if self.storage == "pulpcore.app.models.storage.FileSystem":
cmd = ("ls", os.path.join(self.media_root, artifact.file))
self.cli_client.run(cmd, sudo=True)
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 10})
monitor_task(orphans_response.task)
# assert artifact was not removed
artifacts = artifacts_api.list().count
self.assertEqual(artifacts, 1)
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 0})
monitor_task(orphans_response.task)
with self.assertRaises(ApiException):
artifacts_api.read(artifact.pulp_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def test_all(self):
"""Perform a lazy sync and change to immeditae to force download."""
# delete orphans to assure that no content units are present on the
# file system
delete_orphans()
client = gen_file_client()
artifacts_api = ArtifactsApi(core_client)
repo_api = RepositoriesFileApi(client)
remote_api = RemotesFileApi(client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote(policy=choice(ON_DEMAND_DOWNLOAD_POLICIES))
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository using a lazy download policy
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
artifacts = artifacts_api.list().to_dict()["results"]
self.assertEqual(len(artifacts), 0, artifacts)
# Update the policy to immediate
response = remote_api.partial_update(remote.pulp_href,
{"policy": "immediate"})
monitor_task(response.task)
remote = remote_api.read(remote.pulp_href)
self.assertEqual(remote.policy, "immediate")
# Sync using immediate download policy
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
# Assert that missing artifacts are downloaded
artifacts = artifacts_api.list().to_dict()["results"]
self.assertEqual(len(artifacts), FILE_FIXTURE_COUNT, artifacts)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = gen_file_client()
cls.orphans_api = OrphansCleanupApi(core_client)
cls.reclaim_api = RepositoriesReclaimSpaceApi(core_client)
cls.artifacts_api = ArtifactsApi(core_client)
cls.publication_api = PublicationsFileApi(cls.client)
cls.distributions_api = DistributionsFileApi(cls.client)
cls.repo_api = RepositoriesFileApi(cls.client)
cls.remote_api = RemotesFileApi(cls.client)
orphans_response = cls.orphans_api.cleanup(
{"orphan_protection_time": 0})
monitor_task(orphans_response.task)
def setUpClass(cls):
"""Verify whether dnf or yum are present."""
cls.cfg = config.get_config()
configuration = cls.cfg.get_bindings_config()
core_client = CoreApiClient(configuration)
cls.artifacts_api = ArtifactsApi(core_client)
cls.client = gen_rpm_client()
cls.repo_api = RepositoriesRpmApi(cls.client)
cls.remote_api = RemotesRpmApi(cls.client)
cls.publications = PublicationsRpmApi(cls.client)
cls.distributions = DistributionsRpmApi(cls.client)
cls.before_consumption_artifact_count = 0
cls.pkg_mgr = cli.PackageManager(cls.cfg)
cls.pkg_mgr.raise_if_unsupported(unittest.SkipTest,
"This test requires dnf or yum.")
def test_clean_orphan_content_unit(self):
"""Test whether orphan content units can be clean up.
Do the following:
1. Create, and sync a repo.
2. Remove a content unit from the repo. This will create a second
repository version, and create an orphan content unit.
3. Assert that content unit that was removed from the repo and its
artifact are present on disk.
4. Delete orphans.
5. Assert that the orphan content unit was cleaned up, and its artifact
is not present on disk.
"""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
content = choice(get_content(repo.to_dict())[FILE_CONTENT_NAME])
# Create an orphan content unit.
repo_api.modify(repo.pulp_href,
dict(remove_content_units=[content["pulp_href"]]))
artifacts_api = ArtifactsApi(core_client)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact is present on disk.
artifact_path = os.path.join(
MEDIA_PATH,
artifacts_api.read(content["artifact"]).file)
cmd = ("ls", artifact_path)
self.cli_client.run(cmd, sudo=True)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content["pulp_href"], content_units_href)
delete_orphans()
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content["pulp_href"], content_units_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
artifact = artifacts.read(created_resources[0])
return artifact
# Configure HTTP basic authorization: basic
configuration = Configuration()
configuration.username = '******'
configuration.password = '******'
configuration.safe_chars_for_path_param = '/'
core_client = CoreApiClient(configuration)
file_client = FileApiClient(configuration)
# Create api clients for all resource types
artifacts = ArtifactsApi(core_client)
repositories = RepositoriesApi(core_client)
repoversions = RepositoriesVersionsApi(core_client)
filecontent = ContentFilesApi(file_client)
filedistributions = DistributionsFileApi(core_client)
filepublications = PublicationsFileApi(file_client)
fileremotes = RemotesFileApi(file_client)
tasks = TasksApi(core_client)
uploads = UploadsApi(core_client)
# Test creating an Artifact from a 1mb file uploaded in 200kb chunks
with NamedTemporaryFile() as downloaded_file:
response = requests.get(
'https://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/test_bandwidth_repo/'
'pulp-large_1mb_test-packageA-0.1.1-1.fc14.noarch.rpm')
def setUpClass(cls):
"""Create class-wide variables and delete orphans.
1. Create a repository.
2. Create a remote pointing to external registry with policy=on_demand.
3. Sync the repository using the remote and re-read the repo data.
4. Create a container distribution to serve the repository
5. Create another container distribution to the serve the repository version
This tests targets the following issue:
* `Pulp #4460 <https://pulp.plan.io/issues/4460>`_
"""
cls.cfg = config.get_config()
cls.registry_name = urlparse(cls.cfg.get_base_url()).netloc
client_api = gen_container_client()
cls.repositories_api = RepositoriesContainerApi(client_api)
cls.remotes_api = RemotesContainerApi(client_api)
cls.distributions_api = DistributionsContainerApi(client_api)
cls.teardown_cleanups = []
delete_orphans()
with contextlib.ExitStack() as stack:
# ensure tearDownClass runs if an error occurs here
stack.callback(cls.tearDownClass)
# Step 1
_repo = cls.repositories_api.create(
ContainerContainerRepository(**gen_repo()))
cls.teardown_cleanups.append(
(cls.repositories_api.delete, _repo.pulp_href))
# Step 2
cls.remote = cls.remotes_api.create(
gen_container_remote(policy="on_demand"))
cls.teardown_cleanups.append(
(cls.remotes_api.delete, cls.remote.pulp_href))
# Step 3
sync_data = RepositorySyncURL(remote=cls.remote.pulp_href)
sync_response = cls.repositories_api.sync(_repo.pulp_href,
sync_data)
monitor_task(sync_response.task)
cls.repo = cls.repositories_api.read(_repo.pulp_href)
cls.artifacts_api = ArtifactsApi(core_client)
cls.artifact_count = cls.artifacts_api.list().count
# Step 4.
distribution_response = cls.distributions_api.create(
ContainerContainerDistribution(**gen_distribution(
repository=cls.repo.pulp_href)))
created_resources = monitor_task(
distribution_response.task).created_resources
distribution = cls.distributions_api.read(created_resources[0])
cls.distribution_with_repo = cls.distributions_api.read(
distribution.pulp_href)
cls.teardown_cleanups.append(
(cls.distributions_api.delete,
cls.distribution_with_repo.pulp_href))
# Step 5.
distribution_response = cls.distributions_api.create(
ContainerContainerDistribution(**gen_distribution(
repository_version=cls.repo.latest_version_href)))
created_resources = monitor_task(
distribution_response.task).created_resources
distribution = cls.distributions_api.read(created_resources[0])
cls.distribution_with_repo_version = cls.distributions_api.read(
distribution.pulp_href)
cls.teardown_cleanups.append(
(cls.distributions_api.delete,
cls.distribution_with_repo_version.pulp_href))
# remove callback if everything goes well
stack.pop_all()
RemotesAptApi,
RepositoriesAptApi,
)
skip_if = partial(selectors.skip_if, exc=SkipTest) # pylint:disable=invalid-name
"""The ``@skip_if`` decorator, customized for unittest.
:func:`pulp_smash.selectors.skip_if` is test runner agnostic. This function is
identical, except that ``exc`` has been set to ``unittest.SkipTest``.
"""
cfg = config.get_config()
configuration = cfg.get_bindings_config()
core_client = CoreApiClient(configuration)
artifact_api = ArtifactsApi(core_client)
task_api = TasksApi(core_client)
signing_service_api = SigningServicesApi(core_client)
deb_client = DebApiClient(configuration)
deb_generic_content_api = ContentGenericContentsApi(deb_client)
deb_package_api = ContentPackagesApi(deb_client)
deb_remote_api = RemotesAptApi(deb_client)
deb_repository_api = RepositoriesAptApi(deb_client)
deb_apt_publication_api = PublicationsAptApi(deb_client)
deb_verbatim_publication_api = PublicationsVerbatimApi(deb_client)
deb_distribution_api = DistributionsAptApi(deb_client)
def set_up_module():
"""Skip tests Pulp 3 isn't under test or if pulp_deb isn't installed."""
else:
print("The task did not finish successfully.")
exit()
# Configure HTTP basic authorization: basic
configuration = Configuration()
configuration.username = '******'
configuration.password = '******'
configuration.safe_chars_for_path_param = '/'
core_client = CoreApiClient(configuration)
file_client = FileApiClient(configuration)
# Create api clients for all resource types
artifacts = ArtifactsApi(core_client)
distributions = DistributionsApi(core_client)
repositories = RepositoriesApi(core_client)
filecontent = FileContentApi(file_client)
filepublishers = FilePublishersApi(file_client)
fileremotes = FileRemotesApi(file_client)
tasks = TasksApi(core_client)
# Create a File Remote
remote_url = 'https://repos.fedorapeople.org/pulp/pulp/demo_repos/test_file_repo/PULP_MANIFEST'
remote_data = FileRemote(name='bar15', url=remote_url)
file_remote = fileremotes.remotes_file_file_create(remote_data)
pprint(file_remote)
# Create a Repository
repository_data = Repository(name='foo15')
