def do_test(self, policy):
"""Access lazy synced content on using content endpoint."""
# delete orphans to assure that no content units are present on the
# file system
delete_orphans()
content_api = ContentFilesApi(self.client)
repo_api = RepositoriesFileApi(self.client)
remote_api = RemotesFileApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote(**{"policy": policy})
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
# Assert that no HTTP error was raised.
# Assert that the number of units present is according to the synced
# feed.
content = content_api.list().to_dict()["results"]
self.assertEqual(len(content), FILE_FIXTURE_COUNT, content)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.file_repositories_api = RepositoriesFileApi(
FileApiClient(cls.cfg.get_bindings_config()))
cls.repo = cls.file_repositories_api.create(gen_repo())
cls.file_content_api = ContentFilesApi(
FileApiClient(cls.cfg.get_bindings_config()))
with tempfile.NamedTemporaryFile() as tmp_file:
tmp_file.write(b"not empty")
tmp_file.flush()
monitor_task(
cls.file_content_api.create(relative_path=utils.uuid4(),
file=tmp_file.name).task)
cls.cert_guards_api = ContentGuardsX509Api(
CertGuardApiClient(cls.cfg.get_bindings_config()))
with open(X509_CA_CERT_FILE_PATH, "r") as x509_ca_cert_data_file:
x509_ca_cert_data = x509_ca_cert_data_file.read()
cls.content_guard = cls.cert_guards_api.create({
"name":
utils.uuid4(),
"ca_certificate":
x509_ca_cert_data
})
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.cli_client = cli.Client(cls.cfg)
allowed_imports = get_pulp_setting(cls.cli_client,
"ALLOWED_IMPORT_PATHS")
if not allowed_imports or "/tmp" not in allowed_imports:
raise unittest.SkipTest(
"Cannot run import-tests unless /tmp is in ALLOWED_IMPORT_PATHS ({})."
.format(allowed_imports), )
cls.client = api.Client(cls.cfg, api.json_handler)
cls.core_client = CoreApiClient(
configuration=cls.cfg.get_bindings_config())
cls.file_client = gen_file_client()
cls.repo_api = RepositoriesFileApi(cls.file_client)
cls.remote_api = RemotesFileApi(cls.file_client)
cls.versions_api = RepositoriesFileVersionsApi(cls.file_client)
cls.content_api = ContentFilesApi(cls.file_client)
cls.exporter_api = ExportersPulpApi(cls.core_client)
cls.exports_api = ExportersCoreExportsApi(cls.core_client)
cls.importer_api = ImportersPulpApi(cls.core_client)
cls.imports_api = ImportersCoreImportsApi(cls.core_client)
cls.import_check_api = ImportersCoreImportCheckApi(cls.core_client)
(cls.import_repos, cls.export_repos,
cls.remotes) = cls._setup_repositories()
cls.exporter = cls._create_exporter()
cls.export = cls._create_export()
cls.chunked_export = cls._create_chunked_export()
cls._setup_import_check_directories()
def setUpClass(cls):
"""Clean out Pulp before testing."""
delete_orphans()
client = gen_file_client()
cls.cont_api = ContentFilesApi(client)
cls.repo_api = RepositoriesFileApi(client)
cls.remote_api = RemotesFileApi(client)
def setUpClass(cls):
"""Sets up class"""
client = gen_file_client()
cls.cont_api = ContentFilesApi(client)
cls.repo_api = RepositoriesFileApi(client)
cls.remote_api = RemotesFileApi(client)
cls.pub_api = PublicationsFileApi(client)
cls.dis_api = DistributionsFileApi(client)
cls.repo = cls.repo_api.create(gen_repo(autopublish=True))
cls.remote = cls.remote_api.create(gen_file_remote())
body = RepositorySyncURL(remote=cls.remote.pulp_href)
created = monitor_task(
cls.repo_api.sync(cls.repo.pulp_href, body).task).created_resources
cls.repo = cls.repo_api.read(cls.repo.pulp_href)
cls.pub1 = cls.pub_api.read(created[1])
body = FileFilePublication(repository=cls.repo.pulp_href)
cls.pub2 = cls.pub_api.read(
monitor_task(cls.pub_api.create(body).task).created_resources[0])
cls.pub3 = []
response = cls.dis_api.create(
gen_distribution(repository=cls.repo.pulp_href))
cls.distro = cls.dis_api.read(
monitor_task(response.task).created_resources[0])
cls.distro2 = []
cls.url = urljoin(PULP_CONTENT_BASE_URL, f"{cls.distro.base_path}/")
def test_clean_orphan_content_unit(self):
"""Test whether orphaned content units can be cleaned up."""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
content = choice(get_content(repo.to_dict())[FILE_CONTENT_NAME])
# Create an orphan content unit.
repo_api.modify(repo.pulp_href, dict(remove_content_units=[content["pulp_href"]]))
artifacts_api = ArtifactsApi(core_client)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact is present on disk.
relative_path = artifacts_api.read(content["artifact"]).file
artifact_path = os.path.join(self.media_root, relative_path)
cmd = ("ls", artifact_path)
self.cli_client.run(cmd, sudo=True)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content["pulp_href"], content_units_href)
content_before_cleanup = file_contents_api.list().count
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 10})
monitor_task(orphans_response.task)
# assert content was not removed
content_after_cleanup = file_contents_api.list().count
self.assertEqual(content_after_cleanup, content_before_cleanup)
orphans_response = self.orphans_cleanup_api.cleanup({"orphan_protection_time": 0})
monitor_task(orphans_response.task)
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content["pulp_href"], content_units_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = gen_file_client()
cls.content_api = ContentFilesApi(cls.client)
cls.repo_api = RepositoriesFileApi(cls.client)
cls.remote_api = RemotesFileApi(cls.client)
cls.publications_api = PublicationsFileApi(cls.client)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg, api.json_handler)
cls.file_client = gen_file_client()
cls.content_api = ContentFilesApi(cls.file_client)
cls.repo_api = RepositoriesFileApi(cls.file_client)
cls.versions_api = RepositoriesFileVersionsApi(cls.file_client)
cls.remote_api = RemotesFileApi(cls.file_client)
cls.publication_api = PublicationsFileApi(cls.file_client)
cls.exporter_api = ExportersFilesystemApi(cls.file_client)
cls.exports_api = ExportersFileExportsApi(cls.file_client)
cls.repos, cls.remotes, cls.publications = cls._setup_repositories()
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg, api.json_handler)
cls.core_client = CoreApiClient(configuration=cls.cfg.get_bindings_config())
cls.file_client = gen_file_client()
cls.content_api = ContentFilesApi(cls.file_client)
cls.repo_api = RepositoriesFileApi(cls.file_client)
cls.versions_api = RepositoriesFileVersionsApi(cls.file_client)
cls.remote_api = RemotesFileApi(cls.file_client)
cls.exporter_api = ExportersPulpApi(cls.core_client)
cls.exports_api = ExportersCoreExportsApi(cls.core_client)
(cls.repos, cls.remotes) = cls._setup_repositories()
def setUp(self):
"""Add content to Pulp."""
self.cfg = config.get_config()
self.client = api.Client(self.cfg, api.json_handler)
self.core_client = CoreApiClient(configuration=self.cfg.get_bindings_config())
self.file_client = gen_file_client()
self.content_api = ContentFilesApi(self.file_client)
self.repo_api = RepositoriesFileApi(self.file_client)
self.version_api = RepositoriesFileVersionsApi(self.file_client)
self.distro_api = DistributionsFileApi(self.file_client)
self.publication_api = PublicationsFileApi(self.file_client)
delete_orphans()
populate_pulp(self.cfg, url=FILE_LARGE_FIXTURE_MANIFEST_URL)
self.content = sample(self.content_api.list().results, 3)
self.publications = []
def setUpClass(cls):
"""
Create all the client instances needed to communicate with Pulp.
"""
configuration = Configuration(**BINDINGS_CONFIGURATION)
file_client = FileApiClient(configuration)
migration_client = MigrationApiClient(configuration)
# Create api clients for all resource types
cls.file_repo_api = RepositoriesFileApi(file_client)
cls.file_content_api = ContentFilesApi(file_client)
cls.migration_plans_api = MigrationPlansApi(migration_client)
cls.pulp2content_api = Pulp2ContentApi(migration_client)
cls.pulp2repositories_api = Pulp2RepositoriesApi(migration_client)
set_pulp2_snapshot(name='file_base_4repos')
def test_clean_specific_orphans(self):
"""Test whether the `content_hrefs` param removes specific orphans but not others"""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
# Create two orphaned content units.
content_a = get_content(
repo.to_dict())[FILE_CONTENT_NAME][0]["pulp_href"]
content_b = get_content(
repo.to_dict())[FILE_CONTENT_NAME][1]["pulp_href"]
content_to_remove = dict(remove_content_units=[content_a, content_b])
repo_api.modify(repo.pulp_href, content_to_remove)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content_a, content_units_href)
self.assertIn(content_b, content_units_href)
content_hrefs_dict = {"content_hrefs": [content_a]}
orphans_response = self.orphans_cleanup_api.cleanup(content_hrefs_dict)
monitor_task(orphans_response.task)
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content_a, content_units_href)
self.assertIn(content_b, content_units_href)
def setUpClass(cls):
"""
Create all the client instances needed to communicate with Pulp.
"""
configuration = Configuration(**BINDINGS_CONFIGURATION)
file_client = FileApiClient(configuration)
migration_client = MigrationApiClient(configuration)
# Create api clients for all resource types
cls.file_repo_api = RepositoriesFileApi(file_client)
cls.file_repo_versions_api = RepositoriesFileVersionsApi(file_client)
cls.file_remote_api = RemotesFileApi(file_client)
cls.file_distribution_api = DistributionsFileApi(file_client)
cls.file_publication_api = PublicationsFileApi(file_client)
cls.file_content_api = ContentFilesApi(file_client)
cls.migration_plans_api = MigrationPlansApi(migration_client)
cls.pulp2content_api = Pulp2ContentApi(migration_client)
cls.pulp2repositories_api = Pulp2RepositoriesApi(migration_client)
def setUpClass(cls):
"""
Create all the client instances needed to communicate with Pulp.
"""
configuration = Configuration()
configuration.username = '******'
configuration.password = '******'
configuration.host = 'http://pulp'
configuration.safe_chars_for_path_param = '/'
core_client = CoreApiClient(configuration)
file_client = FileApiClient(configuration)
migration_client = MigrationApiClient(configuration)
# Create api clients for all resource types
cls.file_repo_api = RepositoriesFileApi(file_client)
cls.file_repo_versions_api = RepositoriesFileVersionsApi(file_client)
cls.file_content_api = ContentFilesApi(file_client)
cls.tasks_api = TasksApi(core_client)
cls.migration_plans_api = MigrationPlansApi(migration_client)
def test_second_unit_raises_error(self):
"""Create a duplicate content unit with different ``artifacts`` and same ``relative_path``.
Artifacts are unique by ``relative_path`` and ``file``.
"""
delete_orphans()
content_api = ContentFilesApi(self.client)
repo_api = RepositoriesFileApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
artifact = gen_artifact()
# create first content unit.
content_attrs = gen_file_content_attrs(artifact)
response = content_api.create(**content_attrs)
monitor_task(response.task)
artifact = gen_artifact(file=__file__)
# create second content unit.
second_content_attrs = gen_file_content_attrs(artifact)
second_content_attrs["relative_path"] = content_attrs["relative_path"]
response = content_api.create(**second_content_attrs)
monitor_task(response.task)
data = {
"add_content_units":
[c.pulp_href for c in content_api.list().results]
}
response = repo_api.modify(repo.pulp_href, data)
with self.assertRaises(PulpTaskError) as cm:
monitor_task(response.task)
task = cm.exception.task.to_dict()
error_message = ("Cannot create repository version. "
"More than one file.file content with "
"the duplicate values for relative_path.")
self.assertEqual(task["error"]["description"], error_message)
def setUpClass(cls):
"""
Create all the client instances needed to communicate with Pulp.
"""
configuration = Configuration(**BINDINGS_CONFIGURATION)
file_client = FileApiClient(configuration)
rpm_client = RpmApiClient(configuration)
migration_client = MigrationApiClient(configuration)
# Create api clients for File
cls.file_repo_api = RepositoriesFileApi(file_client)
cls.file_repo_versions_api = RepositoriesFileVersionsApi(file_client)
cls.file_remote_api = RemotesFileApi(file_client)
cls.file_distribution_api = DistributionsFileApi(file_client)
cls.file_publication_api = PublicationsFileApi(file_client)
cls.file_content_api = ContentFilesApi(file_client)
# Create api clients for RPM
cls.rpm_repo_api = RepositoriesRpmApi(rpm_client)
cls.rpm_repo_versions_api = RepositoriesRpmVersionsApi(rpm_client)
cls.rpm_remote_api = RemotesRpmApi(rpm_client)
cls.rpm_distribution_api = DistributionsRpmApi(rpm_client)
cls.rpm_publication_api = PublicationsRpmApi(rpm_client)
cls.rpm_content_apis = {
'advisory': ContentAdvisoriesApi(rpm_client),
'disttree': ContentDistributionTreesApi(rpm_client),
'modulemd': ContentModulemdsApi(rpm_client),
'modulemd-defaults': ContentModulemdDefaultsApi(rpm_client),
'category': ContentPackagecategoriesApi(rpm_client),
'environment': ContentPackageenvironmentsApi(rpm_client),
'group': ContentPackagegroupsApi(rpm_client),
'langpack': ContentPackagelangpacksApi(rpm_client),
'package': ContentPackagesApi(rpm_client),
}
# Create api clients for Migration
cls.migration_plans_api = MigrationPlansApi(migration_client)
cls.pulp2content_api = Pulp2ContentApi(migration_client)
cls.pulp2repositories_api = Pulp2RepositoriesApi(migration_client)
def test_second_unit_replaces_the_first(self):
"""Create a duplicate content unit with different ``artifacts`` and same ``relative_path``.
Artifacts are unique by ``relative_path`` and ``file``.
"""
delete_orphans()
content_api = ContentFilesApi(self.client)
repo_api = RepositoriesFileApi(self.client)
versions_api = RepositoriesFileVersionsApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
artifact = gen_artifact()
# create first content unit.
content_attrs = gen_file_content_attrs(artifact)
content_attrs["repository"] = repo.pulp_href
response = content_api.create(**content_attrs)
monitor_task(response.task)
artifact = gen_artifact(file=__file__)
# create second content unit.
second_content_attrs = gen_file_content_attrs(artifact)
second_content_attrs["repository"] = repo.pulp_href
second_content_attrs["relative_path"] = content_attrs["relative_path"]
response = content_api.create(**second_content_attrs)
monitor_task(response.task)
repo_latest_version = versions_api.read(
repo_api.read(repo.pulp_href).latest_version_href)
self.assertEqual(
repo_latest_version.content_summary.present["file.file"]["count"],
1)
def content_file_api_client(file_client):
return ContentFilesApi(file_client)
def setUpClass(cls):
"""Create class-wide variable."""
delete_orphans()
cls.content_unit = {}
cls.file_content_api = ContentFilesApi(gen_file_client())
cls.artifact = gen_artifact()
def setUpClass(cls):
"""Create class-wide variables."""
cls.file_content_api = ContentFilesApi(gen_file_client())
def test_clean_orphan_content_unit(self):
"""Test whether orphan content units can be clean up.
Do the following:
1. Create, and sync a repo.
2. Remove a content unit from the repo. This will create a second
repository version, and create an orphan content unit.
3. Assert that content unit that was removed from the repo and its
artifact are present on disk.
4. Delete orphans.
5. Assert that the orphan content unit was cleaned up, and its artifact
is not present on disk.
"""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
content = choice(get_content(repo.to_dict())[FILE_CONTENT_NAME])
# Create an orphan content unit.
repo_api.modify(repo.pulp_href,
dict(remove_content_units=[content["pulp_href"]]))
artifacts_api = ArtifactsApi(core_client)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact is present on disk.
artifact_path = os.path.join(
MEDIA_PATH,
artifacts_api.read(content["artifact"]).file)
cmd = ("ls", artifact_path)
self.cli_client.run(cmd, sudo=True)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content["pulp_href"], content_units_href)
delete_orphans()
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content["pulp_href"], content_units_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
# Configure HTTP basic authorization: basic
configuration = Configuration()
configuration.username = '******'
configuration.password = '******'
configuration.safe_chars_for_path_param = '/'
core_client = CoreApiClient(configuration)
file_client = FileApiClient(configuration)
# Create api clients for all resource types
artifacts = ArtifactsApi(core_client)
repositories = RepositoriesApi(core_client)
repoversions = RepositoriesVersionsApi(core_client)
filecontent = ContentFilesApi(file_client)
filedistributions = DistributionsFileApi(core_client)
filepublications = PublicationsFileApi(file_client)
fileremotes = RemotesFileApi(file_client)
tasks = TasksApi(core_client)
uploads = UploadsApi(core_client)
# Test creating an Artifact from a 1mb file uploaded in 200kb chunks
with NamedTemporaryFile() as downloaded_file:
response = requests.get(
'https://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/test_bandwidth_repo/'
'pulp-large_1mb_test-packageA-0.1.1-1.fc14.noarch.rpm')
response.raise_for_status()
downloaded_file.write(response.content)
artifact = upload_file_in_chunks(downloaded_file.name)