def test_on_demand(self):
"""Test whether a particular repository version can be published.
1. Create a repository
2. Create a remote with on_demand sync policy
3. Sync
4. Publish repository
"""
client = gen_python_client()
repo_api = RepositoriesPythonApi(client)
remote_api = RemotesPythonApi(client)
publications = PublicationsPypiApi(client)
body = gen_python_remote(policy="on_demand")
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
publish_data = PythonPythonPublication(repository=repo.pulp_href)
publish_response = publications.create(publish_data)
created_resources = monitor_task(publish_response.task)
publication_href = created_resources[0]
self.addCleanup(publications.delete, publication_href)
publication = publications.read(publication_href)
self.assertEqual(publication.repository_version, repo.latest_version_href)
def test_on_demand_pypi_full_sync(self):
"""This test syncs all of PyPi"""
repo_api = RepositoriesPythonApi(self.client)
remote_api = RemotesPythonApi(self.client)
tasks_api = TasksApi(self.core_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_python_remote("https://pypi.org", includes=[], policy="on_demand")
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
sync_task = tasks_api.read(sync_response.task)
time_diff = sync_task.finished_at - sync_task.started_at
print("Delete time: {} seconds".format(time_diff.seconds))
self.assertIsNotNone(repo.latest_version_href)
# As of August 11 2020, all_packages() returns 253,587 packages,
# only 248,677 of them were downloadable
self.assertTrue(get_content_summary(repo.to_dict())[PYTHON_CONTENT_NAME] > 245000)
def test_workflow_02(self):
"""
Verify workflow 2
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Pip install a package from the pulp repository.
3. Check pip install was successful.
This test targets the following issues:
* `Pulp #4682 <https://pulp.plan.io/issues/4682>`_
* `Pulp #4677 <https://pulp.plan.io/issues/4677>`_
"""
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
body = gen_python_remote(includes=PYTHON_LIST_PROJECT_SPECIFIER)
remote = self.remote_api.create(body)
self.addCleanup(self.remote_api.delete, remote.pulp_href)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href,
repository_sync_data)
monitor_task(sync_response.task)
repo = self.repo_api.read(repo.pulp_href)
distribution = self.gen_pub_dist(repo)
self.check_consume(distribution.to_dict())
def test_sync(self):
"""Sync repositories with the python plugin.
In order to sync a repository a remote has to be associated within
this repository. When a repository is created this version field is set
as None. After a sync the repository version is updated.
Do the following:
1. Create a repository, and a remote.
2. Assert that repository version is None.
3. Sync the remote.
4. Assert that repository version is not None.
5. Assert that the correct number of units were added and are present
in the repo.
6. Sync the remote one more time.
7. Assert that repository version is different from the previous one.
8. Assert that the same number of are present and that no units were
added.
"""
repo_api = RepositoriesPythonApi(self.client)
remote_api = RemotesPythonApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_python_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
self.assertIsNotNone(repo.latest_version_href)
self.assertDictEqual(
get_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY
)
self.assertDictEqual(
get_added_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY
)
# Sync the repository again.
latest_version_href = repo.latest_version_href
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
self.assertEqual(latest_version_href, repo.latest_version_href)
self.assertDictEqual(
get_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY
)
def test_04_fully_update(self):
"""Update a remote using HTTP PUT."""
body = _gen_verbose_remote()
response = self.remote_api.update(self.remote.pulp_href, body)
monitor_task(response.task)
for key in ("username", "password"):
del body[key]
type(self).remote = self.remote_api.read(self.remote.pulp_href)
for key, val in body.items():
with self.subTest(key=key):
self.assertEqual(self.remote.to_dict()[key], val, key)
def test_02_change_policy(self):
"""Verify ability to change policy to value other than the default.
Update the remote policy to a valid value other than `immedaite`
and verify the new set value.
"""
changed_policy = choice(
[item for item in self.policies if item != "immediate"])
response = self.remote_api.partial_update(self.remote["pulp_href"],
{"policy": changed_policy})
monitor_task(response.task)
self.remote.update(
self.remote_api.read(self.remote["pulp_href"]).to_dict())
self.assertEqual(self.remote["policy"], changed_policy, self.remote)
def test_07_upload_duplicate_file_without_repo(self):
"""
1) upload file
2) upload the same file again
3) this should fail/send an error
"""
delete_orphans()
response = self.do_upload()
created_resources = monitor_task(response.task)
content_unit = self.python_content_api.read(created_resources[0])
self.check_package_data(content_unit.to_dict())
task_report = monitor_task(self.do_upload().task)
msg = "This field must be unique"
self.assertTrue(msg in task_report["error"]["description"])
def sync_to_remote(self, body, create=False, mirror=False):
"""Takes a body and creates/updates a remote object, then it performs a sync"""
if create:
type(self).remote = self.remote_api.create(body)
else:
remote_task = self.remote_api.partial_update(self.remote.pulp_href, body)
monitor_task(remote_task.task)
type(self).remote = self.remote_api.read(self.remote.pulp_href)
repository_sync_data = RepositorySyncURL(
remote=self.remote.pulp_href, mirror=mirror
)
sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
type(self).repo = self.repo_api.read(self.repo.pulp_href)
def test_01_create_content_unit(self):
"""Create content unit."""
attrs = gen_python_content_attrs(self.artifact)
response = self.python_content_api.create(**attrs)
created_resources = monitor_task(response.task)
content_unit = self.python_content_api.read(created_resources[0])
self.content_unit.update(content_unit.to_dict())
self.check_package_data(self.content_unit)
def gen_pub_dist(self, repo):
"""Takes a repo and generates a publication and then distributes it"""
publication = publish(repo.to_dict())
self.addCleanup(self.publications_api.delete, publication["pulp_href"])
body = gen_distribution()
body["publication"] = publication["pulp_href"]
distro_response = self.distro_api.create(body)
distro = self.distro_api.read(monitor_task(distro_response.task)[0])
self.addCleanup(self.distro_api.delete, distro.pulp_href)
return distro
def test_05_upload_file_without_repo(self):
"""
1) returns a task
2) check task status complete
3) check created resource of completed task
4) ensure only one resource was created and it's a content unit
"""
delete_orphans()
response = self.do_upload()
created_resources = monitor_task(response.task)
content_unit = self.python_content_api.read(created_resources[0]).to_dict()
self.assertEqual(len(created_resources), 1)
self.check_package_data(content_unit)
def do_test(self, url):
"""Sync a repository given ``url`` on the remote."""
repo_api = RepositoriesPythonApi(self.client)
remote_api = RemotesPythonApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_python_remote(url=url)
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
return monitor_task(sync_response.task)
def test_file_decriptors(self):
"""Test whether file descriptors are closed properly.
This test targets the following issue:
`Pulp #4073 <https://pulp.plan.io/issues/4073>`_
Do the following:
1. Check if 'lsof' is installed. If it is not, skip this test.
2. Create and sync a repo.
3. Run the 'lsof' command to verify that files in the
path ``/var/lib/pulp/`` are closed after the sync.
4. Assert that issued command returns `0` opened files.
"""
cli_client = cli.Client(self.cfg, cli.echo_handler)
# check if 'lsof' is available
if cli_client.run(("which", "lsof")).returncode != 0:
raise unittest.SkipTest("lsof package is not present")
repo_api = RepositoriesPythonApi(self.client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
remote_api = RemotesPythonApi(self.client)
remote = remote_api.create(gen_python_remote())
self.addCleanup(remote_api.delete, remote.pulp_href)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
cmd = "lsof -t +D {}".format(MEDIA_PATH).split()
response = cli_client.run(cmd).stdout
self.assertEqual(len(response), 0, response)
def test_06_upload_file_with_repo(self):
"""
1) returns a task
2) check task status complete
3) check created resource of completed task
4) ensure there was two resources created and are a content unit and a repository version
"""
delete_orphans()
repo_api = RepositoriesPythonApi(gen_python_client())
repo = repo_api.create({"name": "foo"})
self.addCleanup(repo_api.delete, repo.pulp_href)
response = self.do_upload(repository=repo.pulp_href)
created_resources = monitor_task(response.task)
self.assertEqual(len(created_resources), 2)
content_list_search = self.python_content_api.list(
repository_version_added=created_resources[0]).results[0]
content_unit = self.python_content_api.read(created_resources[1])
self.assertEqual(content_unit.pulp_href, content_list_search.pulp_href)
self.check_package_data(content_unit.to_dict())
def test_workflow_01(self):
"""
Verify workflow 1
"""
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
artifacts = []
for pkg in self.PACKAGES_URLS:
artifacts.append(gen_artifact(pkg))
for filename, artifact in zip(PYTHON_FIXTURES_FILENAMES, artifacts):
content_response = self.content_api.create(
**gen_python_content_attrs(artifact, filename))
created_resources = monitor_task(content_response.task)
self.repo_api.modify(repo.pulp_href,
{"add_content_units": created_resources})
repo = self.repo_api.read(repo.pulp_href)
distribution = self.gen_pub_dist(repo)
self.addCleanup(delete_orphans, cfg)
self.check_consume(distribution.to_dict())
def test_05_delete(self):
"""Delete a remote."""
response = self.remote_api.delete(self.remote.pulp_href)
monitor_task(response.task)
with self.assertRaises(ApiException):
self.remote_api.read(self.remote.pulp_href)
def test_all(self):
"""Test whether a particular repository version can be published.
1. Create a repository with at least 2 repository versions.
2. Create a publication by supplying the latest ``repository_version``.
3. Assert that the publication ``repository_version`` attribute points
to the latest repository version.
4. Create a publication by supplying the non-latest ``repository_version``.
5. Assert that the publication ``repository_version`` attribute points
to the supplied repository version.
6. Assert that an exception is raised when providing two different
repository versions to be published at same time.
"""
cfg = config.get_config()
client = gen_python_client()
repo_api = RepositoriesPythonApi(client)
remote_api = RemotesPythonApi(client)
publications = PublicationsPypiApi(client)
body = gen_python_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
# Step 1
repo = repo_api.read(repo.pulp_href)
for python_content in get_content(repo.to_dict())[PYTHON_CONTENT_NAME]:
modify_repo(cfg, repo.to_dict(), add_units=[python_content])
version_hrefs = tuple(ver["pulp_href"] for ver in get_versions(repo.to_dict()))
non_latest = choice(version_hrefs[:-1])
# Step 2
publish_data = PythonPythonPublication(repository=repo.pulp_href)
publish_response = publications.create(publish_data)
created_resources = monitor_task(publish_response.task)
publication_href = created_resources[0]
self.addCleanup(publications.delete, publication_href)
publication = publications.read(publication_href)
# Step 3
self.assertEqual(publication.repository_version, version_hrefs[-1])
# Step 4
publish_data.repository = None
publish_data.repository_version = non_latest
publish_response = publications.create(publish_data)
created_resources = monitor_task(publish_response.task)
publication_href = created_resources[0]
publication = publications.read(publication_href)
# Step 5
self.assertEqual(publication.repository_version, non_latest)
# Step 6
with self.assertRaises(ApiException):
body = {"repository": repo.pulp_href, "repository_version": non_latest}
publications.create(body)
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/foo/`` and
``http://example.com/content/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Pulp-Fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/pulp/pulp-smash/issues/872>`_
"""
client = gen_python_client()
repo_api = RepositoriesPythonApi(client)
remote_api = RemotesPythonApi(client)
publications = PublicationsPypiApi(client)
distributions = DistributionsPypiApi(client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_python_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync a Repository
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
# Create a publication.
publish_data = PythonPythonPublication(repository=repo.pulp_href)
publish_response = publications.create(publish_data)
created_resources = monitor_task(publish_response.task)
publication_href = created_resources[0]
self.addCleanup(publications.delete, publication_href)
# Create a distribution.
body = gen_distribution()
body["publication"] = publication_href
distribution_response = distributions.create(body)
created_resources = monitor_task(distribution_response.task)
distribution = distributions.read(created_resources[0])
self.addCleanup(distributions.delete, distribution.pulp_href)
# Pick a content unit (of each type), and download it from both Pulp Fixtures…
unit_paths = [
choice(paths)
for paths in get_python_content_paths(repo.to_dict()).values()
]
fixtures_hashes = [
hashlib.sha256(
utils.http_get(
urljoin(urljoin(PYTHON_FIXTURE_URL, "packages/"),
unit_path[0]))).hexdigest()
for unit_path in unit_paths
]
# …and Pulp.
pulp_hashes = []
cfg = config.get_config()
for unit_path in unit_paths:
content = download_content_unit(cfg, distribution.to_dict(),
unit_path[1])
pulp_hashes.append(hashlib.sha256(content).hexdigest())
self.assertEqual(fixtures_hashes, pulp_hashes)
