def test_all(self):
"""Verify whether the sync of a repository updates its version.
This test explores the design choice stated in the `Pulp #3308`_ that a
new repository version is created even if the sync does not add or
remove any content units. Even without any changes to the remote if a
new sync occurs, a new repository version is created.
.. _Pulp #3308: https://pulp.plan.io/issues/3308
Do the following:
1. Create a repository, and a remote.
2. Sync the repository an arbitrary number of times.
3. Verify that the repository version is equal to the previous number
of syncs.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
number_of_syncs = randint(1, 10)
for _ in range(number_of_syncs):
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
path = urlsplit(repo['_latest_version_href']).path
latest_repo_version = int(path.split('/')[-2])
self.assertEqual(latest_repo_version, number_of_syncs)
def test_exclude_unavailable_projects(self):
"""
Test that sync doesn't fail if some of the excluded projects aren't available.
Do the following:
1. Update the remote to exclude a specifier that is smaller than the previous one.
2. Sync the remote again.
3. Assert that the content counts in the repo have increased again, to the count
of the smaller excludes specifier.
"""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
body = gen_python_remote(
includes=PYTHON_MD_PROJECT_SPECIFIER,
excludes=PYTHON_UNAVAILABLE_PROJECT_SPECIFIER
)
remote = self.client.post(PYTHON_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertEqual(
get_content_summary(repo)[PYTHON_CONTENT_NAME],
PYTHON_MD_PACKAGE_COUNT - PYTHON_UNAVAILABLE_PACKAGE_COUNT
)
def test_all(self):
"""Test whether the content present in a repo version is immutable.
Do the following:
1. Create a repository that has at least one repository version.
2. Attempt to update the content of a repository version.
3. Assert that an HTTP exception is raised.
4. Assert that the repository version was not updated.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
latest_version_href = client.get(repo['_href'])['_latest_version_href']
with self.assertRaises(HTTPError):
client.post(latest_version_href)
repo = client.get(repo['_href'])
self.assertEqual(latest_version_href, repo['_latest_version_href'])
def test_include_unavailable_projects(self):
"""
Test that the sync doesn't fail if some included projects aren't available.
Do the following:
1. Create a remote.
2. Sync the remote.
3. Assert that the content counts in the repo match the correct count for the specifier.
"""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
body = gen_python_remote(includes=PYTHON_UNAVAILABLE_PROJECT_SPECIFIER)
remote = self.client.post(PYTHON_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertEqual(
get_content_summary(repo)[PYTHON_CONTENT_NAME],
PYTHON_UNAVAILABLE_PACKAGE_COUNT
)
def test_all(self):
"""Test whether sync/publish for content already in Pulp."""
cfg = config.get_config()
client = api.Client(cfg, api.page_handler)
# step 1. delete orphans to assure that no content is present on disk,
# or database.
delete_orphans(cfg)
remote = client.post(
FILE_REMOTE_PATH,
gen_remote(FILE_FIXTURE_MANIFEST_URL)
)
self.addCleanup(client.delete, remote['_href'])
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
publisher = client.post(FILE_PUBLISHER_PATH, gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
for _ in range(2):
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
publish(cfg, publisher, repo)
def test_02_sync_content(self):
"""Sync content into the repository.
Assert that:
* The ``_versions_href`` API call is correct.
* The ``_latest_version_href`` API call is correct.
* The ``_latest_version_href + content/`` API call is correct.
* The ``_latest_version_href + added_content/`` API call is correct.
* The ``_latest_version_href + removed_content/`` API call is correct.
* The ``content_summary`` attribute is correct.
"""
body = gen_remote(urljoin(FILE_FIXTURE_URL, 'PULP_MANIFEST'))
self.remote.update(self.client.post(FILE_REMOTE_PATH, body))
sync(self.cfg, self.remote, self.repo)
repo = self.client.get(self.repo['_href'])
repo_versions = get_versions(repo)
self.assertEqual(len(repo_versions), 1, repo_versions)
self.assertIsNotNone(repo['_latest_version_href'])
content = get_content(repo)
self.assertEqual(len(content), FILE_FIXTURE_COUNT)
added_content = get_added_content(repo)
self.assertEqual(len(added_content), FILE_FIXTURE_COUNT, added_content)
removed_content = get_removed_content(repo)
self.assertEqual(len(removed_content), 0, removed_content)
content_summary = self.get_content_summary(repo)
self.assertEqual(content_summary, {'file': FILE_FIXTURE_COUNT})
def populate_pulp(cfg, remote=None):
"""
Add python content to Pulp.
Args:
cfg (pulp_smash.config.PulpSmashConfig): Information about a Pulp application
remote (dict): A dict of information about the remote.
Returns:
list: A list of dicts, where each dict describes one python content in Pulp.
"""
if remote is None:
remote = gen_python_remote()
client = api.Client(cfg, api.json_handler)
repo = {}
try:
remote.update(client.post(PYTHON_REMOTE_PATH, remote))
repo.update(client.post(REPO_PATH, gen_repo()))
sync(cfg, remote, repo)
finally:
if remote:
client.delete(remote['_href'])
if repo:
client.delete(repo['_href'])
return client.get(PYTHON_CONTENT_PATH)['results']
def test_03_add_subset_exclude(self):
"""
Test that excluding a subset of the packages will reduce the count by that amount.
Do the following:
1. Update the remote to exclude a specifier that is a strict subset of the previous one.
2. Sync the remote again.
3. Assert that that the content counts in the repo decreased by the count of the
excludes specifier.
"""
body = {'excludes': PYTHON_SM_PROJECT_SPECIFIER}
self.client.patch(self.remote['_href'], body)
type(self).remote = self.client.get(self.remote['_href'])
sync(self.cfg, self.remote, self.repo, mirror=True)
type(self).repo = self.client.get(self.repo['_href'])
self.assertEqual(
get_content_summary(self.repo)[PYTHON_CONTENT_NAME],
PYTHON_MD_PACKAGE_COUNT - PYTHON_SM_PACKAGE_COUNT
)
self.assertEqual(
get_removed_content_summary(self.repo)[PYTHON_CONTENT_NAME],
PYTHON_SM_PACKAGE_COUNT
)
def test_all(self):
"""Verify multi-resourcing locking.
Do the following:
1. Create a repository, and a remote.
2. Update the remote to point to a different url.
3. Immediately run a sync. The sync should fire after the update and
sync from the second url.
4. Assert that remote url was updated.
5. Assert that the number of units present in the repository is
according to the updated url.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_file_remote(url=FILE_LARGE_FIXTURE_MANIFEST_URL)
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
url = {'url': FILE_FIXTURE_MANIFEST_URL}
client.patch(remote['_href'], url)
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
remote = client.get(remote['_href'])
self.assertEqual(remote['url'], url['url'])
self.assertDictEqual(get_content_summary(repo), FILE_FIXTURE_SUMMARY)
def populate_pulp(cfg, url=DOCKER_V2_FEED_URL):
"""Add docker contents to Pulp.
:param pulp_smash.config.PulpSmashConfig: Information about a Pulp application.
:param url: The docker repository URL. Defaults to
:data:`pulp_smash.constants.DOCKER_FIXTURE_URL`
:returns: A list of dicts, where each dict describes one file content in Pulp.
"""
client = api.Client(cfg, api.json_handler)
remote = {}
repo = {}
try:
remote.update(
client.post(
DOCKER_REMOTE_PATH,
gen_remote(url, upstream_name=DOCKER_UPSTREAM_NAME)
)
)
repo.update(client.post(REPO_PATH, gen_repo()))
sync(cfg, remote, repo)
finally:
if remote:
client.delete(remote['_href'])
if repo:
client.delete(repo['_href'])
return client.get(DOCKER_CONTENT_PATH)['results']
def test_all(self):
"""Verify publisher and remote can be used with different repos.
This test explores the design choice stated in `Pulp #3341`_ that
remove the FK from publishers and remotes to repository.
Allowing remotes and publishers to be used with different
repositories.
.. _Pulp #3341: https://pulp.plan.io/issues/3341
Do the following:
1. Create a remote, and a publisher.
2. Create 2 repositories.
3. Sync both repositories using the same remote.
4. Assert that the two repositories have the same contents.
5. Publish both repositories using the same publisher.
6. Assert that each generated publication has the same publisher, but
are associated with different repositories.
"""
cfg = config.get_config()
# Create a remote and publisher.
client = api.Client(cfg, api.json_handler)
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
publisher = client.post(FILE_PUBLISHER_PATH, gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Create and sync repos.
repos = []
for _ in range(2):
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
sync(cfg, remote, repo)
repos.append(client.get(repo['_href']))
# Compare contents of repositories.
contents = []
for repo in repos:
contents.append(get_content(repo)[FILE_CONTENT_NAME])
self.assertEqual(
{content['_href'] for content in contents[0]},
{content['_href'] for content in contents[1]},
)
# Publish repositories.
publications = []
for repo in repos:
publications.append(publish(cfg, publisher, repo))
self.assertEqual(
publications[0]['publisher'],
publications[1]['publisher']
)
self.assertNotEqual(
publications[0]['repository_version'],
publications[1]['repository_version']
)
def test_all(self):
"""Test whether content unit used by a repo version can be deleted.
Do the following:
1. Sync content to a repository.
2. Attempt to delete a content unit present in a repository version.
Assert that a HTTP exception was raised.
3. Assert that number of content units present on the repository
does not change after the attempt to delete one content unit.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
body = gen_rpm_remote()
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
content = get_content(repo)
with self.assertRaises(HTTPError):
client.delete(choice(content)['_href'])
self.assertEqual(len(content), len(get_content(repo)))
def test_file_decriptors(self):
"""Test whether file descriptors are closed properly.
This test targets the following issue:
`Pulp #4073 <https://pulp.plan.io/issues/4073>`_
Do the following:
1. Check if 'lsof' is installed. If it is not, skip this test.
2. Create and sync a repo.
3. Run the 'lsof' command to verify that files in the
path ``/var/lib/pulp/`` are closed after the sync.
4. Assert that issued command returns `0` opened files.
"""
cli_client = cli.Client(self.cfg, cli.echo_handler)
# check if 'lsof' is available
if cli_client.run(('which', 'lsof')).returncode != 0:
raise unittest.SkipTest('lsof package is not present')
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
remote = self.client.post(DEB_REMOTE_PATH, gen_deb_remote())
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
cmd = 'lsof -t +D {}'.format(MEDIA_PATH).split()
response = cli_client.run(cmd).stdout
self.assertEqual(len(response), 0, response)
def test_all(self):
"""Sync/publish a repo that ``updateinfo.xml`` contains references.
This test targets the following issue:
`Pulp #3998 <https://pulp.plan.io/issues/3998>`_.
"""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
body = gen_rpm_remote(url=RPM_REFERENCES_UPDATEINFO_URL)
remote = self.client.post(RPM_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertIsNotNone(repo['_latest_version_href'])
content_summary = get_content_summary(repo)
self.assertDictEqual(
content_summary,
RPM_FIXTURE_SUMMARY,
content_summary
)
publication = publish(self.cfg, repo)
self.addCleanup(self.client.delete, publication['_href'])
def populate_pulp(cfg, url=None):
"""Add file contents to Pulp.
:param pulp_smash.config.PulpSmashConfig: Information about a Pulp
application.
:param url: The URL to a file repository's ``PULP_MANIFEST`` file. Defaults
to :data:`pulp_smash.constants.FILE_FEED_URL` + ``PULP_MANIFEST``.
:returns: A list of dicts, where each dict describes one file content in
Pulp.
"""
if url is None:
url = urljoin(FILE_FIXTURE_URL, 'PULP_MANIFEST')
client = api.Client(cfg, api.json_handler)
remote = {}
repo = {}
try:
remote.update(client.post(FILE_REMOTE_PATH, gen_remote(url)))
repo.update(client.post(REPO_PATH, gen_repo()))
sync(cfg, remote, repo)
finally:
if remote:
client.delete(remote['_href'])
if repo:
client.delete(repo['_href'])
return client.get(FILE_CONTENT_PATH)['results']
def test_sync(self):
"""Sync repositories with the docker plugin.
In order to sync a repository a remote has to be associated within
this repository. When a repository is created this version field is set
as None. After a sync the repository version is updated.
Do the following:
1. Create a repository, and a remote.
2. Assert that repository version is None.
3. Sync the remote.
4. Assert that repository version is not None.
5. Sync the remote one more time.
6. Assert that repository version is different from the previous one.
"""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
remote = self.client.post(DOCKER_REMOTE_PATH, gen_docker_remote())
self.addCleanup(self.client.delete, remote['_href'])
# Sync the repository.
self.assertIsNone(repo['_latest_version_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertIsNotNone(repo['_latest_version_href'])
# Sync the repository again.
latest_version_href = repo['_latest_version_href']
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertNotEqual(latest_version_href, repo['_latest_version_href'])
def test_03_removing_units(self):
"""
Sync a Remote, excluding prereleases again.
Just to be sure that the units are being properly removed afterwards.
Do the following:
1. Update the remote to exclude pre-releases again.
2. Sync the remote again.
3. Assert that we're back to the state in test_01_excluding_prereleases.
"""
body = {'prereleases': False}
self.client.patch(self.remote['_href'], body)
type(self).remote = self.client.get(self.remote['_href'])
sync(self.cfg, self.remote, self.repo, mirror=True)
type(self).repo = self.client.get(self.repo['_href'])
self.assertDictEqual(
get_content_summary(self.repo),
PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY
)
self.assertEqual(
get_removed_content_summary(self.repo)[PYTHON_CONTENT_NAME],
PYTHON_WITH_PRERELEASE_COUNT - PYTHON_WITHOUT_PRERELEASE_COUNT
)
def test_all(self):
"""Test whether a particular repository version can be published.
1. Create a repository with at least 2 repository versions.
2. Create a publication by supplying the latest ``repository_version``.
3. Assert that the publication ``repository_version`` attribute points
to the latest repository version.
4. Create a publication by supplying the non-latest
``repository_version``.
5. Assert that the publication ``repository_version`` attribute points
to the supplied repository version.
6. Assert that an exception is raised when providing two different
repository versions to be published at same time.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
body = gen_rpm_remote()
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
sync(cfg, remote, repo)
publisher = client.post(RPM_PUBLISHER_PATH, gen_rpm_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Step 1
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
for rpm_content in client.get(RPM_CONTENT_PATH)['results']:
client.post(
repo['_versions_href'],
{'add_content_units': [rpm_content['_href']]}
)
version_hrefs = tuple(ver['_href'] for ver in get_versions(repo))
non_latest = choice(version_hrefs[:-1])
# Step 2
publication = publish(cfg, publisher, repo)
# Step 3
self.assertEqual(publication['repository_version'], version_hrefs[-1])
# Step 4
publication = publish(cfg, publisher, repo, non_latest)
# Step 5
self.assertEqual(publication['repository_version'], non_latest)
# Step 6
with self.assertRaises(HTTPError):
body = {
'repository': repo['_href'],
'repository_version': non_latest
}
client.post(urljoin(publisher['_href'], 'publish/'), body)
def test_all(self):
"""Sync two copies of the same packages, make sure we end up with only one copy.
Do the following:
1. Create a repository and a remote.
2. Sync the remote.
3. Assert that the content summary matches what is expected.
4. Create a new remote w/ using fixture containing updated errata (packages with the same
NEVRA as the existing package content, but different pkgId)
5. Sync the remote again.
6. Assert that repository version is different from the previous one but has the same
content summary.
7. Assert that the packages have changed since the last sync.
"""
client = api.Client(self.cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
# Create a remote with the unsigned RPM fixture url.
body = gen_rpm_remote(url=RPM_UNSIGNED_FIXTURE_URL)
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
# Sync the repository.
self.assertIsNone(repo['_latest_version_href'])
sync(self.cfg, remote, repo)
repo = client.get(repo['_href'])
self.assertDictEqual(get_content_summary(repo), RPM_FIXTURE_CONTENT_SUMMARY)
# Save a copy of the original packages.
original_packages = {
content['errata_id']: content for content in get_content(repo)
if content['type'] == 'packages'
}
# Create a remote with a different test fixture with the same NEVRA but different digests.
body = gen_rpm_remote(url=RPM_SIGNED_FIXTURE_URL)
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
# Sync the repository again.
sync(self.cfg, remote, repo)
repo = client.get(repo['_href'])
self.assertDictEqual(get_content_summary(repo), RPM_FIXTURE_CONTENT_SUMMARY)
self.assertEqual(len(get_added_content(repo)), 0)
# Test that the packages have been modified.
mutated_packages = {
content['errata_id']: content for content in get_content(repo)
if content['type'] == 'update'
}
self.assertNotEqual(mutated_packages, original_packages)
self.assertNotEqual(mutated_packages[RPM_PACKAGE_NAME]['pkgId'],
original_packages[RPM_PACKAGE_NAME]['pkgId'])
def test_rpm(self):
"""Sync repositories with the rpm plugin.
In order to sync a repository a remote has to be associated within
this repository. When a repository is created this version field is set
as None. After a sync the repository version is updated.
Do the following:
1. Create a repository and a remote.
2. Assert that repository version is None.
3. Sync the remote.
4. Assert that repository version is not None.
5. Assert that the correct number of units were added and are present
in the repo.
6. Sync the remote one more time.
7. Assert that repository version is different from the previous one.
8. Assert that the same number of are present and that no units were
added.
"""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
# Create a remote with the standard test fixture url.
body = gen_rpm_remote()
remote = self.client.post(RPM_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
# Sync the repository.
self.assertIsNone(repo['_latest_version_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
# Check that we have the correct content counts.
self.assertIsNotNone(repo['_latest_version_href'])
self.assertDictEqual(
get_content_summary(repo),
RPM_FIXTURE_SUMMARY
)
self.assertDictEqual(
get_added_content_summary(repo),
RPM_FIXTURE_SUMMARY
)
# Sync the repository again.
latest_version_href = repo['_latest_version_href']
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
# Check that nothing has changed since the last sync.
self.assertNotEqual(latest_version_href, repo['_latest_version_href'])
self.assertDictEqual(
get_content_summary(repo),
RPM_FIXTURE_SUMMARY
)
self.assertDictEqual(get_added_content_summary(repo), {})
def test_all(self):
"""Sync two fixture content with same NEVRA and different checksum.
Make sure we end up with the most recently synced content.
Do the following:
1. Create a repository
2. Create two remotes with same content but different checksums.
Sync the remotes one after the other.
a. Sync remote with packages with SHA256: ``RPM_UNSIGNED_FIXTURE_URL``.
b. Sync remote with packages with SHA512: ``RPM_SHA512_FIXTURE_URL``.
3. Make sure the latest content is only kept.
This test targets the following issues:
* `Pulp #4297 <https://pulp.plan.io/issues/4297>`_
* `Pulp #3954 <https://pulp.plan.io/issues/3954>`_
"""
# Step 1
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
# Step 2.
for body in [
gen_rpm_remote(),
gen_rpm_remote(url=RPM_SHA512_FIXTURE_URL)
]:
remote = self.client.post(RPM_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
# Sync the repository.
sync(self.cfg, remote, repo)
# Step 3
repo = self.client.get(repo['_href'])
added_content = get_content(repo)[RPM_PACKAGE_CONTENT_NAME]
removed_content = get_removed_content(repo)[RPM_PACKAGE_CONTENT_NAME]
# In case of "duplicates" the most recent one is chosen, so the old
# package is removed from and the new one is added to a repo version.
self.assertEqual(
len(added_content),
RPM_PACKAGES_COUNT,
added_content
)
self.assertEqual(
len(removed_content),
RPM_PACKAGES_COUNT,
removed_content
)
# Verifying whether the packages with first checksum is removed and second
# is added.
self.assertEqual(added_content[0]['checksum_type'], 'sha512')
self.assertEqual(removed_content[0]['checksum_type'], 'sha256')
def create_sync_repo(self):
"""Create, and sync a repo."""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
body = gen_file_remote(url=FILE_FIXTURE_MANIFEST_URL)
remote = self.client.post(FILE_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
return self.client.get(repo['_href'])
def test_all(self):
"""Verify the set up of parameters related to auto distribution.
This test targets the following issues:
* `Pulp #3295 <https://pulp.plan.io/issues/3295>`_
* `Pulp #3392 <https://pulp.plan.io/issues/3392>`_
* `Pulp #3394 <https://pulp.plan.io/issues/3394>`_
* `Pulp #3671 <https://pulp.plan.io/issues/3671>`_
* `Pulp Smash #883 <https://github.com/PulpQE/pulp-smash/issues/883>`_
* `Pulp Smash #917 <https://github.com/PulpQE/pulp-smash/issues/917>`_
"""
# Create a repository and a publisher.
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
publisher = self.client.post(FILE_PUBLISHER_PATH, gen_publisher())
self.addCleanup(self.client.delete, publisher['_href'])
# Create a distribution.
self.try_create_distribution(publisher=publisher['_href'])
self.try_create_distribution(repository=repo['_href'])
body = gen_distribution()
body['publisher'] = publisher['_href']
body['repository'] = repo['_href']
distribution = self.client.post(DISTRIBUTION_PATH, body)
self.addCleanup(self.client.delete, distribution['_href'])
# Update the distribution.
self.try_update_distribution(distribution, publisher=None)
self.try_update_distribution(distribution, repository=None)
distribution = self.client.patch(distribution['_href'], {
'publisher': None,
'repository': None,
})
self.assertIsNone(distribution['publisher'], distribution)
self.assertIsNone(distribution['repository'], distribution)
# Publish the repository. Assert that distribution does not point to
# the new publication (because publisher and repository are unset).
remote = self.client.post(
FILE_REMOTE_PATH,
gen_remote(FILE_FIXTURE_MANIFEST_URL),
)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
publication = publish(self.cfg, publisher, repo)
self.addCleanup(self.client.delete, publication['_href'])
distribution = self.client.get(distribution['_href'])
self.assertNotEqual(distribution['publication'], publication['_href'])
def test_all(self):
"""Test whether a particular repository version can be published.
1. Create a repository with at least 2 repository versions.
2. Create a publication by supplying the latest ``repository_version``.
3. Assert that the publication ``repository_version`` attribute points
to the latest repository version.
4. Create a publication by supplying the non-latest
``repository_version``.
5. Assert that the publication ``repository_version`` attribute points
to the supplied repository version.
6. Assert that an exception is raised when providing two different
repository versions to be published at same time.
"""
body = gen_rpm_remote()
remote = self.client.post(RPM_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
sync(self.cfg, remote, repo)
# Step 1
repo = self.client.get(repo['_href'])
for rpm_content in get_content(repo)[RPM_PACKAGE_CONTENT_NAME]:
self.client.post(
repo['_versions_href'],
{'add_content_units': [rpm_content['_href']]}
)
version_hrefs = tuple(ver['_href'] for ver in get_versions(repo))
non_latest = choice(version_hrefs[:-1])
# Step 2
publication = publish(self.cfg, repo)
# Step 3
self.assertEqual(publication['repository_version'], version_hrefs[-1])
# Step 4
publication = publish(self.cfg, repo, non_latest)
# Step 5
self.assertEqual(publication['repository_version'], non_latest)
# Step 6
with self.assertRaises(HTTPError):
body = {
'repository': repo['_href'],
'repository_version': non_latest
}
self.client.post(RPM_PUBLICATION_PATH, body)
def test_all(self):
"""Verify whether package manager can consume content from Pulp.
This test targets the following issue:
`Pulp #3204 <https://pulp.plan.io/issues/3204>`_
"""
cfg = config.get_config()
try:
cli.PackageManager._get_package_manager(cfg) # pylint:disable=protected-access
except NoKnownPackageManagerError:
raise unittest.SkipTest('This test requires dnf or yum.')
client = api.Client(cfg, api.json_handler)
body = gen_rpm_remote()
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
sync(cfg, remote, repo)
publisher = client.post(RPM_PUBLISHER_PATH, gen_rpm_publisher())
self.addCleanup(client.delete, publisher['_href'])
publication = publish(cfg, publisher, repo)
self.addCleanup(client.delete, publication['_href'])
body = gen_distribution()
body['publication'] = publication['_href']
distribution = client.post(DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution['_href'])
repo_path = gen_yum_config_file(
cfg,
baseurl=urljoin(cfg.get_base_url(), urljoin(
'pulp/content/',
distribution['base_path']
)),
name=repo['name'],
repositoryid=repo['name']
)
cli_client = cli.Client(cfg)
self.addCleanup(cli_client.run, ('rm', repo_path), sudo=True)
rpm_name = 'walrus'
pkg_mgr = cli.PackageManager(cfg)
pkg_mgr.install(rpm_name)
self.addCleanup(pkg_mgr.uninstall, rpm_name)
rpm = cli_client.run(('rpm', '-q', rpm_name)).stdout.strip().split('-')
self.assertEqual(rpm_name, rpm[0])
def test_clean_orphan_content_unit(self):
"""Test whether orphan content units can be clean up.
Do the following:
1. Create, and sync a repo.
2. Remove a content unit from the repo. This will create a second
repository version, and create an orphan content unit.
3. Assert that content unit that was removed from the repo and its
artifact are present on disk.
4. Delete orphans.
5. Assert that the orphan content unit was cleaned up, and its artifact
is not present on disk.
"""
repo = self.api_client.post(REPO_PATH, gen_repo())
self.addCleanup(self.api_client.delete, repo['_href'])
body = gen_file_remote()
remote = self.api_client.post(FILE_REMOTE_PATH, body)
self.addCleanup(self.api_client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.api_client.get(repo['_href'])
content = choice(get_content(repo)[FILE_CONTENT_NAME])
# Create an orphan content unit.
self.api_client.post(
repo['_versions_href'],
{'remove_content_units': [content['_href']]}
)
# Verify that the artifact is present on disk.
artifact_path = self.api_client.get(content['_artifact'])['file']
cmd = ('ls', artifact_path)
self.cli_client.run(cmd, sudo=True)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo)[0]['_href'])
content_units = self.api_client.get(FILE_CONTENT_PATH)['results']
self.assertIn(content, content_units)
delete_orphans()
content_units = self.api_client.get(FILE_CONTENT_PATH)['results']
self.assertNotIn(content, content_units)
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def do_test(self, url):
"""Sync and publish an RPM repository given a feed URL."""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
remote = self.client.post(RPM_REMOTE_PATH, gen_rpm_remote(url=url))
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertIsNotNone(repo['_latest_version_href'])
publication = publish(self.cfg, repo)
self.addCleanup(self.client.delete, publication['_href'])
def do_publish(self, download_policy):
"""Publish repository synced with lazy ``download_policy``."""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
remote = self.client.post(
RPM_REMOTE_PATH,
gen_rpm_remote(policy=download_policy)
)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
publication = publish(self.cfg, repo)
self.assertIsNotNone(publication['repository_version'], publication)
def do_publish(self, download_policy):
"""Publish repository synced with lazy ``download_policy``."""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
remote = self.client.post(
RPM_REMOTE_PATH,
gen_rpm_remote(policy=download_policy)
)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
publisher = self.client.post(RPM_PUBLISHER_PATH, gen_rpm_publisher())
self.addCleanup(self.client.delete, publisher['_href'])
def do_sync(self, download_policy):
"""Sync repositories with the different ``download_policy``.
Do the following:
1. Create a repository, and a remote.
2. Assert that repository version is None.
3. Sync the remote.
4. Assert that repository version is not None.
5. Assert that the correct number of possible units to be downloaded
were shown.
6. Sync the remote one more time in order to create another repository
version.
7. Assert that repository version is different from the previous one.
8. Assert that the same number of units are shown, and after the
second sync no extra units should be shown, since the same remote
was synced again.
"""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
remote = self.client.post(
RPM_REMOTE_PATH,
gen_rpm_remote(policy=download_policy)
)
self.addCleanup(self.client.delete, remote['_href'])
# Sync the repository.
self.assertIsNone(repo['_latest_version_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertIsNotNone(repo['_latest_version_href'])
self.assertDictEqual(get_content_summary(repo), RPM_FIXTURE_SUMMARY)
self.assertDictEqual(
get_added_content_summary(repo),
RPM_FIXTURE_SUMMARY
)
# Sync the repository again.
latest_version_href = repo['_latest_version_href']
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
self.assertNotEqual(latest_version_href, repo['_latest_version_href'])
self.assertDictEqual(get_content_summary(repo), RPM_FIXTURE_SUMMARY)
self.assertDictEqual(get_added_content_summary(repo), {})
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/foo/`` and
``http://example.com/content/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Pulp-Fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_gem_remote()
remote = client.post(GEM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
# Create a publisher.
publisher = client.post(GEM_PUBLISHER_PATH, gen_gem_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Create a publication.
publication = publish(cfg, publisher, repo)
self.addCleanup(client.delete, publication['_href'])
# Create a distribution.
body = gen_distribution()
body['publication'] = publication['_href']
distribution = client.post(DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution['_href'])
# Pick a content unit, and download it from both Pulp Fixtures…
unit_path = choice(get_gem_content_paths(repo))
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(GEM_FIXTURE_URL, unit_path))).hexdigest()
# …and Pulp.
client.response_handler = api.safe_handler
unit_url = cfg.get_hosts('api')[0].roles['api']['scheme']
unit_url += '://' + distribution['base_url'] + '/'
unit_url = urljoin(unit_url, unit_path)
pulp_hash = hashlib.sha256(client.get(unit_url).content).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/foo/`` and
``http://example.com/content/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Remote.
NOTE: content unit for docker is `image` or `Layer`
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_docker_remote()
remote = client.post(DOCKER_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
# Create a publisher.
publisher = client.post(DOCKER_PUBLISHER_PATH, gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Create a publication.
publication = publish(cfg, publisher, repo)
self.addCleanup(client.delete, publication['_href'])
# Create a distribution.
body = gen_distribution()
body['publication'] = publication['_href']
distribution = client.post(DOCKER_DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution['_href'])
# Get local checksums for content synced from remote registy
checksums = [
content['digest'] for content
in get_content(repo)['docker.manifest-blob']
]
# Assert that at least one layer is synced from remote:latest
# and the checksum matched with remote
self.assertTrue(
any(
[
result['blobSum'] in checksums
for result in get_docker_hub_remote_blobsums()
]
)
)
def test_all(self):
"""Test whether a particular repository version can be published.
1. Create a repository with at least 2 repository versions.
2. Create a publication without supplying a repository_version (i.e.
take the latest ``repository_version``).
3. Assert that the publication ``repository_version`` attribute points
to the latest repository version.
4. Create a publication by supplying the non-latest
``repository_version``.
5. Assert that the publication ``repository_version`` attribute points
to the supplied repository version.
6. Assert that an exception is raised when providing two different
repository versions to be published at same time.
"""
cfg = config.get_config()
delete_orphans(cfg)
client = api.Client(cfg, api.json_handler)
body = gen_remote(fixture_u1.url,
cookbooks={fixture_u1.example1_name: ""})
remote = client.post(COOKBOOK_REMOTE_PATH, body)
self.addCleanup(client.delete, remote["_href"])
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["_href"])
sync(cfg, remote, repo, mirror=True)
repo = client.get(repo["_href"])
repo_content = get_cookbook_content(repo)
self.assertTrue(repo_content)
# Step 1
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["_href"])
for cookbook in repo_content:
client.post(repo["_versions_href"],
{"add_content_units": [cookbook["_href"]]})
version_hrefs = tuple(ver["_href"] for ver in get_versions(repo))
non_latest = choice(version_hrefs[:-1])
# Step 2
publication = create_publication(cfg, repo)
# Step 3
self.assertEqual(publication["repository_version"], version_hrefs[-1])
# Step 4
publication = create_publication(cfg, repo, version_href=non_latest)
# Step 5
self.assertEqual(publication["repository_version"], non_latest)
# Step 6
with self.assertRaises(HTTPError):
body = {
"repository": repo["_href"],
"repository_version": non_latest
}
client.post(COOKBOOK_PUBLICATION_PATH, body)
def do_test(self, policy):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/foo/`` and
``http://example.com/content/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Pulp-Fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(DEB_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["pulp_href"])
body = gen_deb_remote()
remote = client.post(DEB_REMOTE_PATH, body)
self.addCleanup(client.delete, remote["pulp_href"])
sync(cfg, remote, repo)
repo = client.get(repo["pulp_href"])
# Create a publication.
publication = self.Meta.create_publication(cfg, repo)
self.addCleanup(client.delete, publication["pulp_href"])
# Create a distribution.
body = gen_distribution()
body["publication"] = publication["pulp_href"]
distribution = client.using_handler(api.task_handler).post(
self.Meta.DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution["pulp_href"])
# Pick a content unit (of each type), and download it from both Pulp Fixtures…
unit_paths = [
choice(paths)
for paths in self.Meta.get_content_unit_paths(repo).values()
if paths
]
fixtures_hashes = [
hashlib.sha256(
utils.http_get(urljoin(DEB_FIXTURE_URL,
unit_path[0]))).hexdigest()
for unit_path in unit_paths
]
# …and Pulp.
contents = [
download_content_unit(cfg, distribution, unit_path[1])
for unit_path in unit_paths
]
pulp_hashes = [
hashlib.sha256(content).hexdigest() for content in contents
]
self.assertEqual(fixtures_hashes, pulp_hashes)
def test_all(self):
"""Sync two copies of the same packages.
Make sure we end up with only one copy.
Do the following:
1. Create a repository and a remote.
2. Sync the remote.
3. Assert that the content summary matches what is expected.
4. Create a new remote w/ using fixture containing updated errata
(packages with the same NEVRA as the existing package content, but
different pkgId).
5. Sync the remote again.
6. Assert that repository version is different from the previous one
but has the same content summary.
7. Assert that the packages have changed since the last sync.
"""
client = api.Client(self.cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
# Create a remote with the unsigned RPM fixture url.
body = gen_rpm_remote(url=RPM_UNSIGNED_FIXTURE_URL)
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
# Sync the repository.
self.assertIsNone(repo['_latest_version_href'])
sync(self.cfg, remote, repo)
repo = client.get(repo['_href'])
self.assertDictEqual(get_content_summary(repo),
RPM_FIXTURE_CONTENT_SUMMARY)
# Save a copy of the original packages.
original_packages = {
content['errata_id']: content
for content in get_content(repo) if content['type'] == 'packages'
}
# Create a remote with a different test fixture with the same NEVRA but
# different digests.
body = gen_rpm_remote(url=RPM_SIGNED_FIXTURE_URL)
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
# Sync the repository again.
sync(self.cfg, remote, repo)
repo = client.get(repo['_href'])
self.assertDictEqual(get_content_summary(repo),
RPM_FIXTURE_CONTENT_SUMMARY)
self.assertEqual(len(get_added_content(repo)), 0)
# Test that the packages have been modified.
mutated_packages = {
content['errata_id']: content
for content in get_content(repo) if content['type'] == 'update'
}
self.assertNotEqual(mutated_packages, original_packages)
self.assertNotEqual(mutated_packages[RPM_PACKAGE_NAME]['pkgId'],
original_packages[RPM_PACKAGE_NAME]['pkgId'])
def test_all(self):
"""Test content promotion for a distribution.
This test targets the following issue:
* `Pulp #4186 <https://pulp.plan.io/issues/4186>`_
* `Pulp #8475 <https://pulp.plan.io/issues/8475>`_
Do the following:
1. Create a repository that has at least one repository version.
2. Create a publication.
3. Create 2 distributions - using the same publication. Those
distributions will have different ``base_path``.
4. Assert that distributions have the same publication.
5. Assert that distributions are viewable from base url
6. Assert that content in distributions are viewable
7. Select a content unit. Download that content unit from Pulp using
the two different distributions.
Assert that content unit has the same checksum when fetched from
different distributions.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["pulp_href"])
remote = client.post(FILE_REMOTE_PATH,
gen_remote(FILE_FIXTURE_MANIFEST_URL))
self.addCleanup(client.delete, remote["pulp_href"])
sync(cfg, remote, repo)
repo = client.get(repo["pulp_href"])
publication = create_file_publication(cfg, repo)
self.addCleanup(client.delete, publication["pulp_href"])
distributions = []
for _ in range(2):
body = gen_distribution()
body["publication"] = publication["pulp_href"]
distribution = client.using_handler(api.task_handler).post(
FILE_DISTRIBUTION_PATH, body)
distributions.append(distribution)
self.addCleanup(client.delete, distribution["pulp_href"])
self.assertEqual(distributions[0]["publication"],
distributions[1]["publication"], distributions)
client.response_handler = api.safe_handler
self.assertEqual(client.get(PULP_CONTENT_BASE_URL).status_code, 200)
for distribution in distributions:
self.assertEqual(
client.get(distribution["base_url"]).status_code, 200)
unit_urls = []
unit_path = get_added_content(
repo)[FILE_CONTENT_NAME][0]["relative_path"]
for distribution in distributions:
unit_url = distribution["base_url"]
unit_urls.append(urljoin(unit_url, unit_path))
self.assertEqual(
hashlib.sha256(client.get(unit_urls[0]).content).hexdigest(),
hashlib.sha256(client.get(unit_urls[1]).content).hexdigest(),
unit_urls,
)
def setUpClass(cls):
"""Create class-wide variables.
1. Create a repository.
2. Create a remote pointing to external registry.
3. Sync the repository using the remote and re-read the repo data.
4. Create a container distribution to serve the repository
5. Create another container distribution to the serve the repository version
This tests targets the following issue:
* `Pulp #4460 <https://pulp.plan.io/issues/4460>`_
"""
cls.cfg = config.get_config()
token_auth = cls.cfg.hosts[0].roles['token auth']
client = cli.Client(cls.cfg)
client.run('openssl ecparam -genkey -name prime256v1 -noout -out {}'
.format(token_auth['private key']).split())
client.run('openssl ec -in {} -pubout -out {}'.format(
token_auth['private key'], token_auth['public key']).split())
cls.client = api.Client(cls.cfg, api.page_handler)
cls.teardown_cleanups = []
with contextlib.ExitStack() as stack:
# ensure tearDownClass runs if an error occurs here
stack.callback(cls.tearDownClass)
# Step 1
_repo = cls.client.post(CONTAINER_REPO_PATH, gen_repo())
cls.teardown_cleanups.append((cls.client.delete, _repo['pulp_href']))
# Step 2
cls.remote = cls.client.post(
CONTAINER_REMOTE_PATH, gen_container_remote()
)
cls.teardown_cleanups.append(
(cls.client.delete, cls.remote['pulp_href'])
)
# Step 3
sync(cls.cfg, cls.remote, _repo)
cls.repo = cls.client.get(_repo['pulp_href'])
# Step 4.
response_dict = cls.client.using_handler(api.task_handler).post(
CONTAINER_DISTRIBUTION_PATH,
gen_distribution(repository=cls.repo['pulp_href'])
)
distribution_href = response_dict['pulp_href']
cls.distribution_with_repo = cls.client.get(distribution_href)
cls.teardown_cleanups.append(
(cls.client.delete, cls.distribution_with_repo['pulp_href'])
)
# Step 5.
response_dict = cls.client.using_handler(api.task_handler).post(
CONTAINER_DISTRIBUTION_PATH,
gen_distribution(repository_version=cls.repo['latest_version_href'])
)
distribution_href = response_dict['pulp_href']
cls.distribution_with_repo_version = cls.client.get(distribution_href)
cls.teardown_cleanups.append(
(cls.client.delete, cls.distribution_with_repo_version['pulp_href'])
)
# remove callback if everything goes well
stack.pop_all()
def test_01_create(self):
"""Create a publication distribution.
Do the following:
1. Create a repository and 3 repository versions with at least 1 file
content in it. Create a publication using the second repository
version.
2. Create a distribution with 'publication' field set to
the publication from step (1).
3. Assert the distribution got created correctly with the correct
base_path, name, and publication. Assert that content guard is
unset.
4. Assert that publication has a 'distributions' reference to the
distribution (it's backref).
"""
self.repo.update(self.client.post(REPO_PATH, gen_repo()))
self.remote.update(
self.client.post(FILE_REMOTE_PATH, gen_file_remote())
)
# create 3 repository versions
for _ in range(3):
sync(self.cfg, self.remote, self.repo)
self.repo = self.client.get(self.repo['_href'])
versions = get_versions(self.repo)
self.publication.update(create_file_publication(
self.cfg,
self.repo,
versions[1]['_href']
))
self.distribution.update(
self.client.post(
FILE_DISTRIBUTION_PATH,
gen_distribution(publication=self.publication['_href'])
)
)
self.publication = self.client.get(self.publication['_href'])
# content_guard is the only parameter unset.
for key, val in self.distribution.items():
if key == 'content_guard':
self.assertIsNone(val, self.distribution)
else:
self.assertIsNotNone(val, self.distribution)
self.assertEqual(
self.distribution['publication'],
self.publication['_href'],
self.distribution
)
self.assertEqual(
self.publication['distributions'][0],
self.distribution['_href'],
self.publication
)
def test_all(self):
"""
Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/pub-name/`` and
``https://example.com/content/pub-name/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Pulp-Fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_python_remote(PYTHON_FIXTURES_URL)
remote = client.post(PYTHON_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
# Create a publication
publication = gen_python_publication(cfg, repository=repo)
self.addCleanup(client.delete, publication['_href'])
# Create a distribution.
body = gen_distribution()
body['publication'] = publication['_href']
distribution = client.using_handler(api.task_handler).post(
PYTHON_DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution['_href'])
# Pick a file, and download it from both Pulp Fixtures…
unit_path = choice(get_python_content_paths(repo))
fixtures_hash = hashlib.sha256(
utils.http_get(
urljoin(urljoin(PYTHON_FIXTURES_URL, 'packages/'),
unit_path))).hexdigest()
# …and Pulp.
content = download_content_unit(cfg, distribution, unit_path)
pulp_hash = hashlib.sha256(content).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
def test_all(self):
"""Sync two copies of the same packages.
Make sure we end up with only one copy.
Do the following:
1. Create a repository and a remote.
2. Sync the remote.
3. Assert that the content summary matches what is expected.
4. Create a new remote w/ using fixture containing updated advisory
(packages with the same NEVRA as the existing package content, but
different pkgId).
5. Sync the remote again.
6. Assert that repository version is different from the previous one
but has the same content summary.
7. Assert that the packages have changed since the last sync.
"""
repo = self.client.post(RPM_REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['pulp_href'])
# Create a remote with the unsigned RPM fixture url.
body = gen_rpm_remote(url=RPM_UNSIGNED_FIXTURE_URL)
remote = self.client.post(RPM_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['pulp_href'])
# Sync the repository.
self.assertEqual(repo["latest_version_href"], f"{repo['pulp_href']}versions/0/")
sync(self.cfg, remote, repo)
repo = self.client.get(repo['pulp_href'])
self.assertDictEqual(get_content_summary(repo), RPM_FIXTURE_SUMMARY)
# Save a copy of the original packages.
original_packages = {
(
content['name'],
content['epoch'],
content['version'],
content['release'],
content['arch'],
): content
for content in get_content(repo)[RPM_PACKAGE_CONTENT_NAME]
}
# Create a remote with a different test fixture with the same NEVRA but
# different digests.
body = gen_rpm_remote(url=RPM_SIGNED_FIXTURE_URL)
remote = self.client.post(RPM_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['pulp_href'])
# Sync the repository again.
sync(self.cfg, remote, repo)
repo = self.client.get(repo['pulp_href'])
self.assertDictEqual(get_content_summary(repo), RPM_FIXTURE_SUMMARY)
# In case of "duplicates" the most recent one is chosen, so the old
# package is removed from and the new one is added to a repo version.
self.assertEqual(
len(get_added_content(repo)[RPM_PACKAGE_CONTENT_NAME]),
RPM_PACKAGE_COUNT,
get_added_content(repo)[RPM_PACKAGE_CONTENT_NAME],
)
self.assertEqual(
len(get_removed_content(repo)[RPM_PACKAGE_CONTENT_NAME]),
RPM_PACKAGE_COUNT,
get_removed_content(repo)[RPM_PACKAGE_CONTENT_NAME],
)
# Test that the packages have been modified.
mutated_packages = {
(
content['name'],
content['epoch'],
content['version'],
content['release'],
content['arch'],
): content
for content in get_content(repo)[RPM_PACKAGE_CONTENT_NAME]
}
for nevra in original_packages:
with self.subTest(pkg=nevra):
self.assertNotEqual(
original_packages[nevra]['pkgId'],
mutated_packages[nevra]['pkgId'],
original_packages[nevra]['pkgId'],
)
def test_rpm_kickstart_on_demand(self):
"""Sync repositories with the rpm plugin.
This test targets the following issue:
`Pulp #5202 <https://pulp.plan.io/issues/5202>`_
In order to sync a repository a remote has to be associated within
this repository. When a repository is created this version field is set
as None. After a sync the repository version is updated.
Do the following:
1. Create a repository and a remote.
2. Assert that repository version is None.
3. Sync the remote.
4. Assert that repository version is not None.
5. Assert that the correct number of units were added and are present
in the repo.
6. Sync the remote one more time.
7. Assert that repository version is the same as the previous one.
8. Assert that the same number of packages are present
"""
delete_orphans(self.cfg)
repo = self.client.post(RPM_REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['pulp_href'])
# Create a remote with the standard test fixture url.
body = gen_rpm_remote(
url=RPM_KICKSTART_FIXTURE_URL, policy='on_demand'
)
remote = self.client.post(RPM_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['pulp_href'])
# Sync the repository.
self.assertEqual(repo["latest_version_href"], f"{repo['pulp_href']}versions/0/")
sync(self.cfg, remote, repo)
repo = self.client.get(repo['pulp_href'])
for kickstart_content in get_content(repo)[RPM_KICKSTART_CONTENT_NAME]:
self.addCleanup(self.client.delete, kickstart_content['pulp_href'])
# Check that we have the correct content counts.
self.assertIsNotNone(repo['latest_version_href'])
self.assertDictEqual(
get_content_summary(repo), RPM_KICKSTART_FIXTURE_SUMMARY
)
self.assertDictEqual(
get_added_content_summary(repo), RPM_KICKSTART_FIXTURE_SUMMARY
)
# Sync the repository again.
latest_version_href = repo['latest_version_href']
sync(self.cfg, remote, repo)
repo = self.client.get(repo['pulp_href'])
artifacts = self.client.get(ARTIFACTS_PATH)
self.assertEqual(artifacts['count'], 0, artifacts)
# Check that nothing has changed since the last sync.
self.assertEqual(latest_version_href, repo['latest_version_href'])
self.assertDictEqual(
get_content_summary(repo), RPM_KICKSTART_FIXTURE_SUMMARY
)
def test_repair_repository_version(self):
"""Test whether corrupted files can be redownloaded.
Do the following:
1. Create, and sync a repo.
2. Select a content unit from the repo and change its appearance on disk.
3. Repair the RepositoryVersion.
4. Assert that the repair task reported one corrupted and one repaired unit.
5. Repair the RepositoryVersion.
6. Assert that the repair task reported none corrupted and none repaired unit.
"""
if settings.DEFAULT_FILE_STORAGE not in self.SUPPORTED_STORAGE_FRAMEWORKS:
self.skipTest(
"Cannot simulate bit-rot on this storage platform ({}).".
format(settings.DEFAULT_FILE_STORAGE), )
# STEP 1
delete_orphans()
repo = self.api_client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(self.api_client.delete, repo['pulp_href'])
body = gen_file_remote()
remote = self.api_client.post(FILE_REMOTE_PATH, body)
self.addCleanup(self.api_client.delete, remote['pulp_href'])
sync(self.cfg, remote, repo)
repo = self.api_client.get(repo['pulp_href'])
# STEP 2
content1, content2 = sample(get_content(repo)[FILE_CONTENT_NAME], 2)
if settings.DEFAULT_FILE_STORAGE in self.SUPPORTED_STORAGE_FRAMEWORKS:
# Muddify one artifact on disk.
artifact1_path = os.path.join(
MEDIA_PATH,
self.api_client.get(content1['artifact'])['file'])
cmd1 = ('sed', '-i', '-e', r'$a bit rot', artifact1_path)
self.cli_client.run(cmd1, sudo=True)
# Delete another one from disk.
artifact2_path = os.path.join(
MEDIA_PATH,
self.api_client.get(content2['artifact'])['file'])
cmd2 = ('rm', artifact2_path)
self.cli_client.run(cmd2, sudo=True)
else:
self.fail(
"Corrupting files on this storage platform is not supported.")
# STEP 3
latest_version = get_versions(repo)[-1]['pulp_href']
result = self.api_client.post(latest_version + 'repair/')
# STEP 4
corrupted_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.corrupted'), None)
self.assertEqual(corrupted_units_report['done'], 2,
corrupted_units_report)
repaired_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.repaired'), None)
self.assertEqual(repaired_units_report['done'], 2,
repaired_units_report)
# STEP 5
result = self.api_client.post(latest_version + 'repair/')
# STEP 6
corrupted_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.corrupted'), None)
self.assertEqual(corrupted_units_report['done'], 0,
corrupted_units_report)
repaired_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.repaired'), None)
self.assertEqual(repaired_units_report['done'], 0,
repaired_units_report)
def test_all(self):
"""Sync two copies of the same UpdateRecords.
Make sure we end up with only one copy.
Do the following:
1. Create a repository and a remote.
2. Sync the remote.
3. Assert that the content summary matches what is expected.
4. Create a new remote w/ using fixture containing updated errata
(updaterecords with the ID as the existing updaterecord content, but
different metadata).
5. Sync the remote again.
6. Assert that repository version is different from the previous one
but has the same content summary.
7. Assert that the updaterecords have changed since the last sync.
"""
client = api.Client(self.cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
# Create a remote with the unsigned RPM fixture url.
# We need to use the unsigned fixture because the one used down below
# has unsigned RPMs. Signed and unsigned units have different hashes,
# so they're seen as different units.
body = gen_rpm_remote(url=RPM_UNSIGNED_FIXTURE_URL)
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
# Sync the repository.
self.assertIsNone(repo['_latest_version_href'])
sync(self.cfg, remote, repo)
repo = client.get(repo['_href'])
self.assertDictEqual(get_content_summary(repo),
RPM_FIXTURE_CONTENT_SUMMARY)
# Save a copy of the original updateinfo
original_updaterecords = {
content['errata_id']: content
for content in get_content(repo) if content['type'] == 'update'
}
# Create a remote with a different test fixture, one containing mutated
# updateinfo.
body = gen_rpm_remote(url=RPM_UPDATED_UPDATEINFO_FIXTURE_URL)
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
# Sync the repository again.
sync(self.cfg, remote, repo)
repo = client.get(repo['_href'])
self.assertDictEqual(get_content_summary(repo),
RPM_FIXTURE_CONTENT_SUMMARY)
self.assertEqual(len(get_added_content(repo)), 0)
# Test that the updateinfo have been modified.
mutated_updaterecords = {
content['errata_id']: content
for content in get_content(repo) if content['type'] == 'update'
}
self.assertNotEqual(mutated_updaterecords, original_updaterecords)
self.assertEqual(
mutated_updaterecords[RPM_UPDATERECORD_ID]['description'],
'Updated Gorilla_Erratum and the updated date contains timezone',
mutated_updaterecords[RPM_UPDATERECORD_ID])
def test_all(self):
"""Verify publisher and remote can be used with different repos.
This test explores the design choice stated in `Pulp #3341`_ that
remove the FK from publishers and remotes to repository.
Allowing remotes and publishers to be used with different
repositories.
.. _Pulp #3341: https://pulp.plan.io/issues/3341
Do the following:
1. Create an remote, and a publisher.
2. Create 2 repositories.
3. Sync both repositories using the same remote.
4. Assert that the two repositories have the same contents.
5. Publish both repositories using the same publisher.
6. Assert that each generated publication has the same publisher, but
are associated with different repositories.
"""
cfg = config.get_config()
if not selectors.bug_is_fixed(3502, cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/3502')
# Create an remote and publisher.
client = api.Client(cfg, api.json_handler)
body = gen_remote(urljoin(FILE_FEED_URL, 'PULP_MANIFEST'))
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
publisher = client.post(FILE_PUBLISHER_PATH, gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Create and sync repos.
repos = []
for _ in range(2):
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
sync(cfg, remote, repo)
repos.append(client.get(repo['_href']))
# Compare contents of repositories.
contents = []
for repo in repos:
contents.append(get_content(repo))
self.assertEqual(
{content['_href']
for content in contents[0]},
{content['_href']
for content in contents[1]},
)
# Publish repositories.
publications = []
for repo in repos:
publications.append(publish(cfg, publisher, repo))
if selectors.bug_is_fixed(3354, cfg.pulp_version):
self.addCleanup(client.delete, publications[-1]['_href'])
self.assertEqual(publications[0]['publisher'],
publications[1]['publisher'])
self.assertNotEqual(publications[0]['repository_version'],
publications[1]['repository_version'])
def test_all(self):
"""Test content promotion for a distribution.
This test targets the following issue:
* `Pulp #4186 <https://pulp.plan.io/issues/4186>`_
Do the following:
1. Create a repository that has at least one repository version.
2. Create a publisher, and publication.
3. Create 2 distributions - using the same publication. Those
distributions will have different ``base_path``.
4. Assert that distributions have the same publication.
5. Select a content unit. Download that content unit from Pulp using
the two different distributions.
Assert that content unit has the same checksum when fetched from
different distributions.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
remote = client.post(FILE_REMOTE_PATH,
gen_remote(FILE_FIXTURE_MANIFEST_URL))
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
publisher = client.post(FILE_PUBLISHER_PATH, gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
publication = publish(cfg, publisher, repo)
self.addCleanup(client.delete, publication['_href'])
distributions = []
for _ in range(2):
body = gen_distribution()
body['publication'] = publication['_href']
response_dict = client.post(DISTRIBUTION_PATH, body)
dist_task = client.get(response_dict['task'])
distribution_href = dist_task['created_resources'][0]
distribution = client.get(distribution_href)
distributions.append(distribution)
self.addCleanup(client.delete, distribution['_href'])
self.assertEqual(distributions[0]['publication'],
distributions[1]['publication'], distributions)
unit_urls = []
unit_path = get_added_content(
repo)[FILE_CONTENT_NAME][0]['relative_path']
for distribution in distributions:
unit_url = cfg.get_hosts('api')[0].roles['api']['scheme']
unit_url += '://' + distribution['base_url'] + '/'
unit_urls.append(urljoin(unit_url, unit_path))
client.response_handler = api.safe_handler
self.assertEqual(
hashlib.sha256(client.get(unit_urls[0]).content).hexdigest(),
hashlib.sha256(client.get(unit_urls[1]).content).hexdigest(),
unit_urls,
)
def setUpClass(cls):
"""Create class-wide variables and delete orphans.
1. Create a repository.
2. Create a remote pointing to external registry with policy=on_demand.
3. Sync the repository using the remote and re-read the repo data.
4. Create a docker distribution to serve the repository
5. Create another docker distribution to the serve the repository version
This tests targets the following issue:
* `Pulp #4460 <https://pulp.plan.io/issues/4460>`_
"""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg, api.page_handler)
cls.teardown_cleanups = []
delete_orphans(cls.cfg)
with contextlib.ExitStack() as stack:
# ensure tearDownClass runs if an error occurs here
stack.callback(cls.tearDownClass)
# Step 1
_repo = cls.client.post(REPO_PATH, gen_repo())
cls.teardown_cleanups.append((cls.client.delete, _repo['_href']))
# Step 2
cls.remote = cls.client.post(DOCKER_REMOTE_PATH,
gen_docker_remote(policy='on_demand'))
cls.teardown_cleanups.append(
(cls.client.delete, cls.remote['_href']))
# Step 3
sync(cls.cfg, cls.remote, _repo)
cls.repo = cls.client.get(_repo['_href'])
cls.artifact_count = len(cls.client.get(ARTIFACTS_PATH))
# Step 4.
response_dict = cls.client.using_handler(api.task_handler).post(
DOCKER_DISTRIBUTION_PATH,
gen_distribution(repository=cls.repo['_href']))
distribution_href = response_dict['_href']
cls.distribution_with_repo = cls.client.get(distribution_href)
cls.teardown_cleanups.append(
(cls.client.delete, cls.distribution_with_repo['_href']))
# Step 5.
response_dict = cls.client.using_handler(api.task_handler).post(
DOCKER_DISTRIBUTION_PATH,
gen_distribution(
repository_version=cls.repo['_latest_version_href']))
distribution_href = response_dict['_href']
cls.distribution_with_repo_version = cls.client.get(
distribution_href)
cls.teardown_cleanups.append(
(cls.client.delete,
cls.distribution_with_repo_version['_href']))
# remove callback if everything goes well
stack.pop_all()
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available. For the cookbook plugin
this is below a live API at ``pulp_cookbook/market/`` e.g.
``http://example.com/pulp_cookbook/market/foo/`` and
``http://example.com/pulp_cookbook/market/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_remote(fixture_u1.url)
remote = client.post(COOKBOOK_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
# Create a publisher.
publisher = client.post(COOKBOOK_PUBLISHER_PATH, gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Create a publication.
publication = publish(cfg, publisher, repo)
self.addCleanup(client.delete, publication['_href'])
# Create a distribution.
body = gen_distribution()
body['publication'] = publication['_href']
distribution = client.post(DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution['_href'])
# pulp_cookbook universe live endpoint contains
# all cookbooks
distribution_base_url = cfg.get_hosts('api')[0].roles['api']['scheme']
distribution_base_url += '://' + distribution['base_url'] + '/'
api_host_cfg = cfg.get_hosts('api')[0]
universe_url = api_host_cfg.roles['api']['scheme']
universe_url += '://' + api_host_cfg.hostname
universe_url += ':{}'.format(api_host_cfg.roles['api']['port'])
universe_url += COOKBOOK_BASE_LIVE_API_PATH
universe_url += distribution['base_path'] + '/universe'
universe = client.get(universe_url)
for content, publication_path in get_content_and_unit_paths(repo):
u_url = universe[content['name']][
content['version']]['download_url']
self.assertEqual(u_url,
urljoin(distribution_base_url, publication_path))
# Pick a cookbook, and download it from both Fixtures…
_, unit_path = choice(get_content_and_unit_paths(repo))
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(fixture_u1.url, unit_path))).hexdigest()
# …and Pulp content
client.response_handler = api.safe_handler
unit_url = urljoin(distribution_base_url, unit_path)
pulp_hash = hashlib.sha256(client.get(unit_url).content).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
