def test_all(self):
"""Test whether the content present in a repo version is immutable.
Do the following:
1. Create a repository that has at least one repository version.
2. Attempt to update the content of a repository version.
3. Assert that an HTTP exception is raised.
4. Assert that the repository version was not updated.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
latest_version_href = client.get(repo['_href'])['_latest_version_href']
with self.assertRaises(HTTPError):
client.post(latest_version_href)
repo = client.get(repo['_href'])
self.assertEqual(latest_version_href, repo['_latest_version_href'])
def test_all(self):
"""Verify publisher and remote can be used with different repos.
This test explores the design choice stated in `Pulp #3341`_ that
remove the FK from publishers and remotes to repository.
Allowing remotes and publishers to be used with different
repositories.
.. _Pulp #3341: https://pulp.plan.io/issues/3341
Do the following:
1. Create a remote, and a publisher.
2. Create 2 repositories.
3. Sync both repositories using the same remote.
4. Assert that the two repositories have the same contents.
5. Publish both repositories using the same publisher.
6. Assert that each generated publication has the same publisher, but
are associated with different repositories.
"""
cfg = config.get_config()
# Create a remote and publisher.
client = api.Client(cfg, api.json_handler)
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
publisher = client.post(FILE_PUBLISHER_PATH, gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Create and sync repos.
repos = []
for _ in range(2):
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
sync(cfg, remote, repo)
repos.append(client.get(repo['_href']))
# Compare contents of repositories.
contents = []
for repo in repos:
contents.append(get_content(repo)[FILE_CONTENT_NAME])
self.assertEqual(
{content['_href'] for content in contents[0]},
{content['_href'] for content in contents[1]},
)
# Publish repositories.
publications = []
for repo in repos:
publications.append(publish(cfg, publisher, repo))
self.assertEqual(
publications[0]['publisher'],
publications[1]['publisher']
)
self.assertNotEqual(
publications[0]['repository_version'],
publications[1]['repository_version']
)
def test_all(self):
"""Verify multi-resourcing locking.
Do the following:
1. Create a repository, and a remote.
2. Update the remote to point to a different url.
3. Immediately run a sync. The sync should fire after the update and
sync from the second url.
4. Assert that remote url was updated.
5. Assert that the number of units present in the repository is
according to the updated url.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_file_remote(url=FILE_LARGE_FIXTURE_MANIFEST_URL)
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
url = {'url': FILE_FIXTURE_MANIFEST_URL}
client.patch(remote['_href'], url)
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
remote = client.get(remote['_href'])
self.assertEqual(remote['url'], url['url'])
self.assertDictEqual(get_content_summary(repo), FILE_FIXTURE_SUMMARY)
def test_all(self):
"""Verify whether the sync of a repository updates its version.
This test explores the design choice stated in the `Pulp #3308`_ that a
new repository version is created even if the sync does not add or
remove any content units. Even without any changes to the remote if a
new sync occurs, a new repository version is created.
.. _Pulp #3308: https://pulp.plan.io/issues/3308
Do the following:
1. Create a repository, and a remote.
2. Sync the repository an arbitrary number of times.
3. Verify that the repository version is equal to the previous number
of syncs.
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
number_of_syncs = randint(1, 10)
for _ in range(number_of_syncs):
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
path = urlsplit(repo['_latest_version_href']).path
latest_repo_version = int(path.split('/')[-2])
self.assertEqual(latest_repo_version, number_of_syncs)
def test_negative_create_file_remote_with_invalid_parameter(self):
"""Attempt to create file remote passing invalid parameter.
Assert response returns an error 400 including ["Unexpected field"].
"""
response = api.Client(self.cfg, api.echo_handler).post(
FILE_REMOTE_PATH, gen_file_remote(foo='bar')
)
assert response.status_code == 400
assert response.json()['foo'] == ['Unexpected field']
def create_sync_repo(self):
"""Create, and sync a repo."""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
body = gen_file_remote(url=FILE_FIXTURE_MANIFEST_URL)
remote = self.client.post(FILE_REMOTE_PATH, body)
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
return self.client.get(repo['_href'])
def test_clean_orphan_content_unit(self):
"""Test whether orphan content units can be clean up.
Do the following:
1. Create, and sync a repo.
2. Remove a content unit from the repo. This will create a second
repository version, and create an orphan content unit.
3. Assert that content unit that was removed from the repo and its
artifact are present on disk.
4. Delete orphans.
5. Assert that the orphan content unit was cleaned up, and its artifact
is not present on disk.
"""
repo = self.api_client.post(REPO_PATH, gen_repo())
self.addCleanup(self.api_client.delete, repo['_href'])
body = gen_file_remote()
remote = self.api_client.post(FILE_REMOTE_PATH, body)
self.addCleanup(self.api_client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.api_client.get(repo['_href'])
content = choice(get_content(repo)[FILE_CONTENT_NAME])
# Create an orphan content unit.
self.api_client.post(
repo['_versions_href'],
{'remove_content_units': [content['_href']]}
)
# Verify that the artifact is present on disk.
artifact_path = self.api_client.get(content['_artifact'])['file']
cmd = ('ls', artifact_path)
self.cli_client.run(cmd, sudo=True)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo)[0]['_href'])
content_units = self.api_client.get(FILE_CONTENT_PATH)['results']
self.assertIn(content, content_units)
delete_orphans()
content_units = self.api_client.get(FILE_CONTENT_PATH)['results']
self.assertNotIn(content, content_units)
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def test_02_sync_content(self):
"""Sync content into the repository.
Assert that:
* The ``_versions_href`` API call is correct.
* The ``_latest_version_href`` API call is correct.
* The ``content_hrefs`` attribute is correct.
* The ``content_added_hrefs`` attribute is correct.
* The ``content_removed_hrefs`` attribute is correct.
* The ``content_summary`` attribute is correct.
* The ``content_added_summary`` attribute is correct.
* The ``content_removed_summary`` attribute is correct.
"""
body = gen_file_remote()
self.remote.update(self.client.post(FILE_REMOTE_PATH, body))
sync(self.cfg, self.remote, self.repo)
repo = self.client.get(self.repo['_href'])
repo_versions = get_versions(repo)
self.assertEqual(len(repo_versions), 1, repo_versions)
self.assertIsNotNone(repo['_latest_version_href'])
content_hrefs = get_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(
len(content_hrefs), FILE_FIXTURE_COUNT, content_hrefs
)
content = get_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(len(content), FILE_FIXTURE_COUNT)
content_added = get_added_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(len(content_added), FILE_FIXTURE_COUNT)
content_removed = get_removed_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(len(content_removed), 0)
content_summary = get_content_summary(repo)
self.assertDictEqual(content_summary, FILE_FIXTURE_SUMMARY)
content_added_summary = get_added_content_summary(repo)
self.assertDictEqual(content_added_summary, FILE_FIXTURE_SUMMARY)
content_removed_summary = get_removed_content_summary(repo)
self.assertDictEqual(content_removed_summary, {})
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg, api.page_handler)
cls.remote = {}
cls.publication = {}
cls.publisher = {}
cls.repo = {}
try:
cls.repo.update(cls.client.post(REPO_PATH, gen_repo()))
body = gen_file_remote()
cls.remote.update(cls.client.post(FILE_REMOTE_PATH, body))
cls.publisher.update(
cls.client.post(FILE_PUBLISHER_PATH, gen_publisher()))
sync(cls.cfg, cls.remote, cls.repo)
except Exception:
cls.tearDownClass()
raise
def test_02_sync_content(self):
"""Sync content into the repository.
Assert that:
* The ``versions_href`` API call is correct.
* The ``latest_version_href`` API call is correct.
* The ``content_hrefs`` attribute is correct.
* The ``content_added_hrefs`` attribute is correct.
* The ``content_removed_hrefs`` attribute is correct.
* The ``content_summary`` attribute is correct.
* The ``content_added_summary`` attribute is correct.
* The ``content_removed_summary`` attribute is correct.
"""
body = gen_file_remote()
self.remote.update(self.client.post(FILE_REMOTE_PATH, body))
sync(self.cfg, self.remote, self.repo)
repo = self.client.get(self.repo["pulp_href"])
repo_versions = get_versions(repo)
self.assertEqual(len(repo_versions), 2, repo_versions)
self.assertIsNotNone(repo["latest_version_href"])
content_hrefs = get_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(len(content_hrefs), FILE_FIXTURE_COUNT, content_hrefs)
content = get_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(len(content), FILE_FIXTURE_COUNT)
content_added = get_added_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(len(content_added), FILE_FIXTURE_COUNT)
content_removed = get_removed_content(repo)[FILE_CONTENT_NAME]
self.assertEqual(len(content_removed), 0)
content_summary = get_content_summary(repo)
self.assertDictEqual(content_summary, FILE_FIXTURE_SUMMARY)
content_added_summary = get_added_content_summary(repo)
self.assertDictEqual(content_added_summary, FILE_FIXTURE_SUMMARY)
content_removed_summary = get_removed_content_summary(repo)
self.assertDictEqual(content_removed_summary, {})
def test_clean_specific_orphans(self):
"""Test whether the `content_hrefs` param removes specific orphans but not others"""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
# Create two orphaned content units.
content_a = get_content(
repo.to_dict())[FILE_CONTENT_NAME][0]["pulp_href"]
content_b = get_content(
repo.to_dict())[FILE_CONTENT_NAME][1]["pulp_href"]
content_to_remove = dict(remove_content_units=[content_a, content_b])
repo_api.modify(repo.pulp_href, content_to_remove)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content_a, content_units_href)
self.assertIn(content_b, content_units_href)
content_hrefs_dict = {"content_hrefs": [content_a]}
orphans_response = self.orphans_cleanup_api.cleanup(content_hrefs_dict)
monitor_task(orphans_response.task)
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content_a, content_units_href)
self.assertIn(content_b, content_units_href)
def test_acs_sync_with_paths(self):
"""Test syncing from an ACS using different paths."""
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
remote = self.file_remote_api.create(gen_file_remote(FILE_MANIFEST_ONLY_FIXTURE_URL))
self.addCleanup(self.file_remote_api.delete, remote.pulp_href)
acs = self._create_acs(
paths=("file/PULP_MANIFEST", "file2/PULP_MANIFEST"),
remote_url=PULP_FIXTURES_BASE_URL,
)
resp = self.file_acs_api.refresh(acs.pulp_href, acs)
task_group = monitor_task_group(resp.task_group)
self.assertEquals(len(task_group.tasks), 2)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg, api.page_handler)
cls.client_echo = api.Client(cls.cfg, api.echo_handler)
cls.remote = {}
cls.publication = {}
cls.repo = {}
try:
cls.repo.update(cls.client.post(FILE_REPO_PATH, gen_repo()))
cls.repo_initial_version = cls.repo["latest_version_href"]
body = gen_file_remote()
cls.remote.update(cls.client.post(FILE_REMOTE_PATH, body))
sync(cls.cfg, cls.remote, cls.repo)
# update to get latest_version_href
cls.repo.update(cls.client.get(cls.repo["pulp_href"]))
except Exception:
cls.tearDownClass()
raise
def test_all(self):
"""Verify remotes can be used with different repos.
This test explores the design choice stated in `Pulp #3341`_ that
remove the FK from remotes to repository.
Allowing remotes to be used with different
repositories.
.. _Pulp #3341: https://pulp.plan.io/issues/3341
Do the following:
1. Create a remote.
2. Create 2 repositories.
3. Sync both repositories using the same remote.
4. Assert that the two repositories have the same contents.
"""
cfg = config.get_config()
# Create a remote.
client = api.Client(cfg, api.json_handler)
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
# Create and sync repos.
repos = []
for _ in range(2):
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
sync(cfg, remote, repo)
repos.append(client.get(repo['_href']))
# Compare contents of repositories.
contents = []
for repo in repos:
contents.append(get_content(repo)[FILE_CONTENT_NAME])
self.assertEqual(
{content['_href']
for content in contents[0]},
{content['_href']
for content in contents[1]},
)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg, api.page_handler)
cls.remote = {}
cls.publication = {}
cls.publisher = {}
cls.repo = {}
try:
cls.repo.update(cls.client.post(REPO_PATH, gen_repo()))
body = gen_file_remote()
cls.remote.update(cls.client.post(FILE_REMOTE_PATH, body))
cls.publisher.update(
cls.client.post(FILE_PUBLISHER_PATH, gen_publisher())
)
sync(cls.cfg, cls.remote, cls.repo)
except Exception:
cls.tearDownClass()
raise
def test_03_set_remote_on_repository(self):
"""Test setting remotes on repositories."""
body = gen_file_remote()
remote = self.client.post(FILE_REMOTE_PATH, body)
# verify that syncing with no remote raises an error
with self.assertRaises(HTTPError):
self.client.post(urljoin(self.repo["pulp_href"], "sync/"))
# test setting the remote on the repo
self.client.patch(self.repo["pulp_href"],
{"remote": remote["pulp_href"]})
# test syncing without a remote
self.client.post(urljoin(self.repo["pulp_href"], "sync/"))
repo = self.client.get(self.repo["pulp_href"])
self.assertEqual(repo["latest_version_href"],
f"{repo['pulp_href']}versions/1/")
def test_all(self):
"""Verify whether task report shows repository version was created."""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['pulp_href'])
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['pulp_href'])
call_report = sync(cfg, remote, repo)
for key in ('repositories', 'versions'):
self.assertIn(
key,
call_report['pulp_href'],
call_report
)
def test_all(self):
"""Verify whether task report shows repository version was created."""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_file_remote()
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
call_report = sync(cfg, remote, repo)
last_task = next(api.poll_spawned_tasks(cfg, call_report))
for key in ('repositories', 'versions'):
self.assertIn(
key,
last_task['created_resources'][0],
last_task['created_resources']
)
def _setup_repositories(cls):
"""Create and sync a number of repositories to be exported."""
# create and remember a set of repo
import_repos = []
export_repos = []
remotes = []
for r in range(NUM_REPOS):
import_repo = cls.repo_api.create(gen_repo())
export_repo = cls.repo_api.create(gen_repo())
body = gen_file_remote()
remote = cls.remote_api.create(body)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = cls.repo_api.sync(export_repo.pulp_href,
repository_sync_data)
monitor_task(sync_response.task)
# remember it
export_repos.append(export_repo)
import_repos.append(import_repo)
remotes.append(remote)
return import_repos, export_repos, remotes
def test_filter_last_repository_version(self):
"""Filter by last repository version.
For a repository with more than one version.
"""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['pulp_href'])
for url in [FILE2_FIXTURE_MANIFEST_URL, FILE_FIXTURE_MANIFEST_URL]:
remote = self.client.post(FILE_REMOTE_PATH,
gen_file_remote(url=url))
self.addCleanup(self.client.delete, remote['pulp_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['pulp_href'])
artifacts = self.client.get(
ARTIFACTS_PATH,
params={'repository_version': repo['latest_version_href']})
# Every sync add 3 content units to the repository. Last repository
# version has 6 content units.
self.assertEqual(len(artifacts), FILE_FIXTURE_COUNT * 2, artifacts)
def test_filter_last_repository_version(self):
"""Filter by last repository version.
For a repository with more than one version.
"""
repo = self.client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo["pulp_href"])
for url in [FILE2_FIXTURE_MANIFEST_URL, FILE_FIXTURE_MANIFEST_URL]:
remote = self.client.post(FILE_REMOTE_PATH, gen_file_remote(url=url))
self.addCleanup(self.client.delete, remote["pulp_href"])
sync(self.cfg, remote, repo)
repo = self.client.get(repo["pulp_href"])
artifacts = self.client.get(
ARTIFACTS_PATH, params={"repository_version": repo["latest_version_href"]}
)
# Even though every sync adds 3 content units to the repository the fixture data contains
# the same relative urls so the second sync replaces the first 3, leaving a total of 3 each
# time
self.assertEqual(len(artifacts), FILE_FIXTURE_COUNT, artifacts)
def test_content_served(self):
"""Verify that content is served over publication distribution."""
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
remote = self.client.post(FILE_REMOTE_PATH, gen_file_remote())
self.addCleanup(self.client.delete, remote['_href'])
sync(self.cfg, remote, repo)
repo = self.client.get(repo['_href'])
publication = create_file_publication(self.cfg, repo)
self.addCleanup(self.client.delete, publication['_href'])
distribution = self.client.post(
FILE_DISTRIBUTION_PATH,
gen_distribution(publication=publication['_href'])
)
self.addCleanup(self.client.delete, distribution['_href'])
pulp_manifest = parse_pulp_manifest(
self.download_pulp_manifest(distribution)
)
self.assertEqual(len(pulp_manifest), FILE_FIXTURE_COUNT, pulp_manifest)
added_content = get_added_content(repo)
unit_path = added_content[FILE_CONTENT_NAME][0]['relative_path']
unit_url = self.cfg.get_hosts('api')[0].roles['api']['scheme']
unit_url += '://' + distribution['base_url'] + '/'
unit_url = urljoin(unit_url, unit_path)
pulp_hash = hashlib.sha256(
self.client.using_handler(api.safe_handler).get(unit_url).content
).hexdigest()
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(FILE_URL, unit_path))
).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
def setup_download_test(self, policy, url=None):
# Create a repository
self.repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, self.repo.pulp_href)
# Create a remote
remote_options = {"policy": policy}
if url:
remote_options["url"] = url
self.remote = self.remote_api.create(gen_file_remote(**remote_options))
self.addCleanup(self.remote_api.delete, self.remote.pulp_href)
# Sync the repository.
repository_sync_data = RepositorySyncURL(remote=self.remote.pulp_href)
sync_response = self.repo_api.sync(self.repo.pulp_href,
repository_sync_data)
monitor_task(sync_response.task)
# Create a publication.
publish_data = FileFilePublication(repository=self.repo.pulp_href)
publish_response = self.publications_api.create(publish_data)
publication_href = monitor_task(
publish_response.task).created_resources[0]
self.addCleanup(self.publications_api.delete, publication_href)
# Create a distribution.
response = self.distributions_api.create({
"name":
"foo",
"base_path":
"bar/foo",
"publication":
publication_href
})
distribution_href = monitor_task(response.task).created_resources[0]
self.distribution = self.distributions_api.read(distribution_href)
self.addCleanup(self.distributions_api.delete,
self.distribution.pulp_href)
def setUpClass(cls):
"""Sets up class"""
client = gen_file_client()
cls.cont_api = ContentFilesApi(client)
cls.repo_api = RepositoriesFileApi(client)
cls.remote_api = RemotesFileApi(client)
cls.pub_api = PublicationsFileApi(client)
cls.dis_api = DistributionsFileApi(client)
cls.repo = cls.repo_api.create(gen_repo(autopublish=True))
cls.remote = cls.remote_api.create(gen_file_remote())
body = RepositorySyncURL(remote=cls.remote.pulp_href)
created = monitor_task(cls.repo_api.sync(cls.repo.pulp_href, body).task).created_resources
cls.repo = cls.repo_api.read(cls.repo.pulp_href)
cls.pub1 = cls.pub_api.read(created[1])
body = FileFilePublication(repository=cls.repo.pulp_href)
cls.pub2 = cls.pub_api.read(
monitor_task(cls.pub_api.create(body).task).created_resources[0]
)
cls.pub3 = []
response = cls.dis_api.create(gen_distribution(repository=cls.repo.pulp_href))
cls.distro = cls.dis_api.read(monitor_task(response.task).created_resources[0])
cls.distro2 = []
cls.url = urljoin(PULP_CONTENT_BASE_URL, f"{cls.distro.base_path}/")
def test_reclaim_immediate_content(self):
"""
Test whether immediate repository content can be reclaimed
and then re-populated back after sync.
"""
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
remote = self.remote_api.create(gen_file_remote())
self.addCleanup(self.remote_api.delete, remote.pulp_href)
# sync the repository with immediate policy
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href,
repository_sync_data)
monitor_task(sync_response.task)
# reclaim disk space
reclaim_response = self.reclaim_api.reclaim(
{"repo_hrefs": [repo.pulp_href]})
monitor_task(reclaim_response.task)
# assert no artifacts left
artifacts = self.artifacts_api.list().count
self.assertEqual(artifacts, 0)
# sync repo again
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href,
repository_sync_data)
monitor_task(sync_response.task)
# assert re-sync populated missing artifacts
artifacts = self.artifacts_api.list().count
self.assertGreater(artifacts, 0)
self.addCleanup(self.orphans_api.cleanup,
{"orphan_protection_time": 0})
def test_acs_sync(self):
"""Test syncing from an ACS."""
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
remote = self.file_remote_api.create(gen_file_remote(FILE_MANIFEST_ONLY_FIXTURE_URL))
self.addCleanup(self.file_remote_api.delete, remote.pulp_href)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
# sync should fail as the repo has metadata only (no files)
sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data)
with self.assertRaises(PulpTaskError) as ctx:
monitor_task(sync_response.task)
self.assertIn("404", ctx.exception.task.error["description"])
# create an acs and pull in its remote artifacts
acs = self._create_acs()
resp = self.file_acs_api.refresh(acs.pulp_href, acs)
monitor_task_group(resp.task_group)
# the sync should now work as the files are being pulled from ACS remote
sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
def test_01_create(self):
"""Create a publication distribution.
Do the following:
1. Create a repository and 3 repository versions with at least 1 file
content in it. Create a publication using the second repository
version.
2. Create a distribution with 'publication' field set to
the publication from step (1).
3. Assert the distribution got created correctly with the correct
base_path, name, and publication. Assert that content guard is
unset.
4. Assert that publication has a 'distributions' reference to the
distribution (it's backref).
"""
self.repo.update(self.client.post(REPO_PATH, gen_repo()))
self.remote.update(
self.client.post(FILE_REMOTE_PATH, gen_file_remote())
)
# create 3 repository versions
for _ in range(3):
sync(self.cfg, self.remote, self.repo)
self.repo = self.client.get(self.repo['_href'])
versions = get_versions(self.repo)
self.publication.update(create_file_publication(
self.cfg,
self.repo,
versions[1]['_href']
))
self.distribution.update(
self.client.post(
FILE_DISTRIBUTION_PATH,
gen_distribution(publication=self.publication['_href'])
)
)
self.publication = self.client.get(self.publication['_href'])
# content_guard is the only parameter unset.
for key, val in self.distribution.items():
if key == 'content_guard':
self.assertIsNone(val, self.distribution)
else:
self.assertIsNotNone(val, self.distribution)
self.assertEqual(
self.distribution['publication'],
self.publication['_href'],
self.distribution
)
self.assertEqual(
self.publication['distributions'][0],
self.distribution['_href'],
self.publication
)
def test_clean_orphan_content_unit(self):
"""Test whether orphan content units can be clean up.
Do the following:
1. Create, and sync a repo.
2. Remove a content unit from the repo. This will create a second
repository version, and create an orphan content unit.
3. Assert that content unit that was removed from the repo and its
artifact are present on disk.
4. Delete orphans.
5. Assert that the orphan content unit was cleaned up, and its artifact
is not present on disk.
"""
repo_api = RepositoriesFileApi(self.api_client)
remote_api = RemotesFileApi(self.api_client)
repo = repo_api.create(gen_repo())
self.addCleanup(repo_api.delete, repo.pulp_href)
body = gen_file_remote()
remote = remote_api.create(body)
self.addCleanup(remote_api.delete, remote.pulp_href)
# Sync the repository.
self.assertEqual(repo.latest_version_href,
f"{repo.pulp_href}versions/0/")
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = repo_api.read(repo.pulp_href)
content = choice(get_content(repo.to_dict())[FILE_CONTENT_NAME])
# Create an orphan content unit.
repo_api.modify(repo.pulp_href,
dict(remove_content_units=[content["pulp_href"]]))
artifacts_api = ArtifactsApi(core_client)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact is present on disk.
artifact_path = os.path.join(
MEDIA_PATH,
artifacts_api.read(content["artifact"]).file)
cmd = ("ls", artifact_path)
self.cli_client.run(cmd, sudo=True)
file_contents_api = ContentFilesApi(self.api_client)
# Delete first repo version. The previous removed content unit will be
# an orphan.
delete_version(repo, get_versions(repo.to_dict())[1]["pulp_href"])
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertIn(content["pulp_href"], content_units_href)
delete_orphans()
content_units = file_contents_api.list().to_dict()["results"]
content_units_href = [c["pulp_href"] for c in content_units]
self.assertNotIn(content["pulp_href"], content_units_href)
if self.storage == "pulpcore.app.models.storage.FileSystem":
# Verify that the artifact was removed from disk.
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def test_all(self):
"""Create two publications and check view filter."""
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
remote = self.remote_api.create(gen_file_remote())
self.addCleanup(self.remote_api.delete, remote.pulp_href)
# Sync and update repository data.
repo_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href, repo_sync_data)
monitor_task(sync_response.task)
repo = self.repo_api.read(repo.pulp_href)
# Test content doesn't exists.
non_existant_content_href = (
"/pulp/api/v3/content/file/files/c4ed74cf-a806-490d-a25f-94c3c3dd2dd7/"
)
with self.assertRaises(ApiException) as ctx:
self.publication_api.list(content=non_existant_content_href)
self.assertEqual(ctx.exception.status, 400)
# Test not published content.
content_href = get_content(
repo.to_dict())[FILE_CONTENT_NAME][0]["pulp_href"]
self.assertEqual(
self.publication_api.list(content=content_href).to_dict()["count"],
0)
# Publication
publication_data = FileFilePublication(repository=repo.pulp_href)
publication_response = self.publication_api.create(publication_data)
task_response = monitor_task(publication_response.task)
publication = self.publication_api.read(
task_response.created_resources[0])
self.addCleanup(self.publication_api.delete, publication.pulp_href)
# Second publication
repo_second = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo_second.pulp_href)
body = gen_file_remote(url=FILE_MANY_FIXTURE_MANIFEST_URL)
remote_second = self.remote_api.create(body)
self.addCleanup(self.remote_api.delete, remote_second.pulp_href)
repo_second_sync_data = RepositorySyncURL(
remote=remote_second.pulp_href)
sync_response = self.repo_api.sync(repo_second.pulp_href,
repo_second_sync_data)
monitor_task(sync_response.task)
repo_second = self.repo_api.read(repo_second.pulp_href)
publication_data = FileFilePublication(
repository=repo_second.pulp_href)
publication_response = self.publication_api.create(publication_data)
task_response = monitor_task(publication_response.task)
publication_second = self.publication_api.read(
task_response.created_resources[0])
self.addCleanup(self.publication_api.delete,
publication_second.pulp_href)
# Test there are two publications
self.assertEqual(self.publication_api.list().count, 2)
# Test content match publication
self.assertEqual(
self.publication_api.list(content=content_href).count, 1)
self.assertEqual(
self.publication_api.list(
content=content_href).results[0].repository_version,
repo.latest_version_href,
)
def test_repair_repository_version(self):
"""Test whether corrupted files can be redownloaded.
Do the following:
1. Create, and sync a repo.
2. Select a content unit from the repo and change its appearance on disk.
3. Repair the RepositoryVersion.
4. Assert that the repair task reported one corrupted and one repaired unit.
5. Repair the RepositoryVersion.
6. Assert that the repair task reported none corrupted and none repaired unit.
"""
if settings.DEFAULT_FILE_STORAGE not in self.SUPPORTED_STORAGE_FRAMEWORKS:
self.skipTest(
"Cannot simulate bit-rot on this storage platform ({}).".
format(settings.DEFAULT_FILE_STORAGE), )
# STEP 1
delete_orphans()
repo = self.api_client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(self.api_client.delete, repo['pulp_href'])
body = gen_file_remote()
remote = self.api_client.post(FILE_REMOTE_PATH, body)
self.addCleanup(self.api_client.delete, remote['pulp_href'])
sync(self.cfg, remote, repo)
repo = self.api_client.get(repo['pulp_href'])
# STEP 2
content1, content2 = sample(get_content(repo)[FILE_CONTENT_NAME], 2)
if settings.DEFAULT_FILE_STORAGE in self.SUPPORTED_STORAGE_FRAMEWORKS:
# Muddify one artifact on disk.
artifact1_path = os.path.join(
MEDIA_PATH,
self.api_client.get(content1['artifact'])['file'])
cmd1 = ('sed', '-i', '-e', r'$a bit rot', artifact1_path)
self.cli_client.run(cmd1, sudo=True)
# Delete another one from disk.
artifact2_path = os.path.join(
MEDIA_PATH,
self.api_client.get(content2['artifact'])['file'])
cmd2 = ('rm', artifact2_path)
self.cli_client.run(cmd2, sudo=True)
else:
self.fail(
"Corrupting files on this storage platform is not supported.")
# STEP 3
latest_version = get_versions(repo)[-1]['pulp_href']
result = self.api_client.post(latest_version + 'repair/')
# STEP 4
corrupted_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.corrupted'), None)
self.assertEqual(corrupted_units_report['done'], 2,
corrupted_units_report)
repaired_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.repaired'), None)
self.assertEqual(repaired_units_report['done'], 2,
repaired_units_report)
# STEP 5
result = self.api_client.post(latest_version + 'repair/')
# STEP 6
corrupted_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.corrupted'), None)
self.assertEqual(corrupted_units_report['done'], 0,
corrupted_units_report)
repaired_units_report = next(
(report for report in result['progress_reports']
if report['code'] == 'repair.repaired'), None)
self.assertEqual(repaired_units_report['done'], 0,
repaired_units_report)
