def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.cli_client = cli.Client(cls.cfg)
cls.client = api.Client(cls.cfg)
cls.file = http_get(FILE_TO_BE_CHUNKED_URL)
cls.file_sha256 = hashlib.sha256(cls.file).hexdigest()
cls.size_file = len(cls.file)
first_chunk = http_get(FILE_CHUNKED_PART_1_URL)
header_first_chunk = {
"Content-Range":
"bytes 0-{}/{}".format(len(first_chunk) - 1, cls.size_file)
}
second_chunk = http_get(FILE_CHUNKED_PART_2_URL)
header_second_chunk = {
"Content-Range":
"bytes {}-{}/{}".format(len(first_chunk), cls.size_file - 1,
cls.size_file)
}
cls.chunked_data = [
[first_chunk, header_first_chunk],
[second_chunk, header_second_chunk],
]
shuffle(cls.chunked_data)
def _get_module_yaml_file(path):
"""Return the path to ``modules.yaml``, relative to repository root.
Given a detailed dict of information about a published, repository,
parse that repository's ``repomd.xml`` file and tell the path to the
repository's ``[…]-modules.yaml`` file. The path is likely to be in the
form ``repodata/[…]-modules.yaml.gz``.
"""
repo_path = urljoin(path, 'repodata/repomd.xml')
response = utils.http_get(repo_path)
root_elem = ElementTree.fromstring(response)
# <ns0:repomd xmlns:ns0="http://linux.duke.edu/metadata/repo">
# <ns0:data type="modules">
# <ns0:checksum type="sha256">[…]</ns0:checksum>
# <ns0:location href="repodata/[…]-modules.yaml.gz" />
# …
# </ns0:data>
# …
xpath = '{{{}}}data'.format(RPM_NAMESPACES['metadata/repo'])
data_elements = [
elem for elem in root_elem.findall(xpath)
if elem.get('type') == 'modules'
]
xpath = '{{{}}}location'.format(RPM_NAMESPACES['metadata/repo'])
relative_path = data_elements[0].find(xpath).get('href')
unit = utils.http_get(urljoin(path, relative_path))
return unit
def test_centos8_baseos(self):
"""Publish CentOS 8 BaseOS."""
publication_href = self.rpm_publish(url=CENTOS8_BASEOS_URL)
# Test that the .treeinfo file is available and AppStream sub-repo is published correctly
body = {"publication": publication_href, "name": "centos8", "base_path": "centos8"}
response = self.client.using_handler(api.json_handler).post(RPM_DISTRIBUTION_PATH, body)
distribution_task = self.client.get(response["task"])
distribution_href = distribution_task["created_resources"][0]
self.addCleanup(self.client.delete, distribution_href)
distribution = self.client.get(distribution_href)
treeinfo_file = utils.http_get(urljoin(distribution["base_url"], ".treeinfo"))
treeinfo = TreeInfo()
with NamedTemporaryFile("wb") as temp_file:
temp_file.write(treeinfo_file)
temp_file.flush()
treeinfo.load(f=temp_file.name)
for variant_name, variant in treeinfo.variants.variants.items():
if variant_name == "BaseOS":
self.assertEqual(variant.paths.repository, ".")
self.assertEqual(variant.paths.packages, "Packages")
elif variant_name == "AppStream":
self.assertEqual(variant.paths.repository, "AppStream")
self.assertEqual(variant.paths.packages, "AppStream/Packages")
# Find the first package in the 'AppStream/Packages/a/' directory and download it
parser = PackagesHtmlParser()
a_packages_href = urljoin(
distribution["base_url"], "{}/a/".format(variant.paths.packages)
)
a_packages_listing = utils.http_get(a_packages_href)
parser.feed(a_packages_listing.__str__())
full_package_path = urljoin(a_packages_href, parser.package_href)
utils.http_get(full_package_path)
def test_all(self):
"""Add a content unit to a repo in the middle of several publishes."""
cfg = config.get_config()
if not selectors.bug_is_fixed(2532, cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/2532')
rpms = (utils.http_get(RPM_UNSIGNED_URL),
utils.http_get(RPM2_UNSIGNED_URL))
# Create a user and a repository.
ssh_user, priv_key = self.make_user(cfg)
ssh_identity_file = self.write_private_key(cfg, priv_key)
repo = self.make_repo(
cfg, {
'remote': {
'host': urlparse(cfg.get_base_url()).hostname,
'root': '/home/' + ssh_user,
'ssh_identity_file': ssh_identity_file,
'ssh_user': ssh_user,
}
})
# Add content, publish w/yum, add more content, publish w/rsync.
dists = get_dists_by_type_id(cfg, repo)
for i, key in enumerate(('yum_distributor', 'rpm_rsync_distributor')):
upload_import_unit(cfg, rpms[i], {'unit_type_id': 'rpm'}, repo)
publish_repo(cfg, repo, {'id': dists[key]['id']})
self.verify_remote_units_path(cfg, dists['rpm_rsync_distributor'], 1)
# Publish with yum and rsync, respectively.
for key in 'yum_distributor', 'rpm_rsync_distributor':
publish_repo(cfg, repo, {'id': dists[key]['id']})
self.verify_remote_units_path(cfg, dists['rpm_rsync_distributor'], 1)
def test_second_unit_replaces_the_first(self):
"""Create a duplicate content unit with different ``artifacts`` and same ``relative_path``.
Artifacts are unique by ``relative_path`` and ``file``.
"""
delete_orphans(self.cfg)
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo["pulp_href"])
files = {"file": utils.http_get(FILE_URL)}
artifact = self.client.post(ARTIFACTS_PATH, files=files)
# create first content unit.
content_attrs = gen_file_content_attrs(artifact)
content_attrs["repository"] = repo["pulp_href"]
self.client.post(FILE_CONTENT_PATH, content_attrs)
files = {"file": utils.http_get(FILE_URL2)}
artifact = self.client.post(ARTIFACTS_PATH, files=files)
# create second content unit.
second_content_attrs = gen_file_content_attrs(artifact)
second_content_attrs["repository"] = repo["pulp_href"]
second_content_attrs["relative_path"] = content_attrs["relative_path"]
self.client.post(FILE_CONTENT_PATH, second_content_attrs)
repo_latest_version = self.client.get(
self.client.get(repo["pulp_href"])["latest_version_href"])
self.assertEqual(
repo_latest_version["content_summary"]["present"]["file.file"]
["count"], 1)
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/foo/`` and
``http://example.com/content/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Pulp-Fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/pulp/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(NPM_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["pulp_href"])
body = gen_npm_remote(url="https://registry.npmjs.org/commander/4.0.1")
remote = client.post(NPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote["pulp_href"])
sync(cfg, remote, repo)
repo = client.get(repo["pulp_href"])
# Create a distribution.
body = gen_distribution()
body["repository"] = repo["pulp_href"]
distribution = client.using_handler(api.task_handler).post(
NPM_DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution["pulp_href"])
# Pick a content unit, and download it from both Pulp Fixtures…
unit_path = choice(get_npm_content_paths(repo))
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(NPM_FIXTURE_URL, unit_path))).hexdigest()
# …and Pulp.
pulp_hash = hashlib.sha256(
utils.http_get(
urljoin(distribution["base_url"] + "/",
unit_path.split("/")[-1]))).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
def do_test(self, with_sqlite):
"""Sync and publish an RPM repository."""
# 1. create repo and remote
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
body = gen_rpm_remote(policy="on_demand")
remote = self.remote_api.create(body)
self.addCleanup(self.remote_api.delete, remote.pulp_href)
# 2. Sync it
repository_sync_data = RpmRepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href,
repository_sync_data)
monitor_task(sync_response.task)
# 3. Publish and distribute
publish_data = RpmRpmPublication(repository=repo.pulp_href,
sqlite_metadata=with_sqlite)
publish_response = self.publications.create(publish_data)
created_resources = monitor_task(
publish_response.task).created_resources
publication_href = created_resources[0]
self.addCleanup(self.publications.delete, publication_href)
body = gen_distribution()
body["publication"] = publication_href
distribution_response = self.distributions.create(body)
created_resources = monitor_task(
distribution_response.task).created_resources
distribution = self.distributions.read(created_resources[0])
self.addCleanup(self.distributions.delete, distribution.pulp_href)
repomd = ElementTree.fromstring(
http_get(os.path.join(distribution.base_url,
"repodata/repomd.xml")))
data_xpath = "{{{}}}data".format(RPM_NAMESPACES["metadata/repo"])
data_elems = [elem for elem in repomd.findall(data_xpath)]
sqlite_files = [
elem for elem in data_elems if elem.get("type").endswith("_db")
]
if with_sqlite:
self.assertEqual(3, len(sqlite_files))
for db_elem in sqlite_files:
location_xpath = "{{{}}}location".format(
RPM_NAMESPACES["metadata/repo"])
db_href = db_elem.find(location_xpath).get("href")
http_get(os.path.join(distribution.base_url, db_href))
else:
self.assertEqual(0, len(sqlite_files))
def test_all(self):
"""Sync and publish an RPM repository and verify the checksum."""
# 1. create repo and remote
repo = self.repo_api.create(gen_repo())
self.addCleanup(self.repo_api.delete, repo.pulp_href)
body = gen_rpm_remote()
remote = self.remote_api.create(body)
self.addCleanup(self.remote_api.delete, remote.pulp_href)
# 2. Sync it
repository_sync_data = RpmRepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href,
repository_sync_data)
monitor_task(sync_response.task)
# 3. Publish and distribute
publish_data = RpmRpmPublication(repository=repo.pulp_href)
publish_response = self.publications.create(publish_data)
created_resources = monitor_task(publish_response.task)
publication_href = created_resources[0]
self.addCleanup(self.publications.delete, publication_href)
body = gen_distribution()
body["publication"] = publication_href
distribution_response = self.distributions.create(body)
created_resources = monitor_task(distribution_response.task)
distribution = self.distributions.read(created_resources[0])
self.addCleanup(self.distributions.delete, distribution.pulp_href)
# 4. check the tag 'sum' is not present in updateinfo.xml
repomd = ElementTree.fromstring(
http_get(os.path.join(distribution.base_url,
'repodata/repomd.xml')))
with NamedTemporaryFile() as temp_file:
update_xml_url = self._get_updateinfo_xml_path(repomd)
update_xml_content = http_get(
os.path.join(distribution.base_url, update_xml_url))
temp_file.write(update_xml_content)
temp_file.seek(0)
# TODO: fix this as in CI update_info.xml has '.gz' but
# it is not gzipped
try:
update_xml = gzip.open(temp_file.name).read()
except OSError:
update_xml = temp_file.read()
update_info_content = ElementTree.fromstring(update_xml)
tags = {elem.tag for elem in update_info_content.iter()}
self.assertNotIn('sum', tags, update_info_content)
def test_content_served(self):
"""Verify that content is served over publication distribution."""
repo = self.client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo["pulp_href"])
remote = self.client.post(FILE_REMOTE_PATH, gen_file_remote())
self.addCleanup(self.client.delete, remote["pulp_href"])
sync(self.cfg, remote, repo)
repo = self.client.get(repo["pulp_href"])
publication = create_file_publication(self.cfg, repo)
self.addCleanup(self.client.delete, publication["pulp_href"])
distribution = self.client.post(
FILE_DISTRIBUTION_PATH,
gen_distribution(publication=publication["pulp_href"]))
self.addCleanup(self.client.delete, distribution["pulp_href"])
pulp_manifest = parse_pulp_manifest(
self.download_pulp_manifest(distribution))
self.assertEqual(len(pulp_manifest), FILE_FIXTURE_COUNT, pulp_manifest)
added_content = get_added_content(repo)
unit_path = added_content[FILE_CONTENT_NAME][0]["relative_path"]
unit_url = distribution["base_url"]
unit_url = urljoin(unit_url, unit_path)
pulp_hash = hashlib.sha256(
self.client.get(unit_url).content).hexdigest()
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(FILE_URL, unit_path))).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
def test_raise_error(self):
"""Create a duplicate content unit using same artifact and filename."""
delete_orphans(self.cfg)
files = {'file': utils.http_get(RPM_UNSIGNED_URL)}
artifact = self.client.post(ARTIFACTS_PATH, files=files)
attrs = {
'_artifact': artifact['_href'],
'relative_path': RPM_PACKAGE_FILENAME
}
# create first content unit.
self.client.post(RPM_CONTENT_PATH, attrs)
# using the same attrs used to create the first content unit.
response = api.Client(self.cfg,
api.echo_handler).post(RPM_CONTENT_PATH, attrs)
with self.assertRaises(HTTPError):
response.raise_for_status()
keywords = (
'name',
'epoch',
'version',
'release',
'arch',
'checksum_type',
'pkgId',
)
for key in keywords:
self.assertIn(key.lower(),
response.json()['non_field_errors'][0].lower(),
response.json())
def do_test(self, feed, type_id, body, unit_key=None):
"""Test how well Pulp can deal with duplicate unit uploads.
Do the following:
1. Create a new feed-less repository.
2. Upload content and import it into the repository. Assert the upload
and import was successful.
3. Upload identical content and import it into the repository.
The second upload should silently fail for all Pulp releases in the 2.x
series.
"""
if unit_key is None:
unit_key = {}
client = api.Client(self.cfg, api.json_handler)
unit = utils.http_get(feed)
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
for _ in range(2):
call_report = upload_import_unit(self.cfg, unit, {
'unit_type_id': type_id,
'unit_key': unit_key
}, repo)
self.assertIsNone(call_report['result'])
def test_01_add_unit(self):
"""Add a content unit to the repository. Publish the repository."""
repo_before = self.get_repo()
rpm = utils.http_get(RPM_UNSIGNED_URL)
utils.upload_import_unit(
self.cfg,
rpm,
{'unit_type_id': 'rpm'},
self.repo,
)
utils.publish_repo(self.cfg, repo_before)
repo_after = self.get_repo()
with self.subTest(comment='last_unit_added'):
if selectors.bug_is_untestable(1847, self.cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/1847')
pre = repo_before['last_unit_added']
post = repo_after['last_unit_added']
self.assertIsNone(pre)
self.assertIsNotNone(post)
with self.subTest(comment='last_unit_removed'):
pre = repo_before['last_unit_removed']
post = repo_after['last_unit_removed']
self.assertIsNone(pre)
self.assertIsNone(post)
with self.subTest(comment='last_publish'):
pre = repo_before['distributors'][0]['last_publish']
post = repo_after['distributors'][0]['last_publish']
self.assertIsNone(pre)
self.assertIsNotNone(post)
def do_test(self, distributor_config_update):
"""Implement most of the test logic."""
rpms = tuple(
utils.http_get(url)
for url in (RPM_UNSIGNED_URL, RPM2_UNSIGNED_URL))
# Create a repository.
client = api.Client(self.cfg, api.json_handler)
body = gen_repo()
body['distributors'] = [gen_distributor()]
body['distributors'][0]['distributor_config'].update(
distributor_config_update)
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
repo = client.get(repo['_href'], params={'details': True})
# Upload an RPM, publish the repo, and count metadata files twice.
cli_client = cli.Client(self.cfg)
sudo = () if cli.is_root(self.cfg) else ('sudo', )
find_repodata_cmd = sudo + (
'find',
os.path.join('/var/lib/pulp/published/yum/master/yum_distributor/',
str(repo['id'])), '-type', 'd', '-name', 'repodata')
found = []
for rpm in rpms:
upload_import_unit(self.cfg, rpm, {'unit_type_id': 'rpm'}, repo)
publish_repo(self.cfg, repo)
repodata_path = cli_client.run(find_repodata_cmd).stdout.strip()
found.append(
cli_client.run(sudo + ('find', repodata_path, '-type',
'f')).stdout.splitlines())
return found
def test_iso(self):
"""Upload duplicate ISO content. See :meth:`do_test`.
This test targets the following issues:
* `Pulp Smash #582 <https://github.com/PulpQE/pulp-smash/issues/582>`_
* `Pulp #2274 <https://pulp.plan.io/issues/2274>`_
"""
if not selectors.bug_is_fixed(2274, self.cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/2274')
body = {
'id':
utils.uuid4(),
'importer_type_id':
'iso_importer',
'distributors': [{
'auto_publish': False,
'distributor_id': utils.uuid4(),
'distributor_type_id': 'iso_distributor',
}],
}
iso = utils.http_get(FILE_URL)
unit_key = {
'checksum': hashlib.sha256(iso).hexdigest(),
'name': os.path.basename(urlsplit(FILE_URL).path),
'size': len(iso),
}
self.do_test(FILE_URL, 'iso', body, unit_key)
def test_all(self):
"""Verify ``RPM_LARGE_METADATA`` RPM file can be uploaded.
Specifically, this method does the following:
1. Create an RPM repo.
2. Verify whether the file ``RPM_LARGE_METADATA`` can be uploaded
into the repo without errors.
This test targets:
* `Pulp #723 <https://pulp.plan.io/issues/723>`_
* `Pulp-2-Tests #88 <https://github.com/PulpQE/Pulp-2-Tests/issues/88>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
body = gen_repo(distributors=[gen_distributor()])
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
rpm = utils.http_get(RPM_LARGE_METADATA_FEED)
upload_import_unit(cfg, rpm, {'unit_type_id': 'rpm'}, repo)
repo = client.get(repo['_href'], params={'details': True})
publish_repo(cfg, repo)
rpm_path = get_rpm_published_path(cfg, repo, RPM_LARGE_METADATA)
# Check whether the RPM is uploaded published.
self.assertIn(RPM_LARGE_METADATA, rpm_path, rpm_path)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.client = api.Client(cls.cfg)
cls.file = http_get(FILE_TO_BE_CHUNKED_URL)
cls.file_sha256 = hashlib.sha256(cls.file).hexdigest()
cls.size_file = len(cls.file)
def test_content_remote_delete(self):
"""Assert that an HTTP error is raised when remote is deleted.
Also verify that the content can be downloaded from Pulp once the
remote is recreated and another sync is triggered.
"""
cfg = config.get_config()
delete_orphans(cfg)
client = api.Client(cfg, api.page_handler)
repo = client.post(FILE_REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['pulp_href'])
body = gen_file_remote(policy=choice(ON_DEMAND_DOWNLOAD_POLICIES))
remote = client.post(FILE_REMOTE_PATH, body)
# Sync the repository using a lazy download policy.
sync(cfg, remote, repo)
repo = client.get(repo['pulp_href'])
publication = create_file_publication(cfg, repo)
self.addCleanup(client.delete, publication['pulp_href'])
# Delete the remote.
client.delete(remote['pulp_href'])
body = gen_distribution()
body['publication'] = publication['pulp_href']
distribution = client.using_handler(api.task_handler).post(
FILE_DISTRIBUTION_PATH, body)
self.addCleanup(client.delete, distribution['pulp_href'])
unit_path = choice([
content_unit['relative_path']
for content_unit in get_content(repo)[FILE_CONTENT_NAME]
])
# Assert that an HTTP error is raised when one to fetch content from
# the distribution once the remote was removed.
with self.assertRaises(HTTPError) as ctx:
download_content_unit(cfg, distribution, unit_path)
for key in ('not', 'found'):
self.assertIn(key, ctx.exception.response.reason.lower())
# Recreating a remote and re-triggering a sync will cause these broken
# units to recover again.
body = gen_file_remote(policy=choice(ON_DEMAND_DOWNLOAD_POLICIES))
remote = client.post(FILE_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['pulp_href'])
sync(cfg, remote, repo)
repo = client.get(repo['pulp_href'])
content = download_content_unit(cfg, distribution, unit_path)
pulp_hash = hashlib.sha256(content).hexdigest()
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(FILE_FIXTURE_URL, unit_path))).hexdigest()
self.assertEqual(pulp_hash, fixtures_hash)
def setUpClass(cls):
"""
Check if packages to install through tests are already installed
"""
cls.client = gen_python_client()
cls.cli_client = cli.Client(cfg)
cls.PACKAGES = PYTHON_FIXTURES_PACKAGES
cls.PACKAGES_URLS = [
urljoin(urljoin(PYTHON_FIXTURE_URL, "packages/"), filename)
for filename in PYTHON_FIXTURES_FILENAMES
]
cls.PACKAGES_FILES = [{
"file": http_get(file)
} for file in cls.PACKAGES_URLS]
delete_orphans()
for pkg in cls.PACKAGES:
cls.assertFalse(
cls,
cls.check_install(cls.cli_client, pkg),
"{} is already installed".format(pkg),
)
cls.repo_api = RepositoriesPythonApi(cls.client)
cls.remote_api = RemotesPythonApi(cls.client)
cls.content_api = ContentPackagesApi(cls.client)
cls.publications_api = PublicationsPypiApi(cls.client)
cls.distro_api = DistributionsPypiApi(cls.client)
def test_all(self):
"""Test that uploading DRPM with checksumtype specified works."""
if selectors.bug_is_untestable(1806, self.cfg.pulp_version):
raise unittest.SkipTest('https://pulp.plan.io/issues/1806')
if selectors.bug_is_untestable(2627, self.cfg.pulp_version):
raise unittest.SkipTest('https://pulp.plan.io/issues/2627')
client = api.Client(self.cfg)
repo = client.post(REPOSITORY_PATH, gen_repo()).json()
self.addCleanup(client.delete, repo['_href'])
drpm = utils.http_get(DRPM_UNSIGNED_URL)
utils.upload_import_unit(
self.cfg,
drpm,
{
'unit_type_id': 'drpm',
'unit_metadata': {
'checksumtype': 'sha256'
},
},
repo,
)
units = utils.search_units(self.cfg, repo, {})
self.assertEqual(len(units), 1, units)
# Test if DRPM extracted correct metadata for creating filename.
self.assertEqual(
units[0]['metadata']['filename'],
DRPM,
)
def test_serving_acs_content(self):
"""Test serving of ACS content through the content app."""
cfg = config.get_config()
acs = self._create_acs()
resp = self.file_acs_api.refresh(acs.pulp_href, acs)
monitor_task_group(resp.task_group)
remote = self.file_remote_api.create(
gen_file_remote(FILE_MANIFEST_ONLY_FIXTURE_URL, policy="on_demand")
)
self.addCleanup(self.file_remote_api.delete, remote.pulp_href)
repo = self.repo_api.create(gen_repo(remote=remote.pulp_href, autopublish=True))
self.addCleanup(self.repo_api.delete, repo.pulp_href)
distribution_response = self.distribution_api.create(
gen_distribution(repository=repo.pulp_href)
)
created_resources = monitor_task(distribution_response.task).created_resources
distribution = self.distribution_api.read(created_resources[0])
self.addCleanup(self.distribution_api.delete, distribution.pulp_href)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
repo = self.repo_api.read(repo.pulp_href)
unit_path = choice(get_file_content_paths(repo.to_dict()))
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(FILE_FIXTURE_URL, unit_path))
).hexdigest()
content = download_content_unit(cfg, distribution.to_dict(), unit_path)
pulp_hash = hashlib.sha256(content).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
def test_raise_error(self):
"""Create a duplicate content unit using same relative_path.
Artifacts are unique by ``relative_path`` and ``file``.
In order to raise an HTTP error, the same ``artifact`` and the same
``relative_path`` should be used.
"""
delete_orphans(self.cfg)
files = {'file': utils.http_get(FILE_URL)}
artifact = self.client.post(ARTIFACTS_PATH, files=files)
attrs = gen_file_content_attrs(artifact)
# create first content unit.
self.client.post(FILE_CONTENT_PATH, attrs)
# using the same attrs used to create the first content unit.
response = api.Client(self.cfg,
api.echo_handler).post(FILE_CONTENT_PATH, attrs)
with self.assertRaises(HTTPError):
response.raise_for_status()
for key in ('already', 'content', 'relative', 'path', 'artifact'):
self.assertIn(key,
response.json()['non_field_errors'][0].lower(),
response.json())
def test_raise_error(self):
"""Create a duplicate content unit using same artifact and filename."""
delete_orphans(self.cfg)
files = {'file': utils.http_get(RPM_UNSIGNED_URL)}
artifact = self.client.post(ARTIFACTS_PATH, files=files)
attrs = {
'artifact': artifact['pulp_href'],
'relative_path': RPM_PACKAGE_FILENAME
}
# create first content unit.
self.client.using_handler(api.task_handler).post(
RPM_CONTENT_PATH, attrs)
with self.assertRaises(TaskReportError) as ctx:
# using the same attrs used to create the first content unit.
api.Client(self.cfg,
api.task_handler).post(RPM_CONTENT_PATH, attrs)
keywords = (
'name',
'epoch',
'version',
'release',
'arch',
'checksum_type',
'pkgId',
)
for key in keywords:
self.assertIn(key.lower(),
ctx.exception.task['error']['description'].lower(),
ctx.exception.task['error'])
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of creating a Maven mirror is simple:
1. Create a Maven Remote with a URL pointing to the root of a Maven repository.
2. Create a distribution with the remote set HREF from 1.
Do the following:
1. Create a Maven Remote and a Distribution.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Maven Central.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo["_href"])
body = gen_maven_remote()
remote = client.post(MAVEN_REMOTE_PATH, body)
self.addCleanup(client.delete, remote["_href"])
repo = client.get(repo["_href"])
# Create a distribution.
body = gen_distribution()
body["remote"] = remote["_href"]
response_dict = client.post(MAVEN_DISTRIBUTION_PATH, body)
dist_task = client.get(response_dict["task"])
distribution_href = dist_task["created_resources"][0]
distribution = client.get(distribution_href)
self.addCleanup(client.delete, distribution["_href"])
# Pick a content unit, and download it from both Pulp Fixtures…
unit_path = choice(get_maven_content_paths(repo))
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(MAVEN_FIXTURE_URL, unit_path))).hexdigest()
# …and Pulp.
client.response_handler = api.safe_handler
unit_url = cfg.get_hosts("content")[0].roles["content"]["scheme"]
unit_url += "://" + distribution["base_url"] + "/"
unit_url = urljoin(unit_url, unit_path)
pulp_hash = hashlib.sha256(client.get(unit_url).content).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
# Check that Pulp created a MavenArtifact
content_filter_url = MAVEN_CONTENT_PATH + "?filename=custommatcher-1.0-javadoc.jar.sha1"
content_unit = client.get(content_filter_url)
self.assertEqual(1, content_unit.json()["count"])
def get_metadata_content(base_url, repomd_elem, meta_type):
"""Return the text contents of metadata file.
Provided a url, a repomd root element, and a metadata type, locate the metadata
file's location href, download it from the provided url, un-gzip it, parse it, and
return the root element node.
Don't use this with large repos because it will blow up.
"""
# <ns0:repomd xmlns:ns0="http://linux.duke.edu/metadata/repo">
# <ns0:data type="primary">
# <ns0:checksum type="sha256">[…]</ns0:checksum>
# <ns0:location href="repodata/[…]-primary.xml.gz" />
# …
# </ns0:data>
# …
xpath = "{{{}}}data".format(RPM_NAMESPACES["metadata/repo"])
data_elems = [
elem for elem in repomd_elem.findall(xpath)
if elem.get("type") == meta_type
]
xpath = "{{{}}}location".format(RPM_NAMESPACES["metadata/repo"])
location_href = data_elems[0].find(xpath).get("href")
return read_xml_gz(http_get(os.path.join(base_url, location_href)))
def test_all(self):
"""Delete an artifact, it is removed from the filesystem.
Do the following:
1. Create an artifact, and verify it is present on the filesystem.
2. Delete the artifact, and verify it is absent on the filesystem.
"""
cli_client = cli.Client(config.get_config())
storage = utils.get_pulp_setting(cli_client, "DEFAULT_FILE_STORAGE")
if storage != "pulpcore.app.models.storage.FileSystem":
# this test only works for filesystem storage
return
cfg = config.get_config()
api_client = api.Client(cfg, api.json_handler)
cli_client = cli.Client(cfg)
# create
files = {"file": utils.http_get(FILE_URL)}
artifact = api_client.post(ARTIFACTS_PATH, files=files)
self.addCleanup(api_client.delete, artifact["pulp_href"])
cmd = ("ls", os.path.join(MEDIA_PATH, artifact["file"]))
cli_client.run(cmd, sudo=True)
# delete
self.doCleanups()
with self.assertRaises(CalledProcessError):
cli_client.run(cmd, sudo=True)
def test_single_request_upload(self):
"""Test single request upload."""
cfg = config.get_config()
# Pulp does not support single request upload for a RPM already present
# in Pulp.
delete_orphans(cfg)
file = {'file': utils.http_get(RPM_UNSIGNED_URL)}
client = api.Client(cfg, api.page_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
client.post(
urljoin(BASE_PATH, 'rpm/upload/'),
files=file,
data={'repository': repo['_href']}
)
repo = client.get(repo['_href'])
# Assertion about repo version.
self.assertIsNotNone(repo['_latest_version_href'], repo)
# Assertions about artifcats.
artifact = client.get(ARTIFACTS_PATH)
self.assertEqual(len(artifact), 1, artifact)
self.assertEqual(
artifact[0]['sha256'],
hashlib.sha256(file['file']).hexdigest(),
artifact
)
# Assertion about content unit.
content = client.get(RPM_CONTENT_PATH)
self.assertEqual(len(content), 1, content)
def gen_artifact(url=PYTHON_URL):
"""Creates an artifact."""
with NamedTemporaryFile() as temp_file:
temp_file.write(http_get(url))
temp_file.flush()
artifact = ArtifactsApi(core_client).create(file=temp_file.name)
return artifact.to_dict()
def setUpClass(cls):
"""Create class-wide variable."""
cls.cfg = config.get_config()
delete_orphans(cls.cfg)
cls.content_unit = {}
cls.client = api.Client(cls.cfg, api.json_handler)
files = {'file': utils.http_get(PLUGIN_TEMPLATE_URL)}
cls.artifact = cls.client.post(ARTIFACTS_PATH, files=files)
def setUpClass(cls):
"""Delete orphans and create class-wide variables."""
cfg = config.get_config()
delete_orphans(cfg)
cls.client = api.Client(cfg, api.json_handler)
cls.file = {'file': utils.http_get(FILE_URL)}
cls.file_sha256 = hashlib.sha256(cls.file['file']).hexdigest()
cls.file_size = len(cls.file['file'])
def setUpClass(cls):
"""Delete orphans and create class-wide variables."""
cfg = config.get_config()
delete_orphans(cfg)
cls.client = api.Client(cfg, api.json_handler)
cls.file = {'file': utils.http_get(FILE_URL)}
cls.file_sha256 = hashlib.sha256(cls.file['file']).hexdigest()
cls.file_size = len(cls.file['file'])
def setUpClass(cls):
"""Create class-wide variable."""
cls.cfg = config.get_config()
delete_orphans(cls.cfg)
cls.content_unit = {}
cls.client = api.Client(cls.cfg, api.json_handler)
files = {'file': utils.http_get(RPM_SIGNED_URL)}
cls.artifact = cls.client.post(ARTIFACTS_PATH, files=files)
def setUpClass(cls):
"""Create class-wide variable."""
cls.cfg = config.get_config()
delete_orphans(cls.cfg)
cls.content_unit = {}
cls.client = api.Client(cls.cfg, api.smart_handler)
cls.files = {"file": utils.http_get(FILE_URL)}
cls.attrs = gen_file_content_upload_attrs()
def setUpClass(cls):
"""Create a Docker repository."""
super(DuplicateUploadsTestCase, cls).setUpClass()
unit = utils.http_get(DOCKER_IMAGE_URL)
unit_type_id = 'docker_image'
client = api.Client(cls.cfg, api.json_handler)
repo_href = client.post(REPOSITORY_PATH, gen_repo())['_href']
cls.resources.add(repo_href)
cls.upload_import_unit_args = (cls.cfg, unit, unit_type_id, repo_href)
def setUpClass(cls):
"""Create a Puppet repository."""
super(DuplicateUploadsTestCase, cls).setUpClass()
unit = utils.http_get(PUPPET_MODULE_URL)
unit_type_id = 'puppet_module'
client = api.Client(cls.cfg, api.json_handler)
repo_href = client.post(REPOSITORY_PATH, gen_repo())['_href']
cls.resources.add(repo_href)
cls.upload_import_unit_args = (cls.cfg, unit, unit_type_id, repo_href)
def setUpClass(cls):
"""Create a Python repo. Upload a Python package into it twice."""
super(DuplicateUploadsTestCase, cls).setUpClass()
unit = utils.http_get(PYTHON_EGG_URL)
unit_type_id = "python_package"
client = api.Client(cls.cfg, api.json_handler)
repo_href = client.post(REPOSITORY_PATH, gen_repo())["_href"]
cls.resources.add(repo_href)
cls.call_reports = tuple((utils.upload_import_unit(cls.cfg, unit, unit_type_id, repo_href) for _ in range(2)))
def setUpClass(cls):
"""Create an RPM repository. Upload an RPM into it twice."""
super(DuplicateUploadsTestCase, cls).setUpClass()
utils.reset_pulp(cls.cfg)
unit = utils.http_get(RPM_URL)
unit_type_id = 'rpm'
client = api.Client(cls.cfg, api.json_handler)
repo_href = client.post(REPOSITORY_PATH, gen_repo())['_href']
cls.resources.add(repo_href)
cls.upload_import_unit_args = (cls.cfg, unit, unit_type_id, repo_href)
def test_clean_orphan_artifact(self):
"""Test whether orphan artifacts units can be clean up."""
repo = self.api_client.post(REPO_PATH, gen_repo())
self.addCleanup(self.api_client.delete, repo['_href'])
files = {'file': utils.http_get(FILE2_URL)}
artifact = self.api_client.post(ARTIFACTS_PATH, files=files)
cmd = self.sudo + ('ls', artifact['file'])
self.cli_client.run(cmd)
delete_orphans()
with self.assertRaises(CalledProcessError):
self.cli_client.run(cmd)
def test_all(self):
"""Execute the test case business logic."""
cfg = config.get_config()
self.check_issue_2277(cfg)
self.check_issue_2321(cfg)
repo_href = self.create_repo(cfg, RPM_MIRRORLIST_MIXED, _gen_rel_url())
utils.sync_repo(cfg, repo_href)
self.publish_repo(cfg, repo_href)
actual_rpm = self.get_unit(cfg, repo_href, RPM)
target_rpm = utils.http_get(RPM_UNSIGNED_URL)
self.assertEqual(actual_rpm, target_rpm)
def setUpClass(cls):
"""Create a Python repo. Upload a Python package into it twice."""
super(DuplicateUploadsTestCase, cls).setUpClass()
if (cls.cfg.version >= Version('2.11') and
selectors.bug_is_untestable(2334, cls.cfg.version)):
raise unittest.SkipTest('https://pulp.plan.io/issues/2334')
unit = utils.http_get(PYTHON_EGG_URL)
unit_type_id = 'python_package'
client = api.Client(cls.cfg, api.json_handler)
repo_href = client.post(REPOSITORY_PATH, gen_repo())['_href']
cls.resources.add(repo_href)
cls.upload_import_unit_args = (cls.cfg, unit, unit_type_id, repo_href)
def setUpModule(): # pylint:disable=invalid-name
"""Conditionally skip tests. Cache packages to be uploaded to repos.
Skip the tests in this module if:
* The RPM plugin is unsupported.
* `Pulp #1991 <https://pulp.plan.io/issues/1991>`_ is untestable for the
version of Pulp under test.
"""
cfg = config.get_config()
if selectors.bug_is_untestable(1991, cfg.version):
raise unittest.SkipTest('https://pulp.plan.io/issues/1991')
set_up_module()
try:
_SIGNED_PACKAGES['rpm'] = utils.http_get(RPM_URL)
_SIGNED_PACKAGES['srpm'] = utils.http_get(SRPM_URL)
_UNSIGNED_PACKAGES['rpm'] = utils.http_get(RPM_UNSIGNED_URL)
_UNSIGNED_PACKAGES['srpm'] = utils.http_get(SRPM_UNSIGNED_URL)
if selectors.bug_is_testable(1806, cfg.version):
_SIGNED_PACKAGES['drpm'] = utils.http_get(DRPM_URL)
_UNSIGNED_PACKAGES['drpm'] = utils.http_get(DRPM_UNSIGNED_URL)
except:
_SIGNED_PACKAGES.clear()
_UNSIGNED_PACKAGES.clear()
raise
def setUpModule(): # pylint:disable=invalid-name
"""Conditionally skip tests. Create repositories with fixture data."""
cfg = config.get_config()
if selectors.bug_is_untestable(1991, cfg.version):
raise unittest.SkipTest('https://pulp.plan.io/issues/1991')
set_up_module()
# Fetch RPMs.
_SIGNED_PACKAGES['rpm'] = utils.http_get(RPM_URL)
_SIGNED_PACKAGES['srpm'] = utils.http_get(SRPM_URL)
_UNSIGNED_PACKAGES['rpm'] = utils.http_get(RPM_UNSIGNED_URL)
_UNSIGNED_PACKAGES['srpm'] = utils.http_get(SRPM_UNSIGNED_URL)
if selectors.bug_is_testable(1806, cfg.version):
_SIGNED_PACKAGES['drpm'] = utils.http_get(DRPM_URL)
_UNSIGNED_PACKAGES['drpm'] = utils.http_get(DRPM_UNSIGNED_URL)
# Create repos, and upload RPMs to them.
client = api.Client(cfg, api.json_handler)
try:
repo = client.post(REPOSITORY_PATH, gen_repo())
_REPOS['signed'] = repo
for type_id, pkg in _SIGNED_PACKAGES.items():
utils.upload_import_unit(cfg, pkg, type_id, repo['_href'])
repo = client.post(REPOSITORY_PATH, gen_repo())
_REPOS['unsigned'] = repo
for type_id, pkg in _UNSIGNED_PACKAGES.items():
utils.upload_import_unit(cfg, pkg, type_id, repo['_href'])
except:
_SIGNED_PACKAGES.clear()
_UNSIGNED_PACKAGES.clear()
for _ in range(len(_REPOS)):
client.delete(_REPOS.popitem()[1]['_href'])
raise
def setUpModule(): # pylint:disable=invalid-name
"""Conditionally skip tests. Cache packages to be uploaded to repos.
Skip the tests in this module if:
* The RPM plugin is unsupported.
* `Pulp #1991 <https://pulp.plan.io/issues/1991>`_ is untestable for the
version of Pulp under test.
"""
if selectors.bug_is_untestable(1991, config.get_config().version):
raise unittest2.SkipTest('https://pulp.plan.io/issues/1991')
set_up_module()
global _PACKAGES # pylint:disable=global-statement
try:
_PACKAGES = {
'signed drpm': utils.http_get(DRPM_URL),
'signed rpm': utils.http_get(RPM_URL),
'signed srpm': utils.http_get(SRPM_URL),
'unsigned drpm': utils.http_get(DRPM_UNSIGNED_URL),
'unsigned rpm': utils.http_get(RPM_UNSIGNED_URL),
'unsigned srpm': utils.http_get(SRPM_UNSIGNED_URL),
}
except:
_PACKAGES = None
raise
def test_all(self):
"""Publish a repository with the repoview feature on and off."""
cfg = config.get_config()
if cfg.version < Version('2.9'):
self.skipTest('https://pulp.plan.io/issues/189')
# Create a repo, and add content
client = api.Client(cfg, api.json_handler)
body = gen_repo()
body['distributors'] = [gen_distributor()]
repo_href = client.post(constants.REPOSITORY_PATH, body)['_href']
self.addCleanup(client.delete, repo_href)
rpm = utils.http_get(constants.RPM_UNSIGNED_URL)
utils.upload_import_unit(cfg, rpm, 'rpm', repo_href)
# Gather some facts about the repo distributor
dist = client.get(urljoin(repo_href, 'distributors/'))[0]
dist_url = urljoin('/pulp/repos/', dist['config']['relative_url'])
# Publish the repo
client.response_handler = api.safe_handler
client.post(urljoin(repo_href, 'actions/publish/'), {'id': dist['id']})
response = client.get(dist_url)
with self.subTest(comment='first publish'):
self.assertEqual(len(response.history), 0, response.history)
# Publish the repo a second time
client.post(
urljoin(repo_href, 'actions/publish/'),
{'id': dist['id'], 'override_config': {
'generate_sqlite': True,
'repoview': True,
}},
)
response = client.get(dist_url)
with self.subTest(comment='second publish'):
self.assertEqual(len(response.history), 1, response.history)
self.assertEqual(
response.request.url,
urljoin(response.history[0].request.url, 'repoview/index.html')
)
# Publish the repo a third time
if selectors.bug_is_untestable(2349, cfg.version):
self.skipTest('https://pulp.plan.io/issues/2349')
client.post(urljoin(repo_href, 'actions/publish/'), {'id': dist['id']})
response = client.get(dist_url)
with self.subTest(comment='third publish'):
self.assertEqual(len(response.history), 0, response.history)
def test_all(self):
"""Publish a repository with the repoview feature on and off."""
cfg = config.get_config()
if cfg.version < Version('2.9'):
self.skipTest('https://pulp.plan.io/issues/189')
# Create a repo, and add content
client = api.Client(cfg)
body = gen_repo()
body['distributors'] = [gen_distributor()]
repo = client.post(constants.REPOSITORY_PATH, body).json()
self.addCleanup(client.delete, repo['_href'])
rpm = utils.http_get(constants.RPM_UNSIGNED_URL)
utils.upload_import_unit(cfg, rpm, 'rpm', repo['_href'])
# Get info about the repo distributor
repo = client.get(repo['_href'], params={'details': True}).json()
pub_path = urljoin(
'/pulp/repos/',
repo['distributors'][0]['config']['relative_url']
)
# Publish the repo
utils.publish_repo(cfg, repo)
response = client.get(pub_path)
with self.subTest(comment='first publish'):
self.assertEqual(len(response.history), 0, response.history)
# Publish the repo a second time
utils.publish_repo(cfg, repo, {
'id': repo['distributors'][0]['id'],
'override_config': {'generate_sqlite': True, 'repoview': True},
})
response = client.get(pub_path)
with self.subTest(comment='second publish'):
self.assertEqual(len(response.history), 1, response.history)
self.assertEqual(
response.request.url,
urljoin(response.history[0].request.url, 'repoview/index.html')
)
# Publish the repo a third time
if selectors.bug_is_untestable(2349, cfg.version):
self.skipTest('https://pulp.plan.io/issues/2349')
utils.publish_repo(cfg, repo)
response = client.get(pub_path)
with self.subTest(comment='third publish'):
self.assertEqual(len(response.history), 0, response.history)
def health_check(self):
"""Execute step three of the test plan."""
client = api.Client(self.cfg, api.json_handler)
body = gen_repo()
body['importer_config']['feed'] = RPM_SIGNED_FEED_URL
body['distributors'] = [gen_distributor()]
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
repo = client.get(repo['_href'], params={'details': True})
utils.sync_repo(self.cfg, repo['_href'])
utils.publish_repo(self.cfg, repo)
pulp_rpm = get_unit(self.cfg, repo, RPM).content
# Does this RPM match the original RPM?
rpm = utils.http_get(RPM_SIGNED_URL)
self.assertEqual(rpm, pulp_rpm)
def _create_repo_import_unit(self, pkg_url):
"""Create a repository, and import the given package into it.
Schedule the repository and all orphan content units for deletion.
Return the repository's href.
"""
self.addCleanup(self.client.delete, ORPHANS_PATH)
repo_href = self.client.post(REPOSITORY_PATH, gen_repo())['_href']
self.addCleanup(self.client.delete, repo_href)
pkg = utils.http_get(pkg_url)
pkg_filename = _get_pkg_filename(pkg_url)
pkg_unit_type = _get_pkg_unit_type(pkg_filename)
utils.upload_import_unit(self.cfg, pkg, pkg_unit_type, repo_href)
return repo_href
def health_check(self):
"""Execute step three of the test plan."""
client = api.Client(self.cfg, api.json_handler)
body = gen_repo()
body["importer_config"]["feed"] = RPM_FEED_URL
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(api.Client(self.cfg).delete, repo["_href"])
utils.sync_repo(self.cfg, repo["_href"])
distributor = client.post(urljoin(repo["_href"], "distributors/"), gen_distributor())
client.post(urljoin(repo["_href"], "actions/publish/"), {"id": distributor["id"]})
client.response_handler = api.safe_handler
url = urljoin("/pulp/repos/", distributor["config"]["relative_url"])
url = urljoin(url, RPM)
pulp_rpm = client.get(url).content
# Does this RPM match the original RPM?
rpm = utils.http_get(RPM_URL)
self.assertEqual(rpm, pulp_rpm)
def setUpClass(cls):
"""Import a SRPM into a repository and search it for content units.
Specifically, this method does the following:
1. Create a yum repository.
2. Upload a SRPM into the repository.
3. Search for all content units in the repository.
"""
super(UploadSrpmTestCase, cls).setUpClass()
client = api.Client(cls.cfg)
repo = client.post(REPOSITORY_PATH, gen_repo()).json()
cls.resources.add(repo['_href'])
srpm = utils.http_get(SRPM_UNSIGNED_URL)
upload_import_unit(cls.cfg, srpm, 'srpm', repo['_href'])
cls.repo_units = client.post(
urljoin(repo['_href'], 'search/units/'),
{'criteria': {}},
)
def setUpClass(cls):
"""Import a DRPM into a repository and search it for content units.
Specifically, this method does the following:
1. Create a yum repository.
2. Upload a DRPM into the repository.
3. Search for all content units in the repository.
"""
super(UploadDrpmTestCase, cls).setUpClass()
if selectors.bug_is_untestable(1806, cls.cfg.version):
raise unittest.SkipTest('https://pulp.plan.io/issues/1806')
client = api.Client(cls.cfg)
repo = client.post(REPOSITORY_PATH, gen_repo()).json()
cls.resources.add(repo['_href'])
drpm = utils.http_get(DRPM_UNSIGNED_URL)
upload_import_unit(cls.cfg, drpm, 'drpm', repo['_href'])
cls.repo_units = client.post(
urljoin(repo['_href'], 'search/units/'),
{'criteria': {}},
)
def test_all(self):
"""Delete an artifact, it is removed from the filesystem.
Do the following:
1. Create an artifact, and verify it is present on the filesystem.
2. Delete the artifact, and verify it is absent on the filesystem.
"""
cfg = config.get_config()
api_client = api.Client(cfg, api.json_handler)
cli_client = cli.Client(cfg)
# create
files = {'file': utils.http_get(FILE_URL)}
artifact = api_client.post(ARTIFACTS_PATH, files=files)
self.addCleanup(api_client.delete, artifact['_href'])
cmd = ('ls', artifact['file'])
cli_client.run(cmd, sudo=True)
# delete
self.doCleanups()
with self.assertRaises(CalledProcessError):
cli_client.run(cmd, sudo=True)
def health_check(self):
"""Execute step three of the test plan."""
client = api.Client(self.cfg, api.json_handler)
body = gen_repo()
body['importer_config']['feed'] = RPM_FEED_URL
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(api.Client(self.cfg).delete, repo['_href'])
utils.sync_repo(self.cfg, repo['_href'])
distributor = client.post(
urljoin(repo['_href'], 'distributors/'),
gen_distributor(),
)
client.post(
urljoin(repo['_href'], 'actions/publish/'),
{'id': distributor['id']},
)
client.response_handler = api.safe_handler
url = urljoin('/pulp/repos/', distributor['config']['relative_url'])
url = urljoin(url, RPM)
pulp_rpm = client.get(url).content
# Does this RPM match the original RPM?
rpm = utils.http_get(RPM_URL)
self.assertEqual(rpm, pulp_rpm)
def test_raise_error(self):
"""Create a duplicate content unit using same artifact and filename."""
delete_orphans(self.cfg)
files = {'file': utils.http_get(RPM_UNSIGNED_URL)}
artifact = self.client.post(ARTIFACTS_PATH, files=files)
attrs = {
'_artifact': artifact['_href'],
'filename': RPM_PACKAGE_FILENAME
}
# create first content unit.
self.client.post(RPM_CONTENT_PATH, attrs)
# using the same attrs used to create the first content unit.
response = api.Client(self.cfg, api.echo_handler).post(
RPM_CONTENT_PATH,
attrs
)
with self.assertRaises(HTTPError):
response.raise_for_status()
keywords = (
'name',
'epoch',
'version',
'release',
'arch',
'checksum_type',
'pkgId',
)
for key in keywords:
self.assertIn(
key.lower(),
response.json()['non_field_errors'][0].lower(),
response.json()
)
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/foo/`` and
``http://example.com/content/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Pulp-Fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_deb_remote()
remote = client.post(DEB_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
# Create a publisher.
publisher = client.post(self.Meta.publisher_path, self.Meta.gen_publisher())
self.addCleanup(client.delete, publisher['_href'])
# Create a publication.
publication = publish(cfg, publisher, repo)
self.addCleanup(client.delete, publication['_href'])
# Create a distribution.
body = gen_distribution()
body['publication'] = publication['_href']
response_dict = client.post(DISTRIBUTION_PATH, body)
dist_task = client.get(response_dict['task'])
distribution_href = dist_task['created_resources'][0]
distribution = client.get(distribution_href)
self.addCleanup(client.delete, distribution['_href'])
# Pick a content unit (of each type), and download it from both Pulp Fixtures…
unit_paths = [
choice(paths) for paths in self.Meta.get_content_unit_paths(repo).values() if paths
]
fixtures_hashes = [hashlib.sha256(
utils.http_get(urljoin(DEB_FIXTURE_URL, unit_path[0]))
).hexdigest() for unit_path in unit_paths]
# …and Pulp.
client.response_handler = api.safe_handler
unit_base_url = cfg.get_hosts('api')[0].roles['api']['scheme']
unit_base_url += '://' + distribution['base_url'] + '/'
unit_urls = [urljoin(unit_base_url, unit_path[1]) for unit_path in unit_paths]
pulp_hashes = [hashlib.sha256(client.get(unit_url).content).hexdigest()
for unit_url in unit_urls]
self.assertEqual(fixtures_hashes, pulp_hashes)
def setUpClass(cls):
"""Upload puppet module to a repo, copy it to another, publish and download.
Create two puppet repositories, both without feeds. Upload an module to
the first repository. Copy its content to the second repository. Add
distributors to the repositories, publish repositories and download
modules back from them.
"""
super(PublishTestCase, cls).setUpClass()
utils.reset_pulp(cls.cfg) # See: https://pulp.plan.io/issues/1406
cls.responses = {}
cls.modules = [] # Raw puppet modules.
# Download a puppet module and create two repositories.
client = api.Client(cls.cfg, api.json_handler)
repos = [client.post(REPOSITORY_PATH, gen_repo()) for _ in range(2)]
for repo in repos:
cls.resources.add(repo['_href'])
client.response_handler = api.safe_handler
cls.modules.append(utils.http_get(PUPPET_MODULE_URL))
# Begin an upload request, upload a puppet module, move the puppet
# module into a repository, and end the upload request.
cls.responses['malloc'] = client.post(CONTENT_UPLOAD_PATH)
cls.responses['upload'] = client.put(
urljoin(cls.responses['malloc'].json()['_href'], '0/'),
data=cls.modules[0],
)
cls.responses['import'] = client.post(
urljoin(repos[0]['_href'], 'actions/import_upload/'),
{
'unit_key': {},
'unit_type_id': 'puppet_module',
'upload_id': cls.responses['malloc'].json()['upload_id'],
},
)
cls.responses['free'] = client.delete(
cls.responses['malloc'].json()['_href']
)
# Copy content from the first puppet repository to the second.
cls.responses['copy'] = client.post(
urljoin(repos[1]['_href'], 'actions/associate/'),
{'source_repo_id': repos[0]['id']}
)
# Add a distributor to each repository. Publish each repository.
for key in {'distribute', 'publish'}:
cls.responses[key] = []
for repo in repos:
cls.responses['distribute'].append(client.post(
urljoin(repo['_href'], 'distributors/'),
{
'auto_publish': False,
'distributor_id': utils.uuid4(),
'distributor_type_id': 'puppet_distributor',
'distributor_config': {
'serve_http': True,
'serve_https': True,
'relative_url': '/' + utils.uuid4(),
},
}
))
cls.responses['publish'].append(client.post(
urljoin(repo['_href'], 'actions/publish/'),
{'id': cls.responses['distribute'][-1].json()['id']},
))
# Query both distributors using all three query forms.
cls.responses['puppet releases'] = []
author_name = PUPPET_MODULE['author'] + '/' + PUPPET_MODULE['name']
for repo in repos:
if selectors.bug_is_untestable(1440, cls.cfg.version):
continue
cls.responses['puppet releases'].append(client.get(
'/api/v1/releases.json',
params={'module': author_name},
auth=('.', repo['id']),
))
cls.responses['puppet releases'].append(client.get(
'/pulp_puppet/forge/repository/{}/api/v1/releases.json'
.format(repo['id']),
params={'module': author_name},
))
if cls.cfg.version < Version('2.8'):
continue
cls.responses['puppet releases'].append(client.get(
'/v3/releases',
params={'module': author_name},
auth=('repository', repo['id']),
))
# Download each unit referenced by the queries above.
for response in cls.responses['puppet releases']:
body = response.json()
if set(body.keys()) == {'pagination', 'results'}: # Puppet >= 3.6
path = body['results'][0]['file_uri']
else:
path = body[author_name][0]['file']
cls.modules.append(client.get(path).content)
# Search for all units in each of the two repositories.
body = {'criteria': {}}
cls.responses['repo units'] = [
client.post(urljoin(repo['_href'], 'search/units/'), body)
for repo in repos
]
def test_all(self):
"""Verify whether content served by pulp can be downloaded.
The process of publishing content is more involved in Pulp 3 than it
was under Pulp 2. Given a repository, the process is as follows:
1. Create a publication from the repository. (The latest repository
version is selected if no version is specified.) A publication is a
repository version plus metadata.
2. Create a distribution from the publication. The distribution defines
at which URLs a publication is available, e.g.
``http://example.com/content/foo/`` and
``http://example.com/content/bar/``.
Do the following:
1. Create, populate, publish, and distribute a repository.
2. Select a random content unit in the distribution. Download that
content unit from Pulp, and verify that the content unit has the
same checksum when fetched directly from Pulp-Fixtures.
This test targets the following issues:
* `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
* `Pulp Smash #872 <https://github.com/PulpQE/pulp-smash/issues/872>`_
"""
cfg = config.get_config()
client = api.Client(cfg, api.json_handler)
repo = client.post(REPO_PATH, gen_repo())
self.addCleanup(client.delete, repo['_href'])
body = gen_rpm_remote()
remote = client.post(RPM_REMOTE_PATH, body)
self.addCleanup(client.delete, remote['_href'])
sync(cfg, remote, repo)
repo = client.get(repo['_href'])
# Create a publication.
publication = publish(cfg, repo)
self.addCleanup(client.delete, publication['_href'])
# Create a distribution.
body = gen_distribution()
body['publication'] = publication['_href']
distribution = client.using_handler(api.task_handler).post(
DISTRIBUTION_PATH, body
)
self.addCleanup(client.delete, distribution['_href'])
# Pick a content unit, and download it from both Pulp Fixtures…
unit_path = choice(get_rpm_package_paths(repo))
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(RPM_UNSIGNED_FIXTURE_URL, unit_path))
).hexdigest()
# …and Pulp.
content = download_content_unit(cfg, distribution, unit_path)
pulp_hash = hashlib.sha256(content).hexdigest()
self.assertEqual(fixtures_hash, pulp_hash)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
delete_orphans(cls.cfg)
cls.client = api.Client(cls.cfg, api.page_handler)
cls.file = {'file': utils.http_get(RPM_UNSIGNED_URL)}
def setUpClass(cls):
"""Upload an RPM to a repo, copy it to another, publish and download.
Do the following:
1. Create two RPM repositories, both without feeds.
2. Upload an RPM to the first repository.
3. Associate the first repository with the second, causing the RPM to
be copied.
4. Add a distributor to both repositories and publish them.
"""
super(UploadRpmTestCase, cls).setUpClass()
utils.reset_pulp(cls.cfg) # See: https://pulp.plan.io/issues/1406
cls.responses = {}
# Download an RPM and create two repositories.
client = api.Client(cls.cfg, api.json_handler)
repos = [client.post(REPOSITORY_PATH, gen_repo()) for _ in range(2)]
for repo in repos:
cls.resources.add(repo['_href'])
client.response_handler = api.safe_handler
cls.rpm = utils.http_get(RPM_SIGNED_URL)
# Begin an upload request, upload an RPM, move the RPM into a
# repository, and end the upload request.
cls.responses['malloc'] = client.post(CONTENT_UPLOAD_PATH)
cls.responses['upload'] = client.put(
urljoin(cls.responses['malloc'].json()['_href'], '0/'),
data=cls.rpm,
)
cls.responses['import'] = client.post(
urljoin(repos[0]['_href'], 'actions/import_upload/'),
{
'unit_key': {},
'unit_type_id': 'rpm',
'upload_id': cls.responses['malloc'].json()['upload_id'],
},
)
cls.responses['free'] = client.delete(
cls.responses['malloc'].json()['_href'],
)
# Copy content from the first repository to the second.
cls.responses['copy'] = client.post(
urljoin(repos[1]['_href'], 'actions/associate/'),
{'source_repo_id': repos[0]['id']}
)
# Add a distributor to and publish both repositories.
cls.responses['distribute'] = []
cls.responses['publish'] = []
for repo in repos:
cls.responses['distribute'].append(client.post(
urljoin(repo['_href'], 'distributors/'),
gen_distributor(),
))
cls.responses['publish'].append(client.post(
urljoin(repo['_href'], 'actions/publish/'),
{'id': cls.responses['distribute'][-1].json()['id']},
))
# Search for all units in each of the two repositories.
body = {'criteria': {}}
cls.responses['repo units'] = [
client.post(urljoin(repo['_href'], 'search/units/'), body)
for repo in repos
]
def test_repo_auto_distribution(self):
"""Test auto distribution of a repository.
This test targets the following issue:
* `Pulp Smash #947 <https://github.com/PulpQE/pulp-smash/issues/947>`_
Do the following:
1. Create a repository that has at least one repository version.
2. Create a publisher.
3. Create a distribution and set the repository and publishera to the
previous created ones.
4. Create a publication using the latest repository version.
5. Assert that the previous distribution has a ``publication`` set as
the one created in step 4.
6. Create a new repository version by adding content to the repository.
7. Create another publication using the just created repository
version.
8. Assert that distribution now has the ``publication`` set to the
publication created in the step 7.
9. Verify that content added in the step 7 is now available to download
from distribution, and verify that the content unit has the same
checksum when fetched directly from Pulp-Fixtures.
"""
self.assertGreaterEqual(len(self.contents), 2, self.contents)
# Create a repository.
repo = self.client.post(REPO_PATH, gen_repo())
self.addCleanup(self.client.delete, repo['_href'])
self.client.post(
repo['_versions_href'],
{'add_content_units': [self.contents[0]['_href']]}
)
repo = self.client.get(repo['_href'])
# Create publisher.
publisher = self.client.post(FILE_PUBLISHER_PATH, gen_publisher())
self.addCleanup(self.client.delete, publisher['_href'])
# Create a distribution
body = gen_distribution()
body['repository'] = repo['_href']
body['publisher'] = publisher['_href']
distribution = self.client.post(DISTRIBUTION_PATH, body)
self.addCleanup(self.client.delete, distribution['_href'])
last_version_href = get_versions(repo)[-1]['_href']
publication = publish(self.cfg, publisher, repo, last_version_href)
self.addCleanup(self.client.delete, publication['_href'])
distribution = self.client.get(distribution['_href'])
# Assert that distribution was updated as per step 5.
self.assertEqual(distribution['publication'], publication['_href'])
# Create a new repository version.
self.client.post(
repo['_versions_href'],
{'add_content_units': [self.contents[1]['_href']]}
)
repo = self.client.get(repo['_href'])
last_version_href = get_versions(repo)[-1]['_href']
publication = publish(self.cfg, publisher, repo, last_version_href)
self.addCleanup(self.client.delete, publication['_href'])
distribution = self.client.get(distribution['_href'])
# Assert that distribution was updated as per step 8.
self.assertEqual(distribution['publication'], publication['_href'])
unit_path = get_added_content(repo, last_version_href)[0]['relative_path']
unit_url = self.cfg.get_hosts('api')[0].roles['api']['scheme']
unit_url += '://' + distribution['base_url'] + '/'
unit_url = urljoin(unit_url, unit_path)
self.client.response_handler = api.safe_handler
pulp_hash = hashlib.sha256(
self.client.get(unit_url).content
).hexdigest()
fixtures_hash = hashlib.sha256(
utils.http_get(urljoin(FILE_URL, unit_path))
).hexdigest()
# Verify checksum. Step 9.
self.assertEqual(fixtures_hash, pulp_hash)
