def test_remove_unit_duplicate_nerva(self, mock_criteria,
mock_association):
unit_key = {
'name': 'test-nevra',
'epoch': 0,
'version': 1,
'release': '23',
'arch': 'noarch',
'checksum': '1234abc',
'checksumtype': 'sha256'
}
type_id = 'rpm'
repo_id = 'test-repo'
expected_filters = set([('arch', 'noarch'), ('epoch', 0),
('name', 'test-nevra'), ('release', '23'),
('version', 1)])
purge.remove_unit_duplicate_nevra(unit_key, type_id, repo_id)
# verify
self.assertEqual(mock_criteria.mock_calls[0][2]['type_ids'], 'rpm')
self.assertEqual(mock_criteria.mock_calls[0][2]['unit_filters'].keys(),
['$and'])
result_filters = mock_criteria.mock_calls[0][2]['unit_filters']['$and']
unit_filters = set()
for i in result_filters:
unit_filters.add(i.items()[0])
self.assertEqual(unit_filters, expected_filters)
mock_association.unassociate_by_criteria.assert_called_once_with(
repo_id, mock_criteria.return_value)
def download_succeeded(self, report):
"""
The callback when a download succeeds.
:param report: the report for the succeeded download.
:type report: nectar.report.DownloadReport
"""
model = report.data
try:
self._verify_size(model, report)
self._verify_checksum(model, report)
except verification.VerificationException:
# The verify methods populates the error details of the progress report.
# There is also no need to clean up the bad file as the sync will blow away
# the temp directory after it finishes. Simply punch out so the good unit
# handling below doesn't run.
return
except verification.InvalidChecksumType:
return
# these are the only types we store repo metadata snippets on in the DB
if isinstance(model, (models.RPM, models.SRPM)):
self.metadata_files.add_repodata(model)
purge.remove_unit_duplicate_nevra(model, self.sync_conduit.repo)
model.set_content(report.destination)
model.save()
repo_controller.associate_single_unit(self.sync_conduit.repo, model)
# TODO consider that if an exception occurs before here maybe it shouldn't call success?
self.progress_report['content'].success(model)
self.sync_conduit.set_progress(self.progress_report)
def test_remove_unit_duplicate_nerva(self, mock_criteria, mock_association):
unit_key = {'name': 'test-nevra', 'epoch': 0, 'version': 1, 'release': '23',
'arch': 'noarch', 'checksum': '1234abc', 'checksumtype': 'sha256'}
type_id = 'rpm'
repo_id = 'test-repo'
expected_filters = set([('arch', 'noarch'), ('epoch', 0), ('name', 'test-nevra'),
('release', '23'), ('version', 1)])
purge.remove_unit_duplicate_nevra(unit_key, type_id, repo_id)
# verify
self.assertEqual(mock_criteria.mock_calls[0][2]['type_ids'], 'rpm')
self.assertEqual(mock_criteria.mock_calls[0][2]['unit_filters'].keys(), ['$and'])
result_filters = mock_criteria.mock_calls[0][2]['unit_filters']['$and']
unit_filters = set()
for i in result_filters:
unit_filters.add(i.items()[0])
self.assertEqual(unit_filters, expected_filters)
mock_association.unassociate_by_criteria.assert_called_once_with(
repo_id,
mock_criteria.return_value)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM, SRPM or DRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
if type_id == models.DRPM._content_type_id.default:
rpm_data = _extract_drpm_data(file_path)
else:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
# metadata can be None
metadata = metadata or {}
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
with open(file_path) as fp:
sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES)
# validate checksum if possible
if metadata.get('checksum'):
checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
checksumtype = util.sanitize_checksum_type(checksumtype)
if checksumtype not in sums:
raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype)
if metadata['checksum'] != sums[checksumtype]:
raise PulpCodedException(error_code=platform_errors.PLP1013)
_LOGGER.debug(_('Upload checksum matches.'))
# Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
# types, regardless of what is in the unit key.
rpm_data['checksumtype'] = util.TYPE_SHA256
rpm_data['checksum'] = sums[util.TYPE_SHA256]
# keep all available checksum values on the model
rpm_data['checksums'] = sums
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
if type_id != models.DRPM._content_type_id.default:
# Extract/adjust the repodata snippets
repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit, repodata)
_update_files(unit, repodata)
unit.modify_xml(repodata)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
if rpm_parse.signature_enabled(config):
rpm_parse.filter_signature(unit, config)
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract/adjust the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
_update_location(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit,
config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
# metadata can be None
metadata = metadata or {}
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
with open(file_path) as fp:
sums = util.calculate_checksums(fp,
models.RpmBase.DEFAULT_CHECKSUM_TYPES)
# validate checksum if possible
if metadata.get('checksum'):
checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
checksumtype = util.sanitize_checksum_type(checksumtype)
if checksumtype not in sums:
raise PulpCodedException(error_code=error_codes.RPM1009,
checksumtype=checksumtype)
if metadata['checksum'] != sums[checksumtype]:
raise PulpCodedException(error_code=platform_errors.PLP1013)
_LOGGER.debug(_('Upload checksum matches.'))
# Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
# types, regardless of what is in the unit key.
rpm_data['checksumtype'] = util.TYPE_SHA256
rpm_data['checksum'] = sums[util.TYPE_SHA256]
# keep all available checksum values on the model
rpm_data['checksums'] = sums
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract/adjust the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path,
sumtype=unit.checksumtype)
_update_provides_requires(unit)
unit.modify_xml()
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
unit.save_and_import_content(file_path)
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit,
config):
"""
Handles the upload for an RPM, SRPM or DRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
if type_id == models.DRPM._content_type_id.default:
unit = models.DRPM(**_extract_drpm_data(file_path))
else:
repodata = rpm_parse.get_package_xml(file_path,
sumtype=util.TYPE_SHA256)
package_xml = (utils.fake_xml_element(
repodata['primary'],
constants.COMMON_NAMESPACE).find(primary.PACKAGE_TAG))
unit = primary.process_package_element(package_xml)
except Exception:
raise PulpCodedException(error_codes.RPM1016)
# metadata can be None
metadata = metadata or {}
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
with open(file_path) as fp:
sums = util.calculate_checksums(fp,
models.RpmBase.DEFAULT_CHECKSUM_TYPES)
# validate checksum if possible
if metadata.get('checksum'):
checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
checksumtype = util.sanitize_checksum_type(checksumtype)
if checksumtype not in sums:
raise PulpCodedException(error_code=error_codes.RPM1009,
checksumtype=checksumtype)
if metadata['checksum'] != sums[checksumtype]:
raise PulpCodedException(error_code=platform_errors.PLP1013)
_LOGGER.debug(_('Upload checksum matches.'))
# Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
# types, regardless of what is in the unit key.
unit.checksumtype = util.TYPE_SHA256
unit.checksum = sums[util.TYPE_SHA256]
# keep all available checksum values on the model
unit.checksums = sums
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
for key, value in metadata.items():
setattr(unit, key, value)
for key, value in unit_key.items():
setattr(unit, key, value)
if type_id != models.DRPM._content_type_id.default:
# Extract/adjust the repodata snippets
unit.signing_key = rpm_parse.package_signature(
rpm_parse.package_headers(file_path))
# construct filename from metadata (BZ #1101168)
if type_id == models.SRPM._content_type_id.default:
rpm_basefilename = "%s-%s-%s.src.rpm" % (unit.name, unit.version,
unit.release)
else:
rpm_basefilename = "%s-%s-%s.%s.rpm" % (unit.name, unit.version,
unit.release, unit.arch)
unit.relativepath = rpm_basefilename
unit.filename = rpm_basefilename
_update_files(unit, repodata)
unit.modify_xml(repodata)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except TypeError:
raise ModelInstantiationError()
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
if rpm_parse.signature_enabled(config):
rpm_parse.filter_signature(unit, config)
repo_controller.associate_single_unit(repo, unit)