def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit,
config):
"""
Handles the upload for an RPM or SRPM. For these types, the unit_key
and metadata will only contain additions the user wishes to add. The
typical use case is that the file is uploaded and all of the necessary
data, both unit key and metadata, are extracted in this method.
:type repo: pulp.plugins.model.Repository
:type type_id: str
:type unit_key: dict
:type metadata: dict or None
:type file_path: str
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:type config: pulp.plugins.config.PluginCallConfiguration
"""
# Extract the RPM key and metadata
try:
new_unit_key, new_unit_metadata = _generate_rpm_data(
type_id, file_path, metadata)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
new_unit_key.update(unit_key or {})
new_unit_metadata.update(metadata or {})
# Validate the user specified data by instantiating the model
try:
model_class = models.TYPE_MAP[type_id]
model = model_class(metadata=new_unit_metadata, **new_unit_key)
except TypeError:
raise ModelInstantiationError()
# Move the file to its final storage location in Pulp
try:
unit = conduit.init_unit(model.TYPE, model.unit_key, model.metadata,
model.relative_path)
shutil.move(file_path, unit.storage_path)
except IOError:
raise StoreFileError()
# Extract the repodata snippets
unit.metadata['repodata'] = rpm_parse.get_package_xml(
unit.storage_path, sumtype=new_unit_key['checksumtype'])
_update_provides_requires(unit)
# Save the unit in Pulp
conduit.save_unit(unit)
def _handle_package(type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM. For these types, the unit_key
and metadata will only contain additions the user wishes to add. The
typical use case is that the file is uploaded and all of the necessary
data, both unit key and metadata, are extracted in this method.
:type type_id: str
:type unit_key: dict
:type metadata: dict or None
:type file_path: str
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:type config: pulp.plugins.config.PluginCallConfiguration
"""
# Extract the RPM key and metadata
try:
new_unit_key, new_unit_metadata = _generate_rpm_data(file_path, metadata)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise PackageMetadataError()
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
new_unit_key.update(unit_key or {})
new_unit_metadata.update(metadata or {})
# Validate the user specified data by instantiating the model
try:
model_class = models.TYPE_MAP[type_id]
model = model_class(metadata=new_unit_metadata, **new_unit_key)
except TypeError:
raise ModelInstantiationError()
# Move the file to its final storage location in Pulp
try:
unit = conduit.init_unit(model.TYPE, model.unit_key,
model.metadata, model.relative_path)
shutil.move(file_path, unit.storage_path)
except IOError:
raise StoreFileError()
# Extract the repodata snippets
unit.metadata['repodata'] = rpm_parse.get_package_xml(unit.storage_path,
sumtype=new_unit_key['checksumtype'])
_update_provides_requires(unit)
# Save the unit in Pulp
conduit.save_unit(unit)
def test_get_package_xml(self, change_location_tag, yumbased):
package = Mock()
package.xml_dump_primary_metadata.return_value = self.PRIMARY
yumbased.CreateRepoPackage.return_value = package
rel_path = 'a/b/c/d/rabbit.rpm'
# test
md = rpm.get_package_xml(rel_path)
# validation
change_location_tag.assert_called_once_with(
self.PRIMARY.decode('utf-8', 'replace'),
rel_path)
self.assertEqual(md['primary'], change_location_tag.return_value.encode('utf-8'))
self.assertEqual(md['filelists'], package.xml_dump_filelists_metadata.return_value)
self.assertEqual(md['other'], package.xml_dump_other_metadata.return_value)
def test_get_package_xml(self, change_location_tag, yumbased):
package = Mock()
package.xml_dump_primary_metadata.return_value = self.PRIMARY
yumbased.CreateRepoPackage.return_value = package
rel_path = 'a/b/c/d/rabbit.rpm'
# test
md = rpm.get_package_xml(rel_path)
# validation
change_location_tag.assert_called_once_with(
self.PRIMARY.decode('utf-8', 'replace'), rel_path)
self.assertEqual(md['primary'],
change_location_tag.return_value.encode('utf-8'))
self.assertEqual(md['filelists'],
package.xml_dump_filelists_metadata.return_value)
self.assertEqual(md['other'],
package.xml_dump_other_metadata.return_value)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM, SRPM or DRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
if type_id == models.DRPM._content_type_id.default:
rpm_data = _extract_drpm_data(file_path)
else:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
# metadata can be None
metadata = metadata or {}
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
with open(file_path) as fp:
sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES)
# validate checksum if possible
if metadata.get('checksum'):
checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
checksumtype = util.sanitize_checksum_type(checksumtype)
if checksumtype not in sums:
raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype)
if metadata['checksum'] != sums[checksumtype]:
raise PulpCodedException(error_code=platform_errors.PLP1013)
_LOGGER.debug(_('Upload checksum matches.'))
# Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
# types, regardless of what is in the unit key.
rpm_data['checksumtype'] = util.TYPE_SHA256
rpm_data['checksum'] = sums[util.TYPE_SHA256]
# keep all available checksum values on the model
rpm_data['checksums'] = sums
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
if type_id != models.DRPM._content_type_id.default:
# Extract/adjust the repodata snippets
repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit, repodata)
_update_files(unit, repodata)
unit.modify_xml(repodata)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
if rpm_parse.signature_enabled(config):
rpm_parse.filter_signature(unit, config)
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract/adjust the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
_update_location(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit,
config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
# metadata can be None
metadata = metadata or {}
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
with open(file_path) as fp:
sums = util.calculate_checksums(fp,
models.RpmBase.DEFAULT_CHECKSUM_TYPES)
# validate checksum if possible
if metadata.get('checksum'):
checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
checksumtype = util.sanitize_checksum_type(checksumtype)
if checksumtype not in sums:
raise PulpCodedException(error_code=error_codes.RPM1009,
checksumtype=checksumtype)
if metadata['checksum'] != sums[checksumtype]:
raise PulpCodedException(error_code=platform_errors.PLP1013)
_LOGGER.debug(_('Upload checksum matches.'))
# Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
# types, regardless of what is in the unit key.
rpm_data['checksumtype'] = util.TYPE_SHA256
rpm_data['checksum'] = sums[util.TYPE_SHA256]
# keep all available checksum values on the model
rpm_data['checksums'] = sums
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract/adjust the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path,
sumtype=unit.checksumtype)
_update_provides_requires(unit)
unit.modify_xml()
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
unit.save_and_import_content(file_path)
repo_controller.associate_single_unit(repo, unit)
def test_get_package_xml_yum_exception(self, mock_yumbased):
mock_yumbased.CreateRepoPackage.side_effect = Exception()
result = rpm.get_package_xml("/bad/package/path")
util.compare_dict(result, {})
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit,
config):
"""
Handles the upload for an RPM, SRPM or DRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
if type_id == models.DRPM._content_type_id.default:
unit = models.DRPM(**_extract_drpm_data(file_path))
else:
repodata = rpm_parse.get_package_xml(file_path,
sumtype=util.TYPE_SHA256)
package_xml = (utils.fake_xml_element(
repodata['primary'],
constants.COMMON_NAMESPACE).find(primary.PACKAGE_TAG))
unit = primary.process_package_element(package_xml)
except Exception:
raise PulpCodedException(error_codes.RPM1016)
# metadata can be None
metadata = metadata or {}
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
with open(file_path) as fp:
sums = util.calculate_checksums(fp,
models.RpmBase.DEFAULT_CHECKSUM_TYPES)
# validate checksum if possible
if metadata.get('checksum'):
checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
checksumtype = util.sanitize_checksum_type(checksumtype)
if checksumtype not in sums:
raise PulpCodedException(error_code=error_codes.RPM1009,
checksumtype=checksumtype)
if metadata['checksum'] != sums[checksumtype]:
raise PulpCodedException(error_code=platform_errors.PLP1013)
_LOGGER.debug(_('Upload checksum matches.'))
# Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
# types, regardless of what is in the unit key.
unit.checksumtype = util.TYPE_SHA256
unit.checksum = sums[util.TYPE_SHA256]
# keep all available checksum values on the model
unit.checksums = sums
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
for key, value in metadata.items():
setattr(unit, key, value)
for key, value in unit_key.items():
setattr(unit, key, value)
if type_id != models.DRPM._content_type_id.default:
# Extract/adjust the repodata snippets
unit.signing_key = rpm_parse.package_signature(
rpm_parse.package_headers(file_path))
# construct filename from metadata (BZ #1101168)
if type_id == models.SRPM._content_type_id.default:
rpm_basefilename = "%s-%s-%s.src.rpm" % (unit.name, unit.version,
unit.release)
else:
rpm_basefilename = "%s-%s-%s.%s.rpm" % (unit.name, unit.version,
unit.release, unit.arch)
unit.relativepath = rpm_basefilename
unit.filename = rpm_basefilename
_update_files(unit, repodata)
unit.modify_xml(repodata)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except TypeError:
raise ModelInstantiationError()
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
if rpm_parse.signature_enabled(config):
rpm_parse.filter_signature(unit, config)
repo_controller.associate_single_unit(repo, unit)
def upload(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
:param repo: metadata describing the repository
:type repo: pulp.plugins.model.Repository
:param type_id: type of unit being uploaded
:type type_id: str
:param unit_key: identifier for the unit, specified by the user
:type unit_key: dict
:param metadata: any user-specified metadata for the unit
:type metadata: dict
:param file_path: path on the Pulp server's filesystem to the temporary
location of the uploaded file; may be None in the event that a
unit is comprised entirely of metadata and has no bits associated
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:return: report of the details of the sync
:rtype: dict
"""
if type_id not in (models.RPM.TYPE, models.SRPM.TYPE, models.PackageGroup.TYPE,
models.PackageCategory.TYPE, models.Errata.TYPE):
return _fail_report('%s is not a supported type for upload' % type_id)
model_type = models.TYPE_MAP[type_id]
# get metadata
try:
model = model_type(metadata=metadata, **unit_key)
except TypeError:
return _fail_report('invalid unit key or metadata')
# not all models have a relative path
relative_path = getattr(model, 'relative_path', '')
# both of the below operations perform IO
try:
# init unit
unit = conduit.init_unit(model.TYPE, model.unit_key, model.metadata, relative_path)
# move file to destination
if file_path and unit.storage_path:
shutil.move(file_path, unit.storage_path)
except IOError:
return _fail_report('failed to move file to destination')
# do this for RPMs and SRPMs
if isinstance(model, models.RPM):
# TODO: replace this call with something that doesn't use yum
model.metadata['repodata'] = rpm.get_package_xml(unit.storage_path)
# The provides and requires are not provided by the client, so extract them now
_update_provides_requires(model)
# save unit
conduit.save_unit(unit)
if type_id == models.Errata.TYPE:
link_errata_to_rpms(conduit, model, unit)
report = {'success_flag': True, 'summary': '', 'details': {}}
return report
