def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM or SRPM. For these types, the unit_key and metadata will only contain additions the user wishes to add. The typical use case is that the file is uploaded and all of the necessary data, both unit key and metadata, are extracted in this method. :type repo: pulp.plugins.model.Repository :type type_id: str :type unit_key: dict :type metadata: dict or None :type file_path: str :type conduit: pulp.plugins.conduits.upload.UploadConduit :type config: pulp.plugins.config.PluginCallConfiguration """ # Extract the RPM key and metadata try: new_unit_key, new_unit_metadata = _generate_rpm_data( type_id, file_path, metadata) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. new_unit_key.update(unit_key or {}) new_unit_metadata.update(metadata or {}) # Validate the user specified data by instantiating the model try: model_class = models.TYPE_MAP[type_id] model = model_class(metadata=new_unit_metadata, **new_unit_key) except TypeError: raise ModelInstantiationError() # Move the file to its final storage location in Pulp try: unit = conduit.init_unit(model.TYPE, model.unit_key, model.metadata, model.relative_path) shutil.move(file_path, unit.storage_path) except IOError: raise StoreFileError() # Extract the repodata snippets unit.metadata['repodata'] = rpm_parse.get_package_xml( unit.storage_path, sumtype=new_unit_key['checksumtype']) _update_provides_requires(unit) # Save the unit in Pulp conduit.save_unit(unit)
def _handle_package(type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM or SRPM. For these types, the unit_key and metadata will only contain additions the user wishes to add. The typical use case is that the file is uploaded and all of the necessary data, both unit key and metadata, are extracted in this method. :type type_id: str :type unit_key: dict :type metadata: dict or None :type file_path: str :type conduit: pulp.plugins.conduits.upload.UploadConduit :type config: pulp.plugins.config.PluginCallConfiguration """ # Extract the RPM key and metadata try: new_unit_key, new_unit_metadata = _generate_rpm_data(file_path, metadata) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise PackageMetadataError() # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. new_unit_key.update(unit_key or {}) new_unit_metadata.update(metadata or {}) # Validate the user specified data by instantiating the model try: model_class = models.TYPE_MAP[type_id] model = model_class(metadata=new_unit_metadata, **new_unit_key) except TypeError: raise ModelInstantiationError() # Move the file to its final storage location in Pulp try: unit = conduit.init_unit(model.TYPE, model.unit_key, model.metadata, model.relative_path) shutil.move(file_path, unit.storage_path) except IOError: raise StoreFileError() # Extract the repodata snippets unit.metadata['repodata'] = rpm_parse.get_package_xml(unit.storage_path, sumtype=new_unit_key['checksumtype']) _update_provides_requires(unit) # Save the unit in Pulp conduit.save_unit(unit)
def test_get_package_xml(self, change_location_tag, yumbased): package = Mock() package.xml_dump_primary_metadata.return_value = self.PRIMARY yumbased.CreateRepoPackage.return_value = package rel_path = 'a/b/c/d/rabbit.rpm' # test md = rpm.get_package_xml(rel_path) # validation change_location_tag.assert_called_once_with( self.PRIMARY.decode('utf-8', 'replace'), rel_path) self.assertEqual(md['primary'], change_location_tag.return_value.encode('utf-8')) self.assertEqual(md['filelists'], package.xml_dump_filelists_metadata.return_value) self.assertEqual(md['other'], package.xml_dump_other_metadata.return_value)
def test_get_package_xml(self, change_location_tag, yumbased): package = Mock() package.xml_dump_primary_metadata.return_value = self.PRIMARY yumbased.CreateRepoPackage.return_value = package rel_path = 'a/b/c/d/rabbit.rpm' # test md = rpm.get_package_xml(rel_path) # validation change_location_tag.assert_called_once_with( self.PRIMARY.decode('utf-8', 'replace'), rel_path) self.assertEqual(md['primary'], change_location_tag.return_value.encode('utf-8')) self.assertEqual(md['filelists'], package.xml_dump_filelists_metadata.return_value) self.assertEqual(md['other'], package.xml_dump_other_metadata.return_value)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM, SRPM or DRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: if type_id == models.DRPM._content_type_id.default: rpm_data = _extract_drpm_data(file_path) else: rpm_data = _extract_rpm_data(type_id, file_path) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise # metadata can be None metadata = metadata or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. rpm_data['checksumtype'] = util.TYPE_SHA256 rpm_data['checksum'] = sums[util.TYPE_SHA256] # keep all available checksum values on the model rpm_data['checksums'] = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. rpm_data.update(metadata or {}) rpm_data.update(unit_key or {}) # Validate the user specified data by instantiating the model try: unit = model_class(**rpm_data) except TypeError: raise ModelInstantiationError() if type_id != models.DRPM._content_type_id.default: # Extract/adjust the repodata snippets repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype) _update_provides_requires(unit, repodata) _update_files(unit, repodata) unit.modify_xml(repodata) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() if rpm_parse.signature_enabled(config): rpm_parse.filter_signature(unit, config) repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM or SRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: rpm_data = _extract_rpm_data(type_id, file_path) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) # set checksum and checksumtype if metadata: checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256) rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype) if 'checksum' in metadata: rpm_data['checksum'] = metadata.pop('checksum') try: with open(file_path) as dest_file: verification.verify_checksum(dest_file, rpm_data['checksumtype'], rpm_data['checksum']) except verification.VerificationException: raise PulpCodedException(error_code=platform_errors.PLP1013) else: rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path) else: rpm_data['checksumtype'] = verification.TYPE_SHA256 rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path) # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. rpm_data.update(metadata or {}) rpm_data.update(unit_key or {}) # Validate the user specified data by instantiating the model try: unit = model_class(**rpm_data) except TypeError: raise ModelInstantiationError() # Extract/adjust the repodata snippets unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype) _update_provides_requires(unit) _update_location(unit) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM or SRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: rpm_data = _extract_rpm_data(type_id, file_path) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise # metadata can be None metadata = metadata or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. rpm_data['checksumtype'] = util.TYPE_SHA256 rpm_data['checksum'] = sums[util.TYPE_SHA256] # keep all available checksum values on the model rpm_data['checksums'] = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. rpm_data.update(metadata or {}) rpm_data.update(unit_key or {}) # Validate the user specified data by instantiating the model try: unit = model_class(**rpm_data) except TypeError: raise ModelInstantiationError() # Extract/adjust the repodata snippets unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype) _update_provides_requires(unit) unit.modify_xml() # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM or SRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: rpm_data = _extract_rpm_data(type_id, file_path) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) # set checksum and checksumtype if metadata: checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256) rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype) if 'checksum' in metadata: rpm_data['checksum'] = metadata.pop('checksum') try: with open(file_path) as dest_file: verification.verify_checksum(dest_file, rpm_data['checksumtype'], rpm_data['checksum']) except verification.VerificationException: raise PulpCodedException(error_code=platform_errors.PLP1013) else: rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path) else: rpm_data['checksumtype'] = verification.TYPE_SHA256 rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path) # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. rpm_data.update(metadata or {}) rpm_data.update(unit_key or {}) # Validate the user specified data by instantiating the model try: unit = model_class(**rpm_data) except TypeError: raise ModelInstantiationError() # Extract the repodata snippets unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype) _update_provides_requires(unit) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) unit.save_and_import_content(file_path) repo_controller.associate_single_unit(repo, unit)
def test_get_package_xml_yum_exception(self, mock_yumbased): mock_yumbased.CreateRepoPackage.side_effect = Exception() result = rpm.get_package_xml("/bad/package/path") util.compare_dict(result, {})
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM, SRPM or DRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: if type_id == models.DRPM._content_type_id.default: unit = models.DRPM(**_extract_drpm_data(file_path)) else: repodata = rpm_parse.get_package_xml(file_path, sumtype=util.TYPE_SHA256) package_xml = (utils.fake_xml_element( repodata['primary'], constants.COMMON_NAMESPACE).find(primary.PACKAGE_TAG)) unit = primary.process_package_element(package_xml) except Exception: raise PulpCodedException(error_codes.RPM1016) # metadata can be None metadata = metadata or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. unit.checksumtype = util.TYPE_SHA256 unit.checksum = sums[util.TYPE_SHA256] # keep all available checksum values on the model unit.checksums = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. for key, value in metadata.items(): setattr(unit, key, value) for key, value in unit_key.items(): setattr(unit, key, value) if type_id != models.DRPM._content_type_id.default: # Extract/adjust the repodata snippets unit.signing_key = rpm_parse.package_signature( rpm_parse.package_headers(file_path)) # construct filename from metadata (BZ #1101168) if type_id == models.SRPM._content_type_id.default: rpm_basefilename = "%s-%s-%s.src.rpm" % (unit.name, unit.version, unit.release) else: rpm_basefilename = "%s-%s-%s.%s.rpm" % (unit.name, unit.version, unit.release, unit.arch) unit.relativepath = rpm_basefilename unit.filename = rpm_basefilename _update_files(unit, repodata) unit.modify_xml(repodata) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except TypeError: raise ModelInstantiationError() except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() if rpm_parse.signature_enabled(config): rpm_parse.filter_signature(unit, config) repo_controller.associate_single_unit(repo, unit)
def upload(repo, type_id, unit_key, metadata, file_path, conduit, config): """ :param repo: metadata describing the repository :type repo: pulp.plugins.model.Repository :param type_id: type of unit being uploaded :type type_id: str :param unit_key: identifier for the unit, specified by the user :type unit_key: dict :param metadata: any user-specified metadata for the unit :type metadata: dict :param file_path: path on the Pulp server's filesystem to the temporary location of the uploaded file; may be None in the event that a unit is comprised entirely of metadata and has no bits associated :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :return: report of the details of the sync :rtype: dict """ if type_id not in (models.RPM.TYPE, models.SRPM.TYPE, models.PackageGroup.TYPE, models.PackageCategory.TYPE, models.Errata.TYPE): return _fail_report('%s is not a supported type for upload' % type_id) model_type = models.TYPE_MAP[type_id] # get metadata try: model = model_type(metadata=metadata, **unit_key) except TypeError: return _fail_report('invalid unit key or metadata') # not all models have a relative path relative_path = getattr(model, 'relative_path', '') # both of the below operations perform IO try: # init unit unit = conduit.init_unit(model.TYPE, model.unit_key, model.metadata, relative_path) # move file to destination if file_path and unit.storage_path: shutil.move(file_path, unit.storage_path) except IOError: return _fail_report('failed to move file to destination') # do this for RPMs and SRPMs if isinstance(model, models.RPM): # TODO: replace this call with something that doesn't use yum model.metadata['repodata'] = rpm.get_package_xml(unit.storage_path) # The provides and requires are not provided by the client, so extract them now _update_provides_requires(model) # save unit conduit.save_unit(unit) if type_id == models.Errata.TYPE: link_errata_to_rpms(conduit, model, unit) report = {'success_flag': True, 'summary': '', 'details': {}} return report