Пример #1
0
    def test_remove_unit_duplicate_nerva(self, mock_criteria,
                                         mock_association):
        unit_key = {
            'name': 'test-nevra',
            'epoch': 0,
            'version': 1,
            'release': '23',
            'arch': 'noarch',
            'checksum': '1234abc',
            'checksumtype': 'sha256'
        }
        type_id = 'rpm'
        repo_id = 'test-repo'
        expected_filters = set([('arch', 'noarch'), ('epoch', 0),
                                ('name', 'test-nevra'), ('release', '23'),
                                ('version', 1)])

        purge.remove_unit_duplicate_nevra(unit_key, type_id, repo_id)

        # verify
        self.assertEqual(mock_criteria.mock_calls[0][2]['type_ids'], 'rpm')
        self.assertEqual(mock_criteria.mock_calls[0][2]['unit_filters'].keys(),
                         ['$and'])
        result_filters = mock_criteria.mock_calls[0][2]['unit_filters']['$and']
        unit_filters = set()
        for i in result_filters:
            unit_filters.add(i.items()[0])
        self.assertEqual(unit_filters, expected_filters)
        mock_association.unassociate_by_criteria.assert_called_once_with(
            repo_id, mock_criteria.return_value)
Пример #2
0
    def download_succeeded(self, report):
        """
        The callback when a download succeeds.

        :param report: the report for the succeeded download.
        :type  report: nectar.report.DownloadReport
        """
        model = report.data

        try:
            self._verify_size(model, report)
            self._verify_checksum(model, report)
        except verification.VerificationException:
            # The verify methods populates the error details of the progress report.
            # There is also no need to clean up the bad file as the sync will blow away
            # the temp directory after it finishes. Simply punch out so the good unit
            # handling below doesn't run.
            return
        except verification.InvalidChecksumType:
            return

        # these are the only types we store repo metadata snippets on in the DB
        if isinstance(model, (models.RPM, models.SRPM)):
            self.metadata_files.add_repodata(model)

        purge.remove_unit_duplicate_nevra(model, self.sync_conduit.repo)

        model.set_content(report.destination)
        model.save()

        repo_controller.associate_single_unit(self.sync_conduit.repo, model)

        # TODO consider that if an exception occurs before here maybe it shouldn't call success?
        self.progress_report['content'].success(model)
        self.sync_conduit.set_progress(self.progress_report)
Пример #3
0
    def test_remove_unit_duplicate_nerva(self, mock_criteria, mock_association):
        unit_key = {'name': 'test-nevra', 'epoch': 0, 'version': 1, 'release': '23',
                    'arch': 'noarch', 'checksum': '1234abc', 'checksumtype': 'sha256'}
        type_id = 'rpm'
        repo_id = 'test-repo'
        expected_filters = set([('arch', 'noarch'), ('epoch', 0), ('name', 'test-nevra'),
                                ('release', '23'), ('version', 1)])

        purge.remove_unit_duplicate_nevra(unit_key, type_id, repo_id)

        # verify
        self.assertEqual(mock_criteria.mock_calls[0][2]['type_ids'], 'rpm')
        self.assertEqual(mock_criteria.mock_calls[0][2]['unit_filters'].keys(), ['$and'])
        result_filters = mock_criteria.mock_calls[0][2]['unit_filters']['$and']
        unit_filters = set()
        for i in result_filters:
            unit_filters.add(i.items()[0])
        self.assertEqual(unit_filters, expected_filters)
        mock_association.unassociate_by_criteria.assert_called_once_with(
            repo_id,
            mock_criteria.return_value)
Пример #4
0
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
    """
    Handles the upload for an RPM, SRPM or DRPM.

    This inspects the package contents to determine field values. The unit_key
    and metadata fields overwrite field values determined through package inspection.

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository

    :param type_id: The type_id of the package being uploaded
    :type  type_id: str

    :param unit_key: A dictionary of fields to overwrite introspected field values
    :type  unit_key: dict

    :param metadata: A dictionary of fields to overwrite introspected field values, or None
    :type  metadata: dict or None

    :param file_path: The path to the uploaded package
    :type  file_path: str

    :param conduit: provides access to relevant Pulp functionality
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit

    :param config: plugin configuration for the repository
    :type  config: pulp.plugins.config.PluginCallConfiguration

    :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
    :raises PulpCodedException PLP1013: if the checksum value from the user does not validate
    """
    try:
        if type_id == models.DRPM._content_type_id.default:
            rpm_data = _extract_drpm_data(file_path)
        else:
            rpm_data = _extract_rpm_data(type_id, file_path)
    except:
        _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
        raise

    # metadata can be None
    metadata = metadata or {}

    model_class = plugin_api.get_unit_model_by_id(type_id)
    update_fields_inbound(model_class, unit_key or {})
    update_fields_inbound(model_class, metadata or {})

    with open(file_path) as fp:
        sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES)

    # validate checksum if possible
    if metadata.get('checksum'):
        checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
        checksumtype = util.sanitize_checksum_type(checksumtype)
        if checksumtype not in sums:
            raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype)
        if metadata['checksum'] != sums[checksumtype]:
            raise PulpCodedException(error_code=platform_errors.PLP1013)
        _LOGGER.debug(_('Upload checksum matches.'))

    # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
    # types, regardless of what is in the unit key.
    rpm_data['checksumtype'] = util.TYPE_SHA256
    rpm_data['checksum'] = sums[util.TYPE_SHA256]
    # keep all available checksum values on the model
    rpm_data['checksums'] = sums

    # Update the RPM-extracted data with anything additional the user specified.
    # Allow the user-specified values to override the extracted ones.
    rpm_data.update(metadata or {})
    rpm_data.update(unit_key or {})

    # Validate the user specified data by instantiating the model
    try:
        unit = model_class(**rpm_data)
    except TypeError:
        raise ModelInstantiationError()

    if type_id != models.DRPM._content_type_id.default:
        # Extract/adjust the repodata snippets
        repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
        _update_provides_requires(unit, repodata)
        _update_files(unit, repodata)
        unit.modify_xml(repodata)

    # check if the unit has duplicate nevra
    purge.remove_unit_duplicate_nevra(unit, repo)

    unit.set_storage_path(os.path.basename(file_path))
    try:
        unit.save_and_import_content(file_path)
    except NotUniqueError:
        unit = unit.__class__.objects.filter(**unit.unit_key).first()

    if rpm_parse.signature_enabled(config):
        rpm_parse.filter_signature(unit, config)
    repo_controller.associate_single_unit(repo, unit)
Пример #5
0
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
    """
    Handles the upload for an RPM or SRPM.

    This inspects the package contents to determine field values. The unit_key
    and metadata fields overwrite field values determined through package inspection.

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository

    :param type_id: The type_id of the package being uploaded
    :type  type_id: str

    :param unit_key: A dictionary of fields to overwrite introspected field values
    :type  unit_key: dict

    :param metadata: A dictionary of fields to overwrite introspected field values, or None
    :type  metadata: dict or None

    :param file_path: The path to the uploaded package
    :type  file_path: str

    :param conduit: provides access to relevant Pulp functionality
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit

    :param config: plugin configuration for the repository
    :type  config: pulp.plugins.config.PluginCallConfiguration

    :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
    :raises PulpCodedException PLP1013: if the checksum value from the user does not validate
    """
    try:
        rpm_data = _extract_rpm_data(type_id, file_path)
    except:
        _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
        raise

    model_class = plugin_api.get_unit_model_by_id(type_id)
    update_fields_inbound(model_class, unit_key or {})
    update_fields_inbound(model_class, metadata or {})

    # set checksum and checksumtype
    if metadata:
        checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
        rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
        if 'checksum' in metadata:
            rpm_data['checksum'] = metadata.pop('checksum')
            try:
                with open(file_path) as dest_file:
                    verification.verify_checksum(dest_file, rpm_data['checksumtype'],
                                                 rpm_data['checksum'])
            except verification.VerificationException:
                raise PulpCodedException(error_code=platform_errors.PLP1013)
        else:
            rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
    else:
        rpm_data['checksumtype'] = verification.TYPE_SHA256
        rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)

    # Update the RPM-extracted data with anything additional the user specified.
    # Allow the user-specified values to override the extracted ones.
    rpm_data.update(metadata or {})
    rpm_data.update(unit_key or {})

    # Validate the user specified data by instantiating the model
    try:
        unit = model_class(**rpm_data)
    except TypeError:
        raise ModelInstantiationError()

    # Extract/adjust the repodata snippets
    unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
    _update_provides_requires(unit)
    _update_location(unit)

    # check if the unit has duplicate nevra
    purge.remove_unit_duplicate_nevra(unit, repo)

    unit.set_storage_path(os.path.basename(file_path))
    try:
        unit.save_and_import_content(file_path)
    except NotUniqueError:
        unit = unit.__class__.objects.filter(**unit.unit_key).first()

    repo_controller.associate_single_unit(repo, unit)
Пример #6
0
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit,
                    config):
    """
    Handles the upload for an RPM or SRPM.

    This inspects the package contents to determine field values. The unit_key
    and metadata fields overwrite field values determined through package inspection.

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository

    :param type_id: The type_id of the package being uploaded
    :type  type_id: str

    :param unit_key: A dictionary of fields to overwrite introspected field values
    :type  unit_key: dict

    :param metadata: A dictionary of fields to overwrite introspected field values, or None
    :type  metadata: dict or None

    :param file_path: The path to the uploaded package
    :type  file_path: str

    :param conduit: provides access to relevant Pulp functionality
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit

    :param config: plugin configuration for the repository
    :type  config: pulp.plugins.config.PluginCallConfiguration

    :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
    :raises PulpCodedException PLP1013: if the checksum value from the user does not validate
    """
    try:
        rpm_data = _extract_rpm_data(type_id, file_path)
    except:
        _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
        raise

    # metadata can be None
    metadata = metadata or {}

    model_class = plugin_api.get_unit_model_by_id(type_id)
    update_fields_inbound(model_class, unit_key or {})
    update_fields_inbound(model_class, metadata or {})

    with open(file_path) as fp:
        sums = util.calculate_checksums(fp,
                                        models.RpmBase.DEFAULT_CHECKSUM_TYPES)

    # validate checksum if possible
    if metadata.get('checksum'):
        checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
        checksumtype = util.sanitize_checksum_type(checksumtype)
        if checksumtype not in sums:
            raise PulpCodedException(error_code=error_codes.RPM1009,
                                     checksumtype=checksumtype)
        if metadata['checksum'] != sums[checksumtype]:
            raise PulpCodedException(error_code=platform_errors.PLP1013)
        _LOGGER.debug(_('Upload checksum matches.'))

    # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
    # types, regardless of what is in the unit key.
    rpm_data['checksumtype'] = util.TYPE_SHA256
    rpm_data['checksum'] = sums[util.TYPE_SHA256]
    # keep all available checksum values on the model
    rpm_data['checksums'] = sums

    # Update the RPM-extracted data with anything additional the user specified.
    # Allow the user-specified values to override the extracted ones.
    rpm_data.update(metadata or {})
    rpm_data.update(unit_key or {})

    # Validate the user specified data by instantiating the model
    try:
        unit = model_class(**rpm_data)
    except TypeError:
        raise ModelInstantiationError()

    # Extract/adjust the repodata snippets
    unit.repodata = rpm_parse.get_package_xml(file_path,
                                              sumtype=unit.checksumtype)
    _update_provides_requires(unit)
    unit.modify_xml()

    # check if the unit has duplicate nevra
    purge.remove_unit_duplicate_nevra(unit, repo)

    unit.set_storage_path(os.path.basename(file_path))
    try:
        unit.save_and_import_content(file_path)
    except NotUniqueError:
        unit = unit.__class__.objects.filter(**unit.unit_key).first()

    repo_controller.associate_single_unit(repo, unit)
Пример #7
0
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
    """
    Handles the upload for an RPM or SRPM.

    This inspects the package contents to determine field values. The unit_key
    and metadata fields overwrite field values determined through package inspection.

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository

    :param type_id: The type_id of the package being uploaded
    :type  type_id: str

    :param unit_key: A dictionary of fields to overwrite introspected field values
    :type  unit_key: dict

    :param metadata: A dictionary of fields to overwrite introspected field values, or None
    :type  metadata: dict or None

    :param file_path: The path to the uploaded package
    :type  file_path: str

    :param conduit: provides access to relevant Pulp functionality
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit

    :param config: plugin configuration for the repository
    :type  config: pulp.plugins.config.PluginCallConfiguration

    :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
    :raises PulpCodedException PLP1013: if the checksum value from the user does not validate
    """
    try:
        rpm_data = _extract_rpm_data(type_id, file_path)
    except:
        _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
        raise

    model_class = plugin_api.get_unit_model_by_id(type_id)
    update_fields_inbound(model_class, unit_key or {})
    update_fields_inbound(model_class, metadata or {})

    # set checksum and checksumtype
    if metadata:
        checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
        rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
        if 'checksum' in metadata:
            rpm_data['checksum'] = metadata.pop('checksum')
            try:
                with open(file_path) as dest_file:
                    verification.verify_checksum(dest_file, rpm_data['checksumtype'],
                                                 rpm_data['checksum'])
            except verification.VerificationException:
                raise PulpCodedException(error_code=platform_errors.PLP1013)
        else:
            rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
    else:
        rpm_data['checksumtype'] = verification.TYPE_SHA256
        rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)

    # Update the RPM-extracted data with anything additional the user specified.
    # Allow the user-specified values to override the extracted ones.
    rpm_data.update(metadata or {})
    rpm_data.update(unit_key or {})

    # Validate the user specified data by instantiating the model
    try:
        unit = model_class(**rpm_data)
    except TypeError:
        raise ModelInstantiationError()

    # Extract the repodata snippets
    unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
    _update_provides_requires(unit)

    # check if the unit has duplicate nevra
    purge.remove_unit_duplicate_nevra(unit, repo)

    unit.set_storage_path(os.path.basename(file_path))
    unit.save_and_import_content(file_path)

    repo_controller.associate_single_unit(repo, unit)
Пример #8
0
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit,
                    config):
    """
    Handles the upload for an RPM, SRPM or DRPM.

    This inspects the package contents to determine field values. The unit_key
    and metadata fields overwrite field values determined through package inspection.

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository

    :param type_id: The type_id of the package being uploaded
    :type  type_id: str

    :param unit_key: A dictionary of fields to overwrite introspected field values
    :type  unit_key: dict

    :param metadata: A dictionary of fields to overwrite introspected field values, or None
    :type  metadata: dict or None

    :param file_path: The path to the uploaded package
    :type  file_path: str

    :param conduit: provides access to relevant Pulp functionality
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit

    :param config: plugin configuration for the repository
    :type  config: pulp.plugins.config.PluginCallConfiguration

    :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
    :raises PulpCodedException PLP1013: if the checksum value from the user does not validate
    """
    try:
        if type_id == models.DRPM._content_type_id.default:
            unit = models.DRPM(**_extract_drpm_data(file_path))
        else:
            repodata = rpm_parse.get_package_xml(file_path,
                                                 sumtype=util.TYPE_SHA256)
            package_xml = (utils.fake_xml_element(
                repodata['primary'],
                constants.COMMON_NAMESPACE).find(primary.PACKAGE_TAG))
            unit = primary.process_package_element(package_xml)
    except Exception:
        raise PulpCodedException(error_codes.RPM1016)

    # metadata can be None
    metadata = metadata or {}

    model_class = plugin_api.get_unit_model_by_id(type_id)
    update_fields_inbound(model_class, unit_key or {})
    update_fields_inbound(model_class, metadata or {})

    with open(file_path) as fp:
        sums = util.calculate_checksums(fp,
                                        models.RpmBase.DEFAULT_CHECKSUM_TYPES)

    # validate checksum if possible
    if metadata.get('checksum'):
        checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256)
        checksumtype = util.sanitize_checksum_type(checksumtype)
        if checksumtype not in sums:
            raise PulpCodedException(error_code=error_codes.RPM1009,
                                     checksumtype=checksumtype)
        if metadata['checksum'] != sums[checksumtype]:
            raise PulpCodedException(error_code=platform_errors.PLP1013)
        _LOGGER.debug(_('Upload checksum matches.'))

    # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other
    # types, regardless of what is in the unit key.
    unit.checksumtype = util.TYPE_SHA256
    unit.checksum = sums[util.TYPE_SHA256]
    # keep all available checksum values on the model
    unit.checksums = sums

    # Update the RPM-extracted data with anything additional the user specified.
    # Allow the user-specified values to override the extracted ones.
    for key, value in metadata.items():
        setattr(unit, key, value)
    for key, value in unit_key.items():
        setattr(unit, key, value)

    if type_id != models.DRPM._content_type_id.default:
        # Extract/adjust the repodata snippets
        unit.signing_key = rpm_parse.package_signature(
            rpm_parse.package_headers(file_path))
        # construct filename from metadata (BZ #1101168)
        if type_id == models.SRPM._content_type_id.default:
            rpm_basefilename = "%s-%s-%s.src.rpm" % (unit.name, unit.version,
                                                     unit.release)
        else:
            rpm_basefilename = "%s-%s-%s.%s.rpm" % (unit.name, unit.version,
                                                    unit.release, unit.arch)
        unit.relativepath = rpm_basefilename
        unit.filename = rpm_basefilename
        _update_files(unit, repodata)
        unit.modify_xml(repodata)

    # check if the unit has duplicate nevra
    purge.remove_unit_duplicate_nevra(unit, repo)

    unit.set_storage_path(os.path.basename(file_path))
    try:
        unit.save_and_import_content(file_path)
    except TypeError:
        raise ModelInstantiationError()
    except NotUniqueError:
        unit = unit.__class__.objects.filter(**unit.unit_key).first()

    if rpm_parse.signature_enabled(config):
        rpm_parse.filter_signature(unit, config)
    repo_controller.associate_single_unit(repo, unit)