def signature_filter_passed(self, unit): """ Decide whether to associate unit or not based on its signature. :param unit: A content unit. :type unit: pulp_rpm.plugins.db.models.RpmBase :rtype: bool :return: True if unit passes the signature filter and has to be associated """ if rpm_parse.signature_enabled(self.config): try: rpm_parse.filter_signature(unit, self.config) except PulpCodedException as e: _logger.debug(e) error_report = { constants.NAME: unit.filename, constants.ERROR_CODE: constants.ERROR_KEY_ID_FILTER, } self.progress_report['content'].failure(unit, error_report) self.conduit.set_progress(self.progress_report) return False return True
def test_invalid_package_signature(self): unit = Mock() unit.signing_key = '12345678' config = {"allowed_keys": ['87654321']} with self.assertRaises(PulpCodedException) as cm: rpm.filter_signature(unit, config) self.assertEqual(cm.exception.error_code.code, 'RPM1014')
def test_reject_unsigned_packages(self): unit = Mock() unit.signing_key = None config = {"require_signature": True} with self.assertRaises(PulpCodedException) as cm: rpm.filter_signature(unit, config) self.assertEqual(cm.exception.error_code.code, 'RPM1013')
def _associate_unit(dest_repo, unit, config): """ Associate one particular unit with the destination repository. There are behavioral exceptions based on type: Group, Category, Environment and Yum Metadata File units need to have their "repo_id" attribute set. RPMs are convenient to do all as one block, for the purpose of dependency resolution. So this method skips RPMs and lets them be done together by other means :param dest_repo: destination repo :type dest_repo: pulp.server.db.model.Repository :param unit: Unit to be copied :type unit: pulp.server.db.model.ContentUnit :param config: configuration instance passed to the importer of the destination repo :type config: pulp.plugins.config.PluginCallConfiguration :return: copied unit or None if the unit was not copied :rtype: pulp.server.db.model.ContentUnit """ types_to_be_copied = ( models.PackageGroup, models.PackageCategory, models.PackageEnvironment, models.PackageLangpacks ) if isinstance(unit, types_to_be_copied): return associate_copy_for_repo(unit, dest_repo) elif isinstance(unit, models.RPM): # copy will happen in one batch return unit elif isinstance(unit, models.YumMetadataFile): return associate_copy_for_repo(unit, dest_repo, True) elif isinstance(unit, (models.DRPM, models.SRPM)): if rpm_parse.signature_enabled(config): if unit.downloaded: try: rpm_parse.filter_signature(unit, config) except PulpCodedException as e: _LOGGER.debug(e) return repo_controller.associate_single_unit(repository=dest_repo, unit=unit) return unit else: repo_controller.associate_single_unit(repository=dest_repo, unit=unit) return unit
def check_all_and_associate(wanted, conduit, config, download_deferred, catalog): """ Given a set of unit keys as namedtuples, this function checks if a unit already exists in Pulp and returns the set of tuples that were not found. This checks for the unit in the db as well as for the actual file on the filesystem. If a unit exists in the db and the filesystem, this function also associates the unit to the given repo. Note that the check for the actual file is performed only for the supported unit types. :param wanted: iterable of units as namedtuples :type wanted: iterable :param conduit: repo sync conduit :type conduit: pulp.plugins.conduits.repo_sync.RepoSync :param config: configuration instance passed to the importer :type config: pulp.plugins.config.PluginCallConfiguration :param download_deferred: indicates downloading is deferred (or not). :type download_deferred: bool :param catalog: Deferred downloading catalog. :type catalog: pulp_rpm.plugins.importers.yum.sync.PackageCatalog :return: set of unit keys as namedtuples, identifying which of the named tuples received as input were not found on the server. :rtype: set """ sorted_units = _sort_by_type(wanted) for unit_type, values in sorted_units.iteritems(): model = plugin_api.get_unit_model_by_id(unit_type) # FIXME "fields" does not get used, but it should # fields = model.unit_key_fields + ('_storage_path',) unit_generator = (model(**unit_tuple._asdict()) for unit_tuple in values.copy()) for unit in units_controller.find_units(unit_generator): # Existing RPMs, DRPMs and SRPMs are disqualified when the associated # package file does not exist and downloading is not deferred. if not download_deferred and unit_type in ( ids.TYPE_ID_RPM, ids.TYPE_ID_SRPM, ids.TYPE_ID_DRPM): if unit._storage_path is None or not os.path.isfile(unit._storage_path): continue catalog.add(unit) if rpm_parse.signature_enabled(config): try: rpm_parse.filter_signature(unit, config) except PulpCodedException as e: _LOGGER.debug(e) continue repo_controller.associate_single_unit(conduit.repo, unit) values.discard(unit.unit_key_as_named_tuple) still_wanted = set() still_wanted.update(*sorted_units.values()) return still_wanted
def _associate_unit(dest_repo, unit, config): """ Associate one particular unit with the destination repository. There are behavioral exceptions based on type: Group, Category, Environment and Yum Metadata File units need to have their "repo_id" attribute set. RPMs are convenient to do all as one block, for the purpose of dependency resolution. So this method skips RPMs and lets them be done together by other means :param dest_repo: destination repo :type dest_repo: pulp.server.db.model.Repository :param unit: Unit to be copied :type unit: pulp.server.db.model.ContentUnit :param config: configuration instance passed to the importer of the destination repo :type config: pulp.plugins.config.PluginCallConfiguration :return: copied unit or None if the unit was not copied :rtype: pulp.server.db.model.ContentUnit """ types_to_be_copied = (models.PackageGroup, models.PackageCategory, models.PackageEnvironment, models.PackageLangpacks) if isinstance(unit, types_to_be_copied): return associate_copy_for_repo(unit, dest_repo) elif isinstance(unit, models.RPM): # copy will happen in one batch return unit elif isinstance(unit, models.YumMetadataFile): return associate_copy_for_repo(unit, dest_repo, True) elif isinstance(unit, (models.DRPM, models.SRPM)): if rpm_parse.signature_enabled(config): if unit.downloaded: try: rpm_parse.filter_signature(unit, config) except PulpCodedException as e: _LOGGER.debug(e) return repo_controller.associate_single_unit(repository=dest_repo, unit=unit) return unit else: repo_controller.associate_single_unit(repository=dest_repo, unit=unit) return unit
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM, SRPM or DRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: if type_id == models.DRPM._content_type_id.default: rpm_data = _extract_drpm_data(file_path) else: rpm_data = _extract_rpm_data(type_id, file_path) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise # metadata can be None metadata = metadata or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. rpm_data['checksumtype'] = util.TYPE_SHA256 rpm_data['checksum'] = sums[util.TYPE_SHA256] # keep all available checksum values on the model rpm_data['checksums'] = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. rpm_data.update(metadata or {}) rpm_data.update(unit_key or {}) # Validate the user specified data by instantiating the model try: unit = model_class(**rpm_data) except TypeError: raise ModelInstantiationError() if type_id != models.DRPM._content_type_id.default: # Extract/adjust the repodata snippets repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype) _update_provides_requires(unit, repodata) _update_files(unit, repodata) unit.modify_xml(repodata) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() if rpm_parse.signature_enabled(config): rpm_parse.filter_signature(unit, config) repo_controller.associate_single_unit(repo, unit)
def check_all_and_associate(wanted, conduit, config, download_deferred, catalog): """ Given a set of unit keys as namedtuples, this function checks if a unit already exists in Pulp and returns the set of tuples that were not found. This checks for the unit in the db as well as for the actual file on the filesystem. If a unit exists in the db and the filesystem, this function also associates the unit to the given repo. Note that the check for the actual file is performed only for the supported unit types. :param wanted: dict where keys are units as namedtuples, and values are WantedUnitInfo instances :type wanted: dict :param conduit: repo sync conduit :type conduit: pulp.plugins.conduits.repo_sync.RepoSync :param config: configuration instance passed to the importer :type config: pulp.plugins.config.PluginCallConfiguration :param download_deferred: indicates downloading is deferred (or not). :type download_deferred: bool :param catalog: Deferred downloading catalog. :type catalog: pulp_rpm.plugins.importers.yum.sync.PackageCatalog :return: set of unit keys as namedtuples, identifying which of the named tuples received as input were not found on the server. :rtype: set """ rpm_drpm_srpm = (ids.TYPE_ID_RPM, ids.TYPE_ID_SRPM, ids.TYPE_ID_DRPM) all_associated_units = set() for unit_type in rpm_drpm_srpm: units_generator = repo_controller.get_associated_unit_ids( conduit.repo.repo_id, unit_type) all_associated_units.update(units_generator) sorted_units = _sort_by_type(wanted.iterkeys()) for unit_type, values in sorted_units.iteritems(): model = plugin_api.get_unit_model_by_id(unit_type) # FIXME "fields" does not get used, but it should # fields = model.unit_key_fields + ('_storage_path',) unit_generator = (model(**unit_tuple._asdict()) for unit_tuple in values.copy()) for unit in units_controller.find_units(unit_generator): is_rpm_drpm_srpm = unit_type in rpm_drpm_srpm file_exists = unit._storage_path is not None and os.path.isfile( unit._storage_path) if is_rpm_drpm_srpm: # no matter what is the download policy, if existing unit has a valid storage_path, # we need to set the downloaded flag to True if file_exists and not unit.downloaded: unit.downloaded = True unit.save() # Existing RPMs, DRPMs and SRPMs are disqualified when the associated # package file does not exist and downloading is not deferred. if not download_deferred and not file_exists: continue catalog.add(unit, wanted[unit.unit_key_as_named_tuple].download_path) if unit.id not in all_associated_units: if rpm_parse.signature_enabled(config): try: rpm_parse.filter_signature(unit, config) except PulpCodedException as e: _LOGGER.debug(e) continue repo_controller.associate_single_unit(conduit.repo, unit) values.discard(unit.unit_key_as_named_tuple) still_wanted = set() still_wanted.update(*sorted_units.values()) return still_wanted
def copy_rpms(units, source_repo, dest_repo, import_conduit, config, copy_deps, solver=None): """ Copy RPMs from the source repo to the destination repo, and optionally copy dependencies as well. Dependencies are resolved recursively. :param units: iterable of Units :type units: iterable of pulp_rpm.plugins.db.models.RPM :param source_repo: The repository we are copying units from. :type source_repo: pulp.server.db.model.Repository :param dest_repo: The repository we are copying units to :type dest_repo: pulp.server.db.model.Repository :param import_conduit: import conduit passed to the Importer :type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit :param config: configuration instance passed to the importer of the destination repo :type config: pulp.plugins.config.PluginCallConfiguration :param copy_deps: if True, copies dependencies as specified in "Requires" lines in the RPM metadata. Matches against NEVRAs and Provides declarations that are found in the source repository. Silently skips any dependencies that cannot be resolved within the source repo. :param solver: an object that can be used for dependency solving. this is useful so that data can be cached in the depsolving object and re-used by each iteration of this method. :type solver: pulp_rpm.plugins.importers.yum.depsolve.Solver :return: set of pulp.plugins.models.Unit that were copied :rtype: set """ unit_set = set() failed_signature_check = 0 for unit in units: # we are passing in units that may have flattened "provides" metadata. # This flattened field is not used by associate_single_unit(). if rpm_parse.signature_enabled(config): if unit.downloaded: try: rpm_parse.filter_signature(unit, config) except PulpCodedException as e: _LOGGER.debug(e) failed_signature_check += 1 continue else: continue repo_controller.associate_single_unit(repository=dest_repo, unit=unit) unit_set.add(unit) if failed_signature_check: _LOGGER.warning( _('%s packages failed signature filter and were not imported.' % failed_signature_check)) if copy_deps and unit_set: if solver is None: solver = depsolve.Solver(source_repo) # This returns units that have a flattened 'provides' metadata field # for memory purposes (RHBZ #1185868) deps = solver.find_dependent_rpms(unit_set) # remove rpms already in the destination repo existing_units = set( existing.get_existing_units([dep.unit_key for dep in deps], models.RPM, dest_repo)) # the hash comparison for Units is unit key + type_id, the metadata # field is not used. to_copy = deps - existing_units _LOGGER.debug('Copying deps: %s' % str(sorted([x.name for x in to_copy]))) if to_copy: unit_set |= copy_rpms(to_copy, source_repo, dest_repo, import_conduit, config, copy_deps, solver) return unit_set
def copy_rpms(units, source_repo, dest_repo, import_conduit, config, copy_deps, solver=None): """ Copy RPMs from the source repo to the destination repo, and optionally copy dependencies as well. Dependencies are resolved recursively. :param units: iterable of Units :type units: iterable of pulp_rpm.plugins.db.models.RPM :param source_repo: The repository we are copying units from. :type source_repo: pulp.server.db.model.Repository :param dest_repo: The repository we are copying units to :type dest_repo: pulp.server.db.model.Repository :param import_conduit: import conduit passed to the Importer :type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit :param config: configuration instance passed to the importer of the destination repo :type config: pulp.plugins.config.PluginCallConfiguration :param copy_deps: if True, copies dependencies as specified in "Requires" lines in the RPM metadata. Matches against NEVRAs and Provides declarations that are found in the source repository. Silently skips any dependencies that cannot be resolved within the source repo. :param solver: an object that can be used for dependency solving. this is useful so that data can be cached in the depsolving object and re-used by each iteration of this method. :type solver: pulp_rpm.plugins.importers.yum.depsolve.Solver :return: set of pulp.plugins.models.Unit that were copied :rtype: set """ unit_set = set() failed_signature_check = 0 for unit in units: # we are passing in units that may have flattened "provides" metadata. # This flattened field is not used by associate_single_unit(). if rpm_parse.signature_enabled(config): if unit.downloaded: try: rpm_parse.filter_signature(unit, config) except PulpCodedException as e: _LOGGER.debug(e) failed_signature_check += 1 continue else: continue repo_controller.associate_single_unit(repository=dest_repo, unit=unit) unit_set.add(unit) if failed_signature_check: _LOGGER.warning(_('%s packages failed signature filter and were not imported.' % failed_signature_check)) if copy_deps and unit_set: if solver is None: solver = depsolve.Solver(source_repo) # This returns units that have a flattened 'provides' metadata field # for memory purposes (RHBZ #1185868) deps = solver.find_dependent_rpms(unit_set) # remove rpms already in the destination repo existing_units = set(existing.get_existing_units([dep.unit_key for dep in deps], models.RPM, dest_repo)) # the hash comparison for Units is unit key + type_id, the metadata # field is not used. to_copy = deps - existing_units _LOGGER.debug('Copying deps: %s' % str(sorted([x.name for x in to_copy]))) if to_copy: unit_set |= copy_rpms(to_copy, source_repo, dest_repo, import_conduit, config, copy_deps, solver) return unit_set
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM, SRPM or DRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values, or None :type unit_key: dict or None :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: if type_id == models.DRPM._content_type_id.default: unit = models.DRPM(**_extract_drpm_data(file_path)) else: repodata = rpm_parse.get_package_xml(file_path, sumtype=util.TYPE_SHA256) package_xml = (utils.fake_xml_element(repodata['primary'], constants.COMMON_NAMESPACE) .find(primary.PACKAGE_TAG)) unit = primary.process_package_element(package_xml) except Exception: raise PulpCodedException(error_codes.RPM1016) # metadata and unit_key can be None metadata = metadata or {} unit_key = unit_key or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key) update_fields_inbound(model_class, metadata) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. unit.checksumtype = util.TYPE_SHA256 unit.checksum = sums[util.TYPE_SHA256] # keep all available checksum values on the model unit.checksums = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. for key, value in metadata.items(): setattr(unit, key, value) for key, value in unit_key.items(): setattr(unit, key, value) if type_id != models.DRPM._content_type_id.default: # Extract/adjust the repodata snippets unit.signing_key = rpm_parse.package_signature(rpm_parse.package_headers(file_path)) # construct filename from metadata (BZ #1101168) if type_id == models.SRPM._content_type_id.default: rpm_basefilename = "%s-%s-%s.src.rpm" % (unit.name, unit.version, unit.release) else: rpm_basefilename = "%s-%s-%s.%s.rpm" % (unit.name, unit.version, unit.release, unit.arch) unit.relativepath = rpm_basefilename unit.filename = rpm_basefilename _update_files(unit, repodata) unit.modify_xml(repodata) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except TypeError: raise ModelInstantiationError() except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() if rpm_parse.signature_enabled(config): rpm_parse.filter_signature(unit, config) repo_controller.associate_single_unit(repo, unit)