def _decide_drpms_to_download(self, metadata_files): """ Decide which DRPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes) :rtype: tuple """ if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): _LOGGER.debug('skipping DRPM sync') return set(), 0, 0 presto_file_handle = metadata_files.get_metadata_file_handle(presto.METADATA_FILE_NAME) if presto_file_handle: try: package_info_generator = packages.package_list_generator(presto_file_handle, presto.PACKAGE_TAG, presto.process_package_element) wanted = self._identify_wanted_versions(package_info_generator) to_download = existing.check_repo(wanted.iterkeys(), self.sync_conduit.get_units) count = len(to_download) size = 0 for unit in to_download: size += wanted[unit] finally: presto_file_handle.close() else: to_download = set() count = 0 size = 0 return to_download, count, size
def _decide_rpms_to_download(self, metadata_files): """ Decide which RPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes) :rtype: tuple """ if models.RPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping RPM sync') return set(), 0, 0 primary_file_handle = metadata_files.get_metadata_file_handle(primary.METADATA_FILE_NAME) try: # scan through all the metadata to decide which packages to download package_info_generator = packages.package_list_generator(primary_file_handle, primary.PACKAGE_TAG, primary.process_package_element) wanted = self._identify_wanted_versions(package_info_generator) # check for the units that are already in the repo not_found_in_the_repo = existing.check_repo(wanted.iterkeys(), self.sync_conduit.get_units) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate(not_found_in_the_repo, self.sync_conduit) count = len(to_download) size = 0 for unit in to_download: size += wanted[unit] return to_download, count, size finally: primary_file_handle.close()
def save_fileless_units(self, file_handle, tag, process_func, mutable_type=False, additive_type=False): """ Generic method for saving units parsed from a repo metadata file where the units do not have files to store on disk. For example, groups. :param file_handle: open file-like object containing metadata :type file_handle: file :param tag: XML tag that identifies each unit :type tag: basestring :param process_func: function that processes each unit and returns a dict representing that unit's attribute names and values. The function must take one parameter, which is an ElementTree instance :type process_func: function :param mutable_type: iff True, each unit will be saved regardless of whether it already exists in the repo. this is useful for units like group and category which don't have a version, but could change :type mutable_type: bool :param additive_type: iff True, units will be updated instead of replaced. For example, if you wanted to save an errata and concatenate its package list with an existing errata, you'd set this. Note that mutable_type and additive_type are mutually exclusive. :type additive_type: bool """ if mutable_type and additive_type: raise PulpCodedException(message="The mutable_type and additive_type arguments for " "this method are mutually exclusive.") # iterate through the file and determine what we want to have package_info_generator = packages.package_list_generator(file_handle, tag, process_func) # if units aren't mutable, we don't need to attempt saving units that # we already have if not mutable_type and not additive_type: wanted = (model.as_named_tuple for model in package_info_generator) # given what we want, filter out what we already have to_save = existing.check_repo(wanted, self.sync_conduit.get_units) # rewind, iterate again through the file, and save what we need file_handle.seek(0) all_packages = packages.package_list_generator(file_handle, tag, process_func) package_info_generator = (model for model in all_packages if model.as_named_tuple in to_save) for model in package_info_generator: unit = self.sync_conduit.init_unit(model.TYPE, model.unit_key, model.metadata, None) if additive_type: existing_unit = self.sync_conduit.find_unit_by_unit_key(model.TYPE, model.unit_key) if existing_unit: unit = self._concatenate_units(existing_unit, unit) self.sync_conduit.save_unit(unit)
def _decide_drpms_to_download(self, metadata_files): """ Decide which DRPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes) :rtype: tuple """ if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping DRPM sync') return set(), 0, 0 to_download = set() count = 0 size = 0 # multiple options for deltainfo files depending on the distribution # so we have to go through all of them for metadata_file_name in presto.METADATA_FILE_NAMES: presto_file_handle = metadata_files.get_metadata_file_handle( metadata_file_name) if presto_file_handle: try: package_info_generator = packages.package_list_generator( presto_file_handle, presto.PACKAGE_TAG, presto.process_package_element) wanted = self._identify_wanted_versions( package_info_generator) # check for the units that are already in the repo not_found_in_the_repo = existing.check_repo( wanted.iterkeys(), self.sync_conduit.get_units) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate( not_found_in_the_repo, self.sync_conduit) count += len(to_download) for unit in to_download: size += wanted[unit] finally: presto_file_handle.close() return to_download, count, size
def _decide_drpms_to_download(self, metadata_files): """ Decide which DRPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes) :rtype: tuple """ if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping DRPM sync') return set(), 0, 0 to_download = set() count = 0 size = 0 # multiple options for deltainfo files depending on the distribution # so we have to go through all of them for metadata_file_name in presto.METADATA_FILE_NAMES: presto_file_handle = metadata_files.get_metadata_file_handle(metadata_file_name) if presto_file_handle: try: package_info_generator = packages.package_list_generator( presto_file_handle, presto.PACKAGE_TAG, presto.process_package_element) wanted = self._identify_wanted_versions(package_info_generator) # check for the units that are already in the repo not_found_in_the_repo = existing.check_repo(wanted.iterkeys(), self.sync_conduit.get_units) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate(not_found_in_the_repo, self.sync_conduit) count += len(to_download) for unit in to_download: size += wanted[unit] finally: presto_file_handle.close() return to_download, count, size
def _decide_rpms_to_download(self, metadata_files): """ Decide which RPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes) :rtype: tuple """ if models.RPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping RPM sync') return set(), 0, 0 primary_file_handle = metadata_files.get_metadata_file_handle( primary.METADATA_FILE_NAME) try: # scan through all the metadata to decide which packages to download package_info_generator = packages.package_list_generator( primary_file_handle, primary.PACKAGE_TAG, primary.process_package_element) wanted = self._identify_wanted_versions(package_info_generator) # check for the units that are already in the repo not_found_in_the_repo = existing.check_repo( wanted.iterkeys(), self.sync_conduit.get_units) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate( not_found_in_the_repo, self.sync_conduit) count = len(to_download) size = 0 for unit in to_download: size += wanted[unit] return to_download, count, size finally: primary_file_handle.close()
def save_fileless_units(self, file_handle, tag, process_func, mutable_type=False, additive_type=False): """ Generic method for saving units parsed from a repo metadata file where the units do not have files to store on disk. For example, groups. :param file_handle: open file-like object containing metadata :type file_handle: file :param tag: XML tag that identifies each unit :type tag: basestring :param process_func: function that processes each unit and returns a dict representing that unit's attribute names and values. The function must take one parameter, which is an ElementTree instance :type process_func: function :param mutable_type: iff True, each unit will be saved regardless of whether it already exists in the repo. this is useful for units like group and category which don't have a version, but could change :type mutable_type: bool :param additive_type: iff True, units will be updated instead of replaced. For example, if you wanted to save an errata and concatenate its package list with an existing errata, you'd set this. Note that mutable_type and additive_type are mutually exclusive. :type additive_type: bool """ if mutable_type and additive_type: raise PulpCodedException(message="The mutable_type and additive_type arguments for " "this method are mutually exclusive.") # iterate through the file and determine what we want to have package_info_generator = packages.package_list_generator(file_handle, tag, process_func) # if units aren't mutable, we don't need to attempt saving units that # we already have if not mutable_type and not additive_type: wanted = (model.unit_key_as_named_tuple for model in package_info_generator) # given what we want, filter out what we already have to_save = existing.check_repo(wanted) # rewind, iterate again through the file, and save what we need file_handle.seek(0) all_packages = packages.package_list_generator(file_handle, tag, process_func) package_info_generator = \ (model for model in all_packages if model.unit_key_as_named_tuple in to_save) for model in package_info_generator: # Add repo_id to each collection of the pkglist of the new erratum if isinstance(model, models.Errata): for collection in model.pkglist: collection['_pulp_repo_id'] = self.repo.repo_id existing_unit = model.__class__.objects.filter(**model.unit_key).first() if not existing_unit: model.save() else: if additive_type: model = self._concatenate_units(existing_unit, model) model.save() else: # make sure the associate_unit call gets the existing unit model = existing_unit repo_controller.associate_single_unit(self.repo, model)
def save_fileless_units(self, file_handle, tag, process_func, mutable_type=False, additive_type=False): """ Generic method for saving units parsed from a repo metadata file where the units do not have files to store on disk. For example, groups. :param file_handle: open file-like object containing metadata :type file_handle: file :param tag: XML tag that identifies each unit :type tag: basestring :param process_func: function that processes each unit and returns a dict representing that unit's attribute names and values. The function must take one parameter, which is an ElementTree instance :type process_func: function :param mutable_type: iff True, each unit will be saved regardless of whether it already exists in the repo. this is useful for units like group and category which don't have a version, but could change :type mutable_type: bool :param additive_type: iff True, units will be updated instead of replaced. For example, if you wanted to save an errata and concatenate its package list with an existing errata, you'd set this. Note that mutable_type and additive_type are mutually exclusive. :type additive_type: bool """ if mutable_type and additive_type: raise PulpCodedException( message="The mutable_type and additive_type arguments for " "this method are mutually exclusive.") # iterate through the file and determine what we want to have package_info_generator = packages.package_list_generator( file_handle, tag, process_func) # if units aren't mutable, we don't need to attempt saving units that # we already have if not mutable_type and not additive_type: wanted = (model.unit_key_as_named_tuple for model in package_info_generator) # given what we want, filter out what we already have to_save = existing.check_repo(wanted) # rewind, iterate again through the file, and save what we need file_handle.seek(0) all_packages = packages.package_list_generator( file_handle, tag, process_func) package_info_generator = \ (model for model in all_packages if model.unit_key_as_named_tuple in to_save) for model in package_info_generator: existing_unit = model.__class__.objects.filter( **model.unit_key).first() if not existing_unit: model.save() else: if additive_type: model = self._concatenate_units(existing_unit, model) model.save() else: # make sure the associate_unit call gets the existing unit model = existing_unit repo_controller.associate_single_unit(self.repo, model)
def save_fileless_units(self, file_handle, tag, process_func, mutable_type=False, additive_type=False): """ Generic method for saving units parsed from a repo metadata file where the units do not have files to store on disk. For example, groups. :param file_handle: open file-like object containing metadata :type file_handle: file :param tag: XML tag that identifies each unit :type tag: basestring :param process_func: function that processes each unit and returns a dict representing that unit's attribute names and values. The function must take one parameter, which is an ElementTree instance :type process_func: function :param mutable_type: iff True, each unit will be saved regardless of whether it already exists in the repo. this is useful for units like group and category which don't have a version, but could change :type mutable_type: bool :param additive_type: iff True, units will be updated instead of replaced. For example, if you wanted to save an errata and concatenate its package list with an existing errata, you'd set this. Note that mutable_type and additive_type are mutually exclusive. :type additive_type: bool """ if mutable_type and additive_type: raise PulpCodedException(message="The mutable_type and additive_type arguments for " "this method are mutually exclusive.") # iterate through the file and determine what we want to have package_info_generator = packages.package_list_generator(file_handle, tag, process_func) # if units aren't mutable, we don't need to attempt saving units that # we already have if not mutable_type and not additive_type: wanted = (model.unit_key_as_named_tuple for model in package_info_generator) # given what we want, filter out what we already have to_save = existing.check_repo(wanted) # rewind, iterate again through the file, and save what we need file_handle.seek(0) all_packages = packages.package_list_generator(file_handle, tag, process_func) package_info_generator = \ (model for model in all_packages if model.unit_key_as_named_tuple in to_save) errata_could_not_be_merged_count = 0 for model in package_info_generator: # Add repo_id to each collection of the pkglist of the new erratum if isinstance(model, models.Errata): for collection in model.pkglist: collection['_pulp_repo_id'] = self.repo.repo_id existing_unit = model.__class__.objects.filter(**model.unit_key).first() if not existing_unit: model.save() else: if additive_type: try: model = self._concatenate_units(existing_unit, model) except ValueError: # Sometimes Errata units cannot be merged and a ValueError is raised # Count the errors and log them further down if isinstance(model, models.Errata): msg = _('The Errata "%(errata_id)s" could not be merged from the ' 'remote repository. This is likely due to the existing or new ' 'Errata not containing a valid `updated` field.') _logger.debug(msg % {'errata_id': model.errata_id}) errata_could_not_be_merged_count += 1 continue else: raise model.save() else: # make sure the associate_unit call gets the existing unit model = existing_unit repo_controller.associate_single_unit(self.repo, model) if errata_could_not_be_merged_count != 0: msg = _('There were %(count)d Errata units which could not be merged. This is likely ' 'due to Errata in this repo not containing valid `updated` fields.') _logger.warn(msg % {'count': errata_could_not_be_merged_count})