def _decide_rpms_to_download(self, metadata_files): """ Decide which RPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes) :rtype: tuple """ if ids.TYPE_ID_RPM in self.config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping RPM sync') return set(), 0, 0 primary_file_handle = metadata_files.get_metadata_file_handle( primary.METADATA_FILE_NAME) try: # scan through all the metadata to decide which packages to download package_info_generator = packages.package_list_generator( primary_file_handle, primary.PACKAGE_TAG, primary.process_package_element) wanted = self._identify_wanted_versions(package_info_generator) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate( wanted.iterkeys(), self.conduit, self.download_deferred) count = len(to_download) size = 0 for unit in to_download: size += wanted[unit] return to_download, count, size finally: primary_file_handle.close()
def _decide_rpms_to_download(self, metadata_files): """ Decide which RPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes) :rtype: tuple """ if ids.TYPE_ID_RPM in self.config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping RPM sync') return set(), 0, 0 primary_file_handle = metadata_files.get_metadata_file_handle(primary.METADATA_FILE_NAME) try: # scan through all the metadata to decide which packages to download package_info_generator = packages.package_list_generator( primary_file_handle, primary.PACKAGE_TAG, primary.process_package_element) wanted = self._identify_wanted_versions(package_info_generator) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate( wanted.iterkeys(), self.conduit, self.download_deferred) count = len(to_download) size = 0 for unit in to_download: size += wanted[unit] return to_download, count, size finally: primary_file_handle.close()
def test_drpms_check_all_and_associate_negative(self, mock_isfile, mock_save, mock_search_all_units): mock_search_all_units.return_value = [] mock_isfile.return_value = True units = model_factory.drpm_models(3) input_units = set([unit.as_named_tuple for unit in units]) result = check_all_and_associate(input_units, self.conduit) self.assertEqual(len(list(result)), 3)
def test_drpms_check_all_and_associate_positive(self, mock_isfile, mock_save, mock_search_all_units): units = model_factory.drpm_models(3) mock_search_all_units.return_value = units mock_isfile.return_value = True input_units = set([unit.as_named_tuple for unit in units]) for unit in units: unit.metadata['filename'] = 'test-filename' unit.storage_path = "existing_storage_path" result = check_all_and_associate(input_units, self.conduit) self.assertEqual(len(list(result)), 0)
def _decide_drpms_to_download(self, metadata_files): """ Decide which DRPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes) :rtype: tuple """ if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping DRPM sync') return set(), 0, 0 to_download = set() count = 0 size = 0 # multiple options for deltainfo files depending on the distribution # so we have to go through all of them for metadata_file_name in presto.METADATA_FILE_NAMES: presto_file_handle = metadata_files.get_metadata_file_handle( metadata_file_name) if presto_file_handle: try: package_info_generator = packages.package_list_generator( presto_file_handle, presto.PACKAGE_TAG, presto.process_package_element) wanted = self._identify_wanted_versions( package_info_generator) # check for the units that are already in the repo not_found_in_the_repo = existing.check_repo( wanted.iterkeys(), self.sync_conduit.get_units) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate( not_found_in_the_repo, self.sync_conduit) count += len(to_download) for unit in to_download: size += wanted[unit] finally: presto_file_handle.close() return to_download, count, size
def _decide_drpms_to_download(self, metadata_files): """ Decide which DRPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes) :rtype: tuple """ if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping DRPM sync') return set(), 0, 0 to_download = set() count = 0 size = 0 # multiple options for deltainfo files depending on the distribution # so we have to go through all of them for metadata_file_name in presto.METADATA_FILE_NAMES: presto_file_handle = metadata_files.get_metadata_file_handle(metadata_file_name) if presto_file_handle: try: package_info_generator = packages.package_list_generator( presto_file_handle, presto.PACKAGE_TAG, presto.process_package_element) wanted = self._identify_wanted_versions(package_info_generator) # check for the units that are already in the repo not_found_in_the_repo = existing.check_repo(wanted.iterkeys(), self.sync_conduit.get_units) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate(not_found_in_the_repo, self.sync_conduit) count += len(to_download) for unit in to_download: size += wanted[unit] finally: presto_file_handle.close() return to_download, count, size
def _decide_rpms_to_download(self, metadata_files, catalog): """ Decide which RPMs should be downloaded based on the repo metadata and on the importer config. :param metadata_files: instance of MetadataFiles :type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :param catalog: Deferred downloading catalog. :type catalog: PackageCatalog :return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes) :rtype: tuple :raises PulpCodedException: if there is some inconsistency in metadata """ if ids.TYPE_ID_RPM in self.config.get(constants.CONFIG_SKIP, []): _logger.debug('skipping RPM sync') return set(), 0, 0 primary_file_handle = metadata_files.get_metadata_file_handle(primary.METADATA_FILE_NAME) try: # scan through all the metadata to decide which packages to download package_info_generator = packages.package_list_generator( primary_file_handle, primary.PACKAGE_TAG, primary.process_package_element) # count packages while iterating over primary metadata and deciding on wanted packages counter = itertools.count() wrapped_generator = itertools.imap(lambda x, y: x, package_info_generator, counter) wanted = self._identify_wanted_versions(wrapped_generator) primary_rpm_count = counter.next() if primary_rpm_count != metadata_files.rpm_count: reason = 'metadata is missing for some packages in filelists.xml and in other.xml' raise PulpCodedException(error_code=error_codes.RPM1015, reason=reason) # check for the units that are not in the repo, but exist on the server # and associate them to the repo to_download = existing.check_all_and_associate( wanted, self.conduit, self.config, self.download_deferred, catalog) count = len(to_download) size = 0 for unit in to_download: size += wanted[unit].size return to_download, count, size finally: primary_file_handle.close()