def count_packages(self, channels=None): start_time = datetime.now() reposync.clear_ssl_cache() # Both entitled channels and custom channels with null-org repositories. channel_list = self._list_available_channels() if not channel_list: error_messages = self._msg_array_if_not_activated() if error_messages: log(0, "\n".join(error_messages)) sys.exit(1) channel_list.extend( [c for c in self.synced_channels if self.synced_channels[c]]) # Only some channels specified by parameter if channels: new_channel_list = [] for channel in channels: new_channel_list.extend(fnmatch.filter(channel_list, channel)) channel_list = list(set(new_channel_list)) log(0, "Number of channels: %d" % len(channel_list)) # Prepare repositories repo_tree = {} repository_count = 0 for channel in channel_list: sources = self.cdn_repository_manager.get_content_sources(channel) # Custom channel if not sources: repos = self.cdn_repository_manager.list_associated_repos( channel) sources = [] for index, repo in enumerate(sorted(repos)): repo_label = "%s-%d" % (channel, index) sources.append({ 'relative_url': repo, 'pulp_repo_label_v2': repo_label }) repository_count += len(sources) repo_tree[channel] = sources log(0, "Number of repositories: %d" % repository_count) downloader = ThreadedDownloader() for channel in repo_tree: for source in repo_tree[channel]: yum_repo = self._create_yum_repo(source) params = {} yum_repo.set_download_parameters( params, "repodata/repomd.xml", os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new")) downloader.add(params) progress_bar = ProgressBarLogger("Downloading repomd: ", repository_count) downloader.set_log_obj(progress_bar) # Overwrite existing files downloader.set_force(True) log2background(0, "Downloading repomd started.") downloader.run() log2background(0, "Downloading repomd finished.") progress_bar = ProgressBarLogger("Comparing repomd: ", len(repo_tree)) to_download_count = 0 repo_tree_to_update = {} log2background(0, "Comparing repomd started.") is_missing_repomd = False for channel in repo_tree: cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) packages_num_path = os.path.join(cdn_repodata_path, "packages_num") packages_size_path = os.path.join(cdn_repodata_path, "packages_size") sources = repo_tree[channel] yum_repos = [self._create_yum_repo(source) for source in sources] # check all repomd files were downloaded for yum_repo in yum_repos: new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new") if not os.path.isfile(new_repomd): is_missing_repomd = True # packages_num file exists and all cached repomd files are up to date => skip if os.path.isfile(packages_num_path) and os.path.isfile( packages_size_path) and all( [x.repomd_up_to_date() for x in yum_repos]): progress_bar.log(True, None) continue update_channel = False for yum_repo in yum_repos: # use new repomd new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new") if os.path.isfile(new_repomd): update_channel = True os.rename( new_repomd, os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml")) else: # it wasn't downloaded continue for path, checksum_pair in yum_repo.get_metadata_paths(): params = {} yum_repo.set_download_parameters( params, path, os.path.join(yum_repo.repo.basecachedir, yum_repo.name, os.path.basename(path)), checksum_type=checksum_pair[0], checksum_value=checksum_pair[1]) downloader.add(params) to_download_count += 1 # If there is at least one repo with new repomd, pass through this channel if update_channel: repo_tree_to_update[channel] = sources progress_bar.log(True, None) log2background(0, "Comparing repomd finished.") progress_bar = ProgressBarLogger("Downloading metadata:", to_download_count) downloader.set_log_obj(progress_bar) downloader.set_force(False) log2background(0, "Downloading metadata started.") downloader.run() log2background(0, "Downloading metadata finished.") progress_bar = ProgressBarLogger("Counting packages: ", len(repo_tree_to_update)) log2background(0, "Counting packages started.") for channel in repo_tree_to_update: cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) packages_num_path = os.path.join(cdn_repodata_path, "packages_num") packages_size_path = os.path.join(cdn_repodata_path, "packages_size") sources = repo_tree_to_update[channel] yum_repos = [self._create_yum_repo(source) for source in sources] packages = {} for yum_repo in yum_repos: for pkg in yum_repo.raw_list_packages(): nvrea = str(pkg) packages[nvrea] = pkg.packagesize # create directory for repo data if it doesn't exist try: os.makedirs(cdn_repodata_path) except OSError: exc = sys.exc_info()[1] if exc.errno == errno.EEXIST and os.path.isdir( cdn_repodata_path): pass else: raise f_num_out = open(packages_num_path, 'w') f_size_out = open(packages_size_path, 'w') try: f_num_out.write(str(len(packages))) f_size_out.write(str(sum(packages.values()))) finally: if f_num_out is not None: f_num_out.close() if f_size_out is not None: f_size_out.close() # Delete cache to save space for yum_repo in yum_repos: yum_repo.clear_cache(keep_repomd=True) progress_bar.log(True, None) log2background(0, "Counting packages finished.") end_time = datetime.now() log(0, "Total time: %s" % str(end_time - start_time).split('.')[0]) if is_missing_repomd: raise CountingPackagesError( "Cannot download some repomd.xml files. " "Please, check /var/log/rhn/cdnsync.log for details")
def count_packages(self, channels=None): start_time = datetime.now() reposync.clear_ssl_cache() # Both entitled channels and custom channels with null-org repositories. channel_list = self._list_available_channels() if not channel_list: error_messages = self._msg_array_if_not_activated() if error_messages: log(0, "\n".join(error_messages)) sys.exit(1) channel_list.extend([c for c in self.synced_channels if self.synced_channels[c]]) # Only some channels specified by parameter if channels: new_channel_list = [] for channel in channels: new_channel_list.extend(fnmatch.filter(channel_list, channel)) channel_list = list(set(new_channel_list)) log(0, "Number of channels: %d" % len(channel_list)) # Prepare repositories repo_tree = {} repository_count = 0 for channel in channel_list: sources = self.cdn_repository_manager.get_content_sources(channel) # Custom channel if not sources: repos = self.cdn_repository_manager.list_associated_repos(channel) sources = [] for index, repo in enumerate(sorted(repos)): repo_label = "%s-%d" % (channel, index) sources.append({'relative_url': repo, 'pulp_repo_label_v2': repo_label}) repository_count += len(sources) repo_tree[channel] = sources log(0, "Number of repositories: %d" % repository_count) downloader = ThreadedDownloader() for channel in repo_tree: for source in repo_tree[channel]: yum_repo = self._create_yum_repo(source) params = {} yum_repo.set_download_parameters(params, "repodata/repomd.xml", os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new")) downloader.add(params) progress_bar = ProgressBarLogger("Downloading repomd: ", repository_count) downloader.set_log_obj(progress_bar) # Overwrite existing files downloader.set_force(True) log2background(0, "Downloading repomd started.") downloader.run() log2background(0, "Downloading repomd finished.") progress_bar = ProgressBarLogger("Comparing repomd: ", len(repo_tree)) to_download_count = 0 repo_tree_to_update = {} log2background(0, "Comparing repomd started.") is_missing_repomd = False for channel in repo_tree: cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) packages_num_path = os.path.join(cdn_repodata_path, "packages_num") packages_size_path = os.path.join(cdn_repodata_path, "packages_size") sources = repo_tree[channel] yum_repos = [self._create_yum_repo(source) for source in sources] # check all repomd files were downloaded for yum_repo in yum_repos: new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new") if not os.path.isfile(new_repomd): is_missing_repomd = True # packages_num file exists and all cached repomd files are up to date => skip if os.path.isfile(packages_num_path) and os.path.isfile(packages_size_path) and all( [x.repomd_up_to_date() for x in yum_repos]): progress_bar.log(True, None) continue update_channel = False for yum_repo in yum_repos: # use new repomd new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new") if os.path.isfile(new_repomd): update_channel = True os.rename(new_repomd, os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml")) else: # it wasn't downloaded continue for path, checksum_pair in yum_repo.get_metadata_paths(): params = {} yum_repo.set_download_parameters(params, path, os.path.join(yum_repo.repo.basecachedir, yum_repo.name, os.path.basename(path)), checksum_type=checksum_pair[0], checksum_value=checksum_pair[1]) downloader.add(params) to_download_count += 1 # If there is at least one repo with new repomd, pass through this channel if update_channel: repo_tree_to_update[channel] = sources progress_bar.log(True, None) log2background(0, "Comparing repomd finished.") progress_bar = ProgressBarLogger("Downloading metadata:", to_download_count) downloader.set_log_obj(progress_bar) downloader.set_force(False) log2background(0, "Downloading metadata started.") downloader.run() log2background(0, "Downloading metadata finished.") progress_bar = ProgressBarLogger("Counting packages: ", len(repo_tree_to_update)) log2background(0, "Counting packages started.") for channel in repo_tree_to_update: cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) packages_num_path = os.path.join(cdn_repodata_path, "packages_num") packages_size_path = os.path.join(cdn_repodata_path, "packages_size") sources = repo_tree_to_update[channel] yum_repos = [self._create_yum_repo(source) for source in sources] packages = {} for yum_repo in yum_repos: for pkg in yum_repo.raw_list_packages(): nvrea = str(pkg) packages[nvrea] = pkg.packagesize # create directory for repo data if it doesn't exist try: os.makedirs(cdn_repodata_path) except OSError: exc = sys.exc_info()[1] if exc.errno == errno.EEXIST and os.path.isdir(cdn_repodata_path): pass else: raise f_num_out = open(packages_num_path, 'w') f_size_out = open(packages_size_path, 'w') try: f_num_out.write(str(len(packages))) f_size_out.write(str(sum(packages.values()))) finally: if f_num_out is not None: f_num_out.close() if f_size_out is not None: f_size_out.close() # Delete cache to save space for yum_repo in yum_repos: yum_repo.clear_cache(keep_repomd=True) progress_bar.log(True, None) log2background(0, "Counting packages finished.") end_time = datetime.now() log(0, "Total time: %s" % str(end_time - start_time).split('.')[0]) if is_missing_repomd: raise CountingPackagesError("Cannot download some repomd.xml files. " "Please, check /var/log/rhn/cdnsync.log for details")