def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = list(self.synced_channels)
# Check channel availability before doing anything
not_available = []
available = []
for channel in channels:
if not self._is_channel_available(channel):
not_available.append(channel)
else:
available.append(channel)
channels = available
error_messages = []
# if we have not_available channels log the error immediately
if not_available:
msg = "ERROR: these channels either do not exist or are not available:\n " + "\n ".join(not_available)
error_messages.append(msg)
# BZ 1434913 - let user know satellite may not be activated if all channels are in not_available
if not available:
msg = "WARNING: Is your Red Hat Satellite activated for CDN?\n"
msg += "(to see details about currently used SSL certificates for accessing CDN:"
msg += " /usr/bin/cdn-sync --cdn-certs)"
error_messages.append(msg)
# Need to update channel metadata
self._update_channels_metadata([ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[channel]:
self.cdn_repository_manager.assign_repositories_to_channel(channel)
reposync.clear_ssl_cache()
# Finally, sync channel content
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append("Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel))
if failed_packages > 0:
error_messages.append("%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel,
channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def count_packages(self, channels=None):
start_time = datetime.now()
reposync.clear_ssl_cache()
# Both entitled channels and custom channels with null-org repositories.
channel_list = self._list_available_channels()
if not channel_list:
error_messages = self._msg_array_if_not_activated()
if error_messages:
log(0, "\n".join(error_messages))
sys.exit(1)
channel_list.extend(
[c for c in self.synced_channels if self.synced_channels[c]])
# Only some channels specified by parameter
if channels:
new_channel_list = []
for channel in channels:
new_channel_list.extend(fnmatch.filter(channel_list, channel))
channel_list = list(set(new_channel_list))
log(0, "Number of channels: %d" % len(channel_list))
# Prepare repositories
repo_tree = {}
repository_count = 0
for channel in channel_list:
sources = self.cdn_repository_manager.get_content_sources(channel)
# Custom channel
if not sources:
repos = self.cdn_repository_manager.list_associated_repos(
channel)
sources = []
for index, repo in enumerate(sorted(repos)):
repo_label = "%s-%d" % (channel, index)
sources.append({
'relative_url': repo,
'pulp_repo_label_v2': repo_label
})
repository_count += len(sources)
repo_tree[channel] = sources
log(0, "Number of repositories: %d" % repository_count)
downloader = ThreadedDownloader()
for channel in repo_tree:
for source in repo_tree[channel]:
yum_repo = self._create_yum_repo(source)
params = {}
yum_repo.set_download_parameters(
params, "repodata/repomd.xml",
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
"repomd.xml.new"))
downloader.add(params)
progress_bar = ProgressBarLogger("Downloading repomd: ",
repository_count)
downloader.set_log_obj(progress_bar)
# Overwrite existing files
downloader.set_force(True)
log2background(0, "Downloading repomd started.")
downloader.run()
log2background(0, "Downloading repomd finished.")
progress_bar = ProgressBarLogger("Comparing repomd: ",
len(repo_tree))
to_download_count = 0
repo_tree_to_update = {}
log2background(0, "Comparing repomd started.")
is_missing_repomd = False
for channel in repo_tree:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT,
channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path,
"packages_size")
sources = repo_tree[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
# check all repomd files were downloaded
for yum_repo in yum_repos:
new_repomd = os.path.join(yum_repo.repo.basecachedir,
yum_repo.name, "repomd.xml.new")
if not os.path.isfile(new_repomd):
is_missing_repomd = True
# packages_num file exists and all cached repomd files are up to date => skip
if os.path.isfile(packages_num_path) and os.path.isfile(
packages_size_path) and all(
[x.repomd_up_to_date() for x in yum_repos]):
progress_bar.log(True, None)
continue
update_channel = False
for yum_repo in yum_repos:
# use new repomd
new_repomd = os.path.join(yum_repo.repo.basecachedir,
yum_repo.name, "repomd.xml.new")
if os.path.isfile(new_repomd):
update_channel = True
os.rename(
new_repomd,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
"repomd.xml"))
else:
# it wasn't downloaded
continue
for path, checksum_pair in yum_repo.get_metadata_paths():
params = {}
yum_repo.set_download_parameters(
params,
path,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
os.path.basename(path)),
checksum_type=checksum_pair[0],
checksum_value=checksum_pair[1])
downloader.add(params)
to_download_count += 1
# If there is at least one repo with new repomd, pass through this channel
if update_channel:
repo_tree_to_update[channel] = sources
progress_bar.log(True, None)
log2background(0, "Comparing repomd finished.")
progress_bar = ProgressBarLogger("Downloading metadata:",
to_download_count)
downloader.set_log_obj(progress_bar)
downloader.set_force(False)
log2background(0, "Downloading metadata started.")
downloader.run()
log2background(0, "Downloading metadata finished.")
progress_bar = ProgressBarLogger("Counting packages: ",
len(repo_tree_to_update))
log2background(0, "Counting packages started.")
for channel in repo_tree_to_update:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT,
channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path,
"packages_size")
sources = repo_tree_to_update[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
packages = {}
for yum_repo in yum_repos:
for pkg in yum_repo.raw_list_packages():
nvrea = str(pkg)
packages[nvrea] = pkg.packagesize
# create directory for repo data if it doesn't exist
try:
os.makedirs(cdn_repodata_path)
except OSError:
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST and os.path.isdir(
cdn_repodata_path):
pass
else:
raise
f_num_out = open(packages_num_path, 'w')
f_size_out = open(packages_size_path, 'w')
try:
f_num_out.write(str(len(packages)))
f_size_out.write(str(sum(packages.values())))
finally:
if f_num_out is not None:
f_num_out.close()
if f_size_out is not None:
f_size_out.close()
# Delete cache to save space
for yum_repo in yum_repos:
yum_repo.clear_cache(keep_repomd=True)
progress_bar.log(True, None)
log2background(0, "Counting packages finished.")
end_time = datetime.now()
log(0, "Total time: %s" % str(end_time - start_time).split('.')[0])
if is_missing_repomd:
raise CountingPackagesError(
"Cannot download some repomd.xml files. "
"Please, check /var/log/rhn/cdnsync.log for details")
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = set(self.synced_channels)
# Check channel availability before doing anything
not_available = set()
available = set()
all_channel_list = None
for channel in channels:
# Try to expand wildcards in channel labels
if '*' in channel or '?' in channel or '[' in channel:
if all_channel_list is None:
all_channel_list = self._list_available_channels() + [
c for c in self.synced_channels
if self.synced_channels[c]
]
expanded = fnmatch.filter(all_channel_list, channel)
log(
2, "Expanding channel '%s' to: %s" %
(channel, ", ".join(expanded)))
if expanded:
for expanded_channel in expanded:
if not self._is_channel_available(expanded_channel):
not_available.add(expanded_channel)
else:
available.add(expanded_channel)
else:
not_available.add(channel)
elif not self._is_channel_available(channel):
not_available.add(channel)
else:
available.add(channel)
channels = available
error_messages = []
# if we have not_available channels log the error immediately
if not_available:
msg = "ERROR: these channels either do not exist or are not available for synchronization:\n " + \
"\n ".join(not_available)
error_messages.append(msg)
# BZ 1434913 - let user know if system is not activated if no available channels
if not available:
error_messages.extend(self._msg_array_if_not_activated())
# Need to update channel metadata
self._update_channels_metadata(
[ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[
channel]:
self.cdn_repository_manager.assign_repositories_to_channel(
channel)
reposync.clear_ssl_cache()
# Finally, sync channel content
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append(
"Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(channel, channel))
if failed_packages > 0:
error_messages.append(
"%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(failed_packages, channel, channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def count_packages(self, channels=None):
start_time = datetime.now()
reposync.clear_ssl_cache()
# Both entitled channels and custom channels with null-org repositories.
channel_list = self._list_available_channels()
if not channel_list:
error_messages = self._msg_array_if_not_activated()
if error_messages:
log(0, "\n".join(error_messages))
sys.exit(1)
channel_list.extend([c for c in self.synced_channels if self.synced_channels[c]])
# Only some channels specified by parameter
if channels:
new_channel_list = []
for channel in channels:
new_channel_list.extend(fnmatch.filter(channel_list, channel))
channel_list = list(set(new_channel_list))
log(0, "Number of channels: %d" % len(channel_list))
# Prepare repositories
repo_tree = {}
repository_count = 0
for channel in channel_list:
sources = self.cdn_repository_manager.get_content_sources(channel)
# Custom channel
if not sources:
repos = self.cdn_repository_manager.list_associated_repos(channel)
sources = []
for index, repo in enumerate(sorted(repos)):
repo_label = "%s-%d" % (channel, index)
sources.append({'relative_url': repo, 'pulp_repo_label_v2': repo_label})
repository_count += len(sources)
repo_tree[channel] = sources
log(0, "Number of repositories: %d" % repository_count)
downloader = ThreadedDownloader()
for channel in repo_tree:
for source in repo_tree[channel]:
yum_repo = self._create_yum_repo(source)
params = {}
yum_repo.set_download_parameters(params, "repodata/repomd.xml",
os.path.join(yum_repo.repo.basecachedir,
yum_repo.name, "repomd.xml.new"))
downloader.add(params)
progress_bar = ProgressBarLogger("Downloading repomd: ", repository_count)
downloader.set_log_obj(progress_bar)
# Overwrite existing files
downloader.set_force(True)
log2background(0, "Downloading repomd started.")
downloader.run()
log2background(0, "Downloading repomd finished.")
progress_bar = ProgressBarLogger("Comparing repomd: ", len(repo_tree))
to_download_count = 0
repo_tree_to_update = {}
log2background(0, "Comparing repomd started.")
is_missing_repomd = False
for channel in repo_tree:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path, "packages_size")
sources = repo_tree[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
# check all repomd files were downloaded
for yum_repo in yum_repos:
new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new")
if not os.path.isfile(new_repomd):
is_missing_repomd = True
# packages_num file exists and all cached repomd files are up to date => skip
if os.path.isfile(packages_num_path) and os.path.isfile(packages_size_path) and all(
[x.repomd_up_to_date() for x in yum_repos]):
progress_bar.log(True, None)
continue
update_channel = False
for yum_repo in yum_repos:
# use new repomd
new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new")
if os.path.isfile(new_repomd):
update_channel = True
os.rename(new_repomd,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml"))
else:
# it wasn't downloaded
continue
for path, checksum_pair in yum_repo.get_metadata_paths():
params = {}
yum_repo.set_download_parameters(params, path,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
os.path.basename(path)),
checksum_type=checksum_pair[0], checksum_value=checksum_pair[1])
downloader.add(params)
to_download_count += 1
# If there is at least one repo with new repomd, pass through this channel
if update_channel:
repo_tree_to_update[channel] = sources
progress_bar.log(True, None)
log2background(0, "Comparing repomd finished.")
progress_bar = ProgressBarLogger("Downloading metadata:", to_download_count)
downloader.set_log_obj(progress_bar)
downloader.set_force(False)
log2background(0, "Downloading metadata started.")
downloader.run()
log2background(0, "Downloading metadata finished.")
progress_bar = ProgressBarLogger("Counting packages: ", len(repo_tree_to_update))
log2background(0, "Counting packages started.")
for channel in repo_tree_to_update:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path, "packages_size")
sources = repo_tree_to_update[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
packages = {}
for yum_repo in yum_repos:
for pkg in yum_repo.raw_list_packages():
nvrea = str(pkg)
packages[nvrea] = pkg.packagesize
# create directory for repo data if it doesn't exist
try:
os.makedirs(cdn_repodata_path)
except OSError:
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST and os.path.isdir(cdn_repodata_path):
pass
else:
raise
f_num_out = open(packages_num_path, 'w')
f_size_out = open(packages_size_path, 'w')
try:
f_num_out.write(str(len(packages)))
f_size_out.write(str(sum(packages.values())))
finally:
if f_num_out is not None:
f_num_out.close()
if f_size_out is not None:
f_size_out.close()
# Delete cache to save space
for yum_repo in yum_repos:
yum_repo.clear_cache(keep_repomd=True)
progress_bar.log(True, None)
log2background(0, "Counting packages finished.")
end_time = datetime.now()
log(0, "Total time: %s" % str(end_time - start_time).split('.')[0])
if is_missing_repomd:
raise CountingPackagesError("Cannot download some repomd.xml files. "
"Please, check /var/log/rhn/cdnsync.log for details")
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = set(self.synced_channels)
# Check channel availability before doing anything
not_available = set()
available = set()
all_channel_list = None
for channel in channels:
# Try to expand wildcards in channel labels
if '*' in channel or '?' in channel or '[' in channel:
if all_channel_list is None:
all_channel_list = self._list_available_channels() + [c for c in self.synced_channels
if self.synced_channels[c]]
expanded = fnmatch.filter(all_channel_list, channel)
log(2, "Expanding channel '%s' to: %s" % (channel, ", ".join(expanded)))
if expanded:
for expanded_channel in expanded:
if not self._is_channel_available(expanded_channel):
not_available.add(expanded_channel)
else:
available.add(expanded_channel)
else:
not_available.add(channel)
elif not self._is_channel_available(channel):
not_available.add(channel)
else:
available.add(channel)
channels = available
error_messages = []
# if we have not_available channels log the error immediately
if not_available:
msg = "ERROR: these channels either do not exist or are not available for synchronization:\n " + \
"\n ".join(not_available)
error_messages.append(msg)
# BZ 1434913 - let user know if system is not activated if no available channels
if not available:
error_messages.extend(self._msg_array_if_not_activated())
# Need to update channel metadata
self._update_channels_metadata([ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[channel]:
self.cdn_repository_manager.assign_repositories_to_channel(channel)
reposync.clear_ssl_cache()
# Finally, sync channel content
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append("Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel))
if failed_packages > 0:
error_messages.append("%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel,
channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages