def test_reposync_timeout_minrate_are_passed_to_curl():
# only provide needed params with dummy data, the rest is "None"
params = NoKeyErrorsDict({"http_headers": dict(), "urls": ["http://example.com"]})
CFG = Mock()
CFG.REPOSYNC_TIMEOUT = 42
CFG.REPOSYNC_MINRATE = 42
CFG.REPOSYNC_DOWNLOAD_THREADS = 42 # Throws ValueError if not defined
curl_spy = Mock()
with patch(
"spacewalk.satellite_tools.download.pycurl.Curl", Mock(return_value=curl_spy)
), patch("uyuni.common.context_managers.CFG", CFG):
td = ThreadedDownloader(force=True)
td.add(params)
td.run()
curl_spy.setopt.assert_any_call(pycurl.LOW_SPEED_LIMIT, 42)
curl_spy.setopt.assert_any_call(pycurl.LOW_SPEED_TIME, 42)
def count_packages(self, channels=None):
start_time = datetime.now()
reposync.clear_ssl_cache()
# Both entitled channels and custom channels with null-org repositories.
channel_list = self._list_available_channels()
if not channel_list:
error_messages = self._msg_array_if_not_activated()
if error_messages:
log(0, "\n".join(error_messages))
sys.exit(1)
channel_list.extend(
[c for c in self.synced_channels if self.synced_channels[c]])
# Only some channels specified by parameter
if channels:
new_channel_list = []
for channel in channels:
new_channel_list.extend(fnmatch.filter(channel_list, channel))
channel_list = list(set(new_channel_list))
log(0, "Number of channels: %d" % len(channel_list))
# Prepare repositories
repo_tree = {}
repository_count = 0
for channel in channel_list:
sources = self.cdn_repository_manager.get_content_sources(channel)
# Custom channel
if not sources:
repos = self.cdn_repository_manager.list_associated_repos(
channel)
sources = []
for index, repo in enumerate(sorted(repos)):
repo_label = "%s-%d" % (channel, index)
sources.append({
'relative_url': repo,
'pulp_repo_label_v2': repo_label
})
repository_count += len(sources)
repo_tree[channel] = sources
log(0, "Number of repositories: %d" % repository_count)
downloader = ThreadedDownloader()
for channel in repo_tree:
for source in repo_tree[channel]:
yum_repo = self._create_yum_repo(source)
params = {}
yum_repo.set_download_parameters(
params, "repodata/repomd.xml",
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
"repomd.xml.new"))
downloader.add(params)
progress_bar = ProgressBarLogger("Downloading repomd: ",
repository_count)
downloader.set_log_obj(progress_bar)
# Overwrite existing files
downloader.set_force(True)
log2background(0, "Downloading repomd started.")
downloader.run()
log2background(0, "Downloading repomd finished.")
progress_bar = ProgressBarLogger("Comparing repomd: ",
len(repo_tree))
to_download_count = 0
repo_tree_to_update = {}
log2background(0, "Comparing repomd started.")
is_missing_repomd = False
for channel in repo_tree:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT,
channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path,
"packages_size")
sources = repo_tree[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
# check all repomd files were downloaded
for yum_repo in yum_repos:
new_repomd = os.path.join(yum_repo.repo.basecachedir,
yum_repo.name, "repomd.xml.new")
if not os.path.isfile(new_repomd):
is_missing_repomd = True
# packages_num file exists and all cached repomd files are up to date => skip
if os.path.isfile(packages_num_path) and os.path.isfile(
packages_size_path) and all(
[x.repomd_up_to_date() for x in yum_repos]):
progress_bar.log(True, None)
continue
update_channel = False
for yum_repo in yum_repos:
# use new repomd
new_repomd = os.path.join(yum_repo.repo.basecachedir,
yum_repo.name, "repomd.xml.new")
if os.path.isfile(new_repomd):
update_channel = True
os.rename(
new_repomd,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
"repomd.xml"))
else:
# it wasn't downloaded
continue
for path, checksum_pair in yum_repo.get_metadata_paths():
params = {}
yum_repo.set_download_parameters(
params,
path,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
os.path.basename(path)),
checksum_type=checksum_pair[0],
checksum_value=checksum_pair[1])
downloader.add(params)
to_download_count += 1
# If there is at least one repo with new repomd, pass through this channel
if update_channel:
repo_tree_to_update[channel] = sources
progress_bar.log(True, None)
log2background(0, "Comparing repomd finished.")
progress_bar = ProgressBarLogger("Downloading metadata:",
to_download_count)
downloader.set_log_obj(progress_bar)
downloader.set_force(False)
log2background(0, "Downloading metadata started.")
downloader.run()
log2background(0, "Downloading metadata finished.")
progress_bar = ProgressBarLogger("Counting packages: ",
len(repo_tree_to_update))
log2background(0, "Counting packages started.")
for channel in repo_tree_to_update:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT,
channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path,
"packages_size")
sources = repo_tree_to_update[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
packages = {}
for yum_repo in yum_repos:
for pkg in yum_repo.raw_list_packages():
nvrea = str(pkg)
packages[nvrea] = pkg.packagesize
# create directory for repo data if it doesn't exist
try:
os.makedirs(cdn_repodata_path)
except OSError:
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST and os.path.isdir(
cdn_repodata_path):
pass
else:
raise
f_num_out = open(packages_num_path, 'w')
f_size_out = open(packages_size_path, 'w')
try:
f_num_out.write(str(len(packages)))
f_size_out.write(str(sum(packages.values())))
finally:
if f_num_out is not None:
f_num_out.close()
if f_size_out is not None:
f_size_out.close()
# Delete cache to save space
for yum_repo in yum_repos:
yum_repo.clear_cache(keep_repomd=True)
progress_bar.log(True, None)
log2background(0, "Counting packages finished.")
end_time = datetime.now()
log(0, "Total time: %s" % str(end_time - start_time).split('.')[0])
if is_missing_repomd:
raise CountingPackagesError(
"Cannot download some repomd.xml files. "
"Please, check /var/log/rhn/cdnsync.log for details")
def count_packages(self, channels=None):
start_time = datetime.now()
reposync.clear_ssl_cache()
# Both entitled channels and custom channels with null-org repositories.
channel_list = self._list_available_channels()
if not channel_list:
error_messages = self._msg_array_if_not_activated()
if error_messages:
log(0, "\n".join(error_messages))
sys.exit(1)
channel_list.extend([c for c in self.synced_channels if self.synced_channels[c]])
# Only some channels specified by parameter
if channels:
new_channel_list = []
for channel in channels:
new_channel_list.extend(fnmatch.filter(channel_list, channel))
channel_list = list(set(new_channel_list))
log(0, "Number of channels: %d" % len(channel_list))
# Prepare repositories
repo_tree = {}
repository_count = 0
for channel in channel_list:
sources = self.cdn_repository_manager.get_content_sources(channel)
# Custom channel
if not sources:
repos = self.cdn_repository_manager.list_associated_repos(channel)
sources = []
for index, repo in enumerate(sorted(repos)):
repo_label = "%s-%d" % (channel, index)
sources.append({'relative_url': repo, 'pulp_repo_label_v2': repo_label})
repository_count += len(sources)
repo_tree[channel] = sources
log(0, "Number of repositories: %d" % repository_count)
downloader = ThreadedDownloader()
for channel in repo_tree:
for source in repo_tree[channel]:
yum_repo = self._create_yum_repo(source)
params = {}
yum_repo.set_download_parameters(params, "repodata/repomd.xml",
os.path.join(yum_repo.repo.basecachedir,
yum_repo.name, "repomd.xml.new"))
downloader.add(params)
progress_bar = ProgressBarLogger("Downloading repomd: ", repository_count)
downloader.set_log_obj(progress_bar)
# Overwrite existing files
downloader.set_force(True)
log2background(0, "Downloading repomd started.")
downloader.run()
log2background(0, "Downloading repomd finished.")
progress_bar = ProgressBarLogger("Comparing repomd: ", len(repo_tree))
to_download_count = 0
repo_tree_to_update = {}
log2background(0, "Comparing repomd started.")
is_missing_repomd = False
for channel in repo_tree:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path, "packages_size")
sources = repo_tree[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
# check all repomd files were downloaded
for yum_repo in yum_repos:
new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new")
if not os.path.isfile(new_repomd):
is_missing_repomd = True
# packages_num file exists and all cached repomd files are up to date => skip
if os.path.isfile(packages_num_path) and os.path.isfile(packages_size_path) and all(
[x.repomd_up_to_date() for x in yum_repos]):
progress_bar.log(True, None)
continue
update_channel = False
for yum_repo in yum_repos:
# use new repomd
new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new")
if os.path.isfile(new_repomd):
update_channel = True
os.rename(new_repomd,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml"))
else:
# it wasn't downloaded
continue
for path, checksum_pair in yum_repo.get_metadata_paths():
params = {}
yum_repo.set_download_parameters(params, path,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
os.path.basename(path)),
checksum_type=checksum_pair[0], checksum_value=checksum_pair[1])
downloader.add(params)
to_download_count += 1
# If there is at least one repo with new repomd, pass through this channel
if update_channel:
repo_tree_to_update[channel] = sources
progress_bar.log(True, None)
log2background(0, "Comparing repomd finished.")
progress_bar = ProgressBarLogger("Downloading metadata:", to_download_count)
downloader.set_log_obj(progress_bar)
downloader.set_force(False)
log2background(0, "Downloading metadata started.")
downloader.run()
log2background(0, "Downloading metadata finished.")
progress_bar = ProgressBarLogger("Counting packages: ", len(repo_tree_to_update))
log2background(0, "Counting packages started.")
for channel in repo_tree_to_update:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path, "packages_size")
sources = repo_tree_to_update[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
packages = {}
for yum_repo in yum_repos:
for pkg in yum_repo.raw_list_packages():
nvrea = str(pkg)
packages[nvrea] = pkg.packagesize
# create directory for repo data if it doesn't exist
try:
os.makedirs(cdn_repodata_path)
except OSError:
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST and os.path.isdir(cdn_repodata_path):
pass
else:
raise
f_num_out = open(packages_num_path, 'w')
f_size_out = open(packages_size_path, 'w')
try:
f_num_out.write(str(len(packages)))
f_size_out.write(str(sum(packages.values())))
finally:
if f_num_out is not None:
f_num_out.close()
if f_size_out is not None:
f_size_out.close()
# Delete cache to save space
for yum_repo in yum_repos:
yum_repo.clear_cache(keep_repomd=True)
progress_bar.log(True, None)
log2background(0, "Counting packages finished.")
end_time = datetime.now()
log(0, "Total time: %s" % str(end_time - start_time).split('.')[0])
if is_missing_repomd:
raise CountingPackagesError("Cannot download some repomd.xml files. "
"Please, check /var/log/rhn/cdnsync.log for details")