def download_payloads(payloads, drpm):
# download packages
drpm.err.clear()
targets = [pload.librepo_target() for pload in payloads]
errs = _DownloadErrors()
try:
librepo.download_packages(targets, failfast=True)
except librepo.LibrepoException as e:
errs.fatal = e.args[1] or '<unspecified librepo error>'
drpm.wait()
# process downloading errors
errs.recoverable = drpm.err.copy()
for tgt in targets:
err = tgt.err
if err is None or err.startswith('Not finished'):
continue
payload = tgt.cbdata
pkg = payload.pkg
if err == 'Already downloaded':
errs.skipped.add(pkg)
continue
errs.irrecoverable[pkg] = [err]
return errs
def test_download_packages_with_offline_enabled_01(self):
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h = librepo.Handle()
h.urls = [url]
h.repotype = librepo.YUMREPO
h.offline = True
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs)
# Offline is True, no package should be downloaded successfully
for pkg in pkgs:
self.assertTrue(pkg.err)
h.offline = False
librepo.download_packages(pkgs)
# Offline is False, the package should be downloaded successfully
for pkg in pkgs:
self.assertFalse(pkg.err)
def test_download_packages_from_different_repos(self):
h1 = librepo.Handle()
h2 = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h1.urls = [url]
h1.repotype = librepo.LR_YUMREPO
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
h2.urls = [url]
h2.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h1,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget(config.PACKAGE_03_01,
handle=h2,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.handle == h1)
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
pkg = pkgs[1]
self.assertTrue(pkg.handle == h2)
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_fastestmirror_enabled_1(self):
h = librepo.Handle()
cbdata = {
"called": False,
}
def fastestmirrorstatuscallback(userdata, stage, data):
cbdata["called"] = True
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.fastestmirror = True
h.fastestmirrorcb = fastestmirrorstatuscallback
h.download(config.PACKAGE_01_01)
pkgs = []
pkgs.append(h.new_packagetarget(config.PACKAGE_01_01,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
# There is only one mirror, fastestmirror
# detection should be skiped
self.assertFalse(cbdata["called"])
def test_download_packages_with_checksum_check(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertEqual(pkg.relative_url, config.PACKAGE_01_01)
self.assertEqual(pkg.dest, self.tmpdir)
self.assertEqual(pkg.base_url, None)
self.assertEqual(pkg.checksum_type, librepo.SHA256)
self.assertEqual(pkg.checksum, config.PACKAGE_01_01_SHA256)
self.assertEqual(pkg.resume, 0)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(pkg.local_path, os.path.join(self.tmpdir,
os.path.basename(config.PACKAGE_01_01)))
self.assertTrue(pkg.err is None)
def test_download_packages_with_callback(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
cbdata = {'called': 0}
def cb(cbdata, total, downloaded):
cbdata["called"] += 1
cbdata["total"] = total
cbdata["downloaded"] = downloaded
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
progresscb=cb,
cbdata=cbdata))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(pkg.progresscb == cb)
self.assertTrue(pkg.cbdata == cbdata)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertTrue(cbdata["called"] > 0)
self.assertTrue(cbdata["downloaded"] > 0)
self.assertTrue(cbdata["total"] > 0)
self.assertEqual(cbdata["downloaded"], cbdata["total"])
def _download_payloads(payloads, drpm):
# download packages
def _download_sort_key(payload):
return not hasattr(payload, 'delta')
drpm.err.clear()
targets = [pload._librepo_target()
for pload in sorted(payloads, key=_download_sort_key)]
errs = _DownloadErrors()
try:
librepo.download_packages(targets, failfast=True)
except librepo.LibrepoException as e:
errs._fatal = e.args[1] or '<unspecified librepo error>'
drpm.wait()
# process downloading errors
errs._recoverable = drpm.err.copy()
for tgt in targets:
err = tgt.err
if err is None or err.startswith('Not finished'):
continue
payload = tgt.cbdata
pkg = payload.pkg
if err == _('Already downloaded'):
errs._skipped.add(pkg)
continue
pkg.repo._md_expire_cache()
errs._irrecoverable[pkg] = [err]
return errs
def test_download_packages_with_fastestmirror_enabled_2(self):
h = librepo.Handle()
cbdata = {
"called": False,
"detection": False,
}
def fastestmirrorstatuscallback(cbdata, stage, data):
cbdata["called"] = True
if stage == librepo.FMSTAGE_DETECTION:
cbdata["detection"] = True
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url, "http://foobarblabla"])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, self.tmpdir)
h.fastestmirror = True
h.fastestmirrordata = cbdata
h.fastestmirrorcb = fastestmirrorstatuscallback
h.download(config.PACKAGE_01_01)
pkgs = []
pkgs.append(h.new_packagetarget(config.PACKAGE_01_01,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
# There is only one mirror, fastestmirror
# detection should be skiped
self.assertTrue(cbdata["called"])
self.assertTrue(cbdata["detection"])
def test_download_packages_from_different_mirrors_1(self):
cbdata = {
"failed_urls" : [],
}
def mirrorfailurecb(userdata, msg, url):
cbdata["failed_urls"].append(url)
return None
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)
h.mirrorlist = "%s%s" % (self.MOCKURL, config.METALINK_MIRRORS_01)
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.fastestmirror = True
pkgs = []
for P in [config.PACKAGE_01_01, config.PACKAGE_03_01, config.PACKAGE_04_01]:
pkgs.append(librepo.PackageTarget(P,
handle=h,
dest=self.tmpdir,
mirrorfailurecb=mirrorfailurecb))
librepo.download_packages(pkgs, failfast=True)
# the mirror list should be 1, 3, 4. The metalink file
# defines preference order. This is the order in which
# the URLs are listed in the file
self.assertEquals(h.mirrors, [url1, url2, url3])
# YUM 01 contains P_01
# YUM 03 contains P_03
# YUM_04 contains P_04
# Mirror order of preference is 01 03, 04
# When a package fails to download, librepo moves to the next mirror;
# after a successfull download of the package librepo should continue trying
# to download the remaining packages from the first mirror. e.g.
# always try to download from the fastest mirror containing the package
# Expected download sequence (restart from fastest mirror):
# - P_01 from YUM_01 - PASS
# - P_03 from YUM_01 - FAIL
# - P_03 from YUM_03 - PASS
# - P_04 from YUM_04 - FAIL
# - P_04 from YUM_03 - FAIL
# - P_04 from YUM_04 - PASS
self.assertEquals(len(cbdata["failed_urls"]), 3)
self.assertEquals(cbdata["failed_urls"][0], url1+config.PACKAGE_03_01)
self.assertEquals(cbdata["failed_urls"][1], url1+config.PACKAGE_04_01)
self.assertEquals(cbdata["failed_urls"][2], url2+config.PACKAGE_04_01)
# assert that all packages have been downloaded
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_00(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
librepo.download_packages(pkgs)
def test_download_packages_mirror_penalization_01(self):
# This test is useful for mirror penalization testing
# with debug output on:
# LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1"
#
# You can use it to check adaptive mirror sorting behavior
#
# E.g. from librepo checkout dir:
# LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1" LIBREPO_DEBUG="1" PYTHONPATH=`readlink -f ./build/librepo/python/` python3 -m unittest discover -s tests/python/ -p "test_yum_package_downloading.py" -k "test_download_packages_mirror_penalization_01"
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)
h.urls = [url1, url2, url3]
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.allowedmirrorfailures = -1
h.adaptivemirrorsorting = True
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_03_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_03_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_04_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs, failfast=False)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume_02(self):
# If download that should be resumed fails,
# the original file should not be modified or deleted
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
fn = os.path.join(self.tmpdir, "package.rpm")
# Download first 10 bytes of the package
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=fn,
resume=False,
byterangeend=9))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
# Mark the file as it was downloaded by Librepo
# Otherwise librepo refuse to resume
try:
xattr.setxattr(pkg.local_path,
"user.Librepo.DownloadInProgress".encode("utf-8"),
"".encode("utf-8"))
except IOError as err:
if err.errno == 95:
self.skipTest('extended attributes are not supported')
raise
# Now try to resume from bad URL
pkgs = []
pkgs.append(librepo.PackageTarget("bad_path.rpm",
handle=h,
dest=fn,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
# Download should fail (path is bad, the file doesn't exist)
self.assertRaises(librepo.LibrepoException, librepo.download_packages,
pkgs, failfast=True)
# The package should exists (should not be removed or changed)
pkg = pkgs[0]
self.assertTrue(pkg.err)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
self.assertEqual(fchksum, fchksum_new)
def test_download_packages_00(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
librepo.download_packages(pkgs)
def test_download_packages_00(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
librepo.download_packages(pkgs)
def test_download_packages_with_resume_02(self):
# If download that should be resumed fails,
# the original file should not be modified or deleted
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
fn = os.path.join(self.tmpdir, "package.rpm")
# Download first 10 bytes of the package
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=fn,
resume=False,
byterangeend=9))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
# Mark the file as it was downloaded by Librepo
# Otherwise librepo refuse to resume
xattr.setxattr(pkg.local_path, "user.Librepo.DownloadInProgress", "")
# Now try to resume from bad URL
pkgs = []
pkgs.append(
librepo.PackageTarget("bad_path.rpm",
handle=h,
dest=fn,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
# Download should fail (path is bad, the file doesn't exist)
self.assertRaises(librepo.LibrepoException,
librepo.download_packages,
pkgs,
failfast=True)
# The package should exists (should not be removed or changed)
pkg = pkgs[0]
self.assertTrue(pkg.err)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum_new = hashlib.md5(open(pkg.local_path,
"rb").read()).hexdigest()
self.assertEqual(fchksum, fchksum_new)
def test_download_packages_without_handle(self):
complete_url = "%s%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH,
config.PACKAGE_01_01)
pkgs = []
pkgs.append(librepo.PackageTarget(complete_url, dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_mirror_penalization_01(self):
# This test is useful for mirror penalization testing
# with debug output on:
# LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1"
#
# You can use it to check adaptive mirror sorting behavior
#
# E.g. from librepo checkout dir:
# LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1" LIBREPO_DEBUG="1" PYTHONPATH=`readlink -f ./build/librepo/python/python2/` nosetests -s -v tests/python/tests/test_yum_package_downloading.py:TestCaseYumPackagesDownloading.test_download_packages_mirror_penalization_01
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)
h.urls = [url1, url2, url3]
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.allowedmirrorfailures = -1
h.adaptivemirrorsorting = True
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_03_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget(config.PACKAGE_03_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget(config.PACKAGE_04_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs, failfast=False)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_without_handle(self):
complete_url = "%s%s%s" % (self.MOCKURL,
config.REPO_YUM_01_PATH,
config.PACKAGE_01_01)
pkgs = []
pkgs.append(librepo.PackageTarget(complete_url,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_01(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume_02(self):
# If download that should be resumed fails,
# the original file should not be modified or deleted
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
fn = os.path.join(self.tmpdir, "package.rpm")
# Download first 10 bytes of the package
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=fn,
resume=False,
byterangeend=9))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
# Now try to resume from bad URL
pkgs = []
pkgs.append(librepo.PackageTarget("bad_path.rpm",
handle=h,
dest=fn,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
# Download should fail (path is bad, the file doesn't exist)
self.assertRaises(librepo.LibrepoException, librepo.download_packages,
pkgs, failfast=True)
# The package should exists (should not be removed or changed)
pkg = pkgs[0]
self.assertTrue(pkg.err)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
self.assertEqual(fchksum, fchksum_new)
def test_download_packages_01(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_bad_checksum(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
checksum_type=librepo.SHA256,
checksum="badchecksum"))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err)
def test_download_packages_with_baseurl(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = ["."]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
base_url=url))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_bad_checksum(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
checksum_type=librepo.SHA256,
checksum="badchecksum"))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err)
def test_download_packages_with_baseurl(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = ["."]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
base_url=url))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def _urlopen_progress(url, conf):
handle = _non_repo_handle(conf)
handle.repotype = librepo.LR_YUMREPO
handle.setopt(librepo.LRO_URLS, os.path.dirname(url))
progress = dnf.cli.progress.MultiFileProgressMeter(fo=sys.stdout)
pload = dnf.repo.RemoteRPMPayload(url, conf, handle, progress)
if os.path.exists(pload.local_path):
return pload.local_path
est_remote_size = sum([pload.download_size])
progress.start(1, est_remote_size)
targets = [pload._librepo_target()]
try:
librepo.download_packages(targets, failfast=True)
except librepo.LibrepoException as e:
if conf.strict:
raise IOError(e.args[1])
logger.error(e.args[1])
return pload.local_path
def _urlopen_progress(url, conf):
handle = _non_repo_handle(conf)
handle.repotype = librepo.LR_YUMREPO
handle.setopt(librepo.LRO_URLS, os.path.dirname(url))
progress = dnf.cli.progress.MultiFileProgressMeter(fo=sys.stdout)
pload = dnf.repo.RemoteRPMPayload(url, conf, handle, progress)
if os.path.exists(pload.local_path):
return pload.local_path
est_remote_size = sum([pload.download_size])
progress.start(1, est_remote_size)
targets = [pload._librepo_target()]
try:
librepo.download_packages(targets, failfast=True)
except librepo.LibrepoException as e:
if conf.strict:
raise IOError(e.args[1])
logger.error(e.args[1])
return pload.local_path
def test_download_packages_with_expectedsize(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
expectedsize = 1057084
pkgs = []
pkgs.append(h.new_packagetarget(config.PACKAGE_01_01,
dest=self.tmpdir,
expectedsize=expectedsize))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertEqual(pkg.expectedsize, expectedsize)
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def download_package(self, location, checksum_type, checksum):
if self.librepo_handle.local:
local_path = os.path.join(self._root_path, location)
logger.debug('Using package %s from local filesystem directly', local_path)
return local_path
logger.debug('Loading package %s from repo %s', location, self.name)
target = librepo.PackageTarget(location,
checksum_type=librepo.checksum_str_to_type(checksum_type),
checksum=checksum,
dest=self._root_path,
handle=self.librepo_handle)
librepo.download_packages([target])
if target.err and target.err == 'Already downloaded':
logger.debug('Already downloaded %s', target.local_path)
elif target.err:
raise PackageDownloadError('Failed to download %s from repo %s: %s'
% (location, self.name, target.err))
else:
logger.debug('Saved as %s', target.local_path)
return target.local_path
def test_download_packages_with_expectedsize(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
expectedsize = 1057084
pkgs = []
pkgs.append(h.new_packagetarget(config.PACKAGE_01_01,
dest=self.tmpdir,
expectedsize=expectedsize))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertEqual(pkg.expectedsize, expectedsize)
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def download_package(self, location, baseurl, checksum_type, checksum):
if self.librepo_handle.local:
local_path = os.path.join(self._root_path, location)
logger.debug('Using package %s from local filesystem directly', local_path)
return local_path
logger.debug('Loading package %s from repo %s', location, self.name)
target = librepo.PackageTarget(location,
base_url=baseurl,
checksum_type=librepo.checksum_str_to_type(checksum_type),
checksum=checksum,
dest=self._root_path,
handle=self.librepo_handle)
librepo.download_packages([target])
if target.err and target.err == 'Already downloaded':
logger.debug('Already downloaded %s', target.local_path)
elif target.err:
raise PackageDownloadError('Failed to download %s from repo %s: %s'
% (location, self.name, target.err))
else:
logger.debug('Saved as %s', target.local_path)
return target.local_path
def test_download_packages_one_url_is_bad_01(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget("so_bad_url_of_foo_rpm.rpm",
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs)
self.assertTrue(pkgs[0].err is None)
self.assertTrue(os.path.isfile(pkgs[0].local_path))
self.assertTrue(pkgs[1].err is not None)
self.assertFalse(os.path.isfile(pkgs[1].local_path))
def test_download_packages_one_url_is_bad_01(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget("so_bad_url_of_foo_rpm.rpm",
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs)
self.assertTrue(pkgs[0].err is None)
self.assertTrue(os.path.isfile(pkgs[0].local_path))
self.assertTrue(pkgs[1].err is not None)
self.assertTrue(os.path.isfile(pkgs[1].local_path))
def download_package(step):
remove_fn(CACHE)
for filename in world.filenames:
remove_fn(filename)
h = librepo.Handle()
h.repotype = librepo.YUMREPO
for opt, val in world.handle_options.iteritems():
setattr(h, opt, val)
# Download the package for the first time
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages, failfast=True)
world.t1 = time.time() - t
for package in packages:
assert (package.err is None)
assert not os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
# Download the package for the second time
h.fastestmirror = True
h.fastestmirrorcache = CACHE
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages)
world.t2 = time.time() - t
for package in packages:
assert (package.err is None)
assert os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
# Third download of the package
h.fastestmirror = True
h.fastestmirrorcache = CACHE
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages)
world.t3 = time.time() - t
for package in packages:
assert (package.err is None)
assert os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
def download_package(step):
remove_fn(CACHE)
for filename in world.filenames:
remove_fn(filename)
h = librepo.Handle()
h.repotype = librepo.YUMREPO
for opt, val in world.handle_options.iteritems():
setattr(h, opt, val)
# Download the package for the first time
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages, failfast=True)
world.t1 = time.time() - t
for package in packages:
assert (package.err is None)
assert not os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
# Download the package for the second time
h.fastestmirror = True
h.fastestmirrorcache = CACHE
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages)
world.t2 = time.time() - t
for package in packages:
assert (package.err is None)
assert os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
# Third download of the package
h.fastestmirror = True
h.fastestmirrorcache = CACHE
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages)
world.t3 = time.time() - t
for package in packages:
assert (package.err is None)
assert os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
def test_download_packages_02(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
dest = os.path.join(self.tmpdir, "foo-haha-lol.rpm")
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=dest))
librepo.download_packages(pkgs)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertTrue(os.path.isfile(dest))
def test_download_packages_02_with_failfast(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
dest = os.path.join(self.tmpdir, "foo-haha-lol.rpm")
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=dest))
librepo.download_packages(pkgs, failfast=True)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertTrue(os.path.isfile(dest))
def test_download_packages_02_with_failfast(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
dest = os.path.join(self.tmpdir, "foo-haha-lol.rpm")
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=dest))
librepo.download_packages(pkgs, failfast=True)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertTrue(os.path.isfile(dest))
def test_download_packages_with_fastestmirror_enabled_2(self):
h = librepo.Handle()
cbdata = {
"called": False,
"detection": False,
}
def fastestmirrorstatuscallback(cbdata, stage, data):
cbdata["called"] = True
if stage == librepo.FMSTAGE_DETECTION:
cbdata["detection"] = True
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url, "http://foobarblabla"]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.fastestmirror = True
h.fastestmirrordata = cbdata
h.fastestmirrorcb = fastestmirrorstatuscallback
h.download(config.PACKAGE_01_01)
pkgs = []
pkgs.append(h.new_packagetarget(config.PACKAGE_01_01,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
# There is only one mirror, fastestmirror
# detection should be skiped
self.assertTrue(cbdata["called"])
self.assertTrue(cbdata["detection"])
def download_payloads(payloads, drpm):
# download packages
drpm.err.clear()
targets = [pload.librepo_target() for pload in payloads]
errs = _DownloadErrors()
try:
librepo.download_packages(targets, failfast=True)
except librepo.LibrepoException as e:
errs.fatal = e.args[1] or '<unspecified librepo error>'
drpm.wait()
# process downloading errors
errs.recoverable = drpm.err.copy()
for tgt in targets:
err = tgt.err
if err is None:
continue
if err == 'Already downloaded' or err.startswith('Not finished'):
continue
payload = tgt.cbdata
pkg = payload.pkg
errs.irrecoverable[pkg] = [err]
return errs
def test_download_packages_with_resume(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
resume=False))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
# Again with resume on
# Because the package already exists and checksum matches,
# then download should not be performed!
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertEqual(pkg.err, "Already downloaded")
self.assertTrue(os.path.isfile(pkg.local_path))
## Failfast ignore this type of error (Already downloaded error)
librepo.download_packages(pkgs, failfast=True)
pkg = pkgs[0]
self.assertEqual(pkg.err, "Already downloaded")
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
resume=False))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
# Again with resume on
# Because the package already exists and checksum matches,
# then download should not be performed!
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertEqual(pkg.err, "Already downloaded")
self.assertTrue(os.path.isfile(pkg.local_path))
## Failfast ignore this type of error (Already downloaded error)
librepo.download_packages(pkgs, failfast=True)
pkg = pkgs[0]
self.assertEqual(pkg.err, "Already downloaded")
self.assertTrue(os.path.isfile(pkg.local_path))
# Prepare handle
h = librepo.Handle()
h.urls = [
"http://mirror.karneval.cz/pub/linux/fedora/linux/releases/20/Fedora/x86_64/os/"
]
h.repotype = librepo.YUMREPO
# Prepare list of targets
packages = []
target = librepo.PackageTarget("Packages/a/abrt-2.1.9-1.fc20.x86_64.rpm",
handle=h,
dest="abrt.header",
byterangestart=1384,
byterangeend=100204)
packages.append(target)
librepo.download_packages(packages)
for target in packages:
print "### %s: %s" % (target.local_path, target.err or "OK")
print "Relative URL: ", target.relative_url
print "Destination: ", target.dest
print "Base URL: ", target.base_url
print "Checksum type: ", target.checksum_type
print "Expected checksum: ", target.checksum
print "Resume: ", bool(target.resume)
print "Local path: ", target.local_path
print "Error: ", target.err
print
handle=h,
expectedsize=333333333333333,
cbdata="beaker-client-0.14.1-1.fc18.noarch.rpm (bad size)",
endcb=endcb,
mirrorfailurecb=mirrorfailurecb,
)
packages.append(target)
target = librepo.PackageTarget(
"rhts-4.56-1.fc17.src.rpm_bad_filename",
handle=h,
cbdata="rhts-4.56-1.fc17.src.rpm_bad_filename (bad path)",
endcb=endcb,
mirrorfailurecb=mirrorfailurecb,
)
packages.append(target)
librepo.download_packages(packages, failfast=False)
for target in packages:
print "### %s: %s" % (target.local_path, target.err or "OK")
print "Relative URL: ", target.relative_url
print "Destination: ", target.dest
print "Base URL: ", target.base_url
print "Checksum type: ", target.checksum_type
print "Expected checksum: ", target.checksum
print "Resume: ", bool(target.resume)
print "Local path: ", target.local_path
print "Error: ", target.err
print
def test_download_packages_with_resume_02(self):
return # This test causes issues on Fedora rawhide (F26)
# Flask/werkzeug/SocketServer/socket fails on Broken pipe
# and causes next test (test_download_packages_without_handle)
# to always fail
# If download that should be resumed fails,
# the original file should not be modified or deleted
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
fn = os.path.join(self.tmpdir, "package.rpm")
# Download first 10 bytes of the package
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=fn,
resume=False,
byterangeend=9))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
# Mark the file as it was downloaded by Librepo
# Otherwise librepo refuse to resume
try:
xattr.setxattr(pkg.local_path,
"user.Librepo.DownloadInProgress".encode("utf-8"),
"".encode("utf-8"))
except IOError as err:
if err.errno == 95:
self.skipTest('extended attributes are not supported')
raise
# Now try to resume from bad URL
pkgs = []
pkgs.append(librepo.PackageTarget("bad_path.rpm",
handle=h,
dest=fn,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
# Download should fail (path is bad, the file doesn't exist)
self.assertRaises(librepo.LibrepoException, librepo.download_packages,
pkgs, failfast=True)
# The package should exists (should not be removed or changed)
pkg = pkgs[0]
self.assertTrue(pkg.err)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
self.assertEqual(fchksum, fchksum_new)
def test_download_packages_from_different_mirrors_1(self):
cbdata = {
"failed_urls": [],
}
def mirrorfailurecb(userdata, msg, url):
cbdata["failed_urls"].append(url)
return None
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)
h.mirrorlist = "%s%s" % (self.MOCKURL, config.METALINK_MIRRORS_01)
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.fastestmirror = True
pkgs = []
for P in [
config.PACKAGE_01_01, config.PACKAGE_03_01,
config.PACKAGE_04_01
]:
pkgs.append(
librepo.PackageTarget(P,
handle=h,
dest=self.tmpdir,
mirrorfailurecb=mirrorfailurecb))
librepo.download_packages(pkgs, failfast=True)
# the mirror list should be 1, 3, 4. The metalink file
# defines preference order. This is the order in which
# the URLs are listed in the file
self.assertEqual(h.mirrors, [url1, url2, url3])
# YUM 01 contains P_01
# YUM 03 contains P_03
# YUM_04 contains P_04
# Mirror order of preference is 01 03, 04
# When a package fails to download, librepo moves to the next mirror;
# after a successfull download of the package librepo should continue trying
# to download the remaining packages from the first mirror. e.g.
# always try to download from the fastest mirror containing the package
# Expected download sequence (restart from fastest mirror):
# - P_01 from YUM_01 - PASS
# - P_03 from YUM_01 - FAIL
# - P_03 from YUM_03 - PASS
# - P_04 from YUM_04 - FAIL
# - P_04 from YUM_03 - FAIL
# - P_04 from YUM_04 - PASS
self.assertEqual(len(cbdata["failed_urls"]), 3)
self.assertEqual(cbdata["failed_urls"][0], url1 + config.PACKAGE_03_01)
self.assertEqual(cbdata["failed_urls"][1], url1 + config.PACKAGE_04_01)
self.assertEqual(cbdata["failed_urls"][2], url2 + config.PACKAGE_04_01)
# assert that all packages have been downloaded
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def remove_package():
if os.path.exists(PACKAGE):
print "Removing %s" % PACKAGE
os.remove(PACKAGE)
remove_cache()
remove_package()
print "\n1st run: Basic download"
h = librepo.Handle()
h.interruptible = True
h.metalinkurl = METALINKURL
h.repotype = librepo.YUMREPO
packages = [librepo.PackageTarget(PACKAGEPATH+PACKAGE, handle=h)]
t = time.time()
librepo.download_packages(packages)
t1 = time.time() - t
print "### %s: %s (Time: %s)" % (
packages[0].local_path, packages[0].err or "OK", t1)
remove_package()
print "\n2nd run: Download with fastestmirror - cache will be builded"
h = librepo.Handle()
h.interruptible = True
h.metalinkurl = METALINKURL
h.repotype = librepo.YUMREPO
h.fastestmirror = True
h.fastestmirrorcache = CACHE
h.fastestmirrorcb = fastestmirrorstatuscallback
packages = [librepo.PackageTarget(PACKAGEPATH+PACKAGE, handle=h)]
target = librepo.PackageTarget(
"beaker-client-0.14.1-1.fc18.noarch.rpm",
handle=h,
expectedsize=333333333333333,
cbdata="beaker-client-0.14.1-1.fc18.noarch.rpm (bad size)",
endcb=endcb,
mirrorfailurecb=mirrorfailurecb)
packages.append(target)
target = librepo.PackageTarget(
"rhts-4.56-1.fc17.src.rpm_bad_filename",
handle=h,
cbdata="rhts-4.56-1.fc17.src.rpm_bad_filename (bad path)",
endcb=endcb,
mirrorfailurecb=mirrorfailurecb)
packages.append(target)
librepo.download_packages(packages, failfast=False)
for target in packages:
print "### %s: %s" % (target.local_path, target.err or "OK")
print "Relative URL: ", target.relative_url
print "Destination: ", target.dest
print "Base URL: ", target.base_url
print "Checksum type: ", target.checksum_type
print "Expected checksum: ", target.checksum
print "Resume: ", bool(target.resume)
print "Local path: ", target.local_path
print "Error: ", target.err
print
