def librepo_target(self):
pkg = self.pkg
pkgdir = pkg.repo.pkgdir
dnf.util.ensure_dir(pkgdir)
target_dct = {
'handle' : pkg.repo.get_handle(),
'dest' : pkgdir,
'resume' : True,
'cbdata' : self,
'progresscb' : self._progress_cb,
'endcb' : self._end_cb,
'mirrorfailurecb' : self._mirrorfail_cb,
}
target_dct.update(self._target_params())
return librepo.PackageTarget(**target_dct)
def test_download_packages_01(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume_02(self):
# If download that should be resumed fails,
# the original file should not be modified or deleted
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
h.allowedmirrorfailures = 1
fn = os.path.join(self.tmpdir, "package.rpm")
# Make fake unfinished download
CONTENT = "0123456789"
open(fn, "w").write(CONTENT)
# Mark the file as it was downloaded by Librepo
# Otherwise librepo refuse to resume
try:
xattr.setxattr(fn,
"user.Librepo.DownloadInProgress".encode("utf-8"),
"".encode("utf-8"))
except IOError as err:
if err.errno == 95:
self.skipTest('extended attributes are not supported')
raise
# Now try to resume from bad URL
pkgs = []
pkgs.append(librepo.PackageTarget("bad_path.rpm",
handle=h,
dest=fn,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
# Download should fail (path is bad, the file doesn't exist)
self.assertRaises(librepo.LibrepoException, librepo.download_packages,
pkgs, failfast=True)
# The package should exists (should not be removed or changed)
pkg = pkgs[0]
self.assertTrue(pkg.err)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(open(pkg.local_path, "rb").read(), CONTENT.encode('utf-8'))
def test_download_packages_with_baseurl(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, ["."])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
base_url=url))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_bad_checksum(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
checksum_type=librepo.SHA256,
checksum="badchecksum"))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err)
def test_download_packages_with_bad_checksum_with_failfast(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
checksum_type=librepo.SHA256,
checksum="badchecksum"))
self.assertRaises(librepo.LibrepoException, librepo.download_packages,
pkgs, failfast=True)
pkg = pkgs[0]
self.assertTrue(pkg.err)
def download_package(self, location, checksum_type, checksum):
if self.librepo_handle.local:
local_path = os.path.join(self._root_path, location)
logger.debug('Using package %s from local filesystem directly', local_path)
return local_path
logger.debug('Loading package %s from repo %s', location, self.name)
target = librepo.PackageTarget(location,
checksum_type=librepo.checksum_str_to_type(checksum_type),
checksum=checksum,
dest=self._root_path,
handle=self.librepo_handle)
librepo.download_packages([target])
if target.err and target.err == 'Already downloaded':
logger.debug('Already downloaded %s', target.local_path)
elif target.err:
raise PackageDownloadError('Failed to download %s from repo %s: %s'
% (location, self.name, target.err))
else:
logger.debug('Saved as %s', target.local_path)
return target.local_path
def test_packagetarget_sanity_01(self):
h = librepo.Handle()
cbdata = {"a": "b"}
def progresscb(a, b, c):
return 0
def endcb(a):
return
def mirrorfailurecb(a, b):
return 0
t = librepo.PackageTarget("foo",
dest="bar",
checksum_type=librepo.CHECKSUM_SHA512,
checksum="xxx",
expectedsize=123,
base_url="basefoo",
resume=True,
progresscb=progresscb,
cbdata=cbdata,
handle=h,
endcb=endcb,
mirrorfailurecb=mirrorfailurecb)
self.assertEqual(t.relative_url, "foo")
self.assertEqual(t.dest, "bar")
self.assertEqual(t.checksum_type, librepo.CHECKSUM_SHA512)
self.assertEqual(t.checksum, "xxx")
self.assertEqual(t.expectedsize, 123)
self.assertEqual(t.base_url, "basefoo")
self.assertEqual(t.resume, True)
self.assertEqual(t.progresscb, progresscb)
self.assertEqual(t.cbdata, cbdata)
self.assertEqual(t.handle, h)
self.assertEqual(t.endcb, endcb)
self.assertEqual(t.mirrorfailurecb, mirrorfailurecb)
librepo.set_debug_log_handler(debug_function)
# Prepare handle
h = librepo.Handle()
h.urls = [
"http://mirror.karneval.cz/pub/linux/fedora/linux/releases/20/Fedora/x86_64/os/"
]
h.repotype = librepo.YUMREPO
# Prepare list of targets
packages = []
target = librepo.PackageTarget("Packages/a/abrt-2.1.9-1.fc20.x86_64.rpm",
handle=h,
dest="abrt.header",
byterangestart=1384,
byterangeend=100204)
packages.append(target)
librepo.download_packages(packages)
for target in packages:
print "### %s: %s" % (target.local_path, target.err or "OK")
print "Relative URL: ", target.relative_url
print "Destination: ", target.dest
print "Base URL: ", target.base_url
print "Checksum type: ", target.checksum_type
print "Expected checksum: ", target.checksum
print "Resume: ", bool(target.resume)
print "Local path: ", target.local_path
print(msg)
librepo.set_debug_log_handler(debug_function)
# Prepare handle
h = librepo.Handle()
h.urls = ["http://beaker-project.org/yum/client-testing/Fedora21/"]
h.repotype = librepo.YUMREPO
# Prepare list of targets
packages = []
target = librepo.PackageTarget(
"beaker-19.1-1.fc19.src.rpm",
handle=h,
checksum_type=librepo.SHA256,
checksum=
"3a07327702d15de2707518eb777c0837b2fc7c0cf4cb757331982f5ce6053867",
resume=True)
packages.append(target)
target = librepo.PackageTarget("beaker-client-19.1-1.fc19.noarch.rpm",
handle=h,
expectedsize=381148)
packages.append(target)
target = librepo.PackageTarget("rhts-4.65-1.fc19.src.rpm", handle=h)
packages.append(target)
librepo.download_packages(packages)
if os.path.exists(filename):
os.remove(filename)
remove_pkg(LIBREPOPKG)
remove_pkg(LAMEPKG)
# Prepare list of targets
packages = []
# Prepare first target
h1 = librepo.Handle()
h1.metalinkurl = "https://mirrors.fedoraproject.org/metalink?repo=fedora-20&arch=x86_64"
h1.repotype = librepo.YUMREPO
h1.fastestmirror = True
h1.fastestmirrorcache = CACHE
target = librepo.PackageTarget("Packages/l/" + LIBREPOPKG, handle=h1)
packages.append(target)
# Prepare second target
h2 = librepo.Handle()
h2.mirrorlisturl = "http://mirrors.rpmfusion.org/mirrorlist?repo=free-fedora-19&arch=x86_64"
h2.repotype = librepo.YUMREPO
h2.fastestmirror = True
h2.fastestmirrorcache = CACHE
target = librepo.PackageTarget(LAMEPKG, handle=h2)
packages.append(target)
t = time.time()
librepo.download_packages(packages)
print "Download duration: {0}s\n".format((time.time() - t))
os.remove(CACHE)
def remove_package():
if os.path.exists(PACKAGE):
print "Removing %s" % PACKAGE
os.remove(PACKAGE)
remove_cache()
remove_package()
print "\n1st run: Basic download"
h = librepo.Handle()
h.interruptible = True
h.metalinkurl = METALINKURL
h.repotype = librepo.YUMREPO
packages = [librepo.PackageTarget(PACKAGEPATH+PACKAGE, handle=h)]
t = time.time()
librepo.download_packages(packages)
t1 = time.time() - t
print "### %s: %s (Time: %s)" % (
packages[0].local_path, packages[0].err or "OK", t1)
remove_package()
print "\n2nd run: Download with fastestmirror - cache will be builded"
h = librepo.Handle()
h.interruptible = True
h.metalinkurl = METALINKURL
h.repotype = librepo.YUMREPO
h.fastestmirror = True
h.fastestmirrorcache = CACHE
def test_download_packages_from_different_mirrors_1(self):
cbdata = {
"failed_urls": [],
}
def mirrorfailurecb(userdata, msg, url):
cbdata["failed_urls"].append(url)
return None
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)
h.mirrorlist = "%s%s" % (self.MOCKURL, config.METALINK_MIRRORS_01)
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.fastestmirror = True
pkgs = []
for P in [
config.PACKAGE_01_01, config.PACKAGE_03_01,
config.PACKAGE_04_01
]:
pkgs.append(
librepo.PackageTarget(P,
handle=h,
dest=self.tmpdir,
mirrorfailurecb=mirrorfailurecb))
librepo.download_packages(pkgs, failfast=True)
# the mirror list should be 1, 3, 4. The metalink file
# defines preference order. This is the order in which
# the URLs are listed in the file
self.assertEqual(h.mirrors, [url1, url2, url3])
# YUM 01 contains P_01
# YUM 03 contains P_03
# YUM_04 contains P_04
# Mirror order of preference is 01 03, 04
# When a package fails to download, librepo moves to the next mirror;
# after a successfull download of the package librepo should continue trying
# to download the remaining packages from the first mirror. e.g.
# always try to download from the fastest mirror containing the package
# Expected download sequence (restart from fastest mirror):
# - P_01 from YUM_01 - PASS
# - P_03 from YUM_01 - FAIL
# - P_03 from YUM_03 - PASS
# - P_04 from YUM_04 - FAIL
# - P_04 from YUM_03 - FAIL
# - P_04 from YUM_04 - PASS
self.assertEqual(len(cbdata["failed_urls"]), 3)
self.assertEqual(cbdata["failed_urls"][0], url1 + config.PACKAGE_03_01)
self.assertEqual(cbdata["failed_urls"][1], url1 + config.PACKAGE_04_01)
self.assertEqual(cbdata["failed_urls"][2], url2 + config.PACKAGE_04_01)
# assert that all packages have been downloaded
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume_02(self):
return # This test causes issues on Fedora rawhide (F26)
# Flask/werkzeug/SocketServer/socket fails on Broken pipe
# and causes next test (test_download_packages_without_handle)
# to always fail
# If download that should be resumed fails,
# the original file should not be modified or deleted
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
fn = os.path.join(self.tmpdir, "package.rpm")
# Download first 10 bytes of the package
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=fn,
resume=False,
byterangeend=9))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
# Mark the file as it was downloaded by Librepo
# Otherwise librepo refuse to resume
try:
xattr.setxattr(pkg.local_path,
"user.Librepo.DownloadInProgress".encode("utf-8"),
"".encode("utf-8"))
except IOError as err:
if err.errno == 95:
self.skipTest('extended attributes are not supported')
raise
# Now try to resume from bad URL
pkgs = []
pkgs.append(librepo.PackageTarget("bad_path.rpm",
handle=h,
dest=fn,
resume=True,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
# Download should fail (path is bad, the file doesn't exist)
self.assertRaises(librepo.LibrepoException, librepo.download_packages,
pkgs, failfast=True)
# The package should exists (should not be removed or changed)
pkg = pkgs[0]
self.assertTrue(pkg.err)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(os.path.getsize(pkg.local_path), 10)
fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
self.assertEqual(fchksum, fchksum_new)
print msg
librepo.set_debug_log_handler(debug_function)
# Prepare handle
h = librepo.Handle()
h.urls = ["http://beaker-project.org/yum/client-testing/Fedora19/"]
h.repotype = librepo.YUMREPO
# Prepare list of targets
packages = []
target = librepo.PackageTarget(
"beaker-0.14.0-1.fc18.src.rpm",
handle=h,
checksum_type=librepo.SHA256,
checksum=
"737c974110914a073fb6c736cd7021b0d844c9e47e7d21e37d687dbc86d36538",
resume=True)
packages.append(target)
target = librepo.PackageTarget("beaker-client-0.14.1-1.fc18.noarch.rpm",
handle=h,
expectedsize=325144)
packages.append(target)
target = librepo.PackageTarget("rhts-4.56-1.fc17.src.rpm", handle=h)
packages.append(target)
librepo.download_packages(packages)
print "Already exists"
else:
print "Error: %s" % msg
def mirrorfailurecb(data, msg, url):
print "MirrorFailureCb: Download of %s from %s failed with: %s" % \
(data, url, msg)
# Prepare list of targets
packages = []
target = librepo.PackageTarget(
"beaker-0.14.0-1.fc18.src.rpm",
handle=h,
checksum_type=librepo.SHA256,
checksum=
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
resume=True,
cbdata="beaker-0.14.0-1.fc18.src.rpm",
endcb=endcb,
mirrorfailurecb=mirrorfailurecb)
packages.append(target)
target = librepo.PackageTarget(
"beaker-0.13.2-1.fc17.noarch.rpm",
handle=h,
checksum_type=librepo.SHA256,
checksum="foobar",
cbdata="beaker-0.13.2-1.fc17.noarch.rpm (bad checksum)",
endcb=endcb,
mirrorfailurecb=mirrorfailurecb)
packages.append(target)