def download_payloads(payloads, drpm): # download packages drpm.err.clear() targets = [pload.librepo_target() for pload in payloads] errs = _DownloadErrors() try: librepo.download_packages(targets, failfast=True) except librepo.LibrepoException as e: errs.fatal = e.args[1] or '<unspecified librepo error>' drpm.wait() # process downloading errors errs.recoverable = drpm.err.copy() for tgt in targets: err = tgt.err if err is None or err.startswith('Not finished'): continue payload = tgt.cbdata pkg = payload.pkg if err == 'Already downloaded': errs.skipped.add(pkg) continue errs.irrecoverable[pkg] = [err] return errs
def test_download_packages_with_offline_enabled_01(self): url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h = librepo.Handle() h.urls = [url] h.repotype = librepo.YUMREPO h.offline = True pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) librepo.download_packages(pkgs) # Offline is True, no package should be downloaded successfully for pkg in pkgs: self.assertTrue(pkg.err) h.offline = False librepo.download_packages(pkgs) # Offline is False, the package should be downloaded successfully for pkg in pkgs: self.assertFalse(pkg.err)
def test_download_packages_from_different_repos(self): h1 = librepo.Handle() h2 = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h1.urls = [url] h1.repotype = librepo.LR_YUMREPO url = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH) h2.urls = [url] h2.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h1, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget(config.PACKAGE_03_01, handle=h2, dest=self.tmpdir)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.handle == h1) self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) pkg = pkgs[1] self.assertTrue(pkg.handle == h2) self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_fastestmirror_enabled_1(self): h = librepo.Handle() cbdata = { "called": False, } def fastestmirrorstatuscallback(userdata, stage, data): cbdata["called"] = True url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO h.destdir = self.tmpdir h.fastestmirror = True h.fastestmirrorcb = fastestmirrorstatuscallback h.download(config.PACKAGE_01_01) pkgs = [] pkgs.append(h.new_packagetarget(config.PACKAGE_01_01, dest=self.tmpdir)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) # There is only one mirror, fastestmirror # detection should be skiped self.assertFalse(cbdata["called"])
def test_download_packages_with_checksum_check(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, checksum_type=librepo.SHA256, checksum=config.PACKAGE_01_01_SHA256)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertEqual(pkg.relative_url, config.PACKAGE_01_01) self.assertEqual(pkg.dest, self.tmpdir) self.assertEqual(pkg.base_url, None) self.assertEqual(pkg.checksum_type, librepo.SHA256) self.assertEqual(pkg.checksum, config.PACKAGE_01_01_SHA256) self.assertEqual(pkg.resume, 0) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(pkg.local_path, os.path.join(self.tmpdir, os.path.basename(config.PACKAGE_01_01))) self.assertTrue(pkg.err is None)
def test_download_packages_with_callback(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO cbdata = {'called': 0} def cb(cbdata, total, downloaded): cbdata["called"] += 1 cbdata["total"] = total cbdata["downloaded"] = downloaded pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, progresscb=cb, cbdata=cbdata)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(pkg.progresscb == cb) self.assertTrue(pkg.cbdata == cbdata) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertTrue(cbdata["called"] > 0) self.assertTrue(cbdata["downloaded"] > 0) self.assertTrue(cbdata["total"] > 0) self.assertEqual(cbdata["downloaded"], cbdata["total"])
def _download_payloads(payloads, drpm): # download packages def _download_sort_key(payload): return not hasattr(payload, 'delta') drpm.err.clear() targets = [pload._librepo_target() for pload in sorted(payloads, key=_download_sort_key)] errs = _DownloadErrors() try: librepo.download_packages(targets, failfast=True) except librepo.LibrepoException as e: errs._fatal = e.args[1] or '<unspecified librepo error>' drpm.wait() # process downloading errors errs._recoverable = drpm.err.copy() for tgt in targets: err = tgt.err if err is None or err.startswith('Not finished'): continue payload = tgt.cbdata pkg = payload.pkg if err == _('Already downloaded'): errs._skipped.add(pkg) continue pkg.repo._md_expire_cache() errs._irrecoverable[pkg] = [err] return errs
def test_download_packages_with_fastestmirror_enabled_2(self): h = librepo.Handle() cbdata = { "called": False, "detection": False, } def fastestmirrorstatuscallback(cbdata, stage, data): cbdata["called"] = True if stage == librepo.FMSTAGE_DETECTION: cbdata["detection"] = True url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url, "http://foobarblabla"]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_DESTDIR, self.tmpdir) h.fastestmirror = True h.fastestmirrordata = cbdata h.fastestmirrorcb = fastestmirrorstatuscallback h.download(config.PACKAGE_01_01) pkgs = [] pkgs.append(h.new_packagetarget(config.PACKAGE_01_01, dest=self.tmpdir)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) # There is only one mirror, fastestmirror # detection should be skiped self.assertTrue(cbdata["called"]) self.assertTrue(cbdata["detection"])
def test_download_packages_from_different_mirrors_1(self): cbdata = { "failed_urls" : [], } def mirrorfailurecb(userdata, msg, url): cbdata["failed_urls"].append(url) return None h = librepo.Handle() url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH) url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH) h.mirrorlist = "%s%s" % (self.MOCKURL, config.METALINK_MIRRORS_01) h.repotype = librepo.LR_YUMREPO h.maxparalleldownloads = 1 h.fastestmirror = True pkgs = [] for P in [config.PACKAGE_01_01, config.PACKAGE_03_01, config.PACKAGE_04_01]: pkgs.append(librepo.PackageTarget(P, handle=h, dest=self.tmpdir, mirrorfailurecb=mirrorfailurecb)) librepo.download_packages(pkgs, failfast=True) # the mirror list should be 1, 3, 4. The metalink file # defines preference order. This is the order in which # the URLs are listed in the file self.assertEquals(h.mirrors, [url1, url2, url3]) # YUM 01 contains P_01 # YUM 03 contains P_03 # YUM_04 contains P_04 # Mirror order of preference is 01 03, 04 # When a package fails to download, librepo moves to the next mirror; # after a successfull download of the package librepo should continue trying # to download the remaining packages from the first mirror. e.g. # always try to download from the fastest mirror containing the package # Expected download sequence (restart from fastest mirror): # - P_01 from YUM_01 - PASS # - P_03 from YUM_01 - FAIL # - P_03 from YUM_03 - PASS # - P_04 from YUM_04 - FAIL # - P_04 from YUM_03 - FAIL # - P_04 from YUM_04 - PASS self.assertEquals(len(cbdata["failed_urls"]), 3) self.assertEquals(cbdata["failed_urls"][0], url1+config.PACKAGE_03_01) self.assertEquals(cbdata["failed_urls"][1], url1+config.PACKAGE_04_01) self.assertEquals(cbdata["failed_urls"][2], url2+config.PACKAGE_04_01) # assert that all packages have been downloaded for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_00(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] librepo.download_packages(pkgs)
def test_download_packages_mirror_penalization_01(self): # This test is useful for mirror penalization testing # with debug output on: # LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1" # # You can use it to check adaptive mirror sorting behavior # # E.g. from librepo checkout dir: # LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1" LIBREPO_DEBUG="1" PYTHONPATH=`readlink -f ./build/librepo/python/` python3 -m unittest discover -s tests/python/ -p "test_yum_package_downloading.py" -k "test_download_packages_mirror_penalization_01" h = librepo.Handle() url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH) url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH) h.urls = [url1, url2, url3] h.repotype = librepo.LR_YUMREPO h.maxparalleldownloads = 1 h.allowedmirrorfailures = -1 h.adaptivemirrorsorting = True pkgs = [] pkgs.append( librepo.PackageTarget(config.PACKAGE_03_01, handle=h, dest=self.tmpdir)) pkgs.append( librepo.PackageTarget(config.PACKAGE_03_01, handle=h, dest=self.tmpdir)) pkgs.append( librepo.PackageTarget(config.PACKAGE_04_01, handle=h, dest=self.tmpdir)) pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) librepo.download_packages(pkgs, failfast=False) for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume_02(self): # If download that should be resumed fails, # the original file should not be modified or deleted h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO fn = os.path.join(self.tmpdir, "package.rpm") # Download first 10 bytes of the package pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=fn, resume=False, byterangeend=9)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() # Mark the file as it was downloaded by Librepo # Otherwise librepo refuse to resume try: xattr.setxattr(pkg.local_path, "user.Librepo.DownloadInProgress".encode("utf-8"), "".encode("utf-8")) except IOError as err: if err.errno == 95: self.skipTest('extended attributes are not supported') raise # Now try to resume from bad URL pkgs = [] pkgs.append(librepo.PackageTarget("bad_path.rpm", handle=h, dest=fn, resume=True, checksum_type=librepo.SHA256, checksum=config.PACKAGE_01_01_SHA256)) # Download should fail (path is bad, the file doesn't exist) self.assertRaises(librepo.LibrepoException, librepo.download_packages, pkgs, failfast=True) # The package should exists (should not be removed or changed) pkg = pkgs[0] self.assertTrue(pkg.err) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() self.assertEqual(fchksum, fchksum_new)
def test_download_packages_00(self): h = librepo.Handle() url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) pkgs = [] librepo.download_packages(pkgs)
def test_download_packages_with_resume_02(self): # If download that should be resumed fails, # the original file should not be modified or deleted h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO fn = os.path.join(self.tmpdir, "package.rpm") # Download first 10 bytes of the package pkgs = [] pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=fn, resume=False, byterangeend=9)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() # Mark the file as it was downloaded by Librepo # Otherwise librepo refuse to resume xattr.setxattr(pkg.local_path, "user.Librepo.DownloadInProgress", "") # Now try to resume from bad URL pkgs = [] pkgs.append( librepo.PackageTarget("bad_path.rpm", handle=h, dest=fn, resume=True, checksum_type=librepo.SHA256, checksum=config.PACKAGE_01_01_SHA256)) # Download should fail (path is bad, the file doesn't exist) self.assertRaises(librepo.LibrepoException, librepo.download_packages, pkgs, failfast=True) # The package should exists (should not be removed or changed) pkg = pkgs[0] self.assertTrue(pkg.err) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() self.assertEqual(fchksum, fchksum_new)
def test_download_packages_without_handle(self): complete_url = "%s%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH, config.PACKAGE_01_01) pkgs = [] pkgs.append(librepo.PackageTarget(complete_url, dest=self.tmpdir)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_mirror_penalization_01(self): # This test is useful for mirror penalization testing # with debug output on: # LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1" # # You can use it to check adaptive mirror sorting behavior # # E.g. from librepo checkout dir: # LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1" LIBREPO_DEBUG="1" PYTHONPATH=`readlink -f ./build/librepo/python/python2/` nosetests -s -v tests/python/tests/test_yum_package_downloading.py:TestCaseYumPackagesDownloading.test_download_packages_mirror_penalization_01 h = librepo.Handle() url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH) url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH) h.urls = [url1, url2, url3] h.repotype = librepo.LR_YUMREPO h.maxparalleldownloads = 1 h.allowedmirrorfailures = -1 h.adaptivemirrorsorting = True pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_03_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget(config.PACKAGE_03_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget(config.PACKAGE_04_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) librepo.download_packages(pkgs, failfast=False) for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_01(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) librepo.download_packages(pkgs) for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume_02(self): # If download that should be resumed fails, # the original file should not be modified or deleted h = librepo.Handle() url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) fn = os.path.join(self.tmpdir, "package.rpm") # Download first 10 bytes of the package pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=fn, resume=False, byterangeend=9)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() # Now try to resume from bad URL pkgs = [] pkgs.append(librepo.PackageTarget("bad_path.rpm", handle=h, dest=fn, resume=True, checksum_type=librepo.SHA256, checksum=config.PACKAGE_01_01_SHA256)) # Download should fail (path is bad, the file doesn't exist) self.assertRaises(librepo.LibrepoException, librepo.download_packages, pkgs, failfast=True) # The package should exists (should not be removed or changed) pkg = pkgs[0] self.assertTrue(pkg.err) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() self.assertEqual(fchksum, fchksum_new)
def test_download_packages_01(self): h = librepo.Handle() url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) librepo.download_packages(pkgs) for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_bad_checksum(self): h = librepo.Handle() url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, checksum_type=librepo.SHA256, checksum="badchecksum")) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err)
def test_download_packages_with_baseurl(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = ["."] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, base_url=url)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_bad_checksum(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, checksum_type=librepo.SHA256, checksum="badchecksum")) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err)
def _urlopen_progress(url, conf): handle = _non_repo_handle(conf) handle.repotype = librepo.LR_YUMREPO handle.setopt(librepo.LRO_URLS, os.path.dirname(url)) progress = dnf.cli.progress.MultiFileProgressMeter(fo=sys.stdout) pload = dnf.repo.RemoteRPMPayload(url, conf, handle, progress) if os.path.exists(pload.local_path): return pload.local_path est_remote_size = sum([pload.download_size]) progress.start(1, est_remote_size) targets = [pload._librepo_target()] try: librepo.download_packages(targets, failfast=True) except librepo.LibrepoException as e: if conf.strict: raise IOError(e.args[1]) logger.error(e.args[1]) return pload.local_path
def test_download_packages_with_expectedsize(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO expectedsize = 1057084 pkgs = [] pkgs.append(h.new_packagetarget(config.PACKAGE_01_01, dest=self.tmpdir, expectedsize=expectedsize)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertEqual(pkg.expectedsize, expectedsize) self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def download_package(self, location, checksum_type, checksum): if self.librepo_handle.local: local_path = os.path.join(self._root_path, location) logger.debug('Using package %s from local filesystem directly', local_path) return local_path logger.debug('Loading package %s from repo %s', location, self.name) target = librepo.PackageTarget(location, checksum_type=librepo.checksum_str_to_type(checksum_type), checksum=checksum, dest=self._root_path, handle=self.librepo_handle) librepo.download_packages([target]) if target.err and target.err == 'Already downloaded': logger.debug('Already downloaded %s', target.local_path) elif target.err: raise PackageDownloadError('Failed to download %s from repo %s: %s' % (location, self.name, target.err)) else: logger.debug('Saved as %s', target.local_path) return target.local_path
def test_download_packages_with_expectedsize(self): h = librepo.Handle() url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) expectedsize = 1057084 pkgs = [] pkgs.append(h.new_packagetarget(config.PACKAGE_01_01, dest=self.tmpdir, expectedsize=expectedsize)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertEqual(pkg.expectedsize, expectedsize) self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def download_package(self, location, baseurl, checksum_type, checksum): if self.librepo_handle.local: local_path = os.path.join(self._root_path, location) logger.debug('Using package %s from local filesystem directly', local_path) return local_path logger.debug('Loading package %s from repo %s', location, self.name) target = librepo.PackageTarget(location, base_url=baseurl, checksum_type=librepo.checksum_str_to_type(checksum_type), checksum=checksum, dest=self._root_path, handle=self.librepo_handle) librepo.download_packages([target]) if target.err and target.err == 'Already downloaded': logger.debug('Already downloaded %s', target.local_path) elif target.err: raise PackageDownloadError('Failed to download %s from repo %s: %s' % (location, self.name, target.err)) else: logger.debug('Saved as %s', target.local_path) return target.local_path
def test_download_packages_one_url_is_bad_01(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget("so_bad_url_of_foo_rpm.rpm", handle=h, dest=self.tmpdir)) librepo.download_packages(pkgs) self.assertTrue(pkgs[0].err is None) self.assertTrue(os.path.isfile(pkgs[0].local_path)) self.assertTrue(pkgs[1].err is not None) self.assertFalse(os.path.isfile(pkgs[1].local_path))
def test_download_packages_one_url_is_bad_01(self): h = librepo.Handle() url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) pkgs.append(librepo.PackageTarget("so_bad_url_of_foo_rpm.rpm", handle=h, dest=self.tmpdir)) librepo.download_packages(pkgs) self.assertTrue(pkgs[0].err is None) self.assertTrue(os.path.isfile(pkgs[0].local_path)) self.assertTrue(pkgs[1].err is not None) self.assertTrue(os.path.isfile(pkgs[1].local_path))
def download_package(step): remove_fn(CACHE) for filename in world.filenames: remove_fn(filename) h = librepo.Handle() h.repotype = librepo.YUMREPO for opt, val in world.handle_options.iteritems(): setattr(h, opt, val) # Download the package for the first time packages = [] for package in world.packages: packages.append(librepo.PackageTarget(package, handle=h)) t = time.time() librepo.download_packages(packages, failfast=True) world.t1 = time.time() - t for package in packages: assert (package.err is None) assert not os.path.exists(CACHE) for filename in world.filenames: assert os.path.exists(filename) remove_fn(filename) # Download the package for the second time h.fastestmirror = True h.fastestmirrorcache = CACHE packages = [] for package in world.packages: packages.append(librepo.PackageTarget(package, handle=h)) t = time.time() librepo.download_packages(packages) world.t2 = time.time() - t for package in packages: assert (package.err is None) assert os.path.exists(CACHE) for filename in world.filenames: assert os.path.exists(filename) remove_fn(filename) # Third download of the package h.fastestmirror = True h.fastestmirrorcache = CACHE packages = [] for package in world.packages: packages.append(librepo.PackageTarget(package, handle=h)) t = time.time() librepo.download_packages(packages) world.t3 = time.time() - t for package in packages: assert (package.err is None) assert os.path.exists(CACHE) for filename in world.filenames: assert os.path.exists(filename) remove_fn(filename)
def test_download_packages_02(self): h = librepo.Handle() url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH) h.setopt(librepo.LRO_URLS, [url]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) dest = os.path.join(self.tmpdir, "foo-haha-lol.rpm") pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=dest)) librepo.download_packages(pkgs) for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertTrue(os.path.isfile(dest))
def test_download_packages_02_with_failfast(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) dest = os.path.join(self.tmpdir, "foo-haha-lol.rpm") pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=dest)) librepo.download_packages(pkgs, failfast=True) for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertTrue(os.path.isfile(dest))
def test_download_packages_02_with_failfast(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir)) dest = os.path.join(self.tmpdir, "foo-haha-lol.rpm") pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=dest)) librepo.download_packages(pkgs, failfast=True) for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertTrue(os.path.isfile(dest))
def test_download_packages_with_fastestmirror_enabled_2(self): h = librepo.Handle() cbdata = { "called": False, "detection": False, } def fastestmirrorstatuscallback(cbdata, stage, data): cbdata["called"] = True if stage == librepo.FMSTAGE_DETECTION: cbdata["detection"] = True url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url, "http://foobarblabla"] h.repotype = librepo.LR_YUMREPO h.destdir = self.tmpdir h.fastestmirror = True h.fastestmirrordata = cbdata h.fastestmirrorcb = fastestmirrorstatuscallback h.download(config.PACKAGE_01_01) pkgs = [] pkgs.append(h.new_packagetarget(config.PACKAGE_01_01, dest=self.tmpdir)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) # There is only one mirror, fastestmirror # detection should be skiped self.assertTrue(cbdata["called"]) self.assertTrue(cbdata["detection"])
def download_payloads(payloads, drpm): # download packages drpm.err.clear() targets = [pload.librepo_target() for pload in payloads] errs = _DownloadErrors() try: librepo.download_packages(targets, failfast=True) except librepo.LibrepoException as e: errs.fatal = e.args[1] or '<unspecified librepo error>' drpm.wait() # process downloading errors errs.recoverable = drpm.err.copy() for tgt in targets: err = tgt.err if err is None: continue if err == 'Already downloaded' or err.startswith('Not finished'): continue payload = tgt.cbdata pkg = payload.pkg errs.irrecoverable[pkg] = [err] return errs
def test_download_packages_with_resume(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, resume=False)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) # Again with resume on # Because the package already exists and checksum matches, # then download should not be performed! pkgs = [] pkgs.append( librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, resume=True, checksum_type=librepo.SHA256, checksum=config.PACKAGE_01_01_SHA256)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertEqual(pkg.err, "Already downloaded") self.assertTrue(os.path.isfile(pkg.local_path)) ## Failfast ignore this type of error (Already downloaded error) librepo.download_packages(pkgs, failfast=True) pkg = pkgs[0] self.assertEqual(pkg.err, "Already downloaded") self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_with_resume(self): h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, resume=False)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) # Again with resume on # Because the package already exists and checksum matches, # then download should not be performed! pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=self.tmpdir, resume=True, checksum_type=librepo.SHA256, checksum=config.PACKAGE_01_01_SHA256)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertEqual(pkg.err, "Already downloaded") self.assertTrue(os.path.isfile(pkg.local_path)) ## Failfast ignore this type of error (Already downloaded error) librepo.download_packages(pkgs, failfast=True) pkg = pkgs[0] self.assertEqual(pkg.err, "Already downloaded") self.assertTrue(os.path.isfile(pkg.local_path))
# Prepare handle h = librepo.Handle() h.urls = [ "http://mirror.karneval.cz/pub/linux/fedora/linux/releases/20/Fedora/x86_64/os/" ] h.repotype = librepo.YUMREPO # Prepare list of targets packages = [] target = librepo.PackageTarget("Packages/a/abrt-2.1.9-1.fc20.x86_64.rpm", handle=h, dest="abrt.header", byterangestart=1384, byterangeend=100204) packages.append(target) librepo.download_packages(packages) for target in packages: print "### %s: %s" % (target.local_path, target.err or "OK") print "Relative URL: ", target.relative_url print "Destination: ", target.dest print "Base URL: ", target.base_url print "Checksum type: ", target.checksum_type print "Expected checksum: ", target.checksum print "Resume: ", bool(target.resume) print "Local path: ", target.local_path print "Error: ", target.err print
handle=h, expectedsize=333333333333333, cbdata="beaker-client-0.14.1-1.fc18.noarch.rpm (bad size)", endcb=endcb, mirrorfailurecb=mirrorfailurecb, ) packages.append(target) target = librepo.PackageTarget( "rhts-4.56-1.fc17.src.rpm_bad_filename", handle=h, cbdata="rhts-4.56-1.fc17.src.rpm_bad_filename (bad path)", endcb=endcb, mirrorfailurecb=mirrorfailurecb, ) packages.append(target) librepo.download_packages(packages, failfast=False) for target in packages: print "### %s: %s" % (target.local_path, target.err or "OK") print "Relative URL: ", target.relative_url print "Destination: ", target.dest print "Base URL: ", target.base_url print "Checksum type: ", target.checksum_type print "Expected checksum: ", target.checksum print "Resume: ", bool(target.resume) print "Local path: ", target.local_path print "Error: ", target.err print
def test_download_packages_with_resume_02(self): return # This test causes issues on Fedora rawhide (F26) # Flask/werkzeug/SocketServer/socket fails on Broken pipe # and causes next test (test_download_packages_without_handle) # to always fail # If download that should be resumed fails, # the original file should not be modified or deleted h = librepo.Handle() url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) h.urls = [url] h.repotype = librepo.LR_YUMREPO fn = os.path.join(self.tmpdir, "package.rpm") # Download first 10 bytes of the package pkgs = [] pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01, handle=h, dest=fn, resume=False, byterangeend=9)) librepo.download_packages(pkgs) pkg = pkgs[0] self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() # Mark the file as it was downloaded by Librepo # Otherwise librepo refuse to resume try: xattr.setxattr(pkg.local_path, "user.Librepo.DownloadInProgress".encode("utf-8"), "".encode("utf-8")) except IOError as err: if err.errno == 95: self.skipTest('extended attributes are not supported') raise # Now try to resume from bad URL pkgs = [] pkgs.append(librepo.PackageTarget("bad_path.rpm", handle=h, dest=fn, resume=True, checksum_type=librepo.SHA256, checksum=config.PACKAGE_01_01_SHA256)) # Download should fail (path is bad, the file doesn't exist) self.assertRaises(librepo.LibrepoException, librepo.download_packages, pkgs, failfast=True) # The package should exists (should not be removed or changed) pkg = pkgs[0] self.assertTrue(pkg.err) self.assertTrue(os.path.isfile(pkg.local_path)) self.assertEqual(os.path.getsize(pkg.local_path), 10) fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest() self.assertEqual(fchksum, fchksum_new)
def test_download_packages_from_different_mirrors_1(self): cbdata = { "failed_urls": [], } def mirrorfailurecb(userdata, msg, url): cbdata["failed_urls"].append(url) return None h = librepo.Handle() url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH) url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH) url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH) h.mirrorlist = "%s%s" % (self.MOCKURL, config.METALINK_MIRRORS_01) h.repotype = librepo.LR_YUMREPO h.maxparalleldownloads = 1 h.fastestmirror = True pkgs = [] for P in [ config.PACKAGE_01_01, config.PACKAGE_03_01, config.PACKAGE_04_01 ]: pkgs.append( librepo.PackageTarget(P, handle=h, dest=self.tmpdir, mirrorfailurecb=mirrorfailurecb)) librepo.download_packages(pkgs, failfast=True) # the mirror list should be 1, 3, 4. The metalink file # defines preference order. This is the order in which # the URLs are listed in the file self.assertEqual(h.mirrors, [url1, url2, url3]) # YUM 01 contains P_01 # YUM 03 contains P_03 # YUM_04 contains P_04 # Mirror order of preference is 01 03, 04 # When a package fails to download, librepo moves to the next mirror; # after a successfull download of the package librepo should continue trying # to download the remaining packages from the first mirror. e.g. # always try to download from the fastest mirror containing the package # Expected download sequence (restart from fastest mirror): # - P_01 from YUM_01 - PASS # - P_03 from YUM_01 - FAIL # - P_03 from YUM_03 - PASS # - P_04 from YUM_04 - FAIL # - P_04 from YUM_03 - FAIL # - P_04 from YUM_04 - PASS self.assertEqual(len(cbdata["failed_urls"]), 3) self.assertEqual(cbdata["failed_urls"][0], url1 + config.PACKAGE_03_01) self.assertEqual(cbdata["failed_urls"][1], url1 + config.PACKAGE_04_01) self.assertEqual(cbdata["failed_urls"][2], url2 + config.PACKAGE_04_01) # assert that all packages have been downloaded for pkg in pkgs: self.assertTrue(pkg.err is None) self.assertTrue(os.path.isfile(pkg.local_path))
def remove_package(): if os.path.exists(PACKAGE): print "Removing %s" % PACKAGE os.remove(PACKAGE) remove_cache() remove_package() print "\n1st run: Basic download" h = librepo.Handle() h.interruptible = True h.metalinkurl = METALINKURL h.repotype = librepo.YUMREPO packages = [librepo.PackageTarget(PACKAGEPATH+PACKAGE, handle=h)] t = time.time() librepo.download_packages(packages) t1 = time.time() - t print "### %s: %s (Time: %s)" % ( packages[0].local_path, packages[0].err or "OK", t1) remove_package() print "\n2nd run: Download with fastestmirror - cache will be builded" h = librepo.Handle() h.interruptible = True h.metalinkurl = METALINKURL h.repotype = librepo.YUMREPO h.fastestmirror = True h.fastestmirrorcache = CACHE h.fastestmirrorcb = fastestmirrorstatuscallback packages = [librepo.PackageTarget(PACKAGEPATH+PACKAGE, handle=h)]
target = librepo.PackageTarget( "beaker-client-0.14.1-1.fc18.noarch.rpm", handle=h, expectedsize=333333333333333, cbdata="beaker-client-0.14.1-1.fc18.noarch.rpm (bad size)", endcb=endcb, mirrorfailurecb=mirrorfailurecb) packages.append(target) target = librepo.PackageTarget( "rhts-4.56-1.fc17.src.rpm_bad_filename", handle=h, cbdata="rhts-4.56-1.fc17.src.rpm_bad_filename (bad path)", endcb=endcb, mirrorfailurecb=mirrorfailurecb) packages.append(target) librepo.download_packages(packages, failfast=False) for target in packages: print "### %s: %s" % (target.local_path, target.err or "OK") print "Relative URL: ", target.relative_url print "Destination: ", target.dest print "Base URL: ", target.base_url print "Checksum type: ", target.checksum_type print "Expected checksum: ", target.checksum print "Resume: ", bool(target.resume) print "Local path: ", target.local_path print "Error: ", target.err print