def test_download_packages_from_different_repos(self):
h1 = librepo.Handle()
h2 = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h1.urls = [url]
h1.repotype = librepo.LR_YUMREPO
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
h2.urls = [url]
h2.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h1,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_03_01,
handle=h2,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.handle == h1)
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
pkg = pkgs[1]
self.assertTrue(pkg.handle == h2)
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_locate_incomplete_repo_01_2(self):
# At first, download whole repository
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_URLS, [REPO_YUM_01_PATH])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, self.tmpdir)
h.setopt(librepo.LRO_GPGCHECK, True)
h.setopt(librepo.LRO_YUMDLIST, ["primary"])
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
# Now try to localize the existing repository and all its files
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_URLS, [self.tmpdir])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_LOCAL, True)
h.setopt(librepo.LRO_IGNOREMISSING, True)
h.perform(r)
yum_repo = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD)
# Compare results
yum_repo_downloaded["url"] = None
self.assertEqual(yum_repo, yum_repo_downloaded)
self.assertEqual(yum_repomd, yum_repomd_downloaded)
def test_locate_incomplete_repo_01_2(self):
# At first, download whole repository
h = librepo.Handle()
r = librepo.Result()
h.urls = [REPO_YUM_01_PATH]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.gpgcheck = True
h.yumdlist = ["primary"]
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
# Now try to localize the existing repository and all its files
h = librepo.Handle()
r = librepo.Result()
h.urls = [self.tmpdir]
h.repotype = librepo.LR_YUMREPO
h.local = True
h.ignoremissing = True
h.perform(r)
yum_repo = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD)
# Compare results
yum_repo_downloaded["url"] = None
self.assertEqual(yum_repo, yum_repo_downloaded)
self.assertEqual(yum_repomd, yum_repomd_downloaded)
def test_read_metalink_of_local_repo(self):
# At first, download whole repository
# Check if local metalink.xml will be copied as well
h = librepo.Handle()
r = librepo.Result()
h.urls = [REPO_YUM_01_PATH]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.gpgcheck = True
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
self.assertFalse(yum_repo_downloaded["mirrorlist"])
self.assertTrue(yum_repo_downloaded["metalink"])
self.assertTrue(
yum_repo_downloaded["metalink"].endswith("metalink.xml"))
# Now try to read metalink of the repository
h = librepo.Handle()
r = librepo.Result()
h.urls = [self.tmpdir]
h.repotype = librepo.LR_YUMREPO
h.local = True
h.fetchmirrors = True
h.perform(r)
self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/'])
self.assertEqual(
h.metalink, {
'timestamp':
1347459931,
'hashes':
[('md5', 'f76409f67a84bcd516131d5cc98e57e1'),
('sha1', '75125e73304c21945257d9041a908d0d01d2ca16'),
('sha256',
'bef5d33dc68f47adc7b31df448851b1e9e6bae27840f28700fff144881482a6a'
),
('sha512',
'e40060c747895562e945a68967a04d1279e4bd8507413681f83c322479aa564027fdf3962c2d875089bfcb9317d3a623465f390dc1f4acef294711168b807af0'
)],
'size':
2621,
'urls': [{
'url':
'http://127.0.0.1:5000/yum/static/01/repodata/repomd.xml',
'type': 'http',
'protocol': 'http',
'location': 'CZ',
'preference': 100
}],
'filename':
'repomd.xml'
})
def create(self, repo_id, build_tag):
assert build_tag
self._clean_repo_dir(repo_id)
repo_dir = self._get_repo_dir(repo_id)
temp_dir = repo_dir + ".tmp"
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
try:
for arch in self._arches:
log.debug('Downloading {} repo {} for arch {} from Koji to disk'.
format(build_tag, repo_id, arch))
h = librepo.Handle()
arch_repo_dir = os.path.join(temp_dir, arch)
os.makedirs(arch_repo_dir)
h.destdir = arch_repo_dir
h.repotype = librepo.LR_YUMREPO
url = self._remote_repo.format(repo_id=repo_id, arch=arch,
build_tag=build_tag)
h.urls = [url]
h.yumdlist = ['primary', 'filelists', 'group']
h.perform(librepo.Result())
os.rename(temp_dir, repo_dir)
log.debug('Repo {} successfully downloaded to disk'.format(repo_id))
return repo_dir
except librepo.LibrepoException as e:
if e.args[0] == REPO_404:
log.debug('Repo {} was not found'.format(repo_id))
return None
raise
def create(self, repo_id, repo_dir):
""" Load repo from disk into memory as sack """
if not repo_dir:
return None
try:
sack = hawkey.Sack(arch=self._for_arch)
for arch in self._arches:
log.debug('Loading repo {} for arch {} from disk into memory'.
format(repo_id, arch))
arch_repo_dir = os.path.join(repo_dir, arch)
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = [arch_repo_dir]
h.yumdlist = ['primary', 'filelists', 'group']
repodata = h.perform(librepo.Result()).yum_repo
repo = hawkey.Repo('{}-{}'.format(repo_id, arch))
repo.repomd_fn = repodata['repomd']
repo.primary_fn = repodata['primary']
repo.filelists_fn = repodata['filelists']
sack.load_yum_repo(repo, load_filelists=True)
log.debug('Repo {} successfully loaded into memory'.format(repo_id))
return sack
except (librepo.LibrepoException, IOError):
log.debug('Repo {} could not be loaded'.format(repo_id))
return None
def get_repo(repo_dir, repo_descriptor, download=False):
"""
Obtain hawkey Repo either by loading from disk, or downloading from
Koji.
:repo_dir: path to directory where the repo is/should be stored
:repo_descriptor: which repo to obtain
:download: whether to download or load locally
"""
h = librepo.Handle()
repo_path = os.path.join(repo_dir, str(repo_descriptor))
if download:
h.destdir = repo_path
shutil.rmtree(repo_path, ignore_errors=True)
os.makedirs(os.path.join(repo_path, 'cache'))
h.repotype = librepo.LR_YUMREPO
h.urls = [repo_descriptor.url if download else repo_path]
h.local = not download
h.yumdlist = ['primary', 'filelists', 'group', 'group_gz']
result = librepo.Result()
try:
result = h.perform(result)
except librepo.LibrepoException as e:
if e.args[0] == librepo.LRE_NOURL:
return None
raise
repodata = result.yum_repo
repo = hawkey.Repo(str(repo_descriptor))
repo.repomd_fn = repodata['repomd']
repo.primary_fn = repodata['primary']
repo.filelists_fn = repodata['filelists']
return repo
def from_url(cls, urls=None, mirrorlist=None, metalink=None):
if not urls and not mirrorlist and not metalink:
raise AttributeError("At least one argument must be specified")
tmpdir = tempfile.mkdtemp(prefix="deltarepo-updater-", dir="/tmp")
h = librepo.Handle()
h.repotype = librepo.YUMREPO
h.urls = urls
h.mirrorlisturl = mirrorlist
h.metalinkurl = metalink
h.yumdlist = []
h.destdir = tmpdir
try:
r = h.perform()
except librepo.LibrepoException as e:
shutil.rmtree(tmpdir)
raise DeltaRepoError("Cannot download ({0}, {1}, {2}): {3}".format(
urls, mirrorlist, metalink, e))
repo = cls()
repo._fill_from_path(tmpdir, contenthash=False)
repo.path = None
repo.repodata = None
repo.basename = None
repo.urls = urls
repo.mirrorlist = mirrorlist
repo.metalink = metalink
shutil.rmtree(tmpdir)
return repo
def test_download_packages_one_url_is_bad_with_failfast(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(librepo.PackageTarget("so_bad_url_of_foo_rpm.rpm",
handle=h,
dest=self.tmpdir))
self.assertRaises(librepo.LibrepoException, librepo.download_packages,
pkgs, failfast=True)
# Err state is undefined, it could be error because of interruption
# of could bo None because it was downloaded at one shot before
# second download fails.
#self.assertTrue(pkgs[0].err is not None)
self.assertTrue(os.path.isfile(pkgs[0].local_path))
self.assertTrue(pkgs[1].err is not None)
self.assertTrue(os.path.isfile(pkgs[1].local_path))
def test_download_packages_with_offline_enabled_01(self):
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h = librepo.Handle()
h.urls = [url]
h.repotype = librepo.YUMREPO
h.offline = True
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs)
# Offline is True, no package should be downloaded successfully
for pkg in pkgs:
self.assertTrue(pkg.err)
h.offline = False
librepo.download_packages(pkgs)
# Offline is False, the package should be downloaded successfully
for pkg in pkgs:
self.assertFalse(pkg.err)
def _download_repo(self, repo_id):
repos = {}
try:
for arch, repo_url in self._koji_repos.items():
h = librepo.Handle()
destdir = self._get_repo_dir(repo_id, arch)
if os.path.exists(destdir):
shutil.rmtree(destdir)
os.makedirs(destdir)
h.destdir = destdir
h.repotype = librepo.LR_YUMREPO
assert '{repo_id}' in repo_url
url = repo_url.format(repo_id=repo_id)
h.urls = [url]
h.yumdlist = ['primary', 'filelists', 'group']
log.info("Downloading {arch} repo from {url}".format(arch=arch,
url=url))
result = h.perform(librepo.Result())
repos[arch] = result
self._add_repo(repo_id, repos)
return repos
except librepo.LibrepoException as e:
if e.args[0] == REPO_404:
log.info("Repo id={} not available, skipping".format(repo_id))
return None
raise
def get_repo_metadata(reponame, repopath):
repo_handle = librepo.Handle()
repo_result = librepo.Result()
if repopath.startswith('/'):
repopath = 'file://%s' % repopath
# Set the repository URL.
repo_handle.setopt(librepo.LRO_URL, repopath)
if repopath.startswith('http://') or repopath.startswith('ftp://'):
# Set the metadata destination directory.
destdir = tempfile.mkdtemp(prefix='%s.' % reponame)
repo_handle.setopt(librepo.LRO_DESTDIR, destdir)
elif repopath.startswith('file://'):
repo_handle.setopt(librepo.LRO_LOCAL, True)
destdir = None
else:
raise ValueError("Incorrect repo path '%s'" % repopath)
# Set the repository type.
repo_handle.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
# Download primary.xml and filelists.xml - repomd.xml is downloaded automatically.
repo_handle.setopt(librepo.LRO_YUMDLIST, ['primary', 'filelists'])
repo_handle.perform(repo_result)
return repo_result.getinfo(librepo.LRR_YUM_REPO), destdir
def test_download_packages_with_checksum_check(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
checksum_type=librepo.SHA256,
checksum=config.PACKAGE_01_01_SHA256))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertEqual(pkg.relative_url, config.PACKAGE_01_01)
self.assertEqual(pkg.dest, self.tmpdir)
self.assertEqual(pkg.base_url, None)
self.assertEqual(pkg.checksum_type, librepo.SHA256)
self.assertEqual(pkg.checksum, config.PACKAGE_01_01_SHA256)
self.assertEqual(pkg.resume, 0)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertEqual(
pkg.local_path,
os.path.join(self.tmpdir, os.path.basename(config.PACKAGE_01_01)))
self.assertTrue(pkg.err is None)
def test_packagetarget_sanity_01(self):
h = librepo.Handle()
cbdata = {"a": "b"}
def progresscb(a, b, c): return 0
def endcb(a): return
def mirrorfailurecb(a, b): return 0;
t = librepo.PackageTarget("foo",
dest="bar",
checksum_type=librepo.CHECKSUM_SHA512,
checksum="xxx",
expectedsize=123,
base_url="basefoo",
resume=True,
progresscb=progresscb,
cbdata=cbdata,
handle=h,
endcb=endcb,
mirrorfailurecb=mirrorfailurecb)
self.assertEqual(t.relative_url, "foo")
self.assertEqual(t.dest, "bar")
self.assertEqual(t.checksum_type, librepo.CHECKSUM_SHA512)
self.assertEqual(t.checksum, "xxx")
self.assertEqual(t.expectedsize, 123)
self.assertEqual(t.base_url, "basefoo")
self.assertEqual(t.resume, True)
self.assertEqual(t.progresscb, progresscb)
self.assertEqual(t.cbdata, cbdata)
self.assertEqual(t.handle, h)
self.assertEqual(t.endcb, endcb)
self.assertEqual(t.mirrorfailurecb, mirrorfailurecb)
def test_download_packages_one_url_is_bad_with_failfast(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget("so_bad_url_of_foo_rpm.rpm",
handle=h,
dest=self.tmpdir))
self.assertRaises(librepo.LibrepoException,
librepo.download_packages,
pkgs,
failfast=True)
# Err state of first download is undefined,
# it could be error because of interruption
# of could bo None (when the download was successfull)
# in case that it was downloaded at one shot before
# the second download fails.
# XXX: TODO
#self.assertTrue(os.path.isfile(pkgs[0].local_path))
self.assertTrue(pkgs[1].err is not None)
self.assertFalse(os.path.isfile(pkgs[1].local_path))
def sanity_check_repodata(myurl):
"""
Sanity check the repodata for a given repository.
Args:
myurl (basestring): A path to a repodata directory.
Raises:
bodhi.server.exceptions.RepodataException: If the repodata is not valid or does not exist.
"""
h = librepo.Handle()
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, tempfile.mkdtemp())
if myurl[-1] != '/':
myurl += '/'
if myurl.endswith('repodata/'):
myurl = myurl.replace('repodata/', '')
h.setopt(librepo.LRO_URLS, [myurl])
h.setopt(librepo.LRO_LOCAL, True)
h.setopt(librepo.LRO_CHECKSUM, True)
try:
h.perform()
except librepo.LibrepoException as e:
rc, msg, general_msg = e
raise RepodataException(msg)
updateinfo = os.path.join(myurl, 'updateinfo.xml.gz')
if os.path.exists(updateinfo):
ret = subprocess.call(['zgrep', '<id/>', updateinfo])
if not ret:
raise RepodataException('updateinfo.xml.gz contains empty ID tags')
def test_read_metalink(self):
h = librepo.Handle()
r = librepo.Result()
h.mirrorlist = METALINK
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.fetchmirrors = True
h.perform(r)
self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/'])
self.assertEqual(
h.metalink, {
'timestamp':
1347459931,
'hashes':
[('md5', 'f76409f67a84bcd516131d5cc98e57e1'),
('sha1', '75125e73304c21945257d9041a908d0d01d2ca16'),
('sha256',
'bef5d33dc68f47adc7b31df448851b1e9e6bae27840f28700fff144881482a6a'
),
('sha512',
'e40060c747895562e945a68967a04d1279e4bd8507413681f83c322479aa564027fdf3962c2d875089bfcb9317d3a623465f390dc1f4acef294711168b807af0'
)],
'size':
2621,
'urls': [{
'url':
'http://127.0.0.1:5000/yum/static/01/repodata/repomd.xml',
'type': 'http',
'protocol': 'http',
'location': 'CZ',
'preference': 100
}],
'filename':
'repomd.xml'
})
def download_repodata(self):
logger.debug('Loading repodata for %s from %s', self.name,
self.baseurl or self.metalink)
self.librepo_handle = h = librepo.Handle()
r = librepo.Result()
h.repotype = librepo.LR_YUMREPO
if self.baseurl:
h.urls = [self.baseurl]
if self.metalink:
h.mirrorlist = self.metalink
h.setopt(librepo.LRO_DESTDIR, tempfile.mkdtemp(self.name,
prefix=REPO_CACHE_NAME_PREFIX, dir=REPO_CACHE_DIR))
h.setopt(librepo.LRO_INTERRUPTIBLE, True)
h.setopt(librepo.LRO_YUMDLIST, [])
if self.baseurl and os.path.isdir(self.baseurl):
self._download_metadata_result(h, r)
self._yum_repomd = r.yum_repomd
self._root_path = self.baseurl
self.primary_fn = self.primary_url
self.filelists_fn = self.filelists_url
else:
self._root_path = h.destdir = tempfile.mkdtemp(self.name,
prefix=REPO_CACHE_NAME_PREFIX, dir=REPO_CACHE_DIR)
self._download_metadata_result(h, r)
self._yum_repomd = r.yum_repomd
self.primary_fn = self._download_repodata_file(
self.primary_checksum, self.primary_url)
self.filelists_fn = self._download_repodata_file(
self.filelists_checksum, self.filelists_url)
def test_download_packages_with_fastestmirror_enabled_1(self):
h = librepo.Handle()
cbdata = {
"called": False,
}
def fastestmirrorstatuscallback(userdata, stage, data):
cbdata["called"] = True
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.fastestmirror = True
h.fastestmirrorcb = fastestmirrorstatuscallback
h.download(config.PACKAGE_01_01)
pkgs = []
pkgs.append(h.new_packagetarget(config.PACKAGE_01_01,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
# There is only one mirror, fastestmirror
# detection should be skiped
self.assertFalse(cbdata["called"])
def test_download_packages_with_fastestmirror_enabled_2(self):
h = librepo.Handle()
cbdata = {
"called": False,
"detection": False,
}
def fastestmirrorstatuscallback(cbdata, stage, data):
cbdata["called"] = True
if stage == librepo.FMSTAGE_DETECTION:
cbdata["detection"] = True
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url, "http://foobarblabla"])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, self.tmpdir)
h.fastestmirror = True
h.fastestmirrordata = cbdata
h.fastestmirrorcb = fastestmirrorstatuscallback
h.download(config.PACKAGE_01_01)
pkgs = []
pkgs.append(h.new_packagetarget(config.PACKAGE_01_01,
dest=self.tmpdir))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
# There is only one mirror, fastestmirror
# detection should be skiped
self.assertTrue(cbdata["called"])
self.assertTrue(cbdata["detection"])
def test_download_packages_with_bad_first_mirror_1(self):
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_02_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
h.urls = [url1, url2, url3]
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.allowedmirrorfailures = 1
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_03_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
# Should raise an error, because, the first two mirror
# should be disabled during download of the first package
# and when is the time to download the second package,
# the repo that contains the package is disabled yet.
self.assertRaises(librepo.LibrepoException,
librepo.download_packages,
pkgs,
failfast=True)
self.assertTrue(pkgs[0].err is None)
self.assertTrue(os.path.isfile(pkgs[0].local_path))
self.assertTrue(pkgs[1].err is not None)
self.assertFalse(os.path.isfile(pkgs[1].local_path))
def test_download_packages_with_callback(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
cbdata = {'called': 0}
def cb(cbdata, total, downloaded):
cbdata["called"] += 1
cbdata["total"] = total
cbdata["downloaded"] = downloaded
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir,
progresscb=cb,
cbdata=cbdata))
librepo.download_packages(pkgs)
pkg = pkgs[0]
self.assertTrue(pkg.err is None)
self.assertTrue(pkg.progresscb == cb)
self.assertTrue(pkg.cbdata == cbdata)
self.assertTrue(os.path.isfile(pkg.local_path))
self.assertTrue(cbdata["called"] > 0)
self.assertTrue(cbdata["downloaded"] > 0)
self.assertTrue(cbdata["total"] > 0)
self.assertEqual(cbdata["downloaded"], cbdata["total"])
def test_download_packages_from_different_mirrors_1(self):
cbdata = {
"failed_urls" : [],
}
def mirrorfailurecb(userdata, msg, url):
cbdata["failed_urls"].append(url)
return None
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)
h.mirrorlist = "%s%s" % (self.MOCKURL, config.METALINK_MIRRORS_01)
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.fastestmirror = True
pkgs = []
for P in [config.PACKAGE_01_01, config.PACKAGE_03_01, config.PACKAGE_04_01]:
pkgs.append(librepo.PackageTarget(P,
handle=h,
dest=self.tmpdir,
mirrorfailurecb=mirrorfailurecb))
librepo.download_packages(pkgs, failfast=True)
# the mirror list should be 1, 3, 4. The metalink file
# defines preference order. This is the order in which
# the URLs are listed in the file
self.assertEquals(h.mirrors, [url1, url2, url3])
# YUM 01 contains P_01
# YUM 03 contains P_03
# YUM_04 contains P_04
# Mirror order of preference is 01 03, 04
# When a package fails to download, librepo moves to the next mirror;
# after a successfull download of the package librepo should continue trying
# to download the remaining packages from the first mirror. e.g.
# always try to download from the fastest mirror containing the package
# Expected download sequence (restart from fastest mirror):
# - P_01 from YUM_01 - PASS
# - P_03 from YUM_01 - FAIL
# - P_03 from YUM_03 - PASS
# - P_04 from YUM_04 - FAIL
# - P_04 from YUM_03 - FAIL
# - P_04 from YUM_04 - PASS
self.assertEquals(len(cbdata["failed_urls"]), 3)
self.assertEquals(cbdata["failed_urls"][0], url1+config.PACKAGE_03_01)
self.assertEquals(cbdata["failed_urls"][1], url1+config.PACKAGE_04_01)
self.assertEquals(cbdata["failed_urls"][2], url2+config.PACKAGE_04_01)
# assert that all packages have been downloaded
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def get_repodata(self):
if self._dirty:
self._createrepo()
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = [self._srpm_dir]
return h.perform(librepo.Result())
def test_locate_with_gpgcheck_enabled_but_without_signature(self):
# At first, download whole repository
h = librepo.Handle()
h.urls = [REPO_YUM_02_PATH]
h.repotype = librepo.LR_YUMREPO
h.gpgcheck = True
h.local = True
self.assertRaises(librepo.LibrepoException, h.perform)
def download_package(step):
remove_fn(CACHE)
for filename in world.filenames:
remove_fn(filename)
h = librepo.Handle()
h.repotype = librepo.YUMREPO
for opt, val in world.handle_options.iteritems():
setattr(h, opt, val)
# Download the package for the first time
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages, failfast=True)
world.t1 = time.time() - t
for package in packages:
assert (package.err is None)
assert not os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
# Download the package for the second time
h.fastestmirror = True
h.fastestmirrorcache = CACHE
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages)
world.t2 = time.time() - t
for package in packages:
assert (package.err is None)
assert os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
# Third download of the package
h.fastestmirror = True
h.fastestmirrorcache = CACHE
packages = []
for package in world.packages:
packages.append(librepo.PackageTarget(package, handle=h))
t = time.time()
librepo.download_packages(packages)
world.t3 = time.time() - t
for package in packages:
assert (package.err is None)
assert os.path.exists(CACHE)
for filename in world.filenames:
assert os.path.exists(filename)
remove_fn(filename)
def test_download_packages_00(self):
h = librepo.Handle()
url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
h.urls = [url]
h.repotype = librepo.LR_YUMREPO
pkgs = []
librepo.download_packages(pkgs)
def test_download_packages_mirror_penalization_01(self):
# This test is useful for mirror penalization testing
# with debug output on:
# LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1"
#
# You can use it to check adaptive mirror sorting behavior
#
# E.g. from librepo checkout dir:
# LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING="1" LIBREPO_DEBUG="1" PYTHONPATH=`readlink -f ./build/librepo/python/python2/` nosetests -s -v tests/python/tests/test_yum_package_downloading.py:TestCaseYumPackagesDownloading.test_download_packages_mirror_penalization_01
h = librepo.Handle()
url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)
h.urls = [url1, url2, url3]
h.repotype = librepo.LR_YUMREPO
h.maxparalleldownloads = 1
h.allowedmirrorfailures = -1
h.adaptivemirrorsorting = True
pkgs = []
pkgs.append(
librepo.PackageTarget(config.PACKAGE_03_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_03_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_04_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
pkgs.append(
librepo.PackageTarget(config.PACKAGE_01_01,
handle=h,
dest=self.tmpdir))
librepo.download_packages(pkgs, failfast=False)
for pkg in pkgs:
self.assertTrue(pkg.err is None)
self.assertTrue(os.path.isfile(pkg.local_path))
def test_download_packages_00(self):
h = librepo.Handle()
url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
h.setopt(librepo.LRO_URLS, [url])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
pkgs = []
librepo.download_packages(pkgs)
def test_handle_exceptions(self):
h = librepo.Handle()
self.assertTrue(h)
self.assertRaises(AttributeError, getattr, h, 'foobar_attr')
self.assertRaises(AttributeError, setattr, h, 'foobar_attr', 'xyz')
self.assertRaises(ValueError, h.getinfo, 999999)
self.assertRaises(ValueError, h.setopt, 999999, 'xyz')
