def test_locate_incomplete_repo_01_2(self):
# At first, download whole repository
h = librepo.Handle()
r = librepo.Result()
h.urls = [REPO_YUM_01_PATH]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.gpgcheck = True
h.yumdlist = ["primary"]
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
# Now try to localize the existing repository and all its files
h = librepo.Handle()
r = librepo.Result()
h.urls = [self.tmpdir]
h.repotype = librepo.LR_YUMREPO
h.local = True
h.ignoremissing = True
h.perform(r)
yum_repo = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD)
# Compare results
yum_repo_downloaded["url"] = None
self.assertEqual(yum_repo, yum_repo_downloaded)
self.assertEqual(yum_repomd, yum_repomd_downloaded)
def test_locate_incomplete_repo_01_2(self):
# At first, download whole repository
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_URLS, [REPO_YUM_01_PATH])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, self.tmpdir)
h.setopt(librepo.LRO_GPGCHECK, True)
h.setopt(librepo.LRO_YUMDLIST, ["primary"])
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
# Now try to localize the existing repository and all its files
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_URLS, [self.tmpdir])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_LOCAL, True)
h.setopt(librepo.LRO_IGNOREMISSING, True)
h.perform(r)
yum_repo = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD)
# Compare results
yum_repo_downloaded["url"] = None
self.assertEqual(yum_repo, yum_repo_downloaded)
self.assertEqual(yum_repomd, yum_repomd_downloaded)
def test_read_metalink_of_local_repo(self):
# At first, download whole repository
# Check if local metalink.xml will be copied as well
h = librepo.Handle()
r = librepo.Result()
h.urls = [REPO_YUM_01_PATH]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.gpgcheck = True
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
self.assertFalse(yum_repo_downloaded["mirrorlist"])
self.assertTrue(yum_repo_downloaded["metalink"])
self.assertTrue(
yum_repo_downloaded["metalink"].endswith("metalink.xml"))
# Now try to read metalink of the repository
h = librepo.Handle()
r = librepo.Result()
h.urls = [self.tmpdir]
h.repotype = librepo.LR_YUMREPO
h.local = True
h.fetchmirrors = True
h.perform(r)
self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/'])
self.assertEqual(
h.metalink, {
'timestamp':
1347459931,
'hashes':
[('md5', 'f76409f67a84bcd516131d5cc98e57e1'),
('sha1', '75125e73304c21945257d9041a908d0d01d2ca16'),
('sha256',
'bef5d33dc68f47adc7b31df448851b1e9e6bae27840f28700fff144881482a6a'
),
('sha512',
'e40060c747895562e945a68967a04d1279e4bd8507413681f83c322479aa564027fdf3962c2d875089bfcb9317d3a623465f390dc1f4acef294711168b807af0'
)],
'size':
2621,
'urls': [{
'url':
'http://127.0.0.1:5000/yum/static/01/repodata/repomd.xml',
'type': 'http',
'protocol': 'http',
'location': 'CZ',
'preference': 100
}],
'filename':
'repomd.xml'
})
def create(self, repo_id, repo_dir):
""" Load repo from disk into memory as sack """
if not repo_dir:
return None
try:
sack = hawkey.Sack(arch=self._for_arch)
for arch in self._arches:
log.debug('Loading repo {} for arch {} from disk into memory'.
format(repo_id, arch))
arch_repo_dir = os.path.join(repo_dir, arch)
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = [arch_repo_dir]
h.yumdlist = ['primary', 'filelists', 'group']
repodata = h.perform(librepo.Result()).yum_repo
repo = hawkey.Repo('{}-{}'.format(repo_id, arch))
repo.repomd_fn = repodata['repomd']
repo.primary_fn = repodata['primary']
repo.filelists_fn = repodata['filelists']
sack.load_yum_repo(repo, load_filelists=True)
log.debug('Repo {} successfully loaded into memory'.format(repo_id))
return sack
except (librepo.LibrepoException, IOError):
log.debug('Repo {} could not be loaded'.format(repo_id))
return None
def get_repo_metadata(reponame, repopath):
repo_handle = librepo.Handle()
repo_result = librepo.Result()
if repopath.startswith('/'):
repopath = 'file://%s' % repopath
# Set the repository URL.
repo_handle.setopt(librepo.LRO_URL, repopath)
if repopath.startswith('http://') or repopath.startswith('ftp://'):
# Set the metadata destination directory.
destdir = tempfile.mkdtemp(prefix='%s.' % reponame)
repo_handle.setopt(librepo.LRO_DESTDIR, destdir)
elif repopath.startswith('file://'):
repo_handle.setopt(librepo.LRO_LOCAL, True)
destdir = None
else:
raise ValueError("Incorrect repo path '%s'" % repopath)
# Set the repository type.
repo_handle.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
# Download primary.xml and filelists.xml - repomd.xml is downloaded automatically.
repo_handle.setopt(librepo.LRO_YUMDLIST, ['primary', 'filelists'])
repo_handle.perform(repo_result)
return repo_result.getinfo(librepo.LRR_YUM_REPO), destdir
def create(self, repo_id, build_tag):
assert build_tag
self._clean_repo_dir(repo_id)
repo_dir = self._get_repo_dir(repo_id)
temp_dir = repo_dir + ".tmp"
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
try:
for arch in self._arches:
log.debug('Downloading {} repo {} for arch {} from Koji to disk'.
format(build_tag, repo_id, arch))
h = librepo.Handle()
arch_repo_dir = os.path.join(temp_dir, arch)
os.makedirs(arch_repo_dir)
h.destdir = arch_repo_dir
h.repotype = librepo.LR_YUMREPO
url = self._remote_repo.format(repo_id=repo_id, arch=arch,
build_tag=build_tag)
h.urls = [url]
h.yumdlist = ['primary', 'filelists', 'group']
h.perform(librepo.Result())
os.rename(temp_dir, repo_dir)
log.debug('Repo {} successfully downloaded to disk'.format(repo_id))
return repo_dir
except librepo.LibrepoException as e:
if e.args[0] == REPO_404:
log.debug('Repo {} was not found'.format(repo_id))
return None
raise
def test_read_metalink(self):
h = librepo.Handle()
r = librepo.Result()
h.mirrorlist = METALINK
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.fetchmirrors = True
h.perform(r)
self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/'])
self.assertEqual(
h.metalink, {
'timestamp':
1347459931,
'hashes':
[('md5', 'f76409f67a84bcd516131d5cc98e57e1'),
('sha1', '75125e73304c21945257d9041a908d0d01d2ca16'),
('sha256',
'bef5d33dc68f47adc7b31df448851b1e9e6bae27840f28700fff144881482a6a'
),
('sha512',
'e40060c747895562e945a68967a04d1279e4bd8507413681f83c322479aa564027fdf3962c2d875089bfcb9317d3a623465f390dc1f4acef294711168b807af0'
)],
'size':
2621,
'urls': [{
'url':
'http://127.0.0.1:5000/yum/static/01/repodata/repomd.xml',
'type': 'http',
'protocol': 'http',
'location': 'CZ',
'preference': 100
}],
'filename':
'repomd.xml'
})
def download_repodata(self):
logger.debug('Loading repodata for %s from %s', self.name,
self.baseurl or self.metalink)
self.librepo_handle = h = librepo.Handle()
r = librepo.Result()
h.repotype = librepo.LR_YUMREPO
if self.baseurl:
h.urls = [self.baseurl]
if self.metalink:
h.mirrorlist = self.metalink
h.setopt(librepo.LRO_DESTDIR, tempfile.mkdtemp(self.name,
prefix=REPO_CACHE_NAME_PREFIX, dir=REPO_CACHE_DIR))
h.setopt(librepo.LRO_INTERRUPTIBLE, True)
h.setopt(librepo.LRO_YUMDLIST, [])
if self.baseurl and os.path.isdir(self.baseurl):
self._download_metadata_result(h, r)
self._yum_repomd = r.yum_repomd
self._root_path = self.baseurl
self.primary_fn = self.primary_url
self.filelists_fn = self.filelists_url
else:
self._root_path = h.destdir = tempfile.mkdtemp(self.name,
prefix=REPO_CACHE_NAME_PREFIX, dir=REPO_CACHE_DIR)
self._download_metadata_result(h, r)
self._yum_repomd = r.yum_repomd
self.primary_fn = self._download_repodata_file(
self.primary_checksum, self.primary_url)
self.filelists_fn = self._download_repodata_file(
self.filelists_checksum, self.filelists_url)
def _download_repo(self, repo_id):
repos = {}
try:
for arch, repo_url in self._koji_repos.items():
h = librepo.Handle()
destdir = self._get_repo_dir(repo_id, arch)
if os.path.exists(destdir):
shutil.rmtree(destdir)
os.makedirs(destdir)
h.destdir = destdir
h.repotype = librepo.LR_YUMREPO
assert '{repo_id}' in repo_url
url = repo_url.format(repo_id=repo_id)
h.urls = [url]
h.yumdlist = ['primary', 'filelists', 'group']
log.info("Downloading {arch} repo from {url}".format(arch=arch,
url=url))
result = h.perform(librepo.Result())
repos[arch] = result
self._add_repo(repo_id, repos)
return repos
except librepo.LibrepoException as e:
if e.args[0] == REPO_404:
log.info("Repo id={} not available, skipping".format(repo_id))
return None
raise
def get_repo(repo_dir, repo_descriptor, download=False):
"""
Obtain hawkey Repo either by loading from disk, or downloading from
Koji.
:repo_dir: path to directory where the repo is/should be stored
:repo_descriptor: which repo to obtain
:download: whether to download or load locally
"""
h = librepo.Handle()
repo_path = os.path.join(repo_dir, str(repo_descriptor))
if download:
h.destdir = repo_path
shutil.rmtree(repo_path, ignore_errors=True)
os.makedirs(os.path.join(repo_path, 'cache'))
h.repotype = librepo.LR_YUMREPO
h.urls = [repo_descriptor.url if download else repo_path]
h.local = not download
h.yumdlist = ['primary', 'filelists', 'group', 'group_gz']
result = librepo.Result()
try:
result = h.perform(result)
except librepo.LibrepoException as e:
if e.args[0] == librepo.LRE_NOURL:
return None
raise
repodata = result.yum_repo
repo = hawkey.Repo(str(repo_descriptor))
repo.repomd_fn = repodata['repomd']
repo.primary_fn = repodata['primary']
repo.filelists_fn = repodata['filelists']
return repo
def get_repodata(self):
if self._dirty:
self._createrepo()
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = [self._srpm_dir]
return h.perform(librepo.Result())
def get_repo(name):
# hawkey sacks cannot be easily populated from within python and mocking
# hawkey queries would be too complicated, therefore using real repos
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = [os.path.join('repo', name)]
h.yumdlist = ['primary', 'filelists', 'group']
return h.perform(librepo.Result())
def _download_repo(self, repo_descriptor, repo_dir):
""" Download repo from Koji to disk. """
h = librepo.Handle()
h.destdir = repo_dir
# pylint:disable=no-member
h.repotype = librepo.LR_YUMREPO
h.urls = [repo_descriptor.make_url()]
h.yumdlist = ['primary', 'filelists', 'group', 'group_gz']
h.perform(librepo.Result())
def test_result_getinfo(self):
r = librepo.Result()
self.assertTrue(r)
self.assertRaises(ValueError, r.getinfo, 99999999)
self.assertFalse(r.getinfo(librepo.LRR_YUM_REPO))
self.assertFalse(r.getinfo(librepo.LRR_YUM_REPOMD))
self.assertRaises(LibrepoException, r.getinfo,
librepo.LRR_YUM_TIMESTAMP)
def test_read_mirrorlist(self):
h = librepo.Handle()
r = librepo.Result()
h.mirrorlist = MIRRORLIST
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.fetchmirrors = True
h.perform(r)
self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/'])
self.assertEqual(h.metalink, None)
def test_read_mirrorlist(self):
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_MIRRORLIST, MIRRORLIST)
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, self.tmpdir)
h.setopt(librepo.LRO_FETCHMIRRORS, True)
h.perform(r)
self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/'])
self.assertEqual(h.metalink, None)
def load_local_repos():
repos = {}
for arch in koji_repos.keys():
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = [os.path.join(repodata_dir, arch)]
h.yumdlist = ['primary', 'filelists', 'group']
repos[arch] = h.perform(librepo.Result())
return repos
def __init__(self, url, base, width=2, depth=2):
if base is None:
base = '/dev/null'
save = False
else:
save = True
super(LibrepoPool, self).__init__(base, spec, width, depth)
# Fetch metadata
h = librepo.Handle()
r = librepo.Result()
h.repotype = librepo.LR_YUMREPO
h.urls = [url]
h.local = not url.startswith('http')
self._repodir = h.destdir = tempfile.mkdtemp()
try:
h.perform(r)
except librepo.LibrepoException as e:
pass
basic = r.getinfo(librepo.LRR_RPMMD_REPO)
full = r.getinfo(librepo.LRR_YUM_REPOMD)
# Add repomd to the result
path = basic['repomd']
with open(path, 'r') as f:
csum = gen_hash(f.read())
basic['paths']['repomd'] = path
full['repomd'] = {'checksum': csum, 'location_href': path}
# Generate objects
for key in ['repomd', 'primary']:
path = basic['paths'][key]
csum = full[key]['checksum']
data = self._read(path)
cls = self.detect(data)
obj = cls(csum, data, path, self)
if save:
self.save(obj)
else:
self._table[csum] = obj
# Generate refs
if h.local:
repo = parse_local_repo(url)
else:
repo = parse_url_repo(url)
self.refs = {repo: full['repomd']['checksum']}
if save:
self.save_refs()
# Clean up
if save:
# We already saved our objects into this pool so no need to keep
# the original download around
self.clean()
def test_result_attrs(self):
r = librepo.Result()
self.assertTrue(r)
self.assertRaises(AttributeError, getattr, r, 'foobar_attr')
# Attrs should not be filled (that's why None or
# LibrepoException is expected), but they definitelly
# should exists (not AttributeError should be raised)
self.assertFalse(r.yum_repo)
self.assertFalse(r.yum_repomd)
self.assertRaises(LibrepoException, getattr, r, 'yum_timestamp')
def test_locate_incomplete_repo_01(self):
# At first, download only some files from the repository
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_URLS, [REPO_YUM_01_PATH])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, self.tmpdir)
h.setopt(librepo.LRO_GPGCHECK, True)
h.setopt(librepo.LRO_YUMDLIST, ["primary"])
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
# Now try to localize the existing repository
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_URLS, [self.tmpdir])
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_LOCAL, True)
self.assertRaises(librepo.LibrepoException, h.perform, (r))
def test_locate_incomplete_repo_01(self):
# At first, download only some files from the repository
h = librepo.Handle()
r = librepo.Result()
h.urls = [REPO_YUM_01_PATH]
h.repotype = librepo.LR_YUMREPO
h.destdir = self.tmpdir
h.gpgcheck = True
h.yumdlist = ["primary"]
h.perform(r)
yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD)
# Now try to localize the existing repository
h = librepo.Handle()
r = librepo.Result()
h.urls = [self.tmpdir]
h.repotype = librepo.LR_YUMREPO
h.local = True
self.assertRaises(librepo.LibrepoException, h.perform, (r))
def parse(self, data):
h = librepo.Handle()
r = librepo.Result()
h.repotype = librepo.LR_YUMREPO
d = self._createrepo()
h.urls = [d]
h.local = True
try:
h.perform(r)
except librepo.LibrepoException as e:
pass
result = r.getinfo(librepo.LRR_YUM_REPOMD)
shutil.rmtree(d)
return [result[t]['checksum'] for t in ['primary']]
def _load_from_disk(self, repo_id):
try:
repos = {}
for arch in self._koji_repos.keys():
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = [self._get_repo_dir(repo_id, arch)]
h.yumdlist = ['primary', 'filelists', 'group']
repos[arch] = h.perform(librepo.Result())
self._add_repo(repo_id, repos)
return repos
except (librepo.LibrepoException, IOError):
pass
def get_sack():
# hawkey sacks cannot be easily populated from within python and mocking
# hawkey queries would be too complicated, therefore using real repos
h = librepo.Handle()
h.local = True
h.repotype = librepo.LR_YUMREPO
h.urls = ['repo']
h.yumdlist = ['primary']
repodata = h.perform(librepo.Result()).yum_repo
repo = hawkey.Repo('test')
repo.repomd_fn = repodata['repomd']
repo.primary_fn = repodata['primary']
sack = hawkey.Sack(arch='x86_64')
sack.load_yum_repo(repo)
return sack
def get_comps_path(repo_dir, repo_descriptor):
"""
Returns path to comps for given repo in repo_dir. It needs to be downloaded
already.
:repo_dir: path to directory where the repo is stored
:repo_descriptor: which repo to obtain
"""
h = librepo.Handle()
repo_path = os.path.join(repo_dir, str(repo_descriptor))
h.repotype = librepo.LR_YUMREPO
h.urls = [repo_path]
h.local = True
h.yumdlist = ['group']
result = h.perform(librepo.Result())
return result.yum_repo.get('group')
def librepo_getinfo(self, dirname):
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_URLS, [dirname])
h.setopt(librepo.LRO_LOCAL, True)
h.setopt(librepo.LRO_CHECKSUM, True)
h.setopt(librepo.LRO_IGNOREMISSING, True)
try:
h.perform(r)
except librepo.LibrepoException as e:
utils.die(self.logger, "librepo error: " + dirname + " - " + e.args[1])
rmd = r.getinfo( librepo.LRR_RPMMD_REPOMD )['records']
return( rmd )
def download_koji_repos():
repos = {}
for arch, repo_url in koji_repos.items():
h = librepo.Handle()
h.destdir = os.path.join(repodata_dir, arch)
if os.path.exists(h.destdir):
shutil.rmtree(h.destdir)
os.mkdir(h.destdir)
h.repotype = librepo.LR_YUMREPO
h.urls = [repo_url]
h.yumdlist = ['primary', 'filelists', 'group']
log.info("Downloading {arch} repo from {url}".format(arch=arch,
url=repo_url))
result = h.perform(librepo.Result())
repos[arch] = result
return repos
def download(self, destdir, wanted_metadata=None):
self.destdir = destdir
h = librepo.Handle()
h.urls = self.urls
h.mirrorlisturl = self.mirrorlist
h.metalinkurl = self.metalink
h.repotype = librepo.YUMREPO
h.interruptible = True
h.destdir = destdir
h.yumdlist = wanted_metadata
r = librepo.Result()
# TODO: Catch exceptions
h.perform(r)
self.h = h
self.r = r
def _load_sack(self, repo_descriptor, repo_path, build_cache=False):
""" Load repo from disk into memory as sack. """
cache_dir = os.path.join(repo_path, 'cache')
for_arch = get_config('dependency.resolve_for_arch')
if build_cache:
os.mkdir(cache_dir)
sack = hawkey.Sack(arch=for_arch, cachedir=cache_dir)
h = librepo.Handle()
h.local = True
# pylint:disable=no-member
h.repotype = librepo.LR_YUMREPO
h.urls = [repo_path]
h.yumdlist = ['primary', 'filelists', 'group']
repodata = h.perform(librepo.Result()).yum_repo
repo = hawkey.Repo(str(repo_descriptor))
repo.repomd_fn = repodata['repomd']
repo.primary_fn = repodata['primary']
repo.filelists_fn = repodata['filelists']
sack.load_yum_repo(repo, load_filelists=True, build_cache=build_cache)
return sack
def librepo_getinfo(self, dirname):
h = librepo.Handle()
r = librepo.Result()
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_URLS, [dirname])
h.setopt(librepo.LRO_LOCAL, True)
h.setopt(librepo.LRO_CHECKSUM, True)
h.setopt(librepo.LRO_IGNOREMISSING, True)
try:
h.perform(r)
except librepo.LibrepoException as e:
rc, msg, general_msg = e
if rc == librepo.LRE_BADCHECKSUM:
utils.die(self.logger,
"Corrupted metadata: " + dirname + " " + msg)
else:
utils.die(self.logger, "librepo error: " + dirname + " " + msg)
rmd = r.getinfo(librepo.LRR_RPMMD_REPOMD)['records']
return (rmd)
