def test_locate_incomplete_repo_01_2(self): # At first, download whole repository h = librepo.Handle() r = librepo.Result() h.urls = [REPO_YUM_01_PATH] h.repotype = librepo.LR_YUMREPO h.destdir = self.tmpdir h.gpgcheck = True h.yumdlist = ["primary"] h.perform(r) yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO) yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD) # Now try to localize the existing repository and all its files h = librepo.Handle() r = librepo.Result() h.urls = [self.tmpdir] h.repotype = librepo.LR_YUMREPO h.local = True h.ignoremissing = True h.perform(r) yum_repo = r.getinfo(librepo.LRR_YUM_REPO) yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD) # Compare results yum_repo_downloaded["url"] = None self.assertEqual(yum_repo, yum_repo_downloaded) self.assertEqual(yum_repomd, yum_repomd_downloaded)
def test_locate_incomplete_repo_01_2(self): # At first, download whole repository h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_URLS, [REPO_YUM_01_PATH]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_DESTDIR, self.tmpdir) h.setopt(librepo.LRO_GPGCHECK, True) h.setopt(librepo.LRO_YUMDLIST, ["primary"]) h.perform(r) yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO) yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD) # Now try to localize the existing repository and all its files h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_URLS, [self.tmpdir]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_LOCAL, True) h.setopt(librepo.LRO_IGNOREMISSING, True) h.perform(r) yum_repo = r.getinfo(librepo.LRR_YUM_REPO) yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD) # Compare results yum_repo_downloaded["url"] = None self.assertEqual(yum_repo, yum_repo_downloaded) self.assertEqual(yum_repomd, yum_repomd_downloaded)
def test_read_metalink_of_local_repo(self): # At first, download whole repository # Check if local metalink.xml will be copied as well h = librepo.Handle() r = librepo.Result() h.urls = [REPO_YUM_01_PATH] h.repotype = librepo.LR_YUMREPO h.destdir = self.tmpdir h.gpgcheck = True h.perform(r) yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO) yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD) self.assertFalse(yum_repo_downloaded["mirrorlist"]) self.assertTrue(yum_repo_downloaded["metalink"]) self.assertTrue( yum_repo_downloaded["metalink"].endswith("metalink.xml")) # Now try to read metalink of the repository h = librepo.Handle() r = librepo.Result() h.urls = [self.tmpdir] h.repotype = librepo.LR_YUMREPO h.local = True h.fetchmirrors = True h.perform(r) self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/']) self.assertEqual( h.metalink, { 'timestamp': 1347459931, 'hashes': [('md5', 'f76409f67a84bcd516131d5cc98e57e1'), ('sha1', '75125e73304c21945257d9041a908d0d01d2ca16'), ('sha256', 'bef5d33dc68f47adc7b31df448851b1e9e6bae27840f28700fff144881482a6a' ), ('sha512', 'e40060c747895562e945a68967a04d1279e4bd8507413681f83c322479aa564027fdf3962c2d875089bfcb9317d3a623465f390dc1f4acef294711168b807af0' )], 'size': 2621, 'urls': [{ 'url': 'http://127.0.0.1:5000/yum/static/01/repodata/repomd.xml', 'type': 'http', 'protocol': 'http', 'location': 'CZ', 'preference': 100 }], 'filename': 'repomd.xml' })
def create(self, repo_id, repo_dir): """ Load repo from disk into memory as sack """ if not repo_dir: return None try: sack = hawkey.Sack(arch=self._for_arch) for arch in self._arches: log.debug('Loading repo {} for arch {} from disk into memory'. format(repo_id, arch)) arch_repo_dir = os.path.join(repo_dir, arch) h = librepo.Handle() h.local = True h.repotype = librepo.LR_YUMREPO h.urls = [arch_repo_dir] h.yumdlist = ['primary', 'filelists', 'group'] repodata = h.perform(librepo.Result()).yum_repo repo = hawkey.Repo('{}-{}'.format(repo_id, arch)) repo.repomd_fn = repodata['repomd'] repo.primary_fn = repodata['primary'] repo.filelists_fn = repodata['filelists'] sack.load_yum_repo(repo, load_filelists=True) log.debug('Repo {} successfully loaded into memory'.format(repo_id)) return sack except (librepo.LibrepoException, IOError): log.debug('Repo {} could not be loaded'.format(repo_id)) return None
def get_repo_metadata(reponame, repopath): repo_handle = librepo.Handle() repo_result = librepo.Result() if repopath.startswith('/'): repopath = 'file://%s' % repopath # Set the repository URL. repo_handle.setopt(librepo.LRO_URL, repopath) if repopath.startswith('http://') or repopath.startswith('ftp://'): # Set the metadata destination directory. destdir = tempfile.mkdtemp(prefix='%s.' % reponame) repo_handle.setopt(librepo.LRO_DESTDIR, destdir) elif repopath.startswith('file://'): repo_handle.setopt(librepo.LRO_LOCAL, True) destdir = None else: raise ValueError("Incorrect repo path '%s'" % repopath) # Set the repository type. repo_handle.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) # Download primary.xml and filelists.xml - repomd.xml is downloaded automatically. repo_handle.setopt(librepo.LRO_YUMDLIST, ['primary', 'filelists']) repo_handle.perform(repo_result) return repo_result.getinfo(librepo.LRR_YUM_REPO), destdir
def create(self, repo_id, build_tag): assert build_tag self._clean_repo_dir(repo_id) repo_dir = self._get_repo_dir(repo_id) temp_dir = repo_dir + ".tmp" if os.path.exists(temp_dir): shutil.rmtree(temp_dir) try: for arch in self._arches: log.debug('Downloading {} repo {} for arch {} from Koji to disk'. format(build_tag, repo_id, arch)) h = librepo.Handle() arch_repo_dir = os.path.join(temp_dir, arch) os.makedirs(arch_repo_dir) h.destdir = arch_repo_dir h.repotype = librepo.LR_YUMREPO url = self._remote_repo.format(repo_id=repo_id, arch=arch, build_tag=build_tag) h.urls = [url] h.yumdlist = ['primary', 'filelists', 'group'] h.perform(librepo.Result()) os.rename(temp_dir, repo_dir) log.debug('Repo {} successfully downloaded to disk'.format(repo_id)) return repo_dir except librepo.LibrepoException as e: if e.args[0] == REPO_404: log.debug('Repo {} was not found'.format(repo_id)) return None raise
def test_read_metalink(self): h = librepo.Handle() r = librepo.Result() h.mirrorlist = METALINK h.repotype = librepo.LR_YUMREPO h.destdir = self.tmpdir h.fetchmirrors = True h.perform(r) self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/']) self.assertEqual( h.metalink, { 'timestamp': 1347459931, 'hashes': [('md5', 'f76409f67a84bcd516131d5cc98e57e1'), ('sha1', '75125e73304c21945257d9041a908d0d01d2ca16'), ('sha256', 'bef5d33dc68f47adc7b31df448851b1e9e6bae27840f28700fff144881482a6a' ), ('sha512', 'e40060c747895562e945a68967a04d1279e4bd8507413681f83c322479aa564027fdf3962c2d875089bfcb9317d3a623465f390dc1f4acef294711168b807af0' )], 'size': 2621, 'urls': [{ 'url': 'http://127.0.0.1:5000/yum/static/01/repodata/repomd.xml', 'type': 'http', 'protocol': 'http', 'location': 'CZ', 'preference': 100 }], 'filename': 'repomd.xml' })
def download_repodata(self): logger.debug('Loading repodata for %s from %s', self.name, self.baseurl or self.metalink) self.librepo_handle = h = librepo.Handle() r = librepo.Result() h.repotype = librepo.LR_YUMREPO if self.baseurl: h.urls = [self.baseurl] if self.metalink: h.mirrorlist = self.metalink h.setopt(librepo.LRO_DESTDIR, tempfile.mkdtemp(self.name, prefix=REPO_CACHE_NAME_PREFIX, dir=REPO_CACHE_DIR)) h.setopt(librepo.LRO_INTERRUPTIBLE, True) h.setopt(librepo.LRO_YUMDLIST, []) if self.baseurl and os.path.isdir(self.baseurl): self._download_metadata_result(h, r) self._yum_repomd = r.yum_repomd self._root_path = self.baseurl self.primary_fn = self.primary_url self.filelists_fn = self.filelists_url else: self._root_path = h.destdir = tempfile.mkdtemp(self.name, prefix=REPO_CACHE_NAME_PREFIX, dir=REPO_CACHE_DIR) self._download_metadata_result(h, r) self._yum_repomd = r.yum_repomd self.primary_fn = self._download_repodata_file( self.primary_checksum, self.primary_url) self.filelists_fn = self._download_repodata_file( self.filelists_checksum, self.filelists_url)
def _download_repo(self, repo_id): repos = {} try: for arch, repo_url in self._koji_repos.items(): h = librepo.Handle() destdir = self._get_repo_dir(repo_id, arch) if os.path.exists(destdir): shutil.rmtree(destdir) os.makedirs(destdir) h.destdir = destdir h.repotype = librepo.LR_YUMREPO assert '{repo_id}' in repo_url url = repo_url.format(repo_id=repo_id) h.urls = [url] h.yumdlist = ['primary', 'filelists', 'group'] log.info("Downloading {arch} repo from {url}".format(arch=arch, url=url)) result = h.perform(librepo.Result()) repos[arch] = result self._add_repo(repo_id, repos) return repos except librepo.LibrepoException as e: if e.args[0] == REPO_404: log.info("Repo id={} not available, skipping".format(repo_id)) return None raise
def get_repo(repo_dir, repo_descriptor, download=False): """ Obtain hawkey Repo either by loading from disk, or downloading from Koji. :repo_dir: path to directory where the repo is/should be stored :repo_descriptor: which repo to obtain :download: whether to download or load locally """ h = librepo.Handle() repo_path = os.path.join(repo_dir, str(repo_descriptor)) if download: h.destdir = repo_path shutil.rmtree(repo_path, ignore_errors=True) os.makedirs(os.path.join(repo_path, 'cache')) h.repotype = librepo.LR_YUMREPO h.urls = [repo_descriptor.url if download else repo_path] h.local = not download h.yumdlist = ['primary', 'filelists', 'group', 'group_gz'] result = librepo.Result() try: result = h.perform(result) except librepo.LibrepoException as e: if e.args[0] == librepo.LRE_NOURL: return None raise repodata = result.yum_repo repo = hawkey.Repo(str(repo_descriptor)) repo.repomd_fn = repodata['repomd'] repo.primary_fn = repodata['primary'] repo.filelists_fn = repodata['filelists'] return repo
def get_repodata(self): if self._dirty: self._createrepo() h = librepo.Handle() h.local = True h.repotype = librepo.LR_YUMREPO h.urls = [self._srpm_dir] return h.perform(librepo.Result())
def get_repo(name): # hawkey sacks cannot be easily populated from within python and mocking # hawkey queries would be too complicated, therefore using real repos h = librepo.Handle() h.local = True h.repotype = librepo.LR_YUMREPO h.urls = [os.path.join('repo', name)] h.yumdlist = ['primary', 'filelists', 'group'] return h.perform(librepo.Result())
def _download_repo(self, repo_descriptor, repo_dir): """ Download repo from Koji to disk. """ h = librepo.Handle() h.destdir = repo_dir # pylint:disable=no-member h.repotype = librepo.LR_YUMREPO h.urls = [repo_descriptor.make_url()] h.yumdlist = ['primary', 'filelists', 'group', 'group_gz'] h.perform(librepo.Result())
def test_result_getinfo(self): r = librepo.Result() self.assertTrue(r) self.assertRaises(ValueError, r.getinfo, 99999999) self.assertFalse(r.getinfo(librepo.LRR_YUM_REPO)) self.assertFalse(r.getinfo(librepo.LRR_YUM_REPOMD)) self.assertRaises(LibrepoException, r.getinfo, librepo.LRR_YUM_TIMESTAMP)
def test_read_mirrorlist(self): h = librepo.Handle() r = librepo.Result() h.mirrorlist = MIRRORLIST h.repotype = librepo.LR_YUMREPO h.destdir = self.tmpdir h.fetchmirrors = True h.perform(r) self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/']) self.assertEqual(h.metalink, None)
def test_read_mirrorlist(self): h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_MIRRORLIST, MIRRORLIST) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_DESTDIR, self.tmpdir) h.setopt(librepo.LRO_FETCHMIRRORS, True) h.perform(r) self.assertEqual(h.mirrors, ['http://127.0.0.1:5000/yum/static/01/']) self.assertEqual(h.metalink, None)
def load_local_repos(): repos = {} for arch in koji_repos.keys(): h = librepo.Handle() h.local = True h.repotype = librepo.LR_YUMREPO h.urls = [os.path.join(repodata_dir, arch)] h.yumdlist = ['primary', 'filelists', 'group'] repos[arch] = h.perform(librepo.Result()) return repos
def __init__(self, url, base, width=2, depth=2): if base is None: base = '/dev/null' save = False else: save = True super(LibrepoPool, self).__init__(base, spec, width, depth) # Fetch metadata h = librepo.Handle() r = librepo.Result() h.repotype = librepo.LR_YUMREPO h.urls = [url] h.local = not url.startswith('http') self._repodir = h.destdir = tempfile.mkdtemp() try: h.perform(r) except librepo.LibrepoException as e: pass basic = r.getinfo(librepo.LRR_RPMMD_REPO) full = r.getinfo(librepo.LRR_YUM_REPOMD) # Add repomd to the result path = basic['repomd'] with open(path, 'r') as f: csum = gen_hash(f.read()) basic['paths']['repomd'] = path full['repomd'] = {'checksum': csum, 'location_href': path} # Generate objects for key in ['repomd', 'primary']: path = basic['paths'][key] csum = full[key]['checksum'] data = self._read(path) cls = self.detect(data) obj = cls(csum, data, path, self) if save: self.save(obj) else: self._table[csum] = obj # Generate refs if h.local: repo = parse_local_repo(url) else: repo = parse_url_repo(url) self.refs = {repo: full['repomd']['checksum']} if save: self.save_refs() # Clean up if save: # We already saved our objects into this pool so no need to keep # the original download around self.clean()
def test_result_attrs(self): r = librepo.Result() self.assertTrue(r) self.assertRaises(AttributeError, getattr, r, 'foobar_attr') # Attrs should not be filled (that's why None or # LibrepoException is expected), but they definitelly # should exists (not AttributeError should be raised) self.assertFalse(r.yum_repo) self.assertFalse(r.yum_repomd) self.assertRaises(LibrepoException, getattr, r, 'yum_timestamp')
def test_locate_incomplete_repo_01(self): # At first, download only some files from the repository h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_URLS, [REPO_YUM_01_PATH]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_DESTDIR, self.tmpdir) h.setopt(librepo.LRO_GPGCHECK, True) h.setopt(librepo.LRO_YUMDLIST, ["primary"]) h.perform(r) yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO) yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD) # Now try to localize the existing repository h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_URLS, [self.tmpdir]) h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_LOCAL, True) self.assertRaises(librepo.LibrepoException, h.perform, (r))
def test_locate_incomplete_repo_01(self): # At first, download only some files from the repository h = librepo.Handle() r = librepo.Result() h.urls = [REPO_YUM_01_PATH] h.repotype = librepo.LR_YUMREPO h.destdir = self.tmpdir h.gpgcheck = True h.yumdlist = ["primary"] h.perform(r) yum_repo_downloaded = r.getinfo(librepo.LRR_YUM_REPO) yum_repomd_downloaded = r.getinfo(librepo.LRR_YUM_REPOMD) # Now try to localize the existing repository h = librepo.Handle() r = librepo.Result() h.urls = [self.tmpdir] h.repotype = librepo.LR_YUMREPO h.local = True self.assertRaises(librepo.LibrepoException, h.perform, (r))
def parse(self, data): h = librepo.Handle() r = librepo.Result() h.repotype = librepo.LR_YUMREPO d = self._createrepo() h.urls = [d] h.local = True try: h.perform(r) except librepo.LibrepoException as e: pass result = r.getinfo(librepo.LRR_YUM_REPOMD) shutil.rmtree(d) return [result[t]['checksum'] for t in ['primary']]
def _load_from_disk(self, repo_id): try: repos = {} for arch in self._koji_repos.keys(): h = librepo.Handle() h.local = True h.repotype = librepo.LR_YUMREPO h.urls = [self._get_repo_dir(repo_id, arch)] h.yumdlist = ['primary', 'filelists', 'group'] repos[arch] = h.perform(librepo.Result()) self._add_repo(repo_id, repos) return repos except (librepo.LibrepoException, IOError): pass
def get_sack(): # hawkey sacks cannot be easily populated from within python and mocking # hawkey queries would be too complicated, therefore using real repos h = librepo.Handle() h.local = True h.repotype = librepo.LR_YUMREPO h.urls = ['repo'] h.yumdlist = ['primary'] repodata = h.perform(librepo.Result()).yum_repo repo = hawkey.Repo('test') repo.repomd_fn = repodata['repomd'] repo.primary_fn = repodata['primary'] sack = hawkey.Sack(arch='x86_64') sack.load_yum_repo(repo) return sack
def get_comps_path(repo_dir, repo_descriptor): """ Returns path to comps for given repo in repo_dir. It needs to be downloaded already. :repo_dir: path to directory where the repo is stored :repo_descriptor: which repo to obtain """ h = librepo.Handle() repo_path = os.path.join(repo_dir, str(repo_descriptor)) h.repotype = librepo.LR_YUMREPO h.urls = [repo_path] h.local = True h.yumdlist = ['group'] result = h.perform(librepo.Result()) return result.yum_repo.get('group')
def librepo_getinfo(self, dirname): h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_URLS, [dirname]) h.setopt(librepo.LRO_LOCAL, True) h.setopt(librepo.LRO_CHECKSUM, True) h.setopt(librepo.LRO_IGNOREMISSING, True) try: h.perform(r) except librepo.LibrepoException as e: utils.die(self.logger, "librepo error: " + dirname + " - " + e.args[1]) rmd = r.getinfo( librepo.LRR_RPMMD_REPOMD )['records'] return( rmd )
def download_koji_repos(): repos = {} for arch, repo_url in koji_repos.items(): h = librepo.Handle() h.destdir = os.path.join(repodata_dir, arch) if os.path.exists(h.destdir): shutil.rmtree(h.destdir) os.mkdir(h.destdir) h.repotype = librepo.LR_YUMREPO h.urls = [repo_url] h.yumdlist = ['primary', 'filelists', 'group'] log.info("Downloading {arch} repo from {url}".format(arch=arch, url=repo_url)) result = h.perform(librepo.Result()) repos[arch] = result return repos
def download(self, destdir, wanted_metadata=None): self.destdir = destdir h = librepo.Handle() h.urls = self.urls h.mirrorlisturl = self.mirrorlist h.metalinkurl = self.metalink h.repotype = librepo.YUMREPO h.interruptible = True h.destdir = destdir h.yumdlist = wanted_metadata r = librepo.Result() # TODO: Catch exceptions h.perform(r) self.h = h self.r = r
def _load_sack(self, repo_descriptor, repo_path, build_cache=False): """ Load repo from disk into memory as sack. """ cache_dir = os.path.join(repo_path, 'cache') for_arch = get_config('dependency.resolve_for_arch') if build_cache: os.mkdir(cache_dir) sack = hawkey.Sack(arch=for_arch, cachedir=cache_dir) h = librepo.Handle() h.local = True # pylint:disable=no-member h.repotype = librepo.LR_YUMREPO h.urls = [repo_path] h.yumdlist = ['primary', 'filelists', 'group'] repodata = h.perform(librepo.Result()).yum_repo repo = hawkey.Repo(str(repo_descriptor)) repo.repomd_fn = repodata['repomd'] repo.primary_fn = repodata['primary'] repo.filelists_fn = repodata['filelists'] sack.load_yum_repo(repo, load_filelists=True, build_cache=build_cache) return sack
def librepo_getinfo(self, dirname): h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) h.setopt(librepo.LRO_URLS, [dirname]) h.setopt(librepo.LRO_LOCAL, True) h.setopt(librepo.LRO_CHECKSUM, True) h.setopt(librepo.LRO_IGNOREMISSING, True) try: h.perform(r) except librepo.LibrepoException as e: rc, msg, general_msg = e if rc == librepo.LRE_BADCHECKSUM: utils.die(self.logger, "Corrupted metadata: " + dirname + " " + msg) else: utils.die(self.logger, "librepo error: " + dirname + " " + msg) rmd = r.getinfo(librepo.LRR_RPMMD_REPOMD)['records'] return (rmd)