def fetch(self, fetcher, progress): fetcher.reset() if self._mirrorurl: mirrorlist = self._mirrorurl item = fetcher.enqueue(mirrorlist) fetcher.run(progress=progress) if item.getStatus() is FAILED: progress.add(self.getFetchSteps()-1) if fetcher.getCaching() is NEVER: iface.warning(_("Could not load mirror list. Continuing with base URL only.")) else: self._mirrors.clear() mirrorurls = [] mirrors = self.loadMirrors(item.getTargetPath()) for mirror in mirrors: scheme = mirror["url"].split(":")[0] if not fetcher.getHandler(scheme, None): continue if mirror["type"] != "distrib": continue mirrorurls.append(mirror["url"]) if mirrorurls: self._mirrors[self._baseurl] = mirrorurls fetcher.reset() else: progress.add(1) self._compareurl = self._hdlurl hdlbaseurl, basename = os.path.split(self._hdlurl) infoname = os.path.split(self._infourl)[1] md5url = posixpath.join(hdlbaseurl, "MD5SUM") item = fetcher.enqueue(md5url) fetcher.run(progress=progress) hdlmd5 = None infomd5 = None failed = item.getFailedReason() if not failed: self._compareurl = md5url digest = getFileDigest(item.getTargetPath()) if digest == self._digest: progress.add(3) return True basename = posixpath.basename(self._hdlurl) infoname = posixpath.basename(self._infourl) try: for line in open(item.getTargetPath()): line = line.strip() if line: md5, name = line.split() if name == basename: hdlmd5 = md5 if name == infoname: infomd5 = md5 except ValueError: pass fetcher.reset() hdlitem = fetcher.enqueue(self._hdlurl, md5=hdlmd5, uncomp=True) if infomd5: infoitem = fetcher.enqueue(self._infourl, md5=infomd5, uncomp=True) else: progress.add(1) infoitem = None # do not get "descriptions" on non "update" media if self.getName().find("Updates") == -1: progress.add(1) descitem = None else: descurl = posixpath.join(hdlbaseurl, "descriptions") descitem = fetcher.enqueue(descurl) fetcher.run(progress=progress) if hdlitem.getStatus() == FAILED: hdfailed = hdlitem.getFailedReason() if fetcher.getCaching() is NEVER: # Try reading reconfig.urpmi (should give new path) fetcher.reset() reconfigurl = posixpath.join(hdlbaseurl, "reconfig.urpmi") reconfigitem = fetcher.enqueue(reconfigurl) fetcher.run(progress=progress) if reconfigitem.getStatus() == FAILED: refailed = reconfigitem.getFailedReason() if fetcher.getCaching() is NEVER: lines = [_("Failed acquiring information for '%s':") % self, u"%s: %s" % (hdlitem.getURL(), hdfailed), u"%s: %s" % (reconfigitem.getURL(), refailed)] raise Error, "\n".join(lines) return False else: # Need to inject "/" at the end to avoid buggy urls if not hdlbaseurl.endswith("/"): hdlbaseurl += "/" for line in open(reconfigitem.getTargetPath()): if line.startswith("#"): pass elif line: splitline = line.split() arch = os.uname()[4] if arch == "i686": arch = "i586" reconfarch = re.sub("\$ARCH", arch, splitline[1]) reconfpath = re.sub(splitline[0] + "$", reconfarch, hdlbaseurl) sysconf.set(("channels", self.getAlias(), \ "baseurl"), reconfpath) self._hdlurl = os.path.join(reconfpath, basename) self._infourl = os.path.join(reconfpath, infoname) return self.fetch(fetcher, progress) return False else: localpath = hdlitem.getTargetPath() digestpath = None infopath = None listpath = None if self._compareurl == self._hdlurl: digestpath = localpath if digestpath: digest = getFileDigest(digestpath) if digest == self._digest: return True self.removeLoaders() if localpath.endswith(".cz"): if (not os.path.isfile(localpath[:-3]) or fetcher.getCaching() != ALWAYS): linkpath = fetcher.getLocalPath(hdlitem) linkpath = linkpath[:-2]+"gz" if not os.access(os.path.dirname(linkpath), os.W_OK): dirname = os.path.join(sysconf.get("user-data-dir"), "channels") basename = os.path.basename(linkpath) if not os.path.isdir(dirname): os.makedirs(dirname) linkpath = os.path.join(dirname, basename) if os.path.isfile(linkpath): os.unlink(linkpath) os.symlink(localpath, linkpath) localpath = linkpath uncompressor = fetcher.getUncompressor() uncomphandler = uncompressor.getHandler(linkpath) try: uncomphandler.uncompress(linkpath) except Error, e: # cz file has trailing information which breaks # current gzip module logic. if "Not a gzipped file" not in e[0]: os.unlink(linkpath) raise os.unlink(linkpath) localpath = localpath[:-3] if infoitem and infoitem.getStatus() == SUCCEEDED: infopath = infoitem.getTargetPath() elif infoitem and infoitem.getStatus() == FAILED: lines = [_("Failed acquiring information for '%s':") % self, u"%s: %s" % (infoitem.getURL(), infoitem.getFailedReason())] raise Warning, "\n".join(lines) flagdict = {} if descitem and descitem.getStatus() == SUCCEEDED: descpath = descitem.getTargetPath() errata = RPMDescriptions(descpath) errata.load() #errata.setErrataFlags() <-- done in loader flagdict = errata.getErrataFlags() baseurl = self._baseurl directory = self._directory if directory: baseurl += "/" + directory + "/" if open(localpath).read(4) == "\x8e\xad\xe8\x01": loader = URPMILoader(localpath, baseurl, listpath) else: loader = URPMISynthesisLoader(localpath, baseurl, listpath, infopath) # need to set flags while loading loader.setErrataFlags(flagdict) loader.setChannel(self) self._loaders.append(loader)
def fetch(self, fetcher, progress): fetcher.reset() if self._mirrorurl: mirrorlist = self._mirrorurl item = fetcher.enqueue(mirrorlist) fetcher.run(progress=progress) if item.getStatus() is FAILED: progress.add(self.getFetchSteps()-1) if fetcher.getCaching() is NEVER: iface.warning(_("Could not load mirror list. Continuing with base URL only.")) else: self._mirrors.clear() mirrorurls = [] mirrors = self.loadMirrors(item.getTargetPath()) for mirror in mirrors: scheme = mirror["url"].split(":")[0] if not fetcher.getHandler(scheme, None): continue if mirror["type"] != "distrib": continue mirrorurls.append(mirror["url"]) if mirrorurls: self._mirrors[self._baseurl] = mirrorurls fetcher.reset() else: progress.add(1) self._compareurl = self._hdlurl hdlbaseurl, basename = os.path.split(self._hdlurl) infoname = os.path.split(self._infourl)[1] md5url = posixpath.join(hdlbaseurl, "MD5SUM") item = fetcher.enqueue(md5url) fetcher.run(progress=progress) hdlmd5 = None infomd5 = None failed = item.getFailedReason() if not failed: self._compareurl = md5url digest = getFileDigest(item.getTargetPath()) if digest == self._digest: progress.add(3) return True basename = posixpath.basename(self._hdlurl) infoname = posixpath.basename(self._infourl) try: for line in open(item.getTargetPath()): line = line.strip() if line: md5, name = line.split() if name == basename: hdlmd5 = md5 if name == infoname: infomd5 = md5 except ValueError: pass fetcher.reset() hdlitem = fetcher.enqueue(self._hdlurl, md5=hdlmd5, uncomp=True) if infomd5: infoitem = fetcher.enqueue(self._infourl, md5=infomd5, uncomp=True) else: progress.add(1) infoitem = None # do not get "descriptions" on non "update" media if self.getName().find("Updates") == -1: progress.add(1) descitem = None else: descurl = posixpath.join(hdlbaseurl, "descriptions") descitem = fetcher.enqueue(descurl) fetcher.run(progress=progress) if hdlitem.getStatus() == FAILED: hdfailed = hdlitem.getFailedReason() if fetcher.getCaching() is NEVER: # Try reading reconfig.urpmi (should give new path) fetcher.reset() reconfigurl = posixpath.join(hdlbaseurl, "reconfig.urpmi") reconfigitem = fetcher.enqueue(reconfigurl) fetcher.run(progress=progress) if reconfigitem.getStatus() == FAILED: refailed = reconfigitem.getFailedReason() if fetcher.getCaching() is NEVER: lines = [_("Failed acquiring information for '%s':") % self, u"%s: %s" % (hdlitem.getURL(), hdfailed), u"%s: %s" % (reconfigitem.getURL(), refailed)] raise Error, "\n".join(lines) return False else: # Need to inject "/" at the end to avoid buggy urls if not hdlbaseurl.endswith("/"): hdlbaseurl += "/" for line in open(reconfigitem.getTargetPath()): if line.startswith("#"): pass elif line: splitline = line.split() arch = os.uname()[4] if arch == "i686": arch = "i586" reconfarch = re.sub("\$ARCH", arch, splitline[1]) reconfpath = re.sub(splitline[0] + "$", reconfarch, hdlbaseurl) sysconf.set(("channels", self.getAlias(), \ "baseurl"), reconfpath) self._hdlurl = os.path.join(reconfpath, basename) self._infourl = os.path.join(reconfpath, infoname) return self.fetch(fetcher, progress) return False else: localpath = hdlitem.getTargetPath() digestpath = None infopath = None listpath = None if self._compareurl == self._hdlurl: digestpath = localpath if digestpath: digest = getFileDigest(digestpath) if digest == self._digest: return True self.removeLoaders() if localpath.endswith(".cz"): if (not os.path.isfile(localpath[:-3]) or fetcher.getCaching() != ALWAYS): linkpath = fetcher.getLocalPath(hdlitem) # different old vs new cz format detection #ext = commands.getoutput("file "+linkpath+"|awk -F'cz: ' '{print $2}'|cut -d' ' -f1") ms = magic.open(magic.NONE) ms.load() ext = ms.file(linkpath).split(' ')[0] #os.system('echo ' + ext + '>/tmp/RUN') if ext == "gzip": linkpath = linkpath[:-2]+"gz" else: linkpath = linkpath[:-2]+"xz" ms.close() if not os.access(os.path.dirname(linkpath), os.W_OK): dirname = os.path.join(sysconf.get("user-data-dir"), "channels") basename = os.path.basename(linkpath) if not os.path.isdir(dirname): os.makedirs(dirname) linkpath = os.path.join(dirname, basename) if os.path.isfile(linkpath): os.unlink(linkpath) os.symlink(localpath, linkpath) localpath = linkpath uncompressor = fetcher.getUncompressor() uncomphandler = uncompressor.getHandler(linkpath) try: uncomphandler.uncompress(linkpath) except Error, e: # cz file has trailing information which breaks # current gzip module logic. if "Not a gzipped file" not in e[0]: os.unlink(linkpath) raise os.unlink(linkpath) localpath = localpath[:-3] if infoitem and infoitem.getStatus() == SUCCEEDED: infopath = infoitem.getTargetPath() elif infoitem and infoitem.getStatus() == FAILED: lines = [_("Failed acquiring information for '%s':") % self, u"%s: %s" % (infoitem.getURL(), infoitem.getFailedReason())] raise Warning, "\n".join(lines) flagdict = {} if descitem and descitem.getStatus() == SUCCEEDED: descpath = descitem.getTargetPath() errata = RPMDescriptions(descpath) errata.load() #errata.setErrataFlags() <-- done in loader flagdict = errata.getErrataFlags() baseurl = self._baseurl directory = self._directory if directory: baseurl += "/" + directory + "/" if open(localpath).read(4) == "\x8e\xad\xe8\x01": loader = URPMILoader(localpath, baseurl, listpath) else: loader = URPMISynthesisLoader(localpath, baseurl, listpath, infopath) # need to set flags while loading loader.setErrataFlags(flagdict) loader.setChannel(self) self._loaders.append(loader)
def fetch(self, fetcher, progress): fetcher.reset() self._compareurl = self._hdlurl hdlbaseurl, basename = os.path.split(self._hdlurl) md5url = posixpath.join(hdlbaseurl, "MD5SUM") item = fetcher.enqueue(md5url) fetcher.run(progress=progress) hdlmd5 = None failed = item.getFailedReason() if not failed: self._compareurl = md5url digest = getFileDigest(item.getTargetPath()) if digest == self._digest: progress.add(2) return True basename = posixpath.basename(self._hdlurl) for line in open(item.getTargetPath()): md5, name = line.split() if name == basename: hdlmd5 = md5 break fetcher.reset() hdlitem = fetcher.enqueue(self._hdlurl, md5=hdlmd5, uncomp=True) if self._hdlurl.endswith("/list"): listitem = None else: m = re.compile(r"/(?:synthesis\.)?hdlist(.*)\.") \ .search(self._hdlurl) suffix = m and m.group(1) or "" listurl = posixpath.join(hdlbaseurl, "list%s" % suffix) listitem = fetcher.enqueue(listurl, uncomp=True) fetcher.run(progress=progress) if hdlitem.getStatus() == FAILED: failed = hdlitem.getFailedReason() if fetcher.getCaching() is NEVER: lines = [_("Failed acquiring information for '%s':") % self, u"%s: %s" % (hdlitem.getURL(), failed)] raise Error, "\n".join(lines) return False else: localpath = hdlitem.getTargetPath() digestpath = None if listitem and listitem.getStatus() == SUCCEEDED: if self._compareurl == self._hdlurl: self._compareurl = listurl digestpath = localpath listpath = listitem.getTargetPath() else: listpath = None if self._compareurl == self._hdlurl: digestpath = localpath if digestpath: digest = getFileDigest(digestpath) if digest == self._digest: return True self.removeLoaders() if localpath.endswith(".cz"): if (not os.path.isfile(localpath[:-3]) or fetcher.getCaching() != ALWAYS): linkpath = fetcher.getLocalPath(hdlitem) linkpath = linkpath[:-2]+"gz" if not os.access(os.path.dirname(linkpath), os.W_OK): dirname = os.path.join(sysconf.get("user-data-dir"), "channels") basename = os.path.basename(linkpath) if not os.path.isdir(dirname): os.makedirs(dirname) linkpath = os.path.join(dirname, basename) if os.path.isfile(linkpath): os.unlink(linkpath) os.symlink(localpath, linkpath) localpath = linkpath uncompressor = fetcher.getUncompressor() uncomphandler = uncompressor.getHandler(linkpath) try: uncomphandler.uncompress(linkpath) except Error, e: # cz file has trailing information which breaks # current gzip module logic. if "Not a gzipped file" not in e[0]: os.unlink(linkpath) raise os.unlink(linkpath) localpath = localpath[:-3] if open(localpath).read(4) == "\x8e\xad\xe8\x01": loader = URPMILoader(localpath, self._baseurl, listpath) else: loader = URPMISynthesisLoader(localpath, self._baseurl, listpath) loader.setChannel(self) self._loaders.append(loader)