def getObsoletes(self, channel, msgCallback=None, progressCallback=None): filePath = "%s/%s-obsoletes.%s" % ( self.cfg["storageDir"], channel['label'], channel['version']) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel['label']) self.s.set_progress_callback(progressCallback) obsoletes = self.s.getObsoletes(channel['label'], channel['version']) rpmSourceUtils.saveListToDisk(obsoletes, filePath, globPattern) return obsoletes
def getObsoletes(self, channel, msgCallback = None, progressCallback = None): # well, we've got the headers, might as well create a proper # obslist at this point filePath = "%s/%s-obsoletes.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel['label']) if msgCallback: msgCallback("Fetching obsoletes list for %s" % channel['url']) try: pkgList = self.pkglists[channel['label']] except KeyError: # we just hit the getObsoletes path, with no package info known # figure it out ourselves rd = repoDirector.initRepoDirector() pkgList = rd.listPackages(channel, msgCallback, progressCallback) self.pkglists[channel['label']] = pkgList obsList = [] total = len(pkgList) count = 0 for pkg in pkgList: baseFileName = "%s-%s-%s.%s.hdr" % (pkg[0], pkg[1], pkg[2], pkg[4]) fileName = "%s/%s" % (self.cfg["storageDir"], baseFileName) if os.access(fileName, os.R_OK): fd = open(fileName, "r") try: hdr = rpmUtils.readHeaderBlob(fd.read()) except: continue fd.close() if not hdr['obsoletes']: continue obs = up2dateUtils.genObsoleteTupleFromHdr(hdr) if obs: # print obs obsList = obsList + obs if progressCallback: progressCallback(count, total) count = count + 1 # now we have the package list, convert it to xmlrpc style # presentation and dump it obsList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(obsList, filePath,globPattern) # print obsList return obsList
def listAllPackages(self, channel, msgCallback=None, progressCallback=None): filePath = "%s/%s-all.%s" % (self.cfg["storageDir"], channel["label"], channel["version"]) # a glob used to find the old versions to cleanup globPattern = "%s/%s-all.*" % (self.cfg["storageDir"], channel["label"]) rd = repoDirector.initRepoDirector() pkgList = rd.listPackages(channel, msgCallback, progressCallback) list = pkgList[0] rpmSourceUtils.saveListToDisk(list, filePath, globPattern) return list
def getObsoletes(self, channel, msgCallback = None, progressCallback = None): filePath = "%s/%s-obsoletes.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel['label']) self.s.set_progress_callback(progressCallback) obsoletes = self.s.getObsoletes(channel['label'], channel['version']) rpmSourceUtils.saveListToDisk(obsoletes, filePath, globPattern) return obsoletes
def getObsoletes(self, channel, msgCallback=None, progressCallback=None): # well, we've got the headers, might as well create a proper # obslist at this point filePath = "%s/%s-obsoletes.%s" % ( self.cfg["storageDir"], channel['label'], channel['version']) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel['label']) if msgCallback: msgCallback("Fetching obsoletes list for %s" % channel['url']) try: pkgList = self.pkglists[channel['label']] except KeyError: # we just hit the getObsoletes path, with no package info known # figure it out ourselves rd = repoDirector.initRepoDirector() pkgList = rd.listPackages(channel, msgCallback, progressCallback) self.pkglists[channel['label']] = pkgList obsList = [] total = len(pkgList) count = 0 for pkg in pkgList: baseFileName = "%s-%s-%s.%s.hdr" % (pkg[0], pkg[1], pkg[2], pkg[4]) fileName = "%s/%s" % (self.cfg["storageDir"], baseFileName) if os.access(fileName, os.R_OK): fd = open(fileName, "r") try: hdr = rpmUtils.readHeaderBlob(fd.read()) except: continue fd.close() if not hdr['obsoletes']: continue obs = up2dateUtils.genObsoleteTupleFromHdr(hdr) if obs: # print obs obsList = obsList + obs if progressCallback: progressCallback(count, total) count = count + 1 # now we have the package list, convert it to xmlrpc style # presentation and dump it obsList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(obsList, filePath, globPattern) # print obsList return obsList
def listAllPackages(self, channel, msgCallback = None, progressCallback = None): # yum only knows about the most recent packages. Can't say # I blame them. I wish i only had to know about the most recent... filePath = "%s/%s-all.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) # a glob used to find the old versions to cleanup globPattern = "%s/%s-all.*" % (self.cfg["storageDir"], channel['label']) rd = repoDirector.initRepoDirector() pkgList = rd.listPackages(channel, msgCallback, progressCallback) list = pkgList[0] rpmSourceUtils.saveListToDisk(list, filePath,globPattern) return list
def listAllPackages(self, channel, msgCallback=None, progressCallback=None): filePath = "%s/%s-all.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) # a glob used to find the old versions to cleanup globPattern = "%s/%s-all.*" % (self.cfg["storageDir"], channel['label']) rd = repoDirector.initRepoDirector() pkgList = rd.listPackages(channel, msgCallback, progressCallback) list = pkgList[0] rpmSourceUtils.saveListToDisk(list, filePath, globPattern) return list
def listPackages(self, channel,msgCallback = None, progressCallback = None): filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) # a glob used to find the old versions to cleanup globPattern = "%s/%s.*" % (self.cfg["storageDir"], channel['label']) self.s.set_progress_callback(progressCallback) # FIXME: I still dont like the seemingly arbitrary fact that this # method returns a python structure, and all the other gets return # a file descriptor. list = self.s.listPackages(channel['label'], channel['version']) # do something to save it to disk. rpmSourceUtils.saveListToDisk(list, filePath,globPattern) return list
def listPackages(self, channel, msgCallback=None, progressCallback=None): filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) # a glob used to find the old versions to cleanup globPattern = "%s/%s.*" % (self.cfg["storageDir"], channel['label']) self.s.set_progress_callback(progressCallback) # FIXME: I still dont like the seemingly arbitrary fact that this # method returns a python structure, and all the other gets return # a file descriptor. list = self.s.listPackages(channel['label'], channel['version']) # do something to save it to disk. rpmSourceUtils.saveListToDisk(list, filePath, globPattern) return list
def listAllPackages(self, channel, msgCallback=None, progressCallback=None): # yum only knows about the most recent packages. Can't say # I blame them. I wish i only had to know about the most recent... filePath = "%s/%s-all.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) # a glob used to find the old versions to cleanup globPattern = "%s/%s-all.*" % (self.cfg["storageDir"], channel['label']) rd = repoDirector.initRepoDirector() pkgList = rd.listPackages(channel, msgCallback, progressCallback) list = pkgList[0] rpmSourceUtils.saveListToDisk(list, filePath, globPattern) return list
def _package_list_from_dict(self, pkgsDict, storage_dir, label, name_suffix, version): pkgList = [] names = pkgsDict.keys() names.sort() for name in names: pkgs = pkgsDict[name] for pkg in pkgs: pkgList.append(pkg) # nowi we have the package list, convert it to xmlrpc style # presentation and dump it filePath = "%s/%s%s.%s" % (storage_dir, label, name_suffix, version) fileGlobPattern = "%s/%s.*" % (storage_dir, label) rpmSourceUtils.saveListToDisk(pkgList, filePath, fileGlobPattern) return pkgList
def getObsoletes(self, channel, msgCallback=None, progressCallback=None): filePath = "%s/%s-obsoletes.%s" % ( self.cfg["storageDir"], channel['label'], channel['version']) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel['label']) if msgCallback: msgCallback("Fetching obsoletes list for %s" % channel['url']) fileHdrList = "%s/link-%s" % (self.cfg['storageDir'], channel['label']) #print "fhl: %s" % fileHdrList hdrList = rpm.readHeaderListFromFile(fileHdrList) # FIXME: since we have the package list, and the headers, we dont # have to reload the headerlist... obsList = [] total = len(hdrList) count = 0 for hdr in hdrList: if progressCallback: progressCallback(count, total) count = count + 1 # FIXME: we should share this logic somewhere... # up2dateUtils maybe? if not hdr['obsoletes']: continue obs = up2dateUtils.genObsoleteTupleFromHdr(hdr) if obs: obsList = obsList + obs # now we have the package list, convert it to xmlrpc style # presentation and dump it obsList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(obsList, filePath, globPattern) return obsList
def getObsoletes(self, channel, msgCallback=None, progressCallback=None): filePath = "%s/%s-obsoletes.%s" % ( self.cfg["storageDir"], channel['label'], channel['version']) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel['label']) # if we already founf the list, just statsh it. However it # is possible for it to not exist (ie, user just delets the obsList # from the cache, but since the package list exists we never hit the # above code path. A FIXME if self.obsList: self.obsList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(self.obsList, filePath, globPattern) if progressCallback: progressCallback(1, 1) else: if progressCallback: progressCallback(1, 1) if self.obsList: return self.obsList return []
def getObsoletes(self, channel, msgCallback=None, progressCallback=None): filePath = "%s/%s-obsoletes.%s" % (self.cfg["storageDir"], channel["label"], channel["version"]) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel["label"]) if msgCallback: msgCallback("Fetching obsoletes list for %s" % channel["url"]) fileHdrList = "%s/link-%s" % (self.cfg["storageDir"], channel["label"]) # print "fhl: %s" % fileHdrList hdrList = rpm.readHeaderListFromFile(fileHdrList) # FIXME: since we have the package list, and the headers, we dont # have to reload the headerlist... obsList = [] total = len(hdrList) count = 0 for hdr in hdrList: if progressCallback: progressCallback(count, total) count = count + 1 # FIXME: we should share this logic somewhere... # up2dateUtils maybe? if not hdr["obsoletes"]: continue obs = up2dateUtils.genObsoleteTupleFromHdr(hdr) if obs: obsList = obsList + obs # now we have the package list, convert it to xmlrpc style # presentation and dump it obsList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(obsList, filePath, globPattern) return obsList
def getObsoletes(self, channel, msgCallback = None, progressCallback = None): filePath = "%s/%s-obsoletes.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) globPattern = "%s/%s-obsoletes.*" % (self.cfg["storageDir"], channel['label']) # if we already founf the list, just statsh it. However it # is possible for it to not exist (ie, user just delets the obsList # from the cache, but since the package list exists we never hit the # above code path. A FIXME if self.obsList: self.obsList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(self.obsList, filePath, globPattern) if progressCallback: progressCallback(1,1) else: if progressCallback: progressCallback(1,1) if self.obsList: return self.obsList return []
def listPackages(self, channel, msgCallback=None, progressCallback=None): # TODO: implement cache invalidations, Last-Modified filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel["label"], channel["version"]) # a glob used to find the old versions to cleanup globPattern = "%s/%s.*" % (self.cfg["storageDir"], channel["label"]) oldLists = glob.glob(globPattern) channelTimeStamp = None if oldLists: filename = oldLists[0] filename = os.path.basename(filename) oldVersion = string.split(filename, ".")[-1] channelTimeStamp = time.strptime(oldVersion, "%Y%m%d%H%M%S") # assuming this is always bz2? url = "%s/base/pkglist.%s.bz2" % (channel["url"], channel["dist"]) if msgCallback: msgCallback("Fetching %s" % url) ret = urlUtils.fetchUrl( url, lastModified=channelTimeStamp, progressCallback=progressCallback, agent="Up2date %s/Apt" % up2dateUtils.version(), ) if ret: (buffer, lmtime) = ret else: return None symlinkname = "%s/link-%s" % (self.cfg["storageDir"], channel["label"]) try: os.unlink(symlinkname) except OSError: # file doesnt exist, shrug pass self.version = time.strftime("%Y%m%d%H%M%S", lmtime) filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel["label"], self.version) # sigh, no native bzip2 module... do it the old fashioned way tmpfilename = "%s/tmp-%s-%s" % (self.cfg["storageDir"], channel["label"], self.version) # print "timefilename: %s" % tmpfilename f = open("%s.bz2" % tmpfilename, "w") f.write(buffer) f.close() # FIXME, um, lame... once we settle on what url/http lib # we use, plugin in proper callbacks if progressCallback: progressCallback(1, 1) # well, since we dont have any knowledge about what the # channel version is supposed to be, I'cant really rely # on that (with up2date, the login tells me what channel # versions to look for). So we need a generic name # we symlink to the latest version. pipe = os.popen("/usr/bin/bunzip2 %s.bz2" % tmpfilename) tmp = pipe.read() os.symlink(tmpfilename, symlinkname) hdrList = rpm.readHeaderListFromFile(tmpfilename) # a list of rpm hdr's, handy!! pkgList = [] for hdr in hdrList: epoch = hdr["epoch"] if epoch == None or epoch == "0" or epoch == 0: epoch = "" pkgList.append( [ hdr["name"], hdr["version"], hdr["release"], epoch, hdr["arch"], # we want the actual filesize, but alas... str(hdr["size"]), channel["label"], ] ) # what the hell, this is a little bit of a side effect, but # were already poking at headers, lets just save them while # were at it to same us some trouble rpmSourceUtils.saveHeader(hdr) self.headerCache["%s-%s-%s.%s" % (hdr["name"], hdr["version"], hdr["release"], hdr["arch"])] = hdr # nowwe have the package list, convert it to xmlrpc style # presentation and dump it pkgList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(pkgList, filePath, globPattern) return pkgList
def listPackages(self, channel, msgCallback = None, progressCallback = None): # TODO: where do we implement cache validation? guess we # use http header time stamps to make a best guess since we # dont have any real info about the file format # a glob used to find the old versions to cleanup # FIXME: this is probabaly overkill... Should only have # one version of any given globPattern = "%s/%s.*" % (self.cfg["storageDir"], channel['label']) oldLists = glob.glob(globPattern) channelTimeStamp = None if oldLists: filename = oldLists[0] filename = os.path.basename(filename) oldVersion = string.split(filename, '.')[-1] channelTimeStamp = time.strptime(oldVersion,"%Y%m%d%H%M%S") # for yum stuff, we assume that serverUrl is the base # path, channel is the relative path, and version isnt # user url = "%s/headers/header.info" % (channel['url']) if msgCallback: msgCallback("Fetching %s" % url) # oh, this lame, but implement a fancy url fetcher later # heck, maybe even borrow the one from yum #print urlUtils ret = urlUtils.fetchUrl(url, lastModified=channelTimeStamp, progressCallback = progressCallback, agent = "Up2date %s/Yum" % up2dateUtils.version()) if ret: (buffer, lmtime) = ret else: return None if not lmtime: lmtime = time.gmtime(time.time()) version = time.strftime("%Y%m%d%H%M%S", lmtime) # use the time stamp on the headerlist as the channel "version" filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel['label'], version) # it's possible to get bogus data here, so at least try not # to traceback if buffer: lines = string.split(buffer) else: lines = [] # this gives us the raw yum header list, which is _not_ # in the pretty format up2date likes, so convert it # and sadly, I can no longer proudly state that up2date # at no points attempts to parse rpm filenames into something # useful. At least yum includes the epoch pkgList = [] # yum can have a different path for each rpm. Not exactly # sure how this meets the "keep it simple" idea, but alas self.pkgNamePath = {} for line in lines: if line == "" or line[0] == "#": continue (envra, rpmPath) = string.split(line, '=') rpmPath = string.strip(rpmPath) (epoch, name, ver, rel, arch) = self._stripENVRA(envra) # quite possibly need to encode channel info here as well if epoch == "0" or epoch == 0: epoch = "" # hmm, if an arch doesnt apply, guess no point in # keeping it around, should make package lists smaller # and cut down on some churn if rpm.archscore(arch) == 0: continue self.pkgNamePath[(name,ver,rel,epoch,arch)] = rpmPath # doh, no size info. FIXME size = "1000" # er, yeah... thats not lame at all... pkgList.append([name, ver, rel, epoch, arch, size, channel['label'], rpmPath]) # now we have the package list, convert it to xmlrpc style # presentation and dump it pkgList.sort(lambda a, b: cmp(a[0], b[0])) count = 0 total = len(pkgList) rd = repoDirector.initRepoDirector() for pkg in pkgList: # were deep down in the yum specific bits, but we want to call # the generic getHeader to get it off disc or cache hdr = rd.getHeader([name,ver,rel,epoch,arch, "0",channel['label']]) if progressCallback: progressCallback(count, total) count = count + 1 rpmSourceUtils.saveListToDisk(pkgList, filePath, globPattern) self.pkglists[channel['label']] = pkgList return pkgList
def listPackages(self, channel, msgCallback=None, progressCallback=None): # TODO: where do we implement cache validation? guess we # use http header time stamps to make a best guess since we # dont have any real info about the file format # a glob used to find the old versions to cleanup # FIXME: this is probabaly overkill... Should only have # one version of any given globPattern = "%s/%s.*" % (self.cfg["storageDir"], channel['label']) oldLists = glob.glob(globPattern) channelTimeStamp = None if oldLists: filename = oldLists[0] filename = os.path.basename(filename) oldVersion = string.split(filename, '.')[-1] channelTimeStamp = time.strptime(oldVersion, "%Y%m%d%H%M%S") # for yum stuff, we assume that serverUrl is the base # path, channel is the relative path, and version isnt # user url = "%s/headers/header.info" % (channel['url']) if msgCallback: msgCallback("Fetching %s" % url) # oh, this lame, but implement a fancy url fetcher later # heck, maybe even borrow the one from yum #print urlUtils ret = urlUtils.fetchUrl(url, lastModified=channelTimeStamp, progressCallback=progressCallback, agent="Up2date %s/Yum" % up2dateUtils.version()) if ret: (buffer, lmtime) = ret else: return None if not lmtime: lmtime = time.gmtime(time.time()) version = time.strftime("%Y%m%d%H%M%S", lmtime) # use the time stamp on the headerlist as the channel "version" filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel['label'], version) # it's possible to get bogus data here, so at least try not # to traceback if buffer: lines = string.split(buffer) else: lines = [] # this gives us the raw yum header list, which is _not_ # in the pretty format up2date likes, so convert it # and sadly, I can no longer proudly state that up2date # at no points attempts to parse rpm filenames into something # useful. At least yum includes the epoch pkgList = [] # yum can have a different path for each rpm. Not exactly # sure how this meets the "keep it simple" idea, but alas self.pkgNamePath = {} for line in lines: if line == "" or line[0] == "#": continue (envra, rpmPath) = string.split(line, '=') rpmPath = string.strip(rpmPath) (epoch, name, ver, rel, arch) = self._stripENVRA(envra) # quite possibly need to encode channel info here as well if epoch == "0" or epoch == 0: epoch = "" # hmm, if an arch doesnt apply, guess no point in # keeping it around, should make package lists smaller # and cut down on some churn if rpm.archscore(arch) == 0: continue self.pkgNamePath[(name, ver, rel, epoch, arch)] = rpmPath # doh, no size info. FIXME size = "1000" # er, yeah... thats not lame at all... pkgList.append( [name, ver, rel, epoch, arch, size, channel['label'], rpmPath]) # now we have the package list, convert it to xmlrpc style # presentation and dump it pkgList.sort(lambda a, b: cmp(a[0], b[0])) count = 0 total = len(pkgList) rd = repoDirector.initRepoDirector() for pkg in pkgList: # were deep down in the yum specific bits, but we want to call # the generic getHeader to get it off disc or cache hdr = rd.getHeader( [name, ver, rel, epoch, arch, "0", channel['label']]) if progressCallback: progressCallback(count, total) count = count + 1 rpmSourceUtils.saveListToDisk(pkgList, filePath, globPattern) self.pkglists[channel['label']] = pkgList return pkgList
def listPackages(self, channel, msgCallback=None, progressCallback=None): # TODO: implement cache invalidations, Last-Modified filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel['label'], channel['version']) # a glob used to find the old versions to cleanup globPattern = "%s/%s.*" % (self.cfg["storageDir"], channel['label']) oldLists = glob.glob(globPattern) channelTimeStamp = None if oldLists: filename = oldLists[0] filename = os.path.basename(filename) oldVersion = string.split(filename, '.')[-1] channelTimeStamp = time.strptime(oldVersion, "%Y%m%d%H%M%S") # assuming this is always bz2? url = "%s/base/pkglist.%s.bz2" % (channel['url'], channel['dist']) if msgCallback: msgCallback("Fetching %s" % url) ret = urlUtils.fetchUrl(url, lastModified=channelTimeStamp, progressCallback=progressCallback, agent="Up2date %s/Apt" % up2dateUtils.version()) if ret: (buffer, lmtime) = ret else: return None symlinkname = "%s/link-%s" % (self.cfg["storageDir"], channel['label']) try: os.unlink(symlinkname) except OSError: # file doesnt exist, shrug pass self.version = time.strftime("%Y%m%d%H%M%S", lmtime) filePath = "%s/%s.%s" % (self.cfg["storageDir"], channel['label'], self.version) # sigh, no native bzip2 module... do it the old fashioned way tmpfilename = "%s/tmp-%s-%s" % (self.cfg['storageDir'], channel['label'], self.version) #print "timefilename: %s" % tmpfilename f = open("%s.bz2" % tmpfilename, "w") f.write(buffer) f.close() # FIXME, um, lame... once we settle on what url/http lib # we use, plugin in proper callbacks if progressCallback: progressCallback(1, 1) # well, since we dont have any knowledge about what the # channel version is supposed to be, I'cant really rely # on that (with up2date, the login tells me what channel # versions to look for). So we need a generic name # we symlink to the latest version. pipe = os.popen("/usr/bin/bunzip2 %s.bz2" % tmpfilename) tmp = pipe.read() os.symlink(tmpfilename, symlinkname) hdrList = rpm.readHeaderListFromFile(tmpfilename) # a list of rpm hdr's, handy!! pkgList = [] for hdr in hdrList: epoch = hdr['epoch'] if epoch == None or epoch == "0" or epoch == 0: epoch = "" pkgList.append([ hdr['name'], hdr['version'], hdr['release'], epoch, hdr['arch'], # we want the actual filesize, but alas... str(hdr['size']), channel['label'] ]) # what the hell, this is a little bit of a side effect, but # were already poking at headers, lets just save them while # were at it to same us some trouble rpmSourceUtils.saveHeader(hdr) self.headerCache["%s-%s-%s.%s" % (hdr['name'], hdr['version'], hdr['release'], hdr['arch'])] = hdr # nowwe have the package list, convert it to xmlrpc style # presentation and dump it pkgList.sort(lambda a, b: cmp(a[0], b[0])) rpmSourceUtils.saveListToDisk(pkgList, filePath, globPattern) return pkgList