def __init__(self, systemId, baseURL, channelLabel, savePath): BaseFetch.__init__(self) self.systemId = systemId self.baseURL = baseURL self.rhnComm = RHNComm(baseURL, self.systemId) self.channelLabel = channelLabel self.savePath = savePath
class KickstartFetch(BaseFetch): def __init__(self, systemId, baseURL): BaseFetch.__init__(self) self.baseURL = baseURL self.systemId = systemId self.rhnComm = RHNComm(baseURL, self.systemId) def login(self, refresh=False): return self.rhnComm.login(refresh) def getFetchURL(self, channelLabel, ksLabel, ksFilePath): return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getKickstartFile/" + ksLabel + "/" + ksFilePath; def fetchItem(self, itemInfo): authMap = self.login() fileName = itemInfo['fileName'] itemSize = itemInfo['size'] md5sum = itemInfo['md5sum'] hashType = itemInfo['hashtype'] ksLabel = itemInfo['ksLabel'] channelLabel = itemInfo['channelLabel'] savePath = itemInfo['savePath'] fetchURL = self.getFetchURL(channelLabel, ksLabel, fileName) status = self.fetch(fileName, fetchURL, savePath, itemSize, hashType, md5sum, headers=authMap) if status == BaseFetch.STATUS_UNAUTHORIZED: LOG.warn("Unauthorized request from fetch(). Will attempt to update authentication credentials and retry") authMap = self.login(refresh=True) return self.fetch(fileName, fetchURL, savePath, itemSize, hashType, md5sum, headers=authMap) return status
def __init__(self, url, username, password, cert, systemid, parallel, verbose): self.baseURL = url self.cert = open(cert, 'r').read() self.systemid = open(systemid, 'r').read() self.username = username self.password = password self.parallel = parallel self.fetchAll = False #default is only fetch latest packages self.parallelFetchPkgs = None self.parallelFetchKickstarts = None self.skipProductList = [] self.skipPackageList = [] self.verbose = verbose self.killcount = 0 self.removeOldPackages = False self.numOldPkgsKeep = 1 self.rhnComm = RHNComm(url, self.systemid)
class PackageFetch(BaseFetch): def __init__(self, systemId, baseURL, channelLabel, savePath): BaseFetch.__init__(self) self.systemId = systemId self.baseURL = baseURL self.rhnComm = RHNComm(baseURL, self.systemId) self.channelLabel = channelLabel self.savePath = savePath def login(self, refresh=False): """ Returns authentication headers needed for RHN 'GET' requests. auth data is cached, if data needs to be updated, pass in refresh=True """ return self.rhnComm.login(refresh) def getFetchURL(self, channelLabel, fetchName): return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getPackage/" + fetchName def fetchItem(self, itemInfo): authMap = self.login() fileName = itemInfo['fileName'] fetchName = itemInfo['fetch_name'] itemSize = itemInfo['size'] md5sum = itemInfo['md5sum'] hashType = itemInfo['hashtype'] if itemInfo.has_key('pkgpath'): pkgPath = itemInfo['pkgpath'] else: pkgPath = None fetchURL = self.getFetchURL(self.channelLabel, fetchName) status = self.fetch(fileName, fetchURL, self.savePath, itemSize, hashType, md5sum, headers=authMap) if status == BaseFetch.STATUS_UNAUTHORIZED: LOG.warn( "Unauthorized request from fetch(). Will attempt to update authentication credentials and retry" ) authMap = self.login(refresh=True) return self.fetch(fileName, fetchURL, self.savePath, itemSize, hashType, md5sum, headers=authMap, pkgpath=pkgPath) return status
class KickstartFetch(BaseFetch): def __init__(self, systemId, baseURL): BaseFetch.__init__(self) self.baseURL = baseURL self.systemId = systemId self.rhnComm = RHNComm(baseURL, self.systemId) def login(self, refresh=False): return self.rhnComm.login(refresh) def getFetchURL(self, channelLabel, ksLabel, ksFilePath): return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getKickstartFile/" + ksLabel + "/" + ksFilePath def fetchItem(self, itemInfo): authMap = self.login() fileName = itemInfo['fileName'] itemSize = itemInfo['size'] md5sum = itemInfo['md5sum'] hashType = itemInfo['hashtype'] ksLabel = itemInfo['ksLabel'] channelLabel = itemInfo['channelLabel'] savePath = itemInfo['savePath'] fetchURL = self.getFetchURL(channelLabel, ksLabel, fileName) status = self.fetch(fileName, fetchURL, itemSize, hashType, md5sum, savePath, headers=authMap) if status == BaseFetch.STATUS_UNAUTHORIZED: LOG.warn( "Unauthorized request from fetch(). Will attempt to update authentication credentials and retry" ) authMap = self.login(refresh=True) return self.fetch(fileName, fetchURL, itemSize, hashType, md5sum, savePath, headers=authMap) return status
class PackageFetch(BaseFetch): def __init__(self, systemId, baseURL, channelLabel, savePath): BaseFetch.__init__(self) self.systemId = systemId self.baseURL = baseURL self.rhnComm = RHNComm(baseURL, self.systemId) self.channelLabel = channelLabel self.savePath = savePath def login(self, refresh=False): """ Returns authentication headers needed for RHN 'GET' requests. auth data is cached, if data needs to be updated, pass in refresh=True """ return self.rhnComm.login(refresh) def getFetchURL(self, channelLabel, fetchName): return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getPackage/" + fetchName def fetchItem(self, itemInfo): authMap = self.login() fileName = itemInfo["fileName"] fetchName = itemInfo["fetch_name"] itemSize = itemInfo["size"] md5sum = itemInfo["md5sum"] hashType = itemInfo["hashtype"] if itemInfo.has_key("pkgpath"): pkgPath = itemInfo["pkgpath"] else: pkgPath = None fetchURL = self.getFetchURL(self.channelLabel, fetchName) status = self.fetch(fileName, fetchURL, self.savePath, itemSize, hashType, md5sum, headers=authMap) if status == BaseFetch.STATUS_UNAUTHORIZED: LOG.warn("Unauthorized request from fetch(). Will attempt to update authentication credentials and retry") authMap = self.login(refresh=True) return self.fetch( fileName, fetchURL, self.savePath, itemSize, hashType, md5sum, headers=authMap, pkgpath=pkgPath ) return status
def __init__(self, systemId, baseURL): BaseFetch.__init__(self) self.baseURL = baseURL self.systemId = systemId self.rhnComm = RHNComm(baseURL, self.systemId)
class Grinder: def __init__(self, url, username, password, cert, systemid, parallel, verbose): self.baseURL = url self.cert = open(cert, 'r').read() self.systemid = open(systemid, 'r').read() self.username = username self.password = password self.parallel = parallel self.fetchAll = False #default is only fetch latest packages self.parallelFetchPkgs = None self.parallelFetchKickstarts = None self.skipProductList = [] self.skipPackageList = [] self.verbose = verbose self.killcount = 0 self.removeOldPackages = False self.numOldPkgsKeep = 1 self.rhnComm = RHNComm(url, self.systemid) def setRemoveOldPackages(self, value): self.removeOldPackages = value def getRemoveOldPackages(self): return self.removeOldPackages def getFetchAllPackages(self): return self.fetchAll def setFetchAllPackages(self, val): self.fetchAll = val def getSkipProductList(self): return self.skipProductList def setSkipProductList(self, skipProductList): self.skipProductList = skipProductList def getSkipPackageList(self): return self.skipPackageList def setSkipPackageList(self, skipPackageList): self.skipPackageList = skipPackageList def getNumOldPackagesToKeep(self): return self.numOldPkgsKeep def setNumOldPackagesToKeep(self, num): self.numOldPkgsKeep = num def deactivate(self): SATELLITE_URL = "%s/rpc/api" % (self.baseURL) client = getRhnApi(SATELLITE_URL, verbose=0) key = client.auth.login(self.username, self.password) retval = client.satellite.deactivateSatellite(self.systemid) print "retval from deactivation: %s" % retval client.auth.logout(key) print "Deactivated!" def activate(self): rhn = RHNTransport() satClient = getRhnApi(self.baseURL + "/SAT", verbose=self.verbose, transport=rhn) # First check if we are active active = False retval = satClient.authentication.check(self.systemid) LOG.debug("AUTH CHECK: %s " % str(retval)) if (retval == 1): LOG.debug("We are activated ... continue!") active = True else: LOG.debug("Not active") if (not active): if(not self.username or not self.password): raise SystemNotActivatedException() SATELLITE_URL = "%s/rpc/api" % (self.baseURL) client = RhnApi(SATELLITE_URL, verbose=0) key = client.auth.login(self.username, self.password) retval = client.satellite.activateSatellite(self.systemid, self.cert) LOG.debug("retval from activation: %s" % retval) if (retval != 1): raise CantActivateException() client.auth.logout(key) LOG.debug("Activated!") def stop(self): if (self.parallelFetchPkgs): self.parallelFetchPkgs.stop() if (self.parallelFetchKickstarts): self.parallelFetchKickstarts.stop() def checkChannels(self, channelsToSync): """ Input: channelsToSync - list of channels to sync Output: list containing bad channel names """ satDump = SatDumpClient(self.baseURL) channelFamilies = satDump.getChannelFamilies(self.systemid) badChannel = [] for channelLabel in channelsToSync: found = False for d in channelFamilies.values(): if channelLabel in d["channel_labels"]: LOG.debug("Found %s under %s" % (channelLabel, d["label"])) found = True break if not found: LOG.debug("Unable to find %s, adding it to badChannel list" % (channelLabel)) badChannel.append(channelLabel) return badChannel def getChannelLabels(self): labels = {} satDump = SatDumpClient(self.baseURL) channelFamilies = satDump.getChannelFamilies(self.systemid) for d in channelFamilies.values(): if (d["label"] in self.skipProductList): continue labels[d["label"]] = d["channel_labels"] return labels def displayListOfChannels(self): labels = self.getChannelLabels() print("List of channels:") for lbl in labels: print("\nProduct : %s\n" % (lbl)) for l in labels[lbl]: print(" %s" % (l)) def syncKickstarts(self, channelLabel, savePath, verbose=0): """ channelLabel - channel to sync kickstarts from savePath - path to save kickstarts verbose - if true display more output """ startTime = time.time() satDump = SatDumpClient(self.baseURL, verbose=verbose) ksLabels = satDump.getKickstartLabels(self.systemid, [channelLabel]) LOG.info("Found %s kickstart labels for channel %s" % (len(ksLabels[channelLabel]), channelLabel)) ksFiles = [] for ksLbl in ksLabels[channelLabel]: LOG.info("Syncing kickstart label: %s" % (ksLbl)) metadata = satDump.getKickstartTreeMetadata(self.systemid, [ksLbl]) LOG.info("Retrieved metadata on %s files for kickstart label: %s" % (len(metadata[ksLbl]["files"]), ksLbl)) ksSavePath = os.path.join(savePath, ksLbl) for ksFile in metadata[ksLbl]["files"]: info = {} info["relative-path"] = ksFile["relative-path"] info["size"] = ksFile["file-size"] info["md5sum"] = ksFile["md5sum"] info["ksLabel"] = ksLbl info["channelLabel"] = channelLabel info["savePath"] = ksSavePath ksFiles.append(info) ksFetch = KickstartFetch(self.systemid, self.baseURL) numThreads = int(self.parallel) self.parallelFetchKickstarts = ParallelFetch(ksFetch, numThreads) self.parallelFetchKickstarts.addItemList(ksFiles) self.parallelFetchKickstarts.start() report = self.parallelFetchKickstarts.waitForFinish() endTime = time.time() LOG.info("Processed %s %s %s kickstart files, %s errors, completed in %s seconds" \ % (channelLabel, ksLabels[channelLabel], report.successes, report.errors, (endTime-startTime))) return report def syncPackages(self, channelLabel, savePath, verbose=0): """ channelLabel - channel to sync packages from savePath - path to save packages verbose - if true display more output """ startTime = time.time() if channelLabel == "": LOG.critical("No channel label specified to sync, abort sync.") raise NoChannelLabelException() LOG.info("sync(%s, %s) invoked" % (channelLabel, verbose)) satDump = SatDumpClient(self.baseURL, verbose=verbose) LOG.debug("*** calling product_names ***") packages = satDump.getChannelPackages(self.systemid, channelLabel) LOG.info("%s packages are available, getting list of short metadata now." % (len(packages))) pkgInfo = satDump.getShortPackageInfo(self.systemid, packages, filterLatest = not self.fetchAll) LOG.info("%s packages have been marked to be fetched" % (len(pkgInfo.values()))) numThreads = int(self.parallel) LOG.info("Running in parallel fetch mode with %s threads" % (numThreads)) pkgFetch = PackageFetch(self.systemid, self.baseURL, channelLabel, savePath) self.parallelFetchPkgs = ParallelFetch(pkgFetch, numThreads) self.parallelFetchPkgs.addItemList(pkgInfo.values()) self.parallelFetchPkgs.start() report = self.parallelFetchPkgs.waitForFinish() LOG.debug("Attempting to fetch comps.xml info from RHN") self.fetchCompsXML(savePath, channelLabel) self.fetchUpdateinfo(savePath, channelLabel) endTime = time.time() LOG.info("Processed <%s> %s packages, %s errors, completed in %s seconds" \ % (channelLabel, report.successes, report.errors, (endTime-startTime))) if self.removeOldPackages: LOG.info("Remove old packages from %s" % (savePath)) self.runRemoveOldPackages(savePath) return report def fetchCompsXML(self, savePath, channelLabel): ### # Fetch comps.xml, used by createrepo for "groups" info ### compsxml = "" try: compsxml = self.rhnComm.getRepodata(channelLabel, "comps.xml") except GetRequestException, ge: if (ge.code == 404): LOG.info("Channel has no compsXml") else: raise ge if not savePath: savePath = channelLabel f = open(os.path.join(savePath, "comps.xml"), "w") f.write(compsxml) f.close()