Exemplo n.º 1
0
class PackageFetch(BaseFetch):
    
    def __init__(self, systemId, baseURL, channelLabel, savePath):
        BaseFetch.__init__(self)
        self.systemId = systemId
        self.baseURL = baseURL
        self.rhnComm = RHNComm(baseURL, self.systemId)
        self.channelLabel = channelLabel
        self.savePath = savePath

    def login(self, refresh=False):
        """
        Returns authentication headers needed for RHN 'GET' requests.
        auth data is cached, if data needs to be updated, pass in refresh=True
        """
        return self.rhnComm.login(refresh)

    def getFetchURL(self, channelLabel, fetchName):
        return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getPackage/" + fetchName;

    def fetchItem(self, itemInfo):
        authMap = self.login()
        fileName = itemInfo['filename']
        fetchName = itemInfo['fetch_name']
        itemSize = itemInfo['package_size']
        md5sum = itemInfo['md5sum']
        hashType = itemInfo['hashtype']
        fetchURL = self.getFetchURL(self.channelLabel, fetchName)
        status = self.fetch(fileName, fetchURL, itemSize, hashType, md5sum, self.savePath, headers=authMap)
        if status == BaseFetch.STATUS_UNAUTHORIZED:
            LOG.warn("Unauthorized request from fetch().  Will attempt to update authentication credentials and retry")
            authMap = self.login(refresh=True)
            return self.fetch(fileName, fetchURL, itemSize, md5sum, self.savePath, headers=authMap)
        return status
Exemplo n.º 2
0
 def __init__(self, systemId, baseURL, channelLabel, savePath):
     BaseFetch.__init__(self)
     self.systemId = systemId
     self.baseURL = baseURL
     self.rhnComm = RHNComm(baseURL, self.systemId)
     self.channelLabel = channelLabel
     self.savePath = savePath
Exemplo n.º 3
0
class KickstartFetch(BaseFetch):

    def __init__(self, systemId, baseURL):
        BaseFetch.__init__(self)
        self.baseURL = baseURL
        self.systemId = systemId
        self.rhnComm = RHNComm(baseURL, self.systemId)

    def login(self, refresh=False):
        return self.rhnComm.login(refresh)
    
    def getFetchURL(self, channelLabel, ksLabel, ksFilePath):
        return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getKickstartFile/" + ksLabel + "/" + ksFilePath;

    def fetchItem(self, itemInfo):
        authMap = self.login()

        fileName = itemInfo['relative-path']
        itemSize = itemInfo['size']
        md5sum = itemInfo['md5sum']
        hashType = itemInfo['hashtype']
        ksLabel = itemInfo['ksLabel']
        channelLabel = itemInfo['channelLabel']
        savePath = itemInfo['savePath']
        fetchURL = self.getFetchURL(channelLabel, ksLabel, fileName)
        status = self.fetch(fileName, fetchURL, itemSize, hashType, md5sum, savePath, headers=authMap)
        if status == BaseFetch.STATUS_UNAUTHORIZED:
            LOG.warn("Unauthorized request from fetch().  Will attempt to update authentication credentials and retry")
            authMap = self.login(refresh=True)
            return self.fetch(fileName, fetchURL, itemSize, md5sum, savePath, headers=authMap)
        return status
Exemplo n.º 4
0
 def __init__(self, url, username, password, cert, systemid, parallel, verbose):
     self.baseURL = url
     self.cert = open(cert, 'r').read()
     self.systemid = open(systemid, 'r').read()
     self.username = username
     self.password = password
     self.parallel = parallel
     self.fetchAll = False     #default is only fetch latest packages
     self.parallelFetchPkgs = None
     self.parallelFetchKickstarts = None
     self.skipProductList = []
     self.skipPackageList = []
     self.verbose = verbose
     self.killcount = 0
     self.removeOldPackages = False
     self.numOldPkgsKeep = 1
     self.rhnComm = RHNComm(url, self.systemid)
Exemplo n.º 5
0
 from grinder import GrinderLog
 GrinderLog.setup(True)
 systemId = open("/etc/sysconfig/rhn/systemid").read()
 baseURL = "http://satellite.rhn.redhat.com"
 bf = BaseFetch()
 itemInfo = {}
 fileName = "Virtualization-es-ES-5.2-9.noarch.rpm"
 fetchName = "Virtualization-es-ES-5.2-9:.noarch.rpm"
 channelLabel = "rhel-i386-server-vt-5"
 fetchURL = baseURL + "/SAT/$RHN/" + channelLabel + "/getPackage/" + fetchName;
 itemSize = "1731195"
 md5sum = "91b0f20aeeda88ddae4959797003a173" 
 hashtype = "md5"
 savePath = "./test123"
 from RHNComm import RHNComm
 rhnComm = RHNComm(baseURL, systemId)
 authMap = rhnComm.login()
 status = bf.fetch(fileName, fetchURL, itemSize, hashtype, md5sum, savePath, headers=authMap, retryTimes=2)
 print status
 assert(status in [BaseFetch.STATUS_DOWNLOADED, BaseFetch.STATUS_NOOP])
 print "Test Download or NOOP passed"
 status = bf.fetch(fileName, fetchURL, itemSize, hashtype, md5sum, savePath, headers=authMap, retryTimes=2)
 assert(status == BaseFetch.STATUS_NOOP)
 print "Test for NOOP passed"
 authMap['X-RHN-Auth'] = "Bad Value"
 fileName = "Virtualization-en-US-5.2-9.noarch.rpm"
 fetchName = "Virtualization-en-US-5.2-9:.noarch.rpm"
 status = bf.fetch(fileName, fetchURL, itemSize, hashtype, md5sum, savePath, headers=authMap, retryTimes=2)
 print status
 assert(status == BaseFetch.STATUS_UNAUTHORIZED)
 print "Test for unauthorized passed"
Exemplo n.º 6
0
 def __init__(self, systemId, baseURL):
     BaseFetch.__init__(self)
     self.baseURL = baseURL
     self.systemId = systemId
     self.rhnComm = RHNComm(baseURL, self.systemId)
Exemplo n.º 7
0
class Grinder:
    def __init__(self, url, username, password, cert, systemid, parallel, verbose):
        self.baseURL = url
        self.cert = open(cert, 'r').read()
        self.systemid = open(systemid, 'r').read()
        self.username = username
        self.password = password
        self.parallel = parallel
        self.fetchAll = False     #default is only fetch latest packages
        self.parallelFetchPkgs = None
        self.parallelFetchKickstarts = None
        self.skipProductList = []
        self.skipPackageList = []
        self.verbose = verbose
        self.killcount = 0
        self.removeOldPackages = False
        self.numOldPkgsKeep = 1
        self.rhnComm = RHNComm(url, self.systemid)

    def setRemoveOldPackages(self, value):
        self.removeOldPackages = value

    def getRemoveOldPackages(self):
        return self.removeOldPackages

    def getFetchAllPackages(self):
        return self.fetchAll
    
    def setFetchAllPackages(self, val):
        self.fetchAll = val
    
    def getSkipProductList(self):
        return self.skipProductList

    def setSkipProductList(self, skipProductList):
        self.skipProductList = skipProductList
    
    def getSkipPackageList(self):
        return self.skipPackageList

    def setSkipPackageList(self, skipPackageList):
        self.skipPackageList = skipPackageList

    def getNumOldPackagesToKeep(self):
        return self.numOldPkgsKeep
    
    def setNumOldPackagesToKeep(self, num):
        self.numOldPkgsKeep = num
    

    def deactivate(self):
        SATELLITE_URL = "%s/rpc/api" % (self.baseURL)
        client = getRhnApi(SATELLITE_URL, verbose=0)
        key = client.auth.login(self.username, self.password)
        retval = client.satellite.deactivateSatellite(self.systemid)
        print "retval from deactivation: %s"  % retval
        client.auth.logout(key)        
        print "Deactivated!"

    def activate(self):
        rhn = RHNTransport()    
        satClient = getRhnApi(self.baseURL + "/SAT", 
            verbose=self.verbose, transport=rhn)
        # First check if we are active
        active = False
        retval = satClient.authentication.check(self.systemid)
        LOG.debug("AUTH CHECK: %s " % str(retval))
        if (retval == 1):
            LOG.debug("We are activated ... continue!")
            active = True
        else:
            LOG.debug("Not active")
            
        if (not active): 
            if(not self.username or not self.password):
                raise SystemNotActivatedException()
            SATELLITE_URL = "%s/rpc/api" % (self.baseURL)
            client = RhnApi(SATELLITE_URL, verbose=0)
            key = client.auth.login(self.username, self.password)
            retval = client.satellite.activateSatellite(self.systemid, self.cert)
            LOG.debug("retval from activation: %s"  % retval)
            if (retval != 1):
                raise CantActivateException()
            client.auth.logout(key)        
            LOG.debug("Activated!")

    def stop(self):
        if (self.parallelFetchPkgs):
            self.parallelFetchPkgs.stop()
        if (self.parallelFetchKickstarts):
            self.parallelFetchKickstarts.stop()

    def checkChannels(self, channelsToSync):
        """
        Input:
            channelsToSync - list of channels to sync
        Output:
             list containing bad channel names
        """
        satDump = SatDumpClient(self.baseURL)
        channelFamilies = satDump.getChannelFamilies(self.systemid)
        badChannel = []
        for channelLabel in channelsToSync:
            found = False
            for d in channelFamilies.values():
                if channelLabel in d["channel_labels"]:
                    LOG.debug("Found %s under %s" % (channelLabel, d["label"]))
                    found = True
                    break
            if not found:
                LOG.debug("Unable to find %s, adding it to badChannel list" % (channelLabel))
                badChannel.append(channelLabel)
        return badChannel


    def getChannelLabels(self):
        labels = {}
        satDump = SatDumpClient(self.baseURL)
        channelFamilies = satDump.getChannelFamilies(self.systemid)
        for d in channelFamilies.values():
            if (d["label"] in self.skipProductList):
                continue
            labels[d["label"]] = d["channel_labels"]
        return labels

    def displayListOfChannels(self):
        labels = self.getChannelLabels()
        print("List of channels:")
        for lbl in labels:
            print("\nProduct : %s\n" % (lbl))
            for l in labels[lbl]:
                print("    %s" % (l))

    def syncKickstarts(self, channelLabel, savePath, verbose=0):
        """
        channelLabel - channel to sync kickstarts from
        savePath - path to save kickstarts
        verbose - if true display more output
        """
        startTime = time.time()
        satDump = SatDumpClient(self.baseURL, verbose=verbose)
        ksLabels = satDump.getKickstartLabels(self.systemid, [channelLabel])
        LOG.info("Found %s kickstart labels for channel %s" % (len(ksLabels[channelLabel]), channelLabel))
        ksFiles = []
        for ksLbl in ksLabels[channelLabel]:
            LOG.info("Syncing kickstart label: %s" % (ksLbl))
            metadata = satDump.getKickstartTreeMetadata(self.systemid, [ksLbl])
            LOG.info("Retrieved metadata on %s files for kickstart label: %s" % (len(metadata[ksLbl]["files"]), ksLbl))
            ksSavePath = os.path.join(savePath, ksLbl)
            for ksFile in metadata[ksLbl]["files"]:
                info = {}
                info["relative-path"] = ksFile["relative-path"]
                info["size"] = ksFile["file-size"]
                info["md5sum"] = ksFile["md5sum"]
                info["ksLabel"] = ksLbl
                info["channelLabel"] = channelLabel
                info["savePath"] = ksSavePath
                ksFiles.append(info)
        ksFetch = KickstartFetch(self.systemid, self.baseURL)
        numThreads = int(self.parallel)
        self.parallelFetchKickstarts = ParallelFetch(ksFetch, numThreads)
        self.parallelFetchKickstarts.addItemList(ksFiles)
        self.parallelFetchKickstarts.start()
        report = self.parallelFetchKickstarts.waitForFinish()
        endTime = time.time()
        LOG.info("Processed %s %s %s kickstart files, %s errors, completed in %s seconds" \
                % (channelLabel, ksLabels[channelLabel], report.successes, 
                    report.errors, (endTime-startTime)))
        return report

    def syncPackages(self, channelLabel, savePath, verbose=0):
        """
        channelLabel - channel to sync packages from
        savePath - path to save packages
        verbose - if true display more output
        """
        startTime = time.time()
        if channelLabel == "":
            LOG.critical("No channel label specified to sync, abort sync.")
            raise NoChannelLabelException()
        LOG.info("sync(%s, %s) invoked" % (channelLabel, verbose))
        satDump = SatDumpClient(self.baseURL, verbose=verbose)
        LOG.debug("*** calling product_names ***")
        packages = satDump.getChannelPackages(self.systemid, channelLabel)
        LOG.info("%s packages are available, getting list of short metadata now." % (len(packages)))
        pkgInfo = satDump.getShortPackageInfo(self.systemid, packages, filterLatest = not self.fetchAll)
        LOG.info("%s packages have been marked to be fetched" % (len(pkgInfo.values())))

        numThreads = int(self.parallel)
        LOG.info("Running in parallel fetch mode with %s threads" % (numThreads))
        pkgFetch = PackageFetch(self.systemid, self.baseURL, channelLabel, savePath)
        self.parallelFetchPkgs = ParallelFetch(pkgFetch, numThreads)
        self.parallelFetchPkgs.addItemList(pkgInfo.values())
        self.parallelFetchPkgs.start()
        report = self.parallelFetchPkgs.waitForFinish()
        LOG.debug("Attempting to fetch comps.xml info from RHN")
        self.fetchCompsXML(savePath, channelLabel)
        self.fetchUpdateinfo(savePath, channelLabel)
        endTime = time.time()
        LOG.info("Processed <%s> %s packages, %s errors, completed in %s seconds" \
                % (channelLabel, report.successes, report.errors, (endTime-startTime)))
        if self.removeOldPackages:
            LOG.info("Remove old packages from %s" % (savePath))

            self.runRemoveOldPackages(savePath)
        return report
    
    def fetchCompsXML(self, savePath, channelLabel):
        ###
        # Fetch comps.xml, used by createrepo for "groups" info
        ###
        compsxml = ""
        try:
            compsxml = self.rhnComm.getRepodata(channelLabel, "comps.xml")
        except GetRequestException, ge:
            if (ge.code == 404):
                LOG.info("Channel has no compsXml")
            else:
                raise ge
        if not savePath:
            savePath = channelLabel
        f = open(os.path.join(savePath, "comps.xml"), "w")
        f.write(compsxml)
        f.close()