def fetchYumRepo(self, basepath="./"):
startTime = time.time()
self.yumFetch = RepoFetch(self.repo_label, repourl=self.repo_url, \
cacert=self.sslcacert, clicert=self.sslclientcert, \
clikey=self.sslclientkey, mirrorlist=self.mirrors, \
download_dir=basepath)
self.yumFetch.setupRepo()
LOG.info("Fetching repo metadata...")
# first fetch the metadata
self.yumFetch.getRepoData()
LOG.info("Determining downloadable Content bits...")
# get rpms to fetch
self.prepareRPMS()
# get drpms to fetch
self.prepareDRPMS()
# prepare for download
self.fetchPkgs = ParallelFetch(self.yumFetch, self.numThreads)
self.fetchPkgs.addItemList(self.downloadinfo)
self.fetchPkgs.start()
report = self.fetchPkgs.waitForFinish()
endTime = time.time()
LOG.info("Processed <%s> packages in [%d] seconds" % (len(self.downloadinfo), \
(endTime - startTime)))
return report
def syncPackages(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync packages from
savePath - path to save packages
verbose - if true display more output
"""
startTime = time.time()
if channelLabel == "":
LOG.critical("No channel label specified to sync, abort sync.")
raise NoChannelLabelException()
LOG.info("sync(%s, %s) invoked" % (channelLabel, verbose))
satDump = SatDumpClient(self.baseURL, verbose=verbose)
LOG.debug("*** calling product_names ***")
packages = satDump.getChannelPackages(self.systemid, channelLabel)
LOG.info("%s packages are available, getting list of short metadata now." % (len(packages)))
pkgInfo = satDump.getShortPackageInfo(self.systemid, packages, filterLatest = not self.fetchAll)
LOG.info("%s packages have been marked to be fetched" % (len(pkgInfo.values())))
numThreads = int(self.parallel)
LOG.info("Running in parallel fetch mode with %s threads" % (numThreads))
pkgFetch = PackageFetch(self.systemid, self.baseURL, channelLabel, savePath)
self.parallelFetchPkgs = ParallelFetch(pkgFetch, numThreads)
self.parallelFetchPkgs.addItemList(pkgInfo.values())
self.parallelFetchPkgs.start()
report = self.parallelFetchPkgs.waitForFinish()
LOG.debug("Attempting to fetch comps.xml info from RHN")
self.fetchCompsXML(savePath, channelLabel)
self.fetchUpdateinfo(savePath, channelLabel)
endTime = time.time()
LOG.info("Processed <%s> %s packages, %s errors, completed in %s seconds" \
% (channelLabel, report.successes, report.errors, (endTime-startTime)))
if self.removeOldPackages:
LOG.info("Remove old packages from %s" % (savePath))
self.runRemoveOldPackages(savePath)
return report
def syncKickstarts(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync kickstarts from
savePath - path to save kickstarts
verbose - if true display more output
"""
startTime = time.time()
satDump = SatDumpClient(self.baseURL, verbose=verbose)
ksLabels = satDump.getKickstartLabels(self.systemid, [channelLabel])
LOG.info("Found %s kickstart labels for channel %s" % (len(ksLabels[channelLabel]), channelLabel))
ksFiles = []
for ksLbl in ksLabels[channelLabel]:
LOG.info("Syncing kickstart label: %s" % (ksLbl))
metadata = satDump.getKickstartTreeMetadata(self.systemid, [ksLbl])
LOG.info("Retrieved metadata on %s files for kickstart label: %s" % (len(metadata[ksLbl]["files"]), ksLbl))
ksSavePath = os.path.join(savePath, ksLbl)
for ksFile in metadata[ksLbl]["files"]:
info = {}
info["relative-path"] = ksFile["relative-path"]
info["size"] = ksFile["file-size"]
info["md5sum"] = ksFile["md5sum"]
info["ksLabel"] = ksLbl
info["channelLabel"] = channelLabel
info["savePath"] = ksSavePath
ksFiles.append(info)
ksFetch = KickstartFetch(self.systemid, self.baseURL)
numThreads = int(self.parallel)
self.parallelFetchKickstarts = ParallelFetch(ksFetch, numThreads)
self.parallelFetchKickstarts.addItemList(ksFiles)
self.parallelFetchKickstarts.start()
report = self.parallelFetchKickstarts.waitForFinish()
endTime = time.time()
LOG.info("Processed %s %s %s kickstart files, %s errors, completed in %s seconds" \
% (channelLabel, ksLabels[channelLabel], report.successes,
report.errors, (endTime-startTime)))
return report
class YumRepoGrinder(object):
"""
Driver module to initiate the repo fetching
"""
def __init__(self, repo_label, repo_url, parallel, mirrors=None, \
cacert=None, clicert=None, clikey=None):
self.repo_label = repo_label
self.repo_url = repo_url
self.mirrors = mirrors
self.numThreads = int(parallel)
self.fetchPkgs = None
self.downloadinfo = []
self.yumFetch = None
self.sslcacert = cacert
self.sslclientcert = clicert
self.sslclientkey = clikey
def prepareRPMS(self):
pkglist = self.yumFetch.getPackageList()
for pkg in pkglist:
info = {}
#urljoin doesnt like epoch in rpm name so using string concat
info['fileName'] = pkg.__str__() + ".rpm"
info['downloadurl'] = self.yumFetch.repourl + '/' + pkg.relativepath
info['savepath'] = self.yumFetch.repo_dir + '/' + os.path.dirname(pkg.relativepath)
info['checksumtype'], info['checksum'], status = pkg.checksums[0]
info['size'] = pkg.size
self.downloadinfo.append(info)
LOG.info("%s packages have been marked to be fetched" % len(pkglist))
def prepareDRPMS(self):
deltarpms = self.yumFetch.getDeltaPackageList()
if not deltarpms:
return
for dpkg in deltarpms:
info = {}
relativepath = dpkg.deltas.values()[0].filename
info['fileName'] = dpkg.deltas.values()[0].filename
info['downloadurl'] = self.yumFetch.repourl + '/' + relativepath
info['savepath'] = self.yumFetch.repo_dir + '/' + os.path.dirname(relativepath)
info['checksumtype'] = dpkg.deltas.values()[0].checksum_type
info['checksum'] = dpkg.deltas.values()[0].checksum
info['size'] = dpkg.deltas.values()[0].size
self.downloadinfo.append(info)
LOG.info("%s delta rpms have been marked to be fetched" % len(deltarpms))
def fetchYumRepo(self, basepath="./"):
startTime = time.time()
self.yumFetch = RepoFetch(self.repo_label, repourl=self.repo_url, \
cacert=self.sslcacert, clicert=self.sslclientcert, \
clikey=self.sslclientkey, mirrorlist=self.mirrors, \
download_dir=basepath)
self.yumFetch.setupRepo()
LOG.info("Fetching repo metadata...")
# first fetch the metadata
self.yumFetch.getRepoData()
LOG.info("Determining downloadable Content bits...")
# get rpms to fetch
self.prepareRPMS()
# get drpms to fetch
self.prepareDRPMS()
# prepare for download
self.fetchPkgs = ParallelFetch(self.yumFetch, self.numThreads)
self.fetchPkgs.addItemList(self.downloadinfo)
self.fetchPkgs.start()
report = self.fetchPkgs.waitForFinish()
endTime = time.time()
LOG.info("Processed <%s> packages in [%d] seconds" % (len(self.downloadinfo), \
(endTime - startTime)))
return report
def stop(self):
if self.fetchPkgs:
self.fetchPkgs.stop()
class Grinder:
def __init__(self, url, username, password, cert, systemid, parallel, verbose):
self.baseURL = url
self.cert = open(cert, 'r').read()
self.systemid = open(systemid, 'r').read()
self.username = username
self.password = password
self.parallel = parallel
self.fetchAll = False #default is only fetch latest packages
self.parallelFetchPkgs = None
self.parallelFetchKickstarts = None
self.skipProductList = []
self.skipPackageList = []
self.verbose = verbose
self.killcount = 0
self.removeOldPackages = False
self.numOldPkgsKeep = 1
self.rhnComm = RHNComm(url, self.systemid)
def setRemoveOldPackages(self, value):
self.removeOldPackages = value
def getRemoveOldPackages(self):
return self.removeOldPackages
def getFetchAllPackages(self):
return self.fetchAll
def setFetchAllPackages(self, val):
self.fetchAll = val
def getSkipProductList(self):
return self.skipProductList
def setSkipProductList(self, skipProductList):
self.skipProductList = skipProductList
def getSkipPackageList(self):
return self.skipPackageList
def setSkipPackageList(self, skipPackageList):
self.skipPackageList = skipPackageList
def getNumOldPackagesToKeep(self):
return self.numOldPkgsKeep
def setNumOldPackagesToKeep(self, num):
self.numOldPkgsKeep = num
def deactivate(self):
SATELLITE_URL = "%s/rpc/api" % (self.baseURL)
client = getRhnApi(SATELLITE_URL, verbose=0)
key = client.auth.login(self.username, self.password)
retval = client.satellite.deactivateSatellite(self.systemid)
print "retval from deactivation: %s" % retval
client.auth.logout(key)
print "Deactivated!"
def activate(self):
rhn = RHNTransport()
satClient = getRhnApi(self.baseURL + "/SAT",
verbose=self.verbose, transport=rhn)
# First check if we are active
active = False
retval = satClient.authentication.check(self.systemid)
LOG.debug("AUTH CHECK: %s " % str(retval))
if (retval == 1):
LOG.debug("We are activated ... continue!")
active = True
else:
LOG.debug("Not active")
if (not active):
if(not self.username or not self.password):
raise SystemNotActivatedException()
SATELLITE_URL = "%s/rpc/api" % (self.baseURL)
client = RhnApi(SATELLITE_URL, verbose=0)
key = client.auth.login(self.username, self.password)
retval = client.satellite.activateSatellite(self.systemid, self.cert)
LOG.debug("retval from activation: %s" % retval)
if (retval != 1):
raise CantActivateException()
client.auth.logout(key)
LOG.debug("Activated!")
def stop(self):
if (self.parallelFetchPkgs):
self.parallelFetchPkgs.stop()
if (self.parallelFetchKickstarts):
self.parallelFetchKickstarts.stop()
def checkChannels(self, channelsToSync):
"""
Input:
channelsToSync - list of channels to sync
Output:
list containing bad channel names
"""
satDump = SatDumpClient(self.baseURL)
channelFamilies = satDump.getChannelFamilies(self.systemid)
badChannel = []
for channelLabel in channelsToSync:
found = False
for d in channelFamilies.values():
if channelLabel in d["channel_labels"]:
LOG.debug("Found %s under %s" % (channelLabel, d["label"]))
found = True
break
if not found:
LOG.debug("Unable to find %s, adding it to badChannel list" % (channelLabel))
badChannel.append(channelLabel)
return badChannel
def getChannelLabels(self):
labels = {}
satDump = SatDumpClient(self.baseURL)
channelFamilies = satDump.getChannelFamilies(self.systemid)
for d in channelFamilies.values():
if (d["label"] in self.skipProductList):
continue
labels[d["label"]] = d["channel_labels"]
return labels
def displayListOfChannels(self):
labels = self.getChannelLabels()
print("List of channels:")
for lbl in labels:
print("\nProduct : %s\n" % (lbl))
for l in labels[lbl]:
print(" %s" % (l))
def syncKickstarts(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync kickstarts from
savePath - path to save kickstarts
verbose - if true display more output
"""
startTime = time.time()
satDump = SatDumpClient(self.baseURL, verbose=verbose)
ksLabels = satDump.getKickstartLabels(self.systemid, [channelLabel])
LOG.info("Found %s kickstart labels for channel %s" % (len(ksLabels[channelLabel]), channelLabel))
ksFiles = []
for ksLbl in ksLabels[channelLabel]:
LOG.info("Syncing kickstart label: %s" % (ksLbl))
metadata = satDump.getKickstartTreeMetadata(self.systemid, [ksLbl])
LOG.info("Retrieved metadata on %s files for kickstart label: %s" % (len(metadata[ksLbl]["files"]), ksLbl))
ksSavePath = os.path.join(savePath, ksLbl)
for ksFile in metadata[ksLbl]["files"]:
info = {}
info["relative-path"] = ksFile["relative-path"]
info["size"] = ksFile["file-size"]
info["md5sum"] = ksFile["md5sum"]
info["ksLabel"] = ksLbl
info["channelLabel"] = channelLabel
info["savePath"] = ksSavePath
ksFiles.append(info)
ksFetch = KickstartFetch(self.systemid, self.baseURL)
numThreads = int(self.parallel)
self.parallelFetchKickstarts = ParallelFetch(ksFetch, numThreads)
self.parallelFetchKickstarts.addItemList(ksFiles)
self.parallelFetchKickstarts.start()
report = self.parallelFetchKickstarts.waitForFinish()
endTime = time.time()
LOG.info("Processed %s %s %s kickstart files, %s errors, completed in %s seconds" \
% (channelLabel, ksLabels[channelLabel], report.successes,
report.errors, (endTime-startTime)))
return report
def syncPackages(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync packages from
savePath - path to save packages
verbose - if true display more output
"""
startTime = time.time()
if channelLabel == "":
LOG.critical("No channel label specified to sync, abort sync.")
raise NoChannelLabelException()
LOG.info("sync(%s, %s) invoked" % (channelLabel, verbose))
satDump = SatDumpClient(self.baseURL, verbose=verbose)
LOG.debug("*** calling product_names ***")
packages = satDump.getChannelPackages(self.systemid, channelLabel)
LOG.info("%s packages are available, getting list of short metadata now." % (len(packages)))
pkgInfo = satDump.getShortPackageInfo(self.systemid, packages, filterLatest = not self.fetchAll)
LOG.info("%s packages have been marked to be fetched" % (len(pkgInfo.values())))
numThreads = int(self.parallel)
LOG.info("Running in parallel fetch mode with %s threads" % (numThreads))
pkgFetch = PackageFetch(self.systemid, self.baseURL, channelLabel, savePath)
self.parallelFetchPkgs = ParallelFetch(pkgFetch, numThreads)
self.parallelFetchPkgs.addItemList(pkgInfo.values())
self.parallelFetchPkgs.start()
report = self.parallelFetchPkgs.waitForFinish()
LOG.debug("Attempting to fetch comps.xml info from RHN")
self.fetchCompsXML(savePath, channelLabel)
self.fetchUpdateinfo(savePath, channelLabel)
endTime = time.time()
LOG.info("Processed <%s> %s packages, %s errors, completed in %s seconds" \
% (channelLabel, report.successes, report.errors, (endTime-startTime)))
if self.removeOldPackages:
LOG.info("Remove old packages from %s" % (savePath))
self.runRemoveOldPackages(savePath)
return report
def fetchCompsXML(self, savePath, channelLabel):
###
# Fetch comps.xml, used by createrepo for "groups" info
###
compsxml = ""
try:
compsxml = self.rhnComm.getRepodata(channelLabel, "comps.xml")
except GetRequestException, ge:
if (ge.code == 404):
LOG.info("Channel has no compsXml")
else:
raise ge
if not savePath:
savePath = channelLabel
f = open(os.path.join(savePath, "comps.xml"), "w")
f.write(compsxml)
f.close()
