def __init__(self, systemId, baseURL, channelLabel, savePath):
BaseFetch.__init__(self)
self.systemId = systemId
self.baseURL = baseURL
self.rhnComm = RHNComm(baseURL, self.systemId)
self.channelLabel = channelLabel
self.savePath = savePath
class KickstartFetch(BaseFetch):
def __init__(self, systemId, baseURL):
BaseFetch.__init__(self)
self.baseURL = baseURL
self.systemId = systemId
self.rhnComm = RHNComm(baseURL, self.systemId)
def login(self, refresh=False):
return self.rhnComm.login(refresh)
def getFetchURL(self, channelLabel, ksLabel, ksFilePath):
return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getKickstartFile/" + ksLabel + "/" + ksFilePath;
def fetchItem(self, itemInfo):
authMap = self.login()
fileName = itemInfo['fileName']
itemSize = itemInfo['size']
md5sum = itemInfo['md5sum']
hashType = itemInfo['hashtype']
ksLabel = itemInfo['ksLabel']
channelLabel = itemInfo['channelLabel']
savePath = itemInfo['savePath']
fetchURL = self.getFetchURL(channelLabel, ksLabel, fileName)
status = self.fetch(fileName, fetchURL, savePath, itemSize, hashType, md5sum, headers=authMap)
if status == BaseFetch.STATUS_UNAUTHORIZED:
LOG.warn("Unauthorized request from fetch(). Will attempt to update authentication credentials and retry")
authMap = self.login(refresh=True)
return self.fetch(fileName, fetchURL, savePath, itemSize, hashType, md5sum, headers=authMap)
return status
def __init__(self, systemId, baseURL, channelLabel, savePath):
BaseFetch.__init__(self)
self.systemId = systemId
self.baseURL = baseURL
self.rhnComm = RHNComm(baseURL, self.systemId)
self.channelLabel = channelLabel
self.savePath = savePath
def __init__(self, url, username, password, cert, systemid, parallel, verbose):
self.baseURL = url
self.cert = open(cert, 'r').read()
self.systemid = open(systemid, 'r').read()
self.username = username
self.password = password
self.parallel = parallel
self.fetchAll = False #default is only fetch latest packages
self.parallelFetchPkgs = None
self.parallelFetchKickstarts = None
self.skipProductList = []
self.skipPackageList = []
self.verbose = verbose
self.killcount = 0
self.removeOldPackages = False
self.numOldPkgsKeep = 1
self.rhnComm = RHNComm(url, self.systemid)
class PackageFetch(BaseFetch):
def __init__(self, systemId, baseURL, channelLabel, savePath):
BaseFetch.__init__(self)
self.systemId = systemId
self.baseURL = baseURL
self.rhnComm = RHNComm(baseURL, self.systemId)
self.channelLabel = channelLabel
self.savePath = savePath
def login(self, refresh=False):
"""
Returns authentication headers needed for RHN 'GET' requests.
auth data is cached, if data needs to be updated, pass in refresh=True
"""
return self.rhnComm.login(refresh)
def getFetchURL(self, channelLabel, fetchName):
return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getPackage/" + fetchName
def fetchItem(self, itemInfo):
authMap = self.login()
fileName = itemInfo['fileName']
fetchName = itemInfo['fetch_name']
itemSize = itemInfo['size']
md5sum = itemInfo['md5sum']
hashType = itemInfo['hashtype']
if itemInfo.has_key('pkgpath'):
pkgPath = itemInfo['pkgpath']
else:
pkgPath = None
fetchURL = self.getFetchURL(self.channelLabel, fetchName)
status = self.fetch(fileName,
fetchURL,
self.savePath,
itemSize,
hashType,
md5sum,
headers=authMap)
if status == BaseFetch.STATUS_UNAUTHORIZED:
LOG.warn(
"Unauthorized request from fetch(). Will attempt to update authentication credentials and retry"
)
authMap = self.login(refresh=True)
return self.fetch(fileName,
fetchURL,
self.savePath,
itemSize,
hashType,
md5sum,
headers=authMap,
pkgpath=pkgPath)
return status
class KickstartFetch(BaseFetch):
def __init__(self, systemId, baseURL):
BaseFetch.__init__(self)
self.baseURL = baseURL
self.systemId = systemId
self.rhnComm = RHNComm(baseURL, self.systemId)
def login(self, refresh=False):
return self.rhnComm.login(refresh)
def getFetchURL(self, channelLabel, ksLabel, ksFilePath):
return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getKickstartFile/" + ksLabel + "/" + ksFilePath
def fetchItem(self, itemInfo):
authMap = self.login()
fileName = itemInfo['fileName']
itemSize = itemInfo['size']
md5sum = itemInfo['md5sum']
hashType = itemInfo['hashtype']
ksLabel = itemInfo['ksLabel']
channelLabel = itemInfo['channelLabel']
savePath = itemInfo['savePath']
fetchURL = self.getFetchURL(channelLabel, ksLabel, fileName)
status = self.fetch(fileName,
fetchURL,
itemSize,
hashType,
md5sum,
savePath,
headers=authMap)
if status == BaseFetch.STATUS_UNAUTHORIZED:
LOG.warn(
"Unauthorized request from fetch(). Will attempt to update authentication credentials and retry"
)
authMap = self.login(refresh=True)
return self.fetch(fileName,
fetchURL,
itemSize,
hashType,
md5sum,
savePath,
headers=authMap)
return status
class PackageFetch(BaseFetch):
def __init__(self, systemId, baseURL, channelLabel, savePath):
BaseFetch.__init__(self)
self.systemId = systemId
self.baseURL = baseURL
self.rhnComm = RHNComm(baseURL, self.systemId)
self.channelLabel = channelLabel
self.savePath = savePath
def login(self, refresh=False):
"""
Returns authentication headers needed for RHN 'GET' requests.
auth data is cached, if data needs to be updated, pass in refresh=True
"""
return self.rhnComm.login(refresh)
def getFetchURL(self, channelLabel, fetchName):
return self.baseURL + "/SAT/$RHN/" + channelLabel + "/getPackage/" + fetchName
def fetchItem(self, itemInfo):
authMap = self.login()
fileName = itemInfo["fileName"]
fetchName = itemInfo["fetch_name"]
itemSize = itemInfo["size"]
md5sum = itemInfo["md5sum"]
hashType = itemInfo["hashtype"]
if itemInfo.has_key("pkgpath"):
pkgPath = itemInfo["pkgpath"]
else:
pkgPath = None
fetchURL = self.getFetchURL(self.channelLabel, fetchName)
status = self.fetch(fileName, fetchURL, self.savePath, itemSize, hashType, md5sum, headers=authMap)
if status == BaseFetch.STATUS_UNAUTHORIZED:
LOG.warn("Unauthorized request from fetch(). Will attempt to update authentication credentials and retry")
authMap = self.login(refresh=True)
return self.fetch(
fileName, fetchURL, self.savePath, itemSize, hashType, md5sum, headers=authMap, pkgpath=pkgPath
)
return status
def __init__(self, systemId, baseURL):
BaseFetch.__init__(self)
self.baseURL = baseURL
self.systemId = systemId
self.rhnComm = RHNComm(baseURL, self.systemId)
def __init__(self, systemId, baseURL):
BaseFetch.__init__(self)
self.baseURL = baseURL
self.systemId = systemId
self.rhnComm = RHNComm(baseURL, self.systemId)
class Grinder:
def __init__(self, url, username, password, cert, systemid, parallel, verbose):
self.baseURL = url
self.cert = open(cert, 'r').read()
self.systemid = open(systemid, 'r').read()
self.username = username
self.password = password
self.parallel = parallel
self.fetchAll = False #default is only fetch latest packages
self.parallelFetchPkgs = None
self.parallelFetchKickstarts = None
self.skipProductList = []
self.skipPackageList = []
self.verbose = verbose
self.killcount = 0
self.removeOldPackages = False
self.numOldPkgsKeep = 1
self.rhnComm = RHNComm(url, self.systemid)
def setRemoveOldPackages(self, value):
self.removeOldPackages = value
def getRemoveOldPackages(self):
return self.removeOldPackages
def getFetchAllPackages(self):
return self.fetchAll
def setFetchAllPackages(self, val):
self.fetchAll = val
def getSkipProductList(self):
return self.skipProductList
def setSkipProductList(self, skipProductList):
self.skipProductList = skipProductList
def getSkipPackageList(self):
return self.skipPackageList
def setSkipPackageList(self, skipPackageList):
self.skipPackageList = skipPackageList
def getNumOldPackagesToKeep(self):
return self.numOldPkgsKeep
def setNumOldPackagesToKeep(self, num):
self.numOldPkgsKeep = num
def deactivate(self):
SATELLITE_URL = "%s/rpc/api" % (self.baseURL)
client = getRhnApi(SATELLITE_URL, verbose=0)
key = client.auth.login(self.username, self.password)
retval = client.satellite.deactivateSatellite(self.systemid)
print "retval from deactivation: %s" % retval
client.auth.logout(key)
print "Deactivated!"
def activate(self):
rhn = RHNTransport()
satClient = getRhnApi(self.baseURL + "/SAT",
verbose=self.verbose, transport=rhn)
# First check if we are active
active = False
retval = satClient.authentication.check(self.systemid)
LOG.debug("AUTH CHECK: %s " % str(retval))
if (retval == 1):
LOG.debug("We are activated ... continue!")
active = True
else:
LOG.debug("Not active")
if (not active):
if(not self.username or not self.password):
raise SystemNotActivatedException()
SATELLITE_URL = "%s/rpc/api" % (self.baseURL)
client = RhnApi(SATELLITE_URL, verbose=0)
key = client.auth.login(self.username, self.password)
retval = client.satellite.activateSatellite(self.systemid, self.cert)
LOG.debug("retval from activation: %s" % retval)
if (retval != 1):
raise CantActivateException()
client.auth.logout(key)
LOG.debug("Activated!")
def stop(self):
if (self.parallelFetchPkgs):
self.parallelFetchPkgs.stop()
if (self.parallelFetchKickstarts):
self.parallelFetchKickstarts.stop()
def checkChannels(self, channelsToSync):
"""
Input:
channelsToSync - list of channels to sync
Output:
list containing bad channel names
"""
satDump = SatDumpClient(self.baseURL)
channelFamilies = satDump.getChannelFamilies(self.systemid)
badChannel = []
for channelLabel in channelsToSync:
found = False
for d in channelFamilies.values():
if channelLabel in d["channel_labels"]:
LOG.debug("Found %s under %s" % (channelLabel, d["label"]))
found = True
break
if not found:
LOG.debug("Unable to find %s, adding it to badChannel list" % (channelLabel))
badChannel.append(channelLabel)
return badChannel
def getChannelLabels(self):
labels = {}
satDump = SatDumpClient(self.baseURL)
channelFamilies = satDump.getChannelFamilies(self.systemid)
for d in channelFamilies.values():
if (d["label"] in self.skipProductList):
continue
labels[d["label"]] = d["channel_labels"]
return labels
def displayListOfChannels(self):
labels = self.getChannelLabels()
print("List of channels:")
for lbl in labels:
print("\nProduct : %s\n" % (lbl))
for l in labels[lbl]:
print(" %s" % (l))
def syncKickstarts(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync kickstarts from
savePath - path to save kickstarts
verbose - if true display more output
"""
startTime = time.time()
satDump = SatDumpClient(self.baseURL, verbose=verbose)
ksLabels = satDump.getKickstartLabels(self.systemid, [channelLabel])
LOG.info("Found %s kickstart labels for channel %s" % (len(ksLabels[channelLabel]), channelLabel))
ksFiles = []
for ksLbl in ksLabels[channelLabel]:
LOG.info("Syncing kickstart label: %s" % (ksLbl))
metadata = satDump.getKickstartTreeMetadata(self.systemid, [ksLbl])
LOG.info("Retrieved metadata on %s files for kickstart label: %s" % (len(metadata[ksLbl]["files"]), ksLbl))
ksSavePath = os.path.join(savePath, ksLbl)
for ksFile in metadata[ksLbl]["files"]:
info = {}
info["relative-path"] = ksFile["relative-path"]
info["size"] = ksFile["file-size"]
info["md5sum"] = ksFile["md5sum"]
info["ksLabel"] = ksLbl
info["channelLabel"] = channelLabel
info["savePath"] = ksSavePath
ksFiles.append(info)
ksFetch = KickstartFetch(self.systemid, self.baseURL)
numThreads = int(self.parallel)
self.parallelFetchKickstarts = ParallelFetch(ksFetch, numThreads)
self.parallelFetchKickstarts.addItemList(ksFiles)
self.parallelFetchKickstarts.start()
report = self.parallelFetchKickstarts.waitForFinish()
endTime = time.time()
LOG.info("Processed %s %s %s kickstart files, %s errors, completed in %s seconds" \
% (channelLabel, ksLabels[channelLabel], report.successes,
report.errors, (endTime-startTime)))
return report
def syncPackages(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync packages from
savePath - path to save packages
verbose - if true display more output
"""
startTime = time.time()
if channelLabel == "":
LOG.critical("No channel label specified to sync, abort sync.")
raise NoChannelLabelException()
LOG.info("sync(%s, %s) invoked" % (channelLabel, verbose))
satDump = SatDumpClient(self.baseURL, verbose=verbose)
LOG.debug("*** calling product_names ***")
packages = satDump.getChannelPackages(self.systemid, channelLabel)
LOG.info("%s packages are available, getting list of short metadata now." % (len(packages)))
pkgInfo = satDump.getShortPackageInfo(self.systemid, packages, filterLatest = not self.fetchAll)
LOG.info("%s packages have been marked to be fetched" % (len(pkgInfo.values())))
numThreads = int(self.parallel)
LOG.info("Running in parallel fetch mode with %s threads" % (numThreads))
pkgFetch = PackageFetch(self.systemid, self.baseURL, channelLabel, savePath)
self.parallelFetchPkgs = ParallelFetch(pkgFetch, numThreads)
self.parallelFetchPkgs.addItemList(pkgInfo.values())
self.parallelFetchPkgs.start()
report = self.parallelFetchPkgs.waitForFinish()
LOG.debug("Attempting to fetch comps.xml info from RHN")
self.fetchCompsXML(savePath, channelLabel)
self.fetchUpdateinfo(savePath, channelLabel)
endTime = time.time()
LOG.info("Processed <%s> %s packages, %s errors, completed in %s seconds" \
% (channelLabel, report.successes, report.errors, (endTime-startTime)))
if self.removeOldPackages:
LOG.info("Remove old packages from %s" % (savePath))
self.runRemoveOldPackages(savePath)
return report
def fetchCompsXML(self, savePath, channelLabel):
###
# Fetch comps.xml, used by createrepo for "groups" info
###
compsxml = ""
try:
compsxml = self.rhnComm.getRepodata(channelLabel, "comps.xml")
except GetRequestException, ge:
if (ge.code == 404):
LOG.info("Channel has no compsXml")
else:
raise ge
if not savePath:
savePath = channelLabel
f = open(os.path.join(savePath, "comps.xml"), "w")
f.write(compsxml)
f.close()