def fetchYumRepo(self, basepath="./", callback=None, verify_options=None):
startTime = time.time()
self.yumFetch = RepoFetch(self.repo_label, repourl=self.repo_url, \
cacert=self.sslcacert, clicert=self.sslclientcert, \
clikey=self.sslclientkey, mirrorlist=self.mirrors, \
download_dir=basepath, proxy_url=self.proxy_url, \
proxy_port=self.proxy_port, proxy_user=self.proxy_user, \
proxy_pass=self.proxy_pass, sslverify=self.sslverify,
max_speed=self.max_speed,
verify_options=verify_options)
self.fetchPkgs = ParallelFetch(self.yumFetch, self.numThreads, callback=callback)
try:
if not verify_options:
verify_options = {"size":False, "checksum":False}
self.yumFetch.setupRepo()
# first fetch the metadata
self.fetchPkgs.processCallback(ProgressReport.DownloadMetadata)
self.yumFetch.getRepoData()
if self.stopped:
return None
if not self.skip.has_key('packages') or self.skip['packages'] != 1:
# get rpms to fetch
self.prepareRPMS()
# get drpms to fetch
self.prepareDRPMS()
else:
log("Skipping packages preparation from sync process")
if not self.skip.has_key('distribution') or self.skip['distribution'] != 1:
# get Trees to fetch
self.prepareTrees()
else:
log("Skipping distribution preparation from sync process")
# prepare for download
self.fetchPkgs.addItemList(self.downloadinfo)
self.fetchPkgs.start()
report = self.fetchPkgs.waitForFinish()
self.yumFetch.finalizeMetadata()
endTime = time.time()
#log("Processed <%s> items in [%d] seconds" % (len(self.downloadinfo), \
# (endTime - startTime)))
if not self.skip.has_key('packages') or self.skip['packages'] != 1:
if self.purge_orphaned:
#log("Cleaning any orphaned packages..")
self.fetchPkgs.processCallback(ProgressReport.PurgeOrphanedPackages)
self.purgeOrphanPackages(self.yumFetch.getPackageList(), self.yumFetch.repo_dir)
if self.remove_old:
log("Removing old packages to limit to %s" % self.numOldPackages)
self.fetchPkgs.processCallback(ProgressReport.RemoveOldPackages)
gutils = GrinderUtils()
gutils.runRemoveOldPackages(self.pkgsavepath, self.numOldPackages)
self.yumFetch.deleteBaseCacheDir()
#log("Processed <%s> in %s seconds" % (report, endTime - startTime))
return report, (endTime - startTime)
finally:
self.fetchPkgs.stop()
self.yumFetch.closeRepo()
def setup(self, basepath="./", callback=None, verify_options=None, num_retries=None, retry_delay=None):
"""
Fetches yum metadata and determines what object should be downloaded.
@param basepath: path to store repo data
@type basepath: str
@param callback: progress callback function
@type callback: function which accepts a grinder.GrinderCallback.ProgressReport
@param verify_options: controls verification checks on "size" and "checksum".
@type verify_options: dict{"size":bool,"checksum":bool}
@param num_retries: number of retries to perform if an error occurs
@type num_retries: int
@param retry_delay: delay in seconds between retries, delay = 'retry_attempt' * 'retry_delay'
@type retry_delay: int
"""
self.repo_dir = os.path.join(basepath, self.repo_label)
LOG.info("%s, %s, Calling RepoFetch with: cacert=<%s>, clicert=<%s>, clikey=<%s>, proxy_url=<%s>, proxy_port=<%s>, proxy_user=<%s>, proxy_pass=<NOT_LOGGED>, sslverify=<%s>, max_speed=<%s>, verify_options=<%s>, filter=<%s>" %\
(self.repo_label, self.repo_url, self.sslcacert, self.sslclientcert, self.sslclientkey, self.proxy_url, self.proxy_port, self.proxy_user, self.sslverify, self.max_speed, verify_options, self.filter))
self.repoFetch = RepoFetch(cacert=self.sslcacert, clicert=self.sslclientcert, clikey=self.sslclientkey,\
proxy_url=self.proxy_url, proxy_port=self.proxy_port,
proxy_user=self.proxy_user, proxy_pass=self.proxy_pass,
sslverify=self.sslverify,
max_speed=self.max_speed,
verify_options=verify_options, num_retries=num_retries)
self.fetchPkgs = ParallelFetch(self.repoFetch, self.numThreads, callback=callback)
self.fetchPkgs.processCallback(ProgressReport.DownloadMetadata)
info = YumInfo(
repo_label=self.repo_label, repo_url=self.repo_url,
mirrors = self.mirrors, repo_dir=self.repo_dir,
packages_location=self.pkgpath, newest=self.newest,
remove_old=self.remove_old, numOldPackages=self.numOldPackages,
cacert=self.sslcacert, clicert=self.sslclientcert,
clikey=self.sslclientkey, proxy_url=self.proxy_url,
proxy_port=self.proxy_port, proxy_user=self.proxy_user,
proxy_pass=self.proxy_pass, sslverify=self.sslverify, skip=self.skip,
tmp_path=self.tmp_path, filter=self.filter,
num_retries=num_retries, retry_delay=retry_delay)
info.setUp()
self.rpmlist = info.rpms
self.drpmlist = info.drpms
def fetch(self, basepath="./", callback=None):
LOG.info("fetch basepath = %s" % (basepath))
startTime = time.time()
self.fileFetch = FileFetch(self.repo_label, self.repo_url, cacert=self.sslcacert, \
clicert=self.sslclientcert, clikey=self.sslclientkey, \
download_dir=basepath, proxy_url=self.proxy_url, \
proxy_port=self.proxy_port, proxy_user=self.proxy_user, \
proxy_pass=self.proxy_pass, sslverify=self.sslverify, max_speed=self.max_speed)
self.parallel_fetch_files = ParallelFetch(self.fileFetch,
self.numThreads,
callback=callback)
LOG.info("Determining downloadable Content bits...")
self.parallel_fetch_files.processCallback(
ProgressReport.DownloadMetadata)
self.prepareFiles()
# prepare for download
self.parallel_fetch_files.addItemList(self.downloadinfo)
self.parallel_fetch_files.start()
report = self.parallel_fetch_files.waitForFinish()
endTime = time.time()
LOG.info("Processed <%s> items in [%d] seconds" % (len(self.downloadinfo), \
(endTime - startTime)))
return report
def syncPackages(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync packages from
savePath - path to save packages
verbose - if true display more output
"""
startTime = time.time()
if channelLabel == "":
LOG.critical("No channel label specified to sync, abort sync.")
raise NoChannelLabelException()
LOG.info("sync(%s, %s) invoked" % (channelLabel, verbose))
satDump = SatDumpClient(self.baseURL, verbose=verbose)
LOG.debug("*** calling product_names ***")
packages = satDump.getChannelPackages(self.systemid, channelLabel)
LOG.info("%s packages are available, getting list of short metadata now." % (len(packages)))
pkgInfo = satDump.getShortPackageInfo(self.systemid, packages, filterLatest = not self.fetchAll)
LOG.info("%s packages have been marked to be fetched" % (len(pkgInfo.values())))
numThreads = int(self.parallel)
LOG.info("Running in parallel fetch mode with %s threads" % (numThreads))
pkgFetch = PackageFetch(self.systemid, self.baseURL, channelLabel, savePath)
self.parallelFetchPkgs = ParallelFetch(pkgFetch, numThreads)
self.parallelFetchPkgs.addItemList(pkgInfo.values())
self.parallelFetchPkgs.start()
report = self.parallelFetchPkgs.waitForFinish()
LOG.debug("Attempting to fetch comps.xml info from RHN")
self.fetchCompsXML(savePath, channelLabel)
self.fetchUpdateinfo(savePath, channelLabel)
endTime = time.time()
LOG.info("Processed <%s> %s packages, %s errors, completed in %s seconds" \
% (channelLabel, report.successes, report.errors, (endTime-startTime)))
if self.removeOldPackages:
LOG.info("Remove old packages from %s" % (savePath))
self.runRemoveOldPackages(savePath)
return report
def syncKickstarts(self, channelLabel, savePath, verbose=0):
"""
channelLabel - channel to sync kickstarts from
savePath - path to save kickstarts
verbose - if true display more output
"""
startTime = time.time()
satDump = SatDumpClient(self.baseURL, verbose=verbose)
ksLabels = satDump.getKickstartLabels(self.systemid, [channelLabel])
LOG.info("Found %s kickstart labels for channel %s" % (len(ksLabels[channelLabel]), channelLabel))
ksFiles = []
for ksLbl in ksLabels[channelLabel]:
LOG.info("Syncing kickstart label: %s" % (ksLbl))
metadata = satDump.getKickstartTreeMetadata(self.systemid, [ksLbl])
LOG.info("Retrieved metadata on %s files for kickstart label: %s" % (len(metadata[ksLbl]["files"]), ksLbl))
ksSavePath = os.path.join(savePath, ksLbl)
for ksFile in metadata[ksLbl]["files"]:
info = {}
info["relative-path"] = ksFile["relative-path"]
info["size"] = ksFile["file-size"]
info["md5sum"] = ksFile["md5sum"]
info["ksLabel"] = ksLbl
info["channelLabel"] = channelLabel
info["savePath"] = ksSavePath
ksFiles.append(info)
ksFetch = KickstartFetch(self.systemid, self.baseURL)
numThreads = int(self.parallel)
self.parallelFetchKickstarts = ParallelFetch(ksFetch, numThreads)
self.parallelFetchKickstarts.addItemList(ksFiles)
self.parallelFetchKickstarts.start()
report = self.parallelFetchKickstarts.waitForFinish()
endTime = time.time()
LOG.info("Processed %s %s %s kickstart files, %s errors, completed in %s seconds" \
% (channelLabel, ksLabels[channelLabel], report.successes,
report.errors, (endTime-startTime)))
return report
