def testGetKeyFromPEM(self):
expected_key_file = os.path.join(self.data_dir, "sample_cert.key")
test_pem_file = os.path.join(self.data_dir, "sample_cert.pem")
expected_key = open(expected_key_file).read().rstrip()
test_pem = open(test_pem_file).read()
key = GrinderUtils.getKeyFromPEM(test_pem)
print "Found key = <%s>" % (key)
print "Expected key = <%s>" % (expected_key)
#print "key == expected key, <%s>" % (key == expected_key)
self.assertEqual(key, expected_key)
def testGetCertFromPEM(self):
expected_cert_file = os.path.join(self.data_dir, "sample_cert.cert")
test_pem_file = os.path.join(self.data_dir, "sample_cert.pem")
expected_cert = open(expected_cert_file).read().rstrip()
test_pem = open(test_pem_file).read()
cert = GrinderUtils.getCertFromPEM(test_pem)
#print "Found cert = <%s>" % (cert)
#print "Expected cert = <%s>" % (expected_cert)
#print "cert == expected cert, <%s>" % (cert == expected_cert)
self.assertEqual(cert, expected_cert)
def fetch(self, fileName, fetchURL, savePath, itemSize=None, hashtype=None, checksum=None,
headers=None, retryTimes=None, packages_location=None, verify_options=None, probing=None, force=False):
"""
@param fileName file name
@type fileName str
@param fetchURL url
@type fetchURL str
@param savePath path to save file
@type savePath str
@param itemSize expected size of file
@type itemSize str (will be cast to int)
@param hashtype type of checksum
@type hashtype str
@param checksum value of file
@type checksum str
@param headers optional data to include in headers
@type headers dict{str, str}
@param retryTimes number of times to retry if a problem occurs
@type retryTimes int
@param packages_location path where packages get stored
@type packages_location str
@param verify_options optional parameter to limit the verify operations run on existing files
@type verify_options dict{option=value}, where option is one of "size", "checksum" and value is True/False
@param probing if True will be silent and not log warnings/errors, useful when we are probing to see if a file exists
@type probing bool
@param force if True will force fetch the download file; use this for downloads that dont have checksum and size info for verification
@type force bool
@return true/false if item was fetched successfully
@rtype bool
"""
if retryTimes is None:
retryTimes = self.num_retries
if packages_location is not None:
# this option is to store packages in a central location
# and symlink pkgs to individual repo directories
filePath = os.path.join(packages_location, fileName)
repofilepath = os.path.join(savePath, fileName)
basedir = os.path.dirname(repofilepath)
if basedir and not os.path.exists(basedir):
self.makeDirSafe(basedir)
else:
repofilepath = None
filePath = os.path.join(savePath, fileName)
tempDirPath = os.path.dirname(filePath)
if not os.path.isdir(tempDirPath):
LOG.info("Creating directory: %s" % tempDirPath)
self.makeDirSafe(tempDirPath)
if os.path.exists(filePath) and \
verifyExisting(filePath, itemSize, hashtype, checksum, verify_options) and not force:
LOG.debug("%s exists with expected information, no need to fetch." % (filePath))
if repofilepath is not None and not os.path.exists(repofilepath):
relFilePath = GrinderUtils.get_relative_path(filePath, repofilepath)
LOG.info("Symlink missing in repo directory. Creating link %s to %s" % (repofilepath, relFilePath))
if not os.path.islink(repofilepath):
self.makeSafeSymlink(relFilePath, repofilepath)
return (BaseFetch.STATUS_NOOP,None)
# Acquire a write lock so no other process duplicates the effort
grinder_write_locker = GrinderLock(filePath + '.lock')
existing_lock_pid = grinder_write_locker.readlock()
new_pid = os.getpid()
if existing_lock_pid and int(existing_lock_pid) != new_pid and grinder_write_locker.isvalid(existing_lock_pid):
# If there is an existing write pid
# and if the pid is not same as the current pid
# and pid is valid there is another process alive
# and handling this, exit here.
LOG.debug("another process is already handling this path [%s] and is alive; no need to process this again " % filePath)
return (BaseFetch.STATUS_REQUEUE,None)
# this means either there is no pid or a pid matching current pid exists(retry case),
# verify lock and either skip or re-acquire it
try:
grinder_write_locker.acquire()
existing_lock_pid = grinder_write_locker.readlock()
except:
LOG.debug("Unable to acquire lock.")
return (BaseFetch.STATUS_REQUEUE,None)
if not existing_lock_pid or existing_lock_pid != str(os.getpid()):
# This means, either we still dont have a lock and hence not safe to proceed or
# the acquired lock doesnt match current pid, return and let the next process handle it
return (BaseFetch.STATUS_NOOP,None)
try:
#f = open(filePath, "wb")
curl = pycurl.Curl()
def item_progress_callback(download_total, downloaded, upload_total, uploaded):
#LOG.debug("%s status %s/%s bytes" % (fileName, downloaded, download_total))
self.update_bytes_transferred(fetchURL, download_total, downloaded)
curl.setopt(curl.NOPROGRESS, False)
curl.setopt(curl.PROGRESSFUNCTION, item_progress_callback)
if self.max_speed:
#Convert KB/sec to Bytes/sec for MAC_RECV_SPEED_LARGE
limit = self.max_speed*1024
curl.setopt(curl.MAX_RECV_SPEED_LARGE, limit)
curl.setopt(curl.VERBOSE,0)
# We have seen rare and intermittent problems with grinder syncing against a remote server
# the remote server leaves a socket open but does not send back data. Grinder has been stuck
# for several days looping over a poll of the socket with no data being sent.
# Slower than 1000 byes over 5 minutes will mark the connection as too slow and abort
curl.setopt(curl.LOW_SPEED_LIMIT,1000)
curl.setopt(curl.LOW_SPEED_TIME,60*5)
# When using multiple threads you should set the CURLOPT_NOSIGNAL option to 1 for all handles
# May impact DNS timeouts
curl.setopt(curl.NOSIGNAL, 1)
if type(fetchURL) == types.UnicodeType:
#pycurl does not accept unicode strings for a URL, so we need to convert
fetchURL = unicodedata.normalize('NFKD', fetchURL).encode('ascii','ignore')
curl.setopt(curl.URL, fetchURL)
if self.sslcacert:
curl.setopt(curl.CAINFO, self.sslcacert)
if self.sslclientcert:
curl.setopt(curl.SSLCERT, self.sslclientcert)
if self.sslclientkey:
curl.setopt(curl.SSLKEY, self.sslclientkey)
if not self.sslverify:
curl.setopt(curl.SSL_VERIFYPEER, 0)
if headers:
curl.setopt(pycurl.HTTPHEADER, curlifyHeaders(headers))
if self.proxy_url:
if not self.proxy_port:
raise GrinderException("Proxy url defined, but no port specified")
curl.setopt(pycurl.PROXY, self.proxy_url)
curl.setopt(pycurl.PROXYPORT, int(self.proxy_port))
curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP)
if self.proxy_user:
if not self.proxy_pass:
raise GrinderException("Proxy username is defined, but no password was specified")
curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_BASIC)
curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (self.proxy_user, self.proxy_pass))
# callback logic to save and resume bits
tmp_write_file = get_temp_file_name(filePath)
if itemSize is not None:
itemSize = int(itemSize)
wf = WriteFunction(tmp_write_file, itemSize)
if wf.offset > 0:
# setup file resume
LOG.info("A partial download file already exists; prepare to resume download.")
curl.setopt(pycurl.RESUME_FROM, wf.offset)
curl.setopt(curl.WRITEFUNCTION, wf.callback)
curl.setopt(curl.FOLLOWLOCATION, 1)
if not probing:
LOG.info("Fetching %s bytes: %s from %s" % (itemSize or "Unknown", fileName, fetchURL))
curl.perform()
status = curl.getinfo(curl.HTTP_CODE)
curl.close()
wf.cleanup()
# this tmp file could be closed by other concurrent processes
if os.path.exists(tmp_write_file):
# download complete rename the .part file
os.rename(tmp_write_file, filePath)
# validate the fetched bits
if itemSize is not None and hashtype is not None and checksum is not None:
vstatus = self.validateDownload(filePath, int(itemSize), hashtype, checksum)
else:
vstatus = BaseFetch.STATUS_SKIP_VALIDATE
if status == 401:
LOG.error("Unauthorized request from: %s" % (fetchURL))
grinder_write_locker.release()
cleanup(filePath)
return (BaseFetch.STATUS_UNAUTHORIZED, "HTTP status code of %s received for %s" % (status, fetchURL))
if status not in (0, 200, 206, 226):
# 0 - for local syncs
# 200 - is typical http return code, yet 206 and 226 have also been seen to be returned and valid
if retryTimes > 0 and not fetchURL.startswith("file:"):
retryTimes -= 1
LOG.warn("Retrying fetch of: %s with %s retry attempts left. HTTP status was %s" % (fileName, retryTimes, status))
cleanup(filePath)
self.reset_bytes_transferred(fetchURL)
return self.fetch(fileName, fetchURL, savePath, itemSize, hashtype,
checksum , headers, retryTimes, packages_location)
grinder_write_locker.release()
cleanup(filePath)
LOG.warn("ERROR: Response = %s fetching %s." % (status, fetchURL))
return (BaseFetch.STATUS_ERROR, "HTTP status code of %s received for %s" % (status, fetchURL))
if vstatus in [BaseFetch.STATUS_ERROR, BaseFetch.STATUS_SIZE_MISSMATCH,
BaseFetch.STATUS_MD5_MISSMATCH] and retryTimes > 0:
#
# Incase of a network glitch or issue with RHN, retry the rpm fetch
#
retryTimes -= 1
LOG.error("Retrying fetch of: %s with %s retry attempts left. VerifyStatus was %s" % (fileName, retryTimes, vstatus))
cleanup(filePath)
self.reset_bytes_transferred(fetchURL)
return self.fetch(fileName, fetchURL, savePath, itemSize, hashtype,
checksum, headers, retryTimes, packages_location)
if packages_location and os.path.exists(filePath):
relFilePath = GrinderUtils.get_relative_path(filePath, repofilepath)
LOG.info("Create a link in repo directory for the package at %s to %s" % (repofilepath, relFilePath))
self.makeSafeSymlink(relFilePath, repofilepath)
grinder_write_locker.release()
LOG.debug("Successfully Fetched Package - [%s]" % filePath)
return (vstatus, None)
except Exception, e:
cleanup(tmp_write_file)
cleanup(filePath)
if probing:
LOG.info("Probed for %s and determined it is missing." % (fetchURL))
grinder_write_locker.release()
return BaseFetch.STATUS_ERROR, None
tb_info = traceback.format_exc()
LOG.error("Caught exception<%s> in fetch(%s, %s)" % (e, fileName, fetchURL))
LOG.error("%s" % (tb_info))
if retryTimes > 0 and not fetchURL.startswith("file:"):
retryTimes -= 1
#grinder_write_locker.release()
LOG.error("Retrying fetch of: %s with %s retry attempts left." % (fileName, retryTimes))
self.reset_bytes_transferred(fetchURL)
return self.fetch(fileName, fetchURL, savePath, itemSize, hashtype,
checksum, headers, retryTimes, packages_location)
grinder_write_locker.release()
raise
