def testGetKeyFromPEM(self): expected_key_file = os.path.join(self.data_dir, "sample_cert.key") test_pem_file = os.path.join(self.data_dir, "sample_cert.pem") expected_key = open(expected_key_file).read().rstrip() test_pem = open(test_pem_file).read() key = GrinderUtils.getKeyFromPEM(test_pem) print "Found key = <%s>" % (key) print "Expected key = <%s>" % (expected_key) #print "key == expected key, <%s>" % (key == expected_key) self.assertEqual(key, expected_key)
def testGetCertFromPEM(self): expected_cert_file = os.path.join(self.data_dir, "sample_cert.cert") test_pem_file = os.path.join(self.data_dir, "sample_cert.pem") expected_cert = open(expected_cert_file).read().rstrip() test_pem = open(test_pem_file).read() cert = GrinderUtils.getCertFromPEM(test_pem) #print "Found cert = <%s>" % (cert) #print "Expected cert = <%s>" % (expected_cert) #print "cert == expected cert, <%s>" % (cert == expected_cert) self.assertEqual(cert, expected_cert)
def fetch(self, fileName, fetchURL, savePath, itemSize=None, hashtype=None, checksum=None, headers=None, retryTimes=None, packages_location=None, verify_options=None, probing=None, force=False): """ @param fileName file name @type fileName str @param fetchURL url @type fetchURL str @param savePath path to save file @type savePath str @param itemSize expected size of file @type itemSize str (will be cast to int) @param hashtype type of checksum @type hashtype str @param checksum value of file @type checksum str @param headers optional data to include in headers @type headers dict{str, str} @param retryTimes number of times to retry if a problem occurs @type retryTimes int @param packages_location path where packages get stored @type packages_location str @param verify_options optional parameter to limit the verify operations run on existing files @type verify_options dict{option=value}, where option is one of "size", "checksum" and value is True/False @param probing if True will be silent and not log warnings/errors, useful when we are probing to see if a file exists @type probing bool @param force if True will force fetch the download file; use this for downloads that dont have checksum and size info for verification @type force bool @return true/false if item was fetched successfully @rtype bool """ if retryTimes is None: retryTimes = self.num_retries if packages_location is not None: # this option is to store packages in a central location # and symlink pkgs to individual repo directories filePath = os.path.join(packages_location, fileName) repofilepath = os.path.join(savePath, fileName) basedir = os.path.dirname(repofilepath) if basedir and not os.path.exists(basedir): self.makeDirSafe(basedir) else: repofilepath = None filePath = os.path.join(savePath, fileName) tempDirPath = os.path.dirname(filePath) if not os.path.isdir(tempDirPath): LOG.info("Creating directory: %s" % tempDirPath) self.makeDirSafe(tempDirPath) if os.path.exists(filePath) and \ verifyExisting(filePath, itemSize, hashtype, checksum, verify_options) and not force: LOG.debug("%s exists with expected information, no need to fetch." % (filePath)) if repofilepath is not None and not os.path.exists(repofilepath): relFilePath = GrinderUtils.get_relative_path(filePath, repofilepath) LOG.info("Symlink missing in repo directory. Creating link %s to %s" % (repofilepath, relFilePath)) if not os.path.islink(repofilepath): self.makeSafeSymlink(relFilePath, repofilepath) return (BaseFetch.STATUS_NOOP,None) # Acquire a write lock so no other process duplicates the effort grinder_write_locker = GrinderLock(filePath + '.lock') existing_lock_pid = grinder_write_locker.readlock() new_pid = os.getpid() if existing_lock_pid and int(existing_lock_pid) != new_pid and grinder_write_locker.isvalid(existing_lock_pid): # If there is an existing write pid # and if the pid is not same as the current pid # and pid is valid there is another process alive # and handling this, exit here. LOG.debug("another process is already handling this path [%s] and is alive; no need to process this again " % filePath) return (BaseFetch.STATUS_REQUEUE,None) # this means either there is no pid or a pid matching current pid exists(retry case), # verify lock and either skip or re-acquire it try: grinder_write_locker.acquire() existing_lock_pid = grinder_write_locker.readlock() except: LOG.debug("Unable to acquire lock.") return (BaseFetch.STATUS_REQUEUE,None) if not existing_lock_pid or existing_lock_pid != str(os.getpid()): # This means, either we still dont have a lock and hence not safe to proceed or # the acquired lock doesnt match current pid, return and let the next process handle it return (BaseFetch.STATUS_NOOP,None) try: #f = open(filePath, "wb") curl = pycurl.Curl() def item_progress_callback(download_total, downloaded, upload_total, uploaded): #LOG.debug("%s status %s/%s bytes" % (fileName, downloaded, download_total)) self.update_bytes_transferred(fetchURL, download_total, downloaded) curl.setopt(curl.NOPROGRESS, False) curl.setopt(curl.PROGRESSFUNCTION, item_progress_callback) if self.max_speed: #Convert KB/sec to Bytes/sec for MAC_RECV_SPEED_LARGE limit = self.max_speed*1024 curl.setopt(curl.MAX_RECV_SPEED_LARGE, limit) curl.setopt(curl.VERBOSE,0) # We have seen rare and intermittent problems with grinder syncing against a remote server # the remote server leaves a socket open but does not send back data. Grinder has been stuck # for several days looping over a poll of the socket with no data being sent. # Slower than 1000 byes over 5 minutes will mark the connection as too slow and abort curl.setopt(curl.LOW_SPEED_LIMIT,1000) curl.setopt(curl.LOW_SPEED_TIME,60*5) # When using multiple threads you should set the CURLOPT_NOSIGNAL option to 1 for all handles # May impact DNS timeouts curl.setopt(curl.NOSIGNAL, 1) if type(fetchURL) == types.UnicodeType: #pycurl does not accept unicode strings for a URL, so we need to convert fetchURL = unicodedata.normalize('NFKD', fetchURL).encode('ascii','ignore') curl.setopt(curl.URL, fetchURL) if self.sslcacert: curl.setopt(curl.CAINFO, self.sslcacert) if self.sslclientcert: curl.setopt(curl.SSLCERT, self.sslclientcert) if self.sslclientkey: curl.setopt(curl.SSLKEY, self.sslclientkey) if not self.sslverify: curl.setopt(curl.SSL_VERIFYPEER, 0) if headers: curl.setopt(pycurl.HTTPHEADER, curlifyHeaders(headers)) if self.proxy_url: if not self.proxy_port: raise GrinderException("Proxy url defined, but no port specified") curl.setopt(pycurl.PROXY, self.proxy_url) curl.setopt(pycurl.PROXYPORT, int(self.proxy_port)) curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) if self.proxy_user: if not self.proxy_pass: raise GrinderException("Proxy username is defined, but no password was specified") curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_BASIC) curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (self.proxy_user, self.proxy_pass)) # callback logic to save and resume bits tmp_write_file = get_temp_file_name(filePath) if itemSize is not None: itemSize = int(itemSize) wf = WriteFunction(tmp_write_file, itemSize) if wf.offset > 0: # setup file resume LOG.info("A partial download file already exists; prepare to resume download.") curl.setopt(pycurl.RESUME_FROM, wf.offset) curl.setopt(curl.WRITEFUNCTION, wf.callback) curl.setopt(curl.FOLLOWLOCATION, 1) if not probing: LOG.info("Fetching %s bytes: %s from %s" % (itemSize or "Unknown", fileName, fetchURL)) curl.perform() status = curl.getinfo(curl.HTTP_CODE) curl.close() wf.cleanup() # this tmp file could be closed by other concurrent processes if os.path.exists(tmp_write_file): # download complete rename the .part file os.rename(tmp_write_file, filePath) # validate the fetched bits if itemSize is not None and hashtype is not None and checksum is not None: vstatus = self.validateDownload(filePath, int(itemSize), hashtype, checksum) else: vstatus = BaseFetch.STATUS_SKIP_VALIDATE if status == 401: LOG.error("Unauthorized request from: %s" % (fetchURL)) grinder_write_locker.release() cleanup(filePath) return (BaseFetch.STATUS_UNAUTHORIZED, "HTTP status code of %s received for %s" % (status, fetchURL)) if status not in (0, 200, 206, 226): # 0 - for local syncs # 200 - is typical http return code, yet 206 and 226 have also been seen to be returned and valid if retryTimes > 0 and not fetchURL.startswith("file:"): retryTimes -= 1 LOG.warn("Retrying fetch of: %s with %s retry attempts left. HTTP status was %s" % (fileName, retryTimes, status)) cleanup(filePath) self.reset_bytes_transferred(fetchURL) return self.fetch(fileName, fetchURL, savePath, itemSize, hashtype, checksum , headers, retryTimes, packages_location) grinder_write_locker.release() cleanup(filePath) LOG.warn("ERROR: Response = %s fetching %s." % (status, fetchURL)) return (BaseFetch.STATUS_ERROR, "HTTP status code of %s received for %s" % (status, fetchURL)) if vstatus in [BaseFetch.STATUS_ERROR, BaseFetch.STATUS_SIZE_MISSMATCH, BaseFetch.STATUS_MD5_MISSMATCH] and retryTimes > 0: # # Incase of a network glitch or issue with RHN, retry the rpm fetch # retryTimes -= 1 LOG.error("Retrying fetch of: %s with %s retry attempts left. VerifyStatus was %s" % (fileName, retryTimes, vstatus)) cleanup(filePath) self.reset_bytes_transferred(fetchURL) return self.fetch(fileName, fetchURL, savePath, itemSize, hashtype, checksum, headers, retryTimes, packages_location) if packages_location and os.path.exists(filePath): relFilePath = GrinderUtils.get_relative_path(filePath, repofilepath) LOG.info("Create a link in repo directory for the package at %s to %s" % (repofilepath, relFilePath)) self.makeSafeSymlink(relFilePath, repofilepath) grinder_write_locker.release() LOG.debug("Successfully Fetched Package - [%s]" % filePath) return (vstatus, None) except Exception, e: cleanup(tmp_write_file) cleanup(filePath) if probing: LOG.info("Probed for %s and determined it is missing." % (fetchURL)) grinder_write_locker.release() return BaseFetch.STATUS_ERROR, None tb_info = traceback.format_exc() LOG.error("Caught exception<%s> in fetch(%s, %s)" % (e, fileName, fetchURL)) LOG.error("%s" % (tb_info)) if retryTimes > 0 and not fetchURL.startswith("file:"): retryTimes -= 1 #grinder_write_locker.release() LOG.error("Retrying fetch of: %s with %s retry attempts left." % (fileName, retryTimes)) self.reset_bytes_transferred(fetchURL) return self.fetch(fileName, fetchURL, savePath, itemSize, hashtype, checksum, headers, retryTimes, packages_location) grinder_write_locker.release() raise