def __init__(self, url, headers=None, data=None, onSuccess=None, onFailure=None, toFeed=None): super(_downloader, self).__init__() self._url = url if headers is None: self._headers = httpRequest.headers() else: self._headers = httpRequest.headers(headers) self._headers["Accept-Encoding"] = "gzip, deflate, identity" self._headers["Cache-Control"] = "no-cache" self._headers["User-Agent"] = _downloader._userAgent del self._headers["Connection"] del self._headers["if-modified-since"] del self._headers["etag"] del self._headers["Host"] self._data = data self._onSuccess = onSuccess self._onFailure = onFailure self._toFeed = toFeed
def __init__(self, req, fp, code, msg, hs): self._req = req self._fp = fp self._code = code self._msg = msg self._headers = httpRequest.headers(hs) if "Location" in self._headers: self._location = self._headers["Location"] elif "uri" in self._headers: self._location = self._headers["uri"] else: self._location = None
def lookupResponse(key): if _fetchyCache is None: return None data = _fetchyCache.lookup(key) if data is None: cacheInfo('[MISS] Cache lookup for', key, 'found nothing.') else: cacheInfo('[HIT] Cache lookup for', key, 'found match.') headers = httpRequest.headers(data.getPristineData()) body = data.getCompressibleData(waitForGzip=True) if isinstance(body, _gzippedData): headers['Content-Encoding'] = body.getContentEncoding() body = body.getData() data = httpRequest.httpResponse(headers, body) return data
def do_POST(self): self._initRequest() try: headers = httpRequest.headers(self.headers) contentLength = headers['Content-Length'] if contentLength is None: self.send_error(406, 'POST request must include Content-Length header.') return try: contentLength = int(contentLength) except ValueError: self.send_error(406, 'Content-Length header is not an integer.') return data = self.rfile.read(contentLength) request = httpRequest.httpRequest(self.command, self.path, headers, data) response = fetchy.handleRequest(request) self._writeResponse(response) except IOError: pass finally: try: self.finish() except: pass
except _httpRedirectException, e: nextUrl = e.getFinalUrl() if nextUrl in attempted: clientWarn("Infinite redirect detected! Got redirected to", nextUrl, "after attempts", attempted) return self._fail() clientInfo("Got redirect to", nextUrl, "- Current attempts:", attempted) attempted.append(nextUrl) req = urllib2.Request(nextUrl, postData, headersDict) continue except urllib2.HTTPError, e: return self._fail(e.code) except: return self._fail() successful = True contents = "" responseHeaders = httpRequest.headers(handle.info().headers) contentEncoding = responseHeaders["Content-Encoding"] if contentEncoding is not None and "deflate" in contentEncoding: # Zlib mode; can only do it in one shot contents = self._feed(zlib.decompress(handle.read())) elif contentEncoding is not None and "gzip" in contentEncoding: # Gzip mode; can only do it in one shot due to lack of seek()/tell() contents = self._feed(gzip.GzipFile(fileobj=StringIO(handle.read())).read()) else: # Identity mode while True: try: data = handle.read(_downloader._bufferSize) except: break if not data: