def _fail(self, errorCode=None): clientWarn("Client failed on", self._url, "with error code", errorCode) if self._onFailure is not None: self._onFailure(self, errorCode) if errorCode is None: return None return httpRequest.httpResponse(None, None, responseCode=errorCode)
def lookupResponse(key): if _fetchyCache is None: return None data = _fetchyCache.lookup(key) if data is None: cacheInfo('[MISS] Cache lookup for', key, 'found nothing.') else: cacheInfo('[HIT] Cache lookup for', key, 'found match.') headers = httpRequest.headers(data.getPristineData()) body = data.getCompressibleData(waitForGzip=True) if isinstance(body, _gzippedData): headers['Content-Encoding'] = body.getContentEncoding() body = body.getData() data = httpRequest.httpResponse(headers, body) return data
def processResponse(response): html = _processDocument(_document(response)) return httpRequest.httpResponse(response.getHeaders(), html)
def cacheResource(self, resource, data, headers={}, responseCode=200): if resource in self._resources: self._debug('Resource cached:', resource) response = httpRequest.httpResponse(headers, data, finalUrl=resource, responseCode=responseCode) return fetchy.cacheResponse(cache.generateRequestKey(self._getResourceRequest(resource)), response)
contents = self._feed(zlib.decompress(handle.read())) elif contentEncoding is not None and "gzip" in contentEncoding: # Gzip mode; can only do it in one shot due to lack of seek()/tell() contents = self._feed(gzip.GzipFile(fileobj=StringIO(handle.read())).read()) else: # Identity mode while True: try: data = handle.read(_downloader._bufferSize) except: break if not data: break contents += self._feed(data) result = httpRequest.httpResponse( responseHeaders, contents, responseCode=handle.getcode(), finalUrl=handle.geturl() ) if self._onSuccess is not None: try: self._onSuccess(result) except Exception, e: pass clientInfo("Client successful for", self._url) return result def init(verbose, userAgent, bufferSize, timeout): clientVerbose(verbose) _downloader._userAgent = userAgent _downloader._bufferSize = bufferSize _downloader._timeout = timeout