def _fail(self, errorCode=None):
clientWarn("Client failed on", self._url, "with error code", errorCode)
if self._onFailure is not None:
self._onFailure(self, errorCode)
if errorCode is None:
return None
return httpRequest.httpResponse(None, None, responseCode=errorCode)
def lookupResponse(key):
if _fetchyCache is None:
return None
data = _fetchyCache.lookup(key)
if data is None:
cacheInfo('[MISS] Cache lookup for', key, 'found nothing.')
else:
cacheInfo('[HIT] Cache lookup for', key, 'found match.')
headers = httpRequest.headers(data.getPristineData())
body = data.getCompressibleData(waitForGzip=True)
if isinstance(body, _gzippedData):
headers['Content-Encoding'] = body.getContentEncoding()
body = body.getData()
data = httpRequest.httpResponse(headers, body)
return data
def processResponse(response): html = _processDocument(_document(response)) return httpRequest.httpResponse(response.getHeaders(), html)
def cacheResource(self, resource, data, headers={}, responseCode=200):
if resource in self._resources:
self._debug('Resource cached:', resource)
response = httpRequest.httpResponse(headers, data, finalUrl=resource, responseCode=responseCode)
return fetchy.cacheResponse(cache.generateRequestKey(self._getResourceRequest(resource)), response)
contents = self._feed(zlib.decompress(handle.read()))
elif contentEncoding is not None and "gzip" in contentEncoding:
# Gzip mode; can only do it in one shot due to lack of seek()/tell()
contents = self._feed(gzip.GzipFile(fileobj=StringIO(handle.read())).read())
else:
# Identity mode
while True:
try:
data = handle.read(_downloader._bufferSize)
except:
break
if not data:
break
contents += self._feed(data)
result = httpRequest.httpResponse(
responseHeaders, contents, responseCode=handle.getcode(), finalUrl=handle.geturl()
)
if self._onSuccess is not None:
try:
self._onSuccess(result)
except Exception, e:
pass
clientInfo("Client successful for", self._url)
return result
def init(verbose, userAgent, bufferSize, timeout):
clientVerbose(verbose)
_downloader._userAgent = userAgent
_downloader._bufferSize = bufferSize
_downloader._timeout = timeout
