def prepare_response(self, grab):
if self.body_file:
self.body_file.close()
response = Response()
response.head = b''.join(self.response_header_chunks)
if self.body_path:
response.body_path = self.body_path
else:
response.body = b''.join(self.response_body_chunks)
# Clear memory
self.response_header_chunks = []
self.response_body_chunks = []
response.code = self.curl.getinfo(pycurl.HTTP_CODE)
response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME)
response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME)
response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME)
response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD)
response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD)
response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD)
response.remote_ip = self.curl.getinfo(pycurl.PRIMARY_IP)
response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL)
response.parse(charset=grab.config['document_charset'])
response.cookies = CookieManager(self.extract_cookiejar())
# We do not need anymore cookies stored in the
# curl instance so drop them
self.curl.setopt(pycurl.COOKIELIST, 'ALL')
return response
def prepare_response(self, grab):
if self.body_file:
self.body_file.close()
response = Response()
response.head = b"".join(self.response_header_chunks)
if self.body_path:
response.body_path = self.body_path
else:
response.body = b"".join(self.response_body_chunks)
# Clear memory
self.response_header_chunks = []
self.response_body_chunks = []
response.code = self.curl.getinfo(pycurl.HTTP_CODE)
response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME)
response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME)
response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME)
response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD)
response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD)
response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD)
response.remote_ip = self.curl.getinfo(pycurl.PRIMARY_IP)
response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL)
response.parse(charset=grab.config["document_charset"])
response.cookies = CookieManager(self.extract_cookiejar())
# We do not need anymore cookies stored in the
# curl instance so drop them
self.curl.setopt(pycurl.COOKIELIST, "ALL")
return response
def prepare_response(self, grab):
response = Response()
try:
response.body = MOCK_REGISTRY[self.request_url]['body']
except KeyError:
raise GrabMockNotFoundError(
'Mock registry does not have information about '\
'following URL: %s' % self.request_url)
now_str = datetime.now().strftime('%a, %d %B %Y %H:%M:%S')
response.head = '\r\n'.join((
'Accept-Ranges:bytes',
'Content-Length:%d' % len(response.body),
'Content-Type:text/plain',
'Date:%s GMT' % now_str,
'Last-Modified:%s GMT' % now_str,
'Vary:Accept-Encoding',
))
response.code = 200
response.total_time = 0
response.name_lookup_time = 0
response.connect_time = 0
response.url = self.request_url
response.parse()
response.cookies = CookieManager(self.extract_cookiejar())
return response
def prepare_response(self, grab):
response = Response()
try:
response.body = MOCK_REGISTRY[self.request_url]['body']
except KeyError:
raise GrabMockNotFoundError(
'Mock registry does not have information about '\
'following URL: %s' % self.request_url)
now_str = datetime.now().strftime('%a, %d %B %Y %H:%M:%S')
response.head = '\r\n'.join((
'Accept-Ranges:bytes',
'Content-Length:%d' % len(response.body),
'Content-Type:text/plain',
'Date:%s GMT' % now_str,
'Last-Modified:%s GMT' % now_str,
'Vary:Accept-Encoding',
))
response.code = 200
response.total_time = 0
response.name_lookup_time = 0
response.connect_time = 0
response.url = self.request_url
response.parse()
response.cookies = CookieManager(self.extract_cookiejar())
return response
def prepare_response(self, grab):
# py3 hack
if PY3K:
self.response_head_chunks = decode_list(self.response_head_chunks)
if self.body_file:
self.body_file.close()
response = Response()
response.head = ''.join(self.response_head_chunks)
if self.body_path:
response.body_path = self.body_path
else:
response.body = b''.join(self.response_body_chunks)
# Clear memory
self.response_head_chunks = []
self.response_body_chunks = []
response.code = self.curl.getinfo(pycurl.HTTP_CODE)
response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME)
response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME)
response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME)
response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD)
response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD)
response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD)
response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL)
if grab.config['document_charset'] is not None:
response.parse(charset=grab.config['document_charset'])
else:
response.parse()
response.cookies = CookieManager(self.extract_cookiejar())
# We do not need anymore cookies stored in the
# curl instance so drop them
self.curl.setopt(pycurl.COOKIELIST, 'ALL')
return response