def prepare_response(self, grab): if self.body_file: self.body_file.close() response = Response() response.head = b''.join(self.response_header_chunks) if self.body_path: response.body_path = self.body_path else: response.body = b''.join(self.response_body_chunks) # Clear memory self.response_header_chunks = [] self.response_body_chunks = [] response.code = self.curl.getinfo(pycurl.HTTP_CODE) response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME) response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME) response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME) response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD) response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD) response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD) response.remote_ip = self.curl.getinfo(pycurl.PRIMARY_IP) response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL) response.parse(charset=grab.config['document_charset']) response.cookies = CookieManager(self.extract_cookiejar()) # We do not need anymore cookies stored in the # curl instance so drop them self.curl.setopt(pycurl.COOKIELIST, 'ALL') return response
def prepare_response(self, grab): if self.body_file: self.body_file.close() response = Response() response.head = b"".join(self.response_header_chunks) if self.body_path: response.body_path = self.body_path else: response.body = b"".join(self.response_body_chunks) # Clear memory self.response_header_chunks = [] self.response_body_chunks = [] response.code = self.curl.getinfo(pycurl.HTTP_CODE) response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME) response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME) response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME) response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD) response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD) response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD) response.remote_ip = self.curl.getinfo(pycurl.PRIMARY_IP) response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL) response.parse(charset=grab.config["document_charset"]) response.cookies = CookieManager(self.extract_cookiejar()) # We do not need anymore cookies stored in the # curl instance so drop them self.curl.setopt(pycurl.COOKIELIST, "ALL") return response
def custom_prepare_response_func(transport, g): response = Response() response.head = cache_item['head'] response.body = body response.code = cache_item['response_code'] response.download_size = len(body) response.upload_size = 0 response.download_speed = 0 response.url = cache_item['response_url'] response.parse() response.cookies = CookieManager(transport.extract_cookiejar()) return response
def custom_prepare_response_func(transport, grab): response = Response() response.head = cache_item['head'] response.body = body response.code = cache_item['response_code'] response.download_size = len(body) response.upload_size = 0 response.download_speed = 0 response.url = cache_item['response_url'] response.parse(charset=grab.config['document_charset']) response.cookies = CookieManager(transport.extract_cookiejar()) response.from_cache = True return response
def custom_prepare_response_func(transport, grab): response = Response() response.head = cache_item['head'].decode('utf-8') response.body = body response.code = cache_item['response_code'] response.download_size = len(body) response.upload_size = 0 response.download_speed = 0 response.url = cache_item['response_url'] response.parse(charset=grab.config['document_charset']) response.cookies = CookieManager(transport.extract_cookiejar()) response.from_cache = True return response
def custom_prepare_response_func(transport, g): response = Response() response.head = cache_item['head'] response.body = body response.code = cache_item['response_code'] response.download_size = len(body) response.upload_size = 0 response.download_speed = 0 # Hack for deprecated behaviour if 'response_url' in cache_item: response.url = cache_item['response_url'] else: logger.debug('You cache contains items without `response_url` key. It is depricated data format. Please re-download you cache or build manually `response_url` keys.') response.url = cache_item['url'] response.parse() response.cookies = CookieManager(transport.extract_cookiejar()) return response
def prepare_response(self, grab): # py3 hack if PY3K: self.response_head_chunks = decode_list(self.response_head_chunks) if self.body_file: self.body_file.close() response = Response() response.head = ''.join(self.response_head_chunks) if self.body_path: response.body_path = self.body_path else: response.body = b''.join(self.response_body_chunks) # Clear memory self.response_head_chunks = [] self.response_body_chunks = [] response.code = self.curl.getinfo(pycurl.HTTP_CODE) response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME) response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME) response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME) response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD) response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD) response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD) response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL) if grab.config['document_charset'] is not None: response.parse(charset=grab.config['document_charset']) else: response.parse() response.cookies = CookieManager(self.extract_cookiejar()) # We do not need anymore cookies stored in the # curl instance so drop them self.curl.setopt(pycurl.COOKIELIST, 'ALL') return response