def prepare_response(self, grab): if self.body_file: self.body_file.close() response = Document() response.head = b''.join(self.response_header_chunks) if self.body_path: response.body_path = self.body_path else: response.body = b''.join(self.response_body_chunks) # Clear memory self.response_header_chunks = [] self.response_body_chunks = [] response.code = self.curl.getinfo(pycurl.HTTP_CODE) response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME) response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME) response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME) response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD) response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD) response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD) response.remote_ip = self.curl.getinfo(pycurl.PRIMARY_IP) response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL) response.parse(charset=grab.config['document_charset']) response.cookies = CookieManager(self.extract_cookiejar()) # We do not need anymore cookies stored in the # curl instance so drop them self.curl.setopt(pycurl.COOKIELIST, 'ALL') return response
def setup_document(self, content, **kwargs): """ Setup `response` object without real network requests. Useful for testing and debuging. All ``**kwargs`` will be passed to `Document` constructor. """ self.reset() if isinstance(content, six.text_type): raise error.GrabMisuseError('Method `setup_document` accepts only ' 'byte string in `content` argument.') # Configure Document instance doc = Document(grab=self) doc.body = content doc.status = '' doc.head = b'HTTP/1.1 200 OK\r\n\r\n' doc.parse(charset=kwargs.get('document_charset')) doc.code = 200 doc.total_time = 0 doc.connect_time = 0 doc.name_lookup_time = 0 doc.url = '' for key, value in kwargs.items(): setattr(doc, key, value) self.doc = doc
def setup_document(self, content, **kwargs): """ Setup `response` object without real network requests. Useful for testing and debuging. All ``**kwargs`` will be passed to `Document` constructor. """ self.reset() # Configure Document instance doc = Document(grab=self) doc.body = content doc.status = '' doc.head = '' doc.parse(charset=kwargs.get('document_charset')) doc.code = 200 doc.total_time = 0 doc.connect_time = 0 doc.name_lookup_time = 0 doc.url = '' for key, value in kwargs.items(): setattr(doc, key, value) self.doc = doc