コード例 #1
0
ファイル: curl.py プロジェクト: lyicy/grab
    def prepare_response(self, grab):
        if self.body_file:
            self.body_file.close()
        response = Document()

        response.head = b''.join(self.response_header_chunks)

        if self.body_path:
            response.body_path = self.body_path
        else:
            response.body = b''.join(self.response_body_chunks)

        # Clear memory
        self.response_header_chunks = []
        self.response_body_chunks = []

        response.code = self.curl.getinfo(pycurl.HTTP_CODE)
        response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME)
        response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME)
        response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME)
        response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD)
        response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD)
        response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD)
        response.remote_ip = self.curl.getinfo(pycurl.PRIMARY_IP)

        response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL)

        response.parse(charset=grab.config['document_charset'])

        response.cookies = CookieManager(self.extract_cookiejar())

        # We do not need anymore cookies stored in the
        # curl instance so drop them
        self.curl.setopt(pycurl.COOKIELIST, 'ALL')
        return response
コード例 #2
0
ファイル: curl.py プロジェクト: lorien/grab
    def prepare_response(self, grab):
        if self.body_file:
            self.body_file.close()
        response = Document()

        response.head = b''.join(self.response_header_chunks)

        if self.body_path:
            response.body_path = self.body_path
        else:
            response.body = b''.join(self.response_body_chunks)

        # Clear memory
        self.response_header_chunks = []
        self.response_body_chunks = []

        response.code = self.curl.getinfo(pycurl.HTTP_CODE)
        response.total_time = self.curl.getinfo(pycurl.TOTAL_TIME)
        response.connect_time = self.curl.getinfo(pycurl.CONNECT_TIME)
        response.name_lookup_time = self.curl.getinfo(pycurl.NAMELOOKUP_TIME)
        response.download_size = self.curl.getinfo(pycurl.SIZE_DOWNLOAD)
        response.upload_size = self.curl.getinfo(pycurl.SIZE_UPLOAD)
        response.download_speed = self.curl.getinfo(pycurl.SPEED_DOWNLOAD)
        response.remote_ip = self.curl.getinfo(pycurl.PRIMARY_IP)

        response.url = self.curl.getinfo(pycurl.EFFECTIVE_URL)

        response.parse(charset=grab.config['document_charset'])

        response.cookies = CookieManager(self.extract_cookiejar())

        # We do not need anymore cookies stored in the
        # curl instance so drop them
        self.curl.setopt(pycurl.COOKIELIST, 'ALL')
        return response
コード例 #3
0
ファイル: mysql.py プロジェクト: degustaf/pylint-corpus
 def custom_prepare_response_func(transport, grab):
     doc = Document()
     doc.head = cache_item['head']
     doc.body = body
     doc.code = cache_item['response_code']
     doc.download_size = len(body)
     doc.upload_size = 0
     doc.download_speed = 0
     doc.url = cache_item['response_url']
     doc.parse(charset=grab.config['document_charset'])
     doc.cookies = CookieManager(transport.extract_cookiejar())
     doc.from_cache = True
     return doc
コード例 #4
0
ファイル: postgresql.py プロジェクト: lorien/grab
 def custom_prepare_response_func(transport, grab):
     doc = Document()
     doc.head = cache_item['head']
     doc.body = body
     doc.code = cache_item['response_code']
     doc.download_size = len(body)
     doc.upload_size = 0
     doc.download_speed = 0
     doc.url = cache_item['response_url']
     doc.parse(charset=grab.config['document_charset'])
     doc.cookies = CookieManager(transport.extract_cookiejar())
     doc.from_cache = True
     return doc