Exemplo n.º 1
0
    def _all_lines(self, encoding="utf8"):
        try:
            iterator = self.raw.stream(4096, decode_content=False)

            if self.headers.get('content-encoding') == 'gzip':
                return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding)
            elif self.headers.get('content-type') == 'application/zip':
                return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding)
            elif self.url.endswith(".gz"):
                return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding)
            else:
                return ibytes2ilines(iterator, encoding=encoding, closer=self.close)
        except Exception, e:
            Log.error("Can not read content", cause=e)
Exemplo n.º 2
0
    def get_all_lines(self, encoding="utf8", flexible=False):
        try:
            iterator = self.raw.stream(4096, decode_content=False)

            if self.headers.get('content-encoding') == 'gzip':
                return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible)
            elif self.headers.get('content-type') == 'application/zip':
                return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible)
            elif self.url.endswith(".gz"):
                return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible)
            else:
                return ibytes2ilines(iterator, encoding=encoding, flexible=flexible, closer=self.close)
        except Exception as e:
            Log.error("Can not read content", cause=e)