def read_lines(self, key): source = self.get_meta(key) if source is None: Log.error("{{key}} does not exist", key=key) if source.size < MAX_STRING_SIZE: if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else: return convert.utf82unicode(source.read()).split("\n") if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else: return LazyLines(source)
def read_lines(self, key): source = self.get_meta(key) if source is None: Log.error("{{key}} does not exist", key= key) if source.size < MAX_STRING_SIZE: if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else: return convert.utf82unicode(source.read()).split("\n") if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else: return LazyLines(source)
def _all_lines(self, encoding="utf8"): try: iterator = self.raw.stream(4096, decode_content=False) if self.headers.get('content-encoding') == 'gzip': return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding) elif self.headers.get('content-type') == 'application/zip': return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding) elif self.url.endswith(".gz"): return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding) else: return ibytes2ilines(iterator, encoding=encoding, closer=self.close) except Exception, e: Log.error("Can not read content", cause=e)
def get_all_lines(self, encoding="utf8", flexible=False): try: iterator = self.raw.stream(4096, decode_content=False) if self.headers.get('content-encoding') == 'gzip': return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible) elif self.headers.get('content-type') == 'application/zip': return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible) elif self.url.endswith(".gz"): return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible) else: return ibytes2ilines(iterator, encoding=encoding, flexible=flexible, closer=self.close) except Exception as e: Log.error("Can not read content", cause=e)