def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key=key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return convert.utf82unicode(source.read()).split("\n")
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return LazyLines(source)
def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key= key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return convert.utf82unicode(source.read()).split("\n")
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return LazyLines(source)
def _all_lines(self, encoding="utf8"):
try:
iterator = self.raw.stream(4096, decode_content=False)
if self.headers.get('content-encoding') == 'gzip':
return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding)
elif self.headers.get('content-type') == 'application/zip':
return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding)
elif self.url.endswith(".gz"):
return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding)
else:
return ibytes2ilines(iterator, encoding=encoding, closer=self.close)
except Exception, e:
Log.error("Can not read content", cause=e)
def get_all_lines(self, encoding="utf8", flexible=False):
try:
iterator = self.raw.stream(4096, decode_content=False)
if self.headers.get('content-encoding') == 'gzip':
return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible)
elif self.headers.get('content-type') == 'application/zip':
return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible)
elif self.url.endswith(".gz"):
return ibytes2ilines(icompressed2ibytes(iterator), encoding=encoding, flexible=flexible)
else:
return ibytes2ilines(iterator, encoding=encoding, flexible=flexible, closer=self.close)
except Exception as e:
Log.error("Can not read content", cause=e)
