def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception, e:
Log.error(READ_ERROR, e)
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception, e:
Log.error(READ_ERROR, e)
def zip2bytes(compressed):
"""
UNZIP DATA
"""
if hasattr(compressed, "read"):
return gzip.GzipFile(fileobj=compressed, mode='r')
buff = BytesIO(compressed)
archive = gzip.GzipFile(fileobj=buff, mode='r')
return safe_size(archive)
def zip2bytes(compressed):
"""
UNZIP DATA
"""
if hasattr(compressed, "read"):
return gzip.GzipFile(fileobj=compressed, mode='r')
buff = BytesIO(compressed)
archive = gzip.GzipFile(fileobj=buff, mode='r')
from pyLibrary.env.big_data import safe_size
return safe_size(archive)
def zip2bytes(compressed):
"""
UNZIP DATA
"""
if hasattr(compressed, "read"):
return gzip.GzipFile(fileobj=compressed, mode='r')
buff = BytesIO(compressed)
archive = gzip.GzipFile(fileobj=buff, mode='r')
from pyLibrary.env.big_data import safe_size
return safe_size(archive)
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception as e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return json.decode('utf8')
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception as e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return utf82unicode(json)
def all_content(self):
# Response.content WILL LEAK MEMORY (?BECAUSE OF PYPY"S POOR HANDLING OF GENERATORS?)
# THE TIGHT, SIMPLE, LOOP TO FILL blocks PREVENTS THAT LEAK
if self._cached_content is None:
def read(size):
if self.raw._fp.fp is not None:
return self.raw.read(amt=size, decode_content=True)
else:
self.close()
return None
self._cached_content = safe_size(Dict(read=read))
if hasattr(self._cached_content, "read"):
self._cached_content.seek(0)
return self._cached_content
def all_content(self):
# response.content WILL LEAK MEMORY (?BECAUSE OF PYPY"S POOR HANDLING OF GENERATORS?)
# THE TIGHT, SIMPLE, LOOP TO FILL blocks PREVENTS THAT LEAK
if self._content is not False:
self._cached_content = self._content
elif self._cached_content is None:
def read(size):
if self.raw._fp.fp is not None:
return self.raw.read(amt=size, decode_content=True)
else:
self.close()
return None
self._cached_content = safe_size(Data(read=read))
if hasattr(self._cached_content, "read"):
self._cached_content.seek(0)
return self._cached_content
def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key= key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return GzipLines(source.read())
else:
return convert.utf82unicode(source.read()).split("\n")
if source.key.endswith(".gz"):
bytes = safe_size(source)
if isinstance(bytes, str):
buff = BytesIO(bytes)
else:
# SWAP OUT FILE REFERENCE
bytes.file, buff = None, bytes.file
archive = gzip.GzipFile(fileobj=buff, mode='r')
return LazyLines(archive)
else:
return LazyLines(source)
def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key=key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return GzipLines(source.read())
else:
return convert.utf82unicode(source.read()).split("\n")
if source.key.endswith(".gz"):
bytes = safe_size(source)
if isinstance(bytes, str):
buff = BytesIO(bytes)
else:
# SWAP OUT FILE REFERENCE
bytes.file, buff = None, bytes.file
archive = gzip.GzipFile(fileobj=buff, mode='r')
return LazyLines(archive)
else:
return LazyLines(source)
def read_bytes(self, key):
source = self.get_meta(key)
return safe_size(source)
def read_bytes(self, key):
source = self.get_meta(key)
return safe_size(source)
