def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception, e: Log.error(READ_ERROR, e)
def zip2bytes(compressed): """ UNZIP DATA """ if hasattr(compressed, "read"): return gzip.GzipFile(fileobj=compressed, mode='r') buff = BytesIO(compressed) archive = gzip.GzipFile(fileobj=buff, mode='r') return safe_size(archive)
def zip2bytes(compressed): """ UNZIP DATA """ if hasattr(compressed, "read"): return gzip.GzipFile(fileobj=compressed, mode='r') buff = BytesIO(compressed) archive = gzip.GzipFile(fileobj=buff, mode='r') from pyLibrary.env.big_data import safe_size return safe_size(archive)
def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception as e: Log.error(READ_ERROR, e) if json == None: return None if source.key.endswith(".zip"): json = _unzip(json) elif source.key.endswith(".gz"): json = convert.zip2bytes(json) return json.decode('utf8')
def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception as e: Log.error(READ_ERROR, e) if json == None: return None if source.key.endswith(".zip"): json = _unzip(json) elif source.key.endswith(".gz"): json = convert.zip2bytes(json) return utf82unicode(json)
def all_content(self): # Response.content WILL LEAK MEMORY (?BECAUSE OF PYPY"S POOR HANDLING OF GENERATORS?) # THE TIGHT, SIMPLE, LOOP TO FILL blocks PREVENTS THAT LEAK if self._cached_content is None: def read(size): if self.raw._fp.fp is not None: return self.raw.read(amt=size, decode_content=True) else: self.close() return None self._cached_content = safe_size(Dict(read=read)) if hasattr(self._cached_content, "read"): self._cached_content.seek(0) return self._cached_content
def all_content(self): # response.content WILL LEAK MEMORY (?BECAUSE OF PYPY"S POOR HANDLING OF GENERATORS?) # THE TIGHT, SIMPLE, LOOP TO FILL blocks PREVENTS THAT LEAK if self._content is not False: self._cached_content = self._content elif self._cached_content is None: def read(size): if self.raw._fp.fp is not None: return self.raw.read(amt=size, decode_content=True) else: self.close() return None self._cached_content = safe_size(Data(read=read)) if hasattr(self._cached_content, "read"): self._cached_content.seek(0) return self._cached_content
def read_lines(self, key): source = self.get_meta(key) if source is None: Log.error("{{key}} does not exist", key= key) if source.size < MAX_STRING_SIZE: if source.key.endswith(".gz"): return GzipLines(source.read()) else: return convert.utf82unicode(source.read()).split("\n") if source.key.endswith(".gz"): bytes = safe_size(source) if isinstance(bytes, str): buff = BytesIO(bytes) else: # SWAP OUT FILE REFERENCE bytes.file, buff = None, bytes.file archive = gzip.GzipFile(fileobj=buff, mode='r') return LazyLines(archive) else: return LazyLines(source)
def read_lines(self, key): source = self.get_meta(key) if source is None: Log.error("{{key}} does not exist", key=key) if source.size < MAX_STRING_SIZE: if source.key.endswith(".gz"): return GzipLines(source.read()) else: return convert.utf82unicode(source.read()).split("\n") if source.key.endswith(".gz"): bytes = safe_size(source) if isinstance(bytes, str): buff = BytesIO(bytes) else: # SWAP OUT FILE REFERENCE bytes.file, buff = None, bytes.file archive = gzip.GzipFile(fileobj=buff, mode='r') return LazyLines(archive) else: return LazyLines(source)
def read_bytes(self, key): source = self.get_meta(key) return safe_size(source)