def fetch(self, mode, filename, value, read):
"""
Override from base class diskcache.Disk.
Chunking is due to needing to work on pythons < 2.7.13:
- Issue #27130: In the "zlib" module, fix handling of large buffers
(typically 2 or 4 GiB). Previously, inputs were limited to 2 GiB, and
compression and decompression operations did not properly handle results of
2 or 4 GiB.
:param int mode: value mode raw, binary, text, or pickle
:param str filename: filename of corresponding value
:param value: database value
:param bool read: when True, return an open file handle
:return: corresponding Python value
"""
value = super(GzipDisk, self).fetch(mode, filename, value, read)
if mode == MODE_BINARY:
str_io = BytesIO(value)
gz_file = gzip.GzipFile(mode='rb', fileobj=str_io)
read_csio = BytesIO()
while True:
uncompressed_data = gz_file.read(2**30)
if uncompressed_data:
read_csio.write(uncompressed_data)
else:
break
value = read_csio.getvalue()
return value
def store(self, value, read, key=None):
"""
Override from base class diskcache.Disk.
Chunking is due to needing to work on pythons < 2.7.13:
- Issue #27130: In the "zlib" module, fix handling of large buffers
(typically 2 or 4 GiB). Previously, inputs were limited to 2 GiB, and
compression and decompression operations did not properly handle results of
2 or 4 GiB.
:param value: value to convert
:param bool read: True when value is file-like object
:return: (size, mode, filename, value) tuple for Cache table
"""
# pylint: disable=unidiomatic-typecheck
if type(value) is BytesType:
if read:
value = value.read()
read = False
str_io = BytesIO()
gz_file = gzip.GzipFile(mode='wb', compresslevel=1, fileobj=str_io)
for offset in range(0, len(value), 2**30):
gz_file.write(value[offset:offset + 2**30])
gz_file.close()
value = str_io.getvalue()
return super(GzipDisk, self).store(value, read)
def fetch(self, mode, filename, value, read):
value = super(GzipDisk, self).fetch(mode, filename, value, read)
if mode == MODE_BINARY:
str_io = BytesIO(value)
gz_file = gzip.GzipFile(mode='rb', fileobj=str_io)
read_csio = BytesIO()
while True:
uncompressed_data = gz_file.read(2**30)
if uncompressed_data:
read_csio.write(uncompressed_data)
else:
break
value = read_csio.getvalue()
return value
def store(self, value, read, key=None):
if type(value) is BytesType:
if read:
value = value.read()
read = False
str_io = BytesIO()
gz_file = gzip.GzipFile(mode='wb', compresslevel=1, fileobj=str_io)
for offset in range(0, len(value), 2 ** 30):
gz_file.write(value[offset:offset+2**30])
gz_file.close()
value = str_io.getvalue()
return super(GzipDisk, self).store(value, read)