def _decompress(filename, options):
"""Decompresses the whole file.
It is useful mainly for testing. Normal gunzip is enough
when uncompressing a file from the beginning.
"""
suffix = options.suffix
if not filename.endswith(suffix) or len(filename) == len(suffix):
logging.warn("without %r suffix -- ignored: %r",
suffix, filename)
return False
target = filename[:-len(suffix)]
input = idzip.open(filename)
logging.info("uncompressing %r to %r", filename, target)
output = open(target, "wb")
while True:
data = input.read(1024)
if not data:
break
output.write(data)
_finish_output(output, options)
input.close()
return True
def openall(
filename,
mode='rt',
encoding=None,
errors=None,
newline=None,
buffering=-1,
closefd=True,
opener=None, # for open()
compresslevel=5, # faster default compression
):
"""
Opens all file types known to the Python SL. There are some differences
from the stock functions:
- the default mode is 'rt'
- the default compresslevel is 5, because e.g. gzip does not benefit a lot
from higher values, only becomes slower.
"""
if filename.endswith('.dz') and idzip:
# Unfortunately idzip's API is not very good
f = idzip.open(filename, mode.replace('t', '').replace('b', '') + 'b')
if 't' in mode:
return io.TextIOWrapper(f,
encoding,
errors,
newline,
write_through=True)
else:
return f
elif filename.endswith('.gz') or filename.endswith('.dz'):
# .dz is .gz, so if we don't have idzip installed, we can still read it
return gzip.open(filename, mode, compresslevel, encoding, errors,
newline)
elif filename.endswith('.bz2'):
return bz2.open(filename, mode, compresslevel, encoding, errors,
newline)
else:
return open(filename, mode, buffering, encoding, errors, newline,
closefd, opener)
def _decompress(filename, options):
"""Decompresses the whole file.
It is useful mainly for testing. Normal gunzip is enough
when uncompressing a file from the beginning.
"""
suffix = options.suffix
if not filename.endswith(suffix) or len(filename) == len(suffix):
logging.warn("without %r suffix -- ignored: %r", suffix, filename)
return False
target = filename[:-len(suffix)]
input = idzip.open(filename)
logging.info("uncompressing %r to %r", filename, target)
output = open(target, "wb")
while True:
data = input.read(1024)
if not data:
break
output.write(data)
_finish_output(output, options)
input.close()
return True
def reader(self):
reader = ProcessedMzMLDeserializer(idzip.open(self.path))
return reader
def _create_dictzip_file(fs, config, content):
fs.create_file(config.dict_path)
with idzip.open(config.dict_path, "wb") as f:
f.write(content)
import idzip
if __name__ == "__main__":
src = "test/data/bigrandom"
dest = "test/data/big_out.txt"
with open(src, 'rb') as infh, idzip.open(dest, 'wb') as outfh:
#chunk_size = idzip.MAX_MEMBER_SIZE
chunk_size = 2 ** 28
chunk = infh.read(chunk_size)
while chunk:
outfh.write(chunk)
chunk = infh.read(chunk_size)