def startCompressMessage(self):
if self._isServer:
if self._compressor is None:
self._compressor = bz2.BZ2Compressor(self.server_max_compress_level)
else:
if self._compressor is None:
self._compressor = bz2.BZ2Compressor(self.client_max_compress_level)
def _write(self, raw_data, filename) :
"""
Write raw data to a compressed file.
@arg raw_data: The raw_data to be compressed and written
@type raw_data: byte string
@arg filename: The intended name of the outfile
@type filename: unicode
@return: outfile ; The full path and name of the file written
@rtype: unicode
"""
result = chardet.detect(raw_data)
if result['confidence'] > 0.5:
encoding = result['encoding']
else:
encoding = 'utf-8'
if not util.is_utf8_alias(encoding):
raw_data = raw_data.decode(encoding).encode('utf-8')
# Compress the data to save disk space.
comp = bz2.BZ2Compressor()
data = comp.compress(raw_data)
data += comp.flush()
out_handle = open(self._nametofile(filename), "wb")
out_handle.write(data)
out_handle.close()
return out_handle.name # return the full path to the file
def handle(self):
compressor = bz2.BZ2Compressor()
# Find out what file the client wants
filename = self.request.recv(1024)
self.logger.debug('client asked for: "%s"', filename)
# Send chunks of the file as they are compressed
with open(filename, 'rb') as input:
while True:
block = input.read(BLOCK_SIZE)
if not block:
break
self.logger.debug('RAW "%s"', block)
compressed = compressor.compress(block)
if compressed:
self.logger.debug('SENDING "%s"',
binascii.hexlify(compressed))
self.request.send(compressed)
else:
self.logger.debug('BUFFERING')
# Send any data being buffered by the compressor
remaining = compressor.flush()
while remaining:
to_send = remaining[:BLOCK_SIZE]
remaining = remaining[BLOCK_SIZE:]
self.logger.debug('FLUSHING "%s"', binascii.hexlify(to_send))
self.request.send(to_send)
return
def get(cls):
if support['lzma']:
return b'LZ00', lzma.LZMACompressor()
elif support['bz2']:
return b'BZ00', bz2.BZ2Compressor()
else:
return b'ZL00', zlib.compressobj()
def handle(self):
compressor = bz2.BZ2Compressor()
# Cosa vuole il client?
filename = self.request.recv(1024).decode('utf-8')
self.logger.debug('il client richiede: "%s"', filename)
# Invio di pezzi del file mentre si stanno comprimento
with open(filename, 'rb') as input:
while True:
block = input.read(BLOCK_SIZE)
if not block:
break
self.logger.debug('GREZZI %r', block)
compressed = compressor.compress(block)
if compressed:
self.logger.debug('IN INVIO %r',
binascii.hexlify(compressed))
self.request.send(compressed)
else:
self.logger.debug('ACCUMULO')
# Invia tutti i dati accumulati al compressore
remaining = compressor.flush()
while remaining:
to_send = remaining[:BLOCK_SIZE]
remaining = remaining[BLOCK_SIZE:]
self.logger.debug('SCARICAMENTO %r', binascii.hexlify(to_send))
self.request.send(to_send)
return
def preprocess_data(path: str,
output: Optional[str] = None,
*,
delete_original: bool = True) -> str:
"""Extracts moves from file on given path using :func extract_games: and saves it to compressed file
in .bz2 format to original location or ``output`` if provided
:param path: path to a file
:param output: optional path to save location
:param delete_original: wheather to delete original uncompressed file, default `True`
:return: path to compressed file
"""
compressor = bz2.BZ2Compressor()
output = output if output else os.path.join(
os.path.dirname(path), 'proc_' + os.path.basename(path))
# modifying path is necessary, because otherwise we read and write to the same file
if not output.endswith('.bz2'):
output += '.bz2'
with open(output, 'wb') as f:
for game in extract_games(
path): # extract and compress games one by one
game_str = bytes(' '.join(game) + '\n', encoding='utf-8')
data = compressor.compress(game_str)
f.write(data)
f.write(compressor.flush())
if delete_original:
os.remove(path)
return output
def _write_data(data, filehandle, options):
# Now write data directly
rawdata = data.tostring(order='F')
if options['encoding'] == 'raw':
filehandle.write(rawdata)
elif options['encoding'].lower() in ['ascii', 'text', 'txt']:
# savetxt only works for 1D and 2D arrays, so reshape any > 2 dim arrays into one long 1D array
if data.ndim > 2:
np.savetxt(filehandle, data.ravel(order='F'), '%.17g')
else:
np.savetxt(filehandle, data.T, '%.17g')
else:
if options['encoding'] == 'gzip':
comp_obj = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)
elif options['encoding'] == 'bzip2':
comp_obj = bz2.BZ2Compressor()
else:
raise NrrdError('Unsupported encoding: "%s"' % options['encoding'])
# write data in chunks
start_index = 0
while start_index < len(rawdata):
end_index = start_index + _WRITE_CHUNKSIZE
if end_index > len(rawdata):
end_index = len(rawdata)
filehandle.write(comp_obj.compress(rawdata[start_index:end_index]))
start_index = end_index
filehandle.write(comp_obj.flush())
filehandle.flush()
def _initialize_compressor(self):
if self._compression_type == CompressionTypes.BZIP2:
self._compressor = bz2.BZ2Compressor()
else:
assert self._compression_type == CompressionTypes.GZIP
self._compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, self._gzip_mask)
def _get_encoder(compression):
if compression == 'zlib':
try:
import zlib
except ImportError:
raise ImportError("Your Python does not have the zlib library, "
"therefore the block in this ASDF file "
"can not be compressed.")
return zlib.compressobj()
elif compression == 'bzp2':
try:
import bz2
except ImportError:
raise ImportError("Your Python does not have the bz2 library, "
"therefore the block in this ASDF file "
"can not be compressed.")
return bz2.BZ2Compressor()
elif compression == 'lz4':
try:
import lz4.block
except ImportError:
raise ImportError(
"lz4 library in not installed in your Python environment, "
"therefore the block in this ASDF file "
"can not be compressed.")
return Lz4Compressor(lz4.block)
else:
raise ValueError("Unknown compression type: '{0}'".format(compression))
def bz2_demo():
# 单次压缩
bytes_com = bz2.compress(content)
print("单次压缩: ", bytes_com)
bytes_decom = bz2.decompress(bytes_com)
print("单次解压: ", bytes_decom)
# 增量压缩
bzcom = bz2.BZ2Compressor()
bzdecom = bz2.BZ2Decompressor()
bytes_com = bzcom.compress(content)
bytes_com += bzcom.flush()
print("增量压缩: ", bytes_com)
bytes_decom = bzdecom.decompress(bytes_com)
print("增量解压: ", bytes_decom)
# 读写压缩
with open('file.txt', 'rb') as read, bz2.open('file.txt.gz',
'wb') as write:
shutil.copyfileobj(read, write)
with bz2.open('file.txt.gz', 'rb') as read, open('temp.txt',
'wb') as write:
shutil.copyfileobj(read, write)
def format(self, tokensource, outfile):
if self.compress == 'gz':
import gzip
outfile = gzip.GzipFile('', 'wb', 9, outfile)
write = outfile.write
flush = outfile.flush
elif self.compress == 'bz2':
import bz2
compressor = bz2.BZ2Compressor(9)
def write(text):
outfile.write(compressor.compress(text))
def flush():
outfile.write(compressor.flush())
outfile.flush()
else:
write = outfile.write
flush = outfile.flush
lasttype = None
lastval = u''
if self.error_color:
for ttype, value in tokensource:
line = "%s\t%r\n" % (ttype, value)
if ttype is Token.Error:
write(colorize(self.error_color, line))
else:
write(line)
else:
for ttype, value in tokensource:
write("%s\t%r\n" % (ttype, value))
flush()
def compress(method, data_in):
'''
Compress data in one of a set of methods.
'''
data = None
if method == 'gzip':
import gzip
data = gzip.compress(data_in)
elif method == 'bz2':
import bz2
c = bz2.BZ2Compressor()
data = c.compress(data_in)
data += c.flush()
elif method == 'xz':
import lzma
c = lzma.LZMACompressor(format=lzma.FORMAT_XZ)
data = c.compress(data_in)
data += c.flush()
elif method == 'none':
data = data_in
else:
raise Exception('Invalid compression method')
return data
def bz2_func():
'''
使用bzip2压缩算法压缩和解压
该模块的所有类都是线程安全的
读取和写入压缩文件: open() BZ2File
增量压缩: BZ2Compressor BZ2Decopressor
单次压缩: compress() decompress()
'''
# 读写压缩文件
# 二进制或文本模式打开bzip2压缩文件
# bz2.open(filename, mode='r', compresslevel=9, encoding=None, errors=None, newline=None)
file = bz2.open("box.bz2")
# 单次压缩
# bz2.compress(data, compresslevel=9)
bytes_com = bz2.compress(b'luzhuo.me')
# bz2.decompress(data)
bytes_decom = bz2.decompress(bytes_com)
# 增量压缩
# class bz2.BZ2Compressor(compresslevel=9)
bzcom = bz2.BZ2Compressor()
bytes_com = bzcom.compress(b'luzhuo.me') # 压缩
bytes_com = bzcom.flush() # 完成压缩, 返回剩余的压缩数据
# 增量压缩的解压缩
# class bz2.BZ2Decompressor
bzdecom = bz2.BZ2Decompressor()
# decompress(data, max_length=-1) // 解压
bytes_decom = bzdecom.decompress(bytes_com)
boolean = bzdecom.eof # 是否到达流结束标记
def handle(self):
compressor = bz2.BZ2Compressor()
# Find out what the client wants
filename = self.request.recv(1024)
self.logger.debug("client asked for '%s'", filename)
# Send chunks of the file as they are compressed
with open(filename, 'rb') as input:
while True:
block = input.read(BLOCK_SIZE)
if not block:
break
self.logger.debug("RAW '%s'", block)
compressed = compressor.compress(block)
if compressed:
self.logger.debug("SENDING '%s'",
binascii.hexlify(compressed))
s.request.send(compressed)
else:
self.logger.debug("Buffering...")
# Send any data left in the buffer
remaining = compressor.flush()
while remaining:
to_send = remaining[:BLOCK_SIZE]
remaining = remaining[BLOCK_SIZE:]
self.logger.debug("FLUSHING '%s'", to_send)
self.request.send(to_send)
return
def generate_and_write(self):
prev_point = None
i = 0
if self.strm == "cfile":
self.compressor = bz2.BZ2Compressor()
if self.output_format == "gjson":
self.write_out('{"type": "FeatureCollection", "features": [')
if self.output_format == "wkt":
self.write_out('MULTIPOINT (')
while i < self.card:
point = self.generate_point(i, prev_point)
if self.is_valid_point(point):
prev_point = point
data = prev_point.to_string(self.output_format) + '\n'
self.write_out(data)
i = i + 1
if self.output_format == "gjson" and i < self.card:
self.write_out(',')
if self.output_format == "wkt" and i < self.card:
self.write_out(',')
if self.output_format == "gjson":
self.write_out(']}')
if self.output_format == "wkt":
self.write_out(')')
if self.strm == "cfile":
data = self.compressor.flush(
) # Get the last bits of data remaining in the compressor
sys.stdout.buffer.write(data)
def cereal(self):
bz2.BZ2Compressor(compresslevel=[1-9])
commands.getstatusoutput(1,null)
commands.getstatus(0,null)
#bz2.BZ2File(Medussa[mode=self.terminating, compress = true, compresslevel=[1-9]])
def user(self):
automota.self(classmethod)
world("\t\t\z\y %z /x5z/").author(serialize(self(setattr)))
while world < destroyed:
#create argus Cluster
Paradox.resolved == true
#end
def create(self):
HTMLParser
def solid(self):
#crypt.crypt.__doc__
def state(self):
ConfigParser
def cluster(self):
ColorPicker("0x1239879")
commands
#continue
def paradox(self):
#crypt.crypt.__doc__
def select(self):
#crypt.crypt.__doc__
def anomaly(self):
crypt.crypt(self)
def space(self):
#crypt.crypt.__doc__
def dimension(self):#buildtools,afxres,ConfigParser,Cookie
print "World"
def __init__(self, *args, **kwds):
if not hasattr(self, "compresslevel"):
self.compresslevel = 9
# Compression function with flush and reset.
c = [bz2.BZ2Compressor()]
def compress(data):
if data == "":
return ""
return c[0].compress(data)
def c_flush():
return c[0].flush()
def c_reset():
c[0] = bz2.BZ2Compressor()
compress.flush = c_flush
compress.reset = c_reset
self.compress = compress
# Decompression funtion with reset
d = [bz2.BZ2Decompressor()]
def decompress(data):
if data == "":
return ""
return d[0].decompress(data)
def d_reset():
d[0] = bz2.BZ2Decompressor()
decompress.reset = d_reset
self.decompress = decompress
# These can now be used by superclass constructors
super(BZip2Mixin, self).__init__(*args, **kwds)
def wrapper(*args, **kwargs):
result_iter = f(*args, **kwargs)
assert isinstance(result_iter, GeneratorType), repr(result_iter)
compressor = bz2.BZ2Compressor(BZ2_COMPRESSLEVEL)
size_unpacked = 0
size_packed = 0
# Compress the input data in an iterative way.
for chunk in result_iter:
size_unpacked += len(chunk)
compressed = compressor.compress(chunk)
size_packed += len(compressed)
if compressed:
yield compressed
# Finalize
compressed = compressor.flush()
size_packed += len(compressed)
if compressed:
yield compressed
logger_bzip2.debug(
'Compressed %i/%i (%.2f%%)', size_packed, size_unpacked, 100.0 *
size_packed / size_unpacked if size_unpacked else float('inf'))
def bz2_nnrd(img_list, outfile, scan_name, update, center=''):
"""
Given a list of 2D image paths create nrrd in a temp file and then write this to a bz2 compressed nrrd.
"""
reader = Imreader(img_list)
first_image = reader.imread(img_list[0])
shape = list(first_image.shape)
shape = [shape[1], shape[0]]
if center.lower() == 'tcp':
# The X and Y are determined from the dimensions of a Z slice
# TCP data is on it's side so we need to reverese these shape dimensions
shape.reverse()
shape.append(len(img_list))
compressor = bz2.BZ2Compressor()
compressed_name = outfile + '.bz2'
with open(compressed_name, 'wb') as fh_w:
header_str = nrrd_header_from_dict(
shape,
first_image.dtype,
3,
header=orientations.RAS_HEADER_OPTIONS)
fh_w.write(compressor.compress(header_str))
print(center)
for i, f in enumerate(img_list):
if i % 20 == 0:
done = int((100.0 / len(img_list)) * i)
update.emit('{} {}%'.format(scan_name, done))
img_arr = reader.imread(f)
if center.lower() == 'tcp':
img_arr = np.rot90(img_arr, k=3)
img_arr = np.fliplr(img_arr)
elif center.lower() == 'ucd':
img_arr = np.rot90(img_arr, k=2)
elif center.lower() == 'ccp':
pass # No flip needed for compressor
else: # Get the rest ofthe data into RAS format
img_arr = np.fliplr(img_arr)
rawdata = img_arr.T.tostring(order='F')
compressed = compressor.compress(rawdata)
fh_w.write(compressed)
# Send any data being buffered by the compressor
remaining = compressor.flush()
if remaining:
fh_w.write(remaining)
def transfer(self):
engine = bz2.BZ2Compressor()
while 1:
data = self.read()
if not data:
break
self.write(engine.compress(data))
self.write(engine.flush())
def __init__(self, stream, compression_level=9):
assert isinstance(compression_level, int)
assert 1 <= compression_level <= 9
self.stream = stream
self.EOF = False
self.compressor = bz2.BZ2Compressor(compression_level)
self.buffer = b'' # TODO: bytearray would be more efficient?
self.pos = 0
def test_smart_open_for_bz2_file():
fd, path = tempfile.mkstemp()
comp = bz2.BZ2Compressor()
with os.fdopen(fd, 'wb') as f:
f.write(comp.compress('abcdef'.encode()))
f.write(comp.flush())
assert not isinstance(dumper.smart_open(path), bz2.BZ2File)
os.unlink(path)
def bz2_nnrd(img_list, outfile, scan_name, update):
"""
"""
reader = Imreader(img_list)
first_image = reader.imread(img_list[0])
shape = list(first_image.shape)
shape = [shape[1], shape[0]]
shape.append(len(img_list))
print '++++==== bzp'
tempnrrd = tempfile.TemporaryFile(mode="wb+")
nrrd.write_nrrd_header(tempnrrd, shape, first_image.dtype, 3)
compressor = bz2.BZ2Compressor()
for i, f in enumerate(img_list):
if i % 20 == 0:
done = int((50.0 / len(img_list)) * i)
update.emit('{} {}%'.format(scan_name, done))
img_arr = reader.imread(f)
rawdata = img_arr.T.tostring(order='F')
tempnrrd.write(rawdata)
BLOCK_SIZE = 52428800 # Around 20 MB in memory at one time
# TODO: Check its smaller than image size
compressed_name = outfile + '.bz2'
file_size = os.fstat(tempnrrd.fileno()).st_size
tempnrrd.seek(0)
bytes_read = 0
with open(compressed_name, 'wb') as fh_w:
while True:
block = tempnrrd.read(BLOCK_SIZE)
bytes_read += BLOCK_SIZE
done = int(50 + (50.0 / file_size) * bytes_read)
if done >= 100: # The getsize might not be accurate?
done = 99
update.emit('{} {}%'.format(scan_name, done))
if not block:
break
compressed = compressor.compress(block)
if compressed:
try:
fh_w.write(compressed)
except IOError:
update.emit("Error in compression - job terminated")
print('failed to write bzp chunk')
return
# Send any data being buffered by the compressor
remaining = compressor.flush()
while remaining:
to_send = remaining[:BLOCK_SIZE]
remaining = remaining[BLOCK_SIZE:]
fh_w.write(to_send)
def compressstream(self, it, opts=None):
opts = opts or {}
z = bz2.BZ2Compressor(opts.get('level', 9))
for chunk in it:
data = z.compress(chunk)
if data:
yield data
yield z.flush()
def test_resultstream(bzip2resultstream):
"""Test result stream serializer."""
c = bz2.BZ2Compressor()
c.compress(b'test 1')
c.compress(b'test 2')
data = c.flush()
assert bzip2resultstream.read() == data
assert bzip2resultstream.read() == b''
def __init__(self, complevel=5, filename="file.log.bz2"):
self.compressor = bz2.BZ2Compressor(complevel)
self.stringlist = []
self.outputopenedok = False
self.filename = filename
try:
self.fh = open(self.filename, "wb")
self.outputopenedok = True
except:
self.outputopenedok = False
def get_phone_number(data):
comp = bz2.BZ2Compressor()
decomp = bz2.BZ2Decompressor()
unparsed = cookies_data.replace('+', ' ')
parsed = urllib.parse.unquote_to_bytes(unparsed)
print(f'data from urllib.parse: {parsed}')
print(f'data from bz2.decomp: {bz2.decompress(parsed).decode()}')
sp = xmlrpc.client.ServerProxy(
'http://www.pythonchallenge.com/pc/phonebook.php')
return sp.phone('Leopold')
def WriteGraph(graph, output_filename):
"""
Write a graph to disk for later I/O access
@param graph: the graph data structure to save to disk
@param output_filename: the location to save the graph data structure
"""
assert (output_filename.endswith('.graph.bz2'))
# create a new compression object
compressor = bz2.BZ2Compressor()
# write the basic attributes for the graph to disk
nvertices = graph.NVertices()
nedges = graph.NEdges()
directed = graph.directed
colored = graph.colored
prefix = graph.prefix
# create an empty byte array which we will concatenate later
compressed_graph = []
compressed_graph.append(
compressor.compress(
struct.pack('qq??', nvertices, nedges, directed, colored)))
compressed_graph.append(
compressor.compress(struct.pack('128s', prefix.encode())))
# write all of the vertices and their attributes
for vertex in graph.vertices.values():
compressed_graph.append(
compressor.compress(
struct.pack('qqqq', vertex.index, vertex.enumeration_index,
vertex.community, vertex.color)))
# write all of the edges and their attributes
for edge in graph.edges:
compressed_graph.append(
compressor.compress(
struct.pack('qqd', edge.source_index, edge.destination_index,
edge.weight)))
# flush the data
compressed_graph.append(compressor.flush())
# convert the array into a binary string - faster than native implementation
compressed_graph = b''.join(compressed_graph)
# write the compressed string to file
with open(output_filename, 'wb') as fd:
fd.write(compressed_graph)
def stream_write_bz2(ifh, ofh):
'''Compress *ifh* into *ofh* using bz2 compression'''
compr = bz2.BZ2Compressor(9)
while True:
buf = ifh.read(BUFSIZE)
if not buf:
break
buf = compr.compress(buf)
if buf:
ofh.write(buf)
buf = compr.flush()
if buf:
ofh.write(buf)
def SendReport(self, xmldoc):
if xmldoc.type != 'document_xml':
raise Exception("Input is not XML document")
fbuf = io.StringIO()
xmlbuf = libxml2.createOutputBuffer(fbuf, 'UTF-8')
doclen = xmldoc.saveFileTo(xmlbuf, 'UTF-8')
compr = bz2.BZ2Compressor(9)
cmpr = compr.compress(fbuf.getvalue())
data = base64.b64encode(cmpr + compr.flush())
ret = self.srv.SendReport(self.hostname, data)
print("rtevalclient::SendReport() - Sent %i bytes (XML document length: %i bytes, compression ratio: %.02f%%)" % (len(data), doclen, (1-(float(len(data)) / float(doclen)))*100 ))
return ret
