def create_patch_bsdiff(ffrom, fto, fpatch):
to_size = file_size(fto)
from_data = file_read(ffrom)
suffix_array = sais.sais(from_data)
chunks = bsdiff.create_patch(suffix_array, from_data, file_read(fto))
fctrl = BytesIO()
fdiff = BytesIO()
fextra = BytesIO()
ctrl_compressor = BZ2Compressor()
diff_compressor = BZ2Compressor()
extra_compressor = BZ2Compressor()
for i in range(0, len(chunks), 5):
size = offtout(unpack_size_bytes(chunks[i + 0]))
fctrl.write(ctrl_compressor.compress(size))
fdiff.write(diff_compressor.compress(chunks[i + 1]))
size = offtout(unpack_size_bytes(chunks[i + 2]))
fctrl.write(ctrl_compressor.compress(size))
fextra.write(extra_compressor.compress(chunks[i + 3]))
size = offtout(unpack_size_bytes(chunks[i + 4]))
fctrl.write(ctrl_compressor.compress(size))
fctrl.write(ctrl_compressor.flush())
fdiff.write(diff_compressor.flush())
fextra.write(extra_compressor.flush())
# Write everything to the patch file.
fpatch.write(b'BSDIFF40')
fpatch.write(offtout(fctrl.tell()))
fpatch.write(offtout(fdiff.tell()))
fpatch.write(offtout(to_size))
fpatch.write(fctrl.getvalue())
fpatch.write(fdiff.getvalue())
fpatch.write(fextra.getvalue())
def testCompress4G(self, size):
# "Test BZ2Compressor.compress()/flush() with >4GiB input"
bz2c = BZ2Compressor()
data = b"x" * size
try:
compressed = bz2c.compress(data)
compressed += bz2c.flush()
finally:
data = None # Release memory
data = bz2.decompress(compressed)
try:
self.assertEqual(len(data), size)
self.assertEqual(len(data.strip(b"x")), 0)
finally:
data = None
def __init__(self):
"""Constructor to build the appropriate compressor type."""
if compressor_type == 'bzip2':
from bz2 import BZ2Compressor
self._comp = BZ2Compressor(9)
self._comp_func = self._comp.compress
self._flush_passthru = False
elif compressor_type == 'gzip':
from zlib import compress
self._comp = None
self._comp_func = lambda x: compress(x, 9)
self._flush_passthru = True
else:
self._comp = None
self._comp_func = lambda x: x
self._flush_passthru = True
def thr_func(sock):
chunk_size = 4096
data = sock.recv(chunk_size)
data_size = int(data[:data.find('x')])
data = data[data.find('x') + 1:]
processed = 0
compressor = BZ2Compressor()
while processed != data_size:
processed += len(data)
data_c = compressor.compress(data)
if processed == data_size:
break
if data_c != '':
sock.sendall(data_c)
data = sock.recv(chunk_size)
data_c = compressor.flush()
sock.sendall(data_c)
sock.close()
def create_compressor(compression):
if compression == 'lzma':
compressor = lzma.LZMACompressor(format=lzma.FORMAT_ALONE)
elif compression == 'bz2':
compressor = BZ2Compressor()
elif compression == 'none':
compressor = NoneCompressor()
elif compression == 'crle':
compressor = CrleCompressor()
elif compression == 'heatshrink':
compressor = HeatshrinkCompressor()
elif compression == 'zstd':
compressor = ZstdCompressor()
elif compression == 'lz4':
compressor = Lz4Compressor()
else:
raise Error(format_bad_compression_string(compression))
return compressor
def __init__(self, **args):
self.classobj = BZ2Compressor(args.get('ratio'))
def eval_fitness(self, individual, seed, render=False, record=None, record_force=False, brain_vis_handler=None,
neuron_vis=False, slow_down=0, rounds=None, neuron_vis_width=None, neuron_vis_height=None):
env = self._get_env(record, record_force, render)
set_random_seeds(seed, env)
fitness_total = 0
steps_total = 0
number_of_rounds = self.config.number_fitness_runs if rounds is None else rounds
brain_state_history = []
for i in range(number_of_rounds):
fitness_current = 0
brain = self.brain_class(self.input_space, self.output_space, individual, self.brain_config)
ob = env.reset()
done = False
t = 0
if render:
env.render()
if neuron_vis:
brain_vis = brain_vis_handler.launch_new_visualization(brain=brain, brain_config=self.brain_config,
env_id=self.env_id, initial_observation=ob,
width=neuron_vis_width, height=neuron_vis_height,
color_clipping_range=(255, 2.5, 2.5),
slow_down=slow_down)
else:
brain_vis = None
while not done:
brain_output = brain.step(ob)
action = output_to_action(brain_output, self.output_space)
ob, rew, done, info = env.step(action)
t += 1
fitness_current += rew
if brain_vis:
brain_vis.process_update(in_values=ob, out_values=brain_output)
if slow_down:
time.sleep(slow_down / 1000.0)
if render:
env.render()
if self.config.novelty:
if self.config.novelty.behavior_source == 'brain':
if isinstance(brain, ContinuousTimeRNN):
brain_state_history.append(np.tanh(brain.y))
elif isinstance(brain, CnnCtrnn):
brain_state_history.append(np.tanh(brain.ctrnn.y))
else:
logging.error('behavior_source == "brain" not yet supported for this kind of brain')
if render:
logging.info("steps: " + str(t) + " \tfitness: " + str(fitness_current))
fitness_total += fitness_current
steps_total += t
# print(info['level_seed'])
compressed_behavior = None
if hasattr(env, 'get_compressed_behavior'):
# 'get_compressed_behavior' exists if any wrapper is a BehaviorWrapper
if callable(env.get_compressed_behavior):
compressed_behavior = env.get_compressed_behavior()
if self.config.novelty:
if self.config.novelty.behavior_source == 'brain':
# todo: remove code duplication. This code is also in BehaviorWrapper
compressor = BZ2Compressor(2)
compressed_behavior = b''
if self.config.novelty.behavioral_max_length < 0:
compressor.compress(brain_state_history[-1].astype(np.float16).tobytes())
compressed_behavior += compressor.flush()
else:
for i in range(self.config.novelty.behavioral_max_length):
aggregate = np.zeros(len(brain_state_history[0]), dtype=np.float32)
for j in range(self.config.novelty.behavioral_interval):
if len(brain_state_history) > j + i * self.config.novelty.behavioral_interval:
state = brain_state_history[j + i * self.config.novelty.behavioral_interval]
aggregate += state / self.config.novelty.behavioral_interval
else:
break
compressed_behavior += compressor.compress(aggregate.astype(np.float16).tobytes())
compressed_behavior += compressor.flush()
return fitness_total / number_of_rounds, compressed_behavior, steps_total
def test_buffer(self):
from bz2 import BZ2Compressor
bz2c = BZ2Compressor()
data = bz2c.compress(buffer(self.TEXT))
data = "%s%s" % (data, bz2c.flush())
assert self.decompress(data) == self.TEXT
def testCompress(self):
# "Test BZ2Compressor.compress()/flush()"
bz2c = BZ2Compressor()
data = bz2c.compress(self.TEXT)
data += bz2c.flush()
self.assertEqual(self.decompress(data), self.TEXT)
def _reset_compressor(self):
self.compressed_behavior = b''
self.compressor = BZ2Compressor(2)
self.step_count = 0
self.aggregate = None
# bz2_sample.py
from bz2 import BZ2Compressor as Compressor
compressor = Compressor()
fpath = 'c:\\python33\\README.txt'
with open(fpath) as f:
data = f.read(128)
while data:
c = compressor.compress(data.encode())
if compressed:
print('compressed:', c)
data = f.read(128)
c = compressor.flush()
if c:
print('compressed:', c)
def testPickle(self):
with self.assertRaises(TypeError):
pickle.dumps(BZ2Compressor())
def testPickle(self):
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
with self.assertRaises(TypeError):
pickle.dumps(BZ2Compressor(), proto)
def testCompress(self):
bz2c = BZ2Compressor()
self.assertRaises(TypeError, bz2c.compress)
data = bz2c.compress(self.TEXT)
data += bz2c.flush()
self.assertEqual(self.decompress(data), self.TEXT)
def __init__(self, filename, mode="r", buffering=None, compresslevel=9):
"""Open a bzip2-compressed file.
If filename is a str, bytes or unicode object, it gives the name
of the file to be opened. Otherwise, it should be a file object,
which will be used to read or write the compressed data.
mode can be 'r' for reading (default), 'w' for (over)writing,
'x' for creating exclusively, or 'a' for appending. These can
equivalently be given as 'rb', 'wb', 'xb', and 'ab'.
buffering is ignored. Its use is deprecated.
If mode is 'w', 'x' or 'a', compresslevel can be a number between 1
and 9 specifying the level of compression: 1 produces the least
compression, and 9 (default) produces the most compression.
If mode is 'r', the input file may be the concatenation of
multiple compressed streams.
"""
# This lock must be recursive, so that BufferedIOBase's
# readline(), readlines() and writelines() don't deadlock.
self._lock = RLock()
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
self._pos = 0
self._size = -1
if buffering is not None:
warnings.warn("Use of 'buffering' argument is deprecated",
DeprecationWarning)
if not (1 <= compresslevel <= 9):
raise ValueError("compresslevel must be between 1 and 9")
if mode in ("", "r", "rb"):
mode = "rb"
mode_code = _MODE_READ
self._decompressor = BZ2Decompressor()
self._buffer = b""
self._buffer_offset = 0
elif mode in ("w", "wb"):
mode = "wb"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
elif mode in ("x", "xb") and _HAS_OPEN_X_MODE:
mode = "xb"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
elif mode in ("a", "ab"):
mode = "ab"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
else:
raise ValueError("Invalid mode: %r" % (mode, ))
if isinstance(filename, _STR_TYPES):
self._fp = _builtin_open(filename, mode)
self._closefp = True
self._mode = mode_code
elif hasattr(filename, "read") or hasattr(filename, "write"):
self._fp = filename
self._mode = mode_code
else:
raise TypeError("filename must be a %s or %s object, or a file" %
(_STR_TYPES[0].__name__, _STR_TYPES[1].__name__))
def bz2compress(data):
compressor = BZ2Compressor(9)
return compressor.compress(data) + compressor.flush()
def test_buffer(self):
from bz2 import BZ2Compressor
bz2c = BZ2Compressor()
data = bz2c.compress(memoryview(self.TEXT))
data += bz2c.flush()
assert self.decompress(data) == self.TEXT
def testCompressEmptyString(self):
bz2c = BZ2Compressor()
data = bz2c.compress(b'')
data += bz2c.flush()
self.assertEqual(data, self.EMPTY_DATA)
def __init__(self, text):
self.text = text
self.comp = BZ2Compressor()
def testCompressEmptyString(self):
# "Test BZ2Compressor.compress()/flush() of empty string"
bz2c = BZ2Compressor()
data = bz2c.compress('')
data += bz2c.flush()
self.assertEqual(data, self.EMPTY_DATA)
# bz2_sample.py
from bz2 import BZ2Compressor as Compressor
compressor = Compressor()
fpath = 'c:\\python33\\README.txt'
with open(fpath) as f:
data = f.read(128)
while data:
c = compressor.compress(data.encode())
if compressed:
print('compressed:', c)
data = f.read(128)
c = compressor.flush()
if c:
print('compressed:', c)
