def create_patch_bsdiff(ffrom, fto, fpatch): to_size = file_size(fto) from_data = file_read(ffrom) suffix_array = sais.sais(from_data) chunks = bsdiff.create_patch(suffix_array, from_data, file_read(fto)) fctrl = BytesIO() fdiff = BytesIO() fextra = BytesIO() ctrl_compressor = BZ2Compressor() diff_compressor = BZ2Compressor() extra_compressor = BZ2Compressor() for i in range(0, len(chunks), 5): size = offtout(unpack_size_bytes(chunks[i + 0])) fctrl.write(ctrl_compressor.compress(size)) fdiff.write(diff_compressor.compress(chunks[i + 1])) size = offtout(unpack_size_bytes(chunks[i + 2])) fctrl.write(ctrl_compressor.compress(size)) fextra.write(extra_compressor.compress(chunks[i + 3])) size = offtout(unpack_size_bytes(chunks[i + 4])) fctrl.write(ctrl_compressor.compress(size)) fctrl.write(ctrl_compressor.flush()) fdiff.write(diff_compressor.flush()) fextra.write(extra_compressor.flush()) # Write everything to the patch file. fpatch.write(b'BSDIFF40') fpatch.write(offtout(fctrl.tell())) fpatch.write(offtout(fdiff.tell())) fpatch.write(offtout(to_size)) fpatch.write(fctrl.getvalue()) fpatch.write(fdiff.getvalue()) fpatch.write(fextra.getvalue())
def testCompress4G(self, size): # "Test BZ2Compressor.compress()/flush() with >4GiB input" bz2c = BZ2Compressor() data = b"x" * size try: compressed = bz2c.compress(data) compressed += bz2c.flush() finally: data = None # Release memory data = bz2.decompress(compressed) try: self.assertEqual(len(data), size) self.assertEqual(len(data.strip(b"x")), 0) finally: data = None
def __init__(self): """Constructor to build the appropriate compressor type.""" if compressor_type == 'bzip2': from bz2 import BZ2Compressor self._comp = BZ2Compressor(9) self._comp_func = self._comp.compress self._flush_passthru = False elif compressor_type == 'gzip': from zlib import compress self._comp = None self._comp_func = lambda x: compress(x, 9) self._flush_passthru = True else: self._comp = None self._comp_func = lambda x: x self._flush_passthru = True
def thr_func(sock): chunk_size = 4096 data = sock.recv(chunk_size) data_size = int(data[:data.find('x')]) data = data[data.find('x') + 1:] processed = 0 compressor = BZ2Compressor() while processed != data_size: processed += len(data) data_c = compressor.compress(data) if processed == data_size: break if data_c != '': sock.sendall(data_c) data = sock.recv(chunk_size) data_c = compressor.flush() sock.sendall(data_c) sock.close()
def create_compressor(compression): if compression == 'lzma': compressor = lzma.LZMACompressor(format=lzma.FORMAT_ALONE) elif compression == 'bz2': compressor = BZ2Compressor() elif compression == 'none': compressor = NoneCompressor() elif compression == 'crle': compressor = CrleCompressor() elif compression == 'heatshrink': compressor = HeatshrinkCompressor() elif compression == 'zstd': compressor = ZstdCompressor() elif compression == 'lz4': compressor = Lz4Compressor() else: raise Error(format_bad_compression_string(compression)) return compressor
def __init__(self, **args): self.classobj = BZ2Compressor(args.get('ratio'))
def eval_fitness(self, individual, seed, render=False, record=None, record_force=False, brain_vis_handler=None, neuron_vis=False, slow_down=0, rounds=None, neuron_vis_width=None, neuron_vis_height=None): env = self._get_env(record, record_force, render) set_random_seeds(seed, env) fitness_total = 0 steps_total = 0 number_of_rounds = self.config.number_fitness_runs if rounds is None else rounds brain_state_history = [] for i in range(number_of_rounds): fitness_current = 0 brain = self.brain_class(self.input_space, self.output_space, individual, self.brain_config) ob = env.reset() done = False t = 0 if render: env.render() if neuron_vis: brain_vis = brain_vis_handler.launch_new_visualization(brain=brain, brain_config=self.brain_config, env_id=self.env_id, initial_observation=ob, width=neuron_vis_width, height=neuron_vis_height, color_clipping_range=(255, 2.5, 2.5), slow_down=slow_down) else: brain_vis = None while not done: brain_output = brain.step(ob) action = output_to_action(brain_output, self.output_space) ob, rew, done, info = env.step(action) t += 1 fitness_current += rew if brain_vis: brain_vis.process_update(in_values=ob, out_values=brain_output) if slow_down: time.sleep(slow_down / 1000.0) if render: env.render() if self.config.novelty: if self.config.novelty.behavior_source == 'brain': if isinstance(brain, ContinuousTimeRNN): brain_state_history.append(np.tanh(brain.y)) elif isinstance(brain, CnnCtrnn): brain_state_history.append(np.tanh(brain.ctrnn.y)) else: logging.error('behavior_source == "brain" not yet supported for this kind of brain') if render: logging.info("steps: " + str(t) + " \tfitness: " + str(fitness_current)) fitness_total += fitness_current steps_total += t # print(info['level_seed']) compressed_behavior = None if hasattr(env, 'get_compressed_behavior'): # 'get_compressed_behavior' exists if any wrapper is a BehaviorWrapper if callable(env.get_compressed_behavior): compressed_behavior = env.get_compressed_behavior() if self.config.novelty: if self.config.novelty.behavior_source == 'brain': # todo: remove code duplication. This code is also in BehaviorWrapper compressor = BZ2Compressor(2) compressed_behavior = b'' if self.config.novelty.behavioral_max_length < 0: compressor.compress(brain_state_history[-1].astype(np.float16).tobytes()) compressed_behavior += compressor.flush() else: for i in range(self.config.novelty.behavioral_max_length): aggregate = np.zeros(len(brain_state_history[0]), dtype=np.float32) for j in range(self.config.novelty.behavioral_interval): if len(brain_state_history) > j + i * self.config.novelty.behavioral_interval: state = brain_state_history[j + i * self.config.novelty.behavioral_interval] aggregate += state / self.config.novelty.behavioral_interval else: break compressed_behavior += compressor.compress(aggregate.astype(np.float16).tobytes()) compressed_behavior += compressor.flush() return fitness_total / number_of_rounds, compressed_behavior, steps_total
def test_buffer(self): from bz2 import BZ2Compressor bz2c = BZ2Compressor() data = bz2c.compress(buffer(self.TEXT)) data = "%s%s" % (data, bz2c.flush()) assert self.decompress(data) == self.TEXT
def testCompress(self): # "Test BZ2Compressor.compress()/flush()" bz2c = BZ2Compressor() data = bz2c.compress(self.TEXT) data += bz2c.flush() self.assertEqual(self.decompress(data), self.TEXT)
def _reset_compressor(self): self.compressed_behavior = b'' self.compressor = BZ2Compressor(2) self.step_count = 0 self.aggregate = None
# bz2_sample.py from bz2 import BZ2Compressor as Compressor compressor = Compressor() fpath = 'c:\\python33\\README.txt' with open(fpath) as f: data = f.read(128) while data: c = compressor.compress(data.encode()) if compressed: print('compressed:', c) data = f.read(128) c = compressor.flush() if c: print('compressed:', c)
def testPickle(self): with self.assertRaises(TypeError): pickle.dumps(BZ2Compressor())
def testPickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): pickle.dumps(BZ2Compressor(), proto)
def testCompress(self): bz2c = BZ2Compressor() self.assertRaises(TypeError, bz2c.compress) data = bz2c.compress(self.TEXT) data += bz2c.flush() self.assertEqual(self.decompress(data), self.TEXT)
def __init__(self, filename, mode="r", buffering=None, compresslevel=9): """Open a bzip2-compressed file. If filename is a str, bytes or unicode object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can equivalently be given as 'rb', 'wb', 'xb', and 'ab'. buffering is ignored. Its use is deprecated. If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ # This lock must be recursive, so that BufferedIOBase's # readline(), readlines() and writelines() don't deadlock. self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED self._pos = 0 self._size = -1 if buffering is not None: warnings.warn("Use of 'buffering' argument is deprecated", DeprecationWarning) if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") if mode in ("", "r", "rb"): mode = "rb" mode_code = _MODE_READ self._decompressor = BZ2Decompressor() self._buffer = b"" self._buffer_offset = 0 elif mode in ("w", "wb"): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("x", "xb") and _HAS_OPEN_X_MODE: mode = "xb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("a", "ab"): mode = "ab" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: raise ValueError("Invalid mode: %r" % (mode, )) if isinstance(filename, _STR_TYPES): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename self._mode = mode_code else: raise TypeError("filename must be a %s or %s object, or a file" % (_STR_TYPES[0].__name__, _STR_TYPES[1].__name__))
def bz2compress(data): compressor = BZ2Compressor(9) return compressor.compress(data) + compressor.flush()
def test_buffer(self): from bz2 import BZ2Compressor bz2c = BZ2Compressor() data = bz2c.compress(memoryview(self.TEXT)) data += bz2c.flush() assert self.decompress(data) == self.TEXT
def testCompressEmptyString(self): bz2c = BZ2Compressor() data = bz2c.compress(b'') data += bz2c.flush() self.assertEqual(data, self.EMPTY_DATA)
def __init__(self, text): self.text = text self.comp = BZ2Compressor()
def testCompressEmptyString(self): # "Test BZ2Compressor.compress()/flush() of empty string" bz2c = BZ2Compressor() data = bz2c.compress('') data += bz2c.flush() self.assertEqual(data, self.EMPTY_DATA)