def test_three_bytes(self): buffer = six.BytesIO() write_varint((1 << (7 * 2)), buffer) self.assertSequenceEqual(bytearray(buffer.getvalue()), bytearray([0x80, 0x80, 0x01])) buffer = six.BytesIO() write_varint((1 << (7 * 3)) - 1, buffer) self.assertSequenceEqual(bytearray(buffer.getvalue()), bytearray([0xFF, 0xFF, 0x7F]))
def test_uncompress_exact_len(self): byte_list = [random.randint(0, 255) for i in range(10000)] data = six.binary_type(bytearray(byte_list)) zipped = zlib.compress(data) total = len(zipped) num_of_chunk = 20 chunk_size = total // num_of_chunk in_buffer = six.BytesIO(zipped) reader = ExactSizeReader(ZipCompressedReader(in_buffer)) buff = six.BytesIO() for i in range(num_of_chunk): buff.write(reader.read(chunk_size)) self.assertEqual(buff.getvalue(), data[:(i + 1) * chunk_size]) buff.write(reader.read()) self.assertEqual(buff.getvalue(), data)
def test_default(self): expr_provider = WXFExprProvider(default=list) stream = six.BytesIO() serializer = WXFExprSerializer(stream, expr_provider=expr_provider) serializer.serialize(range(1, 4)) wxf = b"\x38\x3a\x66\x03\x73\x04\x4c\x69\x73\x74\x43\x01\x43\x02\x43\x03" self.assertSequenceEqual(stream.getvalue(), wxf)
def test_uncompress_exact_len_err(self): data = six.binary_type(bytearray(range(100))) zipped = zlib.compress(data) total = len(zipped) reader = ExactSizeReader(ZipCompressedReader(six.BytesIO(zipped))) with self.assertRaises(EOFError): reader.read(size=total + 1)
def encode_image(serializer, img): # some PIL mode are directly mapped to WL ones. Best case fast (de)serialization. try: if img.mode in MODE_MAPPING: wl_data_type, colorspace, interleaving = MODE_MAPPING[img.mode] return serializer.encode( wl.Image( normalize_array(numpy.array(img)), wl_data_type, ColorSpace=colorspace or wl.Automatic, Interleaving=interleaving, ) ) except ImportError: pass # try to use format and import/export, may fail during save() and raise exception. stream = six.BytesIO() img_format = img.format or "PNG" try: img.save(stream, format=img_format) except KeyError: raise NotImplementedError("Format %s is not supported." % img_format) return serializer.serialize_function( serializer.serialize_symbol(b"ImportByteArray"), ( serializer.serialize_bytes(stream.getvalue(), as_byte_array=True), serializer.serialize_string(img_format), ), )
def init(compress=False, enforce=True): expr_provider = WXFExprProvider() stream = six.BytesIO() serializer = WXFExprSerializer(stream, expr_provider=expr_provider, enforce=True, compress=compress) return (serializer, stream)
def init(pa, ra): expr_provider = WXFExprProvider() numpy_encoder = NumPyWXFEncoder(packed_array_support=pa, numeric_array_support=ra) expr_provider.add_encoder(numpy_encoder) expr_provider.add_encoder(DefaultWXFEncoder()) serializer = WXFExprSerializer(six.BytesIO(), expr_provider=expr_provider) return serializer
def test_multi_write_compress(self): byte_list = [random.randint(0, 255) for i in range(10000)] data = six.binary_type(bytearray(byte_list)) stream = six.BytesIO() with ZipCompressedWriter(stream) as z_writer: for i in byte_list: z_writer.write(_bytes(i)) zipped = zlib.compress(data) self.assertSequenceEqual(stream.getvalue(), zipped)
def test_custom_encoder(self): """ test re-entrant calls """ expr_provider = WXFExprProvider() expr_provider.add_encoder(MyClassEncoder()) stream = six.BytesIO() serializer = WXFExprSerializer(stream, expr_provider=expr_provider) myclass2 = MyClass2(True, "foobar") myclass1 = MyClass1(1, None) myclass = MyClass1(1, 2, [myclass2, myclass1]) o = ["foo", [1, {"k1": myclass, "k2": False}]] serializer.serialize(o) data_dir = dir_test_data() filepath = os.path.join(data_dir, "test.wxf") with open(filepath, "wb") as w_file: w_file.write(stream.getvalue()) os.remove(filepath)
def test_custom_encoder(self): ''' test re-entrant calls ''' expr_provider = WXFExprProvider() expr_provider.add_encoder(MyClassEncoder()) stream = six.BytesIO() serializer = WXFExprSerializer(stream, expr_provider=expr_provider) myclass2 = MyClass2(True, 'foobar') myclass1 = MyClass1(1, None) myclass = MyClass1(1, 2, [myclass2, myclass1]) o = ['foo', [1, {'k1': myclass, 'k2': False}]] serializer.serialize(o) data_dir = dir_test_data() filepath = os.path.join(data_dir, 'test.wxf') with open(filepath, 'wb') as w_file: w_file.write(stream.getvalue()) os.remove(filepath)
def __init__(self, wxf_input): """WXF parser returning Python object from a WXF encoded byte sequence. """ self.context = SerializationContext() if isinstance(wxf_input, (six.binary_type, six.buffer_types)): self.reader = six.BytesIO(wxf_input) elif hasattr(wxf_input, 'read'): self.reader = wxf_input else: raise TypeError( 'Class %s neither implements a read method nor is a binary type.' % wxf_input.__class__.__name__) version, compress = self.parse_header() if compress == True: self.reader = ZipCompressedReader(self.reader) else: self.reader = ExactSizeReader(self.reader)
def test_export(self): # checking that export is able to return bytes if no second argument is provided self.assertEqual(export(2), b"2") self.assertEqual(export("foo"), b'"foo"') fd, path = tempfile.mkstemp() # close the file descriptor but keep the path. Prevent error on Windows. os.close(fd) for test in ["foo", wl.Symbol, {"a": [1, 2, 3], 2: 2}]: for export_format in available_formats: expected = export(test, target_format=export_format) # checking that export is able to write to a path if a string is provided export(test, path, target_format=export_format) with open(path, "rb") as stream: self.assertEqual(stream.read(), expected) # checking that export is writing to a byteio stream = six.BytesIO() export(test, stream, target_format=export_format) stream.seek(0) self.assertEqual(stream.read(), expected) # checking that export is able to write to a filelike object with open(path, "wb") as stream: export(test, stream, target_format=export_format) with open(path, "rb") as stream: self.assertEqual(stream.read(), expected) os.remove(path)
def read(self, size=-1): """Read from a compressed stream of bytes and return the inflated byte sequence. Parameter `size` specifies the amount of bytes to return at most. If `size` is set to -1 then the reader is read until EOF is reached. """ if size is None or size < 0: chunk_size = -1 size = -1 else: chunk_size = ZipCompressedReader.CHUNK_SIZE out_data = six.BytesIO() out_len = 0 while True: # first step find try to find some data to uncompress. # sometimes some bytes are left over. We have to send them first to zlib. if self._compressor.unconsumed_tail != b"": data_in = self._compressor.unconsumed_tail else: # read more data from input reader. Read in chunk since we can't guess how # big the inflated result is. data_in = self._reader.read(chunk_size) # no more data is available. if data_in == b"": break # second step, decompress the new chunk if size > 0: chunk = self._compressor.decompress(data_in, size - out_len) else: chunk = self._compressor.decompress(data_in) # increment output len. out_len = out_len + len(chunk) # write to buffer out_data.write(chunk) # check requested size against output length. if size > 0 and out_len == size: break return out_data.getvalue()
def read(self, size=-1): """Read from an underlying readable object. If size is negative the data is read until EOF. If it is 0 then b'' is returned. Otherwise the exact amount of bytes is read from the source. More than one call may be necessary. """ data = self._reader.read(size) # Negative values read until EOF and 0 returns b''. Both remain unchanged. # Also a fast path when the requested amount of bytes is returned in one go. if size <= 0 or len(data) == size: return data # need an intermediary buffer out_len = len(data) data = six.BytesIO(data) while out_len < size: chunk = self._reader.read(size - out_len) if chunk == b"": raise EOFError("Not enough data to read.") data.write(chunk) out_len = out_len + len(chunk) return data.getvalue()
def varint_round_trip_integer(self, int_value): buffer = six.BytesIO() write_varint(int_value, buffer) buffer.seek(0) self.assertEqual(parse_varint(buffer), int_value)
def __init__(self, initial_bytes=None): self.buffer = six.BytesIO(initial_bytes)
def test_zero(self): buffer = six.BytesIO() write_varint(0, buffer) self.assertSequenceEqual(bytearray(buffer.getvalue()), bytearray([0x00]))
def encode_buffer(wxf_input): return six.BytesIO(wxf_input.tobytes())
def test_one_byte(self): buffer = six.BytesIO() write_varint(127, buffer) self.assertSequenceEqual(bytearray(buffer.getvalue()), bytearray([127]))
def test_compress(self): stream = six.BytesIO() with ZipCompressedWriter(stream) as z_writer: z_writer.write(b"abc") zipped = b"x\x9cKLJ\x06\x00\x02M\x01'" self.assertSequenceEqual(stream.getvalue(), zipped)