def test_three_bytes(self):
buffer = six.BytesIO()
write_varint((1 << (7 * 2)), buffer)
self.assertSequenceEqual(bytearray(buffer.getvalue()),
bytearray([0x80, 0x80, 0x01]))
buffer = six.BytesIO()
write_varint((1 << (7 * 3)) - 1, buffer)
self.assertSequenceEqual(bytearray(buffer.getvalue()),
bytearray([0xFF, 0xFF, 0x7F]))
def test_uncompress_exact_len(self):
byte_list = [random.randint(0, 255) for i in range(10000)]
data = six.binary_type(bytearray(byte_list))
zipped = zlib.compress(data)
total = len(zipped)
num_of_chunk = 20
chunk_size = total // num_of_chunk
in_buffer = six.BytesIO(zipped)
reader = ExactSizeReader(ZipCompressedReader(in_buffer))
buff = six.BytesIO()
for i in range(num_of_chunk):
buff.write(reader.read(chunk_size))
self.assertEqual(buff.getvalue(), data[:(i + 1) * chunk_size])
buff.write(reader.read())
self.assertEqual(buff.getvalue(), data)
def test_default(self):
expr_provider = WXFExprProvider(default=list)
stream = six.BytesIO()
serializer = WXFExprSerializer(stream, expr_provider=expr_provider)
serializer.serialize(range(1, 4))
wxf = b"\x38\x3a\x66\x03\x73\x04\x4c\x69\x73\x74\x43\x01\x43\x02\x43\x03"
self.assertSequenceEqual(stream.getvalue(), wxf)
def test_uncompress_exact_len_err(self):
data = six.binary_type(bytearray(range(100)))
zipped = zlib.compress(data)
total = len(zipped)
reader = ExactSizeReader(ZipCompressedReader(six.BytesIO(zipped)))
with self.assertRaises(EOFError):
reader.read(size=total + 1)
def encode_image(serializer, img):
# some PIL mode are directly mapped to WL ones. Best case fast (de)serialization.
try:
if img.mode in MODE_MAPPING:
wl_data_type, colorspace, interleaving = MODE_MAPPING[img.mode]
return serializer.encode(
wl.Image(
normalize_array(numpy.array(img)),
wl_data_type,
ColorSpace=colorspace or wl.Automatic,
Interleaving=interleaving,
)
)
except ImportError:
pass
# try to use format and import/export, may fail during save() and raise exception.
stream = six.BytesIO()
img_format = img.format or "PNG"
try:
img.save(stream, format=img_format)
except KeyError:
raise NotImplementedError("Format %s is not supported." % img_format)
return serializer.serialize_function(
serializer.serialize_symbol(b"ImportByteArray"),
(
serializer.serialize_bytes(stream.getvalue(), as_byte_array=True),
serializer.serialize_string(img_format),
),
)
def init(compress=False, enforce=True):
expr_provider = WXFExprProvider()
stream = six.BytesIO()
serializer = WXFExprSerializer(stream,
expr_provider=expr_provider,
enforce=True,
compress=compress)
return (serializer, stream)
def init(pa, ra):
expr_provider = WXFExprProvider()
numpy_encoder = NumPyWXFEncoder(packed_array_support=pa, numeric_array_support=ra)
expr_provider.add_encoder(numpy_encoder)
expr_provider.add_encoder(DefaultWXFEncoder())
serializer = WXFExprSerializer(six.BytesIO(), expr_provider=expr_provider)
return serializer
def test_multi_write_compress(self):
byte_list = [random.randint(0, 255) for i in range(10000)]
data = six.binary_type(bytearray(byte_list))
stream = six.BytesIO()
with ZipCompressedWriter(stream) as z_writer:
for i in byte_list:
z_writer.write(_bytes(i))
zipped = zlib.compress(data)
self.assertSequenceEqual(stream.getvalue(), zipped)
def test_custom_encoder(self):
""" test re-entrant calls """
expr_provider = WXFExprProvider()
expr_provider.add_encoder(MyClassEncoder())
stream = six.BytesIO()
serializer = WXFExprSerializer(stream, expr_provider=expr_provider)
myclass2 = MyClass2(True, "foobar")
myclass1 = MyClass1(1, None)
myclass = MyClass1(1, 2, [myclass2, myclass1])
o = ["foo", [1, {"k1": myclass, "k2": False}]]
serializer.serialize(o)
data_dir = dir_test_data()
filepath = os.path.join(data_dir, "test.wxf")
with open(filepath, "wb") as w_file:
w_file.write(stream.getvalue())
os.remove(filepath)
def test_custom_encoder(self):
''' test re-entrant calls '''
expr_provider = WXFExprProvider()
expr_provider.add_encoder(MyClassEncoder())
stream = six.BytesIO()
serializer = WXFExprSerializer(stream, expr_provider=expr_provider)
myclass2 = MyClass2(True, 'foobar')
myclass1 = MyClass1(1, None)
myclass = MyClass1(1, 2, [myclass2, myclass1])
o = ['foo', [1, {'k1': myclass, 'k2': False}]]
serializer.serialize(o)
data_dir = dir_test_data()
filepath = os.path.join(data_dir, 'test.wxf')
with open(filepath, 'wb') as w_file:
w_file.write(stream.getvalue())
os.remove(filepath)
def __init__(self, wxf_input):
"""WXF parser returning Python object from a WXF encoded byte sequence.
"""
self.context = SerializationContext()
if isinstance(wxf_input, (six.binary_type, six.buffer_types)):
self.reader = six.BytesIO(wxf_input)
elif hasattr(wxf_input, 'read'):
self.reader = wxf_input
else:
raise TypeError(
'Class %s neither implements a read method nor is a binary type.'
% wxf_input.__class__.__name__)
version, compress = self.parse_header()
if compress == True:
self.reader = ZipCompressedReader(self.reader)
else:
self.reader = ExactSizeReader(self.reader)
def test_export(self):
# checking that export is able to return bytes if no second argument is provided
self.assertEqual(export(2), b"2")
self.assertEqual(export("foo"), b'"foo"')
fd, path = tempfile.mkstemp()
# close the file descriptor but keep the path. Prevent error on Windows.
os.close(fd)
for test in ["foo", wl.Symbol, {"a": [1, 2, 3], 2: 2}]:
for export_format in available_formats:
expected = export(test, target_format=export_format)
# checking that export is able to write to a path if a string is provided
export(test, path, target_format=export_format)
with open(path, "rb") as stream:
self.assertEqual(stream.read(), expected)
# checking that export is writing to a byteio
stream = six.BytesIO()
export(test, stream, target_format=export_format)
stream.seek(0)
self.assertEqual(stream.read(), expected)
# checking that export is able to write to a filelike object
with open(path, "wb") as stream:
export(test, stream, target_format=export_format)
with open(path, "rb") as stream:
self.assertEqual(stream.read(), expected)
os.remove(path)
def read(self, size=-1):
"""Read from a compressed stream of bytes and return the inflated byte sequence.
Parameter `size` specifies the amount of bytes to return at most. If `size` is set to -1
then the reader is read until EOF is reached.
"""
if size is None or size < 0:
chunk_size = -1
size = -1
else:
chunk_size = ZipCompressedReader.CHUNK_SIZE
out_data = six.BytesIO()
out_len = 0
while True:
# first step find try to find some data to uncompress.
# sometimes some bytes are left over. We have to send them first to zlib.
if self._compressor.unconsumed_tail != b"":
data_in = self._compressor.unconsumed_tail
else:
# read more data from input reader. Read in chunk since we can't guess how
# big the inflated result is.
data_in = self._reader.read(chunk_size)
# no more data is available.
if data_in == b"":
break
# second step, decompress the new chunk
if size > 0:
chunk = self._compressor.decompress(data_in, size - out_len)
else:
chunk = self._compressor.decompress(data_in)
# increment output len.
out_len = out_len + len(chunk)
# write to buffer
out_data.write(chunk)
# check requested size against output length.
if size > 0 and out_len == size:
break
return out_data.getvalue()
def read(self, size=-1):
"""Read from an underlying readable object.
If size is negative the data is read until EOF.
If it is 0 then b'' is returned.
Otherwise the exact amount of bytes is read from the source. More than
one call may be necessary.
"""
data = self._reader.read(size)
# Negative values read until EOF and 0 returns b''. Both remain unchanged.
# Also a fast path when the requested amount of bytes is returned in one go.
if size <= 0 or len(data) == size:
return data
# need an intermediary buffer
out_len = len(data)
data = six.BytesIO(data)
while out_len < size:
chunk = self._reader.read(size - out_len)
if chunk == b"":
raise EOFError("Not enough data to read.")
data.write(chunk)
out_len = out_len + len(chunk)
return data.getvalue()
def varint_round_trip_integer(self, int_value):
buffer = six.BytesIO()
write_varint(int_value, buffer)
buffer.seek(0)
self.assertEqual(parse_varint(buffer), int_value)
def __init__(self, initial_bytes=None):
self.buffer = six.BytesIO(initial_bytes)
def test_zero(self):
buffer = six.BytesIO()
write_varint(0, buffer)
self.assertSequenceEqual(bytearray(buffer.getvalue()),
bytearray([0x00]))
def encode_buffer(wxf_input):
return six.BytesIO(wxf_input.tobytes())
def test_one_byte(self):
buffer = six.BytesIO()
write_varint(127, buffer)
self.assertSequenceEqual(bytearray(buffer.getvalue()),
bytearray([127]))
def test_compress(self):
stream = six.BytesIO()
with ZipCompressedWriter(stream) as z_writer:
z_writer.write(b"abc")
zipped = b"x\x9cKLJ\x06\x00\x02M\x01'"
self.assertSequenceEqual(stream.getvalue(), zipped)
