def test_array(self): # invalid length with self.assertRaises(DecoderException): ubjloadb(ARRAY_START + CONTAINER_COUNT + ubjdumpb(-5)) # unencodable type within with self.assertRaises(EncoderException): ubjdumpb([type(None)]) for sequence in list, tuple: self.assertEqual(ubjdumpb(sequence()), ARRAY_START + ARRAY_END) self.assertEqual(ubjdumpb((None,), container_count=True), (ARRAY_START + CONTAINER_COUNT + TYPE_UINT8 + b'\x01' + TYPE_NULL)) obj = [123, 1.25, 43121609.5543, 12345.44e40, Decimal('10e15'), 'a', 'here is a string', None, True, False, [[1, 2], 3, [4, 5, 6], 7], {'a dict': 456}] for opts in ({'container_count': False}, {'container_count': True}): self.check_enc_dec(obj, **opts)
def test_float(self): # insufficient length for float_type in (TYPE_FLOAT32, TYPE_FLOAT64): with self.assertRaises(DecoderException): ubjloadb(float_type + b'\x01') self.check_enc_dec(0.0, 5, expected_type=TYPE_FLOAT32) for type_, value, total_size in ( (TYPE_FLOAT32, 1.18e-38, 5), (TYPE_FLOAT32, 3.4e38, 5), (TYPE_FLOAT64, 2.23e-308, 9), (TYPE_FLOAT64, 12345.44e40, 9), (TYPE_FLOAT64, 1.8e307, 9)): self.check_enc_dec(value, total_size, approximate=True, expected_type=type_, no_float32=False) # using only float64 (default) self.check_enc_dec(value, 9 if type_ == TYPE_FLOAT32 else total_size, approximate=True, expected_type=(TYPE_FLOAT64 if type_ == TYPE_FLOAT32 else type_)) for value in ('nan', '-inf', 'inf'): for no_float32 in (True, False): self.assertEqual(ubjloadb(ubjdumpb(float(value), no_float32=no_float32)), None) # value which results in high_prec usage for no_float32 in (True, False): self.check_enc_dec(2.22e-308, 4, expected_type=TYPE_HIGH_PREC, length_greater_or_equal=True, no_float32=no_float32)
def test_bytes(self): # insufficient length with self.assertRaises(DecoderException): ubjloadb(ARRAY_START + CONTAINER_TYPE + TYPE_UINT8 + CONTAINER_COUNT + TYPE_UINT8 + b'\x02' + b'\x01') self.check_enc_dec(b'') self.check_enc_dec(b'\x01' * 4) self.assertEqual(ubjloadb(ubjdumpb(b'\x04' * 4), no_bytes=True), [4] * 4) self.check_enc_dec(b'largebinary' * 100)
def test_char(self): self.assertEqual(ubjdumpb(u('a')), TYPE_CHAR + 'a'.encode('utf-8')) # no char, char invalid utf-8 for suffix in (b'', b'\xfe'): with self.assertRaises(DecoderException): ubjloadb(TYPE_CHAR + suffix) for char in (u('a'), u('\0'), u('~')): self.check_enc_dec(char, 2)
def test_decoder_fuzz(self): for start, end, fmt in ((0, pow(2, 8), '>B'), (pow(2, 8), pow(2, 16), '>H')): for i in range(start, end): try: ubjloadb(pack(fmt, i)) except DecoderException: pass except Exception as ex: # pragma: no cover pylint: disable=broad-except self.fail('Unexpected failure: %s' % ex)
def test_string(self): self.assertEqual(ubjdumpb(u('ab')), TYPE_STRING + TYPE_UINT8 + b'\x02' + 'ab'.encode('utf-8')) self.check_enc_dec(u(''), 3) # invalid string size, string too short, string invalid utf-8 for suffix in (b'\x81', b'\x01', b'\x01' + b'\xfe'): with self.assertRaises(DecoderException): ubjloadb(TYPE_STRING + TYPE_INT8 + suffix) # Note: In Python 2 plain str type is encoded as byte array for string in ('some ascii', u(r'\u00a9 with extended\u2122'), u('long string') * 100): self.check_enc_dec(string, 4, length_greater_or_equal=True)
def __load(self): if os.path.exists(self.__fn): with self.__lock: with open(self.__fn, 'rb') as f: data = f.read() if len(data): self.__data = ubjloadb(data)
def test_object(self): self.assertEqual(ubjdumpb({}), OBJECT_START + OBJECT_END) self.assertEqual(ubjdumpb({'a': None}, container_count=True), (OBJECT_START + CONTAINER_COUNT + TYPE_UINT8 + b'\x01' + TYPE_UINT8 + b'\x01' + 'a'.encode('utf-8') + TYPE_NULL)) self.check_enc_dec({}) # negative length with self.assertRaises(DecoderException): ubjloadb(OBJECT_START + CONTAINER_COUNT + ubjdumpb(-1)) with self.assertRaises(EncoderException): ubjdumpb({123: 'non-string key'}) with self.assertRaises(EncoderException): ubjdumpb({'fish': type(list)}) # invalid key size type with self.assertRaises(DecoderException): ubjloadb(OBJECT_START + TYPE_NULL) # invalid key size, key too short, key invalid utf-8, no value for suffix in (b'\x81', b'\x01', b'\x01' + b'\xfe', b'\x0101'): with self.assertRaises(DecoderException): ubjloadb(OBJECT_START + TYPE_INT8 + suffix) self.check_enc_dec({'longkey1' * 65: 1}) self.check_enc_dec({'longkey2' * 4096: 1}) obj = {'int': 123, 'longint': 9223372036854775807, 'float': 1.25, 'hp': Decimal('10e15'), 'char': 'a', 'str': 'here is a string', 'unicode': u(r'\u00a9 with extended\u2122'), '': 'empty key', u(r'\u00a9 with extended\u2122'): 'unicode-key', 'null': None, 'true': True, 'false': False, 'array': [1, 2, 3], 'bytes_array': b'1234', 'object': {'another one': 456, 'yet another': {'abc': True}}} for opts in ({'container_count': False}, {'container_count': True}): self.check_enc_dec(obj, **opts) # dictionary key sorting obj1 = OrderedDict.fromkeys('abcdefghijkl') obj2 = OrderedDict.fromkeys('abcdefghijkl'[::-1]) self.assertNotEqual(ubjdumpb(obj1), ubjdumpb(obj2)) self.assertEqual(ubjdumpb(obj1, sort_keys=True), ubjdumpb(obj2, sort_keys=True)) # custom mapping class with self.assertRaises(TypeError): ubjloadb(TYPE_NULL, object_pairs_hook=list) self.assertEqual(ubjloadb(ubjdumpb(obj1), object_pairs_hook=OrderedDict), obj1)
def test_high_precision(self): self.assertEqual(ubjdumpb(Decimal(-1.5)), TYPE_HIGH_PREC + TYPE_UINT8 + b'\x04' + '-1.5'.encode('utf-8')) # insufficient length, invalid utf-8, invalid decimal value for suffix in (b'n', b'\xfe\xfe', b'na'): with self.assertRaises(DecoderException): ubjloadb(TYPE_HIGH_PREC + TYPE_UINT8 + b'\x02' + suffix) self.check_enc_dec('1.8e315') for value in ( '0.0', '2.5', '10e30', '-1.2345e67890'): # minimum length because: marker + length marker + length + value self.check_enc_dec(Decimal(value), 4, length_greater_or_equal=True) # cannot compare equality, so test separately (since these evaluate to "NULL" for value in ('nan', '-inf', 'inf'): self.assertEqual(ubjloadb(ubjdumpb(Decimal(value))), None)
def test_int(self): self.assertEqual(ubjdumpb(Decimal(-1.5)), TYPE_HIGH_PREC + TYPE_UINT8 + b'\x04' + '-1.5'.encode('utf-8')) # insufficient length with self.assertRaises(DecoderException): ubjloadb(TYPE_INT16 + b'\x01') for type_, value, total_size in ( (TYPE_UINT8, 0, 2), (TYPE_UINT8, 255, 2), (TYPE_INT8, -128, 2), (TYPE_INT16, -32768, 3), (TYPE_INT16, 32767, 3), (TYPE_INT32, 2147483647, 5), (TYPE_INT32, -2147483648, 5), (TYPE_INT64, 9223372036854775807, 9), (TYPE_INT64, -9223372036854775808, 9), # HIGH_PREC (marker + length marker + length + value) (TYPE_HIGH_PREC, 9223372036854775808, 22), (TYPE_HIGH_PREC, -9223372036854775809, 23), (TYPE_HIGH_PREC, 9999999999999999999999999999999999999, 40)): self.check_enc_dec(value, total_size, expected_type=type_)
def check_enc_dec(self, obj, # total length of encoded object length=None, # total length is at least the given number of bytes length_greater_or_equal=False, # approximate comparison (e.g. for float) approximate=False, # type marker expected at start of encoded output expected_type=None, # additional arguments to pass to encoder **kwargs): """Black-box test to check whether the provided object is the same once encoded and subsequently decoded.""" encoded = ubjdumpb(obj, **kwargs) if expected_type is not None: self.type_check(encoded[0], expected_type) if length is not None: assert_func = self.assertGreaterEqual if length_greater_or_equal else self.assertEqual assert_func(len(encoded), length, self.__format_in_out(obj, encoded)) if approximate: self.assertTrue(self.numbers_close(ubjloadb(encoded), obj), msg=self.__format_in_out(obj, encoded)) else: self.assertEqual(ubjloadb(encoded), obj, self.__format_in_out(obj, encoded))
def __bytes_to_share_data(cls, payload): """Attempt to auto-decode data""" rbytes = payload['data'] mime = payload['mime'] if mime is None: return rbytes, mime mime = expand_idx_mimetype(mime).lower() try: if mime == 'application/ubjson': return ubjloadb(rbytes), None elif mime == 'text/plain; charset=utf8': return rbytes.decode('utf-8'), None else: return rbytes, mime except: logger.warning('auto-decode failed, returning bytes') return rbytes, mime
def ubjloadb(raw, *args, **kwargs): return ubjloadb(raw, *args, **kwargs)
def test_trailing_input(self): self.assertEqual(ubjloadb(TYPE_BOOL_TRUE * 10), True)
def test_invalid_marker(self): with self.assertRaises(DecoderException): ubjloadb(b'A')
def test_fp(self): obj = {"a": 123, "b": 456} output = BytesIO() ubjdump({"a": 123, "b": 456}, output) self.assertEqual(ubjloadb(output.getvalue()), obj)
def test_no_data(self): with self.assertRaises(DecoderException): ubjloadb(b'')
def test_container_fixed(self): raw_start = ARRAY_START + CONTAINER_TYPE + TYPE_INT8 + CONTAINER_COUNT + TYPE_UINT8 self.assertEqual(ubjloadb(raw_start + b'\x00'), []) # fixed-type + count self.assertEqual(ubjloadb(ARRAY_START + CONTAINER_TYPE + TYPE_NULL + CONTAINER_COUNT + TYPE_UINT8 + b'\x05'), [None] * 5) self.assertEqual(ubjloadb(raw_start + b'\x03' + (b'\x01' * 3)), [1, 1, 1]) # invalid type with self.assertRaises(DecoderException): ubjloadb(ARRAY_START + CONTAINER_TYPE + b'\x01') # type without count with self.assertRaises(DecoderException): ubjloadb(ARRAY_START + CONTAINER_TYPE + TYPE_INT8 + b'\x01') raw_start = OBJECT_START + CONTAINER_TYPE + TYPE_INT8 + CONTAINER_COUNT + TYPE_UINT8 self.assertEqual(ubjloadb(raw_start + b'\x00'), {}) self.assertEqual(ubjloadb(raw_start + b'\x03' + (TYPE_UINT8 + b'\x02' + b'aa' + b'\x01' + TYPE_UINT8 + b'\x02' + b'bb' + b'\x02' + TYPE_UINT8 + b'\x02' + b'cc' + b'\x03')), {'aa': 1, 'bb': 2, 'cc': 3}) # fixed type + count self.assertEqual(ubjloadb(OBJECT_START + CONTAINER_TYPE + TYPE_NULL + CONTAINER_COUNT + TYPE_UINT8 + b'\x02' + TYPE_UINT8 + b'\x02' + b'aa' + TYPE_UINT8 + b'\x02' + b'bb'), {'aa': None, 'bb': None}) # fixed type + count (bytes) self.assertEqual(ubjloadb(OBJECT_START + CONTAINER_TYPE + TYPE_UINT8 + CONTAINER_COUNT + TYPE_UINT8 + b'\x02' + TYPE_UINT8 + b'\x02' + b'aa' + b'\x04' + TYPE_UINT8 + b'\x02' + b'bb' + b'\x05'), {'aa': 4, 'bb': 5})