def loads(b):
""" Transform bytestream back into Python value """
header_length, = struct.unpack('I', b[:4])
if header_length:
header = msgpack.loads(b[4: header_length + 4], encoding='utf8')
else:
header = {}
payload = b[header_length + 4:]
if header.get('compression'):
try:
decompress = compressions[header['compression']]['decompress']
payload = decompress(payload)
except KeyError:
raise ValueError("Data is compressed as %s but we don't have this"
" installed" % header['compression'].decode())
msg = msgpack.loads(payload, encoding='utf8')
if header.get('decode'):
if isinstance(msg, dict) and msg:
msg = keymap(bytes.decode, msg)
elif isinstance(msg, bytes):
msg = msg.decode()
else:
raise TypeError("Asked to decode a %s" % type(msg).__name__)
return msg
def loads(b):
""" Transform bytestream back into Python value """
header_length, = struct.unpack('I', b[:4])
if header_length:
header = msgpack.loads(b[4:header_length + 4], encoding='utf8')
else:
header = {}
payload = b[header_length + 4:]
if header.get('compression'):
try:
decompress = compressions[header['compression']]['decompress']
payload = decompress(payload)
except KeyError:
raise ValueError("Data is compressed as %s but we don't have this"
" installed" % header['compression'].decode())
msg = msgpack.loads(payload, encoding='utf8')
if header.get('decode'):
if isinstance(msg, dict) and msg:
msg = keymap(bytes.decode, msg)
elif isinstance(msg, bytes):
msg = msg.decode()
else:
raise TypeError("Asked to decode a %s" % type(msg).__name__)
return msg
def loads_msgpack(header, payload):
""" Read msgpack header and payload back to Python object
See Also:
dumps_msgpack
"""
if header:
header = msgpack.loads(header, encoding='utf8')
else:
header = {}
if header.get('compression'):
try:
decompress = compressions[header['compression']]['decompress']
payload = decompress(payload)
except KeyError:
raise ValueError("Data is compressed as %s but we don't have this"
" installed" % header['compression'].decode())
return msgpack.loads(payload, encoding='utf8')
def loads_msgpack(header, payload):
""" Read msgpack header and payload back to Python object
See Also:
dumps_msgpack
"""
if header:
header = msgpack.loads(header, encoding='utf8')
else:
header = {}
if header.get('compression'):
try:
decompress = compressions[header['compression']]['decompress']
payload = decompress(payload)
except KeyError:
raise ValueError("Data is compressed as %s but we don't have this"
" installed" % str(header['compression']))
return msgpack.loads(payload, encoding='utf8')
def loads_big_byte_dict(header, *values):
""" Deserialize big-byte frames to large byte dictionary
See Also:
dumps_big_byte_dict
"""
header = msgpack.loads(header, encoding='utf8')
values2 = [compressions[c]['decompress'](v)
for c, v in zip(header['compression'], values)]
return dict(zip(header['keys'], values2))
def loads(b):
""" Transform bytestream back into Python value """
header_length, = struct.unpack('I', b[:4])
if header_length:
header = msgpack.loads(b[4:header_length + 4], encoding='utf8')
else:
header = {}
payload = b[header_length + 4:]
if header.get('compression'):
try:
decompress = compressions[header['compression']]['decompress']
payload = decompress(payload)
except KeyError:
raise ValueError("Data is compressed as %s but we don't have this"
" installed" % header['compression'].decode())
msg = msgpack.loads(payload, encoding='utf8')
return msg
def loads(b):
""" Transform bytestream back into Python value """
header_length, = struct.unpack('I', b[:4])
if header_length:
header = msgpack.loads(b[4: header_length + 4], encoding='utf8')
else:
header = {}
payload = b[header_length + 4:]
if header.get('compression'):
try:
decompress = compressions[header['compression']]['decompress']
payload = decompress(payload)
except KeyError:
raise ValueError("Data is compressed as %s but we don't have this"
" installed" % header['compression'].decode())
msg = msgpack.loads(payload, encoding='utf8')
return msg
def loads_big_byte_dict(header, *values):
""" Deserialize big-byte frames to large byte dictionary
See Also:
dumps_big_byte_dict
"""
header = msgpack.loads(header, encoding='utf8')
values2 = [
compressions[c]['decompress'](v)
for c, v in zip(header['compression'], values)
]
return dict(zip(header['keys'], values2))
def loads_big_byte_dict(header, *values):
""" Deserialize big-byte frames to large byte dictionary
See Also:
dumps_big_byte_dict
"""
header = msgpack.loads(header, encoding='utf8')
values2 = [compressions[c]['decompress'](v)
for c, v in zip(header['compression'], values)]
result = dict(zip(header['keys'], values2))
for k, keys in header.get('shards', {}).items():
result[k] = b''.join(result.pop(kk) for kk in keys)
return result
def loads(frames):
""" Transform bytestream back into Python value """
small_header, small_payload, frames = frames[0], frames[1], frames[2:]
msg = loads_msgpack(small_header, small_payload)
if frames:
header = msgpack.loads(frames[0], encoding='utf8')
values2 = [compressions[c]['decompress'](v)
for c, v in zip(header['compression'], frames[1:])]
lk = dict(zip(header['keys'], values2))
for k, keys in header['shards']:
v = b''.join(lk.pop(kk) for kk in keys)
get_in(k[:-1], msg)[k[-1]] = v
return msg
def loads(frames, deserialize=True):
""" Transform bytestream back into Python value """
frames = frames[::-1] # reverse order to improve pop efficiency
if not isinstance(frames, list):
frames = list(frames)
try:
small_header = frames.pop()
small_payload = frames.pop()
msg = loads_msgpack(small_header, small_payload)
if not frames:
return msg
header = frames.pop()
header = msgpack.loads(header, encoding='utf8', use_list=False)
keys = header['keys']
headers = header['headers']
bytestrings = set(header['bytestrings'])
for key in keys:
head = headers[key]
lengths = head['lengths']
count = head['count']
fs = frames[-count::][::-1]
del frames[-count:]
if deserialize or key in bytestrings:
fs = decompress(head, fs)
fs = merge_frames(head, fs)
value = _deserialize(head, fs)
else:
value = Serialized(head, fs)
get_in(key[:-1], msg)[key[-1]] = value
return msg
except Exception as e:
logger.critical("Failed to deserialize", exc_info=True)
raise