def deserialize(self, msg_list, content=True, copy=True):
"""Unserialize a msg_list to a nested message dict.
This is roughly the inverse of serialize. The serialize/deserialize
methods work with full message lists, whereas pack/unpack work with
the individual message parts in the message list.
Parameters
----------
msg_list : list of bytes or Message objects
The list of message parts of the form [HMAC,p_header,p_parent,
p_metadata,p_content,buffer1,buffer2,...].
content : bool (True)
Whether to unpack the content dict (True), or leave it packed
(False).
copy : bool (True)
Whether msg_list contains bytes (True) or the non-copying Message
objects in each place (False).
Returns
-------
msg : dict
The nested message dict with top-level keys [header, parent_header,
content, buffers]. The buffers are returned as memoryviews.
"""
minlen = 5
message = {}
if not copy:
# pyzmq didn't copy the first parts of the message, so we'll do it
for i in range(minlen):
msg_list[i] = msg_list[i].bytes
if self.auth is not None:
signature = msg_list[0]
if not signature:
raise ValueError("Unsigned Message")
if signature in self.digest_history:
raise ValueError("Duplicate Signature: %r" % signature)
self._add_digest(signature)
check = self.sign(msg_list[1:5])
if not compare_digest(signature, check):
raise ValueError("Invalid Signature: %r" % signature)
if not len(msg_list) >= minlen:
raise TypeError("malformed message, must have at least %i elements"%minlen)
header = self.unpack(msg_list[1])
message['header'] = extract_dates(header)
message['msg_id'] = header['msg_id']
message['msg_type'] = header['msg_type']
message['parent_header'] = extract_dates(self.unpack(msg_list[2]))
message['metadata'] = self.unpack(msg_list[3])
if content:
message['content'] = self.unpack(msg_list[4])
else:
message['content'] = msg_list[4]
buffers = [memoryview(b) for b in msg_list[5:]]
if buffers and buffers[0].shape is None:
# force copy to workaround pyzmq #646
buffers = [memoryview(b.bytes) for b in msg_list[5:]]
message['buffers'] = buffers
# adapt to the current version
return adapt(message)
def deserialize(self, msg_list, content=True, copy=True):
"""Unserialize a msg_list to a nested message dict.
This is roughly the inverse of serialize. The serialize/deserialize
methods work with full message lists, whereas pack/unpack work with
the individual message parts in the message list.
Parameters
----------
msg_list : list of bytes or Message objects
The list of message parts of the form [HMAC,p_header,p_parent,
p_metadata,p_content,buffer1,buffer2,...].
content : bool (True)
Whether to unpack the content dict (True), or leave it packed
(False).
copy : bool (True)
Whether msg_list contains bytes (True) or the non-copying Message
objects in each place (False).
Returns
-------
msg : dict
The nested message dict with top-level keys [header, parent_header,
content, buffers]. The buffers are returned as memoryviews.
"""
minlen = 5
message = {}
if not copy:
# pyzmq didn't copy the first parts of the message, so we'll do it
for i in range(minlen):
msg_list[i] = msg_list[i].bytes
if self.auth is not None:
signature = msg_list[0]
if not signature:
raise ValueError("Unsigned Message")
if signature in self.digest_history:
raise ValueError("Duplicate Signature: %r" % signature)
self._add_digest(signature)
check = self.sign(msg_list[1:5])
if not compare_digest(signature, check):
raise ValueError("Invalid Signature: %r" % signature)
if not len(msg_list) >= minlen:
raise TypeError("malformed message, must have at least %i elements"%minlen)
header = self.unpack(msg_list[1])
message['header'] = extract_dates(header)
message['msg_id'] = header['msg_id']
message['msg_type'] = header['msg_type']
message['parent_header'] = extract_dates(self.unpack(msg_list[2]))
message['metadata'] = self.unpack(msg_list[3])
if content:
message['content'] = self.unpack(msg_list[4])
else:
message['content'] = msg_list[4]
buffers = [memoryview(b) for b in msg_list[5:]]
if buffers and buffers[0].shape is None:
# force copy to workaround pyzmq #646
buffers = [memoryview(b.bytes) for b in msg_list[5:]]
message['buffers'] = buffers
# adapt to the current version
return adapt(message)
def deserialize_binary_message(bmsg):
"""deserialize a message from a binary blog
Header:
4 bytes: number of msg parts (nbufs) as 32b int
4 * nbufs bytes: offset for each buffer as integer as 32b int
Offsets are from the start of the buffer, including the header.
Returns
-------
message dictionary
"""
nbufs = struct.unpack('!i', bmsg[:4])[0]
offsets = list(struct.unpack('!' + 'I' * nbufs, bmsg[4:4 * (nbufs + 1)]))
offsets.append(None)
bufs = []
for start, stop in zip(offsets[:-1], offsets[1:]):
bufs.append(bmsg[start:stop])
msg = json.loads(bufs[0].decode('utf8'))
msg['header'] = extract_dates(msg['header'])
msg['parent_header'] = extract_dates(msg['parent_header'])
msg['buffers'] = bufs[1:]
return msg
def deserialize_binary_message(bmsg):
"""deserialize a message from a binary blog
Header:
4 bytes: number of msg parts (nbufs) as 32b int
4 * nbufs bytes: offset for each buffer as integer as 32b int
Offsets are from the start of the buffer, including the header.
Returns
-------
message dictionary
"""
nbufs = struct.unpack('!i', bmsg[:4])[0]
offsets = list(struct.unpack('!' + 'I' * nbufs, bmsg[4:4*(nbufs+1)]))
offsets.append(None)
bufs = []
for start, stop in zip(offsets[:-1], offsets[1:]):
bufs.append(bmsg[start:stop])
msg = json.loads(bufs[0].decode('utf8'))
msg['header'] = extract_dates(msg['header'])
msg['parent_header'] = extract_dates(msg['parent_header'])
msg['buffers'] = bufs[1:]
return msg
def _convert_dict(ds):
if ds is None:
return ds
else:
if isinstance(ds, bytes):
# If I understand the sqlite doc correctly, this will always be utf8
ds = ds.decode('utf8')
return extract_dates(json.loads(ds))
def _convert_dict(ds):
if ds is None:
return ds
else:
if isinstance(ds, bytes):
# If I understand the sqlite doc correctly, this will always be utf8
ds = ds.decode('utf8')
return extract_dates(json.loads(ds))
def _datetime_test(self, session):
content = dict(t=ss.utcnow())
metadata = dict(t=ss.utcnow())
p = session.msg("msg")
msg = session.msg("msg", content=content, metadata=metadata, parent=p["header"])
smsg = session.serialize(msg)
msg2 = session.deserialize(session.feed_identities(smsg)[1])
assert isinstance(msg2["header"]["date"], datetime)
self.assertEqual(msg["header"], msg2["header"])
self.assertEqual(msg["parent_header"], msg2["parent_header"])
self.assertEqual(msg["parent_header"], msg2["parent_header"])
assert isinstance(msg["content"]["t"], datetime)
assert isinstance(msg["metadata"]["t"], datetime)
assert isinstance(msg2["content"]["t"], str)
assert isinstance(msg2["metadata"]["t"], str)
self.assertEqual(msg["content"], jsonutil.extract_dates(msg2["content"]))
self.assertEqual(msg["content"], jsonutil.extract_dates(msg2["content"]))
def _datetime_test(self, session):
content = dict(t=ss.utcnow())
metadata = dict(t=ss.utcnow())
p = session.msg('msg')
msg = session.msg('msg', content=content, metadata=metadata, parent=p['header'])
smsg = session.serialize(msg)
msg2 = session.deserialize(session.feed_identities(smsg)[1])
assert isinstance(msg2['header']['date'], datetime)
self.assertEqual(msg['header'], msg2['header'])
self.assertEqual(msg['parent_header'], msg2['parent_header'])
self.assertEqual(msg['parent_header'], msg2['parent_header'])
assert isinstance(msg['content']['t'], datetime)
assert isinstance(msg['metadata']['t'], datetime)
assert isinstance(msg2['content']['t'], string_types)
assert isinstance(msg2['metadata']['t'], string_types)
self.assertEqual(msg['content'], jsonutil.extract_dates(msg2['content']))
self.assertEqual(msg['content'], jsonutil.extract_dates(msg2['content']))
def test_date_default():
now = datetime.datetime.now()
utcnow = now.replace(tzinfo=UTC)
data = dict(now=now, utcnow=utcnow)
jsondata = json.dumps(data, default=jsonutil.date_default)
nt.assert_in("+00", jsondata)
nt.assert_equal(jsondata.count("+00"), 1)
extracted = jsonutil.extract_dates(json.loads(jsondata))
for dt in extracted.values():
nt.assert_is_instance(dt, datetime.datetime)
def test_date_default():
now = today=datetime.datetime.now()
utcnow = now.replace(tzinfo=UTC)
data = dict(now=now, utcnow=utcnow)
jsondata = json.dumps(data, default=jsonutil.date_default)
nt.assert_in("+00", jsondata)
nt.assert_equal(jsondata.count("+00"), 1)
extracted = jsonutil.extract_dates(json.loads(jsondata))
for dt in extracted.values():
nt.assert_is_instance(dt, datetime.datetime)
def test_extract_date_from_naive():
ref = REFERENCE_DATETIME
timestamp = "2013-07-03T16:34:52.249482"
with pytest.deprecated_call(match="Interpreting naive datetime as local"):
extracted = jsonutil.extract_dates(timestamp)
assert isinstance(extracted, datetime.datetime)
assert extracted.tzinfo is not None
assert extracted.tzinfo.utcoffset(ref) == tzlocal().utcoffset(ref)
assert extracted == ref
def test_date_default():
naive = datetime.datetime.now()
local = tzoffset('Local', -8 * 3600)
other = tzoffset('Other', 2 * 3600)
data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other))
with mock.patch.object(jsonutil, 'tzlocal', lambda: local):
jsondata = json.dumps(data, default=jsonutil.date_default)
assert "Z" in jsondata
assert jsondata.count("Z") == 1
extracted = jsonutil.extract_dates(json.loads(jsondata))
for dt in extracted.values():
assert isinstance(dt, datetime.datetime)
assert dt.tzinfo != None
def test_date_default():
naive = datetime.datetime.now()
local = tzoffset('Local', -8 * 3600)
other = tzoffset('Other', 2 * 3600)
data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other))
with mock.patch.object(jsonutil, 'tzlocal', lambda : local):
jsondata = json.dumps(data, default=jsonutil.date_default)
nt.assert_in("Z", jsondata)
nt.assert_equal(jsondata.count("Z"), 1)
extracted = jsonutil.extract_dates(json.loads(jsondata))
for dt in extracted.values():
nt.assert_is_instance(dt, datetime.datetime)
nt.assert_not_equal(dt.tzinfo, None)
def test_date_default():
naive = datetime.datetime.now()
local = tzoffset('Local', -8 * 3600)
other = tzoffset('Other', 2 * 3600)
data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other))
with mock.patch.object(jsonutil, 'tzlocal', lambda: local):
jsondata = json.dumps(data, default=jsonutil.date_default)
nt.assert_in("Z", jsondata)
nt.assert_equal(jsondata.count("Z"), 1)
extracted = jsonutil.extract_dates(json.loads(jsondata))
for dt in extracted.values():
nt.assert_is_instance(dt, datetime.datetime)
nt.assert_not_equal(dt.tzinfo, None)
def test_date_default():
naive = datetime.datetime.now()
local = tzoffset('Local', -8 * 3600)
other = tzoffset('Other', 2 * 3600)
data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other))
with mock.patch.object(jsonutil, 'tzlocal', lambda : local):
jsondata = json.dumps(data, default=jsonutil.date_default)
assert "Z" in jsondata
assert jsondata.count("Z") == 1
extracted = jsonutil.extract_dates(json.loads(jsondata))
for dt in extracted.values():
assert isinstance(dt, datetime.datetime)
assert dt.tzinfo != None
def test_date_default():
naive = datetime.datetime.now()
local = tzoffset("Local", -8 * 3600)
other = tzoffset("Other", 2 * 3600)
data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other))
with mock.patch.object(jsonutil, "tzlocal", lambda: local):
with pytest.deprecated_call(match="Please add timezone info"):
jsondata = json.dumps(data, default=jsonutil.date_default)
assert "Z" in jsondata
assert jsondata.count("Z") == 1
extracted = jsonutil.extract_dates(json.loads(jsondata))
for dt in extracted.values():
assert isinstance(dt, datetime.datetime)
assert dt.tzinfo is not None
def test_extract_dates():
timestamps = [
'2013-07-03T16:34:52.249482',
'2013-07-03T16:34:52.249482Z',
'2013-07-03T16:34:52.249482Z-0800',
'2013-07-03T16:34:52.249482Z+0800',
'2013-07-03T16:34:52.249482Z+08:00',
'2013-07-03T16:34:52.249482Z-08:00',
'2013-07-03T16:34:52.249482-0800',
'2013-07-03T16:34:52.249482+0800',
'2013-07-03T16:34:52.249482+08:00',
'2013-07-03T16:34:52.249482-08:00',
]
extracted = jsonutil.extract_dates(timestamps)
ref = extracted[0]
for dt in extracted:
nt.assert_true(isinstance(dt, datetime.datetime))
nt.assert_equal(dt, ref)
def test_extract_dates():
timestamps = [
'2013-07-03T16:34:52.249482',
'2013-07-03T16:34:52.249482Z',
'2013-07-03T16:34:52.249482Z-0800',
'2013-07-03T16:34:52.249482Z+0800',
'2013-07-03T16:34:52.249482Z+08:00',
'2013-07-03T16:34:52.249482Z-08:00',
'2013-07-03T16:34:52.249482-0800',
'2013-07-03T16:34:52.249482+0800',
'2013-07-03T16:34:52.249482+08:00',
'2013-07-03T16:34:52.249482-08:00',
]
extracted = jsonutil.extract_dates(timestamps)
ref = extracted[0]
for dt in extracted:
nt.assert_true(isinstance(dt, datetime.datetime))
nt.assert_equal(dt, ref)
def test_extract_dates():
ref = REFERENCE_DATETIME
timestamps = [
"2013-07-03T16:34:52.249482Z",
"2013-07-03T16:34:52.249482-0800",
"2013-07-03T16:34:52.249482+0800",
"2013-07-03T16:34:52.249482-08:00",
"2013-07-03T16:34:52.249482+08:00",
]
extracted = jsonutil.extract_dates(timestamps)
for dt in extracted:
assert isinstance(dt, datetime.datetime)
assert dt.tzinfo is not None
assert extracted[0].tzinfo.utcoffset(ref) == timedelta(0)
assert extracted[1].tzinfo.utcoffset(ref) == timedelta(hours=-8)
assert extracted[2].tzinfo.utcoffset(ref) == timedelta(hours=8)
assert extracted[3].tzinfo.utcoffset(ref) == timedelta(hours=-8)
assert extracted[4].tzinfo.utcoffset(ref) == timedelta(hours=8)
def test_extract_dates():
timestamps = [
'2013-07-03T16:34:52.249482',
'2013-07-03T16:34:52.249482Z',
'2013-07-03T16:34:52.249482-0800',
'2013-07-03T16:34:52.249482+0800',
'2013-07-03T16:34:52.249482-08:00',
'2013-07-03T16:34:52.249482+08:00',
]
extracted = jsonutil.extract_dates(timestamps)
ref = extracted[0]
for dt in extracted:
assert isinstance(dt, datetime.datetime)
assert dt.tzinfo != None
assert extracted[0].tzinfo.utcoffset(ref) == tzlocal().utcoffset(ref)
assert extracted[1].tzinfo.utcoffset(ref) == timedelta(0)
assert extracted[2].tzinfo.utcoffset(ref) == timedelta(hours=-8)
assert extracted[3].tzinfo.utcoffset(ref) == timedelta(hours=8)
assert extracted[4].tzinfo.utcoffset(ref) == timedelta(hours=-8)
assert extracted[5].tzinfo.utcoffset(ref) == timedelta(hours=8)
def test_extract_dates():
timestamps = [
'2013-07-03T16:34:52.249482',
'2013-07-03T16:34:52.249482Z',
'2013-07-03T16:34:52.249482-0800',
'2013-07-03T16:34:52.249482+0800',
'2013-07-03T16:34:52.249482-08:00',
'2013-07-03T16:34:52.249482+08:00',
]
extracted = jsonutil.extract_dates(timestamps)
ref = extracted[0]
for dt in extracted:
nt.assert_true(isinstance(dt, datetime.datetime))
nt.assert_not_equal(dt.tzinfo, None)
nt.assert_equal(extracted[0].tzinfo.utcoffset(ref), tzlocal().utcoffset(ref))
nt.assert_equal(extracted[1].tzinfo.utcoffset(ref), timedelta(0))
nt.assert_equal(extracted[2].tzinfo.utcoffset(ref), timedelta(hours=-8))
nt.assert_equal(extracted[3].tzinfo.utcoffset(ref), timedelta(hours=8))
nt.assert_equal(extracted[4].tzinfo.utcoffset(ref), timedelta(hours=-8))
nt.assert_equal(extracted[5].tzinfo.utcoffset(ref), timedelta(hours=8))
def test_extract_dates():
timestamps = [
'2013-07-03T16:34:52.249482',
'2013-07-03T16:34:52.249482Z',
'2013-07-03T16:34:52.249482-0800',
'2013-07-03T16:34:52.249482+0800',
'2013-07-03T16:34:52.249482-08:00',
'2013-07-03T16:34:52.249482+08:00',
]
extracted = jsonutil.extract_dates(timestamps)
ref = extracted[0]
for dt in extracted:
assert isinstance(dt, datetime.datetime)
assert dt.tzinfo != None
assert extracted[0].tzinfo.utcoffset(ref) == tzlocal().utcoffset(ref)
assert extracted[1].tzinfo.utcoffset(ref) == timedelta(0)
assert extracted[2].tzinfo.utcoffset(ref) == timedelta(hours=-8)
assert extracted[3].tzinfo.utcoffset(ref) == timedelta(hours=8)
assert extracted[4].tzinfo.utcoffset(ref) == timedelta(hours=-8)
assert extracted[5].tzinfo.utcoffset(ref) == timedelta(hours=8)
def test_extract_dates():
timestamps = [
'2013-07-03T16:34:52.249482',
'2013-07-03T16:34:52.249482Z',
'2013-07-03T16:34:52.249482-0800',
'2013-07-03T16:34:52.249482+0800',
'2013-07-03T16:34:52.249482-08:00',
'2013-07-03T16:34:52.249482+08:00',
]
extracted = jsonutil.extract_dates(timestamps)
ref = extracted[0]
for dt in extracted:
nt.assert_true(isinstance(dt, datetime.datetime))
nt.assert_not_equal(dt.tzinfo, None)
nt.assert_equal(extracted[0].tzinfo.utcoffset(ref),
tzlocal().utcoffset(ref))
nt.assert_equal(extracted[1].tzinfo.utcoffset(ref), timedelta(0))
nt.assert_equal(extracted[2].tzinfo.utcoffset(ref), timedelta(hours=-8))
nt.assert_equal(extracted[3].tzinfo.utcoffset(ref), timedelta(hours=8))
nt.assert_equal(extracted[4].tzinfo.utcoffset(ref), timedelta(hours=-8))
nt.assert_equal(extracted[5].tzinfo.utcoffset(ref), timedelta(hours=8))
def deserialize(
self,
msg_list: t.Union[t.List[bytes], t.List[zmq.Message]],
content: bool = True,
copy: bool = True,
) -> t.Dict[str, t.Any]:
"""Unserialize a msg_list to a nested message dict.
This is roughly the inverse of serialize. The serialize/deserialize
methods work with full message lists, whereas pack/unpack work with
the individual message parts in the message list.
Parameters
----------
msg_list : list of bytes or Message objects
The list of message parts of the form [HMAC,p_header,p_parent,
p_metadata,p_content,buffer1,buffer2,...].
content : bool (True)
Whether to unpack the content dict (True), or leave it packed
(False).
copy : bool (True)
Whether msg_list contains bytes (True) or the non-copying Message
objects in each place (False).
Returns
-------
msg : dict
The nested message dict with top-level keys [header, parent_header,
content, buffers]. The buffers are returned as memoryviews.
"""
minlen = 5
message = {}
if not copy:
# pyzmq didn't copy the first parts of the message, so we'll do it
msg_list = t.cast(t.List[zmq.Message], msg_list)
msg_list_beginning = [
bytes(msg.bytes) for msg in msg_list[:minlen]
]
msg_list = t.cast(t.List[bytes], msg_list)
msg_list = msg_list_beginning + msg_list[minlen:]
msg_list = t.cast(t.List[bytes], msg_list)
if self.auth is not None:
signature = t.cast(bytes, msg_list[0])
if not signature:
raise ValueError("Unsigned Message")
if signature in self.digest_history:
raise ValueError("Duplicate Signature: %r" % signature)
if content:
# Only store signature if we are unpacking content, don't store if just peeking.
self._add_digest(signature)
check = self.sign(msg_list[1:5])
if not compare_digest(signature, check):
raise ValueError("Invalid Signature: %r" % signature)
if not len(msg_list) >= minlen:
raise TypeError(
"malformed message, must have at least %i elements" % minlen)
header = self.unpack(msg_list[1])
message["header"] = extract_dates(header)
message["msg_id"] = header["msg_id"]
message["msg_type"] = header["msg_type"]
message["parent_header"] = extract_dates(self.unpack(msg_list[2]))
message["metadata"] = self.unpack(msg_list[3])
if content:
message["content"] = self.unpack(msg_list[4])
else:
message["content"] = msg_list[4]
buffers = [memoryview(b) for b in msg_list[5:]]
if buffers and buffers[0].shape is None:
# force copy to workaround pyzmq #646
msg_list = t.cast(t.List[zmq.Message], msg_list)
buffers = [memoryview(bytes(b.bytes)) for b in msg_list[5:]]
message["buffers"] = buffers
if self.debug:
pprint.pprint(message)
# adapt to the current version
return adapt(message)
