def deserialize(self, msg_list, content=True, copy=True): """Unserialize a msg_list to a nested message dict. This is roughly the inverse of serialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg_list : list of bytes or Message objects The list of message parts of the form [HMAC,p_header,p_parent, p_metadata,p_content,buffer1,buffer2,...]. content : bool (True) Whether to unpack the content dict (True), or leave it packed (False). copy : bool (True) Whether msg_list contains bytes (True) or the non-copying Message objects in each place (False). Returns ------- msg : dict The nested message dict with top-level keys [header, parent_header, content, buffers]. The buffers are returned as memoryviews. """ minlen = 5 message = {} if not copy: # pyzmq didn't copy the first parts of the message, so we'll do it for i in range(minlen): msg_list[i] = msg_list[i].bytes if self.auth is not None: signature = msg_list[0] if not signature: raise ValueError("Unsigned Message") if signature in self.digest_history: raise ValueError("Duplicate Signature: %r" % signature) self._add_digest(signature) check = self.sign(msg_list[1:5]) if not compare_digest(signature, check): raise ValueError("Invalid Signature: %r" % signature) if not len(msg_list) >= minlen: raise TypeError("malformed message, must have at least %i elements"%minlen) header = self.unpack(msg_list[1]) message['header'] = extract_dates(header) message['msg_id'] = header['msg_id'] message['msg_type'] = header['msg_type'] message['parent_header'] = extract_dates(self.unpack(msg_list[2])) message['metadata'] = self.unpack(msg_list[3]) if content: message['content'] = self.unpack(msg_list[4]) else: message['content'] = msg_list[4] buffers = [memoryview(b) for b in msg_list[5:]] if buffers and buffers[0].shape is None: # force copy to workaround pyzmq #646 buffers = [memoryview(b.bytes) for b in msg_list[5:]] message['buffers'] = buffers # adapt to the current version return adapt(message)
def deserialize_binary_message(bmsg): """deserialize a message from a binary blog Header: 4 bytes: number of msg parts (nbufs) as 32b int 4 * nbufs bytes: offset for each buffer as integer as 32b int Offsets are from the start of the buffer, including the header. Returns ------- message dictionary """ nbufs = struct.unpack('!i', bmsg[:4])[0] offsets = list(struct.unpack('!' + 'I' * nbufs, bmsg[4:4 * (nbufs + 1)])) offsets.append(None) bufs = [] for start, stop in zip(offsets[:-1], offsets[1:]): bufs.append(bmsg[start:stop]) msg = json.loads(bufs[0].decode('utf8')) msg['header'] = extract_dates(msg['header']) msg['parent_header'] = extract_dates(msg['parent_header']) msg['buffers'] = bufs[1:] return msg
def deserialize_binary_message(bmsg): """deserialize a message from a binary blog Header: 4 bytes: number of msg parts (nbufs) as 32b int 4 * nbufs bytes: offset for each buffer as integer as 32b int Offsets are from the start of the buffer, including the header. Returns ------- message dictionary """ nbufs = struct.unpack('!i', bmsg[:4])[0] offsets = list(struct.unpack('!' + 'I' * nbufs, bmsg[4:4*(nbufs+1)])) offsets.append(None) bufs = [] for start, stop in zip(offsets[:-1], offsets[1:]): bufs.append(bmsg[start:stop]) msg = json.loads(bufs[0].decode('utf8')) msg['header'] = extract_dates(msg['header']) msg['parent_header'] = extract_dates(msg['parent_header']) msg['buffers'] = bufs[1:] return msg
def _convert_dict(ds): if ds is None: return ds else: if isinstance(ds, bytes): # If I understand the sqlite doc correctly, this will always be utf8 ds = ds.decode('utf8') return extract_dates(json.loads(ds))
def _datetime_test(self, session): content = dict(t=ss.utcnow()) metadata = dict(t=ss.utcnow()) p = session.msg("msg") msg = session.msg("msg", content=content, metadata=metadata, parent=p["header"]) smsg = session.serialize(msg) msg2 = session.deserialize(session.feed_identities(smsg)[1]) assert isinstance(msg2["header"]["date"], datetime) self.assertEqual(msg["header"], msg2["header"]) self.assertEqual(msg["parent_header"], msg2["parent_header"]) self.assertEqual(msg["parent_header"], msg2["parent_header"]) assert isinstance(msg["content"]["t"], datetime) assert isinstance(msg["metadata"]["t"], datetime) assert isinstance(msg2["content"]["t"], str) assert isinstance(msg2["metadata"]["t"], str) self.assertEqual(msg["content"], jsonutil.extract_dates(msg2["content"])) self.assertEqual(msg["content"], jsonutil.extract_dates(msg2["content"]))
def _datetime_test(self, session): content = dict(t=ss.utcnow()) metadata = dict(t=ss.utcnow()) p = session.msg('msg') msg = session.msg('msg', content=content, metadata=metadata, parent=p['header']) smsg = session.serialize(msg) msg2 = session.deserialize(session.feed_identities(smsg)[1]) assert isinstance(msg2['header']['date'], datetime) self.assertEqual(msg['header'], msg2['header']) self.assertEqual(msg['parent_header'], msg2['parent_header']) self.assertEqual(msg['parent_header'], msg2['parent_header']) assert isinstance(msg['content']['t'], datetime) assert isinstance(msg['metadata']['t'], datetime) assert isinstance(msg2['content']['t'], string_types) assert isinstance(msg2['metadata']['t'], string_types) self.assertEqual(msg['content'], jsonutil.extract_dates(msg2['content'])) self.assertEqual(msg['content'], jsonutil.extract_dates(msg2['content']))
def test_date_default(): now = datetime.datetime.now() utcnow = now.replace(tzinfo=UTC) data = dict(now=now, utcnow=utcnow) jsondata = json.dumps(data, default=jsonutil.date_default) nt.assert_in("+00", jsondata) nt.assert_equal(jsondata.count("+00"), 1) extracted = jsonutil.extract_dates(json.loads(jsondata)) for dt in extracted.values(): nt.assert_is_instance(dt, datetime.datetime)
def test_date_default(): now = today=datetime.datetime.now() utcnow = now.replace(tzinfo=UTC) data = dict(now=now, utcnow=utcnow) jsondata = json.dumps(data, default=jsonutil.date_default) nt.assert_in("+00", jsondata) nt.assert_equal(jsondata.count("+00"), 1) extracted = jsonutil.extract_dates(json.loads(jsondata)) for dt in extracted.values(): nt.assert_is_instance(dt, datetime.datetime)
def test_extract_date_from_naive(): ref = REFERENCE_DATETIME timestamp = "2013-07-03T16:34:52.249482" with pytest.deprecated_call(match="Interpreting naive datetime as local"): extracted = jsonutil.extract_dates(timestamp) assert isinstance(extracted, datetime.datetime) assert extracted.tzinfo is not None assert extracted.tzinfo.utcoffset(ref) == tzlocal().utcoffset(ref) assert extracted == ref
def test_date_default(): naive = datetime.datetime.now() local = tzoffset('Local', -8 * 3600) other = tzoffset('Other', 2 * 3600) data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other)) with mock.patch.object(jsonutil, 'tzlocal', lambda: local): jsondata = json.dumps(data, default=jsonutil.date_default) assert "Z" in jsondata assert jsondata.count("Z") == 1 extracted = jsonutil.extract_dates(json.loads(jsondata)) for dt in extracted.values(): assert isinstance(dt, datetime.datetime) assert dt.tzinfo != None
def test_date_default(): naive = datetime.datetime.now() local = tzoffset('Local', -8 * 3600) other = tzoffset('Other', 2 * 3600) data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other)) with mock.patch.object(jsonutil, 'tzlocal', lambda : local): jsondata = json.dumps(data, default=jsonutil.date_default) nt.assert_in("Z", jsondata) nt.assert_equal(jsondata.count("Z"), 1) extracted = jsonutil.extract_dates(json.loads(jsondata)) for dt in extracted.values(): nt.assert_is_instance(dt, datetime.datetime) nt.assert_not_equal(dt.tzinfo, None)
def test_date_default(): naive = datetime.datetime.now() local = tzoffset('Local', -8 * 3600) other = tzoffset('Other', 2 * 3600) data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other)) with mock.patch.object(jsonutil, 'tzlocal', lambda: local): jsondata = json.dumps(data, default=jsonutil.date_default) nt.assert_in("Z", jsondata) nt.assert_equal(jsondata.count("Z"), 1) extracted = jsonutil.extract_dates(json.loads(jsondata)) for dt in extracted.values(): nt.assert_is_instance(dt, datetime.datetime) nt.assert_not_equal(dt.tzinfo, None)
def test_date_default(): naive = datetime.datetime.now() local = tzoffset('Local', -8 * 3600) other = tzoffset('Other', 2 * 3600) data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other)) with mock.patch.object(jsonutil, 'tzlocal', lambda : local): jsondata = json.dumps(data, default=jsonutil.date_default) assert "Z" in jsondata assert jsondata.count("Z") == 1 extracted = jsonutil.extract_dates(json.loads(jsondata)) for dt in extracted.values(): assert isinstance(dt, datetime.datetime) assert dt.tzinfo != None
def test_date_default(): naive = datetime.datetime.now() local = tzoffset("Local", -8 * 3600) other = tzoffset("Other", 2 * 3600) data = dict(naive=naive, utc=utcnow(), withtz=naive.replace(tzinfo=other)) with mock.patch.object(jsonutil, "tzlocal", lambda: local): with pytest.deprecated_call(match="Please add timezone info"): jsondata = json.dumps(data, default=jsonutil.date_default) assert "Z" in jsondata assert jsondata.count("Z") == 1 extracted = jsonutil.extract_dates(json.loads(jsondata)) for dt in extracted.values(): assert isinstance(dt, datetime.datetime) assert dt.tzinfo is not None
def test_extract_dates(): timestamps = [ '2013-07-03T16:34:52.249482', '2013-07-03T16:34:52.249482Z', '2013-07-03T16:34:52.249482Z-0800', '2013-07-03T16:34:52.249482Z+0800', '2013-07-03T16:34:52.249482Z+08:00', '2013-07-03T16:34:52.249482Z-08:00', '2013-07-03T16:34:52.249482-0800', '2013-07-03T16:34:52.249482+0800', '2013-07-03T16:34:52.249482+08:00', '2013-07-03T16:34:52.249482-08:00', ] extracted = jsonutil.extract_dates(timestamps) ref = extracted[0] for dt in extracted: nt.assert_true(isinstance(dt, datetime.datetime)) nt.assert_equal(dt, ref)
def test_extract_dates(): ref = REFERENCE_DATETIME timestamps = [ "2013-07-03T16:34:52.249482Z", "2013-07-03T16:34:52.249482-0800", "2013-07-03T16:34:52.249482+0800", "2013-07-03T16:34:52.249482-08:00", "2013-07-03T16:34:52.249482+08:00", ] extracted = jsonutil.extract_dates(timestamps) for dt in extracted: assert isinstance(dt, datetime.datetime) assert dt.tzinfo is not None assert extracted[0].tzinfo.utcoffset(ref) == timedelta(0) assert extracted[1].tzinfo.utcoffset(ref) == timedelta(hours=-8) assert extracted[2].tzinfo.utcoffset(ref) == timedelta(hours=8) assert extracted[3].tzinfo.utcoffset(ref) == timedelta(hours=-8) assert extracted[4].tzinfo.utcoffset(ref) == timedelta(hours=8)
def test_extract_dates(): timestamps = [ '2013-07-03T16:34:52.249482', '2013-07-03T16:34:52.249482Z', '2013-07-03T16:34:52.249482-0800', '2013-07-03T16:34:52.249482+0800', '2013-07-03T16:34:52.249482-08:00', '2013-07-03T16:34:52.249482+08:00', ] extracted = jsonutil.extract_dates(timestamps) ref = extracted[0] for dt in extracted: assert isinstance(dt, datetime.datetime) assert dt.tzinfo != None assert extracted[0].tzinfo.utcoffset(ref) == tzlocal().utcoffset(ref) assert extracted[1].tzinfo.utcoffset(ref) == timedelta(0) assert extracted[2].tzinfo.utcoffset(ref) == timedelta(hours=-8) assert extracted[3].tzinfo.utcoffset(ref) == timedelta(hours=8) assert extracted[4].tzinfo.utcoffset(ref) == timedelta(hours=-8) assert extracted[5].tzinfo.utcoffset(ref) == timedelta(hours=8)
def test_extract_dates(): timestamps = [ '2013-07-03T16:34:52.249482', '2013-07-03T16:34:52.249482Z', '2013-07-03T16:34:52.249482-0800', '2013-07-03T16:34:52.249482+0800', '2013-07-03T16:34:52.249482-08:00', '2013-07-03T16:34:52.249482+08:00', ] extracted = jsonutil.extract_dates(timestamps) ref = extracted[0] for dt in extracted: nt.assert_true(isinstance(dt, datetime.datetime)) nt.assert_not_equal(dt.tzinfo, None) nt.assert_equal(extracted[0].tzinfo.utcoffset(ref), tzlocal().utcoffset(ref)) nt.assert_equal(extracted[1].tzinfo.utcoffset(ref), timedelta(0)) nt.assert_equal(extracted[2].tzinfo.utcoffset(ref), timedelta(hours=-8)) nt.assert_equal(extracted[3].tzinfo.utcoffset(ref), timedelta(hours=8)) nt.assert_equal(extracted[4].tzinfo.utcoffset(ref), timedelta(hours=-8)) nt.assert_equal(extracted[5].tzinfo.utcoffset(ref), timedelta(hours=8))
def deserialize( self, msg_list: t.Union[t.List[bytes], t.List[zmq.Message]], content: bool = True, copy: bool = True, ) -> t.Dict[str, t.Any]: """Unserialize a msg_list to a nested message dict. This is roughly the inverse of serialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg_list : list of bytes or Message objects The list of message parts of the form [HMAC,p_header,p_parent, p_metadata,p_content,buffer1,buffer2,...]. content : bool (True) Whether to unpack the content dict (True), or leave it packed (False). copy : bool (True) Whether msg_list contains bytes (True) or the non-copying Message objects in each place (False). Returns ------- msg : dict The nested message dict with top-level keys [header, parent_header, content, buffers]. The buffers are returned as memoryviews. """ minlen = 5 message = {} if not copy: # pyzmq didn't copy the first parts of the message, so we'll do it msg_list = t.cast(t.List[zmq.Message], msg_list) msg_list_beginning = [ bytes(msg.bytes) for msg in msg_list[:minlen] ] msg_list = t.cast(t.List[bytes], msg_list) msg_list = msg_list_beginning + msg_list[minlen:] msg_list = t.cast(t.List[bytes], msg_list) if self.auth is not None: signature = t.cast(bytes, msg_list[0]) if not signature: raise ValueError("Unsigned Message") if signature in self.digest_history: raise ValueError("Duplicate Signature: %r" % signature) if content: # Only store signature if we are unpacking content, don't store if just peeking. self._add_digest(signature) check = self.sign(msg_list[1:5]) if not compare_digest(signature, check): raise ValueError("Invalid Signature: %r" % signature) if not len(msg_list) >= minlen: raise TypeError( "malformed message, must have at least %i elements" % minlen) header = self.unpack(msg_list[1]) message["header"] = extract_dates(header) message["msg_id"] = header["msg_id"] message["msg_type"] = header["msg_type"] message["parent_header"] = extract_dates(self.unpack(msg_list[2])) message["metadata"] = self.unpack(msg_list[3]) if content: message["content"] = self.unpack(msg_list[4]) else: message["content"] = msg_list[4] buffers = [memoryview(b) for b in msg_list[5:]] if buffers and buffers[0].shape is None: # force copy to workaround pyzmq #646 msg_list = t.cast(t.List[zmq.Message], msg_list) buffers = [memoryview(bytes(b.bytes)) for b in msg_list[5:]] message["buffers"] = buffers if self.debug: pprint.pprint(message) # adapt to the current version return adapt(message)