def test_persistence_restart(self):
"""
Test of persistent items load.
"""
queue1 = self.queues_manager.get_queue("testqueue1")
queue1.connect()
queue1.push(Message(b"data a", uuid=b"a", ttl=1,
flags=FLAG_PERSISTENT))
queue1.push(Message(b"data b", uuid=b"b", ttl=1,
flags=FLAG_PERSISTENT))
queue1.push(Message(b"data c", uuid=b"c", ttl=1,
flags=FLAG_PERSISTENT))
queue2 = self.queues_manager.get_queue("testqueue2")
queue2.connect()
queue2.push(Message(b"data d", uuid=b"d", ttl=1,
flags=FLAG_PERSISTENT))
queue2.push(Message(b"data e", uuid=b"e", ttl=1,
flags=FLAG_PERSISTENT))
self.queue_manager_restart()
self.assertEqual(len(self.queues_manager), 2)
queue = self.queues_manager.get_queue("testqueue1")
self.assertEqual(len(queue), 3)
self.assertEqual(queue.get().uuid, b"a")
queue = self.queues_manager.get_queue("testqueue2")
self.assertEqual(len(queue), 2)
self.assertEqual(queue.get().uuid, b"d")
def test_persistence(self):
"""
Without TTL.
"""
queue = self.queues_manager.get_queue("testqueue")
queue.connect()
queue.push(Message(b"data a", uuid=b"a", ttl=1, flags=FLAG_PERSISTENT))
queue.push(Message(b"data b", uuid=b"b"))
queue.push(Message(b"data c", uuid=b"c", ttl=1, flags=FLAG_PERSISTENT))
self.assertEqual(len(queue), 3)
stored_items = self.queues_manager.storage.get_items("testqueue")
self.assertEqual(len(stored_items), 2)
self.assertEqual(stored_items[0].uuid, b"a")
self.assertEqual(stored_items[1].uuid, b"c")
# remove "a"
queue.pop()
stored_items = self.queues_manager.storage.get_items("testqueue")
self.assertEqual(len(stored_items), 1)
self.assertEqual(stored_items[0].uuid, b"c")
# remove "b", "c" remains
queue.pop()
stored_items = self.queues_manager.storage.get_items("testqueue")
self.assertEqual(len(stored_items), 1)
def test_delete_items(self):
items = (Message(b"a"), Message(b"b"), Message(b"c"))
for item in items:
self.storage.push("q1", item)
self.assertEqual(len(self.storage.get_items("q1")), len(items))
self.storage.delete_items([items[0]])
self.assertEqual(len(self.storage.get_items("q1")), len(items) - 1)
self.storage.delete_items(items[1:])
self.assertEqual(len(self.storage.get_items("q1")), 0)
def test_get_queues_grouping(self):
self.storage.push("q1", Message(b"a"))
self.storage.push("q1", Message(b"a"))
self.storage.push("q2", Message(b"a"))
self.storage.push("q2", Message(b"a"))
self.storage.push("q1", Message(b"a"))
queues = self.storage.get_queues()
self.assertEqual(len(queues), 2)
self.assertEqual(set(queues), set(("q1", "q2")))
def test_update_ttl(self):
msg = Message(b"a", ttl=10)
self.storage.push("q1", msg)
self.storage.push("q1", Message(b"b", ttl=5))
msg.ttl = 20
self.storage.update_items_ttl([msg])
items = self.storage.get_items("q1")
self.assertEqual(items[0].ttl, 20) # only this message must be updated
self.assertEqual(items[1].ttl, 5) # this must stay unmodified
def test_update_ttl(self):
msg = Message(b"a", ttl=10)
self.storage.push("q1", msg)
self.storage.push("q1", Message(b"b", ttl=5))
msg.ttl = 20
self.storage.update_items_ttl([msg])
items = self.storage.get_items("q1")
self.assertEqual(items[0].ttl, 20) # only this message must be updated
self.assertEqual(items[1].ttl, 5) # this must stay unmodified
def test_zero_ttl(self):
"""
Push item with ttl=0 into a connected or disconnected queue.
"""
queue = self.queues_manager.get_queue("testqueue")
# disconnected
queue.push(Message(b"data a", uuid=b"a", ttl=0))
self.assertEqual(len(queue), 0)
queue.connect()
queue.push(Message(b"data a", uuid=b"a", ttl=0))
self.assertEqual(len(queue), 1)
def test_ttl(self):
"""
Push 2 items, one will expire on connect.
"""
queue = self.queues_manager.get_queue("testqueue")
with mock.patch("time.time") as time_mock:
time_results = iter([0, 3])
time_mock.side_effect = lambda: next(time_results)
queue.disconnect()
queue.push(Message(b"data a", uuid=b"a", ttl=1))
queue.push(Message(b"data b", uuid=b"b", ttl=5))
queue.connect()
self.assertEqual(len(queue), 1)
msg = queue.get()
self.assertEqual(msg.uuid, b"b")
self.assertEqual(msg.ttl, 2) # 5 - 3
def test_simple_put_get(self):
"""
Few puts, few gets, no TTL or persistence
"""
queue = self.queues_manager.get_queue("testqueue")
queue.connect()
queue.push(Message(b"data a", uuid=b"a"))
queue.push(Message(b"data b", uuid=b"b"))
self.assertEqual(len(queue), 2)
self.assertEqual(queue.get().uuid, b"a")
self.assertEqual(queue.get().uuid, b"a") # must be the same
queue.pop()
self.assertEqual(len(queue), 1)
self.assertEqual(queue.get().uuid, b"b")
queue.pop()
self.assertEqual(len(queue), 0)
self.assertEqual(queue.get(), None)
def send(self, ident, data):
try:
data = self.pickler.dumps(data)
except TypeError as exc:
# TypeError is raised if the object is unpickable
raise self.pickler.PickleError(exc)
message = Message(data=REPLY_PREFIX + data)
self.receive_hook.messaging.send_message(ident, message)
def test_message_ttl_none(self):
old_msg = Message(b"a", ttl=None)
self.storage.push("q1", old_msg)
self.storage.close()
self.storage_factory()
cur_msg = self.storage.get_items("q1")[0]
self.assertEqual(cur_msg.ttl, None)
def test_ttl_none(self):
queue = self.queues_manager.get_queue("testqueue")
queue.disconnect()
queue.push(Message(b"data a", uuid=b"a", ttl=None))
queue.connect()
self.assertEqual(len(queue), 1)
msg = queue.get()
self.assertEqual(msg.uuid, b"a")
self.assertEqual(msg.ttl, None)
def test_persistence_ttl(self):
queue = self.queues_manager.get_queue("testqueue")
queue.connect()
queue.push(Message(b"data a", uuid=b"a", ttl=1, flags=FLAG_PERSISTENT))
queue.push(Message(b"data b", uuid=b"b"))
queue.push(Message(b"data c", uuid=b"c", ttl=5, flags=FLAG_PERSISTENT))
self.assertEqual(len(queue), 3)
self.queue_manager_restart()
queue = self.queues_manager.get_queue("testqueue")
with mock.patch("time.time") as time_mock:
time_results = iter([0, 3])
time_mock.side_effect = lambda: next(time_results)
queue.disconnect()
self.assertEqual(len(queue), 2)
queue.connect()
self.assertEqual(len(queue), 1)
def test_message_attributes_persistence(self):
old_msg = Message(b"a", ttl=100, flags=1234)
self.storage.push("q1", old_msg)
self.storage.close()
self.storage_factory()
cur_msg = self.storage.get_items("q1")[0]
self.assertEqual(old_msg.data, cur_msg.data)
self.assertEqual(old_msg.ttl, cur_msg.ttl)
self.assertEqual(old_msg.uuid, cur_msg.uuid)
self.assertEqual(old_msg.flags, cur_msg.flags)
def get_items(self, queue_name):
items = []
# XXX this might need some explicit sorting
dbitems = self.all_items.find({"queue_name": queue_name})
for item in dbitems:
items.append(
Message(uuid=a2b_base64(item["uuid"]),
data=a2b_base64(item["data"]),
ttl=item["ttl"],
flags=item["flags"]))
return items
def get_items(self, queue_name):
sel = table.select().where(table.c.queue_name == queue_name).order_by(
table.c.id_)
items = []
for res in self.conn.execute(sel).fetchall():
items.append(
Message(uuid=a2b_hex(res.uuid),
data=bytes(res.data),
ttl=res.ttl,
flags=res.flags))
return items
def get_items(self, queue_name):
self.crs.execute(
"""SELECT uuid, data, ttl, flags FROM items
WHERE queue_name = ?""", (queue_name, ))
items = []
for res in self.crs.fetchall():
uuid = a2b_hex(res[0]) # XXX python2 hack
data = bytes(res[1]) # XXX python2 hack
items.append(
Message(uuid=uuid, data=data, ttl=res[2], flags=res[3]))
return items
def test_persistence(self):
self.assertEqual(len(self.storage.get_queues()), 0)
self.storage.push("q1", Message(b"a"))
self.assertEqual(len(self.storage.get_queues()), 1)
self.assertEqual(len(self.storage.get_items("q2")), 0)
self.assertEqual(len(self.storage.get_items("q1")), 1)
self.storage.close()
self.storage_factory()
self.assert_(len(self.storage.get_queues()) >= 1) # at least q1
self.assertEqual(len(self.storage.get_items("q2")), 0)
self.assertEqual(len(self.storage.get_items("q1")), 1)
def parse_message(self, payload, conn_id):
if len(payload) < FRAME_FORMAT_MESSAGE_SIZE:
raise SnakeMQBrokenMessage("message")
try:
ident = self._ident_by_conn[conn_id]
except KeyError:
raise SnakeMQNoIdent(conn_id)
muuid, ttl, flags = struct.unpack(FRAME_FORMAT_MESSAGE,
memstr(payload[:FRAME_FORMAT_MESSAGE_SIZE]))
if ttl == INFINITE_TTL:
ttl = None
message = Message(data=memstr(payload[FRAME_FORMAT_MESSAGE_SIZE:]),
uuid=muuid, ttl=ttl, flags=flags)
self.on_message_recv(conn_id, ident, message)
def send_params(self, remote_ident, params, ttl):
raw = self.pickler.dumps(params)
message = Message(data=REQUEST_PREFIX + raw, ttl=ttl)
self.receive_hook.messaging.send_message(remote_ident, message)
def test_queue_ordering(self):
self.storage.push("q1", Message(b"a"))
self.storage.push("q1", Message(b"b"))
self.assertEqual(self.storage.get_items("q1")[0].data, b"a")
