def test4(self): """Testing rollback reads""" f = queue.fanout_fifo.FanoutFIFO(5) f.add_reader('r1') for token in ['1', '2', '3', '4']: self.assertTrue(f.slots_available(1)) self.assertTrue(f.write(Token(token))) # fifo full self.assertFalse(f.slots_available(1)) self.assertRaises(queue.common.QueueFull, f.write, Token('5')) # tentative reads self.verify_data(['1', '2', '3', '4'], [f.peek("r1") for _ in range(4)]) # check still tentative self.assertTrue(f.tokens_available(0, "r1")) self.assertTrue(f.slots_available(0)) f.cancel("r1") self.assertFalse(f.slots_available(1)) self.assertRaises(queue.common.QueueFull, f.write, Token('5')) self.assertTrue(f.tokens_available(4, "r1")) # re-read self.verify_data(['1'], [f.peek("r1")]) f.commit("r1") self.assertTrue(f.tokens_available(3, "r1")) # one pos free in fifo self.assertTrue(f.slots_available(1)) self.assertTrue(f.write(Token('a'))) self.assertFalse(f.slots_available(1)) self.assertRaises(queue.common.QueueFull, f.write, Token('b'))
def test4(self): """Testing rollback reads""" f = fifo.FIFO(5) f.add_reader('r1') for token in ['1', '2', '3', '4']: self.assertTrue(f.can_write()) self.assertTrue(f.write(Token(token))) # fifo full self.assertFalse(f.can_write()) self.assertFalse(f.write(Token('5'))) # tentative reads self.verify_data(['1', '2', '3', '4'], [f.read("r1") for _ in range(4)]) # len unchanged self.assertEquals(len(f), 4) f.rollback_reads("r1") self.assertFalse(f.can_write()) self.assertFalse(f.write(Token('5'))) self.assertEquals(len(f), 4) # re-read self.verify_data(['1'], [f.read("r1")]) f.commit_reads("r1") self.assertEquals(len(f), 3) # one pos free in fifo self.assertTrue(f.can_write()) self.assertTrue(f.write(Token('a'))) self.assertFalse(f.can_write()) self.assertFalse(f.write(Token('b')))
def test1(self): """Adding reader again (reconnect)""" f = fifo.FIFO(5) f.add_reader('p1.id') data = ['1', '2', '3', '4'] for token in data: self.assertTrue(f.can_write()) self.assertTrue(f.write(Token(token))) self.verify_data(['1', '2'], [f.read('p1.id') for _ in range(2)]) f.commit_reads('p1.id', True) f.add_reader('p1.id') self.verify_data(['3', '4'], [f.read('p1.id') for _ in range(2)]) self.assertEquals(None, f.read('p1.id')) f.commit_reads('p1.id', True) for token in ['5', '6', '7', '8']: self.assertTrue(f.can_write()) self.assertTrue(f.write(Token(token))) self.assertFalse(f.can_write()) self.verify_data(['5', '6', '7', '8'], [f.read('p1.id') for _ in range(4)]) f.commit_reads('p1.id', True)
def testPeek_Exception(self): self.setup_writers(3) for i in [1, 2, 3]: self.inport.write(Token("data-%d" % i), "writer-%d" % i) self.inport.write(Token("data-%d" % (1 + 3)), "writer-%d" % 1) self.inport.write(ExceptionToken(), "writer-%d" % 2) self.inport.write(Token("data-%d" % (3 + 3)), "writer-%d" % 3) for i in [1, 2, 3]: self.inport.write(Token("data-%d" % (i + 6)), "writer-%d" % i) """ w1: 1 4 7 w2: 2 e 8 w3: 3 6 9 """ data_1 = self.inport.peek(None).value self.assertEqual(data_1, {"writer-%d" % i: "data-%d" % i for i in [1, 2, 3]}) """ w1: 4 7 w2: e 8 w3: 6 9 """ data_2 = self.inport.peek(None).value self.assertEqual(data_2, {"writer-2": 'Exception'}) """ w1: 4 7 w2: 8 w3: 6 9 """ data_3 = self.inport.peek(None).value result = {"writer-%d" % i: "data-%d" % (i + 3) for i in [1, 2, 3]} result["writer-2"] = "data-8" self.assertEqual(data_3, result)
def test3(self): """Testing commit reads""" f = fifo.FIFO(5) f.add_reader("r1") for token in ['1', '2', '3', '4']: self.assertTrue(f.can_write()) self.assertTrue(f.write(Token(token))) # Fails, fifo full self.assertFalse(f.can_write()) self.assertFalse(f.write(Token('5'))) # Tentative, fifo still full self.verify_data(['1'], [f.read("r1")]) self.assertFalse(f.can_write()) self.assertFalse(f.write(Token('5'))) # commit previous reads, fifo 1 pos free f.commit_reads('r1') self.assertTrue(f.can_write()) self.assertTrue(f.write(Token('5'))) # fifo full again self.assertFalse(f.can_write()) self.assertFalse(f.write(Token('5')))
def test1(self): """Adding reader again (reconnect)""" f = queue.fanout_fifo.FanoutFIFO(5) f.add_reader('p1.id') data = ['1', '2', '3', '4'] for token in data: self.assertTrue(f.slots_available(1)) self.assertTrue(f.write(Token(token))) self.verify_data(['1', '2'], [f.peek('p1.id') for _ in range(2)]) f.commit('p1.id') f.add_reader('p1.id') self.verify_data(['3', '4'], [f.peek('p1.id') for _ in range(2)]) self.assertRaises(queue.common.QueueEmpty, f.peek, 'p1.id') f.commit('p1.id') for token in ['5', '6', '7', '8']: self.assertTrue(f.slots_available(1)) self.assertTrue(f.write(Token(token))) self.assertFalse(f.slots_available(1)) self.verify_data(['5', '6', '7', '8'], [f.peek('p1.id') for _ in range(4)]) f.commit('p1.id')
def test_collect_unordered3(self): f = queue.collect_unordered.CollectUnordered( { 'routing': 'collect-unordered', 'nbr_peers': 10 }, {}) for i in range(10): f.add_writer("w" + str(i), {}) # Fill queue try: for t in range(40): for i in range(0, 6): f.write(Token(i), "w" + str(i)) except: pass tokens = [] try: for i in range(1000): tokens.append(f.peek(None)) if i % 2 == 1: f.commit(None) else: f.cancel(None) try: f.write(Token(0), "w0") except: pass except: pass print[t.value for t in tokens] s = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5] * 4 + [0] * 12 assert [t.value for t in tokens][:len(s)] == s
def test3(self): """Testing commit reads""" f = queue.fanout_fifo.FanoutFIFO(5) f.add_reader("r1") for token in ['1', '2', '3', '4']: self.assertTrue(f.slots_available(1)) self.assertTrue(f.write(Token(token))) # Fails, fifo full self.assertFalse(f.slots_available(1)) self.assertRaises(queue.common.QueueFull, f.write, Token('5')) # Tentative, fifo still full self.verify_data(['1'], [f.peek("r1")]) self.assertFalse(f.slots_available(1)) self.assertRaises(queue.common.QueueFull, f.write, Token('5')) # commit previous reads, fifo 1 pos free f.commit('r1') self.assertTrue(f.slots_available(1)) self.assertTrue(f.write(Token('5'))) # fifo full again self.assertFalse(f.slots_available(1)) self.assertRaises(queue.common.QueueFull, f.write, Token('5'))
def testTokensAvailable_Normal(self): self.setup_writers(5) self.inport.write(Token("data-1"), "writer-1") self.assertTrue(self.inport.tokens_available(1, None)) self.assertFalse(self.inport.tokens_available(2, None)) self.inport.write(Token("data-1"), "writer-2") self.assertTrue(self.inport.tokens_available(2, None)) self.assertFalse(self.inport.tokens_available(3, None))
def testTokensAvailable_Normal(self): self.setup_writers(5) for i in [1, 2, 3, 4, 5]: self.inport.write(Token("data-%d" % i), "writer-%d" % i) self.assertTrue(self.inport.tokens_available(1, None)) for i in [1, 2, 3]: self.inport.write(Token("data-%d" % i), "writer-%d" % i) self.assertFalse(self.inport.tokens_available(2, None)) for i in [4, 5]: self.inport.write(Token("data-%d" % i), "writer-%d" % i) self.assertTrue(self.inport.tokens_available(2, None))
def test_communicate(self): self.tunnel_out.port.write_token(Token(1)) self.tunnel_out.port.write_token(Token(2)) self.tunnel_out.bulk = False assert self.tunnel_out.communicate() is True assert self.tunnel.send.call_count == 1 assert self.tunnel_out.communicate() is False self.tunnel_out.reply(1, 'ACK') assert self.tunnel_out.communicate() is True assert self.tunnel.send.call_count == 2
def peek(self, metadata): value = {} for writer in self.writers: if self.write_pos[writer] == self.tentative_read_pos[writer]: continue read_pos = self.tentative_read_pos[writer] data = self.fifo[writer][read_pos % self.N] if isinstance(data, ExceptionToken): # We found an exception token, will return it alone # First cancel previous peeks for w in self.writers: if w is writer: break self.tentative_read_pos[w] -= 1 # return exception token alone data = copy.deepcopy(data) data.value = {self.tags[writer]: data.value} self.tentative_read_pos[writer] = read_pos + 1 return data self.tentative_read_pos[writer] = read_pos + 1 value[self.tags[writer]] = data.value if value: return Token(value) else: raise QueueEmpty(reader=metadata)
def peek(self, metadata): if not self.tokens_available(1, metadata): raise QueueEmpty(reader=metadata) value = {} # Collect all tokens for writer in self.writers: read_pos = self.tentative_read_pos[writer] data = self.fifo[writer][read_pos % self.N] if isinstance(data, ExceptionToken): # We found an exception token, will return it alone # First cancel previous peeks for w in self.writers: if w is writer: break self.tentative_read_pos[w] -= 1 # return exception token alone data = copy.deepcopy(data) data.value = {self.tags[writer]: data.value} self.tentative_read_pos[writer] = read_pos + 1 return data self.tentative_read_pos[writer] = read_pos + 1 value[self.tags[writer]] = data.value if self.tags_are_ordering: # ensure values sorted on index in original ordering value = [x for (y, x) in sorted(zip(value.keys(), value.values()))] return Token(value)
def test_collect_unordered1(self): f = queue.collect_unordered.CollectUnordered( { 'routing': 'collect-unordered', 'nbr_peers': 10 }, {}) for i in range(10): f.add_writer("w" + str(i), {}) # Fill queue try: for t in range(40): for i in range(10): f.write(Token(i), "w" + str(i)) except: pass tokens = [] try: for i in range(1000): tokens.append(f.peek(None)) f.commit(None) except: pass print[t.value for t in tokens] assert [t.value for t in tokens] == range(0, 10) * 4
def testPeek_Normal(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) for i in [1, 2, 3]: self.assertEqual(self.inport.peek(None).value, "data-%d" % i) for i in [1, 2, 3]: self.assertEqual(self.inport.peek(None).value, "data-%d" % (i + 3))
def test2(self): """Multiple readers""" f = queue.fanout_fifo.FanoutFIFO({ 'queue_length': 4, 'direction': "in" }, {}) f.add_reader("r1", {}) f.add_reader("r2", {}) # Ensure fifo is empty self.assertRaises(queue.common.QueueEmpty, f.peek, "r1") self.assertTrue(f.tokens_available(0, "r1")) self.assertTrue(f.tokens_available(0, "r2")) # Add something self.assertTrue(f.write(Token('1'), None)) self.assertTrue(f.tokens_available(1, "r1")) self.assertTrue(f.tokens_available(1, "r2")) # Reader r1 read something self.assertTrue(f.peek('r1')) f.commit('r1') self.assertEquals([True] * 3, [f.write(Token(t), None) for t in ['2', '3', '4']]) self.assertRaises(queue.common.QueueFull, f.write, Token('5'), None) self.verify_data(['2', '3', '4'], [f.peek('r1') for _ in range(3)]) f.commit("r1") # Reader r1 all done, ensure reader r2 can still read self.assertTrue(f.tokens_available(4, "r2")) self.assertFalse(f.slots_available(1, None)) self.assertFalse(f.tokens_available(1, "r1")) # Reader r2 reads something self.verify_data(['1', '2', '3'], [f.peek("r2") for _ in range(3)]) f.commit("r2") self.assertTrue(f.tokens_available(1, "r2")) self.assertTrue(f.write(Token('5'), None)) self.verify_data(['4', '5'], [f.peek("r2") for _ in range(2)]) self.assertFalse(f.tokens_available(1, "r2")) self.assertRaises(queue.common.QueueEmpty, f.peek, "r2") self.assertTrue(f.tokens_available(1, "r1")) self.verify_data(['5'], [f.peek("r1")]) f.commit("r2") f.commit("r1") self.assertTrue(f.write(Token('6'), None)) self.assertTrue(f.write(Token('7'), None)) self.assertTrue(f.write(Token('8'), None)) self.assertTrue([f.peek("r1") for _ in range(3)], [f.peek("r2") for _ in range(3)]) self.assertRaises(queue.common.QueueEmpty, f.peek, "r1") self.assertRaises(queue.common.QueueEmpty, f.peek, "r2")
def add_reader(self, reader): if not isinstance(reader, basestring): raise Exception('Not a string: %s' % reader) if reader not in self.readers: self.read_pos[reader] = 0 self.write_pos[reader] = 0 self.tentative_read_pos[reader] = 0 self.fifo.setdefault(reader, [Token(0)] * self.N) self.readers.append(reader) self.readers.sort()
def testTokensAvailable_Normal(self): self.setup_writers(5) for i in [1, 2, 3, 4, 5]: self.inport.write(Token("data-%d" % i), "writer-%d" % i) self.assertTrue(self.inport.tokens_available(1, None)) for i in [1, 2, 3]: self.inport.write(Token("data-%d" % i), "writer-%d" % i) self.assertTrue(self.inport.tokens_available(2, None)) for i in [4, 5]: self.inport.write(Token("data-%d" % i), "writer-%d" % i) self.assertTrue(self.inport.tokens_available(2, None)) i = 0 try: self.inport.write(Token("data-%d" % i), "writer-%d" % 1) self.assertTrue(self.inport.tokens_available(3 + i, None)) i += 1 except QueueFull: pass
def test_nack_reply(self): self.tunnel_out.port.write_token(Token(1)) self.tunnel_out._send_one_token() self.tunnel_out.port.queue.commit(self.port.id) assert self.tunnel_out.port.queue.tentative_read_pos[self.port.id] == 1 assert self.tunnel_out.port.queue.read_pos[self.port.id] == 1 self.tunnel_out.port.write_token(Token(2)) self.tunnel_out.port.write_token(Token(3)) self.tunnel_out._send_one_token() self.tunnel_out._send_one_token() assert self.tunnel_out.port.queue.read_pos[self.port.id] == 1 assert self.tunnel_out.port.queue.tentative_read_pos[self.port.id] == 3 self.tunnel_out.reply(1, 'NACK') assert self.tunnel_out.port.queue.tentative_read_pos[self.port.id] == 1 assert self.tunnel_out.port.queue.read_pos[self.port.id] == 1
def test_reply(self): self.tunnel_out.port.queue.com_commit = Mock() self.tunnel_out.port.queue.com_cancel = Mock() self.tunnel.send = Mock() self.tunnel_out.port.write_token(Token(1)) self.tunnel_out._send_one_token() nbr = self.tunnel.send.call_args_list[-1][0][0]['sequencenbr'] self.tunnel_out.reply(0, 'ACK') self.tunnel_out.port.queue.com_commit.assert_called_with(self.port.id, nbr) assert self.trigger_loop.called self.tunnel_out.port.write_token(Token(1)) self.tunnel_out._send_one_token() nbr = self.tunnel.send.call_args_list[-1][0][0]['sequencenbr'] self.tunnel_out.reply(nbr, 'NACK') assert self.tunnel_out.port.queue.com_cancel.called
def testSerialize(self): self.setup_readers(3) for i in [1,2,3,4,5,6]: self.outport.write(Token("data-%d" % i), None) for i in [1,2,3]: self.outport.peek("reader-%d" % i) state = self.outport._state() port = create_port() port._set_state(state) for i in [1,2,3]: self.assertEqual(port.peek("reader-%d" % i).value, "data-%d" % (i+3))
def testSerialize(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) for i in [1, 2, 3]: self.inport.peek(None) state = self.inport._state() port = create_port() port._set_state(state) for i in [1, 2, 3]: self.assertEqual(port.peek(None).value, "data-%d" % (i + 3))
def _state(self, remap=None): if remap is None: state = { 'queuetype': self._type, 'fifo': { p: [t.encode() for t in tokens] for p, tokens in self.fifo.items() }, 'N': self.N, 'writers': self.writers, 'write_pos': self.write_pos, 'read_pos': self.read_pos, 'tentative_read_pos': self.tentative_read_pos, 'tags': self.tags, 'tags-are-ordering': self.tags_are_ordering } else: state = { 'queuetype': self._type, 'fifo': { remap[p] if p in remap else p: [Token(0).encode() for t in tokens] for p, tokens in self.fifo.items() }, 'N': self.N, 'writers': sorted([ remap[pid] if pid in remap else pid for pid in self.writers ]), 'write_pos': { remap[pid] if pid in remap else pid: 0 for pid in self.write_pos.keys() }, 'read_pos': { remap[pid] if pid in remap else pid: 0 for pid in self.read_pos.keys() }, 'tentative_read_pos': { remap[pid] if pid in remap else pid: 0 for pid in self.tentative_read_pos.keys() }, # TODO Need unique tags, how should these be created 'tags': { remap[pid] if pid in remap else pid: tag for pid, tag in self.tags.items() }, 'tags-are-ordering': self.tags_are_ordering } return state
def testCommit(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) self.inport.peek(None) self.inport.commit(None) data_2 = self.inport.peek(None).value self.assertEqual( data_2, {"writer-%d" % i: "data-%d" % (i + 3) for i in [1, 2, 3]}) with self.assertRaises(QueueEmpty): self.inport.peek(None)
def testCancel(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) data_1 = self.inport.peek(None).value self.inport.cancel(None) data_2 = self.inport.peek(None).value self.assertEqual(data_1, data_2) data_2 = self.inport.peek(None).value self.assertEqual( data_2, {"writer-%d" % i: "data-%d" % (i + 3) for i in [1, 2, 3]})
def testSerialize(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) self.inport.peek(None) state = self.inport._state() port = create_port(routing="collect-all-tagged") port._set_state(state) data = self.inport.peek(None).value self.assertEqual( data, {"writer-%d" % i: "data-%d" % (i + 3) for i in [1, 2, 3]})
def testPeek_Normal(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) for i in [1, 2, 3]: tag, data = unwrap(self.inport.peek(None)) self.assertEqual(tag, "writer-%d" % i) self.assertEqual(data, "data-%d" % i) for i in [1, 2, 3]: tag, data = unwrap(self.inport.peek(None)) self.assertEqual(tag, "writer-%d" % i) self.assertEqual(data, "data-%d" % (i + 3))
def test2(self): """Multiple readers""" f = fifo.FIFO(5) f.add_reader("r1") f.add_reader("r2") # Ensure fifo is empty self.assertEquals(f.read("r1"), None) self.assertEquals(len(f), 0) # Add something self.assertTrue(f.write(Token('1'))) self.assertEquals(len(f), 1) # Reader r1 read something self.assertTrue(f.read('r1')) f.commit_reads('r1') self.assertEquals([True] * 3, [f.write(Token(t)) for t in ['2', '3', '4']]) self.assertFalse(f.write(Token('5'))) self.verify_data(['2', '3', '4'], [f.read('r1') for _ in range(3)]) f.commit_reads("r1") # Reader r1 all done, ensure reader r2 can still read self.assertEquals(len(f), 4) self.assertFalse(f.can_write()) self.assertTrue(f.can_read("r2")) self.assertFalse(f.can_read("r1")) # Reader r2 reads something self.verify_data(['1', '2', '3'], [f.read("r2") for _ in range(3)]) f.commit_reads("r2") self.assertEquals(len(f), 1) self.assertTrue(f.write(Token('5'))) self.verify_data(['4', '5'], [f.read("r2") for _ in range(2)]) self.assertFalse(f.can_read("r2")) self.assertEquals(None, f.read("r2")) self.assertTrue(f.can_read("r1")) self.verify_data(['5'], [f.read("r1")]) f.commit_reads("r2") f.commit_reads("r1") self.assertTrue(f.write(Token('6'))) self.assertTrue(f.write(Token('7'))) self.assertTrue(f.write(Token('8'))) self.assertTrue([f.read("r1") for _ in range(4)], [f.read("r2") for _ in range(4)])
def testSerialize(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) for i in [1, 2, 3]: self.inport.peek(None) state = self.inport._state() port = create_port(routing="collect-tagged") port._set_state(state) for i in [1, 2, 3]: tag, data = unwrap(port.peek(None)) self.assertEqual(tag, "writer-%d" % i) self.assertEqual(data, "data-%d" % (i + 3))
def testCommit(self): self.setup_writers(3) for i in [1, 2, 3]: for j in [0, 3]: self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i) data = [] for i in [1, 2, 3, 4, 5, 6]: data += [self.inport.peek(None).value] self.inport.commit(None) # should be empty now with self.assertRaises(QueueEmpty): self.inport.peek(None) # ensure the data was as expected self.assertEqual(data, ["data-%d" % i for i in [1, 2, 3, 4, 5, 6]])