def test1(self):
"""Adding reader again (reconnect)"""
f = queue.fanout_fifo.FanoutFIFO(5)
f.add_reader('p1.id')
data = ['1', '2', '3', '4']
for token in data:
self.assertTrue(f.slots_available(1))
self.assertTrue(f.write(Token(token)))
self.verify_data(['1', '2'], [f.peek('p1.id') for _ in range(2)])
f.commit('p1.id')
f.add_reader('p1.id')
self.verify_data(['3', '4'], [f.peek('p1.id') for _ in range(2)])
self.assertRaises(queue.common.QueueEmpty, f.peek, 'p1.id')
f.commit('p1.id')
for token in ['5', '6', '7', '8']:
self.assertTrue(f.slots_available(1))
self.assertTrue(f.write(Token(token)))
self.assertFalse(f.slots_available(1))
self.verify_data(['5', '6', '7', '8'],
[f.peek('p1.id') for _ in range(4)])
f.commit('p1.id')
def test4(self):
"""Testing rollback reads"""
f = queue.fanout_fifo.FanoutFIFO(5)
f.add_reader('r1')
for token in ['1', '2', '3', '4']:
self.assertTrue(f.slots_available(1))
self.assertTrue(f.write(Token(token)))
# fifo full
self.assertFalse(f.slots_available(1))
self.assertRaises(queue.common.QueueFull, f.write, Token('5'))
# tentative reads
self.verify_data(['1', '2', '3', '4'],
[f.peek("r1") for _ in range(4)])
# check still tentative
self.assertTrue(f.tokens_available(0, "r1"))
self.assertTrue(f.slots_available(0))
f.cancel("r1")
self.assertFalse(f.slots_available(1))
self.assertRaises(queue.common.QueueFull, f.write, Token('5'))
self.assertTrue(f.tokens_available(4, "r1"))
# re-read
self.verify_data(['1'], [f.peek("r1")])
f.commit("r1")
self.assertTrue(f.tokens_available(3, "r1"))
# one pos free in fifo
self.assertTrue(f.slots_available(1))
self.assertTrue(f.write(Token('a')))
self.assertFalse(f.slots_available(1))
self.assertRaises(queue.common.QueueFull, f.write, Token('b'))
def test1(self):
"""Adding reader again (reconnect)"""
f = fifo.FIFO(5)
f.add_reader('p1.id')
data = ['1', '2', '3', '4']
for token in data:
self.assertTrue(f.can_write())
self.assertTrue(f.write(Token(token)))
self.verify_data(['1', '2'], [f.read('p1.id') for _ in range(2)])
f.commit_reads('p1.id', True)
f.add_reader('p1.id')
self.verify_data(['3', '4'], [f.read('p1.id') for _ in range(2)])
self.assertEquals(None, f.read('p1.id'))
f.commit_reads('p1.id', True)
for token in ['5', '6', '7', '8']:
self.assertTrue(f.can_write())
self.assertTrue(f.write(Token(token)))
self.assertFalse(f.can_write())
self.verify_data(['5', '6', '7', '8'],
[f.read('p1.id') for _ in range(4)])
f.commit_reads('p1.id', True)
def test3(self):
"""Testing commit reads"""
f = fifo.FIFO(5)
f.add_reader("r1")
for token in ['1', '2', '3', '4']:
self.assertTrue(f.can_write())
self.assertTrue(f.write(Token(token)))
# Fails, fifo full
self.assertFalse(f.can_write())
self.assertFalse(f.write(Token('5')))
# Tentative, fifo still full
self.verify_data(['1'], [f.read("r1")])
self.assertFalse(f.can_write())
self.assertFalse(f.write(Token('5')))
# commit previous reads, fifo 1 pos free
f.commit_reads('r1')
self.assertTrue(f.can_write())
self.assertTrue(f.write(Token('5')))
# fifo full again
self.assertFalse(f.can_write())
self.assertFalse(f.write(Token('5')))
def test4(self):
"""Testing rollback reads"""
f = fifo.FIFO(5)
f.add_reader('r1')
for token in ['1', '2', '3', '4']:
self.assertTrue(f.can_write())
self.assertTrue(f.write(Token(token)))
# fifo full
self.assertFalse(f.can_write())
self.assertFalse(f.write(Token('5')))
# tentative reads
self.verify_data(['1', '2', '3', '4'],
[f.read("r1") for _ in range(4)])
# len unchanged
self.assertEquals(len(f), 4)
f.rollback_reads("r1")
self.assertFalse(f.can_write())
self.assertFalse(f.write(Token('5')))
self.assertEquals(len(f), 4)
# re-read
self.verify_data(['1'], [f.read("r1")])
f.commit_reads("r1")
self.assertEquals(len(f), 3)
# one pos free in fifo
self.assertTrue(f.can_write())
self.assertTrue(f.write(Token('a')))
self.assertFalse(f.can_write())
self.assertFalse(f.write(Token('b')))
def testPeek_Exception(self):
self.setup_writers(3)
for i in [1, 2, 3]:
self.inport.write(Token("data-%d" % i), "writer-%d" % i)
self.inport.write(Token("data-%d" % (1 + 3)), "writer-%d" % 1)
self.inport.write(ExceptionToken(), "writer-%d" % 2)
self.inport.write(Token("data-%d" % (3 + 3)), "writer-%d" % 3)
for i in [1, 2, 3]:
self.inport.write(Token("data-%d" % (i + 6)), "writer-%d" % i)
"""
w1: 1 4 7
w2: 2 e 8
w3: 3 6 9
"""
data_1 = self.inport.peek(None).value
self.assertEqual(data_1,
{"writer-%d" % i: "data-%d" % i
for i in [1, 2, 3]})
"""
w1: 4 7
w2: e 8
w3: 6 9
"""
data_2 = self.inport.peek(None).value
self.assertEqual(data_2, {"writer-2": 'Exception'})
"""
w1: 4 7
w2: 8
w3: 6 9
"""
data_3 = self.inport.peek(None).value
result = {"writer-%d" % i: "data-%d" % (i + 3) for i in [1, 2, 3]}
result["writer-2"] = "data-8"
self.assertEqual(data_3, result)
def test_collect_unordered3(self):
f = queue.collect_unordered.CollectUnordered(
{
'routing': 'collect-unordered',
'nbr_peers': 10
}, {})
for i in range(10):
f.add_writer("w" + str(i), {})
# Fill queue
try:
for t in range(40):
for i in range(0, 6):
f.write(Token(i), "w" + str(i))
except:
pass
tokens = []
try:
for i in range(1000):
tokens.append(f.peek(None))
if i % 2 == 1:
f.commit(None)
else:
f.cancel(None)
try:
f.write(Token(0), "w0")
except:
pass
except:
pass
print[t.value for t in tokens]
s = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5] * 4 + [0] * 12
assert [t.value for t in tokens][:len(s)] == s
def test3(self):
"""Testing commit reads"""
f = queue.fanout_fifo.FanoutFIFO(5)
f.add_reader("r1")
for token in ['1', '2', '3', '4']:
self.assertTrue(f.slots_available(1))
self.assertTrue(f.write(Token(token)))
# Fails, fifo full
self.assertFalse(f.slots_available(1))
self.assertRaises(queue.common.QueueFull, f.write, Token('5'))
# Tentative, fifo still full
self.verify_data(['1'], [f.peek("r1")])
self.assertFalse(f.slots_available(1))
self.assertRaises(queue.common.QueueFull, f.write, Token('5'))
# commit previous reads, fifo 1 pos free
f.commit('r1')
self.assertTrue(f.slots_available(1))
self.assertTrue(f.write(Token('5')))
# fifo full again
self.assertFalse(f.slots_available(1))
self.assertRaises(queue.common.QueueFull, f.write, Token('5'))
def testTokensAvailable_Normal(self):
self.setup_writers(5)
self.inport.write(Token("data-1"), "writer-1")
self.assertTrue(self.inport.tokens_available(1, None))
self.assertFalse(self.inport.tokens_available(2, None))
self.inport.write(Token("data-1"), "writer-2")
self.assertTrue(self.inport.tokens_available(2, None))
self.assertFalse(self.inport.tokens_available(3, None))
def testTokensAvailable_Normal(self):
self.setup_writers(5)
for i in [1, 2, 3, 4, 5]:
self.inport.write(Token("data-%d" % i), "writer-%d" % i)
self.assertTrue(self.inport.tokens_available(1, None))
for i in [1, 2, 3]:
self.inport.write(Token("data-%d" % i), "writer-%d" % i)
self.assertFalse(self.inport.tokens_available(2, None))
for i in [4, 5]:
self.inport.write(Token("data-%d" % i), "writer-%d" % i)
self.assertTrue(self.inport.tokens_available(2, None))
def test_communicate(self):
self.tunnel_out.port.write_token(Token(1))
self.tunnel_out.port.write_token(Token(2))
self.tunnel_out.bulk = False
assert self.tunnel_out.communicate() is True
assert self.tunnel.send.call_count == 1
assert self.tunnel_out.communicate() is False
self.tunnel_out.reply(1, 'ACK')
assert self.tunnel_out.communicate() is True
assert self.tunnel.send.call_count == 2
def peek(self, metadata):
value = {}
for writer in self.writers:
if self.write_pos[writer] == self.tentative_read_pos[writer]:
continue
read_pos = self.tentative_read_pos[writer]
data = self.fifo[writer][read_pos % self.N]
if isinstance(data, ExceptionToken):
# We found an exception token, will return it alone
# First cancel previous peeks
for w in self.writers:
if w is writer:
break
self.tentative_read_pos[w] -= 1
# return exception token alone
data = copy.deepcopy(data)
data.value = {self.tags[writer]: data.value}
self.tentative_read_pos[writer] = read_pos + 1
return data
self.tentative_read_pos[writer] = read_pos + 1
value[self.tags[writer]] = data.value
if value:
return Token(value)
else:
raise QueueEmpty(reader=metadata)
def test_collect_unordered1(self):
f = queue.collect_unordered.CollectUnordered(
{
'routing': 'collect-unordered',
'nbr_peers': 10
}, {})
for i in range(10):
f.add_writer("w" + str(i), {})
# Fill queue
try:
for t in range(40):
for i in range(10):
f.write(Token(i), "w" + str(i))
except:
pass
tokens = []
try:
for i in range(1000):
tokens.append(f.peek(None))
f.commit(None)
except:
pass
print[t.value for t in tokens]
assert [t.value for t in tokens] == range(0, 10) * 4
def peek(self, metadata):
if not self.tokens_available(1, metadata):
raise QueueEmpty(reader=metadata)
value = {}
# Collect all tokens
for writer in self.writers:
read_pos = self.tentative_read_pos[writer]
data = self.fifo[writer][read_pos % self.N]
if isinstance(data, ExceptionToken):
# We found an exception token, will return it alone
# First cancel previous peeks
for w in self.writers:
if w is writer:
break
self.tentative_read_pos[w] -= 1
# return exception token alone
data = copy.deepcopy(data)
data.value = {self.tags[writer]: data.value}
self.tentative_read_pos[writer] = read_pos + 1
return data
self.tentative_read_pos[writer] = read_pos + 1
value[self.tags[writer]] = data.value
if self.tags_are_ordering:
# ensure values sorted on index in original ordering
value = [x for (y, x) in sorted(zip(value.keys(), value.values()))]
return Token(value)
def recv_token(self, payload):
try:
r = self.port.queue.com_write(Token.decode(payload['token']), self.peer_id, payload['sequencenbr'])
if r == COMMIT_RESPONSE.handled:
# New token, trigger loop
self.scheduler.tunnel_rx(self)
if r == COMMIT_RESPONSE.invalid:
ok = False
else:
# Either old or new token, ack it
ok = True
_log.debug("recv_token %s %s: %d %s => %s %d" % (self.port.id, self.port.name, payload['sequencenbr'], payload['token'], "True" if ok else "False", r))
except QueueFull:
# Queue full just send NACK
_log.debug("REMOTE QUEUE FULL %d %s" % (self.pressure_count, self.port.id))
ok = False
if self.pressure[(self.pressure_count - 1) % PRESSURE_LENGTH][0] != payload['sequencenbr']:
# Log a QueueFull event
self.pressure[self.pressure_count % PRESSURE_LENGTH] = (payload['sequencenbr'], time.time())
self.pressure_count += 1
# Inform scheduler about potential pressure event
self.scheduler.trigger_pressure_event(self.port.owner.id)
self.pressure_last = payload['sequencenbr']
reply = {
'cmd': 'TOKEN_REPLY',
'port_id': payload['port_id'],
'peer_port_id': payload['peer_port_id'],
'sequencenbr': payload['sequencenbr'],
'value': 'ACK' if ok else 'NACK'
}
self.tunnel.send(reply)
def recv_token(self, payload):
try:
r = self.port.queue.com_write(Token.decode(payload['token']), self.peer_id, payload['sequencenbr'])
if r == COMMIT_RESPONSE.handled:
# New token, trigger loop
self.trigger_loop()
if r == COMMIT_RESPONSE.invalid:
ok = False
else:
# Either old or new token, ack it
ok = True
_log.debug("recv_token %s %s: %d %s => %s %d" % (self.port.id, self.port.name, payload['sequencenbr'], payload['token'], "True" if ok else "False", r))
except QueueFull:
# Queue full just send NACK
ok = False
if self.pressure[(self.pressure_count - 1) % PRESSURE_LENGTH] != payload['sequencenbr']:
self.pressure[self.pressure_count % PRESSURE_LENGTH] = payload['sequencenbr']
self.pressure_count += 1
self.pressure_last = payload['sequencenbr']
reply = {
'cmd': 'TOKEN_REPLY',
'port_id': payload['port_id'],
'peer_port_id': payload['peer_port_id'],
'sequencenbr': payload['sequencenbr'],
'value': 'ACK' if ok else 'NACK'
}
self.tunnel.send(reply)
def _set_state(self, state):
self._type = state.get('queuetype', "fanout_fifo")
self.fifo = [Token.decode(d) for d in state['fifo']]
self.N = state['N']
self.readers = set(state['readers'])
self.write_pos = state['write_pos']
self.read_pos = state['read_pos']
self.tentative_read_pos = state['tentative_read_pos']
def _set_state(self, state):
self._type = state.get('queuetype',"fanout_fifo")
self.fifo = [Token.decode(d) for d in state['fifo']]
self.N = state['N']
self.readers = set(state['readers'])
self.write_pos = state['write_pos']
self.read_pos = state['read_pos']
self.tentative_read_pos = state['tentative_read_pos']
def testPeek_Normal(self):
self.setup_writers(3)
for i in [1, 2, 3]:
for j in [0, 3]:
self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i)
for i in [1, 2, 3]:
self.assertEqual(self.inport.peek(None).value, "data-%d" % i)
for i in [1, 2, 3]:
self.assertEqual(self.inport.peek(None).value, "data-%d" % (i + 3))
def test2(self):
"""Multiple readers"""
f = queue.fanout_fifo.FanoutFIFO({
'queue_length': 4,
'direction': "in"
}, {})
f.add_reader("r1", {})
f.add_reader("r2", {})
# Ensure fifo is empty
self.assertRaises(queue.common.QueueEmpty, f.peek, "r1")
self.assertTrue(f.tokens_available(0, "r1"))
self.assertTrue(f.tokens_available(0, "r2"))
# Add something
self.assertTrue(f.write(Token('1'), None))
self.assertTrue(f.tokens_available(1, "r1"))
self.assertTrue(f.tokens_available(1, "r2"))
# Reader r1 read something
self.assertTrue(f.peek('r1'))
f.commit('r1')
self.assertEquals([True] * 3,
[f.write(Token(t), None) for t in ['2', '3', '4']])
self.assertRaises(queue.common.QueueFull, f.write, Token('5'), None)
self.verify_data(['2', '3', '4'], [f.peek('r1') for _ in range(3)])
f.commit("r1")
# Reader r1 all done, ensure reader r2 can still read
self.assertTrue(f.tokens_available(4, "r2"))
self.assertFalse(f.slots_available(1, None))
self.assertFalse(f.tokens_available(1, "r1"))
# Reader r2 reads something
self.verify_data(['1', '2', '3'], [f.peek("r2") for _ in range(3)])
f.commit("r2")
self.assertTrue(f.tokens_available(1, "r2"))
self.assertTrue(f.write(Token('5'), None))
self.verify_data(['4', '5'], [f.peek("r2") for _ in range(2)])
self.assertFalse(f.tokens_available(1, "r2"))
self.assertRaises(queue.common.QueueEmpty, f.peek, "r2")
self.assertTrue(f.tokens_available(1, "r1"))
self.verify_data(['5'], [f.peek("r1")])
f.commit("r2")
f.commit("r1")
self.assertTrue(f.write(Token('6'), None))
self.assertTrue(f.write(Token('7'), None))
self.assertTrue(f.write(Token('8'), None))
self.assertTrue([f.peek("r1") for _ in range(3)],
[f.peek("r2") for _ in range(3)])
self.assertRaises(queue.common.QueueEmpty, f.peek, "r1")
self.assertRaises(queue.common.QueueEmpty, f.peek, "r2")
def test_nack_reply(self):
self.tunnel_out.port.write_token(Token(1))
self.tunnel_out._send_one_token()
self.tunnel_out.port.queue.commit(self.port.id)
assert self.tunnel_out.port.queue.tentative_read_pos[self.port.id] == 1
assert self.tunnel_out.port.queue.read_pos[self.port.id] == 1
self.tunnel_out.port.write_token(Token(2))
self.tunnel_out.port.write_token(Token(3))
self.tunnel_out._send_one_token()
self.tunnel_out._send_one_token()
assert self.tunnel_out.port.queue.read_pos[self.port.id] == 1
assert self.tunnel_out.port.queue.tentative_read_pos[self.port.id] == 3
self.tunnel_out.reply(1, 'NACK')
assert self.tunnel_out.port.queue.tentative_read_pos[self.port.id] == 1
assert self.tunnel_out.port.queue.read_pos[self.port.id] == 1
def testTokensAvailable_Normal(self):
self.setup_writers(5)
for i in [1, 2, 3, 4, 5]:
self.inport.write(Token("data-%d" % i), "writer-%d" % i)
self.assertTrue(self.inport.tokens_available(1, None))
for i in [1, 2, 3]:
self.inport.write(Token("data-%d" % i), "writer-%d" % i)
self.assertTrue(self.inport.tokens_available(2, None))
for i in [4, 5]:
self.inport.write(Token("data-%d" % i), "writer-%d" % i)
self.assertTrue(self.inport.tokens_available(2, None))
i = 0
try:
self.inport.write(Token("data-%d" % i), "writer-%d" % 1)
self.assertTrue(self.inport.tokens_available(3 + i, None))
i += 1
except QueueFull:
pass
def test_reply(self):
self.tunnel_out.port.queue.com_commit = Mock()
self.tunnel_out.port.queue.com_cancel = Mock()
self.tunnel.send = Mock()
self.tunnel_out.port.write_token(Token(1))
self.tunnel_out._send_one_token()
nbr = self.tunnel.send.call_args_list[-1][0][0]['sequencenbr']
self.tunnel_out.reply(0, 'ACK')
self.tunnel_out.port.queue.com_commit.assert_called_with(self.port.id, nbr)
assert self.trigger_loop.called
self.tunnel_out.port.write_token(Token(1))
self.tunnel_out._send_one_token()
nbr = self.tunnel.send.call_args_list[-1][0][0]['sequencenbr']
self.tunnel_out.reply(nbr, 'NACK')
assert self.tunnel_out.port.queue.com_cancel.called
def _set_state(self, state):
self._type = state.get('queuetype')
self.fifo = {p: [Token.decode(t) for t in tokens] for p, tokens in state['fifo'].items()}
self.N = state['N']
self.writers = state['writers']
self.write_pos = state['write_pos']
self.read_pos = state['read_pos']
self.tentative_read_pos = state['tentative_read_pos']
self.tags = state.get("tags", {})
self.tags_are_ordering = state.get("tags-are-ordering", False)
def add_reader(self, reader):
if not isinstance(reader, basestring):
raise Exception('Not a string: %s' % reader)
if reader not in self.readers:
self.read_pos[reader] = 0
self.write_pos[reader] = 0
self.tentative_read_pos[reader] = 0
self.fifo.setdefault(reader, [Token(0)] * self.N)
self.readers.append(reader)
self.readers.sort()
def _set_state(self, state):
self._type = state.get('queuetype')
self.fifo = {p: [Token.decode(t) for t in tokens] for p, tokens in state['fifo'].items()}
self.N = state['N']
self.readers = state['readers']
self.write_pos = state['write_pos']
self.read_pos = state['read_pos']
self.tentative_read_pos = state['tentative_read_pos']
if len(self.readers) > self.nbr_peers:
# If the peer has been replicated just set it to nbr connected
self.nbr_peers = len(self.readers)
def testSerialize(self):
self.setup_readers(3)
for i in [1,2,3,4,5,6]:
self.outport.write(Token("data-%d" % i), None)
for i in [1,2,3]:
self.outport.peek("reader-%d" % i)
state = self.outport._state()
port = create_port()
port._set_state(state)
for i in [1,2,3]:
self.assertEqual(port.peek("reader-%d" % i).value, "data-%d" % (i+3))
def recv_token(self, payload):
ok =False
# Drop any tokens that we can't write to fifo or is out of sequence
if self.port.fifo.can_write() and self.port.fifo.write_pos == payload['sequencenbr']:
self.port.fifo.write(Token.decode(payload['token']))
self.trigger_loop()
ok = True
elif self.port.fifo.write_pos > payload['sequencenbr']:
# Other side resent a token we already have received (can happen after a reconnect if our previous ACK was lost), just ACK
ok = True
reply = {'cmd': 'TOKEN_REPLY', 'port_id':payload['port_id'], 'peer_port_id': payload['peer_port_id'], 'sequencenbr': payload['sequencenbr'], 'value': 'ACK' if ok else 'NACK'}
self.tunnel.send(reply)
def _set_state(self, state):
self._type = state.get("queuetype")
self.fifo = {p: [Token.decode(t) for t in self.fifo] for p, t in state["fifo"].items()}
self.N = state["N"]
self.readers = state["readers"]
self.write_pos = state["write_pos"]
self.read_pos = state["read_pos"]
self.tentative_read_pos = state["tentative_read_pos"]
self.token_state = state["token_state"]
self.reader_turn = state["reader_turn"]
self.turn_pos = state["turn_pos"]
self._set_turn()
def testSerialize(self):
self.setup_writers(3)
for i in [1, 2, 3]:
for j in [0, 3]:
self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i)
for i in [1, 2, 3]:
self.inport.peek(None)
state = self.inport._state()
port = create_port()
port._set_state(state)
for i in [1, 2, 3]:
self.assertEqual(port.peek(None).value, "data-%d" % (i + 3))
def _state(self, remap=None):
if remap is None:
state = {
'queuetype': self._type,
'fifo': {
p: [t.encode() for t in tokens]
for p, tokens in self.fifo.items()
},
'N': self.N,
'writers': self.writers,
'write_pos': self.write_pos,
'read_pos': self.read_pos,
'tentative_read_pos': self.tentative_read_pos,
'tags': self.tags,
'tags-are-ordering': self.tags_are_ordering
}
else:
state = {
'queuetype':
self._type,
'fifo': {
remap[p] if p in remap else p:
[Token(0).encode() for t in tokens]
for p, tokens in self.fifo.items()
},
'N':
self.N,
'writers':
sorted([
remap[pid] if pid in remap else pid for pid in self.writers
]),
'write_pos': {
remap[pid] if pid in remap else pid: 0
for pid in self.write_pos.keys()
},
'read_pos': {
remap[pid] if pid in remap else pid: 0
for pid in self.read_pos.keys()
},
'tentative_read_pos': {
remap[pid] if pid in remap else pid: 0
for pid in self.tentative_read_pos.keys()
},
# TODO Need unique tags, how should these be created
'tags': {
remap[pid] if pid in remap else pid: tag
for pid, tag in self.tags.items()
},
'tags-are-ordering':
self.tags_are_ordering
}
return state
def testCommit(self):
self.setup_writers(3)
for i in [1, 2, 3]:
for j in [0, 3]:
self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i)
self.inport.peek(None)
self.inport.commit(None)
data_2 = self.inport.peek(None).value
self.assertEqual(
data_2, {"writer-%d" % i: "data-%d" % (i + 3)
for i in [1, 2, 3]})
with self.assertRaises(QueueEmpty):
self.inport.peek(None)
def testCancel(self):
self.setup_writers(3)
for i in [1, 2, 3]:
for j in [0, 3]:
self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i)
data_1 = self.inport.peek(None).value
self.inport.cancel(None)
data_2 = self.inport.peek(None).value
self.assertEqual(data_1, data_2)
data_2 = self.inport.peek(None).value
self.assertEqual(
data_2, {"writer-%d" % i: "data-%d" % (i + 3)
for i in [1, 2, 3]})
def testSerialize(self):
self.setup_writers(3)
for i in [1, 2, 3]:
for j in [0, 3]:
self.inport.write(Token("data-%d" % (i + j)), "writer-%d" % i)
self.inport.peek(None)
state = self.inport._state()
port = create_port(routing="collect-all-tagged")
port._set_state(state)
data = self.inport.peek(None).value
self.assertEqual(
data, {"writer-%d" % i: "data-%d" % (i + 3)
for i in [1, 2, 3]})
def _set_state(self, state):
self._type = state.get('queuetype')
self.fifo = {
p: [Token.decode(t) for t in tokens]
for p, tokens in state['fifo'].items()
}
self.N = state['N']
self.writers = state['writers']
self.write_pos = state['write_pos']
self.read_pos = state['read_pos']
self.tentative_read_pos = state['tentative_read_pos']
self.tags = state.get("tags", {})
self.tags_are_ordering = state.get("tags-are-ordering", False)
def recv_token(self, payload):
ok = False
# Drop any tokens that we can't write to fifo or is out of sequence
if self.port.fifo.can_write() and self.port.fifo.write_pos == payload["sequencenbr"]:
self.port.fifo.write(Token.decode(payload["token"]))
self.trigger_loop()
ok = True
elif self.port.fifo.write_pos > payload["sequencenbr"]:
# Other side resent a token we already have received (can happen after a reconnect if our previous ACK was lost), just ACK
ok = True
reply = {
"cmd": "TOKEN_REPLY",
"port_id": payload["port_id"],
"peer_port_id": payload["peer_port_id"],
"sequencenbr": payload["sequencenbr"],
"value": "ACK" if ok else "NACK",
}
self.tunnel.send(reply)
def recv_token(self, payload):
try:
r = self.port.queue.com_write(Token.decode(payload['token']), payload['sequencenbr'])
if r == COMMIT_RESPONSE.handled:
# New token, trigger loop
self.trigger_loop()
if r == COMMIT_RESPONSE.invalid:
ok = False
else:
# Either old or new token, ack it
ok = True
_log.debug("recv_token %s %s: %d %s => %s %d" % (self.port.id, self.port.name, payload['sequencenbr'], payload['token'], "True" if ok else "False", r))
except QueueFull:
# Queue full just send NACK
ok = False
reply = {
'cmd': 'TOKEN_REPLY',
'port_id': payload['port_id'],
'peer_port_id': payload['peer_port_id'],
'sequencenbr': payload['sequencenbr'],
'value': 'ACK' if ok else 'NACK'
}
self.tunnel.send(reply)
