def test_slave_add_while_waiting(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=25)
dq = DistributedQueue(node,
conn,
numQueues=3,
throw_local_queue_events=True)
dq2 = DistributedQueue(node2,
conn,
numQueues=3,
throw_local_queue_events=True)
dq.connect_to_peer_protocol(dq2, conn)
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
# Add one request for both master and slave
create_id = 0
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq.add(request=request, qid=0)
create_id = 1
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq2.add(request=request, qid=0)
# Wait for slaves add to arrive but not the ack
run_time = dq.comm_delay * (3 / 4)
sim_run(run_time)
# Add request from master
create_id = 2
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq.add(request=request, qid=0)
# Make sure things are added
run_time = dq.comm_delay * 4
sim_run(run_time)
self.ready_items(dq.queueList[0])
self.ready_items(dq2.queueList[0])
self.check_local_queues(dq.queueList[0], dq2.queueList[0])
def test_full_wraparound(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=25)
wSize = 2
maxSeq = 6
dq = DistributedQueue(node,
conn,
numQueues=3,
throw_local_queue_events=True,
myWsize=wSize,
otherWsize=wSize,
maxSeq=maxSeq)
dq2 = DistributedQueue(node2,
conn,
numQueues=3,
throw_local_queue_events=True,
myWsize=wSize,
otherWsize=wSize,
maxSeq=maxSeq)
dq.connect_to_peer_protocol(dq2, conn)
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
for timestep in range(1, maxSeq + 1):
dq.add(timestep, 0)
sim_run(timestep * 1000000)
dq.queueList[0].queue[timestep - 1].ready = True
dq2.queueList[0].queue[timestep - 1].ready = True
dq.queueList[0].pop()
dq2.queueList[0].pop()
dq.add(maxSeq, 0)
sim_run(maxSeq * 100000)
dq.queueList[0].pop()
dq2.queueList[0].pop()
dq.queueList[0].pop()
dq2.queueList[0].pop()
def test_resend_acks(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=25)
dq = DistributedQueue(node,
conn,
numQueues=3,
throw_local_queue_events=True)
dq2 = DistributedQueue(node2,
conn,
numQueues=3,
throw_local_queue_events=True)
dq.connect_to_peer_protocol(dq2, conn)
storage1 = []
storage2 = []
def callback1(result):
storage1.append(result)
def callback2(result):
storage2.append(result)
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
dq.add_callback = callback1
dq2.add_callback = callback2
# Add one request
create_id = 0
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq2.add(request=request, qid=0)
run_time = dq.comm_delay * dq.timeout_factor
sim_run(run_time)
# Set way to short timeout (to force resend)
dq2.timeout_factor = 1 / 2
# Add one request (slave)
create_id = 1
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq2.add(request=request, qid=0)
run_time += (dq.comm_delay + 1) * 4
sim_run(run_time)
# Set to correct factor again
dq2.timeout_factor = 2
create_id = 2
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq.add(request=request, qid=0)
dq2.add(request=request, qid=0)
run_time += dq.comm_delay * dq.timeout_factor
sim_run(run_time)
self.assertEqual(len(storage1), 4)
self.assertEqual(len(storage2), 4)
q_seqs1 = [res[2] for res in storage1]
q_seqs2 = [res[2] for res in storage2]
for qseq in range(4):
for q_seqs in [q_seqs1, q_seqs2]:
# TODO do we care about the ordering?
self.assertIn(qseq, q_seqs)
def test_unordered_subsequent_acks(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=25)
dq = DistributedQueue(node,
conn,
numQueues=3,
throw_local_queue_events=True)
dq2 = DistributedQueue(node2,
conn,
numQueues=3,
throw_local_queue_events=True)
dq.connect_to_peer_protocol(dq2, conn)
storage1 = []
storage2 = []
def callback1(result):
storage1.append(result)
def callback2(result):
storage2.append(result)
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
# Add three requests (master)
dq.add_callback = callback1
dq2.add_callback = callback2
for create_id in range(3):
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True,
False, False, True)
dq.add(request=request, qid=0)
run_time = dq.comm_delay * dq.timeout_factor
sim_run(run_time)
self.assertEqual(len(storage1), 3)
self.assertEqual(len(storage2), 3)
q_seqs1 = [res[2] for res in storage1]
q_seqs2 = [res[2] for res in storage2]
self.assertEqual(q_seqs1, [0, 1, 2])
self.assertEqual(q_seqs2, [0, 1, 2])
# Remove one request (such that next queue seq will be 0 again)
dq.remove_item(0, 1)
dq2.remove_item(0, 1)
storage1 = []
storage2 = []
# Add requests from master and slave
create_id = 3
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq.add(request=request, qid=0)
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq2.add(request=request, qid=0)
run_time += dq.comm_delay * dq.timeout_factor
sim_run(run_time)
self.assertEqual(len(storage1), 2)
self.assertEqual(len(storage2), 2)
q_seqs1 = [res[2] for res in storage1]
q_seqs2 = [res[2] for res in storage2]
self.assertIn(1, q_seqs1)
self.assertIn(3, q_seqs1)
self.assertIn(1, q_seqs2)
self.assertIn(3, q_seqs2)
def test_lossy_comms_wraparound(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=0.1)
dq = DistributedQueue(node,
conn,
numQueues=3,
throw_local_queue_events=True)
dq2 = DistributedQueue(node2,
conn,
numQueues=3,
throw_local_queue_events=True)
dq.connect_to_peer_protocol(dq2, conn)
self.lost_messages = 0
self.lost_seq = dq.myWsize + 1
def faulty_send_msg(cmd, data, clock):
if self.lost_messages == 0 and clock[0] == self.lost_seq:
self.lost_messages += 1
else:
dq.conn.put_from(dq.myID, (cmd, data, clock))
dq.send_msg = faulty_send_msg
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
num_adds = 0
r = 1
curr_time = 0
import pdb
while num_adds < 2 * dq.maxCommsSeq:
add_delay = 2
for i in range(dq.myWsize):
request = SchedulerRequest(0, 0, 0, 0, i, 0, 0, True, False,
False, True)
dq.add(request)
sim_run(curr_time + (i + 1) * add_delay)
curr_time += add_delay
for j in range(dq2.myWsize):
request = SchedulerRequest(0, 0, 0, 0, j, 0, 0, True, False,
False, True)
dq2.add(request)
sim_run(curr_time + (j + 1) * add_delay)
curr_time += add_delay
num_adds += dq.myWsize
num_adds += dq2.myWsize
run_time = r * dq.comm_delay * 4
sim_run(curr_time + run_time)
curr_time += run_time
r += 1
self.ready_items(dq.queueList[0])
self.ready_items(dq2.queueList[0])
self.check_local_queues(dq.queueList[0], dq2.queueList[0])
for i in range(dq.myWsize + dq2.myWsize):
dq.local_pop()
dq2.local_pop()
def test_full_queue(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=0.0001)
dq = DistributedQueue(node, conn, numQueues=3)
dq2 = DistributedQueue(node2, conn, numQueues=3)
dq.connect_to_peer_protocol(dq2, conn)
storage = []
def callback(result):
storage.append(result)
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
for j in range(dq.maxSeq):
dq.add(request=j + 1, qid=0)
sim_run(1000)
with self.assertRaises(LinkLayerException):
dq.add(request=0, qid=0)
dq2.add_callback = callback
for j in range(dq2.maxSeq):
dq2.add(request=j + 1, qid=1)
sim_run(2000)
with self.assertRaises(LinkLayerException):
dq2.add(request=dq2.maxSeq + 1, qid=1)
self.assertEqual(len(storage), 257)
self.assertEqual(storage[-1], (dq2.DQ_ERR, 1, None, dq2.maxSeq + 1))
storage = []
for j in range(dq.maxSeq):
if j % 2:
dq.add(request=j + 1, qid=2)
else:
dq2.add(request=j + 1, qid=2)
dq2.add(request=dq.maxSeq + 1, qid=2)
sim_run(3000)
with self.assertRaises(LinkLayerException):
dq.add(request=0, qid=2)
self.assertEqual(len(storage), 257)
self.assertEqual(storage[-1], (dq2.DQ_REJECT, 2, 0, dq2.maxSeq + 1))
def test_lost_add_slave(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=0.0001)
dq = DistributedQueue(node, conn, numQueues=3)
dq2 = DistributedQueue(node2, conn, numQueues=3)
dq.connect_to_peer_protocol(dq2, conn)
self.lost_messages = defaultdict(int)
def faulty_send_add(cmd, data, clock):
if cmd == dq2.CMD_ADD:
_, cseq, qid, qseq, request = data
if self.lost_messages[(cseq, qid, qseq)] >= 1:
dq2.conn.put_from(dq2.myID, (cmd, data, clock))
else:
self.lost_messages[(cseq, qid, qseq)] += 1
else:
dq2.conn.put_from(dq2.myID, (cmd, data, clock))
def faulty_send_ack(cmd, data, clock):
if cmd == dq.CMD_ADD_ACK:
_, ackd_id, qseq = data
if self.lost_messages[(ackd_id, qseq)] >= 1:
dq.conn.put_from(dq.myID, (cmd, data, clock))
else:
self.lost_messages[(ackd_id, qseq)] += 1
else:
dq.conn.put_from(dq.myID, (cmd, data, clock))
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
dq2.send_msg = faulty_send_add
dq.send_msg = faulty_send_ack
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
reqs = []
num_reqs = 10
for i in range(num_reqs):
req = [node2.nodeID, i]
reqs.append(req)
dq2.add(req)
sim_run(0.1 * (i + 1))
sim_run(200)
# Check that all add and add_ack messages were lost once
for v in self.lost_messages.values():
self.assertEqual(v, 1)
# Check that the item successfully got added
self.assertEqual(len(dq.queueList[0].queue), 10)
self.assertEqual(len(dq2.queueList[0].queue), 10)
for i, expected_req in zip(range(num_reqs), reqs):
item1 = dq.queueList[0].queue[i]
item2 = dq2.queueList[0].queue[i]
self.assertEqual(item1.request, expected_req)
self.assertEqual(item2.request, expected_req)
def test_random_add_remove(self):
sim_reset()
node = QuantumNode("TestNode 1", 1)
node2 = QuantumNode("TestNode 2", 2)
conn = ClassicalFibreConnection(node, node2, length=25)
dq = DistributedQueue(node,
conn,
numQueues=3,
throw_local_queue_events=True)
dq2 = DistributedQueue(node2,
conn,
numQueues=3,
throw_local_queue_events=True)
dq.connect_to_peer_protocol(dq2, conn)
nodes = [
(node, [dq]),
(node2, [dq2]),
]
conns = [(conn, "dq_conn", [dq, dq2])]
network = EasyNetwork(name="DistQueueNetwork",
nodes=nodes,
connections=conns)
network.start()
create_id = 0
for _ in range(20):
# Add random requests to master
num_reqs_master = randint(0, 3)
for _ in range(num_reqs_master):
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True,
False, False, True)
try:
dq.add(request=request, qid=0)
except LinkLayerException:
# Full queue
pass
create_id += 1
# Add random requests to slave
num_reqs_slave = randint(0, 3)
for _ in range(num_reqs_slave):
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True,
False, False, True)
try:
dq2.add(request=request, qid=0)
except LinkLayerException:
# Full queue
pass
create_id += 1
# Randomly remove things for both
num_pop = randint(0, 6)
for _ in range(num_pop):
dq.local_pop(qid=0)
# Run for random fraction of timeout
r = randint(1, 20)
run_time = dq.comm_delay * dq.timeout_factor * (r / 10)
sim_run(run_time)
# Make sure things are not in flight
sim_run()
self.ready_items(dq.queueList[0])
self.ready_items(dq2.queueList[0])
self.check_local_queues(dq.queueList[0], dq2.queueList[0])
# Add one request for both master and slave
create_id = 0
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq.add(request=request, qid=0)
create_id = 1
request = SchedulerRequest(0, 0, 0, 0, create_id, 0, 0, True, False,
False, True)
dq2.add(request=request, qid=0)