def send_to_disconnected(create_stacks, looper):
# 1. create stacks (do not connect)
alpha, beta, beta_msg_handler = create_stacks
# 2. connect Alpha to Beta (it will be connected by TCP/CurveCP,
# but not as remotes)
# Beta is not connected to Alpha at all
connectStack(alpha, beta)
assert alpha.hasRemote(beta.name)
assert not beta.hasRemote(alpha.name)
assert not alpha.isConnectedTo(beta.name)
assert not beta.isConnectedTo(alpha.name)
# 3. send messages from Alpha to Beta
numMsgs = 100
msgSender = MessageSender(numMsgs=numMsgs, fromStack=alpha, toName=beta.name)
looper.add(msgSender)
# 4. check that all messages are sent but not received
looper.run(eventually(msgSender.checkAllSent,
retryWait=1,
timeout=10))
assert not alpha.isConnectedTo(beta.name)
beta_msg_handler.check_received_from(alpha.name, 0)
assert beta.name in alpha._stashed_to_disconnected
assert len(alpha._stashed_to_disconnected[beta.name]) >= NUM_MSGS
return alpha, beta, beta_msg_handler
def test_reconnect_multiple_time(looper, create_stacks):
alpha, beta, beta_msg_handler = create_stacks
prepStacks(looper, *[alpha, beta], connect=True, useKeys=True)
num_msg = 50
send(looper, alpha, beta, num_msg)
check_all_received(looper,
frm=alpha,
to_msg_handler=beta_msg_handler,
num_msg=num_msg)
total = num_msg
for i in range(10):
beta.stop()
num_msg = random.Random().randint(1, 100)
total += num_msg
send(looper, alpha, beta, num_msg)
looper.runFor(random.Random().randint(1, 10))
beta.start()
connectStack(beta, alpha)
check_all_received(looper,
frm=alpha,
to_msg_handler=beta_msg_handler,
num_msg=total)
def test_stash_pongs_from_unknown(looper, create_stacks):
'''
Beta should stash the pong for Alpha on Alpha's ping received when
Alpha is not connected yet.
'''
alpha, beta = create_stacks
connectStack(alpha, beta)
alpha.sendPingPong(beta.name, is_ping=True)
check_stashed_pongs(looper, stack=beta, to=alpha.publicKey)
connectStack(beta, alpha)
check_no_stashed_pongs(looper, stack=beta, to=alpha.publicKey)
check_pong_received(looper, alpha, beta.name)
def test_connect_after_reconnect(looper, tdir, tconf, generated_keys, round):
# create stacks
alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor = create_stack("Beta", looper, tdir, tconf)
# connect Alpha to Beta
connectStack(alpha, beta)
# reconnect Alpha
alpha.reconnectRemoteWithName(beta.name)
# connect Beta to Alpha
connectStack(beta, alpha)
# check connected
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def test_reconnect_multiple_times(looper, tdir, tconf, generated_keys):
# create stacks
alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor = create_stack("Beta", looper, tdir, tconf)
# connect Alpha to Beta
connectStack(alpha, beta)
# reconnect Alpha
alpha.reconnectRemoteWithName(beta.name)
# connect Beta to Alpha
connectStack(beta, alpha)
# check connected
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
for i in range(10):
print(i)
# reconnect Alpha
alpha.reconnectRemoteWithName(beta.name)
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
# reconnect Beta
beta.reconnectRemoteWithName(alpha.name)
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
# reconnect Alpha and then Beta
alpha.reconnectRemoteWithName(beta.name)
beta.reconnectRemoteWithName(alpha.name)
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
# reconnect Beta and then Alpha
beta.reconnectRemoteWithName(alpha.name)
alpha.reconnectRemoteWithName(beta.name)
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def test_reconnect_for_long_time(looper, tdir, tconf, generated_keys):
# create stacks
alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor = create_stack("Beta", looper, tdir, tconf)
# connect both
connectStack(alpha, beta)
connectStack(beta, alpha)
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
for i in range(10):
# 1. stop Beta
looper.removeProdable(beta_motor)
beta_motor.stop()
looper.run(eventually(
checkStackDisonnected, beta, [alpha], retryWait=1, timeout=CONNECT_TIMEOUT))
looper.run(eventually(
checkStackDisonnected, alpha, [beta], retryWait=1, timeout=CONNECT_TIMEOUT))
# 2. wait for some time so that Alpha re-creates the socket multiple time trying to reconnect to Beta
looper.runFor(15)
# 3. start Beta again
beta, beta_motor = create_stack("Beta", looper, tdir, tconf)
connectStack(beta, alpha)
# 4. check that they are connected
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def send_to_disconnected_then_connect(send_to_disconnected, looper,
ping_pong_drop_strategy, reconnect_strategy,
round):
# 1. send to disconnected stack
alpha, beta, beta_msg_handler = send_to_disconnected
# 2. reconnect Alpha
if reconnect_strategy == 'reconnect':
alpha.reconnectRemoteWithName(beta.name)
prepStacks(looper, beta, connect=False)
# 3. Make sure that ping (pong) is the first message received by Alpha
# dropping pong (ping)
if ping_pong_drop_strategy == 'ping_first':
drop_pongs(alpha)
elif ping_pong_drop_strategy == 'pong_first':
drop_pings(alpha)
# 4. Connect beta to alpha
connectStack(beta, alpha)
# 5. make sure that ping (pong) is the first message received by Alpha
if ping_pong_drop_strategy == 'ping_first':
check_ping_received(looper, alpha, beta.name)
assert not alpha.isConnectedTo(beta.name)
elif ping_pong_drop_strategy == 'pong_first':
check_pong_received(looper, alpha, beta.name)
assert beta.name not in alpha.has_ping
# 6. make sure that dropped pong (ping) is received to establish connection
if ping_pong_drop_strategy == 'ping_first':
re_send_pong(alpha, beta)
check_ping_received(looper, alpha, beta.name)
check_pong_received(looper, alpha, beta.name)
elif ping_pong_drop_strategy == 'pong_first':
re_send_ping(alpha, beta)
check_ping_received(looper, alpha, beta.name)
check_pong_received(looper, alpha, beta.name)
return alpha, beta, beta_msg_handler
def test_connect_and_send_after_reconnect(looper, tdir, tconf, generated_keys, round):
# create stacks
alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf)
# connect Alpha to Beta
connectStack(alpha, beta)
# reconnect Alpha
alpha.reconnectRemoteWithName(beta.name)
total = send(looper, alpha, beta)
# connect Beta to Alpha
connectStack(beta, alpha)
# check connected
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total += send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
def test_reconnect_for_long_time_lose_pongs(looper, tdir, tconf, generated_keys):
# create stacks
alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor = create_stack("Beta", looper, tdir, tconf)
# connect both
connectStack(alpha, beta)
connectStack(beta, alpha)
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
for i in range(10):
# 1. stop Beta
looper.removeProdable(beta_motor)
beta_motor.stop()
looper.run(eventually(
checkStackDisonnected, beta, [alpha], retryWait=1, timeout=CONNECT_TIMEOUT))
looper.run(eventually(
checkStackDisonnected, alpha, [beta], retryWait=1, timeout=CONNECT_TIMEOUT))
# 2. wait for some time so that Alpha re-creates the socket multiple time trying to reconnect to Beta
looper.runFor(15)
# 3. start Beta again but drop pongs to emulate a situation when Alpha connected to Beta, but Beta doesn't
beta, beta_motor = create_stack("Beta", looper, tdir, tconf)
beta.drop_pong = True
connectStack(beta, alpha)
# 4. make sure that only Alpha is connected
looper.run(eventually(
checkStackConnected, alpha, [beta], retryWait=1, timeout=CONNECT_TIMEOUT))
looper.run(eventually(
checkStackDisonnected, beta, [alpha], retryWait=1, timeout=CONNECT_TIMEOUT))
# 5. allow beta to connect to Alpha
beta.drop_pong = False
# 6. check that they both connected
looper.run(eventually(
checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def test_reconnect_multiple_time_with_random_waits(looper, tdir, tconf, generated_keys):
# create stacks
alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf)
# connect both
connectStack(alpha, beta)
connectStack(beta, alpha)
total = send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
for i in range(10):
beta.stop()
total += send(looper, alpha, beta)
looper.runFor(random.Random().randint(1, 10))
beta.start()
connectStack(beta, alpha)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
def test_reconnect_for_long_time(looper, tdir, tconf, generated_keys):
# create stacks
alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf)
# connect both
connectStack(alpha, beta)
connectStack(beta, alpha)
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total = send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
for i in range(10):
# 1. stop Beta
beta.stop()
beta_motor.stop()
looper.run(eventually(
checkStackDisonnected, beta, [alpha], timeout=CONNECT_TIMEOUT))
looper.run(eventually(
checkStackDisonnected, alpha, [beta], timeout=CONNECT_TIMEOUT))
# 2. wait for some time
looper.runFor(15)
# 3. start Beta again
beta.start()
connectStack(beta, alpha)
# 4. check that they are connected
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total += send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
def test_use_send_from_zstack_on_resend(func_create_stacks, looper):
aStack, bStack = func_create_stacks(2)
connectStack(aStack, bStack)
"""
Sending some pi msgs for creating a batch on flashOutBox
This function just put 'pi ' message into outBoxes queue, not send
"""
aStack.sendPingPong(bStack.name)
aStack.sendPingPong(bStack.name)
"""
Emulate batch creation and sending. Batch should be added into _stashed_to_disconnected queue
"""
aStack.flushOutBoxes()
assert len(aStack._stashed_to_disconnected[bStack.name]) == 1
batch_to_disconnected = aStack.deserializeMsg(
aStack._stashed_to_disconnected[bStack.name][0])
assert OP_FIELD_NAME in batch_to_disconnected and batch_to_disconnected[
OP_FIELD_NAME] == BATCH
"""
This method call connect method for bStack and put 'pi' message into outBoxes queue
"""
connectStack(bStack, aStack)
"""
Wait for socket's connecting routines
"""
looper.runFor(1)
"""
This instruction get 'pi' message from outBoxes queue, create a batch if needed and send it to aStack
"""
bStack.flushOutBoxes()
"""
It needs for getting 'pi' message from bStack. It process 'pi' message and put 'po' message to outBoxes queue
"""
looper.run(aStack.service())
"""
Send 'po' message to bStack
"""
aStack.flushOutBoxes()
"""
Processing previous sending batch (zmq feature) and 'po'
"""
looper.run(bStack.service())
"""
For sending 'po' message to aStack
"""
bStack.flushOutBoxes()
"""
Append 'pi' msg for checking that batch into batch will not be included
"""
aStack._stashed_to_disconnected[bStack.name].append('pi')
"""
Emulate that aStack got 'po' message from bStack and it must run _resend_to_disconnected
"""
looper.run(aStack.service())
"""
Emulate batch creating and sending
"""
aStack.flushOutBoxes()
looper.run(bStack._serviceStack(bStack.age, None))
"""
rxMsgs queue should contains only one 'pi' message from step 3 and batch
which was failed to sending to disconnected stack from step 2
"""
got_pi = False
got_batch = False
while bStack.rxMsgs:
m, frm = bStack.rxMsgs.popleft()
if m.encode() not in bStack.healthMessages:
msg = bStack.deserializeMsg(m)
else:
got_pi = True
continue
if OP_FIELD_NAME in msg and msg[OP_FIELD_NAME] == BATCH:
if msg == batch_to_disconnected:
"""
Exactly the same batch which should be sent to disconnected node
"""
got_batch = True
continue
else:
"""Check that there is no batches with batch as message"""
batch = Batch(messages=msg[f.MSGS.nm], signature=msg[f.SIG.nm])
for m in batch.messages:
assert OP_FIELD_NAME not in m and BATCH not in m
assert got_pi and got_batch
def test_reconnect_and_send_multi(looper, tdir, tconf, generated_keys):
# create stacks
alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf)
beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf)
# connect Alpha to Beta
connectStack(alpha, beta)
# connect Beta to Alpha
connectStack(beta, alpha)
# check connected
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total_from_alpha = 0
total_from_alpha += send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha)
total_from_beta = 0
total_from_beta += send(looper, beta, alpha)
check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta)
for i in range(10):
# reconnect Alpha
alpha.stop()
total_from_beta += send(looper, beta, alpha)
alpha.start()
connectStack(alpha, beta)
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total_from_beta += send(looper, beta, alpha)
check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta)
total_from_alpha += send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha)
# reconnect Beta
beta.stop()
total_from_alpha += send(looper, alpha, beta)
beta.start()
connectStack(beta, alpha)
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total_from_alpha += send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha)
total_from_beta += send(looper, beta, alpha)
check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta)
# reconnect Alpha and then Beta
alpha.stop()
alpha.start()
connectStack(alpha, beta)
beta.stop()
beta.start()
connectStack(beta, alpha)
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total_from_alpha += send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha)
total_from_beta += send(looper, beta, alpha)
check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta)
# reconnect Beta and then Alpha
beta.stop()
beta.start()
connectStack(beta, alpha)
alpha.stop()
alpha.start()
connectStack(alpha, beta)
looper.run(eventually(
checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT))
total_from_alpha += send(looper, alpha, beta)
check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha)
total_from_beta += send(looper, beta, alpha)
check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta)
