def send_to_disconnected(create_stacks, looper): # 1. create stacks (do not connect) alpha, beta, beta_msg_handler = create_stacks # 2. connect Alpha to Beta (it will be connected by TCP/CurveCP, # but not as remotes) # Beta is not connected to Alpha at all connectStack(alpha, beta) assert alpha.hasRemote(beta.name) assert not beta.hasRemote(alpha.name) assert not alpha.isConnectedTo(beta.name) assert not beta.isConnectedTo(alpha.name) # 3. send messages from Alpha to Beta numMsgs = 100 msgSender = MessageSender(numMsgs=numMsgs, fromStack=alpha, toName=beta.name) looper.add(msgSender) # 4. check that all messages are sent but not received looper.run(eventually(msgSender.checkAllSent, retryWait=1, timeout=10)) assert not alpha.isConnectedTo(beta.name) beta_msg_handler.check_received_from(alpha.name, 0) assert beta.name in alpha._stashed_to_disconnected assert len(alpha._stashed_to_disconnected[beta.name]) >= NUM_MSGS return alpha, beta, beta_msg_handler
def test_reconnect_multiple_time(looper, create_stacks): alpha, beta, beta_msg_handler = create_stacks prepStacks(looper, *[alpha, beta], connect=True, useKeys=True) num_msg = 50 send(looper, alpha, beta, num_msg) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=num_msg) total = num_msg for i in range(10): beta.stop() num_msg = random.Random().randint(1, 100) total += num_msg send(looper, alpha, beta, num_msg) looper.runFor(random.Random().randint(1, 10)) beta.start() connectStack(beta, alpha) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
def test_stash_pongs_from_unknown(looper, create_stacks): ''' Beta should stash the pong for Alpha on Alpha's ping received when Alpha is not connected yet. ''' alpha, beta = create_stacks connectStack(alpha, beta) alpha.sendPingPong(beta.name, is_ping=True) check_stashed_pongs(looper, stack=beta, to=alpha.publicKey) connectStack(beta, alpha) check_no_stashed_pongs(looper, stack=beta, to=alpha.publicKey) check_pong_received(looper, alpha, beta.name)
def test_connect_after_reconnect(looper, tdir, tconf, generated_keys, round): # create stacks alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor = create_stack("Beta", looper, tdir, tconf) # connect Alpha to Beta connectStack(alpha, beta) # reconnect Alpha alpha.reconnectRemoteWithName(beta.name) # connect Beta to Alpha connectStack(beta, alpha) # check connected looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def test_reconnect_multiple_times(looper, tdir, tconf, generated_keys): # create stacks alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor = create_stack("Beta", looper, tdir, tconf) # connect Alpha to Beta connectStack(alpha, beta) # reconnect Alpha alpha.reconnectRemoteWithName(beta.name) # connect Beta to Alpha connectStack(beta, alpha) # check connected looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT)) for i in range(10): print(i) # reconnect Alpha alpha.reconnectRemoteWithName(beta.name) looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT)) # reconnect Beta beta.reconnectRemoteWithName(alpha.name) looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT)) # reconnect Alpha and then Beta alpha.reconnectRemoteWithName(beta.name) beta.reconnectRemoteWithName(alpha.name) looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT)) # reconnect Beta and then Alpha beta.reconnectRemoteWithName(alpha.name) alpha.reconnectRemoteWithName(beta.name) looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def test_reconnect_for_long_time(looper, tdir, tconf, generated_keys): # create stacks alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor = create_stack("Beta", looper, tdir, tconf) # connect both connectStack(alpha, beta) connectStack(beta, alpha) looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT)) for i in range(10): # 1. stop Beta looper.removeProdable(beta_motor) beta_motor.stop() looper.run(eventually( checkStackDisonnected, beta, [alpha], retryWait=1, timeout=CONNECT_TIMEOUT)) looper.run(eventually( checkStackDisonnected, alpha, [beta], retryWait=1, timeout=CONNECT_TIMEOUT)) # 2. wait for some time so that Alpha re-creates the socket multiple time trying to reconnect to Beta looper.runFor(15) # 3. start Beta again beta, beta_motor = create_stack("Beta", looper, tdir, tconf) connectStack(beta, alpha) # 4. check that they are connected looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def send_to_disconnected_then_connect(send_to_disconnected, looper, ping_pong_drop_strategy, reconnect_strategy, round): # 1. send to disconnected stack alpha, beta, beta_msg_handler = send_to_disconnected # 2. reconnect Alpha if reconnect_strategy == 'reconnect': alpha.reconnectRemoteWithName(beta.name) prepStacks(looper, beta, connect=False) # 3. Make sure that ping (pong) is the first message received by Alpha # dropping pong (ping) if ping_pong_drop_strategy == 'ping_first': drop_pongs(alpha) elif ping_pong_drop_strategy == 'pong_first': drop_pings(alpha) # 4. Connect beta to alpha connectStack(beta, alpha) # 5. make sure that ping (pong) is the first message received by Alpha if ping_pong_drop_strategy == 'ping_first': check_ping_received(looper, alpha, beta.name) assert not alpha.isConnectedTo(beta.name) elif ping_pong_drop_strategy == 'pong_first': check_pong_received(looper, alpha, beta.name) assert beta.name not in alpha.has_ping # 6. make sure that dropped pong (ping) is received to establish connection if ping_pong_drop_strategy == 'ping_first': re_send_pong(alpha, beta) check_ping_received(looper, alpha, beta.name) check_pong_received(looper, alpha, beta.name) elif ping_pong_drop_strategy == 'pong_first': re_send_ping(alpha, beta) check_ping_received(looper, alpha, beta.name) check_pong_received(looper, alpha, beta.name) return alpha, beta, beta_msg_handler
def test_connect_and_send_after_reconnect(looper, tdir, tconf, generated_keys, round): # create stacks alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf) # connect Alpha to Beta connectStack(alpha, beta) # reconnect Alpha alpha.reconnectRemoteWithName(beta.name) total = send(looper, alpha, beta) # connect Beta to Alpha connectStack(beta, alpha) # check connected looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total += send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
def test_reconnect_for_long_time_lose_pongs(looper, tdir, tconf, generated_keys): # create stacks alpha, alpha_motor = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor = create_stack("Beta", looper, tdir, tconf) # connect both connectStack(alpha, beta) connectStack(beta, alpha) looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT)) for i in range(10): # 1. stop Beta looper.removeProdable(beta_motor) beta_motor.stop() looper.run(eventually( checkStackDisonnected, beta, [alpha], retryWait=1, timeout=CONNECT_TIMEOUT)) looper.run(eventually( checkStackDisonnected, alpha, [beta], retryWait=1, timeout=CONNECT_TIMEOUT)) # 2. wait for some time so that Alpha re-creates the socket multiple time trying to reconnect to Beta looper.runFor(15) # 3. start Beta again but drop pongs to emulate a situation when Alpha connected to Beta, but Beta doesn't beta, beta_motor = create_stack("Beta", looper, tdir, tconf) beta.drop_pong = True connectStack(beta, alpha) # 4. make sure that only Alpha is connected looper.run(eventually( checkStackConnected, alpha, [beta], retryWait=1, timeout=CONNECT_TIMEOUT)) looper.run(eventually( checkStackDisonnected, beta, [alpha], retryWait=1, timeout=CONNECT_TIMEOUT)) # 5. allow beta to connect to Alpha beta.drop_pong = False # 6. check that they both connected looper.run(eventually( checkStacksConnected, [alpha, beta], retryWait=1, timeout=CONNECT_TIMEOUT))
def test_reconnect_multiple_time_with_random_waits(looper, tdir, tconf, generated_keys): # create stacks alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf) # connect both connectStack(alpha, beta) connectStack(beta, alpha) total = send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total) for i in range(10): beta.stop() total += send(looper, alpha, beta) looper.runFor(random.Random().randint(1, 10)) beta.start() connectStack(beta, alpha) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
def test_reconnect_for_long_time(looper, tdir, tconf, generated_keys): # create stacks alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf) # connect both connectStack(alpha, beta) connectStack(beta, alpha) looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total = send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total) for i in range(10): # 1. stop Beta beta.stop() beta_motor.stop() looper.run(eventually( checkStackDisonnected, beta, [alpha], timeout=CONNECT_TIMEOUT)) looper.run(eventually( checkStackDisonnected, alpha, [beta], timeout=CONNECT_TIMEOUT)) # 2. wait for some time looper.runFor(15) # 3. start Beta again beta.start() connectStack(beta, alpha) # 4. check that they are connected looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total += send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total)
def test_use_send_from_zstack_on_resend(func_create_stacks, looper): aStack, bStack = func_create_stacks(2) connectStack(aStack, bStack) """ Sending some pi msgs for creating a batch on flashOutBox This function just put 'pi ' message into outBoxes queue, not send """ aStack.sendPingPong(bStack.name) aStack.sendPingPong(bStack.name) """ Emulate batch creation and sending. Batch should be added into _stashed_to_disconnected queue """ aStack.flushOutBoxes() assert len(aStack._stashed_to_disconnected[bStack.name]) == 1 batch_to_disconnected = aStack.deserializeMsg( aStack._stashed_to_disconnected[bStack.name][0]) assert OP_FIELD_NAME in batch_to_disconnected and batch_to_disconnected[ OP_FIELD_NAME] == BATCH """ This method call connect method for bStack and put 'pi' message into outBoxes queue """ connectStack(bStack, aStack) """ Wait for socket's connecting routines """ looper.runFor(1) """ This instruction get 'pi' message from outBoxes queue, create a batch if needed and send it to aStack """ bStack.flushOutBoxes() """ It needs for getting 'pi' message from bStack. It process 'pi' message and put 'po' message to outBoxes queue """ looper.run(aStack.service()) """ Send 'po' message to bStack """ aStack.flushOutBoxes() """ Processing previous sending batch (zmq feature) and 'po' """ looper.run(bStack.service()) """ For sending 'po' message to aStack """ bStack.flushOutBoxes() """ Append 'pi' msg for checking that batch into batch will not be included """ aStack._stashed_to_disconnected[bStack.name].append('pi') """ Emulate that aStack got 'po' message from bStack and it must run _resend_to_disconnected """ looper.run(aStack.service()) """ Emulate batch creating and sending """ aStack.flushOutBoxes() looper.run(bStack._serviceStack(bStack.age, None)) """ rxMsgs queue should contains only one 'pi' message from step 3 and batch which was failed to sending to disconnected stack from step 2 """ got_pi = False got_batch = False while bStack.rxMsgs: m, frm = bStack.rxMsgs.popleft() if m.encode() not in bStack.healthMessages: msg = bStack.deserializeMsg(m) else: got_pi = True continue if OP_FIELD_NAME in msg and msg[OP_FIELD_NAME] == BATCH: if msg == batch_to_disconnected: """ Exactly the same batch which should be sent to disconnected node """ got_batch = True continue else: """Check that there is no batches with batch as message""" batch = Batch(messages=msg[f.MSGS.nm], signature=msg[f.SIG.nm]) for m in batch.messages: assert OP_FIELD_NAME not in m and BATCH not in m assert got_pi and got_batch
def test_reconnect_and_send_multi(looper, tdir, tconf, generated_keys): # create stacks alpha, alpha_motor, alpha_msg_handler = create_stack("Alpha", looper, tdir, tconf) beta, beta_motor, beta_msg_handler = create_stack("Beta", looper, tdir, tconf) # connect Alpha to Beta connectStack(alpha, beta) # connect Beta to Alpha connectStack(beta, alpha) # check connected looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total_from_alpha = 0 total_from_alpha += send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha) total_from_beta = 0 total_from_beta += send(looper, beta, alpha) check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta) for i in range(10): # reconnect Alpha alpha.stop() total_from_beta += send(looper, beta, alpha) alpha.start() connectStack(alpha, beta) looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total_from_beta += send(looper, beta, alpha) check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta) total_from_alpha += send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha) # reconnect Beta beta.stop() total_from_alpha += send(looper, alpha, beta) beta.start() connectStack(beta, alpha) looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total_from_alpha += send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha) total_from_beta += send(looper, beta, alpha) check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta) # reconnect Alpha and then Beta alpha.stop() alpha.start() connectStack(alpha, beta) beta.stop() beta.start() connectStack(beta, alpha) looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total_from_alpha += send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha) total_from_beta += send(looper, beta, alpha) check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta) # reconnect Beta and then Alpha beta.stop() beta.start() connectStack(beta, alpha) alpha.stop() alpha.start() connectStack(alpha, beta) looper.run(eventually( checkStacksConnected, [alpha, beta], timeout=CONNECT_TIMEOUT)) total_from_alpha += send(looper, alpha, beta) check_all_received(looper, frm=alpha, to_msg_handler=beta_msg_handler, num_msg=total_from_alpha) total_from_beta += send(looper, beta, alpha) check_all_received(looper, frm=beta, to_msg_handler=alpha_msg_handler, num_msg=total_from_beta)