def test_end_training_duplicated_updates(coordinator_service, participant_stub): # participant can only send updates once in a single round participant_stub.Rendezvous(coordinator_pb2.RendezvousRequest()) participant_stub.EndTraining(coordinator_pb2.EndTrainingRequest()) with pytest.raises(grpc.RpcError): reply = participant_stub.EndTraining( coordinator_pb2.EndTrainingRequest()) assert reply.status_code == grpc.StatusCode.ALREADY_EXISTS
def test_training_finished(): coordinator = Coordinator(minimum_participants_in_round=1, fraction_of_participants=1.0, num_rounds=2) coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant1") # Deliver results for 2 rounds coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant1") coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant1") assert coordinator.state == coordinator_pb2.State.FINISHED
def test_duplicated_update_submit(): # the coordinator should not accept multiples updates from the same participant # in the same round coordinator = Coordinator(minimum_participants_in_round=2, fraction_of_participants=1.0) coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant1") coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant2") coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant1") with pytest.raises(DuplicatedUpdateError): coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant1")
def end_training(channel, theta_n: Tuple[Theta, int], history: History, metrics: Metrics): """Starts a training completion exchange with Coordinator, sending a locally trained model and metadata. Args: channel: gRPC channel to Coordinator. theta_n (obj:`Tuple[Theta, int]`): Locally trained model. history (obj:`History`): History metadata. Metrics (obj:`Metrics`): Metrics metadata. """ # pylint: disable=no-member stub = coordinator_pb2_grpc.CoordinatorStub(channel) # build request starting with theta update theta, num = theta_n theta_n_proto = coordinator_pb2.EndTrainingRequest.ThetaUpdate( theta_prime=[ndarray_to_proto(nda) for nda in theta], num_examples=num) # history data h = { k: coordinator_pb2.EndTrainingRequest.HistoryValue(values=v) for k, v in history.items() } # metrics cid, vbc = metrics m = coordinator_pb2.EndTrainingRequest.Metrics(cid=cid, vol_by_class=vbc) # assemble req req = coordinator_pb2.EndTrainingRequest(theta_update=theta_n_proto, history=h, metrics=m) # send request to end training reply = stub.EndTraining(req) logger.info("Participant received", reply_type=type(reply))
def test_end_training_denied(participant_stub, coordinator_service): # heartbeat requests are only allowed if the participant has already # rendezvous with the coordinator with pytest.raises(grpc.RpcError): reply = participant_stub.EndTraining( coordinator_pb2.EndTrainingRequest()) assert reply.status_code == grpc.StatusCode.PERMISSION_DENIED
def test_end_training_reinitialize_local_models(): coordinator = Coordinator(minimum_participants_in_round=2, fraction_of_participants=1.0, num_rounds=2) coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant1") coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant2") coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant1") # After one participant sends its updates we should have one update in the coordinator assert len(coordinator.round.updates) == 1 coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant2") # once the second participant delivers its updates the round ends and the local models # are reinitialized assert coordinator.round.updates == {}
def test_end_training_round_update(): # Test that the round number is updated once all participants sent their updates coordinator = Coordinator(minimum_participants_in_round=2, fraction_of_participants=1.0, num_rounds=2) coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant1") coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant2") # check that we are currently in round 1 assert coordinator.current_round == 1 coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant1") # check we are still in round 1 assert coordinator.current_round == 1 coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant2") # check that round number was updated assert coordinator.current_round == 2
def test_end_training(): # we need two participants so that we can check the status of the local update mid round # with only one participant it wouldn't work because the local updates state is cleaned at # the end of each round coordinator = Coordinator(minimum_participants_in_round=2, fraction_of_participants=1.0) coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant1") coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant2") coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant1") assert len(coordinator.round.updates) == 1
def test_wrong_participant(): # coordinator should not accept requests from participants that it has not accepted coordinator = Coordinator(minimum_participants_in_round=1, fraction_of_participants=1.0) coordinator.on_message(coordinator_pb2.RendezvousRequest(), "participant1") with pytest.raises(UnknownParticipantError): coordinator.on_message(coordinator_pb2.HeartbeatRequest(), "participant2") with pytest.raises(UnknownParticipantError): coordinator.on_message(coordinator_pb2.StartTrainingRequest(), "participant2") with pytest.raises(UnknownParticipantError): coordinator.on_message(coordinator_pb2.EndTrainingRequest(), "participant2")