def test_titanic(self): """ Test performance on titanic dataset """ corruption_1 = Duplication(proportion=0.8, duplicated_partner_id=0) titanic_scenario_1 = Scenario(2, [0.4, 0.6], epoch_count=3, minibatch_count=1, dataset_name='titanic') titanic_scenario_2 = Scenario(3, [0.2, 0.2, 0.6], corruption_parameters=[ 'not-corrupted', corruption_1, 'not-corrupted' ], epoch_count=3, minibatch_count=1, dataset_name='titanic') exp = Experiment(experiment_name='end_to_end_titanic', scenarios_list=[titanic_scenario_1], nb_repeats=2, is_save=True) exp.add_scenario(titanic_scenario_2) exp.run() titanic_scenario_1.run() assert np.min(titanic_scenario_1.mpl.history.score) > 0.65 result = pd.read_csv(exp.experiment_path / 'results.csv') assert (result.groupby('scenario_index').mean().mpl_test_score > 0.65).all()
def create_Scenario(request): dataset = request.param[0]() samples_split_option = request.param[1] corruption = request.param[2] params = {"dataset": dataset} params.update({ "partners_count": 3, "amounts_per_partner": [0.3, 0.5, 0.2], "samples_split_option": samples_split_option, "corruption_parameters": corruption, }) params.update({ "contributivity_methods": ["Shapley values", "Independent scores"], "multi_partner_learning_approach": "fedavg", "aggregation": "uniform", }) params.update({ "gradient_updates_per_pass_count": 5, "epoch_count": 2, "minibatch_count": 2, "is_early_stopping": True, }) params.update({"init_model_from": "random_initialization"}) params.update({"is_quick_demo": False}) # scenario_.dataset object is created inside the Scenario constructor scenario_ = Scenario(**params, scenario_id=0) scenario_.mpl = scenario_._multi_partner_learning_approach( scenario_, is_save_data=True) return scenario_
def test_titanic(self): """ Test performance on titanic dataset """ titanic_scenario = Scenario(2, [0.4, 0.6], epoch_count=3, minibatch_count=1, dataset_name='titanic') titanic_scenario.run() assert np.min(titanic_scenario.mpl.history.score) > 0.65
def test_add_scenario(self, create_experiment): exp = create_experiment sc = Scenario(2, [0.5, 0.5], dataset='titanic') assert len(exp.scenarios_list ) == 0, 'Scenario list should be empty when initialized' exp.add_scenario(sc) assert exp.scenarios_list[0] is sc, 'Failed to add a scenario'
def create_MultiPartnerLearning(create_all_datasets): data = create_all_datasets # Create partners_list (this is not a fixture): scenario = Scenario(3, [0.3, 0.3, 0.4], dataset=data) mpl = FederatedAverageLearning( scenario, epoch_count=2, minibatch_count=2, dataset=data, aggregation=UniformAggregator, is_early_stopping=True, is_save_data=False, ) yield mpl
def test_all_mpl_approaches(self): """ Test all the mpl approaches """ exp = Experiment() mpl_approaches = multi_partner_learning.MULTI_PARTNER_LEARNING_APPROACHES.copy( ) for approach in mpl_approaches: exp.add_scenario( Scenario(2, [0.25, 0.75], epoch_count=2, minibatch_count=2, dataset_name='mnist', dataset_proportion=0.1, multi_partner_learning_approach=approach, gradient_updates_per_pass_count=3)) exp.run() df = exp.result assert len(df) == len(mpl_approaches)
def create_Scenario(request): dataset = request.param[0]() samples_split_option = request.param[1] params = {"dataset": dataset} params.update({ "partners_count": 3, "amounts_per_partner": [0.2, 0.5, 0.3], "samples_split_option": samples_split_option, "corrupted_datasets": ["not_corrupted"] * 3, }) params.update({ "methods": ["Shapley values", "Independent scores"], "multi_partner_learning_approach": "fedavg", "aggregation": "uniform", }) params.update({ "gradient_updates_per_pass_count": 5, "epoch_count": 2, "minibatch_count": 2, "is_early_stopping": True, }) params.update({"init_model_from": "random_initialization"}) params.update({"is_quick_demo": False}) full_experiment_name = "unit-test-pytest" experiment_path = (Path.cwd() / constants.EXPERIMENTS_FOLDER_NAME / full_experiment_name) # scenario_.dataset object is created inside the Scenario constructor scenario_ = Scenario(**params, experiment_path=experiment_path, scenario_id=0, repeats_count=1) scenario_.mpl = scenario_.multi_partner_learning_approach( scenario_, is_save_data=True) scenario_.instantiate_scenario_partners() # Split data according to scenario and then pre-process successively... # ... train data, early stopping validation data, test data if scenario_.samples_split_type == "basic": scenario_.split_data() elif scenario_.samples_split_type == "advanced": scenario_.split_data_advanced() scenario_.compute_batch_sizes() scenario_.data_corruption() return scenario_