def test_CalibratedEqualOddsPostProcessing(self):
self.post_processor = CalibratedEqualOddsPostProcessing()
result = self.post_processor.post_process(
self.validation_dataset, self.validation_dataset_with_predictions,
self.testset_with_predictions, self.seed, self.privileged_groups,
self.unprivileged_groups)
self.assertEqual(self.post_processor.name(), 'calibrated_eq_odds')
self.assertEqual(type(result), type(self.testset_with_predictions))
def test_RejectOptionPostProcessing(self):
self.post_processor = RejectOptionPostProcessing()
result = self.post_processor.post_process(
self.validation_dataset, self.validation_dataset_with_predictions,
self.testset_with_predictions, self.seed, self.privileged_groups,
self.unprivileged_groups)
self.assertEqual(self.post_processor.name(), 'reject_option')
self.assertEqual(type(result), type(self.testset_with_predictions))
def test_NoPostProcessing(self):
self.post_processor = NoPostProcessing()
result = self.post_processor.post_process(
self.validation_dataset, self.validation_dataset_with_predictions,
self.testset_with_predictions, self.seed, self.privileged_groups,
self.unprivileged_groups)
self.assertEqual(self.post_processor.name(), 'no_post_processing')
self.assertEqual(result, self.testset_with_predictions)
class TestSuitePreProcessors(unittest.TestCase):
def setUp(self):
self.validation_dataset = pickle.load(
open('fp/tests/resource/input/data_validation.obj', 'rb'))
self.validation_dataset_with_predictions = pickle.load(
open(
'fp/tests/resource/input/data_validation_with_predictions.obj',
'rb'))
self.testset_with_predictions = pickle.load(
open('fp/tests/resource/input/data_test_with_predictions.obj',
'rb'))
self.seed = 0xbeef
self.privileged_groups = [{'sex': 1.0}]
self.unprivileged_groups = [{'sex': 0.0}]
def test_NoPostProcessing(self):
self.post_processor = NoPostProcessing()
result = self.post_processor.post_process(
self.validation_dataset, self.validation_dataset_with_predictions,
self.testset_with_predictions, self.seed, self.privileged_groups,
self.unprivileged_groups)
self.assertEqual(self.post_processor.name(), 'no_post_processing')
self.assertEqual(result, self.testset_with_predictions)
def test_RejectOptionPostProcessing(self):
self.post_processor = RejectOptionPostProcessing()
result = self.post_processor.post_process(
self.validation_dataset, self.validation_dataset_with_predictions,
self.testset_with_predictions, self.seed, self.privileged_groups,
self.unprivileged_groups)
self.assertEqual(self.post_processor.name(), 'reject_option')
self.assertEqual(type(result), type(self.testset_with_predictions))
def test_EqualOddsPostProcessing(self):
self.post_processor = EqualOddsPostProcessing()
result = self.post_processor.post_process(
self.validation_dataset, self.validation_dataset_with_predictions,
self.testset_with_predictions, self.seed, self.privileged_groups,
self.unprivileged_groups)
self.assertEqual(self.post_processor.name(), 'eq_odds')
self.assertEqual(type(result), type(self.testset_with_predictions))
def test_CalibratedEqualOddsPostProcessing(self):
self.post_processor = CalibratedEqualOddsPostProcessing()
result = self.post_processor.post_process(
self.validation_dataset, self.validation_dataset_with_predictions,
self.testset_with_predictions, self.seed, self.privileged_groups,
self.unprivileged_groups)
self.assertEqual(self.post_processor.name(), 'calibrated_eq_odds')
self.assertEqual(type(result), type(self.testset_with_predictions))
seeds = [
0xbeef, 0xcafe, 0xdead, 0xdeadcafe, 0xdeadbeef, 0xbeefcafe, 0xcafebeef, 50,
583, 5278, 100000, 0xefac, 0xfeeb, 0xdaed, 0xefacdaed, 0xfeebdead
]
learners = [
NonTunedLogisticRegression(),
LogisticRegression(),
NonTunedDecisionTree(),
DecisionTree()
]
processors = [(NoPreProcessing(), NoPostProcessing()),
(DIRemover(1.0), NoPostProcessing()),
(DIRemover(0.5), NoPostProcessing()),
(Reweighing(), NoPostProcessing()),
(NoPreProcessing(), RejectOptionPostProcessing()),
(NoPreProcessing(), CalibratedEqualOddsPostProcessing())]
def calculate_metrics(seed, learner, pre_processor, post_processor):
'''
Experiment function to run the experiments
'''
exp = GermanCreditDatasetSexExperiment(
fixed_random_seed=seed,
train_data_sampler=CompleteData(),
missing_value_handler=CompleteCaseAnalysis(),
numeric_attribute_scaler=NamedStandardScaler(),
learners=[learner],
pre_processors=[pre_processor],
post_processors=[post_processor])
exp.run()