def test_CreateFairRankingHalfHalf(self):
minProp = 0.5
sigma = 0.1
ranking = fairRanking(self.__k, self.__protectedCandidates,
self.__nonProtectedCandidates, minProp, sigma)[0]
tester = FairnessInRankingsTester(minProp, sigma, self.__k, False)
predecessor = ranking[0]
self.assertFalse(predecessor.isProtected,
"first one should be non-protected")
candidatesNeededAtLastIdx = tester.candidates_needed[0]
for idx in range(1, len(ranking)):
current = ranking[idx]
if candidatesNeededAtLastIdx == tester.candidates_needed[idx]:
# protected candidate should be inserted at each idx, at which the number of
# needed protected candidates changes, otherwise should be the better one (for this
# setting, not in general for the algorithm)
self.assertFalse(current.isProtected)
else:
self.assertTrue(
current.isProtected,
"every candidate on even position should be protected")
predecessor = current
candidatesNeededAtLastIdx = tester.candidates_needed[idx]
def test_CreateFairRankingOnlyProtected(self):
minProp = constants.ESSENTIALLY_ONE
sigma = 0.1
ranking = fairRanking(self.__k, self.__protectedCandidates,
self.__nonProtectedCandidates, minProp, sigma)[0]
predecessor = ranking[0]
self.assertTrue(predecessor.isProtected,
"ranking should contain only protected")
for idx in range(1, len(ranking)):
current = ranking[idx]
self.assertTrue(current.isProtected,
"ranking should contain only protected")
self.assertGreater(
predecessor.originalQualification,
current.originalQualification,
"candidates should be ordered by qualification\
failed at position {0}".format(idx))
predecessor = current
def test_CreateFairRankingSameQualification(self):
# make all protected candidates having the same qualification as non-protected
for candidate in self.__protectedCandidates:
candidate.qualification += self.__k
for idx, candidate in enumerate(self.__protectedCandidates):
# ensure that both candidate pools have same qualifications
self.assertEqual(candidate.qualification,
self.__nonProtectedCandidates[idx].qualification)
# no matter what minProp, we should see a ranking that alternates between protected and
# non-protected
minProp = 0.5
sigma = 0.1
ranking = fairRanking(self.__k, self.__protectedCandidates,
self.__nonProtectedCandidates, minProp, sigma)[0]
predecessor = ranking[0]
self.assertTrue(predecessor.isProtected,
"first one should be protected")
for idx in range(1, len(ranking)):
current = ranking[idx]
if idx % 2 == 0:
# protected candidate should appear on each odd position
self.assertTrue(
current.isProtected,
"every candidate on odd position should be protected")
self.assertGreater(
predecessor.qualification, current.qualification,
"if we see a \
protected candidate the above one should be non-protected and have a higher score"
)
else:
self.assertFalse(current.isProtected,
"failed at position {0}".format(idx))
self.assertEqual(
predecessor.qualification, current.qualification,
"if we see a non-\
protected candidate, the above should be protected and have the same qualification"
)
predecessor = current
def test_CreateFairRankingColorBlind(self):
minProp = constants.ESSENTIALLY_ZERO
sigma = 0.1
ranking = fairRanking(self.__k, self.__protectedCandidates,
self.__nonProtectedCandidates, minProp, sigma)[0]
predecessor = ranking[0]
self.assertFalse(predecessor.isProtected,
"first one should be non-protected")
for idx in range(1, len(ranking)):
current = ranking[idx]
self.assertFalse(current.isProtected,
"all candidates should be non-protected")
self.assertGreater(
predecessor.originalQualification,
current.originalQualification,
"candidates should be ordered by qualification\
failed at position {0}".format(idx))
predecessor = current
def rankAndDump(protected, nonProtected, k, dataSetName, directory, pairsOfPAndAlpha):
"""
creates all rankings we need for one experimental data set and writes them to disk to be used later
@param protected: list of protected candidates, assumed to satisfy in-group monotonicty
@param nonProtected: list of non-protected candidates, assumed to satisfy in-group monotonicty
@param k: length of the rankings we want to create
@param dataSetName: determines which data set is used in this experiment
@param directory: directory in which to store the rankings
@param pairsOfPAndAlpha: contains the mapping of a certain alpha correction to be used for a certain p
The experimental setting is as follows: for a given data set of protected and non-
protected candidates we create the following rankings:
* a colorblind ranking,
* a ranking as in Feldman et al
* ten rankings using our FairRankingCreator, with p varying from 0.1, 0.2 to 0.9, whereas alpha
stays 0.1
"""
print("====================================================================")
print("create rankings of {0}".format(dataSetName))
if not os.path.exists(os.getcwd() + '/' + directory + '/'):
os.makedirs(os.getcwd() + '/' + directory + '/')
print("colorblind ranking", end='', flush=True)
colorblindRanking, colorblindNotSelected = fairRanking(k, protected, nonProtected, ESSENTIALLY_ZERO, 0.1)
print(" [Done]")
print("fair rankings", end='', flush=True)
pair01 = [item for item in pairsOfPAndAlpha if item[0] == 0.1][0]
fairRanking01, fair01NotSelected = fairRanking(k, protected, nonProtected, pair01[0], pair01[1])
pair02 = [item for item in pairsOfPAndAlpha if item[0] == 0.2][0]
fairRanking02, fair02NotSelected = fairRanking(k, protected, nonProtected, pair02[0], pair02[1])
pair03 = [item for item in pairsOfPAndAlpha if item[0] == 0.3][0]
fairRanking03, fair03NotSelected = fairRanking(k, protected, nonProtected, pair03[0], pair03[1])
pair04 = [item for item in pairsOfPAndAlpha if item[0] == 0.4][0]
fairRanking04, fair04NotSelected = fairRanking(k, protected, nonProtected, pair04[0], pair04[1])
pair05 = [item for item in pairsOfPAndAlpha if item[0] == 0.5][0]
fairRanking05, fair05NotSelected = fairRanking(k, protected, nonProtected, pair05[0], pair05[1])
pair06 = [item for item in pairsOfPAndAlpha if item[0] == 0.6][0]
fairRanking06, fair06NotSelected = fairRanking(k, protected, nonProtected, pair06[0], pair06[1])
pair07 = [item for item in pairsOfPAndAlpha if item[0] == 0.7][0]
fairRanking07, fair07NotSelected = fairRanking(k, protected, nonProtected, pair07[0], pair07[1])
pair08 = [item for item in pairsOfPAndAlpha if item[0] == 0.8][0]
fairRanking08, fair08NotSelected = fairRanking(k, protected, nonProtected, pair08[0], pair08[1])
pair09 = [item for item in pairsOfPAndAlpha if item[0] == 0.9][0]
fairRanking09, fair09NotSelected = fairRanking(k, protected, nonProtected, pair09[0], pair09[1])
print(" [Done]")
print("feldman ranking", end='', flush=True)
feldmanRanking, feldmanNotSelected = fair_ranker.create.feldmanRanking(protected, nonProtected, k)
print(" [Done]")
print("Write rankings to disk", end='', flush=True)
writePickleToDisk(colorblindRanking, os.getcwd() + '/' + directory + '/' + dataSetName + 'ColorblindRanking.pickle')
writePickleToDisk(colorblindNotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'ColorblindRankingNotSelected.pickle')
writePickleToDisk(feldmanRanking, os.getcwd() + '/' + directory + '/' + dataSetName + 'FeldmanRanking.pickle')
writePickleToDisk(feldmanNotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FeldmanRankingNotSelected.pickle')
writePickleToDisk(fairRanking01, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking01PercentProtected.pickle')
writePickleToDisk(fair01NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking01NotSelected.pickle')
writePickleToDisk(fairRanking02, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking02PercentProtected.pickle')
writePickleToDisk(fair02NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking02NotSelected.pickle')
writePickleToDisk(fairRanking03, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking03PercentProtected.pickle')
writePickleToDisk(fair03NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking03NotSelected.pickle')
writePickleToDisk(fairRanking04, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking04PercentProtected.pickle')
writePickleToDisk(fair04NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking04NotSelected.pickle')
writePickleToDisk(fairRanking05, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking05PercentProtected.pickle')
writePickleToDisk(fair05NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking05NotSelected.pickle')
writePickleToDisk(fairRanking06, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking06PercentProtected.pickle')
writePickleToDisk(fair06NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking06NotSelected.pickle')
writePickleToDisk(fairRanking07, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking07PercentProtected.pickle')
writePickleToDisk(fair07NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking07NotSelected.pickle')
writePickleToDisk(fairRanking08, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking08PercentProtected.pickle')
writePickleToDisk(fair08NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking08NotSelected.pickle')
writePickleToDisk(fairRanking09, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking09PercentProtected.pickle')
writePickleToDisk(fair09NotSelected, os.getcwd() + '/' + directory + '/' + dataSetName + 'FairRanking09NotSelected.pickle')
print(" [Done]")