def __evaluatePairwiseRanking(self, rankings, rankingType):
"""
performs the actual measurements for a pair of rankings of a given type, e.g. for the
color-blind ranking and the color-blind not selected candidates
"""
print("current ranking name: {0}".format(rankingType),
end='',
flush=True)
# not-selected candidates are needed for determining selection unfairness
ranking, notSelected, percProtDataset = self.__findFilePair(
rankingType, rankings)
util = metrics.ndcp(ranking)
selectUnfair = metrics.selectionUtility(ranking, notSelected)
orderUnfair = metrics.orderingUtility(ranking)
percentProt = metrics.percentageOfProtected(ranking)
print(" [Done]")
return pd.Series({
'util': util,
'selectUnfair': selectUnfair,
'orderUnfair': orderUnfair,
'percentProt': percentProt,
'percentProt_dataset': percProtDataset
})
def __evaluateXingOriginalRanking(self, rankings):
"""
evaluates the original Xing ranking that was obtained from the xing query. Needs special
treatment because there is no pair ranking that contains the non-selected, hence selection
unfairness measure doesn't make sense
"""
for filename, rank in rankings.items():
if self.ORIGINAL in filename.lower():
util = metrics.ndcp(rank)
orderUnfair = metrics.orderingUtility(rank)
percentProt = metrics.percentageOfProtected(rank)
return pd.Series({
'util': util,
'selectUnfair': 0,
'orderUnfair': orderUnfair,
'percentProt': percentProt,
'percentProt_dataset': percentProt
})
def __findFilePair(self, rankingType, rankings):
"""
finds the file pair of one ranking type in a particular experiment (e.g. for the SAT rankings
it finds the colorblind ranking and the respective list of candidates not selected for a
colorblind ranking
@param rankingType: the type of the ranking, e.g. FAIR_RANKING_01
@param rankings: all rankings of one particular directory
"""
for filename, rank in rankings.items():
if rankingType in filename.lower():
if "notselected" in filename.lower():
notSelected = rank
else:
ranking = rank
dataset = ranking + notSelected
percProt = metrics.percentageOfProtected(dataset)
return ranking, notSelected, percProt