def run_eval_alg(algorithm, train, test, dataset, processed_data,
all_sensitive_attributes, single_sensitive, tag, gpu):
"""
Runs the algorithm and gets the resulting metric evaluations.
"""
privileged_vals = dataset.get_privileged_class_names_with_joint(tag)
positive_val = dataset.get_positive_class_val(tag)
# get the actual classifications and sensitive attributes
actual = test[dataset.get_class_attribute()].values.tolist()
params = {**algorithm.get_default_params(), 'gpu': gpu}
# Note: the training and test set here still include the sensitive attributes because
# some fairness aware algorithms may need those in the dataset. They should be removed
# before any model training is done.
predicted, predictions_list = algorithm.run(train, test,
dataset.get_class_attribute(),
positive_val,
all_sensitive_attributes,
single_sensitive,
privileged_vals, params)
# make dictionary mapping sensitive names to sensitive attr test data lists
dict_sensitive_lists = {}
for sens in all_sensitive_attributes:
dict_sensitive_lists[sens] = test[sens].values.tolist()
sensitive_dict = processed_data.get_sensitive_values(tag)
one_run_results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predicted, dict_sensitive_lists,
single_sensitive, privileged_vals, positive_val)
one_run_results.append(result)
# handling the set of predictions returned by ParamGridSearch
results_lol = []
if predictions_list:
for param_name, param_val, predictions in predictions_list:
params_dict = {param_name: param_val}
results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predictions, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val)
results.append(result)
results_lol.append((params_dict, results))
return params, one_run_results, results_lol
def run_eval_alg(algorithm, train, test, dataset, processed_data,
all_sensitive_attributes, single_sensitive, tag):
"""
Runs the algorithm and gets the resulting metric evaluations.
"""
privileged_vals = dataset.get_privileged_class_names_with_joint(tag)
positive_val = dataset.get_positive_class_val(tag)
# get the actual classifications and sensitive attributes
actual = test[dataset.get_class_attribute()].values.tolist()
sensitive = test[single_sensitive].values.tolist()
predicted, params, predictions_list = \
run_alg(algorithm, train, test, dataset, all_sensitive_attributes, single_sensitive,
privileged_vals, positive_val)
# make dictionary mapping sensitive names to sensitive attr test data lists
dict_sensitive_lists = {}
for sens in all_sensitive_attributes:
dict_sensitive_lists[sens] = test[sens].values.tolist()
sensitive_dict = processed_data.get_sensitive_values(tag)
one_run_results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predicted, dict_sensitive_lists,
single_sensitive, privileged_vals, positive_val)
one_run_results.append(result)
# handling the set of predictions returned by ParamGridSearch
results_lol = []
if len(predictions_list) > 0:
for param_name, param_val, predictions in predictions_list:
params_dict = {param_name: param_val}
results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predictions, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val)
results.append(result)
results_lol.append((params_dict, results))
return params, one_run_results, results_lol
def get_metrics_list(dataset, sensitive_dict, tag):
return [
metric.get_name()
for metric in get_metrics(dataset, sensitive_dict, tag)
]