def run_eval_alg(algorithm, train, test, dataset, processed_data,
all_sensitive_attributes, single_sensitive, tag):
"""
Runs the algorithm and gets the resulting metric evaluations.
"""
privileged_vals = dataset.get_privileged_class_names_with_joint(tag)
positive_val = dataset.get_positive_class_val(tag)
# get the actual classifications and sensitive attributes
actual = test[dataset.get_class_attribute()].values.tolist()
sensitive = test[single_sensitive].values.tolist()
predicted, params, predictions_list = \
run_alg(algorithm, train, test, dataset, all_sensitive_attributes, single_sensitive,
privileged_vals, positive_val)
# make dictionary mapping sensitive names to sensitive attr test data lists
dict_sensitive_lists = {}
for sens in all_sensitive_attributes:
dict_sensitive_lists[sens] = test[sens].values.tolist()
sensitive_dict = processed_data.get_sensitive_values(tag)
one_run_results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predicted, dict_sensitive_lists,
single_sensitive, privileged_vals, positive_val)
one_run_results.append(result)
# handling the set of predictions returned by ParamGridSearch
results_lol = []
if len(predictions_list) > 0:
for param_name, param_val, predictions in predictions_list:
params_dict = {param_name: param_val}
results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predictions, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val)
results.append(result)
results_lol.append((params_dict, results))
return params, one_run_results, results_lol
def get_metrics_list(dataset, sensitive_dict, tag):
return [metric.get_name() for metric in get_metrics(dataset, sensitive_dict, tag)]
def run_eval_alg(algorithm, train, test, dataset, processed_data,
all_sensitive_attributes, single_sensitive, tag,
num_bootstrap):
"""
Runs the algorithm and gets the resulting metric evaluations.
"""
privileged_vals = dataset.get_privileged_class_names_with_joint(tag)
positive_val = dataset.get_positive_class_val(tag)
# JPL (08/2020): update this code so that we can bootstrap each train/test split
# run_alg takes data frames for train and test
# so to do a manual bootstrap I'll select indices for each (with replacement)
# let's do 5 for now to make sure it works
NUM_BOOTSTRAP = num_bootstrap
trainLength = len(train)
testLength = len(test)
all_results = []
all_params_lol = []
for i in range(NUM_BOOTSTRAP):
train_idx = resample(list(range(trainLength)),
n_samples=trainLength,
replace=True)
trainDF = train.iloc[train_idx, :]
test_idx = resample(list(range(testLength)),
n_samples=testLength,
replace=True)
testDF = test.iloc[test_idx, :]
# get the actual classifications and sensitive attributes
actual = testDF[dataset.get_class_attribute()].values.tolist()
sensitive = testDF[single_sensitive].values.tolist()
predicted, params, predictions_list = \
run_alg(algorithm, trainDF, testDF, dataset, all_sensitive_attributes, single_sensitive,
privileged_vals, positive_val)
# make dictionary mapping sensitive names to sensitive attr test data lists
dict_sensitive_lists = {}
for sens in all_sensitive_attributes:
dict_sensitive_lists[sens] = testDF[sens].values.tolist()
sensitive_dict = processed_data.get_sensitive_values(tag)
one_run_results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predicted, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val)
one_run_results.append(result)
all_results.append(one_run_results)
# handling the set of predictions returned by ParamGridSearch
results_lol = []
if len(predictions_list) > 0:
for param_name, param_val, predictions in predictions_list:
params_dict = {param_name: param_val}
results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predictions,
dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val)
results.append(result)
results_lol.append((params_dict, results))
all_params_lol.append(results_lol)
#return params, one_run_results, results_lol
return params, all_results, all_params_lol
def run_eval_alg(algorithm, train, test, dataset, processed_data,
all_sensitive_attributes, single_sensitive, tag,
test_indices):
"""
Runs the algorithm and gets the resulting metric evaluations.
"""
features = test.loc[:, test.columns != dataset.get_class_attribute()]
privileged_vals = dataset.get_privileged_class_names_with_joint(tag)
positive_val = dataset.get_positive_class_val(tag)
# get the actual classifications and sensitive attributes
actual = test[dataset.get_class_attribute()].values.tolist()
sensitive = test[single_sensitive].values.tolist()
predicted, params, predictions_list, prediction_probs = \
run_alg(algorithm, train, test, dataset, all_sensitive_attributes, single_sensitive,
privileged_vals, positive_val)
# make dictionary mapping sensitive names to sensitive attr test data lists
dict_sensitive_lists = {}
for sens in all_sensitive_attributes:
dict_sensitive_lists[sens] = test[sens].values.tolist()
sensitive_dict = processed_data.get_sensitive_values(tag)
one_run_results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
# Individual fairness metrics need other definitions
if 'indiv_fairness' in metric.name:
result = metric.calc(actual, predicted, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val, features)
elif 'lipschitz' in metric.name:
if prediction_probs != []:
result = metric.calc(actual, predicted, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val, test_indices,
prediction_probs, dataset.dataset_name)
else:
result = None
else:
result = metric.calc(actual, predicted, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val)
one_run_results.append(result)
# handling the set of predictions returned by ParamGridSearch
results_lol = []
if len(predictions_list) > 0:
for param_name, param_val, predictions in predictions_list:
params_dict = {param_name: param_val}
results = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, predictions, dict_sensitive_lists,
single_sensitive, privileged_vals,
positive_val, features)
results.append(result)
results_lol.append((params_dict, results))
return params, one_run_results, results_lol