def load_and_filter_processed_data(info):
"""
:param info:
:return:
"""
assert isinstance(info, dict)
file_name = '%s_processed.pickle' % info['data_name']
file_name = data_dir / file_name
data, cvindices = load_processed_data(file_name = file_name.with_suffix('.pickle'))
# file_name
if 'fold_id' in info:
if 'fold_num' in info:
data = filter_data_to_fold(data, cvindices = cvindices, fold_id = info['fold_id'], fold_num = info['fold_num'])
else:
data = filter_data_to_fold(data, cvindices = cvindices, fold_id = info['fold_id'], fold_num = 0)
print_log('loaded dataset %s' % file_name)
return data
def test_solution_pool():
s_pool = SolutionPool()
data_name = 'breastcancer'
data_file_name = '%s/%s_processed.csv' % (data_dir, data_name)
data_file_name = Path(data_file_name)
random_seed = 1337
np.random.seed(seed=random_seed)
## load data from disk
data, cvindices = load_processed_data(
file_name=data_file_name.with_suffix('.pickle'))
data = filter_data_to_fold(data,
cvindices=cvindices,
fold_id='K05N01',
fold_num=1)
# about the folds
# fold_id = K[total # of folds]N[replicate number]
# 'K05N01' has 5 total folds, N01 means this is the first replicate
# fold_num = 1 means that we use fold # 1/5 as the test set
# fold_num = 2 means that we use fold # 2/5 as the test set
# fold_num = 0 means that we don't use a fold as the test set (i.e., so filtering with fold_num = 0 just returns the full training dataset)
mip = ZeroOneLossMIP(data)
mip.print_flag = True
mip.set_parallel(True)
out = mip.solve(time_limit=10)
# add solutions manually
s_pool.add(solution=[1, 2, 3],
coefs=[1, 2],
objval=5,
lowerbound=4,
prediction_constraint=('x', 'y'))
s_pool.add(solution=[[0, 0, 0], [1, 1, 1]],
coefs=[[1, 0], [1, 1]],
objval=[0, 1],
lowerbound=[2, 3])
preds = s_pool.get_preds([-2, 3])
small_pool = s_pool.get_solutions_with_pred([-2, 3], 1)
assert all(preds == [1, -1, 1])
assert small_pool._df.shape[0] == 2
s_pool.clear()
s_pool.add_from_mip(mip)
print(s_pool)
s_pool.clear()
s_pool.add_from_mip(mip, add_full_solution_pool=True)
print(s_pool)
def get_compas_subgroups(outcome="arrest"):
compas_file = data_dir / ("compas_%s_processed.pickle" % outcome)
with open(compas_file, "rb") as f:
data = dill.load(f)
data = filter_data_to_fold(data['data'], data['cvindices'], info['fold_id'], fold_num=1)
race_is_caucasian = data['X'][:, data['variable_names'].index('race_is_causasian')]
race_is_african_american = data['X'][:, data['variable_names'].index('race_is_african_american')]
race_is_hispanic = data['X'][:, data['variable_names'].index('race_is_hispanic')]
race_is_other = data['X'][:, data['variable_names'].index('race_is_other')]
subgroups = [
'caucasian' if race_is_caucasian[i] else
'african_american' if race_is_african_american[i] else
'hispanic' if race_is_hispanic[i] else
'other' for i in range(len(data['X']))
]
return np.array(subgroups)
def get_summary_row(info):
# set up directories
working_dir = output_dir / info['data_name']
processed_file = working_dir / get_processed_file_name(info)
discrepancy_file = working_dir / get_discrepancy_file_name(info)
data_file = data_dir / (info['data_name'] + "_processed.pickle")
# get the baseline and flipped models
with open(processed_file, "rb") as f:
processed = dill.load(f)['results_df']
# get the discrepancy models
with open(discrepancy_file, "rb") as f:
discrepancy = dill.load(f)['results_df']
# load data
with open(data_file, "rb") as f:
data = dill.load(f)
data = filter_data_to_fold(data['data'],
data['cvindices'],
fold_id=info['fold_id'],
fold_num=1,
include_validation=True)
X, Y = data['X'], data['Y']
X_test, Y_test = data['X_validation'], data['Y_validation']
# make a classifier for each model in the df
processed['clf'] = processed['coefficients'].apply(
get_classifier_from_coefficients)
discrepancy['clf'] = discrepancy['coefficients'].apply(
get_classifier_from_coefficients)
# get the baseline classifier
baseline = processed.query("model_type == 'baseline'").iloc[0]
baseline_clf = baseline['clf']
baseline_metrics = get_metrics(baseline_clf, X, Y, X_test, Y_test)
baseline_metrics.update({"Dataset": info['data_name']})
return baseline_metrics
def get_overview_table_row(info):
"""
creates a row of the overview data frame
:param info: dictionary containing data_name, fold_id, fold_num
:return: dictionary containing all fields for the overview data frame
"""
## setup file names
output_dir = results_dir / info['data_name']
# file names
file_names = {
'data': '%s/%s_processed.pickle' % (data_dir, info['data_name']),
'baseline': output_dir / get_baseline_file_name(info),
'discrepancy': output_dir / get_discrepancy_file_name(info),
'flipped': output_dir / get_processed_file_name(info),
}
# load data
data, cvindices = load_processed_data(file_name=file_names['data'])
data = filter_data_to_fold(data,
cvindices=cvindices,
fold_id=info['fold_id'],
fold_num=info['fold_num'],
include_validation=True)
XY = np.hstack([data['X'], data['Y'][:, None]])
XY_test = np.hstack([data['X_validation'], data['Y_validation'][:, None]])
U = np.unique(XY, axis=0)
U_test = np.unique(XY_test, axis=0)
observed_idx = get_common_row_indices(U, U_test)
# data-related fields
row = {
'data_name': info['data_name'],
'fold_id': info['fold_id'],
'fold_num': info['fold_num'],
#
'n': data['X'].shape[0],
'd': data['X'].shape[1] - 1,
'n_pos': np.sum(data['Y'] == 1),
'n_neg': np.sum(data['Y'] == -1),
'n_xy_unique': U.shape[0],
'n_xy_unobs_test': U_test.shape[0] - len(observed_idx),
#
'n_test': data['X_validation'].shape[0],
'n_test_pos': np.sum(data['Y_validation'] == 1),
'n_test_neg': np.sum(data['Y_validation'] == -1),
#
'has_baseline_results': file_names['baseline'].exists(),
'has_flipped_results': file_names['flipped'].exists(),
'has_discrepancy_results': file_names['discrepancy'].exists(),
}
# fields from baseline results
baseline_fields = [
'baseline_train_error', 'baseline_test_error', 'baseline_ub',
'baseline_lb', 'baseline_gap', 'baseline_n_equivalent',
'baseline_time_limit'
]
row.update({k: float('nan') for k in baseline_fields})
if file_names['baseline'].exists():
with open(file_names['baseline'], 'rb') as infile:
baseline_results = dill.load(infile)
baseline_coefs = baseline_results['pool_df'].query(
'model_type=="baseline"')['coefficients'].values[0]
out = baseline_results['baseline_output']
err_test = np.sign(data['X_validation'].dot(baseline_coefs))
row.update({
'baseline_train_error':
out['upperbound'],
'baseline_test_error':
np.not_equal(err_test, data['Y_validation']).sum(),
'baseline_n_equivalent':
len(baseline_results['equivalent_output']),
'baseline_time_limit':
baseline_results['info']['time_limit'],
'baseline_ub':
out['upperbound'],
'baseline_lb':
out['lowerbound'],
'baseline_gap':
out['gap'],
})
# initialize discrepancy as 'nan'
disc_fields = [
'disc_instances', 'disc_nnz_instances', 'disc_instances_gap_eq_0',
'disc_instances_gap_leq_0.1', 'disc_instances_gap_leq_0.5',
'disc_instances_gap_gt_0.5', 'disc_min_epsilon',
'disc_discrepancy_ratio_min', 'disc_discrepancy_ratio_med',
'disc_discrepancy_ratio_max', 'disc_discrepancy_min',
'disc_discrepancy_med', 'disc_discrepancy_max'
]
row.update({k: float('nan') for k in disc_fields})
if file_names['discrepancy'].exists():
with open(file_names['discrepancy'], 'rb') as infile:
discrepancy_results = dill.load(infile)
n_instances = len(discrepancy_results['epsilon_values'])
disc_df = discrepancy_results['results_df']
nnz_df = disc_df.query('total_discrepancy > 0')
n_nnz = len(nnz_df)
if len(nnz_df) > 0:
ratio = nnz_df['total_discrepancy'] / nnz_df['epsilon']
row.update({
'disc_min_epsilon': nnz_df['epsilon'].idxmin(),
'disc_discrepancy_ratio_min': np.nanmin(ratio),
'disc_discrepancy_ratio_med': np.nanmedian(ratio),
'disc_discrepancy_ratio_max': np.nanmax(ratio),
})
# stats
row.update({
'disc_instances':
n_instances,
'disc_nnz_instances':
n_nnz,
#
'disc_instances_gap_eq_0':
len(disc_df.query('gap == 0.0')),
'disc_instances_gap_leq_0.1':
len(disc_df.query('gap <= 0.1')),
'disc_instances_gap_leq_0.5':
len(disc_df.query('gap <= 0.5')),
'disc_instances_gap_gt_0.5':
len(disc_df.query('gap > 0.5')),
#
'disc_discrepancy_min':
np.nanmin(disc_df['total_discrepancy']),
'disc_discrepancy_med':
np.nanmedian(disc_df['total_discrepancy']),
'disc_discrepancy_max':
np.nanmax(disc_df['total_discrepancy']),
})
#initialize flipped fields
flipped_fields = [
'flipped_instances', 'flipped_instances_gap_eq_0',
'flipped_instances_gap_leq_0.1', 'flipped_instances_gap_leq_0.5',
'flipped_instances_gap_gt_0.5', 'flipped_change_in_error_min',
'flipped_change_in_error_med', 'flipped_change_in_error_max',
'flipped_change_in_test_error_min', 'flipped_change_in_test_error_med',
'flipped_change_in_test_error_max'
]
row.update({k: float('nan') for k in flipped_fields})
if file_names['flipped'].exists():
with open(file_names['flipped'], 'rb') as infile:
flipped_results = dill.load(infile)
flip_df = flipped_results['results_df']
baseline = flip_df.query('model_type == "baseline"')
flipped = flip_df.query('model_type == "flipped"')
change_in_train_error = flipped['train_error'] - baseline[
'train_error'].values[0]
change_in_test_error = flipped['validation_error'] - baseline[
'validation_error'].values[0]
row.update({
#
'flipped_instances':
len(flipped),
'flipped_instances_missing':
flipped_results['n_missing'],
'flipped_instances_gap_eq_0':
len(flipped.query('gap == 0.0')),
'flipped_instances_gap_leq_0.1':
len(flipped.query('gap <= 0.1')),
'flipped_instances_gap_leq_0.5':
len(flipped.query('gap <= 0.5')),
'flipped_instances_gap_gt_0.5':
len(flipped.query('gap > 0.5')),
#
'flipped_change_in_error_min':
np.nanmin(change_in_train_error),
'flipped_change_in_error_med':
np.nanmedian(change_in_train_error),
'flipped_change_in_error_max':
np.nanmax(change_in_train_error),
#
'flipped_change_in_test_error_min':
np.nanmin(change_in_test_error),
'flipped_change_in_test_error_med':
np.nanmedian(change_in_test_error),
'flipped_change_in_test_error_max':
np.nanmax(change_in_test_error),
})
return row
time_limit_instance = 60
initialize_mip = True
custom_mip_params = True
search_global_equivalent = False
max_iterations = 1e8 #shorten loop during development
fold_id = 'K01N01'
fold_num = 1
# setup seed
np.random.seed(seed = random_seed)
## load data from disk
data_file_name = '%s/%s_processed.csv' % (data_dir, data_name)
data_file_name = Path(data_file_name)
data, cvindices = load_processed_data(file_name = data_file_name.with_suffix('.pickle'))
data = filter_data_to_fold(data, cvindices = cvindices, fold_id = fold_id, fold_num = fold_num, include_validation=True)
# compress dataset into distinct feature vectors and counts
compressed = compress_data(data)
U, N_pos, N_neg, x_to_u_idx, u_to_x_idx = tuple(compressed[var] for var in ('U', 'N_pos', 'N_neg', 'x_to_u_idx', 'u_to_x_idx'))
selection = pd.DataFrame({'n_pos': N_pos, 'n_neg': N_neg, 'idx': x_to_u_idx})
# solve zero-one loss MIP
mip = ZeroOneLossMIP(data, print_flag = True, parallel_flag = True, random_seed = random_seed)
if custom_mip_params:
mip.mip = set_mip_parameters(mip.mip, CPX_MIP_PARAMETERS)
out = mip.solve(time_limit = time_limit_global)
mip.check_solution()
global_coefs = mip.coefficients
global_lb = out['lowerbound']
def aggregate_baseline_and_flipped_results(info):
"""
:param data_name:
:param fold_id:
:param fold_num:
:return:
"""
assert isinstance(info, dict)
assert 'data_name' in info and 'fold_id' in info and 'fold_num' in info
# setup file names
output_dir = results_dir / info['data_name']
baseline_results_file = output_dir / get_baseline_file_name(info)
data_file_name = Path('%s/%s_processed.pickle' % (data_dir, info['data_name']))
# load baseline results
assert baseline_results_file.exists()
with open(baseline_results_file, 'rb') as f:
baseline = dill.load(f)
# load data from disk
assert baseline_results_file.exists()
data, cvindices = load_processed_data(file_name = data_file_name)
data = filter_data_to_fold(data, cvindices = cvindices, fold_id = info['fold_id'], fold_num = info['fold_num'], include_validation = True)
# load flipped results
flipped_raw_results_files = [f for f in output_dir.iterdir() if f.suffix == '.pickle' and 'flipped_raw_results' in f.name]
all_partitions = get_part_id_helper(flipped_raw_results_files)
flipped_partition = select_partition_for_flipped_training(all_partitions)
flipped_results_files = flipped_partition['matched_files']
print_log('partition contains %d raw results files.' % len(flipped_results_files))
# load files
flipped = []
for fname in flipped_results_files:
with open(fname, 'rb') as f:
flipped.append(dill.load(f))
# concatenate data frames
if len(flipped):
flipped_df = pd.concat([info['flipped_df'] for info in flipped], sort = False)
flipped_df['model_type'] = 'flipped'
results_df = pd.concat([baseline['pool_df'], flipped_df], sort=False).reset_index(drop=True)
else:
print("Warning: no flipped model files found during results aggregation.")
results_df = baseline['pool_df']
# combine all results in dataframe
results_df['train_error'] = float('nan')
results_df['validation_error'] = float('nan')
# compute error metrics
W = np.stack(results_df['coefficients'].values)
results_df['train_error'] = compute_error_rate_from_coefficients(W = W, X = data['X'], Y = data['Y'])
if 'X_validation' in data and 'Y_validation' in data:
results_df['validation_error'] = compute_error_rate_from_coefficients(W = W, X = data['X_validation'], Y = data['Y_validation'])
# info to return
now = datetime.now()
out = {
#
'date': now.strftime("%y_%m_%d_%H_%M"),
'info': info,
'training_info': {'baseline': baseline['info'], 'flipped_info': [f['info'] for f in flipped]},
#
'data_file': baseline['data_file'],
'baseline_results_file': baseline_results_file,
'flipped_results_files': flipped_results_files,
'results_df': results_df,
'n_missing': len(flipped_partition['missing_parts']),
#
}
return out
def train_discrepancy_classifier(info):
"""
:param info:
:return:
"""
print_log('entered train_discrepancy_classifier')
# dashboard
for k, v in DISCREPANCY_TRAINING_SETTINGS.items():
if k not in info:
info[k] = v
for k, v in info.items():
print("info['%s'] = %r" % (k, info[k]))
# setup files
output_dir = results_dir / info['data_name']
output_dir.mkdir(exist_ok = True)
results_file_name = output_dir / get_discrepancy_file_name(info)
baseline_file_name = output_dir / get_baseline_file_name(info)
processed_file_name = output_dir / get_processed_file_name(info)
# print file names
print_log('baseline_file_name: %s' % baseline_file_name)
print_log('results_file_name: %s' % results_file_name)
# load baseline file
assert baseline_file_name.exists()
with open(baseline_file_name, 'rb') as infile:
baseline_results = dill.load(infile)
print_log('loaded baseline file %s' % baseline_file_name)
# setup seed
np.random.seed(seed = info['random_seed'])
# load data from disk
data_file_name = '%s_processed.csv' % info['data_name']
data_file_name = data_dir / data_file_name
data, cvindices = load_processed_data(file_name = data_file_name.with_suffix('.pickle'))
data = filter_data_to_fold(data, cvindices = cvindices, fold_id = info['fold_id'], fold_num = info['fold_num'])
print_log('loaded dataset %s' % data_file_name)
# load baseline
initial_pool = baseline_results['pool_df']
baseline_stats = baseline_results['baseline_output']
baseline_coefs = initial_pool.query('model_type == "baseline"')['coefficients'].values[0]
n_samples = data['X'].shape[0]
baseline_ub = baseline_stats['upperbound']
baseline_lb = baseline_stats['lowerbound']
loss_values = np.arange(baseline_ub, n_samples // 2, step=DISCREPANCY_TRAINING_SETTINGS['epsilon_step'])
epsilon_values = sorted(np.array(loss_values - baseline_ub, dtype = int).tolist())
print_log('%1.0f values of epsilon in {%1.0f,...,%1.0f}' % (len(epsilon_values), min(epsilon_values), max(epsilon_values)))
# load existing results file if it exists
if info['load_from_disk'] and results_file_name.exists():
# load from disk
with open(results_file_name, 'rb') as infile:
results = dill.load(infile)
for k in ['data_name', 'fold_id', 'fold_num']:
assert info[k] == results[k], 'mismatch in loaded results'
assert np.isclose(baseline_coefs, results['baseline_coefs']).all()
assert np.isclose(epsilon_values, results['epsilon_values']).all()
print_log('loaded existing results from disk %s' % results_file_name)
else:
results = {
'data_name': info['data_name'],
'fold_id': info['fold_id'],
'fold_num': info['fold_num'],
'baseline_ub': baseline_ub,
'baseline_lb': baseline_lb,
'baseline_coefs': baseline_coefs,
'epsilon_values': epsilon_values,
}
results['output'] = {e: None for e in results['epsilon_values']}
print_log('did not find file with existing results on disk: %s' % results_file_name)
initial_pool = build_coefficient_pool(info) # overwrites initial pool above with superset (also different format)
initial_coefs = np.vstack(initial_pool['coefficients'])
initial_errors = initial_pool['error_ub']
# build discrepancy MIP
mip = DiscrepancyMIP(data, baseline_coefs, baseline_stats, print_flag = info['print_flag'], parallel_flag = True, random_seed = info['random_seed'])
mip.mip = set_mip_parameters(mip.mip, CPX_MIP_PARAMETERS)
mip.bound_error_gap(lb = int(np.floor(baseline_lb - baseline_ub)))
# update date
now = datetime.now()
results['results_file'] = results_file_name
# compute baseline classifier gap
g = mip.get_classifier(coefs = mip.baseline_coefs)
G = g.predict(data['X'])
err_g = np.not_equal(data['Y'], G).sum()
# run training
epsilon_values = results['epsilon_values']
total_iterations = len(epsilon_values)
training_epsilon_values = sorted([e for (e, out) in results['output'].items() if out is None])
remaining_time = info['time_limit']
for e in training_epsilon_values:
i = epsilon_values.index(e)
# record start time
start_time = time.time()
print_log('=' * 70)
print_log('started training for epsilon = %1.0f, iteration %d/%d' % (e, i, total_iterations))
print("\n")
# setup MIP
mip.bound_error_gap(ub = e)
# add initial solutions that are valid
if info['initialize']:
keep_idx = np.less_equal(initial_errors, e + err_g)
feasible_coefs = initial_coefs[keep_idx, :]
print_log('initializing with %d solutions' % feasible_coefs.shape[0])
for w in feasible_coefs:
mip.add_initial_solution_with_coefficients(coefs = w)
train_time = min(remaining_time, info['instance_time_limit'])
out = mip.solve(time_limit = train_time)
# compute current classifier stats
h = mip.get_classifier()
H = h.predict(data['X'])
err_h = np.not_equal(H, data['Y']).sum()
# record stats
out['epsilon'] = e
out['coefficients'] = mip.coefficients
out['total_error_gap'] = mip.solution.get_values(mip.names['total_error_gap'])
out['total_discrepancy'] = np.not_equal(H, G).sum()
out['total_agreement'] = mip.solution.get_values(mip.names['total_agreement'])
# check solution
print_log(('=' * 20) + ' WARNINGS ' + ('=' * 20))
mip.check_solution()
# store solution
results['output'][e] = out
# print details about solution
msg = [('=' * 20) + ' SUMMARY ' + ('=' * 20),
'-' * 70,
'mistakes',
'R(h): %1.0f' % err_h,
'R(g): %1.0f' % err_g,
'R(h)-R(g): %1.0f' % (err_h - err_g),
'-' * 70,
'alignment',
'#[h(x)==g(x)]: %1.0f' % np.equal(H, G).sum(),
'#[h(x)!=g(x)]: %1.0f' % np.not_equal(H, G).sum(),
'-' * 70,
'mip output',
'max error gap (=epsilon): %1.0f' % e,
'total error gap (=epsilon): %1.0f' % out['total_error_gap'],
'total discrepancy (objval): %1.0f' % out['total_discrepancy'],
'total agreement (objval): %1.0f' % out['total_agreement'],
'objval ub: %1.0f' % out['upperbound'],
'objval lb: %1.0f' % out['lowerbound'],
'objval gap: %1.2f%%' % (100.0 * out['gap']),
'=' * 70,
]
msg = '\n'.join(msg)
print_log(msg)
# print completion message
print_log('completed training for epsilon = %1.0f, iteration %d/%d' % (e, i, total_iterations))
# save file
results['date'] = now.strftime("%y_%m_%d_%H_%M")
results_df = [out for out in results['output'].values() if out is not None]
results_df = pd.DataFrame(results_df)
results_df.set_index('epsilon')
results['results_df'] = results_df
save_results_to_disk(results, file_name = results_file_name)
# check whether to stop
time_elapsed = time.time() - start_time
remaining_time = remaining_time - time_elapsed
if remaining_time < 30.0 + info['instance_time_limit']:
print_log('STOPPING TRAINING: out of time')
print(results_df[['epsilon', 'gap', 'total_discrepancy', 'total_agreement']])
break
# print final results
print_log('leaving train_discrepancy_classifier')
return results
def train_flipped_classifiers(info):
"""
:param info:
:return:
"""
print_log('entered train_flipped_classifiers')
print_log('-' * 50)
# dashboard
for k, v in FLIPPED_TRAINING_SETTINGS.items():
if k not in info:
info[k] = v
for k, v in info.items():
print("info['%s'] = %r" % (k, info[k]))
assert "part_id" in info
output_dir = results_dir / info['data_name']
output_dir.mkdir(exist_ok = True)
baseline_file_name = output_dir / get_baseline_file_name(info)
results_file_name = output_dir / get_flipped_file_name(info)
print_log('baseline_file_name: %s' % baseline_file_name)
print_log('results_file_name: %s' % results_file_name)
# load baseline file
assert baseline_file_name.exists()
print_log('loading results from %s' % baseline_file_name)
with open(baseline_file_name, 'rb') as infile:
baseline_results = dill.load(infile)
print_log('loaded results from %s' % baseline_file_name)
# setup pool
pool = SolutionPool(df = baseline_results['pool_df'])
# setup baseline classifier
coefs = baseline_results['pool_df'].query('model_type == "baseline"')['coefficients'].values[0]
h = ClassificationModel(predict_handle = lambda X: 1,
model_info = {'coefficients': coefs[1:], 'intercept': coefs[0]},
model_type = ClassificationModel.LINEAR_MODEL_TYPE)
# load data from disk
data_file_name = '%s_processed.csv' % info['data_name']
data_file_name = data_dir / data_file_name
data, cvindices = load_processed_data(file_name = data_file_name.with_suffix('.pickle'))
data = filter_data_to_fold(data, cvindices = cvindices, fold_id = info['fold_id'], fold_num = info['fold_num'])
print_log('loaded dataset %s' % data_file_name)
# compress dataset into distinct feature vectors and counts
compressed = compress_data(data)
for k, v in compressed.items():
compressed[k] = filter_indices_to_part(v, part_id = info['part_id'])
U, N_pos, N_neg, x_to_u_idx, u_to_x_idx = tuple(compressed[var] for var in ('U', 'N_pos', 'N_neg', 'x_to_u_idx', 'u_to_x_idx'))
# setup MIP
mip = ZeroOneLossMIP(data, print_flag = True, parallel_flag = True, random_seed = info['random_seed'])
mip.mip = set_mip_parameters(mip.mip, CPX_MIP_PARAMETERS)
mip.mip.parameters.emphasis.mip.set(3)
mip.set_total_mistakes(lb = baseline_results['baseline_output']['lowerbound'])
# solve flipped versions
results = []
total_iterations = U.shape[0]
start_time = time.process_time()
#### START INITIALIZATION
# pre-initialize the mip from disk
if info['load_from_disk']:
# load saved coefficients from all methods
initial_pool = build_coefficient_pool(info)
if initial_pool.shape[0] > 0:
print_log('initializing with %d solutions' % len(initial_pool))
# use the best lowerbound we have
max_error_lb = np.ceil(initial_pool['error_lb'].max())
if np.isfinite(max_error_lb):
mip.set_total_mistakes(lb=max_error_lb)
# supply all possible initializations
for k, row in initial_pool.iterrows():
mip.add_initial_solution_with_coefficients(coefs=row['coefficients'])
#### END INITIALIZATION
for k, x in enumerate(U):
print_log('iteration %d/%d' % (k, total_iterations))
print_log('solution pool size: %d' % pool.size)
yhat = int(h.predict(x[None, :]))
# adjust prediction constraints
mip.clear_prediction_constraints()
mip.add_prediction_constraint(x = x, yhat = -yhat, name = 'pred_constraint')
# initialize model
good_pool = pool.get_solutions_with_pred(x, -yhat)
if good_pool.size > 0:
s = good_pool.get_best_solution()
mip.add_initial_solution(solution = s['solution'], objval = s['objval'], name = 'init_from_pred_cons')
mip.add_initial_solution_with_coefficients(coefs = s['coefficients'])
print_log('initialized\nobjval:{}'.format(s['objval']))
# solve MIP
out = mip.solve(time_limit = info['time_limit_flipped'])
mip.check_solution()
# update solution pool
pool.add_from_mip(mip, prediction_constraint = (x, yhat))
# update out
out.update(
{
'k': k,
'i': np.flatnonzero(u_to_x_idx == k).tolist(),
'x': x,
'n_pos': N_pos[k],
'n_neg': N_neg[k],
'coefficients': mip.coefficients,
'elapsed_time': time.process_time() - start_time,
'prediction_constraint': (x, yhat),
'solution': list(out['values']),
}
)
results.append(out)
results_df = pd.DataFrame(results)
results_df.drop(['upperbound', 'values'], axis=1, inplace=True)
# create output dictionary
now = datetime.now()
results = {
'date': now.strftime("%y_%m_%d_%H_%M"),
'info': info,
'part_id': info['part_id'],
'data_file': data_file_name,
'baseline_file': baseline_file_name,
'results_file': results_file_name,
'flipped_df': results_df,
}
print_log('leaving train_flipped_classifier')
return results
def train_baseline_classifier(info):
"""
trains baseline classifier via zero one loss minimization
:param info:
:return:
"""
print_log('entered train_baseline_classifier')
# dashboard
for k, v in BASELINE_TRAINING_SETTINGS.items():
if k not in info:
info[k] = v
print_log('settings')
print_log('-' * 50)
for k, v in info.items():
print_log("info['%s'] = %r" % (k, info[k]))
# name and create output directory
output_dir = results_dir / info['data_name']
output_dir.mkdir(exist_ok = True)
# set output results file
results_file_name = output_dir / get_baseline_file_name(info)
print_log('-' * 50)
print_log('saving results in %s' % results_file_name)
# load dataset
data_file_name = '%s_processed.pickle' % info['data_name']
data_file_name = data_dir / data_file_name
data, cvindices = load_processed_data(file_name = data_file_name.with_suffix('.pickle'))
data = filter_data_to_fold(data, cvindices = cvindices, fold_id = info['fold_id'], fold_num = info['fold_num'])
print_log('loaded dataset %s' % data_file_name)
# solve zero-one loss MIP
mip = ZeroOneLossMIP(data, print_flag = info['print_flag'], parallel_flag = True, random_seed = info['random_seed'], error_constraint_type = info['error_constraint_type'])
mip.mip = set_mip_parameters(mip.mip, CPX_MIP_PARAMETERS)
# load initializations from disk
if info['load_from_disk']:
# load saved coefficients from all methods
initial_pool = build_coefficient_pool(info)
if initial_pool.shape[0] > 0:
print_log('initializing with %d solutions' % len(initial_pool))
# use the best lowerbound we have
max_error_lb = np.ceil(initial_pool['error_lb'].max())
if np.isfinite(max_error_lb):
mip.set_total_mistakes(lb=max_error_lb)
# supply all possible initializations
for k, row in initial_pool.iterrows():
mip.add_initial_solution_with_coefficients(coefs=row['coefficients'])
# report expected objective value
print_log('upperbound should be at most %d' % min(initial_pool['error_ub']))
out = mip.solve(time_limit = info['time_limit'])
baseline_output = mip.solution_info
print_log(str(baseline_output))
mip.check_solution(debug_flag = True)
base_df = {
'solution': list(out['values']),
'objval': out['objval'],
'coefficients': mip.coefficients,
'lowerbound': out['lowerbound'],
'prediction_constraint': None,
'model_type': 'baseline'
}
# initialize solution pool
pool = SolutionPool(mip = mip)
# search for equivalent models
if info['equivalent_time_limit'] > 0:
equivalent_output, pool = mip.enumerate_equivalent_solutions(pool, time_limit = info['equivalent_time_limit'])
print_log('{} global equivalent models found.'.format(len(equivalent_output)))
else:
equivalent_output = []
# generate additional solutions using populate
if info['populate_time_limit'] > 0:
mip.populate(max_gap = 0, time_limit = info['populate_time_limit'])
mip.populate(max_gap = data['X'].shape[0] // 2, time_limit = info['populate_time_limit'])
pool.add_from_mip(mip)
# get alternative models
pool_df = pool.get_df()
pool_df['model_type'] = 'alternative'
pool_df = pool_df.append(base_df, sort = False, ignore_index = True).reset_index(drop = True)
# get time
now = datetime.now()
results = {
'date': now.strftime("%y_%m_%d_%H_%M"),
'info': info,
'data_file': data_file_name,
'results_file': results_file_name,
'pool_df': pool_df,
'baseline_output': baseline_output,
'equivalent_output': equivalent_output,
}
print_log('leaving train_baseline_classifier')
return results
processed_file = output_dir / get_processed_file_name(info)
discrepancy_file = output_dir / get_discrepancy_file_name(info)
data_file = data_dir / (altered_info['data_name'] + "_processed.pickle")
# get the models
with open(processed_file, "rb") as f:
processed = dill.load(f)
with open(discrepancy_file, "rb") as f:
discrepancy = dill.load(f)
# get the data
with open(data_file, "rb") as f:
data = dill.load(f)
data = filter_data_to_fold(data['data'],
data['cvindices'],
info['fold_id'],
fold_num=1,
include_validation=True)
# get the unbalanced data
unbalanced = load_data_from_csv(data_file.with_suffix(".csv"))
#
if "compas" in data_name and "_small" in data_name:
for feat in [
'race_is_causasian', 'race_is_african_american',
'race_is_hispanic', 'race_is_other'
]:
if feat in data['variable_names']:
data = remove_variable(data, feat)
if feat in unbalanced['variable_names']:
def multiplicity_table(info):
# set up directories
working_dir = output_dir / info['data_name']
processed_file = working_dir / get_processed_file_name(info)
discrepancy_file = working_dir / get_discrepancy_file_name(info)
# get the baseline and flipped models
with open(processed_file, "rb") as f:
processed = dill.load(f)['results_df']
# get the discrepancy models
with open(discrepancy_file, "rb") as f:
discrepancy = dill.load(f)['results_df']
# load data
with open(info['data_file'], "rb") as f:
data = dill.load(f)
data = filter_data_to_fold(data['data'], data['cvindices'], fold_id=info['fold_id'], fold_num=1, include_validation=True)
X, Y = data['X'], data['Y']
X_test, Y_test = data['X_validation'], data['Y_validation']
# make a classifier for each model in the df
processed['clf'] = processed['coefficients'].apply(get_classifier_from_coefficients)
discrepancy['clf'] = discrepancy['coefficients'].apply(get_classifier_from_coefficients)
# get the baseline classifier
baseline = processed.query("model_type == 'baseline'").iloc[0]
baseline_clf = baseline['clf']
baseline_scores = baseline_clf.score(X)
baseline_train_preds = baseline_clf.predict(X)
baseline_test_preds = baseline_clf.predict(X_test)
baseline_train_error = np.sum(baseline_train_preds != Y)
baseline_train_error_rate = np.mean(baseline_train_preds != Y)
baseline_test_error = np.sum(baseline_test_preds != Y_test)
baseline_test_error_rate = np.mean(baseline_test_preds != Y_test)
# get the equivalent classifier
epsilon = discrepancy.query("epsilon <= %s" % int(0.01 * X.shape[0]))['epsilon'].max()
equivalent = discrepancy.query("epsilon == %s" % epsilon).iloc[0]
equivalent_clf = equivalent['clf']
equivalent_train_preds = equivalent_clf.predict(X)
equivalent_test_preds = equivalent_clf.predict(X_test)
equivalent_train_error = np.sum(equivalent_train_preds != Y)
equivalent_train_error_rate = np.mean(equivalent_train_preds != Y)
equivalent_test_error = np.sum(equivalent_test_preds != Y_test)
equivalent_test_error_rate = np.mean(equivalent_test_preds != Y_test)
equivalent_train_discrepancy = np.sum(equivalent_train_preds != baseline_train_preds)
equivalent_test_discrepancy = np.sum(equivalent_test_preds != baseline_test_preds)
equivalent_train_discrepancy_rate = np.mean(equivalent_train_preds != baseline_train_preds)
equivalent_test_discrepancy_rate = np.mean(equivalent_test_preds != baseline_test_preds)
is_flipped = np.logical_and(baseline_train_preds == 1, equivalent_train_preds == -1)
is_flipped_nonzero = np.logical_and(is_flipped, abs(baseline_scores) > 0.001)
flipped_train_errors = []
for instance in np.unique(X[is_flipped_nonzero], axis=0):
# get the train error of the best flipped model for each instance
instance = np.expand_dims(instance, 0)
baseline_pred = baseline_clf.predict(instance)[0]
preds = processed.apply(df_predict_handle, axis=1, x=instance)
min_flipped_train_error = processed[preds != baseline_pred]['objval'].min()
flipped_model = processed[preds != baseline_pred].query('objval == %s' % min_flipped_train_error).iloc[0]
flipped_train_errors.append({'min_flipped_train_error': min_flipped_train_error,
'instance': instance,
'model': flipped_model})
# choose the instance to show
flipped_info = min(flipped_train_errors, key=lambda x: x["min_flipped_train_error"])
x_i = flipped_info['instance']
# get the flipped classifier
flipped = flipped_info['model']
flipped_clf = flipped['clf']
flipped_train_preds = flipped_clf.predict(X)
flipped_test_preds = flipped_clf.predict(X_test)
flipped_train_error = np.sum(flipped_train_preds != Y)
flipped_train_error_rate = np.mean(flipped_train_preds != Y)
flipped_test_error = np.sum(flipped_test_preds != Y_test)
flipped_test_error_rate = np.mean(flipped_test_preds != Y_test)
flipped_train_discrepancy = np.sum(flipped_train_preds != baseline_train_preds)
flipped_test_discrepancy = np.sum(flipped_test_preds != baseline_test_preds)
flipped_train_discrepancy_rate = np.mean(flipped_train_preds != baseline_train_preds)
flipped_test_discrepancy_rate = np.mean(flipped_test_preds != baseline_test_preds)
results_df = pd.DataFrame(columns=['model_type', 'coefficients',
'train_error_rate', 'test_error_rate',
'train_discrepancy_rate', 'test_discrepancy_rate',
'score_xi', 'prediction_xi'])
results_df = results_df.append({'model_type': 'baseline',
'coefficients': baseline['coefficients'],
'train_error_rate': baseline_train_error_rate,
'test_error_rate': baseline_test_error_rate,
'train_discrepancy_rate': 0,
'test_discrepancy_rate': 0,
'score_xi': baseline_clf.score(x_i)[0],
'prediction_xi': baseline_clf.predict(x_i)[0]
}, ignore_index=True)
results_df = results_df.append({'model_type': 'discrepancy',
'coefficients': equivalent['coefficients'],
'train_error_rate': equivalent_train_error_rate,
'test_error_rate': equivalent_test_error_rate,
'train_discrepancy_rate': equivalent_train_discrepancy_rate,
'test_discrepancy_rate': equivalent_test_discrepancy_rate,
'score_xi': equivalent_clf.score(x_i)[0],
'prediction_xi': equivalent_clf.predict(x_i)[0]
}, ignore_index=True)
results_df = results_df.append({'model_type': 'flipped',
'coefficients': flipped['coefficients'],
'train_error_rate': flipped_train_error_rate,
'test_error_rate': flipped_test_error_rate,
'train_discrepancy_rate': flipped_train_discrepancy_rate,
'test_discrepancy_rate': flipped_test_discrepancy_rate,
'score_xi': flipped_clf.score(x_i)[0],
'prediction_xi': flipped_clf.predict(x_i)[0]
}, ignore_index=True)
x_i_features = pd.Series(x_i[0], index=data['variable_names'])
results = {'results_df': results_df, 'x_i': x_i_features}
return results
def subgroup_ambiguity_analysis(info, subgroup_getter):
# set up directories
output_dir = paper_dir / info['data_name']
processed_file = output_dir / get_processed_file_name(info)
data_file = data_dir / (info['data_name'] + "_processed.pickle")
# get the models
with open(processed_file, "rb") as f:
processed = dill.load(f)
# get the data
with open(data_file, "rb") as f:
data = dill.load(f)
data = filter_data_to_fold(data['data'], data['cvindices'], info['fold_id'], fold_num=1)
# make a classifier for each model in the df
results = processed['results_df']
results['clf'] = results['coefficients'].apply(get_classifier_from_coefficients)
# get the baseline classifier
assert results.query("model_type == 'baseline'").shape[0] == 1
baseline = results.query("model_type == 'baseline'").iloc[0]
baseline_clf = baseline['clf']
baseline_train_error = baseline['train_error']
# compress data and get subgroups vector for analysis (e.g. race)
U, x_to_u_idxs, counts = np.unique(data['X'], axis=0, return_counts=True, return_index=True)
subgroups = subgroup_getter()
subgroups = subgroups[x_to_u_idxs]
# get the train error of the best flipped model for each instance
flipped_train_errors = []
for i, instance in enumerate(U):
instance = np.expand_dims(instance, 0)
baseline_pred = baseline_clf.predict(instance)[0]
preds = results.apply(df_predict_handle, axis=1, x=instance)
min_flipped_train_error = results[preds != baseline_pred]['train_error'].min()
flipped_train_errors.append(min_flipped_train_error)
# subgroup analysis dataframe
df = pd.DataFrame({
'subgroup': subgroups,
'flipped_error': flipped_train_errors,
'num_instances': counts,
})
# get number of instances per subgroup
group_counts = {grp: df.query("subgroup == '%s'" % grp)['num_instances'].sum() for grp in np.unique(df['subgroup'])}
# check the percent of epsilon-flippable instances per subgroup
epsilon = 0.01
subgroup_multiplicity = {}
for sgroup in np.unique(df['subgroup']):
grp = df.query("subgroup == '%s'" % sgroup)
flippable = grp['flipped_error'] <= baseline_train_error + epsilon
num_flippable = grp[flippable]['num_instances'].sum()
pct_flippable = np.round(num_flippable / group_counts[sgroup], 4)
subgroup_multiplicity[sgroup] = {'num_flippable': num_flippable,
'pct_flippable': pct_flippable,
'num_instances': group_counts[sgroup],
}
return subgroup_multiplicity
def subgroup_disparity_analysis(info, subgroup_getter, protected_subgroup, epsilon=0.01):
# set up directories
output_dir = paper_dir / info['data_name']
processed_file = output_dir / get_processed_file_name(info)
discrepancy_file = output_dir / get_discrepancy_file_name(info)
data_file = data_dir / (info['data_name'] + "_processed.pickle")
# get the models
with open(processed_file, "rb") as f:
processed = dill.load(f)
with open(discrepancy_file, "rb") as f:
discrepancy = dill.load(f)
# get the data
with open(data_file, "rb") as f:
data = dill.load(f)
data = filter_data_to_fold(data['data'], data['cvindices'], info['fold_id'], fold_num=1, include_validation=True)
X, Y = data['X'], data['Y']
X_test, Y_test = data['X_validation'], data['Y_validation']
subgroups = subgroup_getter()
# subgroups = [g if g == "african_american" else "other" for g in subgroups]
# make a classifier for each model in the df
proc_results = processed['results_df']
disc_results = discrepancy['results_df']
proc_results['clf'] = proc_results['coefficients'].apply(get_classifier_from_coefficients)
disc_results['clf'] = disc_results['coefficients'].apply(get_classifier_from_coefficients)
# get the baseline classifier
assert proc_results.query("model_type == 'baseline'").shape[0] == 1
baseline = proc_results.query("model_type == 'baseline'").iloc[0]
baseline_clf = baseline['clf']
baseline_train_error = baseline['train_error']
baseline_disparity = get_disparity(baseline_clf, X=X, subgroups=subgroups, protected_name=protected_subgroup)
proc_metrics = pd.DataFrame.from_records(proc_results['clf'].apply(get_metrics, X=X, Y=Y, X_test=X_test, Y_test=Y_test, metrics=["train_error", "test_error"]))
proc_results = pd.concat([proc_results, proc_metrics], axis=1)
disc_metrics = pd.DataFrame.from_records(disc_results['clf'].apply(get_metrics, X=X, Y=Y, X_test=X_test, Y_test=Y_test, metrics=["train_error", "test_error"]))
disc_results = pd.concat([disc_results, disc_metrics], axis=1)
proc_level_set = proc_results.query('train_error <= %s' % (baseline_train_error + epsilon))
disc_level_set = disc_results.query('train_error <= %s' % (baseline_train_error + epsilon))
level_set_clfs = proc_level_set['clf'].append(disc_level_set['clf'])
all_metrics = pd.concat([proc_metrics, disc_metrics], axis=0)
all_metrics_no_baseline = all_metrics.query("train_error != %s" % all_metrics['train_error'].min())
train_error_of_best_test = all_metrics_no_baseline['train_error'][all_metrics_no_baseline['test_error'].idxmin()]
empirical_epsilon = (train_error_of_best_test - baseline_train_error).to_list()[0]
disparities = level_set_clfs.apply(get_disparity, X=X, subgroups=subgroups, protected_name=protected_subgroup)
smallest_disparity = min(abs(disparities)) * np.sign(min(disparities))
results = {"baseline_disparity": baseline_disparity,
"best_disparity": smallest_disparity,
"difference": baseline_disparity - smallest_disparity,
"epsilon": epsilon,
"empirical_epsilon": empirical_epsilon,
"level_set_size": len(disparities),
"error_&_disparity_level_set_size": sum(abs(disparities - baseline_disparity) <= epsilon),
}
results = {k: num_to_signed(v) for k, v in results.items()}
# ipsh()
return results