indices = list(range(0, len(test_data)))
random.shuffle(indices)
num_samples_to_take = sample_size
indices_to_take = indices[:num_samples_to_take]
test_data_sampled = test_data.iloc[indices_to_take, :]
y_test_sampled = []
for index in indices_to_take:
y_test_sampled.append(y_test[index])
performance_predictor = train_random_forest_regressor(
test_data_sampled, y_test_sampled,
outlier_perturbations(numerical_columns, num_repetitions), model,
scoring)
evaluate_regressor(
target_data, y_target,
outlier_perturbations(numerical_columns, num_repetitions), model,
performance_predictor, scoring, scoring_name, dataset_name,
'missing', learner_name, experiment_name)
# performance_predictor = train_random_forest_regressor(test_data_sampled, y_test_sampled,
# missing_perturbations(categorical_columns,
# 'NULL',
# num_repetitions),
# model, scoring)
#
# evaluate_regressor(target_data, y_target, missing_perturbations(categorical_columns, 'NULL', num_repetitions),
# model, performance_predictor, scoring, scoring_name, dataset_name, 'missing', learner_name,
# experiment_name)
from pp import perturbations
from pp.meta_regressors import train_random_forest_regressor, evaluate_regressor
from pp.serialization import load_black_box
def noise_perturbations():
_perturbations = []
for fraction in [0.0, 0.05, 0.25, 0.5, 0.75, 0.99]:
for _ in range(0, 100):
_perturbations.append(perturbations.NoisyImage(fraction))
return _perturbations
models_to_evaluate = []
for learner in ['convnet']:
for dataset in ['mnist', 'fashion']:
for score in ['roc_auc']:
models_to_evaluate.append(learner + '-' + dataset + '-' + score)
for model_to_evaluate in models_to_evaluate:
experiment_name = 'correct_shift'
(model, scoring, scoring_name, train_data, y_train, test_data, y_test, target_data, y_target, learner_name,
dataset_name) = load_black_box(model_to_evaluate)
predictor = train_random_forest_regressor(test_data, y_test, noise_perturbations(), model, scoring)
evaluate_regressor(target_data, y_target, noise_perturbations(), model,
predictor, scoring, scoring_name, dataset_name, 'noisy', learner_name, experiment_name)
_perturbations = []
for fraction_of_troll_tweets in [0.0, 0.05, 0.25, 0.5, 0.75, 0.99]:
for _ in range(0, 100):
_perturbations.append(
perturbations.Leetspeak(fraction_of_troll_tweets, 'content',
'label', 1))
return _perturbations
models_to_evaluate = []
for learner in ['lr', 'dnn', 'xgb']:
for dataset in ['trolling']:
for score in ['roc_auc']:
models_to_evaluate.append(learner + '-' + dataset + '-' + score)
for model_to_evaluate in models_to_evaluate:
experiment_name = 'correct_shift'
(model, scoring, scoring_name, train_data, y_train, test_data, y_test,
target_data, y_target, learner_name,
dataset_name) = load_black_box(model_to_evaluate)
predictor = train_random_forest_regressor(test_data, y_test,
leet_perturbations(), model,
scoring)
evaluate_regressor(target_data, y_target, leet_perturbations(), model,
predictor, scoring, scoring_name, dataset_name,
'adversarial', learner_name, experiment_name)
from pp.generators import missing_perturbations
threshold = 0.05
num_repetitions = 100
models_to_evaluate = []
for learner in ['xgb']:
for dataset in ['adult']:
for score in ['accuracy']:
models_to_evaluate.append(learner + '-' + dataset + '-' + score)
for model_to_evaluate in models_to_evaluate:
experiment_name = 'playing_correct_shift'
(model, scoring, scoring_name, train_data, y_train, test_data, y_test,
target_data, y_target, learner_name,
dataset_name) = load_black_box(model_to_evaluate)
categorical_columns = DATASETS_CATEGORICAL_COLUMNS[dataset_name]
performance_predictor = train_random_forest_regressor(
test_data, y_test,
missing_perturbations(categorical_columns, 'NULL', num_repetitions),
model, scoring)
evaluate_regressor(
target_data, y_target, threshold,
missing_perturbations(categorical_columns, 'NULL',
num_repetitions), model, performance_predictor,
scoring, scoring_name, dataset_name, 'missing', learner_name)
num_repetitions = 100
models_to_evaluate = []
for learner in ['lr']:
for dataset in ['adult']:
for score in ['accuracy']:
models_to_evaluate.append(learner + '-' + dataset + '-' + score)
for model_to_evaluate in models_to_evaluate:
experiment_name = 'noise_on_features__swapped'
(model, scoring, scoring_name, train_data, y_train, test_data, y_test, target_data, y_target, learner_name,
dataset_name) = load_black_box(model_to_evaluate)
categorical_columns = DATASETS_CATEGORICAL_COLUMNS[dataset_name]
numerical_columns = DATASETS_NUMERICAL_COLUMNS[dataset_name]
affected_column_pairs = list(itertools.chain(
itertools.combinations(numerical_columns, 2), itertools.combinations(categorical_columns, 2)))
affected_column_pairs = random.sample(affected_column_pairs, 5)
performance_predictor = train_random_forest_regressor_with_noise(test_data, y_test, num_repetitions,
model, scoring)
evaluate_regressor(target_data, y_target, swapped_perturbations(affected_column_pairs),
model, performance_predictor, scoring, scoring_name, dataset_name, 'swapped', learner_name,
experiment_name)
perturbations.LassoExperiment(
'workclass', ['Local-gov', 'Never-worked', 'Without-pay']))
return _perturbations
#workclass Local-gov is ignored by classifier
#workclass Never-worked is ignored by classifier
#workclass Without-pay is ignored by classifier
#99 1568 in test data
#109 1569 target data
models_to_evaluate = []
for learner in ['lasso']:
for dataset in ['adult_minimal']:
for score in ['accuracy']:
models_to_evaluate.append(learner + '-' + dataset + '-' + score)
for model_to_evaluate in models_to_evaluate:
experiment_name = 'correct_shift'
(model, scoring, scoring_name, train_data, y_train, test_data, y_test,
target_data, y_target, learner_name,
dataset_name) = load_black_box(model_to_evaluate)
performance_predictor = train_random_forest_regressor(
test_data, y_test, missing_perturbations('NULL'), model, scoring)
evaluate_regressor(target_data, y_target, custom_perturbations(), model,
performance_predictor, scoring, scoring_name,
dataset_name, 'missing', learner_name, experiment_name)