def run_test(config: dict) -> List[float]:
"""
Run runtime experiment with a given configuration
Args:
config (dict): Configuration
Returns:
(list) Runtimes in seconds
"""
# Instantiate model
model_name = config['model']['name']
model_params = config['model']['params']
model = get_model(model_name, model_params)
# Load dataset
dataset_name = config['dataset']['name']
dataset_params = config['dataset'].get('params', {})
dataset = get_dataset(dataset_name, dataset_params)
X = dataset['data']
if type(X) is pd.DataFrame:
X = X.values
y = dataset['target']
if type(y) is pd.DataFrame:
y = y.values
# Train
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=int(config['explanations']['n_explanations']))
model.fit(X_train, y_train)
test_acc = model.score(X_test, y_test)
logger.info(' Test accuracy: {}'.format(test_acc))
# Instantiate explainer
explainer_params = config['explanations']['model_params']
explainer_params['training_data'] = X_train
explainer_params['feature_names'] = dataset.get('feature_names', None)
# Measure time
runtimes = []
logger.info(' Testing with {} runs'.format(
config['experiments']['n_runs']))
for idx in range(int(config['experiments']['n_runs'])):
logger.info(' Trial {}/{}'.format(idx + 1,
config['experiments']['n_runs']))
explainer = get_explainer(name=config['explanations']['type'],
params=explainer_params)
runtime = measure_time(
predict_fn=model.predict_proba,
X=X_test,
explainer=explainer,
inference_params=config['explanations']['inference_params'],
data_row_param_name=config['explanations']['data_row_param_name'],
predict_fn_param_name=config['explanations']
['predict_fn_param_name'])
runtimes.append(runtime)
return runtimes
def run_test(config: dict) -> List[float]:
"""
Run runtime experiment with a given configuration
Args:
config (dict): Configuration
Returns:
(list) Runtimes in seconds
"""
# Instantiate model
model_name = config['model']['name']
model_params = config['model']['params']
model = get_model(model_name, model_params)
# Load dataset
dataset_name = config['dataset']['name']
dataset_params = config['dataset'].get('params', {})
dataset = get_dataset(dataset_name, dataset_params)
raw_train = dataset['raw_train']
raw_test = dataset['raw_test']
vectorizer = TfidfVectorizer()
X_train = vectorizer.fit_transform(raw_train.data)
y_train = raw_train.target
X_test = vectorizer.transform(raw_test.data)
y_test = raw_test.target
model.fit(X_train, y_train)
test_acc = model.score(X_test, y_test)
logger.info(' Test accuracy: {}'.format(test_acc))
def predict_fn(instance):
vec = vectorizer.transform(instance)
return model.predict_proba(vec)
# Instantiate explainer
explainer_params = config['explanations']['model_params']
# Measure time
runtimes = []
logger.info(' Testing with {} runs'.format(
config['experiments']['n_runs']))
for idx in range(int(config['experiments']['n_runs'])):
logger.info(' Trial {}/{}'.format(idx + 1,
config['experiments']['n_runs']))
explainer = get_explainer(name=config['explanations']['type'],
params=explainer_params)
runtime = measure_time(
predict_fn=predict_fn,
X=raw_test.data,
explainer=explainer,
inference_params=config['explanations']['inference_params'],
data_row_param_name=config['explanations']['data_row_param_name'],
predict_fn_param_name=config['explanations']
['predict_fn_param_name'])
runtimes.append(runtime)
return runtimes
def test_unsupported_dataset(self):
"""
Make sure that an unimplemented abstract explainer subclass does not get instantiated
"""
with self.assertRaises(KeyError):
try:
_ = get_dataset('na-dataset', {})
except KeyError as e:
msg = e.args[0]
self.assertEqual(
msg, 'Dataset {} not found or is not supported'.format(
'na-dataset'))
raise KeyError(msg)
def run_test(config: dict) -> List[float]:
"""
Run consistency experiment with a given configuration
Args:
config (dict): configuration
Returns:
(list) Consistency proportion
"""
# Instantiate model
model_name = config['model']['name']
model_params = config['model']['params']
model = get_model(model_name, model_params)
# Load dataset
dataset_name = config['dataset']['name']
dataset_params = config['dataset'].get('params', {})
dataset = get_dataset(dataset_name, dataset_params)
X = dataset['data']
if type(X) is pd.DataFrame:
X = X.values
y = dataset['target']
if type(y) is pd.DataFrame:
y = y.values
# Train
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=int(config['explanations']['n_explanations']))
model.fit(X_train, y_train)
test_acc = model.score(X_test, y_test)
logger.info(' Test accuracy: {}'.format(test_acc))
# Instantiate explainer
explainer_params = config['explanations']['model_params']
explainer_params['training_data'] = X_train
explainer_params['feature_names'] = dataset.get('feature_names', None)
num_exp_per_sample = config['explanations']['num_exp_per_sample']
logger.info(' Testing with {} runs'.format(
config['experiments']['n_runs']))
logger.info(
' Generating {} explanations per sample'.format(num_exp_per_sample))
consistencies = []
for idx in range(int(config['experiments']['n_runs'])):
logger.info(' Trial {}/{}'.format(idx + 1,
config['experiments']['n_runs']))
explainer = get_explainer(name=config['explanations']['type'],
params=explainer_params)
consistency = measure_consistency(
model=model,
X=X_test,
explainer=explainer,
inference_params=config['explanations']['inference_params'],
explainer_type=config['explanations']['type'],
num_exp_per_sample=num_exp_per_sample,
data_row_param_name=config['explanations']['data_row_param_name'],
predict_fn_param_name=config['explanations']
['predict_fn_param_name'])
consistencies.append(consistency)
return consistencies
def run_test(config: dict) -> List[float]:
"""
Run coverage experiment with a given configuration
Args:
config (dict): Configuration
Returns:
(list) coverages
"""
# Instantiate model
model_name = config['model']['name']
model_params = config['model']['params']
model = get_model(model_name, model_params)
# Load dataset
dataset_name = config['dataset']['name']
dataset_params = config['dataset'].get('params', {})
dataset = get_dataset(dataset_name, dataset_params)
X = dataset['data']
if type(X) is pd.DataFrame:
X = X.values
y = dataset['target']
if type(y) is pd.DataFrame:
y = y.values
# Train
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=int(config['explanations']['n_explanations']))
model.fit(X_train, y_train)
test_acc = model.score(X_test, y_test)
logger.info(' Test accuracy: {}'.format(test_acc))
# Instantiate explainer
explainer_params = config['explanations']['model_params']
explainer_params['training_data'] = X_train
explainer_params['feature_names'] = dataset.get('feature_names', None)
logger.info(' Testing with {} runs'.format(
config['experiments']['n_runs']))
coverages = []
for idx in range(int(config['experiments']['n_runs'])):
logger.info(' Trial {}/{}'.format(idx + 1,
config['experiments']['n_runs']))
explainer = get_explainer(name=config['explanations']['type'],
params=explainer_params)
if config['explanations']['type'] == Explainers.LIMETABULAR:
categorical_feature_idxes = explainer_params[
'categorical_features']
elif config['explanations']['type'] in [
Explainers.NUMPYENSEMBLE, Explainers.NUMPYTABULAR,
Explainers.NUMPYROBUSTTABULAR
]:
categorical_feature_idxes = explainer.categorical_feature_idxes
else:
categorical_feature_idxes = []
binarizer = Binarizer(
training_data=X_train,
feature_names=dataset.get('feature_names', None),
categorical_feature_idxes=categorical_feature_idxes)
coverage = measure_coverage(
model=model,
X=X_test,
binarizer=binarizer,
feature_names=dataset.get('feature_names', None),
explainer=explainer,
explainer_type=config['explanations']['type'],
inference_params=config['explanations']['inference_params'],
data_row_param_name=config['explanations']['data_row_param_name'],
predict_fn_param_name=config['explanations']
['predict_fn_param_name'])
coverages.append(coverage)
return coverages
def preprocess_robustness_datasets(dataset, params={}):
data = get_dataset(dataset, params)
if dataset == Datasets.COMPAS:
X, y, _ = data['data'], data['target'], data['cols']
X[DATASET_CONFIGS[Datasets.COMPAS]
['unrelated_column']] = np.random.choice([0, 1], size=X.shape[0])
features = list(X.columns)
categorical_feature_name = [
'two_year_recid', 'c_charge_degree_F', 'c_charge_degree_M',
'sex_Female', 'sex_Male', 'race', 'unrelated_column'
]
categorical_feature_indcs = [
features.index(c) for c in categorical_feature_name
]
X = X.values
elif dataset == Datasets.GERMANCREDIT:
X, y = data['data'], data['target']
X = pd.DataFrame(X, columns=data['feature_names'])
X[DATASET_CONFIGS[Datasets.GERMANCREDIT]
['unrelated_column']] = np.random.choice([0, 1], size=X.shape[0])
features = list(X.columns)
categorical_feature_name = data['categorical_features'] + [
'unrelated_column'
]
categorical_feature_indcs = [
features.index(c) for c in categorical_feature_name
]
X = X.values
elif dataset == Datasets.ADULT:
X, y = data['data'], data['target']
X[DATASET_CONFIGS[Datasets.ADULT]
['unrelated_column']] = np.random.choice([0, 1], size=X.shape[0])
features = list(X.columns)
categorical_feature_name = data['categorical_features'] + [
'unrelated_column'
]
categorical_feature_indcs = [
features.index(c) for c in categorical_feature_name
]
X = X.values
elif dataset == Datasets.COMMUNITY:
X, y = data['data'], data['target']
X[DATASET_CONFIGS[Datasets.COMMUNITY]
['unrelated_column']] = np.random.choice([0, 1],
size=X.shape[0]).astype(int)
features = list(X.columns)
categorical_feature_name = [
DATASET_CONFIGS[Datasets.COMMUNITY]['unrelated_column']
]
categorical_feature_indcs = [
features.index(c) for c in categorical_feature_name
]
X = X.values
else:
raise KeyError('Dataset {} not available'.format(dataset))
numerical_features = [
f for f in features if f not in categorical_feature_name
]
numerical_feature_indcs = [features.index(c) for c in numerical_features]
sc = StandardScaler()
X[:,
numerical_feature_indcs] = sc.fit_transform(X[:,
numerical_feature_indcs])
return X, y, features, categorical_feature_name, categorical_feature_indcs