def run_test(config: dict) -> List[float]: """ Run runtime experiment with a given configuration Args: config (dict): Configuration Returns: (list) Runtimes in seconds """ # Instantiate model model_name = config['model']['name'] model_params = config['model']['params'] model = get_model(model_name, model_params) # Load dataset dataset_name = config['dataset']['name'] dataset_params = config['dataset'].get('params', {}) dataset = get_dataset(dataset_name, dataset_params) X = dataset['data'] if type(X) is pd.DataFrame: X = X.values y = dataset['target'] if type(y) is pd.DataFrame: y = y.values # Train X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=int(config['explanations']['n_explanations'])) model.fit(X_train, y_train) test_acc = model.score(X_test, y_test) logger.info(' Test accuracy: {}'.format(test_acc)) # Instantiate explainer explainer_params = config['explanations']['model_params'] explainer_params['training_data'] = X_train explainer_params['feature_names'] = dataset.get('feature_names', None) # Measure time runtimes = [] logger.info(' Testing with {} runs'.format( config['experiments']['n_runs'])) for idx in range(int(config['experiments']['n_runs'])): logger.info(' Trial {}/{}'.format(idx + 1, config['experiments']['n_runs'])) explainer = get_explainer(name=config['explanations']['type'], params=explainer_params) runtime = measure_time( predict_fn=model.predict_proba, X=X_test, explainer=explainer, inference_params=config['explanations']['inference_params'], data_row_param_name=config['explanations']['data_row_param_name'], predict_fn_param_name=config['explanations'] ['predict_fn_param_name']) runtimes.append(runtime) return runtimes
def run_test(config: dict) -> List[float]: """ Run runtime experiment with a given configuration Args: config (dict): Configuration Returns: (list) Runtimes in seconds """ # Instantiate model model_name = config['model']['name'] model_params = config['model']['params'] model = get_model(model_name, model_params) # Load dataset dataset_name = config['dataset']['name'] dataset_params = config['dataset'].get('params', {}) dataset = get_dataset(dataset_name, dataset_params) raw_train = dataset['raw_train'] raw_test = dataset['raw_test'] vectorizer = TfidfVectorizer() X_train = vectorizer.fit_transform(raw_train.data) y_train = raw_train.target X_test = vectorizer.transform(raw_test.data) y_test = raw_test.target model.fit(X_train, y_train) test_acc = model.score(X_test, y_test) logger.info(' Test accuracy: {}'.format(test_acc)) def predict_fn(instance): vec = vectorizer.transform(instance) return model.predict_proba(vec) # Instantiate explainer explainer_params = config['explanations']['model_params'] # Measure time runtimes = [] logger.info(' Testing with {} runs'.format( config['experiments']['n_runs'])) for idx in range(int(config['experiments']['n_runs'])): logger.info(' Trial {}/{}'.format(idx + 1, config['experiments']['n_runs'])) explainer = get_explainer(name=config['explanations']['type'], params=explainer_params) runtime = measure_time( predict_fn=predict_fn, X=raw_test.data, explainer=explainer, inference_params=config['explanations']['inference_params'], data_row_param_name=config['explanations']['data_row_param_name'], predict_fn_param_name=config['explanations'] ['predict_fn_param_name']) runtimes.append(runtime) return runtimes
def test_unsupported_dataset(self): """ Make sure that an unimplemented abstract explainer subclass does not get instantiated """ with self.assertRaises(KeyError): try: _ = get_dataset('na-dataset', {}) except KeyError as e: msg = e.args[0] self.assertEqual( msg, 'Dataset {} not found or is not supported'.format( 'na-dataset')) raise KeyError(msg)
def run_test(config: dict) -> List[float]: """ Run consistency experiment with a given configuration Args: config (dict): configuration Returns: (list) Consistency proportion """ # Instantiate model model_name = config['model']['name'] model_params = config['model']['params'] model = get_model(model_name, model_params) # Load dataset dataset_name = config['dataset']['name'] dataset_params = config['dataset'].get('params', {}) dataset = get_dataset(dataset_name, dataset_params) X = dataset['data'] if type(X) is pd.DataFrame: X = X.values y = dataset['target'] if type(y) is pd.DataFrame: y = y.values # Train X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=int(config['explanations']['n_explanations'])) model.fit(X_train, y_train) test_acc = model.score(X_test, y_test) logger.info(' Test accuracy: {}'.format(test_acc)) # Instantiate explainer explainer_params = config['explanations']['model_params'] explainer_params['training_data'] = X_train explainer_params['feature_names'] = dataset.get('feature_names', None) num_exp_per_sample = config['explanations']['num_exp_per_sample'] logger.info(' Testing with {} runs'.format( config['experiments']['n_runs'])) logger.info( ' Generating {} explanations per sample'.format(num_exp_per_sample)) consistencies = [] for idx in range(int(config['experiments']['n_runs'])): logger.info(' Trial {}/{}'.format(idx + 1, config['experiments']['n_runs'])) explainer = get_explainer(name=config['explanations']['type'], params=explainer_params) consistency = measure_consistency( model=model, X=X_test, explainer=explainer, inference_params=config['explanations']['inference_params'], explainer_type=config['explanations']['type'], num_exp_per_sample=num_exp_per_sample, data_row_param_name=config['explanations']['data_row_param_name'], predict_fn_param_name=config['explanations'] ['predict_fn_param_name']) consistencies.append(consistency) return consistencies
def run_test(config: dict) -> List[float]: """ Run coverage experiment with a given configuration Args: config (dict): Configuration Returns: (list) coverages """ # Instantiate model model_name = config['model']['name'] model_params = config['model']['params'] model = get_model(model_name, model_params) # Load dataset dataset_name = config['dataset']['name'] dataset_params = config['dataset'].get('params', {}) dataset = get_dataset(dataset_name, dataset_params) X = dataset['data'] if type(X) is pd.DataFrame: X = X.values y = dataset['target'] if type(y) is pd.DataFrame: y = y.values # Train X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=int(config['explanations']['n_explanations'])) model.fit(X_train, y_train) test_acc = model.score(X_test, y_test) logger.info(' Test accuracy: {}'.format(test_acc)) # Instantiate explainer explainer_params = config['explanations']['model_params'] explainer_params['training_data'] = X_train explainer_params['feature_names'] = dataset.get('feature_names', None) logger.info(' Testing with {} runs'.format( config['experiments']['n_runs'])) coverages = [] for idx in range(int(config['experiments']['n_runs'])): logger.info(' Trial {}/{}'.format(idx + 1, config['experiments']['n_runs'])) explainer = get_explainer(name=config['explanations']['type'], params=explainer_params) if config['explanations']['type'] == Explainers.LIMETABULAR: categorical_feature_idxes = explainer_params[ 'categorical_features'] elif config['explanations']['type'] in [ Explainers.NUMPYENSEMBLE, Explainers.NUMPYTABULAR, Explainers.NUMPYROBUSTTABULAR ]: categorical_feature_idxes = explainer.categorical_feature_idxes else: categorical_feature_idxes = [] binarizer = Binarizer( training_data=X_train, feature_names=dataset.get('feature_names', None), categorical_feature_idxes=categorical_feature_idxes) coverage = measure_coverage( model=model, X=X_test, binarizer=binarizer, feature_names=dataset.get('feature_names', None), explainer=explainer, explainer_type=config['explanations']['type'], inference_params=config['explanations']['inference_params'], data_row_param_name=config['explanations']['data_row_param_name'], predict_fn_param_name=config['explanations'] ['predict_fn_param_name']) coverages.append(coverage) return coverages
def preprocess_robustness_datasets(dataset, params={}): data = get_dataset(dataset, params) if dataset == Datasets.COMPAS: X, y, _ = data['data'], data['target'], data['cols'] X[DATASET_CONFIGS[Datasets.COMPAS] ['unrelated_column']] = np.random.choice([0, 1], size=X.shape[0]) features = list(X.columns) categorical_feature_name = [ 'two_year_recid', 'c_charge_degree_F', 'c_charge_degree_M', 'sex_Female', 'sex_Male', 'race', 'unrelated_column' ] categorical_feature_indcs = [ features.index(c) for c in categorical_feature_name ] X = X.values elif dataset == Datasets.GERMANCREDIT: X, y = data['data'], data['target'] X = pd.DataFrame(X, columns=data['feature_names']) X[DATASET_CONFIGS[Datasets.GERMANCREDIT] ['unrelated_column']] = np.random.choice([0, 1], size=X.shape[0]) features = list(X.columns) categorical_feature_name = data['categorical_features'] + [ 'unrelated_column' ] categorical_feature_indcs = [ features.index(c) for c in categorical_feature_name ] X = X.values elif dataset == Datasets.ADULT: X, y = data['data'], data['target'] X[DATASET_CONFIGS[Datasets.ADULT] ['unrelated_column']] = np.random.choice([0, 1], size=X.shape[0]) features = list(X.columns) categorical_feature_name = data['categorical_features'] + [ 'unrelated_column' ] categorical_feature_indcs = [ features.index(c) for c in categorical_feature_name ] X = X.values elif dataset == Datasets.COMMUNITY: X, y = data['data'], data['target'] X[DATASET_CONFIGS[Datasets.COMMUNITY] ['unrelated_column']] = np.random.choice([0, 1], size=X.shape[0]).astype(int) features = list(X.columns) categorical_feature_name = [ DATASET_CONFIGS[Datasets.COMMUNITY]['unrelated_column'] ] categorical_feature_indcs = [ features.index(c) for c in categorical_feature_name ] X = X.values else: raise KeyError('Dataset {} not available'.format(dataset)) numerical_features = [ f for f in features if f not in categorical_feature_name ] numerical_feature_indcs = [features.index(c) for c in numerical_features] sc = StandardScaler() X[:, numerical_feature_indcs] = sc.fit_transform(X[:, numerical_feature_indcs]) return X, y, features, categorical_feature_name, categorical_feature_indcs