def tdi(self,
X_eval,
y_eval,
G,
D=None,
sampler=None,
loss=None,
nr_runs=10,
return_perturbed=False,
train_allowed=True):
"""Computes Relative Feature importance
# TODO(gcsk): allow handing a sample as argument
# (without needing sampler)
Args:
X_eval: data to use for resampling and evaluation.
y_eval: labels for evaluation.
G: relative feature set
D: features, used by the predictive model
sampler: choice of sampler. Default None. Will throw an error
when sampler is None and self.sampler is None as well.
loss: choice of loss. Default None. Will throw an Error when
both loss and self.loss are None.
nr_runs: how often the experiment shall be run
return_perturbed: whether the sampled perturbed versions
shall be returned
train_allowed: whether the explainer is allowed to train
the sampler
Returns:
result: An explanation object with the RFI computation
perturbed_foiss (optional): perturbed features of
interest if return_perturbed
"""
if sampler is None:
if self._sampler_specified():
sampler = self.sampler
logger.debug("Using class specified sampler.")
if loss is None:
if self._loss_specified():
loss = self.loss
logger.debug("Using class specified loss.")
if D is None:
D = X_eval.columns
# check whether the sampler is trained for each fsoi conditional on G
for f in self.fsoi:
if not sampler.is_trained([f], G):
# train if allowed, otherwise raise error
if train_allowed:
sampler.train([f], G)
logger.info('Training sampler on {}|{}'.format([f], G))
else:
raise RuntimeError(
'Sampler is not trained on {}|{}'.format([f], G))
else:
txt = '\tCheck passed: Sampler is already trained on'
txt = txt + '{}|{}'.format([f], G)
logger.debug(txt)
# initialize array for the perturbed samples
nr_obs = X_eval.shape[0]
index = utils.create_multiindex(
['sample', 'i'],
[np.arange(nr_runs), np.arange(nr_obs)])
X_fsoi_pert = pd.DataFrame([], index=index)
# perturbed_foiss = np.zeros((nr_fsoi, nr_runs, nr_obs))
# sample perturbed versions
for foi in self.fsoi:
x_foi_pert = sampler.sample(X_eval, [foi], G, num_samples=nr_runs)
X_fsoi_pert[foi] = x_foi_pert
scores = pd.DataFrame([], index=index)
# lss = np.zeros((nr_fsoi, nr_runs, X_eval.shape[0]))
# compute observasitonwise loss differences for all runs and fois
for foi in self.fsoi:
# copy of the data where perturbed variables are copied into
for kk in np.arange(0, nr_runs, 1):
# replaced with perturbed
X_eval_tilde = X_eval.copy()
arr = X_fsoi_pert.loc[(kk, slice(None)), foi].to_numpy()
X_eval_tilde[foi] = arr
# X_eval_one_perturbed[:, self.fsoi[jj]]
# = perturbed_foiss[jj, kk, :]
# using only seen while training features
# make sure model can handle it (selection and ordering)
X_eval_tilde_model = X_eval_tilde[D]
# X_eval_one_perturbed_model = X_eval_one_perturbed[:, D]
X_eval_model = X_eval[D]
# compute difference in observationwise loss
loss_pert = loss(y_eval, self.model(X_eval_tilde_model))
loss_orig = loss(y_eval, self.model(X_eval_model))
diffs = (loss_pert - loss_orig)
scores.loc[(kk, slice(None)), foi] = diffs
# lss[jj, kk, :] = loss_pert - loss_orig
# return explanation object
ex_name = 'RFI^{}'.format(G)
result = explanation.Explanation(self.fsoi, scores, ex_name=ex_name)
if return_perturbed:
logger.debug('Return both explanation and perturbed.')
return result, X_fsoi_pert
else:
logger.debug('Return explanation object only')
return result
def sage(self,
X_test,
y_test,
fixed_orderings=None,
partial_ordering=None,
nr_orderings=None,
approx=math.sqrt,
type='rfi',
save_orderings=True,
nr_runs=10,
sampler=None,
loss=None,
train_allowed=True,
D=None,
return_test_log_lik=False,
nr_resample_marginalize=10):
"""
Compute Shapley Additive Global Importance values.
Args:
type: either 'rfi' or 'rfa', depending on whether conditional
or marginal resampling of the remaining features shall
be used
X_test: data to use for resampling and evaluation.
y_test: labels for evaluation.
fixed_orderings: list of ready orderings
nr_orderings: number of orderings in which features enter the model
nr_runs: how often each value function shall be computed
sampler: choice of sampler. Default None. Will throw an error
when sampler is None and self.sampler is None as well.
loss: choice of loss. Default None. Will throw an Error when
both loss and self.loss are None.
train_allowed: whether the explainer is allowed to
train the sampler
return_test_log_lik: return log-likelihood of conditional sampler on X_test
Returns:
result: an explanation object containing the respective
pairwise lossdifferences with shape
(nr_fsoi, nr_runs, nr_obs, nr_orderings)
orderings (optional): an array containing the respective
orderings if return_orderings
"""
# the method is currently not build for situations
# where we are only interested in
# a subset of the model's features
if X_test.shape[1] != len(self.fsoi):
logger.debug('self.fsoi: {}'.format(self.fsoi))
logger.debug('#features in model: {}'.format(X_test.shape[1]))
raise RuntimeError('self.fsoi is not identical to all features')
if sampler is None:
if self._sampler_specified():
sampler = self.sampler
logger.debug("Using class specified sampler.")
if loss is None:
if self._loss_specified():
loss = self.loss
logger.debug("Using class specified loss.")
if fixed_orderings is not None:
fixed_orderings = np.array(fixed_orderings)
nr_orderings = len(fixed_orderings)
if nr_orderings is None:
nr_unique = utils.nr_unique_perm(partial_ordering)
if approx is not None:
nr_orderings = math.floor(approx(nr_unique))
else:
nr_orderings = nr_unique
if D is None:
D = X_test.columns
nr_orderings_saved = 1
if save_orderings:
nr_orderings_saved = nr_orderings
# create dataframe for computation results
index = utils.create_multiindex(['ordering', 'sample', 'id'], [
np.arange(nr_orderings_saved),
np.arange(nr_runs),
np.arange(X_test.shape[0])
])
arr = np.zeros(
(nr_orderings_saved * nr_runs * X_test.shape[0], len(self.fsoi)))
scores = pd.DataFrame(arr, index=index, columns=self.fsoi)
test_log_lik = []
for ii in range(nr_orderings):
if fixed_orderings is None:
ordering = utils.sample_partial(partial_ordering)
else:
ordering = fixed_orderings[ii]
logging.info('Ordering : {}'.format(ordering))
# ordering = np.random.permutation(len(self.fsoi))
# resample multiple times
for kk in range(nr_runs):
# enter one feature at a time
y_hat_base = np.repeat(np.mean(self.model(X_test[D])),
X_test.shape[0])
for jj in np.arange(1, len(ordering), 1):
# compute change in performance
# by entering the respective feature
# store the result in the right place
# validate training of sampler
impute, fixed = ordering[jj:], ordering[:jj]
logger.info(
'ordering {}: run {}: split {}: impute, fixed: {} | {}'
.format(ii, kk, jj, impute, fixed))
if not sampler.is_trained(impute, fixed):
# train if allowed, otherwise raise error
if train_allowed:
estimator = sampler.train(impute, fixed)
# Evaluating test log-likelihood for diagnostics
test_inputs = X_test[sampler._order_fset(
impute)].to_numpy()
test_context = X_test[sampler._order_fset(
fixed)].to_numpy()
log_lik = estimator.log_prob(
inputs=test_inputs,
context=test_context).mean()
logger.info(f'Test log-likelihood: {log_lik}')
test_log_lik.append(log_lik)
else:
raise RuntimeError('Sampler is not trained on '
'{}|{}'.format(impute, fixed))
X_test_perturbed = X_test.copy()
# iterate values nr_samples_marginalize times
i_ix = X_test_perturbed.index.values
rn_ix = np.arange(nr_resample_marginalize)
index = utils.create_multiindex(['sample', 'id'],
[rn_ix, i_ix])
tiling = np.tile(np.arange(len(X_test_perturbed)),
nr_resample_marginalize)
vals = X_test_perturbed.iloc[tiling].to_numpy()
cols = X_test_perturbed.columns
X_test_perturbed = pd.DataFrame(vals,
index=index,
columns=cols)
imps = sampler.sample(X_test,
impute,
fixed,
num_samples=nr_resample_marginalize)
X_test_perturbed[impute] = imps[impute].to_numpy()
# sample replacement, create replacement matrix
y_hat_new = self.model(X_test_perturbed[D])
# mean over samples
df_y_hat_new = pd.DataFrame(y_hat_new,
index=index,
columns=['y_hat_new'])
y_hat_new_marg = df_y_hat_new.mean(level='id')
lb = loss(y_test, y_hat_base)
ln = loss(y_test, y_hat_new_marg)
diff = lb - ln
scores.loc[(ii, kk, slice(None)), ordering[jj - 1]] = diff
# lss[self.fsoi[ordering[jj - 1]], kk, :, ii] = lb - ln
y_hat_base = y_hat_new_marg
y_hat_new = self.model(X_test[D])
diff = loss(y_test, y_hat_base) - loss(y_test, y_hat_new)
scores.loc[(ii, kk, slice(None)), ordering[-1]] = diff
# lss[self.fsoi[ordering[-1]], kk, :, ii] = diff
result = explanation.Explanation(self.fsoi, scores, ex_name='SAGE')
if return_test_log_lik:
return result, test_log_lik
return result
def tai(self,
X_eval,
y_eval,
K,
D=None,
sampler=None,
decorrelator=None,
loss=None,
nr_runs=10,
return_perturbed=False,
train_allowed=True,
ex_name=None):
"""Computes Feature Association
Args:
X_eval: data to use for resampling and evaluation.
y_eval: labels for evaluation.
K: features not to be reconstracted
D: model features (including their required ordering)
sampler: choice of sampler. Default None. Will throw an error
when sampler is None and self.sampler is None as well.
loss: choice of loss. Default None. Will throw an Error when
both loss and self.loss are None.
nr_runs: how often the experiment shall be run
return_perturbed: whether the sampled perturbed
versions shall be returned
train_allowed: whether the explainer is allowed
to train the sampler
Returns:
result: An explanation object with the RFI computation
perturbed_foiss (optional): perturbed features of
interest if return_perturbed
"""
if sampler is None:
if self._sampler_specified(): # may throw an error
sampler = self.sampler
logger.debug("Using class specified sampler.")
if decorrelator is None:
if self._decorrelator_specified(): # may throw error
decorrelator = self.decorrelator
logger.debug("Using class specified decorrelator")
if loss is None:
if self._loss_specified(): # may throw an error
loss = self.loss
logger.debug("Using class specified loss.")
if D is None:
D = X_eval.columns
# all_fs = np.arange(X_test.shape[1])
# check whether the sampler is trained for the baseline perturbation
if not sampler.is_trained(D, []):
# train if allowed, otherwise raise error
if train_allowed:
sampler.train(D, [])
logger.info('Training sampler on {}|{}'.format(D, []))
else:
raise RuntimeError('Sampler is not trained on {}|{}'.format(
D, []))
else:
txt = '\tCheck passed: Sampler is already trained on '
txt = txt + '{}|{}'.format(D, [])
logger.debug(txt)
# check for each of the features of interest
for foi in self.fsoi:
if not sampler.is_trained(D, [foi]):
# train if allowed, otherwise raise error
if train_allowed:
sampler.train(D, [foi])
logger.info('Training sampler on {}|{}'.format(D, [foi]))
else:
raise RuntimeError(
'Sampler is not trained on {}|{}'.format(D, [foi]))
else:
txt = '\tCheck passed: Sampler is already trained on '
txt = txt + '{}|{}'.format(D, [foi])
logger.debug(txt)
# check whether decorrelators have been trained
for foi in self.fsoi:
if not decorrelator.is_trained(K, [foi], []):
if train_allowed:
decorrelator.train(K, [foi], [])
txt = 'Training decorrelator on '
txt = txt + '{} idp {} | {}'.format(K, [foi], [])
logger.info(txt)
else:
txt = 'Decorrelator is not trained on '
txt = txt + '{} {} | {}'.format(K, [foi], [])
raise RuntimeError(txt)
else:
logger.debug('\tCheck passed: '
'Decorrelator is already trained on '
'{} {} | {}'.format(K, [foi], []))
# initialize array for the perturbed samples
nr_obs = X_eval.shape[0]
# initialize pandas dataframes for X_eval_tilde baseline
# and X_eval_tilde reconstrcted
index_bsln = utils.create_multiindex(
['sample', 'i'],
[np.arange(nr_runs), np.arange(nr_obs)])
X_eval_tilde_bsln = pd.DataFrame([], index=index_bsln, columns=D)
index_rcnstr = utils.create_multiindex(
['foi', 'sample', 'i'],
[self.fsoi, np.arange(nr_runs),
np.arange(nr_obs)])
X_eval_tilde_rcnstr = pd.DataFrame([], index=index_rcnstr, columns=D)
# sample baseline X^\emptyset
X_eval_tilde_bsln = sampler.sample(X_eval, D, [], num_samples=nr_runs)
# sample perturbed versions
for foi in self.fsoi:
# X^foi
sample = sampler.sample(X_eval, D, [foi], num_samples=nr_runs)
for kk in np.arange(nr_runs):
# X^\emptyset,linked
sample_decorr = decorrelator.decorrelate(
sample.loc[kk, :], K, [foi], [])
sd_np = sample_decorr[D].to_numpy()
X_eval_tilde_rcnstr.loc[(foi, kk, slice(None)), D] = sd_np
# create empty scores data frame
index_scores = utils.create_multiindex(
['sample', 'i'],
[np.arange(nr_runs), np.arange(nr_obs)])
scores = pd.DataFrame([], index=index_scores)
# compute observasitonwise loss differences for all runs and fois
for kk in np.arange(nr_runs):
X_bl = X_eval_tilde_bsln.loc[(kk, slice(None)), D]
l_pb = loss(y_eval, self.model(X_bl))
for foi in self.fsoi:
X_rc = X_eval_tilde_rcnstr.loc[(foi, kk, slice(None)), D]
l_rc = loss(y_eval, self.model(X_rc))
scores.loc[(kk, slice(None)), foi] = l_pb - l_rc
if ex_name is None:
ex_name = 'Unknown tai'
# return explanation object
result = explanation.Explanation(self.fsoi, scores, ex_name=ex_name)
if return_perturbed:
raise NotImplementedError(
'Returning perturbed not implemented yet.')
else:
logger.debug('Return explanation object only')
return result
def tdi_from(self,
K,
B,
J,
X_eval,
y_eval,
D=None,
sampler=None,
decorrelator=None,
loss=None,
nr_runs=10,
return_perturbed=False,
train_allowed=True,
target='Y',
marginalize=False):
"""Computes Relative Feature importance
Args:
K: features of interest
B: baseline features
J: "from" conditioning set
X_eval: data to use for resampling and evaluation.
y_eval: labels for evaluation.
D: features, used by the predictive model
sampler: choice of sampler. Default None. Will throw an error
when sampler is None and self.sampler is None as well.
loss: choice of loss. Default None. Will throw an Error when
both loss and self.loss are None.
nr_runs: how often the experiment shall be run
return_perturbed: whether the sampled perturbed versions
shall be returned
train_allowed: whether the explainer is allowed to train
the sampler
Returns:
result: An explanation object with the RFI computation
perturbed_foiss (optional): perturbed features of
interest if return_perturbed
"""
if target not in ['Y', 'Y_hat']:
raise ValueError('Y and Y_hat are the only valid targets.')
if marginalize:
raise NotImplementedError('Marginalization not implemented yet.')
if sampler is None:
if self._sampler_specified():
sampler = self.sampler
logger.debug("Using class specified sampler.")
if decorrelator is None:
if self._decorrelator_specified():
decorrelator = self.decorrelator
logger.debug("Using class specified decorrelator.")
if loss is None:
if self._loss_specified():
loss = self.loss
logger.debug("Using class specified loss.")
if D is None:
D = X_eval.columns
if not set(K).isdisjoint(set(B)):
raise ValueError('K and B are not disjoint.')
# check whether sampler is trained for baseline dropped features
R = list(set(D) - set(B))
if not sampler.is_trained(R, J):
# train if allowed, otherwise raise error
if train_allowed:
sampler.train(R, J)
logger.info('Training sampler on {}|{}'.format(R, J))
else:
raise RuntimeError('Sampler is not trained on {}|{}'.format(
R, J))
else:
txt = '\tCheck passed: Sampler is already trained on'
txt = txt + '{}|{}'.format(R, J)
logger.debug(txt)
if not decorrelator.is_trained(R, J, []):
# train if allowed, otherwise raise error
if train_allowed:
decorrelator.train(R, J, [])
logger.info('Training decorrelator on {} idp {} |{}'.format(
R, J, []))
else:
raise RuntimeError(
'Sampler is not trained on {} idp {} |{}'.format(R, J, []))
else:
txt = '\tCheck passed: decorrelator is already trained on'
txt = txt + '{} idp {}|{}'.format(R, J, [])
logger.debug(txt)
desc = 'TDI({} | {} <- {})'.format(K, B, J)
# initialize array for the perturbed samples
nr_obs = X_eval.shape[0]
index = utils.create_multiindex(
['sample', 'i'],
[np.arange(nr_runs), np.arange(nr_obs)])
# X_fsoi_pert = pd.DataFrame([], index=index)
# perturbed_foiss = np.zeros((nr_fsoi, nr_runs, nr_obs))
# sample perturbed versions
X_R_J = sampler.sample(X_eval, R, J, num_samples=nr_runs)
breakpoint()
scores = pd.DataFrame([], index=index)
# lss = np.zeros((nr_fsoi, nr_runs, X_eval.shape[0]))
for kk in np.arange(0, nr_runs, 1):
# replaced with perturbed
X_tilde_baseline = X_eval.copy()
X_tilde_foreground = X_eval.copy()
X_R_empty_linked = decorrelator.decorrelate(
X_R_J.loc[kk, :], R, J, [])
arr = X_R_empty_linked[R].to_numpy()
X_tilde_baseline[R] = arr
X_tilde_foreground[R] = arr
X_tilde_foreground[K] = X_R_J.loc[(kk, slice(None)), J].to_numpy()
# make sure model can handle it (selection and ordering)
X_tilde_baseline = X_tilde_baseline[D]
X_tilde_foreground = X_tilde_foreground[D]
# compute difference in observationwise loss
if target == 'Y':
loss_baseline = loss(y_eval, self.model(X_tilde_baseline))
loss_foreground = loss(y_eval, self.model(X_tilde_foreground))
diffs = (loss_baseline - loss_foreground)
scores.loc[(kk, slice(None)), 'score'] = diffs
else:
raise NotImplementedError('Y_hat not implemented yet.')
# lss[jj, kk, :] = loss_pert - loss_orig
# return explanation object
ex_name = desc
result = explanation.Explanation(self.fsoi, scores, ex_name=ex_name)
if return_perturbed:
raise NotImplementedError('return_perturbed=True not implemented.')
else:
logger.debug('Return explanation object only')
return result
def ar_via(self,
J,
C,
K,
X_eval,
y_eval,
D=None,
sampler=None,
decorrelator=None,
loss=None,
nr_runs=10,
return_perturbed=False,
train_allowed=True,
target='Y',
marginalize=False,
nr_resample_marginalize=5):
"""Computes Relative Feature importance
Args:
K: features of interest
B: baseline features
J: "from" conditioning set
X_eval: data to use for resampling and evaluation.
y_eval: labels for evaluation.
D: features, used by the predictive model
sampler: choice of sampler. Default None. Will throw an error
when sampler is None and self.sampler is None as well.
loss: choice of loss. Default None. Will throw an Error when
both loss and self.loss are None.
nr_runs: how often the experiment shall be run
return_perturbed: whether the sampled perturbed versions
shall be returned
train_allowed: whether the explainer is allowed to train
the sampler
Returns:
result: An explanation object with the RFI computation
perturbed_foiss (optional): perturbed features of
interest if return_perturbed
"""
if target not in ['Y', 'Y_hat']:
raise ValueError('Y and Y_hat are the only valid targets.')
# if marginalize:
# raise NotImplementedError('Marginalization not implemented yet.')
if sampler is None:
if self._sampler_specified():
sampler = self.sampler
logger.debug("Using class specified sampler.")
if decorrelator is None:
if self._decorrelator_specified():
decorrelator = self.decorrelator
logger.debug("Using class specified decorrelator.")
if loss is None:
if self._loss_specified():
loss = self.loss
logger.debug("Using class specified loss.")
if D is None:
D = X_eval.columns
if not marginalize:
nr_resample_marginalize = 1
if not set(J).isdisjoint(set(C)):
raise ValueError('J and C are not disjoint.')
# check whether sampler is trained for baseline dropped features
R = list(set(D) - set(C))
R_ = list(set(R) - set(J))
CuJ = list(set(C).union(J))
if not sampler.is_trained(R_, CuJ):
# train if allowed, otherwise raise error
if train_allowed:
sampler.train(R_, CuJ)
logger.info('Training sampler on {}|{}'.format(R_, CuJ))
else:
raise RuntimeError('Sampler is not trained on {}|{}'.format(
R_, CuJ))
else:
txt = '\tCheck passed: Sampler is already trained on'
txt = txt + '{}|{}'.format(R, J)
logger.debug(txt)
if not decorrelator.is_trained(K, J, C):
# train if allowed, otherwise raise error
if train_allowed:
decorrelator.train(K, J, C)
logger.info('Training decorrelator on {} idp {} |{}'.format(
K, J, C))
else:
raise RuntimeError(
'Sampler is not trained on {} idp {} |{}'.format(K, J, C))
else:
txt = '\tCheck passed: decorrelator is already trained on'
txt = txt + '{} idp {}|{}'.format(K, J, C)
logger.debug(txt)
desc = 'AR({} | {} -> {})'.format(J, C, K)
# initialize array for the perturbed samples
nr_obs = X_eval.shape[0]
index = utils.create_multiindex(
['sample', 'i'],
[np.arange(nr_runs), np.arange(nr_obs)])
scores = pd.DataFrame([], index=index)
for kk in np.arange(0, nr_runs, 1):
# sample perturbed versions
X_R_CuJ2 = None
X_R_CuJ = sampler.sample(X_eval,
R_,
CuJ,
num_samples=nr_resample_marginalize)
index = X_R_CuJ.index
df_yh = pd.DataFrame(
index=index, columns=['y_hat_baseline', 'y_hat_foreground'])
for ll in np.arange(0, nr_resample_marginalize, 1):
X_tilde_baseline = X_eval.copy()
X_tilde_foreground = X_eval.copy()
arr_reconstr = X_R_CuJ.loc[(ll, slice(None)), R_].to_numpy()
X_tilde_foreground[R_] = arr_reconstr
X_R_decorr = decorrelator.decorrelate(X_tilde_foreground, K, J,
C)
arr_decorr = X_R_decorr[R].to_numpy()
X_tilde_baseline[R] = arr_decorr
# make sure model can handle it (selection and ordering)
X_tilde_baseline = X_tilde_baseline[D]
X_tilde_foreground = X_tilde_foreground[D]
y_hat_baseline = self.model(X_tilde_baseline)
y_hat_foreground = self.model(X_tilde_foreground)
df_yh.loc[(ll, slice(None)), 'y_hat_baseline'] = y_hat_baseline
df_yh.loc[(ll, slice(None)),
'y_hat_foreground'] = y_hat_foreground
df_yh = df_yh.astype({
'y_hat_baseline': 'float',
'y_hat_foreground': 'float'
})
df_yh = df_yh.mean(level='i')
# compute difference in observationwise loss
if target == 'Y':
loss_baseline = loss(y_eval, df_yh['y_hat_baseline'])
loss_foreground = loss(y_eval, df_yh['y_hat_foreground'])
diffs = (loss_baseline - loss_foreground)
scores.loc[(kk, slice(None)), 'score'] = diffs
elif target == 'Y_hat':
diffs = loss(df_yh['y_hat_baseline'],
df_yh['y_hat_foreground'])
scores.loc[(kk, slice(None)), 'score'] = diffs
# return explanation object
ex_name = desc
result = explanation.Explanation(self.fsoi, scores, ex_name=ex_name)
if return_perturbed:
raise NotImplementedError('return_perturbed=True not implemented.')
else:
logger.debug('Return explanation object only')
return result
def sage(self,
X_test,
y_test,
partial_ordering,
nr_orderings=None,
approx=math.sqrt,
save_orderings=True,
nr_runs=10,
sampler=None,
loss=None,
train_allowed=True,
D=None,
nr_resample_marginalize=10,
target='Y',
G=None,
method='associative',
marginalize=True,
orderings=None,
**kwargs):
"""
Compute Shapley Additive Global Importance values.
Args:
type: either 'rfi' or 'rfa', depending on whether conditional
or marginal resampling of the remaining features shall
be used
X_test: data to use for resampling and evaluation.
y_test: labels for evaluation.
nr_orderings: number of orderings in which features enter the model
nr_runs: how often each value function shall be computed
sampler: choice of sampler. Default None. Will throw an error
when sampler is None and self.sampler is None as well.
loss: choice of loss. Default None. Will throw an Error when
both loss and self.loss are None.
train_allowed: whether the explainer is allowed to
train the sampler
Returns:
result: an explanation object containing the respective
pairwise lossdifferences with shape
(nr_fsoi, nr_runs, nr_obs, nr_orderings)
orderings (optional): an array containing the respective
orderings if return_orderings
"""
if G is None:
G = X_test.columns
if X_test.shape[1] != len(self.fsoi):
logger.debug('self.fsoi: {}'.format(self.fsoi))
logger.debug('#features in model: {}'.format(X_test.shape[1]))
raise RuntimeError('self.fsoi is not identical to all features')
if method not in ['associative', 'direct']:
raise ValueError('only methods associative or direct implemented')
if sampler is None:
if self._sampler_specified():
sampler = self.sampler
logger.debug("Using class specified sampler.")
if loss is None:
if self._loss_specified():
loss = self.loss
logger.debug("Using class specified loss.")
if orderings is None:
if nr_orderings is None:
nr_unique = utils.nr_unique_perm(partial_ordering)
if approx is not None:
nr_orderings = math.floor(approx(nr_unique))
else:
nr_orderings = nr_unique
else:
nr_orderings = orderings.shape[0]
if D is None:
D = X_test.columns
nr_orderings_saved = 1
if save_orderings:
nr_orderings_saved = nr_orderings
# create dataframe for computation results
index = utils.create_multiindex(['ordering', 'sample', 'id'], [
np.arange(nr_orderings_saved),
np.arange(nr_runs),
np.arange(X_test.shape[0])
])
arr = np.zeros(
(nr_orderings_saved * nr_runs * X_test.shape[0], len(self.fsoi)))
scores = pd.DataFrame(arr, index=index, columns=self.fsoi)
orderings_sampled = None
if orderings is None:
orderings_sampled = pd.DataFrame(index=np.arange(nr_orderings),
columns=['ordering'])
# lss = np.zeros(
# (len(self.fsoi), nr_runs, X_test.shape[0], nr_orderings))
# ord hist helps to avoid duplicate histories
ord_hist = None
for ii in range(nr_orderings):
ordering = None
if orderings is None:
ordering, ord_hist = utils.sample_partial(
partial_ordering, ord_hist)
orderings_sampled.loc[ii, 'ordering'] = ordering
else:
ordering = orderings.loc[ii, 'ordering']
logging.info('Ordering : {}'.format(ordering))
for jj in np.arange(1, len(ordering), 1):
# TODO: check if jj in features for which the score shall
# TODO: be computed
# compute change in performance
# by entering the respective feature
# store the result in the right place
# validate training of sampler
J, C = [ordering[jj - 1]], ordering[:jj - 1]
if method == 'associative':
ex = self.ar_via(
J,
C,
G,
X_test,
y_test,
target=target,
marginalize=marginalize,
nr_runs=nr_runs,
nr_resample_marginalize=nr_resample_marginalize,
**kwargs)
elif method == 'direct':
ex = self.dr_from(
J,
C,
G,
X_test,
y_test,
target=target,
marginalize=marginalize,
nr_runs=nr_runs,
nr_resample_marginalize=nr_resample_marginalize,
**kwargs)
scores_arr = ex.scores.to_numpy()
scores.loc[(ii, slice(None), slice(None)),
ordering[jj - 1]] = scores_arr
result = explanation.Explanation(self.fsoi, scores, ex_name='SAGE')
if orderings is None:
orderings = orderings_sampled
return result, orderings