def test_indexed_string_inverse_removing_tokenizer(self):
s = 'This is a good movie. This, it is a great movie.'
def tokenizer(string):
return re.split(r'(?:\W+)|$', string)
indexed_string = IndexedString(s, tokenizer)
self.assertEqual(s, indexed_string.inverse_removing([]))
def test_indexed_string_inverse_removing_tokenizer(self):
s = 'This is a good movie. This, it is a great movie.'
def tokenizer(string):
return re.split(r'(?:\W+)|$', string)
indexed_string = IndexedString(s, tokenizer)
self.assertEqual(s, indexed_string.inverse_removing([]))
def test_indexed_string_regex(self):
s = 'Please, take your time. Please'
tokenized_string = np.array(
['Please', ', ', 'take', ' ', 'your', ' ', 'time', '. ', 'Please'])
inverse_vocab = ['Please', 'take', 'your', 'time']
start_positions = [0, 6, 8, 12, 13, 17, 18, 22, 24]
positions = [[0, 8], [2], [4], [6]]
indexed_string = IndexedString(s)
self.assertTrue(np.array_equal(indexed_string.as_np, tokenized_string))
self.assertTrue(
np.array_equal(indexed_string.string_start, start_positions))
self.assertTrue(indexed_string.inverse_vocab == inverse_vocab)
self.assertTrue(np.array_equal(indexed_string.positions, positions))
def str_to_pair_of_tuples(self, strs):
dataframe = pd.DataFrame(columns=self.columns)
for row in strs:
this_pair_row = defaultdict(str)
row = IndexedString(row, split_expression=" ")
for attr_index in self.schema.keys():
left_tokens = self.__indexes_of_attr(
row, self.__make_attr(attr_index, tuple="left"))
right_tokens = self.__indexes_of_attr(
row, self.__make_attr(attr_index, tuple="right"))
this_pair_row[self.lprefix +
self.schema[attr_index]] = " ".join([
self.__split_attr(row.word(t))[2]
for t in left_tokens
])
this_pair_row[self.rprefix +
self.schema[attr_index]] = " ".join([
self.__split_attr(row.word(t))[2]
for t in right_tokens
])
dataframe = dataframe.append(this_pair_row, ignore_index=True)
return dataframe
def test_indexed_string_callable(self):
s = 'aabbccddaa'
def tokenizer(string):
return [
string[i] + string[i + 1]
for i in range(0,
len(string) - 1, 2)
]
tokenized_string = np.array(['aa', 'bb', 'cc', 'dd', 'aa'])
inverse_vocab = ['aa', 'bb', 'cc', 'dd']
start_positions = [0, 2, 4, 6, 8]
positions = [[0, 4], [1], [2], [3]]
indexed_string = IndexedString(s, tokenizer)
self.assertTrue(np.array_equal(indexed_string.as_np, tokenized_string))
self.assertTrue(
np.array_equal(indexed_string.string_start, start_positions))
self.assertTrue(indexed_string.inverse_vocab == inverse_vocab)
self.assertTrue(np.array_equal(indexed_string.positions, positions))
def test_indexed_string_inverse_removing_regex(self):
s = 'This is a good movie. This is a great movie'
indexed_string = IndexedString(s)
self.assertEqual(s, indexed_string.inverse_removing([]))
def data_labels(self, num_samples, classifier_fn, detection=False):
'''
Steps of this function:
1. generate perturbed text features and image features
2. in a loop, 1) using these features to make instances of perturbed (text, image) pairs,
2) make predictions on these pairs, store labels into 'labels'
3. concatenate text and image features, store into 'data',
also append the original input and prediction of it
4. calculate distances
TODO: add object detection: first run on original image, create feature components,
then run on perturbed images to get corresponding value
:param num_samples:
:param classifier_fn:
:param object_detection:
:return:
'''
''' 1. make text features '''
indexed_string = IndexedString(self.text, bow=True, split_expression=r'\W+', mask_string=None)
domain_mapper = TextDomainMapper(indexed_string)
doc_size = indexed_string.num_words()
sample = self.random_state.randint(1, doc_size + 1, num_samples) # num_samples - 1
data_txt = np.ones((num_samples, doc_size))
# data[0] = np.ones(doc_size)
features_range = range(doc_size)
inverse_data_txt = []
''' 1. make image features '''
random_seed = self.random_state.randint(0, high=1000)
segmentation_fn = SegmentationAlgorithm('quickshift', kernel_size=4,
max_dist=200, ratio=0.2,
random_seed=random_seed)
#segmentation_fn = SegmentationAlgorithm('felzenszwalb', scale=200, sigma=2, min_size=100)
'''segmentation_fn = SegmentationAlgorithm('slic', n_segments=60, compactness=10, sigma=1,
start_label=1)'''
segments = segmentation_fn(self.image) # get segmentation
n_img_features = np.unique(segments).shape[0] # get num of superpixel features
data_img = self.random_state.randint(0, 2, n_img_features * num_samples).reshape(
(num_samples, n_img_features))
data_img_rows = tqdm(data_img)
imgs = []
''' 1. make object detection features
if detection:
predictor, cfg = object_detection_predictor()
ori_label = object_detection_obtain_label(predictor, cfg, self.image)
num_object_detection = ori_label.shape[0]
data_object_detection = np.zeros((num_samples,num_object_detection))'''
# create fudged_image
fudged_image = self.image.copy()
for x in np.unique(segments):
fudged_image[segments == x] = (
np.mean(self.image[segments == x][:, 0]),
np.mean(self.image[segments == x][:, 1]),
np.mean(self.image[segments == x][:, 2]))
# img_features[0, :] = 1 # the first sample is the full image # num_samples
'''2. create data instances and make predictions'''
labels = []
for i, instance in enumerate(zip(sample, data_img_rows)):
size_txt, row_img = instance
# make text instance
inactive = self.random_state.choice(features_range, size_txt,
replace=False)
data_txt[i, inactive] = 0
inverse_data_txt.append(indexed_string.inverse_removing(inactive))
# make image instance
temp = copy.deepcopy(self.image)
zeros = np.where(row_img == 0)[0] # get segment numbers that are turned off in this instance
mask = np.zeros(segments.shape).astype(bool)
for zero in zeros:
mask[segments == zero] = True
temp[mask] = fudged_image[mask]
'''if detection:
label = object_detection_obtain_label(predictor, cfg, temp)
label_diff = compare_labels(ori_label,label)
data_object_detection[i] = label_diff'''
imgs.append(temp)
# make prediction and append result
if len(imgs) == 10:
preds = classifier_fn(self.pred_model, imgs, inverse_data_txt)
labels.extend(preds)
imgs = []
inverse_data_txt = []
if len(imgs) > 0:
preds = classifier_fn(self.pred_model, imgs, inverse_data_txt)
labels.extend(preds)
'''3. concatenate and append features'''
data = np.concatenate((data_txt, data_img), axis=1)
# append the original input to the last
orig_img_f = np.ones((n_img_features,))
orig_txt_f = np.ones(doc_size)
'''if detection:
data = np.concatenate((data, data_object_detection),axis=1)
orig_ot = np.ones(num_object_detection)
data = np.vstack((data, np.concatenate((np.concatenate((orig_txt_f, orig_img_f)),orig_ot))))
else:'''
data = np.vstack((data, np.ones((data.shape[1])))) ###
labels.extend(classifier_fn(self.pred_model, [self.image], [self.text]))
'''4. compute distance# distances[:, :(doc_size-1)] *= 100
use platt scaling t get relative importance of text and image modalities
'''
labels = np.array(labels, dtype=float)
# Modify MMF source code to zero out image / text attributes
#dummy_label_image = np.array(classifier_fn([self.image], [self.text], zero_text=True)) # zero out text
#dummy_label_text = np.array(classifier_fn([self.image], [self.text], zero_image=True)) # zero out image
# perform calibration
try:
labels_for_calib = np.array(labels[:, 0] < 0.5, dtype=float)
calibrated = CalibratedClassifierCV(cv=3)
calibrated.fit(data[:,:doc_size + n_img_features], labels_for_calib)
calib_data = np.ones((3, doc_size + n_img_features), dtype=float)
calib_data[0][:doc_size] = 0 # zero out text
calib_data[1][doc_size:] = 0 # zero out image
calibrated_labels = calibrated.predict_proba(calib_data)
delta_txt = abs(calibrated_labels[-1][0] - calibrated_labels[0][0])
delta_img = abs(calibrated_labels[-1][0] - calibrated_labels[1][0])
ratio_txt_img = max(min(10, delta_txt/delta_img), 0.1)
except:
dummy_text = ""
dummy_image = np.zeros_like(self.image)
try:
label_text_out = np.array(classifier_fn(self.pred_model, [self.image], [self.text], zero_text=True)) # zero out text
label_image_out = np.array(classifier_fn(self.pred_model, [self.image], [self.text], zero_image=True)) # zero out image
except:
label_text_out = np.array(classifier_fn(self.pred_model, [self.image], [dummy_text]))
label_image_out = np.array(classifier_fn(self.pred_model, [dummy_image], [self.text]))
delta_txt = abs(labels[-1][0] - label_text_out[0][0])
delta_img = abs(labels[-1][0] - label_image_out[0][0])
ratio_txt_img = max(min(10, delta_txt / delta_img), 0.1)
# calculate distances
distances_img = sklearn.metrics.pairwise_distances(
data[:, doc_size:],
data[-1, doc_size:].reshape(1, -1),
metric='cosine'
).ravel()
def distance_fn(x):
return sklearn.metrics.pairwise.pairwise_distances(
x, x[-1], metric='cosine').ravel()
distances_txt = distance_fn(sp.sparse.csr_matrix(data[:, :doc_size]))
distances = 1/(1 + ratio_txt_img) * distances_img + (1 - 1/(1 + ratio_txt_img)) * distances_txt
# As required by lime_base, make the first element of data, labels, distances the original data point
data[0] = data[-1]
labels[0] = labels[-1]
distances[0] = distances[-1]
'''if not detection:'''
num_object_detection = 0
ori_label = None
return data, labels, distances, doc_size, n_img_features, \
segments, domain_mapper, num_object_detection, ori_label, ratio_txt_img
def explain_instance(self,
text_instance,
classifier_fn,
labels=(1, ),
top_labels=None,
num_features=25,
num_samples=5000,
distance_metric='cosine',
model_regressor=None):
"""Generates explanations for a prediction.
First, we generate neighborhood data by randomly hiding features from
the instance (see __data_labels_distance_mapping). We then learn
locally weighted linear models on this neighborhood data to explain
each of the classes in an interpretable way (see lime_base.py).
Args:
text_instance: raw tabular data to be explained.
classifier_fn: classifier prediction probability function, which
takes a list of d strings and outputs a (d, k) numpy array with
prediction probabilities, where k is the number of classes.
For ScikitClassifiers , this is classifier.predict_proba.
labels: iterable with labels to be explained.
top_labels: if not None, ignore labels and produce explanations for
the K labels with highest prediction probabilities, where K is
this parameter.
num_features: maximum number of features present in explanation
num_samples: size of the neighborhood to learn the linear model
distance_metric: the distance metric to use for sample weighting,
defaults to cosine similarity
model_regressor: sklearn regressor to use in explanation. Defaults
to Ridge regression in LimeBase. Must have model_regressor.coef_
and 'sample_weight' as a parameter to model_regressor.fit()
Returns:
An Explanation object (see explanation.py) with the corresponding
explanations.
"""
indexed_string = (IndexedCharacters(
text_instance, bow=self.bow, mask_string=self.mask_string)
if self.char_level else IndexedString(
text_instance,
bow=self.bow,
split_expression=self.split_expression,
mask_string=self.mask_string))
domain_mapper = TextDomainMapper(
indexed_string) #change tabel data to string
data, yss, distances = self.__data_labels_distances(
indexed_string,
classifier_fn,
num_samples,
distance_metric=distance_metric)
print('data shape: ', data.shape) #(5000, 63)
print('yss shape: ', yss.shape) #(5000,2)
print('distances shape: ', distances.shape) #(5000,)
if self.class_names is None:
self.class_names = [str(x) for x in range(yss[0].shape[0])]
ret_exp = explanation.Explanation(domain_mapper=domain_mapper,
class_names=self.class_names,
random_state=self.random_state)
ret_exp.predict_proba = yss[0]
if top_labels:
labels = np.argsort(yss[0])[-top_labels:]
ret_exp.top_labels = list(labels)
ret_exp.top_labels.reverse()
for label in labels:
(ret_exp.intercept[label], ret_exp.local_exp[label], ret_exp.score,
ret_exp.local_pred) = self.base.explain_instance_with_data(
data,
yss,
distances,
label,
num_features,
model_regressor=model_regressor,
feature_selection=self.feature_selection)
return ret_exp
def data_labels(self, num_samples, classifier_fn, detection=False):
"""
Steps of this function:
1. generate perturbed text features and image features
2. in a loop, 1) using these features to make instances of perturbed (text, image) pairs,
2) make predictions on these pairs, store labels into 'labels'
3. concatenate text and image features, store into 'data',
also append the original input and prediction of it
4. calculate distances
Arguments:
classifier_fn: classification function to give predictions for given texts and images
num_samples: size of the neighborhood to learn the linear model
detection: Whether object detection method is invoked, default to be false
Return:
data: dense num_samples * num_superpixels
labels: prediction probabilities matrix
distances:distance including text/image distance ratio where
text and image distance are cosine distances between the original instance and
each perturbed instance (computed in the binary 'data'
matrix), times 100.
doc_size: number of words in indexed string, where indexed string is the string with various indexes
n_img_features: number of superpixels to include in explanation
segments:2d numpy array, with the output from skimage.segmentation
domain_mapper:Maps text feature ids to words or word-positions
num_object_detection:number of detected objects to include in explanation
ori_label: numpy including deteced objects in the original image
ratio_txt_img: weight ratio between text and image features
"""
""" 1. make text features """
indexed_string = IndexedString(
self.text, bow=True, split_expression=r"\W+", mask_string=None
)
domain_mapper = TextDomainMapper(indexed_string)
doc_size = indexed_string.num_words()
sample = self.random_state.randint(
1, doc_size + 1, num_samples
) # num_samples - 1
data_txt = np.ones((num_samples, doc_size))
# data[0] = np.ones(doc_size)
features_range = range(doc_size)
inverse_data_txt = []
""" 1. make image features """
random_seed = self.random_state.randint(0, high=1000)
segmentation_fn = SegmentationAlgorithm(
"quickshift",
kernel_size=4,
max_dist=200,
ratio=0.2,
random_seed=random_seed,
)
# segmentation_fn = SegmentationAlgorithm('felzenszwalb', scale=200, sigma=2, min_size=100)
"""segmentation_fn = SegmentationAlgorithm('slic', n_segments=60, compactness=10, sigma=1,
start_label=1)"""
segments = segmentation_fn(self.image) # get segmentation
n_img_features = np.unique(segments).shape[0] # get num of superpixel features
data_img = self.random_state.randint(
0, 2, n_img_features * num_samples
).reshape((num_samples, n_img_features))
data_img_rows = tqdm(data_img)
imgs = []
""" 1. make object detection features
if detection:
predictor, cfg = object_detection_predictor()
ori_label = object_detection_obtain_label(predictor, cfg, self.image)
num_object_detection = ori_label.shape[0]
data_object_detection = np.zeros((num_samples,num_object_detection))"""
# create fudged_image
fudged_image = self.image.copy()
for x in np.unique(segments):
fudged_image[segments == x] = (
np.mean(self.image[segments == x][:, 0]),
np.mean(self.image[segments == x][:, 1]),
np.mean(self.image[segments == x][:, 2]),
)
# img_features[0, :] = 1 # the first sample is the full image # num_samples
"""2. create data instances and make predictions"""
labels = []
for i, instance in enumerate(zip(sample, data_img_rows)):
size_txt, row_img = instance
# make text instance
inactive = self.random_state.choice(features_range, size_txt, replace=False)
data_txt[i, inactive] = 0
inverse_data_txt.append(indexed_string.inverse_removing(inactive))
# make image instance
temp = copy.deepcopy(self.image)
zeros = np.where(row_img == 0)[
0
] # get segment numbers that are turned off in this instance
mask = np.zeros(segments.shape).astype(bool)
for zero in zeros:
mask[segments == zero] = True
temp[mask] = fudged_image[mask]
"""if detection:
label = object_detection_obtain_label(predictor, cfg, temp)
label_diff = compare_labels(ori_label,label)
data_object_detection[i] = label_diff"""
imgs.append(temp)
# make prediction and append result
if len(imgs) == 10:
preds = classifier_fn(self.pred_model, imgs, inverse_data_txt)
labels.extend(preds)
imgs = []
inverse_data_txt = []
if len(imgs) > 0:
preds = classifier_fn(self.pred_model, imgs, inverse_data_txt)
labels.extend(preds)
"""3. concatenate and append features"""
data = np.concatenate((data_txt, data_img), axis=1)
# append the original input to the last
orig_img_f = np.ones((n_img_features,))
orig_txt_f = np.ones(doc_size)
"""if detection:
data = np.concatenate((data, data_object_detection),axis=1)
orig_ot = np.ones(num_object_detection)
data = np.vstack((data, np.concatenate((np.concatenate((orig_txt_f, orig_img_f)),orig_ot))))
else:"""
data = np.vstack((data, np.ones((data.shape[1])))) ###
labels.extend(classifier_fn(self.pred_model, [self.image], [self.text]))
"""4. compute distance# distances[:, :(doc_size-1)] *= 100
use platt scaling t get relative importance of text and image modalities
"""
labels = np.array(labels, dtype=float)
# Modify MMF source code to zero out image / text attributes
# dummy_label_image = np.array(classifier_fn([self.image], [self.text], zero_text=True)) # zero out text
# dummy_label_text = np.array(classifier_fn([self.image], [self.text], zero_image=True)) # zero out image
# perform calibration
try:
labels_for_calib = np.array(labels[:, 0] < 0.5, dtype=float)
calibrated = CalibratedClassifierCV(cv=3)
calibrated.fit(data[:, : doc_size + n_img_features], labels_for_calib)
calib_data = np.ones((3, doc_size + n_img_features), dtype=float)
calib_data[0][:doc_size] = 0 # zero out text
calib_data[1][doc_size:] = 0 # zero out image
calibrated_labels = calibrated.predict_proba(calib_data)
delta_txt = abs(calibrated_labels[-1][0] - calibrated_labels[0][0])
delta_img = abs(calibrated_labels[-1][0] - calibrated_labels[1][0])
ratio_txt_img = max(min(100, delta_txt / delta_img), 0.01)
except:
dummy_text = ""
dummy_image = np.zeros_like(self.image)
label_text_out = np.array(
classifier_fn(
self.pred_model, [self.image], [self.text], zero_text=True
)
) # zero out text
label_image_out = np.array(
classifier_fn(
self.pred_model, [self.image], [self.text], zero_image=True
)
) # zero out image
delta_txt = abs(labels[-1][0] - label_text_out[0][0])
delta_img = abs(labels[-1][0] - label_image_out[0][0])
ratio_txt_img = max(min(10, delta_txt / delta_img), 0.1)
# calculate distances
distances_img = sklearn.metrics.pairwise_distances(
data[:, doc_size:], data[-1, doc_size:].reshape(1, -1), metric="cosine"
).ravel()
def distance_fn(x):
return sklearn.metrics.pairwise.pairwise_distances(
x, x[-1], metric="cosine"
).ravel()
distances_txt = distance_fn(sp.sparse.csr_matrix(data[:, :doc_size]))
distances = (
1 / (1 + ratio_txt_img) * distances_img
+ (1 - 1 / (1 + ratio_txt_img)) * distances_txt
)
# As required by lime_base, make the first element of data, labels, distances the original data point
data[0] = data[-1]
labels[0] = labels[-1]
distances[0] = distances[-1]
"""if not detection:"""
num_object_detection = 0
ori_label = None
return (
data,
labels,
distances,
doc_size,
n_img_features,
segments,
domain_mapper,
num_object_detection,
ori_label,
ratio_txt_img,
)
def test_indexed_string_inverse_removing_regex(self):
s = 'This is a good movie. This is a great movie'
indexed_string = IndexedString(s)
self.assertEqual(s, indexed_string.inverse_removing([]))