def layer_conductance(net, test_input_tensor):
"""
To use Layer Conductance, we create a LayerConductance object passing in the model as well as the module (layer) whose output we would like to understand.
In this case, we choose net.sigmoid1, the output of the first hidden layer.
Now obtain the conductance values for all the test examples by calling attribute on the LayerConductance object.
LayerConductance also requires a target index for networks with mutliple outputs, defining the index of the output for which gradients are computed.
Similar to feature attributions, we provide target = 1, corresponding to survival.
LayerConductance also utilizes a baseline, but we simply use the default zero baseline as in integrated gradients.
"""
cond = LayerConductance(net, net.sigmoid1)
cond_vals = cond.attribute(test_input_tensor, target=1)
cond_vals = cond_vals.detach().numpy()
# We can begin by visualizing the average conductance for each neuron.
neuron_names = ["neuron " + str(x) for x in range(12)]
avg_neuron_imp, neuron_imp_dict = visualize_importances(
neuron_names,
np.mean(cond_vals, axis=0),
title="Average Neuron Importances",
axis_title="Neurons",
)
mlflow.log_metrics(neuron_imp_dict)
mlflow.log_text(str(avg_neuron_imp), "neuron_imp_summary.txt")
# We can also look at the distribution of each neuron's attributions. Below we look at the distributions for neurons 7 and 9,
# and we can confirm that their attribution distributions are very close to 0, suggesting they are not learning substantial features.
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(9, 6))
fig.tight_layout(pad=3)
ax1.hist(cond_vals[:, 9], 100)
ax1.set(title="Neuron 9 Distribution")
ax2.hist(cond_vals[:, 7], 100)
ax2.set(title="Neuron 7 Distribution")
mlflow.log_figure(fig, "Neurons_Distribution.png")
def layer_conductance_pruning(self, p, n_neurons):
_layer_name = p[0].split(".")
if len(_layer_name) == 3:
layer_name = _layer_name[0] + '[' + _layer_name[1] + ']'
elif len(_layer_name) == 2:
layer_name = _layer_name[0]
if len(p[1].data.size()) != 1:
cond = LayerConductance(self.new_model,
eval('self.new_model.' + layer_name))
cond_vals = cond.attribute(self.test_data, target=self.test_target)
cond_vals = np.abs(cond_vals.cpu().detach().numpy())
neuron_values = np.nanmean(cond_vals, axis=0)
# Do we really need visualization?
# visualize_importances(cond_vals.shape[1], neuron_values, p[0] + '{}'.format(time.time()))
try:
prune_idx = np.argpartition(np.array(neuron_values),
-(p[1].data.size()[0] - n_neurons))
prune_idx = prune_idx[-(p[1].data.size()[0] - n_neurons):]
except:
prune_idx = list(range(p[1].data.size()[0]))
#print("Neurons Retained", len(prune_idx))
else:
prune_idx = []
return prune_idx
def layer_importance(self, strategy):
layer_importance_list = []
if strategy == "l1_norm_pruning":
for p in self.new_model.parameters():
if len(p.data.size()) != 1:
normed_weights = p.data.abs()
layer_importance_list.append(
torch.mean(normed_weights).item())
return layer_importance_list
elif strategy == "layer_conductance_pruning":
for p in self.new_model.named_parameters():
_layer_name = p[0].split(".")
if len(_layer_name) == 3:
layer_name = _layer_name[0] + '[' + _layer_name[1] + ']'
elif len(_layer_name) == 2:
layer_name = _layer_name[0]
if len(p[1].data.size()) != 1:
cond = LayerConductance(
self.new_model, eval('self.new_model.' + layer_name))
cond_vals = cond.attribute(self.test_data,
target=self.test_target)
cond_vals = np.abs(cond_vals.cpu().detach().numpy())
layer_importance_val = np.nanmean(cond_vals)
layer_importance_list.append(layer_importance_val)
return layer_importance_list
def layer_conductance_growing(self):
# Number of numbers to add per layer, list of integres
self.number_neurons_per_layer = []
meaned_cond_layer = []
neurons_per_layer = []
for p in [x for x in self.new_model.named_parameters()
][:-2]: # skip output layer
if len(p[1].data.size()) != 1: # skip biases
_layer_name = p[0].split(".")
if len(_layer_name) == 3:
layer_name = _layer_name[0] + '[' + _layer_name[1] + ']'
elif len(_layer_name) == 2:
layer_name = _layer_name[0]
neurons_per_layer.append(p[1].data.shape[0])
cond = LayerConductance(self.new_model,
eval('self.new_model.' + layer_name))
cond_vals = cond.attribute(self.test_data,
target=self.test_target)
cond_vals = cond_vals.cpu().detach().numpy()
layer_value = np.nanmean(np.absolute(cond_vals))
meaned_cond_layer.append(layer_value)
# dont take into account output layer
total_number_neurons = np.nansum(neurons_per_layer)
total_cond = np.nansum(meaned_cond_layer)
add_per_layer = []
for x in meaned_cond_layer:
try:
add_p_layer = int(
round((x / total_cond) * total_number_neurons *
self.growing_perc, 0))
except:
print(
'Could not calculate layer cond value, so 0 used, parameters:'
)
print('total_cond ', total_cond)
print('total number neurons ', total_number_neurons)
print('cond of layer ', x)
add_p_layer = 0
# send_slack_message('growing except has occured')
add_per_layer.append(add_p_layer)
self.number_neurons_per_layer = add_per_layer
print('Test AUROC: {}\n'.format(test_auc))
print('Test AUPR: {}\n'.format(test_aupr))
# Run some simple explanations using Captum
if args.normal:
plotter = VisdomLinePlotter('aa-normal')
type = 'Normal'
else:
plotter = VisdomLinePlotter('aa-adversarial')
type = 'Adversarial'
print("\n============================================== Explanations")
# Let's try looking at the final layer of the model
cond = LayerConductance(model, model.output)
batch_size = 256
# Reset our dataloader
test_loader = DataLoader(dataset=test_set,
batch_size=batch_size,
shuffle=False,
collate_fn=visit_collate_fn,
num_workers=0)
# Get an input batch from our dataset
input_batch = next(iter(test_loader))
inputs, targets, lengths, _ = input_batch
input_var = torch.autograd.Variable(inputs)
target_var = torch.autograd.Variable(targets)
token_type_ids, ref_token_type_ids = construct_input_ref_token_type_pair(
input_ids, len(input_text_ids))
attention_mask = torch.ones_like(input_ids)
input_embeddings = interpretable_embedding.indices_to_embeddings(
input_ids, token_type_ids=token_type_ids, position_ids=position_ids)
ref_input_embeddings = interpretable_embedding.indices_to_embeddings(
ref_input_ids,
token_type_ids=ref_token_type_ids,
position_ids=ref_position_ids)
layer_attrs_start = []
for i in range(len(clone.model.bert.encoder.layer)):
print("2", i, "-" * 100)
lc = LayerConductance(squad_pos_forward_func,
clone.model.bert.encoder.layer[i])
layer_attributions_start = lc.attribute(
inputs=input_embeddings,
baselines=ref_input_embeddings,
additional_forward_args=(token_type_ids, attention_mask))[0]
layer_attrs_start.append(
summarize_attributions(
layer_attributions_start).cpu().detach().tolist())
all_tokens = original_tokenizer.tokenizer.convert_ids_to_tokens(input_ids[0])
fig, ax = plt.subplots(figsize=(15, 5))
xticklabels = all_tokens
yticklabels = list(range(1, 13))
ax = sns.heatmap(np.array(layer_attrs_start),
def captum_heatmap_interactive(request):
if request.method == 'POST':
STORED_POSTS = request.session.get("TextAttackResult")
form = CustomData(request.POST)
if form.is_valid():
input_text, model_name, recipe_name = form.cleaned_data[
'input_text'], form.cleaned_data[
'model_name'], form.cleaned_data['recipe_name']
found = False
if STORED_POSTS:
JSON_STORED_POSTS = json.loads(STORED_POSTS)
for idx, el in enumerate(JSON_STORED_POSTS):
if el["type"] == "heatmap" and el[
"input_string"] == input_text:
tmp = JSON_STORED_POSTS.pop(idx)
JSON_STORED_POSTS.insert(0, tmp)
found = True
break
if found:
request.session["TextAttackResult"] = json.dumps(
JSON_STORED_POSTS[:10])
return HttpResponseRedirect(reverse('webdemo:index'))
original_model = transformers.AutoModelForSequenceClassification.from_pretrained(
"textattack/" + model_name)
original_tokenizer = textattack.models.tokenizers.AutoTokenizer(
"textattack/" + model_name)
model = textattack.models.wrappers.HuggingFaceModelWrapper(
original_model, original_tokenizer)
device = torch.device(
"cuda:2" if torch.cuda.is_available() else "cpu")
clone = deepcopy(model)
clone.model.to(device)
def calculate(input_ids, token_type_ids, attention_mask):
return clone.model(input_ids, token_type_ids,
attention_mask)[0]
interpretable_embedding = configure_interpretable_embedding_layer(
clone.model, 'bert.embeddings')
ref_token_id = original_tokenizer.tokenizer.pad_token_id
sep_token_id = original_tokenizer.tokenizer.sep_token_id
cls_token_id = original_tokenizer.tokenizer.cls_token_id
def summarize_attributions(attributions):
attributions = attributions.sum(dim=-1).squeeze(0)
attributions = attributions / torch.norm(attributions)
return attributions
def construct_attention_mask(input_ids):
return torch.ones_like(input_ids)
def construct_input_ref_pos_id_pair(input_ids):
seq_length = input_ids.size(1)
position_ids = torch.arange(seq_length,
dtype=torch.long,
device=device)
ref_position_ids = torch.zeros(seq_length,
dtype=torch.long,
device=device)
position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
ref_position_ids = ref_position_ids.unsqueeze(0).expand_as(
input_ids)
return position_ids, ref_position_ids
def squad_pos_forward_func(inputs,
token_type_ids=None,
attention_mask=None):
pred = calculate(inputs, token_type_ids, attention_mask)
return pred.max(1).values
def construct_input_ref_token_type_pair(input_ids, sep_ind=0):
seq_len = input_ids.size(1)
token_type_ids = torch.tensor(
[[0 if i <= sep_ind else 1 for i in range(seq_len)]],
device=device)
ref_token_type_ids = torch.zeros_like(token_type_ids,
device=device) # * -1
return token_type_ids, ref_token_type_ids
input_text_ids = original_tokenizer.tokenizer.encode(
input_text, add_special_tokens=False)
input_ids = [cls_token_id] + input_text_ids + [sep_token_id]
input_ids = torch.tensor([input_ids], device=device)
position_ids, ref_position_ids = construct_input_ref_pos_id_pair(
input_ids)
ref_input_ids = [
cls_token_id
] + [ref_token_id] * len(input_text_ids) + [sep_token_id]
ref_input_ids = torch.tensor([ref_input_ids], device=device)
token_type_ids, ref_token_type_ids = construct_input_ref_token_type_pair(
input_ids, len(input_text_ids))
attention_mask = torch.ones_like(input_ids)
input_embeddings = interpretable_embedding.indices_to_embeddings(
input_ids,
token_type_ids=token_type_ids,
position_ids=position_ids)
ref_input_embeddings = interpretable_embedding.indices_to_embeddings(
ref_input_ids,
token_type_ids=ref_token_type_ids,
position_ids=ref_position_ids)
layer_attrs_start = []
for i in range(len(clone.model.bert.encoder.layer)):
lc = LayerConductance(squad_pos_forward_func,
clone.model.bert.encoder.layer[i])
layer_attributions_start = lc.attribute(
inputs=input_embeddings,
baselines=ref_input_embeddings,
additional_forward_args=(token_type_ids,
attention_mask))[0]
layer_attrs_start.append(
summarize_attributions(
layer_attributions_start).cpu().detach().tolist())
all_tokens = original_tokenizer.tokenizer.convert_ids_to_tokens(
input_ids[0])
fig, ax = plt.subplots(figsize=(15, 5))
xticklabels = all_tokens
yticklabels = list(range(1, 13))
ax = sns.heatmap(np.array(layer_attrs_start),
xticklabels=xticklabels,
yticklabels=yticklabels,
linewidth=0.2)
plt.xlabel('Tokens')
plt.ylabel('Layers')
buf = io.BytesIO()
fig.savefig(buf, format='png')
buf.seek(0)
bufferString = base64.b64encode(buf.read())
imageUri = urllib.parse.quote(bufferString)
post = {
"type": "heatmap",
"input_string": input_text,
"model_name": model_name,
"recipe_name": recipe_name,
"image": imageUri,
}
if STORED_POSTS:
JSON_STORED_POSTS = json.loads(STORED_POSTS)
JSON_STORED_POSTS.insert(0, post)
request.session["TextAttackResult"] = json.dumps(
JSON_STORED_POSTS[:10])
else:
request.session["TextAttackResult"] = json.dumps([post])
return HttpResponseRedirect(reverse('webdemo:index'))
else:
return HttpResponseNotFound('Failed')
return HttpResponse('Success')
return HttpResponseNotFound('<h1>Not Found</h1>')
test_input_tensor = torch.from_numpy(X_test).type(torch.FloatTensor)
attr, delta = ig.attribute(test_input_tensor, return_convergence_delta=True)
attr = attr.detach().numpy()
importances = dict(zip(feature_names, np.mean(abs(attr), axis=0)))
outFile = '{0}_FeatureImportance.csv'.format(args.infile[0].split('.')[0])
print('Saving SNP and TF importance to file {}'.format(outFile))
with open(outFile, 'w') as f:
for key in importances.keys():
f.write("%s,%s\n"%(key,importances[key]))
print('Interpreting gene importance...')
cond = LayerConductance(model, model.GRNeQTL)
cond_vals = cond.attribute(test_input_tensor)
cond_vals = cond_vals.detach().numpy()
importances_layer1 = dict(zip(adj.columns.tolist(), np.mean(abs(cond_vals), axis=0)))
outFile = '{0}_GeneImportance.csv'.format(args.infile[0].split('.')[0])
print('Saving gene importance to file {}'.format(outFile))
with open(outFile, 'w') as f:
for key in importances_layer1.keys():
f.write("%s,%s\n"%(key,importances_layer1[key]))
neuron_cond = NeuronConductance(model, model.GRNeQTL)
def plot_signal_attribution_spec_sum(point, code, unit):
point = dataset[labels == code][point]
color_index = 0
colors = list({'r', 'g', 'b'})
fig = plt.figure(figsize=(40, 40))
ax = fig.add_subplot(8, 8, i + 1)
points = np.asarray(pca_result)
deltas = points - point
dist_2 = np.einsum('ij,ij->i', deltas, deltas)
centroid = np.argmin(dist_2)
print(y[centroid].item())
ax.set_title(f"Code: {code}" + " - Label " + str(y[centroid].item()))
for channel, label in zip(
torch.sum(X[centroid], dim=0).unsqueeze(0), channel_labels):
ax.plot(channel, color=colors[2], label="Motion")
color_index += 1
plt.legend()
input = X[centroid]
attr_algo = LayerConductance(model, modules["conv8"])
attributions = attr_algo.attribute(input.unsqueeze(0).cuda(),
attribute_to_layer_input=False)
# print(attributions.shape)
m = torch.nn.functional.upsample(attributions,
size=4000,
mode='linear',
align_corners=True)
# print(m)
attributions = (m[0][unit].cpu().detach().numpy() > 0.001)
# print(attributions)
s, e = get_window(attributions)
ax = fig.add_subplot(211)
# ax.plot(sum(input))
powerSpectrum, freqenciesFound, time, imageAxis = ax.specgram(
torch.sum(X[centroid], dim=0))
ax.set_xlabel('Time')
ax.set_ylabel('Frequency')
rect = Rectangle((s, -30),
e - s,
60,
color='red',
fc=(1, 0, 0, 0.2),
ec=(0, 0, 0, 1))
ax.add_patch(rect)
# plt.ylim((-15,15))
plt.show()
### Neuron Conductance if Layer Conductance is promising
### Run Clustering over vectors of neurons ?
from operator import itemgetter
from captum.attr import LayerActivation, LayerConductance, LayerGradientXActivation, NeuronConductance
from tqdm.notebook import tqdm
from matplotlib.patches import Rectangle
model = model.cuda()
activations = {}
count = 0
attribution_algos = []
for layer in modules:
attribution_algos.append(LayerConductance(model, modules[layer]))
# for idx in tqdm(range(len(X))):
# count+=1
# input = X[idx]
# attributions = attr_algo.attribute(input.unsqueeze(0).cuda(),attribute_to_layer_input=True)
# print(attributions[0].shape)
fig = plt.figure()
for attr_algo in attribution_algos:
input = X[100]
attributions = attr_algo.attribute(input.unsqueeze(0).cuda(),
attribute_to_layer_input=True)
attributions = (sum(
# Try out different layers
# and see observe how the attributions change
layer = model.features[3]
layer_act = LayerActivation(model, layer)
layer_act_attr = compute_attributions(layer_act, X_tensor)
layer_act_attr_sum = layer_act_attr.mean(axis=1, keepdim=True)
# Layer gradcam aggregates across all channels
layer_gradcam = LayerGradCam(model, layer)
layer_gradcam_attr = compute_attributions(layer_gradcam,
X_tensor,
target=y_tensor,
relu_attributions=True)
layer_gradcam_attr_sum = layer_gradcam_attr.mean(axis=1, keepdim=True)
layer_gradcam_attr_sum = layer_gradcam_attr_sum.permute(1, 0, 2, 3)
visualize_attr_maps('visualization/layer_gradcam.png', X, y, class_names,
layer_gradcam_attr_sum, ['layer_gradcam'],
lambda attr: attr.detach().numpy())
layer_conduct = LayerConductance(model, layer)
layer_conduct_attr = compute_attributions(layer_conduct,
X_tensor,
target=y_tensor)
layer_conduct_attr_sum = layer_conduct_attr.mean(axis=1, keepdim=True)
layer_conduct_attr_sum = layer_conduct_attr_sum.permute(1, 0, 2, 3)
visualize_attr_maps('visualization/layer_conduct.png', X, y, class_names,
layer_conduct_attr_sum, ['layer_conductance'],
lambda attr: attr.detach().numpy())