def fit_locality(args, tokenizer, model, inputs, feature):
tokens = feature.tokens
inputs['return_kl'] = False
full_input_ids = inputs.pop('input_ids')
full_positioin_ids = create_position_ids_from_input_ids(
full_input_ids, tokenizer.pad_token_id).to(full_input_ids.device)
# fix position id
inputs['position_ids'] = full_positioin_ids
# fix cls ? maybe
score_fn = partial(predict_with_mask,
tokenizer=tokenizer,
model=model,
base_inputs=inputs,
full_input_ids=full_input_ids)
if args.variant == 'lime':
np_attribution = run_lime_attribution(args, len(tokens),
score_fn).reshape((1, -1))
elif args.variant == 'shap':
np_attribution = run_shap_attribution(args, len(tokens),
score_fn).reshape((1, -1))
else:
raise RuntimeError('Variant must be shap or lime')
return torch.from_numpy(np_attribution)
def forward(self,
input_ids=None,
lang_ids=None,
token_type_ids=None,
position_ids=None,
inputs_embeds=None):
if position_ids is None:
if input_ids is not None:
# Create the position ids from the input token ids. Any padded tokens remain padded.
position_ids = create_position_ids_from_input_ids(
input_ids, self.padding_idx).to(input_ids.device)
else:
position_ids = self.create_position_ids_from_inputs_embeds(
inputs_embeds)
# Copied from transformers.modeling_bert.BertEmbeddings.forward
if input_ids is not None:
input_shape = input_ids.size()
else:
input_shape = inputs_embeds.size()[:-1]
seq_length = input_shape[1]
if position_ids is None:
position_ids = self.position_ids[:, :seq_length]
if token_type_ids is None:
token_type_ids = torch.zeros(input_shape,
dtype=torch.long,
device=self.position_ids.device)
if lang_ids is None:
lang_ids = torch.ones(input_shape,
dtype=torch.long,
device=self.position_ids.device)
elif lang_ids.size(1) == 1:
lang_ids = lang_ids.expand(input_shape)
else:
assert lang_ids.size() == input_shape
if inputs_embeds is None:
inputs_embeds = self.word_embeddings(input_ids)
position_embeddings = self.position_embeddings(position_ids)
token_type_embeddings = self.token_type_embeddings(token_type_ids)
lang_embeddings = self.lang_embeddings(lang_ids)
embeddings = inputs_embeds + position_embeddings + token_type_embeddings + lang_embeddings
embeddings = self.LayerNorm(embeddings)
embeddings = self.dropout(embeddings)
return embeddings
def test_create_position_ids_respects_padding_index(self):
""" Ensure that the default position ids only assign a sequential . This is a regression
test for https://github.com/huggingface/transformers/issues/1761
The position ids should be masked with the embedding object's padding index. Therefore, the
first available non-padding position index is RobertaEmbeddings.padding_idx + 1
"""
config = self.model_tester.prepare_config_and_inputs()[0]
model = RobertaEmbeddings(config=config)
input_ids = torch.as_tensor([[12, 31, 13, model.padding_idx]])
expected_positions = torch.as_tensor(
[[0 + model.padding_idx + 1, 1 + model.padding_idx + 1, 2 + model.padding_idx + 1, model.padding_idx]]
)
position_ids = create_position_ids_from_input_ids(input_ids, model.padding_idx)
self.assertEqual(position_ids.shape, expected_positions.shape)
self.assertTrue(torch.all(torch.eq(position_ids, expected_positions)))
def token_masking_feat_interaction(args, tokenizer, model, inputs, feature):
merged_tokens, segments = merge_tokens_into_words(tokenizer, feature)
inputs['return_kl'] = False
full_input_ids = inputs.pop('input_ids')
full_positioin_ids = create_position_ids_from_input_ids(full_input_ids, tokenizer.pad_token_id).to(full_input_ids.device)
# fix position id
inputs['position_ids'] = full_positioin_ids
# fix cls ? maybe
zero_input_ids = tokenizer.pad_token_id * torch.ones_like(full_input_ids)
full_prob = model.probe_forward(**inputs, input_ids=full_input_ids)
zero_prob = model.probe_forward(**inputs, input_ids=zero_input_ids)
mask_t_probs = [masked_token_prob(tokenizer, model, inputs, full_input_ids, c_range) for c_range in segments]
kept_t_probs = [kept_token_prob(tokenizer, model, inputs, full_input_ids, c_range) for c_range in segments]
# arch_detect
interaction_importance = []
for i in range(len(segments)):
range0 = segments[i]
for j in range(i + 1, len(segments)):
range1 = segments[j]
# full context
mask_t0_t1_prob = masked_token_prob(tokenizer, model, inputs, full_input_ids, range0, range1)
full_importance = full_prob + mask_t0_t1_prob - mask_t_probs[i] - mask_t_probs[j]
# zero context
kept_t0_t1_prob = kept_token_prob(tokenizer, model, inputs, full_input_ids, range0, range1)
zero_importance = kept_t0_t1_prob + zero_prob - kept_t_probs[i] - kept_t_probs[j]
importance = (full_importance + zero_importance) / 2
report = {'full': full_prob, 'mask_i': mask_t_probs[i], 'mask_j': mask_t_probs[j], 'mask_i_j': mask_t0_t1_prob,
'zero': zero_prob, 'keep_i': kept_t_probs[i], 'keep_j': kept_t_probs[j], 'keep_i_j': kept_t0_t1_prob}
interaction_importance.append((i, j, importance, report))
interaction_importance.sort(key=lambda x: abs(x[2]), reverse=True)
interaction_importance = [x for x in interaction_importance]
return interaction_importance
def _get_origin_position_ids(self, input_ids):
return create_position_ids_from_input_ids(
input_ids, self.config.pad_token_id).to(input_ids.device)