def get_attention(model_file, sentence):
model_type = 'bert'
model_version = 'bert-base-uncased'
do_lower_case = True
model = BertForSequenceClassification.from_pretrained(
'bert-base-uncased', output_attentions=True)
model = torch.nn.DataParallel(model)
chpt = torch.load(model_file, map_location=torch.device('cpu'))
model.load_state_dict(chpt['state_dict'])
tokenizer = BertTokenizer.from_pretrained(model_version,
do_lower_case=True)
tokens = [tokenizer.cls_token
] + tokenizer.tokenize(sentence) + [tokenizer.sep_token]
input_ids = tokenizer.convert_tokens_to_ids(tokens)
input_ids = torch.tensor(input_ids).unsqueeze(0)
##head view
model.eval()
outputs = model(input_ids)
attentions = outputs[-1]
head_view(attentions, tokens, None)
####neuron view
show(model, model_type, tokenizer, sentence, None)
model.parameters(), lr=LR, correct_bias=False
) # To reproduce BertAdam specific behavior set correct_bias=False
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=NUM_WARMUP_STEPS,
num_training_steps=NUM_TRAIN_STEPS) # PyTorch scheduler
model.train()
batch = next(iter(train_loader))
input_ids, input_mask, segment_ids, labels = batch
outputs = model(input_ids, input_mask, segment_ids, labels=labels)
loss, logits, attentions = outputs
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
tokens = tokenizer.convert_ids_to_tokens(input_ids.numpy()[0])
tokens = [str(item) for item in tokens]
head_view(attentions, tokens, None)
####neuron view
from bertviz.transformers_neuron_view import BertModel, BertTokenizer
from bertviz.neuron_view import show
model_type = 'bert'
model_version = 'bert-base-uncased'
do_lower_case = True
model = BertModel.from_pretrained(model_version)
tokenizer = BertTokenizer.from_pretrained(model_version,
do_lower_case=do_lower_case)
sentence_a = "date of birth: sex: fservice: medicineallergies:patient recorded as having no known allergies to drugsattending:chief complaint:hematemesismajor surgical or invasive procedure:banding x 4 of esophageal variceshistory of present illness:pt is a 74yo woman with pmh of ms, autoimmune hepatitis"
show(model, model_type, tokenizer, sentence_a, None)
home = expanduser("~")
TEACHER_PATH = os.path.join(home, "Downloads/reasoner_00.model")
UNIFIER_PATH = os.path.join(home, "Downloads/unification_01.model")
device = 'cpu'
model = BackwardChainingInference().to(device)
model.load_state_dict(torch.load(UNIFIER_PATH, map_location=device))
model.teacher.load_state_dict(torch.load(TEACHER_PATH, map_location=device))
model_type = 'bert'
model_version = "bert-base-uncased"
config = BertConfig.from_pretrained(model_version, output_attentions=True)
unification_custom = BertModel.from_pretrained(model_version, config=config)
unification_custom.load_state_dict(model.unification.model.state_dict())
model.unification.model = unification_custom
context = "The dog is blue. If someone is nice and they chase the lion then they visit the bear. If someone chases the dog then they visit the bear. If the bear visits the lion then the lion needs the dog. If someone needs the lion and the lion visits the bear then the lion chases the dog. If someone visits the bear then the bear visits the dog. The bear chases the dog. If someone chases the lion and they visit the bear then the lion visits the dog. The lion visits the dog."
question = "The bear visits the dog." # True
context_question_embeds, context_embeds, masks, types = model.unification.data_prep(
[context], [question])
attn_data = model.unification.model(
inputs_embeds=context_question_embeds.to(model.unification.device),
attention_mask=masks.to(model.unification.device),
token_type_ids=types.to(model.unification.device))[-1]
show(attn_data,
model_type=model_type,
tokenizer=model.unification.tokenizer,
sentence_a=context,
sentence_b=question)