def save_attn_weights(model, bert_version, output_path, prefix):
tokenizer = BertTokenizer.from_pretrained(bert_version)
filename = '/scratch/sb6416/Ling3340/extract_tree/UD_English-PUD/en_pud-ud-test.conllu'
with open(filename, 'r') as f_wsj:
data = json.load(f_wsj)
attention = {}
model.eval()
with torch.no_grad():
for key, datum in tqdm.tqdm(data.items(), total=len(data)):
sent = datum['sentence']
tokens = tokenizer.tokenize(sent)
tokens_a_delim = ['[CLS]'] + tokens + ['[SEP]']
token_ids = tokenizer.convert_tokens_to_ids(tokens_a_delim)
tokens_tensor = torch.tensor([token_ids])
token_type_tensor = torch.LongTensor([[0] * len(tokens_a_delim)])
_, _, attn_data_list = model(tokens_tensor,
token_type_ids=token_type_tensor)
attn_tensor = torch.stack(
[attn_data['attn_probs'] for attn_data in attn_data_list])
attention[prefix + key] = attn_tensor.data.numpy()
print("writing weights to the file!!")
with h5py.File(output_path, 'w') as f:
for idx in attention:
f.create_dataset(idx, data=attention[idx], dtype='float64')
f.close()
print("done")
def main():
bert_version = 'bert-large-uncased'
model = BertModel.from_pretrained(bert_version)
tokenizer = BertTokenizer.from_pretrained(bert_version)
output_path = '/misc/vlgscratch4/BowmanGroup/datasets/ptb_trees/trees_' + bert_version + '_wsj.hdf5'
filename = '/misc/vlgscratch4/BowmanGroup/datasets/ptb_trees/ptb3-wsj-test.json'
attn = create_dict(model, tokenizer, filename, 'test_')
length = len(attn)
print("len attention: ", length)
print("writing weights to the file!!")
with h5py.File(output_path, 'w') as f:
for idx in attn:
f.create_dataset(idx, data=attn[idx], dtype='float64')
f.close()
print("done")
def save_attn_weights(model, bert_version, output_path):
tokenizer = BertTokenizer.from_pretrained(bert_version)
filename = '/scratch/sb6416/Ling3340/extract_tree/UD_English-PUD/en_pud-ud-test.conllu'
with open(filename, 'r') as f:
data = f.readlines()
sentences = {}
for i in range(1, len(data)):
line = data[i]
if line[0] == '#':
if line[2:9] == "sent_id":
sentence_id = line[12:]
if line[2:6] == "text":
sentence = line[9:]
sentences[sentence_id[:-1]] = sentence[:-1]
# print(sentences)
attention = {}
model.eval()
with torch.no_grad():
for sent in tqdm.tqdm(sentences):
tokens_sent = sentences[sent]
tokens = tokenizer.tokenize(tokens_sent)
# print(tokens_sent, tokens)
tokens_a_delim = ['[CLS]'] + tokens + ['[SEP]']
token_ids = tokenizer.convert_tokens_to_ids(tokens_a_delim)
# print(token_ids, len(token_ids))
tokens_tensor = torch.tensor([token_ids])
token_type_tensor = torch.LongTensor([[0] * len(tokens_a_delim)])
# print(token_type_tensor)
_, _, attn_data_list = model(tokens_tensor,
token_type_ids=token_type_tensor)
attn_tensor = torch.stack(
[attn_data['attn_probs'] for attn_data in attn_data_list])
attention[sent] = attn_tensor.data.numpy()
print("writing weights to the file!!")
with h5py.File(output_path, 'w') as f:
for idx in attention:
f.create_dataset(idx, data=attention[idx], dtype='float64')
f.close()
print("done")
for i in range(seq_len):
key_vector = keys[i]
dotProduct = 0
for j in range(config["vector_size"]):
product = query_vector[j] * key_vector[j]
dotProduct += product
dotProducts.append(dotProduct)
return dotProducts
bert_version = '/Users/bayartsogtyadamsuren/Downloads/bert-japanese-files/bert-wiki-ja'
model = BertModel.from_pretrained(bert_version)
tokenizer = BertTokenizer.from_pretrained(bert_version)
print("Head of the csv\n", f.head())
print("Number of lines", len(f))
q_x_k_scores = []
para_tokens = []
too_long = 0
errors = 0
ff.write("title\ttoken\tscore\n")
for i, x in tqdm(f.iterrows()):
sentence_a = str(x["text"]).replace("\n", "。").replace("〝", "").replace(
"〞", "").replace("「", "").replace("」", "").strip()
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir",
default=None,
type=str,
required=True,
help="The input data dir. Shou5ld contain the .tsv files (or other data files) for the task.")
parser.add_argument("--bert_model", default=None, type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name", choices=['PDP', 'MNLI', 'pdp', 'mnli'],
required=True,
help="The name of the task to train, pdp or WNLI.")
parser.add_argument("--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--debug", action='store_true',
help="Set this flag if you are want to print debug infos.")
args = parser.parse_args()
if args.task_name.lower() == 'pdp':
processor = processors.XMLPDPProcessor()
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
all_ambigious, guid_dict = processor.get_train_items(args.data_dir, select_type='ambigious')
all_guids = set([])
for i in range(0,len(all_ambigious)):
all_guids.add(all_ambigious[i].guid)
elif args.task_name.lower() == 'mnli':
processor = processors.XMLMnliProcessor()
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
all_ambigious, guid_dict = processor.get_train_items(args.data_dir, select_type='ambigious')
all_guids = set([])
for i in range(0,len(all_ambigious)):
all_guids.add(all_ambigious[i].guid)
model = BertModel.from_pretrained(args.bert_model)
MAS_list = []
counter = np.zeros((2))
for i in trange(0,len(all_guids)):
_, attention = analyzeAttentionSingleTupleDecoy(model, tokenizer, all_ambigious, guid_dict, select_guid=i, do_debug=args.debug)
ref_word = tokenizer.tokenize(all_ambigious[i].text_b)[all_ambigious[i].reference_idx]
MAS = computeMaximumAttentionScore(attention)
MAS_list.append(MAS[0])
# now count how many time MAX is assigned either the true word or the decoy
if np.argmax(MAS)==0:
counter[0] += 1
else:
counter[1] += 1
if args.debug:
print(str(all_ambigious[i].guid) + ' | ' + str(MAS)+ ' | '+all_ambigious[i].text_a + ' '+all_ambigious[i].text_b + ' || >'+ref_word+'<, '+ str(all_ambigious[i].groundtruth)+', '+str(all_ambigious[i].decoy))
print(args.task_name.upper()+" Accuracy: "+str(counter[0]/np.sum(counter)))