def test_full_tokenizer(self): tokenizer = AlbertTokenizer(SAMPLE_VOCAB, keep_accents=True) tokens = tokenizer.tokenize("This is a test") self.assertListEqual(tokens, ["▁this", "▁is", "▁a", "▁test"]) self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [48, 25, 21, 1289]) tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.") self.assertListEqual(tokens, [ "▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "." ]) ids = tokenizer.convert_tokens_to_ids(tokens) self.assertListEqual( ids, [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]) back_tokens = tokenizer.convert_ids_to_tokens(ids) self.assertListEqual( back_tokens, [ "▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "." ], )
def getRelations(self, tokenizer: AlbertTokenizer, tokenBatch, spanBatch: torch.Tensor): # spanBatch[batchSize, 2, 3] # two most probable entities sentences = [None] * spanBatch.shape[0] sub_idx = [0] * spanBatch.shape[0] obj_idx = [0] * spanBatch.shape[0] markers = tokenizer.convert_tokens_to_ids( ["<e1>", "</e1>", "<e2>", "</e2>"]) for (i, (tokens, spans)) in enumerate(zip(tokenBatch, spanBatch)): i1S = spans[0, 0] i1E = spans[0, 1] i2S = spans[1, 0] i2E = spans[1, 1] if i1S > i2S: temp = (i2S, i2E) (i2S, i2E) = (i1S, i1E) (i1S, i1E) = temp tokens: List = tokens tokens.insert(i2E, markers[3]) tokens.insert(i2S, markers[2]) tokens.insert(i1E, markers[1]) tokens.insert(i1S, markers[0]) sentences[i] = torch.tensor(tokens) sub_idx[i] = i1S obj_idx[i] = i2S input_ids = pad_sequence(sentences, batch_first=True) relations = self( input_ids=input_ids, token_type_ids=(input_ids == 0).type(torch.int), attention_mask=(input_ids != 0).type(torch.int), labels=None, sub_idx=sub_idx, obj_idx=obj_idx, ) relations = relations.argmax(1) return (relations, [relationTypes[rel] for rel in relations])
with open(os.path.join(args.data, 'ontonotes/const/pos/labels.txt')) as f: while True: pos = f.readline().rstrip() if pos == "": break tag[pos] = np.asarray([0, 0]) text_file = open(os.path.join(args.data, 'ontonotes/const/pos/conll-2012-test.json'), 'r') for i, line in tqdm(enumerate(text_file.readlines())): data = json.loads(line) tokens = data['text'].split(' ') labels = data['targets'] re_2_o = [] retokenized = [] for word_id, token in enumerate(tokens): token = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(token)) retokenized.extend(token) re_2_o.extend([word_id for _ in range(len(token))]) model_inputs = [] for span in labels: span1 = [] indexed_tokens = retokenized.copy() for pos in range(span['span1'][0], span['span1'][1]): select = np.where(np.asarray(re_2_o) == pos)[0] span1.extend(select) for i in span1: indexed_tokens[i] = tokenizer.mask_token_id indexed_tokens = [tokenizer.cls_token_id] + indexed_tokens + [tokenizer.sep_token_id] model_inputs.append(indexed_tokens) retokenized.insert(0, tokenizer.cls_token_id)