def test_full_tokenizer(self): tokenizer = AlbertTokenizer(SAMPLE_VOCAB, keep_accents=True) tokens = tokenizer.tokenize("This is a test") self.assertListEqual(tokens, ["▁this", "▁is", "▁a", "▁test"]) self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [48, 25, 21, 1289]) tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.") self.assertListEqual(tokens, [ "▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "." ]) ids = tokenizer.convert_tokens_to_ids(tokens) self.assertListEqual( ids, [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]) back_tokens = tokenizer.convert_ids_to_tokens(ids) self.assertListEqual( back_tokens, [ "▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "." ], )
with open(os.path.join(args.data, 'ontonotes/const/pos/labels.txt')) as f: while True: pos = f.readline().rstrip() if pos == "": break tag[pos] = np.asarray([0, 0]) text_file = open(os.path.join(args.data, 'ontonotes/const/pos/conll-2012-test.json'), 'r') for i, line in tqdm(enumerate(text_file.readlines())): data = json.loads(line) tokens = data['text'].split(' ') labels = data['targets'] re_2_o = [] retokenized = [] for word_id, token in enumerate(tokens): token = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(token)) retokenized.extend(token) re_2_o.extend([word_id for _ in range(len(token))]) model_inputs = [] for span in labels: span1 = [] indexed_tokens = retokenized.copy() for pos in range(span['span1'][0], span['span1'][1]): select = np.where(np.asarray(re_2_o) == pos)[0] span1.extend(select) for i in span1: indexed_tokens[i] = tokenizer.mask_token_id indexed_tokens = [tokenizer.cls_token_id] + indexed_tokens + [tokenizer.sep_token_id] model_inputs.append(indexed_tokens) retokenized.insert(0, tokenizer.cls_token_id)