def test_on_tokenized_line(): data = ds.TextFileDataset("../data/dataset/testVocab/lines.txt", shuffle=False) jieba_op = text.JiebaTokenizer(HMM_FILE, MP_FILE, mode=text.JiebaMode.MP) with open(VOCAB_FILE, 'r') as f: for line in f: word = line.split(',')[0] jieba_op.add_word(word) data = data.map(operations=jieba_op, input_columns=["text"]) vocab = text.Vocab.from_file(VOCAB_FILE, ",", special_tokens=["<pad>", "<unk>"]) lookup = text.Lookup(vocab, "<unk>") data = data.map(operations=lookup, input_columns=["text"]) res = np.array([[10, 1, 11, 1, 12, 1, 15, 1, 13, 1, 14], [11, 1, 12, 1, 10, 1, 14, 1, 13, 1, 15]], dtype=np.int32) for i, d in enumerate(data.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(d["text"], res[i])
def test_on_tokenized_line_with_no_special_tokens(): data = ds.TextFileDataset("../data/dataset/testVocab/lines.txt", shuffle=False) jieba_op = text.JiebaTokenizer(HMM_FILE, MP_FILE, mode=text.JiebaMode.MP) with open(VOCAB_FILE, 'r') as f: for line in f: word = line.split(',')[0] jieba_op.add_word(word) data = data.map(input_columns=["text"], operations=jieba_op) vocab = text.Vocab.from_file(VOCAB_FILE, ",") lookup = text.Lookup(vocab, "not") data = data.map(input_columns=["text"], operations=lookup) res = np.array([[8, 0, 9, 0, 10, 0, 13, 0, 11, 0, 12], [9, 0, 10, 0, 8, 0, 12, 0, 11, 0, 13]], dtype=np.int32) for i, d in enumerate(data.create_dict_iterator()): np.testing.assert_array_equal(d["text"], res[i])