def train(self, train_dset, vocab_size, prefix='sample', **kwargs):
self.vocab_size = vocab_size
df = pd.DataFrame([x[0] for x in train_dset])
df.to_csv('sample.csv', index=False, header=False)
generate_sp_model('sample.csv',
vocab_size=vocab_size,
model_prefix=prefix)
vocab_tokenizer = load_sp_model(prefix + ".model")
self.tokenizer = sentencepiece_tokenizer(sp_model=vocab_tokenizer)
def test_sentencepiece_tokenizer(self):
test_sample = 'SentencePiece is an unsupervised text tokenizer and detokenizer'
model_path = get_asset_path('spm_example.model')
sp_model = load_sp_model(open(model_path, 'rb'))
self.assertEqual(sp_model.GetPieceSize(), 20000)
spm_generator = sentencepiece_tokenizer(sp_model)
ref_results = [
'\u2581Sent', 'ence', 'P', 'ie', 'ce', '\u2581is', '\u2581an',
'\u2581un', 'super', 'vis', 'ed', '\u2581text', '\u2581to', 'ken',
'izer', '\u2581and', '\u2581de', 'to', 'ken', 'izer'
]
self.assertEqual(list(spm_generator([test_sample]))[0], ref_results)
def test_sentencepiece_tokenizer(self):
test_sample = 'SentencePiece is an unsupervised text tokenizer and detokenizer'
model_path = 'test/asset/spm_example.model'
sp_model = load_sp_model(model_path)
self.assertEqual(len(sp_model), 20000)
spm_generator = sentencepiece_tokenizer(sp_model)
# Handle byte string in Python2 and Unicode string in Python3, respectively
if sys.version_info < (3, 0):
ref_results = [
'\xe2\x96\x81Sent', 'ence', 'P', 'ie', 'ce', '\xe2\x96\x81is',
'\xe2\x96\x81an', '\xe2\x96\x81un', 'super', 'vis', 'ed',
'\xe2\x96\x81text', '\xe2\x96\x81to', 'ken', 'izer',
'\xe2\x96\x81and', '\xe2\x96\x81de', 'to', 'ken', 'izer'
]
else:
ref_results = [
'\u2581Sent', 'ence', 'P', 'ie', 'ce', '\u2581is', '\u2581an',
'\u2581un', 'super', 'vis', 'ed', '\u2581text', '\u2581to',
'ken', 'izer', '\u2581and', '\u2581de', 'to', 'ken', 'izer'
]
self.assertEqual(list(spm_generator([test_sample]))[0], ref_results)
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
# lets define already our device and make sure to run on gpu if possible
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# %% [markdown]
# We use the same tokenizer as in the data-preprocessing line
# %%
sp_deu = load_sp_model("preprocessed_data/sp_model/de.wiki.bpe.vs10000.model")
sp_nds = load_sp_model("preprocessed_data/sp_model/nds.wiki.bpe.vs10000.model")
# %%
sp_deu_tokens_generator = sentencepiece_tokenizer(sp_deu)
list_a = [
"Komplizierte Wörter sind Baustelle.",
"Morgen soll es regnen und übermorgen scheint die Sonne"
]
print(list(sp_deu_tokens_generator(list_a)))
sp_numericalize_generator = sentencepiece_numericalizer(sp_deu)
print(list(sp_numericalize_generator(list_a)))
# %%
sp_deu_tokens_generator = sentencepiece_tokenizer(sp_deu)
sp_nds_tokens_generator = sentencepiece_tokenizer(sp_nds)
def tokenize_de(text):
from typing import List, Union
import janome
import spacy
from janome.tokenizer import Tokenizer as JanomeTokenizer
from torchtext.data.functional import load_sp_model, sentencepiece_tokenizer
from src.constants import SENTENCE_PIECE_MODEL_PATH
from src.utils import log_decorator
janome_tokenizer = JanomeTokenizer()
ginza_tokenizer = spacy.load("ja_ginza")
sp_model = load_sp_model(SENTENCE_PIECE_MODEL_PATH)
sp_tokens_generator = sentencepiece_tokenizer(sp_model)
@log_decorator
def wakachi_by_sentencepiece(sentence: str) -> List[str]:
"""SentencePieceを用いた分かち書き
Parameters
----------
sentence : str
分かち書きしたい文章
Returns
-------
List[str]
分かち書き結果
"""