def test_token_level_sampler_swap():
tls = TokenLevelSampler(types=["swap"])
text = "人为什么活着？生而为人必须要有梦想！还要有尽可能多的精神体验。"
res = tls.make_samples(text)
assert type(res) == dict
assert len(res) == 2
def test_token_level_sampler_single_sent():
tls = TokenLevelSampler()
text = "人为什么活着？"
res = tls.make_samples(text)
assert len(res) == 4
def test_token_level_sampler_none():
tls = TokenLevelSampler(types=[])
text = "人为什么活着？生而为人必须要有梦想！还要有尽可能多的精神体验。"
res = tls.make_samples(text)
assert res == {}
def test_token_level_sampler_none_text():
tls = TokenLevelSampler()
text = ""
res = tls.make_samples(text)
assert res == {}
def test_token_level_sampler_token_pos_spliter():
tls = TokenLevelSampler()
text = "人为什么活着？生而为人必须要有梦想！还要有尽可能多的精神体验。"
res = tls.make_samples(text, cut_wps)
assert len(res) == 4