def test_clean_segments(tmpdir):
extractor = DeepTileExtractor(tmpdir, tmpdir, {"tfchannel": True})
assert extractor.clean_segments(["hello world", "foo bar"], p_len=4) == [
"hello world",
"foo bar",
extractor.pad_tok,
extractor.pad_tok,
]
assert extractor.clean_segments(["hello world", "foo bar", "alice", "bob"],
p_len=3) == [
"hello world", "foo bar", "alicebob"
]
def test_deeptiles_clean_segments(tmpdir, dummy_index):
pipeline_config = {
"name": "deeptiles",
"passagelen": 30,
"slicelen": 20,
"tfchannel": True,
"tilechannels": 3
}
extractor = DeepTileExtractor(pipeline_config,
provide={"index": dummy_index})
assert extractor.clean_segments(["hello world", "foo bar"], p_len=4) == [
"hello world",
"foo bar",
extractor.pad_tok,
extractor.pad_tok,
]
assert extractor.clean_segments(["hello world", "foo bar", "alice", "bob"],
p_len=3) == [
"hello world", "foo bar", "alicebob"
]