def test_multiple_output_for_input_list(expected_values):
lower_case_name_field = Field("Lowercase_name", keep_raw=True)
lower_case_name_field.add_pretokenize_hook(str.lower)
upper_case_name_field = Field("Uppercase_name", keep_raw=True)
upper_case_name_field.add_pretokenize_hook(str.upper)
test_field_list = list(field_list)
test_field_list[0] = (
test_field_list[0],
lower_case_name_field,
upper_case_name_field,
)
example_factory = ExampleFactory(test_field_list)
example = example_factory.from_list(expected_values)
raw, tokenized = example["Name"]
assert raw == expected_values[0]
assert tokenized == expected_values[0].split()
raw, tokenized = example["Lowercase_name"]
assert raw == expected_values[0].lower()
assert tokenized == expected_values[0].lower().split()
raw, tokenized = example["Uppercase_name"]
assert raw == expected_values[0].upper()
assert tokenized == expected_values[0].upper().split()
raw, tokenized = example["Score"]
assert raw == expected_values[1]
raw, tokenized = example["Favorite_food"]
assert raw == expected_values[2]
def test_text_clean_up(kwargs, data, expected_output):
pytest.importorskip("cleantext")
field = Field(name="data", tokenizer=None, keep_raw=True)
field.add_pretokenize_hook(TextCleanUp(**kwargs))
example = ExampleFactory([field]).from_list([data])
assert expected_output == example["data"][1]
def test_regex_replace():
data = "This item costs 100$."
field = Field(name="data", tokenizer=None, keep_raw=True)
regex_replace = RegexReplace([(r"\d+", "<NUMBER>"), (r"\s+", "<WHITESPACE>")])
field.add_pretokenize_hook(regex_replace)
example = ExampleFactory([field]).from_list([data])
expected_raw = "This<WHITESPACE>item<WHITESPACE>costs<WHITESPACE><NUMBER>$."
assert expected_raw == example["data"][1]
def test_truecase():
pytest.importorskip("truecase")
data = "hey how are you"
field = Field(name="data", tokenizer=None, keep_raw=True)
field.add_pretokenize_hook(truecase())
example = ExampleFactory([field]).from_list([data])
assert "Hey how are you" == example["data"][0]
def test_moses_normalizer():
pytest.importorskip("sacremoses")
data = "What's up!"
field = Field(name="data", tokenizer=None, keep_raw=True)
normalizer = MosesNormalizer()
field.add_pretokenize_hook(normalizer)
example = ExampleFactory([field]).from_list([data])
assert "What's up!" == example["data"][1]
def test_field_pretokenize_hooks_detach():
f = Field(name="F", tokenizer="split", keep_raw=True)
f.add_pretokenize_hook(str.lower)
f.add_pretokenize_hook(lambda x: x.replace(";", " "))
f.add_pretokenize_hook(lambda x: x.replace(",", " "))
# detaching
f.remove_pretokenize_hooks()
raw_str = "asd;123,BLA"
_, received = f.preprocess(raw_str)[0]
expected = (raw_str, [raw_str])
assert received == expected
def test_field_pretokenize_hooks():
f = Field(name="F", tokenizer="split", keep_raw=True)
f.add_pretokenize_hook(str.lower)
f.add_pretokenize_hook(lambda x: x.replace("bla", "blu"))
f.add_pretokenize_hook(lambda x: x.replace(";", " "))
f.add_pretokenize_hook(lambda x: x.replace(",", " "))
raw_str = "asd;123,BLA"
_, received = f.preprocess(raw_str)[0]
expected = ("asd 123 blu", ["asd", "123", "blu"])
assert received == expected