def test_text_process_lower():
field = TextField(lower=True)
dummy = "justo Praesent luctus justo praesent"
assert list(field.process(dummy)) == [1, 1, 1, 1, 1]
field.setup([dummy])
assert list(field.process(dummy)) == [2, 3, 4, 2, 3]
def test_text_process_list():
field = TextField(lower=True)
field.setup()
dummy = [["justo Praesent luctus", "luctus praesent"],
["justo Praesent luctus", "luctus praesent est"]]
assert recursive_tensor_to_list(field.process(dummy)) == [[[1, 1, 1],
[1, 1]],
[[1, 1, 1],
[1, 1, 1]]]
field.setup(dummy)
assert recursive_tensor_to_list(field.process(dummy)) == [[[2, 3, 4],
[4, 3]],
[[2, 3, 4],
[4, 3, 5]]]
def test_text_process_nested_list_in_dict():
field = TextField(lower=True)
field.setup()
dummy = [{
'text1': ["justo Praesent luctus", "luctus praesent"],
'text2': ["justo Praesent luctus", "luctus praesent est"]
}]
assert recursive_tensor_to_list(field.process(dummy)) == [{
'text1': [[1, 1, 1], [1, 1]],
'text2': [[1, 1, 1], [1, 1, 1]]
}]
field.setup(dummy)
assert recursive_tensor_to_list(field.process(dummy)) == [{
'text1': [[2, 3, 4], [4, 3]],
'text2': [[2, 3, 4], [4, 3, 5]]
}]
def test_text_process_dict():
field = TextField(lower=True)
field.setup()
dummy = {
'text1': "justo Praesent luctus luctus praesent",
'text2': "justo Praesent luctus luctus praesent est"
}
assert recursive_tensor_to_list(field.process(dummy)) == {
'text1': [1, 1, 1, 1, 1],
'text2': [1, 1, 1, 1, 1, 1]
}
field.setup([dummy])
assert recursive_tensor_to_list(field.process(dummy)) == {
'text1': [2, 3, 4, 4, 3],
'text2': [2, 3, 4, 4, 3, 5]
}
def test_text_process_unk():
field = TextField(unk_token=None)
dummy = "justo Praesent luctus justo praesent"
with pytest.raises(Exception):
field.process(dummy)