def test_text_process_lower(): field = TextField(lower=True) dummy = "justo Praesent luctus justo praesent" assert list(field.process(dummy)) == [1, 1, 1, 1, 1] field.setup([dummy]) assert list(field.process(dummy)) == [2, 3, 4, 2, 3]
def test_text_process_list(): field = TextField(lower=True) field.setup() dummy = [["justo Praesent luctus", "luctus praesent"], ["justo Praesent luctus", "luctus praesent est"]] assert recursive_tensor_to_list(field.process(dummy)) == [[[1, 1, 1], [1, 1]], [[1, 1, 1], [1, 1, 1]]] field.setup(dummy) assert recursive_tensor_to_list(field.process(dummy)) == [[[2, 3, 4], [4, 3]], [[2, 3, 4], [4, 3, 5]]]
def test_text_process_nested_list_in_dict(): field = TextField(lower=True) field.setup() dummy = [{ 'text1': ["justo Praesent luctus", "luctus praesent"], 'text2': ["justo Praesent luctus", "luctus praesent est"] }] assert recursive_tensor_to_list(field.process(dummy)) == [{ 'text1': [[1, 1, 1], [1, 1]], 'text2': [[1, 1, 1], [1, 1, 1]] }] field.setup(dummy) assert recursive_tensor_to_list(field.process(dummy)) == [{ 'text1': [[2, 3, 4], [4, 3]], 'text2': [[2, 3, 4], [4, 3, 5]] }]
def test_text_process_dict(): field = TextField(lower=True) field.setup() dummy = { 'text1': "justo Praesent luctus luctus praesent", 'text2': "justo Praesent luctus luctus praesent est" } assert recursive_tensor_to_list(field.process(dummy)) == { 'text1': [1, 1, 1, 1, 1], 'text2': [1, 1, 1, 1, 1, 1] } field.setup([dummy]) assert recursive_tensor_to_list(field.process(dummy)) == { 'text1': [2, 3, 4, 4, 3], 'text2': [2, 3, 4, 4, 3, 5] }
def test_text_process_unk(): field = TextField(unk_token=None) dummy = "justo Praesent luctus justo praesent" with pytest.raises(Exception): field.process(dummy)