def test_field_pad_to_length_custom_pad(row, length, expected_row):
f = Field(name="F", numericalizer=None)
row_arr = np.array(row)
received_row = f._pad_to_length(row_arr, length, custom_pad_symbol=CUSTOM_PAD)
assert received_row.tolist() == expected_row
def test_field_pad_to_length(row, length, expected_row, pad_left, truncate_left):
vocab = MockVocab()
f = Field(name="F", numericalizer=vocab)
received_row = f._pad_to_length(
np.array(row), length, pad_left=pad_left, truncate_left=truncate_left
)
assert received_row.tolist() == expected_row
def test_field_pad_custom_numericalize():
custom_padding_token = -999
f = Field(
"test_field",
numericalizer=int,
padding_token=custom_padding_token,
tokenizer="split",
)
mock_numericalization = np.array([1, 2, 3, 4])
expected_numericalization = np.array([1, 2, 3, 4] + [custom_padding_token] * 6)
padded = f._pad_to_length(mock_numericalization, 10, pad_left=False)
assert np.all(padded == expected_numericalization)