def get_default_padding_callback(
cls,
fixed_length_left: int = 10,
fixed_length_right: int = 40,
pad_word_value: typing.Union[int, str] = 0,
pad_word_mode: str = 'pre',
with_ngram: bool = True,
fixed_ngram_length: int = None,
pad_ngram_value: typing.Union[int, str] = 0,
pad_ngram_mode: str = 'pre'
) -> BaseCallback:
"""
Model default padding callback.
The padding callback's on_batch_unpacked would pad a batch of data to
a fixed length.
:return: Default padding callback.
"""
return callbacks.BasicPadding(
fixed_length_left=fixed_length_left,
fixed_length_right=fixed_length_right,
pad_word_value=pad_word_value,
pad_word_mode=pad_word_mode,
with_ngram=with_ngram,
fixed_ngram_length=fixed_ngram_length,
pad_ngram_value=pad_ngram_value,
pad_ngram_mode=pad_ngram_mode
)
def get_default_padding_callback(
cls,
fixed_length_left: int = None,
fixed_length_right: int = None
):
""":return: Default padding callback."""
return callbacks.BasicPadding(
fixed_length_left=fixed_length_left,
fixed_length_right=fixed_length_right,
)
def get_default_padding_callback(cls,
fixed_length_left: int = 10,
fixed_length_right: int = 100,
pad_value: typing.Union[int, str] = 0,
pad_mode: str = 'pre'):
""":return: Default padding callback."""
return callbacks.BasicPadding(fixed_length_left=fixed_length_left,
fixed_length_right=fixed_length_right,
pad_value=pad_value,
pad_mode=pad_mode)
def test_basic_padding(train_raw):
preprocessor = preprocessors.BasicPreprocessor()
data_preprocessed = preprocessor.fit_transform(train_raw, verbose=0)
dataset = Dataset(data_preprocessed, mode='point')
pre_fixed_padding = callbacks.BasicPadding(fixed_length_left=5,
fixed_length_right=5,
pad_mode='pre')
dataloader = DataLoader(dataset, batch_size=5, callback=pre_fixed_padding)
for batch in dataloader:
assert batch[0]['text_left'].shape == (5, 5)
assert batch[0]['text_right'].shape == (5, 5)
post_padding = callbacks.BasicPadding(pad_mode='post')
dataloader = DataLoader(dataset, batch_size=5, callback=post_padding)
for batch in dataloader:
max_left_len = max(batch[0]['length_left'].numpy())
max_right_len = max(batch[0]['length_right'].numpy())
assert batch[0]['text_left'].shape == (5, max_left_len)
assert batch[0]['text_right'].shape == (5, max_right_len)
def get_default_padding_callback(
cls,
fixed_length_left: int = 10,
fixed_length_right: int = 100,
pad_value: typing.Union[int, str] = 0,
pad_mode: str = 'pre'
):
"""
Model default padding callback.
The padding callback's on_batch_unpacked would pad a batch of data to
a fixed length.
:return: Default padding callback.
"""
return callbacks.BasicPadding(
fixed_length_left=fixed_length_left,
fixed_length_right=fixed_length_right,
pad_value=pad_value,
pad_mode=pad_mode)
def get_default_padding_callback(cls):
""":return: Default padding callback."""
return callbacks.BasicPadding()