def build_dataloader(self,
data,
transform: TransformList = None,
training=False,
device=None,
logger: logging.Logger = None,
gradient_accumulation=1,
**kwargs) -> DataLoader:
transform.insert(0, append_bos)
dataset = BiaffineDependencyParser.build_dataset(self, data, transform)
if isinstance(data, str):
dataset.purge_cache()
if self.vocabs.mutable:
BiaffineDependencyParser.build_vocabs(self,
dataset,
logger,
transformer=True)
if dataset.cache:
timer = CountdownTimer(len(dataset))
BiaffineDependencyParser.cache_dataset(self, dataset, timer,
training, logger)
max_seq_len = self.config.get('max_seq_len', None)
if max_seq_len and isinstance(data, str):
dataset.prune(lambda x: len(x['token_input_ids']) > 510, logger)
return PadSequenceDataLoader(batch_sampler=self.sampler_builder.build(
self.compute_lens(data, dataset, length_field='FORM'),
shuffle=training,
gradient_accumulation=gradient_accumulation),
device=device,
dataset=dataset,
pad=self.get_pad_dict())
def build_dataloader(self,
data,
transform: TransformList = None,
training=False,
device=None,
logger: logging.Logger = None,
cache=False,
gradient_accumulation=1,
**kwargs) -> DataLoader:
args = dict((k, self.config[k]) for k in [
'delimiter', 'max_seq_len', 'sent_delimiter', 'char_level',
'hard_constraint'
] if k in self.config)
# We only need those transforms before TransformerTokenizer
transformer_index = transform.index_by_type(
TransformerSequenceTokenizer)
assert transformer_index is not None
transform = transform[:transformer_index + 1]
if self.transform:
transform.insert(0, self.transform)
transform.append(self.last_transform())
dataset = self.build_dataset(data,
cache=cache,
transform=transform,
**args)
if self.vocabs.mutable:
self.build_vocabs(dataset, logger)
return PadSequenceDataLoader(batch_sampler=self.sampler_builder.build(
self.compute_lens(data, dataset, 'token_input_ids'),
shuffle=training,
gradient_accumulation=gradient_accumulation),
device=device,
dataset=dataset)
def build_transform(self, task: Task) -> Tuple[TransformerSequenceTokenizer, TransformList]:
encoder: ContextualWordEmbedding = self.config.encoder
encoder_transform: TransformerSequenceTokenizer = task.build_tokenizer(encoder.transform())
length_transform = FieldLength('token', 'token_length')
transform = TransformList(encoder_transform, length_transform)
extra_transform = self.config.get('transform', None)
if extra_transform:
transform.insert(0, extra_transform)
return encoder_transform, transform
class Transformable(ABC):
def __init__(self, transform: Union[Callable, List] = None) -> None:
"""An object which can be transformed with a list of functions. It can be treated as an objected being passed
through a list of functions, while these functions are kept in a list.
Args:
transform: A transform function or a list of functions.
"""
super().__init__()
if isinstance(transform,
list) and not isinstance(transform, TransformList):
transform = TransformList(*transform)
self.transform: Union[Callable, TransformList] = transform
def append_transform(self, transform: Callable):
"""Append a transform to its list of transforms.
Args:
transform: A new transform to be appended.
Returns:
Itself.
"""
assert transform is not None, 'None transform not allowed'
if not self.transform:
self.transform = TransformList(transform)
elif not isinstance(self.transform, TransformList):
if self.transform != transform:
self.transform = TransformList(self.transform, transform)
else:
if transform not in self.transform:
self.transform.append(transform)
return self
def insert_transform(self, index: int, transform: Callable):
"""Insert a transform to a certain position.
Args:
index: A certain position.
transform: A new transform.
Returns:
Itself.
"""
assert transform is not None, 'None transform not allowed'
if not self.transform:
self.transform = TransformList(transform)
elif not isinstance(self.transform, TransformList):
if self.transform != transform:
self.transform = TransformList(self.transform)
self.transform.insert(index, transform)
else:
if transform not in self.transform:
self.transform.insert(index, transform)
return self
def transform_sample(self, sample: dict, inplace=False) -> dict:
"""Apply transforms to a sample.
Args:
sample: A sample, which is a ``dict`` holding features.
inplace: ``True`` to apply transforms inplace.
.. Attention::
If any transform modifies existing features, it will modify again and again when ``inplace=True``.
For example, if a transform insert a ``BOS`` token to a list inplace, and it is called twice,
then 2 ``BOS`` will be inserted which might not be an intended result.
Returns:
Transformed sample.
"""
if not inplace:
sample = copy(sample)
if self.transform:
sample = self.transform(sample)
return sample