def append_transform(self, transform: Callable):
"""Append a transform to its list of transforms.
Args:
transform: A new transform to be appended.
Returns: Itself.
"""
assert transform is not None, 'None transform not allowed'
if not self.transform:
self.transform = TransformList(transform)
elif not isinstance(self.transform, TransformList):
if self.transform != transform:
self.transform = TransformList(self.transform, transform)
else:
if transform not in self.transform:
self.transform.append(transform)
return self
def __init__(self, transform: Union[Callable, List] = None) -> None:
"""An object which can be transformed with a list of functions. It can be treated as an objected being passed
through a list of functions, while these functions are kept in a list.
Args:
transform: A transform function or a list of functions.
"""
super().__init__()
if isinstance(transform, list) and not isinstance(transform, TransformList):
transform = TransformList(*transform)
self.transform: Union[Callable, TransformList] = transform
def build_transform(
self,
task: Task) -> Tuple[TransformerSequenceTokenizer, TransformList]:
encoder: ContextualWordEmbedding = self.config.encoder
encoder_transform: TransformerSequenceTokenizer = task.build_tokenizer(
encoder.transform())
length_transform = FieldLength('token', 'token_length')
transform = TransformList(encoder_transform, length_transform)
extra_transform = self.config.get('transform', None)
if extra_transform:
transform.insert(0, extra_transform)
return encoder_transform, transform
def insert_transform(self, index: int, transform: Callable):
"""Insert a transform to a certain position.
Args:
index: A certain position.
transform: A new transform.
Returns: Dataset itself.
"""
assert transform is not None, 'None transform not allowed'
if not self.transform:
self.transform = TransformList(transform)
elif not isinstance(self.transform, TransformList):
if self.transform != transform:
self.transform = TransformList(self.transform)
self.transform.insert(index, transform)
else:
if transform not in self.transform:
self.transform.insert(index, transform)
return self
def build_dataset(self, data, bos_transform=None):
transform = TransformList(
functools.partial(append_bos, pos_key='UPOS'),
functools.partial(unpack_deps_to_head_deprel,
pad_rel=self.config.pad_rel,
arc_key='arc_2nd',
rel_key='rel_2nd'))
if self.config.joint:
transform.append(merge_head_deprel_with_2nd)
if bos_transform:
transform.append(bos_transform)
return super().build_dataset(data, transform)
def build_dataloader(self,
data,
shuffle,
device,
embed: Embedding,
training=False,
logger=None,
gradient_accumulation=1,
sampler_builder=None,
batch_size=None,
bos='\0',
**kwargs) -> DataLoader:
first_transform = TransformList(functools.partial(append_bos, bos=bos))
embed_transform = embed.transform(vocabs=self.vocabs)
transformer_transform = self._get_transformer_transform_from_transforms(
embed_transform)
if embed_transform:
if transformer_transform and isinstance(embed_transform,
TransformList):
embed_transform.remove(transformer_transform)
first_transform.append(embed_transform)
dataset = self.build_dataset(data, first_transform=first_transform)
if self.config.get('transform', None):
dataset.append_transform(self.config.transform)
if self.vocabs.mutable:
self.build_vocabs(dataset, logger, self._transformer_trainable())
if transformer_transform and isinstance(embed_transform,
TransformList):
embed_transform.append(transformer_transform)
dataset.append_transform(FieldLength('token', 'sent_length'))
if isinstance(data, str):
dataset.purge_cache()
if len(dataset) > 1000 and isinstance(data, str):
timer = CountdownTimer(len(dataset))
self.cache_dataset(dataset, timer, training, logger)
if sampler_builder:
lens = [sample['sent_length'] for sample in dataset]
sampler = sampler_builder.build(lens, shuffle,
gradient_accumulation)
else:
sampler = None
loader = PadSequenceDataLoader(dataset=dataset,
batch_sampler=sampler,
batch_size=batch_size,
pad=self.get_pad_dict(),
device=device,
vocabs=self.vocabs)
return loader
def last_transform(self):
return TransformList(
functools.partial(generate_tags_for_subtokens,
tagging_scheme=self.config.tagging_scheme),
super().last_transform())
def build_dataset(self, data, transform=None):
transforms = TransformList(functools.partial(append_bos_to_form_pos, pos_key='UPOS'),
functools.partial(unpack_deps_to_head_deprel, pad_rel=self.config.pad_rel))
if transform:
transforms.append(transform)
return super(BiaffineSemanticDependencyParser, self).build_dataset(data, transforms)
def transform(self, **kwargs):
transforms = [e.transform(**kwargs) for e in self._embeddings]
transforms = [t for t in transforms if t]
return TransformList(*transforms)
def last_transform(self):
return TransformList(self.vocabs, FieldLength(self.config.token_key))
def build_dataset(self, data, transform=None, **kwargs):
if not isinstance(transform, list):
transform = TransformList()
transform.append(add_lemma_rules_to_sample)
return super().build_dataset(data, transform, **kwargs)
def transform(self, **kwargs) -> Callable:
vocab = Vocab()
vocab.load(os.path.join(get_resource(self.path), 'vocab.json'))
return TransformList(ContextualStringEmbeddingTransform(self.field),
FieldToIndex(f'{self.field}_f_char', vocab),
FieldToIndex(f'{self.field}_b_char', vocab))