def from_config(
cls,
config: Config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Vocabulary] = None,
):
if labels is not None:
vocab = list(labels)
vocab_dict = labels.idx
else:
vocab = metadata.vocab.itos
vocab_dict = metadata.vocab.stoi
label_weights = (get_label_weights(vocab_dict, config.label_weights)
if config.label_weights else None)
loss = create_loss(config.loss, weight=label_weights)
if isinstance(loss, BinaryCrossEntropyLoss):
cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
cls = MultiLabelOutputLayer
else:
cls = MulticlassOutputLayer
return cls(vocab, create_loss(config.loss, weight=label_weights),
config)
def __init__(self, config, ignore_index=1):
self.beta = config.beta
self.label_loss_fn = create_loss(config.label_loss,
ignore_index=ignore_index)
self.smoothing_loss_fn = create_loss(config.smoothing_loss,
ignore_index=ignore_index)
self.ignore_index = ignore_index
# Tracking variables.
self.label_loss = 0
self.smoothing_loss = 0
def from_config(cls, config: Config, metadata: FieldMeta = None, labels=None):
label_weights = getattr(metadata, "label_weights", None)
if label_weights is not None:
label_weights = FloatTensor(label_weights)
vocab = metadata.vocab.itos if metadata else labels
loss = create_loss(config.loss, weight=label_weights)
cls = (
BinaryClassificationOutputLayer
if isinstance(loss, BinaryCrossEntropyLoss)
else MulticlassOutputLayer
)
return cls(vocab, create_loss(config.loss, weight=label_weights), config)
def __init__(self, config, ignore_index=1):
self.beta = config.beta
self.assert_valid_targets = config.assert_valid_targets
self.label_type = config.label_type
self.length_type = config.length_type
# We can't use a structured loss for optimizing lengths.
if isinstance(config.length_loss.label_loss, StructuredLoss):
raise ValueError("StructuredLoss can't be used as a length loss")
self.label_loss_fn = create_loss(config.label_loss,
ignore_index=ignore_index)
self.length_loss_fn = create_loss(config.length_loss,
ignore_index=ignore_index)
def from_config(
cls,
config: Config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Vocabulary] = None,
):
if labels is not None:
vocab = list(labels)
vocab_dict = labels.idx
pad_token_idx = labels.idx.get(labels.pad_token,
Padding.DEFAULT_LABEL_PAD_IDX)
else:
vocab = metadata.vocab.itos
pad_token_idx = metadata.pad_token_idx
vocab_dict = metadata.vocab.stoi
label_weights = (get_label_weights(vocab_dict, config.label_weights)
if config.label_weights else None)
return cls(
vocab,
create_loss(
config.loss,
weight=label_weights,
ignore_index=pad_token_idx
if config.ignore_pad_in_loss else -1,
),
)
def from_config(cls, config: Config, tensorizers: Dict[str, Tensorizer]):
labels = tensorizers["labels"].vocab
if not labels:
raise ValueError("Labels were not created, see preceding errors")
vocab = tensorizers["tokens"].vocab
encoder = create_module(config.encoder,
padding_idx=vocab.get_pad_index(),
vocab_size=len(vocab))
right_dense_dim = tensorizers["right_dense"].dim
left_dense_dim = tensorizers["left_dense"].dim
decoder = create_module(
config.decoder,
right_dim=encoder.representation_dim + right_dense_dim,
left_dim=left_dense_dim,
to_dim=len(labels),
)
label_weights = (get_label_weights(labels.idx,
config.output_layer.label_weights)
if config.output_layer.label_weights else None)
loss = create_loss(config.output_layer.loss, weight=label_weights)
if isinstance(loss, BinaryCrossEntropyLoss):
output_layer_cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
output_layer_cls = MultiLabelOutputLayer
else:
output_layer_cls = MulticlassOutputLayer
output_layer = output_layer_cls(list(labels), loss)
return cls(encoder, decoder, output_layer)
def from_config(cls, config: Config, tensorizers: Dict[str, Tensorizer]):
labels = tensorizers["labels"].vocab
embedding = cls.create_embedding(config, tensorizers)
representation = create_module(
config.representation, embed_dim=embedding.embedding_dim
)
decoder = cls.create_decoder(
config, representation.representation_dim, len(labels)
)
label_weights = (
get_label_weights(labels.idx, config.output_layer.label_weights)
if config.output_layer.label_weights
else None
)
loss = create_loss(config.output_layer.loss, weight=label_weights)
if isinstance(loss, BinaryCrossEntropyLoss):
output_layer_cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
output_layer_cls = MultiLabelOutputLayer
else:
output_layer_cls = MulticlassOutputLayer
output_layer = output_layer_cls(list(labels), loss)
return cls(embedding, representation, decoder, output_layer)
def from_config(cls, config: Config, metadata: FieldMeta):
label_weights = getattr(metadata, "label_weights", None)
if label_weights is not None:
label_weights = FloatTensor(label_weights)
return cls(
metadata.vocab.itos, create_loss(config.loss, weight=label_weights), config
)
def from_config(
cls,
config: Config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Vocabulary] = None,
):
if labels is not None:
vocab = list(labels)
vocab_dict = labels.idx
pad_token_idx = labels.idx.get(labels.pad_token,
Padding.DEFAULT_LABEL_PAD_IDX)
else:
vocab = metadata.vocab.itos
vocab_dict = metadata.vocab.stoi
pad_token_idx = getattr(metadata, "pad_token_idx", -1)
label_weights = (get_label_weights(vocab_dict, config.label_weights)
if config.label_weights else None)
loss = create_loss(config.loss,
weight=label_weights,
ignore_index=pad_token_idx)
if isinstance(loss, BinaryCrossEntropyLoss):
cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
cls = MultiLabelOutputLayer
else:
cls = MulticlassOutputLayer
return cls(vocab, loss, config)
def from_config(cls, config: Config, tensorizers: Dict[str, Tensorizer]):
labels = tensorizers["labels"].vocab
vocab = tensorizers["tokens"].vocab
encoder = create_module(
config.encoder, padding_idx=vocab.get_pad_index(), vocab_size=len(vocab)
)
dense_dim = tensorizers["dense"].dim if "dense" in tensorizers else 0
decoder = create_module(
config.decoder,
in_dim=encoder.representation_dim + dense_dim,
out_dim=len(labels),
)
label_weights = (
get_label_weights(labels.idx, config.output_layer.label_weights)
if config.output_layer.label_weights
else None
)
loss = create_loss(config.output_layer.loss, weight=label_weights)
if isinstance(loss, BinaryCrossEntropyLoss):
output_layer_cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
output_layer_cls = MultiLabelOutputLayer
else:
output_layer_cls = MulticlassOutputLayer
output_layer = output_layer_cls(list(labels), loss)
return cls(encoder, decoder, output_layer)
def from_config(
cls,
config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Vocabulary] = None,
):
return cls(list(labels), create_loss(config.loss),
config.cosine_distance_threshold)
def __init__(self, config, ignore_index=1, *args, **kwargs):
super().__init__(config, ignore_index)
self.cost_scale = config.cost_scale
self.cost_fn = get_cost_fn(config.cost_fn)
self.label_loss_fn = create_loss(config.label_loss,
ignore_index=ignore_index)
def from_config(cls, config: Config, metadata=None, labels=None):
vocab = labels or metadata.vocab.itos
pad_token_idx = metadata.pad_token_idx if metadata else vocab.idx[PAD]
return cls(
vocab,
create_loss(config.loss, ignore_index=pad_token_idx),
pad_token_idx=pad_token_idx,
)
def from_config(cls, config: Config, tensorizers: Dict[str, Tensorizer]):
labels = tensorizers["labels"].labels
embedding = cls.create_embedding(config, tensorizers)
representation = create_module(config.representation,
embed_dim=embedding.embedding_dim)
decoder = cls.create_decoder(config, representation.representation_dim,
len(labels))
# TODO change from_config function of ClassificationOutputLayer after migriting to new design
output_layer = ClassificationOutputLayer(
list(labels), create_loss(config.output_layer.loss))
return cls(embedding, representation, decoder, output_layer)
def from_config(cls, config: Config, tensorizers: Dict[str, Tensorizer]):
labels = tensorizers["labels"].vocab
embedding = cls.create_embedding(config, tensorizers)
representation = create_module(config.representation,
embed_dim=embedding.embedding_dim)
decoder = cls.create_decoder(config, representation.representation_dim,
len(labels))
loss = create_loss(config.output_layer.loss)
output_layer_cls = (BinaryClassificationOutputLayer if isinstance(
loss, BinaryCrossEntropyLoss) else MulticlassOutputLayer)
output_layer = output_layer_cls(list(labels), loss)
return cls(embedding, representation, decoder, output_layer)
def from_config(
cls,
config: Config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Vocabulary] = None,
):
if labels is not None:
vocab = list(labels)
pad_token_idx = labels.idx[PAD]
else:
vocab = metadata.vocab.itos
pad_token_idx = metadata.pad_token_idx
return cls(vocab, create_loss(config.loss, ignore_index=pad_token_idx))
def create_output_layer(cls, config: Config, labels: VocabConfig):
label_weights = (get_label_weights(labels.idx,
config.output_layer.label_weights)
if config.output_layer.label_weights else None)
loss = create_loss(config.output_layer.loss, weight=label_weights)
if isinstance(loss, BinaryCrossEntropyLoss):
output_layer_cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
output_layer_cls = MultiLabelOutputLayer
else:
output_layer_cls = MulticlassOutputLayer
return output_layer_cls(list(labels), loss)
def from_config(
cls,
config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Iterable[str]] = None,
):
return cls(
loss_fn=create_loss(config.loss, ignore_index=-100),
ignore_impossible=config.ignore_impossible,
pos_loss_weight=config.pos_loss_weight,
has_answer_loss_weight=config.has_answer_loss_weight,
has_answer_labels=labels,
false_label=config.false_label,
)
def from_config(cls, config: Config, labels: Vocabulary):
vocab = list(labels)
vocab_dict = labels.idx
pad_token_idx = labels.idx.get(labels.pad_token,
Padding.DEFAULT_LABEL_PAD_IDX)
label_weights = (get_label_weights(vocab_dict, config.label_weights)
if config.label_weights else None)
return cls(
vocab,
create_loss(
config.loss,
weight=label_weights,
ignore_index=pad_token_idx
if config.ignore_pad_in_loss else -1,
),
)
def from_config(
cls,
config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Vocabulary] = None,
):
label_weights = (get_label_weights(labels.idx, config.label_weights)
if config.label_weights else None)
assert (
config.score_type == OutputScore.raw_cosine
or config.score_type == OutputScore.norm_cosine
or config.score_type == OutputScore.sigmoid_cosine
), f"Invalid score_type {config.score_type}. See OutputScore enum."
return cls(
list(labels),
create_loss(config.loss, weight=label_weights),
config.score_threshold,
config.score_type,
)
def from_config(
cls,
config,
metadata: Optional[FieldMeta] = None,
labels: Optional[Iterable[str]] = None,
is_kd: bool = False,
):
return cls(
loss_fn=create_loss(config.loss, ignore_index=-100),
ignore_impossible=config.ignore_impossible,
pos_loss_weight=config.pos_loss_weight,
has_answer_loss_weight=config.has_answer_loss_weight,
has_answer_labels=labels,
false_label=config.false_label,
max_answer_len=config.max_answer_len,
hard_weight=config.hard_weight,
use_zero_answer=config.use_zero_answer,
is_kd=is_kd,
)
def from_config(cls, config: Config, tensorizers: Dict[str, Tensorizer]):
labels = tensorizers["labels"].vocab
if not labels:
raise ValueError("Labels were not created, see preceding errors")
vocab = tensorizers["tokens"].vocab
encoder = create_module(
config.encoder, padding_idx=vocab.get_pad_index(), vocab_size=len(vocab)
)
if getattr(config, "use_selfie", False):
# No MLP fusion in SELFIE
dense_dim = 0
else:
dense_dim = tensorizers["dense"].dim if "dense" in tensorizers else 0
decoder = create_module(
config.decoder,
in_dim=encoder.representation_dim + dense_dim,
out_dim=len(labels),
)
label_weights = (
get_label_weights(labels.idx, config.output_layer.label_weights)
if config.output_layer.label_weights
else None
)
loss = create_loss(config.output_layer.loss, weight=label_weights)
if isinstance(loss, BinaryCrossEntropyLoss):
output_layer_cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
output_layer_cls = MultiLabelOutputLayer
else:
output_layer_cls = MulticlassOutputLayer
additional_kwargs = {}
if hasattr(config, "r3f_options"):
additional_kwargs["r3f_options"] = config.r3f_options
output_layer = output_layer_cls(list(labels), loss)
return cls(encoder, decoder, output_layer, **additional_kwargs)
def from_config(cls, config: Config, **kwargs):
return cls(create_loss(config.loss))
def from_config(cls, config):
return cls(None, create_loss(config.loss), config)
def from_config(cls, config: Config, tensorizers: Dict[str, Tensorizer]):
labels = tensorizers["labels"].vocab
if not labels:
raise ValueError("Labels were not created, see preceding errors")
if config.use_shared_embedding:
token_embedding = torch.nn.Embedding(
config.vocab_size, config.hidden_dim, padding_idx=config.padding_idx
)
else:
token_embedding = None
right_vocab = tensorizers["right_tokens"].vocab
right_encoder = create_module(
config.right_encoder,
token_embedding=token_embedding,
padding_idx=right_vocab.get_pad_index(),
vocab_size=len(right_vocab),
)
left_vocab = tensorizers["left_tokens"].vocab
left_encoder = create_module(
config.left_encoder,
token_embedding=token_embedding,
padding_idx=left_vocab.get_pad_index(),
vocab_size=len(left_vocab),
)
right_dense_dim = tensorizers["right_dense"].dim
left_dense_dim = tensorizers["left_dense"].dim
decoder = create_module(
config.decoder,
right_dim=right_encoder.representation_dim + right_dense_dim,
left_dim=left_encoder.representation_dim + left_dense_dim,
to_dim=len(labels),
)
label_weights = (
get_label_weights(labels.idx, config.output_layer.label_weights)
if config.output_layer.label_weights
else None
)
loss = create_loss(config.output_layer.loss, weight=label_weights)
if isinstance(loss, BinaryCrossEntropyLoss):
output_layer_cls = BinaryClassificationOutputLayer
elif isinstance(loss, MultiLabelSoftMarginLoss):
output_layer_cls = MultiLabelOutputLayer
else:
output_layer_cls = MulticlassOutputLayer
output_layer = output_layer_cls(list(labels), loss)
return cls(
right_encoder,
left_encoder,
decoder,
output_layer,
config.use_shared_encoder,
config.use_shared_embedding,
config.vocab_size,
config.hidden_dim,
config.padding_idx,
use_dense_in_decoder=config.use_dense_in_decoder,
)
def from_config(cls, config: Config, metadata: FieldMeta):
return cls(
metadata.vocab.itos,
create_loss(config.loss, ignore_index=metadata.pad_token_idx),
)
def from_config(cls, config, vocab, pad_token):
return cls(vocab, create_loss(config.loss, ignore_index=pad_token))
def from_config(cls, config: Config):
return cls(create_loss(config.loss), config.squash_to_unit_range)
def from_config(cls, config: Config, vocab: Vocabulary):
return cls(vocab._vocab, create_loss(config.loss,
vocab.get_pad_index()))
def from_config(cls, config, metadata: FieldMeta):
return cls(metadata.vocab.itos, create_loss(config.loss), config)
