def from_params(cls, vocab: Vocabulary, params: Params) -> 'SpanConstituencyParser':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params)
span_extractor = SpanExtractor.from_params(params.pop("span_extractor"))
encoder = Seq2SeqEncoder.from_params(params.pop("encoder"))
feed_forward_params = params.pop("feedforward", None)
if feed_forward_params is not None:
feedforward_layer = FeedForward.from_params(feed_forward_params)
else:
feedforward_layer = None
pos_tag_embedding_params = params.pop("pos_tag_embedding", None)
if pos_tag_embedding_params is not None:
pos_tag_embedding = Embedding.from_params(vocab, pos_tag_embedding_params)
else:
pos_tag_embedding = None
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
evalb_directory_path = params.pop("evalb_directory_path", None)
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
span_extractor=span_extractor,
encoder=encoder,
feedforward_layer=feedforward_layer,
pos_tag_embedding=pos_tag_embedding,
initializer=initializer,
regularizer=regularizer,
evalb_directory_path=evalb_directory_path)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'DecomposableAttention':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params)
premise_encoder_params = params.pop("premise_encoder", None)
if premise_encoder_params is not None:
premise_encoder = Seq2SeqEncoder.from_params(premise_encoder_params)
else:
premise_encoder = None
hypothesis_encoder_params = params.pop("hypothesis_encoder", None)
if hypothesis_encoder_params is not None:
hypothesis_encoder = Seq2SeqEncoder.from_params(hypothesis_encoder_params)
else:
hypothesis_encoder = None
attend_feedforward = FeedForward.from_params(params.pop('attend_feedforward'))
similarity_function = SimilarityFunction.from_params(params.pop("similarity_function"))
compare_feedforward = FeedForward.from_params(params.pop('compare_feedforward'))
aggregate_feedforward = FeedForward.from_params(params.pop('aggregate_feedforward'))
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
attend_feedforward=attend_feedforward,
similarity_function=similarity_function,
compare_feedforward=compare_feedforward,
aggregate_feedforward=aggregate_feedforward,
premise_encoder=premise_encoder,
hypothesis_encoder=hypothesis_encoder,
initializer=initializer,
regularizer=regularizer)
def test_regex_matches_are_initialized_correctly(self):
class Net(torch.nn.Module):
def __init__(self):
super(Net, self).__init__()
self.linear_1_with_funky_name = torch.nn.Linear(5, 10)
self.linear_2 = torch.nn.Linear(10, 5)
self.conv = torch.nn.Conv1d(5, 5, 5)
def forward(self, inputs): # pylint: disable=arguments-differ
pass
# pyhocon does funny things if there's a . in a key. This test makes sure that we
# handle these kinds of regexes correctly.
json_params = """{"initializer": [
["conv", {"type": "constant", "val": 5}],
["funky_na.*bi", {"type": "constant", "val": 7}]
]}
"""
params = Params(pyhocon.ConfigFactory.parse_string(json_params))
initializers = InitializerApplicator.from_params(params['initializer'])
model = Net()
initializers(model)
for parameter in model.conv.parameters():
assert torch.equal(parameter.data, torch.ones(parameter.size()) * 5)
parameter = model.linear_1_with_funky_name.bias
assert torch.equal(parameter.data, torch.ones(parameter.size()) * 7)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'SimpleTagger':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params)
encoder = Seq2SeqEncoder.from_params(params.pop("encoder"))
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
encoder=encoder,
initializer=initializer,
regularizer=regularizer)
def _get_applicator(self,
regex: str,
weights_file_path: str,
parameter_name_overrides: Optional[Dict[str, str]] = None) -> InitializerApplicator:
parameter_name_overrides = parameter_name_overrides or {}
initializer_params = Params({
"type": "pretrained",
"weights_file_path": weights_file_path,
"parameter_name_overrides": parameter_name_overrides
})
params = Params({
"initializer": [(regex, initializer_params)]
})
return InitializerApplicator.from_params(params["initializer"])
def from_params(cls, vocab: Vocabulary, params: Params) -> 'CrfTagger':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params)
encoder = Seq2SeqEncoder.from_params(params.pop("encoder"))
label_namespace = params.pop("label_namespace", "labels")
constraint_type = params.pop("constraint_type", None)
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
encoder=encoder,
label_namespace=label_namespace,
constraint_type=constraint_type,
initializer=initializer,
regularizer=regularizer)
def from_params(cls, vocab, params):
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params)
num_highway_layers = params.pop("num_highway_layers")
phrase_layer = Seq2SeqEncoder.from_params(params.pop("phrase_layer"))
similarity_function = SimilarityFunction.from_params(params.pop("similarity_function"))
modeling_layer = Seq2SeqEncoder.from_params(params.pop("modeling_layer"))
dropout = params.pop('dropout', 0.2)
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
mask_lstms = params.pop('mask_lstms', True)
params.assert_empty(cls.__name__)
return cls(vocab=vocab, text_field_embedder=text_field_embedder,
num_highway_layers=num_highway_layers, phrase_layer=phrase_layer,
attention_similarity_function=similarity_function, modeling_layer=modeling_layer,
dropout=dropout, mask_lstms=mask_lstms,
initializer=initializer, regularizer=regularizer)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'BiattentiveClassificationNetwork': # type: ignore
# pylint: disable=arguments-differ
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(vocab=vocab, params=embedder_params)
embedding_dropout = params.pop("embedding_dropout")
pre_encode_feedforward = FeedForward.from_params(params.pop("pre_encode_feedforward"))
encoder = Seq2SeqEncoder.from_params(params.pop("encoder"))
integrator = Seq2SeqEncoder.from_params(params.pop("integrator"))
integrator_dropout = params.pop("integrator_dropout")
output_layer_params = params.pop("output_layer")
if "activations" in output_layer_params:
output_layer = FeedForward.from_params(output_layer_params)
else:
output_layer = Maxout.from_params(output_layer_params)
elmo = params.pop("elmo", None)
if elmo is not None:
elmo = Elmo.from_params(elmo)
use_input_elmo = params.pop_bool("use_input_elmo", False)
use_integrator_output_elmo = params.pop_bool("use_integrator_output_elmo", False)
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
embedding_dropout=embedding_dropout,
pre_encode_feedforward=pre_encode_feedforward,
encoder=encoder,
integrator=integrator,
integrator_dropout=integrator_dropout,
output_layer=output_layer,
elmo=elmo,
use_input_elmo=use_input_elmo,
use_integrator_output_elmo=use_integrator_output_elmo,
initializer=initializer,
regularizer=regularizer)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
binary_feature_dim: int,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
label_smoothing: float = None,
ignore_span_metric: bool = False) -> None:
super(SemanticRoleLabeler, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SpanBasedF1Measure(vocab,
tag_namespace="labels",
ignore_classes=["V"])
self.encoder = encoder
# There are exactly 2 binary features for the verb predicate embedding.
self.binary_feature_embedding = Embedding(2, binary_feature_dim)
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
check_dimensions_match(
text_field_embedder.get_output_dim() + binary_feature_dim,
encoder.get_input_dim(),
"text embedding dim + verb indicator embedding dim",
"encoder input dim")
initializer(self)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'HierarchicalCRF':
embedder_params = params.pop('text_field_embedder')
text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params)
inner_encoder = Seq2VecEncoder.from_params(params.pop('inner_encoder'))
outer_encoder = Seq2SeqEncoder.from_params(params.pop('outer_encoder'))
label_namespace = params.pop('label_namespace', 'labels')
constraint_type = params.pop('constraint_type', None)
dropout = params.pop('dropout', None)
include_start_end_transitions = params.pop('include_start_end_transitions', True)
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
inner_encoder=inner_encoder,
outer_encoder=outer_encoder,
label_namespace=label_namespace,
constraint_type=constraint_type,
dropout=dropout,
include_start_end_transitions=include_start_end_transitions,
initializer=initializer,
regularizer=regularizer)
def from_params(cls, vocab: Vocabulary,
params: Params) -> 'BidirectionalAttentionFlow':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(
vocab, embedder_params)
num_highway_layers = params.pop("num_highway_layers")
phrase_layer = Seq2SeqEncoder.from_params(params.pop("phrase_layer"))
similarity_function = SimilarityFunction.from_params(
params.pop("similarity_function"))
modeling_layer = Seq2SeqEncoder.from_params(
params.pop("modeling_layer"))
span_end_encoder = Seq2SeqEncoder.from_params(
params.pop("span_end_encoder"))
initializer = InitializerApplicator.from_params(
params.pop("initializer", []))
dropout = params.pop('dropout', 0.2)
# TODO: Remove the following when fully deprecated
evaluation_json_file = params.pop('evaluation_json_file', None)
if evaluation_json_file is not None:
logger.warning(
"the 'evaluation_json_file' model parameter is deprecated, please remove"
)
mask_lstms = params.pop('mask_lstms', True)
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
num_highway_layers=num_highway_layers,
phrase_layer=phrase_layer,
attention_similarity_function=similarity_function,
modeling_layer=modeling_layer,
span_end_encoder=span_end_encoder,
initializer=initializer,
dropout=dropout,
mask_lstms=mask_lstms)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'BIOLabeler':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(vocab=vocab, params=embedder_params)
stacked_encoder = Seq2SeqEncoder.from_params(params.pop("stacked_encoder"))
predicate_feature_dim = params.pop("predicate_feature_dim", 100)
dim_hidden = params.pop("hidden_dim", 100)
question_generator = QuestionGenerator.from_params(vocab, params.pop("question_generator"))
initializer = InitializerApplicator.from_params(params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(params.pop('regularizer', []))
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
stacked_encoder=stacked_encoder,
question_generator=question_generator,
predicate_feature_dim=predicate_feature_dim,
dim_hidden=dim_hidden,
initializer=initializer,
regularizer=regularizer)
def __init__(
self,
vocab: Vocabulary,
bert_model: str,
dropout: float = 0.0,
requires_grad: str = "none",
pos_weight: float = 1.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
):
super(BernoulliBertGenerator, self).__init__(vocab, initializer,
regularizer)
self._vocabulary = vocab
self._bert_model = AutoModel.from_pretrained(bert_model)
self._dropout = torch.nn.Dropout(p=dropout)
self._classification_layer = torch.nn.Linear(
self._bert_model.config.hidden_size, 1, bias=False)
if requires_grad in ["none", "all"]:
for param in self._bert_model.parameters():
param.requires_grad = requires_grad == "all"
else:
model_name_regexes = requires_grad.split(",")
for name, param in self._bert_model.named_parameters():
found = any([regex in name for regex in model_name_regexes])
param.requires_grad = found
for n, v in self._bert_model.named_parameters():
if n.startswith("classifier"):
v.requires_grad = True
self._pos_weight = 1.0 / pos_weight - 1
self._token_prf = F1Measure(1)
initializer(self)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'Tagger':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(
vocab, embedder_params)
stacked_encoder = Seq2SeqEncoder.from_params(
params.pop("stacked_encoder"))
tasks = params.pop("tasks")
source_namespace = params.pop("source_namespace", "tokens")
label_namespace = params.pop("label_namespace", "labels")
is_crf = params.pop("is_crf", False)
initializer = InitializerApplicator.from_params(
params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(
params.pop('regularizer', []))
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
stacked_encoder=stacked_encoder,
tasks=tasks,
source_namespace=source_namespace,
label_namespace=label_namespace,
is_crf=is_crf,
initializer=initializer,
regularizer=regularizer)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2VecEncoder,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(SequenceClassifier, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
self.encoder = encoder
self.projection_layer = Linear(self.encoder.get_output_dim(),
self.num_classes)
if text_field_embedder.get_output_dim() != encoder.get_input_dim():
raise ConfigurationError(
"The output dimension of the text_field_embedder must match the "
"input dimension of the sequence encoder. Found {} and {}, "
"respectively.".format(text_field_embedder.get_output_dim(),
encoder.get_input_dim()))
self._accuracy = CategoricalAccuracy()
self._loss = torch.nn.CrossEntropyLoss()
initializer(self)
def __init__(self,
vocab: Vocabulary,
input_size: int,
hidden_size: int,
label_namespace: str = "labels",
negative_label: str = "0",
evaluated_labels: List[str] = None,
loss_gamma: float = None,
label_smoothing: float = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self._idx_to_label = self.vocab.get_index_to_token_vocabulary(
label_namespace)
self._num_pos_classes = self.vocab.get_vocab_size(label_namespace) - 1
self._negative_label_idx = self.vocab.get_token_index(
negative_label, namespace=label_namespace)
assert self._negative_label_idx == 0, "negative label idx is supposed to be 0"
self._loss_gamma = loss_gamma
self._label_smoothing = label_smoothing
# projection layers
self._U = Linear(input_size, hidden_size, bias=False)
self._W = Linear(input_size, hidden_size, bias=False)
self._V = Linear(hidden_size, self._num_pos_classes, bias=False)
self._T = Linear(hidden_size, 1, bias=False)
self._b = Parameter(torch.Tensor(hidden_size))
nn.init.normal_(self._b)
self._accuracy = CategoricalAccuracy()
self._f1_metric = F1Measure(vocabulary=vocab,
negative_label=negative_label,
label_namespace=label_namespace,
evaluated_labels=evaluated_labels)
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
document_encoder: Seq2SeqEncoder,
word_attention: Seq2SeqEncoder,
output_logit: FeedForward,
pre_document_encoder_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
self._word_attention = word_attention
self._pre_document_encoder_dropout = torch.nn.Dropout(
p=pre_document_encoder_dropout)
self._document_encoder = document_encoder
self._output_logit = output_logit
self._num_labels = vocab.get_vocab_size(namespace="labels")
self._accuracy = CategoricalAccuracy()
self._loss = torch.nn.CrossEntropyLoss()
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
dropout: float = 0.0,
max_span_length: int = 30,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
self._max_span_length = max_span_length
self.qa_outputs = torch.nn.Linear(
self._text_field_embedder.get_output_dim(), 2)
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._span_accuracy = BooleanAccuracy()
self._span_qa_metrics = SquadEmAndF1()
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
initializer(self)
def __init__(self,
vocab,
text_field_embedder,
encoder,
initializer=InitializerApplicator(),
regularizer=None):
super(SimpleTagger, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size(u"labels")
self.encoder = encoder
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_classes))
check_dimensions_match(text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
u"text field embedding dim",
u"encoder input dim")
self.metrics = {
u"accuracy": CategoricalAccuracy(),
u"accuracy3": CategoricalAccuracy(top_k=3)
}
initializer(self)
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
stacked_encoder: Seq2SeqEncoder,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(SimpleTagger, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
self.stacked_encoder = stacked_encoder
self.tag_projection_layer = TimeDistributed(Linear(self.stacked_encoder.get_output_dim(),
self.num_classes))
if text_field_embedder.get_output_dim() != stacked_encoder.get_input_dim():
raise ConfigurationError("The output dimension of the text_field_embedder must match the "
"input dimension of the phrase_encoder. Found {} and {}, "
"respectively.".format(text_field_embedder.get_output_dim(),
stacked_encoder.get_input_dim()))
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3)
}
initializer(self)
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
stacked_encoder: Seq2SeqEncoder,
predicate_feature_dim: int,
dim_hidden: int = 100,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None):
super(SpanDetector, self).__init__(vocab, regularizer)
self.dim_hidden = dim_hidden
self.text_field_embedder = text_field_embedder
self.predicate_feature_embedding = Embedding(2, predicate_feature_dim)
self.embedding_dropout = Dropout(p=embedding_dropout)
self.threshold_metric = ThresholdMetric()
self.stacked_encoder = stacked_encoder
self.span_hidden = SpanRepAssembly(self.stacked_encoder.get_output_dim(), self.stacked_encoder.get_output_dim(),
self.dim_hidden)
self.pred = TimeDistributed(Linear(self.dim_hidden, 1))
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
similarity_function: SimilarityFunction,
projection_feedforward: FeedForward,
inference_encoder: Seq2SeqEncoder,
output_feedforward: FeedForwardPair,
dropout: float = 0.5,
margin: float = 1.25,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
self._encoder = encoder
self._matrix_attention = LegacyMatrixAttention(similarity_function)
self._projection_feedforward = projection_feedforward
self._inference_encoder = inference_encoder
if dropout:
self.dropout = torch.nn.Dropout(dropout)
self.rnn_input_dropout = InputVariationalDropout(dropout)
else:
self.dropout = None
self.rnn_input_dropout = None
self._output_feedforward = output_feedforward
self._margin = margin
self._accuracy = BooleanAccuracy()
initializer(self)
def from_params(cls, vocab: Vocabulary,
params: Params) -> 'SpanConstituencyParser':
embedder_params = params.pop("text_field_embedder")
text_field_embedder = TextFieldEmbedder.from_params(
vocab, embedder_params)
span_extractor = SpanExtractor.from_params(
params.pop("span_extractor"))
encoder = Seq2SeqEncoder.from_params(params.pop("encoder"))
feed_forward_params = params.pop("feedforward", None)
if feed_forward_params is not None:
feedforward_layer = FeedForward.from_params(feed_forward_params)
else:
feedforward_layer = None
pos_tag_embedding_params = params.pop("pos_tag_embedding", None)
if pos_tag_embedding_params is not None:
pos_tag_embedding = Embedding.from_params(
vocab, pos_tag_embedding_params)
else:
pos_tag_embedding = None
initializer = InitializerApplicator.from_params(
params.pop('initializer', []))
regularizer = RegularizerApplicator.from_params(
params.pop('regularizer', []))
evalb_directory_path = params.pop("evalb_directory_path", None)
params.assert_empty(cls.__name__)
return cls(vocab=vocab,
text_field_embedder=text_field_embedder,
span_extractor=span_extractor,
encoder=encoder,
feedforward_layer=feedforward_layer,
pos_tag_embedding=pos_tag_embedding,
initializer=initializer,
regularizer=regularizer,
evalb_directory_path=evalb_directory_path)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
include_start_end_transitions: bool = True,
dropout: Optional[float] = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self.label_namespace = 'labels'
self.num_tags = self.vocab.get_vocab_size(self.label_namespace)
# encode text
self.text_field_embedder = text_field_embedder
self.encoder = encoder
self.dropout = torch.nn.Dropout(dropout) if dropout else None
# crf
output_dim = self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(
Linear(output_dim, self.num_tags))
self.crf = ConditionalRandomField(
self.num_tags,
constraints=None,
include_start_end_transitions=include_start_end_transitions)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3)
}
for index, label in self.vocab.get_index_to_token_vocabulary(
self.label_namespace).items():
self.metrics['F1_' + label] = F1Measure(positive_label=index)
initializer(self)
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
sentence_encoder: Optional[Seq2SeqEncoder] = None,
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(SeqLabeler, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=['SENT_BOUND'])
# Metrics just for specific classes
all_labels = self.vocab.get_index_to_token_vocabulary("labels").values()
logger.info(f'Labels: {all_labels}')
mwe_labels = [label.replace('B-', '').replace('I-', '') for label in all_labels if 'MWE' in label]
self.mwe_span_metric = None
if len(mwe_labels) > 0:
logger.info(f'Instantiating MWE only evaluation: {mwe_labels}')
self.mwe_span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels",
ignore_classes=ne_labels + ['SENT_BOUND'])
else:
logger.info('No MWE only evaluation instantiated.')
self.sentence_encoder = sentence_encoder
# Encode the sentence with a biLSTM / self-attention
if sentence_encoder:
check_dimensions_match(text_field_embedder.get_output_dim(), sentence_encoder.get_input_dim(),
"text embedding dim", "encoder input dim")
self.tag_projection_layer = TimeDistributed(Linear(self.sentence_encoder.get_output_dim(), self.num_classes))
# Just use embeddings
else:
self.tag_projection_layer = TimeDistributed(Linear(text_field_embedder.get_output_dim(), self.num_classes))
InitializerApplicator()(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
fc_dim: int,
nonlinear_fc: bool,
dropout_fc: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(NLIEncoder, self).__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
self._encoder = encoder
self._fc_dim = fc_dim
self._nonlinear_fc = nonlinear_fc
self._dropout_fc = dropout_fc
if nonlinear_fc:
self._classifier = nn.Sequential(
nn.Dropout(p=dropout_fc),
nn.Linear(4 * encoder.get_output_dim(), fc_dim),
nn.Tanh(),
nn.Dropout(p=dropout_fc),
nn.Linear(fc_dim, fc_dim),
nn.Tanh(),
nn.Dropout(p=dropout_fc),
nn.Linear(fc_dim, 3),
)
else:
self._classifier = nn.Sequential(
nn.Linear(4 * encoder.get_output_dim(), fc_dim),
nn.Linear(fc_dim, fc_dim), nn.Linear(fc_dim, 3))
self._num_labels = vocab.get_vocab_size(namespace="labels")
self._accuracy = CategoricalAccuracy()
self._loss = torch.nn.CrossEntropyLoss()
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
seq2vec_encoder: Seq2VecEncoder,
seq2seq_encoder: Seq2SeqEncoder = None,
dropout: float = None,
num_labels: int = None,
label_namespace: str = "labels",
initializer: InitializerApplicator = InitializerApplicator()) -> None:
super().__init__(vocab)
self._text_field_embedder = text_field_embedder
if seq2seq_encoder:
self._seq2seq_encoder = seq2seq_encoder
else:
self._seq2seq_encoder = None
self._seq2vec_encoder = seq2vec_encoder
self._classifier_input_dim = self._seq2vec_encoder.get_output_dim()
if dropout:
self._dropout = torch.nn.Dropout(dropout)
else:
self._dropout = None
self._label_namespace = label_namespace
if num_labels:
self._num_labels = num_labels
else:
self._num_labels = vocab.get_vocab_size(namespace=self._label_namespace)
self._classification_layer = torch.nn.Linear(self._classifier_input_dim, self._num_labels)
self._accuracy = CategoricalAccuracy()
self._loss = torch.nn.CrossEntropyLoss()
initializer(self)
def __init__(
self,
vocab: Vocabulary,
bert_model: str,
dropout: float = 0.0,
requires_grad: str = "none",
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
):
super(BertClassifier, self).__init__(vocab, initializer, regularizer)
self._vocabulary = vocab
self._num_labels = self._vocabulary.get_vocab_size("labels")
self._bert_config = AutoConfig.from_pretrained(bert_model,
output_attentions=True)
self._bert_model = AutoModel.from_pretrained(bert_model,
config=self._bert_config)
# self._bert_model = Longformer.from_pretrained(bert_model, output_attentions=True)
self._dropout = nn.Dropout(dropout)
self._classifier = nn.Linear(self._bert_model.config.hidden_size,
self._num_labels)
self.embedding_layers = [self._bert_model.embeddings]
if requires_grad in ["none", "all"]:
for param in self._bert_model.parameters():
param.requires_grad = requires_grad == "all"
else:
model_name_regexes = requires_grad.split(",")
for name, param in self._bert_model.named_parameters():
found = any([regex in name for regex in model_name_regexes])
param.requires_grad = found
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
left_text_encoder: Seq2VecEncoder,
right_text_encoder: Seq2VecEncoder,
feedforward: Optional[FeedForward] = None,
target_field_embedder: Optional[TextFieldEmbedder] = None,
target_encoder: Optional[Seq2VecEncoder] = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
word_dropout: float = 0.0,
dropout: float = 0.0) -> None:
super().__init__(vocab, regularizer)
'''
:param vocab: vocab : A Vocabulary, required in order to compute sizes
for input/output projections.
:param text_field_embedder: Used to embed the text and target text if
target_field_embedder is None but the
target_encoder is not None.
:param left_text_encoder: Encoder that will create the representation
of the tokens left of the target and
the target itself if included from the
dataset reader.
:param right_text_encoder: Encoder that will create the representation
of the tokens right of the target and the
target itself if included from the
dataset reader.
:param feedforward: An optional feed forward layer to apply after the
encoder.
:param target_field_embedder: Used to embed the target text to give as
input to the target_encoder. Thus this
allows a seperate embedding for text and
target text.
:param target_encoder: Encoder that will create the representation of
target text tokens.
:param initializer: Used to initialize the model parameters.
:param regularizer: If provided, will be used to calculate the
regularization penalty during training.
:param word_dropout: Dropout that is applied after the embedding of the
tokens/words. It will drop entire words with this
probabilty.
:param dropout: To apply dropout after each layer apart from the last
layer. All dropout that is applied to timebased data
will be `variational dropout`_ all else will be
standard dropout.
Without the target encoder this will be the standard TDLSTM method
from `Effective LSTM's for Target-Dependent Sentiment classification`_
. With the target encoder this will then become the TCLSTM method
from `Effective LSTM's for Target-Dependent Sentiment classification`_.
.. _variational dropout:
https://papers.nips.cc/paper/6241-a-theoretically-grounded-application-of-dropout-in-recurrent-neural-networks.pdf
.. _Effective LSTM's for Target-Dependent Sentiment classification:
https://aclanthology.coli.uni-saarland.de/papers/C16-1311/c16-1311
'''
self.text_field_embedder = text_field_embedder
self.target_field_embedder = target_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
self.left_text_encoder = left_text_encoder
self.right_text_encoder = right_text_encoder
self.target_encoder = target_encoder
self.feedforward = feedforward
if feedforward is not None:
output_dim = self.feedforward.get_output_dim()
else:
left_out_dim = self.left_text_encoder.get_output_dim()
right_out_dim = self.right_text_encoder.get_output_dim()
output_dim = left_out_dim + right_out_dim
self.label_projection = Linear(output_dim, self.num_classes)
self.metrics = {"accuracy": CategoricalAccuracy()}
self.f1_metrics = {}
# F1 Scores
label_index_name = self.vocab.get_index_to_token_vocabulary('labels')
for label_index, label_name in label_index_name.items():
label_name = f'F1_{label_name.capitalize()}'
self.f1_metrics[label_name] = F1Measure(label_index)
# Dropout
self._word_dropout = WordDrouput(word_dropout)
self._variational_dropout = InputVariationalDropout(dropout)
self._naive_dropout = Dropout(dropout)
self.loss = torch.nn.CrossEntropyLoss()
# Ensure that the input to the right_text_encoder and left_text_encoder
# is the size of the target encoder output plus the size of the text
# embedding output.
if self.target_encoder:
right_text_out_dim = self.right_text_encoder.get_input_dim()
left_text_out_dim = self.left_text_encoder.get_input_dim()
target_dim = self.target_encoder.get_output_dim()
text_dim = self.text_field_embedder.get_output_dim()
total_out_dim = target_dim + text_dim
config_err_msg = (
"As the target is being encoded the output of the"
" target encoder is concatenated onto each word "
" vector for the left and right contexts "
"therefore the input of the right_text_encoder"
"/left_text_encoder is the output dimension of "
"the target encoder + the dimension of the word "
"embeddings for the left and right contexts.")
if (total_out_dim != right_text_out_dim
or total_out_dim != left_text_out_dim):
raise ConfigurationError(config_err_msg)
# Ensure that the target field embedder has an output dimension the
# same as the input dimension to the target encoder.
if self.target_encoder and self.target_field_embedder:
target_embed_out = self.target_field_embedder.get_output_dim()
target_in = self.target_encoder.get_input_dim()
config_embed_err_msg = ("The Target field embedder should have"
" the same output size "
f"{target_embed_out} as the input to "
f"the target encoder {target_in}")
if target_embed_out != target_in:
raise ConfigurationError(config_embed_err_msg)
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
embedding_dropout: float,
pre_encode_feedforward: FeedForward,
encoder: Seq2SeqEncoder,
integrator: Seq2SeqEncoder,
integrator_dropout: float,
output_layer: Union[FeedForward, Maxout],
elmo: Elmo = None,
use_input_elmo: bool = False,
use_integrator_output_elmo: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
self._text_field_embedder = text_field_embedder
if "elmo" in self._text_field_embedder._token_embedders.keys():
raise ConfigurationError(
"To use ELMo in the BiattentiveClassificationNetwork input, "
"remove elmo from the text_field_embedder and pass an "
"Elmo object to the BiattentiveClassificationNetwork and set the "
"'use_input_elmo' and 'use_integrator_output_elmo' flags accordingly."
)
self._embedding_dropout = nn.Dropout(embedding_dropout)
self._num_classes = self.vocab.get_vocab_size("labels")
self._pre_encode_feedforward = pre_encode_feedforward
self._encoder = encoder
self._integrator = integrator
self._integrator_dropout = nn.Dropout(integrator_dropout)
self._elmo = elmo
self._use_input_elmo = use_input_elmo
self._use_integrator_output_elmo = use_integrator_output_elmo
self._num_elmo_layers = int(self._use_input_elmo) + int(
self._use_integrator_output_elmo)
# Check that, if elmo is None, none of the elmo flags are set.
if self._elmo is None and self._num_elmo_layers != 0:
raise ConfigurationError(
"One of 'use_input_elmo' or 'use_integrator_output_elmo' is True, "
"but no Elmo object was provided upon construction. Pass in an Elmo "
"object to use Elmo.")
if self._elmo is not None:
# Check that, if elmo is not None, we use it somewhere.
if self._num_elmo_layers == 0:
raise ConfigurationError(
"Elmo object provided upon construction, but both 'use_input_elmo' "
"and 'use_integrator_output_elmo' are 'False'. Set one of them to "
"'True' to use Elmo, or do not provide an Elmo object upon construction."
)
# Check that the number of flags set is equal to the num_output_representations of the Elmo object
if len(self._elmo._scalar_mixes) != self._num_elmo_layers:
raise ConfigurationError(
f"Elmo object has num_output_representations={len(self._elmo._scalar_mixes)}, but this "
f"does not match the number of use_*_elmo flags set to true. use_input_elmo "
f"is {self._use_input_elmo}, and use_integrator_output_elmo "
f"is {self._use_integrator_output_elmo}")
# Calculate combined integrator output dim, taking into account elmo
if self._use_integrator_output_elmo:
self._combined_integrator_output_dim = (
self._integrator.get_output_dim() +
self._elmo.get_output_dim())
else:
self._combined_integrator_output_dim = self._integrator.get_output_dim(
)
self._self_attentive_pooling_projection = nn.Linear(
self._combined_integrator_output_dim, 1)
self._output_layer = output_layer
if self._use_input_elmo:
check_dimensions_match(
text_field_embedder.get_output_dim() +
self._elmo.get_output_dim(),
self._pre_encode_feedforward.get_input_dim(),
"text field embedder output dim + ELMo output dim",
"Pre-encoder feedforward input dim",
)
else:
check_dimensions_match(
text_field_embedder.get_output_dim(),
self._pre_encode_feedforward.get_input_dim(),
"text field embedder output dim",
"Pre-encoder feedforward input dim",
)
check_dimensions_match(
self._pre_encode_feedforward.get_output_dim(),
self._encoder.get_input_dim(),
"Pre-encoder feedforward output dim",
"Encoder input dim",
)
check_dimensions_match(
self._encoder.get_output_dim() * 3,
self._integrator.get_input_dim(),
"Encoder output dim * 3",
"Integrator input dim",
)
if self._use_integrator_output_elmo:
check_dimensions_match(
self._combined_integrator_output_dim * 4,
self._output_layer.get_input_dim(),
"(Integrator output dim + ELMo output dim) * 4",
"Output layer input dim",
)
else:
check_dimensions_match(
self._integrator.get_output_dim() * 4,
self._output_layer.get_input_dim(),
"Integrator output dim * 4",
"Output layer input dim",
)
check_dimensions_match(
self._output_layer.get_output_dim(),
self._num_classes,
"Output layer output dim",
"Number of classes.",
)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3),
}
self.loss = torch.nn.CrossEntropyLoss()
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
context_layer: Seq2SeqEncoder,
mention_feedforward: FeedForward,
antecedent_feedforward: FeedForward,
feature_size: int,
max_span_width: int,
spans_per_word: float,
max_antecedents: int,
coarse_to_fine: bool = False,
inference_order: int = 1,
lexical_dropout: float = 0.2,
initializer: InitializerApplicator = InitializerApplicator(),
**kwargs
) -> None:
super().__init__(vocab, **kwargs)
self._text_field_embedder = text_field_embedder
self._context_layer = context_layer
self._mention_feedforward = TimeDistributed(mention_feedforward)
self._mention_scorer = TimeDistributed(
torch.nn.Linear(mention_feedforward.get_output_dim(), 1)
)
self._antecedent_feedforward = TimeDistributed(antecedent_feedforward)
self._antecedent_scorer = TimeDistributed(
torch.nn.Linear(antecedent_feedforward.get_output_dim(), 1)
)
self._endpoint_span_extractor = EndpointSpanExtractor(
context_layer.get_output_dim(),
combination="x,y",
num_width_embeddings=max_span_width,
span_width_embedding_dim=feature_size,
bucket_widths=False,
)
self._attentive_span_extractor = SelfAttentiveSpanExtractor(
input_dim=text_field_embedder.get_output_dim()
)
# 10 possible distance buckets.
self._num_distance_buckets = 10
self._distance_embedding = Embedding(
embedding_dim=feature_size, num_embeddings=self._num_distance_buckets
)
self._max_span_width = max_span_width
self._spans_per_word = spans_per_word
self._max_antecedents = max_antecedents
self._coarse_to_fine = coarse_to_fine
if self._coarse_to_fine:
self._coarse2fine_scorer = torch.nn.Linear(
mention_feedforward.get_input_dim(), mention_feedforward.get_input_dim()
)
self._inference_order = inference_order
if self._inference_order > 1:
self._span_updating_gated_sum = GatedSum(mention_feedforward.get_input_dim())
self._mention_recall = MentionRecall()
self._conll_coref_scores = ConllCorefScores()
if lexical_dropout > 0:
self._lexical_dropout = torch.nn.Dropout(p=lexical_dropout)
else:
self._lexical_dropout = lambda x: x
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
label_namespace: str = "labels",
feedforward: Optional[FeedForward] = None,
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
constrain_crf_decoding: bool = None,
calculate_span_f1: bool = None,
dropout: Optional[float] = None,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
top_k: int = 1,
) -> None:
super().__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self.top_k = top_k
self._verbose_metrics = verbose_metrics
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self._feedforward = feedforward
if feedforward is not None:
output_dim = feedforward.get_output_dim()
else:
output_dim = self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(
Linear(output_dim, self.num_tags))
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError("constrain_crf_decoding is True, but "
"no label_encoding was specified.")
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_tags,
constraints,
include_start_end_transitions=include_start_end_transitions)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3),
}
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace,
label_encoding=label_encoding)
check_dimensions_match(
text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim",
)
if feedforward is not None:
check_dimensions_match(
encoder.get_output_dim(),
feedforward.get_input_dim(),
"encoder output dim",
"feedforward input dim",
)
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
tag_representation_dim: int,
arc_representation_dim: int,
tag_feedforward: FeedForward = None,
arc_feedforward: FeedForward = None,
lemma_tag_embedding: Embedding = None,
upos_tag_embedding: Embedding = None,
xpos_tag_embedding: Embedding = None,
feats_tag_embedding: Embedding = None,
head_information_embedding: Embedding = None,
head_tag_embedding: Embedding = None,
dropout: float = 0.0,
input_dropout: float = 0.0,
edge_prediction_threshold: float = 0.5,
initializer: InitializerApplicator = InitializerApplicator(),
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
self.text_field_embedder = text_field_embedder
self.encoder = encoder
self.edge_prediction_threshold = edge_prediction_threshold
if not 0 < edge_prediction_threshold < 1:
raise ConfigurationError(
f"edge_prediction_threshold must be between "
f"0 and 1 (exclusive) but found {edge_prediction_threshold}.")
encoder_dim = encoder.get_output_dim()
self.head_arc_feedforward = arc_feedforward or FeedForward(
encoder_dim, 1, arc_representation_dim,
Activation.by_name("elu")())
self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward)
self.arc_attention = BilinearMatrixAttention(arc_representation_dim,
arc_representation_dim,
use_input_biases=True)
num_labels = self.vocab.get_vocab_size("deps")
self.head_tag_feedforward = tag_feedforward or FeedForward(
encoder_dim, 1, tag_representation_dim,
Activation.by_name("elu")())
self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward)
self.tag_bilinear = BilinearMatrixAttention(tag_representation_dim,
tag_representation_dim,
label_dim=num_labels)
self._lemma_tag_embedding = lemma_tag_embedding or None
self._upos_tag_embedding = upos_tag_embedding or None
self._xpos_tag_embedding = xpos_tag_embedding or None
self._feats_tag_embedding = feats_tag_embedding or None
self._head_tag_embedding = head_tag_embedding or None
self._head_information_embedding = head_information_embedding or None
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
# add a head sentinel to accommodate for extra root token in EUD graphs
self._head_sentinel = torch.nn.Parameter(
torch.randn([1, 1, encoder.get_output_dim()]))
representation_dim = text_field_embedder.get_output_dim()
if lemma_tag_embedding is not None:
representation_dim += lemma_tag_embedding.get_output_dim()
if upos_tag_embedding is not None:
representation_dim += upos_tag_embedding.get_output_dim()
if xpos_tag_embedding is not None:
representation_dim += xpos_tag_embedding.get_output_dim()
if feats_tag_embedding is not None:
representation_dim += feats_tag_embedding.get_output_dim()
if head_tag_embedding is not None:
representation_dim += head_tag_embedding.get_output_dim()
if head_information_embedding is not None:
representation_dim += head_information_embedding.get_output_dim()
check_dimensions_match(
representation_dim,
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim",
)
check_dimensions_match(
tag_representation_dim,
self.head_tag_feedforward.get_output_dim(),
"tag representation dim",
"tag feedforward output dim",
)
check_dimensions_match(
arc_representation_dim,
self.head_arc_feedforward.get_output_dim(),
"arc representation dim",
"arc feedforward output dim",
)
self._enhanced_attachment_scores = EnhancedAttachmentScores()
self._arc_loss = torch.nn.BCEWithLogitsLoss(reduction="none")
self._tag_loss = torch.nn.CrossEntropyLoss(reduction="none")
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
num_highway_layers: int,
phrase_layer: Seq2SeqEncoder,
matrix_attention: MatrixAttention,
modeling_layer: Seq2SeqEncoder,
span_end_encoder: Seq2SeqEncoder,
dropout: float = 0.2,
mask_lstms: bool = True,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
) -> None:
super().__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
self._highway_layer = TimeDistributed(
Highway(text_field_embedder.get_output_dim(), num_highway_layers))
self._phrase_layer = phrase_layer
self._matrix_attention = matrix_attention
self._modeling_layer = modeling_layer
self._span_end_encoder = span_end_encoder
encoding_dim = phrase_layer.get_output_dim()
modeling_dim = modeling_layer.get_output_dim()
span_start_input_dim = encoding_dim * 4 + modeling_dim
self._span_start_predictor = TimeDistributed(
torch.nn.Linear(span_start_input_dim, 1))
span_end_encoding_dim = span_end_encoder.get_output_dim()
span_end_input_dim = encoding_dim * 4 + span_end_encoding_dim
self._span_end_predictor = TimeDistributed(
torch.nn.Linear(span_end_input_dim, 1))
# Bidaf has lots of layer dimensions which need to match up - these aren't necessarily
# obvious from the configuration files, so we check here.
check_dimensions_match(
modeling_layer.get_input_dim(),
4 * encoding_dim,
"modeling layer input dim",
"4 * encoding dim",
)
check_dimensions_match(
text_field_embedder.get_output_dim(),
phrase_layer.get_input_dim(),
"text field embedder output dim",
"phrase layer input dim",
)
check_dimensions_match(
span_end_encoder.get_input_dim(),
4 * encoding_dim + 3 * modeling_dim,
"span end encoder input dim",
"4 * encoding dim + 3 * modeling dim",
)
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._span_accuracy = BooleanAccuracy()
self._squad_metrics = SquadEmAndF1()
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._mask_lstms = mask_lstms
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
relation_scorer: RelationScorer,
ner_tag_namespace: str = 'tags',
evaluated_ner_labels: List[str] = None,
re_loss_weight: float = 1.0,
ner_tag_embedder: TokenEmbedder = None,
use_aux_ner_labels: bool = False,
aux_coarse_namespace: str = 'coarse_tags',
aux_modifier_namespace: str = 'modifier_tags',
aux_loss_weight: float = 1.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab=vocab, regularizer=regularizer)
self.text_field_embedder = text_field_embedder
self.encoder = encoder
# NER subtask 2
self._ner_label_encoding = 'BIO'
self._ner_tag_namespace = ner_tag_namespace
ner_input_dim = self.encoder.get_output_dim()
num_ner_tags = self.vocab.get_vocab_size(ner_tag_namespace)
self.tag_projection_layer = TimeDistributed(
Linear(ner_input_dim, num_ner_tags))
self._use_aux_ner_labels = use_aux_ner_labels
if self._use_aux_ner_labels:
self._coarse_tag_namespace = aux_coarse_namespace
self._num_coarse_tags = self.vocab.get_vocab_size(
self._coarse_tag_namespace)
self._coarse_projection_layer = TimeDistributed(
Linear(ner_input_dim, self._num_coarse_tags))
self._modifier_tag_namespace = aux_modifier_namespace
self._num_modifier_tags = self.vocab.get_vocab_size(
self._modifier_tag_namespace)
self._modifier_projection_layer = TimeDistributed(
Linear(ner_input_dim, self._num_modifier_tags))
self._coarse_acc = CategoricalAccuracy()
self._modifier_acc = CategoricalAccuracy()
self._aux_loss_weight = aux_loss_weight
self.ner_accuracy = CategoricalAccuracy()
if evaluated_ner_labels is None:
ignored_classes = None
else:
assert self._ner_label_encoding == 'BIO', 'expected BIO encoding'
all_ner_tags = self.vocab.get_token_to_index_vocabulary(
ner_tag_namespace).keys()
ner_tag_classes = set(
[bio_tag[2:] for bio_tag in all_ner_tags if len(bio_tag) > 2])
ignored_classes = list(
set(ner_tag_classes).difference(evaluated_ner_labels))
self.ner_f1 = SpanBasedF1Measure(
vocabulary=vocab,
tag_namespace=ner_tag_namespace,
label_encoding=self._ner_label_encoding,
ignore_classes=ignored_classes)
# Use constrained crf decoding with the BIO labeling scheme
ner_labels = self.vocab.get_index_to_token_vocabulary(
ner_tag_namespace)
constraints = allowed_transitions(self._ner_label_encoding, ner_labels)
self.crf = ConditionalRandomField(num_ner_tags,
constraints,
include_start_end_transitions=True)
# RE subtask 3
self.ner_tag_embedder = ner_tag_embedder
self.relation_scorer = relation_scorer
self._re_loss_weight = re_loss_weight
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
tag_representation_dim: int,
arc_representation_dim: int,
model_name: str = None,
tag_feedforward: FeedForward = None,
arc_feedforward: FeedForward = None,
pos_tag_embedding: Embedding = None,
use_mst_decoding_for_validation: bool = True,
dropout: float = 0.0,
input_dropout: float = 0.0,
word_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
) -> None:
super().__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.encoder = encoder
if model_name:
from src.data.token_indexers import PretrainedAutoTokenizer
self._tokenizer = PretrainedAutoTokenizer.load(model_name)
encoder_dim = encoder.get_output_dim()
self.head_arc_feedforward = arc_feedforward or FeedForward(
encoder_dim, 1, arc_representation_dim,
Activation.by_name("elu")())
self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward)
self.arc_attention = BilinearMatrixAttention(arc_representation_dim,
arc_representation_dim,
use_input_biases=True)
num_labels = self.vocab.get_vocab_size("head_tags")
self.head_tag_feedforward = tag_feedforward or FeedForward(
encoder_dim, 1, tag_representation_dim,
Activation.by_name("elu")())
self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward)
self.tag_bilinear = torch.nn.modules.Bilinear(tag_representation_dim,
tag_representation_dim,
num_labels)
self._pos_tag_embedding = pos_tag_embedding or None
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
self._word_dropout = word_dropout
self._head_sentinel = torch.nn.Parameter(
torch.randn([1, 1, encoder.get_output_dim()]))
representation_dim = text_field_embedder.get_output_dim()
if pos_tag_embedding is not None:
representation_dim += pos_tag_embedding.get_output_dim()
check_dimensions_match(
representation_dim,
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim",
)
check_dimensions_match(
tag_representation_dim,
self.head_tag_feedforward.get_output_dim(),
"tag representation dim",
"tag feedforward output dim",
)
check_dimensions_match(
arc_representation_dim,
self.head_arc_feedforward.get_output_dim(),
"arc representation dim",
"arc feedforward output dim",
)
self.use_mst_decoding_for_validation = use_mst_decoding_for_validation
tags = self.vocab.get_token_to_index_vocabulary("pos")
punctuation_tag_indices = {
tag: index
for tag, index in tags.items() if tag in POS_TO_IGNORE
}
self._pos_to_ignore = set(punctuation_tag_indices.values())
logger.info(
f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. "
"Ignoring words with these POS tags for evaluation.")
self._attachment_scores = AttachmentScores()
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
tag_representation_dim: int,
arc_representation_dim: int,
tag_feedforward: FeedForward = None,
arc_feedforward: FeedForward = None,
pos_tag_embedding: Embedding = None,
treebank_embedding: Embedding = None,
use_mst_decoding_for_validation: bool = True,
use_treebank_embedding: bool = False,
dropout: float = 0.0,
input_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(BiaffineDependencyParserMonolingual, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.encoder = encoder
encoder_dim = encoder.get_output_dim()
self.head_arc_feedforward = arc_feedforward or \
FeedForward(encoder_dim, 1,
arc_representation_dim,
Activation.by_name("elu")())
self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward)
self.arc_attention = BilinearMatrixAttention(arc_representation_dim,
arc_representation_dim,
use_input_biases=True)
num_labels = self.vocab.get_vocab_size("head_tags")
self.head_tag_feedforward = tag_feedforward or \
FeedForward(encoder_dim, 1,
tag_representation_dim,
Activation.by_name("elu")())
self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward)
self.tag_bilinear = torch.nn.modules.Bilinear(tag_representation_dim,
tag_representation_dim,
num_labels)
self._pos_tag_embedding = pos_tag_embedding or None
self._treebank_embedding = treebank_embedding or None
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
self._head_sentinel = torch.nn.Parameter(torch.randn([1, 1, encoder.get_output_dim()]))
representation_dim = text_field_embedder.get_output_dim()
if pos_tag_embedding is not None:
representation_dim += pos_tag_embedding.get_output_dim()
if treebank_embedding is not None:
representation_dim += treebank_embedding.get_output_dim()
check_dimensions_match(representation_dim, encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
check_dimensions_match(tag_representation_dim, self.head_tag_feedforward.get_output_dim(),
"tag representation dim", "tag feedforward output dim")
check_dimensions_match(arc_representation_dim, self.head_arc_feedforward.get_output_dim(),
"arc representation dim", "arc feedforward output dim")
self.use_mst_decoding_for_validation = use_mst_decoding_for_validation
self.use_treebank_embedding = use_treebank_embedding
tags = self.vocab.get_token_to_index_vocabulary("pos")
punctuation_tag_indices = {tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE}
self._pos_to_ignore = set(punctuation_tag_indices.values())
logger.info(f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. "
"Ignoring words with these POS tags for evaluation.")
if self.use_treebank_embedding:
tbids = self.vocab.get_token_to_index_vocabulary("tbids")
tbid_indices = {tb: index for tb, index in tbids.items()}
self._tbids = set(tbid_indices.values())
logger.info(f"Found TBIDs corresponding to the following treebanks : {tbid_indices}. "
"Embedding these as additional features.")
self._attachment_scores = AttachmentScores()
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
pos_field_embedder: TextFieldEmbedder,
sent_pos_field_embedder: TextFieldEmbedder,
modeling_layer: Seq2SeqEncoder,
span_end_encoder_before: Seq2SeqEncoder,
span_start_encoder_after: Seq2SeqEncoder,
span_end_encoder_after: Seq2SeqEncoder,
dropout: float = 0.2,
mask_lstms: bool = True,
initializer: InitializerApplicator = InitializerApplicator()
) -> None:
"""
This ``Model`` takes as input a dataset read by ProGlobalDatasetReader
Input: a list of sentences, a participant
Output: location category for the participant, location span
The basic outline of this model is to
1. get an embedded representation for paragraph tokens,
2. apply bi-LSTM to get contextual embeddings,
3. apply three-category classification and location span prediction to predict the location state
:param vocab: Vocabulary
:param text_field_embedder: ``TextFieldEmbedder`` used to embed the ``sentence tokens``
:param pos_field_embedder: ``TextFieldEmbedder`` used to embed the ``word positions``
:param sent_pos_field_embedder: ``TextFieldEmbedder`` used to embed the sent indicators
:param modeling_layer: ``Seq2SeqEncoder`` to encode the sequence of paragraph
:param span_end_encoder_bef: ``Seq2SeqEncoder`` to encode the sequence for before location end prediction
:param span_start_encoder_aft: ``Seq2SeqEncoder`` to encode the sequence for after location start prediction
:param span_end_encoder_aft: ``Seq2SeqEncoder`` to encode the sequence for after location end prediction
:param dropout:
:param mask_lstms:
:param initializer: ``InitializerApplicator`` We will use this to initialize the parameters in the model
Sample commandline
------------------
python processes/run.py train -s /output_folder experiment_config/ProGlobal.json
"""
super(ProGlobal, self).__init__(vocab)
# embedders for text, word positions, and sentence indicators
self._text_field_embedder = text_field_embedder
self._pos_field_embedder = pos_field_embedder
self._sent_pos_field_embedder = sent_pos_field_embedder
# bi-LSTM: to generate the contextual embedding
self._modeling_layer = modeling_layer
modeling_dim = modeling_layer.get_output_dim()
# three category classifier for before location
self._category_before_predictor = torch.nn.Linear(modeling_dim, 3)
# LSTM encoder for before location end: encode the contextual embedding and before location start scores
self._span_end_encoder_before = span_end_encoder_before
# linear function for before location start
span_start_before_input_dim = modeling_dim
self._span_start_predictor_before = TimeDistributed(
torch.nn.Linear(span_start_before_input_dim, 1))
# linear function for before location end
span_end_before_encoding_dim = span_end_encoder_before.get_output_dim()
span_end_before_input_dim = modeling_dim + span_end_before_encoding_dim
self._span_end_predictor_before = TimeDistributed(
torch.nn.Linear(span_end_before_input_dim, 1))
# three category classifier for after location
self._category_after_predictor = torch.nn.Linear(modeling_dim + 3, 3)
# LSTM encoder for after location start: encode the contextual embedding and
# previous before location start scores
self._span_start_encoder_after = span_start_encoder_after
# linear function for after location start
span_start_after_encoding_dim = span_start_encoder_after.get_output_dim(
)
span_start_after_input_dim = modeling_dim + span_start_after_encoding_dim
self._span_start_predictor_after = TimeDistributed(
torch.nn.Linear(span_start_after_input_dim, 1))
# LSTM encoder for after location end: encode the contextual embedding and
# current before location start scores
self._span_end_encoder_after = span_end_encoder_after
span_end_after_encoding_dim = span_end_encoder_after.get_output_dim()
span_end_after_input_dim = modeling_dim + span_end_after_encoding_dim
# linear function for after location end
self._span_end_predictor_after = TimeDistributed(
torch.nn.Linear(span_end_after_input_dim, 1))
self._dropout = torch.nn.Dropout(p=dropout)
self._mask_lstms = mask_lstms
initializer(self)
