def from_params(cls, vocab: Vocabulary, params: Params) -> 'SpanConstituencyParser': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params) span_extractor = SpanExtractor.from_params(params.pop("span_extractor")) encoder = Seq2SeqEncoder.from_params(params.pop("encoder")) feed_forward_params = params.pop("feedforward", None) if feed_forward_params is not None: feedforward_layer = FeedForward.from_params(feed_forward_params) else: feedforward_layer = None pos_tag_embedding_params = params.pop("pos_tag_embedding", None) if pos_tag_embedding_params is not None: pos_tag_embedding = Embedding.from_params(vocab, pos_tag_embedding_params) else: pos_tag_embedding = None initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) evalb_directory_path = params.pop("evalb_directory_path", None) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, span_extractor=span_extractor, encoder=encoder, feedforward_layer=feedforward_layer, pos_tag_embedding=pos_tag_embedding, initializer=initializer, regularizer=regularizer, evalb_directory_path=evalb_directory_path)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'DecomposableAttention': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params) premise_encoder_params = params.pop("premise_encoder", None) if premise_encoder_params is not None: premise_encoder = Seq2SeqEncoder.from_params(premise_encoder_params) else: premise_encoder = None hypothesis_encoder_params = params.pop("hypothesis_encoder", None) if hypothesis_encoder_params is not None: hypothesis_encoder = Seq2SeqEncoder.from_params(hypothesis_encoder_params) else: hypothesis_encoder = None attend_feedforward = FeedForward.from_params(params.pop('attend_feedforward')) similarity_function = SimilarityFunction.from_params(params.pop("similarity_function")) compare_feedforward = FeedForward.from_params(params.pop('compare_feedforward')) aggregate_feedforward = FeedForward.from_params(params.pop('aggregate_feedforward')) initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, attend_feedforward=attend_feedforward, similarity_function=similarity_function, compare_feedforward=compare_feedforward, aggregate_feedforward=aggregate_feedforward, premise_encoder=premise_encoder, hypothesis_encoder=hypothesis_encoder, initializer=initializer, regularizer=regularizer)
def test_regex_matches_are_initialized_correctly(self): class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() self.linear_1_with_funky_name = torch.nn.Linear(5, 10) self.linear_2 = torch.nn.Linear(10, 5) self.conv = torch.nn.Conv1d(5, 5, 5) def forward(self, inputs): # pylint: disable=arguments-differ pass # pyhocon does funny things if there's a . in a key. This test makes sure that we # handle these kinds of regexes correctly. json_params = """{"initializer": [ ["conv", {"type": "constant", "val": 5}], ["funky_na.*bi", {"type": "constant", "val": 7}] ]} """ params = Params(pyhocon.ConfigFactory.parse_string(json_params)) initializers = InitializerApplicator.from_params(params['initializer']) model = Net() initializers(model) for parameter in model.conv.parameters(): assert torch.equal(parameter.data, torch.ones(parameter.size()) * 5) parameter = model.linear_1_with_funky_name.bias assert torch.equal(parameter.data, torch.ones(parameter.size()) * 7)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'SimpleTagger': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params) encoder = Seq2SeqEncoder.from_params(params.pop("encoder")) initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, encoder=encoder, initializer=initializer, regularizer=regularizer)
def _get_applicator(self, regex: str, weights_file_path: str, parameter_name_overrides: Optional[Dict[str, str]] = None) -> InitializerApplicator: parameter_name_overrides = parameter_name_overrides or {} initializer_params = Params({ "type": "pretrained", "weights_file_path": weights_file_path, "parameter_name_overrides": parameter_name_overrides }) params = Params({ "initializer": [(regex, initializer_params)] }) return InitializerApplicator.from_params(params["initializer"])
def from_params(cls, vocab: Vocabulary, params: Params) -> 'CrfTagger': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params) encoder = Seq2SeqEncoder.from_params(params.pop("encoder")) label_namespace = params.pop("label_namespace", "labels") constraint_type = params.pop("constraint_type", None) initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, encoder=encoder, label_namespace=label_namespace, constraint_type=constraint_type, initializer=initializer, regularizer=regularizer)
def from_params(cls, vocab, params): embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params) num_highway_layers = params.pop("num_highway_layers") phrase_layer = Seq2SeqEncoder.from_params(params.pop("phrase_layer")) similarity_function = SimilarityFunction.from_params(params.pop("similarity_function")) modeling_layer = Seq2SeqEncoder.from_params(params.pop("modeling_layer")) dropout = params.pop('dropout', 0.2) initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) mask_lstms = params.pop('mask_lstms', True) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, num_highway_layers=num_highway_layers, phrase_layer=phrase_layer, attention_similarity_function=similarity_function, modeling_layer=modeling_layer, dropout=dropout, mask_lstms=mask_lstms, initializer=initializer, regularizer=regularizer)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'BiattentiveClassificationNetwork': # type: ignore # pylint: disable=arguments-differ embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params(vocab=vocab, params=embedder_params) embedding_dropout = params.pop("embedding_dropout") pre_encode_feedforward = FeedForward.from_params(params.pop("pre_encode_feedforward")) encoder = Seq2SeqEncoder.from_params(params.pop("encoder")) integrator = Seq2SeqEncoder.from_params(params.pop("integrator")) integrator_dropout = params.pop("integrator_dropout") output_layer_params = params.pop("output_layer") if "activations" in output_layer_params: output_layer = FeedForward.from_params(output_layer_params) else: output_layer = Maxout.from_params(output_layer_params) elmo = params.pop("elmo", None) if elmo is not None: elmo = Elmo.from_params(elmo) use_input_elmo = params.pop_bool("use_input_elmo", False) use_integrator_output_elmo = params.pop_bool("use_integrator_output_elmo", False) initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, embedding_dropout=embedding_dropout, pre_encode_feedforward=pre_encode_feedforward, encoder=encoder, integrator=integrator, integrator_dropout=integrator_dropout, output_layer=output_layer, elmo=elmo, use_input_elmo=use_input_elmo, use_integrator_output_elmo=use_integrator_output_elmo, initializer=initializer, regularizer=regularizer)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, binary_feature_dim: int, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, label_smoothing: float = None, ignore_span_metric: bool = False) -> None: super(SemanticRoleLabeler, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") # For the span based evaluation, we don't want to consider labels # for verb, because the verb index is provided to the model. self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=["V"]) self.encoder = encoder # There are exactly 2 binary features for the verb predicate embedding. self.binary_feature_embedding = Embedding(2, binary_feature_dim) self.tag_projection_layer = TimeDistributed( Linear(self.encoder.get_output_dim(), self.num_classes)) self.embedding_dropout = Dropout(p=embedding_dropout) self._label_smoothing = label_smoothing self.ignore_span_metric = ignore_span_metric check_dimensions_match( text_field_embedder.get_output_dim() + binary_feature_dim, encoder.get_input_dim(), "text embedding dim + verb indicator embedding dim", "encoder input dim") initializer(self)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'HierarchicalCRF': embedder_params = params.pop('text_field_embedder') text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params) inner_encoder = Seq2VecEncoder.from_params(params.pop('inner_encoder')) outer_encoder = Seq2SeqEncoder.from_params(params.pop('outer_encoder')) label_namespace = params.pop('label_namespace', 'labels') constraint_type = params.pop('constraint_type', None) dropout = params.pop('dropout', None) include_start_end_transitions = params.pop('include_start_end_transitions', True) initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, inner_encoder=inner_encoder, outer_encoder=outer_encoder, label_namespace=label_namespace, constraint_type=constraint_type, dropout=dropout, include_start_end_transitions=include_start_end_transitions, initializer=initializer, regularizer=regularizer)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'BidirectionalAttentionFlow': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params( vocab, embedder_params) num_highway_layers = params.pop("num_highway_layers") phrase_layer = Seq2SeqEncoder.from_params(params.pop("phrase_layer")) similarity_function = SimilarityFunction.from_params( params.pop("similarity_function")) modeling_layer = Seq2SeqEncoder.from_params( params.pop("modeling_layer")) span_end_encoder = Seq2SeqEncoder.from_params( params.pop("span_end_encoder")) initializer = InitializerApplicator.from_params( params.pop("initializer", [])) dropout = params.pop('dropout', 0.2) # TODO: Remove the following when fully deprecated evaluation_json_file = params.pop('evaluation_json_file', None) if evaluation_json_file is not None: logger.warning( "the 'evaluation_json_file' model parameter is deprecated, please remove" ) mask_lstms = params.pop('mask_lstms', True) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, num_highway_layers=num_highway_layers, phrase_layer=phrase_layer, attention_similarity_function=similarity_function, modeling_layer=modeling_layer, span_end_encoder=span_end_encoder, initializer=initializer, dropout=dropout, mask_lstms=mask_lstms)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'BIOLabeler': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params(vocab=vocab, params=embedder_params) stacked_encoder = Seq2SeqEncoder.from_params(params.pop("stacked_encoder")) predicate_feature_dim = params.pop("predicate_feature_dim", 100) dim_hidden = params.pop("hidden_dim", 100) question_generator = QuestionGenerator.from_params(vocab, params.pop("question_generator")) initializer = InitializerApplicator.from_params(params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params(params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, stacked_encoder=stacked_encoder, question_generator=question_generator, predicate_feature_dim=predicate_feature_dim, dim_hidden=dim_hidden, initializer=initializer, regularizer=regularizer)
def __init__( self, vocab: Vocabulary, bert_model: str, dropout: float = 0.0, requires_grad: str = "none", pos_weight: float = 1.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(BernoulliBertGenerator, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._bert_model = AutoModel.from_pretrained(bert_model) self._dropout = torch.nn.Dropout(p=dropout) self._classification_layer = torch.nn.Linear( self._bert_model.config.hidden_size, 1, bias=False) if requires_grad in ["none", "all"]: for param in self._bert_model.parameters(): param.requires_grad = requires_grad == "all" else: model_name_regexes = requires_grad.split(",") for name, param in self._bert_model.named_parameters(): found = any([regex in name for regex in model_name_regexes]) param.requires_grad = found for n, v in self._bert_model.named_parameters(): if n.startswith("classifier"): v.requires_grad = True self._pos_weight = 1.0 / pos_weight - 1 self._token_prf = F1Measure(1) initializer(self)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'Tagger': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params( vocab, embedder_params) stacked_encoder = Seq2SeqEncoder.from_params( params.pop("stacked_encoder")) tasks = params.pop("tasks") source_namespace = params.pop("source_namespace", "tokens") label_namespace = params.pop("label_namespace", "labels") is_crf = params.pop("is_crf", False) initializer = InitializerApplicator.from_params( params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params( params.pop('regularizer', [])) return cls(vocab=vocab, text_field_embedder=text_field_embedder, stacked_encoder=stacked_encoder, tasks=tasks, source_namespace=source_namespace, label_namespace=label_namespace, is_crf=is_crf, initializer=initializer, regularizer=regularizer)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(SequenceClassifier, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") self.encoder = encoder self.projection_layer = Linear(self.encoder.get_output_dim(), self.num_classes) if text_field_embedder.get_output_dim() != encoder.get_input_dim(): raise ConfigurationError( "The output dimension of the text_field_embedder must match the " "input dimension of the sequence encoder. Found {} and {}, " "respectively.".format(text_field_embedder.get_output_dim(), encoder.get_input_dim())) self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, vocab: Vocabulary, input_size: int, hidden_size: int, label_namespace: str = "labels", negative_label: str = "0", evaluated_labels: List[str] = None, loss_gamma: float = None, label_smoothing: float = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self._idx_to_label = self.vocab.get_index_to_token_vocabulary( label_namespace) self._num_pos_classes = self.vocab.get_vocab_size(label_namespace) - 1 self._negative_label_idx = self.vocab.get_token_index( negative_label, namespace=label_namespace) assert self._negative_label_idx == 0, "negative label idx is supposed to be 0" self._loss_gamma = loss_gamma self._label_smoothing = label_smoothing # projection layers self._U = Linear(input_size, hidden_size, bias=False) self._W = Linear(input_size, hidden_size, bias=False) self._V = Linear(hidden_size, self._num_pos_classes, bias=False) self._T = Linear(hidden_size, 1, bias=False) self._b = Parameter(torch.Tensor(hidden_size)) nn.init.normal_(self._b) self._accuracy = CategoricalAccuracy() self._f1_metric = F1Measure(vocabulary=vocab, negative_label=negative_label, label_namespace=label_namespace, evaluated_labels=evaluated_labels) initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, document_encoder: Seq2SeqEncoder, word_attention: Seq2SeqEncoder, output_logit: FeedForward, pre_document_encoder_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder self._word_attention = word_attention self._pre_document_encoder_dropout = torch.nn.Dropout( p=pre_document_encoder_dropout) self._document_encoder = document_encoder self._output_logit = output_logit self._num_labels = vocab.get_vocab_size(namespace="labels") self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, dropout: float = 0.0, max_span_length: int = 30, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder self._max_span_length = max_span_length self.qa_outputs = torch.nn.Linear( self._text_field_embedder.get_output_dim(), 2) self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._span_qa_metrics = SquadEmAndF1() if dropout > 0: self._dropout = torch.nn.Dropout(p=dropout) else: self._dropout = lambda x: x initializer(self)
def __init__(self, vocab, text_field_embedder, encoder, initializer=InitializerApplicator(), regularizer=None): super(SimpleTagger, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size(u"labels") self.encoder = encoder self.tag_projection_layer = TimeDistributed( Linear(self.encoder.get_output_dim(), self.num_classes)) check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), u"text field embedding dim", u"encoder input dim") self.metrics = { u"accuracy": CategoricalAccuracy(), u"accuracy3": CategoricalAccuracy(top_k=3) } initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, stacked_encoder: Seq2SeqEncoder, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(SimpleTagger, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") self.stacked_encoder = stacked_encoder self.tag_projection_layer = TimeDistributed(Linear(self.stacked_encoder.get_output_dim(), self.num_classes)) if text_field_embedder.get_output_dim() != stacked_encoder.get_input_dim(): raise ConfigurationError("The output dimension of the text_field_embedder must match the " "input dimension of the phrase_encoder. Found {} and {}, " "respectively.".format(text_field_embedder.get_output_dim(), stacked_encoder.get_input_dim())) self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3) } initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, stacked_encoder: Seq2SeqEncoder, predicate_feature_dim: int, dim_hidden: int = 100, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None): super(SpanDetector, self).__init__(vocab, regularizer) self.dim_hidden = dim_hidden self.text_field_embedder = text_field_embedder self.predicate_feature_embedding = Embedding(2, predicate_feature_dim) self.embedding_dropout = Dropout(p=embedding_dropout) self.threshold_metric = ThresholdMetric() self.stacked_encoder = stacked_encoder self.span_hidden = SpanRepAssembly(self.stacked_encoder.get_output_dim(), self.stacked_encoder.get_output_dim(), self.dim_hidden) self.pred = TimeDistributed(Linear(self.dim_hidden, 1))
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, similarity_function: SimilarityFunction, projection_feedforward: FeedForward, inference_encoder: Seq2SeqEncoder, output_feedforward: FeedForwardPair, dropout: float = 0.5, margin: float = 1.25, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder self._encoder = encoder self._matrix_attention = LegacyMatrixAttention(similarity_function) self._projection_feedforward = projection_feedforward self._inference_encoder = inference_encoder if dropout: self.dropout = torch.nn.Dropout(dropout) self.rnn_input_dropout = InputVariationalDropout(dropout) else: self.dropout = None self.rnn_input_dropout = None self._output_feedforward = output_feedforward self._margin = margin self._accuracy = BooleanAccuracy() initializer(self)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'SpanConstituencyParser': embedder_params = params.pop("text_field_embedder") text_field_embedder = TextFieldEmbedder.from_params( vocab, embedder_params) span_extractor = SpanExtractor.from_params( params.pop("span_extractor")) encoder = Seq2SeqEncoder.from_params(params.pop("encoder")) feed_forward_params = params.pop("feedforward", None) if feed_forward_params is not None: feedforward_layer = FeedForward.from_params(feed_forward_params) else: feedforward_layer = None pos_tag_embedding_params = params.pop("pos_tag_embedding", None) if pos_tag_embedding_params is not None: pos_tag_embedding = Embedding.from_params( vocab, pos_tag_embedding_params) else: pos_tag_embedding = None initializer = InitializerApplicator.from_params( params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params( params.pop('regularizer', [])) evalb_directory_path = params.pop("evalb_directory_path", None) params.assert_empty(cls.__name__) return cls(vocab=vocab, text_field_embedder=text_field_embedder, span_extractor=span_extractor, encoder=encoder, feedforward_layer=feedforward_layer, pos_tag_embedding=pos_tag_embedding, initializer=initializer, regularizer=regularizer, evalb_directory_path=evalb_directory_path)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, include_start_end_transitions: bool = True, dropout: Optional[float] = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self.label_namespace = 'labels' self.num_tags = self.vocab.get_vocab_size(self.label_namespace) # encode text self.text_field_embedder = text_field_embedder self.encoder = encoder self.dropout = torch.nn.Dropout(dropout) if dropout else None # crf output_dim = self.encoder.get_output_dim() self.tag_projection_layer = TimeDistributed( Linear(output_dim, self.num_tags)) self.crf = ConditionalRandomField( self.num_tags, constraints=None, include_start_end_transitions=include_start_end_transitions) self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3) } for index, label in self.vocab.get_index_to_token_vocabulary( self.label_namespace).items(): self.metrics['F1_' + label] = F1Measure(positive_label=index) initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, sentence_encoder: Optional[Seq2SeqEncoder] = None, regularizer: Optional[RegularizerApplicator] = None) -> None: super(SeqLabeler, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=['SENT_BOUND']) # Metrics just for specific classes all_labels = self.vocab.get_index_to_token_vocabulary("labels").values() logger.info(f'Labels: {all_labels}') mwe_labels = [label.replace('B-', '').replace('I-', '') for label in all_labels if 'MWE' in label] self.mwe_span_metric = None if len(mwe_labels) > 0: logger.info(f'Instantiating MWE only evaluation: {mwe_labels}') self.mwe_span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=ne_labels + ['SENT_BOUND']) else: logger.info('No MWE only evaluation instantiated.') self.sentence_encoder = sentence_encoder # Encode the sentence with a biLSTM / self-attention if sentence_encoder: check_dimensions_match(text_field_embedder.get_output_dim(), sentence_encoder.get_input_dim(), "text embedding dim", "encoder input dim") self.tag_projection_layer = TimeDistributed(Linear(self.sentence_encoder.get_output_dim(), self.num_classes)) # Just use embeddings else: self.tag_projection_layer = TimeDistributed(Linear(text_field_embedder.get_output_dim(), self.num_classes)) InitializerApplicator()(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, fc_dim: int, nonlinear_fc: bool, dropout_fc: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(NLIEncoder, self).__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder self._encoder = encoder self._fc_dim = fc_dim self._nonlinear_fc = nonlinear_fc self._dropout_fc = dropout_fc if nonlinear_fc: self._classifier = nn.Sequential( nn.Dropout(p=dropout_fc), nn.Linear(4 * encoder.get_output_dim(), fc_dim), nn.Tanh(), nn.Dropout(p=dropout_fc), nn.Linear(fc_dim, fc_dim), nn.Tanh(), nn.Dropout(p=dropout_fc), nn.Linear(fc_dim, 3), ) else: self._classifier = nn.Sequential( nn.Linear(4 * encoder.get_output_dim(), fc_dim), nn.Linear(fc_dim, fc_dim), nn.Linear(fc_dim, 3)) self._num_labels = vocab.get_vocab_size(namespace="labels") self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, seq2vec_encoder: Seq2VecEncoder, seq2seq_encoder: Seq2SeqEncoder = None, dropout: float = None, num_labels: int = None, label_namespace: str = "labels", initializer: InitializerApplicator = InitializerApplicator()) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder if seq2seq_encoder: self._seq2seq_encoder = seq2seq_encoder else: self._seq2seq_encoder = None self._seq2vec_encoder = seq2vec_encoder self._classifier_input_dim = self._seq2vec_encoder.get_output_dim() if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = None self._label_namespace = label_namespace if num_labels: self._num_labels = num_labels else: self._num_labels = vocab.get_vocab_size(namespace=self._label_namespace) self._classification_layer = torch.nn.Linear(self._classifier_input_dim, self._num_labels) self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__( self, vocab: Vocabulary, bert_model: str, dropout: float = 0.0, requires_grad: str = "none", initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(BertClassifier, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._num_labels = self._vocabulary.get_vocab_size("labels") self._bert_config = AutoConfig.from_pretrained(bert_model, output_attentions=True) self._bert_model = AutoModel.from_pretrained(bert_model, config=self._bert_config) # self._bert_model = Longformer.from_pretrained(bert_model, output_attentions=True) self._dropout = nn.Dropout(dropout) self._classifier = nn.Linear(self._bert_model.config.hidden_size, self._num_labels) self.embedding_layers = [self._bert_model.embeddings] if requires_grad in ["none", "all"]: for param in self._bert_model.parameters(): param.requires_grad = requires_grad == "all" else: model_name_regexes = requires_grad.split(",") for name, param in self._bert_model.named_parameters(): found = any([regex in name for regex in model_name_regexes]) param.requires_grad = found initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, left_text_encoder: Seq2VecEncoder, right_text_encoder: Seq2VecEncoder, feedforward: Optional[FeedForward] = None, target_field_embedder: Optional[TextFieldEmbedder] = None, target_encoder: Optional[Seq2VecEncoder] = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, word_dropout: float = 0.0, dropout: float = 0.0) -> None: super().__init__(vocab, regularizer) ''' :param vocab: vocab : A Vocabulary, required in order to compute sizes for input/output projections. :param text_field_embedder: Used to embed the text and target text if target_field_embedder is None but the target_encoder is not None. :param left_text_encoder: Encoder that will create the representation of the tokens left of the target and the target itself if included from the dataset reader. :param right_text_encoder: Encoder that will create the representation of the tokens right of the target and the target itself if included from the dataset reader. :param feedforward: An optional feed forward layer to apply after the encoder. :param target_field_embedder: Used to embed the target text to give as input to the target_encoder. Thus this allows a seperate embedding for text and target text. :param target_encoder: Encoder that will create the representation of target text tokens. :param initializer: Used to initialize the model parameters. :param regularizer: If provided, will be used to calculate the regularization penalty during training. :param word_dropout: Dropout that is applied after the embedding of the tokens/words. It will drop entire words with this probabilty. :param dropout: To apply dropout after each layer apart from the last layer. All dropout that is applied to timebased data will be `variational dropout`_ all else will be standard dropout. Without the target encoder this will be the standard TDLSTM method from `Effective LSTM's for Target-Dependent Sentiment classification`_ . With the target encoder this will then become the TCLSTM method from `Effective LSTM's for Target-Dependent Sentiment classification`_. .. _variational dropout: https://papers.nips.cc/paper/6241-a-theoretically-grounded-application-of-dropout-in-recurrent-neural-networks.pdf .. _Effective LSTM's for Target-Dependent Sentiment classification: https://aclanthology.coli.uni-saarland.de/papers/C16-1311/c16-1311 ''' self.text_field_embedder = text_field_embedder self.target_field_embedder = target_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") self.left_text_encoder = left_text_encoder self.right_text_encoder = right_text_encoder self.target_encoder = target_encoder self.feedforward = feedforward if feedforward is not None: output_dim = self.feedforward.get_output_dim() else: left_out_dim = self.left_text_encoder.get_output_dim() right_out_dim = self.right_text_encoder.get_output_dim() output_dim = left_out_dim + right_out_dim self.label_projection = Linear(output_dim, self.num_classes) self.metrics = {"accuracy": CategoricalAccuracy()} self.f1_metrics = {} # F1 Scores label_index_name = self.vocab.get_index_to_token_vocabulary('labels') for label_index, label_name in label_index_name.items(): label_name = f'F1_{label_name.capitalize()}' self.f1_metrics[label_name] = F1Measure(label_index) # Dropout self._word_dropout = WordDrouput(word_dropout) self._variational_dropout = InputVariationalDropout(dropout) self._naive_dropout = Dropout(dropout) self.loss = torch.nn.CrossEntropyLoss() # Ensure that the input to the right_text_encoder and left_text_encoder # is the size of the target encoder output plus the size of the text # embedding output. if self.target_encoder: right_text_out_dim = self.right_text_encoder.get_input_dim() left_text_out_dim = self.left_text_encoder.get_input_dim() target_dim = self.target_encoder.get_output_dim() text_dim = self.text_field_embedder.get_output_dim() total_out_dim = target_dim + text_dim config_err_msg = ( "As the target is being encoded the output of the" " target encoder is concatenated onto each word " " vector for the left and right contexts " "therefore the input of the right_text_encoder" "/left_text_encoder is the output dimension of " "the target encoder + the dimension of the word " "embeddings for the left and right contexts.") if (total_out_dim != right_text_out_dim or total_out_dim != left_text_out_dim): raise ConfigurationError(config_err_msg) # Ensure that the target field embedder has an output dimension the # same as the input dimension to the target encoder. if self.target_encoder and self.target_field_embedder: target_embed_out = self.target_field_embedder.get_output_dim() target_in = self.target_encoder.get_input_dim() config_embed_err_msg = ("The Target field embedder should have" " the same output size " f"{target_embed_out} as the input to " f"the target encoder {target_in}") if target_embed_out != target_in: raise ConfigurationError(config_embed_err_msg) initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, embedding_dropout: float, pre_encode_feedforward: FeedForward, encoder: Seq2SeqEncoder, integrator: Seq2SeqEncoder, integrator_dropout: float, output_layer: Union[FeedForward, Maxout], elmo: Elmo = None, use_input_elmo: bool = False, use_integrator_output_elmo: bool = False, initializer: InitializerApplicator = InitializerApplicator(), **kwargs, ) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = text_field_embedder if "elmo" in self._text_field_embedder._token_embedders.keys(): raise ConfigurationError( "To use ELMo in the BiattentiveClassificationNetwork input, " "remove elmo from the text_field_embedder and pass an " "Elmo object to the BiattentiveClassificationNetwork and set the " "'use_input_elmo' and 'use_integrator_output_elmo' flags accordingly." ) self._embedding_dropout = nn.Dropout(embedding_dropout) self._num_classes = self.vocab.get_vocab_size("labels") self._pre_encode_feedforward = pre_encode_feedforward self._encoder = encoder self._integrator = integrator self._integrator_dropout = nn.Dropout(integrator_dropout) self._elmo = elmo self._use_input_elmo = use_input_elmo self._use_integrator_output_elmo = use_integrator_output_elmo self._num_elmo_layers = int(self._use_input_elmo) + int( self._use_integrator_output_elmo) # Check that, if elmo is None, none of the elmo flags are set. if self._elmo is None and self._num_elmo_layers != 0: raise ConfigurationError( "One of 'use_input_elmo' or 'use_integrator_output_elmo' is True, " "but no Elmo object was provided upon construction. Pass in an Elmo " "object to use Elmo.") if self._elmo is not None: # Check that, if elmo is not None, we use it somewhere. if self._num_elmo_layers == 0: raise ConfigurationError( "Elmo object provided upon construction, but both 'use_input_elmo' " "and 'use_integrator_output_elmo' are 'False'. Set one of them to " "'True' to use Elmo, or do not provide an Elmo object upon construction." ) # Check that the number of flags set is equal to the num_output_representations of the Elmo object if len(self._elmo._scalar_mixes) != self._num_elmo_layers: raise ConfigurationError( f"Elmo object has num_output_representations={len(self._elmo._scalar_mixes)}, but this " f"does not match the number of use_*_elmo flags set to true. use_input_elmo " f"is {self._use_input_elmo}, and use_integrator_output_elmo " f"is {self._use_integrator_output_elmo}") # Calculate combined integrator output dim, taking into account elmo if self._use_integrator_output_elmo: self._combined_integrator_output_dim = ( self._integrator.get_output_dim() + self._elmo.get_output_dim()) else: self._combined_integrator_output_dim = self._integrator.get_output_dim( ) self._self_attentive_pooling_projection = nn.Linear( self._combined_integrator_output_dim, 1) self._output_layer = output_layer if self._use_input_elmo: check_dimensions_match( text_field_embedder.get_output_dim() + self._elmo.get_output_dim(), self._pre_encode_feedforward.get_input_dim(), "text field embedder output dim + ELMo output dim", "Pre-encoder feedforward input dim", ) else: check_dimensions_match( text_field_embedder.get_output_dim(), self._pre_encode_feedforward.get_input_dim(), "text field embedder output dim", "Pre-encoder feedforward input dim", ) check_dimensions_match( self._pre_encode_feedforward.get_output_dim(), self._encoder.get_input_dim(), "Pre-encoder feedforward output dim", "Encoder input dim", ) check_dimensions_match( self._encoder.get_output_dim() * 3, self._integrator.get_input_dim(), "Encoder output dim * 3", "Integrator input dim", ) if self._use_integrator_output_elmo: check_dimensions_match( self._combined_integrator_output_dim * 4, self._output_layer.get_input_dim(), "(Integrator output dim + ELMo output dim) * 4", "Output layer input dim", ) else: check_dimensions_match( self._integrator.get_output_dim() * 4, self._output_layer.get_input_dim(), "Integrator output dim * 4", "Output layer input dim", ) check_dimensions_match( self._output_layer.get_output_dim(), self._num_classes, "Output layer output dim", "Number of classes.", ) self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3), } self.loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, context_layer: Seq2SeqEncoder, mention_feedforward: FeedForward, antecedent_feedforward: FeedForward, feature_size: int, max_span_width: int, spans_per_word: float, max_antecedents: int, coarse_to_fine: bool = False, inference_order: int = 1, lexical_dropout: float = 0.2, initializer: InitializerApplicator = InitializerApplicator(), **kwargs ) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = text_field_embedder self._context_layer = context_layer self._mention_feedforward = TimeDistributed(mention_feedforward) self._mention_scorer = TimeDistributed( torch.nn.Linear(mention_feedforward.get_output_dim(), 1) ) self._antecedent_feedforward = TimeDistributed(antecedent_feedforward) self._antecedent_scorer = TimeDistributed( torch.nn.Linear(antecedent_feedforward.get_output_dim(), 1) ) self._endpoint_span_extractor = EndpointSpanExtractor( context_layer.get_output_dim(), combination="x,y", num_width_embeddings=max_span_width, span_width_embedding_dim=feature_size, bucket_widths=False, ) self._attentive_span_extractor = SelfAttentiveSpanExtractor( input_dim=text_field_embedder.get_output_dim() ) # 10 possible distance buckets. self._num_distance_buckets = 10 self._distance_embedding = Embedding( embedding_dim=feature_size, num_embeddings=self._num_distance_buckets ) self._max_span_width = max_span_width self._spans_per_word = spans_per_word self._max_antecedents = max_antecedents self._coarse_to_fine = coarse_to_fine if self._coarse_to_fine: self._coarse2fine_scorer = torch.nn.Linear( mention_feedforward.get_input_dim(), mention_feedforward.get_input_dim() ) self._inference_order = inference_order if self._inference_order > 1: self._span_updating_gated_sum = GatedSum(mention_feedforward.get_input_dim()) self._mention_recall = MentionRecall() self._conll_coref_scores = ConllCorefScores() if lexical_dropout > 0: self._lexical_dropout = torch.nn.Dropout(p=lexical_dropout) else: self._lexical_dropout = lambda x: x initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, label_namespace: str = "labels", feedforward: Optional[FeedForward] = None, label_encoding: Optional[str] = None, include_start_end_transitions: bool = True, constrain_crf_decoding: bool = None, calculate_span_f1: bool = None, dropout: Optional[float] = None, verbose_metrics: bool = False, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, top_k: int = 1, ) -> None: super().__init__(vocab, regularizer) self.label_namespace = label_namespace self.text_field_embedder = text_field_embedder self.num_tags = self.vocab.get_vocab_size(label_namespace) self.encoder = encoder self.top_k = top_k self._verbose_metrics = verbose_metrics if dropout: self.dropout = torch.nn.Dropout(dropout) else: self.dropout = None self._feedforward = feedforward if feedforward is not None: output_dim = feedforward.get_output_dim() else: output_dim = self.encoder.get_output_dim() self.tag_projection_layer = TimeDistributed( Linear(output_dim, self.num_tags)) # if constrain_crf_decoding and calculate_span_f1 are not # provided, (i.e., they're None), set them to True # if label_encoding is provided and False if it isn't. if constrain_crf_decoding is None: constrain_crf_decoding = label_encoding is not None if calculate_span_f1 is None: calculate_span_f1 = label_encoding is not None self.label_encoding = label_encoding if constrain_crf_decoding: if not label_encoding: raise ConfigurationError("constrain_crf_decoding is True, but " "no label_encoding was specified.") labels = self.vocab.get_index_to_token_vocabulary(label_namespace) constraints = allowed_transitions(label_encoding, labels) else: constraints = None self.include_start_end_transitions = include_start_end_transitions self.crf = ConditionalRandomField( self.num_tags, constraints, include_start_end_transitions=include_start_end_transitions) self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3), } self.calculate_span_f1 = calculate_span_f1 if calculate_span_f1: if not label_encoding: raise ConfigurationError("calculate_span_f1 is True, but " "no label_encoding was specified.") self._f1_metric = SpanBasedF1Measure(vocab, tag_namespace=label_namespace, label_encoding=label_encoding) check_dimensions_match( text_field_embedder.get_output_dim(), encoder.get_input_dim(), "text field embedding dim", "encoder input dim", ) if feedforward is not None: check_dimensions_match( encoder.get_output_dim(), feedforward.get_input_dim(), "encoder output dim", "feedforward input dim", ) initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, tag_representation_dim: int, arc_representation_dim: int, tag_feedforward: FeedForward = None, arc_feedforward: FeedForward = None, lemma_tag_embedding: Embedding = None, upos_tag_embedding: Embedding = None, xpos_tag_embedding: Embedding = None, feats_tag_embedding: Embedding = None, head_information_embedding: Embedding = None, head_tag_embedding: Embedding = None, dropout: float = 0.0, input_dropout: float = 0.0, edge_prediction_threshold: float = 0.5, initializer: InitializerApplicator = InitializerApplicator(), **kwargs, ) -> None: super().__init__(vocab, **kwargs) self.text_field_embedder = text_field_embedder self.encoder = encoder self.edge_prediction_threshold = edge_prediction_threshold if not 0 < edge_prediction_threshold < 1: raise ConfigurationError( f"edge_prediction_threshold must be between " f"0 and 1 (exclusive) but found {edge_prediction_threshold}.") encoder_dim = encoder.get_output_dim() self.head_arc_feedforward = arc_feedforward or FeedForward( encoder_dim, 1, arc_representation_dim, Activation.by_name("elu")()) self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward) self.arc_attention = BilinearMatrixAttention(arc_representation_dim, arc_representation_dim, use_input_biases=True) num_labels = self.vocab.get_vocab_size("deps") self.head_tag_feedforward = tag_feedforward or FeedForward( encoder_dim, 1, tag_representation_dim, Activation.by_name("elu")()) self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward) self.tag_bilinear = BilinearMatrixAttention(tag_representation_dim, tag_representation_dim, label_dim=num_labels) self._lemma_tag_embedding = lemma_tag_embedding or None self._upos_tag_embedding = upos_tag_embedding or None self._xpos_tag_embedding = xpos_tag_embedding or None self._feats_tag_embedding = feats_tag_embedding or None self._head_tag_embedding = head_tag_embedding or None self._head_information_embedding = head_information_embedding or None self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) # add a head sentinel to accommodate for extra root token in EUD graphs self._head_sentinel = torch.nn.Parameter( torch.randn([1, 1, encoder.get_output_dim()])) representation_dim = text_field_embedder.get_output_dim() if lemma_tag_embedding is not None: representation_dim += lemma_tag_embedding.get_output_dim() if upos_tag_embedding is not None: representation_dim += upos_tag_embedding.get_output_dim() if xpos_tag_embedding is not None: representation_dim += xpos_tag_embedding.get_output_dim() if feats_tag_embedding is not None: representation_dim += feats_tag_embedding.get_output_dim() if head_tag_embedding is not None: representation_dim += head_tag_embedding.get_output_dim() if head_information_embedding is not None: representation_dim += head_information_embedding.get_output_dim() check_dimensions_match( representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim", ) check_dimensions_match( tag_representation_dim, self.head_tag_feedforward.get_output_dim(), "tag representation dim", "tag feedforward output dim", ) check_dimensions_match( arc_representation_dim, self.head_arc_feedforward.get_output_dim(), "arc representation dim", "arc feedforward output dim", ) self._enhanced_attachment_scores = EnhancedAttachmentScores() self._arc_loss = torch.nn.BCEWithLogitsLoss(reduction="none") self._tag_loss = torch.nn.CrossEntropyLoss(reduction="none") initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, num_highway_layers: int, phrase_layer: Seq2SeqEncoder, matrix_attention: MatrixAttention, modeling_layer: Seq2SeqEncoder, span_end_encoder: Seq2SeqEncoder, dropout: float = 0.2, mask_lstms: bool = True, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ) -> None: super().__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder self._highway_layer = TimeDistributed( Highway(text_field_embedder.get_output_dim(), num_highway_layers)) self._phrase_layer = phrase_layer self._matrix_attention = matrix_attention self._modeling_layer = modeling_layer self._span_end_encoder = span_end_encoder encoding_dim = phrase_layer.get_output_dim() modeling_dim = modeling_layer.get_output_dim() span_start_input_dim = encoding_dim * 4 + modeling_dim self._span_start_predictor = TimeDistributed( torch.nn.Linear(span_start_input_dim, 1)) span_end_encoding_dim = span_end_encoder.get_output_dim() span_end_input_dim = encoding_dim * 4 + span_end_encoding_dim self._span_end_predictor = TimeDistributed( torch.nn.Linear(span_end_input_dim, 1)) # Bidaf has lots of layer dimensions which need to match up - these aren't necessarily # obvious from the configuration files, so we check here. check_dimensions_match( modeling_layer.get_input_dim(), 4 * encoding_dim, "modeling layer input dim", "4 * encoding dim", ) check_dimensions_match( text_field_embedder.get_output_dim(), phrase_layer.get_input_dim(), "text field embedder output dim", "phrase layer input dim", ) check_dimensions_match( span_end_encoder.get_input_dim(), 4 * encoding_dim + 3 * modeling_dim, "span end encoder input dim", "4 * encoding dim + 3 * modeling dim", ) self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._squad_metrics = SquadEmAndF1() if dropout > 0: self._dropout = torch.nn.Dropout(p=dropout) else: self._dropout = lambda x: x self._mask_lstms = mask_lstms initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, relation_scorer: RelationScorer, ner_tag_namespace: str = 'tags', evaluated_ner_labels: List[str] = None, re_loss_weight: float = 1.0, ner_tag_embedder: TokenEmbedder = None, use_aux_ner_labels: bool = False, aux_coarse_namespace: str = 'coarse_tags', aux_modifier_namespace: str = 'modifier_tags', aux_loss_weight: float = 1.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab=vocab, regularizer=regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder # NER subtask 2 self._ner_label_encoding = 'BIO' self._ner_tag_namespace = ner_tag_namespace ner_input_dim = self.encoder.get_output_dim() num_ner_tags = self.vocab.get_vocab_size(ner_tag_namespace) self.tag_projection_layer = TimeDistributed( Linear(ner_input_dim, num_ner_tags)) self._use_aux_ner_labels = use_aux_ner_labels if self._use_aux_ner_labels: self._coarse_tag_namespace = aux_coarse_namespace self._num_coarse_tags = self.vocab.get_vocab_size( self._coarse_tag_namespace) self._coarse_projection_layer = TimeDistributed( Linear(ner_input_dim, self._num_coarse_tags)) self._modifier_tag_namespace = aux_modifier_namespace self._num_modifier_tags = self.vocab.get_vocab_size( self._modifier_tag_namespace) self._modifier_projection_layer = TimeDistributed( Linear(ner_input_dim, self._num_modifier_tags)) self._coarse_acc = CategoricalAccuracy() self._modifier_acc = CategoricalAccuracy() self._aux_loss_weight = aux_loss_weight self.ner_accuracy = CategoricalAccuracy() if evaluated_ner_labels is None: ignored_classes = None else: assert self._ner_label_encoding == 'BIO', 'expected BIO encoding' all_ner_tags = self.vocab.get_token_to_index_vocabulary( ner_tag_namespace).keys() ner_tag_classes = set( [bio_tag[2:] for bio_tag in all_ner_tags if len(bio_tag) > 2]) ignored_classes = list( set(ner_tag_classes).difference(evaluated_ner_labels)) self.ner_f1 = SpanBasedF1Measure( vocabulary=vocab, tag_namespace=ner_tag_namespace, label_encoding=self._ner_label_encoding, ignore_classes=ignored_classes) # Use constrained crf decoding with the BIO labeling scheme ner_labels = self.vocab.get_index_to_token_vocabulary( ner_tag_namespace) constraints = allowed_transitions(self._ner_label_encoding, ner_labels) self.crf = ConditionalRandomField(num_ner_tags, constraints, include_start_end_transitions=True) # RE subtask 3 self.ner_tag_embedder = ner_tag_embedder self.relation_scorer = relation_scorer self._re_loss_weight = re_loss_weight initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, tag_representation_dim: int, arc_representation_dim: int, model_name: str = None, tag_feedforward: FeedForward = None, arc_feedforward: FeedForward = None, pos_tag_embedding: Embedding = None, use_mst_decoding_for_validation: bool = True, dropout: float = 0.0, input_dropout: float = 0.0, word_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ) -> None: super().__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder if model_name: from src.data.token_indexers import PretrainedAutoTokenizer self._tokenizer = PretrainedAutoTokenizer.load(model_name) encoder_dim = encoder.get_output_dim() self.head_arc_feedforward = arc_feedforward or FeedForward( encoder_dim, 1, arc_representation_dim, Activation.by_name("elu")()) self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward) self.arc_attention = BilinearMatrixAttention(arc_representation_dim, arc_representation_dim, use_input_biases=True) num_labels = self.vocab.get_vocab_size("head_tags") self.head_tag_feedforward = tag_feedforward or FeedForward( encoder_dim, 1, tag_representation_dim, Activation.by_name("elu")()) self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward) self.tag_bilinear = torch.nn.modules.Bilinear(tag_representation_dim, tag_representation_dim, num_labels) self._pos_tag_embedding = pos_tag_embedding or None self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self._word_dropout = word_dropout self._head_sentinel = torch.nn.Parameter( torch.randn([1, 1, encoder.get_output_dim()])) representation_dim = text_field_embedder.get_output_dim() if pos_tag_embedding is not None: representation_dim += pos_tag_embedding.get_output_dim() check_dimensions_match( representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim", ) check_dimensions_match( tag_representation_dim, self.head_tag_feedforward.get_output_dim(), "tag representation dim", "tag feedforward output dim", ) check_dimensions_match( arc_representation_dim, self.head_arc_feedforward.get_output_dim(), "arc representation dim", "arc feedforward output dim", ) self.use_mst_decoding_for_validation = use_mst_decoding_for_validation tags = self.vocab.get_token_to_index_vocabulary("pos") punctuation_tag_indices = { tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE } self._pos_to_ignore = set(punctuation_tag_indices.values()) logger.info( f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. " "Ignoring words with these POS tags for evaluation.") self._attachment_scores = AttachmentScores() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, tag_representation_dim: int, arc_representation_dim: int, tag_feedforward: FeedForward = None, arc_feedforward: FeedForward = None, pos_tag_embedding: Embedding = None, treebank_embedding: Embedding = None, use_mst_decoding_for_validation: bool = True, use_treebank_embedding: bool = False, dropout: float = 0.0, input_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(BiaffineDependencyParserMonolingual, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder encoder_dim = encoder.get_output_dim() self.head_arc_feedforward = arc_feedforward or \ FeedForward(encoder_dim, 1, arc_representation_dim, Activation.by_name("elu")()) self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward) self.arc_attention = BilinearMatrixAttention(arc_representation_dim, arc_representation_dim, use_input_biases=True) num_labels = self.vocab.get_vocab_size("head_tags") self.head_tag_feedforward = tag_feedforward or \ FeedForward(encoder_dim, 1, tag_representation_dim, Activation.by_name("elu")()) self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward) self.tag_bilinear = torch.nn.modules.Bilinear(tag_representation_dim, tag_representation_dim, num_labels) self._pos_tag_embedding = pos_tag_embedding or None self._treebank_embedding = treebank_embedding or None self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self._head_sentinel = torch.nn.Parameter(torch.randn([1, 1, encoder.get_output_dim()])) representation_dim = text_field_embedder.get_output_dim() if pos_tag_embedding is not None: representation_dim += pos_tag_embedding.get_output_dim() if treebank_embedding is not None: representation_dim += treebank_embedding.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") check_dimensions_match(tag_representation_dim, self.head_tag_feedforward.get_output_dim(), "tag representation dim", "tag feedforward output dim") check_dimensions_match(arc_representation_dim, self.head_arc_feedforward.get_output_dim(), "arc representation dim", "arc feedforward output dim") self.use_mst_decoding_for_validation = use_mst_decoding_for_validation self.use_treebank_embedding = use_treebank_embedding tags = self.vocab.get_token_to_index_vocabulary("pos") punctuation_tag_indices = {tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE} self._pos_to_ignore = set(punctuation_tag_indices.values()) logger.info(f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. " "Ignoring words with these POS tags for evaluation.") if self.use_treebank_embedding: tbids = self.vocab.get_token_to_index_vocabulary("tbids") tbid_indices = {tb: index for tb, index in tbids.items()} self._tbids = set(tbid_indices.values()) logger.info(f"Found TBIDs corresponding to the following treebanks : {tbid_indices}. " "Embedding these as additional features.") self._attachment_scores = AttachmentScores() initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, pos_field_embedder: TextFieldEmbedder, sent_pos_field_embedder: TextFieldEmbedder, modeling_layer: Seq2SeqEncoder, span_end_encoder_before: Seq2SeqEncoder, span_start_encoder_after: Seq2SeqEncoder, span_end_encoder_after: Seq2SeqEncoder, dropout: float = 0.2, mask_lstms: bool = True, initializer: InitializerApplicator = InitializerApplicator() ) -> None: """ This ``Model`` takes as input a dataset read by ProGlobalDatasetReader Input: a list of sentences, a participant Output: location category for the participant, location span The basic outline of this model is to 1. get an embedded representation for paragraph tokens, 2. apply bi-LSTM to get contextual embeddings, 3. apply three-category classification and location span prediction to predict the location state :param vocab: Vocabulary :param text_field_embedder: ``TextFieldEmbedder`` used to embed the ``sentence tokens`` :param pos_field_embedder: ``TextFieldEmbedder`` used to embed the ``word positions`` :param sent_pos_field_embedder: ``TextFieldEmbedder`` used to embed the sent indicators :param modeling_layer: ``Seq2SeqEncoder`` to encode the sequence of paragraph :param span_end_encoder_bef: ``Seq2SeqEncoder`` to encode the sequence for before location end prediction :param span_start_encoder_aft: ``Seq2SeqEncoder`` to encode the sequence for after location start prediction :param span_end_encoder_aft: ``Seq2SeqEncoder`` to encode the sequence for after location end prediction :param dropout: :param mask_lstms: :param initializer: ``InitializerApplicator`` We will use this to initialize the parameters in the model Sample commandline ------------------ python processes/run.py train -s /output_folder experiment_config/ProGlobal.json """ super(ProGlobal, self).__init__(vocab) # embedders for text, word positions, and sentence indicators self._text_field_embedder = text_field_embedder self._pos_field_embedder = pos_field_embedder self._sent_pos_field_embedder = sent_pos_field_embedder # bi-LSTM: to generate the contextual embedding self._modeling_layer = modeling_layer modeling_dim = modeling_layer.get_output_dim() # three category classifier for before location self._category_before_predictor = torch.nn.Linear(modeling_dim, 3) # LSTM encoder for before location end: encode the contextual embedding and before location start scores self._span_end_encoder_before = span_end_encoder_before # linear function for before location start span_start_before_input_dim = modeling_dim self._span_start_predictor_before = TimeDistributed( torch.nn.Linear(span_start_before_input_dim, 1)) # linear function for before location end span_end_before_encoding_dim = span_end_encoder_before.get_output_dim() span_end_before_input_dim = modeling_dim + span_end_before_encoding_dim self._span_end_predictor_before = TimeDistributed( torch.nn.Linear(span_end_before_input_dim, 1)) # three category classifier for after location self._category_after_predictor = torch.nn.Linear(modeling_dim + 3, 3) # LSTM encoder for after location start: encode the contextual embedding and # previous before location start scores self._span_start_encoder_after = span_start_encoder_after # linear function for after location start span_start_after_encoding_dim = span_start_encoder_after.get_output_dim( ) span_start_after_input_dim = modeling_dim + span_start_after_encoding_dim self._span_start_predictor_after = TimeDistributed( torch.nn.Linear(span_start_after_input_dim, 1)) # LSTM encoder for after location end: encode the contextual embedding and # current before location start scores self._span_end_encoder_after = span_end_encoder_after span_end_after_encoding_dim = span_end_encoder_after.get_output_dim() span_end_after_input_dim = modeling_dim + span_end_after_encoding_dim # linear function for after location end self._span_end_predictor_after = TimeDistributed( torch.nn.Linear(span_end_after_input_dim, 1)) self._dropout = torch.nn.Dropout(p=dropout) self._mask_lstms = mask_lstms initializer(self)