def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, binary_feature_dim: int, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, label_smoothing: float = None, ignore_span_metric: bool = False) -> None: super(SemanticRoleLabeler, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") # For the span based evaluation, we don't want to consider labels # for verb, because the verb index is provided to the model. self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=["V"]) self.encoder = encoder # There are exactly 2 binary features for the verb predicate embedding. self.binary_feature_embedding = Embedding(2, binary_feature_dim) self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(), self.num_classes)) self.embedding_dropout = Dropout(p=embedding_dropout) self._label_smoothing = label_smoothing self.ignore_span_metric = ignore_span_metric check_dimensions_match(text_field_embedder.get_output_dim() + binary_feature_dim, encoder.get_input_dim(), "text embedding dim + verb indicator embedding dim", "encoder input dim") initializer(self)
def __init__(self, vocab: Vocabulary, bert_model: Union[str, BertModel], embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, label_smoothing: float = None, ignore_span_metric: bool = False, srl_eval_path: str = DEFAULT_SRL_EVAL_PATH) -> None: super().__init__(vocab, regularizer) if isinstance(bert_model, str): self.bert_model = BertModel.from_pretrained(bert_model) else: self.bert_model = bert_model self.num_classes = self.vocab.get_vocab_size("labels") if srl_eval_path is not None: # For the span based evaluation, we don't want to consider labels # for verb, because the verb index is provided to the model. self.span_metric = SrlEvalScorer(srl_eval_path, ignore_classes=["V"]) else: self.span_metric = None self.tag_projection_layer = Linear(self.bert_model.config.hidden_size, self.num_classes) self.embedding_dropout = Dropout(p=embedding_dropout) self._label_smoothing = label_smoothing self.ignore_span_metric = ignore_span_metric initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, question_generator: QuestionGenerator, stacked_encoder: Seq2SeqEncoder = None, predicate_feature_dim: int = 100, dim_hidden: int = 100, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None): super(QuestionPredictor, self).__init__(vocab, regularizer) self.dim_hidden = dim_hidden self.text_field_embedder = text_field_embedder self.predicate_feature_embedding = Embedding(2, predicate_feature_dim) self.embedding_dropout = Dropout(p=embedding_dropout) self.stacked_encoder = stacked_encoder self.span_extractor = EndpointSpanExtractor(self.stacked_encoder.get_output_dim(), combination="x,y") self.question_generator = question_generator self.slot_labels = question_generator.get_slot_labels() self.question_metric = QuestionPredictionMetric(vocab, question_generator.get_slot_labels())
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, tag_representation_dim: int, arc_representation_dim: int, tag_feedforward: FeedForward = None, arc_feedforward: FeedForward = None, dropout: float = 0.5, input_dropout: float = 0.5, head_tag_temperature: Optional[float] = None, head_temperature: Optional[float] = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(Supertagger, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder encoder_dim = encoder.get_output_dim() self.head_arc_feedforward = \ arc_feedforward or FeedForward(encoder_dim, 1, arc_representation_dim, Activation.by_name("elu")(), dropout=dropout) self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward) self.arc_attention = BilinearMatrixAttention(arc_representation_dim, arc_representation_dim, use_input_biases=True) num_labels = self.vocab.get_vocab_size("head_tags") self.head_tag_feedforward = \ tag_feedforward or FeedForward(encoder_dim, 1, tag_representation_dim, Activation.by_name("elu")(), dropout=dropout) self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward) self.tag_bilinear = BilinearWithBias(tag_representation_dim, tag_representation_dim, num_labels) self._head_sentinel = torch.nn.Parameter(torch.randn([1, 1, encoder.get_output_dim()])) representation_dim = text_field_embedder.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") check_dimensions_match(tag_representation_dim, self.head_tag_feedforward.get_output_dim(), "tag representation dim", "tag feedforward output dim") check_dimensions_match(arc_representation_dim, self.head_arc_feedforward.get_output_dim(), "arc representation dim", "arc feedforward output dim") self._input_dropout = Dropout(input_dropout) self._attachment_scores = CategoricalAccuracy() self._tagging_accuracy = CategoricalAccuracy() self.head_tag_temperature = head_tag_temperature self.head_temperature = head_temperature initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, edge_model: graph_dependency_parser.components.edge_models. EdgeModel, loss_function: graph_dependency_parser.components.losses.EdgeLoss, pos_tag_embedding: Embedding = None, use_mst_decoding_for_validation: bool = True, dropout: float = 0.0, input_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, validation_evaluator: Optional[ValidationEvaluator] = None ) -> None: super(GraphDependencyParser, self).__init__(vocab, regularizer) self.validation_evaluator = validation_evaluator self.text_field_embedder = text_field_embedder self.encoder = encoder self._pos_tag_embedding = pos_tag_embedding or None self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self._head_sentinel = torch.nn.Parameter( torch.randn([1, 1, encoder.get_output_dim()])) representation_dim = text_field_embedder.get_output_dim() if pos_tag_embedding is not None: representation_dim += pos_tag_embedding.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") check_dimensions_match(encoder.get_output_dim(), edge_model.encoder_dim(), "encoder output dim", "input dim edge model") self.use_mst_decoding_for_validation = use_mst_decoding_for_validation tags = self.vocab.get_token_to_index_vocabulary("pos") punctuation_tag_indices = { tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE } self._pos_to_ignore = set(punctuation_tag_indices.values()) logger.info( f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. " "Ignoring words with these POS tags for evaluation.") self._attachment_scores = AttachmentScores() initializer(self) self.edge_model = edge_model self.loss_function = loss_function #Being able to detect what state we are in, probably not the best idea. self.current_epoch = 1 self.pass_over_data_just_started = True
def __init__(self, options_file , weight_file , num_output_representations , requires_grad = False, do_layer_norm = False, dropout = 0.5, vocab_to_cache = None, module = None) : super(Elmo, self).__init__() logging.info(u"Initializing ELMo") if module is not None: if options_file is not None or weight_file is not None: raise ConfigurationError( u"Don't provide options_file or weight_file with module") self._elmo_lstm = module else: self._elmo_lstm = _ElmoBiLm(options_file, weight_file, requires_grad=requires_grad, vocab_to_cache=vocab_to_cache) self._has_cached_vocab = vocab_to_cache is not None self._dropout = Dropout(p=dropout) self._scalar_mixes = [] for k in range(num_output_representations): scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm) self.add_module(u'scalar_mix_{}'.format(k), scalar_mix) self._scalar_mixes.append(scalar_mix)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, stacked_encoder: Seq2SeqEncoder, predicate_feature_dim: int, dim_hidden: int = 100, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None): super(SpanDetector, self).__init__(vocab, regularizer) self.dim_hidden = dim_hidden self.text_field_embedder = text_field_embedder self.predicate_feature_embedding = Embedding(2, predicate_feature_dim) self.embedding_dropout = Dropout(p=embedding_dropout) self.threshold_metric = ThresholdMetric() self.stacked_encoder = stacked_encoder self.span_hidden = SpanRepAssembly( self.stacked_encoder.get_output_dim(), self.stacked_encoder.get_output_dim(), self.dim_hidden) self.pred = TimeDistributed(Linear(self.dim_hidden, 1))
def __init__(self, serialization_dir, cuda_device=0) -> None: super(DependencyParsingEmbedding2, self).__init__() from allennlp.models.archival import load_archive self.serialization_dir = serialization_dir self.parameter_filename = os.path.join(serialization_dir, "config.json") self.weights_filename = os.path.join(serialization_dir, "weights.th") self.cuda_device = cuda_device self.config = Params.from_file(self.parameter_filename) self.archive = load_archive(self.serialization_dir) self.model = self.archive.model self.model.eval() self.dataset_reader_params = self.config["dataset_reader"] self.dataset_reader = DatasetReader.from_params( self.dataset_reader_params) self.tokenizer = SpacyWordSplitter(language='en_core_web_sm', pos_tags=True, wst=True) num_output_representations = 1 do_layer_norm = False num_layers = 3 self._keep_sentence_boundaries = False self._dropout = Dropout(p=0.5) self._scalar_mixes: Any = [] for k in range(num_output_representations): scalar_mix = ScalarMix(num_layers, do_layer_norm=do_layer_norm) self.add_module('scalar_mix_{}'.format(k), scalar_mix) self._scalar_mixes.append(scalar_mix)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, stacked_encoder: Seq2SeqEncoder = None, predicate_feature_dim: int = 0, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None): super(SentenceEncoder, self).__init__() self._text_field_embedder = text_field_embedder self._stacked_encoder = stacked_encoder self._predicate_feature_dim = predicate_feature_dim self._embedding_dropout = Dropout(p=embedding_dropout) if self._predicate_feature_dim > 0: self._predicate_feature_embedding = Embedding( 2, predicate_feature_dim) if self._stacked_encoder is not None: embedding_dim_with_predicate_feature = self._text_field_embedder.get_output_dim( ) + self._predicate_feature_dim if embedding_dim_with_predicate_feature != self._stacked_encoder.get_input_dim( ): raise ConfigurationError( ("Input dimension of sentence encoder (%s) must be " % self._stacked_encoder.get_input_dim()) + \ ("the sum of predicate feature dim and text embedding dim (%s)." % (embedding_dim_with_predicate_feature))) self._metric = BinaryF1()
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, # binary_feature_dim: int, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(LstmSwag, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder # For the span based evaluation, we don't want to consider labels # for verb, because the verb index is provided to the model. self.encoder = encoder self.embedding_dropout = Dropout(p=embedding_dropout) self.output_prediction = Linear(self.encoder.get_output_dim(), 1, bias=False) check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), "text embedding dim", "eq encoder input dim") self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, input_size, output_size, convs_configs, dropout=.5, normalize=True): super(MultiCNN, self).__init__() input_size = input_size concat_cnn_output_size = 0 output_size = output_size self._normalize = normalize self.cnn_modules = nn.ModuleList() for configs in convs_configs: channel_size = configs['channel_size'] kernel_size = configs['kernel_size'] padding = configs['padding'] concat_cnn_output_size += channel_size module = Conv1d(input_size, channel_size, kernel_size=kernel_size, padding=padding) self.cnn_modules.append(module) self.batch_normlize = BatchNorm1d(concat_cnn_output_size) self.dropout = Dropout(dropout) self.output_linear = Linear(concat_cnn_output_size, output_size)
def __init__(self, options_file: str, weight_file: str, num_output_representations: int, requires_grad: bool = False, do_layer_norm: bool = False, dropout: float = 0.5, module: torch.nn.Module = None) -> None: super(Elmo, self).__init__() logging.info("Initializing ELMo") if module is not None: if options_file is not None or weight_file is not None: raise ConfigurationError( "Don't provide options_file or weight_file with module") self._elmo_lstm = module else: self._elmo_lstm = _ElmoBiLm(options_file, weight_file, requires_grad=requires_grad) self._dropout = Dropout(p=dropout) ################################################## # Eq.1 in the paper: gamma * sum(s_k * tensor_k) # ################################################## self._scalar_mixes: Any = [] for k in range(num_output_representations): # gamma * sum(s_k * tensor_k) scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm) self.add_module('scalar_mix_{}'.format(k), scalar_mix) self._scalar_mixes.append(scalar_mix)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, dropout: float = 0.0, input_dropout: float = 0.0, label_namespace: str = "pos", initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(PosTaggerMonolingual, self).__init__(vocab, regularizer) self.label_namespace = label_namespace self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size(label_namespace) self.encoder = encoder self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(), self.num_classes)) check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), "text field embedding dim", "encoder input dim") self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3) } initializer(self)
def __init__(self, options_file: str, weight_file: str, num_output_representations: int, requires_grad: bool = False, do_layer_norm: bool = False, dropout: float = 0.5, vocab_to_cache: List[str] = None, keep_sentence_boundaries: bool = True, scalar_mix_parameters: List[float] = None, module: torch.nn.Module = None, combine_method="weighted-sum", random_init=False) -> None: super(Elmo, self).__init__() logger.info("Initializing ELMo") self._elmo_lstm = _LatticeElmoBiLm(options_file, weight_file, requires_grad=requires_grad, vocab_to_cache=vocab_to_cache, combine_method=combine_method, random_init=random_init) self._has_cached_vocab = vocab_to_cache is not None self._keep_sentence_boundaries = keep_sentence_boundaries self._dropout = Dropout(p=dropout) self._scalar_mixes: Any = [] for k in range(num_output_representations): scalar_mix = ScalarMix( self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm, initial_scalar_parameters=scalar_mix_parameters, trainable=scalar_mix_parameters is None, ) self.add_module("scalar_mix_{}".format(k), scalar_mix) self._scalar_mixes.append(scalar_mix)
def __init__( self, options_file: str, weight_file: str, num_output_representations: int, requires_grad: bool = False, do_layer_norm: bool = False, dropout: float = 0.5, vocab_to_cache: List[str] = None, module: torch.nn.Module = None, ) -> None: super(Elmo, self).__init__() logging.info("Initializing ELMo") if module is not None: if options_file is not None or weight_file is not None: raise ValueError("Don't provide options_file or weight_file with module") self._elmo_lstm = module else: self._elmo_lstm = _ElmoBiLm( options_file, weight_file, requires_grad=requires_grad, vocab_to_cache=vocab_to_cache, ) self._has_cached_vocab = vocab_to_cache is not None self._dropout = Dropout(p=dropout) self._scalar_mixes: Any = [] for k in range(num_output_representations): scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm) self.add_module("scalar_mix_{}".format(k), scalar_mix) self._scalar_mixes.append(scalar_mix)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, stacked_encoder: Seq2SeqEncoder, binary_feature_dim: int, initializer: InitializerApplicator, embedding_dropout: float = 0.0) -> None: super(SemanticRoleLabeler, self).__init__(vocab) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") # For the span based evaluation, we don't want to consider labels # for verb, because the verb index is provided to the model. self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=["V"]) self.stacked_encoder = stacked_encoder # There are exactly 2 binary features for the verb predicate embedding. self.binary_feature_embedding = Embedding(2, binary_feature_dim) self.tag_projection_layer = TimeDistributed( Linear(self.stacked_encoder.get_output_dim(), self.num_classes)) self.embedding_dropout = Dropout(p=embedding_dropout) initializer(self) if text_field_embedder.get_output_dim( ) + binary_feature_dim != stacked_encoder.get_input_dim(): raise ConfigurationError( "The SRL Model uses a binary verb indicator feature, meaning " "the input dimension of the stacked_encoder must be equal to " "the output dimension of the text_field_embedder + 1.")
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, feedforward: Optional[FeedForward] = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, dropout: float = 0.0, label_name: str = 'target-sentiment-labels') -> None: super().__init__(vocab, regularizer) ''' :param vocab: A Vocabulary, required in order to compute sizes for input/output projections. :param embedder: Used to embed the text. :param encoder: Encodes the sentence/text. E.g. LSTM :param feedforward: An optional feed forward layer to apply after the encoder :param initializer: Used to initialize the model parameters. :param regularizer: If provided, will be used to calculate the regularization penalty during training. :param dropout: To apply dropout after each layer apart from the last layer. All dropout that is applied to timebased data will be `variational dropout`_ all else will be standard dropout. :param label_name: Name of the label name space. This is based on the LSTM model by `Tang et al. 2016 <https://www.aclweb.org/anthology/C16-1311.pdf>`_ ''' self.label_name = label_name self.embedder = embedder self.encoder = encoder self.num_classes = self.vocab.get_vocab_size(self.label_name) self.feedforward = feedforward if feedforward is not None: output_dim = self.feedforward.get_output_dim() else: output_dim = self.encoder.get_output_dim() self.label_projection = Linear(output_dim, self.num_classes) self.metrics = {"accuracy": CategoricalAccuracy()} self.f1_metrics = {} # F1 Scores label_index_name = self.vocab.get_index_to_token_vocabulary( self.label_name) for label_index, _label_name in label_index_name.items(): _label_name = f'F1_{_label_name.capitalize()}' self.f1_metrics[_label_name] = F1Measure(label_index) self._variational_dropout = InputVariationalDropout(dropout) self._naive_dropout = Dropout(dropout) check_dimensions_match(embedder.get_output_dim(), encoder.get_input_dim(), 'Embedding', 'Encoder') if self.feedforward is not None: check_dimensions_match(encoder.get_output_dim(), feedforward.get_input_dim(), 'Encoder', 'FeedForward') initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, tag_representation_dim: int, arc_representation_dim: int, pos_tag_embedding: Embedding = None, use_mst_decoding_for_validation: bool = True, dropout: float = 0.0, input_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(BiaffineDependencyParser, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder encoder_dim = encoder.get_output_dim() self.head_arc_projection = torch.nn.Linear(encoder_dim, arc_representation_dim) self.child_arc_projection = torch.nn.Linear(encoder_dim, arc_representation_dim) self.arc_attention = BilinearMatrixAttention(arc_representation_dim, arc_representation_dim, use_input_biases=True) num_labels = self.vocab.get_vocab_size("head_tags") self.head_tag_projection = torch.nn.Linear(encoder_dim, tag_representation_dim) self.child_tag_projection = torch.nn.Linear(encoder_dim, tag_representation_dim) self.tag_bilinear = torch.nn.modules.Bilinear(tag_representation_dim, tag_representation_dim, num_labels) self._pos_tag_embedding = pos_tag_embedding or None self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) representation_dim = text_field_embedder.get_output_dim() if pos_tag_embedding is not None: representation_dim += pos_tag_embedding.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") self.use_mst_decoding_for_validation = use_mst_decoding_for_validation tags = self.vocab.get_token_to_index_vocabulary("pos") punctuation_tag_indices = { tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE } self._pos_to_ignore = set(punctuation_tag_indices.values()) logger.info( f"Found POS tags correspoding to the following punctuation : {punctuation_tag_indices}. " "Ignoring words with these POS tags for evaluation.") self._attachment_scores = AttachmentScores() initializer(self)
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu"): super(TransformerEncoderLayer, self).__init__() self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout) # Implementation of Feedforward model self.linear1 = Linear(d_model, dim_feedforward) self.dropout = Dropout(dropout) self.linear2 = Linear(dim_feedforward, d_model) self.norm1 = LayerNorm(d_model) self.norm2 = LayerNorm(d_model) self.dropout1 = Dropout(dropout) self.dropout2 = Dropout(dropout) self.activation = _get_activation_fn(activation)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, binary_feature_dim: int, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, label_smoothing: float = None, label_namespace: str = "labels", ignore_span_metric: bool = False, label_encoding: Optional[str] = 'BIO', include_start_end_transitions: bool = True, constrain_crf_decoding: bool = True) -> None: super(OieLabelerCRF, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") # For the span based evaluation, we don't want to consider labels # for verb, because the verb index is provided to the model. self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=["V"]) self.label_namespace = label_namespace self.encoder = encoder # There are exactly 2 binary features for the verb predicate embedding. self.binary_feature_embedding = Embedding(2, binary_feature_dim) self.tag_projection_layer = TimeDistributed( Linear(self.encoder.get_output_dim(), self.num_classes)) self.embedding_dropout = Dropout(p=embedding_dropout) self._label_smoothing = label_smoothing self.ignore_span_metric = ignore_span_metric self.include_start_end_transitions = include_start_end_transitions if constrain_crf_decoding is None: constrain_crf_decoding = label_encoding is not None if constrain_crf_decoding: labels = self.vocab.get_index_to_token_vocabulary(label_namespace) print(labels) constraints = allowed_transitions(label_encoding, labels) else: constraints = None self.crf = ConditionalRandomField( self.num_classes, constraints, include_start_end_transitions=include_start_end_transitions) check_dimensions_match( text_field_embedder.get_output_dim() + binary_feature_dim, encoder.get_input_dim(), "text embedding dim + verb indicator embedding dim", "encoder input dim") initializer(self)
def __init__(self, vocab: Vocabulary, source_embedder: TextFieldEmbedder, sentence_encoder: Seq2VecEncoder, sentence_accumulator: Seq2SeqEncoder, use_salience: bool, use_pos_embedding: bool, use_output_bias: bool, use_novelty: bool, dropout: float = 0.3, pos_embedding_num: int = 50, pos_embedding_size: int = 128) -> None: super(SummaRuNNer, self).__init__(vocab) self._source_embedder = source_embedder self._sentence_encoder = sentence_encoder self._se_output_dim = self._sentence_encoder.get_output_dim() self._sentence_accumulator = sentence_accumulator self._h_sentence_dim = self._sentence_accumulator.get_output_dim() self._dropout_layer = Dropout(dropout) self._content_projection_layer = Linear(self._h_sentence_dim, 1) self._use_salience = use_salience if use_salience: self._document_linear_layer = Linear(self._h_sentence_dim, self._h_sentence_dim, bias=True) self._salience_linear_layer = Linear(self._h_sentence_dim, self._h_sentence_dim, bias=False) self._use_pos_embedding = use_pos_embedding if use_pos_embedding: self._pos_embedding_num = pos_embedding_num self._pos_embedding_size = pos_embedding_size self._pos_embedding_layer = Embedding(pos_embedding_num, pos_embedding_size) self._pos_projection_layer = Linear(pos_embedding_size, 1) self._use_output_bias = use_output_bias if use_output_bias: self._output_bias = Parameter(torch.zeros(1).uniform_(-0.1, 0.1), requires_grad=True) self._use_novelty = use_novelty if use_novelty: self._novelty_linear_layer = Linear(self._h_sentence_dim, self._h_sentence_dim, bias=False)
def __init__( self, vocab: Vocabulary, rnn: Seq2SeqEncoder, latent_dim: int, target_embedder: TextFieldEmbedder, target_namespace: str = 'tokens', dropout_p: float = 0.5, ) -> None: super().__init__(vocab) self._target_embedder = target_embedder self._target_namespace = target_namespace self.rnn = rnn self.dec_num_layers = self.rnn._module.num_layers # noqa: WPS437 self.dec_hidden = self.rnn._module.hidden_size # noqa: WPS437 self._latent_to_dec_hidden = Linear(latent_dim, self.dec_hidden) self._dec_linear = Linear(self.rnn.get_output_dim(), self.vocab.get_vocab_size()) self._emb_dropout = Dropout(dropout_p) self._dec_dropout = Dropout(dropout_p)
def __init__(self, rnn_size, dropout=DROPOUT, max_len=5000): super(PositionalEncoding, self).__init__() self.dropout = Dropout(p=dropout) # Compute the positional encodings once in log space pe = torch.zeros(max_len, rnn_size) position = torch.arange(0.0, max_len).unsqueeze(1) div_term = torch.exp( torch.arange(0.0, rnn_size, 2) * -(math.log(10000.0) / rnn_size)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0) self.register_buffer('pe', pe)
def __init__(self, vocab: Vocabulary, target_embedder: TextFieldEmbedder, rnn: Seq2SeqEncoder, latent_dim: int, dropout_p: float = 0.5) -> None: super(VariationalDecoder, self).__init__() self.vocab = vocab self._target_embedder = target_embedder self.rnn = rnn self.dec_num_layers = self.rnn._module.num_layers # pylint: disable=protected-access self.dec_hidden = self.rnn._module.hidden_size # pylint: disable=protected-access self._latent_to_dec_hidden = Linear(latent_dim, self.dec_hidden) self._dec_linear = Linear(self.rnn.get_output_dim(), self.vocab.get_vocab_size()) self._emb_dropout = Dropout(dropout_p) self._dec_dropout = Dropout(dropout_p) self._start_index = self.vocab.get_token_index(START_SYMBOL) self._end_index = self.vocab.get_token_index(END_SYMBOL)
def __init__(self, vocab: Vocabulary, source_embedder: TextFieldEmbedder, sentence_encoder: Seq2VecEncoder, sentence_accumulator: Seq2SeqEncoder, use_salience: bool, use_pos_embedding: bool, use_output_bias: bool, use_novelty: bool, dropout: float = 0.3, pos_embedding_num: int = 50, pos_embedding_size: int = 128) -> None: super().__init__(vocab) self._source_embedder = source_embedder self._sentence_encoder = sentence_encoder self._se_output_dim = self._sentence_encoder.get_output_dim() self._sentence_accumulator = sentence_accumulator self._h_sentence_dim = self._sentence_accumulator.get_output_dim() self._dropout_layer = Dropout(dropout) self._content_projection_layer = Linear(self._h_sentence_dim, 1) # options to train the system on self._use_pos_embedding = use_pos_embedding self._use_salience = use_salience self._use_novelty = use_novelty if use_salience: self._document_linear_layer = Linear(self._h_sentence_dim, self._h_sentence_dim, bias=True) self._salience_linear_layer = Linear(self._h_sentence_dim, self._h_sentence_dim, bias=False) if use_pos_embedding: self._pos_embedding_num = pos_embedding_num self._pos_embedding_size = pos_embedding_size self._pos_embedding_layer = Embedding(pos_embedding_num, pos_embedding_size) self._pos_projection_layer = Linear(pos_embedding_size, 1) self._use_novelty = use_novelty if use_novelty: self._novelty_linear_layer = Linear(self._h_sentence_dim, self._h_sentence_dim, bias=False)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: MTLWeightSharer, tasks: List[AMTask], pos_tag_embedding: Embedding = None, lemma_embedding: Embedding = None, ne_embedding: Embedding = None, input_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, tok2vec: Optional[TokenToVec] = None) -> None: super(GraphDependencyParser, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder self.tok2vec = tok2vec self._pos_tag_embedding = pos_tag_embedding or None self._lemma_embedding = lemma_embedding self._ne_embedding = ne_embedding self._input_dropout = Dropout(input_dropout) self._head_sentinel = torch.nn.Parameter( torch.randn([1, 1, encoder.get_output_dim()])) representation_dim = text_field_embedder.get_output_dim() if pos_tag_embedding is not None: representation_dim += pos_tag_embedding.get_output_dim() if self._lemma_embedding is not None: representation_dim += lemma_embedding.get_output_dim() if self._ne_embedding is not None: representation_dim += ne_embedding.get_output_dim() assert len(tasks) > 0, "List of tasks must not be empty" self.tasks: Dict[str, AMTask] = {t.name: t for t in tasks} if self.tok2vec: representation_dim += self.tok2vec.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") for t in tasks: t.check_all_dimensions_match(encoder.get_output_dim()) for formalism, task in sorted(self.tasks.items(), key=lambda nt: nt[0]): #sort by name of formalism for consistent ordering self.add_module(formalism, task) initializer(self)
def __init__(self, options_file: str, weight_file: str, num_output_representations: int, do_layer_norm: bool = False, dropout: float = 0.5) -> None: super(Elmo, self).__init__() self._elmo_lstm = _ElmoBiLm(options_file, weight_file) self._dropout = Dropout(p=dropout) self._scalar_mixes: Any = [] for k in range(num_output_representations): scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm) self.add_module('scalar_mix_{}'.format(k), scalar_mix) self._scalar_mixes.append(scalar_mix)
def __init__(self, in_size: int, out_size: int, dropout: float) -> None: """ :param in_size: dimensionality of input vectors :param out_size: dimensionality of hidden states and output vectors :param dropout: dropout ratio """ super().__init__() self.in_size = in_size self.state_size = out_size if dropout == 0.0: self._dropout = lambda x: x else: self._dropout = Dropout(dropout) self._topdown_lstm = ChildSumTreeLSTM(in_size, out_size) self._bottomup_lstm = ChildSumTreeLSTM(in_size, out_size)
def __init__(self, vocab: Vocabulary, bert_model: Union[str, BertModel], pretrained_conditional_semanticbert: str, embedding_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, label_smoothing: float = None, ignore_span_metric: bool = False) -> None: super(SrlBert, self).__init__(vocab, regularizer) if isinstance(bert_model, str): self.bert_model = BertModel.from_pretrained(bert_model) else: self.bert_model = bert_model # load pre-trained conditional semanticbert self.conditional_semanticbert = ConditionalSemanticBert.from_pretrained( "bert-base-uncased") print('load a pre-trained model from ' + pretrained_conditional_semanticbert) pretrained_state_dict = torch.load(pretrained_conditional_semanticbert) model_state_dict = self.conditional_semanticbert.state_dict() print('pretrained_state_dict', pretrained_state_dict.keys()) print('model_state_dict', model_state_dict.keys()) pretrained_state = { k: v for k, v in pretrained_state_dict.items() if k in model_state_dict and v.size() == model_state_dict[k].size() } model_state_dict.update(pretrained_state) print('updated_state_dict', model_state_dict.keys()) self.conditional_semanticbert.load_state_dict(model_state_dict) self.conditional_semanticbert.to('cuda') self.num_classes = self.vocab.get_vocab_size("labels") # For the span based evaluation, we don't want to consider labels # for verb, because the verb index is provided to the model. self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=["V"]) self.tag_projection_layer = Linear( self.bert_model.config.hidden_size * 2, self.num_classes) self.embedding_dropout = Dropout(p=embedding_dropout) self._label_smoothing = label_smoothing self.ignore_span_metric = ignore_span_metric initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, dropout: float = 0.0, input_dropout: float = 0.0, label_namespace: str = "pos", treebank_embedding: Embedding = None, use_treebank_embedding: bool = True, langs_for_early_stop: List[str] = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(PosTaggerTbemb, self).__init__(vocab, regularizer) self.label_namespace = label_namespace self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size(label_namespace) self.encoder = encoder self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self._langs_for_early_stop = langs_for_early_stop or [] self._treebank_embedding = treebank_embedding or None self._use_treebank_embedding = use_treebank_embedding self._lang_accuracy_scores: Dict[ str, CategoricalAccuracy] = defaultdict(CategoricalAccuracy) self.tag_projection_layer = TimeDistributed( Linear(self.encoder.get_output_dim(), self.num_classes)) representation_dim = text_field_embedder.get_output_dim() if treebank_embedding is not None: representation_dim += treebank_embedding.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") if self._use_treebank_embedding: tbids = self.vocab.get_token_to_index_vocabulary("tbids") tbid_indices = {tb: index for tb, index in tbids.items()} self._tbids = set(tbid_indices.values()) logger.info( f"Found TBIDs corresponding to the following treebanks : {tbid_indices}. " "Embedding these as additional features.") initializer(self)