def __init__( self, vocab: Vocabulary, sequence_field_embedder: TextFieldEmbedder, structure_field_embedder: TextFieldEmbedder, predicted_loop_type_field_embedder: TextFieldEmbedder, seq2seq_encoder: Seq2SeqEncoder, loss: Loss, structure_field_attention: Optional[MatrixAttention] = None, predicted_loop_type_field_attention: Optional[MatrixAttention] = None, masked_lm: Optional[Model] = None, # MaskedLanguageModel lm_is_trainable: bool = False, lm_matrix_attention: Optional[MatrixAttention] = None, lm_dropout: float = 0.0, emb_dropout: float = 0.0, bpps_aggegator: Optional[Aggregator] = None, bpp_dropout: float = 0.0, regularizer: RegularizerApplicator = None, ) -> None: super().__init__(vocab, regularizer) # embedders self._sequence_field_embedder = sequence_field_embedder self._structure_field_embedder = structure_field_embedder self._predicted_loop_type_field_embedder = predicted_loop_type_field_embedder self._seq2seq_encoder = seq2seq_encoder self._loss = loss self._structure_field_attention = structure_field_attention self._predicted_loop_type_field_attention = predicted_loop_type_field_attention # masked language models self._masked_lm = masked_lm if self._masked_lm is not None: self._masked_lm._tokens_masker = None if not lm_is_trainable and self._masked_lm is not None: self._masked_lm = self._masked_lm.eval() self._lm_matrix_attention = lm_matrix_attention self._lm_dropout = InputVariationalDropout(p=lm_dropout) self._emb_dropout = InputVariationalDropout(p=emb_dropout) self._bpps_aggegator = bpps_aggegator self._bpp_dropout = InputVariationalDropout(p=bpp_dropout) hidden_dim = self._seq2seq_encoder.get_output_dim() if self._masked_lm is not None: hidden_dim += self._masked_lm.get_output_dim() # we predict reactivity, deg_Mg_pH10, deg_Mg_50C, deg_pH10, deg_50C self._linear = torch.nn.Linear(hidden_dim, 5)
def __init__(self, config: ParsingConfig): assert isinstance(config, ParsingConfig) super().__init__(config) self.config = config encoder_dim = config.decoder_config.output_dim if self.config.use_pos: self.pos_embedding = nn.Embedding(config.num_pos, config.pos_dim, padding_idx=0) encoder_dim += config.pos_dim self.head_arc_feedforward = FeedForward(encoder_dim, 1, config.arc_dim, Activation.by_name("elu")()) self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward) self.arc_attention = BilinearMatrixAttention(config.arc_dim, config.arc_dim, use_input_biases=True) self.head_tag_feedforward = FeedForward(encoder_dim, 1, config.tag_dim, Activation.by_name("elu")()) self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward) self.tag_bilinear = torch.nn.modules.Bilinear(config.tag_dim, config.tag_dim, config.num_labels) self.dropout = InputVariationalDropout(config.dropout) self.use_mst_decoding_for_validation = config.use_mst_decoding_for_validation
def __init__(self, vocab: Vocabulary, token_embedder: TokenEmbedder, num_labels: int) -> None: super().__init__(vocab) self.word_embedders = BasicTextFieldEmbedder( {"tokens": token_embedder}) self._encoder = LstmSeq2SeqEncoder(300, 300, 2) # self._encoder = PytorchTransformer(300, 3, 300, 4) self._matrix_attention = DotProductMatrixAttention() self._projection_feedforward = FeedForward(300 * 4, 1, 300, torch.nn.ReLU(), 0.2) self._inference_encoder = LstmSeq2SeqEncoder(300, 300, 2) # self._inference_encoder = PytorchTransformer(300, 3, 300, 4) self.dropout = torch.nn.Dropout(0.3) self.rnn_input_dropout = InputVariationalDropout(0.3) self._output_feedforward = FeedForward(1200, 1, 300, torch.nn.ReLU(), 0.2) self._output_logit = FeedForward(300, 1, num_labels, lambda x: x) self._num_labels = num_labels self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss()
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, phrase_layer: Seq2SeqEncoder, pq_attention: MatrixAttention, p_selfattention: MatrixAttention, supports_pooling: Seq2VecEncoder, query_pooling: Seq2VecEncoder, candidates_pooling: Seq2VecEncoder, decoder: Decoder, dropout: float = 0.2, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(MultiStepParaRankModel, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.phrase_layer = phrase_layer self.pq_attention = pq_attention self.p_selfattention = p_selfattention self.supports_pooling = supports_pooling self.query_pooling = query_pooling self.candidates_pooling = candidates_pooling self.decoder = decoder self.dropout = InputVariationalDropout(p=dropout) self._support_accuracy = Auc() self._candidate_accuracy = CategoricalAccuracy() initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, matrix_attention: MatrixAttention, projection_feedforward: FeedForward, inference_encoder: Seq2SeqEncoder, output_feedforward: FeedForward, output_logit: FeedForward, dropout: float = 0.5, initializer: InitializerApplicator = InitializerApplicator(), **kwargs, ) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = text_field_embedder self._encoder = encoder self._matrix_attention = matrix_attention self._projection_feedforward = projection_feedforward self._inference_encoder = inference_encoder if dropout: self.dropout = torch.nn.Dropout(dropout) self.rnn_input_dropout = InputVariationalDropout(dropout) else: self.dropout = None self.rnn_input_dropout = None self._output_feedforward = output_feedforward self._output_logit = output_logit self._num_labels = vocab.get_vocab_size(namespace="labels") check_dimensions_match( text_field_embedder.get_output_dim(), encoder.get_input_dim(), "text field embedding dim", "encoder input dim", ) check_dimensions_match( encoder.get_output_dim() * 4, projection_feedforward.get_input_dim(), "encoder output dim", "projection feedforward input", ) check_dimensions_match( projection_feedforward.get_output_dim(), inference_encoder.get_input_dim(), "proj feedforward output dim", "inference lstm input dim", ) self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() self._debug = 2 initializer(self)
def __init__(self, config, num_labels: int, num_pos: int, use_pos: bool, arc_representation_dim: int, arc_feedforward: FeedForward = None, use_mst_decoding_for_validation: bool = True, dropout: float = 0.) -> None: super(DistanceDependencyParser, self).__init__(config) self.bert = BertModel(config) self.apply(self.init_bert_weights) encoder_dim = config.hidden_size self.arc_feedforward = arc_feedforward or \ FeedForward(encoder_dim, 1, arc_representation_dim, Activation.by_name("linear")()) self.arc_attention = DistanceAttention() self._dropout = InputVariationalDropout(dropout) self.use_mst_decoding_for_validation = use_mst_decoding_for_validation self._attachment_scores = UndirectedAttachmentScores()
def __init__(self, device): super(UnbiasedTopicalExtractorELMo, self).__init__() self.device = device self.tokenizer = WordTokenizer() options_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" self.elmo_model = Elmo(options_file, weight_file, 1, do_layer_norm=False, dropout=0.5) self.rnn_input_dropout = InputVariationalDropout(0.0) self.encoder = _Seq2SeqWrapper(torch.nn.LSTM)(input_size=1024, hidden_size=300, num_layers=1, bidirectional=True) self._feature_feedforward_layer = torch.nn.Linear(600, 300) self._feature_feedforward_dropout = torch.nn.Dropout(0.5) self._feature_feedforward_activation = torch.nn.ReLU() self._class_classification_layer = torch.nn.Linear(300, 2) self._group_classification_layer = torch.nn.Linear(300, 2) init.xavier_uniform_(self._feature_feedforward_layer.weight) init.zeros_(self._feature_feedforward_layer.bias) init.xavier_uniform_(self._class_classification_layer.weight) init.zeros_(self._class_classification_layer.bias) init.xavier_uniform_(self._group_classification_layer.weight) init.zeros_(self._group_classification_layer.bias) self._class_loss = torch.nn.CrossEntropyLoss() self._group_loss = torch.nn.CrossEntropyLoss()
def __init__(self, input_size: int, hidden_size: int, decoder_layer: MisoTransformerDecoderLayer, num_layers: int, source_attention_layer: AttentionLayer, target_attention_layer: AttentionLayer, norm=None, dropout=0.1, use_coverage=True): super(MisoTransformerDecoder, self).__init__() self.input_proj_layer = torch.nn.Linear(input_size, hidden_size) self.layers = _get_clones(decoder_layer, num_layers) self.num_layers = num_layers self.norm = norm self.dropout = InputVariationalDropout(dropout) self.source_attn_layer = source_attention_layer self.target_attn_layer = target_attention_layer self.use_coverage = use_coverage self.prenorm = isinstance(decoder_layer, MisoPreNormTransformerDecoderLayer) if self.prenorm: self.final_norm = copy.deepcopy(decoder_layer.norm4)
def __init__(self, device, vocab_dir): super(UnbiasedTopicalExtractorGloVe, self).__init__() self.device = device vocab = Vocabulary() self.vocab = vocab.from_files(vocab_dir) self.embedding = Embedding( embedding_dim=300, trainable=True, num_embeddings=self.vocab.get_vocab_size("tokens"), pretrained_file= "https://allennlp.s3.amazonaws.com/datasets/glove/glove.840B.300d.txt.gz" ) self.tokenizer = WordTokenizer() self.token_indexer = SingleIdTokenIndexer(lowercase_tokens=True) self.rnn_input_dropout = InputVariationalDropout(0.5) self.encoder = _Seq2SeqWrapper(torch.nn.LSTM)(input_size=300, hidden_size=300, num_layers=1, bidirectional=True) self._feature_feedforward_layer = torch.nn.Linear(600, 300) self._feature_feedforward_dropout = torch.nn.Dropout(0.5) self._feature_feedforward_activation = torch.nn.ReLU() self._class_classification_layer = torch.nn.Linear(300, 2) self._group_classification_layer = torch.nn.Linear(300, 2) init.xavier_uniform_(self._feature_feedforward_layer.weight) init.zeros_(self._feature_feedforward_layer.bias) init.xavier_uniform_(self._class_classification_layer.weight) init.zeros_(self._class_classification_layer.bias) init.xavier_uniform_(self._group_classification_layer.weight) init.zeros_(self._group_classification_layer.bias) self._class_loss = torch.nn.CrossEntropyLoss() self._group_loss = torch.nn.CrossEntropyLoss()
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, dropout: float = 0.0, input_dropout: float = 0.0, label_namespace: str = "pos", initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(PosTaggerMonolingual, self).__init__(vocab, regularizer) self.label_namespace = label_namespace self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size(label_namespace) self.encoder = encoder self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(), self.num_classes)) check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), "text field embedding dim", "encoder input dim") self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3) } initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, output_feedforward: FeedForward, output_logit: FeedForward, dropout: float = 0.5, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder self._encoder = encoder if dropout: self.dropout = torch.nn.Dropout(dropout) self.rnn_input_dropout = InputVariationalDropout(dropout) else: self.dropout = None self.rnn_input_dropout = None self._output_feedforward = output_feedforward self._output_logit = output_logit self._num_labels = vocab.get_vocab_size(namespace="labels") check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), "text field embedding dim", "encoder input dim") self._f1 = F1Measure( positive_label=vocab._token_to_index["labels"]["1"]) self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, edge_model: graph_dependency_parser.components.edge_models. EdgeModel, loss_function: graph_dependency_parser.components.losses.EdgeLoss, pos_tag_embedding: Embedding = None, use_mst_decoding_for_validation: bool = True, dropout: float = 0.0, input_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, validation_evaluator: Optional[ValidationEvaluator] = None ) -> None: super(GraphDependencyParser, self).__init__(vocab, regularizer) self.validation_evaluator = validation_evaluator self.text_field_embedder = text_field_embedder self.encoder = encoder self._pos_tag_embedding = pos_tag_embedding or None self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self._head_sentinel = torch.nn.Parameter( torch.randn([1, 1, encoder.get_output_dim()])) representation_dim = text_field_embedder.get_output_dim() if pos_tag_embedding is not None: representation_dim += pos_tag_embedding.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") check_dimensions_match(encoder.get_output_dim(), edge_model.encoder_dim(), "encoder output dim", "input dim edge model") self.use_mst_decoding_for_validation = use_mst_decoding_for_validation tags = self.vocab.get_token_to_index_vocabulary("pos") punctuation_tag_indices = { tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE } self._pos_to_ignore = set(punctuation_tag_indices.values()) logger.info( f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. " "Ignoring words with these POS tags for evaluation.") self._attachment_scores = AttachmentScores() initializer(self) self.edge_model = edge_model self.loss_function = loss_function #Being able to detect what state we are in, probably not the best idea. self.current_epoch = 1 self.pass_over_data_just_started = True
def __init__( self, vocab: Vocabulary, option_encoder: Seq2SeqEncoder, input_dropout: float = 0.3, initializer: InitializerApplicator = InitializerApplicator(), ): super(LSTMBatchNormBUANonTagGlobalFullNoFinalImage, self).__init__(vocab) self.rnn_input_dropout = TimeDistributed( InputVariationalDropout( input_dropout)) if input_dropout > 0 else None self.obj_downsample = torch.nn.Sequential( torch.nn.Dropout(p=0.1), torch.nn.Linear(2048, 512), torch.nn.ReLU(inplace=True), ) self.image_BN = BatchNorm1d(512) self.option_encoder = TimeDistributed(option_encoder) self.option_BN = torch.nn.Sequential(BatchNorm1d(512)) self.query_BN = torch.nn.Sequential(BatchNorm1d(512)) self.final_mlp = torch.nn.Sequential( torch.nn.Linear(1024, 512), torch.nn.ReLU(inplace=True), ) self.final_BN = torch.nn.Sequential(BatchNorm1d(512)) self.final_mlp_linear = torch.nn.Sequential(torch.nn.Linear(512, 1)) self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, decoders: Dict[str, Model], tasks: List[str], task_types: List[str], dropout: float = None, **kwargs) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = text_field_embedder self.encoder = encoder self._classifier_input_dim = self.encoder.get_output_dim() if dropout: self._dropout = InputVariationalDropout(dropout) self._dropout_sents = torch.nn.Dropout(dropout) else: self._dropout = None self.decoders = torch.nn.ModuleDict(decoders) self.tasks = tasks self.task_types = task_types self.counter = 0 self.metrics = {}
def __init__(self, config: BaseDecoderConfig): assert isinstance(config, BaseDecoderConfig) super().__init__() self.config = config self.weight = nn.Parameter(torch.zeros(config.num_encoder_layers)) self.mapping: nn.Module = nn.Identity() self.dropout = InputVariationalDropout(config.input_dropout)
def __init__( self, similarity_function: SimilarityFunction, response_projection_feedforward: FeedForward, response_inference_encoder: Seq2SeqEncoder, response_input_feedforward: Optional[FeedForward] = None, source_input_feedforward: Optional[FeedForward] = None, source_projection_feedforward: Optional[FeedForward] = None, source_inference_encoder: Optional[Seq2SeqEncoder] = None, dropout: float = 0.5, #whether to only consider the response and alignments from the source to response response_only=False ) -> None: super().__init__() self._response_input_feedforward = response_input_feedforward self._response_projection_feedforward = response_projection_feedforward self._response_inference_encoder = response_inference_encoder self._source_input_feedforward = source_input_feedforward or response_input_feedforward self._source_projection_feedforward = source_projection_feedforward or response_projection_feedforward self._source_inference_encoder = source_inference_encoder or response_inference_encoder self._matrix_attention = LegacyMatrixAttention(similarity_function) if dropout: self.dropout = torch.nn.Dropout(dropout) self.rnn_input_dropout = InputVariationalDropout(dropout) else: self.dropout = None self.rnn_input_dropout = None self._response_only = response_only
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, feedforward: Optional[FeedForward] = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, dropout: float = 0.0, label_name: str = 'target-sentiment-labels') -> None: super().__init__(vocab, regularizer) ''' :param vocab: A Vocabulary, required in order to compute sizes for input/output projections. :param embedder: Used to embed the text. :param encoder: Encodes the sentence/text. E.g. LSTM :param feedforward: An optional feed forward layer to apply after the encoder :param initializer: Used to initialize the model parameters. :param regularizer: If provided, will be used to calculate the regularization penalty during training. :param dropout: To apply dropout after each layer apart from the last layer. All dropout that is applied to timebased data will be `variational dropout`_ all else will be standard dropout. :param label_name: Name of the label name space. This is based on the LSTM model by `Tang et al. 2016 <https://www.aclweb.org/anthology/C16-1311.pdf>`_ ''' self.label_name = label_name self.embedder = embedder self.encoder = encoder self.num_classes = self.vocab.get_vocab_size(self.label_name) self.feedforward = feedforward if feedforward is not None: output_dim = self.feedforward.get_output_dim() else: output_dim = self.encoder.get_output_dim() self.label_projection = Linear(output_dim, self.num_classes) self.metrics = {"accuracy": CategoricalAccuracy()} self.f1_metrics = {} # F1 Scores label_index_name = self.vocab.get_index_to_token_vocabulary( self.label_name) for label_index, _label_name in label_index_name.items(): _label_name = f'F1_{_label_name.capitalize()}' self.f1_metrics[_label_name] = F1Measure(label_index) self._variational_dropout = InputVariationalDropout(dropout) self._naive_dropout = Dropout(dropout) check_dimensions_match(embedder.get_output_dim(), encoder.get_input_dim(), 'Embedding', 'Encoder') if self.feedforward is not None: check_dimensions_match(encoder.get_output_dim(), feedforward.get_input_dim(), 'Encoder', 'FeedForward') initializer(self)
def __init__(self, vocab: Vocabulary, encoder_dim: int, label_dim: int, edge_dim: int, dropout: float, tag_feedforward: FeedForward = None, arc_feedforward: FeedForward = None) -> None: """ Parameters ---------- vocab : ``Vocabulary``, required A Vocabulary, required in order to compute sizes for input/output projections. encoder_dim : ``int``, required. The output dimension of the encoder. label_dim : ``int``, required. The dimension of the MLPs used for dependency tag prediction. edge_dim : ``int``, required. The dimension of the MLPs used for head arc prediction. tag_feedforward : ``FeedForward``, optional, (default = None). The feedforward network used to produce tag representations. By default, a 1 layer feedforward network with an elu activation is used. arc_feedforward : ``FeedForward``, optional, (default = None). The feedforward network used to produce arc representations. By default, a 1 layer feedforward network with an elu activation is used. dropout : ``float``, optional, (default = 0.0) The variational dropout applied to the output of the encoder and MLP layers. """ super(DMEdges, self).__init__(vocab) self._encoder_dim = encoder_dim self.head_arc_feedforward = arc_feedforward or \ FeedForward(encoder_dim, 1, edge_dim, Activation.by_name("elu")()) self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward) self.arc_attention = BilinearMatrixAttention(edge_dim, edge_dim, use_input_biases=True) num_labels = vocab.get_vocab_size("head_tags") #= edge labels self.head_tag_feedforward = tag_feedforward or \ FeedForward(encoder_dim, 1, label_dim, Activation.by_name("elu")()) self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward) self.tag_bilinear = torch.nn.modules.Bilinear(label_dim, label_dim, num_labels) self._dropout = InputVariationalDropout(dropout) check_dimensions_match(label_dim, self.head_tag_feedforward.get_output_dim(), "tag representation dim", "tag feedforward output dim") check_dimensions_match(edge_dim, self.head_arc_feedforward.get_output_dim(), "arc representation dim", "arc feedforward output dim")
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, tag_representation_dim: int, arc_representation_dim: int, pos_tag_embedding: Embedding = None, use_mst_decoding_for_validation: bool = True, dropout: float = 0.0, input_dropout: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(BiaffineDependencyParser, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder encoder_dim = encoder.get_output_dim() self.head_arc_projection = torch.nn.Linear(encoder_dim, arc_representation_dim) self.child_arc_projection = torch.nn.Linear(encoder_dim, arc_representation_dim) self.arc_attention = BilinearMatrixAttention(arc_representation_dim, arc_representation_dim, use_input_biases=True) num_labels = self.vocab.get_vocab_size("head_tags") self.head_tag_projection = torch.nn.Linear(encoder_dim, tag_representation_dim) self.child_tag_projection = torch.nn.Linear(encoder_dim, tag_representation_dim) self.tag_bilinear = torch.nn.modules.Bilinear(tag_representation_dim, tag_representation_dim, num_labels) self._pos_tag_embedding = pos_tag_embedding or None self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) representation_dim = text_field_embedder.get_output_dim() if pos_tag_embedding is not None: representation_dim += pos_tag_embedding.get_output_dim() check_dimensions_match(representation_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") self.use_mst_decoding_for_validation = use_mst_decoding_for_validation tags = self.vocab.get_token_to_index_vocabulary("pos") punctuation_tag_indices = { tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE } self._pos_to_ignore = set(punctuation_tag_indices.values()) logger.info( f"Found POS tags correspoding to the following punctuation : {punctuation_tag_indices}. " "Ignoring words with these POS tags for evaluation.") self._attachment_scores = AttachmentScores() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, text_encoder: Seq2SeqEncoder, classifier_feedforward: Optional[FeedForward] = None, dropout: Optional[float] = 0.0, code_switching_regularizer: Optional[float] = 0.0, bivalency_regularizer: Optional[float] = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: ''' :param dropout: The amount of dropout to apply. Dropout is applied after each non-linear layer and the word embeddings lookup. Two types of dropout are applied, variational dropout is applied if the input is to the dropout is a sequence of vectors (each vector in the sequence representing a word), and normal dropout if the input is a vector. :param code_switching_regularizer: The weight associated to the code switching lexicon regulisation the lower the less affect it has. This requires that the dataset reader is going to supply the code switching arrays for the forward function of this class. If set a good values is 0.001 :param bivalency_regularizer: The weight associated to the bivalency regulisation the lower the less affect it has. This requires that the dataset reader is going to supply the bivalency arrays for the forward function of this class. ''' super().__init__(vocab, regularizer) self._naive_dropout = Dropout(dropout) self._variational_dropout = InputVariationalDropout(dropout) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") self.text_encoder = text_encoder text_encoder_dim = text_encoder.get_output_dim() # Attention parameters self.project_encoded_text = TimeDistributed( Linear(text_encoder_dim, text_encoder_dim)) self.attention_vector = Parameter(torch.Tensor(text_encoder_dim)) self.reset_parameters() self.attention_layer = DotProductAttention(normalize=True) self.classifier_feedforward = classifier_feedforward output_dim = text_encoder_dim if classifier_feedforward: output_dim = classifier_feedforward.get_output_dim() self.label_projection = Linear(output_dim, self.num_classes) self.metrics = {"accuracy": CategoricalAccuracy()} self.code_switching_regularizer = code_switching_regularizer self.bivalency_regularizer = bivalency_regularizer self.loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, share_encoder: _EncoderBase, private_encoder: _EncoderBase, domain_embeddings: Embedding, s_domain_discriminator: Discriminator, p_domain_discriminator: Discriminator, valid_discriminator: Discriminator, dropout: float = 0.0, input_dropout: float = 0.0, label_smoothing: float = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(SentimentClassifier, self).__init__(vocab, regularizer) if isinstance(share_encoder, Seq2VecEncoder) and isinstance( private_encoder, Seq2VecEncoder): self._encoder = CNNEncoder(vocab, text_field_embedder, share_encoder, private_encoder, with_domain_embedding=True, domain_embeddings=domain_embeddings, input_dropout=input_dropout) else: self._encoder = RNNEncoder(vocab, text_field_embedder, share_encoder, private_encoder, with_domain_embedding=True, domain_embeddings=domain_embeddings, input_dropout=input_dropout) self._num_classes = self.vocab.get_vocab_size("label") self._sentiment_discriminator = Discriminator( self._encoder.get_output_dim(), self._num_classes) self._s_domain_discriminator = s_domain_discriminator self._p_domain_discriminator = p_domain_discriminator self._valid_discriminator = valid_discriminator self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self._label_smoothing = label_smoothing self.metrics = { "sentiment_acc": CategoricalAccuracy(), "p_domain_acc": CategoricalAccuracy(), "s_domain_acc": CategoricalAccuracy(), "valid_acc": CategoricalAccuracy() } self._loss = torch.nn.CrossEntropyLoss() self._domain_loss = torch.nn.CrossEntropyLoss() # TODO torch.nn.BCELoss self._valid_loss = torch.nn.BCEWithLogitsLoss() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, similarity_function: SimilarityFunction, projection_feedforward: FeedForward, inference_encoder: Seq2SeqEncoder, output_feedforward: FeedForward, output_logit: FeedForward, parser_model_path: str, parser_cuda_device: int, freeze_parser: bool, dropout: float = 0.5, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder self._encoder = encoder self._matrix_attention = LegacyMatrixAttention(similarity_function) self._projection_feedforward = projection_feedforward self._inference_encoder = inference_encoder if dropout: self.dropout = torch.nn.Dropout(dropout) self.rnn_input_dropout = InputVariationalDropout(dropout) else: self.dropout = None self.rnn_input_dropout = None self._output_feedforward = output_feedforward self._output_logit = output_logit self._num_labels = vocab.get_vocab_size(namespace="labels") check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), "text field embedding dim", "encoder input dim") check_dimensions_match(encoder.get_output_dim() * 4, projection_feedforward.get_input_dim(), "encoder output dim", "projection feedforward input") check_dimensions_match(projection_feedforward.get_output_dim(), inference_encoder.get_input_dim(), "proj feedforward output dim", "inference lstm input dim") self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() self._parser = load_archive(parser_model_path, cuda_device=parser_cuda_device).model self._parser._head_sentinel.requires_grad = False for child in self._parser.children(): for param in child.parameters(): param.requires_grad = False if not freeze_parser: for param in self._parser.encoder.parameters(): param.requires_grad = True initializer(self)
def __init__(self, encoder: Seq2SeqEncoder, dropout: float = 0.5) -> None: super().__init__() self._encoder = encoder if dropout: self.dropout = torch.nn.Dropout(dropout) self.rnn_input_dropout = InputVariationalDropout(dropout) else: self.dropout = None self.rnn_input_dropout = None
def __init__(self, vocab: Vocabulary, span_encoder: Seq2SeqEncoder, reasoning_encoder: Seq2SeqEncoder, input_dropout: float = 0.3, hidden_dim_maxpool: int = 1024, class_embs: bool=True, reasoning_use_obj: bool=True, reasoning_use_answer: bool=True, reasoning_use_question: bool=True, pool_reasoning: bool = True, pool_answer: bool = True, pool_question: bool = False, initializer: InitializerApplicator = InitializerApplicator(), ): super(AttentionQA, self).__init__(vocab) self.detector = SimpleDetector(pretrained=True, average_pool=True, semantic=class_embs, final_dim=512) ################################################################################################### self.rnn_input_dropout = TimeDistributed(InputVariationalDropout(input_dropout)) if input_dropout > 0 else None self.span_encoder = TimeDistributed(span_encoder) self.reasoning_encoder = TimeDistributed(reasoning_encoder) self.span_attention = BilinearMatrixAttention( matrix_1_dim=span_encoder.get_output_dim(), matrix_2_dim=span_encoder.get_output_dim(), ) self.obj_attention = BilinearMatrixAttention( matrix_1_dim=span_encoder.get_output_dim(), matrix_2_dim=self.detector.final_dim, ) self.reasoning_use_obj = reasoning_use_obj self.reasoning_use_answer = reasoning_use_answer self.reasoning_use_question = reasoning_use_question self.pool_reasoning = pool_reasoning self.pool_answer = pool_answer self.pool_question = pool_question dim = sum([d for d, to_pool in [(reasoning_encoder.get_output_dim(), self.pool_reasoning), (span_encoder.get_output_dim(), self.pool_answer), (span_encoder.get_output_dim(), self.pool_question)] if to_pool]) self.final_mlp = torch.nn.Sequential( torch.nn.Dropout(input_dropout, inplace=False), torch.nn.Linear(dim, hidden_dim_maxpool), torch.nn.ReLU(inplace=True), torch.nn.Dropout(input_dropout, inplace=False), torch.nn.Linear(hidden_dim_maxpool, 1), ) self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, vocab: Vocabulary, name:str, edge_model: EdgeModel, loss_function: EdgeLoss, supertagger: Supertagger, lexlabeltagger: Supertagger, supertagger_loss : SupertaggingLoss, lexlabel_loss : SupertaggingLoss, output_null_lex_label : bool = True, loss_mixing : Dict[str,float] = None, dropout : float = 0.0, validation_evaluator: Optional[Evaluator] = None, regularizer: Optional[RegularizerApplicator] = None): super().__init__(vocab,regularizer) self.name = name self.edge_model = edge_model self.supertagger = supertagger self.lexlabeltagger = lexlabeltagger self.supertagger_loss = supertagger_loss self.lexlabel_loss = lexlabel_loss self.loss_function = loss_function self.loss_mixing = loss_mixing or dict() self.validation_evaluator = validation_evaluator self.output_null_lex_label = output_null_lex_label self._dropout = InputVariationalDropout(dropout) loss_names = ["edge_existence", "edge_label", "supertagging", "lexlabel"] for loss_name in loss_names: if loss_name not in self.loss_mixing: self.loss_mixing[loss_name] = 1.0 logger.info(f"Loss name {loss_name} not found in loss_mixing, using a weight of 1.0") else: if self.loss_mixing[loss_name] is None: if loss_name not in ["supertagging", "lexlabel"]: raise ConfigurationError("Only the loss mixing coefficients for supertagging and lexlabel may be None, but not "+loss_name) not_contained = set(self.loss_mixing.keys()) - set(loss_names) if len(not_contained): logger.critical(f"The following loss name(s) are unknown: {not_contained}") raise ValueError(f"The following loss name(s) are unknown: {not_contained}") self._supertagging_acc = CategoricalAccuracy() self._top_6supertagging_acc = CategoricalAccuracy(top_k=6) self._lexlabel_acc = CategoricalAccuracy() self._attachment_scores = AttachmentScores() self.current_epoch = 0 tags = self.vocab.get_token_to_index_vocabulary("pos") punctuation_tag_indices = {tag: index for tag, index in tags.items() if tag in POS_TO_IGNORE} self._pos_to_ignore = set(punctuation_tag_indices.values()) logger.info(f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. " "Ignoring words with these POS tags for evaluation.")
def __init__(self, vocab: Vocabulary, response_embedder: TextFieldEmbedder, response_word_attention: Seq2VecEncoder, response_encoder: Seq2SeqEncoder, response_sentence_attention: Seq2VecEncoder, output_feedforward: FeedForward, op_embedder: Optional[TextFieldEmbedder] = None, op_word_attention: Optional[Seq2VecEncoder] = None, op_encoder: Optional[Seq2SeqEncoder] = None, dropout: float = 0.5, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, feature_feedforward: FeedForward = None) -> None: super().__init__(vocab, regularizer) self._response_embedder = response_embedder self._response_word_attention = response_word_attention self._response_encoder = response_encoder self._response_sentence_attention = response_sentence_attention self._op_embedder = op_embedder or response_embedder self._op_word_attention = op_word_attention or response_word_attention self._op_encoder = op_encoder or response_encoder if dropout: self.dropout = torch.nn.Dropout(dropout) self.rnn_input_dropout = InputVariationalDropout(dropout) else: self.dropout = None self.rnn_input_dropout = None self._output_feedforward = output_feedforward self._feature_feedforward = feature_feedforward self._num_labels = vocab.get_vocab_size(namespace="labels") #check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), # "text field embedding dim", "encoder input dim") #check_dimensions_match(encoder.get_output_dim() * 4, projection_feedforward.get_input_dim(), # "encoder output dim", "projection feedforward input") #check_dimensions_match(projection_feedforward.get_output_dim(), inference_encoder.get_input_dim(), # "proj feedforward output dim", "inference lstm input dim") self._accuracy = BooleanAccuracy() self._fscore = F1Measure(positive_label=1) self._fake_accuracy = BooleanAccuracy() self._fake_fscore = F1Measure(positive_label=1) self._loss = torch.nn.functional.binary_cross_entropy_with_logits initializer(self)
def __init__(self, rnn_cell: StackedLstm, source_attention_layer: AttentionLayer, target_attention_layer: AttentionLayer = None, dropout: float = 0.0) -> None: super().__init__() self.rnn_cell = rnn_cell self.source_attention_layer = source_attention_layer self.target_attention_layer = target_attention_layer self.dropout = InputVariationalDropout(dropout) self.use_coverage = source_attention_layer.attention.use_coverage self.hidden_vector_dim = self.rnn_cell.hidden_size
def __init__(self, vocab:Vocabulary, encoder_dim:int, label_dim:int, edge_dim:int, dropout: float, edge_label_namespace: str, activation : Activation = None) -> None: """ Parameters ---------- vocab : ``Vocabulary``, required A Vocabulary, required in order to compute sizes for input/output projections. encoder_dim : ``int``, required. The output dimension of the encoder. label_dim : ``int``, required. The dimension of the hidden layer of the MLP used for predicting the edge labels. edge_dim : ``int``, required. The dimension of the hidden layer of the MLP used for predicting edge existence. edge_label_namespace: str, The namespace of the edge labels: a combination of the task name + _head_tags activation : ``Activation``, optional, (default = tanh). The activation function used in the MLPs. dropout : ``float``, optional, (default = 0.0) The variational dropout applied to the output of the encoder and MLP layers. """ super(KGEdges, self).__init__(vocab) self._encoder_dim = encoder_dim if activation is None: self.activation = Activation.by_name("tanh")() else: self.activation = activation #edge existence: #these two matrices together form the feed forward network which takes the vectors of the two words in question and makes predictions from that #this is the trick described by Kiperwasser and Goldberg to make training faster. self.head_arc_feedforward = torch.nn.Linear(encoder_dim, edge_dim) self.child_arc_feedforward = torch.nn.Linear(encoder_dim, edge_dim, bias=False) #bias is already added by head_arc_feedforward self.arc_out_layer = torch.nn.Linear(edge_dim, 1, bias=False) # K&G don't use a bias for the output layer #edge labels: num_labels = vocab.get_vocab_size(edge_label_namespace) #same trick again self.head_label_feedforward = torch.nn.Linear(encoder_dim, label_dim) self.child_label_feedforward = torch.nn.Linear(encoder_dim, label_dim, bias=False) self.label_out_layer = torch.nn.Linear(edge_dim, num_labels) #output layer for edge labels self._dropout = InputVariationalDropout(dropout)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, dropout: float = 0.0, input_dropout: float = 0.0, label_smoothing: float = 0.1, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(SentimentClassifier, self).__init__(vocab, regularizer) self._text_field_embedder = text_field_embedder share_rnn = nn.LSTM(input_size=self._text_field_embedder.get_output_dim(), hidden_size=150, batch_first=True, # dropout=dropout, bidirectional=True) share_encoder = PytorchSeq2SeqWrapper(share_rnn) self._encoder = RNNEncoder(vocab, share_encoder, input_dropout, regularizer) self._seq_vec = CnnEncoder(self._encoder.get_output_dim(), 25) self._de_dim = len(TASKS_NAME) weight = torch.empty(self._de_dim, self._text_field_embedder.get_output_dim()) torch.nn.init.orthogonal_(weight) self._domain_embeddings = Embedding(self._de_dim, self._text_field_embedder.get_output_dim(), weight=weight) self._de_attention = BilinearAttention(self._seq_vec.get_output_dim(), self._domain_embeddings.get_output_dim()) self._de_feedforward = FeedForward(self._domain_embeddings.get_output_dim(), 1, self._seq_vec.get_output_dim(), Activation.by_name("elu")()) self._num_classes = self.vocab.get_vocab_size("label") self._sentiment_discriminator = Discriminator(self._seq_vec.get_output_dim(), self._num_classes) self._s_domain_discriminator = Discriminator(self._seq_vec.get_output_dim(), len(TASKS_NAME)) self._valid_discriminator = Discriminator(self._domain_embeddings.get_output_dim(), 2) self._dropout = InputVariationalDropout(dropout) self._input_dropout = Dropout(input_dropout) self._label_smoothing = label_smoothing self.metrics = { "s_domain_acc": CategoricalAccuracy(), "valid_acc": CategoricalAccuracy() } for task_name in TASKS_NAME: self.metrics["{}_stm_acc".format(task_name)] = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() self._domain_loss = torch.nn.CrossEntropyLoss() # TODO torch.nn.BCELoss self._valid_loss = torch.nn.BCEWithLogitsLoss() initializer(self)
def __init__(self, vocab: Vocabulary, source_text_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, decoder: SeqDecoder, noisy_prediction: NoisyPredictionModel, dropout: float = 0.2, tied_source_embedder_key: Optional[str] = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(SalienceSeq2Seq, self).__init__(vocab, regularizer) self._source_text_embedder = source_text_embedder self._encoder = encoder self._decoder = decoder self._dropout = InputVariationalDropout(dropout) self._noisy_prediction = noisy_prediction if self._encoder.get_output_dim() != self._decoder.get_output_dim(): raise ConfigurationError( f"Encoder output dimension {self._encoder.get_output_dim()} should be" f" equal to decoder dimension {self._decoder.get_output_dim()}." ) if tied_source_embedder_key: # A bit of a ugly hack to tie embeddings. # Works only for `BasicTextFieldEmbedder`, and since # it can have multiple embedders, and `SeqDecoder` contains only a single embedder, we need # the key to select the source embedder to replace it with the target embedder from the decoder. if not isinstance(self._source_text_embedder, BasicTextFieldEmbedder): raise ConfigurationError( "Unable to tie embeddings," "Source text embedder is not an instance of `BasicTextFieldEmbedder`." ) # pylint: disable=protected-access source_embedder = self._source_text_embedder._token_embedders[ tied_source_embedder_key] if not isinstance(source_embedder, Embedding): raise ConfigurationError( "Unable to tie embeddings," "Selected source embedder is not an instance of `Embedding`." ) if source_embedder.get_output_dim( ) != self._decoder.target_embedder.get_output_dim(): raise ConfigurationError( f"Output Dimensions mismatch between" f"source embedder and target embedder.") self._source_text_embedder._token_embedders[ tied_source_embedder_key] = self._decoder.target_embedder initializer(self)