def __init__(self,
bert_path: Path,
encoder: Seq2SeqEncoder,
vocab: Vocabulary,
hidden_dim: int = 100,
encoder_dropout: float = 0.0,
train_bert: bool = False) -> None:
# We have to pass the vocabulary to the constructor.
super().__init__(vocab)
self.word_embeddings = bert_embeddings(pretrained_model=bert_path,
training=train_bert)
self.encoder_dropout: torch.nn.Module
if encoder_dropout > 0:
self.encoder_dropout = torch.nn.Dropout(p=encoder_dropout)
else:
self.encoder_dropout = torch.nn.Identity()
self.pooler = BertPooler(pretrained_model=str(bert_path))
self.dense1 = torch.nn.Linear(in_features=self.pooler.get_output_dim(),
out_features=hidden_dim)
self.encoder = encoder
self.self_attn = LinearSelfAttention(
input_dim=self.encoder.get_output_dim(), bias=True)
self.dense2 = torch.nn.Linear(
in_features=self.encoder.get_output_dim(), out_features=1)
def get_word_embeddings(vocabulary: Vocabulary) -> TextFieldEmbedder:
"Instatiates the word embeddings based on config."
if ARGS.EMBEDDING_TYPE == 'glove':
return glove_embeddings(vocabulary,
ARGS.GLOVE_PATH,
ARGS.GLOVE_EMBEDDING_DIM,
training=True)
if ARGS.EMBEDDING_TYPE == 'bert':
return bert_embeddings(pretrained_model=ARGS.BERT_PATH,
training=ARGS.finetune_embeddings)
if ARGS.EMBEDDING_TYPE == 'xlnet':
return xlnet_embeddings(config_path=ARGS.xlnet_config_path,
model_path=ARGS.xlnet_model_path,
training=ARGS.finetune_embeddings,
window_size=ARGS.xlnet_window_size)
raise ValueError(f'Invalid word embedding type: {ARGS.EMBEDDING_TYPE}')
def build_hierarchical_bert(vocab: Vocabulary) -> Model:
"""
Builds the HierarchicalBert.
Parameters
---------
vocab : Vocabulary built from the problem dataset.
Returns
-------
A `HierarchicalBert` model ready to be trained.
"""
if ARGS.ENCODER_TYPE == 'lstm':
encoder_fn = lstm_encoder
elif ARGS.ENCODER_TYPE == 'gru':
encoder_fn = gru_encoder
else:
raise ValueError('Invalid RNN type')
bert = bert_embeddings(ARGS.BERT_PATH)
embedding_dim = bert.get_output_dim()
# To prevent the warning on single-layer, as the dropout is only
# between layers of the stacked RNN.
dropout = ARGS.RNN_DROPOUT if ARGS.RNN_LAYERS > 1 else 0
if ARGS.ENCODER_TYPE in ['lstm', 'gru']:
sentence_encoder = encoder_fn(input_dim=embedding_dim,
output_dim=ARGS.HIDDEN_DIM,
num_layers=ARGS.RNN_LAYERS,
bidirectional=ARGS.BIDIRECTIONAL,
dropout=dropout)
document_encoder = encoder_fn(
input_dim=sentence_encoder.get_output_dim(),
output_dim=ARGS.HIDDEN_DIM,
num_layers=1,
bidirectional=ARGS.BIDIRECTIONAL,
dropout=dropout)
# Instantiate modele with our embedding, encoder and vocabulary
model = models.HierarchicalBert(bert_path=ARGS.BERT_PATH,
sentence_encoder=sentence_encoder,
document_encoder=document_encoder,
vocab=vocab,
encoder_dropout=ARGS.RNN_DROPOUT)
return model
def __init__(self,
bert_path: Path,
vocab: Vocabulary,
train_bert: bool = False
) -> None:
# We have to pass the vocabulary to the constructor.
super().__init__(vocab)
self.word_embeddings = bert_embeddings(pretrained_model=bert_path,
training=train_bert)
self.pooler = BertPooler(pretrained_model=str(bert_path))
hidden_dim = self.pooler.get_output_dim()
self.hidden2logit = torch.nn.Linear(
in_features=hidden_dim,
out_features=1
)
def __init__(self,
bert_path: Path,
sentence_encoder: Seq2VecEncoder,
document_encoder: Seq2VecEncoder,
vocab: Vocabulary,
encoder_dropout: float = 0.0,
train_bert: bool = False
) -> None:
# We have to pass the vocabulary to the constructor.
super().__init__(vocab)
self.word_embeddings = bert_embeddings(pretrained_model=bert_path,
training=train_bert)
if encoder_dropout > 0:
self.encoder_dropout = torch.nn.Dropout(p=encoder_dropout)
else:
self.encoder_dropout = lambda x: x
self.sentence_encoder = sentence_encoder
self.document_encoder = document_encoder
self.dense = torch.nn.Linear(
in_features=document_encoder.get_output_dim(),
out_features=1
)