def __init__(self, embedding_matrix, opt):
super(LCA_GLOVE, self).__init__()
# Only few of the parameters are necessary in the config.json, such as hidden_size, num_attention_heads
self.config = BertConfig.from_json_file("modules/utils/bert_config.json")
self.opt = opt
self.embed = nn.Embedding.from_pretrained(torch.tensor(embedding_matrix, dtype=torch.float))
self.lc_embed = nn.Embedding(2, opt.embed_dim)
self.global_encoder1 = SelfAttention(self.config, opt)
self.local_encoder1 = SelfAttention(self.config, opt)
self.local_encoder2 = SelfAttention(self.config, opt)
self.mha = SelfAttention(self.config, opt)
self.pool = BertPooler(self.config)
self.dropout = nn.Dropout(opt.dropout)
self.linear = nn.Linear(opt.embed_dim * 2, opt.embed_dim)
self.dense = nn.Linear(opt.embed_dim, opt.polarities_dim)
self.classifier = nn.Linear(opt.embed_dim, 2)
def setup_method(self):
self.monkeypatch = MonkeyPatch()
# monkeypatch the PretrainedBertModel to return the tiny test fixture model
config_path = FIXTURES_ROOT / "structured_prediction" / "srl" / "bert" / "config.json"
vocab_path = FIXTURES_ROOT / "structured_prediction" / "srl" / "bert" / "vocab.txt"
config = BertConfig.from_json_file(config_path)
self.monkeypatch.setattr(BertModel, "from_pretrained", lambda _: BertModel(config))
self.monkeypatch.setattr(
BertTokenizer, "from_pretrained", lambda _: BertTokenizer(vocab_path)
)
super().setup_method()
self.set_up_model(
FIXTURES_ROOT / "structured_prediction" / "srl" / "bert_srl.jsonnet",
FIXTURES_ROOT / "structured_prediction" / "srl" / "conll_2012",
)
def __init__(self, embedding_matrix, opt):
super(LCF_GLOVE, self).__init__()
self.config = BertConfig.from_json_file(
"modules/utils/bert_config.json")
self.opt = opt
self.embed = nn.Embedding.from_pretrained(
torch.tensor(embedding_matrix, dtype=torch.float))
self.mha_global = SelfAttention(self.config, opt)
self.mha_local = SelfAttention(self.config, opt)
self.ffn_global = PositionwiseFeedForward(self.opt.embed_dim,
dropout=self.opt.dropout)
self.ffn_local = PositionwiseFeedForward(self.opt.embed_dim,
dropout=self.opt.dropout)
self.mha_local_SA = SelfAttention(self.config, opt)
self.mha_global_SA = SelfAttention(self.config, opt)
self.pool = BertPooler(self.config)
self.dropout = nn.Dropout(opt.dropout)
self.linear = nn.Linear(opt.embed_dim * 2, opt.embed_dim)
self.dense = nn.Linear(opt.embed_dim, opt.polarities_dim)
