class getElmo(nn.Module):
def __init__(self, layer=2, dropout=0, out_dim=100, gpu=True):
super(getElmo, self).__init__()
options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
self.dropout = dropout
self.gpu = gpu
self.Elmo = Elmo(options_file, weight_file, layer, dropout=dropout)
self.Elmo.eval()
self.layers2one = nn.Linear(
layer, 1).cuda() if self.gpu else nn.Linear(layer, 1)
self.optLinear = nn.Linear(
1024, out_dim).cuda() if self.gpu else nn.Linear(1024, out_dim)
def forward(self, texts):
word_idxs = batch_to_ids(texts).cuda() if self.gpu else batch_to_ids(
texts)
elmo_embs = self.Elmo.forward(word_idxs)
elmo_reps = torch.stack(elmo_embs['elmo_representations'],
dim=-1).cuda() if self.gpu else torch.stack(
elmo_embs['elmo_representations'], dim=-1)
elmo_decrease_layer = self.layers2one(elmo_reps).squeeze()
elmo_fit_hidden = self.optLinear(elmo_decrease_layer)
mask = elmo_embs['mask']
return elmo_fit_hidden, mask
class ElmoHandler(nn.Module):
def __init__(self, d_content, elmo_dropout, weights):
super().__init__()
self.d_content = d_content
orig_json = "/scratch/skulick/downloaded/embeddings/elmo_2x4096_512_2048cnn_2xhighway_options.json",
orig_hdf5 = "/scratch/skulick/downloaded/embeddings/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5",
eebo_json = "/scratch/skulick/downloaded/embeddings/eebo-options1.json"
eebo_hdf5 = "/scratch/skulick/downloaded/embeddings/eebo-weights2.hdf5"
assert weights in ('eebo', 'orig'), f'unknown weights {weights}'
if weights == 'eebo':
json_fname = eebo_json
hdf5_fname = eebo_hdf5
else:
json_fname = orig_json
hdf5_fname = orig_hdf5
print(f'using embeddings {json_fname} {hdf5_fname}')
self.elmo = Elmo(
options_file=json_fname,
weight_file=hdf5_fname,
num_output_representations=1,
requires_grad=False,
do_layer_norm=False,
keep_sentence_boundaries=True,
dropout=elmo_dropout,
)
d_elmo_annotations = 1024
# Don't train gamma parameter for ELMo - the projection can do any
# necessary scaling
self.elmo.scalar_mix_0.gamma.requires_grad = False
# Reshapes the embeddings to match the model dimension, and making
# the projection trainable appears to improve parsing accuracy
self.project_elmo = nn.Linear(d_elmo_annotations,
self.d_content,
bias=False)
def forward(self, char_idxs):
elmo_out = self.elmo.forward(char_idxs)
elmo_rep0 = elmo_out['elmo_representations'][0]
elmo_mask = elmo_out['mask']
#elmo_annotations_packed = elmo_rep0[elmo_mask.byte()].view(packed_len, -1)
extra_content_annotations = self.project_elmo(elmo_rep0)
return extra_content_annotations
from allennlp.modules.elmo import Elmo, batch_to_ids
weights_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5'
options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json'
elmo = Elmo(options_file, weights_file, 1, dropout=0)
text = [
'Backgammon', 'is', 'one', 'of', 'the', 'oldest', 'known', 'board', 'games'
]
batch = batch_to_ids(text)
print(batch)
dict = elmo.forward(batch)
print(dict['elmo_representations'])
