def get_elmo_embedder():
options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json'
weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5'
elmo_embedder = ElmoTokenEmbedder(options_file, weight_file)
word_embeddings = BasicTextFieldEmbedder({"tokens": elmo_embedder})
return word_embeddings
def load_elmo_embeddings(large=True):
"""
Loads pre-trained ELMo embeddings ('large' model by default).
Parameters
----------
large: bool
Set to True to load the Large ELMo model; False for small ELMo model
Returns
-------
TextFieldEmbedder
"""
if large: # use the Large pre-trained model
print("Loading LARGE ELMo..")
options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json'
weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5'
else: # use the Small pre-trained model
print("Loading SMALL ELMo..")
options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json'
weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5'
elmo_embedder = ElmoTokenEmbedder(options_file, weight_file)
word_embeddings = BasicTextFieldEmbedder({"tokens": elmo_embedder})
print("Pre-trained ELMo loaded..")
return word_embeddings
def get_embedder_info(
embedder_type: str
) -> Tuple[TokenEmbedder, TokenIndexer, str, Dict[str, Any]]:
embedder_type = embedder_type.lower()
text_field_embedder_kwargs: Dict[str, Any] = {}
if embedder_type == 'ner_elmo':
return NERElmoTokenEmbedder(), ELMoTokenCharactersIndexer(
), text_field_embedder_kwargs
elif embedder_type == 'elmo':
return ElmoTokenEmbedder(ELMO_OPTIONS_FILE,
ELMO_WEIGHT_FILE), ELMoTokenCharactersIndexer(
), text_field_embedder_kwargs
elif embedder_type == 'bert':
bert_embedder = PretrainedBertEmbedder(
pretrained_model="bert-base-uncased",
top_layer_only=True, # conserve memory
)
token_indexer = PretrainedBertIndexer(
pretrained_model="bert-base-uncased",
max_pieces=512, # max pieces allowed for positional embeddings
do_lowercase=True,
use_starting_offsets=True,
)
text_field_embedder_kwargs['allow_unmatched_keys'] = True
text_field_embedder_kwargs['embedder_to_indexer_map'] = {
"tokens": ["tokens", "tokens-offsets"]
}
return bert_embedder, token_indexer, text_field_embedder_kwargs
else:
raise Exception(f'Unknown embedder type: {embedder_type}')
def _run_test(self, requires_grad):
options_file = os.path.join(FIXTURES, 'options.json')
weight_file = os.path.join(FIXTURES, 'lm_weights.hdf5')
embedder = ElmoTokenEmbedder(options_file,
weight_file,
requires_grad=requires_grad)
batch_size = 3
seq_len = 4
char_ids = Variable(
torch.from_numpy(
numpy.random.randint(0, 262, (batch_size, seq_len, 50))))
embeddings = embedder(char_ids)
loss = embeddings.sum()
loss.backward()
elmo_grads = [
param.grad for name, param in embedder.named_parameters()
if '_elmo_lstm' in name
]
if requires_grad:
# None of the elmo grads should be None.
assert all([grad is not None for grad in elmo_grads])
else:
# All of the elmo grads should be None.
assert all([grad is None for grad in elmo_grads])
def get_embeddings(embedder_type,
vocab,
embedding_dim=300,
bert_trainable=True):
if embedder_type not in valid_embedders:
raise Exception(f'Unknown embedder type {embedder_type}')
vocab_size = vocab.get_vocab_size('tokens')
token_embedders = {}
if embedder_type == 'random':
token_embedding = Embedding(vocab_size, embedding_dim, trainable=True)
token_embedders['tokens'] = token_embedding
if embedder_type in ['glove', 'elmo_and_glove']:
weights = load_glove_weights(vocab)
token_embedding = Embedding(vocab_size,
embedding_dim,
weight=weights,
trainable=True)
token_embedders['tokens'] = token_embedding
if embedder_type in ['elmo', 'elmo_and_glove']:
elmo_token_embedder = ElmoTokenEmbedder(
'embeddings/elmo_2x4096_512_2048cnn_2xhighway_options.json',
'embeddings/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5',
do_layer_norm=False,
dropout=0.5)
token_embedders['elmo'] = elmo_token_embedder
if 'bert' in embedder_type:
token_embedders['bert'] = BertEmbedder(bert_type=embedder_type,
trainable=bert_trainable)
word_embeddings = BasicTextFieldEmbedder(token_embedders)
return word_embeddings
def build_elmo_model(vocab: Vocabulary) -> Model:
print("Building the model")
vocab_size = vocab.get_vocab_size("tokens")
embedding = ElmoTokenEmbedder()
embedder = BasicTextFieldEmbedder(token_embedders={'bert_tokens': embedding})
encoder = BagOfEmbeddingsEncoder(embedding_dim=embedder.get_output_dim(), averaged=True)
return SimpleClassifier(vocab, embedder, encoder)
def run_ELMo_RSA(stim_file, header=False, filter_file=None):
EXP = data.Stim(stim_file, header, filter_file, VOCAB_FILE)
#Get tokenizer
tokenizer = WhitespaceTokenizer()
#Load model
##ELMo OG
elmo_weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5'
elmo_options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json'
#ELMo Small
#elmo_weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5'
#elmo_options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json'
#ELMo Medium
#elmo_weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5'
#elmo_options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_options.json'
#ELMo OG (5.5B)
#elmo_weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5'
#elmo_options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json'
elmo_embedding = ElmoTokenEmbedder(options_file=elmo_options_file,
weight_file=elmo_weight_file,
dropout=0.0)
embedder = BasicTextFieldEmbedder(
token_embedders={'elmo_tokens': elmo_embedding})
for x in range(len(EXP.SENTS)):
sentences = list(EXP.SENTS[x])
target = sentences[0]
sentence = sentences[1]
#GET BASELINE
token_indexer = ELMoTokenCharactersIndexer()
vocab = Vocabulary()
target_tokens = tokenizer.tokenize(target)
target_text_field = TextField(target_tokens,
{'elmo_tokens': token_indexer})
target_text_field.index(vocab)
target_token_tensor = target_text_field.as_tensor(
target_text_field.get_padding_lengths())
target_tensor_dict = target_text_field.batch_tensors(
[target_token_tensor])
target_embedding = embedder(target_tensor_dict)[0]
baseline = target_embedding[-1].data.cpu().squeeze()
#GET SIMS
sims = get_ELMo_sims(sentence, baseline, tokenizer, embedder)
values = get_dummy_values(sentence)
EXP.load_IT('elmo', x, values, False, sims)
return EXP
def build_model(options_file, weight_file):
vocab = Vocabulary()
iterator = BucketIterator(batch_size=config.batch_size, sorting_keys=[("tokens", "num_tokens")])
iterator.index_with(vocab)
elmo_embedder = ElmoTokenEmbedder(options_file, weight_file)
word_embeddings = BasicTextFieldEmbedder({"tokens": elmo_embedder})
encoder: Seq2VecEncoder = PytorchSeq2VecWrapper(nn.LSTM(word_embeddings.get_output_dim(), config.hidden_size, bidirectional=True, batch_first=True))
model = BaselineModel(word_embeddings, encoder, vocab)
return model, iterator, vocab
def _run_test(self, requires_grad):
embedder = ElmoTokenEmbedder(self.options_file, self.weight_file, requires_grad=requires_grad)
batch_size = 3
seq_len = 4
char_ids = torch.from_numpy(numpy.random.randint(0, 262, (batch_size, seq_len, 50)))
for _ in range(2):
embeddings = embedder(char_ids)
loss = embeddings.sum()
loss.backward()
elmo_grads = [param.grad for name, param in embedder.named_parameters() if '_elmo_lstm' in name]
if requires_grad:
# None of the elmo grads should be None.
assert all([grad is not None for grad in elmo_grads])
else:
# All of the elmo grads should be None.
assert all([grad is None for grad in elmo_grads])
def __init__(self,
options_file: str,
weight_file: str,
vocab_to_cache: List[str],
do_layer_norm: bool = False,
dropout: float = 0.5,
requires_grad: bool = False,
projection_dim: int = None) -> None:
super(ElmoWordEmbedding, self).__init__()
self._elmo = ElmoTokenEmbedder(options_file=options_file,
weight_file=weight_file,
do_layer_norm=do_layer_norm,
dropout=dropout,
requires_grad=requires_grad,
projection_dim=projection_dim,
vocab_to_cache=vocab_to_cache)
self._projection = self._elmo._projection
def load_elmo_model():
elmo_embedders = ElmoTokenEmbedder(OPTION_FILE, WEIGHT_FILE)
word_embeddings = BasicTextFieldEmbedder({"tokens": elmo_embedders})
encoder = PytorchSeq2VecWrapper(
torch.nn.LSTM(word_embeddings.get_output_dim(),
HIDDEN_DIM,
bidirectional=True,
batch_first=True))
vocabulary = Vocabulary()
model = BaseModel(word_embeddings=word_embeddings,
encoder=encoder,
vocabulary=vocabulary)
output_elmo_model_file = os.path.join(PRETRAINED_ELMO,
"lstm_elmo_model.bin")
model.load_state_dict(torch.load(output_elmo_model_file))
return model
def sequence_labelling():
# Index each token as a sequence of character Ids (ELMo)
token_indexers = {"tokens": ELMoTokenCharactersIndexer()}
# Read the data
reader = SequenceLabellingDatasetReader(token_indexers)
training_data = reader.read(path='data/sequence_labelling/train.txt')
validation_data = reader.read(path='data/sequence_labelling/test.txt')
test_data = reader.read(path='data/sequence_labelling/test.txt')
# Create a vocabulary
vocabulary = Vocabulary.from_instances(training_data + validation_data +
test_data)
# Use ELMo embeddings
elmo = ElmoTokenEmbedder(options_file=ELMO_OPTIONS_FILE,
weight_file=ELMO_WEIGHTS_FILE)
embedder = BasicTextFieldEmbedder(token_embedders={"tokens": elmo})
# Our text classifier will use a CNN encoder
lstm_layer = LSTM(input_size=ELMO_EMBEDDING_DIM,
hidden_size=HIDDEN_SIZE,
bidirectional=True,
batch_first=True)
lstm_encoder = PytorchSeq2SeqWrapper(module=lstm_layer)
model = SequenceLabeller(vocabulary=vocabulary,
embedder=embedder,
encoder=lstm_encoder)
print("\nModel :\n")
print(model)
# Training
train_model(model, training_data, validation_data, vocabulary)
# Evaluation
evaluate_sequence_labelling_model(model, test_data)
def get_model(vocab, params):
emb_d = params["embedding_dim"]
hidden_d = params["hidden_dim"]
use_elmo_embeddings = params['use_elmo']
use_lstm = params['use_lstm']
n_layers = params["num_layers"]
bidirectional = params['bidirectional']
if use_elmo_embeddings:
token_embedder = ElmoTokenEmbedder(ELMO_OPTIONS_FILE,
ELMO_WEIGHTS_FILE)
else:
token_embedder = Embedding(
num_embeddings=vocab.get_vocab_size('tokens'), embedding_dim=emb_d)
word_embedder = BasicTextFieldEmbedder({"tokens": token_embedder})
emb_d = word_embedder.get_output_dim()
if use_lstm:
encoder = PytorchSeq2SeqWrapper(
torch.nn.LSTM(emb_d,
hidden_d,
num_layers=n_layers,
batch_first=True,
bidirectional=bidirectional))
else:
encoder = PytorchSeq2SeqWrapper(
torch.nn.GRU(emb_d,
hidden_d,
num_layers=n_layers,
batch_first=True,
bidirectional=bidirectional))
model = NerModel(word_embedder,
encoder,
vocab,
num_categories=(3 if params["dataset"] == "senti" else 4))
return model
def _run_test_with_vocab_to_cache(self, requires_grad):
vocab_to_cache = ['<pad>', 'hello', 'world']
embedder = ElmoTokenEmbedder(self.options_file,
self.weight_file,
requires_grad=requires_grad,
vocab_to_cache=vocab_to_cache)
word_tensor = torch.LongTensor([[[1, 2]]])
for _ in range(2):
embeddings = embedder(word_tensor, word_tensor)
loss = embeddings.sum()
loss.backward()
elmo_grads = [param.grad for name, param in embedder.named_parameters()
if '_elmo_lstm' in name and '_token_embedder' not in name]
if requires_grad:
# None of the elmo grads should be None.
assert all([grad is not None for grad in elmo_grads])
else:
# All of the elmo grads should be None.
assert all([grad is None for grad in elmo_grads])
assert all([param.grad is None for name, param in embedder.named_parameters()
if '_token_embedder' in name])
def get_predictor():
EMBEDDING_DIM = 128
HIDDEN_DIM = 60 #128
MAX_LEN = 70
dropout = 0.25
lstm_layers = 2
# pre-trained model
options_file = (
'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo'
'/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json'
)
weight_file = (
'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo'
'/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5'
)
elmo_embedder = ElmoTokenEmbedder(options_file, weight_file)
vocab = Vocabulary.from_files(data_dir +
"vocabulary_allennlp_imdb_twoclass")
word_embeddings = BasicTextFieldEmbedder({"tokens": elmo_embedder})
elmo_embedding_dim = 256
lstm = PytorchSeq2VecWrapper(
torch.nn.LSTM(elmo_embedding_dim,
HIDDEN_DIM,
bidirectional=True,
num_layers=lstm_layers,
dropout=dropout,
batch_first=True))
model = LstmTwoClassifier(word_embeddings, lstm, vocab)
net = torch.load("model_allen_imdb_twoclass.th", map_location=str(device))
model.load_state_dict(net)
elmo_token_indexer = ELMoTokenCharactersIndexer()
readerSentence = SentenceDatasetReader(
token_indexers={'tokens': elmo_token_indexer})
return SentimentPredictor(model, dataset_reader=readerSentence)
def build_model(vocab: Vocabulary) -> Model:
print("Building the model")
vocab_size_tokens = vocab.get_vocab_size("tokens")
vocab_size_chars = vocab.get_vocab_size("token_characters")
embedder = BasicTextFieldEmbedder({"tokens": Embedding(embedding_dim=embedding_dim, pretrained_file=f"{cur_dir}/glove/glove.6B.200d.txt", trainable=False, num_embeddings=vocab_size_tokens, vocab=vocab),\
"elmo": ElmoTokenEmbedder(weight_file="https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5", options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json", do_layer_norm=False, dropout=0.0),\
"token_characters":TokenCharactersEncoder(embedding=Embedding(embedding_dim=16, num_embeddings=vocab_size_chars, vocab=vocab), \
encoder=CnnEncoder(embedding_dim=16, num_filters=128, ngram_filter_sizes=[3]))})
encoder = PytorchTransformer(input_dim=1352,
num_layers=6,
positional_encoding="sinusoidal")
# embedder = BasicTextFieldEmbedder({"tokens": Embedding(embedding_dim=embedding_dim, num_embeddings=vocab_size)})
# encoder = BagOfEmbeddingsEncoder(embedding_dim=embedding_dim)
# embedder = BasicTextFieldEmbedder({"tokens": PretrainedTransformerMismatchedEmbedder("bert-large-uncased")})
# encoder = LstmSeq2SeqEncoder(input_size=1024, hidden_size=1024, num_layers=2, dropout=0.5, bidirectional=True)
if args.pseudo:
return PseudoCrfTagger(vocab, embedder, encoder, \
label_encoding="BIOUL", include_start_end_transitions=False, num_virtual_models = num_virtual_models)
else:
return CrfTagger(vocab, embedder, encoder, \
label_encoding="BIOUL", include_start_end_transitions=False)
def embeddings_returner(self, vocab=None):
'''
Either the name of the pretrained model to use (e.g. bert-base-uncased),or the path to the .tar.gz
file with the model weights.
:param args: vocab_size and vocab is needed only when pretrained embeddings is used.
:return: embedder
'''
'''
"bert-base-uncased", do_lower_case=True
"bert-base-cased" , do_lower_case=False
https://github.com/huggingface/pytorch-transformers/issues/712
https://qiita.com/uedake722/items/b7f4b75b4d77d9bd358b
'''
if self.embedding_strategy == 'bert':
self.bertmodel_dir = ''
if self.ifbert_use_whichmodel == 'general':
self.bertmodel_dir += 'bert-base-uncased/' # recomendded ver is uncased, in original repository
self.bertmodel_relative_dirpath = self.bert_src_dir + self.bertmodel_dir
# included in pytorch_transformers, so we replace it with model name itself
self.bert_weight_filepath = copy.copy('bert-base-uncased')
elif self.ifbert_use_whichmodel == 'scibert':
self.bertmodel_dir += 'scibert_scivocab_uncased/' # recomendded ver is uncased, in original repository
self.bertmodel_relative_dirpath = self.bert_src_dir + self.bertmodel_dir
self.bert_weight_filepath = self.bertmodel_relative_dirpath + 'weights.tar.gz'
elif self.ifbert_use_whichmodel == 'biobert':
self.bertmodel_dir += 'biobert_v1.1_pubmed/' # currently cased version only supported
self.bertmodel_relative_dirpath = self.bert_src_dir + self.bertmodel_dir
self.bert_weight_filepath = self.bertmodel_relative_dirpath + 'weights.tar.gz' # including bert_config.json and bin.
# Load embedder
bert_embedder = PretrainedBertEmbedder(
pretrained_model=self.bert_weight_filepath,
top_layer_only=self.bert_top_layer_only,
requires_grad=self.emb_requires_grad)
return bert_embedder, bert_embedder.get_output_dim(
), BasicTextFieldEmbedder({'tokens': bert_embedder},
allow_unmatched_keys=True)
elif self.embedding_strategy == 'elmo':
if self.ifelmo_use_whichmodel == 'general':
options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json'
weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5'
elif self.ifelmo_use_whichmodel == 'pubmed':
options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/contributed/pubmed/elmo_2x4096_512_2048cnn_2xhighway_options.json'
weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/contributed/pubmed/elmo_2x4096_512_2048cnn_2xhighway_weights_PubMed_only.hdf5'
elif self.ifelmo_use_whichmodel == 'bioelmo':
options_file = self.elmo_src_dir + 'BioELMo/weights/biomed_elmo_options.json'
weight_file = self.elmo_src_dir + 'BioELMo/weights/biomed_elmo_weights.hdf5'
else:
options_file = -1
weight_file = -1
assert options_file != -1
elmo_embedder = ElmoTokenEmbedder(
options_file=options_file,
weight_file=weight_file,
requires_grad=self.emb_requires_grad)
return elmo_embedder, elmo_embedder.get_output_dim(
), BasicTextFieldEmbedder({'tokens': elmo_embedder})
elif self.embedding_strategy == 'pretrained':
print('\nGloVe pretrained vocab loading\n')
if 'glove' in self.args.ifpretrained_use_whichmodel:
embedding_dim = 300
else:
embedding_dim = 200
pretrain_emb_embedder = Embedding.from_params(
vocab=vocab,
params=Params({
'pretrained_file': self.glove_embeddings_file,
'embedding_dim': embedding_dim,
'trainable': False,
'padding_index': 0
}))
return pretrain_emb_embedder, pretrain_emb_embedder.get_output_dim(
), BasicTextFieldEmbedder({'tokens': pretrain_emb_embedder})
def build_model(vocab: Vocabulary) -> Model:
print("Building the model")
EMBEDDING_DIM = 300
HIDDEN_DIM = 300
NUM_FILTERS = 60
NGRAM_FILTER_SIZES = (2, 3, 4, 5, 6)
#out_dim for char = len(NGRAM_FILTER_SIZES) * NUM_FILTERS
F_OUT = 200
elmo_options_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
elmo_weight_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
elmo_embedding = ElmoTokenEmbedder(options_file=elmo_options_file,
weight_file=elmo_weight_file)
character_embedding = Embedding(vocab=vocab,
embedding_dim=EMBEDDING_DIM,
vocab_namespace='character_vocab')
cnn_encoder = CnnEncoder(embedding_dim=EMBEDDING_DIM,
num_filters=NUM_FILTERS,
ngram_filter_sizes=NGRAM_FILTER_SIZES)
token_encoder = TokenCharactersEncoder(character_embedding, cnn_encoder)
pos_tag_embedding = Embedding(vocab=vocab,
embedding_dim=EMBEDDING_DIM,
vocab_namespace='pos_tag_vocab')
ner_tag_embedding = Embedding(vocab=vocab,
embedding_dim=EMBEDDING_DIM,
vocab_namespace='ner_tag_vocab')
word_embedding = Embedding(vocab=vocab,
embedding_dim=EMBEDDING_DIM,
vocab_namespace='token_vocab')
utterance_embedder = BasicTextFieldEmbedder(
token_embedders={
'elmo_tokens': elmo_embedding,
'token_characters': token_encoder,
'pos_tags': pos_tag_embedding,
'ner_tags': ner_tag_embedding
})
#slot embed
slot_embedder = BasicTextFieldEmbedder(token_embedders={
'elmo_tokens': elmo_embedding,
'token_characters': token_encoder,
})
utterance_lstm = PytorchSeq2SeqWrapper(
torch.nn.LSTM(2 * EMBEDDING_DIM + 1024 +
len(NGRAM_FILTER_SIZES) * NUM_FILTERS,
HIDDEN_DIM,
num_layers=2,
batch_first=True,
bidirectional=True))
slot_lstm = PytorchSeq2SeqWrapper(
torch.nn.LSTM(1024 + len(NGRAM_FILTER_SIZES) * NUM_FILTERS,
HIDDEN_DIM,
num_layers=2,
batch_first=True,
bidirectional=True))
similarity = LinearMatrixAttention(tensor_1_dim=2 * HIDDEN_DIM,
tensor_2_dim=2 * HIDDEN_DIM,
combination="x,y,x*y",
activation=Activation.by_name('tanh')())
modeling_lstm = PytorchSeq2SeqWrapper(
torch.nn.LSTM(
2 * 5 * HIDDEN_DIM, # bi-direction
HIDDEN_DIM,
num_layers=2,
batch_first=True,
bidirectional=True))
#step1_utterance
utterance_embedder2 = BasicTextFieldEmbedder(
token_embedders={
'elmo_tokens': elmo_embedding,
'token_characters': token_encoder,
'pos_tags': pos_tag_embedding,
'ner_tags': ner_tag_embedding
})
utterance_lstm2 = PytorchSeq2SeqWrapper(
torch.nn.LSTM(2 * EMBEDDING_DIM + 1024 +
len(NGRAM_FILTER_SIZES) * NUM_FILTERS,
HIDDEN_DIM,
num_layers=2,
batch_first=True,
bidirectional=True))
## FF to combines two lstm inputs
final_linear_layer = FeedForward(2 * HIDDEN_DIM, 2, [HIDDEN_DIM, F_OUT],
torch.nn.ReLU(), 0.3)
#CRF model
model = CrfTagger(vocab=vocab,
utterance_embedder=utterance_embedder,
utterance_embedder2=utterance_embedder2,
slot_embedder=slot_embedder,
utterance_encoder=utterance_lstm,
utterance_encoder2=utterance_lstm2,
slot_encoder=slot_lstm,
matrix_attention=similarity,
modeling_layer=modeling_lstm,
fc_ff_layer=final_linear_layer)
return model
def build_embeddings(args, vocab, pretrained_embs=None):
''' Build embeddings according to options in args '''
d_emb, d_char = 0, args.d_char
token_embedder = {}
# Word embeddings
if args.word_embs != 'none':
if args.word_embs in ['glove', 'fastText'] and pretrained_embs is not None:
log.info("\tUsing word embeddings from %s", args.word_embs_file)
word_embs = pretrained_embs
d_word = pretrained_embs.size()[-1]
else:
log.info("\tLearning word embeddings from scratch!")
word_embs = None
d_word = args.d_word
embeddings = Embedding(vocab.get_vocab_size('tokens'), d_word,
weight=word_embs, trainable=False,
padding_index=vocab.get_token_index('@@PADDING@@'))
token_embedder["words"] = embeddings
d_emb += d_word
else:
log.info("\tNot using word embeddings!")
# Handle cove
if args.cove:
sys.path.append(args.path_to_cove)
try:
from cove import MTLSTM as cove_lstm
cove_emb = cove_lstm(n_vocab=vocab.get_vocab_size('tokens'),
vectors=embeddings.weight.data)
d_emb += 600
log.info("\tUsing CoVe embeddings!")
except ImportError:
log.info("Failed to import CoVE!")
else:
cove_emb = None
# Character embeddings
if args.char_embs:
log.info("\tUsing character embeddings!")
char_embeddings = Embedding(vocab.get_vocab_size('chars'), d_char)
filter_sizes = tuple([int(i) for i in args.char_filter_sizes.split(',')])
char_encoder = CnnEncoder(d_char, num_filters=args.n_char_filters,
ngram_filter_sizes=filter_sizes,
output_dim=d_char)
char_embedder = TokenCharactersEncoder(char_embeddings, char_encoder,
dropout=args.dropout_embs)
d_emb += d_char
token_embedder["chars"] = char_embedder
else:
log.info("\tNot using character embeddings!")
# Handle elmo
if args.elmo:
log.info("Loading ELMo from files:")
log.info("ELMO_OPT_PATH = %s", ELMO_OPT_PATH)
log.info("ELMO_WEIGHTS_PATH = %s", ELMO_WEIGHTS_PATH)
if args.elmo_chars_only:
log.info("\tUsing ELMo character CNN only!")
#elmo_embedder = elmo_embedder._elmo._elmo_lstm._token_embedder
elmo_embedder = ElmoCharacterEncoder(options_file=ELMO_OPT_PATH,
weight_file=ELMO_WEIGHTS_PATH,
requires_grad=False)
d_emb += 512
else:
log.info("\tUsing full ELMo!")
elmo_embedder = ElmoTokenEmbedder(options_file=ELMO_OPT_PATH,
weight_file=ELMO_WEIGHTS_PATH,
dropout=args.dropout)
d_emb += 1024
token_embedder["elmo"] = elmo_embedder
embedder = BasicTextFieldEmbedder(token_embedder)
assert d_emb, "You turned off all the embeddings, ya goof!"
return d_emb, embedder, cove_emb
Params({
"pretrained_file": config["pre_trained_embedding"],
"embedding_dim": config["pre_trained_embedding_dim"],
"trainable": config["train_embedding"],
"padding_index": 0,
"sparse": config["sparse_gradient_embedding"]
}))
elif config["token_embedder_type"] == "fasttext":
vocab = None #FastTextVocab(config["fasttext_vocab_mapping"])
tokens_embedder = FastTextEmbeddingBag(numpy.load(
config["fasttext_weights"]),
sparse=True)
elif config["token_embedder_type"] == "elmo":
vocab = None
tokens_embedder = ElmoTokenEmbedder(config["elmo_options_file"],
config["elmo_weights_file"])
else:
logger.error("token_embedder_type %s not known",
config["token_embedder_type"])
exit(1)
word_embedder = BasicTextFieldEmbedder({"tokens": tokens_embedder})
if config["model"] == "knrm":
model = KNRM(word_embedder,
n_kernels=config["knrm_kernels"]).cuda(cuda_device)
elif config["model"] == "conv_knrm":
model = Conv_KNRM(
word_embedder,
n_grams=config["conv_knrm_ngrams"],
batch_size=config.eval_batch_size,
sorting_keys=[("text", "num_tokens")],
)
val_iterator.index_with(vocab)
if args.embedding_type == 'glove':
param_dict = {
"pretrained_file":
"(https://nlp.stanford.edu/data/glove.6B.zip)#glove.6B.300d.txt",
"embedding_dim": 300
}
params = Params(params=param_dict)
token_embedding = Embedding.from_params(vocab=vocab, params=params)
elif args.embedding_type == 'elmo':
token_embedding = ElmoTokenEmbedder(args.options_file,
args.weights_file,
requires_grad=args.finetune_embeddings)
word_embeddings = BasicTextFieldEmbedder({"tokens": token_embedding})
if args.encoder_type == 'bag':
encoder = BagOfEmbeddingsEncoder(word_embeddings.get_output_dim())
elif args.encoder_type == 'lstm':
encoder = PytorchSeq2VecWrapper(
torch.nn.LSTM(word_embeddings.get_output_dim(),
config.hidden_sz,
bidirectional=True,
batch_first=True))
num_classes = vocab.get_vocab_size("labels")
decoder_input_dim = encoder.get_output_dim()
def main():
elmo_token_indexer = ELMoTokenCharactersIndexer()
reader = Seq2SeqDatasetReader(
source_tokenizer=WordTokenizer(),
target_tokenizer=WordTokenizer(),
source_token_indexers={'tokens': elmo_token_indexer},
target_token_indexers={
'tokens': SingleIdTokenIndexer(namespace='target_tokens')
})
train_dataset, test_dataset, dev_dataset = (
reader.read(DATA_ROOT + "/" + fname) for fname in
["train_all_seq.txt", "test_all_seq.txt", "val_all_seq.txt"])
vocab = Vocabulary.from_instances(train_dataset + dev_dataset +
test_dataset,
min_count={
'tokens': 1,
'target_tokens': 1
})
# en_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
# embedding_dim=256)
# en_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
# embedding_dim=elmo_embedding_dim)
#elmo_embedder = Elmo(options_file, weight_file, 2, dropout=0.5)
elmo_embedder = ElmoTokenEmbedder(options_file, weight_file)
# word_embeddings = BasicTextFieldEmbedder({'tokens': elmo_embedder})
# en_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
# embedding_dim=256)
source_embedder = BasicTextFieldEmbedder({"tokens": elmo_embedder})
#Initializing the model
max_decoding_steps = 20
encoder = PytorchSeq2SeqWrapper(
torch.nn.LSTM(elmo_embedding_dim, hidden_dim, batch_first=True))
# encoder = StackedSelfAttentionEncoder(input_dim=elmo_embedding_dim, hidden_dim=hidden_dim, projection_dim=128, feedforward_hidden_dim=128, num_layers=1, num_attention_heads=8)
attention = DotProductAttention()
model = SimpleSeq2Seq(vocab,
source_embedder,
encoder,
max_decoding_steps,
target_embedding_dim=elmo_embedding_dim,
target_namespace='target_tokens',
attention=attention,
beam_size=8,
use_bleu=True)
if USE_GPU: model.cuda()
else: model
# Training the model
optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-5)
iterator = BucketIterator(batch_size=32,
sorting_keys=[("source_tokens", "num_tokens")])
iterator.index_with(vocab)
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=dev_dataset,
patience=10,
num_epochs=1,
cuda_device=0 if USE_GPU else -1)
for i in range(20):
print('Epoch: {}'.format(i))
trainer.train()
predictor = SimpleSeq2SeqPredictor(model, reader)
for instance in itertools.islice(dev_dataset, 10):
print('SOURCE:', instance.fields['source_tokens'].tokens)
print('GOLD:', instance.fields['target_tokens'].tokens)
print('PRED:',
predictor.predict_instance(instance)['predicted_tokens'])
#Saving the model
with open("model_seq2seq.th", 'wb') as f:
torch.save(model.state_dict(), f)
vocab.save_to_files("vocabulary_seq2seq")
predictor = SimpleSeq2SeqPredictor(model, reader)
with open('predict_seq2seq.txt', 'w+') as f:
for instance in itertools.islice(test_dataset, 10):
preds = predictor.predict_instance(instance)['predicted_tokens']
f.write(" ".join(preds) + "\n")