def make_ELMo(self):
# Location of pretrained BiLM for the specified language
# TBD check if ELMo language resources are present
description = self._get_description('elmo-en')
if description is not None:
self.lang = description["lang"]
vocab_file = description["path-vocab"]
options_file = description["path-config"]
weight_file = description["path_weights"]
print('init ELMo')
# Create a Batcher to map text to character ids
self.batcher = Batcher(vocab_file, 50)
# Build the biLM graph.
self.bilm = BidirectionalLanguageModel(options_file, weight_file)
# Input placeholders to the biLM.
self.character_ids = tf.placeholder('int32',
shape=(None, None, 50))
self.embeddings_op = self.bilm(self.character_ids)
with tf.variable_scope('', reuse=tf.AUTO_REUSE):
# the reuse=True scope reuses weights from the whole context
self.elmo_input = weight_layers('input',
self.embeddings_op,
l2_coef=0.0)
def get_sentence_vector_ELMo_with_token_dump(self, token_list):
if not self.use_ELMo:
print(
"Warning: ELMo embeddings requested but embeddings object wrongly initialised"
)
return
with tf.variable_scope('', reuse=tf.AUTO_REUSE):
# the reuse=True scope reuses weights from the whole context
self.elmo_input_token_dump = weight_layers(
'input', self.embeddings_op_token_dump, l2_coef=0.0)
# Create batches of data
local_token_ids = self.batcher_token_dump.batch_sentences(token_list)
with tf.Session() as sess:
# weird, for this cpu is faster than gpu (1080Ti !)
with tf.device("/cpu:0"):
# It is necessary to initialize variables once before running inference
sess.run(tf.global_variables_initializer())
# Compute ELMo representations
elmo_result = sess.run(
self.elmo_input_token_dump['weighted_op'],
feed_dict={self.token_ids: local_token_ids})
return elmo_result
def get_elmo_embedding(character_ids, elmo, sent_len):
embeddings_op = elmo.bilm(character_ids)
with tf.variable_scope('', reuse=tf.AUTO_REUSE):
# the reuse=True scope reuses weights from the whole context
elmo_input = weight_layers('input', embeddings_op, l2_coef=0.0)
elmo_embed = elmo_input['weighted_op']
return elmo_embed
