def create_transfo_xl_model(self, config, input_ids_1, input_ids_2,
lm_labels):
model = TransfoXLModel(config)
model.eval()
hidden_states_1, mems_1 = model(input_ids_1)
hidden_states_2, mems_2 = model(input_ids_2, mems_1)
outputs = {
"hidden_states_1": hidden_states_1,
"mems_1": mems_1,
"hidden_states_2": hidden_states_2,
"mems_2": mems_2,
}
return outputs
output = output.reshape(output.shape[0], -1,
output.shape[1])
print(output.shape)
output = np.swapaxes(output, 0, 1)
list_output.append(output)
# ====== Construct Cache ====== #
temp_cache = {}
for i, sent in enumerate(mini_batch):
hask_key = hashlib.sha256(sent.encode()).hexdigest()
temp_cache[hask_key] = output[i]
self.cache.update(temp_cache)
idx += mini_batch_size
self.count += mini_batch_size
output = np.concatenate(list_output, 0)
te = time.time()
print('encoding with model', len(sentences), 'processed',
self.count, 'took', '{:4.1f}'.format(te - ts))
te = time.time()
embedding = self.get_multi_head_embedding(output, heads, head_size)
return embedding
if __name__ == '__main__':
model = TransfoXLModel('bert-base-uncased')
model.prepare('Length')
model.construct_encoder()