model = BertModel.from_pretrained('bert-base-uncased')
token_embedding = BertEmbedder(model)
PROJECT_DIM = 768
else:
print("Error: Some weird Embedding type", EMBEDDING_TYPE)
exit()
word_embeddings = BasicTextFieldEmbedder({"tokens": token_embedding})
HIDDEN_DIM = 200
params = Params({
'input_dim': PROJECT_DIM,
'hidden_dims': HIDDEN_DIM,
'activations': 'relu',
'num_layers': NUM_LAYERS,
'dropout': DROPOUT
})
attend_feedforward = FeedForward.from_params(params)
similarity_function = DotProductSimilarity()
params = Params({
'input_dim': 2 * PROJECT_DIM,
'hidden_dims': HIDDEN_DIM,
'activations': 'relu',
'num_layers': NUM_LAYERS,
'dropout': DROPOUT
})
compare_feedforward = FeedForward.from_params(params)
params = Params({
'input_dim': 2 * HIDDEN_DIM,
'hidden_dims': 1,
'activations': 'linear',
'num_layers': 1
})
def load_decomposable_attention_elmo_softmax_model():
NEGATIVE_PERCENTAGE = 100
# EMBEDDING_TYPE = ""
# LOSS_TYPE = "" # NLL
# LOSS_TYPE = "_nll" # NLL
LOSS_TYPE = "_mse" # MSE
# EMBEDDING_TYPE = ""
# EMBEDDING_TYPE = "_glove"
# EMBEDDING_TYPE = "_bert"
EMBEDDING_TYPE = "_elmo"
# EMBEDDING_TYPE = "_elmo_retrained"
# EMBEDDING_TYPE = "_elmo_retrained_2"
token_indexers = None
if EMBEDDING_TYPE == "_elmo" or EMBEDDING_TYPE == "_elmo_retrained" or EMBEDDING_TYPE == "_elmo_retrained_2":
token_indexers = {"tokens": ELMoTokenCharactersIndexer()}
MAX_BATCH_SIZE = 0
# MAX_BATCH_SIZE = 150 # for bert and elmo
reader = QuestionResponseSoftmaxReader(token_indexers=token_indexers,
max_batch_size=MAX_BATCH_SIZE)
model_file = os.path.join(
"saved_softmax_models",
"decomposable_attention{}{}_model_{}.th".format(
LOSS_TYPE, EMBEDDING_TYPE, NEGATIVE_PERCENTAGE))
vocabulary_filepath = os.path.join(
"saved_softmax_models",
"vocabulary{}{}_{}".format(LOSS_TYPE, EMBEDDING_TYPE,
NEGATIVE_PERCENTAGE))
print("LOADING VOCABULARY")
# Load vocabulary
vocab = Vocabulary.from_files(vocabulary_filepath)
EMBEDDING_DIM = 300
PROJECT_DIM = 200
DROPOUT = 0.2
NUM_LAYERS = 2
if EMBEDDING_TYPE == "":
token_embedding = Embedding(
num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=EMBEDDING_DIM,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_glove":
token_embedding = Embedding.from_params(vocab=vocab,
params=Params({
'pretrained_file':
glove_embeddings_file,
'embedding_dim':
EMBEDDING_DIM,
'projection_dim':
PROJECT_DIM,
'trainable':
False
}))
elif EMBEDDING_TYPE == "_elmo":
# options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_options.json"
# weights_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5"
options_file = os.path.join(
"data", "elmo", "elmo_2x2048_256_2048cnn_1xhighway_options.json")
weights_file = os.path.join(
"data", "elmo", "elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5")
# NOTE: using Small size as medium size gave CUDA out of memory error
# options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json"
# weights_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5"
# options_file = os.path.join("data", "elmo", "elmo_2x1024_128_2048cnn_1xhighway_options.json")
# weights_file = os.path.join("data", "elmo", "elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5")
token_embedding = ElmoTokenEmbedder(options_file,
weights_file,
dropout=DROPOUT,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_elmo_retrained":
options_file = os.path.join("data", "bilm-tf", "elmo_retrained",
"options.json")
weights_file = os.path.join("data", "bilm-tf", "elmo_retrained",
"weights.hdf5")
token_embedding = ElmoTokenEmbedder(options_file,
weights_file,
dropout=DROPOUT,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_elmo_retrained_2":
options_file = os.path.join("data", "bilm-tf", "elmo_retrained",
"options_2.json")
weights_file = os.path.join("data", "bilm-tf", "elmo_retrained",
"weights_2.hdf5")
token_embedding = ElmoTokenEmbedder(options_file,
weights_file,
dropout=DROPOUT,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_bert":
print("Loading bert model")
model = BertModel.from_pretrained('bert-base-uncased')
token_embedding = BertEmbedder(model)
PROJECT_DIM = 768
else:
print("Error: Some weird Embedding type", EMBEDDING_TYPE)
exit()
word_embeddings = BasicTextFieldEmbedder({"tokens": token_embedding})
HIDDEN_DIM = 200
params = Params({
'input_dim': PROJECT_DIM,
'hidden_dims': HIDDEN_DIM,
'activations': 'relu',
'num_layers': NUM_LAYERS,
'dropout': DROPOUT
})
attend_feedforward = FeedForward.from_params(params)
similarity_function = DotProductSimilarity()
params = Params({
'input_dim': 2 * PROJECT_DIM,
'hidden_dims': HIDDEN_DIM,
'activations': 'relu',
'num_layers': NUM_LAYERS,
'dropout': DROPOUT
})
compare_feedforward = FeedForward.from_params(params)
params = Params({
'input_dim': 2 * HIDDEN_DIM,
'hidden_dims': 1,
'activations': 'linear',
'num_layers': 1
})
aggregate_feedforward = FeedForward.from_params(params)
model = DecomposableAttentionSoftmax(vocab, word_embeddings,
attend_feedforward,
similarity_function,
compare_feedforward,
aggregate_feedforward)
print("MODEL CREATED")
# Load model state
with open(model_file, 'rb') as f:
model.load_state_dict(torch.load(f, map_location='cuda:0'))
print("MODEL LOADED!")
if torch.cuda.is_available():
# cuda_device = 3
# model = model.cuda(cuda_device)
cuda_device = -1
else:
cuda_device = -1
predictor = DecomposableAttentionSoftmaxPredictor(model,
dataset_reader=reader)
return model, predictor
def __init__(
self,
vocab: Vocabulary,
source_embedder: TextFieldEmbedder, # just Embedding layer
encoder1: Seq2SeqEncoder, # user encoder
encoder2: Seq2SeqEncoder, # system encoder
attention: Attention, # decoding attention
max_decoding_steps: int = 200, # max timesteps of decoder
beam_size: int = 3, # beam search parameter
target_namespace: str = "target_tokens", # two separate vocabulary
target_embedding_dim: int = None, # target word embedding dimension
scheduled_sampling_ratio: float = 0., # maybe unnecessary
projection_dim: int = None, #
use_coverage: bool = False, # coverage penalty, optional
coverage_loss_weight: float = None,
domain_lambda:
float = 0.5, # the penalty weight in final loss function, need to be tuned
initializer: InitializerApplicator = InitializerApplicator()
) -> None:
super(SPNet, self).__init__(vocab)
# General variables
# target_namespace: target_tokens; source_namespace: tokens;
self._target_namespace = target_namespace
self._start_index = self.vocab.get_token_index(START_SYMBOL,
self._target_namespace)
self._end_index = self.vocab.get_token_index(END_SYMBOL,
self._target_namespace)
self._source_unk_index = self.vocab.get_token_index(DEFAULT_OOV_TOKEN)
self._target_unk_index = self.vocab.get_token_index(
DEFAULT_OOV_TOKEN, self._target_namespace)
self._source_vocab_size = self.vocab.get_vocab_size()
self._target_vocab_size = self.vocab.get_vocab_size(
self._target_namespace)
# Encoder setting
self._source_embedder = source_embedder
self._encoder1 = encoder1
self._encoder2 = encoder2
# We assume that the 2 encoders have the same hidden state size
self._encoder_output_dim = self._encoder1.get_output_dim()
# Decoder setting
self._target_embedding_dim = target_embedding_dim or source_embedder.get_output_dim(
)
self._num_classes = self.vocab.get_vocab_size(self._target_namespace)
self._target_embedder = Embedding(self._num_classes,
self._target_embedding_dim)
self._decoder_input_dim = self._encoder_output_dim * 2 # default as the decoder_output_dim
# input projection of decoder: [context_attn, target_emb] -> [decoder_input_dim]
self._input_projection_layer = Linear(
self._target_embedding_dim + self._encoder_output_dim * 2,
self._decoder_input_dim)
self._decoder_output_dim = self._encoder_output_dim * 2
self._decoder_cell = LSTMCell(self._decoder_input_dim,
self._decoder_output_dim)
self._projection_dim = projection_dim or self._source_embedder.get_output_dim(
)
self._output_projection_layer = Linear(self._decoder_output_dim,
self._num_classes)
self._p_gen_layer = Linear(
self._encoder_output_dim * 2 + self._decoder_output_dim * 2 +
self._decoder_input_dim, 1)
self._attention = attention
# coverage penalty setting
self._use_coverage = use_coverage
self._coverage_loss_weight = coverage_loss_weight
self._eps = 1e-45
# Decoding strategy setting
self._scheduled_sampling_ratio = scheduled_sampling_ratio
self._max_decoding_steps = max_decoding_steps
self._beam_search = BeamSearch(self._end_index,
max_steps=max_decoding_steps,
beam_size=beam_size)
# multitasking of domain classification
self._domain_penalty = domain_lambda # penalty term = 0.5 as default
self._classifier_params = Params({
"input_dim": self._decoder_output_dim,
"hidden_dims": [128, 7],
"activations": ["relu", "linear"],
"dropout": [0.2, 0.0],
"num_layers": 2
})
self._domain_classifier = FeedForward.from_params(
self._classifier_params)
initializer(self)
seq2seq_encoder = PytorchSeq2SeqWrapper(
torch.nn.LSTM(EMBEDDING_DIM, HIDDEN_DIM, batch_first=True))
# In[1152]:
classifier_params = Params({
"input_dim": HIDDEN_DIM * 2,
"num_layers": 2,
"hidden_dims": [50, 3],
"activations": ["sigmoid", "linear"],
"dropout": [0.2, 0.0]
})
# In[1153]:
classifier_feedforward = FeedForward.from_params(classifier_params)
# In[1154]:
parse_label = {
'word': torch.LongTensor([[1, 0, 3, 7, 2, 9, 4], [0, 0, 5, 0, 0, 0, 4]])
}
embedded_parse_label = field_type2embedder['word'](parse_label)
# In[1155]:
feature_mask = util.get_text_field_mask(parse_label)
# In[1156]:
encoded_feature = encoder(embedded_parse_label, feature_mask)
def save_top_results(process_no, start_index, end_index):
print("Starting process {} with start at {} and end at {}".format(
process_no, start_index, end_index))
DATA_FOLDER = "train_data"
# EMBEDDING_TYPE = ""
LOSS_TYPE = "" # NLL
LOSS_TYPE = "_mse" # MSE
# EMBEDDING_TYPE = ""
# EMBEDDING_TYPE = "_glove"
# EMBEDDING_TYPE = "_bert"
EMBEDDING_TYPE = "_elmo"
# EMBEDDING_TYPE = "_elmo_retrained"
# EMBEDDING_TYPE = "_elmo_retrained_2"
token_indexers = None
if EMBEDDING_TYPE == "_elmo" or EMBEDDING_TYPE == "_elmo_retrained" or EMBEDDING_TYPE == "_elmo_retrained_2":
token_indexers = {"tokens": ELMoTokenCharactersIndexer()}
MAX_BATCH_SIZE = 0
# MAX_BATCH_SIZE = 150 # for bert and elmo
# q_file = os.path.join("squad_seq2seq_train", "rule_based_system_squad_seq2seq_train_case_sensitive_saved_questions_lexparser_sh.txt")
# r_file = os.path.join("squad_seq2seq_train", "rule_based_system_squad_seq2seq_train_case_sensitive_generated_answers_lexparser_sh.txt")
# rules_file = os.path.join("squad_seq2seq_train", "rule_based_system_squad_seq2seq_train_case_sensitive_generated_answer_rules_lexparser_sh.txt")
#NOTE: Squad dev test set
q_file = os.path.join(
"squad_seq2seq_dev_moses_tokenized",
"rule_based_system_squad_seq2seq_dev_test_saved_questions.txt")
r_file = os.path.join(
"squad_seq2seq_dev_moses_tokenized",
"rule_based_system_squad_seq2seq_dev_test_generated_answers.txt")
rules_file = os.path.join(
"squad_seq2seq_dev_moses_tokenized",
"rule_based_system_squad_seq2seq_dev_test_generated_answer_rules.txt")
reader = QuestionResponseSoftmaxReader(q_file,
r_file,
token_indexers=token_indexers,
max_batch_size=MAX_BATCH_SIZE)
glove_embeddings_file = os.path.join("data", "glove",
"glove.840B.300d.txt")
# RESULTS_DIR = "squad_seq2seq_train2"
#NOTE: All other experiments
# RESULTS_DIR = "squad_seq2seq_train_moses_tokenized"
# make_dir_if_not_exists(RESULTS_DIR)
# all_results_save_file = os.path.join(RESULTS_DIR, "squad_seq2seq_train_predictions_start_{}_end_{}.txt".format(start_index, end_index))
#NOTE: Squad dev test set
RESULTS_DIR = "squad_seq2seq_dev_moses_tokenized"
make_dir_if_not_exists(RESULTS_DIR)
all_results_save_file = os.path.join(
RESULTS_DIR,
"squad_seq2seq_dev_test_predictions_start_{}_end_{}.txt".format(
start_index, end_index))
with open(all_results_save_file, "w") as all_writer:
print("Testing out model with", EMBEDDING_TYPE, "embeddings")
print("Testing out model with", LOSS_TYPE, "loss")
# for NEGATIVE_PERCENTAGE in [100,50,20,10,5,1]:
for NEGATIVE_PERCENTAGE in [100]:
model_file = os.path.join(
"saved_softmax_models",
"decomposable_attention{}{}_model_{}.th".format(
LOSS_TYPE, EMBEDDING_TYPE, NEGATIVE_PERCENTAGE))
vocabulary_filepath = os.path.join(
"saved_softmax_models",
"vocabulary{}{}_{}".format(LOSS_TYPE, EMBEDDING_TYPE,
NEGATIVE_PERCENTAGE))
print("LOADING VOCABULARY")
# Load vocabulary
vocab = Vocabulary.from_files(vocabulary_filepath)
EMBEDDING_DIM = 300
PROJECT_DIM = 200
DROPOUT = 0.2
NUM_LAYERS = 2
if EMBEDDING_TYPE == "":
token_embedding = Embedding(
num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=EMBEDDING_DIM,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_glove":
token_embedding = Embedding.from_params(
vocab=vocab,
params=Params({
'pretrained_file': glove_embeddings_file,
'embedding_dim': EMBEDDING_DIM,
'projection_dim': PROJECT_DIM,
'trainable': False
}))
elif EMBEDDING_TYPE == "_elmo":
# options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_options.json"
# weights_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5"
options_file = os.path.join(
"data", "elmo",
"elmo_2x2048_256_2048cnn_1xhighway_options.json")
weights_file = os.path.join(
"data", "elmo",
"elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5")
# NOTE: using Small size as medium size gave CUDA out of memory error
# options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json"
# weights_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5"
# options_file = os.path.join("data", "elmo", "elmo_2x1024_128_2048cnn_1xhighway_options.json")
# weights_file = os.path.join("data", "elmo", "elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5")
token_embedding = ElmoTokenEmbedder(options_file,
weights_file,
dropout=DROPOUT,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_elmo_retrained":
options_file = os.path.join("data", "bilm-tf",
"elmo_retrained", "options.json")
weights_file = os.path.join("data", "bilm-tf",
"elmo_retrained", "weights.hdf5")
token_embedding = ElmoTokenEmbedder(options_file,
weights_file,
dropout=DROPOUT,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_elmo_retrained_2":
options_file = os.path.join("data", "bilm-tf",
"elmo_retrained", "options_2.json")
weights_file = os.path.join("data", "bilm-tf",
"elmo_retrained", "weights_2.hdf5")
token_embedding = ElmoTokenEmbedder(options_file,
weights_file,
dropout=DROPOUT,
projection_dim=PROJECT_DIM)
elif EMBEDDING_TYPE == "_bert":
print("Loading bert model")
model = BertModel.from_pretrained('bert-base-uncased')
token_embedding = BertEmbedder(model)
PROJECT_DIM = 768
else:
print("Error: Some weird Embedding type", EMBEDDING_TYPE)
exit()
word_embeddings = BasicTextFieldEmbedder(
{"tokens": token_embedding})
HIDDEN_DIM = 200
params = Params({
'input_dim': PROJECT_DIM,
'hidden_dims': HIDDEN_DIM,
'activations': 'relu',
'num_layers': NUM_LAYERS,
'dropout': DROPOUT
})
attend_feedforward = FeedForward.from_params(params)
similarity_function = DotProductSimilarity()
params = Params({
'input_dim': 2 * PROJECT_DIM,
'hidden_dims': HIDDEN_DIM,
'activations': 'relu',
'num_layers': NUM_LAYERS,
'dropout': DROPOUT
})
compare_feedforward = FeedForward.from_params(params)
params = Params({
'input_dim': 2 * HIDDEN_DIM,
'hidden_dims': 1,
'activations': 'linear',
'num_layers': 1
})
aggregate_feedforward = FeedForward.from_params(params)
model = DecomposableAttentionSoftmax(vocab, word_embeddings,
attend_feedforward,
similarity_function,
compare_feedforward,
aggregate_feedforward)
print("MODEL CREATED")
# Load model state
with open(model_file, 'rb') as f:
device = torch.device('cpu')
model.load_state_dict(torch.load(f, map_location=device))
print("MODEL LOADED!")
if torch.cuda.is_available():
# cuda_device = 3
# model = model.cuda(cuda_device)
cuda_device = -1
else:
cuda_device = -1
predictor = DecomposableAttentionSoftmaxPredictor(
model, dataset_reader=reader)
# Read test file and get predictions
gold = list()
predicted_labels = list()
probs = list()
total_time = avg_time = 0.0
print("Started Testing:", NEGATIVE_PERCENTAGE)
# before working on anything just save all the questions and responses in a list
all_data = list()
examples_count = processed_examples_count = 0
with open(q_file,
'r') as q_reader, open(r_file, "r") as r_reader, open(
rules_file, "r") as rule_reader:
logger.info("Reading questions from : %s", q_file)
logger.info("Reading responses from : %s", r_file)
q = next(q_reader).lower().strip()
q = mt.tokenize(q, return_str=True, escape=False)
current_qa = (q, "")
current_rules_and_responses = list()
for i, (response,
rule) in enumerate(zip(r_reader, rule_reader)):
response = response.strip()
rule = rule.strip()
if response and rule:
# get current_answer from response
a = get_answer_from_response(response)
if not current_qa[1]:
current_qa = (q, a)
else:
# verify if the a is same as the one in current_qa
if a != current_qa[1]:
# print("answer phrase mismatch!!", current_qa, ":::", a, ":::", response)
current_qa = (current_qa[0], a)
# print(current_rules_and_responses)
# exit()
# Add it to the current responses
current_rules_and_responses.append((response, rule))
elif len(current_rules_and_responses) > 0:
# Create a instance
# print(current_qa)
# print(current_rules_and_responses)
# exit()
if rule or response:
print("Rule Response mismatch")
print(current_qa)
print(response)
print(rule)
print(examples_count)
print(i)
exit()
if examples_count < start_index:
examples_count += 1
q = next(q_reader).lower().strip()
q = mt.tokenize(q, return_str=True, escape=False)
current_qa = (q, "")
current_rules_and_responses = list()
continue
elif examples_count > end_index:
break
all_data.append(
(current_qa, current_rules_and_responses))
try:
q = next(q_reader).lower().strip()
q = mt.tokenize(q, return_str=True, escape=False)
except StopIteration:
# previous one was the last question
q = ""
current_qa = (q, "")
current_rules_and_responses = list()
examples_count += 1
# if(examples_count%100 == 0):
# print(examples_count)
else:
# Serious Bug
print("Serious BUG!!")
print(current_qa)
print(response)
print(rule)
print(examples_count)
print(i)
exit()
print("{}:\tFINISHED IO".format(process_no))
examples_count = start_index
processed_examples_count = 0
for current_qa, responses_and_rules in all_data:
start_time = time.time()
# Tokenize and preprocess the responses
preprocessed_responses = [
mt.tokenize(remove_answer_brackets(response),
return_str=True,
escape=False)
for response, rule in responses_and_rules
]
# predictions = predictor.predict(current_qa[0], [remove_answer_brackets(response) for response, rule in responses_and_rules])
predictions = predictor.predict(current_qa[0],
preprocessed_responses)
label_probs = predictions["label_probs"]
tuples = zip(responses_and_rules, label_probs)
sorted_by_score = sorted(tuples,
key=lambda tup: tup[1],
reverse=True)
count = 0
all_writer.write("{}\n".format(current_qa[0]))
all_writer.write("{}\n".format(current_qa[1]))
for index, ((response, rule),
label_prob) in enumerate(sorted_by_score):
if index == 3:
break
all_writer.write("{}\t{}\t{}\t{}\n".format(
response,
mt.tokenize(remove_answer_brackets(response),
return_str=True,
escape=False), rule, label_prob))
all_writer.write("\n")
all_writer.flush()
end_time = time.time()
processed_examples_count += 1
examples_count += 1
total_time += end_time - start_time
avg_time = total_time / float(processed_examples_count)
print(
"{}:\ttime to write {} with {} responses is {} secs. {} avg time"
.format(process_no, examples_count,
len(responses_and_rules), end_time - start_time,
avg_time))