def load_model(self):
select_device = 'GPU'
restore_model = True
# Create dictionary of filenames
self.download_model()
data_path = self.data_path
# Paths for preprcessed files
path_gen = data_path # data is actually in mrc_data/data not, mrc_data
train_para_ids = os.path.join(path_gen,
self.file_name_dict['train_para_ids'])
train_ques_ids = os.path.join(path_gen,
self.file_name_dict['train_ques_ids'])
answer_file = os.path.join(path_gen,
self.file_name_dict['train_answer'])
val_paras_ids = os.path.join(path_gen,
self.file_name_dict['val_para_ids'])
val_ques_ids = os.path.join(path_gen,
self.file_name_dict['val_ques_ids'])
val_ans_file = os.path.join(path_gen, self.file_name_dict['val_ans'])
vocab_file = os.path.join(path_gen, self.file_name_dict['vocab_file'])
model_dir = self.model_path
# Create model dir if it doesn't exist
if not os.path.exists(model_dir):
os.makedirs(model_dir)
model_path = model_dir
# Create lists for train and validation sets
data_train = create_squad_training(train_para_ids, train_ques_ids,
answer_file)
data_dev = create_squad_training(val_paras_ids, val_ques_ids,
val_ans_file)
with open(vocab_file, encoding='UTF-8') as fp:
vocab_list = fp.readlines()
self.vocab_dict = {}
self.vocab_rev = {}
for i in range(len(vocab_list)):
self.vocab_dict[i] = vocab_list[i].strip()
self.vocab_rev[vocab_list[i].strip()] = i
self.params_dict['train_set_size'] = len(data_train)
# Combine train and dev data
data_total = data_train + data_dev
# obtain maximum length of question
_, max_question = max_values_squad(data_total)
self.params_dict['max_question'] = max_question
# Load embeddings for vocab
print('Loading Embeddings')
embeddingz = np.load(
os.path.join(path_gen, self.file_name_dict['embedding']))
embeddings = embeddingz['glove']
# Create train and dev sets
print("Creating training and development sets")
self.dev = get_data_array_squad(self.params_dict,
data_dev,
set_val='val')
# Define Reading Comprehension model
with tf.device('/device:' + select_device + ':0'):
self.model = MatchLSTM_AnswerPointer(self.params_dict, embeddings)
# Define Configs for training
run_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
# Create session run training
self.sess = tf.Session(config=run_config)
init = tf.global_variables_initializer()
# Model Saver
model_saver = tf.train.Saver()
model_ckpt = tf.train.get_checkpoint_state(model_path)
idx_path = model_ckpt.model_checkpoint_path + ".index" if model_ckpt else ""
# Intitialze with random or pretrained weights
if model_ckpt and restore_model and (tf.gfile.Exists(
model_ckpt.model_checkpoint_path)
or tf.gfile.Exists(idx_path)):
model_saver.restore(self.sess, model_ckpt.model_checkpoint_path)
print("Loading from previously stored session")
else:
self.sess.run(init)
shuffle(self.dev)
return
# Combine train and dev data
data_total = data_train + data_dev
# obtain maximum length of question
_, max_question = max_values_squad(data_total)
params_dict['max_question'] = max_question
# Load embeddings for vocab
print('Loading Embeddings')
embeddingz = np.load(os.path.join(path_gen, file_name_dict['embedding']))
embeddings = embeddingz['glove']
# Create train and dev sets
print("Creating training and development sets")
train = get_data_array_squad(params_dict, data_train, set_val='train')
dev = get_data_array_squad(params_dict, data_dev, set_val='val')
# Define Reading Comprehension model
with tf.device('/device:' + args.select_device + ':0'):
model = MatchLSTM_AnswerPointer(params_dict, embeddings)
# Define Configs for training
run_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
# Create session run training
with tf.Session(config=run_config) as sess:
init = tf.global_variables_initializer()
# Model Saver
model_saver = tf.train.Saver()