def load_model(self): select_device = 'GPU' restore_model = True # Create dictionary of filenames self.download_model() data_path = self.data_path # Paths for preprcessed files path_gen = data_path # data is actually in mrc_data/data not, mrc_data train_para_ids = os.path.join(path_gen, self.file_name_dict['train_para_ids']) train_ques_ids = os.path.join(path_gen, self.file_name_dict['train_ques_ids']) answer_file = os.path.join(path_gen, self.file_name_dict['train_answer']) val_paras_ids = os.path.join(path_gen, self.file_name_dict['val_para_ids']) val_ques_ids = os.path.join(path_gen, self.file_name_dict['val_ques_ids']) val_ans_file = os.path.join(path_gen, self.file_name_dict['val_ans']) vocab_file = os.path.join(path_gen, self.file_name_dict['vocab_file']) model_dir = self.model_path # Create model dir if it doesn't exist if not os.path.exists(model_dir): os.makedirs(model_dir) model_path = model_dir # Create lists for train and validation sets data_train = create_squad_training(train_para_ids, train_ques_ids, answer_file) data_dev = create_squad_training(val_paras_ids, val_ques_ids, val_ans_file) with open(vocab_file, encoding='UTF-8') as fp: vocab_list = fp.readlines() self.vocab_dict = {} self.vocab_rev = {} for i in range(len(vocab_list)): self.vocab_dict[i] = vocab_list[i].strip() self.vocab_rev[vocab_list[i].strip()] = i self.params_dict['train_set_size'] = len(data_train) # Combine train and dev data data_total = data_train + data_dev # obtain maximum length of question _, max_question = max_values_squad(data_total) self.params_dict['max_question'] = max_question # Load embeddings for vocab print('Loading Embeddings') embeddingz = np.load( os.path.join(path_gen, self.file_name_dict['embedding'])) embeddings = embeddingz['glove'] # Create train and dev sets print("Creating training and development sets") self.dev = get_data_array_squad(self.params_dict, data_dev, set_val='val') # Define Reading Comprehension model with tf.device('/device:' + select_device + ':0'): self.model = MatchLSTM_AnswerPointer(self.params_dict, embeddings) # Define Configs for training run_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) # Create session run training self.sess = tf.Session(config=run_config) init = tf.global_variables_initializer() # Model Saver model_saver = tf.train.Saver() model_ckpt = tf.train.get_checkpoint_state(model_path) idx_path = model_ckpt.model_checkpoint_path + ".index" if model_ckpt else "" # Intitialze with random or pretrained weights if model_ckpt and restore_model and (tf.gfile.Exists( model_ckpt.model_checkpoint_path) or tf.gfile.Exists(idx_path)): model_saver.restore(self.sess, model_ckpt.model_checkpoint_path) print("Loading from previously stored session") else: self.sess.run(init) shuffle(self.dev) return
# Combine train and dev data data_total = data_train + data_dev # obtain maximum length of question _, max_question = max_values_squad(data_total) params_dict['max_question'] = max_question # Load embeddings for vocab print('Loading Embeddings') embeddingz = np.load(os.path.join(path_gen, file_name_dict['embedding'])) embeddings = embeddingz['glove'] # Create train and dev sets print("Creating training and development sets") train = get_data_array_squad(params_dict, data_train, set_val='train') dev = get_data_array_squad(params_dict, data_dev, set_val='val') # Define Reading Comprehension model with tf.device('/device:' + args.select_device + ':0'): model = MatchLSTM_AnswerPointer(params_dict, embeddings) # Define Configs for training run_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) # Create session run training with tf.Session(config=run_config) as sess: init = tf.global_variables_initializer() # Model Saver model_saver = tf.train.Saver()