def test(self, data): """Test routine""" # Check if model exists if not os.path.exists(self.save_file_best + ".index"): print("Model File {} does not exist! Quiting".format( self.save_file_best)) exit(1) # Restore model print("Restoring from {}...".format(self.save_file_best)) self.saver_best.restore(self.sess, self.save_file_best) # Run Test cur_global_step = 0 # dummy if self.config.vis_dump: test_mode_list = ["test"] else: # test_mode_list = ["valid", "test"] test_mode_list = ["test"] # Only run testing for test_mode in test_mode_list: test_process(test_mode, self.sess, cur_global_step, self.summary_op, getattr(self, "summary_" + test_mode[:2]), self.x_in, self.y_in, self.R_in, self.t_in, self.is_training, None, None, None, self.logits, self.e_hat, self.loss, data[test_mode], getattr(self, "res_dir_" + test_mode[:2]), self.config)
def test(self, data): """Test routine""" # Check if model exists if not os.path.exists(self.save_file_best + ".index"): print("Model File {} does not exist! Quiting".format( self.save_file_best)) exit(1) # Restore model print("Restoring from {}...".format(self.save_file_best)) self.saver_best.restore(self.sess, self.save_file_best) # Run Test cur_global_step = 0 # dummy test_mode_list = ["test"] for test_mode in test_mode_list: te_res = test_process(test_mode, self.sess, cur_global_step, self.summary_va, self.x_in, self.R_in, self.t_in, self.is_training, data[test_mode], self.res_dir_va, self.config, False, w=self.w, delta=None)
def val(self, data_loader, epoch): va_res = 0 cur_global_step = self.global_step va_res, summary_t = test_process(self.net, "valid", data_loader, self.res_dir_va, self.config, True) for entry in summary_t: self.tensorboardX.add_scalars(entry['tag'], {'val': entry['val']}, self.global_step) # Higher the better if va_res > self.best_va_res: print("Saving best model with va_res = {}".format(va_res)) self.best_va_res = va_res # Save best validation result with open(self.va_res_file, "w") as ofp: ofp.write("{:e}\n".format(self.best_va_res)) # Save best model cprint('save best model: %s' % epoch, 'yellow') self._save_checkpoint(self.net, self.optimizer, os.path.join(self.config.EXP_DIR, 'best_checkpoint_G.pth.tar'), do_cleaning=False, epoch=epoch)
def test(self, data): """Test routine""" # Check if model exists if not os.path.exists(self.save_file_best + ".index"): print("Model File {} does not exist! Quiting".format( self.save_file_best)) exit(1) # Restore model print("Restoring from {}...".format(self.save_file_best)) self.saver_best.restore(self.sess, self.save_file_best) # Run Test cur_global_step = 0 # dummy test_mode_list = ["test"] # Only evaluate on test set for test_mode in test_mode_list: score = self.last_logit if self.bool_use_weight_for_score: print("score is from weights!") score = self.last_weights if self.config.weight_opt == "sigmoid_softmax": score = [ self.last_logit, self.logit_softmax, self.last_weights ] test_process_ins = [ self.x_in, self.y_in, self.R_in, self.t_in, self.is_training ] if self.config.use_fundamental > 0: test_process_ins += [ self.T1_in, self.T2_in, self.K1_in, self.K2_in ] test_process(test_mode, self.sess, cur_global_step, self.summary_op, getattr(self, "summary_" + test_mode[:2]), test_process_ins, None, None, None, self.logits, self.e_hat, self.loss, self.precision, self.recall, self.last_e_hat, score, self.last_x_in, data[test_mode], getattr(self, "res_dir_" + test_mode[:2]), self.config)
def train(self, data): """Training function. Parameters ---------- data_tr : tuple Training data. data_va : tuple Validation data. x_va : ndarray Validation data. y_va : ndarray Validation labels. """ print("Initializing...") self.sess.run(tf.global_variables_initializer()) # ---------------------------------------- # Resume data if it already exists latest_checkpoint = tf.train.latest_checkpoint(self.res_dir_tr) b_resume = latest_checkpoint is not None if b_resume: # Restore network print("Restoring from {}...".format(self.res_dir_tr)) self.saver_cur.restore(self.sess, latest_checkpoint) # restore number of steps so far step = self.sess.run(self.global_step) # restore best validation result if os.path.exists(self.va_res_file): with open(self.va_res_file, "r") as ifp: dump_res = ifp.read() dump_res = parse("{best_va_res:e}\n", dump_res) best_va_res = dump_res["best_va_res"] else: print("Starting from scratch...") step = 0 best_va_res = -1 # ---------------------------------------- # Unpack some data for simple coding xs_tr = data["train"]["xs"] ys_tr = data["train"]["ys"] Rs_tr = data["train"]["Rs"] ts_tr = data["train"]["ts"] # ---------------------------------------- # The training loop batch_size = self.config.train_batch_size max_iter = self.config.train_iter for step in trange(step, max_iter, ncols=self.config.tqdm_width): # ---------------------------------------- # Batch construction # Get a random training batch ind_cur = np.random.choice(len(xs_tr), batch_size, replace=False) # Use minimum kp in batch to construct the batch numkps = np.array([xs_tr[_i].shape[1] for _i in ind_cur]) cur_num_kp = numkps.min() # Actual construction of the batch xs_b = np.array([xs_tr[_i][:, :cur_num_kp, :] for _i in ind_cur ]).reshape(batch_size, 1, cur_num_kp, 4) ys_b = np.array([ys_tr[_i][:cur_num_kp, :] for _i in ind_cur ]).reshape(batch_size, cur_num_kp, 2) Rs_b = np.array([Rs_tr[_i] for _i in ind_cur]).reshape(batch_size, 9) ts_b = np.array([ts_tr[_i] for _i in ind_cur]).reshape(batch_size, 3) # ---------------------------------------- # Train # Feed Dict feed_dict = { self.x_in: xs_b, self.y_in: ys_b, self.R_in: Rs_b, self.t_in: ts_b, self.is_training: True, } # Fetch fetch = { "optim": self.optim, } # Check if we want to write summary and check validation b_write_summary = ((step + 1) % self.config.report_intv) == 0 b_validate = ((step + 1) % self.config.val_intv) == 0 if b_write_summary or b_validate: fetch["summary"] = self.summary_op fetch["global_step"] = self.global_step # Run optimization try: res = self.sess.run(fetch, feed_dict=feed_dict) except (ValueError, tf.errors.InvalidArgumentError): print("Backward pass had numerical errors. " "This training batch is skipped!") continue # Write summary and save current model if b_write_summary: self.summary_tr.add_summary(res["summary"], global_step=res["global_step"]) self.saver_cur.save(self.sess, self.save_file_cur, global_step=self.global_step, write_meta_graph=False) # ---------------------------------------- # Validation if b_validate: va_res = 0 cur_global_step = res["global_step"] va_res = test_process("valid", self.sess, cur_global_step, self.summary_op, self.summary_va, self.x_in, self.y_in, self.R_in, self.t_in, self.is_training, None, None, None, self.logits, self.e_hat, self.loss, data["valid"], self.res_dir_va, self.config, True) # Higher the better if va_res > best_va_res: print("Saving best model with va_res = {}".format(va_res)) best_va_res = va_res # Save best validation result with open(self.va_res_file, "w") as ofp: ofp.write("{:e}\n".format(best_va_res)) # Save best model self.saver_best.save( self.sess, self.save_file_best, write_meta_graph=False, )
def train(self, data): """Training function. Parameters ---------- data_tr : tuple Training data. data_va : tuple Validation data. x_va : ndarray Validation data. y_va : ndarray Validation labels. """ print("Initializing...") self.sess.run(tf.global_variables_initializer()) # ---------------------------------------- # Resume data if it already exists latest_checkpoint = tf.train.latest_checkpoint(self.res_dir_tr) b_resume = latest_checkpoint is not None if b_resume: # Restore network print("Restoring from {}...".format(self.res_dir_tr)) self.saver_cur.restore(self.sess, latest_checkpoint) # restore number of steps so far step = self.sess.run(self.global_step) # restore best validation result if os.path.exists(self.va_res_file): with open(self.va_res_file, "r") as ifp: dump_res = ifp.read() dump_res = parse("{best_va_res:e}\n", dump_res) best_va_res = dump_res["best_va_res"] if os.path.exists(self.va_res_file_ours_ransac): with open(self.va_res_file_ours_ransac, "r") as ifp: dump_res = ifp.read() dump_res = parse("{best_va_res:e}\n", dump_res) best_va_res_ours_ransac = dump_res["best_va_res"] else: print("Starting from scratch...") step = 0 best_va_res = -1 best_va_res_ours_ransac = -1 # ---------------------------------------- if self.config.data_name.startswith("oan"): data_loader = iter(data["train"]) else: # Unpack some data for simple coding xs_tr = data["train"]["xs"] ys_tr = data["train"]["ys"] Rs_tr = data["train"]["Rs"] ts_tr = data["train"]["ts"] T1s_tr = data["train"]["T1s"] T2s_tr = data["train"]["T2s"] K1s_tr = data["train"]["K1s"] K2s_tr = data["train"]["K2s"] # ---------------------------------------- # The training loop batch_size = self.config.train_batch_size max_iter = self.config.train_iter for step in trange(step, max_iter, ncols=self.config.tqdm_width): # ---------------------------------------- # Batch construction # Get a random training batch if self.config.data_name.startswith("oan"): try: data_dict = next(data_loader) except StopIteration: data_loader = iter(data["train"]) data_dict = next(data_loader) xs_b = data_dict["xs"] ys_b = data_dict["ys"] Rs_b = data_dict["Rs"].reshape(-1, 9) ts_b = data_dict["ts"].reshape(-1, 3) T1s_b = data_dict["T1s"] T2s_b = data_dict["T2s"] K1s_b = data_dict["K1s"] K2s_b = data_dict["K2s"] else: ind_cur = np.random.choice(len(xs_tr), batch_size, replace=False) # Use minimum kp in batch to construct the batch numkps = np.array([xs_tr[_i].shape[1] for _i in ind_cur]) cur_num_kp = numkps.min() # Actual construction of the batch xs_b = np.array([ xs_tr[_i][:, :cur_num_kp, :] for _i in ind_cur ]).reshape(batch_size, 1, cur_num_kp, 4) ys_b = np.array([ys_tr[_i][:cur_num_kp, :] for _i in ind_cur ]).reshape(batch_size, cur_num_kp, 2) Rs_b = np.array([Rs_tr[_i] for _i in ind_cur]).reshape(batch_size, 9) ts_b = np.array([ts_tr[_i] for _i in ind_cur]).reshape(batch_size, 3) if self.config.use_fundamental > 0: T1s_b = np.array([T1s_tr[_i] for _i in ind_cur]) T2s_b = np.array([T2s_tr[_i] for _i in ind_cur]) K1s_b = np.array([K1s_tr[_i] for _i in ind_cur]) K2s_b = np.array([K2s_tr[_i] for _i in ind_cur]) # ---------------------------------------- # Train # Feed Dict feed_dict = { self.x_in: xs_b, self.y_in: ys_b, self.R_in: Rs_b, self.t_in: ts_b, self.is_training: True, } # add use_fundamental if self.config.use_fundamental > 0: feed_dict[self.T1_in] = T1s_b feed_dict[self.T2_in] = T2s_b feed_dict[self.K1_in] = K1s_b feed_dict[self.K2_in] = K2s_b # Fetch fetch = { "optim": self.optim, "loss": self.loss, "precision": self.precision, "recall": self.recall, } # Check if we want to write summary and check validation b_write_summary = ((step + 1) % self.config.report_intv) == 0 b_validate = ((step + 1) % self.config.val_intv) == 0 if b_write_summary or b_validate: fetch["summary"] = self.summary_op fetch["global_step"] = self.global_step # Run optimization # res = self.sess.run(fetch, feed_dict=feed_dict) try: res = self.sess.run(fetch, feed_dict=feed_dict) except (ValueError, tf.errors.InvalidArgumentError): print("Backward pass had numerical errors. " "This training batch is skipped!") continue # Write summary and save current model if b_write_summary: self.summary_tr.add_summary(res["summary"], global_step=res["global_step"]) self.saver_cur.save(self.sess, self.save_file_cur, global_step=self.global_step, write_meta_graph=False) # ---------------------------------------- # Validation if b_validate: va_res = 0 cur_global_step = res["global_step"] score = self.last_logit # defaul score: local attention if self.config.weight_opt == "sigmoid_softmax": score = [ self.last_logit, self.logit_softmax, self.last_weights ] test_process_ins = [ self.x_in, self.y_in, self.R_in, self.t_in, self.is_training ] if self.config.use_fundamental > 0: test_process_ins += [ self.T1_in, self.T2_in, self.K1_in, self.K2_in ] va_res, va_res_ours_ransac = test_process( "valid", self.sess, cur_global_step, self.summary_op, self.summary_va, test_process_ins, None, None, None, self.logits, self.e_hat, self.loss, self.precision, self.recall, self.last_e_hat, score, self.last_x_in, data["valid"], self.res_dir_va, self.config, True) # Higher the better if va_res > best_va_res: print("Saving best model with va_res = {}".format(va_res)) best_va_res = va_res # Save best validation result with open(self.va_res_file, "w") as ofp: ofp.write("{:e}\n".format(best_va_res)) # Save best model self.saver_best.save( self.sess, self.save_file_best, write_meta_graph=False, ) if va_res_ours_ransac > best_va_res_ours_ransac: print("Saving best model with va_res_ours_ransac = {}". format(va_res_ours_ransac)) best_va_res_ours_ransac = va_res_ours_ransac # Save best validation result with open(self.va_res_file_ours_ransac, "w") as ofp: ofp.write("{:e}\n".format(best_va_res_ours_ransac)) # Save best model self.saver_best.save( self.sess, self.save_file_best_ours_ransac, write_meta_graph=False, )