# evaluation self.seq_eval_engine.train_eval_seq(self.valid_data, self.test_data, engine, epoch) def train(self): """ Train and test NARM Returns: None """ self.monitor = Monitor(log_dir=self.config["run_dir"], delay=1, gpu_id=self.gpu_id) train_loader = self.load_train_data self.engine = NARMEngine(self.config) self.narm_save_dir = os.path.join(self.config["model_save_dir"], self.config["save_name"]) self._train(self.engine, train_loader, self.narm_save_dir) self.config["run_time"] = self.monitor.stop() self.seq_eval_engine.test_eval_seq(self.test_data, self.engine) if __name__ == "__main__": args = parse_args() config = {} update_args(config, args) narm = NARM_train(config) narm.train() # narm.test() have already implemented in train()
def prepare_env(self): """Prepare running environment. * Load parameters from json files. * Initialize system folders, model name and the paths to be saved. * Initialize resource monitor. * Initialize random seed. * Initialize logging. """ # Load config file from json with open(self.args.config_file) as config_params: print(f"loading config file {self.args.config_file}") config = json.load(config_params) # Update configs based on the received args from the command line . update_args(config, self.args) # obtain abspath for the project config["system"]["root_dir"] = os.path.abspath( config["system"]["root_dir"]) # construct unique model run id, which consist of model name, config id and a timestamp timestamp_str = datetime.now().strftime("%Y%m%d_%H%M%S") random_str = "".join( [random.choice(string.ascii_lowercase) for n in range(6)]) config["system"]["model_run_id"] = (config["model"]["model"] + "_" + config["model"]["config_id"] + "_" + timestamp_str + "_" + random_str) # Initialize random seeds set_seed(config["system"]["seed"] if "seed" in config["system"] else 2020) # Initialize working folders self.initialize_folders(config) config["system"]["process_dir"] = os.path.join( config["system"]["root_dir"], config["system"]["process_dir"]) # Initialize log file config["system"]["log_file"] = os.path.join( config["system"]["root_dir"], config["system"]["log_dir"], config["system"]["model_run_id"], ) logger.init_std_logger(config["system"]["log_file"]) print("Python version:", sys.version) print("pytorch version:", torch.__version__) # File paths to be saved config["model"]["run_dir"] = os.path.join( config["system"]["root_dir"], config["system"]["run_dir"], config["system"]["model_run_id"], ) config["system"]["run_dir"] = config["model"]["run_dir"] print( "The intermediate running statuses will be reported in folder:", config["system"]["run_dir"], ) config["system"]["tune_dir"] = os.path.join( config["system"]["root_dir"], config["system"]["tune_dir"]) def get_user_temp_dir(): tempdir = os.path.join(config["system"]["root_dir"], "tmp") print(f"ray temp dir {tempdir}") return tempdir ray.utils.get_user_temp_dir = get_user_temp_dir # Model checkpoints paths to be saved config["system"]["model_save_dir"] = os.path.join( config["system"]["root_dir"], config["system"]["checkpoint_dir"], config["system"]["model_run_id"], ) ensureDir(config["system"]["model_save_dir"]) print("Model checkpoint will save in file:", config["system"]["model_save_dir"]) config["system"]["result_file"] = os.path.join( config["system"]["root_dir"], config["system"]["result_dir"], config["system"]["result_file"], ) print("Performance result will save in file:", config["system"]["result_file"]) print_dict_as_table(config["system"], "System configs") return config