def main(params): # load configuration of pre-trained models exp_path = os.path.join("results", params["dataset"], params["model_type"], params["exp"]) config_path = os.path.join(exp_path, "config.yml") config = io_utils.load_yaml(config_path) params["config_path"] = config_path config = M.override_config_from_params(config, params) config["exp_path"] = exp_path cmf.create_save_dirs(config["misc"]) # create logger logger_path = os.path.join(config["exp_path"], "evaluation.log") logger = io_utils.get_logger("Evaluate", log_file_path=logger_path) """ Build data loader """ loader_config = io_utils.load_yaml(params["loader_config_path"]) dset, L = cmf.get_loader(dataset, ["test"], [loader_config], num_workers=params["num_workers"]) config = M.override_config_from_loader(config, dset["test"]) """ Build network """ ckpt_path = os.path.join( exp_path, "checkpoints", "checkpoint_epoch_{:03d}.pkl".format(params["epoch"])) net = cmf.factory_model(config, M, dset["test"], ckpt_path) """ Test networks """ cmf.test_inference(config, L["test"], net)
def main(params): # Obtain configuration path exp_path = os.path.join("results", params["dataset"], params["model_type"], params["experiment"]) config_path = os.path.join(exp_path, "config.yml") params["config_path"] = config_path # prepare model and dataset M, dataset, config = cmf.prepare_experiment(params) # evaluate on GT config["evaluation"]["use_gt"] = params["evaluate_on_gt"] # evaluate on Top 1000 proposals if params["evaluate_on_top1000"]: config["evaluation"]["use_gt"] = False config["evaluation"]["apply_nms"] = False if len(params["proposal"]) > 0: config["evaluation"]["precomputed_proposal_sequence"] = params[ "proposal"] # create logger epoch_logger = cmf.create_logger(config, "EPOCH", "test.log") """ Build data loader """ loader_config = io_utils.load_yaml(params["loader_config_path"]) if params["test_on_server"]: loader_config = loader_config["test_loader"] test_on = "Test_Server" else: loader_config = loader_config["val_loader"] test_on = "Test" dsets, L = cmf.get_loader(dataset, split=["test"], loader_configs=[loader_config], num_workers=params["num_workers"]) config = M.override_config_from_dataset(config, dsets["test"], mode="Test") config["model"]["resume"] = True tensorboard_path = config["misc"]["tensorboard_dir"] config["misc"]["tensorboard_dir"] = "" # config["misc"]["debug"] = params["debug_mode"] """ Evaluating networks """ e0 = params["start_epoch"] e1 = params["end_epoch"] es = params["epoch_stride"] io_utils.check_and_create_dir(tensorboard_path + "_test_s{}_e{}".format(e0, e1)) summary = PytorchSummary(tensorboard_path + "_test_s{}_e{}".format(e0, e1)) for epoch in range(e0, e1 + 1, es): """ Build network """ config["model"]["checkpoint_path"] = \ os.path.join(exp_path, "checkpoints", "epoch_{:03d}.pkl".format(epoch)) net, _ = cmf.factory_model(config, M, dsets["test"], None) net.set_tensorboard_summary(summary) cmf.test(config, L["test"], net, epoch, None, epoch_logger, on=test_on)
def main(params): # loading configuration and setting environment config = io_utils.load_yaml(params["config_path"]) config["debug_mode"] = params["debug_mode"] config["out"] = params["output_filename"] config["assignment_path"] = params["assignment_path"] config["save_logits"] = params["save_logits"] # ensemble networks ensemble(config)
def prepare_experiment(params): M = get_model(params["model_type"]) D = get_dataset(params["dataset"]) # loading configuration and setting environment config = io_utils.load_yaml(params["config_path"]) config = update_config_from_params(config, params) create_save_dirs(config["misc"]) return M, D, config
def main(): # get parameters from cmd params = _get_argument_params() global M, dataset M = cmf.get_model(params["model_type"]) dataset = cmf.get_dataset(params["dataset"]) # loading configuration and setting environment config = io_utils.load_yaml(params["config_path"]) config = M.override_config_from_params(config, params) cmf.create_save_dirs(config["misc"]) # create loggers global logger logger = cmf.create_logger(config) # train network train(config)
def main(params): config = io_utils.load_yaml(params["config"]) # prepare dataset D = cmf.get_dataset(params["dataset"]) dsets, L = cmf.get_loader(D, split=["test"], loader_configs=[config["test_loader"]], num_workers=params["num_workers"]) # Build network M = cmf.get_method(params["method"]) net = M(config, logger=None) net.load_checkpoint(params["checkpoint"], True) if config["model"]["use_gpu"]: net.gpu_mode() # Evaluating networks cmf.test(config, L["test"], net, -1, None, mode="Test")
def ensemble(config): """ Build data loader """ dset = dataset.DataSet(config["test_loader"]) L = data.DataLoader( \ dset, batch_size=config["test_loader"]["batch_size"], \ num_workers=config["num_workers"], \ shuffle=False, collate_fn=dataset.collate_fn) """ Load assignments if exists """ with_assignment = False if config["assignment_path"] != "None": with_assignment = True assignment_file = io_utils.load_hdf5(config["assignment_path"], verbose=False) assignments = assignment_file["assignments"][:] cnt_mapping = np.zeros((3,3)) """ Build network """ nets = [] net_configs = [] for i in range(len(config["checkpoint_paths"])): net_configs.append(io_utils.load_yaml(config["config_paths"][i])) net_configs[i] = M.override_config_from_loader(net_configs[i], dset) nets.append(M(net_configs[i])) nets[i].bring_loader_info(dset) apply_cc_after = utils.get_value_from_dict( net_configs[i]["model"], "apply_curriculum_learning_after", -1) # load checkpoint if exists nets[i].load_checkpoint(config["checkpoint_paths"][i]) start_epoch = int(utils.get_filename_from_path( config["checkpoint_paths"][i]).split("_")[-1]) # If checkpoint use curriculum learning if (apply_cc_after > 0) and (start_epoch >= apply_cc_after): nets[i].apply_curriculum_learning() # ship network to use gpu if config["use_gpu"]: for i in range(len(nets)): nets[i].gpu_mode() for i in range(len(nets)): nets[i].eval_mode() # initialize counters for different tau metrics = ["top1-avg", "top1-max", "oracle"] for i in range(len(nets)): modelname = "M{}".format(i) metrics.append(modelname) tau = [1.0, 1.2, 1.5, 2.0, 5.0, 10.0, 50.0, 100.0] counters = OrderedDict() for T in tau: tau_name = "tau-"+str(T) counters[tau_name] = OrderedDict() for mt in metrics: counters[tau_name][mt] = accumulator.Accumulator(mt) """ Run training network """ ii = 0 itoa = dset.get_itoa() predictions = [] for batch in tqdm(L): # Forward networks probs = 0 B = batch[0][0].size(0) if type(batch[0][-1]) == type(list()): gt = batch[0][-1][0] else: gt = batch[0][-1] correct = 0 probs = {} for T in tau: tau_name = "tau-"+str(T) probs[tau_name] = 0 prob_list = [] for i in range(len(nets)): outputs = nets[i].evaluate(batch) prob_list.append(outputs[1]) # m*[B,A] if config["save_logits"]: TODO = True for T in tau: tau_name = "tau-"+str(T) probs = [net_utils.get_data(F.softmax(logits/T, dim=1)) \ for logits in prob_list] # m*[B,A] # count correct numbers for each model for i in range(len(nets)): val, idx = probs[i].max(dim=1) correct = torch.eq(idx, gt) num_correct = torch.sum(correct) modelname = "M{}".format(i) counters[tau_name][modelname].add(num_correct, B) # add prob of each model if i == 0: oracle_correct = correct else: oracle_correct = oracle_correct + correct # top1-max accuracy for ensemble ens_probs, ens_idx = torch.stack(probs,0).max(0) # [B,A] max_val, max_idx = ens_probs.max(dim=1) num_correct = torch.sum(torch.eq(max_idx, gt)) counters[tau_name]["top1-max"].add(num_correct, B) # top1-avg accuracy for ensemble ens_probs = sum(probs) # [B,A] max_val, max_idx = ens_probs.max(dim=1) num_correct = torch.sum(torch.eq(max_idx, gt)) counters[tau_name]["top1-avg"].add(num_correct, B) # oracle accuracy for ensemble num_oracle_correct = torch.sum(torch.ge(oracle_correct, 1)) counters[tau_name]["oracle"].add(num_oracle_correct, B) # attach predictions for i in range(len(batch[1])): qid = batch[1][i] predictions.append({ "question_id": qid, "answer": utils.label2string(itoa, max_idx[i]) }) # epoch done # print accuracy for cnt_k,cnt_v in counters.items(): txt = cnt_k + " " for k,v in cnt_v.items(): txt += ", {} = {:.5f}".format(v.get_name(), v.get_average()) print(txt) save_dir = os.path.join("results", "ensemble_predictions") io_utils.check_and_create_dir(save_dir) io_utils.write_json(os.path.join(save_dir, config["out"]+".json"), predictions)
help="Train the model in debug mode.") params = vars(parser.parse_args()) print(json.dumps(params, indent=4)) return params if __name__ == "__main__": # load parameters params = _get_argument_params() global M, dataset M = cmf.get_model(params["model_type"]) dataset = cmf.get_dataset(params["dataset"]) # loading configuration and setting environment config = io_utils.load_yaml(params["config_path"]) config = M.override_config_from_params(config, params) cmf.create_save_dirs(config["misc"]) """ Build data loader """ if params["mode"] == "train": dset = dataset.DataSet(config["train_loader"]) else: dset = dataset.DataSet(config["test_loader"]) L = data.DataLoader(dset, batch_size=64, \ num_workers=config["misc"]["num_workers"], \ shuffle=False, collate_fn=dataset.collate_fn) config = M.override_config_from_loader(config, dset) """ Build network """ net = M(config) net.bring_loader_info(dset)
def main(params): # load configuration of pre-trained models exp_path = os.path.join("results", params["dataset"], params["model_type"], params["exp"]) config_path = os.path.join(exp_path, "config.yml") config = io_utils.load_yaml(config_path) params["config_path"] = config_path config = M.override_config_from_params(config, params) config["exp_path"] = exp_path cmf.create_save_dirs(config["misc"]) # create logger logger_path = os.path.join(config["exp_path"], "evaluation.log") logger = io_utils.get_logger("Evaluate", log_file_path=logger_path) """ Build data loader """ loader_config = io_utils.load_yaml(params["loader_config_path"]) dset = dataset.DataSet(loader_config) L = data.DataLoader(dset, batch_size=loader_config["batch_size"], \ num_workers=params["num_workers"], \ shuffle=False, collate_fn=dataset.collate_fn) config = M.override_config_from_loader(config, dset) if params["mode"] == "eval": """ Evaluating networks """ e0 = params["start_epoch"] e1 = params["end_epoch"] e_stride = params["epoch_stride"] sample_data = dset.get_samples(5) for epoch in range(e0, e1+1, e_stride): """ Build network """ net = M(config) net.bring_loader_info(dset) # ship network to use gpu if config["model"]["use_gpu"]: net.gpu_mode() # load checkpoint if not (net.classname == "ENSEMBLE" and config["model"]["version"] == "IE"): ckpt_path = os.path.join(exp_path, "checkpoints", "checkpoint_epoch_{:03d}.pkl".format(epoch)) assert os.path.exists(ckpt_path), \ "Checkpoint does not exists ({})".format(ckpt_path) net.load_checkpoint(ckpt_path) # If checkpoint is already applied with curriculum learning apply_cc_after = utils.get_value_from_dict( config["model"], "apply_curriculum_learning_after", -1) if (apply_cc_after > 0) and (epoch >= apply_cc_after): net.apply_curriculum_learning() cmf.evaluate(config, L, net, epoch-1, logger_name="eval", mode="Evaluation", verbose_every=100) elif params["mode"] == "selection": epoch = params["start_epoch"] """ Build network """ net = M(config) net.bring_loader_info(dset) # ship network to use gpu if config["model"]["use_gpu"]: net.gpu_mode() # load checkpoint ckpt_path = os.path.join(exp_path, "checkpoints", "checkpoint_epoch_{:03d}.pkl".format(epoch)) assert os.path.exists(ckpt_path), "Checkpoint does not exists ({})".format(ckpt_path) net.load_checkpoint(ckpt_path) apply_cc_after = utils.get_value_from_dict( config["model"], "apply_curriculum_learning_after", -1) # If checkpoint use curriculum learning if (apply_cc_after > 0) and (epoch >= apply_cc_after): net.apply_curriculum_learning() cmf.get_selection_values(config, L, net, epoch-1, logger_name="eval", mode="Evaluation", verbose_every=100)