def main(): # setup arguments parser = utils.ArgParser(description=__doc__) arguments.add_multi_experiment_args(parser) # support multi experiment groups and search arguments.add_show_args(parser) # options for the output table arguments.add_path_args(parser, dataset_path=False) # source path for experiments arguments.add_default_args(parser) args = parser.parse_args() utils.create_logger_without_file(utils.LOGGER_NAME, log_level=args.log_level, no_print=True) # find experiments to show depending on arguments exp_groups_names = utils.match_folder(args.log_dir, EXP_TYPE, args.exp_group, args.exp_list, args.search) collector = collect_results_data( EXP_TYPE, exp_groups_names, log_dir=args.log_dir, read_last_epoch=args.last, add_group=args.add_group) # ---------- Define the custom retrieval metrics to show for these experiments ---------- # define the retrieval validation metrics to show retrieval_metrics = {} # retrieval validation metrics must be constructed as product of two lists re_retrieval_at = re.compile(r"r[0-9]+") for modality, shortcut in zip(CootMetersConst.RET_MODALITIES, CootMetersConst.RET_MODALITIES_SHORT): # modality: retrieval from where to where for metric in CootMetersConst.RET_METRICS: # metric: retrieval@1, mean, ... if metric == "r1": # log r1 metric to the overview class metric_class = "val_base" else: # log all other metrics to the detail class metric_class = "val_ret" decimals = 2 formatting = "%" if re_retrieval_at.match(metric) else "f" key = f"{metric_class}/{modality}-{metric}" print_group = CootPrintGroupConst.VID if "vid" in modality else CootPrintGroupConst.CLIP retrieval_metrics[f"{shortcut}-{metric}"] = PrintMetric(key, formatting, decimals, print_group) # define average of R@1 text->video, video->text to get a single metric. same for clip->sentence, sentence->clip retrieval_metrics["vp-r1"] = PrintMetric("vp-r1", "%", 2, CootPrintGroupConst.RETRIEVAL) retrieval_metrics["cs-r1"] = PrintMetric("cs-r1", "%", 2, CootPrintGroupConst.RETRIEVAL) # calculate those R@1 averages for each model for model_name, metrics in collector.items(): try: metrics["vp-r1"] = (metrics[f"val_base/vid2par-r1"] + metrics[f"val_base/par2vid-r1"]) / 2 # only calculate average clip-sentence r1 if clips where evaluated if f"val_base/cli2sen-r1" in metrics: metrics["cs-r1"] = (metrics[f"val_base/cli2sen-r1"] + metrics[f"val_base/sen2cli-r1"]) / 2 except KeyError as e: print(f"WARNING: {e} for {model_name}") # ---------- Define which metrics to print ---------- default_metrics = [] default_fields = ["v2p-r1", "p2v-r1", "c2s-r1", "s2c-r1", "time (h)"] output_results(collector, custom_metrics=retrieval_metrics, metrics=args.metrics, default_metrics=default_metrics, fields=args.fields, default_fields=default_fields, mean=args.mean, mean_all=args.mean_all, sort=args.sort, sort_asc=args.sort_asc, compact=args.compact)
def main(): # setup arguments parser = utils.ArgParser(description=__doc__) arguments.add_multi_experiment_args( parser) # support multi experiment groups and search arguments.add_show_args(parser) # options for the output table arguments.add_path_args(parser, dataset_path=False) # source path for experiments arguments.add_default_args(parser) args = parser.parse_args() utils.create_logger_without_file(utils.LOGGER_NAME, log_level=args.log_level, no_print=True) # find experiments to show depending on arguments exp_groups_names = utils.match_folder(args.log_dir, EXP_TYPE, args.exp_group, args.exp_list, args.search) collector = collect_results_data(EXP_TYPE, exp_groups_names, log_dir=args.log_dir, read_last_epoch=args.last, add_group=args.add_group) collector = update_performance_profile(collector) # ---------- Define which metrics to print ---------- default_metrics = [] default_fields = ["bleu4", "meteo", "rougl", "cider", "re4"] output_results(collector, custom_metrics=TEXT_METRICS, metrics=args.metrics, default_metrics=default_metrics, fields=args.fields, default_fields=default_fields, mean=args.mean, mean_all=args.mean_all, sort=args.sort, sort_asc=args.sort_asc, compact=args.compact)
def main(): # setup arguments parser = utils.ArgParser(description=__doc__) arguments.add_default_args(parser) arguments.add_trainer_args(parser) arguments.add_exp_identifier_args(parser) arguments.add_dataset_test_arg(parser) args = parser.parse_args() # load experiment config exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args( args, EXP_TYPE) config = load_yaml_config_file(config_file) # update experiment config and dataset path given the script arguments config = arguments.update_config_from_args(config, args) dataset_path = arguments.update_path_from_args(args) # create configuration object cfg = MLPMNISTExperimentConfig(config) if args.print_config: print(cfg) # set seed if cfg.random_seed is not None: print(f"Set seed to {cfg.random_seed}") set_seed( cfg.random_seed, set_deterministic=False) # set deterministic via config if needed # create datasets train_set = MNIST(str(dataset_path), train=True, download=True, transform=ToTensor()) val_set = MNIST(str(dataset_path), train=False, download=True, transform=ToTensor()) # make datasets smaller if requested in config if cfg.dataset_train.max_datapoints > -1: train_set.data = train_set.data[:cfg.dataset_train.max_datapoints] if cfg.dataset_val.max_datapoints > -1: val_set.data = val_set.data[:cfg.dataset_val.max_datapoints] # create dataloaders train_loader = create_loader(train_set, cfg.dataset_train, batch_size=cfg.train.batch_size) val_loader = create_loader(val_set, cfg.dataset_val, batch_size=cfg.val.batch_size) # create model model_mgr = MLPModelManager(cfg) if args.test_dataset: # run dataset test and exit run_mlpmnist_dataset_test(train_set, train_loader) return # always load best epoch during validation load_best = args.load_best or args.validate # create trainer trainer = MLPMNISTTrainer(cfg, model_mgr, exp_group, exp_name, args.run_name, len(train_loader), log_dir=args.log_dir, log_level=args.log_level, logger=None, print_graph=args.print_graph, reset=args.reset, load_best=load_best, load_epoch=args.load_epoch, inference_only=args.validate) if args.validate: # run validation trainer.validate_epoch(val_loader) else: # run training trainer.train_model(train_loader, val_loader)
def main(): # ---------- Setup script arguments. ---------- parser = utils.ArgParser(description=__doc__) arguments.add_default_args(parser) # logging level etc. arguments.add_exp_identifier_args( parser) # arguments to identify the experiment to run arguments.add_trainer_args(parser, dataset_path=False) # general trainer arguments parser.add_argument("--preload", action="store_true", help="Preload everything.") # feature preloading arguments_mart.add_mart_args(parser) # some more paths for mart parser.add_argument("--load_model", type=str, default=None, help="Load model from file.") parser.add_argument("--print_model", action="store_true", help=f"Print model") args = parser.parse_args() # load repository config yaml file to dict exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args( args, EXP_TYPE) config = load_yaml_config_file(config_file) # update experiment config given the script arguments config = arguments.update_config_from_args(config, args) config = arguments_mart.update_mart_config_from_args(config, args) # read experiment config dict cfg = Config(config) if args.print_config: print(cfg) # set seed if cfg.random_seed is not None: print(f"Set seed to {cfg.random_seed}") set_seed( cfg.random_seed, set_deterministic=False) # set deterministic via config if needed # create dataset train_set, val_set, train_loader, val_loader = create_mart_datasets_and_loaders( cfg, args.coot_feat_dir, args.annotations_dir, args.video_feature_dir) for i, run_number in enumerate( range(args.start_run, args.start_run + args.num_runs)): run_name = f"{args.run_name}{run_number}" # create model from config model = create_mart_model(cfg, len(train_set.word2idx), cache_dir=args.cache_dir) # print model for debug if requested if args.print_model and i == 0: print(model) # always load best epoch during validation load_best = args.load_best or args.validate # create trainer trainer = MartTrainer(cfg, model, exp_group, exp_name, run_name, len(train_loader), log_dir=args.log_dir, log_level=args.log_level, logger=None, print_graph=args.print_graph, reset=args.reset, load_best=load_best, load_epoch=args.load_epoch, load_model=args.load_model, inference_only=args.validate, annotations_dir=args.annotations_dir) if args.validate: # run validation if not trainer.load and not args.ignore_untrained: raise ValueError( "Validating an untrained model! No checkpoints were loaded. Add --ignore_untrained " "to ignore this error.") trainer.validate_epoch(val_loader) else: # run training trainer.train_model(train_loader, val_loader) # done with this round trainer.close() del model del trainer
def main(): # ---------- Setup script arguments. ---------- parser = utils.ArgParser(description=__doc__) arguments.add_default_args(parser) # logging level etc. arguments.add_exp_identifier_args( parser) # arguments to identify the experiment to run arguments.add_trainer_args(parser) # general trainer arguments arguments.add_dataset_test_arg(parser) # flag for dataset testing arguments_coot.add_dataloader_args(parser) # feature preloading parser.add_argument("--load_model", type=str, default=None, help="Load model from file.") parser.add_argument("--save_embeddings", action="store_true", help="Save generated COOT embeddings.") args = parser.parse_args() if args.save_embeddings: assert args.validate, "Saving embeddings only works in validation with --validate" # load repository config yaml file to dict exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args( args, EXP_TYPE) config = load_yaml_config_file(config_file) # update experiment config and dataset path given the script arguments path_data = arguments.update_path_from_args(args) config = arguments.update_config_from_args(config, args) config = arguments_coot.update_coot_config_from_args(config, args) # read experiment config dict cfg = Config(config, is_train=not args.validate and not args.test_dataset) if args.print_config: print(cfg) # set seed if cfg.random_seed is not None: print(f"Set seed to {cfg.random_seed}") set_seed( cfg.random_seed, set_deterministic=False) # set deterministic via config if needed # create dataset and dataloader if (cfg.dataset_train.preload_vid_feat or cfg.dataset_train.preload_text_feat or cfg.dataset_val.preload_vid_feat or cfg.dataset_val.preload_text_feat): cmd = "ulimit -n 100000" print(f"Run system command to avoid TooManyFiles error:\n{cmd}") os.system(cmd) ################## CREATE DATASETS FROM PATH DATA ################ train_set, val_set, train_loader, val_loader = create_retrieval_datasets_and_loaders( cfg, path_data) if args.test_dataset: # run dataset test and exit run_retrieval_dataset_test(train_set, train_loader) return print("---------- Setup done!") for run_number in range(1, args.num_runs + 1): run_name = f"{args.run_name}{run_number}" # create coot models model_mgr = ModelManager(cfg) # always load best epoch during validation load_best = args.load_best or args.validate # create trainer trainer = Trainer(cfg, model_mgr, exp_group, exp_name, run_name, len(train_loader), log_dir=args.log_dir, log_level=args.log_level, logger=None, print_graph=args.print_graph, reset=args.reset, load_best=load_best, load_epoch=args.load_epoch, load_model=args.load_model, inference_only=args.validate) if args.validate: # run validation trainer.validate_epoch(val_loader, val_clips=cfg.val.val_clips, save_embs=args.save_embeddings) else: # run training train_loss = trainer.train_model(train_loader, val_loader) # save train loss ipdb.set_trace() # done with this round trainer.close() del model_mgr del trainer
def main(): # setup arguments parser = utils.ArgParser(description=__doc__) arguments.add_default_args(parser) arguments.add_exp_identifier_args(parser) arguments.add_trainer_args(parser) arguments.add_dataset_test_arg(parser) args = parser.parse_args() # load repository config yaml file to dict exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args( args, EXP_TYPE) config = load_yaml_config_file(config_file) # update experiment config and dataset path given the script arguments config = arguments.update_config_from_args(config, args) dataset_path = arguments.update_path_from_args(args) # read experiment config dict cfg = MLPMNISTExperimentConfig(config) if args.print_config: print(cfg) # set seed verb = "Set seed" if cfg.random_seed is None: cfg.random_seed = np.random.randint(0, 2**15, dtype=np.int32) verb = "Randomly generated seed" print(f"{verb} {cfg.random_seed} deterministic {cfg.cudnn_deterministic} " f"benchmark {cfg.cudnn_benchmark}") set_seed(cfg.random_seed, cudnn_deterministic=cfg.cudnn_deterministic, cudnn_benchmark=cfg.cudnn_benchmark) # create datasets train_set = MNIST(str(dataset_path), train=True, download=True, transform=ToTensor()) val_set = MNIST(str(dataset_path), train=False, download=True, transform=ToTensor()) # make datasets smaller if requested in config if cfg.dataset_train.max_datapoints > -1: train_set.data = train_set.data[:cfg.dataset_train.max_datapoints] if cfg.dataset_val.max_datapoints > -1: val_set.data = val_set.data[:cfg.dataset_val.max_datapoints] # create dataloaders train_loader = create_loader(train_set, cfg.dataset_train, batch_size=cfg.train.batch_size) val_loader = create_loader(val_set, cfg.dataset_val, batch_size=cfg.val.batch_size) if args.test_dataset: # run dataset test and exit run_mlpmnist_dataset_test(train_set, train_loader) return print("---------- Setup done!") for run_number in range(1, args.num_runs + 1): run_name = f"{args.run_name}{run_number}" # create model model_mgr = MLPModelManager(cfg) # always load best epoch during validation load_best = args.load_best or args.validate # create trainer trainer = MLPMNISTTrainer(cfg, model_mgr, exp_group, exp_name, run_name, len(train_loader), log_dir=args.log_dir, log_level=args.log_level, logger=None, print_graph=args.print_graph, reset=args.reset, load_best=load_best, load_epoch=args.load_epoch, inference_only=args.validate) if args.validate: # run validation trainer.validate_epoch(val_loader) else: # run training trainer.train_model(train_loader, val_loader) # done with this round trainer.close() del model_mgr del trainer