def main():
# setup arguments
parser = utils.ArgParser(description=__doc__)
arguments.add_multi_experiment_args(parser) # support multi experiment groups and search
arguments.add_show_args(parser) # options for the output table
arguments.add_path_args(parser, dataset_path=False) # source path for experiments
arguments.add_default_args(parser)
args = parser.parse_args()
utils.create_logger_without_file(utils.LOGGER_NAME, log_level=args.log_level, no_print=True)
# find experiments to show depending on arguments
exp_groups_names = utils.match_folder(args.log_dir, EXP_TYPE, args.exp_group, args.exp_list, args.search)
collector = collect_results_data(
EXP_TYPE, exp_groups_names, log_dir=args.log_dir, read_last_epoch=args.last, add_group=args.add_group)
# ---------- Define the custom retrieval metrics to show for these experiments ----------
# define the retrieval validation metrics to show
retrieval_metrics = {}
# retrieval validation metrics must be constructed as product of two lists
re_retrieval_at = re.compile(r"r[0-9]+")
for modality, shortcut in zip(CootMetersConst.RET_MODALITIES, CootMetersConst.RET_MODALITIES_SHORT):
# modality: retrieval from where to where
for metric in CootMetersConst.RET_METRICS:
# metric: retrieval@1, mean, ...
if metric == "r1":
# log r1 metric to the overview class
metric_class = "val_base"
else:
# log all other metrics to the detail class
metric_class = "val_ret"
decimals = 2
formatting = "%" if re_retrieval_at.match(metric) else "f"
key = f"{metric_class}/{modality}-{metric}"
print_group = CootPrintGroupConst.VID if "vid" in modality else CootPrintGroupConst.CLIP
retrieval_metrics[f"{shortcut}-{metric}"] = PrintMetric(key, formatting, decimals, print_group)
# define average of R@1 text->video, video->text to get a single metric. same for clip->sentence, sentence->clip
retrieval_metrics["vp-r1"] = PrintMetric("vp-r1", "%", 2, CootPrintGroupConst.RETRIEVAL)
retrieval_metrics["cs-r1"] = PrintMetric("cs-r1", "%", 2, CootPrintGroupConst.RETRIEVAL)
# calculate those R@1 averages for each model
for model_name, metrics in collector.items():
try:
metrics["vp-r1"] = (metrics[f"val_base/vid2par-r1"] + metrics[f"val_base/par2vid-r1"]) / 2
# only calculate average clip-sentence r1 if clips where evaluated
if f"val_base/cli2sen-r1" in metrics:
metrics["cs-r1"] = (metrics[f"val_base/cli2sen-r1"] + metrics[f"val_base/sen2cli-r1"]) / 2
except KeyError as e:
print(f"WARNING: {e} for {model_name}")
# ---------- Define which metrics to print ----------
default_metrics = []
default_fields = ["v2p-r1", "p2v-r1", "c2s-r1", "s2c-r1", "time (h)"]
output_results(collector, custom_metrics=retrieval_metrics, metrics=args.metrics, default_metrics=default_metrics,
fields=args.fields, default_fields=default_fields, mean=args.mean, mean_all=args.mean_all,
sort=args.sort, sort_asc=args.sort_asc,
compact=args.compact)
def main():
# setup arguments
parser = utils.ArgParser(description=__doc__)
arguments.add_multi_experiment_args(
parser) # support multi experiment groups and search
arguments.add_show_args(parser) # options for the output table
arguments.add_path_args(parser,
dataset_path=False) # source path for experiments
arguments.add_default_args(parser)
args = parser.parse_args()
utils.create_logger_without_file(utils.LOGGER_NAME,
log_level=args.log_level,
no_print=True)
# find experiments to show depending on arguments
exp_groups_names = utils.match_folder(args.log_dir, EXP_TYPE,
args.exp_group, args.exp_list,
args.search)
collector = collect_results_data(EXP_TYPE,
exp_groups_names,
log_dir=args.log_dir,
read_last_epoch=args.last,
add_group=args.add_group)
collector = update_performance_profile(collector)
# ---------- Define which metrics to print ----------
default_metrics = []
default_fields = ["bleu4", "meteo", "rougl", "cider", "re4"]
output_results(collector,
custom_metrics=TEXT_METRICS,
metrics=args.metrics,
default_metrics=default_metrics,
fields=args.fields,
default_fields=default_fields,
mean=args.mean,
mean_all=args.mean_all,
sort=args.sort,
sort_asc=args.sort_asc,
compact=args.compact)
def main():
# setup arguments
parser = utils.ArgParser(description=__doc__)
arguments.add_default_args(parser)
arguments.add_trainer_args(parser)
arguments.add_exp_identifier_args(parser)
arguments.add_dataset_test_arg(parser)
args = parser.parse_args()
# load experiment config
exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args(
args, EXP_TYPE)
config = load_yaml_config_file(config_file)
# update experiment config and dataset path given the script arguments
config = arguments.update_config_from_args(config, args)
dataset_path = arguments.update_path_from_args(args)
# create configuration object
cfg = MLPMNISTExperimentConfig(config)
if args.print_config:
print(cfg)
# set seed
if cfg.random_seed is not None:
print(f"Set seed to {cfg.random_seed}")
set_seed(
cfg.random_seed,
set_deterministic=False) # set deterministic via config if needed
# create datasets
train_set = MNIST(str(dataset_path),
train=True,
download=True,
transform=ToTensor())
val_set = MNIST(str(dataset_path),
train=False,
download=True,
transform=ToTensor())
# make datasets smaller if requested in config
if cfg.dataset_train.max_datapoints > -1:
train_set.data = train_set.data[:cfg.dataset_train.max_datapoints]
if cfg.dataset_val.max_datapoints > -1:
val_set.data = val_set.data[:cfg.dataset_val.max_datapoints]
# create dataloaders
train_loader = create_loader(train_set,
cfg.dataset_train,
batch_size=cfg.train.batch_size)
val_loader = create_loader(val_set,
cfg.dataset_val,
batch_size=cfg.val.batch_size)
# create model
model_mgr = MLPModelManager(cfg)
if args.test_dataset:
# run dataset test and exit
run_mlpmnist_dataset_test(train_set, train_loader)
return
# always load best epoch during validation
load_best = args.load_best or args.validate
# create trainer
trainer = MLPMNISTTrainer(cfg,
model_mgr,
exp_group,
exp_name,
args.run_name,
len(train_loader),
log_dir=args.log_dir,
log_level=args.log_level,
logger=None,
print_graph=args.print_graph,
reset=args.reset,
load_best=load_best,
load_epoch=args.load_epoch,
inference_only=args.validate)
if args.validate:
# run validation
trainer.validate_epoch(val_loader)
else:
# run training
trainer.train_model(train_loader, val_loader)
def main():
# ---------- Setup script arguments. ----------
parser = utils.ArgParser(description=__doc__)
arguments.add_default_args(parser) # logging level etc.
arguments.add_exp_identifier_args(
parser) # arguments to identify the experiment to run
arguments.add_trainer_args(parser,
dataset_path=False) # general trainer arguments
parser.add_argument("--preload",
action="store_true",
help="Preload everything.") # feature preloading
arguments_mart.add_mart_args(parser) # some more paths for mart
parser.add_argument("--load_model",
type=str,
default=None,
help="Load model from file.")
parser.add_argument("--print_model",
action="store_true",
help=f"Print model")
args = parser.parse_args()
# load repository config yaml file to dict
exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args(
args, EXP_TYPE)
config = load_yaml_config_file(config_file)
# update experiment config given the script arguments
config = arguments.update_config_from_args(config, args)
config = arguments_mart.update_mart_config_from_args(config, args)
# read experiment config dict
cfg = Config(config)
if args.print_config:
print(cfg)
# set seed
if cfg.random_seed is not None:
print(f"Set seed to {cfg.random_seed}")
set_seed(
cfg.random_seed,
set_deterministic=False) # set deterministic via config if needed
# create dataset
train_set, val_set, train_loader, val_loader = create_mart_datasets_and_loaders(
cfg, args.coot_feat_dir, args.annotations_dir, args.video_feature_dir)
for i, run_number in enumerate(
range(args.start_run, args.start_run + args.num_runs)):
run_name = f"{args.run_name}{run_number}"
# create model from config
model = create_mart_model(cfg,
len(train_set.word2idx),
cache_dir=args.cache_dir)
# print model for debug if requested
if args.print_model and i == 0:
print(model)
# always load best epoch during validation
load_best = args.load_best or args.validate
# create trainer
trainer = MartTrainer(cfg,
model,
exp_group,
exp_name,
run_name,
len(train_loader),
log_dir=args.log_dir,
log_level=args.log_level,
logger=None,
print_graph=args.print_graph,
reset=args.reset,
load_best=load_best,
load_epoch=args.load_epoch,
load_model=args.load_model,
inference_only=args.validate,
annotations_dir=args.annotations_dir)
if args.validate:
# run validation
if not trainer.load and not args.ignore_untrained:
raise ValueError(
"Validating an untrained model! No checkpoints were loaded. Add --ignore_untrained "
"to ignore this error.")
trainer.validate_epoch(val_loader)
else:
# run training
trainer.train_model(train_loader, val_loader)
# done with this round
trainer.close()
del model
del trainer
def main():
# ---------- Setup script arguments. ----------
parser = utils.ArgParser(description=__doc__)
arguments.add_default_args(parser) # logging level etc.
arguments.add_exp_identifier_args(
parser) # arguments to identify the experiment to run
arguments.add_trainer_args(parser) # general trainer arguments
arguments.add_dataset_test_arg(parser) # flag for dataset testing
arguments_coot.add_dataloader_args(parser) # feature preloading
parser.add_argument("--load_model",
type=str,
default=None,
help="Load model from file.")
parser.add_argument("--save_embeddings",
action="store_true",
help="Save generated COOT embeddings.")
args = parser.parse_args()
if args.save_embeddings:
assert args.validate, "Saving embeddings only works in validation with --validate"
# load repository config yaml file to dict
exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args(
args, EXP_TYPE)
config = load_yaml_config_file(config_file)
# update experiment config and dataset path given the script arguments
path_data = arguments.update_path_from_args(args)
config = arguments.update_config_from_args(config, args)
config = arguments_coot.update_coot_config_from_args(config, args)
# read experiment config dict
cfg = Config(config, is_train=not args.validate and not args.test_dataset)
if args.print_config:
print(cfg)
# set seed
if cfg.random_seed is not None:
print(f"Set seed to {cfg.random_seed}")
set_seed(
cfg.random_seed,
set_deterministic=False) # set deterministic via config if needed
# create dataset and dataloader
if (cfg.dataset_train.preload_vid_feat
or cfg.dataset_train.preload_text_feat
or cfg.dataset_val.preload_vid_feat
or cfg.dataset_val.preload_text_feat):
cmd = "ulimit -n 100000"
print(f"Run system command to avoid TooManyFiles error:\n{cmd}")
os.system(cmd)
################## CREATE DATASETS FROM PATH DATA ################
train_set, val_set, train_loader, val_loader = create_retrieval_datasets_and_loaders(
cfg, path_data)
if args.test_dataset:
# run dataset test and exit
run_retrieval_dataset_test(train_set, train_loader)
return
print("---------- Setup done!")
for run_number in range(1, args.num_runs + 1):
run_name = f"{args.run_name}{run_number}"
# create coot models
model_mgr = ModelManager(cfg)
# always load best epoch during validation
load_best = args.load_best or args.validate
# create trainer
trainer = Trainer(cfg,
model_mgr,
exp_group,
exp_name,
run_name,
len(train_loader),
log_dir=args.log_dir,
log_level=args.log_level,
logger=None,
print_graph=args.print_graph,
reset=args.reset,
load_best=load_best,
load_epoch=args.load_epoch,
load_model=args.load_model,
inference_only=args.validate)
if args.validate:
# run validation
trainer.validate_epoch(val_loader,
val_clips=cfg.val.val_clips,
save_embs=args.save_embeddings)
else:
# run training
train_loss = trainer.train_model(train_loader, val_loader)
# save train loss
ipdb.set_trace()
# done with this round
trainer.close()
del model_mgr
del trainer
def main():
# setup arguments
parser = utils.ArgParser(description=__doc__)
arguments.add_default_args(parser)
arguments.add_exp_identifier_args(parser)
arguments.add_trainer_args(parser)
arguments.add_dataset_test_arg(parser)
args = parser.parse_args()
# load repository config yaml file to dict
exp_group, exp_name, config_file = arguments.setup_experiment_identifier_from_args(
args, EXP_TYPE)
config = load_yaml_config_file(config_file)
# update experiment config and dataset path given the script arguments
config = arguments.update_config_from_args(config, args)
dataset_path = arguments.update_path_from_args(args)
# read experiment config dict
cfg = MLPMNISTExperimentConfig(config)
if args.print_config:
print(cfg)
# set seed
verb = "Set seed"
if cfg.random_seed is None:
cfg.random_seed = np.random.randint(0, 2**15, dtype=np.int32)
verb = "Randomly generated seed"
print(f"{verb} {cfg.random_seed} deterministic {cfg.cudnn_deterministic} "
f"benchmark {cfg.cudnn_benchmark}")
set_seed(cfg.random_seed,
cudnn_deterministic=cfg.cudnn_deterministic,
cudnn_benchmark=cfg.cudnn_benchmark)
# create datasets
train_set = MNIST(str(dataset_path),
train=True,
download=True,
transform=ToTensor())
val_set = MNIST(str(dataset_path),
train=False,
download=True,
transform=ToTensor())
# make datasets smaller if requested in config
if cfg.dataset_train.max_datapoints > -1:
train_set.data = train_set.data[:cfg.dataset_train.max_datapoints]
if cfg.dataset_val.max_datapoints > -1:
val_set.data = val_set.data[:cfg.dataset_val.max_datapoints]
# create dataloaders
train_loader = create_loader(train_set,
cfg.dataset_train,
batch_size=cfg.train.batch_size)
val_loader = create_loader(val_set,
cfg.dataset_val,
batch_size=cfg.val.batch_size)
if args.test_dataset:
# run dataset test and exit
run_mlpmnist_dataset_test(train_set, train_loader)
return
print("---------- Setup done!")
for run_number in range(1, args.num_runs + 1):
run_name = f"{args.run_name}{run_number}"
# create model
model_mgr = MLPModelManager(cfg)
# always load best epoch during validation
load_best = args.load_best or args.validate
# create trainer
trainer = MLPMNISTTrainer(cfg,
model_mgr,
exp_group,
exp_name,
run_name,
len(train_loader),
log_dir=args.log_dir,
log_level=args.log_level,
logger=None,
print_graph=args.print_graph,
reset=args.reset,
load_best=load_best,
load_epoch=args.load_epoch,
inference_only=args.validate)
if args.validate:
# run validation
trainer.validate_epoch(val_loader)
else:
# run training
trainer.train_model(train_loader, val_loader)
# done with this round
trainer.close()
del model_mgr
del trainer
