if args.mode != 'local':
config_args + ns.parse({'monitoring.frequency_seconds': 0})
config_args = ns({k: v for k, v in config_args if v is not None})
log.debug("Config args: %s.", config_args)
# merging all configuration files
amlb.resources.from_configs(config, config_user, config_args)
try:
if args.mode == 'local':
bench = amlb.Benchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'docker':
bench = amlb.DockerBenchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'singularity':
bench = amlb.SingularityBenchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'aws':
bench = amlb.AWSBenchmark(args.framework, args.benchmark, args.constraint)
# bench = amlb.AWSBenchmark(args.framework, args.benchmark, args.constraint, region=args.region)
# elif args.mode == "aws-remote":
# bench = amlb.AWSRemoteBenchmark(args.framework, args.benchmark, args.constraint, region=args.region)
else:
raise ValueError("`mode` must be one of 'aws', 'docker', 'singularity' or 'local'.")
if args.setup == 'only':
log.warning("Setting up %s environment only for %s, no benchmark will be run.", args.mode, args.framework)
if not args.keep_scores and args.mode != 'local':
log.warning("`keep_scores` parameter is currently ignored in %s mode, scores are always saved in this mode.", args.mode)
bench.setup(amlb.SetupMode[args.setup])
if args.setup != 'only':
res = bench.run(args.task, args.fold)
log.debug("Config args: %s.", config_args)
# merging all configuration files
amlb.resources.from_configs(config, config_user, config_args)
try:
if args.mode == 'local':
bench = amlb.Benchmark(args.framework,
args.benchmark,
parallel_jobs=args.parallel)
elif args.mode == 'docker':
bench = amlb.DockerBenchmark(args.framework,
args.benchmark,
parallel_jobs=args.parallel)
elif args.mode == 'aws':
bench = amlb.AWSBenchmark(args.framework,
args.benchmark,
parallel_jobs=args.parallel)
# bench = amlb.AWSBenchmark(args.framework, args.benchmark, parallel_jobs=args.parallel, region=args.region)
# elif args.mode == "aws-remote":
# bench = amlb.AWSRemoteBenchmark(args.framework, args.benchmark, parallel_jobs=args.parallel, region=args.region)
else:
raise ValueError("`mode` must be one of 'aws', 'docker' or 'local'.")
if args.setup == 'only':
log.warning(
"Setting up %s environment only for %s, no benchmark will be run.",
args.mode, args.framework)
if not args.keep_scores and args.mode != 'local':
log.warning(
"`keep_scores` parameter is currently ignored in %s mode, scores are always saved in this mode.",