run_mode=args.mode,
parallel_jobs=args.parallel,
sid=sid,
) + ns.parse(extras)
if args.mode != 'local':
config_args + ns.parse({'monitoring.frequency_seconds': 0})
config_args = ns({k: v for k, v in config_args if v is not None})
log.debug("Config args: %s.", config_args)
# merging all configuration files
amlb.resources.from_configs(config, config_user, config_args)
try:
if args.mode == 'local':
bench = amlb.Benchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'docker':
bench = amlb.DockerBenchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'singularity':
bench = amlb.SingularityBenchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'aws':
bench = amlb.AWSBenchmark(args.framework, args.benchmark, args.constraint)
# bench = amlb.AWSBenchmark(args.framework, args.benchmark, args.constraint, region=args.region)
# elif args.mode == "aws-remote":
# bench = amlb.AWSRemoteBenchmark(args.framework, args.benchmark, args.constraint, region=args.region)
else:
raise ValueError("`mode` must be one of 'aws', 'docker', 'singularity' or 'local'.")
if args.setup == 'only':
log.warning("Setting up %s environment only for %s, no benchmark will be run.", args.mode, args.framework)
if not args.keep_scores and args.mode != 'local':
log.warning("`keep_scores` parameter is currently ignored in %s mode, scores are always saved in this mode.", args.mode)
) + ns.parse(extras)
if args.mode != 'local':
config_args + ns.parse({'monitoring.frequency_seconds': 0})
config_args = ns({k: v for k, v in config_args if v is not None})
log.debug("Config args: %s.", config_args)
# merging all configuration files
amlb.resources.from_configs(config, config_user, config_args)
try:
if args.mode == 'local':
bench = amlb.Benchmark(args.framework,
args.benchmark,
parallel_jobs=args.parallel)
elif args.mode == 'docker':
bench = amlb.DockerBenchmark(args.framework,
args.benchmark,
parallel_jobs=args.parallel)
elif args.mode == 'aws':
bench = amlb.AWSBenchmark(args.framework,
args.benchmark,
parallel_jobs=args.parallel)
# bench = amlb.AWSBenchmark(args.framework, args.benchmark, parallel_jobs=args.parallel, region=args.region)
# elif args.mode == "aws-remote":
# bench = amlb.AWSRemoteBenchmark(args.framework, args.benchmark, parallel_jobs=args.parallel, region=args.region)
else:
raise ValueError("`mode` must be one of 'aws', 'docker' or 'local'.")
if args.setup == 'only':
log.warning(
"Setting up %s environment only for %s, no benchmark will be run.",
args.mode, args.framework)