def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = args.revision
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [
path for path in [exp.path, exp.eval_dir] if os.path.exists(path)
]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step('rm-exp-dir', shutil.rmtree, exp.path)
exp.add_step('rm-eval-dir', shutil.rmtree, exp.eval_dir)
exp.run_steps()
def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = cached_revision.get_global_rev(REPO,
vcs=cached_revision.MERCURIAL,
rev=args.revision)
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
def result_handler(success):
regression_test_handler(args.test, rev, success)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS,
result_handler),
name='regression-check')
exp.run_steps()
for format in ["png", "tex"]:
exp.add_report(
ScatterPlotReport(
attributes=["cost"],
format=format,
filter=only_two_algorithms,
get_category=get_domain,
scale="linear",
matplotlib_options=matplotlib_options,
),
outfile=os.path.join("plots", "scatter-domain." + format),
)
exp.add_report(
ComparativeReport([("lama11", "iter-hadd")], attributes=["coverage"]),
name="report-compare",
outfile="compare.html",
)
exp.add_report(
TaskwiseReport(attributes=["cost", "coverage"], filter_algorithm=["ipdb"]),
name="report-taskwise",
outfile="taskwise.html",
)
exp.add_report(AbsoluteReport(), name="report-abs-p")
exp.add_step("finished", call, ["echo", "Experiment", "finished."])
if __name__ == "__main__":
exp.run_steps()
