def main(): args = parse_custom_args() if args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = args.revision name = rev exp = FastDownwardExperiment(path=get_exp_dir(name, args.test), revision_cache=REVISION_CACHE) exp.add_suite(BENCHMARKS_DIR, SUITES[args.test]) for config_nick, config in CONFIGS[args.test]: exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER) exp.add_parser(exp.PLANNER_PARSER) exp.add_step('build', exp.build) exp.add_step('start', exp.start_runs) exp.add_fetcher(name='fetch') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: dirty_paths = [ path for path in [exp.path, exp.eval_dir] if os.path.exists(path) ] if dirty_paths: logging.critical( 'The last run found a regression. Please inspect what ' 'went wrong and then delete the following directories ' 'manually: %s' % dirty_paths) exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval', dest=exp.eval_dir, merge=True, name='fetch-baseline-results') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') # We abort if there is a regression and keep the directories. exp.add_step('rm-exp-dir', shutil.rmtree, exp.path) exp.add_step('rm-eval-dir', shutil.rmtree, exp.eval_dir) exp.run_steps()
def main(): args = parse_custom_args() if args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = cached_revision.get_global_rev(REPO, vcs=cached_revision.MERCURIAL, rev=args.revision) name = rev exp = FastDownwardExperiment(path=get_exp_dir(name, args.test), revision_cache=REVISION_CACHE) exp.add_suite(BENCHMARKS_DIR, SUITES[args.test]) for config_nick, config in CONFIGS[args.test]: exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER) exp.add_parser(exp.PLANNER_PARSER) exp.add_step('build', exp.build) exp.add_step('start', exp.start_runs) exp.add_fetcher(name='fetch') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: def result_handler(success): regression_test_handler(args.test, rev, success) exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval', dest=exp.eval_dir, merge=True, name='fetch-baseline-results') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS, result_handler), name='regression-check') exp.run_steps()
for format in ["png", "tex"]: exp.add_report( ScatterPlotReport( attributes=["cost"], format=format, filter=only_two_algorithms, get_category=get_domain, scale="linear", matplotlib_options=matplotlib_options, ), outfile=os.path.join("plots", "scatter-domain." + format), ) exp.add_report( ComparativeReport([("lama11", "iter-hadd")], attributes=["coverage"]), name="report-compare", outfile="compare.html", ) exp.add_report( TaskwiseReport(attributes=["cost", "coverage"], filter_algorithm=["ipdb"]), name="report-taskwise", outfile="taskwise.html", ) exp.add_report(AbsoluteReport(), name="report-abs-p") exp.add_step("finished", call, ["echo", "Experiment", "finished."]) if __name__ == "__main__": exp.run_steps()