def main(): args = parse_custom_args() if args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = args.revision name = rev exp = FastDownwardExperiment(path=get_exp_dir(name, args.test), revision_cache=REVISION_CACHE) exp.add_suite(BENCHMARKS_DIR, SUITES[args.test]) for config_nick, config in CONFIGS[args.test]: exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER) exp.add_parser(exp.PLANNER_PARSER) exp.add_step('build', exp.build) exp.add_step('start', exp.start_runs) exp.add_fetcher(name='fetch') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: dirty_paths = [ path for path in [exp.path, exp.eval_dir] if os.path.exists(path) ] if dirty_paths: logging.critical( 'The last run found a regression. Please inspect what ' 'went wrong and then delete the following directories ' 'manually: %s' % dirty_paths) exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval', dest=exp.eval_dir, merge=True, name='fetch-baseline-results') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') # We abort if there is a regression and keep the directories. exp.add_step('rm-exp-dir', shutil.rmtree, exp.path) exp.add_step('rm-eval-dir', shutil.rmtree, exp.eval_dir) exp.run_steps()
def main(): args = parse_custom_args() if args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = cached_revision.get_global_rev(REPO, vcs=cached_revision.MERCURIAL, rev=args.revision) name = rev exp = FastDownwardExperiment(path=get_exp_dir(name, args.test), revision_cache=REVISION_CACHE) exp.add_suite(BENCHMARKS_DIR, SUITES[args.test]) for config_nick, config in CONFIGS[args.test]: exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER) exp.add_parser(exp.PLANNER_PARSER) exp.add_step('build', exp.build) exp.add_step('start', exp.start_runs) exp.add_fetcher(name='fetch') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: def result_handler(success): regression_test_handler(args.test, rev, success) exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval', dest=exp.eval_dir, merge=True, name='fetch-baseline-results') exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS, result_handler), name='regression-check') exp.run_steps()
# Showcase some fetcher options. def eval_dir(num): return os.path.join(exp.eval_dir, "test%d" % num) exp.add_fetcher(dest=eval_dir(1), name="fetcher-test1", filter=only_two_algorithms) exp.add_fetcher(dest=eval_dir(2), name="fetcher-test2", filter_algorithm="lama11") # Add report steps. exp.add_report(AbsoluteReport(attributes=["coverage", "cost"]), name="report-abs-d") quality_filters = QualityFilters() exp.add_report( AbsoluteReport( attributes=[ "coverage", "cost", Attribute("quality", function=reports.arithmetic_mean), ], filter=[quality_filters.store_costs, quality_filters.add_quality], ), name="report-abs-builtin-filters", ) exp.add_report( AbsoluteReport(attributes=["coverage"], filter=only_two_algorithms), name="report-abs-p-filter",
exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER) exp.add_parser(exp.PLANNER_PARSER) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_algorithm("blind", REPO, REV, ["--search", "astar(blind())"]) exp.add_algorithm("lmcut", REPO, REV, ["--search", "astar(lmcut())"]) # Add step that writes experiment files to disk. exp.add_step("build", exp.build) # Add step that executes all runs. exp.add_step("start", exp.start_runs) # Add step that collects properties from run directories and # writes them to *-eval/properties. exp.add_fetcher(name="fetch") # Add report step (AbsoluteReport is the standard report). exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile="report.html") # Add scatter plot report step. exp.add_report( ScatterPlotReport(attributes=["expansions"], filter_algorithm=["blind", "lmcut"]), outfile="scatterplot.png", ) # Parse the commandline and show or run experiment steps. exp.run_steps()
# Showcase some fetcher options. def eval_dir(num): return os.path.join(exp.eval_dir, 'test%d' % num) exp.add_fetcher(dest=eval_dir(1), name='fetcher-test1', filter=only_two_algorithms) exp.add_fetcher(dest=eval_dir(2), name='fetcher-test2', filter_algorithm='lama11') # Add report steps. exp.add_report(AbsoluteReport(attributes=['coverage', 'cost']), name='report-abs-d') quality_filters = QualityFilters() exp.add_report(AbsoluteReport( attributes=['coverage', 'cost', 'quality'], filter=[quality_filters.store_costs, quality_filters.add_quality]), name='report-abs-builtin-filters') exp.add_report(AbsoluteReport(attributes=['coverage'], filter=only_two_algorithms), name='report-abs-p-filter') exp.add_report(AbsoluteReport(attributes=['coverage', 'error'], format='tex'), outfile='report-abs-combined.tex') exp.add_report(AbsoluteReport(attributes=['coverage', 'error'], format='html'), outfile='report-abs-combined.html') exp.add_report(FilterReport(), outfile=os.path.join(exp.eval_dir, 'filter-eval', 'properties'))
# baseline 1: LAMA-2011 (executed with vanilla fast-downward) exp.add_algorithm( "lama", VANILLAREPO, "default", [], build_options=["release64"], driver_options=["--build", "release64", "--alias", "seq-sat-lama-2011"]) # baseline 2: A* with LM-Cut exp.add_algorithm("astar_lmcut", NEURALREPO, "default", ["--search", "astar(lmcut())"], build_options=["release64dynamic"], driver_options=["--build", "release64dynamic"]) # baseline 3: lazy GBFS with FF-heuristic - not optimal (executed with vanilla fast-downward) exp.add_algorithm( "gbfs_ff", VANILLAREPO, "default", [ "--heuristic", "hff=ff(transform=adapt_costs(cost_type=1))", "--search", "lazy_greedy([hff], preferred=[hff])" ], build_options=["release64"], driver_options=["--build", "release64"]) report = os.path.join(exp.eval_dir, 'report.html') exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile=report) exp.run_steps()