def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = args.revision
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [
path for path in [exp.path, exp.eval_dir] if os.path.exists(path)
]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step('rm-exp-dir', shutil.rmtree, exp.path)
exp.add_step('rm-eval-dir', shutil.rmtree, exp.eval_dir)
exp.run_steps()
def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = cached_revision.get_global_rev(REPO,
vcs=cached_revision.MERCURIAL,
rev=args.revision)
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
def result_handler(success):
regression_test_handler(args.test, rev, success)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS,
result_handler),
name='regression-check')
exp.run_steps()
# Showcase some fetcher options.
def eval_dir(num):
return os.path.join(exp.eval_dir, "test%d" % num)
exp.add_fetcher(dest=eval_dir(1),
name="fetcher-test1",
filter=only_two_algorithms)
exp.add_fetcher(dest=eval_dir(2),
name="fetcher-test2",
filter_algorithm="lama11")
# Add report steps.
exp.add_report(AbsoluteReport(attributes=["coverage", "cost"]),
name="report-abs-d")
quality_filters = QualityFilters()
exp.add_report(
AbsoluteReport(
attributes=[
"coverage",
"cost",
Attribute("quality", function=reports.arithmetic_mean),
],
filter=[quality_filters.store_costs, quality_filters.add_quality],
),
name="report-abs-builtin-filters",
)
exp.add_report(
AbsoluteReport(attributes=["coverage"], filter=only_two_algorithms),
name="report-abs-p-filter",
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_algorithm("blind", REPO, REV, ["--search", "astar(blind())"])
exp.add_algorithm("lmcut", REPO, REV, ["--search", "astar(lmcut())"])
# Add step that writes experiment files to disk.
exp.add_step("build", exp.build)
# Add step that executes all runs.
exp.add_step("start", exp.start_runs)
# Add step that collects properties from run directories and
# writes them to *-eval/properties.
exp.add_fetcher(name="fetch")
# Add report step (AbsoluteReport is the standard report).
exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile="report.html")
# Add scatter plot report step.
exp.add_report(
ScatterPlotReport(attributes=["expansions"], filter_algorithm=["blind", "lmcut"]),
outfile="scatterplot.png",
)
# Parse the commandline and show or run experiment steps.
exp.run_steps()
# Showcase some fetcher options.
def eval_dir(num):
return os.path.join(exp.eval_dir, 'test%d' % num)
exp.add_fetcher(dest=eval_dir(1),
name='fetcher-test1',
filter=only_two_algorithms)
exp.add_fetcher(dest=eval_dir(2),
name='fetcher-test2',
filter_algorithm='lama11')
# Add report steps.
exp.add_report(AbsoluteReport(attributes=['coverage', 'cost']),
name='report-abs-d')
quality_filters = QualityFilters()
exp.add_report(AbsoluteReport(
attributes=['coverage', 'cost', 'quality'],
filter=[quality_filters.store_costs, quality_filters.add_quality]),
name='report-abs-builtin-filters')
exp.add_report(AbsoluteReport(attributes=['coverage'],
filter=only_two_algorithms),
name='report-abs-p-filter')
exp.add_report(AbsoluteReport(attributes=['coverage', 'error'], format='tex'),
outfile='report-abs-combined.tex')
exp.add_report(AbsoluteReport(attributes=['coverage', 'error'], format='html'),
outfile='report-abs-combined.html')
exp.add_report(FilterReport(),
outfile=os.path.join(exp.eval_dir, 'filter-eval', 'properties'))
# baseline 1: LAMA-2011 (executed with vanilla fast-downward)
exp.add_algorithm(
"lama",
VANILLAREPO,
"default", [],
build_options=["release64"],
driver_options=["--build", "release64", "--alias", "seq-sat-lama-2011"])
# baseline 2: A* with LM-Cut
exp.add_algorithm("astar_lmcut",
NEURALREPO,
"default", ["--search", "astar(lmcut())"],
build_options=["release64dynamic"],
driver_options=["--build", "release64dynamic"])
# baseline 3: lazy GBFS with FF-heuristic - not optimal (executed with vanilla fast-downward)
exp.add_algorithm(
"gbfs_ff",
VANILLAREPO,
"default", [
"--heuristic", "hff=ff(transform=adapt_costs(cost_type=1))",
"--search", "lazy_greedy([hff], preferred=[hff])"
],
build_options=["release64"],
driver_options=["--build", "release64"])
report = os.path.join(exp.eval_dir, 'report.html')
exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile=report)
exp.run_steps()
