def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = args.revision
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [
path for path in [exp.path, exp.eval_dir] if os.path.exists(path)
]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step('rm-exp-dir', shutil.rmtree, exp.path)
exp.add_step('rm-eval-dir', shutil.rmtree, exp.eval_dir)
exp.run_steps()
def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = cached_revision.get_global_rev(REPO,
vcs=cached_revision.MERCURIAL,
rev=args.revision)
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
def result_handler(success):
regression_test_handler(args.test, rev, success)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS,
result_handler),
name='regression-check')
exp.run_steps()
def store_costs(self, run):
cost = run.get("cost")
if cost is not None:
assert run["coverage"]
self.tasks_to_costs[self._get_task(run)].append(cost)
return True
def add_quality(self, run):
run["quality"] = self._compute_quality(
run.get("cost"), self.tasks_to_costs[self._get_task(run)])
return run
exp = FastDownwardExperiment(environment=ENV, revision_cache=REV_CACHE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.ANYTIME_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, ["gripper:prob01.pddl", "miconic:s1-0.pddl"])
exp.add_algorithm(
"iter-hadd",
REPO,
REV,
[
"--heuristic",
"hadd=add()",
"--search",
"iterated([lazy_greedy([hadd]),lazy_wastar([hadd])],repeat_last=true)",
],