def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = args.revision
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [
path for path in [exp.path, exp.eval_dir] if os.path.exists(path)
]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step('rm-exp-dir', shutil.rmtree, exp.path)
exp.add_step('rm-eval-dir', shutil.rmtree, exp.eval_dir)
exp.run_steps()
def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = cached_revision.get_global_rev(REPO,
vcs=cached_revision.MERCURIAL,
rev=args.revision)
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
def result_handler(success):
regression_test_handler(args.test, rev, success)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
merge=True,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS,
result_handler),
name='regression-check')
exp.run_steps()
return True
def add_quality(self, run):
run["quality"] = self._compute_quality(
run.get("cost"), self.tasks_to_costs[self._get_task(run)])
return run
exp = FastDownwardExperiment(environment=ENV, revision_cache=REV_CACHE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.ANYTIME_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, ["gripper:prob01.pddl", "miconic:s1-0.pddl"])
exp.add_algorithm(
"iter-hadd",
REPO,
REV,
[
"--heuristic",
"hadd=add()",
"--search",
"iterated([lazy_greedy([hadd]),lazy_wastar([hadd])],repeat_last=true)",
],
)
exp.add_algorithm(
"ipdb",
REPO,
REV,
REPO = os.environ["DOWNWARD_REPO"]
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
# If REVISION_CACHE is None, the default ./data/revision-cache is used.
REVISION_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE")
VCS = cached_revision.get_version_control_system(REPO)
REV = "default" if VCS == cached_revision.MERCURIAL else "main"
exp = FastDownwardExperiment(environment=ENV, revision_cache=REVISION_CACHE)
# Add built-in parsers to the experiment.
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_algorithm("blind", REPO, REV, ["--search", "astar(blind())"])
exp.add_algorithm("lmcut", REPO, REV, ["--search", "astar(lmcut())"])
# Add step that writes experiment files to disk.
exp.add_step("build", exp.build)
# Add step that executes all runs.
exp.add_step("start", exp.start_runs)
# Add step that collects properties from run directories and
# writes them to *-eval/properties.
exp.add_fetcher(name="fetch")
# Add report step (AbsoluteReport is the standard report).
exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile="report.html")
return True
def add_quality(self, run):
run["quality"] = self._compute_quality(
run.get("cost"), self.tasks_to_costs[self._get_task(run)])
return run
exp = FastDownwardExperiment(environment=ENV, revision_cache=REV_CACHE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.ANYTIME_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, ["gripper:prob01.pddl", "miconic:s1-0.pddl"])
exp.add_suite(DIR / ".." / "tests" / "benchmarks",
["blocks", "gripper:p01.sas"])
exp.add_algorithm(
"iter-hadd",
REPO,
REV,
[
"--heuristic",
"hadd=add()",
"--search",
"iterated([lazy_greedy([hadd]),lazy_wastar([hadd])],repeat_last=true)",
],
)
exp.add_algorithm(
"ipdb",
return True
def add_quality(self, run):
run['quality'] = self._compute_quality(
run.get('cost'), self.tasks_to_costs[self._get_task(run)])
return run
exp = FastDownwardExperiment(environment=ENV, revision_cache=REV_CACHE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.ANYTIME_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, ['gripper:prob01.pddl', 'miconic:s1-0.pddl'])
exp.add_algorithm('iter-hadd', REPO, REV, [
'--heuristic', 'hadd=add()', '--search',
'iterated([lazy_greedy([hadd]),lazy_wastar([hadd])],repeat_last=true)'
])
exp.add_algorithm('ipdb',
REPO,
REV, ["--search", "astar(ipdb())"],
driver_options=['--search-time-limit', 10])
exp.add_algorithm('ff', REPO, REV, ["--search", "eager_greedy([ff()])"])
exp.add_algorithm(
'lama11',
REPO,
REV, [],
driver_options=['--alias', 'seq-sat-lama-2011', '--plan-file', 'sas_plan'])
exp.add_algorithm('sat-fdss-1',
BENCHMARK = '/mnt/data_server/schaefer/asnetsfastdownward/benchmarks/evaluation_domains'
ENV = OracleGridEngineEnvironment(queue='all.q@@fai0x')
REVISION_CACHE = os.path.expanduser('~/lab/revision-cache')
SUITE = [
'turnandopen', 'tyreworld', 'sokoban', 'hanoi', 'floortile', 'blocksworld',
'elevator', 'parcprinter'
]
ATTRIBUTES = [
'unsolvable', 'memory', 'total_search_time', 'total_time', 'plan_length',
'cost', 'coverage', 'error'
]
exp = FastDownwardExperiment(environment=ENV, revision_cache=REVISION_CACHE)
exp.add_suite(BENCHMARK, SUITE)
# baseline planners:
# baseline 1: LAMA-2011 (executed with vanilla fast-downward)
exp.add_algorithm(
"lama",
VANILLAREPO,
"default", [],
build_options=["release64"],
driver_options=["--build", "release64", "--alias", "seq-sat-lama-2011"])
# baseline 2: A* with LM-Cut
exp.add_algorithm("astar_lmcut",
NEURALREPO,
"default", ["--search", "astar(lmcut())"],
build_options=["release64dynamic"],
