return run["algorithm"] in ["lama11", "iter-hadd"]
# Showcase some fetcher options.
def eval_dir(num):
return os.path.join(exp.eval_dir, f"test{num}")
exp.add_fetcher(dest=eval_dir(1), name="fetcher-test1", filter=only_two_algorithms)
exp.add_fetcher(dest=eval_dir(2), name="fetcher-test2", filter_algorithm="lama11")
# Add report steps.
exp.add_report(AbsoluteReport(attributes=["coverage", "cost"]), name="report-abs-d")
quality_filters = QualityFilters()
exp.add_report(
AbsoluteReport(
attributes=[
"coverage",
"cost",
Attribute("quality", function=reports.arithmetic_mean),
],
filter=[quality_filters.store_costs, quality_filters.add_quality],
),
name="report-abs-builtin-filters",
)
exp.add_report(
AbsoluteReport(attributes=["coverage"], filter=only_two_algorithms),
name="report-abs-p-filter",
PROPERTIES = {
"ff-gripper-prob01.pddl": {
"domain": "gripper",
"problem": "prob01.pddl",
"algorithm": "ff",
"coverage": 1,
"expansions": 1234,
},
"blind-gripper-prob01.pddl": {
"domain": "gripper",
"problem": "prob01.pddl",
"algorithm": "blind",
"coverage": 1,
"expansions": 6543,
},
}
def write_properties(eval_dir):
tools.makedirs(eval_dir)
with open(os.path.join(eval_dir, "properties"), "w") as f:
json.dump(PROPERTIES, f)
exp = Experiment()
exp.add_report(AbsoluteReport(attributes=["coverage", "expansions"]))
write_properties(exp.eval_dir)
exp.run_steps()
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_algorithm("blind", REPO, REV, ["--search", "astar(blind())"])
exp.add_algorithm("lmcut", REPO, REV, ["--search", "astar(lmcut())"])
# Add step that writes experiment files to disk.
exp.add_step("build", exp.build)
# Add step that executes all runs.
exp.add_step("start", exp.start_runs)
# Add step that collects properties from run directories and
# writes them to *-eval/properties.
exp.add_fetcher(name="fetch")
# Add report step (AbsoluteReport is the standard report).
exp.add_report(AbsoluteReport(attributes=ATTRIBUTES, format="html"),
outfile="report.html")
# Add scatter plot report step.
exp.add_report(
ScatterPlotReport(attributes=["expansions"],
filter_algorithm=["blind", "lmcut"]),
outfile="scatterplot.png",
)
# Parse the commandline and show or run experiment steps.
exp.run_steps()
def __init__(self,
path=None,
repo=None,
environment=None,
combinations=None,
limits=None,
attributes=None,
derived_properties=None,
priority=0,
queue=None,
processes=2,
email=None,
cache_dir=CACHE_DIR,
**kwargs):
if path is None:
path = os.path.splitext(os.path.basename(sys.argv[0]))[0]
expname = os.path.basename(path)
remote_exppath = os.path.join(REMOTE_EXPS, path)
local_exppath = os.path.join(LOCAL_EXPS, path)
if REMOTE:
exppath = remote_exppath
repo = repo or REMOTE_REPO
environment = environment or MaiaEnvironment(
priority=priority, queue=queue, email=email)
else:
exppath = local_exppath
repo = repo or LOCAL_REPO
environment = environment or LocalEnvironment(processes=processes)
DownwardExperiment.__init__(self,
path=exppath,
environment=environment,
repo=repo,
combinations=combinations,
limits=limits,
cache_dir=cache_dir,
**kwargs)
self.set_path_to_python(PYTHON)
if attributes is None:
attributes = ATTRIBUTES
# Add report steps
abs_report_file = os.path.join(self.eval_dir, '%s-abs.html' % expname)
self.add_report(AbsoluteReport(attributes=attributes,
colored=True,
derived_properties=derived_properties),
name='report-abs',
outfile=abs_report_file)
if REMOTE:
# Compress the experiment directory
self.add_step(Step.zip_exp_dir(self))
self.add_step(
Step('zip-eval-dir',
call, [
'tar', '-cjf', self.name + '-eval.tar.bz2',
self.name + '-eval'
],
cwd=os.path.dirname(self.path)))
self.add_step(Step.remove_exp_dir(self))
self.add_step(
Step('remove-eval-dir',
shutil.rmtree,
self.eval_dir,
ignore_errors=True))
if not REMOTE:
# Copy the results to local directory
self.add_step(
Step('scp-eval-dir', call, [
'scp', '-r',
'%s:%s-eval' % (SCP_LOGIN, remote_exppath),
'%s-eval' % local_exppath
]))
# Copy the results to local directory
self.add_step(
Step('scp-zipped-eval-dir', call, [
'scp', '-r',
'%s:%s-eval.tar.bz2' % (SCP_LOGIN, remote_exppath),
'%s-eval.tar.bz2' % local_exppath
]))
# Copy the zipped experiment directory to local directory
self.add_step(
Step('scp-exp-dir', call, [
'scp', '-r',
'%s:%s.tar.bz2' % (SCP_LOGIN, remote_exppath),
'%s.tar.bz2' % local_exppath
]))
# Unzip the experiment directory
self.add_step(Step.unzip_exp_dir(self))
self.add_step(
Step('unzip-eval-dir',
call, ['tar', '-xjf', self.name + '-eval.tar.bz2'],
cwd=os.path.dirname(self.path)))
return True
def filter_tasks_with_equal_values(self, run):
values = self._tasks_to_values[self._get_task(run)]
return len(set(values)) != 1
exp.add_fetcher(src='data/issue939-base-eval')
exp.add_fetcher(src='data/issue939-v1-eval', merge=True)
ATTRIBUTES = ["error", "run_dir", "translator_*", "translator_output_sas_hash"]
#exp.add_comparison_table_step(attributes=ATTRIBUTES)
same_value_filters = SameValueFilters("translator_output_sas_hash")
# exp.add_comparison_table_step(
# name="filtered",
# attributes=ATTRIBUTES,
# filter=[same_value_filters.store_values, same_value_filters.filter_tasks_with_equal_values])
exp.add_report(TranslatorDiffReport(
attributes=["domain", "problem", "algorithm", "run_dir"]),
outfile="different_output_sas.csv")
exp.add_report(AbsoluteReport(attributes=ATTRIBUTES))
exp.add_report(
ComparativeReport(
[('issue939-base-translate-only', 'issue939-v1-translate-only')],
attributes=ATTRIBUTES))
exp.run_steps()
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_algorithm("blind", REPO, REV, ["--search", "astar(blind())"])
exp.add_algorithm("lmcut", REPO, REV, ["--search", "astar(lmcut())"])
# Add step that writes experiment files to disk.
exp.add_step("build", exp.build)
# Add step that executes all runs.
exp.add_step("start", exp.start_runs)
# Add step that collects properties from run directories and
# writes them to *-eval/properties.
exp.add_fetcher(name="fetch")
# Add report step (AbsoluteReport is the standard report).
exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile="report.html")
# Add scatter plot report step.
exp.add_report(
ScatterPlotReport(attributes=["expansions"],
filter_algorithm=["blind", "lmcut"]),
outfile="scatterplot.png",
)
# Parse the commandline and show or run experiment steps.
exp.run_steps()
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_report(AbsoluteReport(attributes=attributes,filter_algorithm=[
'%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v5-clean',
]),outfile='issue668-v5-clean-abp.html')
exp.add_report(AbsoluteReport(attributes=attributes,filter_algorithm=[
'%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-l-rnd-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rnd-otn-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v5-clean',
'%s-sbf-miasm-rnd-rnd-pba-b50k' % 'issue668-v5-clean',
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_report(AbsoluteReport(
attributes=attributes,
filter_algorithm=[
'%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v2',
'%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v2',
]),
outfile='issue668-v2-abp.html')
exp.add_report(AbsoluteReport(
attributes=attributes,
filter_algorithm=[
'%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v2',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v2',
'%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v2',
'%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v2',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v2',