return run["algorithm"] in ["lama11", "iter-hadd"] # Showcase some fetcher options. def eval_dir(num): return os.path.join(exp.eval_dir, f"test{num}") exp.add_fetcher(dest=eval_dir(1), name="fetcher-test1", filter=only_two_algorithms) exp.add_fetcher(dest=eval_dir(2), name="fetcher-test2", filter_algorithm="lama11") # Add report steps. exp.add_report(AbsoluteReport(attributes=["coverage", "cost"]), name="report-abs-d") quality_filters = QualityFilters() exp.add_report( AbsoluteReport( attributes=[ "coverage", "cost", Attribute("quality", function=reports.arithmetic_mean), ], filter=[quality_filters.store_costs, quality_filters.add_quality], ), name="report-abs-builtin-filters", ) exp.add_report( AbsoluteReport(attributes=["coverage"], filter=only_two_algorithms), name="report-abs-p-filter",
PROPERTIES = { "ff-gripper-prob01.pddl": { "domain": "gripper", "problem": "prob01.pddl", "algorithm": "ff", "coverage": 1, "expansions": 1234, }, "blind-gripper-prob01.pddl": { "domain": "gripper", "problem": "prob01.pddl", "algorithm": "blind", "coverage": 1, "expansions": 6543, }, } def write_properties(eval_dir): tools.makedirs(eval_dir) with open(os.path.join(eval_dir, "properties"), "w") as f: json.dump(PROPERTIES, f) exp = Experiment() exp.add_report(AbsoluteReport(attributes=["coverage", "expansions"])) write_properties(exp.eval_dir) exp.run_steps()
exp.add_parser(exp.SINGLE_SEARCH_PARSER) exp.add_parser(exp.PLANNER_PARSER) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_algorithm("blind", REPO, REV, ["--search", "astar(blind())"]) exp.add_algorithm("lmcut", REPO, REV, ["--search", "astar(lmcut())"]) # Add step that writes experiment files to disk. exp.add_step("build", exp.build) # Add step that executes all runs. exp.add_step("start", exp.start_runs) # Add step that collects properties from run directories and # writes them to *-eval/properties. exp.add_fetcher(name="fetch") # Add report step (AbsoluteReport is the standard report). exp.add_report(AbsoluteReport(attributes=ATTRIBUTES, format="html"), outfile="report.html") # Add scatter plot report step. exp.add_report( ScatterPlotReport(attributes=["expansions"], filter_algorithm=["blind", "lmcut"]), outfile="scatterplot.png", ) # Parse the commandline and show or run experiment steps. exp.run_steps()
def __init__(self, path=None, repo=None, environment=None, combinations=None, limits=None, attributes=None, derived_properties=None, priority=0, queue=None, processes=2, email=None, cache_dir=CACHE_DIR, **kwargs): if path is None: path = os.path.splitext(os.path.basename(sys.argv[0]))[0] expname = os.path.basename(path) remote_exppath = os.path.join(REMOTE_EXPS, path) local_exppath = os.path.join(LOCAL_EXPS, path) if REMOTE: exppath = remote_exppath repo = repo or REMOTE_REPO environment = environment or MaiaEnvironment( priority=priority, queue=queue, email=email) else: exppath = local_exppath repo = repo or LOCAL_REPO environment = environment or LocalEnvironment(processes=processes) DownwardExperiment.__init__(self, path=exppath, environment=environment, repo=repo, combinations=combinations, limits=limits, cache_dir=cache_dir, **kwargs) self.set_path_to_python(PYTHON) if attributes is None: attributes = ATTRIBUTES # Add report steps abs_report_file = os.path.join(self.eval_dir, '%s-abs.html' % expname) self.add_report(AbsoluteReport(attributes=attributes, colored=True, derived_properties=derived_properties), name='report-abs', outfile=abs_report_file) if REMOTE: # Compress the experiment directory self.add_step(Step.zip_exp_dir(self)) self.add_step( Step('zip-eval-dir', call, [ 'tar', '-cjf', self.name + '-eval.tar.bz2', self.name + '-eval' ], cwd=os.path.dirname(self.path))) self.add_step(Step.remove_exp_dir(self)) self.add_step( Step('remove-eval-dir', shutil.rmtree, self.eval_dir, ignore_errors=True)) if not REMOTE: # Copy the results to local directory self.add_step( Step('scp-eval-dir', call, [ 'scp', '-r', '%s:%s-eval' % (SCP_LOGIN, remote_exppath), '%s-eval' % local_exppath ])) # Copy the results to local directory self.add_step( Step('scp-zipped-eval-dir', call, [ 'scp', '-r', '%s:%s-eval.tar.bz2' % (SCP_LOGIN, remote_exppath), '%s-eval.tar.bz2' % local_exppath ])) # Copy the zipped experiment directory to local directory self.add_step( Step('scp-exp-dir', call, [ 'scp', '-r', '%s:%s.tar.bz2' % (SCP_LOGIN, remote_exppath), '%s.tar.bz2' % local_exppath ])) # Unzip the experiment directory self.add_step(Step.unzip_exp_dir(self)) self.add_step( Step('unzip-eval-dir', call, ['tar', '-xjf', self.name + '-eval.tar.bz2'], cwd=os.path.dirname(self.path)))
return True def filter_tasks_with_equal_values(self, run): values = self._tasks_to_values[self._get_task(run)] return len(set(values)) != 1 exp.add_fetcher(src='data/issue939-base-eval') exp.add_fetcher(src='data/issue939-v1-eval', merge=True) ATTRIBUTES = ["error", "run_dir", "translator_*", "translator_output_sas_hash"] #exp.add_comparison_table_step(attributes=ATTRIBUTES) same_value_filters = SameValueFilters("translator_output_sas_hash") # exp.add_comparison_table_step( # name="filtered", # attributes=ATTRIBUTES, # filter=[same_value_filters.store_values, same_value_filters.filter_tasks_with_equal_values]) exp.add_report(TranslatorDiffReport( attributes=["domain", "problem", "algorithm", "run_dir"]), outfile="different_output_sas.csv") exp.add_report(AbsoluteReport(attributes=ATTRIBUTES)) exp.add_report( ComparativeReport( [('issue939-base-translate-only', 'issue939-v1-translate-only')], attributes=ATTRIBUTES)) exp.run_steps()
exp.add_parser(exp.TRANSLATOR_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER) exp.add_parser(exp.PLANNER_PARSER) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_algorithm("blind", REPO, REV, ["--search", "astar(blind())"]) exp.add_algorithm("lmcut", REPO, REV, ["--search", "astar(lmcut())"]) # Add step that writes experiment files to disk. exp.add_step("build", exp.build) # Add step that executes all runs. exp.add_step("start", exp.start_runs) # Add step that collects properties from run directories and # writes them to *-eval/properties. exp.add_fetcher(name="fetch") # Add report step (AbsoluteReport is the standard report). exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile="report.html") # Add scatter plot report step. exp.add_report( ScatterPlotReport(attributes=["expansions"], filter_algorithm=["blind", "lmcut"]), outfile="scatterplot.png", ) # Parse the commandline and show or run experiment steps. exp.run_steps()
ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_report(AbsoluteReport(attributes=attributes,filter_algorithm=[ '%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v5-clean', ]),outfile='issue668-v5-clean-abp.html') exp.add_report(AbsoluteReport(attributes=attributes,filter_algorithm=[ '%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-l-rnd-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rnd-otn-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v5-clean', '%s-sbf-miasm-rnd-rnd-pba-b50k' % 'issue668-v5-clean',
ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_report(AbsoluteReport( attributes=attributes, filter_algorithm=[ '%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v2', '%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v2', ]), outfile='issue668-v2-abp.html') exp.add_report(AbsoluteReport( attributes=attributes, filter_algorithm=[ '%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v2', '%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v2', '%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v2', '%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v2', '%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v2',