CONFIGS = [('lmcut', ['--search', 'astar(lmcut())']) ] ATTRIBUTES = ['coverage', 'expansions','initial_h_value','cost','hstar_to_h','statistics','commualtive_hstar_to_h'] exp = DownwardExperiment(path=EXPPATH, repo=REPO, environment=ENV, limits={'search_time': 300}) exp.add_suite({'freecell'}) for nick, config in CONFIGS: exp.add_config(nick, config) # Make a report containing absolute numbers (this is the most common report). file_name_for_report = 'report_' + nick +'.html' report = os.path.join(exp.eval_dir, file_name_for_report) file_name_for_preprocess = os.path.join(exp.eval_dir, 'preprocess') exp.add_report(HstarToHRatioAndStatistics(nick,file_name_for_preprocess,attributes=ATTRIBUTES), outfile=report) # Plot sub_dir = 'plots_' + nick exp.add_step(Step('report-plot-cat', ProblemPlotReport(), exp.eval_dir, os.path.join(exp.eval_dir, sub_dir))) # "Publish" the results with "cat" for demonstration purposes. exp.add_step(Step('publish-report', subprocess.call, ['cat', report])) # Compress the experiment directory. exp.add_step(Step.zip_exp_dir(exp)) # Parse the commandline and show or run experiment steps. exp()
ENV = LocalEnvironment(processes=6) CONFIGS = [('lmcut', ['--search', 'astar(lmcut())'])] ATTRIBUTES = [ 'coverage', 'expansions', 'initial_h_value', 'cost', 'hstar_to_h' ] #All with timeout #exp = DownwardExperiment(path=EXPPATH, repo=REPO, environment=ENV, limits={'search_time': 100}) #exp.add_suite(suites.suite_all()) #Only lmcut domains without timeout exp = DownwardExperiment(path=EXPPATH, repo=REPO, environment=ENV) exp.add_suite(suites.suite_lmcut_domains()) for nick, config in CONFIGS: exp.add_config(nick, config) # Make a report containing absolute numbers with h*/h values. report = os.path.join(exp.eval_dir, 'report.html') exp.add_report(HstarToHRatio(attributes=ATTRIBUTES), outfile=report) # "Publish" the results with "cat" for demonstration purposes. exp.add_step(Step('publish-report', subprocess.call, ['cat', report])) # Compress the experiment directory. exp.add_step(Step.zip_exp_dir(exp)) # Parse the commandline and show or run experiment steps. exp()
# exp.add_suite({'blocks:probBLOCKS-8-0.pddl','blocks:probBLOCKS-8-1.pddl','blocks:probBLOCKS-4-0.pddl','blocks:probBLOCKS-5-1.pddl'}) #exp.add_suite({'gripper:prob02.pddl'}) # exp.add_suite({'zenotravel','trucks-strips','sokoban-sat11-strips','philosophers'}) for nick, config in CONFIGS: exp.add_config(nick, config) # Make a report containing absolute numbers (this is the most common report). file_name_for_report = 'report_' + nick + '.html' report = os.path.join(exp.eval_dir, file_name_for_report) file_name_for_preprocess = os.path.join(exp.eval_dir, 'preprocess') exp.add_report(HstarToHRatioAndStatistics(nick, file_name_for_preprocess, attributes=ATTRIBUTES), outfile=report) # Plot sub_dir = 'plots_' + nick exp.add_step( Step('report-plot-cat', ProblemPlotReport(), exp.eval_dir, os.path.join(exp.eval_dir, sub_dir))) # "Publish" the results with "cat" for demonstration purposes. # exp.add_step(Step('publish-report', subprocess.call, ['cat', report])) # Compress the experiment directory. # exp.add_step(Step.zip_exp_dir(exp)) # Parse the commandline and show or run experiment steps. exp()
environment=ENV, limits={'search_time': 300}) exp.add_suite({'airport', 'blocks', 'freecell'}) for nick, config in CONFIGS: exp.add_config(nick, config) # Make a report containing absolute numbers (this is the most common report). file_name_for_report = 'report_' + nick + '.html' report = os.path.join(exp.eval_dir, file_name_for_report) file_name_for_preprocess = os.path.join(exp.eval_dir, 'preprocess') exp.add_report(HstarToHRatioAndStatistics(nick, file_name_for_preprocess, attributes=ATTRIBUTES), outfile=report) # Plot sub_dir = 'plots_' + nick exp.add_step( Step('report-plot-cat', ProblemPlotReport(), exp.eval_dir, os.path.join(exp.eval_dir, sub_dir))) # "Publish" the results with "cat" for demonstration purposes. exp.add_step(Step('publish-report', subprocess.call, ['cat', report])) # Compress the experiment directory. exp.add_step(Step.zip_exp_dir(exp)) # Parse the commandline and show or run experiment steps. exp()
run['config'] = config return run def filter_and_transform(run): """Remove "WORK-" from the configs and only include certain configurations. This also demonstrates a nested filter (one that calls another filter).""" if not only_two_configs(run): return False return remove_work_tag(run) # Check that the various fetcher options work. def eval_dir(num): return os.path.join(exp.eval_dir, 'test%d' % num) exp.add_step(Step('fetcher-test1', Fetcher(), exp.path, eval_dir(1), copy_all=True)) exp.add_step(Step('fetcher-test2', Fetcher(), exp.path, eval_dir(2), copy_all=True, write_combined_props=True)) exp.add_step(Step('fetcher-test3', Fetcher(), exp.path, eval_dir(3), filter_config_nick='lama11')) exp.add_step(Step('fetcher-test4', Fetcher(), exp.path, eval_dir(4), parsers=os.path.join(DIR, 'simple', 'simple-parser.py'))) # Add report steps abs_domain_report_file = os.path.join(exp.eval_dir, '%s-abs-d.html' % EXPNAME) abs_problem_report_file = os.path.join(exp.eval_dir, '%s-abs-p.html' % EXPNAME) abs_combined_report_file = os.path.join(exp.eval_dir, '%s-abs-c.tex' % EXPNAME) exp.add_step(Step('report-abs-d', AbsoluteReport('domain', attributes=ATTRIBUTES + ['expansions', 'cost']), exp.eval_dir, abs_domain_report_file)) exp.add_step(Step('report-abs-p-filter', AbsoluteReport('problem', attributes=ATTRIBUTES, filter=filter_and_transform), exp.eval_dir, abs_problem_report_file)) exp.add_step(Step('report-abs-combined', AbsoluteReport(attributes=None, format='tex'),
# combos = [ # (Translator(REPO, rev='WORK'), Preprocessor(REPO, rev=3097), Planner(REPO)), # (Translator(REPO, rev='WORK'), Preprocessor(REPO, rev=3097), Planner(MYOTHER_REPO)), # ] exp = DownwardExperiment(EXPPATH, REPO, # combinations=combos, limits={'search_time': 60}) exp.add_suite(['gripper:prob01.pddl']) exp.add_suite('zenotravel:pfile2') exp.add_config('ff', ['--search', 'lazy(single(ff()))']) exp.add_config('add', ['--search', 'lazy(single(add()))']) exp.add_portfolio(os.path.join(REPO, 'src', 'search', 'downward-seq-sat-fdss-1.py')) exp.add_report(AbsoluteReport('problem'), name='make-report', outfile='report-abs-p.html') def solved(run): return run['coverage'] == 1 exp.add_step(Step('suite', SuiteReport(filter=solved), exp.eval_dir, os.path.join(exp.eval_dir, 'suite.py'))) exp.add_step(Step('scatter', ScatterPlotReport(filter_config_nick=['ff', 'add'], attributes='expansions', format='png'), exp.eval_dir, os.path.join(exp.eval_dir, 'scatter'))) exp()
exp.add_suite(['gripper:prob01.pddl']) exp.add_suite('zenotravel:pfile2') exp.add_config('ff', ['--search', 'lazy(single(ff()))']) exp.add_config('add', ['--search', 'lazy(single(add()))']) exp.add_portfolio( os.path.join(REPO, 'src', 'search', 'downward-seq-sat-fdss-1.py')) exp.add_report(AbsoluteReport('problem'), name='make-report', outfile='report-abs-p.html') def solved(run): return run['coverage'] == 1 exp.add_step( Step('suite', SuiteReport(filter=solved), exp.eval_dir, os.path.join(exp.eval_dir, 'suite.py'))) exp.add_step( Step( 'scatter', ScatterPlotReport(filter_config_nick=['ff', 'add'], attributes='expansions', format='png'), exp.eval_dir, os.path.join(exp.eval_dir, 'scatter'))) exp()
from lab.steps import Step from lab.environments import LocalEnvironment from downward.experiment import DownwardExperiment from downward.reports.absolute import AbsoluteReport EXPPATH = 'exp-lmcut' REPO = os.path.expanduser('~/downward') ENV = LocalEnvironment(processes=2) SUITE = ['airport'] CONFIGS = [('lmcut', ['--search', 'astar(lmcut())'])] ATTRIBUTES = ['coverage', 'expansions'] exp = DownwardExperiment(path=EXPPATH, repo=REPO, environment=ENV) exp.add_suite(SUITE) for nick, config in CONFIGS: exp.add_config(nick, config) # Make a report containing absolute numbers (this is the most common report). report = os.path.join(exp.eval_dir, 'report.html') exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile=report) # "Publish" the results with "cat" for demonstration purposes. exp.add_step(Step('publish-report', subprocess.call, ['cat', report])) # Compress the experiment directory. exp.add_step(Step.zip_exp_dir(exp)) # Parse the commandline and show or run experiment steps. exp()
def filter_and_transform(run): """Remove "WORK-" from the configs and only include certain configurations. This also demonstrates a nested filter (one that calls another filter).""" if not only_two_configs(run): return False return remove_work_tag(run) # Check that the various fetcher options work. def eval_dir(num): return os.path.join(exp.eval_dir, 'test%d' % num) exp.add_step( Step('fetcher-test1', Fetcher(), exp.path, eval_dir(1), copy_all=True)) exp.add_step( Step('fetcher-test2', Fetcher(), exp.path, eval_dir(2), copy_all=True, write_combined_props=True)) exp.add_step( Step('fetcher-test3', Fetcher(), exp.path, eval_dir(3), filter_config_nick='lama11')) exp.add_step( Step('fetcher-test4',