def make_comparison_tables():
for rev1, rev2 in itertools.combinations(self._revisions, 2):
compared_configs = []
for config in self._configs:
config_nick = config.nick
compared_configs.append(("%s-%s" % (rev1, config_nick),
"%s-%s" % (rev2, config_nick),
"Diff (%s)" % config_nick))
report = CompareConfigsReport(compared_configs, **kwargs)
outfile = os.path.join(
self.eval_dir, "%s-%s-%s-compare" %
(self.name, rev1, rev2) + "." + report.output_format)
report(self.eval_dir, outfile)
def make_comparison_tables():
for rev1, rev2 in itertools.combinations(self._revisions, 2):
compared_configs = []
for config in self._configs:
config_nick = config.nick
compared_configs.append(
("{rev1}-{config_nick}".format(**locals()),
"{rev2}-{config_nick}".format(**locals()),
"Diff ({config_nick})".format(**locals())))
report = CompareConfigsReport(compared_configs, **kwargs)
outfile = os.path.join(
self.eval_dir,
"{name}-{rev1}-{rev2}-compare.html".format(name=self.name,
**locals()))
report(self.eval_dir, outfile)
def add_custom_comparison_table_step(self, name, **kwargs):
"""Add a step that compares the configurations given in
*compared_configs*.
*compared_configs* must be specified. See CompareConfigsReport class.
*name* is a custom name for the report.
All *kwargs* will be passed to the CompareConfigsReport class.
If the keyword argument *attributes* is not specified, a
default list of attributes is used. ::
exp.add_comparison_table_step(attributes=["coverage"])
"""
kwargs.setdefault("attributes", self.DEFAULT_TABLE_ATTRIBUTES)
report = CompareConfigsReport(**kwargs)
outfile = os.path.join(self.eval_dir,
name + "." + report.output_format)
self.add_report(report, outfile=outfile)
self.add_step(
Step('publish-custom-comparison-report', subprocess.call,
['publish', outfile]))
if config.startswith('issue571-base-'):
return True
return False
def filter_base2(run):
config = run['config']
if config.startswith('issue571-base-issue571-base-issue571-base-v2-'):
return True
return False
exp.add_fetcher('data/issue571-v1-eval', filter=filter_base1)
exp.add_fetcher('data/issue571-v2-eval', filter=filter_base2)
# TODO: the following does not work, presumably because issue571-base
# is not represented as issue571-base-issue571-base-issue571-base
#exp.add_comparison_table_step(attributes=attributes)
compared_configs = []
configs1 = ['issue571-base-%s' % conf for conf in CONFIGS]
configs2 = [
'issue571-base-issue571-base-issue571-base-v2-%s' % conf
for conf in CONFIGS
]
for index in range(len(configs1)):
compared_configs.append((configs1[index], configs2[index]))
exp.add_report(
CompareConfigsReport(attributes=attributes,
compared_configs=compared_configs))
exp()
exp.add_absolute_report_step()
exp.add_comparison_table_step()
def grouped_configs_to_compare(config_nicks):
grouped_configs = []
for config_nick in config_nicks:
col_names = ['%s-%s' % (r, config_nick) for r in REVISIONS]
grouped_configs.append(
(col_names[0], col_names[1], 'Diff - %s' % config_nick))
return grouped_configs
exp.add_report(CompareConfigsReport(
compared_configs=grouped_configs_to_compare(
configs.configs_optimal_core()),
attributes=common_setup.IssueExperiment.DEFAULT_TABLE_ATTRIBUTES,
),
outfile="issue462-opt-compare-core-configs.html")
def add_first_run_search_time(run):
if run.get("search_time_all", []):
run["first_run_search_time"] = run["search_time_all"][0]
return run
exp.add_report(CompareConfigsReport(
compared_configs=grouped_configs_to_compare(configs.configs_optimal_ipc()),
attributes=common_setup.IssueExperiment.DEFAULT_TABLE_ATTRIBUTES +
["first_run_search_time"],
import os
DATADIR = os.path.join(os.path.dirname(__file__), 'data')
exp = Experiment(get_data_dir())
exp.add_fetcher(os.path.join(DATADIR, 'e2013101802-pho-seq-constraints-eval'),
filter_config_nick="astar_pho_seq_no_onesafe")
exp.add_fetcher(os.path.join(DATADIR, 'issue527-v2-eval'),
filter_config_nick="astar_occ_seq")
exp.add_report(
CompareConfigsReport(
[
('869fec6f843b-astar_pho_seq_no_onesafe',
'issue527-v2-astar_occ_seq'),
],
attributes=[
'coverage',
'total_time',
'expansions',
'evaluations',
'generated',
'expansions_until_last_jump',
'error',
],
))
exp()
import common_setup
REPO = common_setup.get_repo_base()
REV_BASE = 'issue585-base'
REV_V1 = 'issue585-v2'
SUITE = suites.suite_optimal_with_ipc11()
ALGORITHMS = {
'astar_ipdb_base': (REV_BASE, ['--search', 'astar(ipdb())']),
'astar_ipdb_v2': (REV_V1, ['--search', 'astar(ipdb())']),
}
COMPARED_ALGORITHMS = [
('astar_ipdb_base', 'astar_ipdb_v2', 'Diff (ipdb)'),
]
exp = common_setup.IssueExperiment(
revisions=[],
configs={},
suite=SUITE,
)
for nick, (rev, cmd) in ALGORITHMS.items():
exp.add_algorithm(nick, REPO, rev, cmd)
exp.add_report(
CompareConfigsReport(
COMPARED_ALGORITHMS,
attributes=common_setup.IssueExperiment.DEFAULT_TABLE_ATTRIBUTES))
exp()
exp.add_step(
Step(
'report-relative-d',
RelativeReport('domain',
attributes=['expansions'],
filter_config=['WORK-lama11', 'WORK-iter-hadd'],
rel_change=0.1,
abs_change=20), exp.eval_dir,
os.path.join(exp.eval_dir, 'relative.html')))
exp.add_report(RelativeReport('problem',
attributes=['quality', 'coverage', 'expansions'],
filter_config_nick=['lama11', 'iter-hadd']),
name='report-relative-p',
outfile='relative.html')
exp.add_report(CompareConfigsReport(
[('WORK-lama11', 'WORK-iter-hadd')],
attributes=['quality', 'coverage', 'expansions']),
name='report-compare',
outfile='compare.html')
# Write suite of solved problems
suite_file = os.path.join(exp.eval_dir, '%s_solved_suite.py' % EXPNAME)
exp.add_step(
Step('report-suite', SuiteReport(filter=solved), exp.eval_dir, suite_file))
exp.add_report(AbsoluteReport('problem',
colored=True,
attributes=[
'coverage', 'search_time', 'cost', 'memory',
'error', 'cost_all', 'limit_search_time'
]),
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('cggl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('rl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('cggl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('rl-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('cggl-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta',
absolute=False,
min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue604-v4-eval')
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-v3-rl-b50k', 'issue604-v5-rl-b50k'),
('issue604-v3-cggl-b50k', 'issue604-v5-cggl-b50k'),
('issue604-v3-dfp-b50k', 'issue604-v5-dfp-b50k'),
('issue604-v3-rl-ginf', 'issue604-v5-rl-ginf'),
('issue604-v3-cggl-ginf', 'issue604-v5-cggl-ginf'),
('issue604-v3-dfp-ginf', 'issue604-v5-dfp-ginf'),
('issue604-v3-rl-f50k', 'issue604-v5-rl-f50k'),
('issue604-v3-cggl-f50k', 'issue604-v5-cggl-f50k'),
('issue604-v3-dfp-f50k', 'issue604-v5-dfp-f50k'),
],
attributes=attributes),
outfile=os.path.join(exp.eval_dir,
'issue604-v3-v5-comparison.html'))
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-v4-rl-b50k', 'issue604-v5-rl-b50k'),
('issue604-v4-cggl-b50k', 'issue604-v5-cggl-b50k'),
('issue604-v4-dfp-b50k', 'issue604-v5-dfp-b50k'),
('issue604-v4-rl-ginf', 'issue604-v5-rl-ginf'),
('issue604-v4-cggl-ginf', 'issue604-v5-cggl-ginf'),
('issue604-v4-dfp-ginf', 'issue604-v5-dfp-ginf'),
('issue604-v4-rl-f50k', 'issue604-v5-rl-f50k'),
('issue604-v4-cggl-f50k', 'issue604-v5-cggl-f50k'),
('issue604-v4-dfp-f50k', 'issue604-v5-dfp-f50k'),
],
attributes=attributes),
outfile=os.path.join(exp.eval_dir,
'issue604-v4-v5-comparison.html'))
exp()
def main(revisions=[]):
suite = suites.suite_optimal_with_ipc11()
configs = {}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta',
absolute=False,
min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue604-v1-eval',
filter_config=[
'issue604-base-rl-b50k',
'issue604-base-cggl-b50k',
'issue604-base-dfp-b50k',
'issue604-base-rl-ginf',
'issue604-base-cggl-ginf',
'issue604-base-dfp-ginf',
'issue604-base-rl-f50k',
'issue604-base-cggl-f50k',
'issue604-base-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-eval',
filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-rest-eval',
filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-base-rl-b50k', 'issue604-v7-rl-b50k'),
('issue604-base-cggl-b50k', 'issue604-v7-cggl-b50k'),
('issue604-base-dfp-b50k', 'issue604-v7-dfp-b50k'),
('issue604-base-rl-ginf', 'issue604-v7-rl-ginf'),
('issue604-base-cggl-ginf', 'issue604-v7-cggl-ginf'),
('issue604-base-dfp-ginf', 'issue604-v7-dfp-ginf'),
('issue604-base-rl-f50k', 'issue604-v7-rl-f50k'),
('issue604-base-cggl-f50k', 'issue604-v7-cggl-f50k'),
('issue604-base-dfp-f50k', 'issue604-v7-dfp-f50k'),
],
attributes=attributes),
outfile=os.path.join(exp.eval_dir,
'issue604-base-v7-comparison.html'))
exp()