def _get_warnings_text_and_table(self):
"""
Return a :py:class:`Table <lab.reports.Table>` containing one line for
each run where an unexplained error occured.
"""
if not self.ERROR_ATTRIBUTES:
logging.critical("The list of error attributes must not be empty.")
table = reports.Table(title="Unexplained errors")
table.set_column_order(self.ERROR_ATTRIBUTES)
wrote_to_slurm_err = any(
"output-to-slurm.err" in run.get("unexplained_errors", [])
for run in self.runs.values())
for run in self.runs.values():
error_message = tools.get_unexplained_errors_message(run)
if error_message:
logging.error(error_message)
run_dir = run["run_dir"]
for attr in self.ERROR_ATTRIBUTES:
value = run.get(attr, "?")
if attr == "unexplained_errors":
value = self._format_unexplained_errors(value)
# Use formatted value as-is.
table.cell_formatters[run_dir][
attr] = reports.CellFormatter()
table.add_cell(run_dir, attr, value)
errors = []
if wrote_to_slurm_err:
src_dir = self.eval_dir.rstrip("/")[:-len("-eval")]
slurm_err_file = src_dir + "-grid-steps/slurm.err"
try:
slurm_err_content = tools.get_slurm_err_content(src_dir)
except FileNotFoundError:
slurm_err_file = "*-grid-steps/slurm.err"
errors.append(
f"There was output to {slurm_err_file}, but the file was missing "
f"when this report was made.")
else:
slurm_err_content = tools.filter_slurm_err_content(
slurm_err_content)
errors.append(
f"There was output to {slurm_err_file}. Below is the output without"
f'"memory cg" errors:\n```\n{slurm_err_content}\n```')
logging.error(f"There was output to {slurm_err_file}.")
if table:
errors.append(str(table))
infai_1_nodes = {f"ase{i:02d}.cluster.bc2.ch" for i in range(1, 25)}
infai_2_nodes = {f"ase{i:02d}.cluster.bc2.ch" for i in range(31, 55)}
nodes = self._get_node_names()
if nodes & infai_1_nodes and nodes & infai_2_nodes:
errors.append(
"Report combines runs from infai_1 and infai_2 partitions.")
return "\n".join(errors)
def _get_empty_table(self, attribute=None, title=None, columns=None):
"""Return an empty table."""
if title is None:
assert attribute is not None
title = attribute
if self.output_format == 'tex':
title = title.capitalize().replace('_', ' ')
if columns is None:
columns = self.algorithms
if attribute is not None and self.attribute_is_numeric(attribute):
# Decide whether we want to highlight minima or maxima.
kwargs = {
'min_wins': attribute.min_wins,
'colored': self.colored and attribute.min_wins is not None,
'digits': attribute.digits,
}
else:
# Do not highlight anything.
kwargs = {}
table = reports.Table(title=title, **kwargs)
table.set_column_order(columns)
link = '#%s' % title
formatter = reports.CellFormatter(link=link)
table.cell_formatters[table.header_row][
table.header_column] = formatter
return table
def _get_suite_table(self, attribute):
assert self.attribute_is_numeric(attribute), attribute
table = self._get_empty_table(attribute)
self._add_summary_functions(table, attribute)
func_name, func = self._get_aggregation_function(attribute)
num_probs = 0
self._add_table_info(attribute, func_name, table)
domain_algo_values = {}
for domain in self.domains:
for algorithm in self.algorithms:
domain_algo_values[(domain, algorithm)] = []
for (domain, _), runs in self.problem_runs.items():
# If the attribute is absolute, no runs must have been filtered and
# no values must be missing.
if not attribute.absolute and (len(runs) < len(self.algorithms)
or any(
run.get(attribute) is None
for run in runs)):
continue
num_probs += 1
for run in runs:
value = run.get(attribute)
if value is not None:
domain_algo_values[(domain,
run["algorithm"])].append(value)
# If the attribute is absolute (e.g. coverage) we may have
# added problems for which not all algorithms have a value. Therefore, we
# can only print the number of instances (in brackets after the domain
# name) if that number is the same for all algorithms. If not all algorithms
# have values for the same number of problems, we write the full list of
# different problem numbers.
for domain in self.domains:
task_counts = [
str(len(domain_algo_values[(domain, algo)]))
for algo in self.algorithms
]
if len(set(task_counts)) == 1:
count = task_counts[0]
else:
count = ", ".join(task_counts)
link = None
if self.use_domain_links:
link = f"#{attribute}-{domain}"
formatter = reports.CellFormatter(link=link, count=count)
table.cell_formatters[domain][table.header_column] = formatter
for (domain, algo), values in domain_algo_values.items():
domain_value = func(values) if values else None
table.add_cell(domain, algo, domain_value)
table.num_values = num_probs
return table
def _get_suite_table(self, attribute):
assert self.attribute_is_numeric(attribute), attribute
table = self._get_empty_table(attribute)
self._add_summary_functions(table, attribute)
# The first group function is used for aggregation.
func_name, func = self._get_group_functions(attribute)[0]
num_probs = 0
self._add_table_info(attribute, func_name, table)
domain_config_values = defaultdict(list)
for domain, problems in self.domains.items():
for problem in problems:
runs = self.problem_runs[(domain, problem)]
if (not attribute.absolute and
any(run.get(attribute) is None for run in runs)):
continue
num_probs += 1
for run in runs:
value = run.get(attribute)
if value is not None:
domain_config_values[(domain, run['config'])].append(value)
# If the attribute is absolute (e.g. coverage) we may have
# added problems for which not all configs have a value. Therefore, we
# can only print the number of instances (in brackets after the domain
# name) if that number is the same for all configs. If not all configs
# have values for the same number of problems, we write the full list of
# different problem numbers.
num_values_lists = defaultdict(list)
for domain in self.domains:
for config in self.configs:
values = domain_config_values.get((domain, config), [])
num_values_lists[domain].append(str(len(values)))
for domain, num_values_list in num_values_lists.items():
if len(set(num_values_list)) == 1:
count = num_values_list[0]
else:
count = ','.join(num_values_list)
link = None
if self.resolution == 'combined':
link = '#%s-%s' % (attribute, domain)
formatter = reports.CellFormatter(link=link, count=count)
table.cell_formatters[domain][table.header_column] = formatter
for (domain, config), values in domain_config_values.items():
table.add_cell(domain, config, func(values))
table.num_values = num_probs
return table
def _get_suite_table(self, attribute):
assert self.attribute_is_numeric(attribute), attribute
table = self._get_empty_table(attribute)
self._add_summary_functions(table, attribute)
func_name, func = self._get_aggregation_function(attribute)
num_probs = 0
self._add_table_info(attribute, func_name, table)
domain_algo_values = defaultdict(list)
for (domain, _), runs in self.problem_runs.items():
if (not attribute.absolute
and any(run.get(attribute) is None for run in runs)):
continue
num_probs += 1
for run in runs:
value = run.get(attribute)
if value is not None:
domain_algo_values[(domain,
run['algorithm'])].append(value)
# If the attribute is absolute (e.g. coverage) we may have
# added problems for which not all algorithms have a value. Therefore, we
# can only print the number of instances (in brackets after the domain
# name) if that number is the same for all algorithms. If not all algorithms
# have values for the same number of problems, we write the full list of
# different problem numbers.
num_values_lists = defaultdict(list)
for domain in self.domains:
for algo in self.algorithms:
values = domain_algo_values.get((domain, algo), [])
num_values_lists[domain].append(str(len(values)))
for domain, num_values_list in num_values_lists.items():
if len(set(num_values_list)) == 1:
count = num_values_list[0]
else:
count = ','.join(num_values_list)
link = None
if self.use_domain_links:
link = '#{}-{}'.format(attribute, domain)
formatter = reports.CellFormatter(link=link, count=count)
table.cell_formatters[domain][table.header_column] = formatter
for (domain, algo), values in domain_algo_values.items():
table.add_cell(domain, algo, func(values))
table.num_values = num_probs
return table
def _get_empty_table(self, attribute=None, title=None, columns=None):
"""Return an empty table."""
if title is None:
assert attribute is not None
title = attribute
if columns is None:
columns = self.configs
if attribute is not None and self.attribute_is_numeric(attribute):
# Decide whether we want to highlight minima or maxima.
min_wins = attribute.min_wins
colored = self.colored and min_wins is not None
else:
# Do not highlight anything.
min_wins = None
colored = False
table = reports.Table(title=title, min_wins=min_wins, colored=colored)
table.set_column_order(columns)
if self.resolution == 'combined':
link = '#%s' % title
formatter = reports.CellFormatter(link=link)
table.cell_formatters[table.header_row][table.header_column] = formatter
return table
def _get_commulative_table(self, domain):
#init new table
title = 'Commulative'
columns = {'Percentage', 'h*/h(s)'}
min_wins = False
colored = True
table = reports.Table(title=title, min_wins=min_wins, colored=colored)
table.set_column_order(columns)
link = '#%s' % title
formatter = reports.CellFormatter(link=link)
table.cell_formatters[table.header_row][
table.header_column] = formatter
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Commulative_ratio.csv'
file = open(domain_file, "w")
#get relevant value from original table
ratio_attr = Attribute('hstar_to_h', min_wins=False, absolute=True)
ratio_table = AbsoluteReport._get_table(self, ratio_attr, domain)
#define arrays to work
ratios = [
0.75, 0.8, 0.85, 0.9, 0.95, 1, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3,
1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7, 1.75, 1.8, 1.85, 1.9,
1.95, 2, 2.05, 2.1, 2.15, 2.2, 2.25, 2.3, 2.35, 2.4, 2.45, 2.5,
2.55, 2.6, 2.65, 2.7, 2.75, 2.8, 2.85, 2.9, 2.95, 3.0, 3.05, 3.1,
3.15, 3.2, 3.25, 3.3, 3.35, 3.4, 3.45, 3.5, 3.55, 3.6, 3.65, 3.7,
3.75, 3.80, 3.85, 3.9, 3.95, 4.0, 4.05, 4.1, 4.15, 4.2, 2.25, 4.3,
4.35, 4.4, 4.45, 4.5
]
names = [
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
'za', 'zb', 'zc', 'zd', 'ze', 'zf', 'zg', 'zh', 'zi', 'zj', 'zk',
'zl', 'zm', 'zn', 'zo', 'aa', 'ab', 'ac', 'ad', 'ae', 'af', 'ag',
'ah', 'ai', 'aj', 'ak', 'al', 'am', 'an', 'ao', 'ap', 'aq', 'ar',
'as', 'at', 'au', 'av', 'aw', 'ax', 'ay', 'az', 'ba', 'bb', 'bc',
'bd', 'be', 'bf', 'bg', 'bh', 'bi'
]
counter = 0
#calculate number of solved problems
total_solved = 0
for row in ratio_table.row_names:
curr_val = ratio_table.get(row)
val = curr_val[self.nick]
if val > 0:
total_solved = total_solved + 1
#for each ratio (1,1.05...), find the number of problems with this ratio, calc percentage and add row
for ratio in ratios:
_sum = 0
for row in ratio_table.row_names:
curr_val = ratio_table.get(row)
val = curr_val[self.nick]
if val <= ratio and val > 0:
_sum = _sum + 1
if total_solved == 0:
_sum_percent = 0
else:
_sum_percent = _sum * 100 / total_solved
#add new row
row_to_add = {}
row_to_add['Percentage'] = _sum_percent
row_to_add['h*/h(s)'] = ratio
table.add_row(names[counter], row_to_add)
counter = counter + 1
#TODO - save only one ratio per percentage
toWrite = str(ratio) + ',' + str(_sum_percent) + '\n'
file.write(toWrite)
file.close()
self.create_commulative_h_star_table(domain)
return table
def get_markup(self):
sections = []
toc_lines = []
warnings = self._get_warnings_text_and_table()
if warnings:
toc_lines.append('- **['
'Unexplained Errors'
' #unexplained-errors]**')
sections.append(('unexplained-errors', warnings))
toc_lines.append('- **[Info #info]**')
sections.append(('info', self._get_general_info()))
# Index of summary section.
summary_index = len(sections)
# Build a table containing summary functions of all other tables.
# The actual section is added at position summary_index after creating
# all other tables.
summary = self._get_empty_table(title='Summary')
summary.colored = self.colored
toc_lines.append('- **[Summary #summary]**')
for attribute in self.attributes:
logging.info('Creating table(s) for %s' % attribute)
tables = []
if attribute == 'error':
seen_errors = set()
error_counter = defaultdict(int)
for run in self.runs.values():
error = run.get('error', 'attribute-error-missing')
seen_errors.add(error)
error_counter[(run["algorithm"], run["domain"],
error)] += 1
error_to_min_wins = {
outcome.msg: outcome.min_wins
for outcome in outcomes.OUTCOMES
}
for error in sorted(seen_errors):
# Txt2tags seems to only allow letters, "-" and "_" in anchors.
pseudo_attribute = 'error-' + error
table = self._get_empty_table(title=pseudo_attribute)
min_wins = error_to_min_wins.get(error, None)
table.min_wins = min_wins
table.colored = min_wins is not None
for domain in self.domains:
if self.use_domain_links:
table.cell_formatters[domain][
table.header_column] = (reports.CellFormatter(
link='#error-{domain}'.format(**locals())))
for algorithm in self.algorithms:
count = error_counter.get(
(algorithm, domain, error), 0)
table.add_cell(domain, algorithm, count)
table.add_summary_function('Sum', sum)
reports.extract_summary_rows(table,
summary,
link='#' + 'error-' +
pseudo_attribute)
tables.append((pseudo_attribute, table))
elif self.attribute_is_numeric(attribute):
domain_table = self._get_table(attribute)
tables.append(('', domain_table))
reports.extract_summary_rows(domain_table,
summary,
link='#' + attribute)
else:
tables.append(
('', 'Domain-wise reports only support numeric '
'attributes, but %s has type %s.' %
(attribute, self._all_attributes[attribute].__name__)))
for domain in sorted(self.domains.keys()):
tables.append((domain, self._get_table(attribute, domain)))
parts = []
toc_line = []
for (domain, table) in tables:
if domain:
assert table
toc_line.append(
"[''%(domain)s'' #%(attribute)s-%(domain)s]" %
locals())
parts.append('== %(domain)s ==[%(attribute)s-%(domain)s]\n'
'%(table)s\n' % locals())
else:
if table:
parts.append('%(table)s\n' % locals())
else:
parts.append('No task was found where all algorithms '
'have a value for "%s". Therefore no '
'domain-wise table can be generated.\n' %
attribute)
toc_lines.append("- **[''{}'' #{}]**".format(attribute, attribute))
toc_lines.append(' - ' + ' '.join(toc_line))
sections.append((attribute, '\n'.join(parts)))
# Add summary before main content. This is done after creating the main content
# because the summary table is extracted from all other tables.
sections.insert(summary_index, ('summary', summary))
toc = '\n'.join(toc_lines)
content = '\n'.join('= {} =[{}]\n\n{}'.format(attr, attr, section)
for (attr, section) in sections)
return '{}\n\n\n{}'.format(toc, content)
def get_markup(self):
sections = []
toc_lines = []
warnings = self._get_warnings_text_and_table()
if warnings:
toc_lines.append("- **["
"Unexplained Errors"
" #unexplained-errors]**")
sections.append(("unexplained-errors", warnings))
toc_lines.append("- **[Info #info]**")
sections.append(("info", self._get_general_info()))
# Index of summary section.
summary_index = len(sections)
# Build a table containing summary functions of all other tables.
# The actual section is added at position summary_index after creating
# all other tables.
summary = self._get_empty_table(title="Summary")
summary.colored = self.colored
toc_lines.append("- **[Summary #summary]**")
for attribute in self.attributes:
logging.info(f"Creating table(s) for {attribute}")
tables = []
if attribute == "error":
seen_errors = set()
error_counter = defaultdict(int)
for run in self.runs.values():
error = run.get("error", "attribute-error-missing")
seen_errors.add(error)
error_counter[(run["algorithm"], run["domain"],
error)] += 1
error_to_min_wins = {
outcome.msg: outcome.min_wins
for outcome in outcomes.OUTCOMES
}
for error in sorted(seen_errors):
# Txt2tags seems to only allow letters, "-" and "_" in anchors.
pseudo_attribute = "error-" + error
table = self._get_empty_table(title=pseudo_attribute)
min_wins = error_to_min_wins.get(error, None)
table.min_wins = min_wins
table.colored = min_wins is not None
for domain in self.domains:
if self.use_domain_links:
table.cell_formatters[domain][
table.header_column] = reports.CellFormatter(
link=f"#error-{domain}")
for algorithm in self.algorithms:
count = error_counter.get(
(algorithm, domain, error), 0)
table.add_cell(domain, algorithm, count)
table.add_summary_function("Sum", sum)
reports.extract_summary_rows(table,
summary,
link="#" + "error-" +
pseudo_attribute)
tables.append((pseudo_attribute, table))
elif self.attribute_is_numeric(attribute):
domain_table = self._get_suite_table(attribute)
tables.append(("", domain_table))
reports.extract_summary_rows(domain_table,
summary,
link="#" + attribute)
else:
tables.append((
"",
f"Domain-wise reports only support numeric "
f"attributes, but {attribute} has type "
f"{self._all_attributes[attribute].__name__}.",
))
for domain in sorted(self.domains.keys()):
tables.append(
(domain, self._get_domain_table(attribute, domain)))
parts = []
toc_line = []
for (domain, table) in tables:
if domain:
assert table
toc_line.append(f"[''{domain}'' #{attribute}-{domain}]")
parts.append(
f"== {domain} ==[{attribute}-{domain}]\n{table}\n")
else:
if table:
parts.append(f"{table}\n")
else:
parts.append(
f"No task was found where all algorithms "
f'have a value for "{attribute}". Therefore no '
f"domain-wise table can be generated.\n")
toc_lines.append(f"- **[''{attribute}'' #{attribute}]**")
toc_lines.append(" - " + " ".join(toc_line))
sections.append((attribute, "\n".join(parts)))
# Add summary before main content. This is done after creating the main content
# because the summary table is extracted from all other tables.
sections.insert(summary_index, ("summary", summary))
toc = "\n".join(toc_lines)
content = "\n".join(f"= {attr} =[{attr}]\n\n{section}"
for (attr, section) in sections)
return f"{toc}\n\n\n{content}"
