def get_markup(self):
if not len(self.configs) == 1:
logging.critical('The number of configs has to be one for taskwise reports.')
table = Table()
for (domain, problem), runs in self.problem_runs.items():
assert len(runs) == 1, len(runs)
run = runs[0]
for attr in self.attributes:
table.add_cell('%s:%s' % (domain, problem), attr, run.get(attr))
return str(table)
def get_markup(self):
if not len(self.configs) == 1:
logging.critical(
'The number of configs has to be one for taskwise reports.')
table = Table()
for (domain, problem), runs in self.problem_runs.items():
assert len(runs) == 1, len(runs)
run = runs[0]
for attr in self.attributes:
table.add_cell('%s:%s' % (domain, problem), attr,
run.get(attr))
return str(table)
def _get_warnings_text_and_table(self):
"""
Return a :py:class:`Table <lab.reports.Table>` containing one line for
each run where an unexplained error occured.
"""
if not self.ERROR_ATTRIBUTES:
logging.critical("The list of error attributes must not be empty.")
table = Table(title="Unexplained errors")
table.set_column_order(self.ERROR_ATTRIBUTES)
wrote_to_slurm_err = any(
"output-to-slurm.err" in run.get("unexplained_errors", [])
for run in self.runs.values()
)
for run in self.runs.values():
error_message = tools.get_unexplained_errors_message(run)
if error_message:
logging.error(error_message)
run_dir = run["run_dir"]
for attr in self.ERROR_ATTRIBUTES:
value = run.get(attr, "?")
if attr == "unexplained_errors":
value = self._format_unexplained_errors(value)
# Use formatted value as-is.
table.cell_formatters[run_dir][attr] = CellFormatter()
table.add_cell(run_dir, attr, value)
errors = []
if wrote_to_slurm_err:
src_dir = self.eval_dir.rstrip("/")[: -len("-eval")]
slurm_err_file = src_dir + "-grid-steps/slurm.err"
try:
slurm_err_content = tools.get_slurm_err_content(src_dir)
except OSError:
slurm_err_content = (
"The slurm.err file was missing while creating the report."
)
else:
slurm_err_content = tools.filter_slurm_err_content(slurm_err_content)
logging.error("There was output to {slurm_err_file}.".format(**locals()))
errors.append(
' Contents of {slurm_err_file} without "memory cg"'
" errors:\n```\n{slurm_err_content}\n```".format(**locals())
)
if table:
errors.append(str(table))
infai_1_nodes = {f"ase{i:02d}.cluster.bc2.ch" for i in range(1, 25)}
infai_2_nodes = {f"ase{i:02d}.cluster.bc2.ch" for i in range(31, 55)}
nodes = self._get_node_names()
if nodes & infai_1_nodes and nodes & infai_2_nodes:
errors.append("Report combines runs from infai_1 and infai_2 partitions.")
return "\n".join(errors)
def get_markup(self):
"""
Return `txt2tags <http://txt2tags.org/>`_ markup for the report.
"""
table = Table()
row_sort_module = RowSortModule(self._sort_spec)
table.dynamic_data_modules.append(row_sort_module)
for run_id, run in self.props.items():
row = {}
for key, value in run.items():
if key not in self.attributes:
continue
if isinstance(value, (list, tuple)):
key = '-'.join([str(item) for item in value])
row[key] = value
table.add_row(run_id, row)
return str(table)
def get_markup(self):
tables = []
for (domain, problem) in self.problems:
for config_nick in self.config_nicks:
runs = [self.runs[(domain, problem, rev + "-" + config_nick)]
for rev in self.revision_nicks]
if any(runs[0][self.regression_attribute] >
runs[i][self.regression_attribute]
for i in range(1, len(self.revision_nicks))):
print "\"%s:%s\"," % (domain, problem)
table = Table()
for rev, run in zip(self.revision_nicks, runs):
for attr in self.attributes:
table.add_cell(rev, attr, run.get(attr))
table_name = ":".join((domain, problem, config_nick))
tables.append((table_name, table))
return "\n".join(name + "\n" + str(table) for name, table in tables)
def get_markup(self):
tables = []
for (domain, problem) in self.problems:
for config_nick in self.config_nicks:
runs = [
self.runs[(domain, problem, rev + "-" + config_nick)]
for rev in self.revision_nicks
]
if any(runs[0][self.regression_attribute] > runs[i][
self.regression_attribute]
for i in range(1, len(self.revision_nicks))):
print "\"%s:%s\"," % (domain, problem)
table = Table()
for rev, run in zip(self.revision_nicks, runs):
for attr in self.attributes:
table.add_cell(rev, attr, run.get(attr))
table_name = ":".join((domain, problem, config_nick))
tables.append((table_name, table))
return "\n".join(name + "\n" + str(table) for name, table in tables)
def get_markup(self):
num_tasks_better = defaultdict(int)
num_tasks = 0
for (domain, problem), runs in sorted(self.problem_runs.items()):
if all(self.attribute in run for run in runs):
num_tasks += 1
else:
continue
for run1, run2 in itertools.combinations(runs, 2):
algo1 = run1["algorithm"]
algo2 = run2["algorithm"]
val1 = run1.get(self.attribute)
val2 = run2.get(self.attribute)
if val1 is not None and val2 is not None:
order = None
if self.attribute.min_wins:
if val1 < val2:
order = (algo1, algo2)
elif val1 > val2:
order = (algo2, algo1)
else:
assert not self.attribute.min_wins
if val1 > val2:
order = (algo1, algo2)
elif val1 < val2:
order = (algo2, algo1)
if order is not None:
num_tasks_better[order] += 1
def get_wins(algo1):
num_wins = 0
for algo2 in self.algorithms:
if algo1 == algo2:
continue
num_algo1_better = num_tasks_better[(algo1, algo2)]
num_algo2_better = num_tasks_better[(algo2, algo1)]
if num_algo1_better >= num_algo2_better:
num_wins += 1
return num_wins
algorithms = self.algorithms[:]
if self.sort:
algorithms.sort(key=get_wins)
comparison_table = Table()
comparison_table.set_row_order(algorithms)
comparison_table.set_column_order(algorithms)
for algo1, algo2 in itertools.permutations(algorithms, 2):
num_algo1_better = num_tasks_better[(algo1, algo2)]
num_algo2_better = num_tasks_better[(algo2, algo1)]
if num_algo1_better >= num_algo2_better:
if self.output_format == "tex":
content = r" ''\textbf{{{}}}''".format(num_algo1_better)
else:
content = r" ''<b>{}</b>''".format(num_algo1_better)
else:
content = num_algo1_better
comparison_table.add_cell(algo1, algo2, content)
for algo in algorithms:
comparison_table.add_cell(algo, algo, " ''--''")
print("Number of tasks for which all algorithms report {}: {}".format(
self.attribute, num_tasks))
return str(comparison_table)
def _get_table(self, domain, runs):
table = Table(title=domain)
for run in runs:
for attr in self.attributes:
table.add_cell(run['problem'], attr, run.get(attr))
return table
