def _get_warnings_text_and_table(self): """ Return a :py:class:`Table <lab.reports.Table>` containing one line for each run where an unexplained error occured. """ if not self.ERROR_ATTRIBUTES: logging.critical("The list of error attributes must not be empty.") table = reports.Table(title="Unexplained errors") table.set_column_order(self.ERROR_ATTRIBUTES) wrote_to_slurm_err = any( "output-to-slurm.err" in run.get("unexplained_errors", []) for run in self.runs.values()) for run in self.runs.values(): error_message = tools.get_unexplained_errors_message(run) if error_message: logging.error(error_message) run_dir = run["run_dir"] for attr in self.ERROR_ATTRIBUTES: value = run.get(attr, "?") if attr == "unexplained_errors": value = self._format_unexplained_errors(value) # Use formatted value as-is. table.cell_formatters[run_dir][ attr] = reports.CellFormatter() table.add_cell(run_dir, attr, value) errors = [] if wrote_to_slurm_err: src_dir = self.eval_dir.rstrip("/")[:-len("-eval")] slurm_err_file = src_dir + "-grid-steps/slurm.err" try: slurm_err_content = tools.get_slurm_err_content(src_dir) except FileNotFoundError: slurm_err_file = "*-grid-steps/slurm.err" errors.append( f"There was output to {slurm_err_file}, but the file was missing " f"when this report was made.") else: slurm_err_content = tools.filter_slurm_err_content( slurm_err_content) errors.append( f"There was output to {slurm_err_file}. Below is the output without" f'"memory cg" errors:\n```\n{slurm_err_content}\n```') logging.error(f"There was output to {slurm_err_file}.") if table: errors.append(str(table)) infai_1_nodes = {f"ase{i:02d}.cluster.bc2.ch" for i in range(1, 25)} infai_2_nodes = {f"ase{i:02d}.cluster.bc2.ch" for i in range(31, 55)} nodes = self._get_node_names() if nodes & infai_1_nodes and nodes & infai_2_nodes: errors.append( "Report combines runs from infai_1 and infai_2 partitions.") return "\n".join(errors)
def _get_empty_table(self, attribute=None, title=None, columns=None): """Return an empty table.""" if title is None: assert attribute is not None title = attribute if self.output_format == 'tex': title = title.capitalize().replace('_', ' ') if columns is None: columns = self.algorithms if attribute is not None and self.attribute_is_numeric(attribute): # Decide whether we want to highlight minima or maxima. kwargs = { 'min_wins': attribute.min_wins, 'colored': self.colored and attribute.min_wins is not None, 'digits': attribute.digits, } else: # Do not highlight anything. kwargs = {} table = reports.Table(title=title, **kwargs) table.set_column_order(columns) link = '#%s' % title formatter = reports.CellFormatter(link=link) table.cell_formatters[table.header_row][ table.header_column] = formatter return table
def _get_suite_table(self, attribute): assert self.attribute_is_numeric(attribute), attribute table = self._get_empty_table(attribute) self._add_summary_functions(table, attribute) func_name, func = self._get_aggregation_function(attribute) num_probs = 0 self._add_table_info(attribute, func_name, table) domain_algo_values = {} for domain in self.domains: for algorithm in self.algorithms: domain_algo_values[(domain, algorithm)] = [] for (domain, _), runs in self.problem_runs.items(): # If the attribute is absolute, no runs must have been filtered and # no values must be missing. if not attribute.absolute and (len(runs) < len(self.algorithms) or any( run.get(attribute) is None for run in runs)): continue num_probs += 1 for run in runs: value = run.get(attribute) if value is not None: domain_algo_values[(domain, run["algorithm"])].append(value) # If the attribute is absolute (e.g. coverage) we may have # added problems for which not all algorithms have a value. Therefore, we # can only print the number of instances (in brackets after the domain # name) if that number is the same for all algorithms. If not all algorithms # have values for the same number of problems, we write the full list of # different problem numbers. for domain in self.domains: task_counts = [ str(len(domain_algo_values[(domain, algo)])) for algo in self.algorithms ] if len(set(task_counts)) == 1: count = task_counts[0] else: count = ", ".join(task_counts) link = None if self.use_domain_links: link = f"#{attribute}-{domain}" formatter = reports.CellFormatter(link=link, count=count) table.cell_formatters[domain][table.header_column] = formatter for (domain, algo), values in domain_algo_values.items(): domain_value = func(values) if values else None table.add_cell(domain, algo, domain_value) table.num_values = num_probs return table
def _get_suite_table(self, attribute): assert self.attribute_is_numeric(attribute), attribute table = self._get_empty_table(attribute) self._add_summary_functions(table, attribute) # The first group function is used for aggregation. func_name, func = self._get_group_functions(attribute)[0] num_probs = 0 self._add_table_info(attribute, func_name, table) domain_config_values = defaultdict(list) for domain, problems in self.domains.items(): for problem in problems: runs = self.problem_runs[(domain, problem)] if (not attribute.absolute and any(run.get(attribute) is None for run in runs)): continue num_probs += 1 for run in runs: value = run.get(attribute) if value is not None: domain_config_values[(domain, run['config'])].append(value) # If the attribute is absolute (e.g. coverage) we may have # added problems for which not all configs have a value. Therefore, we # can only print the number of instances (in brackets after the domain # name) if that number is the same for all configs. If not all configs # have values for the same number of problems, we write the full list of # different problem numbers. num_values_lists = defaultdict(list) for domain in self.domains: for config in self.configs: values = domain_config_values.get((domain, config), []) num_values_lists[domain].append(str(len(values))) for domain, num_values_list in num_values_lists.items(): if len(set(num_values_list)) == 1: count = num_values_list[0] else: count = ','.join(num_values_list) link = None if self.resolution == 'combined': link = '#%s-%s' % (attribute, domain) formatter = reports.CellFormatter(link=link, count=count) table.cell_formatters[domain][table.header_column] = formatter for (domain, config), values in domain_config_values.items(): table.add_cell(domain, config, func(values)) table.num_values = num_probs return table
def _get_suite_table(self, attribute): assert self.attribute_is_numeric(attribute), attribute table = self._get_empty_table(attribute) self._add_summary_functions(table, attribute) func_name, func = self._get_aggregation_function(attribute) num_probs = 0 self._add_table_info(attribute, func_name, table) domain_algo_values = defaultdict(list) for (domain, _), runs in self.problem_runs.items(): if (not attribute.absolute and any(run.get(attribute) is None for run in runs)): continue num_probs += 1 for run in runs: value = run.get(attribute) if value is not None: domain_algo_values[(domain, run['algorithm'])].append(value) # If the attribute is absolute (e.g. coverage) we may have # added problems for which not all algorithms have a value. Therefore, we # can only print the number of instances (in brackets after the domain # name) if that number is the same for all algorithms. If not all algorithms # have values for the same number of problems, we write the full list of # different problem numbers. num_values_lists = defaultdict(list) for domain in self.domains: for algo in self.algorithms: values = domain_algo_values.get((domain, algo), []) num_values_lists[domain].append(str(len(values))) for domain, num_values_list in num_values_lists.items(): if len(set(num_values_list)) == 1: count = num_values_list[0] else: count = ','.join(num_values_list) link = None if self.use_domain_links: link = '#{}-{}'.format(attribute, domain) formatter = reports.CellFormatter(link=link, count=count) table.cell_formatters[domain][table.header_column] = formatter for (domain, algo), values in domain_algo_values.items(): table.add_cell(domain, algo, func(values)) table.num_values = num_probs return table
def _get_empty_table(self, attribute=None, title=None, columns=None): """Return an empty table.""" if title is None: assert attribute is not None title = attribute if columns is None: columns = self.configs if attribute is not None and self.attribute_is_numeric(attribute): # Decide whether we want to highlight minima or maxima. min_wins = attribute.min_wins colored = self.colored and min_wins is not None else: # Do not highlight anything. min_wins = None colored = False table = reports.Table(title=title, min_wins=min_wins, colored=colored) table.set_column_order(columns) if self.resolution == 'combined': link = '#%s' % title formatter = reports.CellFormatter(link=link) table.cell_formatters[table.header_row][table.header_column] = formatter return table
def _get_commulative_table(self, domain): #init new table title = 'Commulative' columns = {'Percentage', 'h*/h(s)'} min_wins = False colored = True table = reports.Table(title=title, min_wins=min_wins, colored=colored) table.set_column_order(columns) link = '#%s' % title formatter = reports.CellFormatter(link=link) table.cell_formatters[table.header_row][ table.header_column] = formatter domain_dir = self.outFile + '/' + domain tools.makedirs(domain_dir) domain_file = domain_dir + '/' + 'PAC_Commulative_ratio.csv' file = open(domain_file, "w") #get relevant value from original table ratio_attr = Attribute('hstar_to_h', min_wins=False, absolute=True) ratio_table = AbsoluteReport._get_table(self, ratio_attr, domain) #define arrays to work ratios = [ 0.75, 0.8, 0.85, 0.9, 0.95, 1, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7, 1.75, 1.8, 1.85, 1.9, 1.95, 2, 2.05, 2.1, 2.15, 2.2, 2.25, 2.3, 2.35, 2.4, 2.45, 2.5, 2.55, 2.6, 2.65, 2.7, 2.75, 2.8, 2.85, 2.9, 2.95, 3.0, 3.05, 3.1, 3.15, 3.2, 3.25, 3.3, 3.35, 3.4, 3.45, 3.5, 3.55, 3.6, 3.65, 3.7, 3.75, 3.80, 3.85, 3.9, 3.95, 4.0, 4.05, 4.1, 4.15, 4.2, 2.25, 4.3, 4.35, 4.4, 4.45, 4.5 ] names = [ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'za', 'zb', 'zc', 'zd', 'ze', 'zf', 'zg', 'zh', 'zi', 'zj', 'zk', 'zl', 'zm', 'zn', 'zo', 'aa', 'ab', 'ac', 'ad', 'ae', 'af', 'ag', 'ah', 'ai', 'aj', 'ak', 'al', 'am', 'an', 'ao', 'ap', 'aq', 'ar', 'as', 'at', 'au', 'av', 'aw', 'ax', 'ay', 'az', 'ba', 'bb', 'bc', 'bd', 'be', 'bf', 'bg', 'bh', 'bi' ] counter = 0 #calculate number of solved problems total_solved = 0 for row in ratio_table.row_names: curr_val = ratio_table.get(row) val = curr_val[self.nick] if val > 0: total_solved = total_solved + 1 #for each ratio (1,1.05...), find the number of problems with this ratio, calc percentage and add row for ratio in ratios: _sum = 0 for row in ratio_table.row_names: curr_val = ratio_table.get(row) val = curr_val[self.nick] if val <= ratio and val > 0: _sum = _sum + 1 if total_solved == 0: _sum_percent = 0 else: _sum_percent = _sum * 100 / total_solved #add new row row_to_add = {} row_to_add['Percentage'] = _sum_percent row_to_add['h*/h(s)'] = ratio table.add_row(names[counter], row_to_add) counter = counter + 1 #TODO - save only one ratio per percentage toWrite = str(ratio) + ',' + str(_sum_percent) + '\n' file.write(toWrite) file.close() self.create_commulative_h_star_table(domain) return table
def get_markup(self): sections = [] toc_lines = [] warnings = self._get_warnings_text_and_table() if warnings: toc_lines.append('- **[' 'Unexplained Errors' ' #unexplained-errors]**') sections.append(('unexplained-errors', warnings)) toc_lines.append('- **[Info #info]**') sections.append(('info', self._get_general_info())) # Index of summary section. summary_index = len(sections) # Build a table containing summary functions of all other tables. # The actual section is added at position summary_index after creating # all other tables. summary = self._get_empty_table(title='Summary') summary.colored = self.colored toc_lines.append('- **[Summary #summary]**') for attribute in self.attributes: logging.info('Creating table(s) for %s' % attribute) tables = [] if attribute == 'error': seen_errors = set() error_counter = defaultdict(int) for run in self.runs.values(): error = run.get('error', 'attribute-error-missing') seen_errors.add(error) error_counter[(run["algorithm"], run["domain"], error)] += 1 error_to_min_wins = { outcome.msg: outcome.min_wins for outcome in outcomes.OUTCOMES } for error in sorted(seen_errors): # Txt2tags seems to only allow letters, "-" and "_" in anchors. pseudo_attribute = 'error-' + error table = self._get_empty_table(title=pseudo_attribute) min_wins = error_to_min_wins.get(error, None) table.min_wins = min_wins table.colored = min_wins is not None for domain in self.domains: if self.use_domain_links: table.cell_formatters[domain][ table.header_column] = (reports.CellFormatter( link='#error-{domain}'.format(**locals()))) for algorithm in self.algorithms: count = error_counter.get( (algorithm, domain, error), 0) table.add_cell(domain, algorithm, count) table.add_summary_function('Sum', sum) reports.extract_summary_rows(table, summary, link='#' + 'error-' + pseudo_attribute) tables.append((pseudo_attribute, table)) elif self.attribute_is_numeric(attribute): domain_table = self._get_table(attribute) tables.append(('', domain_table)) reports.extract_summary_rows(domain_table, summary, link='#' + attribute) else: tables.append( ('', 'Domain-wise reports only support numeric ' 'attributes, but %s has type %s.' % (attribute, self._all_attributes[attribute].__name__))) for domain in sorted(self.domains.keys()): tables.append((domain, self._get_table(attribute, domain))) parts = [] toc_line = [] for (domain, table) in tables: if domain: assert table toc_line.append( "[''%(domain)s'' #%(attribute)s-%(domain)s]" % locals()) parts.append('== %(domain)s ==[%(attribute)s-%(domain)s]\n' '%(table)s\n' % locals()) else: if table: parts.append('%(table)s\n' % locals()) else: parts.append('No task was found where all algorithms ' 'have a value for "%s". Therefore no ' 'domain-wise table can be generated.\n' % attribute) toc_lines.append("- **[''{}'' #{}]**".format(attribute, attribute)) toc_lines.append(' - ' + ' '.join(toc_line)) sections.append((attribute, '\n'.join(parts))) # Add summary before main content. This is done after creating the main content # because the summary table is extracted from all other tables. sections.insert(summary_index, ('summary', summary)) toc = '\n'.join(toc_lines) content = '\n'.join('= {} =[{}]\n\n{}'.format(attr, attr, section) for (attr, section) in sections) return '{}\n\n\n{}'.format(toc, content)
def get_markup(self): sections = [] toc_lines = [] warnings = self._get_warnings_text_and_table() if warnings: toc_lines.append("- **[" "Unexplained Errors" " #unexplained-errors]**") sections.append(("unexplained-errors", warnings)) toc_lines.append("- **[Info #info]**") sections.append(("info", self._get_general_info())) # Index of summary section. summary_index = len(sections) # Build a table containing summary functions of all other tables. # The actual section is added at position summary_index after creating # all other tables. summary = self._get_empty_table(title="Summary") summary.colored = self.colored toc_lines.append("- **[Summary #summary]**") for attribute in self.attributes: logging.info(f"Creating table(s) for {attribute}") tables = [] if attribute == "error": seen_errors = set() error_counter = defaultdict(int) for run in self.runs.values(): error = run.get("error", "attribute-error-missing") seen_errors.add(error) error_counter[(run["algorithm"], run["domain"], error)] += 1 error_to_min_wins = { outcome.msg: outcome.min_wins for outcome in outcomes.OUTCOMES } for error in sorted(seen_errors): # Txt2tags seems to only allow letters, "-" and "_" in anchors. pseudo_attribute = "error-" + error table = self._get_empty_table(title=pseudo_attribute) min_wins = error_to_min_wins.get(error, None) table.min_wins = min_wins table.colored = min_wins is not None for domain in self.domains: if self.use_domain_links: table.cell_formatters[domain][ table.header_column] = reports.CellFormatter( link=f"#error-{domain}") for algorithm in self.algorithms: count = error_counter.get( (algorithm, domain, error), 0) table.add_cell(domain, algorithm, count) table.add_summary_function("Sum", sum) reports.extract_summary_rows(table, summary, link="#" + "error-" + pseudo_attribute) tables.append((pseudo_attribute, table)) elif self.attribute_is_numeric(attribute): domain_table = self._get_suite_table(attribute) tables.append(("", domain_table)) reports.extract_summary_rows(domain_table, summary, link="#" + attribute) else: tables.append(( "", f"Domain-wise reports only support numeric " f"attributes, but {attribute} has type " f"{self._all_attributes[attribute].__name__}.", )) for domain in sorted(self.domains.keys()): tables.append( (domain, self._get_domain_table(attribute, domain))) parts = [] toc_line = [] for (domain, table) in tables: if domain: assert table toc_line.append(f"[''{domain}'' #{attribute}-{domain}]") parts.append( f"== {domain} ==[{attribute}-{domain}]\n{table}\n") else: if table: parts.append(f"{table}\n") else: parts.append( f"No task was found where all algorithms " f'have a value for "{attribute}". Therefore no ' f"domain-wise table can be generated.\n") toc_lines.append(f"- **[''{attribute}'' #{attribute}]**") toc_lines.append(" - " + " ".join(toc_line)) sections.append((attribute, "\n".join(parts))) # Add summary before main content. This is done after creating the main content # because the summary table is extracted from all other tables. sections.insert(summary_index, ("summary", summary)) toc = "\n".join(toc_lines) content = "\n".join(f"= {attr} =[{attr}]\n\n{section}" for (attr, section) in sections) return f"{toc}\n\n\n{content}"