def fetch_dir(self, run_dir, eval_dir, copy_all=False, run_filter=None, parsers=None): run_filter = run_filter or tools.RunFilter() parsers = parsers or [] # Allow specyfing a list of multiple parsers or a single parser. if not isinstance(parsers, (tuple, list)): parsers = [parsers] # Make sure parsers is a list. parsers = list(parsers) prop_file = os.path.join(run_dir, 'properties') # Somehow '../..' gets inserted into sys.path and more strangely the # system lab.tools module gets called. # TODO: This HACK should be removed once the source of the error is clear. props = tools.Properties(filename=prop_file) if props.get('search_returncode' ) is not None and props.get("coverage") is None: logging.warning('search_parser.py exited abnormally for %s' % run_dir) logging.info('Rerunning search_parser.py') parsers.append(os.path.join(run_dir, '../../search_parser.py')) for parser in parsers: rel_parser = os.path.relpath(parser, start=run_dir) subprocess.call([rel_parser], cwd=run_dir) props = tools.Properties(filename=prop_file) props = run_filter.apply_to_run(props) if not props: return None, None run_id = props.get('id') # Abort if an id cannot be read. if not run_id: logging.critical('id is not set in %s.' % prop_file) if copy_all: dest_dir = os.path.join(eval_dir, *run_id) tools.makedirs(dest_dir) tools.fast_updatetree(run_dir, dest_dir, symlinks=True) return run_id, props
def __call__(self, src_dir, dest_dir=None, copy_all=False, write_combined_props=True, filter=None, parsers=None, **kwargs): if os.path.isdir(src_dir): run_filter = tools.RunFilter(filter, **kwargs) logging.info('Fetching files from %s -> %s' % (src_dir, dest_dir)) # Get all run_dirs. None will be found if we fetch from an eval dir. run_dirs = sorted(glob(os.path.join(dest_dir, 'runs-*-*', '*'))) total_dirs = len(run_dirs) logging.info('Scanning properties from %d run directories' % total_dirs) unxeplained_errors = 0 for index, run_dir in enumerate(run_dirs, start=1): loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG logging.log(loglevel, 'Scanning: %6d/%d' % (index, total_dirs)) run_id, props = self.fetch_dir(run_dir, src_dir, copy_all=copy_all, run_filter=run_filter, parsers=parsers) if props is None: continue assert run_id, 'Dir %s has no id' % props.get('run_dir') if write_combined_props: combined_props['-'.join(run_id)] = props if props.get('error', '').startswith('unexplained'): logging.warning('Unexplained error in {run_dir}: {error}'.format(**props)) unxeplained_errors += 1 if unxeplained_errors: logging.warning('There were %d runs with unexplained errors.' % unxeplained_errors) #tools.makedirs(eval_dir) else: logging.info('%s is not a valid directory, not using PAC commulative info' % src_dir)
def __init__(self, attributes=None, format="html", filter=None, **kwargs): """ Inherit from this or a child class to implement a custom report. Depending on the type of output you want to make, you will have to overwrite the :meth:`.write`, :meth:`.get_text` or :meth:`.get_markup` method. *attributes* is the list of attributes you want to include in your report. If omitted, use all numerical attributes. Globbing characters * and ? are allowed. Example: >>> report = Report(attributes=["coverage", "translator_*"]) When a report is made, both the available and the selected attributes are printed on the commandline. *format* can be one of e.g. html, tex, wiki (MediaWiki), doku (DokuWiki), pmw (PmWiki), moin (MoinMoin) and txt (Plain text). Subclasses may allow additional formats. If given, *filter* must be a function or a list of functions that are passed a dictionary of a run's attribute keys and values. Filters must return True, False or a new dictionary. Depending on the returned value, the run is included or excluded from the report, or replaced by the new dictionary, respectively. Filters for properties can be given in shorter form without defining a function. To include only runs where attribute ``foo`` has value v, use ``filter_foo=v``. To include only runs where attribute ``foo`` has value v1, v2 or v3, use ``filter_foo=[v1, v2, v3]``. Filters are applied sequentially, i.e., the first filter is applied to all runs before the second filter is executed. Filters given as ``filter_*`` kwargs are applied *after* all filters passed via the ``filter`` kwarg. Examples: Include only the "cost" attribute in a LaTeX report: >>> report = Report(attributes=["cost"], format="tex") Only include successful runs in the report: >>> report = Report(filter_coverage=1) Only include runs in the report where the initial h value is at most 100: >>> def low_init_h(run): ... return run["initial_h_value"] <= 100 ... >>> report = Report(filter=low_init_h) Only include runs from "blocks" and "barman" with a timeout: >>> report = Report(filter_domain=["blocks", "barman"], filter_search_timeout=1) Add a new attribute: >>> def add_expansions_per_time(run): ... expansions = run.get("expansions") ... time = run.get("search_time") ... if expansions is not None and time: ... run["expansions_per_time"] = expansions / time ... return run ... >>> report = Report( ... attributes=["expansions_per_time"], filter=[add_expansions_per_time] ... ) Rename, filter and sort algorithms: >>> def rename_algorithms(run): ... name = run["algorithm"] ... paper_names = {"lama11": "LAMA 2011", "fdss_sat1": "FDSS 1"} ... run["algorithm"] = paper_names[name] ... return run ... >>> # We want LAMA 2011 to be the leftmost column. >>> # filter_* filters are evaluated last, so we use the updated >>> # algorithm names here. >>> algorithms = ["LAMA 2011", "FDSS 1"] >>> report = Report(filter=rename_algorithms, filter_algorithm=algorithms) """ self.attributes = tools.make_list(attributes) if format not in txt2tags.TARGETS + ["eps", "pdf", "pgf", "png", "py"]: raise ValueError(f"invalid format: {format}") self.output_format = format self.toc = True self.run_filter = tools.RunFilter(filter, **kwargs)
def __call__(self, src_dir, eval_dir=None, merge=None, filter=None, **kwargs): """ This method can be used to copy properties from an exp-dir or eval-dir into an eval-dir. If the destination eval-dir already exist, the data will be merged. This means *src_dir* can either be an exp-dir or an eval-dir and *eval_dir* can be a new or existing directory. We recommend using lab.Experiment.add_fetcher() to add fetchers to an experiment. See the method's documentation for a description of the parameters. """ if not os.path.isdir(src_dir): logging.critical( "{} is missing or not a directory".format(src_dir)) run_filter = tools.RunFilter(filter, **kwargs) eval_dir = eval_dir or src_dir.rstrip("/") + "-eval" logging.info("Fetching properties from {} to {}".format( src_dir, eval_dir)) if merge is None: _check_eval_dir(eval_dir) elif merge: # No action needed, data will be merged. pass else: tools.remove_path(eval_dir) # Load properties in the eval_dir if there are any already. combined_props = tools.Properties(os.path.join(eval_dir, "properties")) fetch_from_eval_dir = not os.path.exists( os.path.join(src_dir, "runs-00001-00100")) if fetch_from_eval_dir: src_props = tools.Properties( filename=os.path.join(src_dir, "properties")) run_filter.apply(src_props) combined_props.update(src_props) logging.info("Fetched properties of {} runs.".format( len(src_props))) else: slurm_err_content = tools.get_slurm_err_content(src_dir) if slurm_err_content: logging.error("There was output to *-grid-steps/slurm.err") new_props = tools.Properties() run_dirs = sorted(glob(os.path.join(src_dir, "runs-*-*", "*"))) total_dirs = len(run_dirs) logging.info( "Scanning properties from {:d} run directories".format( total_dirs)) for index, run_dir in enumerate(run_dirs, start=1): loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG logging.log(loglevel, "Scanning: {:6d}/{:d}".format(index, total_dirs)) props = self.fetch_dir(run_dir) if slurm_err_content: props.add_unexplained_error("output-to-slurm.err") id_string = "-".join(props["id"]) new_props[id_string] = props run_filter.apply(new_props) combined_props.update(new_props) unexplained_errors = 0 for props in combined_props.values(): error_message = tools.get_unexplained_errors_message(props) if error_message: logging.error(error_message) unexplained_errors += 1 tools.makedirs(eval_dir) combined_props.write() logging.info("Wrote properties file (contains {unexplained_errors} " "runs with unexplained errors).".format(**locals()))
def __init__(self, attributes=None, format='html', filter=None, **kwargs): """ *attributes* is a list of the attributes you want to include in your report. If omitted, use all found numerical attributes. Globbing characters * and ? are allowed. Example: :: Report(attributes=['translator_time_*']) When a report is made, both the available and the selected attributes are printed on the commandline. *format* can be one of e.g. html, tex, wiki (MediaWiki), gwiki (Google Code Wiki), doku (DokuWiki), pmw (PmWiki), moin (MoinMoin), txt (Plain text) and art (ASCII art). Subclasses may allow additional formats. If given, *filter* must be a function or a list of functions that are passed a dictionary of a run's keys and values and return True or False. Depending on the returned value, the run is included or excluded from the report. Alternatively, the function can return a dictionary that will overwrite the old run's dictionary for the report. Filters for properties can be given in shorter form without defining a function To include only runs where property p has value v, use *filter_p=v*. To include only runs where property p has value v1, v2 or v3, use *filter_p=[v1, v2, v3]*. Examples: Include only *coverage* and *expansions* in the report and write a LaTeX file at ``myreport.tex``:: report = Report(attributes=['coverage', 'expansions'], format='tex') report(path_to_eval_dir, 'myreport.tex') Only include successful runs in the report:: report = Report(filter_coverage=1) report(path_to_eval_dir, 'myreport.html') Only include runs in the report where the time score is better than the memory score:: def better_time_than_memory_score(run): return run['score_search_time'] > run['score_memory'] report = Report(filter=better_time_than_memory_score) report(path_to_eval_dir, 'myreport.html') Filter function that filters and renames configs with additional sorting:: def rename_configs(run): config = run['config'].replace('WORK-', '') paper_names = {'lama11': 'LAMA 2011', 'fdss_sat1': 'FDSS 1', 'fdss_sat2': 'FDSS 2'} run['config'] = paper_names.get(config, 'unknown') return run # We want LAMA 2011 to be the leftmost column. # Filters defined with key word arguments are evaluated last, # so we use the updated config names here. configs = ['LAMA 2011', 'FDSS 1', 'FDSS 2'] Report(filter=rename_configs, filter_config=configs) Filter function that only allows runs with a timeout in one of two domains:: report = Report(attributes=['coverage'], filter_domain=['blocks', 'barman'], filter_search_timeout=1) """ if isinstance(attributes, basestring): attributes = [attributes] self.attributes = attributes or [] assert format in txt2tags.TARGETS + ['eps', 'pdf', 'pgf', 'png', 'py'] self.output_format = format self.toc = True self.run_filter = tools.RunFilter(filter, **kwargs)
def fetch_dir(self, run_dir, src_dir, copy_all=False, run_filter=None, parsers=None): logging.info('Now I will copy for PAC') run_filter = run_filter or tools.RunFilter() parsers = parsers or [] # Allow specyfing a list of multiple parsers or a single parser. if not isinstance(parsers, (tuple, list)): parsers = [parsers] # Make sure parsers is a list. parsers = list(parsers) prop_file = os.path.join(run_dir, 'properties') # Somehow '../..' gets inserted into sys.path and more strangely the # system lab.tools module gets called. # TODO: This HACK should be removed once the source of the error is clear. props = tools.Properties(filename=prop_file) if props.get('search_returncode') is not None and props.get("coverage") is None: logging.warning('search_parser.py exited abnormally for %s' % run_dir) logging.info('Rerunning search_parser.py') parsers.append(os.path.join(run_dir, '../../search_parser.py')) logging.info(props.get('domain')) for parser in parsers: rel_parser = os.path.relpath(parser, start=run_dir) subprocess.call([rel_parser], cwd=run_dir) props = tools.Properties(filename=prop_file) props = run_filter.apply_to_run(props) if not props: return None, None run_id = props.get('id') # Abort if an id cannot be read. if not run_id: logging.critical('id is not set in %s.' % prop_file) #dest_dir = os.path.join(run_dir, *run_id) src_dir_hstar2h = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_ratio.csv' src_dir_hstar = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_hstar.csv' src_dir_stats = src_dir + '/' + props.get('domain') + '/' + 'PAC_Statistics.csv' src_dir_hffToh = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_h-ff_to_h-star.csv' run_dir_hstar2h = run_dir + '/' + 'PAC_Commulative_ratio.csv' run_dir_hstar = run_dir + '/' + 'PAC_Commulative_hstar.csv' run_dir_stats = run_dir + '/' + 'PAC_Statistics.csv' run_dir_hffToh = run_dir + '/' + 'PAC_Commulative_h-ff_to_h-star.csv' logging.info("run_dir: " + run_dir) logging.info("src_dir: " + src_dir) shutil.copy2(src_dir_hstar2h, run_dir_hstar2h) shutil.copy2(src_dir_hstar, run_dir_hstar) shutil.copy2(src_dir_stats, run_dir_stats) shutil.copy2(src_dir_hffToh, run_dir_hffToh) return run_id, props
def __call__(self, src_dir, eval_dir=None, copy_all=False, write_combined_props=True, filter=None, parsers=None, **kwargs): """ This method can be used to copy properties from an exp-dir or eval-dir into an eval-dir. If the destination eval-dir already exist, the data will be merged. This means *src_dir* can either be an exp-dir or an eval-dir and *eval_dir* can be a new or existing directory. If *copy_all* is True (default: False), copy all files from the run dirs to a new directory tree at *eval_dir*. Without this option only the combined properties file is written do disk. If *write_combined_props* is True (default), write the combined properties file. You can include only specific domains or configurations by using :py:class:`filters <.Report>`. *parsers* can be a list of paths to parser scripts. If given, each parser is called in each run directory and the results are added to the properties file which is fetched afterwards. This option is useful if you haven't parsed all or some values already during the experiment. Examples: Fetch all results and write a single combined properties file to the default evaluation directory (this step is added by default):: exp.add_step(Step('fetch', Fetcher(), exp.path)) Read the combined properties file at ``<eval_dir1>/properties`` and merge it into the combined properties file at ``<combined_eval_dir>/properties``:: exp.add_step(Step('combine', Fetcher(), eval_dir1, combined_eval_dir)) Fetch only the runs for certain configuration from an older experiment:: exp.add_step(Step('fetch', Fetcher(), src_dir, filter_config_nick=['config_1', 'config_5'])) """ if not os.path.isdir(src_dir): logging.critical('%s is not a valid directory' % src_dir) run_filter = tools.RunFilter(filter, **kwargs) src_props = tools.Properties( filename=os.path.join(src_dir, 'properties')) fetch_from_eval_dir = 'runs' not in src_props or src_dir.endswith( '-eval') if fetch_from_eval_dir: src_props = run_filter.apply(src_props) for prop in src_props.values(): if prop.get('error', '').startswith('unexplained'): logging.warning("Unexplained error in '%s': %s" % (prop.get('run_dir'), prop.get('error'))) eval_dir = eval_dir or src_dir.rstrip('/') + '-eval' logging.info('Fetching files from %s -> %s' % (src_dir, eval_dir)) logging.info('Fetching from evaluation dir: %s' % fetch_from_eval_dir) if write_combined_props: # Load properties in the eval_dir if there are any already. combined_props = tools.Properties( os.path.join(eval_dir, 'properties')) if fetch_from_eval_dir: combined_props.update(src_props) # Get all run_dirs. None will be found if we fetch from an eval dir. run_dirs = sorted(glob(os.path.join(src_dir, 'runs-*-*', '*'))) total_dirs = len(run_dirs) logging.info('Scanning properties from %d run directories' % total_dirs) unxeplained_errors = 0 for index, run_dir in enumerate(run_dirs, start=1): loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG logging.log(loglevel, 'Scanning: %6d/%d' % (index, total_dirs)) run_id, props = self.fetch_dir(run_dir, eval_dir, copy_all=copy_all, run_filter=run_filter, parsers=parsers) if props is None: continue assert run_id, 'Dir %s has no id' % props.get('run_dir') if write_combined_props: combined_props['-'.join(run_id)] = props if props.get('error', '').startswith('unexplained'): logging.warning( 'Unexplained error in {run_dir}: {error}'.format(**props)) unxeplained_errors += 1 if unxeplained_errors: logging.warning('There were %d runs with unexplained errors.' % unxeplained_errors) tools.makedirs(eval_dir) if write_combined_props: combined_props.write()