def fetch_dir(self,
run_dir,
eval_dir,
copy_all=False,
run_filter=None,
parsers=None):
run_filter = run_filter or tools.RunFilter()
parsers = parsers or []
# Allow specyfing a list of multiple parsers or a single parser.
if not isinstance(parsers, (tuple, list)):
parsers = [parsers]
# Make sure parsers is a list.
parsers = list(parsers)
prop_file = os.path.join(run_dir, 'properties')
# Somehow '../..' gets inserted into sys.path and more strangely the
# system lab.tools module gets called.
# TODO: This HACK should be removed once the source of the error is clear.
props = tools.Properties(filename=prop_file)
if props.get('search_returncode'
) is not None and props.get("coverage") is None:
logging.warning('search_parser.py exited abnormally for %s' %
run_dir)
logging.info('Rerunning search_parser.py')
parsers.append(os.path.join(run_dir, '../../search_parser.py'))
for parser in parsers:
rel_parser = os.path.relpath(parser, start=run_dir)
subprocess.call([rel_parser], cwd=run_dir)
props = tools.Properties(filename=prop_file)
props = run_filter.apply_to_run(props)
if not props:
return None, None
run_id = props.get('id')
# Abort if an id cannot be read.
if not run_id:
logging.critical('id is not set in %s.' % prop_file)
if copy_all:
dest_dir = os.path.join(eval_dir, *run_id)
tools.makedirs(dest_dir)
tools.fast_updatetree(run_dir, dest_dir, symlinks=True)
return run_id, props
def __call__(self, src_dir, dest_dir=None, copy_all=False, write_combined_props=True,
filter=None, parsers=None, **kwargs):
if os.path.isdir(src_dir):
run_filter = tools.RunFilter(filter, **kwargs)
logging.info('Fetching files from %s -> %s' % (src_dir, dest_dir))
# Get all run_dirs. None will be found if we fetch from an eval dir.
run_dirs = sorted(glob(os.path.join(dest_dir, 'runs-*-*', '*')))
total_dirs = len(run_dirs)
logging.info('Scanning properties from %d run directories' % total_dirs)
unxeplained_errors = 0
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel, 'Scanning: %6d/%d' % (index, total_dirs))
run_id, props = self.fetch_dir(run_dir, src_dir, copy_all=copy_all,
run_filter=run_filter, parsers=parsers)
if props is None:
continue
assert run_id, 'Dir %s has no id' % props.get('run_dir')
if write_combined_props:
combined_props['-'.join(run_id)] = props
if props.get('error', '').startswith('unexplained'):
logging.warning('Unexplained error in {run_dir}: {error}'.format(**props))
unxeplained_errors += 1
if unxeplained_errors:
logging.warning('There were %d runs with unexplained errors.'
% unxeplained_errors)
#tools.makedirs(eval_dir)
else:
logging.info('%s is not a valid directory, not using PAC commulative info' % src_dir)
def __init__(self, attributes=None, format="html", filter=None, **kwargs):
"""
Inherit from this or a child class to implement a custom report.
Depending on the type of output you want to make, you will have
to overwrite the :meth:`.write`, :meth:`.get_text` or
:meth:`.get_markup` method.
*attributes* is the list of attributes you want to include in
your report. If omitted, use all numerical attributes. Globbing
characters * and ? are allowed. Example:
>>> report = Report(attributes=["coverage", "translator_*"])
When a report is made, both the available and the selected
attributes are printed on the commandline.
*format* can be one of e.g. html, tex, wiki (MediaWiki), doku
(DokuWiki), pmw (PmWiki), moin (MoinMoin) and txt (Plain text).
Subclasses may allow additional formats.
If given, *filter* must be a function or a list of functions
that are passed a dictionary of a run's attribute keys and
values. Filters must return True, False or a new dictionary.
Depending on the returned value, the run is included or excluded
from the report, or replaced by the new dictionary,
respectively.
Filters for properties can be given in shorter form without
defining a function. To include only runs where attribute
``foo`` has value v, use ``filter_foo=v``. To include only runs
where attribute ``foo`` has value v1, v2 or v3, use
``filter_foo=[v1, v2, v3]``.
Filters are applied sequentially, i.e., the first filter is
applied to all runs before the second filter is executed.
Filters given as ``filter_*`` kwargs are applied *after* all
filters passed via the ``filter`` kwarg.
Examples:
Include only the "cost" attribute in a LaTeX report:
>>> report = Report(attributes=["cost"], format="tex")
Only include successful runs in the report:
>>> report = Report(filter_coverage=1)
Only include runs in the report where the initial h value is
at most 100:
>>> def low_init_h(run):
... return run["initial_h_value"] <= 100
...
>>> report = Report(filter=low_init_h)
Only include runs from "blocks" and "barman" with a timeout:
>>> report = Report(filter_domain=["blocks", "barman"], filter_search_timeout=1)
Add a new attribute:
>>> def add_expansions_per_time(run):
... expansions = run.get("expansions")
... time = run.get("search_time")
... if expansions is not None and time:
... run["expansions_per_time"] = expansions / time
... return run
...
>>> report = Report(
... attributes=["expansions_per_time"], filter=[add_expansions_per_time]
... )
Rename, filter and sort algorithms:
>>> def rename_algorithms(run):
... name = run["algorithm"]
... paper_names = {"lama11": "LAMA 2011", "fdss_sat1": "FDSS 1"}
... run["algorithm"] = paper_names[name]
... return run
...
>>> # We want LAMA 2011 to be the leftmost column.
>>> # filter_* filters are evaluated last, so we use the updated
>>> # algorithm names here.
>>> algorithms = ["LAMA 2011", "FDSS 1"]
>>> report = Report(filter=rename_algorithms, filter_algorithm=algorithms)
"""
self.attributes = tools.make_list(attributes)
if format not in txt2tags.TARGETS + ["eps", "pdf", "pgf", "png", "py"]:
raise ValueError(f"invalid format: {format}")
self.output_format = format
self.toc = True
self.run_filter = tools.RunFilter(filter, **kwargs)
def __call__(self,
src_dir,
eval_dir=None,
merge=None,
filter=None,
**kwargs):
"""
This method can be used to copy properties from an exp-dir or
eval-dir into an eval-dir. If the destination eval-dir already
exist, the data will be merged. This means *src_dir* can either
be an exp-dir or an eval-dir and *eval_dir* can be a new or
existing directory.
We recommend using lab.Experiment.add_fetcher() to add fetchers
to an experiment. See the method's documentation for a
description of the parameters.
"""
if not os.path.isdir(src_dir):
logging.critical(
"{} is missing or not a directory".format(src_dir))
run_filter = tools.RunFilter(filter, **kwargs)
eval_dir = eval_dir or src_dir.rstrip("/") + "-eval"
logging.info("Fetching properties from {} to {}".format(
src_dir, eval_dir))
if merge is None:
_check_eval_dir(eval_dir)
elif merge:
# No action needed, data will be merged.
pass
else:
tools.remove_path(eval_dir)
# Load properties in the eval_dir if there are any already.
combined_props = tools.Properties(os.path.join(eval_dir, "properties"))
fetch_from_eval_dir = not os.path.exists(
os.path.join(src_dir, "runs-00001-00100"))
if fetch_from_eval_dir:
src_props = tools.Properties(
filename=os.path.join(src_dir, "properties"))
run_filter.apply(src_props)
combined_props.update(src_props)
logging.info("Fetched properties of {} runs.".format(
len(src_props)))
else:
slurm_err_content = tools.get_slurm_err_content(src_dir)
if slurm_err_content:
logging.error("There was output to *-grid-steps/slurm.err")
new_props = tools.Properties()
run_dirs = sorted(glob(os.path.join(src_dir, "runs-*-*", "*")))
total_dirs = len(run_dirs)
logging.info(
"Scanning properties from {:d} run directories".format(
total_dirs))
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel,
"Scanning: {:6d}/{:d}".format(index, total_dirs))
props = self.fetch_dir(run_dir)
if slurm_err_content:
props.add_unexplained_error("output-to-slurm.err")
id_string = "-".join(props["id"])
new_props[id_string] = props
run_filter.apply(new_props)
combined_props.update(new_props)
unexplained_errors = 0
for props in combined_props.values():
error_message = tools.get_unexplained_errors_message(props)
if error_message:
logging.error(error_message)
unexplained_errors += 1
tools.makedirs(eval_dir)
combined_props.write()
logging.info("Wrote properties file (contains {unexplained_errors} "
"runs with unexplained errors).".format(**locals()))
def __init__(self, attributes=None, format='html', filter=None, **kwargs):
"""
*attributes* is a list of the attributes you want to include in your
report. If omitted, use all found numerical attributes. Globbing
characters * and ? are allowed. Example: ::
Report(attributes=['translator_time_*'])
When a report is made, both the available and the selected attributes
are printed on the commandline.
*format* can be one of e.g. html, tex, wiki (MediaWiki),
gwiki (Google Code Wiki), doku (DokuWiki), pmw (PmWiki),
moin (MoinMoin), txt (Plain text) and art (ASCII art). Subclasses may
allow additional formats.
If given, *filter* must be a function or a list of functions that
are passed a dictionary of a run's keys and values and return
True or False. Depending on the returned value, the run is included
or excluded from the report.
Alternatively, the function can return a dictionary that will overwrite
the old run's dictionary for the report.
Filters for properties can be given in shorter form without defining a function
To include only runs where property p has value v, use *filter_p=v*.
To include only runs where property p has value v1, v2 or v3, use
*filter_p=[v1, v2, v3]*.
Examples:
Include only *coverage* and *expansions* in the report and write a
LaTeX file at ``myreport.tex``::
report = Report(attributes=['coverage', 'expansions'], format='tex')
report(path_to_eval_dir, 'myreport.tex')
Only include successful runs in the report::
report = Report(filter_coverage=1)
report(path_to_eval_dir, 'myreport.html')
Only include runs in the report where the time score is better than the
memory score::
def better_time_than_memory_score(run):
return run['score_search_time'] > run['score_memory']
report = Report(filter=better_time_than_memory_score)
report(path_to_eval_dir, 'myreport.html')
Filter function that filters and renames configs with additional sorting::
def rename_configs(run):
config = run['config'].replace('WORK-', '')
paper_names = {'lama11': 'LAMA 2011', 'fdss_sat1': 'FDSS 1',
'fdss_sat2': 'FDSS 2'}
run['config'] = paper_names.get(config, 'unknown')
return run
# We want LAMA 2011 to be the leftmost column.
# Filters defined with key word arguments are evaluated last,
# so we use the updated config names here.
configs = ['LAMA 2011', 'FDSS 1', 'FDSS 2']
Report(filter=rename_configs, filter_config=configs)
Filter function that only allows runs with a timeout in one of two domains::
report = Report(attributes=['coverage'],
filter_domain=['blocks', 'barman'],
filter_search_timeout=1)
"""
if isinstance(attributes, basestring):
attributes = [attributes]
self.attributes = attributes or []
assert format in txt2tags.TARGETS + ['eps', 'pdf', 'pgf', 'png', 'py']
self.output_format = format
self.toc = True
self.run_filter = tools.RunFilter(filter, **kwargs)
def fetch_dir(self, run_dir, src_dir, copy_all=False, run_filter=None, parsers=None):
logging.info('Now I will copy for PAC')
run_filter = run_filter or tools.RunFilter()
parsers = parsers or []
# Allow specyfing a list of multiple parsers or a single parser.
if not isinstance(parsers, (tuple, list)):
parsers = [parsers]
# Make sure parsers is a list.
parsers = list(parsers)
prop_file = os.path.join(run_dir, 'properties')
# Somehow '../..' gets inserted into sys.path and more strangely the
# system lab.tools module gets called.
# TODO: This HACK should be removed once the source of the error is clear.
props = tools.Properties(filename=prop_file)
if props.get('search_returncode') is not None and props.get("coverage") is None:
logging.warning('search_parser.py exited abnormally for %s' % run_dir)
logging.info('Rerunning search_parser.py')
parsers.append(os.path.join(run_dir, '../../search_parser.py'))
logging.info(props.get('domain'))
for parser in parsers:
rel_parser = os.path.relpath(parser, start=run_dir)
subprocess.call([rel_parser], cwd=run_dir)
props = tools.Properties(filename=prop_file)
props = run_filter.apply_to_run(props)
if not props:
return None, None
run_id = props.get('id')
# Abort if an id cannot be read.
if not run_id:
logging.critical('id is not set in %s.' % prop_file)
#dest_dir = os.path.join(run_dir, *run_id)
src_dir_hstar2h = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_ratio.csv'
src_dir_hstar = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_hstar.csv'
src_dir_stats = src_dir + '/' + props.get('domain') + '/' + 'PAC_Statistics.csv'
src_dir_hffToh = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_h-ff_to_h-star.csv'
run_dir_hstar2h = run_dir + '/' + 'PAC_Commulative_ratio.csv'
run_dir_hstar = run_dir + '/' + 'PAC_Commulative_hstar.csv'
run_dir_stats = run_dir + '/' + 'PAC_Statistics.csv'
run_dir_hffToh = run_dir + '/' + 'PAC_Commulative_h-ff_to_h-star.csv'
logging.info("run_dir: " + run_dir)
logging.info("src_dir: " + src_dir)
shutil.copy2(src_dir_hstar2h, run_dir_hstar2h)
shutil.copy2(src_dir_hstar, run_dir_hstar)
shutil.copy2(src_dir_stats, run_dir_stats)
shutil.copy2(src_dir_hffToh, run_dir_hffToh)
return run_id, props
def __call__(self,
src_dir,
eval_dir=None,
copy_all=False,
write_combined_props=True,
filter=None,
parsers=None,
**kwargs):
"""
This method can be used to copy properties from an exp-dir or eval-dir
into an eval-dir. If the destination eval-dir already exist, the data
will be merged. This means *src_dir* can either be an exp-dir or an
eval-dir and *eval_dir* can be a new or existing directory.
If *copy_all* is True (default: False), copy all files from the run
dirs to a new directory tree at *eval_dir*. Without this option only
the combined properties file is written do disk.
If *write_combined_props* is True (default), write the combined
properties file.
You can include only specific domains or configurations by using
:py:class:`filters <.Report>`.
*parsers* can be a list of paths to parser scripts. If given, each
parser is called in each run directory and the results are added to
the properties file which is fetched afterwards. This option is
useful if you haven't parsed all or some values already during the
experiment.
Examples:
Fetch all results and write a single combined properties file to the
default evaluation directory (this step is added by default)::
exp.add_step(Step('fetch', Fetcher(), exp.path))
Read the combined properties file at ``<eval_dir1>/properties`` and
merge it into the combined properties file at
``<combined_eval_dir>/properties``::
exp.add_step(Step('combine', Fetcher(), eval_dir1, combined_eval_dir))
Fetch only the runs for certain configuration from an older experiment::
exp.add_step(Step('fetch', Fetcher(), src_dir,
filter_config_nick=['config_1', 'config_5']))
"""
if not os.path.isdir(src_dir):
logging.critical('%s is not a valid directory' % src_dir)
run_filter = tools.RunFilter(filter, **kwargs)
src_props = tools.Properties(
filename=os.path.join(src_dir, 'properties'))
fetch_from_eval_dir = 'runs' not in src_props or src_dir.endswith(
'-eval')
if fetch_from_eval_dir:
src_props = run_filter.apply(src_props)
for prop in src_props.values():
if prop.get('error', '').startswith('unexplained'):
logging.warning("Unexplained error in '%s': %s" %
(prop.get('run_dir'), prop.get('error')))
eval_dir = eval_dir or src_dir.rstrip('/') + '-eval'
logging.info('Fetching files from %s -> %s' % (src_dir, eval_dir))
logging.info('Fetching from evaluation dir: %s' % fetch_from_eval_dir)
if write_combined_props:
# Load properties in the eval_dir if there are any already.
combined_props = tools.Properties(
os.path.join(eval_dir, 'properties'))
if fetch_from_eval_dir:
combined_props.update(src_props)
# Get all run_dirs. None will be found if we fetch from an eval dir.
run_dirs = sorted(glob(os.path.join(src_dir, 'runs-*-*', '*')))
total_dirs = len(run_dirs)
logging.info('Scanning properties from %d run directories' %
total_dirs)
unxeplained_errors = 0
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel, 'Scanning: %6d/%d' % (index, total_dirs))
run_id, props = self.fetch_dir(run_dir,
eval_dir,
copy_all=copy_all,
run_filter=run_filter,
parsers=parsers)
if props is None:
continue
assert run_id, 'Dir %s has no id' % props.get('run_dir')
if write_combined_props:
combined_props['-'.join(run_id)] = props
if props.get('error', '').startswith('unexplained'):
logging.warning(
'Unexplained error in {run_dir}: {error}'.format(**props))
unxeplained_errors += 1
if unxeplained_errors:
logging.warning('There were %d runs with unexplained errors.' %
unxeplained_errors)
tools.makedirs(eval_dir)
if write_combined_props:
combined_props.write()
