def fetch_dir(self, run_dir):
static_props = tools.Properties(filename=os.path.join(
run_dir, lab.experiment.STATIC_RUN_PROPERTIES_FILENAME))
dynamic_props = tools.Properties(
filename=os.path.join(run_dir, "properties"))
props = tools.Properties()
props.update(static_props)
props.update(dynamic_props)
driver_log = os.path.join(run_dir, "driver.log")
if not os.path.exists(driver_log):
props.add_unexplained_error(
"driver.log is missing. Probably the run was never started.")
driver_err = os.path.join(run_dir, "driver.err")
run_err = os.path.join(run_dir, "run.err")
for logfile in [driver_err, run_err]:
if os.path.exists(logfile):
with open(logfile) as f:
content = f.read()
if content:
props.add_unexplained_error("{}: {}".format(
os.path.basename(logfile), content))
return props
def parse(self):
"""Search all patterns and apply all functions.
The found values are written to the run's ``properties`` file.
"""
run_dir = os.path.abspath('.')
prop_file = os.path.join(run_dir, 'properties')
self.props = tools.Properties(filename=prop_file)
for filename, file_parser in self.file_parsers.items():
# If filename is absolute it will not be changed here.
path = os.path.join(run_dir, filename)
try:
file_parser.load_file(path)
except IOError as err:
if err.errno == errno.ENOENT:
logging.info(
'File "{path}" is missing and thus not parsed.'.format(
**locals()))
del self.file_parsers[filename]
else:
logging.error(
'Failed to read "{path}": {err}'.format(**locals()))
for file_parser in self.file_parsers.values():
self.props.update(file_parser.search_patterns())
for file_parser in self.file_parsers.values():
file_parser.apply_functions(self.props)
self.props.write()
def fetch_dir(self,
run_dir,
eval_dir,
copy_all=False,
run_filter=None,
parsers=None):
run_filter = run_filter or tools.RunFilter()
parsers = parsers or []
# Allow specyfing a list of multiple parsers or a single parser.
if not isinstance(parsers, (tuple, list)):
parsers = [parsers]
# Make sure parsers is a list.
parsers = list(parsers)
prop_file = os.path.join(run_dir, 'properties')
# Somehow '../..' gets inserted into sys.path and more strangely the
# system lab.tools module gets called.
# TODO: This HACK should be removed once the source of the error is clear.
props = tools.Properties(filename=prop_file)
if props.get('search_returncode'
) is not None and props.get("coverage") is None:
logging.warning('search_parser.py exited abnormally for %s' %
run_dir)
logging.info('Rerunning search_parser.py')
parsers.append(os.path.join(run_dir, '../../search_parser.py'))
for parser in parsers:
rel_parser = os.path.relpath(parser, start=run_dir)
subprocess.call([rel_parser], cwd=run_dir)
props = tools.Properties(filename=prop_file)
props = run_filter.apply_to_run(props)
if not props:
return None, None
run_id = props.get('id')
# Abort if an id cannot be read.
if not run_id:
logging.critical('id is not set in %s.' % prop_file)
if copy_all:
dest_dir = os.path.join(eval_dir, *run_id)
tools.makedirs(dest_dir)
tools.fast_updatetree(run_dir, dest_dir, symlinks=True)
return run_id, props
def _load_data(self):
props_file = os.path.join(self.eval_dir, "properties")
if not os.path.exists(props_file):
logging.critical(f"Properties file not found at {props_file}")
logging.info("Reading properties file")
self.props = tools.Properties(filename=props_file)
logging.info("Reading properties file finished")
if not self.props:
logging.critical("properties file in evaluation dir is empty.")
def _load_data(self):
props_file = os.path.join(self.eval_dir, 'properties')
if not os.path.exists(props_file):
logging.critical('Properties file not found at %s' % props_file)
logging.info('Reading properties file')
self.props = tools.Properties(filename=props_file)
logging.info('Reading properties file finished')
if not self.props:
logging.critical('properties file in evaluation dir is empty.')
def __init__(self, key_value_patterns=False):
"""
If *key_value_patterns* is True, the parser will parse all lines with the
following format automatically (underlying regex: r'^(.+): (\d+)$')::
My attribute: 89 --> props['my_attribute'] = 89
other attribute name: 1234 --> props['other_attribute_name'] = 1234
"""
self.file_parsers = defaultdict(_FileParser)
self.run_dir = os.path.abspath('.')
prop_file = os.path.join(self.run_dir, 'properties')
if not os.path.exists(prop_file):
logging.critical('No properties file found at "%s"' % prop_file)
self.props = tools.Properties(filename=prop_file)
if key_value_patterns:
self.add_function(parse_key_value_patterns)
def _build_properties_file(self, properties_filename):
combined_props = tools.Properties(
self._get_abs_path(properties_filename))
combined_props.update(self.properties)
combined_props.write()
def __init__(self):
self.resources = []
self.new_files = []
self.env_vars_relative = {}
self.commands = OrderedDict()
self.properties = tools.Properties()
def __call__(self,
src_dir,
eval_dir=None,
merge=None,
filter=None,
**kwargs):
"""
This method can be used to copy properties from an exp-dir or
eval-dir into an eval-dir. If the destination eval-dir already
exist, the data will be merged. This means *src_dir* can either
be an exp-dir or an eval-dir and *eval_dir* can be a new or
existing directory.
We recommend using lab.Experiment.add_fetcher() to add fetchers
to an experiment. See the method's documentation for a
description of the parameters.
"""
if not os.path.isdir(src_dir):
logging.critical(
"{} is missing or not a directory".format(src_dir))
run_filter = tools.RunFilter(filter, **kwargs)
eval_dir = eval_dir or src_dir.rstrip("/") + "-eval"
logging.info("Fetching properties from {} to {}".format(
src_dir, eval_dir))
if merge is None:
_check_eval_dir(eval_dir)
elif merge:
# No action needed, data will be merged.
pass
else:
tools.remove_path(eval_dir)
# Load properties in the eval_dir if there are any already.
combined_props = tools.Properties(os.path.join(eval_dir, "properties"))
fetch_from_eval_dir = not os.path.exists(
os.path.join(src_dir, "runs-00001-00100"))
if fetch_from_eval_dir:
src_props = tools.Properties(
filename=os.path.join(src_dir, "properties"))
run_filter.apply(src_props)
combined_props.update(src_props)
logging.info("Fetched properties of {} runs.".format(
len(src_props)))
else:
slurm_err_content = tools.get_slurm_err_content(src_dir)
if slurm_err_content:
logging.error("There was output to *-grid-steps/slurm.err")
new_props = tools.Properties()
run_dirs = sorted(glob(os.path.join(src_dir, "runs-*-*", "*")))
total_dirs = len(run_dirs)
logging.info(
"Scanning properties from {:d} run directories".format(
total_dirs))
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel,
"Scanning: {:6d}/{:d}".format(index, total_dirs))
props = self.fetch_dir(run_dir)
if slurm_err_content:
props.add_unexplained_error("output-to-slurm.err")
id_string = "-".join(props["id"])
new_props[id_string] = props
run_filter.apply(new_props)
combined_props.update(new_props)
unexplained_errors = 0
for props in combined_props.values():
error_message = tools.get_unexplained_errors_message(props)
if error_message:
logging.error(error_message)
unexplained_errors += 1
tools.makedirs(eval_dir)
combined_props.write()
logging.info("Wrote properties file (contains {unexplained_errors} "
"runs with unexplained errors).".format(**locals()))
def __init__(self):
self.resources = []
self.new_files = []
# List of glob-style patterns used to exclude files (not full paths).
self.ignores = []
self.properties = tools.Properties()
def fetch_dir(self, run_dir, src_dir, copy_all=False, run_filter=None, parsers=None):
logging.info('Now I will copy for PAC')
run_filter = run_filter or tools.RunFilter()
parsers = parsers or []
# Allow specyfing a list of multiple parsers or a single parser.
if not isinstance(parsers, (tuple, list)):
parsers = [parsers]
# Make sure parsers is a list.
parsers = list(parsers)
prop_file = os.path.join(run_dir, 'properties')
# Somehow '../..' gets inserted into sys.path and more strangely the
# system lab.tools module gets called.
# TODO: This HACK should be removed once the source of the error is clear.
props = tools.Properties(filename=prop_file)
if props.get('search_returncode') is not None and props.get("coverage") is None:
logging.warning('search_parser.py exited abnormally for %s' % run_dir)
logging.info('Rerunning search_parser.py')
parsers.append(os.path.join(run_dir, '../../search_parser.py'))
logging.info(props.get('domain'))
for parser in parsers:
rel_parser = os.path.relpath(parser, start=run_dir)
subprocess.call([rel_parser], cwd=run_dir)
props = tools.Properties(filename=prop_file)
props = run_filter.apply_to_run(props)
if not props:
return None, None
run_id = props.get('id')
# Abort if an id cannot be read.
if not run_id:
logging.critical('id is not set in %s.' % prop_file)
#dest_dir = os.path.join(run_dir, *run_id)
src_dir_hstar2h = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_ratio.csv'
src_dir_hstar = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_hstar.csv'
src_dir_stats = src_dir + '/' + props.get('domain') + '/' + 'PAC_Statistics.csv'
src_dir_hffToh = src_dir + '/' + props.get('domain') + '/' + 'PAC_Commulative_h-ff_to_h-star.csv'
run_dir_hstar2h = run_dir + '/' + 'PAC_Commulative_ratio.csv'
run_dir_hstar = run_dir + '/' + 'PAC_Commulative_hstar.csv'
run_dir_stats = run_dir + '/' + 'PAC_Statistics.csv'
run_dir_hffToh = run_dir + '/' + 'PAC_Commulative_h-ff_to_h-star.csv'
logging.info("run_dir: " + run_dir)
logging.info("src_dir: " + src_dir)
shutil.copy2(src_dir_hstar2h, run_dir_hstar2h)
shutil.copy2(src_dir_hstar, run_dir_hstar)
shutil.copy2(src_dir_stats, run_dir_stats)
shutil.copy2(src_dir_hffToh, run_dir_hffToh)
return run_id, props
def __call__(self,
src_dir,
eval_dir=None,
copy_all=False,
write_combined_props=True,
filter=None,
parsers=None,
**kwargs):
"""
This method can be used to copy properties from an exp-dir or eval-dir
into an eval-dir. If the destination eval-dir already exist, the data
will be merged. This means *src_dir* can either be an exp-dir or an
eval-dir and *eval_dir* can be a new or existing directory.
If *copy_all* is True (default: False), copy all files from the run
dirs to a new directory tree at *eval_dir*. Without this option only
the combined properties file is written do disk.
If *write_combined_props* is True (default), write the combined
properties file.
You can include only specific domains or configurations by using
:py:class:`filters <.Report>`.
*parsers* can be a list of paths to parser scripts. If given, each
parser is called in each run directory and the results are added to
the properties file which is fetched afterwards. This option is
useful if you haven't parsed all or some values already during the
experiment.
Examples:
Fetch all results and write a single combined properties file to the
default evaluation directory (this step is added by default)::
exp.add_step(Step('fetch', Fetcher(), exp.path))
Read the combined properties file at ``<eval_dir1>/properties`` and
merge it into the combined properties file at
``<combined_eval_dir>/properties``::
exp.add_step(Step('combine', Fetcher(), eval_dir1, combined_eval_dir))
Fetch only the runs for certain configuration from an older experiment::
exp.add_step(Step('fetch', Fetcher(), src_dir,
filter_config_nick=['config_1', 'config_5']))
"""
if not os.path.isdir(src_dir):
logging.critical('%s is not a valid directory' % src_dir)
run_filter = tools.RunFilter(filter, **kwargs)
src_props = tools.Properties(
filename=os.path.join(src_dir, 'properties'))
fetch_from_eval_dir = 'runs' not in src_props or src_dir.endswith(
'-eval')
if fetch_from_eval_dir:
src_props = run_filter.apply(src_props)
for prop in src_props.values():
if prop.get('error', '').startswith('unexplained'):
logging.warning("Unexplained error in '%s': %s" %
(prop.get('run_dir'), prop.get('error')))
eval_dir = eval_dir or src_dir.rstrip('/') + '-eval'
logging.info('Fetching files from %s -> %s' % (src_dir, eval_dir))
logging.info('Fetching from evaluation dir: %s' % fetch_from_eval_dir)
if write_combined_props:
# Load properties in the eval_dir if there are any already.
combined_props = tools.Properties(
os.path.join(eval_dir, 'properties'))
if fetch_from_eval_dir:
combined_props.update(src_props)
# Get all run_dirs. None will be found if we fetch from an eval dir.
run_dirs = sorted(glob(os.path.join(src_dir, 'runs-*-*', '*')))
total_dirs = len(run_dirs)
logging.info('Scanning properties from %d run directories' %
total_dirs)
unxeplained_errors = 0
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel, 'Scanning: %6d/%d' % (index, total_dirs))
run_id, props = self.fetch_dir(run_dir,
eval_dir,
copy_all=copy_all,
run_filter=run_filter,
parsers=parsers)
if props is None:
continue
assert run_id, 'Dir %s has no id' % props.get('run_dir')
if write_combined_props:
combined_props['-'.join(run_id)] = props
if props.get('error', '').startswith('unexplained'):
logging.warning(
'Unexplained error in {run_dir}: {error}'.format(**props))
unxeplained_errors += 1
if unxeplained_errors:
logging.warning('There were %d runs with unexplained errors.' %
unxeplained_errors)
tools.makedirs(eval_dir)
if write_combined_props:
combined_props.write()
