def _build_resources(self):
for name, dest, content in self.new_files:
filename = self._get_abs_path(dest)
tools.makedirs(os.path.dirname(filename))
with open(filename, 'w') as file:
logging.debug('Writing file "%s"' % filename)
file.write(content)
if dest == 'run':
# Make run script executable.
# TODO: Replace by adding an "executable" kwarg in add_new_file().
os.chmod(filename, 0755)
for name, source, dest, required, symlink in self.resources:
if required and not os.path.exists(source):
logging.critical('Required resource not found: %s' % source)
dest = self._get_abs_path(dest)
if not dest.startswith(self.path):
# Only copy resources that reside in the experiment/run dir.
continue
if symlink:
# Do not create a symlink if the file doesn't exist.
if not os.path.exists(source):
continue
source = self._get_rel_path(source)
os.symlink(source, dest)
logging.debug('Linking from %s to %s' % (source, dest))
continue
# Even if the directory containing a resource has already been added,
# we copy the resource since we might want to overwrite it.
logging.debug('Copying %s to %s' % (source, dest))
tools.copy(source, dest, required, self.ignores)
def _build_resources(self):
for name, dest, content in self.new_files:
filename = self._get_abs_path(dest)
tools.makedirs(os.path.dirname(filename))
with open(filename, 'w') as file:
logging.debug('Writing file "%s"' % filename)
file.write(content)
if dest == 'run':
# Make run script executable.
# TODO: Replace by adding an "executable" kwarg in add_new_file().
os.chmod(filename, 0755)
for name, source, dest, required, symlink in self.resources:
if required and not os.path.exists(source):
logging.critical('Required resource not found: %s' % source)
dest = self._get_abs_path(dest)
if not dest.startswith(self.path):
# Only copy resources that reside in the experiment/run dir.
continue
if symlink:
# Do not create a symlink if the file doesn't exist.
if not os.path.exists(source):
continue
source = self._get_rel_path(source)
os.symlink(source, dest)
logging.debug('Linking from %s to %s' % (source, dest))
continue
# Even if the directory containing a resource has already been added,
# we copy the resource since we might want to overwrite it.
logging.debug('Copying %s to %s' % (source, dest))
tools.copy(source, dest, required, self.ignores)
def create_commulative_h_ff_to_h_table(self, domain, table):
print "---------------------------"
print table
print "---------------------------"
#calculate number of solved problems
total_solved = len(table)
ratio_set = list(set(table)) # remove duplicate element
ratio_dict = {}
for value in sorted(ratio_set):
smaller_than_value_counter = 0
for compared_value in table:
if compared_value <= value:
smaller_than_value_counter += 1
ratio_dict[value] = smaller_than_value_counter * 100 / total_solved
print ratio_dict
print "---------------------------"
#write results into .cvs file:
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Commulative_h-ff_to_h-star.csv'
file = open(domain_file, "w")
sorted_ratio_dict_keys = sorted(ratio_dict.keys())
for hstar in sorted_ratio_dict_keys:
toWrite = str(hstar) + ',' + str(ratio_dict[hstar]) + '\n'
file.write(toWrite)
file.close()
def build(self, overwrite=False, only_main_script=False, no_main_script=False):
"""Apply all the actions to the filesystem.
If *overwrite* is True and the experiment directory exists, it is
overwritten without prior confirmation.
"""
logging.info('Exp Dir: "%s"' % self.path)
self._set_run_dirs()
# TODO: Currently no_main_script is always False.
if not no_main_script:
# This is the first part where we only write the main script.
# We only overwrite the exp dir in the first part.
if os.path.exists(self.path):
runs_exist = any(path.startswith('runs')
for path in os.listdir(self.path))
logging.info('The directory "%s" contains run directories: %s' %
(self.path, runs_exist))
# Overwrite if overwrite is True or if no runs exist.
tools.overwrite_dir(self.path, overwrite or not runs_exist)
else:
tools.makedirs(self.path)
self._build_main_script()
if only_main_script:
return
# This is the second part where we write everything else
self._build_resources()
self._build_runs()
self._build_properties_file()
def create_commulative_h_ff_to_h_table(self, domain, table):
print "---------------------------"
print table
print "---------------------------"
#calculate number of solved problems
total_solved = len(table)
ratio_set = list(set(table)) # remove duplicate element
ratio_dict = {}
for value in sorted(ratio_set):
smaller_than_value_counter = 0;
for compared_value in table:
if compared_value <= value:
smaller_than_value_counter += 1
ratio_dict[value] = smaller_than_value_counter*100 / total_solved
print ratio_dict
print "---------------------------"
#write results into .cvs file:
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Commulative_h-ff_to_h-star.csv'
file = open(domain_file, "w")
sorted_ratio_dict_keys = sorted(ratio_dict.keys())
for hstar in sorted_ratio_dict_keys:
toWrite = str(hstar) + ',' + str(ratio_dict[hstar]) + '\n'
file.write(toWrite)
file.close()
def save_stat_table_to_file(self, domain, table):
#write results into .cvs file:
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Statistics.csv'
outfile = open(domain_file, "w")
first_line = 'Problem,isSolved,h*,ff_h(s),h(s),h*/h(s)\n'
outfile.write(first_line)
for prob in table:
row = table[prob]
# print '----------------------'
# print row
# print '----------------------'
probName = prob
isSolved = row['WORK-lmcut']
h_star = row['h*']
ff_h = row['ff_h(s)']
h = row['h(s)']
ratio = row['h*/h(s)']
curr_line = str(probName) + ',' + str(isSolved) + ',' + str(h_star) + ',' + str(ff_h) + ',' + str(h) + ',' + str(ratio) + '\n'
outfile.write(curr_line)
def cache(self, revision_cache):
self._path = os.path.join(revision_cache, self._hashed_name)
if os.path.exists(self.path):
logging.info('Revision is already cached: "%s"' % self.path)
if not os.path.exists(self._get_sentinel_file()):
logging.critical(
'The build for the cached revision at {} is corrupted '
'or was made with an older Lab version. Please delete '
'it and try again.'.format(self.path))
else:
tools.makedirs(self.path)
if not os.path.exists(os.path.join(self.repo, 'export.sh')):
logging.critical(
'export.sh script not found. Make sure you\'re using a recent version of the planner.'
)
# First export the main repo
script = os.path.join(self.repo, "export.sh")
retcode = tools.run_command((script, self.global_rev, self.path),
cwd=self.repo)
if retcode != 0:
shutil.rmtree(self.path)
logging.critical('Failed to make checkout.')
self._compile()
self._cleanup()
def _build_new_files(self):
for dest, content, permissions in self.new_files:
filename = self._get_abs_path(dest)
tools.makedirs(os.path.dirname(filename))
logging.debug(f'Writing file "{filename}"')
tools.write_file(filename, content)
os.chmod(filename, permissions)
def cache(self, revision_cache):
self.path = os.path.join(revision_cache, self.name)
if os.path.exists(self.path):
logging.info(f'Revision is already cached: "{self.path}"')
if not os.path.exists(self._get_sentinel_file()):
logging.critical(
f"The build for the cached revision at {self.path} is corrupted. "
f"Please delete it and try again.")
else:
tools.makedirs(self.path)
tar_archive = os.path.join(self.path, "solver.tgz")
cmd = ["git", "archive", "--format", "tar", self.global_rev]
with open(tar_archive, "w") as f:
retcode = tools.run_command(cmd, stdout=f, cwd=self.repo)
if retcode == 0:
with tarfile.open(tar_archive) as tf:
tf.extractall(self.path)
tools.remove_path(tar_archive)
for exclude_dir in self.exclude:
path = os.path.join(self.path, exclude_dir)
if os.path.exists(path):
tools.remove_path(path)
if retcode != 0:
shutil.rmtree(self.path)
logging.critical("Failed to make checkout.")
self._compile()
self._cleanup()
def build(self,
overwrite=False,
only_main_script=False,
no_main_script=False):
"""Apply all the actions to the filesystem.
If *overwrite* is True and the experiment directory exists, it is
overwritten without prior confirmation.
"""
logging.info('Exp Dir: "%s"' % self.path)
self._set_run_dirs()
# TODO: Currently no_main_script is always False.
if not no_main_script:
# This is the first part where we only write the main script.
# We only overwrite the exp dir in the first part.
if os.path.exists(self.path):
runs_exist = any(
path.startswith('runs') for path in os.listdir(self.path))
logging.info(
'The directory "%s" contains run directories: %s' %
(self.path, runs_exist))
# Overwrite if overwrite is True or if no runs exist.
tools.overwrite_dir(self.path, overwrite or not runs_exist)
else:
tools.makedirs(self.path)
self._build_main_script()
if only_main_script:
return
# This is the second part where we write everything else
self._build_resources()
self._build_runs()
self._build_properties_file()
def save_stat_table_to_file(self, domain, table):
#write results into .cvs file:
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Statistics.csv'
outfile = open(domain_file, "w")
first_line = 'Problem,isSolved,h*,ff_h(s),h(s),h*/h(s)\n'
outfile.write(first_line)
for prob in table:
row = table[prob]
# print '----------------------'
# print row
# print '----------------------'
probName = prob
isSolved = row['WORK-lmcut']
h_star = row['h*']
ff_h = row['ff_h(s)']
h = row['h(s)']
ratio = row['h*/h(s)']
curr_line = str(probName) + ',' + str(isSolved) + ',' + str(
h_star) + ',' + str(ff_h) + ',' + str(h) + ',' + str(
ratio) + '\n'
outfile.write(curr_line)
outfile.close()
def write(cls, report, filename):
lines = ([
r'\documentclass[tikz]{standalone}', r'\usepackage{pgfplots}',
r'\begin{document}', r'\begin{tikzpicture}'
] + cls._get_plot(report) + [r'\end{tikzpicture}', r'\end{document}'])
tools.makedirs(os.path.dirname(filename))
tools.write_file(filename, '\n'.join(lines))
logging.info('Wrote file://%s' % filename)
def write(self):
if not len(self.algorithms) == 2:
logging.critical(
f"Scatter plots need exactly 2 algorithms: {self.algorithms}")
suffix = "." + self.output_format
if not self.outfile.endswith(suffix):
self.outfile += suffix
tools.makedirs(os.path.dirname(self.outfile))
self._write_plot(self.runs.values(), self.outfile)
def write(cls, report, filename, scatter=False):
lines = []
lines.append('\\begin{tikzpicture}')
lines.extend(cls._get_plot(report))
lines.append('\\end{tikzpicture}')
tools.makedirs(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write('\n'.join(lines))
logging.info('Wrote file://%s' % filename)
def write(cls, report, filename, scatter=False):
lines = []
lines.append('\\begin{tikzpicture}')
lines.extend(cls._get_plot(report))
lines.append('\\end{tikzpicture}')
tools.makedirs(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write('\n'.join(lines))
logging.info('Wrote file://%s' % filename)
def write(cls, report, filename):
lines = ([
r"\documentclass[tikz]{standalone}",
r"\usepackage{pgfplots}",
r"\begin{document}",
r"\begin{tikzpicture}",
] + cls._get_plot(report) + [r"\end{tikzpicture}", r"\end{document}"])
tools.makedirs(os.path.dirname(filename))
tools.write_file(filename, "\n".join(lines))
logging.info(f"Wrote file://{filename}")
def write(self):
"""
Overwrite this method if you want to write the report directly. You
should write the report to *self.outfile*.
"""
content = self.get_text()
tools.makedirs(os.path.dirname(self.outfile))
with open(self.outfile, 'w') as file:
file.write(content)
logging.info('Wrote file://%s' % self.outfile)
def write(self):
"""
Overwrite this method if you want to write the report directly. You
should write the report to *self.outfile*.
"""
content = self.get_text()
tools.makedirs(os.path.dirname(self.outfile))
with open(self.outfile, 'w') as file:
file.write(content)
logging.info('Wrote file://%s' % self.outfile)
def write(self):
if not len(self.configs) == 2:
logging.critical('Scatterplots need exactly 2 configs: %s' % self.configs)
self.xlabel = self.xlabel or self.configs[0]
self.ylabel = self.ylabel or self.configs[1]
suffix = '.' + self.output_format
if not self.outfile.endswith(suffix):
self.outfile += suffix
tools.makedirs(os.path.dirname(self.outfile))
self._write_plot(self.runs.values(), self.outfile)
def write(self):
if not len(self.algorithms) == 2:
logging.critical('Scatter plots need exactly 2 algorithms: %s' %
self.algorithms)
self.xlabel = self.xlabel or self.algorithms[0]
self.ylabel = self.ylabel or self.algorithms[1]
suffix = '.' + self.output_format
if not self.outfile.endswith(suffix):
self.outfile += suffix
tools.makedirs(os.path.dirname(self.outfile))
self._write_plot(self.runs.values(), self.outfile)
def write(self):
if len(self.algorithms) < 1:
logging.critical('Cactus plots need >=1 algorithms: %s' %
self.algorithms)
self.xlabel = self.xlabel or "Time (s)"
self.ylabel = self.ylabel or "Number/ Count"
suffix = '.' + self.output_format
if not self.outfile.endswith(suffix):
self.outfile += suffix
tools.makedirs(os.path.dirname(self.outfile))
self._write_plot(self.runs.values(), self.outfile)
def write(self):
logging.info('Using score attribute "%s"' % self.score)
logging.info('Adding column with best value: %s' %
self.best_value_column)
self.total_scores = self._compute_total_scores()
tools.makedirs(os.path.dirname(self.outfile))
with open(self.outfile, 'w') as file:
sys.stdout = file
self.print_report()
sys.stdout = sys.__stdout__
logging.info('Wrote file://%s' % self.outfile)
def write(self):
if not (len(self.algorithms) == 1 and self.x_algo == self.algorithms[0]
and self.y_algo == self.algorithms[0]):
logging.critical(
'Scatter plots need exactly 1 algorithm that must match x_algo and y_algo: %s, %s, %s'
% (self.algorithms, self.x_algo, self.y_algo))
self.xlabel = self.xlabel or self.x_algo + ": " + self.x_attribute
self.ylabel = self.ylabel or self.y_algo + ": " + self.y_attribute
suffix = '.' + self.output_format
if not self.outfile.endswith(suffix):
self.outfile += suffix
tools.makedirs(os.path.dirname(self.outfile))
self._write_plot(self.runs.values(), self.outfile)
def write(self):
"""
Write the report files.
By default this method calls :meth:`.get_text` and writes the
obtained text to *outfile*.
Overwrite this method if you want to write the report file(s)
directly. You should write them to *self.outfile*.
"""
content = self.get_text()
tools.makedirs(os.path.dirname(self.outfile))
tools.write_file(self.outfile, content)
logging.info(f"Wrote file://{self.outfile}")
def fetch_dir(self,
run_dir,
eval_dir,
copy_all=False,
run_filter=None,
parsers=None):
run_filter = run_filter or tools.RunFilter()
parsers = parsers or []
# Allow specyfing a list of multiple parsers or a single parser.
if not isinstance(parsers, (tuple, list)):
parsers = [parsers]
# Make sure parsers is a list.
parsers = list(parsers)
prop_file = os.path.join(run_dir, 'properties')
# Somehow '../..' gets inserted into sys.path and more strangely the
# system lab.tools module gets called.
# TODO: This HACK should be removed once the source of the error is clear.
props = tools.Properties(filename=prop_file)
if props.get('search_returncode'
) is not None and props.get("coverage") is None:
logging.warning('search_parser.py exited abnormally for %s' %
run_dir)
logging.info('Rerunning search_parser.py')
parsers.append(os.path.join(run_dir, '../../search_parser.py'))
for parser in parsers:
rel_parser = os.path.relpath(parser, start=run_dir)
subprocess.call([rel_parser], cwd=run_dir)
props = tools.Properties(filename=prop_file)
props = run_filter.apply_to_run(props)
if not props:
return None, None
run_id = props.get('id')
# Abort if an id cannot be read.
if not run_id:
logging.critical('id is not set in %s.' % prop_file)
if copy_all:
dest_dir = os.path.join(eval_dir, *run_id)
tools.makedirs(dest_dir)
tools.fast_updatetree(run_dir, dest_dir, symlinks=True)
return run_id, props
def _cache(self, compilation_options):
path = self.get_path()
if os.path.exists(path):
logging.info('Revision is already cached: "%s"' % path)
if not os.path.exists(self._sentinel_file):
logging.critical(
'The build for the cached revision at "%s" is corrupted '
'or was made with an older lab version. Please delete '
'it and try again.' % path)
else:
tools.makedirs(path)
retcode = tools.run_command(
['hg', 'archive', '-r', self.rev, '-I', 'src', path], cwd=self.repo)
if retcode != 0:
shutil.rmtree(path)
logging.critical('Failed to make checkout.')
self._compile(compilation_options)
self._cleanup()
def _cache(self, compilation_options):
path = self.get_path()
if os.path.exists(path):
logging.info('Revision is already cached: "%s"' % path)
if not os.path.exists(self._sentinel_file):
logging.critical(
'The build for the cached revision at "%s" is corrupted '
'or was made with an older lab version. Please delete '
'it and try again.' % path)
else:
tools.makedirs(path)
retcode = tools.run_command(
['hg', 'archive', '-r', self.rev, '-I', 'src', path],
cwd=self.repo)
if retcode != 0:
shutil.rmtree(path)
logging.critical('Failed to make checkout.')
self._compile(compilation_options)
self._cleanup()
def run_steps(self, steps):
"""
We can't submit jobs from within the grid, so we submit them
all at once with dependencies. We also can't rewrite the job
files after they have been submitted.
"""
self.exp.build(write_to_disk=False)
# Prepare job dir.
job_dir = self.exp.path + "-grid-steps"
if os.path.exists(job_dir):
tools.confirm_or_abort(
f'The path "{job_dir}" already exists, so the experiment has '
f"already been submitted. Are you sure you want to "
f"delete the grid-steps and submit it again?"
)
tools.remove_path(job_dir)
# Overwrite exp dir if it exists.
if any(is_build_step(step) for step in steps):
self.exp._remove_experiment_dir()
# Remove eval dir if it exists.
if os.path.exists(self.exp.eval_dir):
tools.confirm_or_abort(
f'The evaluation directory "{self.exp.eval_dir}" already exists. '
f"Do you want to remove it?"
)
tools.remove_path(self.exp.eval_dir)
# Create job dir only when we need it.
tools.makedirs(job_dir)
prev_job_id = None
for step in steps:
job_name = self._get_job_name(step)
job_file = os.path.join(job_dir, job_name)
job_content = self._get_job(step, is_last=(step == steps[-1]))
tools.write_file(job_file, job_content)
prev_job_id = self._submit_job(
job_name, job_file, job_dir, dependency=prev_job_id
)
def cache(self, revision_cache):
self.path = os.path.join(revision_cache, self.name)
if os.path.exists(self.path):
logging.info('Revision is already cached: "%s"' % self.path)
if not os.path.exists(self._get_sentinel_file()):
logging.critical(
"The build for the cached revision at {} is corrupted. "
"Please delete it and try again.".format(self.path)
)
else:
tools.makedirs(self.path)
vcs = get_version_control_system(self.repo)
if vcs == MERCURIAL:
retcode = tools.run_command(
["hg", "archive", "-r", self.global_rev]
+ [f"-X{d}" for d in self.exclude]
+ [self.path],
cwd=self.repo,
)
elif vcs == GIT:
tar_archive = os.path.join(self.path, "solver.tgz")
cmd = ["git", "archive", "--format", "tar", self.global_rev]
with open(tar_archive, "w") as f:
retcode = tools.run_command(cmd, stdout=f, cwd=self.repo)
if retcode == 0:
with tarfile.open(tar_archive) as tf:
tf.extractall(self.path)
tools.remove_path(tar_archive)
for exclude_dir in self.exclude:
path = os.path.join(self.path, exclude_dir)
if os.path.exists(path):
tools.remove_path(path)
else:
_raise_unknown_vcs_error(vcs)
if retcode != 0:
shutil.rmtree(self.path)
logging.critical("Failed to make checkout.")
self._compile()
self._cleanup()
def regression_test_handler(test, rev, success):
if not success:
tools.makedirs(REGRESSIONS_DIR)
tarball = os.path.join(REGRESSIONS_DIR,
"{test}-{rev}.tar.gz".format(**locals()))
subprocess.check_call([
"tar", "-czf", tarball, "-C", BASE_DIR,
os.path.relpath(EXPERIMENTS_DIR, start=BASE_DIR)
])
logging.error(
"Regression found. To inspect the experiment data for the failed regression test, run\n"
"sudo ./extract-regression-experiment.sh {test}-{rev}\n"
"in the ~/infrastructure/hosts/linux-buildbot-worker directory "
"on the Linux buildbot computer.".format(**locals()))
exp_dir = get_exp_dir(rev, test)
eval_dir = exp_dir + "-eval"
shutil.rmtree(exp_dir)
shutil.rmtree(eval_dir)
if not success:
sys.exit(1)
def __init__(self, path, environment=None, cache_dir=None):
"""
Create a new experiment that will be built at *path* using the methods
provided by :ref:`Environment <environments>` *environment*. If
*environment* is None, ``LocalEnvironment`` is used (default).
Lab will use the *cache_dir* for storing temporary files.
In case you run :py:class:`Fast Downward experiments
<downward.experiments.DownwardExperiment>` this directory can become
very large (tens of GB) since it is used to cache revisions and
preprocessed tasks. By default *cache_dir* points to ``~/lab``.
An experiment consists of multiple steps. Every experiment will need at
least the following steps:
* Build the experiment.
* Run it.
* Fetch the results.
* Make a report.
In the "Run it" step all runs that have been added to the experiment
will be executed. Each run consists of one or multiple commands.
"""
_Buildable.__init__(self)
self.path = os.path.abspath(path)
if any(char in self.path for char in (':', ',')):
logging.critical('Path contains commas or colons: %s' % self.path)
self.environment = environment or LocalEnvironment()
self.environment.exp = self
self.cache_dir = cache_dir or tools.DEFAULT_USER_DIR
tools.makedirs(self.cache_dir)
self.shard_size = SHARD_SIZE
self.runs = []
self.set_property('experiment_file', self._script)
self.steps = Sequence()
self.add_step(Step('build', self.build))
self.add_step(Step('start', self.run))
self.add_fetcher(name='fetch')
def __init__(self, path, environment=None, cache_dir=None):
"""
Create a new experiment that will be built at *path* using the methods
provided by :ref:`Environment <environments>` *environment*. If
*environment* is None, ``LocalEnvironment`` is used (default).
Lab will use the *cache_dir* for storing temporary files.
In case you run :py:class:`Fast Downward experiments
<downward.experiments.DownwardExperiment>` this directory can become
very large (tens of GB) since it is used to cache revisions and
preprocessed tasks. By default *cache_dir* points to ``~/lab``.
An experiment consists of multiple steps. Every experiment will need at
least the following steps:
* Build the experiment.
* Run it.
* Fetch the results.
* Make a report.
In the "Run it" step all runs that have been added to the experiment
will be executed. Each run consists of one or multiple commands.
"""
_Buildable.__init__(self)
self.path = os.path.abspath(path)
if any(char in self.path for char in (':', ',')):
logging.critical('Path contains commas or colons: %s' % self.path)
self.environment = environment or LocalEnvironment()
self.environment.exp = self
self.cache_dir = cache_dir or tools.DEFAULT_USER_DIR
tools.makedirs(self.cache_dir)
self.shard_size = SHARD_SIZE
self.runs = []
self.set_property('experiment_file', self._script)
self.steps = Sequence()
self.add_step(Step('build', self.build))
self.add_step(Step('start', self.run))
self.add_fetcher(name='fetch')
def cache(self, revision_cache):
self._path = os.path.join(revision_cache, self._hashed_name)
if os.path.exists(self.path):
logging.info('Revision is already cached: "%s"' % self.path)
if not os.path.exists(self._get_sentinel_file()):
logging.critical(
'The build for the cached revision at {} is corrupted '
'or was made with an older Lab version. Please delete '
'it and try again.'.format(self.path))
else:
tools.makedirs(self.path)
excludes = ['-X{}'.format(d) for d in ['experiments', 'misc']]
retcode = tools.run_command(
['hg', 'archive', '-r', self.global_rev] + excludes +
[self.path],
cwd=self.repo)
if retcode != 0:
shutil.rmtree(self.path)
logging.critical('Failed to make checkout.')
self._compile()
self._cleanup()
def create_commulative_h_star_table(self, domain):
#get relevant value from original table
cost_attr = Attribute('cost', min_wins=False, absolute=True)
cost_table = AbsoluteReport._get_table(self, cost_attr, domain)
#define list of costs:
cost_list = []
#calculate number of solved problems
total_solved = 0
for row in cost_table.row_names:
curr_val = cost_table.get(row)
val = curr_val[self.nick]
if val > 0:
total_solved = total_solved + 1
cost_list.append(val)
cost_set = list(set(cost_list)) # remove duplicate element
cost_dict = {}
for value in sorted(cost_set):
smaller_than_value_counter = 0;
for compared_value in cost_list:
if compared_value <= value:
smaller_than_value_counter += 1
cost_dict[value] = smaller_than_value_counter*100 / total_solved
#write results into .cvs file:
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Commulative_hstar.csv'
file = open(domain_file, "w")
sorted_cost_dict_keys = sorted(cost_dict.keys())
for hstar in sorted_cost_dict_keys:
toWrite = str(hstar) + ',' + str(cost_dict[hstar]) + '\n'
file.write(toWrite)
file.close()
def fetch_dir(self, run_dir, eval_dir, copy_all=False, run_filter=None, parsers=None):
run_filter = run_filter or tools.RunFilter()
parsers = parsers or []
# Allow specyfing a list of multiple parsers or a single parser.
if not isinstance(parsers, (tuple, list)):
parsers = [parsers]
# Make sure parsers is a list.
parsers = list(parsers)
prop_file = os.path.join(run_dir, 'properties')
# Somehow '../..' gets inserted into sys.path and more strangely the
# system lab.tools module gets called.
# TODO: This HACK should be removed once the source of the error is clear.
props = tools.Properties(filename=prop_file)
if props.get('search_returncode') is not None and props.get("coverage") is None:
logging.warning('search_parser.py exited abnormally for %s' % run_dir)
logging.info('Rerunning search_parser.py')
parsers.append(os.path.join(run_dir, '../../search_parser.py'))
for parser in parsers:
rel_parser = os.path.relpath(parser, start=run_dir)
subprocess.call([rel_parser], cwd=run_dir)
props = tools.Properties(filename=prop_file)
props = run_filter.apply_to_run(props)
if not props:
return None, None
run_id = props.get('id')
# Abort if an id cannot be read.
if not run_id:
logging.critical('id is not set in %s.' % prop_file)
if copy_all:
dest_dir = os.path.join(eval_dir, *run_id)
tools.makedirs(dest_dir)
tools.fast_updatetree(run_dir, dest_dir, symlinks=True)
return run_id, props
def create_commulative_h_star_table(self, domain):
#get relevant value from original table
cost_attr = Attribute('cost', min_wins=False, absolute=True)
cost_table = AbsoluteReport._get_table(self, cost_attr, domain)
#define list of costs:
cost_list = []
#calculate number of solved problems
total_solved = 0
for row in cost_table.row_names:
curr_val = cost_table.get(row)
val = curr_val[self.nick]
if val > 0:
total_solved = total_solved + 1
cost_list.append(val)
cost_set = list(set(cost_list)) # remove duplicate element
cost_dict = {}
for value in sorted(cost_set):
smaller_than_value_counter = 0
for compared_value in cost_list:
if compared_value <= value:
smaller_than_value_counter += 1
cost_dict[value] = smaller_than_value_counter * 100 / total_solved
#write results into .cvs file:
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Commulative_hstar.csv'
file = open(domain_file, "w")
sorted_cost_dict_keys = sorted(cost_dict.keys())
for hstar in sorted_cost_dict_keys:
toWrite = str(hstar) + ',' + str(cost_dict[hstar]) + '\n'
file.write(toWrite)
file.close()
def build(self, write_to_disk=True):
"""
Finalize the internal data structures, then write all files
needed for the experiment to disk.
If *write_to_disk* is False, only compute the internal data
structures. This is only needed on grids for
FastDownwardExperiments.build() which turns the added algorithms
and benchmarks into Runs.
"""
if not write_to_disk:
return
logging.info(f'Experiment path: "{self.path}"')
self._remove_experiment_dir()
tools.makedirs(self.path)
self.environment.write_main_script()
self._build_new_files()
self._build_resources()
self._build_runs()
self._build_properties_file(STATIC_EXPERIMENT_PROPERTIES_FILENAME)
def __call__(self,
src_dir,
eval_dir=None,
merge=None,
filter=None,
**kwargs):
"""
This method can be used to copy properties from an exp-dir or
eval-dir into an eval-dir. If the destination eval-dir already
exist, the data will be merged. This means *src_dir* can either
be an exp-dir or an eval-dir and *eval_dir* can be a new or
existing directory.
We recommend using lab.Experiment.add_fetcher() to add fetchers
to an experiment. See the method's documentation for a
description of the parameters.
"""
if not os.path.isdir(src_dir):
logging.critical(
"{} is missing or not a directory".format(src_dir))
run_filter = tools.RunFilter(filter, **kwargs)
eval_dir = eval_dir or src_dir.rstrip("/") + "-eval"
logging.info("Fetching properties from {} to {}".format(
src_dir, eval_dir))
if merge is None:
_check_eval_dir(eval_dir)
elif merge:
# No action needed, data will be merged.
pass
else:
tools.remove_path(eval_dir)
# Load properties in the eval_dir if there are any already.
combined_props = tools.Properties(os.path.join(eval_dir, "properties"))
fetch_from_eval_dir = not os.path.exists(
os.path.join(src_dir, "runs-00001-00100"))
if fetch_from_eval_dir:
src_props = tools.Properties(
filename=os.path.join(src_dir, "properties"))
run_filter.apply(src_props)
combined_props.update(src_props)
logging.info("Fetched properties of {} runs.".format(
len(src_props)))
else:
slurm_err_content = tools.get_slurm_err_content(src_dir)
if slurm_err_content:
logging.error("There was output to *-grid-steps/slurm.err")
new_props = tools.Properties()
run_dirs = sorted(glob(os.path.join(src_dir, "runs-*-*", "*")))
total_dirs = len(run_dirs)
logging.info(
"Scanning properties from {:d} run directories".format(
total_dirs))
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel,
"Scanning: {:6d}/{:d}".format(index, total_dirs))
props = self.fetch_dir(run_dir)
if slurm_err_content:
props.add_unexplained_error("output-to-slurm.err")
id_string = "-".join(props["id"])
new_props[id_string] = props
run_filter.apply(new_props)
combined_props.update(new_props)
unexplained_errors = 0
for props in combined_props.values():
error_message = tools.get_unexplained_errors_message(props)
if error_message:
logging.error(error_message)
unexplained_errors += 1
tools.makedirs(eval_dir)
combined_props.write()
logging.info("Wrote properties file (contains {unexplained_errors} "
"runs with unexplained errors).".format(**locals()))
def __init__(self, path, repo, environment=None, combinations=None,
compact=True, limits=None, cache_dir=None):
"""
The experiment will be built at *path*.
*repo* must be the path to a Fast Downward repository. Among other things
this repository is used to search for benchmark files.
*environment* must be an :ref:`Environment <environments>` instance.
By default the experiment is run locally.
If given, *combinations* must be a list of :ref:`Checkout <checkouts>`
tuples of the form (Translator, Preprocessor, Planner). If combinations
is None (default), perform an experiment with the working copy in *repo*.
The *compact* parameter is only relevant for the search
stage. If *compact* is ``False``, the preprocessed task and
the two PDDL files are **copied** into the respective run
directories for all configurations. This requires a lot of
space (tens of GB), so it is strongly recommended to use the
default (``compact=True``) which only references these
files. Use ``compact=False`` only if you really need a
portable experiment.
If *limits* is given, it must be a dictionary that maps a
subset of the keys below to seconds and MiB. It will be used
to overwrite the default limits::
default_limits = {
'translate_time': 7200,
'translate_memory': 8192,
'preprocess_time': 7200,
'preprocess_memory': 8192,
'search_time': 1800,
'search_memory': 2048,
}
*cache_dir* is used to cache Fast Downward clones and preprocessed
tasks. By default it points to ``~/lab``.
.. note::
The directory *cache_dir* can grow very large (tens of GB).
Example: ::
repo = '/path/to/downward-repo'
env = GkiGridEnvironment(queue='xeon_core.q', priority=-2)
combos = [(Translator(repo, rev=123),
Preprocessor(repo, rev='e2a018c865f7'),
Planner(repo, rev='tip')]
exp = DownwardExperiment('/tmp/path', repo, environment=env,
combinations=combos,
limits={'search_time': 30,
'search_memory': 1024})
"""
Experiment.__init__(self, path, environment=environment, cache_dir=cache_dir)
if not repo or not os.path.isdir(repo):
logging.critical('The path "%s" is not a local Fast Downward '
'repository.' % repo)
self.repo = repo
self.orig_path = self.path
self.search_exp_path = self.path
self.preprocess_exp_path = self.path + '-p'
self._path_to_python = None
Checkout.REV_CACHE_DIR = os.path.join(self.cache_dir, 'revision-cache')
self.preprocessed_tasks_dir = os.path.join(self.cache_dir, 'preprocessed-tasks')
tools.makedirs(self.preprocessed_tasks_dir)
self.combinations = (combinations or
[(Translator(repo), Preprocessor(repo), Planner(repo))])
self.compact = compact
self.suites = defaultdict(list)
self._algorithms = []
self._portfolios = []
limits = limits or {}
for key, value in limits.items():
if key not in LIMITS:
logging.critical('Unknown limit: %s' % key)
self.limits = LIMITS
self.limits.update(limits)
# Save if this is a compact experiment i.e. preprocessed tasks are referenced.
self.set_property('compact', compact)
# TODO: Integrate this into the API.
self.include_preprocess_results_in_search_runs = True
self.compilation_options = ['-j%d' % self._jobs]
self._search_parsers = []
self.add_search_parser(os.path.join(DOWNWARD_SCRIPTS_DIR, 'search_parser.py'))
# Remove the default experiment steps
self.steps = Sequence()
self.add_step(Step('build-preprocess-exp', self.build, stage='preprocess'))
self.add_step(Step('run-preprocess-exp', self.run, stage='preprocess'))
self.add_fetcher(src=self.preprocess_exp_path,
dest=self.preprocessed_tasks_dir,
name='fetch-preprocess-results',
copy_all=True,
write_combined_props=False)
self.add_step(Step('build-search-exp', self.build, stage='search'))
self.add_PAC_fetcher(src='/home/gal-d/downward/lab/examples/PAC_Preprocess_Output-eval/preprocess',#TODO change to be parameter
dest=self.search_exp_path,
name='fetch-preprocess-results',
copy_all=True,
write_combined_props=False)#new featcher to copy preprocess for PAC results
self.add_step(Step('run-search-exp', self.run, stage='search'))
self.add_fetcher(src=self.search_exp_path, name='fetch-search-results')
def write(self):
if os.path.isfile(self.outfile):
logging.critical('outfile must be a directory for this report.')
tools.makedirs(self.outfile)
self._write_plots(self.outfile)
def _get_commulative_table(self, domain):
#init new table
title = 'Commulative'
columns = {'Percentage','h*/h(s)'}
min_wins = False
colored = True
table = reports.Table(title=title, min_wins=min_wins, colored=colored)
table.set_column_order(columns)
link = '#%s' % title
formatter = reports.CellFormatter(link=link)
table.cell_formatters[table.header_row][table.header_column] = formatter
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Commulative_ratio.csv'
file = open(domain_file, "w")
#get relevant value from original table
ratio_attr = Attribute('hstar_to_h', min_wins=False, absolute=True)
ratio_table = AbsoluteReport._get_table(self, ratio_attr, domain)
#define arrays to work
ratios = [0.75,0.8,0.85,0.9,0.95,1,1.05,1.1,1.15,1.2,1.25,1.3,1.35,1.4,1.45,1.5,1.55,1.6,1.65,1.7,1.75,1.8,1.85,1.9,1.95,2,2.05,2.1,2.15,2.2,2.25,2.3,2.35,2.4,2.45,2.5,2.55,2.6,2.65,2.7,2.75,2.8,2.85,2.9,2.95,3.0,3.05,3.1,3.15,3.2,3.25,3.3,3.35,3.4,3.45,3.5,3.55,3.6,3.65,3.7,3.75,3.80,3.85,3.9,3.95,4.0,4.05,4.1,4.15,4.2,2.25,4.3,4.35,4.4,4.45,4.5]
names = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','za','zb','zc','zd','ze','zf','zg','zh','zi','zj','zk','zl','zm','zn','zo','aa','ab','ac','ad','ae','af','ag','ah','ai','aj','ak','al','am','an','ao','ap','aq','ar','as','at', 'au','av','aw','ax','ay','az','ba','bb','bc','bd','be','bf','bg','bh','bi']
counter = 0
#calculate number of solved problems
total_solved = 0
for row in ratio_table.row_names:
curr_val = ratio_table.get(row)
val = curr_val[self.nick]
if val > 0:
total_solved = total_solved + 1
#for each ratio (1,1.05...), find the number of problems with this ratio, calc percentage and add row
for ratio in ratios:
_sum = 0
for row in ratio_table.row_names:
curr_val = ratio_table.get(row)
val = curr_val[self.nick]
if val <= ratio and val > 0:
_sum = _sum + 1
if total_solved == 0:
_sum_percent = 0
else:
_sum_percent = _sum*100 / total_solved
#add new row
row_to_add = {}
row_to_add['Percentage'] = _sum_percent
row_to_add['h*/h(s)'] = ratio
table.add_row(names[counter],row_to_add)
counter = counter + 1
#TODO - save only one ratio per percentage
toWrite = str(ratio)+','+str(_sum_percent)+'\n'
file.write(toWrite)
file.close()
self.create_commulative_h_star_table(domain)
return table
def write_properties(eval_dir):
tools.makedirs(eval_dir)
with open(os.path.join(eval_dir, 'properties'), 'w') as f:
json.dump(PROPERTIES, f)
def _get_commulative_table(self, domain):
#init new table
title = 'Commulative'
columns = {'Percentage', 'h*/h(s)'}
min_wins = False
colored = True
table = reports.Table(title=title, min_wins=min_wins, colored=colored)
table.set_column_order(columns)
link = '#%s' % title
formatter = reports.CellFormatter(link=link)
table.cell_formatters[table.header_row][
table.header_column] = formatter
domain_dir = self.outFile + '/' + domain
tools.makedirs(domain_dir)
domain_file = domain_dir + '/' + 'PAC_Commulative_ratio.csv'
file = open(domain_file, "w")
#get relevant value from original table
ratio_attr = Attribute('hstar_to_h', min_wins=False, absolute=True)
ratio_table = AbsoluteReport._get_table(self, ratio_attr, domain)
#define arrays to work
ratios = [
0.75, 0.8, 0.85, 0.9, 0.95, 1, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3,
1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7, 1.75, 1.8, 1.85, 1.9,
1.95, 2, 2.05, 2.1, 2.15, 2.2, 2.25, 2.3, 2.35, 2.4, 2.45, 2.5,
2.55, 2.6, 2.65, 2.7, 2.75, 2.8, 2.85, 2.9, 2.95, 3.0, 3.05, 3.1,
3.15, 3.2, 3.25, 3.3, 3.35, 3.4, 3.45, 3.5, 3.55, 3.6, 3.65, 3.7,
3.75, 3.80, 3.85, 3.9, 3.95, 4.0, 4.05, 4.1, 4.15, 4.2, 2.25, 4.3,
4.35, 4.4, 4.45, 4.5
]
names = [
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
'za', 'zb', 'zc', 'zd', 'ze', 'zf', 'zg', 'zh', 'zi', 'zj', 'zk',
'zl', 'zm', 'zn', 'zo', 'aa', 'ab', 'ac', 'ad', 'ae', 'af', 'ag',
'ah', 'ai', 'aj', 'ak', 'al', 'am', 'an', 'ao', 'ap', 'aq', 'ar',
'as', 'at', 'au', 'av', 'aw', 'ax', 'ay', 'az', 'ba', 'bb', 'bc',
'bd', 'be', 'bf', 'bg', 'bh', 'bi'
]
counter = 0
#calculate number of solved problems
total_solved = 0
for row in ratio_table.row_names:
curr_val = ratio_table.get(row)
val = curr_val[self.nick]
if val > 0:
total_solved = total_solved + 1
#for each ratio (1,1.05...), find the number of problems with this ratio, calc percentage and add row
for ratio in ratios:
_sum = 0
for row in ratio_table.row_names:
curr_val = ratio_table.get(row)
val = curr_val[self.nick]
if val <= ratio and val > 0:
_sum = _sum + 1
if total_solved == 0:
_sum_percent = 0
else:
_sum_percent = _sum * 100 / total_solved
#add new row
row_to_add = {}
row_to_add['Percentage'] = _sum_percent
row_to_add['h*/h(s)'] = ratio
table.add_row(names[counter], row_to_add)
counter = counter + 1
#TODO - save only one ratio per percentage
toWrite = str(ratio) + ',' + str(_sum_percent) + '\n'
file.write(toWrite)
file.close()
self.create_commulative_h_star_table(domain)
return table
def __init__(self, path, repo, environment=None, combinations=None,
compact=True, limits=None, cache_dir=None):
"""
The experiment will be built at *path*.
*repo* must be the path to a Fast Downward repository. Among other things
this repository is used to search for benchmark files.
*environment* must be an :ref:`Environment <environments>` instance.
By default the experiment is run locally.
If given, *combinations* must be a list of :ref:`Checkout <checkouts>`
tuples of the form (Translator, Preprocessor, Planner). If combinations
is None (default), perform an experiment with the working copy in *repo*.
The *compact* parameter is only relevant for the search
stage. If *compact* is ``False``, the preprocessed task and
the two PDDL files are **copied** into the respective run
directories for all configurations. This requires a lot of
space (tens of GB), so it is strongly recommended to use the
default (``compact=True``) which only references these
files. Use ``compact=False`` only if you really need a
portable experiment.
If *limits* is given, it must be a dictionary that maps a
subset of the keys below to seconds and MiB. It will be used
to overwrite the default limits::
default_limits = {
'translate_time': 7200,
'translate_memory': 8192,
'preprocess_time': 7200,
'preprocess_memory': 8192,
'search_time': 1800,
'search_memory': 2048,
}
*cache_dir* is used to cache Fast Downward clones and preprocessed
tasks. By default it points to ``~/lab``.
.. note::
The directory *cache_dir* can grow very large (tens of GB).
Example: ::
repo = '/path/to/downward-repo'
env = GkiGridEnvironment(queue='xeon_core.q', priority=-2)
combos = [(Translator(repo, rev=123),
Preprocessor(repo, rev='e2a018c865f7'),
Planner(repo, rev='tip')]
exp = DownwardExperiment('/tmp/path', repo, environment=env,
combinations=combos,
limits={'search_time': 30,
'search_memory': 1024})
"""
Experiment.__init__(self, path, environment=environment, cache_dir=cache_dir)
#if not repo or not os.path.isdir(repo):
# logging.critical('The path "%s" is not a local Fast Downward '
# 'repository.' % repo)
self.repo = repo
self.orig_path = self.path
self.search_exp_path = self.path
self.preprocess_exp_path = self.path + '-p'
self._path_to_python = None
Checkout.REV_CACHE_DIR = os.path.join(self.cache_dir, 'revision-cache')
self.preprocessed_tasks_dir = os.path.join(self.cache_dir, 'preprocessed-tasks')
tools.makedirs(self.preprocessed_tasks_dir)
self.combinations = (combinations or
[(Translator(repo), Preprocessor(repo), Planner(repo))])
self.compact = compact
self.suites = defaultdict(list)
self._algorithms = []
self._portfolios = []
limits = limits or {}
for key, value in limits.items():
if key not in LIMITS:
logging.critical('Unknown limit: %s' % key)
self.limits = LIMITS
self.limits.update(limits)
# Save if this is a compact experiment i.e. preprocessed tasks are referenced.
self.set_property('compact', compact)
# TODO: Integrate this into the API.
self.include_preprocess_results_in_search_runs = True
self.compilation_options = ['-j%d' % self._jobs]
self._search_parsers = []
self.add_search_parser(os.path.join(DOWNWARD_SCRIPTS_DIR, 'search_parser.py'))
# Remove the default experiment steps
self.steps = Sequence()
self.add_step(Step('build-preprocess-exp', self.build, stage='preprocess'))
self.add_step(Step('run-preprocess-exp', self.run, stage='preprocess'))
self.add_fetcher(src=self.preprocess_exp_path,
dest=self.preprocessed_tasks_dir,
name='fetch-preprocess-results',
copy_all=True,
write_combined_props=False)
self.add_step(Step('build-search-exp', self.build, stage='search'))
self.add_PAC_fetcher(src='/home/sternron/gal-dreiman/downward/lab/examples/PAC_Preprocess_Output-eval/preprocess',#TODO change to be parameter
dest=self.search_exp_path,
name='fetch-preprocess-results',
copy_all=True,
write_combined_props=False)#new featcher to copy preprocess for PAC results
self.add_step(Step('run-search-exp', self.run, stage='search'))
self.add_fetcher(src=self.search_exp_path, name='fetch-search-results')
def __call__(self, src_dir, eval_dir=None, copy_all=False, write_combined_props=True,
filter=None, parsers=None, **kwargs):
"""
This method can be used to copy properties from an exp-dir or eval-dir
into an eval-dir. If the destination eval-dir already exist, the data
will be merged. This means *src_dir* can either be an exp-dir or an
eval-dir and *eval_dir* can be a new or existing directory.
If *copy_all* is True (default: False), copy all files from the run
dirs to a new directory tree at *eval_dir*. Without this option only
the combined properties file is written do disk.
If *write_combined_props* is True (default), write the combined
properties file.
You can include only specific domains or configurations by using
:py:class:`filters <.Report>`.
*parsers* can be a list of paths to parser scripts. If given, each
parser is called in each run directory and the results are added to
the properties file which is fetched afterwards. This option is
useful if you haven't parsed all or some values already during the
experiment.
Examples:
Fetch all results and write a single combined properties file to the
default evaluation directory (this step is added by default)::
exp.add_step(Step('fetch', Fetcher(), exp.path))
Read the combined properties file at ``<eval_dir1>/properties`` and
merge it into the combined properties file at
``<combined_eval_dir>/properties``::
exp.add_step(Step('combine', Fetcher(), eval_dir1, combined_eval_dir))
Fetch only the runs for certain configuration from an older experiment::
exp.add_step(Step('fetch', Fetcher(), src_dir,
filter_config_nick=['config_1', 'config_5']))
"""
if not os.path.isdir(src_dir):
logging.critical('%s is not a valid directory' % src_dir)
run_filter = tools.RunFilter(filter, **kwargs)
src_props = tools.Properties(filename=os.path.join(src_dir, 'properties'))
fetch_from_eval_dir = 'runs' not in src_props or src_dir.endswith('-eval')
if fetch_from_eval_dir:
src_props = run_filter.apply(src_props)
for prop in src_props.values():
if prop.get('error', '').startswith('unexplained'):
logging.warning("Unexplained error in '%s': %s" %
(prop.get('run_dir'), prop.get('error')))
eval_dir = eval_dir or src_dir.rstrip('/') + '-eval'
logging.info('Fetching files from %s -> %s' % (src_dir, eval_dir))
logging.info('Fetching from evaluation dir: %s' % fetch_from_eval_dir)
if write_combined_props:
# Load properties in the eval_dir if there are any already.
combined_props = tools.Properties(os.path.join(eval_dir, 'properties'))
if fetch_from_eval_dir:
combined_props.update(src_props)
# Get all run_dirs. None will be found if we fetch from an eval dir.
run_dirs = sorted(glob(os.path.join(src_dir, 'runs-*-*', '*')))
total_dirs = len(run_dirs)
logging.info('Scanning properties from %d run directories' % total_dirs)
unxeplained_errors = 0
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel, 'Scanning: %6d/%d' % (index, total_dirs))
run_id, props = self.fetch_dir(run_dir, eval_dir, copy_all=copy_all,
run_filter=run_filter, parsers=parsers)
if props is None:
continue
assert run_id, 'Dir %s has no id' % props.get('run_dir')
if write_combined_props:
combined_props['-'.join(run_id)] = props
if props.get('error', '').startswith('unexplained'):
logging.warning('Unexplained error in {run_dir}: {error}'.format(**props))
unxeplained_errors += 1
if unxeplained_errors:
logging.warning('There were %d runs with unexplained errors.'
% unxeplained_errors)
tools.makedirs(eval_dir)
if write_combined_props:
combined_props.write()
