def run_parsers():
if not os.path.isdir(self.path):
logging.critical(f"{self.path} is missing or not a directory")
# Copy all parsers from their source to their destination again.
self._build_resources(only_parsers=True)
run_dirs = sorted(glob(os.path.join(self.path, "runs-*-*", "*")))
total_dirs = len(run_dirs)
logging.info(
f"Parsing properties in {total_dirs:d} run directories")
for index, run_dir in enumerate(run_dirs, start=1):
if os.path.exists(os.path.join(run_dir, "properties")):
tools.remove_path(os.path.join(run_dir, "properties"))
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel,
f"Parsing run: {index:6d}/{total_dirs:d}")
for resource in self.resources:
if resource.is_parser:
parser_filename = self.env_vars_relative[resource.name]
rel_parser = os.path.join("../../", parser_filename)
# Since parsers often produce output which we would
# rather not want to see for each individual run, we
# suppress it here.
subprocess.check_call(
[tools.get_python_executable(), rel_parser],
cwd=run_dir,
stdout=subprocess.DEVNULL,
)
def run_parsers():
if not os.path.isdir(self.path):
logging.critical('{} is missing or not a directory'.format(
self.path))
# Copy all parsers from their source to their destination again.
self._build_resources(only_parsers=True)
run_dirs = sorted(glob(os.path.join(self.path, 'runs-*-*', '*')))
total_dirs = len(run_dirs)
logging.info('Parsing properties in {:d} run directories'.format(
total_dirs))
for index, run_dir in enumerate(run_dirs, start=1):
if os.path.exists(os.path.join(run_dir, 'properties')):
# print "removing path {}".format(os.path.join(run_dir, 'properties'))
tools.remove_path(os.path.join(run_dir, 'properties'))
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(
loglevel,
'Parsing run: {:6d}/{:d}'.format(index, total_dirs))
for resource in self.resources:
if resource.is_parser:
parser_filename = self.env_vars_relative[resource.name]
rel_parser = os.path.join('../../', parser_filename)
with open(os.devnull, 'w') as devnull:
# Since parsers often produce output which we would
# rather not want to see for each individual run, we
# suppress it here.
subprocess.check_call([rel_parser],
cwd=run_dir,
stdout=devnull)
def cache(self, revision_cache):
self.path = os.path.join(revision_cache, self.name)
if os.path.exists(self.path):
logging.info(f'Revision is already cached: "{self.path}"')
if not os.path.exists(self._get_sentinel_file()):
logging.critical(
f"The build for the cached revision at {self.path} is corrupted. "
f"Please delete it and try again.")
else:
tools.makedirs(self.path)
tar_archive = os.path.join(self.path, "solver.tgz")
cmd = ["git", "archive", "--format", "tar", self.global_rev]
with open(tar_archive, "w") as f:
retcode = tools.run_command(cmd, stdout=f, cwd=self.repo)
if retcode == 0:
with tarfile.open(tar_archive) as tf:
tf.extractall(self.path)
tools.remove_path(tar_archive)
for exclude_dir in self.exclude:
path = os.path.join(self.path, exclude_dir)
if os.path.exists(path):
tools.remove_path(path)
if retcode != 0:
shutil.rmtree(self.path)
logging.critical("Failed to make checkout.")
self._compile()
self._cleanup()
def _check_eval_dir(eval_dir):
if os.path.exists(eval_dir):
answer = (input(
f"{eval_dir} already exists. Do you want to (o)verwrite it, "
f"(m)erge the results, or (c)ancel? ").strip().lower())
if answer == "o":
tools.remove_path(eval_dir)
elif answer == "m":
pass
elif answer == "c":
sys.exit()
else:
# Abort for "cancel" and invalid answers.
logging.critical(f'Invalid answer: "{answer}"')
def cache(self, revision_cache):
self.path = os.path.join(revision_cache, self.name)
if os.path.exists(self.path):
logging.info('Revision is already cached: "%s"' % self.path)
if not os.path.exists(self._get_sentinel_file()):
logging.critical(
"The build for the cached revision at {} is corrupted. "
"Please delete it and try again.".format(self.path)
)
else:
tools.makedirs(self.path)
vcs = get_version_control_system(self.repo)
if vcs == MERCURIAL:
retcode = tools.run_command(
["hg", "archive", "-r", self.global_rev]
+ [f"-X{d}" for d in self.exclude]
+ [self.path],
cwd=self.repo,
)
elif vcs == GIT:
tar_archive = os.path.join(self.path, "solver.tgz")
cmd = ["git", "archive", "--format", "tar", self.global_rev]
with open(tar_archive, "w") as f:
retcode = tools.run_command(cmd, stdout=f, cwd=self.repo)
if retcode == 0:
with tarfile.open(tar_archive) as tf:
tf.extractall(self.path)
tools.remove_path(tar_archive)
for exclude_dir in self.exclude:
path = os.path.join(self.path, exclude_dir)
if os.path.exists(path):
tools.remove_path(path)
else:
_raise_unknown_vcs_error(vcs)
if retcode != 0:
shutil.rmtree(self.path)
logging.critical("Failed to make checkout.")
self._compile()
self._cleanup()
def run_steps(self, steps):
"""
We can't submit jobs from within the grid, so we submit them
all at once with dependencies. We also can't rewrite the job
files after they have been submitted.
"""
self.exp.build(write_to_disk=False)
# Prepare job dir.
job_dir = self.exp.path + "-grid-steps"
if os.path.exists(job_dir):
tools.confirm_or_abort(
f'The path "{job_dir}" already exists, so the experiment has '
f"already been submitted. Are you sure you want to "
f"delete the grid-steps and submit it again?"
)
tools.remove_path(job_dir)
# Overwrite exp dir if it exists.
if any(is_build_step(step) for step in steps):
self.exp._remove_experiment_dir()
# Remove eval dir if it exists.
if os.path.exists(self.exp.eval_dir):
tools.confirm_or_abort(
f'The evaluation directory "{self.exp.eval_dir}" already exists. '
f"Do you want to remove it?"
)
tools.remove_path(self.exp.eval_dir)
# Create job dir only when we need it.
tools.makedirs(job_dir)
prev_job_id = None
for step in steps:
job_name = self._get_job_name(step)
job_file = os.path.join(job_dir, job_name)
job_content = self._get_job(step, is_last=(step == steps[-1]))
tools.write_file(job_file, job_content)
prev_job_id = self._submit_job(
job_name, job_file, job_dir, dependency=prev_job_id
)
def _cleanup(self):
# Only keep the bin directories in "builds" dir.
for path in glob.glob(os.path.join(self.path, "builds", "*", "*")):
if os.path.basename(path) != 'bin':
tools.remove_path(path)
# Remove unneeded files.
tools.remove_path(self.get_cached_path('build.py'))
# Strip binaries.
binaries = []
for path in glob.glob(
os.path.join(self.path, "builds", "*", "bin", "*")):
if os.path.basename(path) in ['downward', 'preprocess']:
binaries.append(path)
subprocess.call(['strip'] + binaries)
# Compress src directory.
subprocess.call(['tar', '-cf', 'src.tar', '--remove-files', 'src'],
cwd=self.path)
subprocess.call(['xz', 'src.tar'], cwd=self.path)
def _cleanup(self):
# Only keep the bin directories in "builds" dir.
for path in glob.glob(os.path.join(self.path, "builds", "*", "*")):
if os.path.basename(path) != "bin":
tools.remove_path(path)
# Remove unneeded files.
tools.remove_path(self.get_cached_path("build.py"))
# Strip binaries.
binaries = []
for path in glob.glob(
os.path.join(self.path, "builds", "*", "bin", "*")):
if os.path.basename(path) in ["downward", "preprocess"]:
binaries.append(path)
subprocess.call(["strip"] + binaries)
# Compress src directory.
subprocess.call(["tar", "-cf", "src.tar", "--remove-files", "src"],
cwd=self.path)
subprocess.call(["xz", "src.tar"], cwd=self.path)
def _cleanup(self):
# Remove unneeded files.
tools.remove_path(self.get_cached_path('.build'))
tools.remove_path(self.get_cached_path('build'))
tools.remove_path(self.get_cached_path('vendor'))
# Strip binaries.
binaries = []
for path in glob.glob(os.path.join(self.path, "*.bin")):
binaries.append(path)
subprocess.call(['strip'] + binaries)
# Compress src directory.
subprocess.call(
['tar', '-cf', 'src.tar', '--remove-files', 'src', 'submodules'],
cwd=self.path)
subprocess.call(['xz', 'src.tar'], cwd=self.path)
def _remove_experiment_dir(self):
if os.path.exists(self.path):
tools.confirm_overwrite_or_abort(self.path)
tools.remove_path(self.path)
def __call__(self,
src_dir,
eval_dir=None,
merge=None,
filter=None,
**kwargs):
"""
This method can be used to copy properties from an exp-dir or
eval-dir into an eval-dir. If the destination eval-dir already
exist, the data will be merged. This means *src_dir* can either
be an exp-dir or an eval-dir and *eval_dir* can be a new or
existing directory.
We recommend using lab.Experiment.add_fetcher() to add fetchers
to an experiment. See the method's documentation for a
description of the parameters.
"""
if not os.path.isdir(src_dir):
logging.critical(
"{} is missing or not a directory".format(src_dir))
run_filter = tools.RunFilter(filter, **kwargs)
eval_dir = eval_dir or src_dir.rstrip("/") + "-eval"
logging.info("Fetching properties from {} to {}".format(
src_dir, eval_dir))
if merge is None:
_check_eval_dir(eval_dir)
elif merge:
# No action needed, data will be merged.
pass
else:
tools.remove_path(eval_dir)
# Load properties in the eval_dir if there are any already.
combined_props = tools.Properties(os.path.join(eval_dir, "properties"))
fetch_from_eval_dir = not os.path.exists(
os.path.join(src_dir, "runs-00001-00100"))
if fetch_from_eval_dir:
src_props = tools.Properties(
filename=os.path.join(src_dir, "properties"))
run_filter.apply(src_props)
combined_props.update(src_props)
logging.info("Fetched properties of {} runs.".format(
len(src_props)))
else:
slurm_err_content = tools.get_slurm_err_content(src_dir)
if slurm_err_content:
logging.error("There was output to *-grid-steps/slurm.err")
new_props = tools.Properties()
run_dirs = sorted(glob(os.path.join(src_dir, "runs-*-*", "*")))
total_dirs = len(run_dirs)
logging.info(
"Scanning properties from {:d} run directories".format(
total_dirs))
for index, run_dir in enumerate(run_dirs, start=1):
loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG
logging.log(loglevel,
"Scanning: {:6d}/{:d}".format(index, total_dirs))
props = self.fetch_dir(run_dir)
if slurm_err_content:
props.add_unexplained_error("output-to-slurm.err")
id_string = "-".join(props["id"])
new_props[id_string] = props
run_filter.apply(new_props)
combined_props.update(new_props)
unexplained_errors = 0
for props in combined_props.values():
error_message = tools.get_unexplained_errors_message(props)
if error_message:
logging.error(error_message)
unexplained_errors += 1
tools.makedirs(eval_dir)
combined_props.write()
logging.info("Wrote properties file (contains {unexplained_errors} "
"runs with unexplained errors).".format(**locals()))
