def run_parsers(): if not os.path.isdir(self.path): logging.critical(f"{self.path} is missing or not a directory") # Copy all parsers from their source to their destination again. self._build_resources(only_parsers=True) run_dirs = sorted(glob(os.path.join(self.path, "runs-*-*", "*"))) total_dirs = len(run_dirs) logging.info( f"Parsing properties in {total_dirs:d} run directories") for index, run_dir in enumerate(run_dirs, start=1): if os.path.exists(os.path.join(run_dir, "properties")): tools.remove_path(os.path.join(run_dir, "properties")) loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG logging.log(loglevel, f"Parsing run: {index:6d}/{total_dirs:d}") for resource in self.resources: if resource.is_parser: parser_filename = self.env_vars_relative[resource.name] rel_parser = os.path.join("../../", parser_filename) # Since parsers often produce output which we would # rather not want to see for each individual run, we # suppress it here. subprocess.check_call( [tools.get_python_executable(), rel_parser], cwd=run_dir, stdout=subprocess.DEVNULL, )
def run_parsers(): if not os.path.isdir(self.path): logging.critical('{} is missing or not a directory'.format( self.path)) # Copy all parsers from their source to their destination again. self._build_resources(only_parsers=True) run_dirs = sorted(glob(os.path.join(self.path, 'runs-*-*', '*'))) total_dirs = len(run_dirs) logging.info('Parsing properties in {:d} run directories'.format( total_dirs)) for index, run_dir in enumerate(run_dirs, start=1): if os.path.exists(os.path.join(run_dir, 'properties')): # print "removing path {}".format(os.path.join(run_dir, 'properties')) tools.remove_path(os.path.join(run_dir, 'properties')) loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG logging.log( loglevel, 'Parsing run: {:6d}/{:d}'.format(index, total_dirs)) for resource in self.resources: if resource.is_parser: parser_filename = self.env_vars_relative[resource.name] rel_parser = os.path.join('../../', parser_filename) with open(os.devnull, 'w') as devnull: # Since parsers often produce output which we would # rather not want to see for each individual run, we # suppress it here. subprocess.check_call([rel_parser], cwd=run_dir, stdout=devnull)
def cache(self, revision_cache): self.path = os.path.join(revision_cache, self.name) if os.path.exists(self.path): logging.info(f'Revision is already cached: "{self.path}"') if not os.path.exists(self._get_sentinel_file()): logging.critical( f"The build for the cached revision at {self.path} is corrupted. " f"Please delete it and try again.") else: tools.makedirs(self.path) tar_archive = os.path.join(self.path, "solver.tgz") cmd = ["git", "archive", "--format", "tar", self.global_rev] with open(tar_archive, "w") as f: retcode = tools.run_command(cmd, stdout=f, cwd=self.repo) if retcode == 0: with tarfile.open(tar_archive) as tf: tf.extractall(self.path) tools.remove_path(tar_archive) for exclude_dir in self.exclude: path = os.path.join(self.path, exclude_dir) if os.path.exists(path): tools.remove_path(path) if retcode != 0: shutil.rmtree(self.path) logging.critical("Failed to make checkout.") self._compile() self._cleanup()
def _check_eval_dir(eval_dir): if os.path.exists(eval_dir): answer = (input( f"{eval_dir} already exists. Do you want to (o)verwrite it, " f"(m)erge the results, or (c)ancel? ").strip().lower()) if answer == "o": tools.remove_path(eval_dir) elif answer == "m": pass elif answer == "c": sys.exit() else: # Abort for "cancel" and invalid answers. logging.critical(f'Invalid answer: "{answer}"')
def cache(self, revision_cache): self.path = os.path.join(revision_cache, self.name) if os.path.exists(self.path): logging.info('Revision is already cached: "%s"' % self.path) if not os.path.exists(self._get_sentinel_file()): logging.critical( "The build for the cached revision at {} is corrupted. " "Please delete it and try again.".format(self.path) ) else: tools.makedirs(self.path) vcs = get_version_control_system(self.repo) if vcs == MERCURIAL: retcode = tools.run_command( ["hg", "archive", "-r", self.global_rev] + [f"-X{d}" for d in self.exclude] + [self.path], cwd=self.repo, ) elif vcs == GIT: tar_archive = os.path.join(self.path, "solver.tgz") cmd = ["git", "archive", "--format", "tar", self.global_rev] with open(tar_archive, "w") as f: retcode = tools.run_command(cmd, stdout=f, cwd=self.repo) if retcode == 0: with tarfile.open(tar_archive) as tf: tf.extractall(self.path) tools.remove_path(tar_archive) for exclude_dir in self.exclude: path = os.path.join(self.path, exclude_dir) if os.path.exists(path): tools.remove_path(path) else: _raise_unknown_vcs_error(vcs) if retcode != 0: shutil.rmtree(self.path) logging.critical("Failed to make checkout.") self._compile() self._cleanup()
def run_steps(self, steps): """ We can't submit jobs from within the grid, so we submit them all at once with dependencies. We also can't rewrite the job files after they have been submitted. """ self.exp.build(write_to_disk=False) # Prepare job dir. job_dir = self.exp.path + "-grid-steps" if os.path.exists(job_dir): tools.confirm_or_abort( f'The path "{job_dir}" already exists, so the experiment has ' f"already been submitted. Are you sure you want to " f"delete the grid-steps and submit it again?" ) tools.remove_path(job_dir) # Overwrite exp dir if it exists. if any(is_build_step(step) for step in steps): self.exp._remove_experiment_dir() # Remove eval dir if it exists. if os.path.exists(self.exp.eval_dir): tools.confirm_or_abort( f'The evaluation directory "{self.exp.eval_dir}" already exists. ' f"Do you want to remove it?" ) tools.remove_path(self.exp.eval_dir) # Create job dir only when we need it. tools.makedirs(job_dir) prev_job_id = None for step in steps: job_name = self._get_job_name(step) job_file = os.path.join(job_dir, job_name) job_content = self._get_job(step, is_last=(step == steps[-1])) tools.write_file(job_file, job_content) prev_job_id = self._submit_job( job_name, job_file, job_dir, dependency=prev_job_id )
def _cleanup(self): # Only keep the bin directories in "builds" dir. for path in glob.glob(os.path.join(self.path, "builds", "*", "*")): if os.path.basename(path) != 'bin': tools.remove_path(path) # Remove unneeded files. tools.remove_path(self.get_cached_path('build.py')) # Strip binaries. binaries = [] for path in glob.glob( os.path.join(self.path, "builds", "*", "bin", "*")): if os.path.basename(path) in ['downward', 'preprocess']: binaries.append(path) subprocess.call(['strip'] + binaries) # Compress src directory. subprocess.call(['tar', '-cf', 'src.tar', '--remove-files', 'src'], cwd=self.path) subprocess.call(['xz', 'src.tar'], cwd=self.path)
def _cleanup(self): # Only keep the bin directories in "builds" dir. for path in glob.glob(os.path.join(self.path, "builds", "*", "*")): if os.path.basename(path) != "bin": tools.remove_path(path) # Remove unneeded files. tools.remove_path(self.get_cached_path("build.py")) # Strip binaries. binaries = [] for path in glob.glob( os.path.join(self.path, "builds", "*", "bin", "*")): if os.path.basename(path) in ["downward", "preprocess"]: binaries.append(path) subprocess.call(["strip"] + binaries) # Compress src directory. subprocess.call(["tar", "-cf", "src.tar", "--remove-files", "src"], cwd=self.path) subprocess.call(["xz", "src.tar"], cwd=self.path)
def _cleanup(self): # Remove unneeded files. tools.remove_path(self.get_cached_path('.build')) tools.remove_path(self.get_cached_path('build')) tools.remove_path(self.get_cached_path('vendor')) # Strip binaries. binaries = [] for path in glob.glob(os.path.join(self.path, "*.bin")): binaries.append(path) subprocess.call(['strip'] + binaries) # Compress src directory. subprocess.call( ['tar', '-cf', 'src.tar', '--remove-files', 'src', 'submodules'], cwd=self.path) subprocess.call(['xz', 'src.tar'], cwd=self.path)
def _remove_experiment_dir(self): if os.path.exists(self.path): tools.confirm_overwrite_or_abort(self.path) tools.remove_path(self.path)
def __call__(self, src_dir, eval_dir=None, merge=None, filter=None, **kwargs): """ This method can be used to copy properties from an exp-dir or eval-dir into an eval-dir. If the destination eval-dir already exist, the data will be merged. This means *src_dir* can either be an exp-dir or an eval-dir and *eval_dir* can be a new or existing directory. We recommend using lab.Experiment.add_fetcher() to add fetchers to an experiment. See the method's documentation for a description of the parameters. """ if not os.path.isdir(src_dir): logging.critical( "{} is missing or not a directory".format(src_dir)) run_filter = tools.RunFilter(filter, **kwargs) eval_dir = eval_dir or src_dir.rstrip("/") + "-eval" logging.info("Fetching properties from {} to {}".format( src_dir, eval_dir)) if merge is None: _check_eval_dir(eval_dir) elif merge: # No action needed, data will be merged. pass else: tools.remove_path(eval_dir) # Load properties in the eval_dir if there are any already. combined_props = tools.Properties(os.path.join(eval_dir, "properties")) fetch_from_eval_dir = not os.path.exists( os.path.join(src_dir, "runs-00001-00100")) if fetch_from_eval_dir: src_props = tools.Properties( filename=os.path.join(src_dir, "properties")) run_filter.apply(src_props) combined_props.update(src_props) logging.info("Fetched properties of {} runs.".format( len(src_props))) else: slurm_err_content = tools.get_slurm_err_content(src_dir) if slurm_err_content: logging.error("There was output to *-grid-steps/slurm.err") new_props = tools.Properties() run_dirs = sorted(glob(os.path.join(src_dir, "runs-*-*", "*"))) total_dirs = len(run_dirs) logging.info( "Scanning properties from {:d} run directories".format( total_dirs)) for index, run_dir in enumerate(run_dirs, start=1): loglevel = logging.INFO if index % 100 == 0 else logging.DEBUG logging.log(loglevel, "Scanning: {:6d}/{:d}".format(index, total_dirs)) props = self.fetch_dir(run_dir) if slurm_err_content: props.add_unexplained_error("output-to-slurm.err") id_string = "-".join(props["id"]) new_props[id_string] = props run_filter.apply(new_props) combined_props.update(new_props) unexplained_errors = 0 for props in combined_props.values(): error_message = tools.get_unexplained_errors_message(props) if error_message: logging.error(error_message) unexplained_errors += 1 tools.makedirs(eval_dir) combined_props.write() logging.info("Wrote properties file (contains {unexplained_errors} " "runs with unexplained errors).".format(**locals()))