def __clone_needed__(repository: str, directory: str) -> bool: """ Do we need to create a fresh clone of the given repository. Args: repository: the repository we want to clone. directory: the directory we expect the clone to live. Returns: True, if the clone is required. False, if the directory is a valid clone. """ from benchbuild.utils.cmd import git, rm git_dir = local.path(directory) / '.git' if not git_dir.exists(): return True requires_clone = True with local.cwd(directory): repo_origin_url = git('config', '--get', 'remote.origin.url') requires_clone = repo_origin_url.strip('\n') != repository if requires_clone: rm('-r', directory)
def source_required(src_file): """ Check, if a download is required. Args: src_file: The filename to check for. src_root: The path we find the file in. Returns: True, if we need to download something, False otherwise. """ if not src_file.exists(): return True required = True hash_file = src_file.with_suffix(".hash", depth=0) LOG.debug("Hash file location: %s", hash_file) if hash_file.exists(): new_hash = get_hash_of_dirs(src_file) with open(hash_file, 'r') as h_file: old_hash = h_file.readline() required = not new_hash == old_hash if required: from benchbuild.utils.cmd import rm rm("-r", src_file) rm(hash_file) if required: LOG.info("Source required for: %s", src_file) LOG.debug("Reason: src-exists: %s hash-exists: %s", src_file.exists(), hash_file.exists()) return required
def source_required(src_file, src_root): """ Check, if a download is required. Args: src_file: The filename to check for. src_root: The path we find the file in. Returns: True, if we need to download something, False otherwise. """ from os import path # Check if we need to do something src_dir = path.join(src_root, src_file) hash_file = path.join(src_root, src_file + ".hash") required = True if path.exists(src_dir) and path.exists(hash_file): new_hash = get_hash_of_dirs(src_dir) with open(hash_file, 'r') as h_file: old_hash = h_file.readline() required = not new_hash == old_hash if required: from benchbuild.utils.cmd import rm rm("-r", src_dir) rm(hash_file) return required
def setup_container(builddir, _container): """Prepare the container and returns the path where it can be found.""" build_dir = local.path(builddir) in_dir = build_dir / "container-in" container_path = local.path(_container) with local.cwd(builddir): container_bin = container_path.basename container_in = in_dir / container_bin download.Copy(_container, container_in) uchrt = uchroot.no_args() with local.cwd("container-in"): uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash[ "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format( container_in)] has_erlent = (has_erlent & TF) # Unpack input container to: container-in if not has_erlent: cmd = local["/bin/tar"]["xf"] cmd = uchrt[cmd[container_bin]] else: cmd = tar["xf"] cmd = cmd[container_in] with local.cwd("container-in"): cmd("--exclude=dev/*") rm(container_in) return in_dir
def setup_container(builddir, container): """Prepare the container and returns the path where it can be found.""" with local.cwd(builddir): container_filename = str(container).split(os.path.sep)[-1] container_in = os.path.join("container-in", container_filename) Copy(container, container_in) uchroot = uchroot_no_args() with local.cwd("container-in"): uchroot = uchroot["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash["-c", "tar --list -f './{0}' | grep --silent '.erlent'". format(container_in)] has_erlent = (has_erlent & TF) # Unpack input container to: container-in if not has_erlent: cmd = local["/bin/tar"]["xf"] cmd = uchroot[cmd[container_filename]] else: cmd = tar["xf"] cmd = cmd[os.path.abspath(container_in)] with local.cwd("container-in"): cmd("--exclude=dev/*") rm(container_in) return os.path.join(builddir, "container-in")
def setup_container(builddir, _container): """Prepare the container and returns the path where it can be found.""" build_dir = local.path(builddir) in_dir = build_dir / "container-in" container_path = local.path(_container) with local.cwd(builddir): container_bin = container_path.basename container_in = in_dir / container_bin download.Copy(_container, container_in) uchrt = uchroot.no_args() with local.cwd("container-in"): uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash["-c", "tar --list -f './{0}' | grep --silent '.erlent'". format(container_in)] has_erlent = (has_erlent & TF) # Unpack input container to: container-in if not has_erlent: cmd = local["/bin/tar"]["xf"] cmd = uchrt[cmd[container_bin]] else: cmd = tar["xf"] cmd = cmd[container_in] with local.cwd("container-in"): cmd("--exclude=dev/*") rm(container_in) return in_dir
def compile(self): self.download() download.Git(self.test_suite_uri, self.test_suite_dir) venv_path = local.cwd / "local" virtualenv(venv_path, "--python=python2") pip_path = local.cwd / "local" / "bin" / "pip" pip = local[pip_path] with local.cwd(self.SRC_FILE): pip("install", "--no-cache-dir", "--disable-pip-version-check", "-e", ".") self.sandbox_dir = local.cwd / "run" if self.sandbox_dir.exists(): rm("-rf", self.sandbox_dir) mkdir(self.sandbox_dir) self.lnt = local[local.path("./local/bin/lnt")] self.clang = compiler.cc(self, detect_project=True) self.clang_cxx = compiler.cxx(self, detect_project=True) self.lnt("runtest", "test-suite", "-v", "-j1", "--sandbox", self.sandbox_dir, "--benchmarking-only", "--only-compile", "--cc", str(self.clang), "--cxx", str(self.clang_cxx), "--test-suite", self.test_suite_dir, "--only-test=" + self.SUBDIR)
def __call__(self, binary_command, *args, may_wrap=True, **kwargs): self.project.name = kwargs.get("project_name", self.project.name) likwid_f = self.project.name + ".txt" jobs = self.config['jobs'] res = [] for group in ["CLOCK"]: run_cmd = \ likwid_perfctr["-O", "-o", likwid_f, "-m", "-C", "0-{0:d}".format(jobs), "-g", group, binary_command] with pb.local.env(POLLI_ENABLE_LIKWID=1): res.extend(self.call_next(run_cmd, *args, **kwargs)) likwid_measurement = likwid.perfcounters(likwid_f) for run_info in res: persist_likwid(run_info.db_run, run_info.session, likwid_measurement) db.persist_config(run_info.db_run, run_info.session, { "cores": str(jobs), "likwid.group": group }) rm("-f", likwid_f) return res
def __call__(self): if not CFG['clean'].value(): return paths = CFG["cleanup_paths"].value() for p in paths: if os.path.exists(p): rm("-r", p)
def __call__(self): if not CFG['clean']: return StepResult.OK paths = CFG["cleanup_paths"].value for p in paths: if os.path.exists(p): rm("-r", p) self.status = StepResult.OK
def clean_directories(builddir, in_dir=True, out_dir=True): """Remove the in and out of the container if confirmed by the user.""" with local.cwd(builddir): if in_dir and os.path.exists("container-in") and ask( "Should I delete '{0}'?".format( os.path.abspath("container-in"))): rm("-rf", "container-in") if out_dir and os.path.exists("container-out") and ask( "Should I delete '{0}'?".format( os.path.abspath("container-out"))): rm("-rf", "container-out")
def __call__(self): if not CFG['clean'].value(): return if not self._obj: return obj_builddir = os.path.abspath(self._obj.builddir) if os.path.exists(obj_builddir): self.__clean_mountpoints__(obj_builddir) if self.check_empty: rmdir(obj_builddir, retcode=None) else: rm("-rf", obj_builddir)
def run_with_likwid(project, experiment, config, jobs, run_f, args, **kwargs): """ Run the given file wrapped by likwid. Args: project: The benchbuild.project. experiment: The benchbuild.experiment. config: The benchbuild.settings.config. jobs: Number of cores we should use for this exection. run_f: The file we want to execute. args: List of arguments that should be passed to the wrapped binary. **kwargs: Dictionary with our keyword args. We support the following entries: project_name: The real name of our project. This might not be the same as the configured project name, if we got wrapped with ::benchbuild.project.wrap_dynamic has_stdin: Signals whether we should take care of stdin. """ from benchbuild.settings import CFG from benchbuild.utils.run import track_execution, handle_stdin from benchbuild.utils.db import persist_likwid, persist_config from benchbuild.likwid import get_likwid_perfctr CFG.update(config) project.name = kwargs.get("project_name", project.name) likwid_f = project.name + ".txt" for group in ["CLOCK"]: likwid_path = path.join(CFG["likwiddir"], "bin") likwid_perfctr = local[path.join(likwid_path, "likwid-perfctr")] run_cmd = \ likwid_perfctr["-O", "-o", likwid_f, "-m", "-C", "0-{0:d}".format(jobs), "-g", group, run_f] run_cmd = handle_stdin(run_cmd[args], kwargs) with local.env(POLLI_ENABLE_LIKWID=1): with track_execution(run_cmd, project, experiment) as run: ri = run() likwid_measurement = get_likwid_perfctr(likwid_f) persist_likwid(run, ri.session, likwid_measurement) persist_config(run, ri.session, { "cores": str(jobs), "likwid.group": group }) rm("-f", likwid_f)
def unpack(container, path): """ Unpack a container usable by uchroot. Method that checks if a directory for the container exists, checks if erlent support is needed and then unpacks the container accordingly. Args: path: The location where the container is, that needs to be unpacked. """ from benchbuild.utils.run import run from benchbuild.utils.uchroot import no_args path = local.path(path) c_filename = local.path(container.filename) name = c_filename.basename if not path.exists(): path.mkdir() with local.cwd(path): Wget(container.remote, name) uchroot = no_args() uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash[ "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(name)] has_erlent = (has_erlent & TF) untar = local["/bin/tar"]["xf", "./" + name] if not has_erlent: untar = uchroot[untar] run(untar["--exclude=dev/*"]) if not os.path.samefile(name, container.filename): rm(name) else: LOG.warning("File contents do not match: %s != %s", name, container.filename) cp(container.filename + ".hash", path)
def unpack(container, path): """ Unpack a container usable by uchroot. Method that checks if a directory for the container exists, checks if erlent support is needed and then unpacks the container accordingly. Args: path: The location where the container is, that needs to be unpacked. """ from benchbuild.utils.run import run from benchbuild.utils.uchroot import no_args path = local.path(path) c_filename = local.path(container.filename) name = c_filename.basename if not path.exists(): path.mkdir() with local.cwd(path): Wget(container.remote, name) uchroot = no_args() uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash[ "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format( name)] has_erlent = (has_erlent & TF) untar = local["/bin/tar"]["xf", "./" + name] if not has_erlent: untar = uchroot[untar] run(untar["--exclude=dev/*"]) if not os.path.samefile(name, container.filename): rm(name) else: LOG.warning("File contents do not match: %s != %s", name, container.filename) cp(container.filename + ".hash", path)
def unpack_container(container, path): """ Unpack a container usable by uchroot. Method that checks if a directory for the container exists, checks if erlent support is needed and then unpacks the container accordingly. Args: path: The location where the container is, that needs to be unpacked. """ from benchbuild.utils.run import run, uchroot_no_args path = os.path.abspath(path) name = os.path.basename(os.path.abspath(container.filename)) if not os.path.exists(path): mkdir("-p", path) with local.cwd(path): Wget(container.remote, name) uchroot = uchroot_no_args() uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash[ "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(name)] has_erlent = (has_erlent & TF) cmd = local["/bin/tar"]["xf"] if not has_erlent: cmd = uchroot[cmd["./" + name]] else: cmd = cmd[name] run(cmd["--exclude=dev/*"]) if not os.path.samefile(name, container.filename): rm(name) else: logging.warning("File contents do not match: {0} != {1}", name, container.filename) cp(container.filename + ".hash", path)
def __call__(self): if not CFG['clean']: LOG.warning("Clean disabled by config.") return if not self.obj: LOG.warning("No object assigned to this action.") return obj_builddir = os.path.abspath(self.obj.builddir) if os.path.exists(obj_builddir): LOG.debug("Path %s exists", obj_builddir) Clean.clean_mountpoints(obj_builddir) if self.check_empty: rmdir(obj_builddir, retcode=None) else: rm("-rf", obj_builddir) else: LOG.debug("Path %s did not exist anymore", obj_builddir) self.status = StepResult.OK
def configure(self): sandbox_dir = path.join(self.builddir, "run") if path.exists(sandbox_dir): rm("-rf", sandbox_dir) mkdir(sandbox_dir)
def clean(self): """Clean the project build directory.""" if path.exists(self.builddir) and listdir(self.builddir) == []: rmdir(self.builddir) elif path.exists(self.builddir) and listdir(self.builddir) != []: rm("-rf", self.builddir)
def build(self): povray_binary = path.join(self.SRC_FILE, "unix", self.name) with local.cwd(self.SRC_FILE): rm("-f", povray_binary) run(make["clean", "all"])