def __clone_needed__(repository: str, directory: str) -> bool:
"""
Do we need to create a fresh clone of the given repository.
Args:
repository: the repository we want to clone.
directory: the directory we expect the clone to live.
Returns:
True, if the clone is required.
False, if the directory is a valid clone.
"""
from benchbuild.utils.cmd import git, rm
git_dir = local.path(directory) / '.git'
if not git_dir.exists():
return True
requires_clone = True
with local.cwd(directory):
repo_origin_url = git('config', '--get', 'remote.origin.url')
requires_clone = repo_origin_url.strip('\n') != repository
if requires_clone:
rm('-r', directory)
def source_required(src_file):
"""
Check, if a download is required.
Args:
src_file: The filename to check for.
src_root: The path we find the file in.
Returns:
True, if we need to download something, False otherwise.
"""
if not src_file.exists():
return True
required = True
hash_file = src_file.with_suffix(".hash", depth=0)
LOG.debug("Hash file location: %s", hash_file)
if hash_file.exists():
new_hash = get_hash_of_dirs(src_file)
with open(hash_file, 'r') as h_file:
old_hash = h_file.readline()
required = not new_hash == old_hash
if required:
from benchbuild.utils.cmd import rm
rm("-r", src_file)
rm(hash_file)
if required:
LOG.info("Source required for: %s", src_file)
LOG.debug("Reason: src-exists: %s hash-exists: %s", src_file.exists(),
hash_file.exists())
return required
def source_required(src_file):
"""
Check, if a download is required.
Args:
src_file: The filename to check for.
src_root: The path we find the file in.
Returns:
True, if we need to download something, False otherwise.
"""
if not src_file.exists():
return True
required = True
hash_file = src_file.with_suffix(".hash", depth=0)
LOG.debug("Hash file location: %s", hash_file)
if hash_file.exists():
new_hash = get_hash_of_dirs(src_file)
with open(hash_file, 'r') as h_file:
old_hash = h_file.readline()
required = not new_hash == old_hash
if required:
from benchbuild.utils.cmd import rm
rm("-r", src_file)
rm(hash_file)
if required:
LOG.info("Source required for: %s", src_file)
LOG.debug("Reason: src-exists: %s hash-exists: %s", src_file.exists(),
hash_file.exists())
return required
def source_required(src_file, src_root):
"""
Check, if a download is required.
Args:
src_file: The filename to check for.
src_root: The path we find the file in.
Returns:
True, if we need to download something, False otherwise.
"""
from os import path
# Check if we need to do something
src_dir = path.join(src_root, src_file)
hash_file = path.join(src_root, src_file + ".hash")
required = True
if path.exists(src_dir) and path.exists(hash_file):
new_hash = get_hash_of_dirs(src_dir)
with open(hash_file, 'r') as h_file:
old_hash = h_file.readline()
required = not new_hash == old_hash
if required:
from benchbuild.utils.cmd import rm
rm("-r", src_dir)
rm(hash_file)
return required
def setup_container(builddir, _container):
"""Prepare the container and returns the path where it can be found."""
build_dir = local.path(builddir)
in_dir = build_dir / "container-in"
container_path = local.path(_container)
with local.cwd(builddir):
container_bin = container_path.basename
container_in = in_dir / container_bin
download.Copy(_container, container_in)
uchrt = uchroot.no_args()
with local.cwd("container-in"):
uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/",
"-w",
os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash[
"-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
container_in)]
has_erlent = (has_erlent & TF)
# Unpack input container to: container-in
if not has_erlent:
cmd = local["/bin/tar"]["xf"]
cmd = uchrt[cmd[container_bin]]
else:
cmd = tar["xf"]
cmd = cmd[container_in]
with local.cwd("container-in"):
cmd("--exclude=dev/*")
rm(container_in)
return in_dir
def setup_container(builddir, container):
"""Prepare the container and returns the path where it can be found."""
with local.cwd(builddir):
container_filename = str(container).split(os.path.sep)[-1]
container_in = os.path.join("container-in", container_filename)
Copy(container, container_in)
uchroot = uchroot_no_args()
with local.cwd("container-in"):
uchroot = uchroot["-E", "-A", "-u", "0", "-g", "0", "-C", "-r",
"/", "-w",
os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash["-c",
"tar --list -f './{0}' | grep --silent '.erlent'".
format(container_in)]
has_erlent = (has_erlent & TF)
# Unpack input container to: container-in
if not has_erlent:
cmd = local["/bin/tar"]["xf"]
cmd = uchroot[cmd[container_filename]]
else:
cmd = tar["xf"]
cmd = cmd[os.path.abspath(container_in)]
with local.cwd("container-in"):
cmd("--exclude=dev/*")
rm(container_in)
return os.path.join(builddir, "container-in")
def setup_container(builddir, _container):
"""Prepare the container and returns the path where it can be found."""
build_dir = local.path(builddir)
in_dir = build_dir / "container-in"
container_path = local.path(_container)
with local.cwd(builddir):
container_bin = container_path.basename
container_in = in_dir / container_bin
download.Copy(_container, container_in)
uchrt = uchroot.no_args()
with local.cwd("container-in"):
uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/",
"-w",
os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash["-c",
"tar --list -f './{0}' | grep --silent '.erlent'".
format(container_in)]
has_erlent = (has_erlent & TF)
# Unpack input container to: container-in
if not has_erlent:
cmd = local["/bin/tar"]["xf"]
cmd = uchrt[cmd[container_bin]]
else:
cmd = tar["xf"]
cmd = cmd[container_in]
with local.cwd("container-in"):
cmd("--exclude=dev/*")
rm(container_in)
return in_dir
def compile(self):
self.download()
download.Git(self.test_suite_uri, self.test_suite_dir)
venv_path = local.cwd / "local"
virtualenv(venv_path, "--python=python2")
pip_path = local.cwd / "local" / "bin" / "pip"
pip = local[pip_path]
with local.cwd(self.SRC_FILE):
pip("install", "--no-cache-dir", "--disable-pip-version-check",
"-e", ".")
self.sandbox_dir = local.cwd / "run"
if self.sandbox_dir.exists():
rm("-rf", self.sandbox_dir)
mkdir(self.sandbox_dir)
self.lnt = local[local.path("./local/bin/lnt")]
self.clang = compiler.cc(self, detect_project=True)
self.clang_cxx = compiler.cxx(self, detect_project=True)
self.lnt("runtest", "test-suite", "-v", "-j1", "--sandbox",
self.sandbox_dir, "--benchmarking-only",
"--only-compile", "--cc", str(self.clang), "--cxx",
str(self.clang_cxx), "--test-suite", self.test_suite_dir,
"--only-test=" + self.SUBDIR)
def __call__(self, binary_command, *args, may_wrap=True, **kwargs):
self.project.name = kwargs.get("project_name", self.project.name)
likwid_f = self.project.name + ".txt"
jobs = self.config['jobs']
res = []
for group in ["CLOCK"]:
run_cmd = \
likwid_perfctr["-O", "-o", likwid_f, "-m",
"-C", "0-{0:d}".format(jobs),
"-g", group, binary_command]
with pb.local.env(POLLI_ENABLE_LIKWID=1):
res.extend(self.call_next(run_cmd, *args, **kwargs))
likwid_measurement = likwid.perfcounters(likwid_f)
for run_info in res:
persist_likwid(run_info.db_run, run_info.session,
likwid_measurement)
db.persist_config(run_info.db_run, run_info.session, {
"cores": str(jobs),
"likwid.group": group
})
rm("-f", likwid_f)
return res
def __call__(self):
if not CFG['clean'].value():
return
paths = CFG["cleanup_paths"].value()
for p in paths:
if os.path.exists(p):
rm("-r", p)
def __call__(self):
if not CFG['clean']:
return StepResult.OK
paths = CFG["cleanup_paths"].value
for p in paths:
if os.path.exists(p):
rm("-r", p)
self.status = StepResult.OK
def clean_directories(builddir, in_dir=True, out_dir=True):
"""Remove the in and out of the container if confirmed by the user."""
with local.cwd(builddir):
if in_dir and os.path.exists("container-in") and ask(
"Should I delete '{0}'?".format(
os.path.abspath("container-in"))):
rm("-rf", "container-in")
if out_dir and os.path.exists("container-out") and ask(
"Should I delete '{0}'?".format(
os.path.abspath("container-out"))):
rm("-rf", "container-out")
def __call__(self):
if not CFG['clean'].value():
return
if not self._obj:
return
obj_builddir = os.path.abspath(self._obj.builddir)
if os.path.exists(obj_builddir):
self.__clean_mountpoints__(obj_builddir)
if self.check_empty:
rmdir(obj_builddir, retcode=None)
else:
rm("-rf", obj_builddir)
def run_with_likwid(project, experiment, config, jobs, run_f, args, **kwargs):
"""
Run the given file wrapped by likwid.
Args:
project: The benchbuild.project.
experiment: The benchbuild.experiment.
config: The benchbuild.settings.config.
jobs: Number of cores we should use for this exection.
run_f: The file we want to execute.
args: List of arguments that should be passed to the wrapped binary.
**kwargs: Dictionary with our keyword args. We support the following
entries:
project_name: The real name of our project. This might not
be the same as the configured project name, if we got wrapped
with ::benchbuild.project.wrap_dynamic
has_stdin: Signals whether we should take care of stdin.
"""
from benchbuild.settings import CFG
from benchbuild.utils.run import track_execution, handle_stdin
from benchbuild.utils.db import persist_likwid, persist_config
from benchbuild.likwid import get_likwid_perfctr
CFG.update(config)
project.name = kwargs.get("project_name", project.name)
likwid_f = project.name + ".txt"
for group in ["CLOCK"]:
likwid_path = path.join(CFG["likwiddir"], "bin")
likwid_perfctr = local[path.join(likwid_path, "likwid-perfctr")]
run_cmd = \
likwid_perfctr["-O", "-o", likwid_f, "-m",
"-C", "0-{0:d}".format(jobs),
"-g", group, run_f]
run_cmd = handle_stdin(run_cmd[args], kwargs)
with local.env(POLLI_ENABLE_LIKWID=1):
with track_execution(run_cmd, project, experiment) as run:
ri = run()
likwid_measurement = get_likwid_perfctr(likwid_f)
persist_likwid(run, ri.session, likwid_measurement)
persist_config(run, ri.session, {
"cores": str(jobs),
"likwid.group": group
})
rm("-f", likwid_f)
def unpack(container, path):
"""
Unpack a container usable by uchroot.
Method that checks if a directory for the container exists,
checks if erlent support is needed and then unpacks the
container accordingly.
Args:
path: The location where the container is, that needs to be unpacked.
"""
from benchbuild.utils.run import run
from benchbuild.utils.uchroot import no_args
path = local.path(path)
c_filename = local.path(container.filename)
name = c_filename.basename
if not path.exists():
path.mkdir()
with local.cwd(path):
Wget(container.remote, name)
uchroot = no_args()
uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w",
os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash[
"-c",
"tar --list -f './{0}' | grep --silent '.erlent'".format(name)]
has_erlent = (has_erlent & TF)
untar = local["/bin/tar"]["xf", "./" + name]
if not has_erlent:
untar = uchroot[untar]
run(untar["--exclude=dev/*"])
if not os.path.samefile(name, container.filename):
rm(name)
else:
LOG.warning("File contents do not match: %s != %s", name,
container.filename)
cp(container.filename + ".hash", path)
def unpack(container, path):
"""
Unpack a container usable by uchroot.
Method that checks if a directory for the container exists,
checks if erlent support is needed and then unpacks the
container accordingly.
Args:
path: The location where the container is, that needs to be unpacked.
"""
from benchbuild.utils.run import run
from benchbuild.utils.uchroot import no_args
path = local.path(path)
c_filename = local.path(container.filename)
name = c_filename.basename
if not path.exists():
path.mkdir()
with local.cwd(path):
Wget(container.remote, name)
uchroot = no_args()
uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w",
os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash[
"-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
name)]
has_erlent = (has_erlent & TF)
untar = local["/bin/tar"]["xf", "./" + name]
if not has_erlent:
untar = uchroot[untar]
run(untar["--exclude=dev/*"])
if not os.path.samefile(name, container.filename):
rm(name)
else:
LOG.warning("File contents do not match: %s != %s", name,
container.filename)
cp(container.filename + ".hash", path)
def unpack_container(container, path):
"""
Unpack a container usable by uchroot.
Method that checks if a directory for the container exists,
checks if erlent support is needed and then unpacks the
container accordingly.
Args:
path: The location where the container is, that needs to be unpacked.
"""
from benchbuild.utils.run import run, uchroot_no_args
path = os.path.abspath(path)
name = os.path.basename(os.path.abspath(container.filename))
if not os.path.exists(path):
mkdir("-p", path)
with local.cwd(path):
Wget(container.remote, name)
uchroot = uchroot_no_args()
uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w",
os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash[
"-c",
"tar --list -f './{0}' | grep --silent '.erlent'".format(name)]
has_erlent = (has_erlent & TF)
cmd = local["/bin/tar"]["xf"]
if not has_erlent:
cmd = uchroot[cmd["./" + name]]
else:
cmd = cmd[name]
run(cmd["--exclude=dev/*"])
if not os.path.samefile(name, container.filename):
rm(name)
else:
logging.warning("File contents do not match: {0} != {1}", name,
container.filename)
cp(container.filename + ".hash", path)
def __call__(self):
if not CFG['clean']:
LOG.warning("Clean disabled by config.")
return
if not self.obj:
LOG.warning("No object assigned to this action.")
return
obj_builddir = os.path.abspath(self.obj.builddir)
if os.path.exists(obj_builddir):
LOG.debug("Path %s exists", obj_builddir)
Clean.clean_mountpoints(obj_builddir)
if self.check_empty:
rmdir(obj_builddir, retcode=None)
else:
rm("-rf", obj_builddir)
else:
LOG.debug("Path %s did not exist anymore", obj_builddir)
self.status = StepResult.OK
def configure(self):
sandbox_dir = path.join(self.builddir, "run")
if path.exists(sandbox_dir):
rm("-rf", sandbox_dir)
mkdir(sandbox_dir)
def clean(self):
"""Clean the project build directory."""
if path.exists(self.builddir) and listdir(self.builddir) == []:
rmdir(self.builddir)
elif path.exists(self.builddir) and listdir(self.builddir) != []:
rm("-rf", self.builddir)
def build(self):
povray_binary = path.join(self.SRC_FILE, "unix", self.name)
with local.cwd(self.SRC_FILE):
rm("-f", povray_binary)
run(make["clean", "all"])
