def main(self, *args):
builddir = str(CFG["build_dir"])
in_container = CFG["container"]["input"].value()
if (in_container is None) or not os.path.exists(in_container):
in_is_file = False
in_container = Gentoo().local
else:
in_is_file = os.path.isfile(in_container)
if in_is_file:
clean_directories(builddir)
setup_directories(builddir)
in_container = setup_container(builddir, in_container)
run_in_container(args, in_container)
clean_directories(builddir, in_is_file, False)
class GentooGroup(project.Project):
"""Gentoo ProjectGroup is the base class for every portage build."""
GROUP = 'gentoo'
CONTAINER = Gentoo()
SRC_FILE = None
def build(self):
pass
def download(self):
if not CFG["unionfs"]["enable"].value():
container.unpack_container(self.container, self.builddir)
def configure(self):
write_bashrc("etc/portage/bashrc")
write_makeconfig("etc/portage/make.conf")
write_wgetrc("etc/wgetrc")
write_layout("etc/portage/metadata/layout.conf")
mkfile_uchroot("/etc/resolv.conf")
cp("/etc/resolv.conf", "etc/resolv.conf")
config_file = CFG["config_file"].value()
if os.path.exists(str(config_file)):
paths, libs = \
uchroot_env(
uchroot_mounts(
"mnt",
CFG["container"]["mounts"].value()))
uchroot_cfg = CFG
uchroot_cfg["plugins"]["projects"] = []
uchroot_cfg["env"]["path"] = paths
uchroot_cfg["env"]["ld_library_path"] = libs
mkfile_uchroot("/.benchbuild.yml")
uchroot_cfg.store(".benchbuild.yml")
wrap_cc_in_uchroot(self.cflags, self.ldflags, self.compiler_extension)
wrap_cxx_in_uchroot(self.cflags, self.ldflags, self.compiler_extension)
def main(self, *args):
builddir = CFG["build_dir"].value()
in_container = CFG["container"]["input"].value()
out_container = CFG["container"]["output"].value()
mounts = CFG["container"]["mounts"].value()
shell = CFG["container"]["shell"].value()
if (in_container is None) or not os.path.exists(in_container):
in_container = Gentoo().local
in_is_file = os.path.isfile(in_container)
if in_is_file:
in_container = setup_container(builddir, in_container)
self._strategy.run(
MockObj(builddir=builddir,
in_container=in_container,
out_container=out_container,
mounts=mounts,
shell=shell))
clean_directories(builddir, in_is_file, True)
class Project(object, metaclass=ProjectDecorator):
"""Abstract class for benchbuild projects.
A project is an arbitrary software system usable by benchbuild in
experiments.
Subclasses of Project are registered automatically by benchbuild, if
imported in the same interpreter session. For this to happen, you must list
the in the settings under plugins -> projects.
A project implementation *must* provide the following methods:
download: Download the sources into the build directory.
configure: Configure the sources, replace the compiler with our wrapper,
if possible.
build: Build the sources, with the wrapper compiler.
A project implementation *may* provide the following functions:
run_tests: Wrap any binary that has to be run under the
runtime_extension wrapper and execute an implementation defined
set of run-time tests.
Defaults to a call of a binary with the name `run_f` in the
build directory without arguments.
prepare: Prepare the project's build directory. Defaults to a
simple call to 'mkdir'.
clean: Clean the project's build directory. Defaults to
recursive 'rm' on the build directory and can be disabled
by setting the environment variable ``BB_CLEAN=false``.
Raises:
AttributeError: Class definition raises an attribute error, if
the implementation does not provide a value for the attributes
`NAME`, `DOMAIN`, and `GROUP`
TypeError: Validation of properties may throw a TypeError.
Attributes:
experiment (benchbuild.experiment.Experiment):
The experiment this project is assigned to.
name (str, optional):
The name of this project. Defaults to `NAME`.
domain (str, optional):
The application domain of this project. Defaults to `DOMAIN`.
group (str, optional):
The group this project belongs to. Defaults to `GROUP`.
src_file (str, optional):
A main src_file this project is assigned to. Defaults to `SRC_FILE`
container (benchbuild.utils.container.Container, optional):
A uchroot compatible container that we can use for this project.
Defaults to `benchbuild.utils.container.Gentoo`.
version (str, optional):
A version information for this project. Defaults to `VERSION`.
builddir (str, optional):
The build directory for this project. Auto generated, if not set.
testdir (str, optional):
The location of any additional test-files for this project,
usually stored out of tree. Auto generated, if not set. Usually a
project implementation
will define this itself.
cflags (:obj:`list` of :obj:`str`, optional)
A list of cflags used, for compilation of this project.
ldflags (:obj:`list` of :obj:`str`, optional)
A list of ldflags used, for compilation of this project.
run_f (str, optional):
A filename that points to the binary we want to track.
Usually a project implementation will define this itself.
run_uuid (uuid.UUID, optional):
An UUID that identifies all binaries executed by a single run of
this project. In the database schema this is named the 'run_group'.
compiler_extension (Callable[str, iterable[str], RunInfo], optional):
A composable extension that will be used in place of the real
compiler. Defaults to running the compiler with a timeout command
wrapped around it.
runtime_extension (Callable[str, iterable[str], RunInfo], optional):
A composable extension that will be used in place of any binary
this project
wants to execute. Which binaries to replace is defined by the
implementation using `benchbuild.utils.wrapping.wrap`.
Defaults to None.
"""
NAME = None
DOMAIN = None
GROUP = None
VERSION = None
SRC_FILE = None
def __new__(cls, *_):
"""Create a new project instance and set some defaults."""
new_self = super(Project, cls).__new__(cls)
if cls.NAME is None:
raise AttributeError(
"{0} @ {1} does not define a NAME class attribute.".format(
cls.__name__, cls.__module__))
if cls.DOMAIN is None:
raise AttributeError(
"{0} @ {1} does not define a DOMAIN class attribute.".format(
cls.__name__, cls.__module__))
if cls.GROUP is None:
raise AttributeError(
"{0} @ {1} does not define a GROUP class attribute.".format(
cls.__name__, cls.__module__))
return new_self
experiment = attr.ib()
name = attr.ib(
default=attr.Factory(lambda self: type(self).NAME, takes_self=True))
domain = attr.ib(
default=attr.Factory(lambda self: type(self).DOMAIN, takes_self=True))
group = attr.ib(
default=attr.Factory(lambda self: type(self).GROUP, takes_self=True))
src_file = attr.ib(default=attr.Factory(lambda self: type(self).SRC_FILE,
takes_self=True))
container = attr.ib(default=Gentoo())
version = attr.ib(default=attr.Factory(
lambda self: get_version_from_cache_dir(self.src_file),
takes_self=True))
builddir = attr.ib(default=attr.Factory(lambda self: path.join(
str(CFG["build_dir"]), "{0}-{1}-{2}-{3}".format(
self.experiment.name, self.name, self.group, self.experiment.id)),
takes_self=True))
testdir = attr.ib()
@testdir.default
def __default_testdir(self):
if self.group:
return path.join(str(CFG["test_dir"]), self.domain, self.group,
self.name)
else:
return path.join(str(CFG["test_dir"]), self.domain, self.name)
cflags = attr.ib(default=attr.Factory(list))
ldflags = attr.ib(default=attr.Factory(list))
run_f = attr.ib(default=attr.Factory(
lambda self: path.join(self.builddir, self.name), takes_self=True))
run_uuid = attr.ib()
@run_uuid.default
def __default_run_uuid(self):
return getenv("BB_DB_RUN_GROUP", uuid.uuid4())
@run_uuid.validator
def __check_if_uuid(self, _, value):
if not isinstance(value, uuid.UUID):
raise TypeError("{attribute} must be a valid UUID object")
compiler_extension = attr.ib(
default=attr.Factory(lambda self: ext.RunWithTimeout(
ext.RunCompiler(self, self.experiment)),
takes_self=True))
runtime_extension = attr.ib(default=None)
def __attrs_post_init__(self):
persist_project(self)
def run_tests(self, runner):
"""
Run the tests of this project.
Clients override this method to provide customized run-time tests.
Args:
experiment: The experiment we run this project under
run: A function that takes the run command.
"""
exp = wrap(self.run_f, self)
runner(exp)
def run(self):
"""Run the tests of this project.
This method initializes the default environment and takes care of
cleaning up the mess we made, after a successfull run.
Args:
experiment: The experiment we run this project under
"""
from benchbuild.utils.run import (begin_run_group, end_run_group,
fail_run_group)
CFG["experiment"] = self.experiment.name
CFG["project"] = self.NAME
CFG["domain"] = self.DOMAIN
CFG["group"] = self.GROUP
CFG["version"] = self.VERSION
CFG["db"]["run_group"] = str(self.run_uuid)
with local.cwd(self.builddir):
group, session = begin_run_group(self)
signals.handlers.register(fail_run_group, group, session)
try:
self.run_tests(ur.run)
end_run_group(group, session)
except ProcessExecutionError:
fail_run_group(group, session)
raise
except KeyboardInterrupt:
fail_run_group(group, session)
raise
finally:
signals.handlers.deregister(fail_run_group)
def clean(self):
"""Clean the project build directory."""
if path.exists(self.builddir) and listdir(self.builddir) == []:
rmdir(self.builddir)
elif path.exists(self.builddir) and listdir(self.builddir) != []:
rm("-rf", self.builddir)
def prepare(self):
"""Prepare the build diretory."""
if not path.exists(self.builddir):
mkdir(self.builddir)
@abstractmethod
def download(self):
"""Download the input source for this project."""
@abstractmethod
def configure(self):
"""Configure the project."""
@abstractmethod
def build(self):
"""Build the project."""
def clone(self):
"""Create a deepcopy of ourself."""
new_p = copy.deepcopy(self)
new_p.run_uuid = uuid.uuid4()
return new_p
@property
def id(self):
return "{name}-{group}-{id}".format(name=self.name,
group=self.group,
id=self.run_uuid)
class GentooGroup(project.Project):
"""Gentoo ProjectGroup is the base class for every portage build."""
GROUP = 'gentoo'
CONTAINER = Gentoo()
SRC_FILE = None
def __init__(self, exp):
super(GentooGroup, self).__init__(exp, "gentoo")
def build(self):
pass
def download(self):
if not CFG["unionfs"]["enable"].value():
container.unpack_container(project.Project.CONTAINER,
self.builddir)
def write_wgetrc(self, path):
mkfile_uchroot("/etc/wgetrc")
with open(path, 'w') as wgetrc:
hp = CFG["gentoo"]["http_proxy"].value()
fp = CFG["gentoo"]["ftp_proxy"].value()
if hp is not None:
http_s = "http_proxy = {0}".format(str(hp))
https_s = "https_proxy = {0}".format(str(hp))
wgetrc.write("use_proxy = on\n")
wgetrc.write(http_s + "\n")
wgetrc.write(https_s + "\n")
if fp is not None:
fp_s = "ftp_proxy={0}".format(str(fp))
wgetrc.write(fp_s + "\n")
def write_makeconfig(self, path):
mkfile_uchroot("/etc/portage/make.conf")
with open(path, 'w') as makeconf:
lines = '''
PORTAGE_USERNAME=root
PORTAGE_GROUPNAME=root
CFLAGS="-O2 -pipe"
CXXFLAGS="${CFLAGS}"
FEATURES="-xattr"
CC="/clang"
CXX="/clang++"
CHOST="x86_64-pc-linux-gnu"
USE="bindist mmx sse sse2"
PORTDIR="/usr/portage"
DISTDIR="${PORTDIR}/distfiles"
PKGDIR="${PORTDIR}/packages"
'''
makeconf.write(lines)
hp = CFG["gentoo"]["http_proxy"].value()
if hp is not None:
http_s = "http_proxy={0}".format(str(hp))
https_s = "https_proxy={0}".format(str(hp))
makeconf.write(http_s + "\n")
makeconf.write(https_s + "\n")
fp = CFG["gentoo"]["ftp_proxy"].value()
if fp is not None:
fp_s = "ftp_proxy={0}".format(str(fp))
makeconf.write(fp_s + "\n")
rp = CFG["gentoo"]["rsync_proxy"].value()
if rp is not None:
rp_s = "RSYNC_PROXY={0}".format(str(rp))
makeconf.write(rp_s + "\n")
def write_bashrc(self, path):
mkfile_uchroot("/etc/portage/bashrc")
paths, libs = uchroot_env(
uchroot_mounts("mnt", CFG["container"]["mounts"].value()))
with open(path, 'w') as bashrc:
lines = '''
export PATH="{0}:${{PATH}}"
export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}"
'''.format(list_to_path(paths), list_to_path(libs))
bashrc.write(lines)
def write_layout(self, path):
mkdir_uchroot("/etc/portage/metadata")
mkfile_uchroot("/etc/portage/metadata/layout.conf")
with open(path, 'w') as layoutconf:
lines = '''masters = gentoo'''
layoutconf.write(lines)
def configure(self):
self.write_bashrc("etc/portage/bashrc")
self.write_makeconfig("etc/portage/make.conf")
self.write_wgetrc("etc/wgetrc")
self.write_layout("etc/portage/metadata/layout.conf")
mkfile_uchroot("/etc/resolv.conf")
cp("/etc/resolv.conf", "etc/resolv.conf")
config_file = CFG["config_file"].value()
if path.exists(str(config_file)):
with open("log.file", 'w') as outf:
paths, libs = \
uchroot_env(
uchroot_mounts("mnt",
CFG["container"]["mounts"].value()))
UCHROOT_CFG = CFG
UCHROOT_CFG["plugins"]["projects"] = []
UCHROOT_CFG["env"]["compiler_path"] = paths
UCHROOT_CFG["env"]["compiler_ld_library_path"] = libs
UCHROOT_CFG["env"]["binary_path"] = paths
UCHROOT_CFG["env"]["binary_ld_library_path"] = libs
UCHROOT_CFG["env"]["lookup_path"] = paths
UCHROOT_CFG["env"]["lookup_ld_library_path"] = libs
mkfile_uchroot("/.benchbuild.json")
UCHROOT_CFG.store(".benchbuild.json")
wrap_cc_in_uchroot(self.cflags, self.ldflags, self.compiler_extension)
wrap_cxx_in_uchroot(self.cflags, self.ldflags, self.compiler_extension)
class Project(object, metaclass=ProjectDecorator):
"""
benchbuild's Project class.
A project defines how run-time testing and cleaning is done for this
IR project
"""
NAME = None
DOMAIN = None
GROUP = None
VERSION = None
SRC_FILE = None
CONTAINER = Gentoo()
def __new__(cls, *args, **kwargs):
"""Create a new project instance and set some defaults."""
new_self = super(Project, cls).__new__(cls)
if cls.NAME is None:
raise AttributeError(
"{0} @ {1} does not define a NAME class attribute.".format(
cls.__name__, cls.__module__))
if cls.DOMAIN is None:
raise AttributeError(
"{0} @ {1} does not define a DOMAIN class attribute.".format(
cls.__name__, cls.__module__))
if cls.GROUP is None:
raise AttributeError(
"{0} @ {1} does not define a GROUP class attribute.".format(
cls.__name__, cls.__module__))
if cls.SRC_FILE is None:
warnings.warn("{0} @ {1} does not offer a source file yet.".format(
cls.__name__, cls.__module__))
cls.SRC_FILE = "<not-set>"
if cls.CONTAINER is None:
warnings.warn("{0} @ {1} does not offer a container yet.".format(
cls.__name__, cls.__module__))
new_self.name = cls.NAME
new_self.domain = cls.DOMAIN
new_self.group = cls.GROUP
new_self.src_file = cls.SRC_FILE
new_self.version = lambda: get_version_from_cache_dir(cls.SRC_FILE)
new_self.container = cls.CONTAINER
return new_self
def __init__(self, exp, group=None):
"""
Setup a new project.
Args:
exp: The experiment this project belongs to.
group: The group this project belongs to. This is useful for
finding group specific test input files.
"""
self.experiment = exp
self.group_name = group
self.sourcedir = path.join(str(CFG["src_dir"]), self.name)
self.builddir = path.join(str(CFG["build_dir"]), exp.name, self.name)
if group:
self.testdir = path.join(str(CFG["test_dir"]), self.domain, group,
self.name)
else:
self.testdir = path.join(str(CFG["test_dir"]), self.domain,
self.name)
self.cflags = []
self.ldflags = []
self.setup_derived_filenames()
persist_project(self)
def setup_derived_filenames(self):
"""Construct all derived file names."""
self.run_f = path.join(self.builddir, self.name)
def run_tests(self, experiment, run):
"""
Run the tests of this project.
Clients override this method to provide customized run-time tests.
Args:
experiment: The experiment we run this project under
run: A function that takes the run command.
"""
exp = wrap(self.run_f, experiment)
with local.cwd(self.builddir):
run(exp)
def run(self, experiment):
"""
Run the tests of this project.
This method initializes the default environment and takes care of
cleaning up the mess we made, after a successfull run.
Args:
experiment: The experiment we run this project under
"""
from benchbuild.utils.run import GuardedRunException
from benchbuild.utils.run import (begin_run_group, end_run_group,
fail_run_group)
CFG["experiment"] = self.experiment.name
CFG["project"] = self.NAME
CFG["domain"] = self.DOMAIN
CFG["group"] = self.GROUP
CFG["version"] = self.VERSION
CFG["use_database"] = 1
CFG["db"]["run_group"] = str(self.run_uuid)
with local.cwd(self.builddir):
group, session = begin_run_group(self)
try:
self.run_tests(experiment, ur.run)
end_run_group(group, session)
except GuardedRunException:
fail_run_group(group, session)
except KeyboardInterrupt:
fail_run_group(group, session)
raise
if CFG["clean"].value():
self.clean()
def clean(self):
"""Clean the project build directory."""
if path.exists(self.builddir) and listdir(self.builddir) == []:
rmdir(self.builddir)
elif path.exists(self.builddir) and listdir(self.builddir) != []:
rm("-rf", self.builddir)
@property
def compiler_extension(self):
"""Return the compiler extension registered to this project."""
try:
return self._compiler_extension
except AttributeError:
self._compiler_extension = None
return self._compiler_extension
@compiler_extension.setter
def compiler_extension(
self, func: Callable[[str, Iterable[str]], RunInfo]) -> None:
"""
Set a function as compiler extension.
Args:
func: The compiler extension function. The minimal signature that
is required is ::
f(cc, **kwargs)
where cc is the original compiler command.
"""
self._compiler_extension = func
@property
def runtime_extension(self):
"""Return the runtime extension registered for this project."""
try:
return self._runtime_extension
except AttributeError:
self._runtime_extension = None
return self._runtime_extension
@runtime_extension.setter
def runtime_extension(
self, func: Callable[[str, Iterable[str]], RunInfo]) -> None:
"""
Set a function as compiler extension.
Args:
func: The compiler extension function. The minimal signature that
is required is ::
f(cc, **kwargs)
where cc is the original compiler command.
"""
self._runtime_extension = func
@property
def run_uuid(self):
"""
Get the UUID that groups all tests for one project run.
Args:
create_new: Create a fresh UUID (Default: False)
"""
from os import getenv
from uuid import uuid4
try:
if self._run_uuid is None:
self._run_uuid = getenv("BB_DB_RUN_GROUP", uuid4())
except AttributeError:
self._run_uuid = getenv("BB_DB_RUN_GROUP", uuid4())
return self._run_uuid
@run_uuid.setter
def run_uuid(self, value):
"""
Set a new UUID for this project.
Args:
value: The new value to set.
"""
from uuid import UUID
if isinstance(value, UUID):
self._run_uuid = value
def prepare(self):
"""Prepare the build diretory."""
if not path.exists(self.builddir):
mkdir(self.builddir)
@abstractmethod
def download(self):
"""Download the input source for this project."""
@abstractmethod
def configure(self):
"""Configure the project."""
@abstractmethod
def build(self):
"""Build the project."""