def setup_basic_blame_experiment(
experiment: VersionExperiment, project: Project,
report_type: tp.Type[BaseReport]
) -> None:
"""
Setup the project for a blame experiment.
- run time extensions
- compile time extensions
- prepare compiler
- configure C/CXX flags
"""
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, experiment) \
<< time.RunWithTime()
# Add the required compiler extensions to the project(s).
project.compiler_extension = compiler.RunCompiler(project, experiment) \
<< RunWLLVM() \
<< run.WithTimeout()
# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
experiment.get_handle(), project, report_type
)
# This c-flag is provided by VaRA and it suggests to use the git-blame
# annotation.
project.cflags += ["-fvara-GB"]
def actions_for_project(
self, project: Project) -> tp.MutableSequence[actions.Step]:
"""Returns the specified steps to run the project(s) specified in the
call in a fixed order."""
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, self) \
<< time.RunWithTime()
# Add the required compiler extensions to the project(s).
project.compiler_extension = compiler.RunCompiler(project, self) \
<< RunWLLVM() \
<< run.WithTimeout()
# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, CR)
# This c-flag is provided by VaRA and it suggests to use the git-blame
# annotation.
project.cflags = ["-fvara-GB"]
analysis_actions = []
analysis_actions += get_bc_cache_actions(
project,
extraction_error_handler=create_default_compiler_error_handler(
self.get_handle(), project, self.REPORT_SPEC.main_report))
analysis_actions.append(CRAnalysis(project, self.get_handle()))
analysis_actions.append(actions.Clean(project))
return analysis_actions
def actions_for_project(self, project):
"""
Create & Run a papi-instrumented version of the project.
This experiment uses the -jitable flag of libPolyJIT to generate
dynamic SCoP coverage.
"""
project.ldflags = project.ldflags + ["-lpjit", "-lpprof", "-lpapi"]
project.cflags = [
"-O3", "-Xclang", "-load", "-Xclang", "LLVMPolyJIT.so", "-mllvm",
"-polli", "-mllvm", "-polli-instrument", "-mllvm",
"-polli-no-recompilation", "-mllvm", "-polly-detect-keep-going"
]
project.compiler_extension = \
run.WithTimeout(compilestats.ExtractCompileStats(project, self))
project.runtime_extension = \
time.RunWithTime(
run.RuntimeExtension(project, self, config={'jobs': 1}))
def evaluate_calibration(e):
from benchbuild.utils.cmd import pprof_calibrate
papi_calibration = e.get_papi_calibration(pprof_calibrate)
e.persist_calibration(project, pprof_calibrate, papi_calibration)
actns = self.default_runtime_actions(project)
actns.append(Calibrate(self, evaluate_calibration))
return self.default_runtime_actions(project)
def actions_for_project(self, project):
project.cflags = [
"-O3", "-Xclang", "-load", "-Xclang", "LLVMPolly.so", "-mllvm",
"-polly"
]
project.compiler_extension = \
run.WithTimeout(ExtractCompileStats(project, self))
return CompilestatsExperiment.default_compiletime_actions(project)
def actions_for_project(
self, project: Project
) -> tp.MutableSequence[actions.Step]:
"""
Returns the specified steps to run the project(s) specified in the call
in a fixed order.
Args:
project: to analyze
"""
# Try, to build the project without optimizations to get more precise
# blame annotations. Note: this does not guarantee that a project is
# build without optimizations because the used build tool/script can
# still add optimizations flags after the experiment specified cflags.
project.cflags += ["-O1", "-Xclang", "-disable-llvm-optzns", "-g0"]
bc_file_extensions = [
BCFileExtensions.NO_OPT,
BCFileExtensions.TBAA,
]
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, self) \
<< time.RunWithTime()
# Add the required compiler extensions to the project(s).
project.compiler_extension = compiler.RunCompiler(project, self) \
<< RunWLLVM() \
<< run.WithTimeout()
# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, self.REPORT_SPEC.main_report
)
analysis_actions = get_bc_cache_actions(
project, bc_file_extensions,
create_default_compiler_error_handler(
self.get_handle(), project, self.REPORT_SPEC.main_report
)
)
for _ in range(0, 10):
analysis_actions.append(
RunGlobalsTestAnalysis(project, self.get_handle(), True)
)
analysis_actions.append(
RunGlobalsTestAnalysis(project, self.get_handle(), False)
)
# Clean up the generated files afterwards
analysis_actions.append(actions.Clean(project))
return analysis_actions
def actions_for_project(
self, project: Project
) -> tp.MutableSequence[actions.Step]:
"""
Returns the specified steps to run the project(s) specified in the call
in a fixed order.
Args:
project: to analyze
"""
# Add tracing markers.
fm_provider = FeatureModelProvider.create_provider_for_project(project)
if fm_provider is None:
raise Exception("Could not get FeatureModelProvider!")
fm_path = fm_provider.get_feature_model_path(project.version_of_primary)
if fm_path is None or not fm_path.exists():
raise FeatureModelNotFound(project, fm_path)
# Sets vara tracing flags
project.cflags += [
"-fvara-feature", f"-fvara-fm-path={fm_path.absolute()}",
"-fsanitize=vara", "-fvara-instr=usdt", "-flto", "-fuse-ld=lld"
]
project.ldflags += ["-flto"]
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, self) \
<< bbtime.RunWithTime()
# Add the required compiler extensions to the project(s).
project.compiler_extension = compiler.RunCompiler(project, self) \
<< run.WithTimeout()
# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, VaraInstrumentationStatsReport
)
analysis_actions = []
analysis_actions.append(actions.Compile(project))
analysis_actions.append(
CaptureInstrumentationStats(project, self.get_handle())
)
analysis_actions.append(actions.Clean(project))
return analysis_actions
def actions_for_project(self, project: Project) -> tp.List[actions.Step]:
"""
Returns the specified steps to run the project(s) specified in the call
in a fixed order.
Args:
project: to analyze
"""
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, self) \
<< time.RunWithTime()
# Add the required compiler extensions to the project(s).
project.compiler_extension = compiler.RunCompiler(project, self) \
<< RunWLLVM() \
<< run.WithTimeout()
# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, self.REPORT_SPEC.main_report)
fm_provider = FeatureModelProvider.get_provider_for_project(project)
fm_path = fm_provider.get_feature_model_path(
project.version_of_primary)
project.cflags += [
"-O1", "-Xclang", "-disable-llvm-optzns", "-fvara-feature",
"-fvara-fm-path=" + str(fm_path), "-g"
]
bc_file_extensions = [
BCFileExtensions.NO_OPT, BCFileExtensions.TBAA,
BCFileExtensions.FEATURE, BCFileExtensions.DEBUG
]
analysis_actions = []
analysis_actions += get_bc_cache_actions(
project,
bc_file_extensions,
extraction_error_handler=create_default_compiler_error_handler(
self.get_handle(), project, self.REPORT_SPEC.main_report))
analysis_actions.append(
PhASARFTACheck(project, self.get_handle(), bc_file_extensions))
analysis_actions.append(actions.Clean(project))
return analysis_actions
def actions_for_project(
self, project: Project) -> tp.MutableSequence[actions.Step]:
"""Returns the specified steps to run the project(s) specified in the
call in a fixed order."""
fm_provider = FeatureModelProvider.create_provider_for_project(project)
if fm_provider is None:
raise Exception("Could not get FeatureModelProvider!")
fm_path = fm_provider.get_feature_model_path(
project.version_of_primary)
if fm_path is None or not fm_path.exists():
raise FeatureModelNotFound(project, fm_path)
# We need debug info to later determine source code locations in a
# utility pass. Also, include feature information.
project.cflags += [
"-g", "-fvara-feature", f"-fvara-fm-path={fm_path.absolute()}"
]
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, self) \
<< time.RunWithTime()
# Add the required compiler extensions to the project(s). We want to
# transfer the whole project into LLVM-IR.
project.compiler_extension = compiler.RunCompiler(project, self) \
<< RunWLLVM() \
<< run.WithTimeout()
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, self.REPORT_SPEC.main_report)
bc_file_extensions = [BCFileExtensions.DEBUG, BCFileExtensions.FEATURE]
analysis_actions = []
analysis_actions.append(actions.Compile(project))
analysis_actions.append(Extract(project, bc_file_extensions))
analysis_actions.append(
CollectInstrumentationPoints(project, self.get_handle()))
analysis_actions.append(actions.Clean(project))
return analysis_actions
def actions_for_project(
self, project: Project
) -> tp.MutableSequence[actions.Step]:
"""
Returns the specified steps to run the project(s) specified in the call
in a fixed order.
Args:
project: to analyze
"""
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, self) \
<< time.RunWithTime()
# Add the required compiler extensions to the project(s).
project.compiler_extension = compiler.RunCompiler(project, self) \
<< RunWLLVM() \
<< run.WithTimeout()
# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, EmptyReport
)
analysis_actions = []
analysis_actions += get_bc_cache_actions(
project,
extraction_error_handler=create_default_compiler_error_handler(
self.get_handle(), project, self.REPORT_SPEC.main_report
)
)
analysis_actions.append(
IDELinearConstantAnalysis(project, self.get_handle())
)
analysis_actions.append(actions.Clean(project))
return analysis_actions
def actions_for_project(
self, project: Project) -> tp.MutableSequence[actions.Step]:
"""Returns the specified steps to run the project(s) specified in the
call in a fixed order."""
# Add the required runtime extensions to the project(s).
project.runtime_extension = run.RuntimeExtension(project, self) \
<< time.RunWithTime()
# Add the required compiler extensions to the project(s).
project.compiler_extension = compiler.RunCompiler(project, self) \
<< RunWLLVM() \
<< run.WithTimeout()
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, self.REPORT_SPEC.main_report)
analysis_actions = []
analysis_actions.append(actions.Compile(project))
analysis_actions.append(EmptyAnalysis(project, self.get_handle()))
analysis_actions.append(actions.Clean(project))
return analysis_actions
class Project(metaclass=ProjectDecorator):
"""Abstract class for benchbuild projects.
A project is an arbitrary software system usable by benchbuild in
experiments.
Subclasses of Project are registered automatically by benchbuild, if
imported in the same interpreter session. For this to happen, you must list
the in the settings under plugins -> projects.
A project implementation *must* provide the following method:
compile: Downloads & Compiles the source.
A project implementation *may* provide the following functions:
run_tests: Wrap any binary that has to be run under the
runtime_extension wrapper and execute an implementation defined
set of run-time tests.
Defaults to a call of a binary with the name `run_f` in the
build directory without arguments.
clean: Clean the project's build directory. Defaults to
recursive 'rm' on the build directory and can be disabled
by setting the environment variable ``BB_CLEAN=false``.
Raises:
AttributeError: Class definition raises an attribute error, if
the implementation does not provide a value for the attributes
`NAME`, `DOMAIN`, and `GROUP`
TypeError: Validation of properties may throw a TypeError.
Attributes:
experiment (benchbuild.experiment.Experiment):
The experiment this project is assigned to.
name (str, optional):
The name of this project. Defaults to `NAME`.
domain (str, optional):
The application domain of this project. Defaults to `DOMAIN`.
group (str, optional):
The group this project belongs to. Defaults to `GROUP`.
src_file (str, optional):
A main src_file this project is assigned to. Defaults to `SRC_FILE`
container (benchbuild.utils.container.Container, optional):
A uchroot compatible container that we can use for this project.
Defaults to `benchbuild.utils.container.Gentoo`.
version (str, optional):
A version information for this project. Defaults to `VERSION`.
builddir (str, optional):
The build directory for this project. Auto generated, if not set.
testdir (str, optional):
The location of any additional test-files for this project,
usually stored out of tree. Auto generated, if not set. Usually a
project implementation
will define this itself.
cflags (:obj:`list` of :obj:`str`, optional)
A list of cflags used, for compilation of this project.
ldflags (:obj:`list` of :obj:`str`, optional)
A list of ldflags used, for compilation of this project.
run_f (str, optional):
A filename that points to the binary we want to track.
Usually a project implementation will define this itself.
run_uuid (uuid.UUID, optional):
An UUID that identifies all binaries executed by a single run of
this project. In the database schema this is named the 'run_group'.
compiler_extension (Callable[str, iterable[str], RunInfo], optional):
A composable extension that will be used in place of the real
compiler. Defaults to running the compiler with a timeout command
wrapped around it.
runtime_extension (Callable[str, iterable[str], RunInfo], optional):
A composable extension that will be used in place of any binary
this project
wants to execute. Which binaries to replace is defined by the
implementation using `benchbuild.utils.wrapping.wrap`.
Defaults to None.
"""
NAME = None
DOMAIN = None
GROUP = None
VERSION = None
SRC_FILE = None
CONTAINER = None
def __new__(cls, *args, **kwargs):
"""Create a new project instance and set some defaults."""
new_self = super(Project, cls).__new__(cls)
if cls.NAME is None:
raise AttributeError(
"{0} @ {1} does not define a NAME class attribute.".format(
cls.__name__, cls.__module__))
if cls.DOMAIN is None:
raise AttributeError(
"{0} @ {1} does not define a DOMAIN class attribute.".format(
cls.__name__, cls.__module__))
if cls.GROUP is None:
raise AttributeError(
"{0} @ {1} does not define a GROUP class attribute.".format(
cls.__name__, cls.__module__))
return new_self
experiment = attr.ib()
name = attr.ib(
default=attr.Factory(lambda self: type(self).NAME, takes_self=True))
domain = attr.ib(
default=attr.Factory(lambda self: type(self).DOMAIN, takes_self=True))
group = attr.ib(
default=attr.Factory(lambda self: type(self).GROUP, takes_self=True))
src_file = attr.ib(default=attr.Factory(lambda self: type(self).SRC_FILE,
takes_self=True))
container = attr.ib(default=attr.Factory(lambda self: type(self).CONTAINER,
takes_self=True))
version = attr.ib(
default=attr.Factory(lambda self: type(self).VERSION, takes_self=True))
testdir = attr.ib()
@testdir.default
def __default_testdir(self):
if self.group:
return local.path(str(
CFG["test_dir"])) / self.domain / self.group / self.name
return local.path(str(CFG["test_dir"])) / self.domain / self.name
cflags = attr.ib(default=attr.Factory(list))
ldflags = attr.ib(default=attr.Factory(list))
run_uuid = attr.ib()
@run_uuid.default
def __default_run_uuid(self):
run_group = getenv("BB_DB_RUN_GROUP", None)
if run_group:
return uuid.UUID(run_group)
return uuid.uuid4()
@run_uuid.validator
def __check_if_uuid(self, _, value):
if not isinstance(value, uuid.UUID):
raise TypeError("{attribute} must be a valid UUID object")
builddir = attr.ib(default=attr.Factory(lambda self: local.path(
str(CFG["build_dir"])) / "{}-{}".format(self.experiment.name, self.id),
takes_self=True))
run_f = attr.ib(default=attr.Factory(
lambda self: local.path(self.builddir) / self.name, takes_self=True))
compiler_extension = attr.ib(
default=attr.Factory(lambda self: ext_run.WithTimeout(
compiler.RunCompiler(self, self.experiment)),
takes_self=True))
runtime_extension = attr.ib(default=None)
def __attrs_post_init__(self):
db.persist_project(self)
def run_tests(self, runner):
"""
Run the tests of this project.
Clients override this method to provide customized run-time tests.
Args:
experiment: The experiment we run this project under
run: A function that takes the run command.
"""
exp = wrapping.wrap(self.run_f, self)
runner(exp)
def run(self):
"""Run the tests of this project.
This method initializes the default environment and takes care of
cleaning up the mess we made, after a successfull run.
Args:
experiment: The experiment we run this project under
"""
from benchbuild.utils.run import (begin_run_group, end_run_group,
fail_run_group)
CFG["experiment"] = self.experiment.name
CFG["project"] = self.NAME
CFG["domain"] = self.DOMAIN
CFG["group"] = self.GROUP
CFG["version"] = self.VERSION
CFG["db"]["run_group"] = str(self.run_uuid)
group, session = begin_run_group(self)
signals.handlers.register(fail_run_group, group, session)
try:
self.run_tests(run.run)
end_run_group(group, session)
except ProcessExecutionError:
fail_run_group(group, session)
raise
except KeyboardInterrupt:
fail_run_group(group, session)
raise
finally:
signals.handlers.deregister(fail_run_group)
def clean(self):
"""Clean the project build directory."""
builddir_p = local.path(self.builddir)
builddir_p.delete()
def prepare(self):
"""Prepare the build diretory."""
builddir_p = local.path(self.builddir)
if not builddir_p.exists():
builddir_p.mkdir()
@abstractmethod
def compile(self):
"""Compile the project."""
def download(self, version=None):
"""Auto-generated by with_* decorators."""
del version
LOG.error("Not implemented.")
def redirect(self):
"""Redirect execution to a containerized benchbuild instance."""
LOG.error("Redirection not supported by project.")
def clone(self):
"""Create a deepcopy of ourself."""
new_p = copy.deepcopy(self)
new_p.run_uuid = uuid.uuid4()
return new_p
@property
def id(self):
return "{name}-{group}@{version}_uuid_{id}".format(
name=self.name,
group=self.group,
version=self.version,
id=self.run_uuid)
@classmethod
def versions(cls):
"""Return all available versions."""
if cls.VERSION:
return [cls.VERSION]
return ["unknown"]
def actions_for_project(self, project):
actns = []
project.cflags = ["-Xclang", "-load", "-Xclang", "LLVMPolly.so"]
newp = copy.deepcopy(project)
newp.run_uuid = uuid.uuid4()
newp.cflags = newp.cflags + ["-O3"]
cfg = {"name": "-O3"}
newp.compiler_extension = \
compilestats.ExtractCompileStats(newp, self, config=cfg) \
<< run.WithTimeout() \
<< statistics.Statistics(newp, self, config=cfg)
actns.extend(self.default_compiletime_actions(newp))
newp = copy.deepcopy(project)
newp.run_uuid = uuid.uuid4()
newp.cflags = newp.cflags + ["-O3", "-mllvm", "-polly"]
cfg = {"name": "-O3 -polly"}
newp.compiler_extension = \
compilestats.ExtractCompileStats(newp, self, config=cfg) \
<< run.WithTimeout()\
<< statistics.Statistics(newp, self, config=cfg)
actns.extend(self.default_compiletime_actions(newp))
newp = copy.deepcopy(project)
newp.run_uuid = uuid.uuid4()
newp.cflags = newp.cflags + [
"-O3", "-mllvm", "-polly", "-mllvm",
"-polly-position=before-vectorizer"
]
cfg = {"name": "-O3 -polly -polly-position=before-vectorizer"}
newp.compiler_extension = \
compilestats.ExtractCompileStats(newp, self, config=cfg) \
<< run.WithTimeout() \
<< statistics.Statistics(newp, self, config=cfg)
actns.extend(self.default_compiletime_actions(newp))
newp = copy.deepcopy(project)
newp.run_uuid = uuid.uuid4()
newp.cflags = newp.cflags + [
"-O3", "-mllvm", "-polly", "-mllvm", "-polly-process-unprofitable",
"-mllvm", "-polly-position=before-vectorizer"
]
cfg = {
"name":
"-O3 -polly -polly-position=before-vectorizer "
"-polly-process-unprofitable"
}
newp.compiler_extension = \
compilestats.ExtractCompileStats(newp, self, config=cfg) \
<< run.WithTimeout() \
<< statistics.Statistics(newp, self, config=cfg)
actns.extend(self.default_compiletime_actions(newp))
newp = copy.deepcopy(project)
newp.run_uuid = uuid.uuid4()
newp.cflags = newp.cflags + [
"-O3", "-mllvm", "-polly", "-mllvm", "-polly-process-unprofitable"
]
cfg = {"name": "-O3 -polly -polly-process-unprofitable"}
newp.compiler_extension = \
compilestats.ExtractCompileStats(newp, self, config=cfg) \
<< run.WithTimeout() \
<< statistics.Statistics(newp, self, config=cfg)
actns.extend(self.default_compiletime_actions(newp))
return actns
def actions_for_project(self, project):
project = polyjit.PolyJIT.init_project(project)
project.compiler_extension = \
run.WithTimeout(compilestats.ExtractCompileStats(project, self))
return self.default_compiletime_actions(project)
def actions_for_project(self, project):
project.compiler_extension = \
run.WithTimeout(ExtractCompileStats(project, self))
return CompilestatsExperiment.default_compiletime_actions(project)
