def __getattr__(self, command):
"""Proxy getter for plumbum commands."""
from os import getenv
from plumbum import local
from benchbuild.settings import CFG
from benchbuild.utils.path import list_to_path
from benchbuild.utils.path import path_to_list
check = [command]
if command in self.__overrides__:
check = self.__overrides__[command]
if command in __ALIASES__:
check = __ALIASES__[command]
path = path_to_list(getenv("PATH", default=""))
path = CFG["env"]["binary_path"].value() + path
libs_path = path_to_list(getenv("LD_LIBRARY_PATH", default=""))
libs_path = CFG["env"]["binary_ld_library_path"].value() + libs_path
if self.__override_all__ is not None:
check = [self.__override_all__]
for alias_command in check:
try:
cmd = local[alias_command]
cmd = cmd.with_env(PATH=list_to_path(path),
LD_LIBRARY_PATH=list_to_path(libs_path))
return cmd
except AttributeError:
pass
raise AttributeError(command)
def wrap_dynamic(project,
name,
sprefix=None,
python=sys.executable,
name_filters=None):
"""
Wrap the binary :name with the function :runner.
This module generates a python tool :name: that can replace
a yet unspecified binary.
It behaves similar to the :wrap: function. However, the first
argument is the actual binary name.
Args:
name: name of the python module
runner: Function that should run the real binary
sprefix: Prefix that should be used for commands.
python: The python executable that should be used.
name_filters:
List of regex expressions that are used to filter the
real project name. Make sure to include a match group named
'name' in the regex, e.g.,
[
r'foo(?P<name>.)-flt'
]
Returns: plumbum command, readty to launch.
"""
from jinja2 import Environment, PackageLoader
env = Environment(trim_blocks=True,
lstrip_blocks=True,
loader=PackageLoader('benchbuild', 'utils/templates'))
template = env.get_template('run_dynamic.py.inc')
name_absolute = os.path.abspath(name)
real_f = name_absolute + PROJECT_BIN_F_EXT
project_file = persist(project, suffix=".project")
bin_path = list_to_path(CFG["env"]["path"].value())
bin_path = list_to_path([bin_path, os.environ["PATH"]])
bin_lib_path = \
list_to_path(CFG["env"]["ld_library_path"].value())
bin_lib_path = \
list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]])
with open(name_absolute, 'w') as wrapper:
wrapper.write(
template.render(runf=strip_path_prefix(real_f, sprefix),
project_file=strip_path_prefix(
project_file, sprefix),
path=str(bin_path),
ld_library_path=str(bin_lib_path),
python=python,
name_filters=name_filters))
chmod("+x", name_absolute)
return local[name_absolute]
def write_bashrc(_path):
"""
Write a valid gentoo bashrc file to :path:.
Args:
path - The output path of the make.conf
"""
cfg_mounts = CFG["container"]["mounts"].value
cfg_prefix = CFG["container"]["prefixes"].value
path.mkfile_uchroot("/etc/portage/bashrc")
mounts = uchroot.mounts("mnt", cfg_mounts)
p_paths, p_libs = uchroot.env(cfg_prefix)
paths, libs = uchroot.env(mounts)
paths = paths + p_paths
libs = libs + p_libs
with open(_path, 'w') as bashrc:
lines = '''
export PATH="{0}:${{PATH}}"
export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}"
'''.format(path.list_to_path(paths), path.list_to_path(libs))
bashrc.write(lines)
def write_bashrc(_path):
"""
Write a valid gentoo bashrc file to :path:.
Args:
path - The output path of the make.conf
"""
cfg_mounts = CFG["container"]["mounts"].value
cfg_prefix = CFG["container"]["prefixes"].value
path.mkfile_uchroot("/etc/portage/bashrc")
mounts = uchroot.mounts("mnt", cfg_mounts)
p_paths, p_libs = uchroot.env(cfg_prefix)
paths, libs = uchroot.env(mounts)
paths = paths + p_paths
libs = libs + p_libs
with open(_path, 'w') as bashrc:
lines = '''
export PATH="{0}:${{PATH}}"
export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}"
'''.format(path.list_to_path(paths), path.list_to_path(libs))
bashrc.write(lines)
def wrap_dynamic(project,
name,
sprefix=None,
python=sys.executable,
name_filters=None):
"""
Wrap the binary :name with the function :runner.
This module generates a python tool :name: that can replace
a yet unspecified binary.
It behaves similar to the :wrap: function. However, the first
argument is the actual binary name.
Args:
name: name of the python module
runner: Function that should run the real binary
sprefix: Prefix that should be used for commands.
python: The python executable that should be used.
name_filters:
List of regex expressions that are used to filter the
real project name. Make sure to include a match group named
'name' in the regex, e.g.,
[
r'foo(?P<name>.)-flt'
]
Returns: plumbum command, readty to launch.
"""
env = __create_jinja_env()
template = env.get_template('run_dynamic.py.inc')
name_absolute = os.path.abspath(name)
real_f = name_absolute + PROJECT_BIN_F_EXT
project_file = persist(project, suffix=".project")
env = CFG['env'].value
bin_path = list_to_path(env.get('PATH', []))
bin_path = list_to_path([bin_path, os.environ["PATH"]])
bin_lib_path = \
list_to_path(env.get('LD_LIBRARY_PATH', []))
bin_lib_path = \
list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]])
with open(name_absolute, 'w') as wrapper:
wrapper.write(
template.render(
runf=strip_path_prefix(real_f, sprefix),
project_file=strip_path_prefix(project_file, sprefix),
path=str(bin_path),
ld_library_path=str(bin_lib_path),
python=python,
name_filters=name_filters))
chmod("+x", name_absolute)
return local[name_absolute]
def test_list_to_path(self):
from benchbuild.utils.path import list_to_path
p = list_to_path(["a", "b"])
self.assertEqual(p, "a:b")
p = list_to_path(["a"])
self.assertEqual(p, "a")
p = list_to_path([""])
self.assertEqual(p, "")
def test_list_to_path(self):
from benchbuild.utils.path import list_to_path
p = list_to_path(["a", "b"])
self.assertEqual(p, "a:b")
p = list_to_path(["a"])
self.assertEqual(p, "a")
p = list_to_path([""])
self.assertEqual(p, "")
def uchroot_with_mounts(*args, **kwargs):
"""Return a uchroot command with all mounts enabled."""
uchroot_cmd = uchroot_no_args(*args, **kwargs)
uchroot_cmd, mounts = \
_uchroot_mounts("mnt", CFG["container"]["mounts"].value(), uchroot_cmd)
paths, libs = uchroot_env(mounts)
uchroot_cmd = with_env_recursive(uchroot_cmd,
LD_LIBRARY_PATH=list_to_path(libs),
PATH=list_to_path(paths))
return uchroot_cmd
def uchroot_no_args():
"""Return the uchroot command without any customizations."""
from benchbuild.utils.cmd import uchroot as uchrt
prefixes = CFG["container"]["prefixes"].value()
p_paths, p_libs = uchroot_env(prefixes)
uchrt = with_env_recursive(uchrt,
LD_LIBRARY_PATH=list_to_path(p_libs),
PATH=list_to_path(p_paths))
return uchrt
def wrap_dynamic(self, name, runner, sprefix=None, **template_vars):
"""
Wrap the binary :name with the function :runner.
This module generates a python tool :name: that can replace
a yet unspecified binary.
It behaves similar to the :wrap: function. However, the first
argument is the actual binary name.
Args:
name: name of the python module
runner: Function that should run the real binary
base_class: The base_class of our project.
base_module: The module of base_class.
Returns: plumbum command, readty to launch.
"""
base_class = self.__class__.__name__
base_module = self.__module__
name_absolute = os.path.abspath(name)
blob_f = name_absolute + PROJECT_BLOB_F_EXT
real_f = name_absolute + PROJECT_BIN_F_EXT
with open(blob_f, 'wb') as blob:
blob.write(dill.dumps(runner))
bin_path = list_to_path(CFG["env"]["binary_path"].value())
bin_path = list_to_path([bin_path, os.environ["PATH"]])
bin_lib_path = list_to_path(CFG["env"]["binary_ld_library_path"].value())
bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]])
template_vars['db_host'] = str(CFG["db"]["host"])
template_vars['db_name'] = str(CFG["db"]["name"])
template_vars['db_port'] = str(CFG["db"]["port"])
template_vars['db_pass'] = str(CFG["db"]["pass"])
template_vars['db_user'] = str(CFG["db"]["user"])
template_vars['path'] = bin_path
template_vars['ld_lib_path'] = bin_lib_path
template_vars['blobf'] = strip_path_prefix(blob_f, sprefix)
template_vars['runf'] = strip_path_prefix(real_f, sprefix)
template_vars['base_class'] = base_class
template_vars['base_module'] = base_module
if 'python' not in template_vars:
template_vars['python'] = sys.executable
with open(name_absolute, 'w') as wrapper:
lines = template_str("templates/run_dynamic.py.inc")
lines = lines.format(**template_vars)
wrapper.write(lines)
chmod("+x", name_absolute)
return local[name_absolute]
def wrap(name, runner, sprefix=None, **template_vars):
""" Wrap the binary :name: with the function :runner:.
This module generates a python tool that replaces :name:
The function in runner only accepts the replaced binaries
name as argument. We use the cloudpickle package to
perform the serialization, make sure :runner: can be serialized
with it and you're fine.
Args:
name: Binary we want to wrap
runner: Function that should run instead of :name:
Returns:
A plumbum command, ready to launch.
"""
name_absolute = os.path.abspath(name)
real_f = name_absolute + PROJECT_BIN_F_EXT
if sprefix:
run(uchroot()["/bin/mv",
strip_path_prefix(name_absolute, sprefix),
strip_path_prefix(real_f, sprefix)])
else:
run(mv[name_absolute, real_f])
blob_f = name_absolute + PROJECT_BLOB_F_EXT
with open(blob_f, 'wb') as blob:
dill.dump(runner, blob, protocol=-1, recurse=True)
bin_path = list_to_path(CFG["env"]["binary_path"].value())
bin_path = list_to_path([bin_path, os.environ["PATH"]])
bin_lib_path = list_to_path(CFG["env"]["binary_ld_library_path"].value())
bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]])
template_vars['db_host'] = str(CFG["db"]["host"])
template_vars['db_name'] = str(CFG["db"]["name"])
template_vars['db_port'] = str(CFG["db"]["port"])
template_vars['db_pass'] = str(CFG["db"]["pass"])
template_vars['db_user'] = str(CFG["db"]["user"])
template_vars['path'] = bin_path
template_vars['ld_lib_path'] = bin_lib_path
template_vars['blobf'] = strip_path_prefix(blob_f, sprefix)
template_vars['runf'] = strip_path_prefix(real_f, sprefix)
if 'python' not in template_vars:
template_vars['python'] = sys.executable
with open(name_absolute, 'w') as wrapper:
lines = template_str("templates/run_static.py.inc")
lines = lines.format(**template_vars)
wrapper.write(lines)
run(chmod["+x", name_absolute])
return local[name_absolute]
def write_bashrc(self, path):
"""Write inside a bash and update the shell if necessary."""
mkfile_uchroot("/etc/portage/bashrc")
paths, libs = uchroot_env(
uchroot_mounts("mnt", CFG["container"]["mounts"].value()))
with open(path, 'w') as bashrc:
lines = '''
export PATH="{0}:${{PATH}}"
export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}"
'''.format(list_to_path(paths), list_to_path(libs))
bashrc.write(lines)
def wrap(name, project, sprefix=None, python=sys.executable):
""" Wrap the binary :name: with the runtime extension of the project.
This module generates a python tool that replaces :name:
The function in runner only accepts the replaced binaries
name as argument. We use the cloudpickle package to
perform the serialization, make sure :runner: can be serialized
with it and you're fine.
Args:
name: Binary we want to wrap
project: The project that contains the runtime_extension we want
to run instead of the binary.
Returns:
A plumbum command, ready to launch.
"""
env = __create_jinja_env()
template = env.get_template('run_static.py.inc')
name_absolute = os.path.abspath(name)
real_f = name_absolute + PROJECT_BIN_F_EXT
if sprefix:
run(uchroot()["/bin/mv",
strip_path_prefix(name_absolute, sprefix),
strip_path_prefix(real_f, sprefix)])
else:
run(mv[name_absolute, real_f])
project_file = persist(project, suffix=".project")
env = CFG['env'].value
bin_path = list_to_path(env.get('PATH', []))
bin_path = list_to_path([bin_path, os.environ["PATH"]])
bin_lib_path = list_to_path(env.get('LD_LIBRARY_PATH', []))
bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]])
home = env.get("HOME", os.getenv("HOME", ""))
with open(name_absolute, 'w') as wrapper:
wrapper.write(
template.render(
runf=strip_path_prefix(real_f, sprefix),
project_file=strip_path_prefix(project_file, sprefix),
path=str(bin_path),
ld_library_path=str(bin_lib_path),
home=str(home),
python=python,
))
run(chmod["+x", name_absolute])
return local[name_absolute]
def no_args(**kwargs):
"""Return the uchroot command without any customizations."""
from benchbuild.utils.cmd import uchroot as uchrt
prefixes = CFG["container"]["prefixes"].value
p_paths, p_libs = env(prefixes)
uchrt = run.with_env_recursive(
uchrt,
LD_LIBRARY_PATH=path.list_to_path(p_libs),
PATH=path.list_to_path(p_paths))
return uchrt
def wrap(name, project, sprefix=None, python=sys.executable):
""" Wrap the binary :name: with the runtime extension of the project.
This module generates a python tool that replaces :name:
The function in runner only accepts the replaced binaries
name as argument. We use the cloudpickle package to
perform the serialization, make sure :runner: can be serialized
with it and you're fine.
Args:
name: Binary we want to wrap
project: The project that contains the runtime_extension we want
to run instead of the binary.
Returns:
A plumbum command, ready to launch.
"""
env = __create_jinja_env()
template = env.get_template('run_static.py.inc')
name_absolute = os.path.abspath(name)
real_f = name_absolute + PROJECT_BIN_F_EXT
if sprefix:
run(uchroot()["/bin/mv",
strip_path_prefix(name_absolute, sprefix),
strip_path_prefix(real_f, sprefix)])
else:
run(mv[name_absolute, real_f])
project_file = persist(project, suffix=".project")
env = CFG['env'].value
bin_path = list_to_path(env.get('PATH', []))
bin_path = list_to_path([bin_path, os.environ["PATH"]])
bin_lib_path = list_to_path(env.get('LD_LIBRARY_PATH', []))
bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]])
with open(name_absolute, 'w') as wrapper:
wrapper.write(
template.render(
runf=strip_path_prefix(real_f, sprefix),
project_file=strip_path_prefix(project_file, sprefix),
path=str(bin_path),
ld_library_path=str(bin_lib_path),
python=python,
))
run(chmod["+x", name_absolute])
return local[name_absolute]
def __get_compiler_paths():
from os import getenv
from benchbuild.utils.path import list_to_path
path = getenv("PATH", "")
lib_path = getenv("LD_LIBRARY_PATH", "")
_lib_path = CFG["env"]["compiler_ld_library_path"].value()
_path = CFG["env"]["compiler_path"].value()
_lib_path = list_to_path(_lib_path)
_path = list_to_path(_path)
path = list_to_path([_path, path])
lib_path = list_to_path([_lib_path, lib_path])
return {"ld_library_path": lib_path, "path": path}
def with_mounts(*args, uchroot_cmd_fn=no_args, **kwargs):
"""Return a uchroot command with all mounts enabled."""
mounts = CFG["container"]["mounts"].value
prefixes = CFG["container"]["prefixes"].value
uchroot_opts, mounts = __mounts__("mnt", mounts)
uchroot_cmd = uchroot_cmd_fn(**kwargs)
uchroot_cmd = uchroot_cmd[uchroot_opts]
uchroot_cmd = uchroot_cmd[args]
paths, libs = env(mounts)
prefix_paths, prefix_libs = env(prefixes)
uchroot_cmd = run.with_env_recursive(
uchroot_cmd,
LD_LIBRARY_PATH=path.list_to_path(libs + prefix_libs),
PATH=path.list_to_path(paths + prefix_paths))
return uchroot_cmd
def __get_paths():
from os import getenv
from benchbuild.utils.path import list_to_path
path = getenv("PATH", "")
lib_path = getenv("LD_LIBRARY_PATH", "")
env = CFG["env"].value
_lib_path = env.get("LD_LIBRARY_PATH", "")
_path = env.get("PATH", "")
_lib_path = list_to_path(_lib_path)
_path = list_to_path(_path)
path = list_to_path([_path, path])
lib_path = list_to_path([_lib_path, lib_path])
return {"ld_library_path": lib_path, "path": path}
def uchroot(*args, **kwargs):
"""
Return a customizable uchroot command.
Args:
args: List of additional arguments for uchroot (typical: mounts)
Return:
chroot_cmd
"""
mkdir("-p", "llvm")
uchroot_cmd = uchroot_no_llvm(*args, **kwargs)
uchroot_cmd, mounts = _uchroot_mounts("mnt",
CFG["container"]["mounts"].value(),
uchroot_cmd)
paths, libs = uchroot_env(mounts)
uchroot_cmd = uchroot_cmd.with_env(LD_LIBRARY_PATH=list_to_path(libs),
PATH=list_to_path(paths))
return uchroot_cmd["--"]
def __call__(self, compiler: cc, *args: tp.Any,
**kwargs: tp.Any) -> tp.Any:
if str(compiler).endswith("clang++"):
wllvm = local["wllvm++"]
else:
wllvm = local["wllvm"]
env = bb_cfg()["env"].value
env_path_list = path_to_list(getenv("PATH", ""))
env_path_list = env.get("PATH", []) + env_path_list
libs_path = path_to_list(getenv("LD_LIBRARY_PATH", ""))
libs_path = env.get("LD_LIBRARY_PATH", []) + libs_path
wllvm = wllvm.with_env(LLVM_COMPILER="clang",
PATH=list_to_path(env_path_list),
LD_LIBRARY_PATH=list_to_path(libs_path))
return self.call_next(wllvm, *args, **kwargs)
def write_bashrc(path):
"""
Write a valid gentoo bashrc file to :path:.
Args:
path - The output path of the make.conf
"""
mkfile_uchroot("/etc/portage/bashrc")
paths, libs = uchroot_env(
uchroot_mounts("mnt", CFG["container"]["mounts"].value()))
p_paths, p_libs = uchroot_env(CFG["container"]["prefixes"].value())
with open(path, 'w') as bashrc:
lines = '''
export PATH="{0}:${{PATH}}"
export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}"
'''.format(list_to_path(paths + p_paths), list_to_path(libs + p_libs))
bashrc.write(lines)
def __get_paths():
from os import getenv
from benchbuild.utils.path import list_to_path
path = getenv("PATH", "")
lib_path = getenv("LD_LIBRARY_PATH", "")
env = CFG["env"].value
_lib_path = env.get("LD_LIBRARY_PATH", "")
_path = env.get("PATH", "")
_lib_path = list_to_path(_lib_path)
_path = list_to_path(_path)
path = list_to_path([_path, path])
lib_path = list_to_path([_lib_path, lib_path])
home = env.get("HOME", getenv("HOME", ""))
return {"ld_library_path": lib_path, "path": path, "home": home}
def __getattr__(self, command):
"""Proxy getter for plumbum commands."""
from os import getenv
from plumbum import local
from benchbuild.settings import CFG
from benchbuild.utils.path import list_to_path
from benchbuild.utils.path import path_to_list
check = [command]
if command in self.__overrides__:
check = self.__overrides__[command]
if command in __ALIASES__:
check = __ALIASES__[command]
env = CFG["env"].value
path = path_to_list(getenv("PATH", ""))
path.extend(env.get("PATH", []))
libs_path = path_to_list(getenv("LD_LIBRARY_PATH", ""))
libs_path.extend(env.get("LD_LIBRARY_PATH", []))
home = env.get("HOME", getenv("HOME", ""))
if self.__override_all__ is not None:
check = [self.__override_all__]
for alias_command in check:
try:
alias_cmd = local[alias_command]
alias_cmd = alias_cmd.with_env(
PATH=list_to_path(path),
LD_LIBRARY_PATH=list_to_path(libs_path),
HOME=home)
return alias_cmd
except AttributeError:
pass
LOG.warning("'%s' cannot be found. Import failed.", command)
return ERROR
def compile(self):
self.download()
download.Wget(self.libmcrypt_uri, self.libmcrypt_file)
download.Wget(self.mhash_uri, self.mhash_file)
tar('xfz', self.src_file)
tar('xfz', self.libmcrypt_file)
tar('xfz', self.mhash_file)
builddir = local.path(self.builddir)
mcrypt_dir = builddir / "mcrypt-2.6.8"
mhash_dir = builddir / self.mhash_dir
libmcrypt_dir = builddir / self.libmcrypt_dir
_cc = compiler.cc(self)
_cxx = compiler.cxx(self)
# Build mhash dependency
with local.cwd(mhash_dir):
configure = local["./configure"]
with local.env(CC=_cc, CXX=_cxx):
run.run(configure["--prefix=" + builddir])
run.run(make["-j", CFG["jobs"], "install"])
# Builder libmcrypt dependency
with local.cwd(libmcrypt_dir):
configure = local["./configure"]
with local.env(CC=_cc, CXX=_cxx):
run.run(configure["--prefix=" + builddir])
run.run(make["-j", CFG["jobs"], "install"])
with local.cwd(mcrypt_dir):
configure = local["./configure"]
lib_dir = builddir / "lib"
inc_dir = builddir / "include"
env = CFG["env"].value
mod_env = dict(
CC=_cc,
CXX=_cxx,
LD_LIBRARY_PATH=path.list_to_path(
[str(lib_dir)] + env.get("LD_LIBRARY_PATH", [])),
LDFLAGS="-L" + str(lib_dir),
CFLAGS="-I" + str(inc_dir))
env.update(mod_env)
with local.env(**env):
run.run(configure["--disable-dependency-tracking",
"--enable-static", "--disable-shared",
"--with-libmcrypt=" +
builddir, "--with-libmhash=" + builddir])
run.run(make["-j", CFG["jobs"]])
def __getattr__(self, command):
"""Proxy getter for plumbum commands."""
from os import getenv
from plumbum import local
from benchbuild.settings import CFG
from benchbuild.utils.path import list_to_path
from benchbuild.utils.path import path_to_list
check = [command]
if command in self.__overrides__:
check = self.__overrides__[command]
if command in __ALIASES__:
check = __ALIASES__[command]
env = CFG["env"].value
path = path_to_list(getenv("PATH", ""))
path.extend(env.get("PATH", []))
libs_path = path_to_list(getenv("LD_LIBRARY_PATH", ""))
libs_path.extend(env.get("LD_LIBRARY_PATH", []))
if self.__override_all__ is not None:
check = [self.__override_all__]
for alias_command in check:
try:
alias_cmd = local[alias_command]
alias_cmd = alias_cmd.with_env(
PATH=list_to_path(path),
LD_LIBRARY_PATH=list_to_path(libs_path))
return alias_cmd
except AttributeError:
pass
LOG.warning("'%s' cannot be found. Import failed.", command)
return ERROR
def __ld_library_path():
host_path = os.getenv('LD_LIBRARY_PATH', default='')
env = CFG['env'].value
benchbuild_path = list_to_path(env.get('LD_LIBRARY_PATH', []))
return os.path.pathsep.join([benchbuild_path, host_path])
def __ld_library_path():
host_path = os.getenv('LD_LIBRARY_PATH', default='')
env = CFG['env'].value
benchbuild_path = list_to_path(env.get('LD_LIBRARY_PATH', []))
return os.path.pathsep.join([benchbuild_path, host_path])
