def __getattr__(self, command): """Proxy getter for plumbum commands.""" from os import getenv from plumbum import local from benchbuild.settings import CFG from benchbuild.utils.path import list_to_path from benchbuild.utils.path import path_to_list check = [command] if command in self.__overrides__: check = self.__overrides__[command] if command in __ALIASES__: check = __ALIASES__[command] path = path_to_list(getenv("PATH", default="")) path = CFG["env"]["binary_path"].value() + path libs_path = path_to_list(getenv("LD_LIBRARY_PATH", default="")) libs_path = CFG["env"]["binary_ld_library_path"].value() + libs_path if self.__override_all__ is not None: check = [self.__override_all__] for alias_command in check: try: cmd = local[alias_command] cmd = cmd.with_env(PATH=list_to_path(path), LD_LIBRARY_PATH=list_to_path(libs_path)) return cmd except AttributeError: pass raise AttributeError(command)
def wrap_dynamic(project, name, sprefix=None, python=sys.executable, name_filters=None): """ Wrap the binary :name with the function :runner. This module generates a python tool :name: that can replace a yet unspecified binary. It behaves similar to the :wrap: function. However, the first argument is the actual binary name. Args: name: name of the python module runner: Function that should run the real binary sprefix: Prefix that should be used for commands. python: The python executable that should be used. name_filters: List of regex expressions that are used to filter the real project name. Make sure to include a match group named 'name' in the regex, e.g., [ r'foo(?P<name>.)-flt' ] Returns: plumbum command, readty to launch. """ from jinja2 import Environment, PackageLoader env = Environment(trim_blocks=True, lstrip_blocks=True, loader=PackageLoader('benchbuild', 'utils/templates')) template = env.get_template('run_dynamic.py.inc') name_absolute = os.path.abspath(name) real_f = name_absolute + PROJECT_BIN_F_EXT project_file = persist(project, suffix=".project") bin_path = list_to_path(CFG["env"]["path"].value()) bin_path = list_to_path([bin_path, os.environ["PATH"]]) bin_lib_path = \ list_to_path(CFG["env"]["ld_library_path"].value()) bin_lib_path = \ list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]]) with open(name_absolute, 'w') as wrapper: wrapper.write( template.render(runf=strip_path_prefix(real_f, sprefix), project_file=strip_path_prefix( project_file, sprefix), path=str(bin_path), ld_library_path=str(bin_lib_path), python=python, name_filters=name_filters)) chmod("+x", name_absolute) return local[name_absolute]
def write_bashrc(_path): """ Write a valid gentoo bashrc file to :path:. Args: path - The output path of the make.conf """ cfg_mounts = CFG["container"]["mounts"].value cfg_prefix = CFG["container"]["prefixes"].value path.mkfile_uchroot("/etc/portage/bashrc") mounts = uchroot.mounts("mnt", cfg_mounts) p_paths, p_libs = uchroot.env(cfg_prefix) paths, libs = uchroot.env(mounts) paths = paths + p_paths libs = libs + p_libs with open(_path, 'w') as bashrc: lines = ''' export PATH="{0}:${{PATH}}" export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}" '''.format(path.list_to_path(paths), path.list_to_path(libs)) bashrc.write(lines)
def write_bashrc(_path): """ Write a valid gentoo bashrc file to :path:. Args: path - The output path of the make.conf """ cfg_mounts = CFG["container"]["mounts"].value cfg_prefix = CFG["container"]["prefixes"].value path.mkfile_uchroot("/etc/portage/bashrc") mounts = uchroot.mounts("mnt", cfg_mounts) p_paths, p_libs = uchroot.env(cfg_prefix) paths, libs = uchroot.env(mounts) paths = paths + p_paths libs = libs + p_libs with open(_path, 'w') as bashrc: lines = ''' export PATH="{0}:${{PATH}}" export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}" '''.format(path.list_to_path(paths), path.list_to_path(libs)) bashrc.write(lines)
def wrap_dynamic(project, name, sprefix=None, python=sys.executable, name_filters=None): """ Wrap the binary :name with the function :runner. This module generates a python tool :name: that can replace a yet unspecified binary. It behaves similar to the :wrap: function. However, the first argument is the actual binary name. Args: name: name of the python module runner: Function that should run the real binary sprefix: Prefix that should be used for commands. python: The python executable that should be used. name_filters: List of regex expressions that are used to filter the real project name. Make sure to include a match group named 'name' in the regex, e.g., [ r'foo(?P<name>.)-flt' ] Returns: plumbum command, readty to launch. """ env = __create_jinja_env() template = env.get_template('run_dynamic.py.inc') name_absolute = os.path.abspath(name) real_f = name_absolute + PROJECT_BIN_F_EXT project_file = persist(project, suffix=".project") env = CFG['env'].value bin_path = list_to_path(env.get('PATH', [])) bin_path = list_to_path([bin_path, os.environ["PATH"]]) bin_lib_path = \ list_to_path(env.get('LD_LIBRARY_PATH', [])) bin_lib_path = \ list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]]) with open(name_absolute, 'w') as wrapper: wrapper.write( template.render( runf=strip_path_prefix(real_f, sprefix), project_file=strip_path_prefix(project_file, sprefix), path=str(bin_path), ld_library_path=str(bin_lib_path), python=python, name_filters=name_filters)) chmod("+x", name_absolute) return local[name_absolute]
def test_list_to_path(self): from benchbuild.utils.path import list_to_path p = list_to_path(["a", "b"]) self.assertEqual(p, "a:b") p = list_to_path(["a"]) self.assertEqual(p, "a") p = list_to_path([""]) self.assertEqual(p, "")
def test_list_to_path(self): from benchbuild.utils.path import list_to_path p = list_to_path(["a", "b"]) self.assertEqual(p, "a:b") p = list_to_path(["a"]) self.assertEqual(p, "a") p = list_to_path([""]) self.assertEqual(p, "")
def uchroot_with_mounts(*args, **kwargs): """Return a uchroot command with all mounts enabled.""" uchroot_cmd = uchroot_no_args(*args, **kwargs) uchroot_cmd, mounts = \ _uchroot_mounts("mnt", CFG["container"]["mounts"].value(), uchroot_cmd) paths, libs = uchroot_env(mounts) uchroot_cmd = with_env_recursive(uchroot_cmd, LD_LIBRARY_PATH=list_to_path(libs), PATH=list_to_path(paths)) return uchroot_cmd
def uchroot_no_args(): """Return the uchroot command without any customizations.""" from benchbuild.utils.cmd import uchroot as uchrt prefixes = CFG["container"]["prefixes"].value() p_paths, p_libs = uchroot_env(prefixes) uchrt = with_env_recursive(uchrt, LD_LIBRARY_PATH=list_to_path(p_libs), PATH=list_to_path(p_paths)) return uchrt
def wrap_dynamic(self, name, runner, sprefix=None, **template_vars): """ Wrap the binary :name with the function :runner. This module generates a python tool :name: that can replace a yet unspecified binary. It behaves similar to the :wrap: function. However, the first argument is the actual binary name. Args: name: name of the python module runner: Function that should run the real binary base_class: The base_class of our project. base_module: The module of base_class. Returns: plumbum command, readty to launch. """ base_class = self.__class__.__name__ base_module = self.__module__ name_absolute = os.path.abspath(name) blob_f = name_absolute + PROJECT_BLOB_F_EXT real_f = name_absolute + PROJECT_BIN_F_EXT with open(blob_f, 'wb') as blob: blob.write(dill.dumps(runner)) bin_path = list_to_path(CFG["env"]["binary_path"].value()) bin_path = list_to_path([bin_path, os.environ["PATH"]]) bin_lib_path = list_to_path(CFG["env"]["binary_ld_library_path"].value()) bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]]) template_vars['db_host'] = str(CFG["db"]["host"]) template_vars['db_name'] = str(CFG["db"]["name"]) template_vars['db_port'] = str(CFG["db"]["port"]) template_vars['db_pass'] = str(CFG["db"]["pass"]) template_vars['db_user'] = str(CFG["db"]["user"]) template_vars['path'] = bin_path template_vars['ld_lib_path'] = bin_lib_path template_vars['blobf'] = strip_path_prefix(blob_f, sprefix) template_vars['runf'] = strip_path_prefix(real_f, sprefix) template_vars['base_class'] = base_class template_vars['base_module'] = base_module if 'python' not in template_vars: template_vars['python'] = sys.executable with open(name_absolute, 'w') as wrapper: lines = template_str("templates/run_dynamic.py.inc") lines = lines.format(**template_vars) wrapper.write(lines) chmod("+x", name_absolute) return local[name_absolute]
def wrap(name, runner, sprefix=None, **template_vars): """ Wrap the binary :name: with the function :runner:. This module generates a python tool that replaces :name: The function in runner only accepts the replaced binaries name as argument. We use the cloudpickle package to perform the serialization, make sure :runner: can be serialized with it and you're fine. Args: name: Binary we want to wrap runner: Function that should run instead of :name: Returns: A plumbum command, ready to launch. """ name_absolute = os.path.abspath(name) real_f = name_absolute + PROJECT_BIN_F_EXT if sprefix: run(uchroot()["/bin/mv", strip_path_prefix(name_absolute, sprefix), strip_path_prefix(real_f, sprefix)]) else: run(mv[name_absolute, real_f]) blob_f = name_absolute + PROJECT_BLOB_F_EXT with open(blob_f, 'wb') as blob: dill.dump(runner, blob, protocol=-1, recurse=True) bin_path = list_to_path(CFG["env"]["binary_path"].value()) bin_path = list_to_path([bin_path, os.environ["PATH"]]) bin_lib_path = list_to_path(CFG["env"]["binary_ld_library_path"].value()) bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]]) template_vars['db_host'] = str(CFG["db"]["host"]) template_vars['db_name'] = str(CFG["db"]["name"]) template_vars['db_port'] = str(CFG["db"]["port"]) template_vars['db_pass'] = str(CFG["db"]["pass"]) template_vars['db_user'] = str(CFG["db"]["user"]) template_vars['path'] = bin_path template_vars['ld_lib_path'] = bin_lib_path template_vars['blobf'] = strip_path_prefix(blob_f, sprefix) template_vars['runf'] = strip_path_prefix(real_f, sprefix) if 'python' not in template_vars: template_vars['python'] = sys.executable with open(name_absolute, 'w') as wrapper: lines = template_str("templates/run_static.py.inc") lines = lines.format(**template_vars) wrapper.write(lines) run(chmod["+x", name_absolute]) return local[name_absolute]
def write_bashrc(self, path): """Write inside a bash and update the shell if necessary.""" mkfile_uchroot("/etc/portage/bashrc") paths, libs = uchroot_env( uchroot_mounts("mnt", CFG["container"]["mounts"].value())) with open(path, 'w') as bashrc: lines = ''' export PATH="{0}:${{PATH}}" export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}" '''.format(list_to_path(paths), list_to_path(libs)) bashrc.write(lines)
def wrap(name, project, sprefix=None, python=sys.executable): """ Wrap the binary :name: with the runtime extension of the project. This module generates a python tool that replaces :name: The function in runner only accepts the replaced binaries name as argument. We use the cloudpickle package to perform the serialization, make sure :runner: can be serialized with it and you're fine. Args: name: Binary we want to wrap project: The project that contains the runtime_extension we want to run instead of the binary. Returns: A plumbum command, ready to launch. """ env = __create_jinja_env() template = env.get_template('run_static.py.inc') name_absolute = os.path.abspath(name) real_f = name_absolute + PROJECT_BIN_F_EXT if sprefix: run(uchroot()["/bin/mv", strip_path_prefix(name_absolute, sprefix), strip_path_prefix(real_f, sprefix)]) else: run(mv[name_absolute, real_f]) project_file = persist(project, suffix=".project") env = CFG['env'].value bin_path = list_to_path(env.get('PATH', [])) bin_path = list_to_path([bin_path, os.environ["PATH"]]) bin_lib_path = list_to_path(env.get('LD_LIBRARY_PATH', [])) bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]]) home = env.get("HOME", os.getenv("HOME", "")) with open(name_absolute, 'w') as wrapper: wrapper.write( template.render( runf=strip_path_prefix(real_f, sprefix), project_file=strip_path_prefix(project_file, sprefix), path=str(bin_path), ld_library_path=str(bin_lib_path), home=str(home), python=python, )) run(chmod["+x", name_absolute]) return local[name_absolute]
def no_args(**kwargs): """Return the uchroot command without any customizations.""" from benchbuild.utils.cmd import uchroot as uchrt prefixes = CFG["container"]["prefixes"].value p_paths, p_libs = env(prefixes) uchrt = run.with_env_recursive( uchrt, LD_LIBRARY_PATH=path.list_to_path(p_libs), PATH=path.list_to_path(p_paths)) return uchrt
def wrap(name, project, sprefix=None, python=sys.executable): """ Wrap the binary :name: with the runtime extension of the project. This module generates a python tool that replaces :name: The function in runner only accepts the replaced binaries name as argument. We use the cloudpickle package to perform the serialization, make sure :runner: can be serialized with it and you're fine. Args: name: Binary we want to wrap project: The project that contains the runtime_extension we want to run instead of the binary. Returns: A plumbum command, ready to launch. """ env = __create_jinja_env() template = env.get_template('run_static.py.inc') name_absolute = os.path.abspath(name) real_f = name_absolute + PROJECT_BIN_F_EXT if sprefix: run(uchroot()["/bin/mv", strip_path_prefix(name_absolute, sprefix), strip_path_prefix(real_f, sprefix)]) else: run(mv[name_absolute, real_f]) project_file = persist(project, suffix=".project") env = CFG['env'].value bin_path = list_to_path(env.get('PATH', [])) bin_path = list_to_path([bin_path, os.environ["PATH"]]) bin_lib_path = list_to_path(env.get('LD_LIBRARY_PATH', [])) bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]]) with open(name_absolute, 'w') as wrapper: wrapper.write( template.render( runf=strip_path_prefix(real_f, sprefix), project_file=strip_path_prefix(project_file, sprefix), path=str(bin_path), ld_library_path=str(bin_lib_path), python=python, )) run(chmod["+x", name_absolute]) return local[name_absolute]
def __get_compiler_paths(): from os import getenv from benchbuild.utils.path import list_to_path path = getenv("PATH", "") lib_path = getenv("LD_LIBRARY_PATH", "") _lib_path = CFG["env"]["compiler_ld_library_path"].value() _path = CFG["env"]["compiler_path"].value() _lib_path = list_to_path(_lib_path) _path = list_to_path(_path) path = list_to_path([_path, path]) lib_path = list_to_path([_lib_path, lib_path]) return {"ld_library_path": lib_path, "path": path}
def with_mounts(*args, uchroot_cmd_fn=no_args, **kwargs): """Return a uchroot command with all mounts enabled.""" mounts = CFG["container"]["mounts"].value prefixes = CFG["container"]["prefixes"].value uchroot_opts, mounts = __mounts__("mnt", mounts) uchroot_cmd = uchroot_cmd_fn(**kwargs) uchroot_cmd = uchroot_cmd[uchroot_opts] uchroot_cmd = uchroot_cmd[args] paths, libs = env(mounts) prefix_paths, prefix_libs = env(prefixes) uchroot_cmd = run.with_env_recursive( uchroot_cmd, LD_LIBRARY_PATH=path.list_to_path(libs + prefix_libs), PATH=path.list_to_path(paths + prefix_paths)) return uchroot_cmd
def __get_paths(): from os import getenv from benchbuild.utils.path import list_to_path path = getenv("PATH", "") lib_path = getenv("LD_LIBRARY_PATH", "") env = CFG["env"].value _lib_path = env.get("LD_LIBRARY_PATH", "") _path = env.get("PATH", "") _lib_path = list_to_path(_lib_path) _path = list_to_path(_path) path = list_to_path([_path, path]) lib_path = list_to_path([_lib_path, lib_path]) return {"ld_library_path": lib_path, "path": path}
def uchroot(*args, **kwargs): """ Return a customizable uchroot command. Args: args: List of additional arguments for uchroot (typical: mounts) Return: chroot_cmd """ mkdir("-p", "llvm") uchroot_cmd = uchroot_no_llvm(*args, **kwargs) uchroot_cmd, mounts = _uchroot_mounts("mnt", CFG["container"]["mounts"].value(), uchroot_cmd) paths, libs = uchroot_env(mounts) uchroot_cmd = uchroot_cmd.with_env(LD_LIBRARY_PATH=list_to_path(libs), PATH=list_to_path(paths)) return uchroot_cmd["--"]
def __call__(self, compiler: cc, *args: tp.Any, **kwargs: tp.Any) -> tp.Any: if str(compiler).endswith("clang++"): wllvm = local["wllvm++"] else: wllvm = local["wllvm"] env = bb_cfg()["env"].value env_path_list = path_to_list(getenv("PATH", "")) env_path_list = env.get("PATH", []) + env_path_list libs_path = path_to_list(getenv("LD_LIBRARY_PATH", "")) libs_path = env.get("LD_LIBRARY_PATH", []) + libs_path wllvm = wllvm.with_env(LLVM_COMPILER="clang", PATH=list_to_path(env_path_list), LD_LIBRARY_PATH=list_to_path(libs_path)) return self.call_next(wllvm, *args, **kwargs)
def write_bashrc(path): """ Write a valid gentoo bashrc file to :path:. Args: path - The output path of the make.conf """ mkfile_uchroot("/etc/portage/bashrc") paths, libs = uchroot_env( uchroot_mounts("mnt", CFG["container"]["mounts"].value())) p_paths, p_libs = uchroot_env(CFG["container"]["prefixes"].value()) with open(path, 'w') as bashrc: lines = ''' export PATH="{0}:${{PATH}}" export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}" '''.format(list_to_path(paths + p_paths), list_to_path(libs + p_libs)) bashrc.write(lines)
def __get_paths(): from os import getenv from benchbuild.utils.path import list_to_path path = getenv("PATH", "") lib_path = getenv("LD_LIBRARY_PATH", "") env = CFG["env"].value _lib_path = env.get("LD_LIBRARY_PATH", "") _path = env.get("PATH", "") _lib_path = list_to_path(_lib_path) _path = list_to_path(_path) path = list_to_path([_path, path]) lib_path = list_to_path([_lib_path, lib_path]) home = env.get("HOME", getenv("HOME", "")) return {"ld_library_path": lib_path, "path": path, "home": home}
def __getattr__(self, command): """Proxy getter for plumbum commands.""" from os import getenv from plumbum import local from benchbuild.settings import CFG from benchbuild.utils.path import list_to_path from benchbuild.utils.path import path_to_list check = [command] if command in self.__overrides__: check = self.__overrides__[command] if command in __ALIASES__: check = __ALIASES__[command] env = CFG["env"].value path = path_to_list(getenv("PATH", "")) path.extend(env.get("PATH", [])) libs_path = path_to_list(getenv("LD_LIBRARY_PATH", "")) libs_path.extend(env.get("LD_LIBRARY_PATH", [])) home = env.get("HOME", getenv("HOME", "")) if self.__override_all__ is not None: check = [self.__override_all__] for alias_command in check: try: alias_cmd = local[alias_command] alias_cmd = alias_cmd.with_env( PATH=list_to_path(path), LD_LIBRARY_PATH=list_to_path(libs_path), HOME=home) return alias_cmd except AttributeError: pass LOG.warning("'%s' cannot be found. Import failed.", command) return ERROR
def compile(self): self.download() download.Wget(self.libmcrypt_uri, self.libmcrypt_file) download.Wget(self.mhash_uri, self.mhash_file) tar('xfz', self.src_file) tar('xfz', self.libmcrypt_file) tar('xfz', self.mhash_file) builddir = local.path(self.builddir) mcrypt_dir = builddir / "mcrypt-2.6.8" mhash_dir = builddir / self.mhash_dir libmcrypt_dir = builddir / self.libmcrypt_dir _cc = compiler.cc(self) _cxx = compiler.cxx(self) # Build mhash dependency with local.cwd(mhash_dir): configure = local["./configure"] with local.env(CC=_cc, CXX=_cxx): run.run(configure["--prefix=" + builddir]) run.run(make["-j", CFG["jobs"], "install"]) # Builder libmcrypt dependency with local.cwd(libmcrypt_dir): configure = local["./configure"] with local.env(CC=_cc, CXX=_cxx): run.run(configure["--prefix=" + builddir]) run.run(make["-j", CFG["jobs"], "install"]) with local.cwd(mcrypt_dir): configure = local["./configure"] lib_dir = builddir / "lib" inc_dir = builddir / "include" env = CFG["env"].value mod_env = dict( CC=_cc, CXX=_cxx, LD_LIBRARY_PATH=path.list_to_path( [str(lib_dir)] + env.get("LD_LIBRARY_PATH", [])), LDFLAGS="-L" + str(lib_dir), CFLAGS="-I" + str(inc_dir)) env.update(mod_env) with local.env(**env): run.run(configure["--disable-dependency-tracking", "--enable-static", "--disable-shared", "--with-libmcrypt=" + builddir, "--with-libmhash=" + builddir]) run.run(make["-j", CFG["jobs"]])
def __getattr__(self, command): """Proxy getter for plumbum commands.""" from os import getenv from plumbum import local from benchbuild.settings import CFG from benchbuild.utils.path import list_to_path from benchbuild.utils.path import path_to_list check = [command] if command in self.__overrides__: check = self.__overrides__[command] if command in __ALIASES__: check = __ALIASES__[command] env = CFG["env"].value path = path_to_list(getenv("PATH", "")) path.extend(env.get("PATH", [])) libs_path = path_to_list(getenv("LD_LIBRARY_PATH", "")) libs_path.extend(env.get("LD_LIBRARY_PATH", [])) if self.__override_all__ is not None: check = [self.__override_all__] for alias_command in check: try: alias_cmd = local[alias_command] alias_cmd = alias_cmd.with_env( PATH=list_to_path(path), LD_LIBRARY_PATH=list_to_path(libs_path)) return alias_cmd except AttributeError: pass LOG.warning("'%s' cannot be found. Import failed.", command) return ERROR
def __ld_library_path(): host_path = os.getenv('LD_LIBRARY_PATH', default='') env = CFG['env'].value benchbuild_path = list_to_path(env.get('LD_LIBRARY_PATH', [])) return os.path.pathsep.join([benchbuild_path, host_path])
def __ld_library_path(): host_path = os.getenv('LD_LIBRARY_PATH', default='') env = CFG['env'].value benchbuild_path = list_to_path(env.get('LD_LIBRARY_PATH', [])) return os.path.pathsep.join([benchbuild_path, host_path])