def test_vara_test_repo_dir_creation(self) -> None: """Test if the needed directories of the main repo and its submodules are present.""" mkdir("-p", self.bb_result_report_path) self.elementalist.version( f"{self.bb_result_report_path}/" f"TwoLibsOneProjectInteractionDiscreteLibsSingleProject" f"-cpp_projects@{self.revision}", version=self.revision) # Are directories present? self.assertTrue( isdir(f"{str(bb_cfg()['tmp_dir'])}/" f"TwoLibsOneProjectInteractionDiscreteLibsSingleProject")) self.assertTrue(isdir(self.bb_result_lib_path)) self.assertTrue(isdir(self.bb_result_lib_path / "Elementalist")) self.assertTrue(isdir(self.bb_result_lib_path / "fire_lib")) self.assertTrue(isdir(self.bb_result_lib_path / "water_lib")) self.assertTrue( isdir(self.bb_result_lib_path / "Elementalist" / "external" / "fire_lib")) self.assertTrue( isdir(self.bb_result_lib_path / "Elementalist" / "external" / "water_lib"))
def compile(self): self.download() js_dir = local.path(self.src_file) / "js" / "src" clang = compiler.cc(self) clang_cxx = compiler.cxx(self) with local.cwd(js_dir): make_src_pkg = local["./make-source-package.sh"] with local.env(DIST=self.builddir, MOZJS_MAJOR_VERSION=0, MOZJS_MINOR_VERSION=0, MOZJS_PATCH_VERSION=0): make_src_pkg() mozjs_dir = local.path("mozjs-0.0.0") mozjs_src_dir = mozjs_dir / "js" / "src" tar("xfj", mozjs_dir + ".tar.bz2") with local.cwd(mozjs_src_dir): mkdir("obj") autoconf = local["autoconf-2.13"] autoconf() with local.cwd("obj"): with local.env(CC=str(clang), CXX=str(clang_cxx)): configure = local["../configure"] configure = configure["--without-system-zlib"] run.run(configure) mozjs_obj_dir = mozjs_src_dir / "obj" with local.cwd(mozjs_obj_dir): run.run(make["-j", str(CFG["jobs"])])
def compile(self) -> None: """Compile the project.""" gzip_version_source = local.path(self.source_of_primary) # Build binaries in separate dir because executing the binary with path # 'gzip' will execute '/usr/bin/gzip' independent of the current working # directory. mkdir("-p", gzip_version_source / "build") self.cflags += [ "-Wno-error=string-plus-int", "-Wno-error=shift-negative-value", "-Wno-string-plus-int", "-Wno-shift-negative-value" ] with local.cwd(gzip_version_source): bb.watch(local["./bootstrap"])() c_compiler = bb.compiler.cc(self) with local.cwd(gzip_version_source / "build"), local.env(CC=str(c_compiler)): bb.watch(local["../configure"])() bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) with local.cwd(gzip_version_source): verify_binaries(self)
def compile(self): self.download() makefile_config = local.path(self.src_file) / "config" / "make.config" clang = compiler.cc(self) with open(makefile_config, 'w') as config: lines = [ "LABEL=benchbuild", "ENABLE_OMPSS=", "OMPSSC=", "OMPC=", "CC={cc}", "OMPSSLINK=", "OMPLINK={cc} -fopenmp", "CLINK={cc}", "OPT_FLAGS=", "CC_FLAGS=", "OMPC_FLAGS=", "OMPSSC_FLAGS=", "OMPC_FINAL_FLAGS=", "OMPSSC_FINAL_FLAG=", "CLINK_FLAGS=", "OMPLINK_FLAGS=", "OMPSSLINK_FLAGS=", ] lines = [l.format(cc=clang) + "\n" for l in lines] config.writelines(lines) mkdir(local.path(self.src_file) / "bin") with local.cwd(self.src_file): run.run(make["-C", self.path_dict[self.name]])
def pack_container(in_container, out_file): """ Pack a container image into a .tar.bz2 archive. Args: in_container (str): Path string to the container image. out_file (str): Output file name. """ container_filename = os.path.split(out_file)[-1] out_container = os.path.join("container-out", container_filename) out_container = os.path.abspath(out_container) out_tmp_filename = os.path.basename(out_container) out_dir = os.path.dirname(out_container) # Pack the results to: container-out with local.cwd(in_container): tar("cjf", out_container, ".") c_hash = update_hash(out_tmp_filename, out_dir) if not os.path.exists(out_dir): mkdir("-p", out_dir) mv(out_container, out_file) mv(out_container + ".hash", out_file + ".hash") new_container = {"path": out_file, "hash": str(c_hash)} settings.CFG["container"]["known"].value().append(new_container)
def compile(self) -> None: """Compile the project.""" brotli_version_source = local.path(self.source_of_primary) brotli_git_path = get_local_project_git_path(self.NAME) brotli_version = ShortCommitHash(self.version_of_primary) with local.cwd(brotli_git_path): configure_revisions = get_all_revisions_between( "f9ab24a7aaee93d5932ba212e5e3d32e4306f748", "5814438791fb2d4394b46e5682a96b68cd092803", ShortCommitHash) simple_make_revisions = get_all_revisions_between( "e1739826c04a9944672b99b98249dda021bdeb36", "378485b097fd7b80a5e404a3cb912f7b18f78cdb", ShortCommitHash) c_compiler = bb.compiler.cc(self) if brotli_version in simple_make_revisions: with local.cwd(brotli_version_source / "tools"): bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) elif brotli_version in configure_revisions: with local.cwd(brotli_version_source): with local.env(CC=str(c_compiler)): bb.watch(local["./configure"])() bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) else: mkdir(brotli_version_source / "out") with local.cwd(brotli_version_source / "out"): with local.env(CC=str(c_compiler)): bb.watch(local["../configure-cmake"])() bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) with local.cwd(brotli_version_source): verify_binaries(self)
def setup_directories(builddir): """Create the in and out directories of the container.""" with local.cwd(builddir): if not os.path.exists("container-in"): mkdir("-p", "container-in") if not os.path.exists("container-out"): mkdir("-p", "container-out")
def unionfs_set_up(ro_base, rw_image, mountpoint): """ Setup a unionfs via unionfs-fuse. Args: ro_base: base_directory of the project rw_image: virtual image of actual file system mountpoint: location where ro_base and rw_image merge """ log = logging.getLogger("benchbuild") if not os.path.exists(mountpoint): mkdir("-p", mountpoint) if not os.path.exists(ro_base): log.error("Base dir does not exist: '{0}'".format(ro_base)) raise ValueError("Base directory does not exist") if not os.path.exists(rw_image): log.error("Image dir does not exist: '{0}'".format(ro_base)) raise ValueError("Image directory does not exist") from benchbuild.utils.cmd import unionfs ro_base = os.path.abspath(ro_base) rw_image = os.path.abspath(rw_image) mountpoint = os.path.abspath(mountpoint) unionfs("-o", "allow_other,cow", rw_image + "=RW:" + ro_base + "=RO", mountpoint)
def compile(self): self.download() js_dir = local.path(self.src_file) / "js" / "src" clang = compiler.cc(self) clang_cxx = compiler.cxx(self) with local.cwd(js_dir): make_src_pkg = local["./make-source-package.sh"] with local.env( DIST=self.builddir, MOZJS_MAJOR_VERSION=0, MOZJS_MINOR_VERSION=0, MOZJS_PATCH_VERSION=0): make_src_pkg() mozjs_dir = local.path("mozjs-0.0.0") mozjs_src_dir = mozjs_dir / "js" / "src" tar("xfj", mozjs_dir + ".tar.bz2") with local.cwd(mozjs_src_dir): mkdir("obj") autoconf = local["autoconf-2.13"] autoconf() with local.cwd("obj"): with local.env(CC=str(clang), CXX=str(clang_cxx)): configure = local["../configure"] configure = configure["--without-system-zlib"] run.run(configure) mozjs_obj_dir = mozjs_src_dir / "obj" with local.cwd(mozjs_obj_dir): run.run(make["-j", str(CFG["jobs"])])
def compile(self): self.download() download.Git(self.test_suite_uri, self.test_suite_dir) venv_path = local.cwd / "local" virtualenv(venv_path, "--python=python2") pip_path = local.cwd / "local" / "bin" / "pip" pip = local[pip_path] with local.cwd(self.SRC_FILE): pip("install", "--no-cache-dir", "--disable-pip-version-check", "-e", ".") self.sandbox_dir = local.cwd / "run" if self.sandbox_dir.exists(): rm("-rf", self.sandbox_dir) mkdir(self.sandbox_dir) self.lnt = local[local.path("./local/bin/lnt")] self.clang = compiler.cc(self, detect_project=True) self.clang_cxx = compiler.cxx(self, detect_project=True) self.lnt("runtest", "test-suite", "-v", "-j1", "--sandbox", self.sandbox_dir, "--benchmarking-only", "--only-compile", "--cc", str(self.clang), "--cxx", str(self.clang_cxx), "--test-suite", self.test_suite_dir, "--only-test=" + self.SUBDIR)
def configure(self): clang = cc(self) with local.cwd(self.src_dir): mkdir("build") with local.cwd("build"): configure = local["../configure"] run(configure["--cc=" + str(clang), "--with-libgcc"])
def setup_build_link(self) -> None: """Setup build-config folder link for VaRA's default build setup scripts.""" llvm_project_dir = self.base_dir / self.get_sub_project( "vara-llvm-project").path mkdir(llvm_project_dir / "build/") with local.cwd(llvm_project_dir / "build/"): ln("-s", "../vara/utils/vara/builds/", "build_cfg")
def prepare_directories(dirs): """ Make sure that the required directories exist. Args: dirs - the directories we want. """ for directory in dirs: mkdir("-p", directory, retcode=None)
def _uchroot_mounts(prefix, mounts, uchroot): i = 0 new_uchroot = uchroot mntpoints = [] for mount in mounts: mntpoint = "{0}/{1}".format(prefix, str(i)) mkdir("-p", mntpoint) new_uchroot = new_uchroot["-M", "{0}:/{1}".format(mount, mntpoint)] mntpoints.append(mntpoint) i = i + 1 return new_uchroot, mntpoints
def download(self): book_file = "book.bin" book_bin = "http://www.craftychess.com/downloads/book/" + book_file Wget(self.src_uri, self.SRC_FILE) Wget(book_bin, book_file) mkdir(self.src_dir) with local.cwd(self.src_dir): unzip(path.join("..", self.SRC_FILE)) mv(book_file, self.src_dir)
def compile(self) -> None: """Compile the project.""" libpng_source = local.path(self.source_of(self.primary_source)) compiler = bb.compiler.cc(self) mkdir(libpng_source / "build") with local.cwd(libpng_source / "build"): with local.env(CC=str(compiler)): bb.watch(cmake)("-G", "Unix Makefiles", "..") bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) with local.cwd(libpng_source): verify_binaries(self)
def main(self, *args): log.configure() builddir = local.path(str(CFG["build_dir"])) if not builddir.exists(): response = ui.ask( "The build directory {dirname} does not exist yet. " "Should I create it?".format(dirname=builddir)) if response: mkdir("-p", builddir) print("Created directory {0}.".format(builddir)) setup_directories(builddir)
def compile(self) -> None: """Contains instructions on how to build the project.""" version_source = local.path(self.source_of_primary) c_compiler = bb.compiler.cc(self) cxx_compiler = bb.compiler.cxx(self) mkdir(version_source / "build") with local.cwd(version_source / "build"): with local.env(CC=str(c_compiler), CXX=str(cxx_compiler)): bb.watch(cmake)("-G", "Unix Makefiles", "..") bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) verify_binaries(self)
def compile(self): self.download() tar("xf", self.src_file) unpack_dir = local.path('tcc-{0}.tar.bz2'.format(self.version)) clang = compiler.cc(self) with local.cwd(unpack_dir): mkdir("build") with local.cwd("build"): configure = local["../configure"] run.run(configure["--cc=" + str(clang), "--with-libgcc"]) run.run(make)
def version(self, target_dir: str, version: str = 'HEAD') -> pb.LocalPath: """Overrides ``Git`` s version to create a new git worktree pointing to the requested version.""" main_repo_src_local = self.fetch() tgt_loc = pb.local.path(target_dir) / self.local vara_test_repos_path = self.__vara_test_repos_git.fetch() main_repo_src_remote = vara_test_repos_path / self.remote mkdir('-p', tgt_loc) # Extract main repository cp("-r", main_repo_src_local + "/.", tgt_loc) # Skip submodule extraction if none exist if not Path(tgt_loc / ".gitmodules").exists(): with pb.local.cwd(tgt_loc): git("checkout", "--detach", version) return tgt_loc # Extract submodules with pb.local.cwd(tgt_loc): # Get submodule entries submodule_url_entry_list = git( "config", "--file", ".gitmodules", "--name-only", "--get-regexp", "url" ).split('\n') # Remove empty strings submodule_url_entry_list = list( filter(None, submodule_url_entry_list) ) for entry in submodule_url_entry_list: relative_submodule_url = Path( git("config", "--file", ".gitmodules", "--get", entry).replace('\n', '') ) copy_renamed_git_to_dest( main_repo_src_remote / relative_submodule_url, relative_submodule_url ) git("checkout", "--detach", version) git("submodule", "update") return tgt_loc
def _uchroot_mounts(prefix, mounts, uchrt): i = 0 new_uchroot = uchrt mntpoints = [] for mount in mounts: src_mount = mount if isinstance(mount, dict): src_mount = mount["src"] tgt_mount = mount["tgt"] else: tgt_mount = "{0}/{1}".format(prefix, str(i)) i = i + 1 mkdir("-p", tgt_mount) new_uchroot = new_uchroot["-M", "{0}:/{1}".format(src_mount, tgt_mount)] mntpoints.append(tgt_mount) return new_uchroot, mntpoints
def unpack_container(container, path): """ Unpack a container usable by uchroot. Method that checks if a directory for the container exists, checks if erlent support is needed and then unpacks the container accordingly. Args: path: The location where the container is, that needs to be unpacked. """ from benchbuild.utils.run import run, uchroot_no_args path = os.path.abspath(path) name = os.path.basename(os.path.abspath(container.filename)) if not os.path.exists(path): mkdir("-p", path) with local.cwd(path): Wget(container.remote, name) uchroot = uchroot_no_args() uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash[ "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(name)] has_erlent = (has_erlent & TF) cmd = local["/bin/tar"]["xf"] if not has_erlent: cmd = uchroot[cmd["./" + name]] else: cmd = cmd[name] run(cmd["--exclude=dev/*"]) if not os.path.samefile(name, container.filename): rm(name) else: logging.warning("File contents do not match: {0} != {1}", name, container.filename) cp(container.filename + ".hash", path)
def compile(self) -> None: """Compile the project.""" bzip2_source = local.path(self.source_of(self.primary_source)) cc_compiler = bb.compiler.cc(self) cxx_compiler = bb.compiler.cxx(self) mkdir("-p", bzip2_source / "build") with local.cwd(bzip2_source / "build"): with local.env(CC=str(cc_compiler), CXX=str(cxx_compiler)): bb.watch(cmake)("..") bb.watch(cmake)("--build", ".", "--config", "Release", "-j", get_number_of_jobs(bb_cfg())) with local.cwd(bzip2_source): verify_binaries(self)
def uchroot(*args, **kwargs): """ Return a customizable uchroot command. Args: args: List of additional arguments for uchroot (typical: mounts) Return: chroot_cmd """ mkdir("-p", "llvm") uchroot_cmd = uchroot_no_llvm(*args, **kwargs) uchroot_cmd, mounts = _uchroot_mounts("mnt", CFG["container"]["mounts"].value(), uchroot_cmd) paths, libs = uchroot_env(mounts) uchroot_cmd = uchroot_cmd.with_env(LD_LIBRARY_PATH=list_to_path(libs), PATH=list_to_path(paths)) return uchroot_cmd["--"]
def compile(self): self.download() book_file = "book.bin" book_bin = "http://www.craftychess.com/downloads/book/" + book_file download.Wget(book_bin, book_file) unpack_dir = "crafty.src" mkdir(unpack_dir) with local.cwd(unpack_dir): unzip(local.path("..") / self.src_file) mv(book_file, unpack_dir) clang = compiler.cc(self) with local.cwd(unpack_dir): target_opts = ["-DCPUS=1", "-DSYZYGY", "-DTEST"] crafty_make = make["target=UNIX", "CC=" + str(clang), "opt=" + " ".join(target_opts), "crafty-make"] run.run(crafty_make)
def pack_container(in_container, out_file): container_filename = os.path.split(out_file)[-1] out_container = os.path.join("container-out", container_filename) out_container = os.path.abspath(out_container) out_tmp_filename = os.path.basename(out_container) out_dir = os.path.dirname(out_container) # Pack the results to: container-out with local.cwd(in_container): tar("cjf", out_container, ".") c_hash = update_hash(out_tmp_filename, out_dir) if not os.path.exists(out_dir): mkdir("-p", out_dir) mv(out_container, out_file) mv(out_container + ".hash", out_file + ".hash") new_container = {"path": out_file, "hash": str(c_hash)} CFG["container"]["known"].value().append(new_container)
def get_varats_result_folder(project: Project) -> Path: """ Get the project specific path to the varats result folder. Args: project: to lookup the result folder for Returns: path to the project specific result folder """ result_folder_template = "{result_dir}/{project_dir}" vara_result_folder = result_folder_template.format( result_dir=str(bb_cfg()["varats"]["outfile"]), project_dir=str(project.name) ) mkdir("-p", vara_result_folder) return Path(vara_result_folder)
def main(self, *args): log.configure() _log = logging.getLogger() _log.setLevel({ 3: logging.DEBUG, 2: logging.INFO, 1: logging.WARNING, 0: logging.ERROR }[self.verbosity]) builddir = os.path.abspath(str(CFG["build_dir"])) if not os.path.exists(builddir): response = ask("The build directory {dirname} does not exist yet. " "Should I create it?".format(dirname=builddir)) if response: mkdir("-p", builddir) print("Created directory {0}.".format(builddir)) setup_directories(builddir)
def mkdir_interactive(dirpath): """ Create a directory if required. This will query the user for a confirmation. Args: dirname: The path to create. """ from benchbuild.utils.cmd import mkdir if os.path.exists(dirpath): return response = ui.ask("The directory {dirname} does not exist yet. " "Should I create it?".format(dirname=dirpath), default_answer=True, default_answer_str="yes") if response: mkdir("-p", dirpath) print("Created directory {0}.".format(dirpath))
def run_tests(self, experiment, run): povray_binary = path.join(self.SRC_FILE, "unix", self.name) tmpdir = "tmp" povini = path.join("cfg", ".povray", "3.6", "povray.ini") scene_dir = path.join("share", "povray-3.6", "scenes") mkdir(tmpdir, retcode=None) povray = wrap(povray_binary, experiment) pov_files = find(scene_dir, "-name", "*.pov").splitlines() for pov_f in pov_files: with local.env(POVRAY=povray_binary, INSTALL_DIR='.', OUTPUT_DIR=tmpdir, POVINI=povini): options = ((((head["-n", "50", "\"" + pov_f + "\""] | grep["-E", "'^//[ ]+[-+]{1}[^ -]'"]) | head["-n", "1"]) | sed["s?^//[ ]*??"]) & FG) run(povray["+L" + scene_dir, "+L" + tmpdir, "-i" + pov_f, "-o" + tmpdir, options, "-p"], retcode=None)
def install_uchroot(): from benchbuild.utils.cmd import git, mkdir builddir = settings.CFG["build_dir"].value() with local.cwd(builddir): if not os.path.exists("erlent/.git"): git("clone", settings.CFG["uchroot"]["repo"].value()) else: with local.cwd("erlent"): git("pull", "--rebase") mkdir("-p", "erlent/build") with local.cwd("erlent/build"): from benchbuild.utils.cmd import cmake from benchbuild.utils.cmd import make cmake("../") make() erlent_path = os.path.abspath(os.path.join(builddir, "erlent", "build")) os.environ["PATH"] = os.path.pathsep.join( [erlent_path, os.environ["PATH"]]) local.env.update(PATH=os.environ["PATH"]) if not find_package("uchroot"): sys.exit(-1) settings.CFG["env"]["lookup_path"].value().append(erlent_path)
def mkdir_interactive(dirpath): """ Create a directory if required. This will query the user for a confirmation. Args: dirname: The path to create. """ from benchbuild.utils.cmd import mkdir if os.path.exists(dirpath): return response = ui.ask( "The directory {dirname} does not exist yet. " "Should I create it?".format(dirname=dirpath), default_answer=True, default_answer_str="yes") if response: mkdir("-p", dirpath) print("Created directory {0}.".format(dirpath))
def pack_container(in_container, out_file): """ Pack a container image into a .tar.bz2 archive. Args: in_container (str): Path string to the container image. out_file (str): Output file name. """ container_filename = local.path(out_file).basename out_container = local.cwd / "container-out" / container_filename out_dir = out_container.dirname # Pack the results to: container-out with local.cwd(in_container): tar("cjf", out_container, ".") c_hash = download.update_hash(out_container) if out_dir.exists(): mkdir("-p", out_dir) mv(out_container, out_file) mv(out_container + ".hash", out_file + ".hash") new_container = {"path": out_file, "hash": str(c_hash)} CFG["container"]["known"] += new_container
def compile(self): self.download() download.Wget(self.boost_src_uri, self.boost_src_file) tar("xfj", self.boost_src_file) cp("-ar", local.path(self.testdir) / "cfg", '.') cp("-ar", local.path(self.testdir) / "etc", '.') cp("-ar", local.path(self.testdir) / "scenes", '.') cp("-ar", local.path(self.testdir) / "share", '.') cp("-ar", local.path(self.testdir) / "test", '.') clang = compiler.cc(self) clang_cxx = compiler.cxx(self) # First we have to prepare boost for lady povray... boost_prefix = "boost-install" with local.cwd(self.boost_src_dir): mkdir(boost_prefix) bootstrap = local["./bootstrap.sh"] run.run(bootstrap["--with-toolset=clang", "--prefix=\"{0}\"".format(boost_prefix)]) _b2 = local["./b2"] run.run( _b2["--ignore-site-config", "variant=release", "link=static", "threading=multi", "optimization=speed", "install"]) src_file = local.path(self.src_file) with local.cwd(src_file): with local.cwd("unix"): sh("prebuild.sh") configure = local["./configure"] with local.env( COMPILED_BY="BB <*****@*****.**>", CC=str(clang), CXX=str(clang_cxx)): run.run(configure["--with-boost=" + boost_prefix]) run.run(make["all"])