def configure(self): from benchbuild.utils.run import run # First we have to prepare boost for lady povray... boost_dir = path.join(self.builddir, self.boost_src_dir) boost_prefix = path.join(self.builddir, "boost-install") with local.cwd(boost_dir): from plumbum.cmd import mkdir mkdir(boost_prefix) bootstrap = local["./bootstrap.sh"] run(bootstrap["--with-toolset=clang", "--prefix=\"{0}\"".format( boost_prefix)]) b2 = local["./b2"] run(b2["--ignore-site-config", "variant=release", "link=static", "threading=multi", "optimization=speed", "install"]) povray_dir = path.join(self.builddir, self.src_dir) with local.cwd(path.join(povray_dir, "unix")): from plumbum.cmd import sh sh("prebuild.sh") with local.cwd(povray_dir): from benchbuild.utils.compiler import lt_clang, lt_clang_cxx with local.cwd(self.builddir): clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) clang_cxx = lt_clang_cxx(self.cflags, self.ldflags, self.compiler_extension) configure = local["./configure"] with local.env(COMPILED_BY="BB <*****@*****.**>", CC=str(clang), CXX=str(clang_cxx)): run(configure["--with-boost=" + boost_prefix])
def compile(self): self.download() js_dir = local.path(self.src_file) / "js" / "src" clang = compiler.cc(self) clang_cxx = compiler.cxx(self) with local.cwd(js_dir): make_src_pkg = local["./make-source-package.sh"] with local.env( DIST=self.builddir, MOZJS_MAJOR_VERSION=0, MOZJS_MINOR_VERSION=0, MOZJS_PATCH_VERSION=0): make_src_pkg() mozjs_dir = local.path("mozjs-0.0.0") mozjs_src_dir = mozjs_dir / "js" / "src" tar("xfj", mozjs_dir + ".tar.bz2") with local.cwd(mozjs_src_dir): mkdir("obj") autoconf = local["autoconf-2.13"] autoconf() with local.cwd("obj"): with local.env(CC=str(clang), CXX=str(clang_cxx)): configure = local["../configure"] configure = configure["--without-system-zlib"] run.run(configure) mozjs_obj_dir = mozjs_src_dir / "obj" with local.cwd(mozjs_obj_dir): run.run(make["-j", str(CFG["jobs"])])
def compile(self): self.download() download.Git(self.gdal_uri, self.gdal_dir) rasdaman_dir = local.path(self.src_file) gdal_dir = local.path(self.gdal_dir) / self.gdal_dir clang = compiler.cc(self) clang_cxx = compiler.cxx(self) with local.cwd(gdal_dir): configure = local["./configure"] with local.env(CC=str(clang), CXX=str(clang_cxx)): run.run(configure["--with-pic", "--enable-static", "--disable-debug", "--with-gnu-ld", "--without-ld-shared", "--without-libtool"]) run.run(make["-j", CFG["jobs"]]) with local.cwd(rasdaman_dir): autoreconf("-i") configure = local["./configure"] with local.env(CC=str(clang), CXX=str(clang_cxx)): run.run(configure["--without-debug-symbols", "--enable-benchmark", "--with-static-libs", "--disable-java", "--with-pic", "--disable-debug", "--without-docs"]) run.run(make["clean", "all", "-j", CFG["jobs"]])
def run_makefile(self, makefile=None): """ Run the whole build process with designated Makefile. """ # cd into git repository directory with local.cwd(self.repo.working_dir): # cd into working directory inside git repository with local.cwd(self.workingdir): # Run Makefile to start the compilation process make('--file', makefile, 'all-plus-firmware-info', stdout=self.stream, stderr=self.stream) # Slurp output of build process try: self.stream.seek(0) output = self.stream.read() except IOError: make_firmware_info = make['--file', makefile, 'firmware-info'] | grep['TARGET_'] output = make_firmware_info() # Grep "TARGET_HEX", "TARGET_ELF" (for AVR) as well as "TARGET_BIN" and "TARGET_CHIP" (for ESP) paths # from build output and store into "self.build_result" target_matcher = re.compile('(?P<name>TARGET_.+):(?: (?P<value>.+))?') for m in target_matcher.finditer(output): match = m.groupdict() name = match['name'] value = match['value'] if value: self.build_result[name] = value # Add build path to build result self.build_result['build_path'] = pwd().strip()
def setup_container(builddir, container): with local.cwd(builddir): container_filename = str(container).split(os.path.sep)[-1] container_in = os.path.join("container-in", container_filename) Copy(container, container_in) uchroot = uchroot_no_args() with local.cwd("container-in"): uchroot = uchroot["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash[ "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format( container_in)] has_erlent = (has_erlent & TF) # Unpack input container to: container-in if not has_erlent: cmd = local["/bin/tar"]["xf"] cmd = uchroot[cmd[container_filename]] else: cmd = tar["xf"] cmd = cmd[os.path.abspath(container_in)] with local.cwd("container-in"): cmd("--exclude=dev/*") rm(container_in) return os.path.join(builddir, "container-in")
def configure_openmp(self, openmp_path): """ Configure LLVM/Clang's own OpenMP runtime. """ from plumbum.cmd import cmake with local.cwd(openmp_path): builddir = os.path.join(openmp_path, "build") if not os.path.exists(builddir): mkdir(builddir) with local.cwd(builddir): cmake_cache = os.path.join(builddir, "CMakeCache.txt") install_path = os.path.join(self._builddir, "install") openmp_cmake = cmake[ "-DCMAKE_INSTALL_PREFIX=" + install_path, "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_USE_RELATIVE_PATHS=On", "-DLIBOMP_ENABLE_ASSERTIONS=Off"] if self._use_make: openmp_cmake = openmp_cmake["-G", "Unix Makefiles"] else: openmp_cmake = openmp_cmake["-G", "Ninja"] if not os.path.exists(cmake_cache): openmp_cmake = configure_compiler(openmp_cmake, use_gcc=False) openmp_cmake = openmp_cmake[openmp_path] else: openmp_cmake = openmp_cmake["."] openmp_cmake()
def configure_llvm(self, llvm_path): """ Configure LLVM and all subprojects. """ with local.cwd(llvm_path): builddir = os.path.join(llvm_path, "build") if not os.path.exists(builddir): mkdir(builddir) with local.cwd(builddir): cmake_cache = os.path.join(builddir, "CMakeCache.txt") install_path = os.path.join(self._builddir, "install") llvm_cmake = cmake[ "-DCMAKE_INSTALL_PREFIX=" + install_path, "-DCMAKE_BUILD_TYPE=Release", "-DBUILD_SHARED_LIBS=Off", "-DCMAKE_USE_RELATIVE_PATHS=On", "-DPOLLY_BUILD_POLLI=On", "-DLLVM_TARGETS_TO_BUILD=X86", "-DLLVM_BINUTILS_INCDIR=/usr/include/", "-DLLVM_ENABLE_PIC=On", "-DLLVM_ENABLE_ASSERTIONS=On", "-DCLANG_DEFAULT_OPENMP_RUNTIME=libomp", "-DCMAKE_CXX_FLAGS_RELEASE='-O3 -DNDEBUG -fno-omit-frame-pointer'"] if self._use_make: llvm_cmake = llvm_cmake["-G", "Unix Makefiles"] else: llvm_cmake = llvm_cmake["-G", "Ninja"] llvm_cmake = configure_papi(llvm_cmake, self._papidir) llvm_cmake = configure_likwid(llvm_cmake, self._likwiddir) llvm_cmake = configure_isl(llvm_cmake, self._isldir) if not os.path.exists(cmake_cache): llvm_cmake = configure_compiler(llvm_cmake, self._use_gcc) llvm_cmake = llvm_cmake[llvm_path] else: llvm_cmake = llvm_cmake["."] llvm_cmake()
def install_uchroot(_): """Installer for erlent (contains uchroot).""" builddir = local.path(str(CFG["build_dir"].value)) with local.cwd(builddir): erlent_src = local.path('erlent') erlent_git = erlent_src / '.git' erlent_repo = str(CFG['uchroot']['repo']) erlent_build = erlent_src / 'build' if not erlent_git.exists(): git("clone", erlent_repo) else: with local.cwd(erlent_src): git("pull", "--rebase") erlent_build.mkdir() with local.cwd(erlent_build): cmake("../") make() os.environ["PATH"] = os.path.pathsep.join( [erlent_build, os.environ["PATH"]]) local.env.update(PATH=os.environ["PATH"]) if not find_package("uchroot"): LOG.error('uchroot not found, after updating PATH to %s', os.environ['PATH']) sys.exit(-1) env = CFG['env'].value if 'PATH' not in env: env['PATH'] = [] env['PATH'].append(str(erlent_build))
def configure(self): lapack_dir = path.join(self.builddir, self.src_dir) from benchbuild.utils.compiler import lt_clang, lt_clang_cxx with local.cwd(self.builddir): clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) clang_cxx = lt_clang_cxx(self.cflags, self.ldflags, self.compiler_extension) with local.cwd(lapack_dir): with open("make.inc", 'w') as makefile: content = [ "SHELL = /bin/sh\n", "PLAT = _LINUX\n", "CC = " + str(clang) + "\n", "CXX = " + str(clang_cxx) + "\n", "CFLAGS = -I$(TOPDIR)/INCLUDE\n", "LOADER = " + str(clang) + "\n", "LOADOPTS = \n", "NOOPT = -O0 -I$(TOPDIR)/INCLUDE\n", "DRVCFLAGS = $(CFLAGS)\n", "F2CCFLAGS = $(CFLAGS)\n", "TIMER = INT_CPU_TIME\n", "ARCH = ar\n", "ARCHFLAGS = cr\n", "RANLIB = ranlib\n", "BLASLIB = ../../blas$(PLAT).a\n", "XBLASLIB = \n", "LAPACKLIB = lapack$(PLAT).a\n", "F2CLIB = ../../F2CLIBS/libf2c.a\n", "TMGLIB = tmglib$(PLAT).a\n", "EIGSRCLIB = eigsrc$(PLAT).a\n", "LINSRCLIB = linsrc$(PLAT).a\n", "F2CLIB = ../../F2CLIBS/libf2c.a\n" ] makefile.writelines(content)
def setup_bash_in_container(builddir, container, outfile, mounts, shell): with local.cwd(builddir): # Switch to bash inside uchroot print("Entering bash inside User-Chroot. Prepare your image and " "type 'exit' when you are done. If bash exits with a non-zero" "exit code, no new container will be stored.") store_new_container = True try: run_in_container(shell, container, mounts) except ProcessExecutionError: store_new_container = False if store_new_container: # pylint: disable=W0104 print("Packing new container image.") container_filename = os.path.split(container)[-1] container_out = os.path.join("container-out", container_filename) container_out = os.path.abspath(container_out) # Pack the results to: container-out with local.cwd("container-in"): tar("cjf", container_out, ".") update_hash(container_filename, os.path.dirname(container_out)) outdir = os.path.dirname(outfile) if not os.path.exists(outdir): mkdir("-p", outdir) mv(container_out, outfile)
def run_tests(self, experiment): from benchbuild.project import wrap from benchbuild.utils.run import run lapack_dir = path.join(self.builddir, self.src_dir) with local.cwd(lapack_dir): with local.cwd(path.join("BLAS")): xblat2s = wrap("xblat2s", experiment) xblat2d = wrap("xblat2d", experiment) xblat2c = wrap("xblat2c", experiment) xblat2z = wrap("xblat2z", experiment) xblat3s = wrap("xblat3s", experiment) xblat3d = wrap("xblat3d", experiment) xblat3c = wrap("xblat3c", experiment) xblat3z = wrap("xblat3z", experiment) run((xblat2s < "sblat2.in")) run((xblat2d < "dblat2.in")) run((xblat2c < "cblat2.in")) run((xblat2z < "zblat2.in")) run((xblat3s < "sblat3.in")) run((xblat3d < "dblat3.in")) run((xblat3c < "cblat3.in")) run((xblat3z < "zblat3.in"))
def compile(self): self.download() tar("xfz", self.src_file) unpack_dir = "CLAPACK-{0}".format(self.version) clang = compiler.cc(self) clang_cxx = compiler.cxx(self) with local.cwd(unpack_dir): with open("make.inc", 'w') as makefile: content = [ "SHELL = /bin/sh\n", "PLAT = _LINUX\n", "CC = " + str(clang) + "\n", "CXX = " + str(clang_cxx) + "\n", "CFLAGS = -I$(TOPDIR)/INCLUDE\n", "LOADER = " + str(clang) + "\n", "LOADOPTS = \n", "NOOPT = -O0 -I$(TOPDIR)/INCLUDE\n", "DRVCFLAGS = $(CFLAGS)\n", "F2CCFLAGS = $(CFLAGS)\n", "TIMER = INT_CPU_TIME\n", "ARCH = ar\n", "ARCHFLAGS = cr\n", "RANLIB = ranlib\n", "BLASLIB = ../../blas$(PLAT).a\n", "XBLASLIB = \n", "LAPACKLIB = lapack$(PLAT).a\n", "F2CLIB = ../../F2CLIBS/libf2c.a\n", "TMGLIB = tmglib$(PLAT).a\n", "EIGSRCLIB = eigsrc$(PLAT).a\n", "LINSRCLIB = linsrc$(PLAT).a\n", "F2CLIB = ../../F2CLIBS/libf2c.a\n" ] makefile.writelines(content) run.run(make["-j", CFG["jobs"], "f2clib", "blaslib"]) with local.cwd(local.path("BLAS") / "TESTING"): run.run(make["-j", CFG["jobs"], "-f", "Makeblat2"]) run.run(make["-j", CFG["jobs"], "-f", "Makeblat3"])
def setup_container(builddir, _container): """Prepare the container and returns the path where it can be found.""" build_dir = local.path(builddir) in_dir = build_dir / "container-in" container_path = local.path(_container) with local.cwd(builddir): container_bin = container_path.basename container_in = in_dir / container_bin download.Copy(_container, container_in) uchrt = uchroot.no_args() with local.cwd("container-in"): uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/", "-w", os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. has_erlent = bash[ "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format( container_in)] has_erlent = (has_erlent & TF) # Unpack input container to: container-in if not has_erlent: cmd = local["/bin/tar"]["xf"] cmd = uchrt[cmd[container_bin]] else: cmd = tar["xf"] cmd = cmd[container_in] with local.cwd("container-in"): cmd("--exclude=dev/*") rm(container_in) return in_dir
def configure(self): from benchbuild.utils.compiler import lt_clang, lt_clang_cxx from benchbuild.utils.run import run from plumbum.cmd import autoreconf, make rasdaman_dir = path.join(self.builddir, self.src_dir) gdal_dir = path.join(self.builddir, self.gdal_dir, self.gdal_dir) with local.cwd(self.builddir): clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) clang_cxx = lt_clang_cxx(self.cflags, self.ldflags, self.compiler_extension) with local.cwd(gdal_dir): configure = local["./configure"] with local.env(CC=str(clang), CXX=str(clang_cxx)): run(configure["--with-pic", "--enable-static", "--disable-debug", "--with-gnu-ld", "--without-ld-shared", "--without-libtool"]) run(make["-j", CFG["jobs"]]) with local.cwd(rasdaman_dir): autoreconf("-i") configure = local["./configure"] with local.env(CC=str(clang), CXX=str(clang_cxx)): run(configure["--without-debug-symbols", "--enable-benchmark", "--with-static-libs", "--disable-java", "--with-pic", "--disable-debug", "--without-docs"])
def in_merge_conflict(tmpdir_factory): path = make_consuming_repo(tmpdir_factory, 'script_hooks_repo') with local.cwd(path): local['touch']('dummy') git('add', 'dummy') git('add', C.CONFIG_FILE) git('commit', '-m', 'Add config.') conflict_path = tmpdir_factory.get() git('clone', path, conflict_path) with local.cwd(conflict_path): git('checkout', 'origin/master', '-b', 'foo') with io.open('conflict_file', 'w') as conflict_file: conflict_file.write('herp\nderp\n') git('add', 'conflict_file') with io.open('foo_only_file', 'w') as foo_only_file: foo_only_file.write('foo') git('add', 'foo_only_file') git('commit', '-m', 'conflict_file') git('checkout', 'origin/master', '-b', 'bar') with io.open('conflict_file', 'w') as conflict_file: conflict_file.write('harp\nddrp\n') git('add', 'conflict_file') with io.open('bar_only_file', 'w') as bar_only_file: bar_only_file.write('bar') git('add', 'bar_only_file') git('commit', '-m', 'conflict_file') git('merge', 'foo', retcode=None) yield os.path.join(conflict_path)
def run_tests(self, runner): unpack_dir = local.path('tcc-{0}.tar.bz2'.format(self.version)) with local.cwd(unpack_dir): with local.cwd("build"): wrapping.wrap("tcc", self) inc_path = path.abspath("..") runner(make["TCCFLAGS=-B{}".format(inc_path), "test", "-i"])
def clone_or_pull(repo_dict, to_dir): """ Clone or pull a repository and switch to the desired branch. If the directory already exists, we will try to du a pull with --rebase. In case anything goes wrong, we exit and print a simple diagnostic message. Args: url (str): Where is the remote repository? to_dir (str): Where should we clone/update to? branch (str): Which branch should we check out? Defaults to the repo's master. """ url = repo_dict["url"] branch = repo_dict.get("branch") commit_hash = repo_dict.get("commit_hash") from plumbum.cmd import git if not os.path.exists(os.path.join(to_dir, ".git/")): git_clone = git["clone", url, to_dir, "--recursive", "--depth=1"] if branch: git_clone = git_clone["--branch", branch] git_clone() elif not commit_hash: # If we haven't specified a commit hash, # fetch the latest version. with local.cwd(to_dir): git("remote", "update") locl = git("rev-parse", "@{0}") remote = git("rev-parse", "@{u}") base = git("merge-base", "@", "@{u}") if locl == remote: print("{:s} is up-to-date.".format(url)) elif locl == base: git("pull", "--rebase") git("submodule", "update") elif remote == base: print("push required") exit(1) else: print("{:s} has diverged from its remote.".format(to_dir)) exit(1) if commit_hash: with local.cwd(to_dir): # We only need to do something if we aren't already at the # latest commit hash current_hash = git("rev-parse", "--verify", "HEAD").rstrip("\n") if current_hash != commit_hash: # Make sure we have a full history, not just depth 1 print(( "HEAD for repository {:s} is not at configured commit hash {:s}, fetching and checking out.".format( url, commit_hash))) git("fetch", "--unshallow") git_checkout = git("checkout", commit_hash)
def download(self): from pprof.utils.downloader import Wget, Rsync from plumbum.cmd import tar with local.cwd(self.builddir): Wget(self.src_uri, self.src_file) tar('xfj', path.join(self.builddir, self.src_file)) with local.cwd(self.src_dir): Rsync(self.fate_uri, self.fate_dir)
def download(self): from pprof.utils.downloader import Git from plumbum.cmd import git minisat_dir = path.join(self.builddir, self.src_dir) with local.cwd(self.builddir): Git(self.src_uri, self.src_dir) with local.cwd(minisat_dir): git("fetch", "origin", "pull/17/head:clang") git("checkout", "clang")
def run_tests(self, experiment): from pprof.project import wrap from pprof.utils.run import run libav_dir = path.join(self.builddir, self.src_dir) with local.cwd(libav_dir): wrap(self.name, experiment) with local.cwd(self.src_dir): run(make["V=1", "-i", "fate"])
def build(self): from plumbum.cmd import make from benchbuild.utils.compiler import lt_clang from benchbuild.utils.run import run blas_dir = path.join(self.builddir, self.src_dir) with local.cwd(self.builddir): clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) with local.cwd(blas_dir): run(make["CC=" + str(clang)])
def build(self): from plumbum.cmd import make from pprof.utils.compiler import lt_clang from pprof.utils.run import run xz_dir = path.join(self.builddir, self.src_dir) with local.cwd(self.builddir): clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) with local.cwd(xz_dir): run(make["CC=" + str(clang), "clean", "all"])
def build(self): from plumbum.cmd import make from benchbuild.utils.run import run lapack_dir = path.join(self.builddir, self.src_dir) with local.cwd(lapack_dir): run(make["-j", CFG["jobs"], "f2clib", "blaslib"]) with local.cwd(path.join("BLAS", "TESTING")): run(make["-j", CFG["jobs"], "-f", "Makeblat2"]) run(make["-j", CFG["jobs"], "-f", "Makeblat3"])
def configure(self): from benchbuild.utils.compiler import lt_clang from benchbuild.utils.run import run openssl_dir = path.join(self.builddir, self.src_dir) configure = local[path.join(openssl_dir, "configure")] with local.cwd(self.builddir): clang = lt_clang(self.cflags, self.ldflags) with local.cwd(openssl_dir): with local.env(CC=str(clang)): run(configure["--disable-asm"])
def build(self): from plumbum.cmd import make from benchbuild.utils.compiler import lt_clang_cxx crocopat_dir = path.join(self.builddir, self.src_dir, "src") with local.cwd(crocopat_dir): cflags = self.cflags + ["-I.", "-ansi"] ldflags = self.ldflags + ["-L.", "-lrelbdd"] with local.cwd(self.builddir): clang_cxx = lt_clang_cxx(cflags, ldflags, self.compiler_extension) make("CXX=" + str(clang_cxx))
def run_tests(self, experiment): from plumbum.cmd import find, make from benchbuild.project import wrap from benchbuild.utils.run import run with local.cwd(path.join(self.src_dir, "tests", ".libs")): files = find(".", "-type", "f", "-executable") for wrap_f in files.split("\n"): if wrap_f: wrap(wrap_f, experiment) with local.cwd(self.src_dir): run(make["V=1", "check"])
def download(self): from pprof.settings import config with local.cwd(self.builddir): Wget(self.src_uri, self.src_file) cp(config["sourcedir"] + "/bin/uchroot", "uchroot") run(fakeroot["tar", "xfj", self.src_file]) rm(self.src_file) with local.cwd(self.builddir + "/usr"): Wget(self.src_uri_portage, self.src_file_portage) run(tar["xfj", self.src_file_portage]) rm(self.src_file_portage)
def compile(self): self.download() self.cflags = ['-DUSE_MPI=0', '-fopenmp'] cxx_files = local.cwd / self.src_file // "*.cc" clang = compiler.cxx(self) with local.cwd(self.src_file): for src_file in cxx_files: clang("-c", "-o", src_file + '.o', src_file) obj_files = local.cwd / self.src_file // "*.cc.o" with local.cwd(self.src_file): clang(obj_files, "-lm", "-o", "../lulesh")
def configure(self): from pprof.utils.compiler import lt_clang from pprof.utils.run import run libav_dir = path.join(self.builddir, self.src_dir) with local.cwd(self.builddir): clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) with local.cwd(libav_dir): configure = local["./configure"] run(configure["--disable-shared", "--cc=" + str( clang), "--extra-ldflags=" + " ".join(self.ldflags), "--samples=" + self.fate_dir])
def configure(self): from benchbuild.utils.compiler import lt_clang from benchbuild.utils.run import run from plumbum.cmd import mkdir tcc_dir = path.join(self.builddir, self.src_dir) with local.cwd(self.builddir): mkdir("build") clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) with local.cwd(path.join(self.builddir, "build")): configure = local[path.join(tcc_dir, "configure")] run(configure["--cc=" + str(clang), "--libdir=/usr/lib64"])
def error(self, e=None): """Write the error state""" with local.cwd(self.dst_dir): if self.wrote_state: # Do not double write return self.wrote_state = True with open(local.path(PipelineState.error), "w") as f: f.write(str(e))
def success(self): """Write the success state without over-writing""" with local.cwd(self.dst_dir): if self.wrote_state: # Do not double write return self.wrote_state = True local.path(PipelineState.success).touch() utils.json_save(local.path("config.json"), self.config)
def compile(self): self.download() book_file = "book.bin" book_bin = "http://www.craftychess.com/downloads/book/" + book_file download.Wget(book_bin, book_file) unpack_dir = "crafty.src" mkdir(unpack_dir) with local.cwd(unpack_dir): unzip(local.path("..") / self.src_file) mv(book_file, unpack_dir) clang = compiler.cc(self) with local.cwd(unpack_dir): target_opts = ["-DCPUS=1", "-DSYZYGY", "-DTEST"] crafty_make = make["target=UNIX", "CC=" + str(clang), "opt=" + " ".join(target_opts), "crafty-make"] run.run(crafty_make)
def test_no_vscode_in_private(cloned_template: Path, tmp_path: Path): """Make sure .vscode folders are git-ignored in private folder.""" copy(str(cloned_template), str(tmp_path), vcs_ref="HEAD", force=True) with local.cwd(tmp_path): git("add", ".") git("commit", "--no-verify", "-am", "hello world") vscode = tmp_path / "odoo" / "custom" / "src" / "private" / ".vscode" vscode.mkdir() (vscode / "something").touch() assert not git("status", "--porcelain")
def test_copy_default_advertised(tmp_path_factory, monkeypatch, capsys, name): """Test that the questions for the user are OK""" monkeypatch.setattr("sys.stdin", StringIO("\n" * 3)) template, subproject = ( tmp_path_factory.mktemp("template"), tmp_path_factory.mktemp("subproject"), ) with local.cwd(template): build_file_tree(MARIO_TREE) git("init") git("add", ".") git("commit", "-m", "v1") git("tag", "v1") git("commit", "--allow-empty", "-m", "v2") git("tag", "v2") with local.cwd(subproject): # Copy the v1 template kwargs = {} if name is not DEFAULT: kwargs["data"] = {"your_name": name} else: name = "Mario" # Default in the template copy(str(template), ".", vcs_ref="v1", **kwargs) # Check what was captured captured = capsys.readouterr() assert "in_love? Format: bool\n🎤? [Y/n] " in captured.out assert ("Secret enemy name\nyour_enemy? Format: str\n🕵️ [Bowser]: " in captured.out) assert (f"what_enemy_does? Format: str\n🎤 [Bowser hates {name}]: " in captured.out) # We already sent the name through 'data', so it shouldn't be prompted assert name == "Mario" or "your_name" not in captured.out assert "_commit: v1" in Path(".copier-answers.yml").read_text() # Update subproject git("init") git("add", ".") git("commit", "-m", "v1") copy(dst_path=".") # Check what was captured captured = capsys.readouterr() assert ( f"If you have a name, tell me now.\nyour_name? Format: str\n🎤 [{name}]: " in captured.out)
def run_tests(self, experiment, run): wrap(path.join(self.builddir, "usr/bin/lmp_serial"), experiment, self.builddir) lammps = uchroot()["/usr/bin/lmp_serial"] lammps_dir = path.join(self.builddir, "lammps") with local.cwd("lammps"): tests = glob(path.join(lammps_dir, "in.*")) for test in tests: run((lammps < strip_path_prefix(test, self.builddir)))
def setUpClass(cls): with local.cwd(local.cwd / ".."): print("Building image") local["./hooks/build"] & FG cls.image = "tecnativa/postgres-autoconf:{}".format( local.env["DOCKER_TAG"]) cls.cert_files = { "client.ca.cert.pem", "server.cert.pem", "server.key.pem" } return super().setUpClass()
def update(self): """ Updates the git repo of the program. """ print("Updating " + self.name) from plumbum.cmd import git if path.exists(self.install_loc): with local.cwd(self.install_loc): git["pull"]()
def test_default_settings(tmp_path: Path, any_odoo_version: float, cloned_template: Path): """Test that a template can be rendered from zero for each version.""" with local.cwd(cloned_template): copy( ".", str(tmp_path), vcs_ref="test", force=True, data={"odoo_version": any_odoo_version}, ) with local.cwd(tmp_path): # TODO When copier runs pre-commit before extracting diff, make sure # here that it works as expected Path(tmp_path, "odoo", "auto", "addons").rmdir() Path(tmp_path, "odoo", "auto").rmdir() git("add", ".") git("commit", "-am", "Hello World", retcode=1) # pre-commit fails git("commit", "-am", "Hello World")
def get_nonword_count(target): """ Run git diff and check lines added """ regex = re.compile("[+][^+]") path = get_unanimous(target) git = local["git"] with local.cwd(str(path.parent)): output = git("diff", path.name) return len([line for line in output.splitlines() if regex.match(line)])
def test_start( cloned_template: Path, docker: LocalCommand, supported_odoo_version: float, tmp_path: Path, ): """Test the start task. On this test flow, other downsream tasks are also tested: - img-build - git-aggregate - stop --purge """ try: with local.cwd(tmp_path): copy( src_path=str(cloned_template), vcs_ref="HEAD", force=True, data={"odoo_version": supported_odoo_version}, ) # Imagine the user is in the src subfolder for these tasks with local.cwd(tmp_path / "odoo" / "custom" / "src"): invoke("img-build") invoke("git-aggregate") # Test normal call stdout = invoke("start") print(stdout) assert "Reinitialized existing Git repository" in stdout assert "pre-commit installed" in stdout # Test "--debugpy and wait time call invoke("stop") stdout = invoke("start", "--debugpy") assert socket_is_open("127.0.0.1", int(supported_odoo_version) * 1000 + 899) # Check if auto-reload is disabled container_logs = docker_compose("logs", "odoo") assert "dev=reload" not in container_logs finally: # Imagine the user is in the odoo subrepo for this command with local.cwd(tmp_path / "odoo" / "custom" / "src" / "odoo"): invoke("stop", "--purge")
def test_file_conflicts_with_committed_file(temp_git_dir): with local.cwd(temp_git_dir): write_file('f.py', "print('hello world')") local['git']('add', 'f.py') local['git']('commit', '--no-verify', '-m', 'Add f.py') write_file('F.py', "print('hello world')") local['git']('add', 'F.py') assert find_conflicting_filenames(['F.py']) == 1
def configure(self): clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension) clang_cxx = lt_clang_cxx(self.cflags, self.ldflags, self.compiler_extension) with local.cwd(self.SRC_FILE): configure = local["./configure"] with local.env(CC=str(clang), CXX=str(clang_cxx)): run(configure["--without-ccache", "--disable-pic14-port", "--disable-pic16-port"])
def examine_repo(repodir): """ Inspect an available repository """ print("Opening editor") editor = local[get_editor()] # plumbum bug workaround os.chdir(pathlib.Path.home()) with local.cwd(repodir): _ = editor["spelling.txt"] & FG
def configure(self): clang = cc(self) clang_cxx = cxx(self) ccrypt_dir = path.join('.', self.src_dir) with local.cwd(ccrypt_dir): configure = local["./configure"] with local.env(CC=str(clang), CXX=str(clang_cxx)): run(configure)
def main(self, *src): if not src: assert self.source, "You must use the --source flag to run over GooFit's source" git = local['git'] with local.cwd(DIR / '../'): src = [ local.path(n) for n in git('ls-files', '--', '*.cpp', '*.h', '*.cu').splitlines() ] fix_files(src, self.set_version)
def stow_dotfiles(): with local.cwd(THIS_DIR): for path in THIS_DIR.iterdir(): if not path.is_dir(): continue if path.name.startswith("."): continue cmd.stow(path.name, "-R", "--no-folding")
def test_update_choice(tmp_path_factory, spawn, choices): """Choices are properly remembered and selected in TUI when updating.""" template, subproject = ( tmp_path_factory.mktemp("template"), tmp_path_factory.mktemp("subproject"), ) # Create template build_file_tree({ template / "copier.yml": f""" _templates_suffix: {SUFFIX_TMPL} _envops: {BRACKET_ENVOPS_JSON} pick_one: type: float default: 3 choices: {choices} """, template / "[[ _copier_conf.answers_file ]].tmpl": "[[ _copier_answers|to_nice_yaml ]]", }) with local.cwd(template): git("init") git("add", ".") git("commit", "-m one") git("tag", "v1") # Copy tui = spawn(COPIER_PATH + (str(template), str(subproject)), timeout=10) tui.expect_exact("pick_one?") tui.sendline(Keyboard.Up) tui.expect_exact(pexpect.EOF) answers = yaml.safe_load((subproject / ".copier-answers.yml").read_text()) assert answers["pick_one"] == 2.0 with local.cwd(subproject): git("init") git("add", ".") git("commit", "-m1") # Update tui = spawn(COPIER_PATH + (str(subproject), ), timeout=10) tui.expect_exact("pick_one?") tui.sendline(Keyboard.Down) tui.expect_exact(pexpect.EOF) answers = yaml.safe_load((subproject / ".copier-answers.yml").read_text()) assert answers["pick_one"] == 3.0
def test_symlink(builddir, runner): builder = BuildFile(build_dir=builddir, runner=runner) with local.cwd(builddir): sh.touch('testfile') sh.mkdir('testdir') ###### First build ########## builder.main(command_line=['-D', 'build']) expected_json = { ".deps_version": 2, "ln -s nofile testlink_nofile": { "testlink_nofile": "output-" }, "ln -s testdir testlink_dir": { "testlink_dir": "output-" }, "ln -s testfile testlink": { "testlink": "output-" } } # assertions with local.cwd(builddir): assert_same_json('.deps', expected_json) assert os.path.islink('testlink') assert os.path.realpath('testlink').endswith('/testfile') assert os.path.islink('testlink_dir') assert os.path.realpath('testlink_dir').endswith('/testdir') assert os.path.islink('testlink_nofile') assert not os.path.isfile('testlink_nofile') sys.exit.assert_called_once_with(0) ###### Cleaning ########## builder.main(command_line=['-D', 'clean']) with local.cwd(builddir): assert not os.path.isfile('.deps') assert not os.path.islink('testlink') assert not os.path.islink('testlink_dir') assert not os.path.islink('testlink_nofile')
def make(commit=DEFAULT_HASH): softdir = getSoftDir() blddir = softdir / "ANTs-build" with local.cwd(softdir): repo = downloadGithubRepo('ANTsX/ANTs', commit) sha, date = getCommitInfo(repo) out = get_path(sha) # if output binary directory already exists, then return if checkExists(out): return logging.info("Build code:") blddir.mkdir() with local.cwd(blddir): cmake(repo) import plumbum.cmd plumbum.cmd.make['-j', psutil.cpu_count(logical=False)] & FG # copy ANTs scripts cp('-a', (softdir / 'ANTs' / 'Scripts').list(), blddir / 'ANTS-build' / 'Examples') # move binary directory (blddir / 'ANTS-build' / 'Examples').move(out) # write PATH and ANTSPATH with open(out / 'env.sh', 'w') as f: f.write("export PATH={}:$PATH\n".format(out)) f.write("export ANTSPATH={}\n".format(out)) # generate symbolink links symlink = get_path(date) print("Make symlink: {} -> {}".format(symlink, get_path(sha))) symlink.unlink() get_path(sha).symlink(symlink) logging.info("Made '{}'".format(out))
def compile(self) -> None: """Compile the project.""" libjpeg_version_source = local.path(self.source_of_primary) c_compiler = bb.compiler.cc(self) with local.cwd(libjpeg_version_source): with local.env(CC=str(c_compiler)): bb.watch(cmake)("-G", "Unix Makefiles", ".") bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) verify_binaries(self)
def clean_directories(builddir, in_dir=True, out_dir=True): """Remove the in and out of the container if confirmed by the user.""" with local.cwd(builddir): if in_dir and os.path.exists("container-in") and ask( "Should I delete '{0}'?".format( os.path.abspath("container-in"))): rm("-rf", "container-in") if out_dir and os.path.exists("container-out") and ask( "Should I delete '{0}'?".format( os.path.abspath("container-out"))): rm("-rf", "container-out")
def configure(self): clang = cc(self) with local.cwd(self.src_dir): configure = local["./configure"] with local.env(CC=str(clang)): run(configure["--enable-threads=no", "--with-gnu-ld=yes", "--disable-shared", "--disable-dependency-tracking", "--disable-xzdec", "--disable-lzmadec", "--disable-lzmainfo", "--disable-lzma-links", "--disable-scripts", "--disable-doc"])
def test_update_subdirectory(demo_template, tmp_path): copier.copy(demo_template, tmp_path, force=True) with local.cwd(tmp_path): git_init() copier.copy(dst_path=tmp_path, force=True) assert not (tmp_path / "conf_project").exists() assert not (tmp_path / "api_project").exists() assert not (tmp_path / "api_readme.md").exists() assert (tmp_path / "conf_readme.md").exists()
def push_commit(repodir, add_word): """ Create commit and push """ git = local["git"] with local.cwd(repodir): to_branch = git("symbolic-ref", "--short", "HEAD").strip() from_branch = f"bugfix_typo_{add_word.replace(' ', '_')}" git("commit", "-F", "__commit__.txt") git("push", "origin", f"{to_branch}:{from_branch}") return from_branch, to_branch
def wrap_in_builddir_func(self, *args, **kwargs): """The actual function inside the wrapper for the new builddir.""" p = local.path(self.builddir) / sub if not p.exists(): LOG.error("%s does not exist.", p) if p == local.cwd: LOG.debug("CWD already is %s", p) return func(self, *args, *kwargs) with local.cwd(p): return func(self, *args, **kwargs)
def compile(self) -> None: """Compile the project.""" openssl_source = local.path(self.source_of_primary) compiler = bb.compiler.cc(self) with local.cwd(openssl_source): with local.env(CC=str(compiler)): bb.watch(local['./config'])() bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) verify_binaries(self)
def test_no_vscode_in_private(tmp_path: Path): """Asegurese que las carpetas .vscode se git-ignoren en la carpeta privada. """ copy(".", str(tmp_path), vcs_ref="HEAD", force=True) with local.cwd(tmp_path): git("add", ".") git("commit", "--no-verify", "-am", "hello world") vscode = tmp_path / "odoo" / "custom" / "src" / "private" / ".vscode" vscode.mkdir() (vscode / "something").touch() assert not git("status", "--porcelain")
def test_multiple_domains_prod( cloned_template: Path, supported_odoo_version: float, tmp_path: Path, traefik_host: str, ): """Test multiple domains are produced properly.""" data = { "traefik_cert_resolver": None, "odoo_version": supported_odoo_version, "project_name": uuid.uuid4().hex, "domains_prod": { f"main.{traefik_host}.sslip.io": [ f"alt0.main.{traefik_host}.sslip.io", f"alt1.main.{traefik_host}.sslip.io", ], f"secondary.{traefik_host}.sslip.io": [ f"alt0.secondary.{traefik_host}.sslip.io", f"alt1.secondary.{traefik_host}.sslip.io", ], f"third.{traefik_host}.sslip.io": [], }, } dc = docker_compose["-f", "prod.yaml"] with local.cwd(tmp_path): copy( src_path=str(cloned_template), dst_path=".", vcs_ref="test", force=True, data=data, ) try: dc("build") dc( "run", "--rm", "odoo", "--stop-after-init", "-i", "base", ) dc("up", "-d") time.sleep(10) for main_domain, alt_list in data["domains_prod"].items(): for alt_domain in alt_list + [main_domain]: response = requests.get(f"http://{alt_domain}/web/login") assert response.ok assert response.url == f"http://{main_domain}/web/login" bad_response = requests.get( f"http://missing.{traefik_host}.sslip.io") assert bad_response.status_code == 404 finally: dc("down", "--volumes", "--remove-orphans")
def compile(self) -> None: grep_source = local.path(self.source_of_primary) compiler = bb.compiler.cc(self) with local.cwd(grep_source): with local.env(CC=str(compiler)): bb.watch(local["./bootstrap"])() bb.watch(local["./configure"])("--disable-gcc-warnings") bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) verify_binaries(self)