def configure(self):
from benchbuild.utils.run import run
# First we have to prepare boost for lady povray...
boost_dir = path.join(self.builddir, self.boost_src_dir)
boost_prefix = path.join(self.builddir, "boost-install")
with local.cwd(boost_dir):
from plumbum.cmd import mkdir
mkdir(boost_prefix)
bootstrap = local["./bootstrap.sh"]
run(bootstrap["--with-toolset=clang", "--prefix=\"{0}\"".format(
boost_prefix)])
b2 = local["./b2"]
run(b2["--ignore-site-config", "variant=release", "link=static",
"threading=multi", "optimization=speed", "install"])
povray_dir = path.join(self.builddir, self.src_dir)
with local.cwd(path.join(povray_dir, "unix")):
from plumbum.cmd import sh
sh("prebuild.sh")
with local.cwd(povray_dir):
from benchbuild.utils.compiler import lt_clang, lt_clang_cxx
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
clang_cxx = lt_clang_cxx(self.cflags, self.ldflags,
self.compiler_extension)
configure = local["./configure"]
with local.env(COMPILED_BY="BB <*****@*****.**>",
CC=str(clang),
CXX=str(clang_cxx)):
run(configure["--with-boost=" + boost_prefix])
def compile(self):
self.download()
js_dir = local.path(self.src_file) / "js" / "src"
clang = compiler.cc(self)
clang_cxx = compiler.cxx(self)
with local.cwd(js_dir):
make_src_pkg = local["./make-source-package.sh"]
with local.env(
DIST=self.builddir,
MOZJS_MAJOR_VERSION=0,
MOZJS_MINOR_VERSION=0,
MOZJS_PATCH_VERSION=0):
make_src_pkg()
mozjs_dir = local.path("mozjs-0.0.0")
mozjs_src_dir = mozjs_dir / "js" / "src"
tar("xfj", mozjs_dir + ".tar.bz2")
with local.cwd(mozjs_src_dir):
mkdir("obj")
autoconf = local["autoconf-2.13"]
autoconf()
with local.cwd("obj"):
with local.env(CC=str(clang), CXX=str(clang_cxx)):
configure = local["../configure"]
configure = configure["--without-system-zlib"]
run.run(configure)
mozjs_obj_dir = mozjs_src_dir / "obj"
with local.cwd(mozjs_obj_dir):
run.run(make["-j", str(CFG["jobs"])])
def compile(self):
self.download()
download.Git(self.gdal_uri, self.gdal_dir)
rasdaman_dir = local.path(self.src_file)
gdal_dir = local.path(self.gdal_dir) / self.gdal_dir
clang = compiler.cc(self)
clang_cxx = compiler.cxx(self)
with local.cwd(gdal_dir):
configure = local["./configure"]
with local.env(CC=str(clang), CXX=str(clang_cxx)):
run.run(configure["--with-pic", "--enable-static",
"--disable-debug", "--with-gnu-ld",
"--without-ld-shared", "--without-libtool"])
run.run(make["-j", CFG["jobs"]])
with local.cwd(rasdaman_dir):
autoreconf("-i")
configure = local["./configure"]
with local.env(CC=str(clang), CXX=str(clang_cxx)):
run.run(configure["--without-debug-symbols",
"--enable-benchmark", "--with-static-libs",
"--disable-java", "--with-pic",
"--disable-debug", "--without-docs"])
run.run(make["clean", "all", "-j", CFG["jobs"]])
def run_makefile(self, makefile=None):
"""
Run the whole build process with designated Makefile.
"""
# cd into git repository directory
with local.cwd(self.repo.working_dir):
# cd into working directory inside git repository
with local.cwd(self.workingdir):
# Run Makefile to start the compilation process
make('--file', makefile, 'all-plus-firmware-info', stdout=self.stream, stderr=self.stream)
# Slurp output of build process
try:
self.stream.seek(0)
output = self.stream.read()
except IOError:
make_firmware_info = make['--file', makefile, 'firmware-info'] | grep['TARGET_']
output = make_firmware_info()
# Grep "TARGET_HEX", "TARGET_ELF" (for AVR) as well as "TARGET_BIN" and "TARGET_CHIP" (for ESP) paths
# from build output and store into "self.build_result"
target_matcher = re.compile('(?P<name>TARGET_.+):(?: (?P<value>.+))?')
for m in target_matcher.finditer(output):
match = m.groupdict()
name = match['name']
value = match['value']
if value:
self.build_result[name] = value
# Add build path to build result
self.build_result['build_path'] = pwd().strip()
def setup_container(builddir, container):
with local.cwd(builddir):
container_filename = str(container).split(os.path.sep)[-1]
container_in = os.path.join("container-in", container_filename)
Copy(container, container_in)
uchroot = uchroot_no_args()
with local.cwd("container-in"):
uchroot = uchroot["-E", "-A", "-u", "0", "-g", "0", "-C", "-r",
"/", "-w", os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash[
"-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
container_in)]
has_erlent = (has_erlent & TF)
# Unpack input container to: container-in
if not has_erlent:
cmd = local["/bin/tar"]["xf"]
cmd = uchroot[cmd[container_filename]]
else:
cmd = tar["xf"]
cmd = cmd[os.path.abspath(container_in)]
with local.cwd("container-in"):
cmd("--exclude=dev/*")
rm(container_in)
return os.path.join(builddir, "container-in")
def configure_openmp(self, openmp_path):
""" Configure LLVM/Clang's own OpenMP runtime. """
from plumbum.cmd import cmake
with local.cwd(openmp_path):
builddir = os.path.join(openmp_path, "build")
if not os.path.exists(builddir):
mkdir(builddir)
with local.cwd(builddir):
cmake_cache = os.path.join(builddir, "CMakeCache.txt")
install_path = os.path.join(self._builddir, "install")
openmp_cmake = cmake[
"-DCMAKE_INSTALL_PREFIX=" + install_path,
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_USE_RELATIVE_PATHS=On",
"-DLIBOMP_ENABLE_ASSERTIONS=Off"]
if self._use_make:
openmp_cmake = openmp_cmake["-G", "Unix Makefiles"]
else:
openmp_cmake = openmp_cmake["-G", "Ninja"]
if not os.path.exists(cmake_cache):
openmp_cmake = configure_compiler(openmp_cmake,
use_gcc=False)
openmp_cmake = openmp_cmake[openmp_path]
else:
openmp_cmake = openmp_cmake["."]
openmp_cmake()
def configure_llvm(self, llvm_path):
""" Configure LLVM and all subprojects. """
with local.cwd(llvm_path):
builddir = os.path.join(llvm_path, "build")
if not os.path.exists(builddir):
mkdir(builddir)
with local.cwd(builddir):
cmake_cache = os.path.join(builddir, "CMakeCache.txt")
install_path = os.path.join(self._builddir, "install")
llvm_cmake = cmake[
"-DCMAKE_INSTALL_PREFIX=" + install_path,
"-DCMAKE_BUILD_TYPE=Release", "-DBUILD_SHARED_LIBS=Off",
"-DCMAKE_USE_RELATIVE_PATHS=On", "-DPOLLY_BUILD_POLLI=On",
"-DLLVM_TARGETS_TO_BUILD=X86",
"-DLLVM_BINUTILS_INCDIR=/usr/include/",
"-DLLVM_ENABLE_PIC=On", "-DLLVM_ENABLE_ASSERTIONS=On",
"-DCLANG_DEFAULT_OPENMP_RUNTIME=libomp",
"-DCMAKE_CXX_FLAGS_RELEASE='-O3 -DNDEBUG -fno-omit-frame-pointer'"]
if self._use_make:
llvm_cmake = llvm_cmake["-G", "Unix Makefiles"]
else:
llvm_cmake = llvm_cmake["-G", "Ninja"]
llvm_cmake = configure_papi(llvm_cmake, self._papidir)
llvm_cmake = configure_likwid(llvm_cmake, self._likwiddir)
llvm_cmake = configure_isl(llvm_cmake, self._isldir)
if not os.path.exists(cmake_cache):
llvm_cmake = configure_compiler(llvm_cmake, self._use_gcc)
llvm_cmake = llvm_cmake[llvm_path]
else:
llvm_cmake = llvm_cmake["."]
llvm_cmake()
def install_uchroot(_):
"""Installer for erlent (contains uchroot)."""
builddir = local.path(str(CFG["build_dir"].value))
with local.cwd(builddir):
erlent_src = local.path('erlent')
erlent_git = erlent_src / '.git'
erlent_repo = str(CFG['uchroot']['repo'])
erlent_build = erlent_src / 'build'
if not erlent_git.exists():
git("clone", erlent_repo)
else:
with local.cwd(erlent_src):
git("pull", "--rebase")
erlent_build.mkdir()
with local.cwd(erlent_build):
cmake("../")
make()
os.environ["PATH"] = os.path.pathsep.join(
[erlent_build, os.environ["PATH"]])
local.env.update(PATH=os.environ["PATH"])
if not find_package("uchroot"):
LOG.error('uchroot not found, after updating PATH to %s',
os.environ['PATH'])
sys.exit(-1)
env = CFG['env'].value
if 'PATH' not in env:
env['PATH'] = []
env['PATH'].append(str(erlent_build))
def configure(self):
lapack_dir = path.join(self.builddir, self.src_dir)
from benchbuild.utils.compiler import lt_clang, lt_clang_cxx
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
clang_cxx = lt_clang_cxx(self.cflags, self.ldflags,
self.compiler_extension)
with local.cwd(lapack_dir):
with open("make.inc", 'w') as makefile:
content = [
"SHELL = /bin/sh\n", "PLAT = _LINUX\n",
"CC = " + str(clang) + "\n",
"CXX = " + str(clang_cxx) + "\n",
"CFLAGS = -I$(TOPDIR)/INCLUDE\n",
"LOADER = " + str(clang) + "\n", "LOADOPTS = \n",
"NOOPT = -O0 -I$(TOPDIR)/INCLUDE\n",
"DRVCFLAGS = $(CFLAGS)\n", "F2CCFLAGS = $(CFLAGS)\n",
"TIMER = INT_CPU_TIME\n", "ARCH = ar\n",
"ARCHFLAGS = cr\n", "RANLIB = ranlib\n",
"BLASLIB = ../../blas$(PLAT).a\n", "XBLASLIB = \n",
"LAPACKLIB = lapack$(PLAT).a\n",
"F2CLIB = ../../F2CLIBS/libf2c.a\n",
"TMGLIB = tmglib$(PLAT).a\n",
"EIGSRCLIB = eigsrc$(PLAT).a\n",
"LINSRCLIB = linsrc$(PLAT).a\n",
"F2CLIB = ../../F2CLIBS/libf2c.a\n"
]
makefile.writelines(content)
def setup_bash_in_container(builddir, container, outfile, mounts, shell):
with local.cwd(builddir):
# Switch to bash inside uchroot
print("Entering bash inside User-Chroot. Prepare your image and "
"type 'exit' when you are done. If bash exits with a non-zero"
"exit code, no new container will be stored.")
store_new_container = True
try:
run_in_container(shell, container, mounts)
except ProcessExecutionError:
store_new_container = False
if store_new_container: # pylint: disable=W0104
print("Packing new container image.")
container_filename = os.path.split(container)[-1]
container_out = os.path.join("container-out", container_filename)
container_out = os.path.abspath(container_out)
# Pack the results to: container-out
with local.cwd("container-in"):
tar("cjf", container_out, ".")
update_hash(container_filename, os.path.dirname(container_out))
outdir = os.path.dirname(outfile)
if not os.path.exists(outdir):
mkdir("-p", outdir)
mv(container_out, outfile)
def run_tests(self, experiment):
from benchbuild.project import wrap
from benchbuild.utils.run import run
lapack_dir = path.join(self.builddir, self.src_dir)
with local.cwd(lapack_dir):
with local.cwd(path.join("BLAS")):
xblat2s = wrap("xblat2s", experiment)
xblat2d = wrap("xblat2d", experiment)
xblat2c = wrap("xblat2c", experiment)
xblat2z = wrap("xblat2z", experiment)
xblat3s = wrap("xblat3s", experiment)
xblat3d = wrap("xblat3d", experiment)
xblat3c = wrap("xblat3c", experiment)
xblat3z = wrap("xblat3z", experiment)
run((xblat2s < "sblat2.in"))
run((xblat2d < "dblat2.in"))
run((xblat2c < "cblat2.in"))
run((xblat2z < "zblat2.in"))
run((xblat3s < "sblat3.in"))
run((xblat3d < "dblat3.in"))
run((xblat3c < "cblat3.in"))
run((xblat3z < "zblat3.in"))
def compile(self):
self.download()
tar("xfz", self.src_file)
unpack_dir = "CLAPACK-{0}".format(self.version)
clang = compiler.cc(self)
clang_cxx = compiler.cxx(self)
with local.cwd(unpack_dir):
with open("make.inc", 'w') as makefile:
content = [
"SHELL = /bin/sh\n", "PLAT = _LINUX\n",
"CC = " + str(clang) + "\n",
"CXX = " + str(clang_cxx) + "\n",
"CFLAGS = -I$(TOPDIR)/INCLUDE\n",
"LOADER = " + str(clang) + "\n", "LOADOPTS = \n",
"NOOPT = -O0 -I$(TOPDIR)/INCLUDE\n",
"DRVCFLAGS = $(CFLAGS)\n", "F2CCFLAGS = $(CFLAGS)\n",
"TIMER = INT_CPU_TIME\n", "ARCH = ar\n",
"ARCHFLAGS = cr\n", "RANLIB = ranlib\n",
"BLASLIB = ../../blas$(PLAT).a\n", "XBLASLIB = \n",
"LAPACKLIB = lapack$(PLAT).a\n",
"F2CLIB = ../../F2CLIBS/libf2c.a\n",
"TMGLIB = tmglib$(PLAT).a\n",
"EIGSRCLIB = eigsrc$(PLAT).a\n",
"LINSRCLIB = linsrc$(PLAT).a\n",
"F2CLIB = ../../F2CLIBS/libf2c.a\n"
]
makefile.writelines(content)
run.run(make["-j", CFG["jobs"], "f2clib", "blaslib"])
with local.cwd(local.path("BLAS") / "TESTING"):
run.run(make["-j", CFG["jobs"], "-f", "Makeblat2"])
run.run(make["-j", CFG["jobs"], "-f", "Makeblat3"])
def setup_container(builddir, _container):
"""Prepare the container and returns the path where it can be found."""
build_dir = local.path(builddir)
in_dir = build_dir / "container-in"
container_path = local.path(_container)
with local.cwd(builddir):
container_bin = container_path.basename
container_in = in_dir / container_bin
download.Copy(_container, container_in)
uchrt = uchroot.no_args()
with local.cwd("container-in"):
uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/",
"-w",
os.path.abspath("."), "--"]
# Check, if we need erlent support for this archive.
has_erlent = bash[
"-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
container_in)]
has_erlent = (has_erlent & TF)
# Unpack input container to: container-in
if not has_erlent:
cmd = local["/bin/tar"]["xf"]
cmd = uchrt[cmd[container_bin]]
else:
cmd = tar["xf"]
cmd = cmd[container_in]
with local.cwd("container-in"):
cmd("--exclude=dev/*")
rm(container_in)
return in_dir
def configure(self):
from benchbuild.utils.compiler import lt_clang, lt_clang_cxx
from benchbuild.utils.run import run
from plumbum.cmd import autoreconf, make
rasdaman_dir = path.join(self.builddir, self.src_dir)
gdal_dir = path.join(self.builddir, self.gdal_dir, self.gdal_dir)
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
clang_cxx = lt_clang_cxx(self.cflags, self.ldflags,
self.compiler_extension)
with local.cwd(gdal_dir):
configure = local["./configure"]
with local.env(CC=str(clang), CXX=str(clang_cxx)):
run(configure["--with-pic", "--enable-static",
"--disable-debug", "--with-gnu-ld",
"--without-ld-shared", "--without-libtool"])
run(make["-j", CFG["jobs"]])
with local.cwd(rasdaman_dir):
autoreconf("-i")
configure = local["./configure"]
with local.env(CC=str(clang), CXX=str(clang_cxx)):
run(configure["--without-debug-symbols", "--enable-benchmark",
"--with-static-libs", "--disable-java",
"--with-pic", "--disable-debug",
"--without-docs"])
def in_merge_conflict(tmpdir_factory):
path = make_consuming_repo(tmpdir_factory, 'script_hooks_repo')
with local.cwd(path):
local['touch']('dummy')
git('add', 'dummy')
git('add', C.CONFIG_FILE)
git('commit', '-m', 'Add config.')
conflict_path = tmpdir_factory.get()
git('clone', path, conflict_path)
with local.cwd(conflict_path):
git('checkout', 'origin/master', '-b', 'foo')
with io.open('conflict_file', 'w') as conflict_file:
conflict_file.write('herp\nderp\n')
git('add', 'conflict_file')
with io.open('foo_only_file', 'w') as foo_only_file:
foo_only_file.write('foo')
git('add', 'foo_only_file')
git('commit', '-m', 'conflict_file')
git('checkout', 'origin/master', '-b', 'bar')
with io.open('conflict_file', 'w') as conflict_file:
conflict_file.write('harp\nddrp\n')
git('add', 'conflict_file')
with io.open('bar_only_file', 'w') as bar_only_file:
bar_only_file.write('bar')
git('add', 'bar_only_file')
git('commit', '-m', 'conflict_file')
git('merge', 'foo', retcode=None)
yield os.path.join(conflict_path)
def run_tests(self, runner):
unpack_dir = local.path('tcc-{0}.tar.bz2'.format(self.version))
with local.cwd(unpack_dir):
with local.cwd("build"):
wrapping.wrap("tcc", self)
inc_path = path.abspath("..")
runner(make["TCCFLAGS=-B{}".format(inc_path), "test", "-i"])
def clone_or_pull(repo_dict, to_dir):
"""
Clone or pull a repository and switch to the desired branch.
If the directory already exists, we will try to du a pull with
--rebase. In case anything goes wrong, we exit and print a simple
diagnostic message.
Args:
url (str): Where is the remote repository?
to_dir (str): Where should we clone/update to?
branch (str): Which branch should we check out? Defaults to the repo's
master.
"""
url = repo_dict["url"]
branch = repo_dict.get("branch")
commit_hash = repo_dict.get("commit_hash")
from plumbum.cmd import git
if not os.path.exists(os.path.join(to_dir, ".git/")):
git_clone = git["clone", url, to_dir, "--recursive", "--depth=1"]
if branch:
git_clone = git_clone["--branch", branch]
git_clone()
elif not commit_hash:
# If we haven't specified a commit hash,
# fetch the latest version.
with local.cwd(to_dir):
git("remote", "update")
locl = git("rev-parse", "@{0}")
remote = git("rev-parse", "@{u}")
base = git("merge-base", "@", "@{u}")
if locl == remote:
print("{:s} is up-to-date.".format(url))
elif locl == base:
git("pull", "--rebase")
git("submodule", "update")
elif remote == base:
print("push required")
exit(1)
else:
print("{:s} has diverged from its remote.".format(to_dir))
exit(1)
if commit_hash:
with local.cwd(to_dir):
# We only need to do something if we aren't already at the
# latest commit hash
current_hash = git("rev-parse", "--verify", "HEAD").rstrip("\n")
if current_hash != commit_hash:
# Make sure we have a full history, not just depth 1
print((
"HEAD for repository {:s} is not at configured commit hash {:s}, fetching and checking out.".format(
url, commit_hash)))
git("fetch", "--unshallow")
git_checkout = git("checkout", commit_hash)
def download(self):
from pprof.utils.downloader import Wget, Rsync
from plumbum.cmd import tar
with local.cwd(self.builddir):
Wget(self.src_uri, self.src_file)
tar('xfj', path.join(self.builddir, self.src_file))
with local.cwd(self.src_dir):
Rsync(self.fate_uri, self.fate_dir)
def download(self):
from pprof.utils.downloader import Git
from plumbum.cmd import git
minisat_dir = path.join(self.builddir, self.src_dir)
with local.cwd(self.builddir):
Git(self.src_uri, self.src_dir)
with local.cwd(minisat_dir):
git("fetch", "origin", "pull/17/head:clang")
git("checkout", "clang")
def run_tests(self, experiment):
from pprof.project import wrap
from pprof.utils.run import run
libav_dir = path.join(self.builddir, self.src_dir)
with local.cwd(libav_dir):
wrap(self.name, experiment)
with local.cwd(self.src_dir):
run(make["V=1", "-i", "fate"])
def build(self):
from plumbum.cmd import make
from benchbuild.utils.compiler import lt_clang
from benchbuild.utils.run import run
blas_dir = path.join(self.builddir, self.src_dir)
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
with local.cwd(blas_dir):
run(make["CC=" + str(clang)])
def build(self):
from plumbum.cmd import make
from pprof.utils.compiler import lt_clang
from pprof.utils.run import run
xz_dir = path.join(self.builddir, self.src_dir)
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
with local.cwd(xz_dir):
run(make["CC=" + str(clang), "clean", "all"])
def build(self):
from plumbum.cmd import make
from benchbuild.utils.run import run
lapack_dir = path.join(self.builddir, self.src_dir)
with local.cwd(lapack_dir):
run(make["-j", CFG["jobs"], "f2clib", "blaslib"])
with local.cwd(path.join("BLAS", "TESTING")):
run(make["-j", CFG["jobs"], "-f", "Makeblat2"])
run(make["-j", CFG["jobs"], "-f", "Makeblat3"])
def configure(self):
from benchbuild.utils.compiler import lt_clang
from benchbuild.utils.run import run
openssl_dir = path.join(self.builddir, self.src_dir)
configure = local[path.join(openssl_dir, "configure")]
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags)
with local.cwd(openssl_dir):
with local.env(CC=str(clang)):
run(configure["--disable-asm"])
def build(self):
from plumbum.cmd import make
from benchbuild.utils.compiler import lt_clang_cxx
crocopat_dir = path.join(self.builddir, self.src_dir, "src")
with local.cwd(crocopat_dir):
cflags = self.cflags + ["-I.", "-ansi"]
ldflags = self.ldflags + ["-L.", "-lrelbdd"]
with local.cwd(self.builddir):
clang_cxx = lt_clang_cxx(cflags, ldflags,
self.compiler_extension)
make("CXX=" + str(clang_cxx))
def run_tests(self, experiment):
from plumbum.cmd import find, make
from benchbuild.project import wrap
from benchbuild.utils.run import run
with local.cwd(path.join(self.src_dir, "tests", ".libs")):
files = find(".", "-type", "f", "-executable")
for wrap_f in files.split("\n"):
if wrap_f:
wrap(wrap_f, experiment)
with local.cwd(self.src_dir):
run(make["V=1", "check"])
def download(self):
from pprof.settings import config
with local.cwd(self.builddir):
Wget(self.src_uri, self.src_file)
cp(config["sourcedir"] + "/bin/uchroot", "uchroot")
run(fakeroot["tar", "xfj", self.src_file])
rm(self.src_file)
with local.cwd(self.builddir + "/usr"):
Wget(self.src_uri_portage, self.src_file_portage)
run(tar["xfj", self.src_file_portage])
rm(self.src_file_portage)
def compile(self):
self.download()
self.cflags = ['-DUSE_MPI=0', '-fopenmp']
cxx_files = local.cwd / self.src_file // "*.cc"
clang = compiler.cxx(self)
with local.cwd(self.src_file):
for src_file in cxx_files:
clang("-c", "-o", src_file + '.o', src_file)
obj_files = local.cwd / self.src_file // "*.cc.o"
with local.cwd(self.src_file):
clang(obj_files, "-lm", "-o", "../lulesh")
def configure(self):
from pprof.utils.compiler import lt_clang
from pprof.utils.run import run
libav_dir = path.join(self.builddir, self.src_dir)
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
with local.cwd(libav_dir):
configure = local["./configure"]
run(configure["--disable-shared", "--cc=" + str(
clang), "--extra-ldflags=" + " ".join(self.ldflags),
"--samples=" + self.fate_dir])
def configure(self):
from benchbuild.utils.compiler import lt_clang
from benchbuild.utils.run import run
from plumbum.cmd import mkdir
tcc_dir = path.join(self.builddir, self.src_dir)
with local.cwd(self.builddir):
mkdir("build")
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
with local.cwd(path.join(self.builddir, "build")):
configure = local[path.join(tcc_dir, "configure")]
run(configure["--cc=" + str(clang), "--libdir=/usr/lib64"])
def error(self, e=None):
"""Write the error state"""
with local.cwd(self.dst_dir):
if self.wrote_state:
# Do not double write
return
self.wrote_state = True
with open(local.path(PipelineState.error), "w") as f:
f.write(str(e))
def success(self):
"""Write the success state without over-writing"""
with local.cwd(self.dst_dir):
if self.wrote_state:
# Do not double write
return
self.wrote_state = True
local.path(PipelineState.success).touch()
utils.json_save(local.path("config.json"), self.config)
def compile(self):
self.download()
book_file = "book.bin"
book_bin = "http://www.craftychess.com/downloads/book/" + book_file
download.Wget(book_bin, book_file)
unpack_dir = "crafty.src"
mkdir(unpack_dir)
with local.cwd(unpack_dir):
unzip(local.path("..") / self.src_file)
mv(book_file, unpack_dir)
clang = compiler.cc(self)
with local.cwd(unpack_dir):
target_opts = ["-DCPUS=1", "-DSYZYGY", "-DTEST"]
crafty_make = make["target=UNIX", "CC=" + str(clang),
"opt=" + " ".join(target_opts), "crafty-make"]
run.run(crafty_make)
def test_no_vscode_in_private(cloned_template: Path, tmp_path: Path):
"""Make sure .vscode folders are git-ignored in private folder."""
copy(str(cloned_template), str(tmp_path), vcs_ref="HEAD", force=True)
with local.cwd(tmp_path):
git("add", ".")
git("commit", "--no-verify", "-am", "hello world")
vscode = tmp_path / "odoo" / "custom" / "src" / "private" / ".vscode"
vscode.mkdir()
(vscode / "something").touch()
assert not git("status", "--porcelain")
def test_copy_default_advertised(tmp_path_factory, monkeypatch, capsys, name):
"""Test that the questions for the user are OK"""
monkeypatch.setattr("sys.stdin", StringIO("\n" * 3))
template, subproject = (
tmp_path_factory.mktemp("template"),
tmp_path_factory.mktemp("subproject"),
)
with local.cwd(template):
build_file_tree(MARIO_TREE)
git("init")
git("add", ".")
git("commit", "-m", "v1")
git("tag", "v1")
git("commit", "--allow-empty", "-m", "v2")
git("tag", "v2")
with local.cwd(subproject):
# Copy the v1 template
kwargs = {}
if name is not DEFAULT:
kwargs["data"] = {"your_name": name}
else:
name = "Mario" # Default in the template
copy(str(template), ".", vcs_ref="v1", **kwargs)
# Check what was captured
captured = capsys.readouterr()
assert "in_love? Format: bool\n🎤? [Y/n] " in captured.out
assert ("Secret enemy name\nyour_enemy? Format: str\n🕵️ [Bowser]: "
in captured.out)
assert (f"what_enemy_does? Format: str\n🎤 [Bowser hates {name}]: "
in captured.out)
# We already sent the name through 'data', so it shouldn't be prompted
assert name == "Mario" or "your_name" not in captured.out
assert "_commit: v1" in Path(".copier-answers.yml").read_text()
# Update subproject
git("init")
git("add", ".")
git("commit", "-m", "v1")
copy(dst_path=".")
# Check what was captured
captured = capsys.readouterr()
assert (
f"If you have a name, tell me now.\nyour_name? Format: str\n🎤 [{name}]: "
in captured.out)
def run_tests(self, experiment, run):
wrap(path.join(self.builddir, "usr/bin/lmp_serial"), experiment,
self.builddir)
lammps = uchroot()["/usr/bin/lmp_serial"]
lammps_dir = path.join(self.builddir, "lammps")
with local.cwd("lammps"):
tests = glob(path.join(lammps_dir, "in.*"))
for test in tests:
run((lammps < strip_path_prefix(test, self.builddir)))
def setUpClass(cls):
with local.cwd(local.cwd / ".."):
print("Building image")
local["./hooks/build"] & FG
cls.image = "tecnativa/postgres-autoconf:{}".format(
local.env["DOCKER_TAG"])
cls.cert_files = {
"client.ca.cert.pem", "server.cert.pem", "server.key.pem"
}
return super().setUpClass()
def update(self):
"""
Updates the git repo of the program.
"""
print("Updating " + self.name)
from plumbum.cmd import git
if path.exists(self.install_loc):
with local.cwd(self.install_loc):
git["pull"]()
def test_default_settings(tmp_path: Path, any_odoo_version: float,
cloned_template: Path):
"""Test that a template can be rendered from zero for each version."""
with local.cwd(cloned_template):
copy(
".",
str(tmp_path),
vcs_ref="test",
force=True,
data={"odoo_version": any_odoo_version},
)
with local.cwd(tmp_path):
# TODO When copier runs pre-commit before extracting diff, make sure
# here that it works as expected
Path(tmp_path, "odoo", "auto", "addons").rmdir()
Path(tmp_path, "odoo", "auto").rmdir()
git("add", ".")
git("commit", "-am", "Hello World", retcode=1) # pre-commit fails
git("commit", "-am", "Hello World")
def get_nonword_count(target):
"""
Run git diff and check lines added
"""
regex = re.compile("[+][^+]")
path = get_unanimous(target)
git = local["git"]
with local.cwd(str(path.parent)):
output = git("diff", path.name)
return len([line for line in output.splitlines() if regex.match(line)])
def test_start(
cloned_template: Path,
docker: LocalCommand,
supported_odoo_version: float,
tmp_path: Path,
):
"""Test the start task.
On this test flow, other downsream tasks are also tested:
- img-build
- git-aggregate
- stop --purge
"""
try:
with local.cwd(tmp_path):
copy(
src_path=str(cloned_template),
vcs_ref="HEAD",
force=True,
data={"odoo_version": supported_odoo_version},
)
# Imagine the user is in the src subfolder for these tasks
with local.cwd(tmp_path / "odoo" / "custom" / "src"):
invoke("img-build")
invoke("git-aggregate")
# Test normal call
stdout = invoke("start")
print(stdout)
assert "Reinitialized existing Git repository" in stdout
assert "pre-commit installed" in stdout
# Test "--debugpy and wait time call
invoke("stop")
stdout = invoke("start", "--debugpy")
assert socket_is_open("127.0.0.1",
int(supported_odoo_version) * 1000 + 899)
# Check if auto-reload is disabled
container_logs = docker_compose("logs", "odoo")
assert "dev=reload" not in container_logs
finally:
# Imagine the user is in the odoo subrepo for this command
with local.cwd(tmp_path / "odoo" / "custom" / "src" / "odoo"):
invoke("stop", "--purge")
def test_file_conflicts_with_committed_file(temp_git_dir):
with local.cwd(temp_git_dir):
write_file('f.py', "print('hello world')")
local['git']('add', 'f.py')
local['git']('commit', '--no-verify', '-m', 'Add f.py')
write_file('F.py', "print('hello world')")
local['git']('add', 'F.py')
assert find_conflicting_filenames(['F.py']) == 1
def configure(self):
clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension)
clang_cxx = lt_clang_cxx(self.cflags, self.ldflags,
self.compiler_extension)
with local.cwd(self.SRC_FILE):
configure = local["./configure"]
with local.env(CC=str(clang), CXX=str(clang_cxx)):
run(configure["--without-ccache", "--disable-pic14-port",
"--disable-pic16-port"])
def examine_repo(repodir):
"""
Inspect an available repository
"""
print("Opening editor")
editor = local[get_editor()]
# plumbum bug workaround
os.chdir(pathlib.Path.home())
with local.cwd(repodir):
_ = editor["spelling.txt"] & FG
def configure(self):
clang = cc(self)
clang_cxx = cxx(self)
ccrypt_dir = path.join('.', self.src_dir)
with local.cwd(ccrypt_dir):
configure = local["./configure"]
with local.env(CC=str(clang),
CXX=str(clang_cxx)):
run(configure)
def main(self, *src):
if not src:
assert self.source, "You must use the --source flag to run over GooFit's source"
git = local['git']
with local.cwd(DIR / '../'):
src = [
local.path(n) for n in git('ls-files', '--', '*.cpp',
'*.h', '*.cu').splitlines()
]
fix_files(src, self.set_version)
def stow_dotfiles():
with local.cwd(THIS_DIR):
for path in THIS_DIR.iterdir():
if not path.is_dir():
continue
if path.name.startswith("."):
continue
cmd.stow(path.name, "-R", "--no-folding")
def test_update_choice(tmp_path_factory, spawn, choices):
"""Choices are properly remembered and selected in TUI when updating."""
template, subproject = (
tmp_path_factory.mktemp("template"),
tmp_path_factory.mktemp("subproject"),
)
# Create template
build_file_tree({
template / "copier.yml":
f"""
_templates_suffix: {SUFFIX_TMPL}
_envops: {BRACKET_ENVOPS_JSON}
pick_one:
type: float
default: 3
choices: {choices}
""",
template / "[[ _copier_conf.answers_file ]].tmpl":
"[[ _copier_answers|to_nice_yaml ]]",
})
with local.cwd(template):
git("init")
git("add", ".")
git("commit", "-m one")
git("tag", "v1")
# Copy
tui = spawn(COPIER_PATH + (str(template), str(subproject)), timeout=10)
tui.expect_exact("pick_one?")
tui.sendline(Keyboard.Up)
tui.expect_exact(pexpect.EOF)
answers = yaml.safe_load((subproject / ".copier-answers.yml").read_text())
assert answers["pick_one"] == 2.0
with local.cwd(subproject):
git("init")
git("add", ".")
git("commit", "-m1")
# Update
tui = spawn(COPIER_PATH + (str(subproject), ), timeout=10)
tui.expect_exact("pick_one?")
tui.sendline(Keyboard.Down)
tui.expect_exact(pexpect.EOF)
answers = yaml.safe_load((subproject / ".copier-answers.yml").read_text())
assert answers["pick_one"] == 3.0
def test_symlink(builddir, runner):
builder = BuildFile(build_dir=builddir, runner=runner)
with local.cwd(builddir):
sh.touch('testfile')
sh.mkdir('testdir')
###### First build ##########
builder.main(command_line=['-D', 'build'])
expected_json = {
".deps_version": 2,
"ln -s nofile testlink_nofile": {
"testlink_nofile": "output-"
},
"ln -s testdir testlink_dir": {
"testlink_dir": "output-"
},
"ln -s testfile testlink": {
"testlink": "output-"
}
}
# assertions
with local.cwd(builddir):
assert_same_json('.deps', expected_json)
assert os.path.islink('testlink')
assert os.path.realpath('testlink').endswith('/testfile')
assert os.path.islink('testlink_dir')
assert os.path.realpath('testlink_dir').endswith('/testdir')
assert os.path.islink('testlink_nofile')
assert not os.path.isfile('testlink_nofile')
sys.exit.assert_called_once_with(0)
###### Cleaning ##########
builder.main(command_line=['-D', 'clean'])
with local.cwd(builddir):
assert not os.path.isfile('.deps')
assert not os.path.islink('testlink')
assert not os.path.islink('testlink_dir')
assert not os.path.islink('testlink_nofile')
def make(commit=DEFAULT_HASH):
softdir = getSoftDir()
blddir = softdir / "ANTs-build"
with local.cwd(softdir):
repo = downloadGithubRepo('ANTsX/ANTs', commit)
sha, date = getCommitInfo(repo)
out = get_path(sha)
# if output binary directory already exists, then return
if checkExists(out):
return
logging.info("Build code:")
blddir.mkdir()
with local.cwd(blddir):
cmake(repo)
import plumbum.cmd
plumbum.cmd.make['-j', psutil.cpu_count(logical=False)] & FG
# copy ANTs scripts
cp('-a', (softdir / 'ANTs' / 'Scripts').list(),
blddir / 'ANTS-build' / 'Examples')
# move binary directory
(blddir / 'ANTS-build' / 'Examples').move(out)
# write PATH and ANTSPATH
with open(out / 'env.sh', 'w') as f:
f.write("export PATH={}:$PATH\n".format(out))
f.write("export ANTSPATH={}\n".format(out))
# generate symbolink links
symlink = get_path(date)
print("Make symlink: {} -> {}".format(symlink, get_path(sha)))
symlink.unlink()
get_path(sha).symlink(symlink)
logging.info("Made '{}'".format(out))
def compile(self) -> None:
"""Compile the project."""
libjpeg_version_source = local.path(self.source_of_primary)
c_compiler = bb.compiler.cc(self)
with local.cwd(libjpeg_version_source):
with local.env(CC=str(c_compiler)):
bb.watch(cmake)("-G", "Unix Makefiles", ".")
bb.watch(make)("-j", get_number_of_jobs(bb_cfg()))
verify_binaries(self)
def clean_directories(builddir, in_dir=True, out_dir=True):
"""Remove the in and out of the container if confirmed by the user."""
with local.cwd(builddir):
if in_dir and os.path.exists("container-in") and ask(
"Should I delete '{0}'?".format(
os.path.abspath("container-in"))):
rm("-rf", "container-in")
if out_dir and os.path.exists("container-out") and ask(
"Should I delete '{0}'?".format(
os.path.abspath("container-out"))):
rm("-rf", "container-out")
def configure(self):
clang = cc(self)
with local.cwd(self.src_dir):
configure = local["./configure"]
with local.env(CC=str(clang)):
run(configure["--enable-threads=no", "--with-gnu-ld=yes",
"--disable-shared",
"--disable-dependency-tracking",
"--disable-xzdec", "--disable-lzmadec",
"--disable-lzmainfo", "--disable-lzma-links",
"--disable-scripts", "--disable-doc"])
def test_update_subdirectory(demo_template, tmp_path):
copier.copy(demo_template, tmp_path, force=True)
with local.cwd(tmp_path):
git_init()
copier.copy(dst_path=tmp_path, force=True)
assert not (tmp_path / "conf_project").exists()
assert not (tmp_path / "api_project").exists()
assert not (tmp_path / "api_readme.md").exists()
assert (tmp_path / "conf_readme.md").exists()
def push_commit(repodir, add_word):
"""
Create commit and push
"""
git = local["git"]
with local.cwd(repodir):
to_branch = git("symbolic-ref", "--short", "HEAD").strip()
from_branch = f"bugfix_typo_{add_word.replace(' ', '_')}"
git("commit", "-F", "__commit__.txt")
git("push", "origin", f"{to_branch}:{from_branch}")
return from_branch, to_branch
def wrap_in_builddir_func(self, *args, **kwargs):
"""The actual function inside the wrapper for the new builddir."""
p = local.path(self.builddir) / sub
if not p.exists():
LOG.error("%s does not exist.", p)
if p == local.cwd:
LOG.debug("CWD already is %s", p)
return func(self, *args, *kwargs)
with local.cwd(p):
return func(self, *args, **kwargs)
def compile(self) -> None:
"""Compile the project."""
openssl_source = local.path(self.source_of_primary)
compiler = bb.compiler.cc(self)
with local.cwd(openssl_source):
with local.env(CC=str(compiler)):
bb.watch(local['./config'])()
bb.watch(make)("-j", get_number_of_jobs(bb_cfg()))
verify_binaries(self)
def test_no_vscode_in_private(tmp_path: Path):
"""Asegurese que las carpetas .vscode se git-ignoren en la carpeta privada.
"""
copy(".", str(tmp_path), vcs_ref="HEAD", force=True)
with local.cwd(tmp_path):
git("add", ".")
git("commit", "--no-verify", "-am", "hello world")
vscode = tmp_path / "odoo" / "custom" / "src" / "private" / ".vscode"
vscode.mkdir()
(vscode / "something").touch()
assert not git("status", "--porcelain")
def test_multiple_domains_prod(
cloned_template: Path,
supported_odoo_version: float,
tmp_path: Path,
traefik_host: str,
):
"""Test multiple domains are produced properly."""
data = {
"traefik_cert_resolver": None,
"odoo_version": supported_odoo_version,
"project_name": uuid.uuid4().hex,
"domains_prod": {
f"main.{traefik_host}.sslip.io": [
f"alt0.main.{traefik_host}.sslip.io",
f"alt1.main.{traefik_host}.sslip.io",
],
f"secondary.{traefik_host}.sslip.io": [
f"alt0.secondary.{traefik_host}.sslip.io",
f"alt1.secondary.{traefik_host}.sslip.io",
],
f"third.{traefik_host}.sslip.io": [],
},
}
dc = docker_compose["-f", "prod.yaml"]
with local.cwd(tmp_path):
copy(
src_path=str(cloned_template),
dst_path=".",
vcs_ref="test",
force=True,
data=data,
)
try:
dc("build")
dc(
"run",
"--rm",
"odoo",
"--stop-after-init",
"-i",
"base",
)
dc("up", "-d")
time.sleep(10)
for main_domain, alt_list in data["domains_prod"].items():
for alt_domain in alt_list + [main_domain]:
response = requests.get(f"http://{alt_domain}/web/login")
assert response.ok
assert response.url == f"http://{main_domain}/web/login"
bad_response = requests.get(
f"http://missing.{traefik_host}.sslip.io")
assert bad_response.status_code == 404
finally:
dc("down", "--volumes", "--remove-orphans")
def compile(self) -> None:
grep_source = local.path(self.source_of_primary)
compiler = bb.compiler.cc(self)
with local.cwd(grep_source):
with local.env(CC=str(compiler)):
bb.watch(local["./bootstrap"])()
bb.watch(local["./configure"])("--disable-gcc-warnings")
bb.watch(make)("-j", get_number_of_jobs(bb_cfg()))
verify_binaries(self)
