def compile( self, source_dir, build_dir, install_dir ):
package_source_dir = os.path.join( source_dir, self.dirname )
assert( os.path.exists( package_source_dir ) )
package_build_dir = os.path.join( build_dir, self.dirname )
runpath_dir = os.path.join( package_source_dir, 'RunPath' )
if ( not os.path.exists( os.path.join( runpath_dir, 'media.zip' ) ) ):
sh.cd( runpath_dir )
sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/media.zip' )
sh.unzip( 'media.zip' )
if ( not os.path.exists( os.path.join( runpath_dir, 'projects.zip' ) ) ):
sh.cd( runpath_dir )
sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/projects.zip' )
sh.unzip( 'projects.zip' )
sh.mkdir( '-p', package_build_dir )
sh.cd( package_build_dir )
if ( platform.system() == 'Darwin' ):
sh.cmake(
'-G', 'Xcode',
'-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'CMake' ),
package_source_dir,
_out = sys.stdout )
sh.xcodebuild( '-configuration', 'Release', _out = sys.stdout )
else:
sh.cmake(
'-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'lib/OGRE/cmake' ),
package_source_dir,
_out = sys.stdout )
sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
sh.make.install( _out = sys.stdout )
def compile(self, source_dir, build_dir, install_dir):
package_source_dir = os.path.join(source_dir, self.dirname)
assert (os.path.exists(package_source_dir))
package_build_dir = os.path.join(build_dir, self.dirname)
sh.mkdir('-p', package_build_dir)
sh.cd(package_build_dir)
if (platform.system() == 'Darwin'):
sh.cmake('-G',
'Xcode',
'-D',
'CMAKE_INSTALL_PREFIX=%s' % install_dir,
package_source_dir,
_out=sys.stdout)
sh.xcodebuild('-scheme',
'install',
'-configuration',
'Release',
_out=sys.stdout)
else:
sh.cmake('-D',
'CMAKE_INSTALL_PREFIX=%s' % install_dir,
package_source_dir,
_out=sys.stdout)
sh.make('-j4', 'VERBOSE=1', _out=sys.stdout)
sh.make.install(_out=sys.stdout)
def generate_template():
template_file = ""
if not isdir(build_dir):
mkdir(build_dir)
if isdir(build_dir):
template_file = build_dir + "/dekko.dekkoproject.pot"
print("TemplateFile: " + template_file)
cd(build_dir)
print("Running cmake to generate updated template")
cmake('..')
print("Running make")
make("-j2")
if isfile(template_file):
if isdir(po_dir):
print("Moving template to po dir: " + po_dir)
mv(template_file, po_dir)
else:
print("Couldn't find po dir: " + po_dir)
cleanup()
return
else:
cleanup()
print("No template found for: " + template_file)
return
print("Cleaning up")
cleanup()
print("YeeHaa!")
print("All done, you need to commit & push this to bitbucket now :-)")
print("NOTE: this would also be a good time to sync with launchpad, run")
print(" $ python3 launchpad_sync.py")
def BuildYcmdLibs(cmake_args):
build_dir = unicode(sh.mktemp('-d', '-t', 'ycm_build.XXXXXX')).strip()
try:
full_cmake_args = ['-G', 'Unix Makefiles']
if OnMac():
full_cmake_args.extend(CustomPythonCmakeArgs())
full_cmake_args.extend(cmake_args)
full_cmake_args.append(p.join(DIR_OF_THIS_SCRIPT, 'cpp'))
sh.cd(build_dir)
sh.cmake(*full_cmake_args, _out=sys.stdout)
build_target = ('ycm_support_libs' if 'YCM_TESTRUN' not in os.environ
else 'ycm_core_tests')
sh.make('-j',
NumCores(),
build_target,
_out=sys.stdout,
_err=sys.stderr)
if 'YCM_TESTRUN' in os.environ:
RunYcmdTests(build_dir)
finally:
sh.cd(DIR_OF_THIS_SCRIPT)
sh.rm('-rf', build_dir)
def build_opencv():
sh.pip.install("numpy")
clone_if_not_exists("opencv", "https://github.com/PolarNick239/opencv.git", branch="stable_3.0.0)
clone_if_not_exists("opencv_contrib", "https://github.com/PolarNick239/opencv_contrib.git", branch="stable_3.0.0")
sh.rm("-rf", "build")
sh.mkdir("build")
sh.cd("build")
python_path = pathlib.Path(sh.pyenv.which("python").stdout.decode()).parent.parent
version = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
sh.cmake(
"..",
"-DCMAKE_BUILD_TYPE=RELEASE",
"-DCMAKE_INSTALL_PREFIX={}/usr/local".format(python_path),
"-DWITH_CUDA=OFF",
"-DWITH_FFMPEG=OFF",
"-DINSTALL_C_EXAMPLES=OFF",
"-DBUILD_opencv_legacy=OFF",
"-DBUILD_NEW_PYTHON_SUPPORT=ON",
"-DBUILD_opencv_python3=ON",
"-DOPENCV_EXTRA_MODULES_PATH=~/opencv_contrib-4.3.2/modules",
"-DBUILD_EXAMPLES=ON",
"-DPYTHON_EXECUTABLE={}/bin/python".format(python_path),
"-DPYTHON3_LIBRARY={}/lib/libpython{}m.so".format(python_path, version),
"-DPYTHON3_PACKAGES_PATH={}/lib/python{}/site-packages/".format(python_path, version),
"-DPYTHON3_NUMPY_INCLUDE_DIRS={}/lib/python{}/site-packages/numpy/core/include".format(python_path, version),
"-DPYTHON_INCLUDE_DIR={}/include/python{}m".format(python_path, version),
_out=sys.stdout,
)
sh.make("-j4", _out=sys.stdout)
sh.make.install(_out=sys.stdout)
def compile( self, source_dir, build_dir, install_dir ):
package_source_dir = os.path.join( source_dir, self.dirname )
assert( os.path.exists( package_source_dir ) )
package_build_dir = os.path.join( build_dir, self.dirname )
sh.cd( os.path.join( package_source_dir, 'scripts/Resources' ) )
sh.sh( './copyresources.sh' )
# the install target doesn't copy the stuff that copyresources.sh puts in place
sh.cp( '-v', os.path.join( package_source_dir, 'bin/Release/Readme.txt' ), os.path.join( install_dir, 'Readme.meshy.txt' ) )
sh.cp( '-v', '-r', os.path.join( package_source_dir, 'bin/Release_Linux/Resources/' ), install_dir )
sh.mkdir( '-p', package_build_dir )
sh.cd( package_build_dir )
if ( platform.system() == 'Darwin' ):
sh.cmake(
'-G', 'Xcode',
'-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'CMake' ),
package_source_dir,
_out = sys.stdout )
sh.xcodebuild( '-configuration', 'Release', _out = sys.stdout )
else:
sh.cmake(
'-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'lib/OGRE/cmake' ),
package_source_dir,
_out = sys.stdout )
sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
sh.make.install( _out = sys.stdout )
def build_opencv():
sh.pip.install("numpy")
clone_if_not_exists("opencv", "https://github.com/PolarNick239/opencv.git", branch="stable_3.0.0)
clone_if_not_exists("opencv_contrib", "https://github.com/PolarNick239/opencv_contrib.git", branch="stable_3.0.0")
sh.rm("-rf", "build")
sh.mkdir("build")
sh.cd("build")
python_path = pathlib.Path(sh.pyenv.which("python").stdout.decode()).parent.parent
version = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
sh.cmake(
"..",
"-DCMAKE_BUILD_TYPE=RELEASE",
"-DCMAKE_INSTALL_PREFIX={}/usr/local".format(python_path),
"-DWITH_CUDA=OFF",
"-DWITH_FFMPEG=OFF",
"-DINSTALL_C_EXAMPLES=OFF",
"-DBUILD_opencv_legacy=OFF",
"-DBUILD_NEW_PYTHON_SUPPORT=ON",
"-DBUILD_opencv_python3=ON",
"-DOPENCV_EXTRA_MODULES_PATH=~/opencv_contrib-3.4.1/modules",
"-DBUILD_EXAMPLES=ON",
"-DPYTHON_EXECUTABLE={}/bin/python".format(python_path),
"-DPYTHON3_LIBRARY={}/lib/libpython{}m.so".format(python_path, version),
"-DPYTHON3_PACKAGES_PATH={}/lib/python{}/site-packages/".format(python_path, version),
"-DPYTHON3_NUMPY_INCLUDE_DIRS={}/lib/python{}/site-packages/numpy/core/include".format(python_path, version),
"-DPYTHON_INCLUDE_DIR={}/include/python{}m".format(python_path, version),
_out=sys.stdout,
)
sh.make("-j4", _out=sys.stdout)
sh.make.install(_out=sys.stdout)
def _init_build_directory(config, initialised, directory, tute_directory, output=None, config_dict=PLAT_CONFIG):
if not initialised:
tute_dir = "-DTUTORIAL_DIR=" + os.path.basename(tute_directory)
args = ['-DCMAKE_TOOLCHAIN_FILE=../kernel/gcc.cmake', '-G', 'Ninja'] + config_dict[config] + [tute_dir]
result = sh.cmake(args + ['..'], _cwd = directory, _out=output, _err=output)
if result.exit_code != 0:
return result
result = sh.cmake(['..'], _cwd = directory, _out=output, _err=output)
if result.exit_code != 0:
return result
return sh.cmake(['..'], _cwd = directory, _out=output, _err=output)
def compile( self, source_dir, build_dir, install_dir ):
package_source_dir = os.path.join( source_dir, self.dirname )
assert( os.path.exists( package_source_dir ) )
package_build_dir = os.path.join( build_dir, self.dirname )
sh.mkdir( '-p', package_build_dir )
sh.cd( package_build_dir )
if ( platform.system() == 'Darwin' ):
sh.cmake( '-G', 'Xcode', '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir, package_source_dir, _out = sys.stdout )
sh.xcodebuild( '-scheme', 'install', '-configuration', 'Release', _out = sys.stdout )
else:
sh.cmake( '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir, package_source_dir, _out = sys.stdout )
sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
sh.make.install( _out = sys.stdout )
def main():
parser = argparse.ArgumentParser(
description="Initialize a build directory for completing a tutorial. Invoke from "
"an empty sub directory, or the tutorials directory, in which case a "
"new build directory will be created for you.")
parser.add_argument('--verbose', action='store_true',
help="Output everything including debug info", default=False)
parser.add_argument('--quiet', action='store_true',
help="Suppress output except for tutorial completion information", default=True)
parser.add_argument('--plat', type=str, choices=common.ALL_CONFIGS, required=True)
parser.add_argument('--tut', type=str, choices=common.ALL_TUTORIALS, required=True)
parser.add_argument('--solution', action='store_true', help="Generate pre-made solutions", default=False)
args = parser.parse_args()
common.setup_logger(__name__)
common.set_log_level(args.verbose, True)
# Additional config/tutorial combination validation
if args.plat not in common.TUTORIALS[args.tut]:
logging.error("Tutorial %s not supported by platform %s. Valid platforms are %s: ", args.tut, args.plat, common.TUTORIALS[args.tut])
return -1
# Check that the current working directory is empty. If not create a suitably
# named build directory and switch to it
if len(os.listdir(os.getcwd())) != 0:
# Check that we are in the tutorial root directory before we decide to start
# Making new directories
if not os.access(os.getcwd() + "/init", os.X_OK):
logging.error("Current dir %s is invalid" % os.getcwd())
parser.print_help(sys.stderr)
return -1
build_dir = tempfile.mkdtemp(dir=os.getcwd(), prefix=('build_%s_%s' % (args.plat, args.tut)))
os.chdir(build_dir)
else:
build_dir = None
# Check that our parent directory is an expected tutorial root directory
if not os.access(os.getcwd() + "/../init", os.X_OK):
logging.error("Parent directory is not tutorials root directory")
return -1
# Initialize cmake. Output will be supressed as it defaults to the background
result = common.init_build_directory(args.plat, args.tut, args.solution, os.getcwd())
if result.exit_code != 0:
logging.error("Failed to initialize build directory.")
return -1
# Run cmake again but show the tutorial information (to do this we run it in the foreground)
sh.cmake(['-DTUTORIALS_PRINT_INSTRUCTIONS=TRUE', '..'], _fg=True)
# Run again in the background but turn of printing
sh.cmake(['-DTUTORIALS_PRINT_INSTRUCTIONS=FALSE', '..'])
# Inform the user about any subdirectory we might have made
if build_dir is not None:
print("Tutorials created in subdirectory \"%s\". Switch to this directory and follow the instructions above to complete." % os.path.basename(build_dir))
return 0
def setup(flamegraph_directory, flamegraph_path, perf_map_agent_path, java_home):
"""Setup deps for flamegraph"""
# Create the flamegraph directory and clean the directory
if not os.path.exists(flamegraph_directory):
os.mkdir(flamegraph_directory)
for f in os.listdir(flamegraph_directory):
file_path = os.path.join(flamegraph_directory, f)
sh.sudo.rm(file_path)
if not os.path.exists(perf_map_agent_path):
sh.git('clone', 'https://github.com/jrudolph/perf-map-agent', perf_map_agent_path)
sh.cmake('.', _cwd=perf_map_agent_path, _env={'JAVA_HOME': java_home})
sh.make(_cwd=perf_map_agent_path)
if not os.path.exists(flamegraph_path):
sh.git('clone', 'https://github.com/brendangregg/FlameGraph', flamegraph_path)
def build(self, branch, sourceDir, buildDir, rebuild):
msgcmd("Building branch {}".format(branch))
try:
if not path.exists(buildDir):
mkdir("-p", buildDir)
elif path.exists(buildDir + "/bin") and rebuild:
rm("-rf", buildDir + "/bin")
cd(buildDir)
cmake("-C", self.configFile, sourceDir, _out=debugSh, _err=debugSh)
cd(sourceDir)
make("BUILD_DIR=" + buildDir, "NPROCS=4", _out=debugSh, _err=debugSh)
except ErrorReturnCode as e:
msgerr("Failed build branch {}".format(branch))
return False
else:
msgstat("Sucess build branch {}".format(branch))
return True
def build_host(extra_utilities):
binaries = {}
for utility in ["vfctrl_usb"] + extra_utilities:
sh_print = lambda x: print_output(x, False)
CMakeCache_file = host_utility_locations[utility] / "CMakeCache.txt"
if CMakeCache_file.is_file():
sh.rm(CMakeCache_file)
print("Building %s..." % utility)
with pushd(host_utility_locations[utility]):
if utility != "vfctrl_json":
sh.cmake(".")
else:
sh.cmake([".", "-DJSON=ON"])
sh.make(_out=sh_print)
print()
path = host_utility_locations[utility] / "bin" / utility
assert path.is_file()
binaries[utility] = sh.Command(str(path))
return binaries
def check_cmake():
"""Check that cmake is installed."""
from sh import which
shOut = which('cmake')
if not shOut is None:
from sh import cmake
version = str(cmake('--version'))
return (version, True)
else:
return (None, False)
def setup(flamegraph_directory, flamegraph_path, perf_map_agent_path,
java_home):
"""Setup deps for flamegraph"""
# Create the flamegraph directory and clean the directory
if not os.path.exists(flamegraph_directory):
os.mkdir(flamegraph_directory)
for f in os.listdir(flamegraph_directory):
file_path = os.path.join(flamegraph_directory, f)
sh.sudo.rm(file_path)
if not os.path.exists(perf_map_agent_path):
sh.git('clone', 'https://github.com/jrudolph/perf-map-agent',
perf_map_agent_path)
sh.cmake('.', _cwd=perf_map_agent_path, _env={'JAVA_HOME': java_home})
sh.make(_cwd=perf_map_agent_path)
if not os.path.exists(flamegraph_path):
sh.git('clone', 'https://github.com/brendangregg/FlameGraph',
flamegraph_path)
def build(version, to_build):
saved_cwd = os.getcwd()
root_directory = os.path.join(config.WORKING_DIRECTORY, version)
source_directory = os.path.join(root_directory, config.SOURCE_PREFIX)
for (compiler_name, targets) in to_build.items():
directory = os.path.join(root_directory, compiler_name)
build_directory = os.path.join(directory, config.BUILD_PREFIX)
log_directory = os.path.join(directory, "log")
compiler = config.COMPILERS[compiler_name]
try:
os.makedirs(build_directory)
os.mkdir(log_directory)
except FileExistsError:
# ignore -- we have already built before, directories exist
pass
os.chdir(build_directory)
if not os.path.exists(os.path.join(build_directory, "CMakeCache.txt")):
# N.B. Must NOT run CMake twice. This can break things!
# In particular, CMake will think that the compilers are being changed
# (even though they're remaining the same); it will issue a warning,
# and hose the CMakeCache completely (e.g. compiler flags get reset)
c_flags = compiler.get("flags", "") + " " + compiler.get("cflags", "")
cxx_flags = compiler.get("flags", "") + " " + compiler.get("cxxflags", "")
args = [source_directory, "-DCMAKE_BUILD_TYPE=Custom",
"-DCMAKE_C_COMPILER=" + compiler["cc"],
"-DCMAKE_CXX_COMPILER=" + compiler["cxx"],
"-DCMAKE_C_FLAGS_CUSTOM=" + c_flags,
"-DCMAKE_CXX_FLAGS_CUSTOM=" + cxx_flags]
sh.cmake(*args)
# run make
sh.make(config.MAKE_FLAGS + list(targets),
_out=os.path.join(log_directory, "makefile.out"),
_err=os.path.join(log_directory, "makefile.err"))
os.chdir(saved_cwd)
def BuildYcmdLibs( cmake_args ):
build_dir = unicode( sh.mktemp( '-d', '-t', 'ycm_build.XXXXXX' ) ).strip()
try:
full_cmake_args = [ '-G', 'Unix Makefiles' ]
if OnMac():
full_cmake_args.extend( CustomPythonCmakeArgs() )
full_cmake_args.extend( cmake_args )
full_cmake_args.append( p.join( DIR_OF_THIS_SCRIPT, 'cpp' ) )
sh.cd( build_dir )
sh.cmake( *full_cmake_args, _out = sys.stdout )
build_target = ( 'ycm_support_libs' if 'YCM_TESTRUN' not in os.environ else
'ycm_core_tests' )
sh.make( '-j', NumCores(), build_target, _out = sys.stdout, _err = sys.stderr )
if 'YCM_TESTRUN' in os.environ:
RunYcmdTests( build_dir )
finally:
sh.cd( DIR_OF_THIS_SCRIPT )
sh.rm( '-rf', build_dir )
def _init_build_directory(config,
initialised,
directory,
tute_directory,
output=None,
config_dict=PLAT_CONFIG):
args = []
if not initialised:
tute_dir = "-DTUTORIAL_DIR=" + os.path.basename(tute_directory)
args = [
'-DCMAKE_TOOLCHAIN_FILE=../kernel/gcc.cmake', '-G', 'Ninja'
] + config_dict[config] + [tute_dir] + ["-C", "../settings.cmake"]
return sh.cmake(args + ['..'], _cwd=directory, _out=output, _err=output)
def compile(self, source_dir, build_dir, install_dir):
package_source_dir = os.path.join(source_dir, self.dirname)
assert (os.path.exists(package_source_dir))
package_build_dir = os.path.join(build_dir, self.dirname)
sh.cd(os.path.join(package_source_dir, 'scripts/Resources'))
sh.sh('./copyresources.sh')
# the install target doesn't copy the stuff that copyresources.sh puts in place
sh.cp('-v', os.path.join(package_source_dir, 'bin/Release/Readme.txt'),
os.path.join(install_dir, 'Readme.meshy.txt'))
sh.cp('-v', '-r',
os.path.join(package_source_dir, 'bin/Release_Linux/Resources/'),
install_dir)
sh.mkdir('-p', package_build_dir)
sh.cd(package_build_dir)
if (platform.system() == 'Darwin'):
sh.cmake('-G',
'Xcode',
'-D',
'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D',
'CMAKE_MODULE_PATH=%s' %
os.path.join(install_dir, 'CMake'),
package_source_dir,
_out=sys.stdout)
sh.xcodebuild('-configuration', 'Release', _out=sys.stdout)
else:
sh.cmake('-D',
'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D',
'CMAKE_MODULE_PATH=%s' %
os.path.join(install_dir, 'lib/OGRE/cmake'),
package_source_dir,
_out=sys.stdout)
sh.make('-j4', 'VERBOSE=1', _out=sys.stdout)
sh.make.install(_out=sys.stdout)
def _init_build_directory(config,
initialised,
directory,
tute_directory,
output=None,
config_dict=PLAT_CONFIG):
args = []
if not initialised:
tute_dir = "-DTUTORIAL_DIR=" + os.path.basename(tute_directory)
args = ['-G', 'Ninja'] + config_dict[config] + [tute_dir] + \
["-C", "../projects/sel4-tutorials/settings.cmake"]
return sh.cmake(args + [tute_directory],
_cwd=directory,
_out=output,
_err=output)
def determine_compiler_name(compiler_executable_name):
tmpdir = tempfile.gettempdir() + '/fruit-determine-compiler-version-dir'
ensure_empty_dir(tmpdir)
with open(tmpdir + '/CMakeLists.txt', 'w') as file:
file.write('message("@@@${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}@@@")\n')
modified_env = os.environ.copy()
modified_env['CXX'] = compiler_executable_name
# By converting to a list, we force all output to be read (so the command execution is guaranteed to be complete after this line).
# Otherwise, subsequent calls to determine_compiler_name might have trouble deleting the temporary directory because the cmake
# process is still writing files in there.
cmake_output = list(sh.cmake('.', _cwd=tmpdir, _env=modified_env, _iter='err'))
for line in cmake_output:
re_result = re.search('@@@(.*)@@@', line)
if re_result:
pretty_name = re_result.group(1)
# CMake calls GCC 'GNU', change it into 'GCC'.
return pretty_name.replace('GNU ', 'GCC ')
raise Exception('Unable to determine compiler. CMake output was: \n', cmake_output)
def init_build_directory(config, tut, solution, directory, output=None):
tut_arg = "-DTUTORIAL=" + tut
args = ['-DCMAKE_TOOLCHAIN_FILE=../kernel/gcc.cmake', '-G', 'Ninja'
] + PLAT_CONFIG[config] + [tut_arg]
if solution:
args = args + ["-DBUILD_SOLUTIONS=TRUE"]
result = sh.cmake(args + ['..'], _cwd=directory, _out=output)
if result.exit_code != 0:
return result
sh.cmake(['..'], _cwd=directory, _out=output)
if result.exit_code != 0:
return result
sh.cmake(['..'], _cwd=directory, _out=output)
if result.exit_code != 0:
return result
return sh.cmake(['..'], _cwd=directory, _out=output)
#!/usr/bin/env python
from contextlib import contextmanager
import os
from sh import git, cmake, make, mv
import sys
@contextmanager
def cd(newdir):
prevdir = os.getcwd()
os.chdir(os.path.expanduser(newdir))
try:
yield
finally:
os.chdir(prevdir)
if __name__ == "__main__":
# git clone
with cd("tool"):
print(cmake(".", _out=sys.stdout))
print(make(_out=sys.stdout))
mv("src/sigrefmc", "..")
mv("src/sigrefmc_ht", "..")
def main():
# This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead.
numpy.seterr(all='raise')
scipy.seterr(all='raise')
parser = argparse.ArgumentParser(
description='Runs a set of benchmarks defined in a YAML file.')
parser.add_argument(
'--fruit-benchmark-sources-dir',
help='Path to the fruit sources (used for benchmarking code only)')
parser.add_argument('--fruit-sources-dir',
help='Path to the fruit sources')
parser.add_argument('--boost-di-sources-dir',
help='Path to the Boost.DI sources')
parser.add_argument(
'--output-file',
help=
'The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.'
)
parser.add_argument(
'--benchmark-definition',
help=
'The YAML file that defines the benchmarks (see fruit_wiki_benchs.yml for an example).'
)
args = parser.parse_args()
if args.output_file is None:
raise Exception('You must specify --output_file')
sh.rm('-f', args.output_file)
fruit_build_tmpdir = tempfile.gettempdir() + '/fruit-benchmark-build-dir'
with open(args.benchmark_definition, 'r') as f:
yaml_file_content = yaml.load(f)
global_definitions = yaml_file_content['global']
benchmark_definitions = expand_benchmark_definitions(
yaml_file_content['benchmarks'])
benchmark_index = 0
all_compilers = {
benchmarks_definition['compiler']
for benchmarks_definition in benchmark_definitions
}
for compiler_executable_name in all_compilers:
print('Preparing for benchmarks with the compiler %s' %
compiler_executable_name)
compiler_name = determine_compiler_name(compiler_executable_name)
# Build Fruit in fruit_build_tmpdir, so that fruit_build_tmpdir points to a built Fruit (useful for e.g. the config header).
shutil.rmtree(fruit_build_tmpdir, ignore_errors=True)
os.makedirs(fruit_build_tmpdir)
modified_env = os.environ.copy()
modified_env['CXX'] = compiler_executable_name
sh.cmake(args.fruit_sources_dir,
'-DCMAKE_BUILD_TYPE=Release',
_cwd=fruit_build_tmpdir,
_env=modified_env)
make_command(_cwd=fruit_build_tmpdir)
for benchmark_definition in benchmark_definitions:
if benchmark_definition['compiler'] != compiler_executable_name:
continue
# The 'compiler_name' dimension is synthesized automatically from the 'compiler' dimension.
# We put the compiler name/version in the results because the same 'compiler' value might refer to different compiler versions
# (e.g. if GCC 6.0.0 is installed when benchmarks are run, then it's updated to GCC 6.0.1 and finally the results are formatted, we
# want the formatted results to say "GCC 6.0.0" instead of "GCC 6.0.1").
benchmark_definition = benchmark_definition.copy()
benchmark_definition['compiler_name'] = compiler_name
benchmark_index += 1
print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions),
benchmark_definition))
benchmark_name = benchmark_definition['name']
if (benchmark_name in {
'boost_di_compile_time', 'boost_di_run_time',
'boost_di_executable_size'
} and args.boost_di_sources_dir is None):
raise Exception(
'Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.'
)
if benchmark_name == 'new_delete_run_time':
benchmark = NewDeleteRunTimeBenchmark(
benchmark_definition,
fruit_benchmark_sources_dir=args.
fruit_benchmark_sources_dir)
elif benchmark_name == 'fruit_single_file_compile_time':
benchmark = FruitSingleFileCompileTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_benchmark_sources_dir=args.
fruit_benchmark_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'fruit_compile_time':
benchmark = FruitCompileTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'fruit_run_time':
benchmark = FruitRunTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'fruit_executable_size':
benchmark = FruitExecutableSizeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'boost_di_compile_time':
benchmark = BoostDiCompileTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir,
boost_di_sources_dir=args.boost_di_sources_dir)
elif benchmark_name == 'boost_di_run_time':
benchmark = BoostDiRunTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir,
boost_di_sources_dir=args.boost_di_sources_dir)
elif benchmark_name == 'boost_di_executable_size':
benchmark = BoostDiExecutableSizeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir,
boost_di_sources_dir=args.boost_di_sources_dir)
else:
raise Exception("Unrecognized benchmark: %s" % benchmark_name)
run_benchmark(benchmark,
output_file=args.output_file,
max_runs=global_definitions['max_runs'])
def cmakeVT(self):
cmake(*self.cmakeParams + ['..'], _out=_print_line, _err=_print_line)
#!/usr/bin/env python
from contextlib import contextmanager
import os
from sh import cmake, make, mv
import sys
@contextmanager
def cd(newdir):
prevdir = os.getcwd()
os.chdir(os.path.expanduser(newdir))
try:
yield
finally:
os.chdir(prevdir)
if __name__ == "__main__":
# git clone
with cd("tool"):
print(cmake('.', _out=sys.stdout))
print(make(_out=sys.stdout))
mv("src/sigrefmc", "..")
mv("src/sigrefmc_ht", "..")
{fullcmd}
from {BUILDDIR} (i.e., CMake's config+generate stages).
Scroll up for details or look at the build log via
less -R {BUILDDIR}
Exiting...
""")
else:
rich.print(f"The config+generate stage was skipped as CMakeCache.txt was already found in {BUILDDIR}")
if args.cmake_graphviz:
output = sh.cmake(["--graphviz=graphviz/targets.dot", "."])
sys.exit(output.exit_code)
if args.n_jobs:
nprocs_argument = f"-j {args.n_jobs}"
else:
nprocs = multiprocessing.cpu_count()
nprocs_argument = f"-j {nprocs}"
rich.print(f"This script believes you have {nprocs} processors available on this system, and will use as many of them as it can")
starttime_build_d=get_time("as_date")
starttime_build_s=get_time("as_seconds_since_epoch")
build_options=""
if args.cpp_verbose:
try:
if gitCloneSuccFlag:
print("git clone, successed, skip")
else:
print("git cloning...")
sh.contrib.git.clone(cfg.repository, cfg.projectname)
gitCloneSuccFlag = True
sh.cd(cfg.projectname)
if cmakeSuccFlag:
print("cmake, successed, skip")
else:
print('cmake is running')
sh.cmake('.')
cmakeSuccFlag = True
if makeSuccFlag:
print("make, successed, skip")
else:
print('make is running...')
sh.make('-j{}'.format(cfg.thread))
makeSuccFlag = True
break
except Exception as e:
print("Build Failed")
print("Try : ", tryTimes)
print('Exception Caught:', e)
def main():
# This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead.
numpy.seterr(all='raise')
scipy.seterr(all='raise')
parser = argparse.ArgumentParser(description='Runs a set of benchmarks defined in a YAML file.')
parser.add_argument('--fruit-benchmark-sources-dir', help='Path to the fruit sources (used for benchmarking code only)')
parser.add_argument('--fruit-sources-dir', help='Path to the fruit sources')
parser.add_argument('--boost-di-sources-dir', help='Path to the Boost.DI sources')
parser.add_argument('--output-file',
help='The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.')
parser.add_argument('--benchmark-definition', help='The YAML file that defines the benchmarks (see fruit_wiki_benchs_fruit.yml for an example).')
parser.add_argument('--continue-benchmark', help='If this is \'true\', continues a previous benchmark run instead of starting from scratch (taking into account the existing benchmark results in the file specified with --output-file).')
args = parser.parse_args()
if args.output_file is None:
raise Exception('You must specify --output_file')
if args.continue_benchmark == 'true':
with open(args.output_file, 'r') as f:
previous_run_completed_benchmarks = [json.loads(line)['benchmark'] for line in f.readlines()]
else:
previous_run_completed_benchmarks = []
sh.rm('-f', args.output_file)
fruit_build_tmpdir = tempfile.gettempdir() + '/fruit-benchmark-build-dir'
with open(args.benchmark_definition, 'r') as f:
yaml_file_content = yaml.load(f)
global_definitions = yaml_file_content['global']
benchmark_definitions = expand_benchmark_definitions(yaml_file_content['benchmarks'])
benchmark_index = 0
all_compilers = {benchmarks_definition['compiler'] for benchmarks_definition in benchmark_definitions}
for compiler_executable_name in all_compilers:
print('Preparing for benchmarks with the compiler %s' % compiler_executable_name)
# We compute this here (and memoize the result) so that the benchmark's describe() will retrieve the cached
# value instantly.
determine_compiler_name(compiler_executable_name)
# Build Fruit in fruit_build_tmpdir, so that fruit_build_tmpdir points to a built Fruit (useful for e.g. the config header).
shutil.rmtree(fruit_build_tmpdir, ignore_errors=True)
os.makedirs(fruit_build_tmpdir)
modified_env = os.environ.copy()
modified_env['CXX'] = compiler_executable_name
sh.cmake(args.fruit_sources_dir, '-DCMAKE_BUILD_TYPE=Release', _cwd=fruit_build_tmpdir, _env=modified_env)
make_command(_cwd=fruit_build_tmpdir)
for benchmark_definition in benchmark_definitions:
if benchmark_definition['compiler'] != compiler_executable_name:
continue
benchmark_index += 1
print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions), benchmark_definition))
benchmark_name = benchmark_definition['name']
if (benchmark_name in {'boost_di_compile_time', 'boost_di_run_time', 'boost_di_executable_size'}
and args.boost_di_sources_dir is None):
raise Exception('Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.')
if benchmark_name == 'new_delete_run_time':
benchmark = NewDeleteRunTimeBenchmark(
benchmark_definition,
fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir)
elif benchmark_name == 'fruit_single_file_compile_time':
benchmark = FruitSingleFileCompileTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'fruit_compile_time':
benchmark = FruitCompileTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'fruit_run_time':
benchmark = FruitRunTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'fruit_executable_size':
benchmark = FruitExecutableSizeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir)
elif benchmark_name == 'boost_di_compile_time':
benchmark = BoostDiCompileTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir,
boost_di_sources_dir=args.boost_di_sources_dir)
elif benchmark_name == 'boost_di_run_time':
benchmark = BoostDiRunTimeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir,
boost_di_sources_dir=args.boost_di_sources_dir)
elif benchmark_name == 'boost_di_executable_size':
benchmark = BoostDiExecutableSizeBenchmark(
benchmark_definition,
fruit_sources_dir=args.fruit_sources_dir,
fruit_build_tmpdir=fruit_build_tmpdir,
boost_di_sources_dir=args.boost_di_sources_dir)
else:
raise Exception("Unrecognized benchmark: %s" % benchmark_name)
if benchmark.describe() in previous_run_completed_benchmarks:
print("Skipping benchmark that was already run previously (due to --continue-benchmark):", benchmark.describe())
continue
run_benchmark(benchmark, output_file=args.output_file, max_runs=global_definitions['max_runs'])
