def _init_armpl(self):
if not self.armhpc_version:
raise ConfigurationError(
'ARM Perf Libs are only supported in conjunction with the ARM HPC compilers'
+ version)
else:
# ARMPL module name has the form
# Generic-AArch64/Ubuntu/16.04/gcc-X.Y.Z/armpl/J.K.L, where we assume Z == L == 0
# Generic-AArch64/Ubuntu/16.04/arm-hpc-compiler-J.K/armpl/J.K.L, where L == 0
module_load = '. /usr/share/modules/init/sh && module load '
if self.compiler == Compiler.ARMCLANG:
self.run_env_script(
module_load +
'Generic-AArch64/Ubuntu/16.04/arm-hpc-compiler-' +
self.compiler_version + '/armpl/' + self.armhpc_version +
'.0')
elif self.compiler == Compiler.GCC:
self.run_env_script(module_load +
'Generic-AArch64/Ubuntu/16.04/gcc-' +
self.compiler_version + '.0/armpl/' +
self.armhpc_version + '.0')
else:
raise ConfigurationError(
'ARM Perf Libs will only work with armclang or gcc, but the compiler in use is: '
+ self.cxx_compiler)
self.armpl_dir = self.get_env_var('ARMPL_DIR')
def _process_options(self, handlers, opts):
opts = list(opts)
for handler in handlers:
found_opts = [x for x in opts if handler.matches(x)]
if not handler.allow_multiple and len(found_opts) > 1:
raise ConfigurationError('conflicting options found: ' + ' '.join(found_opts))
for found_opt in found_opts:
opts.remove(found_opt)
self._handle_option(handler, found_opt)
if opts:
raise ConfigurationError('unknown options: ' + ' '.join(opts))
def _manage_stdlib(self, use_stdlib_through_env_vars):
"""Coordinates the C++ standard library to use in the build
Artefacts built by all C++ compilers require link-time access
to a C++ standard library. This works differently for each
compiler, and can be influenced by build options or the
operating system.
Often the standard library comes from an installation of gcc,
and it is generally necessary to avoid the main compiler
finding the system default gcc, which may be too old to be
useful. A particular gcc installation is pre-specified for
each agent to cater for this.
Requires that self.compiler and self.system are specified
accurately.
Args:
use_stdlib_through_env_vars(bool): Whether to use CFLAGS/CXXFLAGS
environment variables to set the C++ standard library for
compilation, as used before GROMACS 2020
"""
if self.compiler == Compiler.CLANG:
# The clang compiler can use either libstdc++ or libc++.
if self.libcxx_version is None:
# Use libstdc++ from the pre-specified gcc for this
# agent.
self._locate_gcc('--gcc-toolchain={gcctoolchain}',
use_stdlib_through_env_vars)
return
else:
# Use libc++ from this clang installation
if self.compiler_version != self.libcxx_version:
raise ConfigurationError(
'libcxx version must match clang version')
self.append_to_env_var('CXXFLAGS', '-stdlib=libc++')
return
else:
if self.libcxx_version is not None:
raise ConfigurationError('libcxx only supported with clang')
if self.compiler == Compiler.INTEL and self.system != System.WINDOWS:
# The Intel compiler on Linux must use a libstdc++ from
# the pre-specified gcc for this agent.
self._locate_gcc('-gcc-name={gcctoolchain}/bin/gcc',
use_stdlib_through_env_vars)
return
if self.compiler == Compiler.GCC:
# Newer gcc needs to be linked against compatible standard
# libraries from the pre-specified gcc for this agent.
self._locate_gcc(None, use_stdlib_through_env_vars)
return
def _init_msvc(self, version):
self.compiler = Compiler.MSVC
self.compiler_version = version
if version == '2010':
self.run_env_script(
r'"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" amd64'
)
elif version == '2013':
self.run_env_script(
r'"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" amd64'
)
elif version == '2015':
self.run_env_script(
r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 -vcvars_ver=14.0'
)
elif version == '2017':
self.run_env_script(
r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64'
)
else:
raise ConfigurationError(
'only Visual Studio 2010, 2013, 2015, and 2017 are supported, got msvc-'
+ version)
self._cmd_runner.copy_env_var(
"caexcludepath",
"INCLUDE") #fragile because function is case sensitive
def get_cmake_version(cmd_runner, cmake_executable):
version_re = 'cmake version\s*([\d.]+)'
output = cmd_runner.check_output([cmake_executable, '--version'])
match = re.match(version_re, output)
if match:
return match.group(1)
raise ConfigurationError('Could not parse CMake version:\n' + output)
def build_dir(self):
"""Build directory for building gromacs.
Returns either the gromacs project directory or a separate build
directory, depending on whether the build is in- or out-of-source.
"""
if self._build_dir is None:
raise ConfigurationError('build directory not initialized before access')
return self._build_dir
def _init_armhpc(self, version):
self.armhpc_version = version
module_load = '. /usr/share/modules/init/sh && module load '
if version == '18.3' or version == '18.4':
self.run_env_script(
module_load +
'Generic-AArch64/Ubuntu/16.04/arm-hpc-compiler/' + version)
else:
raise ConfigurationError(
'Only the ARM HPC compiler 18.3 and 18.4 are supported, was passed '
+ version)
def parse(self, opt):
if opt == self.name:
return True
if opt == 'no-' + self.name:
return False
if opt.startswith(self.name + '='):
value = opt[len(self.name)+1:].lower()
if value in ('1', 'on', 'true'):
return True
if value in ('0', 'off', 'false'):
return False
raise ConfigurationError('invalid build option: ' + opt)
def read_cmake_minimum_version(executor, root):
version_re = r'(?i)cmake_minimum_required\s*\(\s*VERSION\s+([\d.]+)\s*\)'
path = os.path.join(root, 'CMakeLists.txt')
if os.path.isfile(path):
for line in executor.read_file(path):
match = re.match(version_re, line)
if match:
return match.group(1)
raise ConfigurationError(
'Could not parse minimum required CMake version from CMakeLists.txt'
)
return None
def __init__(self, executor, path):
"""Loads build script from a given path.
Args:
executor (Executor): Executor for reading the build script.
path (str): Path to the file from which the build script is loaded.
"""
build_globals = dict()
# Inject some globals to make the enums and exceptions easily usable in
# the build script.
build_globals['BuildError'] = BuildError
build_globals['Enum'] = Enum
build_globals['Option'] = OptionTypes
build_globals['Parameter'] = ParameterTypes
build_globals['BuildType'] = BuildType
build_globals['Compiler'] = Compiler
build_globals['FftLibrary'] = FftLibrary
build_globals['JobType'] = JobType
build_globals['Project'] = Project
build_globals['Simd'] = Simd
build_globals['Gpuhw'] = Gpuhw
build_globals['System'] = System
try:
source = ''.join(executor.read_file(path))
except IOError:
raise ConfigurationError('error reading build script: ' + path)
# TODO: Capture errors and try to report reasonably
code = compile(source, path, 'exec')
exec(code, build_globals)
do_build = build_globals.get('do_build', None)
if do_build is None or not callable(do_build):
raise ConfigurationError(
'build script does not define do_build(): ' + path)
self._do_build = do_build
self.settings = BuildScriptSettings()
self.settings.init_from_script_globals(build_globals)
def process_clang_analyzer_results(self):
"""Processes results from clang analyzer."""
html_dir = self.env.clang_analyzer_output_dir
html_dir = self._cwd.to_abs_path(html_dir)
# The analyzer produces a subdirectory for each run with a dynamic name.
# To make it easier to process in Jenkins, we rename it to a fixed name.
subdirs = os.listdir(html_dir)
output_dir = os.path.join(html_dir, 'final')
if not subdirs:
os.makedirs(output_dir)
with open(os.path.join(output_dir, 'index.html'), 'w') as fp:
fp.write("No errors\n")
return
if len(subdirs) > 1:
raise ConfigurationError(
"unexpected multiple clang analyzer results in " + html_dir)
# TODO: Count the issues and possibly report the files they are in etc.
self.mark_unstable('analyzer found issues')
shutil.move(os.path.join(html_dir, subdirs[0]), output_dir)
def make_archive(self, path, root_dir=None, use_git=False, prefix=None):
"""Creates a tar.gz archive.
Args:
path (str): Path to the archive to create without extension.
root_dir (str): Root directory from which the archive should be
created.
"""
if prefix:
prefix += '/'
if use_git:
if root_dir:
raise ConfigurationError(
"archiving with root dir with git not implemented")
cmd = ['git', 'archive', '-o', path + '.tar.gz']
cmd.append('--prefix=' + prefix)
cmd.extend(['-9', 'HEAD'])
self.run_cmd(cmd)
else:
path = self._cwd.to_abs_path(path)
# TODO: Check that root_dir is a subdirectory of the workspace
# (this all does not work if it is the workspace itself).
if not os.path.isabs(root_dir):
root_dir = os.path.join(self._cwd.cwd, root_dir)
org_dir = root_dir
root_dir, base_dir = os.path.split(root_dir)
# TODO: Instead of renaming the directory twice, we could use
# tarfile directly to create the archive.
if prefix:
base_dir = prefix
shutil.move(org_dir, os.path.join(root_dir, prefix))
if not base_dir:
base_dir = '.'
shutil.make_archive(path, 'gztar', root_dir, base_dir)
if prefix:
shutil.move(os.path.join(root_dir, prefix), org_dir)
def _init_icc(self, version):
if self.system == System.WINDOWS:
if self.compiler is None or self.compiler != Compiler.MSVC:
raise ConfigurationError(
'need to specify msvc version for icc on Windows')
self.c_compiler = 'icl'
self.cxx_compiler = 'icl'
self.compiler = Compiler.INTEL
self.extra_cmake_options[
'CMAKE_EXE_LINKER_FLAGS'] = '"/machine:x64"'
if version == '15.0':
self.run_env_script(
r'"C:\Program Files (x86)\Intel\Composer XE 2015\bin\compilervars.bat" intel64 vs'
+ self.compiler_version)
# TODO remove the next clause when no matrices use it any more
elif version == '16.0':
self.run_env_script(
r'"C:\Program Files (x86)\IntelSWTools\compilers_and_libraries_2016\windows\bin\compilervars.bat" intel64 vs'
+ self.compiler_version)
elif re.match('^(\d\d)$', version):
self.run_env_script(
r'"C:\Program Files (x86)\IntelSWTools\compilers_and_libraries_20{0}\windows\bin\compilervars.bat" intel64 vs{1}'
.format(version, self.compiler_version))
else:
raise ConfigurationError(
'invalid icc version: got icc-{0}. Try a version with two digits, e.g. 18 for 2018 release.'
.format(version))
else:
self.c_compiler = 'icc'
self.cxx_compiler = 'icpc'
self.compiler = Compiler.INTEL
if re.match('^(\d\d)$', version):
self.run_env_script(
'. /opt/intel/compilers_and_libraries_20{0}/linux/bin/compilervars.sh intel64'
.format(version))
# TODO remove the next clause when no matrices use it any more
elif version == '16.0':
self.run_env_script(
'. /opt/intel/compilers_and_libraries_2016/linux/bin/compilervars.sh intel64'
)
elif version == '15.0':
self.run_env_script(
'. /opt/intel/composer_xe_2015/bin/compilervars.sh intel64'
)
elif version == '14.0':
self.run_env_script(
'. /opt/intel/composer_xe_2013_sp1/bin/compilervars.sh intel64'
)
elif version == '13.0':
self.run_env_script(
'. /opt/intel/composer_xe_2013/bin/compilervars.sh intel64'
)
elif version == '12.1':
self.run_env_script(
'. /opt/intel/composer_xe_2011_sp1/bin/compilervars.sh intel64'
)
else:
raise ConfigurationError(
'invalid icc version: got icc-{0}. Try a version with two digits, e.g. 18 for 2018 release.'
.format(version))
self.compiler_version = version
def _get_checkout_info(self, project):
"""Returns the project info for a project that has been checked
out from git."""
if project not in self._checkouts:
raise ConfigurationError('accessing project {0} before checkout'.format(project))
return self._checkouts[project]
def _verify_project(self, project, expect_checkout=False):
if project not in self._projects:
raise ConfigurationError(project.upper() + '_REFSPEC is not set')
if expect_checkout and not self._projects[project].is_checked_out:
raise ConfigurationError(
'accessing project {0} before checkout'.format(project))
def get_triggering_refspec(self):
refspec = self._env.get('GERRIT_REFSPEC', None)
if refspec is None:
raise ConfigurationError('GERRIT_REFSPEC not set')
return RefSpec(refspec)
def _define_handlers(e, extra_options):
"""Defines the list of recognized build options."""
# The options are processed in the order they are in the tuple, to support
# cross-dependencies between the options (there are a few). Complex
# cross-dependencies may be managed better from functions called by
# the finalize_ method of BuildEnvironment.
#
# If you add options here, please also update the documentation for the
# options in docs/releng.rst.
#
# Labels need to be specified for options that require specific features
# from the build host (e.g., required software versions or special
# hardware support). They need to match with the labels defined in
# agents.py.
handlers = [
_IntOptionHandler('build-jobs', e._set_build_jobs),
_SimpleOptionHandler('out-of-source'),
_VersionOptionHandler('cmake', e._init_cmake, label=OPT),
_VersionOptionHandler('gcc', e._init_gcc, label=OPT),
_VersionOptionHandler('gcov', label=OPT),
_VersionOptionHandler('armclang', e._init_armclang, label=OPT),
_VersionOptionHandler('clang', e._init_clang, label=OPT),
_VersionOptionHandler('libcxx', e._init_libcxx, label=OPT),
_VersionOptionHandler('clang-static-analyzer', e._init_clang_static_analyzer, label=OPT),
_VersionOptionHandler('msvc', e._init_msvc, label=OPT),
_VersionOptionHandler('icc', e._init_icc, label=OPT),
_VersionOptionHandler('doxygen', e._init_doxygen, label=OPT),
_VersionOptionHandler('sphinx', label=OPT),
_VersionOptionHandler('cuda', e._init_cuda, label=OPT),
_VersionOptionHandler('amdappsdk', e._init_amdappsdk, label=OPT), # TODO: remove
_VersionOptionHandler('clFFT', e._init_clFFT, label=OPT),
_VersionOptionHandler('armhpc', e._init_armhpc, label=OPT),
_SimpleOptionHandler('phi', e._init_phi, label=OPT),
_SimpleOptionHandler('tsan', label=OPT),
_SimpleOptionHandler('atlas', e._init_atlas),
_SimpleOptionHandler('x11', label=OPT),
_EnumOptionHandler('simd', Simd, label=simd_label),
_EnumOptionHandler('gpuhw', Gpuhw, label=gpuhw_label),
_SimpleOptionHandler('mpi', e._init_mpi, label=OPT),
_SimpleOptionHandler('armpl', e._init_armpl, label=OPT),
_SimpleOptionHandler('tidy', label=OPT)
]
if extra_options and "opencl" in extra_options:
# This build is running an old script where opencl was a bool,
# so we handle it like that using a handler that will set the
# version to a suitable value.
#
# TODO Remove this when legacy matrices are no longer supported
handlers.append(_BoolOptionHandler('opencl', e._init_opencl_legacy, label=OPT))
else:
# TODO Once the legacy matrices are no longer supported, move
# this handler into the initialization of handlers above.
handlers.append(_VersionOptionHandler('opencl', e._init_opencl, label=OPT))
if extra_options:
for name, builder in extra_options.iteritems():
new_handler = builder(name)
existing_handlers = [x for x in handlers if x.name == name]
assert len(existing_handlers) <= 1
if existing_handlers:
if type(new_handler) != type(existing_handlers[0]):
raise ConfigurationError('Option {} redeclared with a different type'.format(name))
continue
handlers.append(new_handler)
return handlers
def _check_matrix_configs(configs):
for config in configs:
if config.host and not agents.is_matrix_host(config.host):
raise ConfigurationError(
'non-matrix agent would execute this combination: ' +
' '.join(config.opts))