def build_dependency(self, dep: Dependency) -> None:
self.init_flags(dep)
# This is needed at least for glog to be able to find gflags.
self.add_rpath(
os.path.join(self.fs_layout.tp_installed_dir, self.build_type,
'lib'))
if self.build_type != BUILD_TYPE_COMMON:
# Needed to find libunwind for Clang 10 when using compiler-rt.
self.add_rpath(
os.path.join(self.fs_layout.tp_installed_dir,
BUILD_TYPE_COMMON, 'lib'))
if self.args.download_extract_only:
log(
"Skipping build of dependency %s, build type %s, --download-extract-only is "
"specified.", dep.name, self.build_type)
return
env_vars: Dict[str, Optional[str]] = {
"CPPFLAGS": " ".join(self.preprocessor_flags)
}
log_and_set_env_var_to_list(env_vars, 'CXXFLAGS',
self.get_effective_cxx_flags(dep))
log_and_set_env_var_to_list(env_vars, 'CFLAGS',
self.get_effective_c_flags(dep))
log_and_set_env_var_to_list(env_vars, 'LDFLAGS',
self.get_effective_ld_flags(dep))
log_and_set_env_var_to_list(env_vars, 'LIBS', self.libs)
log_and_set_env_var_to_list(env_vars, 'CPPFLAGS',
self.get_effective_preprocessor_flags(dep))
if self.build_type == BUILD_TYPE_ASAN:
# To avoid errors similar to:
# https://gist.githubusercontent.com/mbautin/4b8eec566f54bcc35706dcd97cab1a95/raw
#
# This could also be fixed to some extent by the compiler flags
# -mllvm -asan-use-private-alias=1
# but applying that flag to all builds is complicated in practice and is probably
# best done using a compiler wrapper script, which would slow things down.
env_vars["ASAN_OPTIONS"] = "detect_odr_violation=0"
with PushDir(self.create_build_dir_and_prepare(dep)):
with EnvVarContext(**env_vars):
write_env_vars('yb_dependency_env.sh')
dep.build(self)
self.save_build_stamp_for_dependency(dep)
log("")
log("Finished building %s (%s)", dep.name, self.build_type)
log("")
def perform_pre_build_steps(self, dep: Dependency) -> None:
log("")
colored_log(YELLOW_COLOR, SEPARATOR)
colored_log(YELLOW_COLOR, "Building %s (%s)", dep.name,
self.build_type)
colored_log(YELLOW_COLOR, SEPARATOR)
self.download_manager.download_dependency(
dep=dep,
src_path=self.fs_layout.get_source_path(dep),
archive_path=self.fs_layout.get_archive_path(dep))
archive_name = dep.get_archive_name()
if archive_name:
archive_path = os.path.join('downloads', archive_name)
self.fossa_modules.append({
"fossa_module": {
"name": f"{dep.name}-{dep.version}",
"type": "raw",
"target": os.path.basename(archive_path)
},
"yb_metadata": {
"url":
dep.download_url,
"sha256sum":
self.download_manager.get_expected_checksum(archive_name)
}
})
def get_common_cmake_flag_args(self, dep: Dependency) -> List[str]:
c_flags_str = ' '.join(self.get_effective_c_flags(dep))
cxx_flags_str = ' '.join(self.get_effective_cxx_flags(dep))
preprocessor_flags_str = ' '.join(self.get_effective_preprocessor_flags(dep))
ld_flags_str = ' '.join(self.get_effective_ld_flags(dep))
exe_ld_flags_str = ' '.join(self.get_effective_executable_ld_flags(dep))
return [
'-DCMAKE_C_FLAGS={}'.format(c_flags_str),
'-DCMAKE_CXX_FLAGS={}'.format(cxx_flags_str),
'-DCMAKE_SHARED_LINKER_FLAGS={}'.format(ld_flags_str),
'-DCMAKE_EXE_LINKER_FLAGS={}'.format(exe_ld_flags_str),
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DCMAKE_INSTALL_PREFIX={}'.format(dep.get_install_prefix(self)),
'-DCMAKE_POSITION_INDEPENDENT_CODE=ON'
]
def get_effective_executable_ld_flags(self, dep: Dependency) -> List[str]:
return self.ld_flags + self.executable_only_ld_flags + dep.get_additional_ld_flags(
self)
def get_effective_c_flags(self, dep: Dependency) -> List[str]:
return (self.c_flags + self.get_effective_compiler_flags(dep) +
dep.get_additional_c_flags(self))
def build_with_cmake(self,
dep: Dependency,
extra_args: List[str] = [],
use_ninja_if_available: bool = True,
src_subdir_name: Optional[str] = None,
extra_build_tool_args: List[str] = [],
should_install: bool = True,
install_targets: List[str] = ['install'],
shared_and_static: bool = False) -> None:
build_tool = 'make'
if use_ninja_if_available:
ninja_available = is_ninja_available()
log('Ninja is %s',
'available' if ninja_available else 'unavailable')
if ninja_available:
build_tool = 'ninja'
log("Building dependency %s using CMake. Build tool: %s", dep,
build_tool)
log_prefix = self.log_prefix(dep)
os.environ["YB_REMOTE_COMPILATION"] = "0"
remove_path('CMakeCache.txt')
remove_path('CMakeFiles')
src_path = self.fs_layout.get_source_path(dep)
if src_subdir_name is not None:
src_path = os.path.join(src_path, src_subdir_name)
args = ['cmake', src_path]
if build_tool == 'ninja':
args += ['-G', 'Ninja']
args += self.get_common_cmake_flag_args(dep)
if extra_args is not None:
args += extra_args
args += dep.get_additional_cmake_args(self)
if shared_and_static and any(
arg.startswith('-DBUILD_SHARED_LIBS=') for arg in args):
raise ValueError(
"shared_and_static=True is specified but CMake arguments already mention "
"-DBUILD_SHARED_LIBS: %s" % args)
if '-DBUILD_SHARED_LIBS=OFF' not in args and not shared_and_static:
# TODO: a better approach for setting CMake arguments from multiple places.
args.append('-DBUILD_SHARED_LIBS=ON')
def build_internal(even_more_cmake_args: List[str] = []) -> None:
final_cmake_args = args + even_more_cmake_args
log("CMake command line (one argument per line):\n%s" %
"\n".join([(" " * 4 + sanitize_flags_line_for_log(line))
for line in final_cmake_args]))
log_output(log_prefix, final_cmake_args)
if build_tool == 'ninja':
dep.postprocess_ninja_build_file(self, 'build.ninja')
build_tool_cmd = [
build_tool, '-j{}'.format(get_make_parallelism())
] + extra_build_tool_args
log_output(log_prefix, build_tool_cmd)
if should_install:
log_output(log_prefix, [build_tool] + install_targets)
with open('compile_commands.json') as compile_commands_file:
compile_commands = json.load(compile_commands_file)
for command_item in compile_commands:
command_args = command_item['command'].split()
if self.build_type == BUILD_TYPE_ASAN:
assert_list_contains(command_args, '-fsanitize=address')
assert_list_contains(command_args, '-fsanitize=undefined')
if self.build_type == BUILD_TYPE_TSAN:
assert_list_contains(command_args, '-fsanitize=thread')
if shared_and_static:
for build_shared_libs_value, subdir_name in (('ON', 'shared'),
('OFF', 'static')):
build_dir = os.path.join(os.getcwd(), subdir_name)
mkdir_if_missing(build_dir)
build_shared_libs_cmake_arg = '-DBUILD_SHARED_LIBS=%s' % build_shared_libs_value
log(
"Building dependency '%s' for build type '%s' with option: %s",
dep.name, self.build_type, build_shared_libs_cmake_arg)
with PushDir(build_dir):
build_internal([build_shared_libs_cmake_arg])
else:
build_internal()
def get_source_path(self, dep: Dependency) -> str:
return os.path.join(self.tp_src_dir, dep.get_source_dir_basename())
def get_archive_path(self, dep: Dependency) -> Optional[str]:
archive_name = dep.get_archive_name()
if archive_name is None:
return None
return os.path.join(self.tp_download_dir, archive_name)
