def activate_devtoolset(devtoolset_number: int) -> None:
devtoolset_enable_script = ('/opt/rh/devtoolset-%d/enable' %
devtoolset_number)
log("Enabling devtoolset-%s by sourcing the script %s", devtoolset_number,
devtoolset_enable_script)
if not os.path.exists(devtoolset_enable_script):
raise IOError("Devtoolset script does not exist: %s" %
devtoolset_enable_script)
cmd_args = ['bash', '-c', '. "%s" && env' % devtoolset_enable_script]
log("Running command: %s", cmd_args)
devtoolset_env_str = subprocess.check_output(cmd_args).decode('utf-8')
found_vars = set()
for line in devtoolset_env_str.split("\n"):
line = line.strip()
if not line:
continue
k, v = line.split("=", 1)
if k in DEVTOOLSET_ENV_VARS:
log("Setting %s to: %s", k, v)
os.environ[k] = v
found_vars.add(k)
missing_vars = set()
for var_name in DEVTOOLSET_ENV_VARS:
if var_name not in found_vars:
log("Did not set env var %s for devtoolset-%d", var_name,
devtoolset_number)
if var_name not in DEVTOOLSET_ENV_VARS_OK_IF_UNSET:
missing_vars.add(var_name)
if missing_vars:
raise IOError(
"Invalid environment after running devtoolset script %s. Did not set vars: %s"
% (devtoolset_enable_script, ', '.join(sorted(missing_vars))))
def activate_devtoolset(devtoolset_number: int) -> None:
devtoolset_enable_script_candidates = [
f'/opt/rh/{toolset_name_prefix}-{devtoolset_number}/enable'
for toolset_name_prefix in DEVTOOLSET_DIR_NAMES
]
existing_devtoolset_enable_scripts = [
script_path for script_path in devtoolset_enable_script_candidates
if os.path.exists(script_path)
]
if len(existing_devtoolset_enable_scripts) != 1:
fatal(
f"Expected exactly one of the scripts to exist: {devtoolset_enable_script_candidates}. "
f"Found that {len(existing_devtoolset_enable_scripts)} exist.")
devtoolset_enable_script = existing_devtoolset_enable_scripts[0]
log("Enabling devtoolset-%s by sourcing the script %s",
devtoolset_number, devtoolset_enable_script)
if not os.path.exists(devtoolset_enable_script):
raise IOError("Devtoolset script does not exist: %s" % devtoolset_enable_script)
echo_env_vars_str = '; '.join(
['echo %s=$%s' % (k, shlex.quote(k)) for k in DEVTOOLSET_ENV_VARS])
cmd_args = ['bash', '-c', '. "%s" && ( %s )' % (devtoolset_enable_script, echo_env_vars_str)]
log("Running command: %s", cmd_args)
devtoolset_env_str = subprocess.check_output(cmd_args).decode('utf-8')
found_vars = set()
for line in devtoolset_env_str.split("\n"):
line = line.strip()
if not line:
continue
k, v = line.split("=", 1)
if k in DEVTOOLSET_ENV_VARS:
log("Setting %s to: %s", k, v)
os.environ[k] = v
found_vars.add(k)
missing_vars = set()
for var_name in DEVTOOLSET_ENV_VARS:
if var_name not in found_vars:
log("Did not set env var %s for devtoolset-%d", var_name, devtoolset_number)
if var_name not in DEVTOOLSET_ENV_VARS_OK_IF_UNSET:
missing_vars.add(var_name)
if missing_vars:
raise IOError(
"Invalid environment after running devtoolset script %s. Did not set vars: %s" % (
devtoolset_enable_script, ', '.join(sorted(missing_vars))
))
def parse_cmd_line_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument(
'--build-type',
default=None,
type=str,
choices=BUILD_TYPES,
help='Build only specific part of thirdparty dependencies.')
parser.add_argument(
'--skip-sanitizers',
action='store_true',
help='Do not build ASAN and TSAN instrumented dependencies.')
parser.add_argument('--clean',
action='store_true',
default=False,
help='Clean, but keep downloads.')
parser.add_argument('--clean-downloads',
action='store_true',
default=False,
help='Clean, including downloads.')
parser.add_argument('--add_checksum',
help='Compute and add unknown checksums to %s' %
CHECKSUM_FILE_NAME,
action='store_true')
parser.add_argument('--skip', help='Dependencies to skip')
parser.add_argument(
'--single-compiler-type',
type=str,
choices=['gcc', 'clang'],
default=None,
help=
'Produce a third-party dependencies build using only a single compiler. '
'This also implies that we are not using Linuxbrew.')
parser.add_argument(
'--compiler-prefix',
type=str,
help=
'The prefix directory for looking for compiler executables. We will look for '
'compiler executable in the bin subdirectory of this directory.')
parser.add_argument(
'--compiler-suffix',
type=str,
default='',
help=
'Suffix to append to compiler executables, such as the version number, '
'potentially prefixed with a dash, to obtain names such as gcc-8, g++-8, '
'clang-10, or clang++-10.')
parser.add_argument('--devtoolset',
type=int,
help='Specifies a CentOS devtoolset')
parser.add_argument(
'-j',
'--make-parallelism',
help='How many cores should the build use. This is passed to '
'Make/Ninja child processes. This can also be specified using the '
'YB_MAKE_PARALLELISM environment variable.',
type=int)
parser.add_argument('--use-ccache',
action='store_true',
help='Use ccache to speed up compilation')
parser.add_argument(
'--use-compiler-wrapper',
action='store_true',
help='Use a compiler wrapper script. Allows additional validation but '
'makes the build slower.')
parser.add_argument(
'--llvm-version',
default=None,
help='Version (tag) to use for dependencies based on LLVM codebase')
parser.add_argument(
'--remote-build-server',
help=
'Build third-party dependencies remotely on this server. The default value is '
'determined by YB_THIRDPARTY_REMOTE_BUILD_SERVER environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_SERVER'))
parser.add_argument(
'--remote-build-dir',
help=
'The directory on the remote server to build third-party dependencies in. The '
'value is determined by the YB_THIRDPARTY_REMOTE_BUILD_DIR environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_DIR'))
parser.add_argument(
'--local',
help=
'Forces the local build even if --remote-... options are specified or the '
'corresponding environment variables are set.',
action='store_true')
parser.add_argument(
'--download-extract-only',
help=
'Only download and extract archives. Do not build any dependencies.',
action='store_true')
parser.add_argument(
'--multi-build',
action='store_true',
help=
'Build multiple configurations in parallel. Most other arguments will get ignored '
'if this is specified. Configurations to build are taken from .circleci/config.yml. '
'The set of configurations to build can be customized using the '
'--multi-build-conf-name-pattern flag.')
parser.add_argument(
'--multi-build-conf-name-pattern',
help=
'Only build configurations matching this glob-style pattern, anchored on both ends. '
'This implies --multi-build.')
parser.add_argument('--license-report',
action='store_true',
help='Generate a license report.')
parser.add_argument(
'--toolchain',
help='Automatically download, install and use the given toolchain',
choices=TOOLCHAIN_TYPES)
parser.add_argument('dependencies',
nargs=argparse.REMAINDER,
help='Dependencies to build.')
args = parser.parse_args()
# ---------------------------------------------------------------------------------------------
# Validating arguments
# ---------------------------------------------------------------------------------------------
if args.dependencies and args.skip:
raise ValueError(
"--skip is not compatible with specifying a list of dependencies to build"
)
if is_mac():
if args.single_compiler_type not in [None, 'clang']:
raise ValueError(
"--single-compiler-type=%s is not allowed on macOS" %
args.single_compiler_type)
args.single_compiler_type = 'clang'
if args.local and (args.remote_build_server is not None
or args.remote_build_dir is not None):
log("Forcing a local build")
args.remote_build_server = None
args.remote_build_dir = None
if (args.remote_build_server is None) != (args.remote_build_dir is None):
raise ValueError(
'--remote-build-server and --remote-build-dir have to be specified or unspecified '
'at the same time. Note that their default values are provided by corresponding '
'environment variables, YB_THIRDPARTY_REMOTE_BUILD_SERVER and '
'YB_THIRDPARTY_REMOTE_BUILD_DIR.')
if args.remote_build_dir is not None:
assert os.path.isabs(args.remote_build_dir), (
'Remote build directory path must be an absolute path: %s' %
args.remote_build_dir)
is_remote_build = args.remote_build_server is not None
if args.devtoolset is not None and not is_remote_build:
if not is_centos():
raise ValueError("--devtoolset can only be used on CentOS Linux")
if args.single_compiler_type not in [None, 'gcc']:
raise ValueError(
"--devtoolset is not compatible with compiler type: %s" %
args.single_compiler_type)
args.single_compiler_type = 'gcc'
if args.toolchain:
if args.devtoolset:
raise ValueError("--devtoolset and --toolchain are incompatible")
if args.compiler_prefix:
raise ValueError(
"--compiler-prefix and --toolchain are incompatible")
if args.compiler_suffix:
raise ValueError(
"--compiler-suffix and --toolchain are incompatible")
if args.multi_build_conf_name_pattern:
args.multi_build = True
return args
def run_remote_bash_script(bash_script: str) -> None:
bash_script = bash_script.strip()
log("Running script remotely: %s", bash_script)
# TODO: why exactly do we need shlex.quote here?
run_ssh_cmd(['bash', '-c', shlex.quote(bash_script)])
def build_remotely(remote_server: str, remote_build_code_path: str) -> None:
assert remote_server is not None
assert remote_build_code_path is not None
assert remote_build_code_path.startswith('/')
def run_ssh_cmd(ssh_args: List[str]) -> None:
log_and_run_cmd(['ssh', remote_server] + ssh_args)
def run_remote_bash_script(bash_script: str) -> None:
bash_script = bash_script.strip()
log("Running script remotely: %s", bash_script)
# TODO: why exactly do we need shlex.quote here?
run_ssh_cmd(['bash', '-c', shlex.quote(bash_script)])
quoted_remote_path = shlex.quote(remote_build_code_path)
# Ensure the remote directory exists. We are not attempting to create it if it does not.
run_remote_bash_script('[[ -d %s ]]' % quoted_remote_path)
with PushDir(YB_THIRDPARTY_DIR):
local_branch_name = get_current_git_branch_name()
local_git_remotes = subprocess.check_output(
shlex.split('git remote -v')).decode('utf-8')
remote_url = '%s:%s' % (remote_server, remote_build_code_path)
preferred_remote_name = 'remote-build-%s' % remote_server
remote_name = None
for remote_line in local_git_remotes.split('\n'):
remote_line = remote_line.strip()
if not remote_line:
continue
remote_components = remote_line.split('\t')
if remote_components[1].endswith(' (push)'):
parsed_remote_url = remote_components[1][:-7].strip()
if parsed_remote_url == remote_url:
remote_name = remote_components[0]
log("Found existing remote %s for %s", remote_name,
remote_url)
break
if remote_name is None:
log_and_run_cmd(
['git', 'remote', 'add', preferred_remote_name, remote_url])
remote_name = preferred_remote_name
log("Local branch name: %s, checking it out remotely",
local_branch_name)
run_remote_bash_script(f"""
set -euo pipefail
cd {quoted_remote_path}
git reset --hard HEAD
git clean -df
git checkout master
""")
log_and_run_cmd([
'git', 'push', '--force', remote_name,
'%s:%s' % (local_branch_name, local_branch_name)
])
run_remote_bash_script(
'cd %s && git checkout %s' %
(quoted_remote_path, shlex.quote(local_branch_name)))
rsync_code_to('%s:%s' % (remote_server, remote_build_code_path))
remote_bash_script = 'cd %s && ./build_thirdparty.sh %s' % (
quoted_remote_path, shlex_join(sys.argv[1:]))
run_remote_bash_script(remote_bash_script)
def parse_cmd_line_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument(
'--build-type',
default=None,
type=str,
choices=BUILD_TYPES,
help='Build only specific part of thirdparty dependencies.')
parser.add_argument(
'--skip-sanitizers',
action='store_true',
help='Do not build ASAN and TSAN instrumented dependencies.')
parser.add_argument('--clean',
action='store_true',
default=False,
help='Clean, but keep downloads.')
parser.add_argument('--clean-downloads',
action='store_true',
default=False,
help='Clean, including downloads.')
parser.add_argument('--add-checksum',
help='Compute and add unknown checksums to %s' %
CHECKSUM_FILE_NAME,
action='store_true')
parser.add_argument('--skip', help='Dependencies to skip')
parser.add_argument(
'--single-compiler-type',
type=str,
choices=['gcc', 'clang'],
default=None,
help=
'Produce a third-party dependencies build using only a single compiler. '
'This also implies that we are not using Linuxbrew.')
parser.add_argument(
'--compiler-prefix',
type=str,
help=
'The prefix directory for looking for compiler executables. We will look for '
'compiler executable in the bin subdirectory of this directory.')
parser.add_argument(
'--compiler-suffix',
type=str,
default='',
help=
'Suffix to append to compiler executables, such as the version number, '
'potentially prefixed with a dash, to obtain names such as gcc-8, g++-8, '
'clang-10, or clang++-10.')
parser.add_argument('--devtoolset',
type=int,
help='Specifies a CentOS devtoolset')
parser.add_argument(
'-j',
'--make-parallelism',
help='How many cores should the build use. This is passed to '
'Make/Ninja child processes. This can also be specified using the '
'YB_MAKE_PARALLELISM environment variable.',
type=int)
parser.add_argument('--use-ccache',
action='store_true',
help='Use ccache to speed up compilation')
parser.add_argument(
'--use-compiler-wrapper',
action='store_true',
help='Use a compiler wrapper script. Allows additional validation but '
'makes the build slower.')
parser.add_argument(
'--llvm-version',
default=None,
help='Version (tag) to use for dependencies based on LLVM codebase')
parser.add_argument(
'--remote-build-server',
help=
'Build third-party dependencies remotely on this server. The default value is '
'determined by YB_THIRDPARTY_REMOTE_BUILD_SERVER environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_SERVER'))
parser.add_argument(
'--remote-build-dir',
help=
'The directory on the remote server to build third-party dependencies in. The '
'value is determined by the YB_THIRDPARTY_REMOTE_BUILD_DIR environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_DIR'))
parser.add_argument(
'--local',
help=
'Forces the local build even if --remote-... options are specified or the '
'corresponding environment variables are set.',
action='store_true')
parser.add_argument(
'--download-extract-only',
help=
'Only download and extract archives. Do not build any dependencies.',
action='store_true')
parser.add_argument('--license-report',
action='store_true',
help='Generate a license report.')
parser.add_argument(
'--toolchain',
help='Automatically download, install and use the given toolchain',
choices=TOOLCHAIN_TYPES)
parser.add_argument('--create-package',
help='Create the package tarball',
action='store_true')
parser.add_argument(
'--upload-as-tag',
help='Upload the package tarball as a GitHub release under this tag. '
'Implies --create-package. Requires GITHUB_TOKEN to be set. If GITHUB_TOKEN is not '
'set, this is a no-op (with success exit code).')
parser.add_argument(
'--expected-major-compiler-version',
type=int,
help='Expect the major version of the compiler to be as specified')
parser.add_argument('--verbose',
help='Show verbose output',
action='store_true')
parser.add_argument('dependencies',
nargs=argparse.REMAINDER,
help='Dependencies to build.')
parser.add_argument(
'--enforce_arch',
help=
'Ensure that we use the given architecture, such as arm64. Useful for macOS systems '
'with Apple Silicon CPUs and Rosetta 2 installed that can switch between '
'architectures.')
parser.add_argument(
'--force',
help=
'Build dependencies even though the system does not detect any changes compared '
'to an earlier completed build.',
action='store_true')
parser.add_argument(
'--delete-build-dir',
help=
"Delete each dependency's build directory to start each build from scratch. "
"Note that this does not affect the corresponding source directory.",
action='store_true')
args = parser.parse_args()
# ---------------------------------------------------------------------------------------------
# Validating arguments
# ---------------------------------------------------------------------------------------------
if args.dependencies and args.skip:
raise ValueError(
"--skip is not compatible with specifying a list of dependencies to build"
)
if is_macos():
if args.single_compiler_type not in [None, 'clang']:
raise ValueError(
"--single-compiler-type=%s is not allowed on macOS" %
args.single_compiler_type)
args.single_compiler_type = 'clang'
if args.local and (args.remote_build_server is not None
or args.remote_build_dir is not None):
log("Forcing a local build")
args.remote_build_server = None
args.remote_build_dir = None
if (args.remote_build_server is None) != (args.remote_build_dir is None):
raise ValueError(
'--remote-build-server and --remote-build-dir have to be specified or unspecified '
'at the same time. Note that their default values are provided by corresponding '
'environment variables, YB_THIRDPARTY_REMOTE_BUILD_SERVER and '
'YB_THIRDPARTY_REMOTE_BUILD_DIR.')
if args.remote_build_dir is not None:
assert os.path.isabs(args.remote_build_dir), (
'Remote build directory path must be an absolute path: %s' %
args.remote_build_dir)
is_remote_build = args.remote_build_server is not None
if args.devtoolset is not None and not is_remote_build:
if not local_sys_conf().is_redhat_family():
raise ValueError(
"--devtoolset can only be used on Red Hat Enterprise Linux OS family"
)
if args.single_compiler_type not in [None, 'gcc']:
raise ValueError(
"--devtoolset is not compatible with compiler type: %s" %
args.single_compiler_type)
args.single_compiler_type = 'gcc'
if args.toolchain:
if args.devtoolset:
raise ValueError("--devtoolset and --toolchain are incompatible")
if args.compiler_prefix:
raise ValueError(
"--compiler-prefix and --toolchain are incompatible")
if args.compiler_suffix:
raise ValueError(
"--compiler-suffix and --toolchain are incompatible")
if args.enforce_arch and platform.machine() != args.enforce_arch:
raise ValueError("Machine architecture is %s but we expect %s" %
(platform.machine(), args.enforce_arch))
return args