def test_quote_for_bash(self):
for s in [
"\\" * 1, "\\" * 2, "\\" * 3, "\\" * 4, "\\" * 5, "\\" * 6, '',
'a', 'a b'
'"foo"'
'foo"bar', 'foo " bar', "foo'bar", '$foo $bar', '"' + "'",
'"' + "'" + '"' + "'", r"""'\''""", 'foo bar', ' ', 'foo ',
' foo', ' foo bar '
';', ' ;', '; ', ' ; '
'.', '*', ' .', '. ', '* ', ' *'
]:
quoted_s = quote_for_bash(s)
tmp_file_path = None
try:
with tempfile.NamedTemporaryFile(suffix='.sh',
delete=False) as tmp_file:
cmd = 'echo -n ' + quoted_s
tmp_file.write(cmd.encode('utf-8'))
tmp_file_path = tmp_file.name
result = subprocess.check_output(['bash', tmp_file_path
]).decode('utf-8')
self.assertEqual(
s, result,
"For input string: [[ {} ]], quote_for_bash produced [[ {} ]], "
"but echo -n with that argument returned: [[ {} ]]".format(
s, quoted_s, result))
finally:
if False and tmp_file_path and os.path.exists(tmp_file_path):
os.remove(tmp_file_path)
def make_postgres(self):
self.set_env_vars('make')
make_cmd = ['make']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_parallelism = int(make_parallelism)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = cpu_count
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
os.environ['YB_COMPILER_TYPE'] = self.compiler_type
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
make_script_content = "#!/usr/bin/env bash\n"
for env_var_name in [
'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS',
'PATH']:
env_var_value = os.environ[env_var_name]
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" % env_var_name)
make_script_content += "export %s=%s\n" % (env_var_name, quote_for_bash(env_var_value))
make_script_content += 'make "$@"\n'
for work_dir in [self.pg_build_root, os.path.join(self.pg_build_root, 'contrib')]:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write(make_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
make_result = run_program(make_cmd)
write_program_output_to_file('make', make_result, work_dir)
make_install_result = run_program(['make', 'install'])
write_program_output_to_file('make_install', make_install_result, work_dir)
logging.info("Successfully ran make in the %s directory", work_dir)
def make_postgres(self):
self.set_env_vars('make')
make_cmd = ['make']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
os.environ['YB_COMPILER_TYPE'] = self.compiler_type
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
make_script_content = "#!/usr/bin/env bash\n"
for env_var_name in [
'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS',
'CXXFLAGS', 'LDFLAGS', 'PATH'
]:
env_var_value = os.environ[env_var_name]
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
make_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
make_script_content += 'make "$@"\n'
for work_dir in [
self.pg_build_root,
os.path.join(self.pg_build_root, 'contrib')
]:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write(make_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
make_result = run_program(make_cmd)
write_program_output_to_file('make', make_result, work_dir)
make_install_result = run_program(['make', 'install'])
write_program_output_to_file('make_install',
make_install_result, work_dir)
logging.info("Successfully ran make in the %s directory",
work_dir)
def make_postgres(self):
self.set_env_vars('make')
make_cmd = ['make']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
make_script_content = "#!/usr/bin/env bash\n"
for env_var_name in [
'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS',
'PATH']:
env_var_value = os.environ[env_var_name]
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" % env_var_name)
make_script_content += "export %s=%s\n" % (env_var_name, quote_for_bash(env_var_value))
make_script_content += 'make "$@"\n'
for work_dir in [self.pg_build_root, os.path.join(self.pg_build_root, 'contrib')]:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write(make_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
make_result = run_program(make_cmd)
write_program_output_to_file('make', make_result, work_dir)
make_install_result = run_program(['make', 'install'])
write_program_output_to_file('make_install', make_install_result, work_dir)
logging.info("Successfully ran make in the %s directory", work_dir)
def make_postgres(self):
self.set_env_vars('make')
# Postgresql requires MAKELEVEL to be 0 or non-set when calling its make.
# But in case YB project is built with make, MAKELEVEL is not 0 at this point.
make_cmd = ['make', 'MAKELEVEL=0']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_parallelism = int(make_parallelism)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = cpu_count
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
self.set_env_var('YB_COMPILER_TYPE', self.compiler_type)
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
env_script_content = ''
for env_var_name in CONFIG_ENV_VARS:
env_var_value = os.environ.get(env_var_name)
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
env_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
compile_commands_files = []
work_dirs = [self.pg_build_root]
if self.build_type != 'compilecmds':
work_dirs.append(os.path.join(self.pg_build_root, 'contrib'))
for work_dir in work_dirs:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
if self.should_build:
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write('#!/usr/bin/env bash\n'
'. "${BASH_SOURCE%/*}"/env.sh\n'
'make "$@"\n')
with open('env.sh', 'w') as out_f:
out_f.write(env_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory",
work_dir)
run_program(make_cmd,
stdout_stderr_prefix='make',
cwd=work_dir,
shell=True,
error_ok=True
).print_output_and_raise_error_if_failed()
if self.build_type == 'compilecmds':
logging.info(
"Not running make install in the %s directory since we are only "
"generating the compilation database", work_dir)
else:
run_program('make install',
stdout_stderr_prefix='make_install',
cwd=work_dir,
shell=True,
error_ok=True
).print_output_and_raise_error_if_failed()
logging.info("Successfully ran make in the %s directory",
work_dir)
if self.export_compile_commands:
logging.info(
"Generating the compilation database in directory '%s'",
work_dir)
compile_commands_path = os.path.join(
work_dir, 'compile_commands.json')
self.set_env_var('YB_PG_SKIP_CONFIG_STATUS', '1')
if (not os.path.exists(compile_commands_path)
or not self.export_compile_commands_lazily):
run_program(['compiledb', 'make', '-n'],
capture_output=False)
del os.environ['YB_PG_SKIP_CONFIG_STATUS']
if not os.path.exists(compile_commands_path):
raise RuntimeError(
"Failed to generate compilation database at: %s" %
compile_commands_path)
compile_commands_files.append(compile_commands_path)
if self.export_compile_commands:
self.combine_compile_commands(compile_commands_files)
def make_postgres(self) -> None:
self.set_env_vars('make')
# Postgresql requires MAKELEVEL to be 0 or non-set when calling its make.
# But in case YB project is built with make, MAKELEVEL is not 0 at this point.
make_cmd = ['make', 'MAKELEVEL=0']
if is_macos_arm64():
make_cmd = ['arch', '-arm64'] + make_cmd
make_parallelism_str: Optional[str] = os.environ.get(
'YB_MAKE_PARALLELISM')
make_parallelism: Optional[int] = None
if make_parallelism_str is not None:
make_parallelism = int(make_parallelism_str)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = parallelism_cap
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
self.set_env_var('YB_COMPILER_TYPE', self.compiler_type)
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
env_script_content = ''
for env_var_name in CONFIG_ENV_VARS:
env_var_value = os.environ.get(env_var_name)
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
env_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
pg_compile_commands_paths = []
third_party_extensions_dir = os.path.join(self.pg_build_root,
'third-party-extensions')
work_dirs = [
self.pg_build_root,
os.path.join(self.pg_build_root, 'contrib'),
third_party_extensions_dir
]
for work_dir in work_dirs:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write('#!/usr/bin/env bash\n'
'. "${BASH_SOURCE%/*}"/env.sh\n'
'make "$@"\n')
with open('env.sh', 'w') as out_f:
out_f.write(env_script_content)
run_program(['chmod', 'u+x', make_script_path])
make_cmd_suffix = []
if work_dir == third_party_extensions_dir:
make_cmd_suffix = ['PG_CONFIG=' + self.pg_config_path]
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
complete_make_cmd = make_cmd + make_cmd_suffix
complete_make_cmd_str = shlex_join(complete_make_cmd)
complete_make_install_cmd = make_cmd + ['install'
] + make_cmd_suffix
attempt = 0
while attempt <= TRANSIENT_BUILD_RETRIES:
attempt += 1
make_result = run_program(
complete_make_cmd_str,
stdout_stderr_prefix='make',
cwd=work_dir,
error_ok=True,
shell=True # TODO: get rid of shell=True.
)
if make_result.failure():
transient_err = False
stderr_lines = make_result.get_stderr().split('\n')
for line in stderr_lines:
if any(transient_error_pattern in line
for transient_error_pattern in
TRANSIENT_BUILD_ERRORS):
transient_err = True
logging.info(f'Transient error: {line}')
break
if transient_err:
logging.info(
f"Transient error during build attempt {attempt}. "
f"Re-trying make command: {complete_make_cmd_str}."
)
else:
make_result.print_output_to_stdout()
raise RuntimeError("PostgreSQL compilation failed")
else:
logging.info(
"Successfully ran 'make' in the %s directory",
work_dir)
break # No error, break out of retry loop
else:
raise RuntimeError(
f"Maximum build attempts reached ({TRANSIENT_BUILD_RETRIES} attempts)."
)
if self.build_type != 'compilecmds' or work_dir == self.pg_build_root:
run_program(
' '.join(
shlex.quote(arg)
for arg in complete_make_install_cmd),
stdout_stderr_prefix='make_install',
cwd=work_dir,
error_ok=True,
shell=True # TODO: get rid of shell=True.
).print_output_and_raise_error_if_failed()
logging.info(
"Successfully ran 'make install' in the %s directory",
work_dir)
else:
logging.info(
"Not running 'make install' in the %s directory since we are only "
"generating the compilation database", work_dir)
if self.export_compile_commands:
logging.info(
"Generating the compilation database in directory '%s'",
work_dir)
compile_commands_path = os.path.join(
work_dir, 'compile_commands.json')
self.set_env_var('YB_PG_SKIP_CONFIG_STATUS', '1')
if not os.path.exists(compile_commands_path):
run_program(['compiledb', 'make', '-n'] +
make_cmd_suffix,
capture_output=False)
del os.environ['YB_PG_SKIP_CONFIG_STATUS']
if not os.path.exists(compile_commands_path):
raise RuntimeError(
"Failed to generate compilation database at: %s" %
compile_commands_path)
pg_compile_commands_paths.append(compile_commands_path)
if self.export_compile_commands:
self.write_compile_commands_files(pg_compile_commands_paths)
def make_postgres(self):
self.set_env_vars('make')
make_cmd = ['make']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_parallelism = int(make_parallelism)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = cpu_count
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
os.environ['YB_COMPILER_TYPE'] = self.compiler_type
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
env_script_content = ''
for env_var_name in [
'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS',
'CXXFLAGS', 'LDFLAGS', 'PATH'
]:
env_var_value = os.environ[env_var_name]
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
env_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
compile_commands_files = []
for work_dir in [
self.pg_build_root,
os.path.join(self.pg_build_root, 'contrib')
]:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write('#!/usr/bin/env bash\n'
'. "${BASH_SOURCE%/*}"/env.sh\n'
'make "$@"\n')
with open('env.sh', 'w') as out_f:
out_f.write(env_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
make_result = run_program(make_cmd)
write_program_output_to_file('make', make_result, work_dir)
make_install_result = run_program(['make', 'install'])
write_program_output_to_file('make_install',
make_install_result, work_dir)
logging.info("Successfully ran make in the %s directory",
work_dir)
if self.export_compile_commands:
logging.info(
"Generating the compilation database in directory '%s'",
work_dir)
compile_commands_path = os.path.join(
work_dir, 'compile_commands.json')
os.environ['YB_PG_SKIP_CONFIG_STATUS'] = '1'
if (not os.path.exists(compile_commands_path)
or not self.export_compile_commands_lazily):
run_program(['compiledb', 'make', '-n'],
capture_output=False)
del os.environ['YB_PG_SKIP_CONFIG_STATUS']
if not os.path.exists(compile_commands_path):
raise RuntimeError(
"Failed to generate compilation database at: %s" %
compile_commands_path)
compile_commands_files.append(compile_commands_path)
if self.export_compile_commands:
self.combine_compile_commands(compile_commands_files)
