def test_quote_for_bash(self): for s in [ "\\" * 1, "\\" * 2, "\\" * 3, "\\" * 4, "\\" * 5, "\\" * 6, '', 'a', 'a b' '"foo"' 'foo"bar', 'foo " bar', "foo'bar", '$foo $bar', '"' + "'", '"' + "'" + '"' + "'", r"""'\''""", 'foo bar', ' ', 'foo ', ' foo', ' foo bar ' ';', ' ;', '; ', ' ; ' '.', '*', ' .', '. ', '* ', ' *' ]: quoted_s = quote_for_bash(s) tmp_file_path = None try: with tempfile.NamedTemporaryFile(suffix='.sh', delete=False) as tmp_file: cmd = 'echo -n ' + quoted_s tmp_file.write(cmd.encode('utf-8')) tmp_file_path = tmp_file.name result = subprocess.check_output(['bash', tmp_file_path ]).decode('utf-8') self.assertEqual( s, result, "For input string: [[ {} ]], quote_for_bash produced [[ {} ]], " "but echo -n with that argument returned: [[ {} ]]".format( s, quoted_s, result)) finally: if False and tmp_file_path and os.path.exists(tmp_file_path): os.remove(tmp_file_path)
def make_postgres(self): self.set_env_vars('make') make_cmd = ['make'] make_parallelism = os.environ.get('YB_MAKE_PARALLELISM') if make_parallelism: make_parallelism = int(make_parallelism) if self.build_uses_remote_compilation and not self.remote_compilation_allowed: # Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is # likely specified for distributed compilation, cap it at some factor times the number # of CPU cores. parallelism_cap = multiprocessing.cpu_count() * 2 if make_parallelism: make_parallelism = min(parallelism_cap, make_parallelism) else: make_parallelism = cpu_count if make_parallelism: make_cmd += ['-j', str(int(make_parallelism))] os.environ['YB_COMPILER_TYPE'] = self.compiler_type # Create a script allowing to easily run "make" from the build directory with the right # environment. make_script_content = "#!/usr/bin/env bash\n" for env_var_name in [ 'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'PATH']: env_var_value = os.environ[env_var_name] if env_var_value is None: raise RuntimeError("Expected env var %s to be set" % env_var_name) make_script_content += "export %s=%s\n" % (env_var_name, quote_for_bash(env_var_value)) make_script_content += 'make "$@"\n' for work_dir in [self.pg_build_root, os.path.join(self.pg_build_root, 'contrib')]: with WorkDirContext(work_dir): # Create a script to run Make easily with the right environment. make_script_path = 'make.sh' with open(make_script_path, 'w') as out_f: out_f.write(make_script_content) run_program(['chmod', 'u+x', make_script_path]) # Actually run Make. if is_verbose_mode(): logging.info("Running make in the %s directory", work_dir) make_result = run_program(make_cmd) write_program_output_to_file('make', make_result, work_dir) make_install_result = run_program(['make', 'install']) write_program_output_to_file('make_install', make_install_result, work_dir) logging.info("Successfully ran make in the %s directory", work_dir)
def make_postgres(self): self.set_env_vars('make') make_cmd = ['make'] make_parallelism = os.environ.get('YB_MAKE_PARALLELISM') if make_parallelism: make_cmd += ['-j', str(int(make_parallelism))] os.environ['YB_COMPILER_TYPE'] = self.compiler_type # Create a script allowing to easily run "make" from the build directory with the right # environment. make_script_content = "#!/usr/bin/env bash\n" for env_var_name in [ 'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'PATH' ]: env_var_value = os.environ[env_var_name] if env_var_value is None: raise RuntimeError("Expected env var %s to be set" % env_var_name) make_script_content += "export %s=%s\n" % ( env_var_name, quote_for_bash(env_var_value)) make_script_content += 'make "$@"\n' for work_dir in [ self.pg_build_root, os.path.join(self.pg_build_root, 'contrib') ]: with WorkDirContext(work_dir): # Create a script to run Make easily with the right environment. make_script_path = 'make.sh' with open(make_script_path, 'w') as out_f: out_f.write(make_script_content) run_program(['chmod', 'u+x', make_script_path]) # Actually run Make. if is_verbose_mode(): logging.info("Running make in the %s directory", work_dir) make_result = run_program(make_cmd) write_program_output_to_file('make', make_result, work_dir) make_install_result = run_program(['make', 'install']) write_program_output_to_file('make_install', make_install_result, work_dir) logging.info("Successfully ran make in the %s directory", work_dir)
def make_postgres(self): self.set_env_vars('make') make_cmd = ['make'] make_parallelism = os.environ.get('YB_MAKE_PARALLELISM') if make_parallelism: make_cmd += ['-j', str(int(make_parallelism))] # Create a script allowing to easily run "make" from the build directory with the right # environment. make_script_content = "#!/usr/bin/env bash\n" for env_var_name in [ 'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'PATH']: env_var_value = os.environ[env_var_name] if env_var_value is None: raise RuntimeError("Expected env var %s to be set" % env_var_name) make_script_content += "export %s=%s\n" % (env_var_name, quote_for_bash(env_var_value)) make_script_content += 'make "$@"\n' for work_dir in [self.pg_build_root, os.path.join(self.pg_build_root, 'contrib')]: with WorkDirContext(work_dir): # Create a script to run Make easily with the right environment. make_script_path = 'make.sh' with open(make_script_path, 'w') as out_f: out_f.write(make_script_content) run_program(['chmod', 'u+x', make_script_path]) # Actually run Make. if is_verbose_mode(): logging.info("Running make in the %s directory", work_dir) make_result = run_program(make_cmd) write_program_output_to_file('make', make_result, work_dir) make_install_result = run_program(['make', 'install']) write_program_output_to_file('make_install', make_install_result, work_dir) logging.info("Successfully ran make in the %s directory", work_dir)
def make_postgres(self): self.set_env_vars('make') # Postgresql requires MAKELEVEL to be 0 or non-set when calling its make. # But in case YB project is built with make, MAKELEVEL is not 0 at this point. make_cmd = ['make', 'MAKELEVEL=0'] make_parallelism = os.environ.get('YB_MAKE_PARALLELISM') if make_parallelism: make_parallelism = int(make_parallelism) if self.build_uses_remote_compilation and not self.remote_compilation_allowed: # Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is # likely specified for distributed compilation, cap it at some factor times the number # of CPU cores. parallelism_cap = multiprocessing.cpu_count() * 2 if make_parallelism: make_parallelism = min(parallelism_cap, make_parallelism) else: make_parallelism = cpu_count if make_parallelism: make_cmd += ['-j', str(int(make_parallelism))] self.set_env_var('YB_COMPILER_TYPE', self.compiler_type) # Create a script allowing to easily run "make" from the build directory with the right # environment. env_script_content = '' for env_var_name in CONFIG_ENV_VARS: env_var_value = os.environ.get(env_var_name) if env_var_value is None: raise RuntimeError("Expected env var %s to be set" % env_var_name) env_script_content += "export %s=%s\n" % ( env_var_name, quote_for_bash(env_var_value)) compile_commands_files = [] work_dirs = [self.pg_build_root] if self.build_type != 'compilecmds': work_dirs.append(os.path.join(self.pg_build_root, 'contrib')) for work_dir in work_dirs: with WorkDirContext(work_dir): # Create a script to run Make easily with the right environment. if self.should_build: make_script_path = 'make.sh' with open(make_script_path, 'w') as out_f: out_f.write('#!/usr/bin/env bash\n' '. "${BASH_SOURCE%/*}"/env.sh\n' 'make "$@"\n') with open('env.sh', 'w') as out_f: out_f.write(env_script_content) run_program(['chmod', 'u+x', make_script_path]) # Actually run Make. if is_verbose_mode(): logging.info("Running make in the %s directory", work_dir) run_program(make_cmd, stdout_stderr_prefix='make', cwd=work_dir, shell=True, error_ok=True ).print_output_and_raise_error_if_failed() if self.build_type == 'compilecmds': logging.info( "Not running make install in the %s directory since we are only " "generating the compilation database", work_dir) else: run_program('make install', stdout_stderr_prefix='make_install', cwd=work_dir, shell=True, error_ok=True ).print_output_and_raise_error_if_failed() logging.info("Successfully ran make in the %s directory", work_dir) if self.export_compile_commands: logging.info( "Generating the compilation database in directory '%s'", work_dir) compile_commands_path = os.path.join( work_dir, 'compile_commands.json') self.set_env_var('YB_PG_SKIP_CONFIG_STATUS', '1') if (not os.path.exists(compile_commands_path) or not self.export_compile_commands_lazily): run_program(['compiledb', 'make', '-n'], capture_output=False) del os.environ['YB_PG_SKIP_CONFIG_STATUS'] if not os.path.exists(compile_commands_path): raise RuntimeError( "Failed to generate compilation database at: %s" % compile_commands_path) compile_commands_files.append(compile_commands_path) if self.export_compile_commands: self.combine_compile_commands(compile_commands_files)
def make_postgres(self) -> None: self.set_env_vars('make') # Postgresql requires MAKELEVEL to be 0 or non-set when calling its make. # But in case YB project is built with make, MAKELEVEL is not 0 at this point. make_cmd = ['make', 'MAKELEVEL=0'] if is_macos_arm64(): make_cmd = ['arch', '-arm64'] + make_cmd make_parallelism_str: Optional[str] = os.environ.get( 'YB_MAKE_PARALLELISM') make_parallelism: Optional[int] = None if make_parallelism_str is not None: make_parallelism = int(make_parallelism_str) if self.build_uses_remote_compilation and not self.remote_compilation_allowed: # Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is # likely specified for distributed compilation, cap it at some factor times the number # of CPU cores. parallelism_cap = multiprocessing.cpu_count() * 2 if make_parallelism: make_parallelism = min(parallelism_cap, make_parallelism) else: make_parallelism = parallelism_cap if make_parallelism: make_cmd += ['-j', str(int(make_parallelism))] self.set_env_var('YB_COMPILER_TYPE', self.compiler_type) # Create a script allowing to easily run "make" from the build directory with the right # environment. env_script_content = '' for env_var_name in CONFIG_ENV_VARS: env_var_value = os.environ.get(env_var_name) if env_var_value is None: raise RuntimeError("Expected env var %s to be set" % env_var_name) env_script_content += "export %s=%s\n" % ( env_var_name, quote_for_bash(env_var_value)) pg_compile_commands_paths = [] third_party_extensions_dir = os.path.join(self.pg_build_root, 'third-party-extensions') work_dirs = [ self.pg_build_root, os.path.join(self.pg_build_root, 'contrib'), third_party_extensions_dir ] for work_dir in work_dirs: with WorkDirContext(work_dir): # Create a script to run Make easily with the right environment. make_script_path = 'make.sh' with open(make_script_path, 'w') as out_f: out_f.write('#!/usr/bin/env bash\n' '. "${BASH_SOURCE%/*}"/env.sh\n' 'make "$@"\n') with open('env.sh', 'w') as out_f: out_f.write(env_script_content) run_program(['chmod', 'u+x', make_script_path]) make_cmd_suffix = [] if work_dir == third_party_extensions_dir: make_cmd_suffix = ['PG_CONFIG=' + self.pg_config_path] # Actually run Make. if is_verbose_mode(): logging.info("Running make in the %s directory", work_dir) complete_make_cmd = make_cmd + make_cmd_suffix complete_make_cmd_str = shlex_join(complete_make_cmd) complete_make_install_cmd = make_cmd + ['install' ] + make_cmd_suffix attempt = 0 while attempt <= TRANSIENT_BUILD_RETRIES: attempt += 1 make_result = run_program( complete_make_cmd_str, stdout_stderr_prefix='make', cwd=work_dir, error_ok=True, shell=True # TODO: get rid of shell=True. ) if make_result.failure(): transient_err = False stderr_lines = make_result.get_stderr().split('\n') for line in stderr_lines: if any(transient_error_pattern in line for transient_error_pattern in TRANSIENT_BUILD_ERRORS): transient_err = True logging.info(f'Transient error: {line}') break if transient_err: logging.info( f"Transient error during build attempt {attempt}. " f"Re-trying make command: {complete_make_cmd_str}." ) else: make_result.print_output_to_stdout() raise RuntimeError("PostgreSQL compilation failed") else: logging.info( "Successfully ran 'make' in the %s directory", work_dir) break # No error, break out of retry loop else: raise RuntimeError( f"Maximum build attempts reached ({TRANSIENT_BUILD_RETRIES} attempts)." ) if self.build_type != 'compilecmds' or work_dir == self.pg_build_root: run_program( ' '.join( shlex.quote(arg) for arg in complete_make_install_cmd), stdout_stderr_prefix='make_install', cwd=work_dir, error_ok=True, shell=True # TODO: get rid of shell=True. ).print_output_and_raise_error_if_failed() logging.info( "Successfully ran 'make install' in the %s directory", work_dir) else: logging.info( "Not running 'make install' in the %s directory since we are only " "generating the compilation database", work_dir) if self.export_compile_commands: logging.info( "Generating the compilation database in directory '%s'", work_dir) compile_commands_path = os.path.join( work_dir, 'compile_commands.json') self.set_env_var('YB_PG_SKIP_CONFIG_STATUS', '1') if not os.path.exists(compile_commands_path): run_program(['compiledb', 'make', '-n'] + make_cmd_suffix, capture_output=False) del os.environ['YB_PG_SKIP_CONFIG_STATUS'] if not os.path.exists(compile_commands_path): raise RuntimeError( "Failed to generate compilation database at: %s" % compile_commands_path) pg_compile_commands_paths.append(compile_commands_path) if self.export_compile_commands: self.write_compile_commands_files(pg_compile_commands_paths)
def make_postgres(self): self.set_env_vars('make') make_cmd = ['make'] make_parallelism = os.environ.get('YB_MAKE_PARALLELISM') if make_parallelism: make_parallelism = int(make_parallelism) if self.build_uses_remote_compilation and not self.remote_compilation_allowed: # Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is # likely specified for distributed compilation, cap it at some factor times the number # of CPU cores. parallelism_cap = multiprocessing.cpu_count() * 2 if make_parallelism: make_parallelism = min(parallelism_cap, make_parallelism) else: make_parallelism = cpu_count if make_parallelism: make_cmd += ['-j', str(int(make_parallelism))] os.environ['YB_COMPILER_TYPE'] = self.compiler_type # Create a script allowing to easily run "make" from the build directory with the right # environment. env_script_content = '' for env_var_name in [ 'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'PATH' ]: env_var_value = os.environ[env_var_name] if env_var_value is None: raise RuntimeError("Expected env var %s to be set" % env_var_name) env_script_content += "export %s=%s\n" % ( env_var_name, quote_for_bash(env_var_value)) compile_commands_files = [] for work_dir in [ self.pg_build_root, os.path.join(self.pg_build_root, 'contrib') ]: with WorkDirContext(work_dir): # Create a script to run Make easily with the right environment. make_script_path = 'make.sh' with open(make_script_path, 'w') as out_f: out_f.write('#!/usr/bin/env bash\n' '. "${BASH_SOURCE%/*}"/env.sh\n' 'make "$@"\n') with open('env.sh', 'w') as out_f: out_f.write(env_script_content) run_program(['chmod', 'u+x', make_script_path]) # Actually run Make. if is_verbose_mode(): logging.info("Running make in the %s directory", work_dir) make_result = run_program(make_cmd) write_program_output_to_file('make', make_result, work_dir) make_install_result = run_program(['make', 'install']) write_program_output_to_file('make_install', make_install_result, work_dir) logging.info("Successfully ran make in the %s directory", work_dir) if self.export_compile_commands: logging.info( "Generating the compilation database in directory '%s'", work_dir) compile_commands_path = os.path.join( work_dir, 'compile_commands.json') os.environ['YB_PG_SKIP_CONFIG_STATUS'] = '1' if (not os.path.exists(compile_commands_path) or not self.export_compile_commands_lazily): run_program(['compiledb', 'make', '-n'], capture_output=False) del os.environ['YB_PG_SKIP_CONFIG_STATUS'] if not os.path.exists(compile_commands_path): raise RuntimeError( "Failed to generate compilation database at: %s" % compile_commands_path) compile_commands_files.append(compile_commands_path) if self.export_compile_commands: self.combine_compile_commands(compile_commands_files)