def parse_ninja_metadata(self) -> None:
with WorkDirContext(self.conf.build_root):
ninja_path = os.environ.get('YB_NINJA_PATH', 'ninja')
logging.info("Ninja executable path: %s", ninja_path)
logging.info("Running 'ninja -t commands'")
subprocess.check_call('{} -t commands >ninja_commands.txt'.format(
pipes.quote(ninja_path)),
shell=True)
logging.info(
"Parsing the output of 'ninja -t commands' for linker commands"
)
start_time_sec = time.time()
self.parse_link_txt_file('ninja_commands.txt')
logging.info("Parsing linker commands took %.1f seconds",
time.time() - start_time_sec)
logging.info("Running 'ninja -t deps'")
subprocess.check_call('{} -t deps >ninja_deps.txt'.format(
pipes.quote(ninja_path)),
shell=True)
start_time = time.time()
logging.info(
"Parsing the output of 'ninja -t deps' to infer dependencies")
logging.info("Parsing dependencies took %.1f seconds",
time.time() - start_time_sec)
self.parse_depend_file('ninja_deps.txt')
def build_postgres(self):
if self.args.clean:
self.clean_postgres()
mkdir_p(self.pg_build_root)
self.set_env_vars('configure')
saved_build_stamp = self.get_saved_build_stamp()
initial_build_stamp = self.get_build_stamp(include_env_vars=True)
initial_build_stamp_no_env = self.get_build_stamp(include_env_vars=False)
logging.info("PostgreSQL build stamp:\n%s", initial_build_stamp)
if initial_build_stamp == saved_build_stamp:
logging.info(
"PostgreSQL is already up-to-date in directory %s, not rebuilding.",
self.pg_build_root)
return
with WorkDirContext(self.pg_build_root):
self.sync_postgres_source()
if os.environ.get('YB_PG_SKIP_CONFIGURE', '0') != '1':
self.configure_postgres()
self.make_postgres()
final_build_stamp_no_env = self.get_build_stamp(include_env_vars=False)
if final_build_stamp_no_env == initial_build_stamp_no_env:
logging.info("Updating build stamp file at %s", self.build_stamp_path)
with open(self.build_stamp_path, 'w') as build_stamp_file:
build_stamp_file.write(initial_build_stamp)
else:
logging.warning("PostgreSQL build stamp changed during the build! Not updating.")
def link_whole_program(dep_graph: DependencyGraph,
initial_nodes: Iterable[Node],
link_cmd_out_file: Optional[str], run_linker: bool,
lto_output_suffix: Optional[str],
lto_type: str) -> None:
initial_node_list = list(initial_nodes)
assert len(initial_node_list) == 1
initial_node = initial_node_list[0]
link_helper = LinkHelper(dep_graph=dep_graph,
initial_node=initial_node,
lto_output_suffix=lto_output_suffix)
# We stop recursive traversal at executables because those are just code generators
# (protoc-gen-insertions, protoc-gen-yrpc, bfql_codegen, bfpg_codegen).
conf = dep_graph.conf
link_helper.consume_original_link_cmd()
link_helper.add_leaf_object_files()
link_helper.add_final_args(lto_type=lto_type)
with WorkDirContext(conf.build_root):
if link_cmd_out_file:
link_helper.write_link_cmd_file(link_cmd_out_file)
if not run_linker:
return
link_helper.run_linker()
def add_leaf_object_files(self) -> None:
"""
Goes over all the object files that the original node transitively depends on, and adds
them to the link command if they have not already been added.
"""
transitive_deps = self.initial_node.get_recursive_deps(
skip_node_types=set([NodeType.EXECUTABLE]))
with WorkDirContext(self.build_root):
# Sort nodes by path for determinism.
for node in sorted(list(transitive_deps),
key=lambda dep: dep.path):
if node in self.obj_file_graph_nodes:
# Dedup .cc.o files already existing on the command line.
continue
if node.node_type == NodeType.OBJECT:
self.new_args.add_new_arg(node.path)
for arg in self.yb_pgbackend_link_cmd:
if arg.endswith('.o'):
if os.path.basename(arg) == 'main_cpp_wrapper.cc.o':
# TOOD: why is this file even linked into libyb_pgbackend?
continue
self.new_args.append(
os.path.join(self.pg_backend_build_dir, arg))
continue
if (arg.startswith('-l') and not self.new_args.contains(arg)
and not arg.startswith('-lyb_')):
self.process_arg(arg)
def get_build_stamp(self, include_env_vars):
"""
Creates a "build stamp" that tries to capture all inputs that might affect the PostgreSQL
code. This is needed to avoid needlessly rebuilding PostgreSQL, as it takes ~10 seconds
even if there are no code changes.
"""
with WorkDirContext(YB_SRC_ROOT):
code_subset = [
'src/postgres', 'src/yb/yql/pggate',
'python/yb/build_postgres.py', 'build-support/build_postgres',
'CMakeLists.txt'
]
git_hash = subprocess.check_output(
['git', '--no-pager', 'log', '-n', '1', '--pretty=%H'] +
code_subset).strip()
git_diff = subprocess.check_output(['git', 'diff'] + code_subset)
git_diff_cached = subprocess.check_output(
['git', 'diff', '--cached'] + code_subset)
env_vars_str = self.get_env_vars_str(self.env_vars_for_build_stamp)
build_stamp = "\n".join([
"git_commit_sha1=%s" % git_hash,
"git_diff_sha256=%s" % sha256(git_diff),
"git_diff_cached_sha256=%s" % sha256(git_diff_cached)
])
if include_env_vars:
build_stamp += "\nenv_vars_sha256=%s" % hashlib.sha256(
env_vars_str).hexdigest()
return build_stamp.strip()
def test_work_dir_context(self):
old_work_dir = os.getcwd()
for d in ['/tmp', os.path.expanduser('~')]:
with WorkDirContext(d):
self.assertEquals(d, os.getcwd())
self.assertEquals(d, run_program('pwd').stdout.strip())
self.assertEquals(old_work_dir, os.getcwd())
def build_postgres(self):
start_time_sec = time.time()
if self.args.clean:
self.clean_postgres()
mkdir_p(self.pg_build_root)
self.set_env_vars('configure')
saved_build_stamp = self.get_saved_build_stamp()
initial_build_stamp = self.get_build_stamp(include_env_vars=True)
initial_build_stamp_no_env = self.get_build_stamp(
include_env_vars=False)
logging.info("PostgreSQL build stamp:\n%s", initial_build_stamp)
if initial_build_stamp == saved_build_stamp:
if self.export_compile_commands:
logging.info(
"Even though PostgreSQL is already up-to-date in directory %s, we still need "
"to create compile_commands.json, so proceeding with %s",
self.pg_build_root, self.steps_description())
else:
logging.info(
"PostgreSQL is already up-to-date in directory %s, skipping %s.",
self.pg_build_root, self.steps_description())
return
with WorkDirContext(self.pg_build_root):
if self.should_configure:
self.sync_postgres_source()
configure_start_time_sec = time.time()
self.configure_postgres()
logging.info(
"The configure step of building PostgreSQL took %.1f sec",
time.time() - configure_start_time_sec)
if self.should_make:
make_start_time_sec = time.time()
self.make_postgres()
logging.info(
"The make step of building PostgreSQL took %.1f sec",
time.time() - make_start_time_sec)
if self.should_make:
# Guard against the code having changed while we were building it.
final_build_stamp_no_env = self.get_build_stamp(
include_env_vars=False)
if final_build_stamp_no_env == initial_build_stamp_no_env:
logging.info("Updating build stamp file at %s",
self.build_stamp_path)
with open(self.build_stamp_path, 'w') as build_stamp_file:
build_stamp_file.write(initial_build_stamp)
else:
logging.warning(
"PostgreSQL build stamp changed during the build! Not updating."
)
logging.info("PostgreSQL build (%s) took %.1f sec",
self.steps_description(),
time.time() - start_time_sec)
def build_postgres(self):
if self.args.clean:
self.clean_postgres()
mkdir_p(self.pg_build_root)
with WorkDirContext(self.pg_build_root):
self.sync_postgres_source()
self.configure_postgres()
self.make_postgres()
def run_linker(self) -> None:
with WorkDirContext(self.build_root):
start_time_sec = time.time()
logging.info("Running linker")
try:
subprocess.check_call(self.new_args.args)
except subprocess.CalledProcessError as ex:
# Avoid printing the extremely long command line.
logging.error("Linker returned exit code %d", ex.returncode)
elapsed_time_sec = time.time() - start_time_sec
logging.info("Linking finished in %.1f sec", elapsed_time_sec)
def get_build_stamp(self, include_env_vars: bool) -> str:
"""
Creates a "build stamp" that tries to capture all inputs that might affect the PostgreSQL
code. This is needed to avoid needlessly rebuilding PostgreSQL, as it takes ~10 seconds
even if there are no code changes.
"""
with WorkDirContext(YB_SRC_ROOT):
# Postgres files.
pathspec = [
'src/postgres',
'src/yb/yql/pggate',
'python/yb/build_postgres.py',
'build-support/build_postgres',
'CMakeLists.txt',
]
git_version = self.get_git_version()
if git_version and git_version >= semantic_version.Version(
'1.9.0'):
# Git version 1.8.5 allows specifying glob pathspec, and Git version 1.9.0 allows
# specifying negative pathspec. Use them to exclude changes to regress test files
# not needed for build.
pathspec.extend([
':(glob,exclude)src/postgres/**/*_schedule',
':(glob,exclude)src/postgres/**/data/*.csv',
':(glob,exclude)src/postgres/**/data/*.data',
':(glob,exclude)src/postgres/**/expected/*.out',
':(glob,exclude)src/postgres/**/input/*.source',
':(glob,exclude)src/postgres/**/output/*.source',
':(glob,exclude)src/postgres/**/specs/*.spec',
':(glob,exclude)src/postgres/**/sql/*.sql',
':(glob,exclude)src/postgres/.clang-format',
':(glob,exclude)src/postgres/src/test/regress/README',
':(glob,exclude)src/postgres/src/test/regress/yb_lint_regress_schedule.sh',
])
# Get the most recent commit that touched postgres files.
git_hash = subprocess.check_output(
['git', '--no-pager', 'log', '-n', '1', '--format=%H', '--'] +
pathspec).decode('utf-8').strip()
# Get uncommitted changes to tracked postgres files.
git_diff = subprocess.check_output(['git', 'diff', 'HEAD', '--'] +
pathspec)
env_vars_str = self.get_env_vars_str(self.env_vars_for_build_stamp)
build_stamp = "\n".join([
"git_commit_sha1=%s" % git_hash,
"git_diff_sha256=%s" % hashlib.sha256(git_diff).hexdigest(),
])
if include_env_vars:
build_stamp += "\nenv_vars_sha256=%s" % hashlib.sha256(
env_vars_str.encode('utf-8')).hexdigest()
return build_stamp.strip()
def make_postgres(self):
self.set_env_vars('make')
make_cmd = ['make']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_parallelism = int(make_parallelism)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = cpu_count
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
os.environ['YB_COMPILER_TYPE'] = self.compiler_type
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
make_script_content = "#!/usr/bin/env bash\n"
for env_var_name in [
'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS',
'PATH']:
env_var_value = os.environ[env_var_name]
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" % env_var_name)
make_script_content += "export %s=%s\n" % (env_var_name, quote_for_bash(env_var_value))
make_script_content += 'make "$@"\n'
for work_dir in [self.pg_build_root, os.path.join(self.pg_build_root, 'contrib')]:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write(make_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
make_result = run_program(make_cmd)
write_program_output_to_file('make', make_result, work_dir)
make_install_result = run_program(['make', 'install'])
write_program_output_to_file('make_install', make_install_result, work_dir)
logging.info("Successfully ran make in the %s directory", work_dir)
def main():
if os.environ.get('YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT', '0') == '1':
logging.info(
'YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT is set, skipping initdb')
return
build_root = os.environ['YB_BUILD_ROOT']
tool_name = 'create_initial_sys_catalog_snapshot'
tool_path = os.path.join(build_root, 'tests-pgwrapper', tool_name)
snapshot_dest_path = os.path.join(build_root, 'share',
'initial_sys_catalog_snapshot')
file_to_check = os.path.join(snapshot_dest_path,
'exported_tablet_metadata_changes')
if (os.path.exists(file_to_check) and os.environ.get(
'YB_RECREATE_INITIAL_SYS_CATALOG_SNAPSHOT', '') != '1'):
logging.info(
"Initial sys catalog snapshot already exists, not re-creating: %s",
snapshot_dest_path)
return
if os.path.exists(snapshot_dest_path):
logging.info("Removing initial sys catalog snapshot data at: %s",
snapshot_dest_path)
shutil.rmtree(snapshot_dest_path)
mkdir_p(os.path.dirname(snapshot_dest_path))
start_time_sec = time.time()
logging.info("Starting creating initial system catalog snapshot data")
logging.info("Logging to: %s", os.path.join(build_root,
tool_name + '.err'))
os.environ['YB_EXTRA_GTEST_FLAGS'] = (
'--initial_sys_catalog_snapshot_dest_path=' + snapshot_dest_path)
os.environ['YB_CTEST_VERBOSE'] = '1'
with WorkDirContext(build_root):
initdb_result = run_program([
os.path.join(YB_SRC_ROOT, 'build-support', 'run-test.sh'),
tool_path,
],
stdout_stderr_prefix=tool_name,
shell=True,
error_ok=True)
elapsed_time_sec = time.time() - start_time_sec
if initdb_result.failure():
initdb_result.print_output_to_stdout()
raise RuntimeError("initdb failed in %.1f sec" % elapsed_time_sec)
logging.info(
"Initial system catalog snapshot data creation took %1.f sec. Wrote data to: %s",
elapsed_time_sec, snapshot_dest_path)
def make_postgres(self):
self.set_env_vars('make')
make_cmd = ['make']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
os.environ['YB_COMPILER_TYPE'] = self.compiler_type
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
make_script_content = "#!/usr/bin/env bash\n"
for env_var_name in [
'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS',
'CXXFLAGS', 'LDFLAGS', 'PATH'
]:
env_var_value = os.environ[env_var_name]
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
make_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
make_script_content += 'make "$@"\n'
for work_dir in [
self.pg_build_root,
os.path.join(self.pg_build_root, 'contrib')
]:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write(make_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
make_result = run_program(make_cmd)
write_program_output_to_file('make', make_result, work_dir)
make_install_result = run_program(['make', 'install'])
write_program_output_to_file('make_install',
make_install_result, work_dir)
logging.info("Successfully ran make in the %s directory",
work_dir)
def build_postgres(self):
start_time_sec = time.time()
if self.args.clean:
self.clean_postgres()
mkdir_p(self.pg_build_root)
self.set_env_vars('configure')
saved_build_stamp = self.get_saved_build_stamp()
initial_build_stamp = self.get_build_stamp(include_env_vars=True)
initial_build_stamp_no_env = self.get_build_stamp(include_env_vars=False)
logging.info("PostgreSQL build stamp:\n%s", initial_build_stamp)
if initial_build_stamp == saved_build_stamp:
logging.info(
"PostgreSQL is already up-to-date in directory %s, not rebuilding.",
self.pg_build_root)
if self.export_compile_commands:
self.should_build = False
logging.info("Still need to create compile_commands.json, proceeding.")
else:
return
with WorkDirContext(self.pg_build_root):
if self.should_build:
self.sync_postgres_source()
if os.environ.get('YB_PG_SKIP_CONFIGURE', '0') != '1':
configure_start_time_sec = time.time()
self.configure_postgres()
logging.info("The configure step of building PostgreSQL took %.1f sec",
time.time() - configure_start_time_sec)
make_start_time_sec = time.time()
self.make_postgres()
logging.info("The make step of building PostgreSQL took %.1f sec",
time.time() - make_start_time_sec)
final_build_stamp_no_env = self.get_build_stamp(include_env_vars=False)
if final_build_stamp_no_env == initial_build_stamp_no_env:
logging.info("Updating build stamp file at %s", self.build_stamp_path)
with open(self.build_stamp_path, 'w') as build_stamp_file:
build_stamp_file.write(initial_build_stamp)
else:
logging.warning("PostgreSQL build stamp changed during the build! Not updating.")
logging.info("PostgreSQL build took %.1f sec", time.time() - start_time_sec)
def add_final_args(self) -> None:
self.new_args.extend([
'-L%s' % os.path.join(self.build_root, 'postgres', 'lib'),
'-l:libpgcommon.a',
'-l:libpgport.a',
'-l:libpq.a',
'-fwhole-program',
'-Wl,-v',
'-nostdlib++',
'-flto=full',
])
for lib_name in ['libc++.a', 'libc++abi.a']:
self.new_args.append(
os.path.join(self.thirdparty_path, 'installed',
'uninstrumented', 'libcxx', 'lib', lib_name))
with WorkDirContext(self.build_root):
self.write_link_cmd_file(self.final_output_name +
'_lto_link_cmd_args.txt')
def consume_original_link_cmd(self) -> None:
"""
Goes over the original linker command and reuses some of its arguments for the new command.
"""
with WorkDirContext(self.build_root):
expect_output_name = False
output_name: Optional[str] = None
for arg in self.original_link_args:
if arg == '-o':
expect_output_name = True
continue
if expect_output_name:
if output_name:
raise ValueError(
"Found multiple output names in the original link command: "
"%s and %s" % (output_name, arg))
output_name = arg
expect_output_name = False
continue
expect_output_name = False
if is_yb_library(arg):
logging.info("Skipping YB library: %s", arg)
continue
if arg.endswith('.cc.o'):
# E.g. tablet_server_main.cc.o.
# Remember this node for later deduplication.
self.obj_file_graph_nodes.add(
self.dep_graph.find_node(os.path.realpath(arg)))
self.process_arg(arg)
if not output_name:
raise ValueError(
"Did not find an output name in the original link command")
self.final_output_name = os.path.abspath(output_name)
logging.info("Final output file name: %s", self.final_output_name)
if self.lto_output_suffix is not None:
self.final_output_name += self.lto_output_suffix
self.new_args.extend(['-o', self.final_output_name])
def add_final_args(self, lto_type: str) -> None:
assert lto_type in ['full', 'thin']
for static_lib_path in sorted(self.static_libs_from_ldd):
if not self.new_args.contains(static_lib_path):
logging.info(
"Adding a static library determined using shared library dependencies: %s "
"(needed by: %s)",
static_lib_path,
# The static_libs_from_ldd dictionary stores the set of shared library paths
# that caused us to add a particular static library dependency as the value
# corresponding to that static library's path in the key.
', '.join(
sorted(self.static_libs_from_ldd[static_lib_path])))
self.new_args.append(static_lib_path)
self.new_args.extend([
'-L%s' % os.path.join(self.build_root, 'postgres', 'lib'),
'-l:libpgcommon.a',
'-l:libpgport.a',
'-l:libpq.a',
'-fwhole-program',
'-Wl,-v',
'-nostdlib++',
# For __res_nsearch, ns_initparse, ns_parserr, ns_name_uncompress.
# See https://github.com/yugabyte/yugabyte-db/issues/12738 for details.
'-lresolv',
'-flto=' + lto_type,
])
for lib_name in LIBCXX_STATIC_LIB_NAMES:
self.new_args.append(
os.path.join(self.thirdparty_path, 'installed',
'uninstrumented', 'libcxx', 'lib', lib_name))
with WorkDirContext(self.build_root):
self.write_link_cmd_file(self.final_output_name +
'_lto_link_cmd_args.txt')
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--program_name', type=str, help="Program name")
parser.add_argument('--output_file_path',
type=str,
help="Path to output file")
args = parser.parse_args()
start_time_sec = time.time()
build_root = os.environ['YB_BUILD_ROOT']
with WorkDirContext(build_root):
content = run_program([
os.path.join(build_root, 'bin', args.program_name),
"--dump_flags_xml",
],
shell=True)
with open(args.output_file_path, 'w+', encoding='utf-8') as f:
f.write(content.stdout)
elapsed_time_sec = time.time() - start_time_sec
logging.info("Generated flags_metadata for %s in %.1f sec",
args.program_name, elapsed_time_sec)
def make_postgres(self):
self.set_env_vars('make')
# Postgresql requires MAKELEVEL to be 0 or non-set when calling its make.
# But in case YB project is built with make, MAKELEVEL is not 0 at this point.
make_cmd = ['make', 'MAKELEVEL=0']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_parallelism = int(make_parallelism)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = cpu_count
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
self.set_env_var('YB_COMPILER_TYPE', self.compiler_type)
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
env_script_content = ''
for env_var_name in CONFIG_ENV_VARS:
env_var_value = os.environ.get(env_var_name)
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
env_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
compile_commands_files = []
work_dirs = [self.pg_build_root]
if self.build_type != 'compilecmds':
work_dirs.append(os.path.join(self.pg_build_root, 'contrib'))
for work_dir in work_dirs:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
if self.should_build:
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write('#!/usr/bin/env bash\n'
'. "${BASH_SOURCE%/*}"/env.sh\n'
'make "$@"\n')
with open('env.sh', 'w') as out_f:
out_f.write(env_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory",
work_dir)
run_program(make_cmd,
stdout_stderr_prefix='make',
cwd=work_dir,
shell=True,
error_ok=True
).print_output_and_raise_error_if_failed()
if self.build_type == 'compilecmds':
logging.info(
"Not running make install in the %s directory since we are only "
"generating the compilation database", work_dir)
else:
run_program('make install',
stdout_stderr_prefix='make_install',
cwd=work_dir,
shell=True,
error_ok=True
).print_output_and_raise_error_if_failed()
logging.info("Successfully ran make in the %s directory",
work_dir)
if self.export_compile_commands:
logging.info(
"Generating the compilation database in directory '%s'",
work_dir)
compile_commands_path = os.path.join(
work_dir, 'compile_commands.json')
self.set_env_var('YB_PG_SKIP_CONFIG_STATUS', '1')
if (not os.path.exists(compile_commands_path)
or not self.export_compile_commands_lazily):
run_program(['compiledb', 'make', '-n'],
capture_output=False)
del os.environ['YB_PG_SKIP_CONFIG_STATUS']
if not os.path.exists(compile_commands_path):
raise RuntimeError(
"Failed to generate compilation database at: %s" %
compile_commands_path)
compile_commands_files.append(compile_commands_path)
if self.export_compile_commands:
self.combine_compile_commands(compile_commands_files)
def make_postgres(self):
self.set_env_vars('make')
make_cmd = ['make']
make_parallelism = os.environ.get('YB_MAKE_PARALLELISM')
if make_parallelism:
make_parallelism = int(make_parallelism)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = cpu_count
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
os.environ['YB_COMPILER_TYPE'] = self.compiler_type
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
env_script_content = ''
for env_var_name in [
'YB_SRC_ROOT', 'YB_BUILD_ROOT', 'YB_BUILD_TYPE', 'CFLAGS',
'CXXFLAGS', 'LDFLAGS', 'PATH'
]:
env_var_value = os.environ[env_var_name]
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
env_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
compile_commands_files = []
for work_dir in [
self.pg_build_root,
os.path.join(self.pg_build_root, 'contrib')
]:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write('#!/usr/bin/env bash\n'
'. "${BASH_SOURCE%/*}"/env.sh\n'
'make "$@"\n')
with open('env.sh', 'w') as out_f:
out_f.write(env_script_content)
run_program(['chmod', 'u+x', make_script_path])
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
make_result = run_program(make_cmd)
write_program_output_to_file('make', make_result, work_dir)
make_install_result = run_program(['make', 'install'])
write_program_output_to_file('make_install',
make_install_result, work_dir)
logging.info("Successfully ran make in the %s directory",
work_dir)
if self.export_compile_commands:
logging.info(
"Generating the compilation database in directory '%s'",
work_dir)
compile_commands_path = os.path.join(
work_dir, 'compile_commands.json')
os.environ['YB_PG_SKIP_CONFIG_STATUS'] = '1'
if (not os.path.exists(compile_commands_path)
or not self.export_compile_commands_lazily):
run_program(['compiledb', 'make', '-n'],
capture_output=False)
del os.environ['YB_PG_SKIP_CONFIG_STATUS']
if not os.path.exists(compile_commands_path):
raise RuntimeError(
"Failed to generate compilation database at: %s" %
compile_commands_path)
compile_commands_files.append(compile_commands_path)
if self.export_compile_commands:
self.combine_compile_commands(compile_commands_files)
def main() -> None:
parser = argparse.ArgumentParser(
description='A tool for working with the dependency graph')
parser.add_argument('--verbose',
action='store_true',
help='Enable debug output')
parser.add_argument(
'-r',
'--rebuild-graph',
action='store_true',
help='Rebuild the dependecy graph and save it to a file')
parser.add_argument('--node-type',
help='Node type to look for',
type=NodeType,
choices=list(NodeType),
default=NodeType.ANY)
parser.add_argument(
'--file-regex',
help='Regular expression for file names to select as initial nodes for '
'querying the dependency graph.')
parser.add_argument(
'--file-name-glob',
help='Like file-regex, but applies only to file name and uses the glob '
'syntax instead of regex.')
parser.add_argument(
'--git-diff',
help='Figure out the list of files to use as starting points in the '
'dependency graph traversal by diffing the current state of the code '
'against this commit. This could also be anything that could be '
'passed to "git diff" as a single argument.')
parser.add_argument(
'--git-commit',
help='Similar to --git-diff, but takes a git commit ref (e.g. sha1 or '
'branch) and uses the set of files from that commit.')
parser.add_argument(
'--build-root',
required=True,
help='E.g. <some_root>/build/debug-gcc-dynamic-community')
parser.add_argument('command',
type=Command,
choices=list(Command),
help='Command to perform')
parser.add_argument(
'--output-test-config',
help=
'Output a "test configuration file", which is a JSON containing the '
'resulting list of C++ tests to run to this file, a flag indicating '
'wheter to run Java tests or not, etc.')
parser.add_argument(
'--incomplete-build',
action='store_true',
help='Skip checking for file existence. Allows using the tool after '
'build artifacts have been deleted.')
parser.add_argument(
'--build-args',
help='Extra arguments to pass to yb_build.sh. The build is invoked e.g. '
'if the compilation database file is missing.')
parser.add_argument(
'--link-cmd-out-file',
help='For the %s command, write the linker arguments (one per line ) '
'to the given file.')
parser.add_argument(
'--lto-output-suffix',
default="-lto",
help='The suffix to append to LTO-enabled binaries produced by '
'the %s command' % Command.LINK_WHOLE_PROGRAM.value)
parser.add_argument(
'--run-linker',
help=
'Whether to actually run the linker. Setting this to false might be useful when '
'debugging, combined with --link-cmd-out-file.',
type=arg_str_to_bool,
default=True)
args = parser.parse_args()
if args.file_regex and args.file_name_glob:
raise RuntimeError(
'--file-regex and --file-name-glob are incompatible')
cmd = args.command
if (not args.file_regex and not args.file_name_glob
and not args.rebuild_graph and not args.git_diff
and not args.git_commit
and cmd not in COMMANDS_NOT_NEEDING_TARGET_SET):
raise RuntimeError(
"Neither of --file-regex, --file-name-glob, --git-{diff,commit}, or "
"--rebuild-graph are specified, and the command is not one of: " +
", ".join([cmd.value for cmd in COMMANDS_NOT_NEEDING_TARGET_SET]))
log_level = logging.INFO
logging.basicConfig(
level=log_level,
format=
"[%(filename)s:%(lineno)d] %(asctime)s %(levelname)s: %(message)s")
conf = DepGraphConf(verbose=args.verbose,
build_root=args.build_root,
incomplete_build=args.incomplete_build,
file_regex=args.file_regex,
file_name_glob=args.file_name_glob,
build_args=args.build_args)
if conf.file_regex and args.git_diff:
raise RuntimeError(
"--git-diff is incompatible with --file-{regex,name-glob}")
if args.git_diff and args.git_commit:
raise RuntimeError('--git-diff and --git-commit are incompatible')
if args.git_commit:
args.git_diff = "{}^..{}".format(args.git_commit, args.git_commit)
graph_cache_path = os.path.join(args.build_root, 'dependency_graph.json')
if args.rebuild_graph or not os.path.isfile(graph_cache_path):
logging.info(
"Generating a dependency graph at '{}'".format(graph_cache_path))
dep_graph_builder = DependencyGraphBuilder(conf)
dep_graph = dep_graph_builder.build()
dep_graph.save_as_json(graph_cache_path)
else:
start_time = datetime.now()
with open(graph_cache_path) as graph_input_file:
dep_graph = DependencyGraph(conf,
json_data=json.load(graph_input_file))
logging.info("Loaded dependency graph from '%s' in %.2f sec" %
(graph_cache_path,
(datetime.now() - start_time).total_seconds()))
dep_graph.validate_node_existence()
# ---------------------------------------------------------------------------------------------
# Commands that do not require an "initial set of targets"
# ---------------------------------------------------------------------------------------------
if cmd == Command.SELF_TEST:
run_self_test(dep_graph)
return
if cmd == Command.DEBUG_DUMP:
dep_graph.dump_debug_info()
return
# ---------------------------------------------------------------------------------------------
# Figure out the initial set of targets based on a git commit, a regex, etc.
# ---------------------------------------------------------------------------------------------
updated_categories: Set[SourceFileCategory] = set()
file_changes = []
initial_nodes: Iterable[Node]
if args.git_diff:
old_working_dir = os.getcwd()
with WorkDirContext(conf.yb_src_root):
git_diff_output = subprocess.check_output(
['git', 'diff', args.git_diff, '--name-only']).decode('utf-8')
initial_nodes = set()
file_paths = set()
for file_path in git_diff_output.split("\n"):
file_path = file_path.strip()
if not file_path:
continue
file_changes.append(file_path)
# It is important that we invoke os.path.realpath with the current directory set to
# the git repository root.
file_path = os.path.realpath(file_path)
file_paths.add(file_path)
node = dep_graph.node_by_path.get(file_path)
if node:
initial_nodes.add(node)
if not initial_nodes:
logging.warning(
"Did not find any graph nodes for this set of files: %s",
file_paths)
for basename in set(
[os.path.basename(file_path) for file_path in file_paths]):
logging.warning("Nodes for basename '{}': {}".format(
basename, dep_graph.find_nodes_by_basename(basename)))
elif conf.file_regex:
logging.info("Using file name regex: {}".format(conf.file_regex))
initial_nodes = dep_graph.find_nodes_by_regex(conf.file_regex)
if not initial_nodes:
logging.warning(
"Did not find any graph nodes for this pattern: %s",
conf.file_regex)
for node in initial_nodes:
file_changes.append(node.path)
else:
raise RuntimeError(
"Could not figure out how to generate the initial set of files")
file_changes = [(os.path.relpath(file_path, conf.yb_src_root)
if os.path.isabs(file_path) else file_path)
for file_path in file_changes]
if cmd == Command.LINK_WHOLE_PROGRAM:
link_whole_program(dep_graph=dep_graph,
initial_nodes=initial_nodes,
link_cmd_out_file=args.link_cmd_out_file,
run_linker=args.run_linker,
lto_output_suffix=args.lto_output_suffix)
return
file_changes_by_category: Dict[SourceFileCategory, List[str]] = group_by(
file_changes, get_file_category)
# Same as file_changes_by_category, but with string values of categories instead of enum
# elements.
file_changes_by_category_str: Dict[str, List[str]] = {}
for category, changes in file_changes_by_category.items():
logging.info("File changes in category '%s':", category)
for change in sorted(changes):
logging.info(" %s", change)
file_changes_by_category_str[category.value] = changes
updated_categories = set(file_changes_by_category.keys())
results: Set[Node] = set()
if cmd == Command.AFFECTED:
results = dep_graph.find_affected_nodes(set(initial_nodes),
args.node_type)
elif cmd == Command.DEPS:
for node in initial_nodes:
results.update(node.deps)
elif cmd == Command.REVERSE_DEPS:
for node in initial_nodes:
results.update(node.reverse_deps)
else:
raise ValueError("Unimplemented command '{}'".format(cmd))
if args.output_test_config:
test_basename_list = sorted([
os.path.basename(node.path) for node in results
if node.node_type == NodeType.TEST
])
affected_basenames = set(
[os.path.basename(node.path) for node in results])
# These are ALL tests, not just tests affected by the changes in question, used mostly
# for logging.
all_test_programs = [
node for node in dep_graph.get_nodes()
if node.node_type == NodeType.TEST
]
all_test_basenames = set(
[os.path.basename(node.path) for node in all_test_programs])
# A very conservative way to decide whether to run all tests. If there are changes in any
# categories (meaning the changeset is non-empty), and there are changes in categories other
# than C++ / Java / files known not to affect unit tests, we force re-running all tests.
unsafe_categories = updated_categories - CATEGORIES_NOT_CAUSING_RERUN_OF_ALL_TESTS
user_said_all_tests = get_bool_env_var('YB_RUN_ALL_TESTS')
test_filter_re = os.getenv('YB_TEST_EXECUTION_FILTER_RE')
manual_test_filtering_with_regex = bool(test_filter_re)
select_all_tests_for_now = (bool(unsafe_categories)
or user_said_all_tests
or manual_test_filtering_with_regex)
user_said_all_cpp_tests = get_bool_env_var('YB_RUN_ALL_CPP_TESTS')
user_said_all_java_tests = get_bool_env_var('YB_RUN_ALL_JAVA_TESTS')
cpp_files_changed = SourceFileCategory.CPP in updated_categories
java_files_changed = SourceFileCategory.JAVA in updated_categories
yb_master_or_tserver_changed = bool(affected_basenames
& set(['yb-master', 'yb-tserver']))
run_cpp_tests = select_all_tests_for_now or cpp_files_changed or user_said_all_cpp_tests
run_java_tests = (select_all_tests_for_now or java_files_changed
or yb_master_or_tserver_changed
or user_said_all_java_tests)
if select_all_tests_for_now:
if user_said_all_tests:
logging.info(
"User explicitly specified that all tests should be run")
elif manual_test_filtering_with_regex:
logging.info(
"YB_TEST_EXECUTION_FILTER_RE specified: %s, will filter tests at a later step",
test_filter_re)
else:
logging.info(
"All tests should be run based on file changes in these categories: {}"
.format(', '.join(
sorted([
category.value for category in unsafe_categories
]))))
else:
if run_cpp_tests:
if user_said_all_cpp_tests:
logging.info(
"User explicitly specified that all C++ tests should be run"
)
else:
logging.info(
'Will run some C++ tests, some C++ files changed')
if run_java_tests:
if user_said_all_java_tests:
logging.info(
"User explicitly specified that all Java tests should be run"
)
else:
logging.info('Will run all Java tests, ' + ' and '.join(
(['some Java files changed'] if java_files_changed else
[]) + (['yb-{master,tserver} binaries changed']
if yb_master_or_tserver_changed else [])))
if run_cpp_tests and not test_basename_list and not select_all_tests_for_now:
logging.info(
'There are no C++ test programs affected by the changes, '
'will skip running C++ tests.')
run_cpp_tests = False
test_conf = dict(run_cpp_tests=run_cpp_tests,
run_java_tests=run_java_tests,
file_changes_by_category=file_changes_by_category_str)
if test_filter_re:
test_conf.update(test_filter_re=test_filter_re)
if not select_all_tests_for_now:
# We only have this kind of fine-grained filtering for C++ test programs, and for Java
# tests we either run all of them or none.
test_conf['cpp_test_programs'] = test_basename_list
if len(all_test_basenames) > 0:
logging.info(
"{} C++ test programs should be run (out of {} possible, {}%)"
.format(
len(test_basename_list), len(all_test_basenames),
"%.1f" % (100.0 * len(test_basename_list) /
len(all_test_basenames))))
if len(test_basename_list) != len(all_test_basenames):
logging.info(
"The following C++ test programs will be run: {}".format(
", ".join(sorted(test_basename_list))))
with open(args.output_test_config, 'w') as output_file:
output_file.write(json.dumps(test_conf, indent=2) + "\n")
logging.info("Wrote a test configuration to {}".format(
args.output_test_config))
else:
# For ad-hoc command-line use, mostly for testing and sanity-checking.
for node in sorted(results,
key=lambda node: [node.node_type.value, node.path]):
print(node)
logging.info("Found {} results".format(len(results)))
def make_postgres(self) -> None:
self.set_env_vars('make')
# Postgresql requires MAKELEVEL to be 0 or non-set when calling its make.
# But in case YB project is built with make, MAKELEVEL is not 0 at this point.
make_cmd = ['make', 'MAKELEVEL=0']
if is_macos_arm64():
make_cmd = ['arch', '-arm64'] + make_cmd
make_parallelism_str: Optional[str] = os.environ.get(
'YB_MAKE_PARALLELISM')
make_parallelism: Optional[int] = None
if make_parallelism_str is not None:
make_parallelism = int(make_parallelism_str)
if self.build_uses_remote_compilation and not self.remote_compilation_allowed:
# Since we're building everything locally in this case, and YB_MAKE_PARALLELISM is
# likely specified for distributed compilation, cap it at some factor times the number
# of CPU cores.
parallelism_cap = multiprocessing.cpu_count() * 2
if make_parallelism:
make_parallelism = min(parallelism_cap, make_parallelism)
else:
make_parallelism = parallelism_cap
if make_parallelism:
make_cmd += ['-j', str(int(make_parallelism))]
self.set_env_var('YB_COMPILER_TYPE', self.compiler_type)
# Create a script allowing to easily run "make" from the build directory with the right
# environment.
env_script_content = ''
for env_var_name in CONFIG_ENV_VARS:
env_var_value = os.environ.get(env_var_name)
if env_var_value is None:
raise RuntimeError("Expected env var %s to be set" %
env_var_name)
env_script_content += "export %s=%s\n" % (
env_var_name, quote_for_bash(env_var_value))
pg_compile_commands_paths = []
third_party_extensions_dir = os.path.join(self.pg_build_root,
'third-party-extensions')
work_dirs = [
self.pg_build_root,
os.path.join(self.pg_build_root, 'contrib'),
third_party_extensions_dir
]
for work_dir in work_dirs:
with WorkDirContext(work_dir):
# Create a script to run Make easily with the right environment.
make_script_path = 'make.sh'
with open(make_script_path, 'w') as out_f:
out_f.write('#!/usr/bin/env bash\n'
'. "${BASH_SOURCE%/*}"/env.sh\n'
'make "$@"\n')
with open('env.sh', 'w') as out_f:
out_f.write(env_script_content)
run_program(['chmod', 'u+x', make_script_path])
make_cmd_suffix = []
if work_dir == third_party_extensions_dir:
make_cmd_suffix = ['PG_CONFIG=' + self.pg_config_path]
# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)
complete_make_cmd = make_cmd + make_cmd_suffix
complete_make_cmd_str = shlex_join(complete_make_cmd)
complete_make_install_cmd = make_cmd + ['install'
] + make_cmd_suffix
attempt = 0
while attempt <= TRANSIENT_BUILD_RETRIES:
attempt += 1
make_result = run_program(
complete_make_cmd_str,
stdout_stderr_prefix='make',
cwd=work_dir,
error_ok=True,
shell=True # TODO: get rid of shell=True.
)
if make_result.failure():
transient_err = False
stderr_lines = make_result.get_stderr().split('\n')
for line in stderr_lines:
if any(transient_error_pattern in line
for transient_error_pattern in
TRANSIENT_BUILD_ERRORS):
transient_err = True
logging.info(f'Transient error: {line}')
break
if transient_err:
logging.info(
f"Transient error during build attempt {attempt}. "
f"Re-trying make command: {complete_make_cmd_str}."
)
else:
make_result.print_output_to_stdout()
raise RuntimeError("PostgreSQL compilation failed")
else:
logging.info(
"Successfully ran 'make' in the %s directory",
work_dir)
break # No error, break out of retry loop
else:
raise RuntimeError(
f"Maximum build attempts reached ({TRANSIENT_BUILD_RETRIES} attempts)."
)
if self.build_type != 'compilecmds' or work_dir == self.pg_build_root:
run_program(
' '.join(
shlex.quote(arg)
for arg in complete_make_install_cmd),
stdout_stderr_prefix='make_install',
cwd=work_dir,
error_ok=True,
shell=True # TODO: get rid of shell=True.
).print_output_and_raise_error_if_failed()
logging.info(
"Successfully ran 'make install' in the %s directory",
work_dir)
else:
logging.info(
"Not running 'make install' in the %s directory since we are only "
"generating the compilation database", work_dir)
if self.export_compile_commands:
logging.info(
"Generating the compilation database in directory '%s'",
work_dir)
compile_commands_path = os.path.join(
work_dir, 'compile_commands.json')
self.set_env_var('YB_PG_SKIP_CONFIG_STATUS', '1')
if not os.path.exists(compile_commands_path):
run_program(['compiledb', 'make', '-n'] +
make_cmd_suffix,
capture_output=False)
del os.environ['YB_PG_SKIP_CONFIG_STATUS']
if not os.path.exists(compile_commands_path):
raise RuntimeError(
"Failed to generate compilation database at: %s" %
compile_commands_path)
pg_compile_commands_paths.append(compile_commands_path)
if self.export_compile_commands:
self.write_compile_commands_files(pg_compile_commands_paths)
def main():
parser = argparse.ArgumentParser(
description='A tool for working with the dependency graph')
parser.add_argument('--verbose',
action='store_true',
help='Enable debug output')
parser.add_argument(
'-r',
'--rebuild-graph',
action='store_true',
help='Rebuild the dependecy graph and save it to a file')
parser.add_argument('--node-type',
help='Node type to look for',
default='any',
choices=['test', 'object', 'library', 'source', 'any'])
parser.add_argument(
'--file-regex',
help='Regular expression for file names to select as initial nodes for '
'querying the dependency graph.')
parser.add_argument(
'--file-name-glob',
help='Like file-regex, but applies only to file name and uses the glob '
'syntax instead of regex.')
parser.add_argument(
'--git-diff',
help='Figure out the list of files to use as starting points in the '
'dependency graph traversal by diffing the current state of the code '
'against this commit. This could also be anything that could be '
'passed to "git diff" as a single argument.')
parser.add_argument(
'--git-commit',
help='Similar to --git-diff, but takes a git commit ref (e.g. sha1 or '
'branch) and uses the set of files from that commit.')
parser.add_argument(
'--build-root',
required=True,
help='E.g. <some_root>/build/debug-gcc-dynamic-community')
parser.add_argument('command', choices=COMMANDS, help='Command to perform')
parser.add_argument(
'--output-test-config',
help=
'Output a "test configuration file", which is a JSON containing the '
'resulting list of C++ tests to run to this file, a flag indicating '
'wheter to run Java tests or not, etc.')
parser.add_argument(
'--incomplete-build',
action='store_true',
help='Skip checking for file existence. Allows using the tool after '
'build artifacts have been deleted.')
args = parser.parse_args()
if args.file_regex and args.file_name_glob:
raise RuntimeError(
'--file-regex and --file-name-glob are incompatible')
cmd = args.command
if (not args.file_regex and not args.file_name_glob
and not args.rebuild_graph and not args.git_diff
and not args.git_commit and cmd != SELF_TEST_CMD):
raise RuntimeError(
"Neither of --file-regex, --file-name-glob, --git-{diff,commit}, or "
"--rebuild-graph are specified, and the command is not " +
SELF_TEST_CMD)
log_level = logging.INFO
logging.basicConfig(
level=log_level,
format=
"[%(filename)s:%(lineno)d] %(asctime)s %(levelname)s: %(message)s")
conf = Configuration(args)
if conf.file_regex and args.git_diff:
raise RuntimeError(
"--git-diff is incompatible with --file-{regex,name-glob}")
if args.git_diff and args.git_commit:
raise RuntimeError('--git-diff and --git-commit are incompatible')
if args.git_commit:
args.git_diff = "{}^..{}".format(args.git_commit, args.git_commit)
graph_cache_path = os.path.join(args.build_root, 'dependency_graph.json')
if args.rebuild_graph or not os.path.isfile(graph_cache_path):
logging.info(
"Generating a dependency graph at '{}'".format(graph_cache_path))
dep_graph_builder = DependencyGraphBuilder(conf)
dep_graph = dep_graph_builder.build()
dep_graph.save_as_json(graph_cache_path)
else:
start_time = datetime.now()
with open(graph_cache_path) as graph_input_file:
dep_graph = DependencyGraph(conf,
json_data=json.load(graph_input_file))
logging.info("Loaded dependency graph from '%s' in %.2f sec" %
(graph_cache_path,
(datetime.now() - start_time).total_seconds()))
dep_graph.validate_node_existence()
if cmd == SELF_TEST_CMD:
run_self_test(dep_graph)
return
updated_categories = None
file_changes = []
if args.git_diff:
old_working_dir = os.getcwd()
with WorkDirContext(conf.yb_src_root):
git_diff_output = subprocess.check_output(
['git', 'diff', args.git_diff, '--name-only'])
initial_nodes = set()
file_paths = set()
for file_path in git_diff_output.split("\n"):
file_path = file_path.strip()
if not file_path:
continue
file_changes.append(file_path)
# It is important that we invoke os.path.realpath with the current directory set to
# the git repository root.
file_path = os.path.realpath(file_path)
file_paths.add(file_path)
node = dep_graph.node_by_path.get(file_path)
if node:
initial_nodes.add(node)
if not initial_nodes:
logging.warning(
"Did not find any graph nodes for this set of files: {}".
format(file_paths))
for basename in set(
[os.path.basename(file_path) for file_path in file_paths]):
logging.warning("Nodes for basename '{}': {}".format(
basename, dep_graph.find_nodes_by_basename(basename)))
file_changes_by_category = group_by(file_changes, get_file_category)
for category, changes in file_changes_by_category.items():
logging.info("File changes in category '{}':".format(category))
for change in sorted(changes):
logging.info(" {}".format(change))
updated_categories = set(file_changes_by_category.keys())
elif conf.file_regex:
logging.info("Using file name regex: {}".format(conf.file_regex))
initial_nodes = dep_graph.find_nodes_by_regex(conf.file_regex)
else:
raise RuntimeError(
"Could not figure out how to generate the initial set of files")
results = set()
if cmd == LIST_AFFECTED_CMD:
results = dep_graph.find_affected_nodes(initial_nodes, args.node_type)
elif cmd == LIST_DEPS_CMD:
for node in initial_nodes:
results.update(node.deps)
elif cmd == LIST_REVERSE_DEPS_CMD:
for node in initial_nodes:
results.update(node.reverse_deps)
else:
raise RuntimeError("Unimplemented command '{}'".format(command))
if args.output_test_config:
test_basename_list = sorted([
os.path.basename(node.path) for node in results
if node.node_type == 'test'
])
affected_basenames = set(
[os.path.basename(node.path) for node in results])
# These are ALL tests, not just tests affected by the changes in question, used mostly
# for logging.
all_test_programs = [
node for node in dep_graph.get_nodes() if node.node_type == 'test'
]
all_test_basenames = set(
[os.path.basename(node.path) for node in all_test_programs])
# A very conservative way to decide whether to run all tests. If there are changes in any
# categories (meaning the changeset is non-empty), and there are changes in categories other
# than C++ / Java / files known not to affect unit tests, we force re-running all tests.
unsafe_categories = updated_categories - CATEGORIES_NOT_CAUSING_RERUN_OF_ALL_TESTS
user_said_all_tests = get_bool_env_var('YB_RUN_ALL_TESTS')
run_all_tests = bool(unsafe_categories) or user_said_all_tests
user_said_all_cpp_tests = get_bool_env_var('YB_RUN_ALL_CPP_TESTS')
user_said_all_java_tests = get_bool_env_var('YB_RUN_ALL_JAVA_TESTS')
cpp_files_changed = 'c++' in updated_categories
java_files_changed = 'java' in updated_categories
yb_master_or_tserver_changed = bool(affected_basenames
& set(['yb-master', 'yb-tserver']))
run_cpp_tests = run_all_tests or cpp_files_changed or user_said_all_cpp_tests
run_java_tests = (run_all_tests or java_files_changed
or yb_master_or_tserver_changed
or user_said_all_java_tests)
if run_all_tests:
if user_said_all_tests:
logging.info(
"User explicitly specified that all tests should be run")
else:
logging.info(
"All tests should be run based on file changes in these categories: {}"
.format(', '.join(sorted(unsafe_categories))))
else:
if run_cpp_tests:
if user_said_all_cpp_tests:
logging.info(
"User explicitly specified that all C++ tests should be run"
)
else:
logging.info(
'Will run some C++ tests, some C++ files changed')
if run_java_tests:
if user_said_all_java_tests:
logging.info(
"User explicitly specified that all Java tests should be run"
)
else:
logging.info('Will run all Java tests, ' + ' and '.join(
(['some Java files changed'] if java_files_changed else
[]) + (['yb-{master,tserver} binaries changed']
if yb_master_or_tserver_changed else [])))
if run_cpp_tests and not test_basename_list and not run_all_tests:
logging.info(
'There are no C++ test programs affected by the changes, '
'will skip running C++ tests.')
run_cpp_tests = False
test_conf = dict(run_cpp_tests=run_cpp_tests,
run_java_tests=run_java_tests,
file_changes_by_category=file_changes_by_category)
if not run_all_tests:
test_conf['cpp_test_programs'] = test_basename_list
logging.info(
"{} C++ test programs should be run (out of {} possible, {}%)".
format(
len(test_basename_list), len(all_test_basenames),
"%.1f" % (100.0 * len(test_basename_list) /
len(all_test_basenames))))
if len(test_basename_list) != len(all_test_basenames):
logging.info(
"The following C++ test programs will be run: {}".format(
", ".join(sorted(test_basename_list))))
with open(args.output_test_config, 'w') as output_file:
output_file.write(json.dumps(test_conf, indent=2) + "\n")
logging.info("Wrote a test configuration to {}".format(
args.output_test_config))
else:
# For ad-hoc command-line use, mostly for testing and sanity-checking.
for node in sorted(results,
key=lambda node: [node.node_type, node.path]):
print(node)
logging.info("Found {} results".format(len(results)))
