def expand_value(self, old_value):
"""
Expand old_value with the following changes:
- Replace ${project.version} with the Java version from pom.xml.
- Replace the leading "thirdparty/" with the respective YB_THIRDPARTY_DIR from the build.
- Replace $BUILD_ROOT with the actual build_root.
"""
# Substitution for Java.
new_value = old_value.replace('${project.version}',
self.java_project_version)
# Substitution for thirdparty.
thirdparty_prefix_match = THIRDPARTY_PREFIX_RE.match(new_value)
if thirdparty_prefix_match:
new_value = os.path.join(get_thirdparty_dir(),
thirdparty_prefix_match.group(1))
# Substitution for BUILD_ROOT.
new_value = new_value.replace("$BUILD_ROOT", self.build_root)
thirdparty_intrumentation = "clang_uninstrumented" if is_macos(
) else "uninstrumented"
new_value = new_value.replace(
"$THIRDPARTY_BUILD_SPECIFIC_DIR",
os.path.join(get_thirdparty_dir(), "installed",
thirdparty_intrumentation))
if new_value != old_value:
logging.info("Substituting '{}' -> '{}' in manifest".format(
old_value, new_value))
return new_value
def get_category(self) -> str:
"""
Categorizes binaries into a few buckets:
- yb -- built as part of YugabyteDB
- yb-thirdparty -- built as part of yugabyte-db-thirdparty
- linuxbrew -- built using Linuxbrew
- system -- a library residing in a system-wide library directory. We do not copy these.
- postgres -- libraries inside the postgres directory
"""
if self.category:
return self.category
linuxbrew_home = get_linuxbrew_home()
if linuxbrew_home is not None and linuxbrew_home.path_is_in_linuxbrew_dir(
self.target):
self.category = 'linuxbrew'
elif self.target.startswith(get_thirdparty_dir() + '/'):
self.category = 'yb-thirdparty'
elif self.target.startswith(self.context.build_dir + '/postgres/'):
self.category = 'postgres'
elif self.target.startswith(self.context.build_dir + '/'):
self.category = 'yb'
elif (self.target.startswith(YB_SCRIPT_BIN_DIR + '/')
or self.target.startswith(YB_BUILD_SUPPORT_DIR + '/')):
self.category = 'yb-scripts'
if not self.category:
for system_library_path in SYSTEM_LIBRARY_PATHS:
if self.target.startswith(system_library_path + '/'):
self.category = 'system'
break
if self.category:
if self.category not in LIBRARY_CATEGORIES:
raise RuntimeError((
"Internal error: library category computed as '{}', must be one of: {}. "
+ "Dependency: {}").format(self.category,
LIBRARY_CATEGORIES, self))
return self.category
if linuxbrew_home:
linuxbrew_dir_str = linuxbrew_home.get_human_readable_dirs()
else:
linuxbrew_dir_str = 'N/A'
raise RuntimeError((
"Could not determine the category of this binary "
"(yugabyte / yb-thirdparty / linuxbrew / system): '{}'. "
"Does not reside in the Linuxbrew directory ({}), "
"YB third-party directory ('{}'), "
"YB build directory ('{}'), "
"YB general-purpose script directory ('{}'), "
"YB build support script directory ('{}'), "
"and does not appear to be a system library (does not start with any of {})."
).format(self.target, linuxbrew_dir_str, get_thirdparty_dir(),
self.context.build_dir, YB_SCRIPT_BIN_DIR,
YB_BUILD_SUPPORT_DIR, SYSTEM_LIBRARY_PATHS))
def _rewrite_manifest(self):
"""
Rewrite the release manifest with the following changes:
- Replace ${project.version} with the Java version from pom.xml.
- Replace the leading "thirdparty/" with the respective YB_THIRDPARTY_DIR from the build.
- Replace $BUILD_ROOT with the actual build_root.
"""
pom_file = os.path.join(self.repo, 'java', 'pom.xml')
java_project_version = minidom.parse(pom_file).getElementsByTagName(
'version')[0].firstChild.nodeValue
logging.info("Java project version from pom.xml: {}".format(
java_project_version))
for key, value_list in self.release_manifest.iteritems():
for i in xrange(len(value_list)):
old_value = value_list[i]
# Substitution for Java.
new_value = old_value.replace('${project.version}',
java_project_version)
# Substitution for thirdparty.
thirdparty_prefix_match = THIRDPARTY_PREFIX_RE.match(new_value)
if thirdparty_prefix_match:
new_value = os.path.join(get_thirdparty_dir(),
thirdparty_prefix_match.group(1))
# Substitution for BUILD_ROOT.
new_value = new_value.replace("$BUILD_ROOT", self.build_root)
if new_value != value_list[i]:
logging.info(
"Substituting '{}' -> '{}' in manifest".format(
value_list[i], new_value))
value_list[i] = new_value
def rewrite_manifest(self, build_root):
"""
Rewrite the release manifest with the following changes:
- Replace ${project.version} with the Java version from pom.xml.
- Replace the leading "thirdparty/" with the respective YB_THIRDPARTY_DIR from the build.
- Replace $BUILD_ROOT with the actual build_root.
"""
pom_file = os.path.join(self.repo, 'java', 'pom.xml')
java_project_version = minidom.parse(pom_file).getElementsByTagName(
'version')[0].firstChild.nodeValue
logging.info("Java project version from pom.xml: {}".format(java_project_version))
for key, value_list in self.release_manifest.iteritems():
for i in xrange(len(value_list)):
old_value = value_list[i]
# Substitution for Java.
new_value = old_value.replace('${project.version}', java_project_version)
# Substitution for thirdparty.
thirdparty_prefix_match = THIRDPARTY_PREFIX_RE.match(new_value)
if thirdparty_prefix_match:
new_value = os.path.join(get_thirdparty_dir(), thirdparty_prefix_match.group(1))
# Substitution for BUILD_ROOT.
new_value = new_value.replace("$BUILD_ROOT", build_root)
if new_value != value_list[i]:
logging.info("Substituting '{}' -> '{}' in manifest".format(
value_list[i], new_value))
value_list[i] = new_value
def get_category(self):
"""
Categorizes binaries into a few buckets:
- yb -- YugaByte product itself
- yb-thirdparty -- built with YugaByte
- linuxbrew -- built using Linuxbrew
- system -- grabbed from a system-wide library directory
"""
if self.category:
return self.category
if self.target.startswith(LINUXBREW_HOME + '/'):
self.category = 'linuxbrew'
elif self.target.startswith(get_thirdparty_dir() + '/'):
self.category = 'yb-thirdparty'
elif self.target.startswith(self.context.build_dir + '/'):
self.category = 'yb'
elif (self.target.startswith(YB_SCRIPT_BIN_DIR + '/')
or self.target.startswith(YB_BUILD_SUPPORT_DIR + '/')):
self.category = 'yb-scripts'
if not self.category:
for system_library_path in SYSTEM_LIBRARY_PATHS:
if self.target.startswith(system_library_path + '/'):
self.category = 'system'
break
if self.category:
if self.category not in LIBRARY_CATEGORIES:
raise RuntimeError((
"Internal error: library category computed as '{}', must be one of: {}. "
+ "Dependency: {}").format(self.category,
LIBRARY_CATEGORIES, self))
return self.category
raise RuntimeError((
"Could not determine the category of this binary "
"(yugabyte / yb-thirdparty / linuxbrew / system): '{}'. "
"Does not reside in the Linuxbrew directory ('{}'), "
"YB third-party directory ('{}'), "
"YB build directory ('{}'), "
"YB general-purpose script directory ('{}'), "
"YB build support script directory ('{}'), "
"and does not appear to be a system library (does not start with any of {})."
).format(self.target, LINUXBREW_HOME, get_thirdparty_dir(),
self.context.build_dir, YB_SCRIPT_BIN_DIR,
YB_BUILD_SUPPORT_DIR, SYSTEM_LIBRARY_PATHS))
def get_category(self):
"""
Categorizes binaries into a few buckets:
- yb -- YugaByte product itself
- yb-thirdparty -- built with YugaByte
- linuxbrew -- built using Linuxbrew
- system -- grabbed from a system-wide library directory
"""
if self.category:
return self.category
if self.target.startswith(LINUXBREW_HOME + '/'):
self.category = 'linuxbrew'
elif self.target.startswith(get_thirdparty_dir() + '/'):
self.category = 'yb-thirdparty'
elif self.target.startswith(self.context.build_dir + '/'):
self.category = 'yb'
elif (self.target.startswith(YB_SCRIPT_BIN_DIR + '/') or
self.target.startswith(YB_BUILD_SUPPORT_DIR + '/')):
self.category = 'yb-scripts'
if not self.category:
for system_library_path in SYSTEM_LIBRARY_PATHS:
if self.target.startswith(system_library_path + '/'):
self.category = 'system'
break
if self.category:
if self.category not in LIBRARY_CATEGORIES:
raise RuntimeError(
("Internal error: library category computed as '{}', must be one of: {}. " +
"Dependency: {}").format(self.category, LIBRARY_CATEGORIES, self))
return self.category
raise RuntimeError(
("Could not determine the category of this binary "
"(yugabyte / yb-thirdparty / linuxbrew / system): '{}'. "
"Does not reside in the Linuxbrew directory ('{}'), "
"YB third-party directory ('{}'), "
"YB build directory ('{}'), "
"YB general-purpose script directory ('{}'), "
"YB build support script directory ('{}'), "
"and does not appear to be a system library (does not start with any of {})."
).format(self.target, LINUXBREW_HOME, get_thirdparty_dir(), self.context.build_dir,
YB_SCRIPT_BIN_DIR, YB_BUILD_SUPPORT_DIR, SYSTEM_LIBRARY_PATHS))
def main():
parser = argparse.ArgumentParser(
description='Run FOSSA analysis (open source license compliance).')
parser.add_argument('--verbose', action='store_true', help='Enable verbose output')
parser.add_argument(
'fossa_cli_args',
nargs='*',
help='These arguments are passed directly to fossa-cli')
args = parser.parse_args()
init_env(args.verbose)
# TODO: We may also want to try using the v2 option --unpack-archives
# Though that may be going to deeper level than we want.
fossa_cmd_line = ['fossa', 'analyze']
fossa_cmd_line.extend(args.fossa_cli_args)
should_upload = not any(
arg in args.fossa_cli_args for arg in ('--show-output', '--output', '-o'))
if should_upload and not os.getenv('FOSSA_API_KEY'):
# --output is used for local analysis only, without uploading the results. In all other
# cases we would like .
raise RuntimeError('FOSSA_API_KEY must be specified in order to upload analysis results.')
logging.info(
f"FOSSA CLI command line: {shlex_join(fossa_cmd_line)}")
fossa_version_str = subprocess.check_output(['fossa', '--version']).decode('utf-8')
fossa_version_match = FOSSA_VERSION_RE.match(fossa_version_str)
if not fossa_version_match:
raise RuntimeError(f"Cannot parse fossa-cli version: {fossa_version_str}")
fossa_version = fossa_version_match.group(1)
if version.parse(fossa_version) < version.parse(MIN_FOSSA_CLI_VERSION):
raise RuntimeError(
f"fossa version too old: {fossa_version} "
f"(expected {MIN_FOSSA_CLI_VERSION} or later)")
download_cache_path = get_download_cache_dir()
logging.info(f"Using the download cache directory {download_cache_path}")
download_config = DownloadConfig(
verbose=args.verbose,
cache_dir_path=download_cache_path
)
downloader = Downloader(download_config)
fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa-local.yml')
fossa_yml_data = load_yaml_file(fossa_yml_path)
modules = fossa_yml_data['analyze']['modules']
# fossa v2.6.1 does not pick up project name from config file version 2 format.
# TODO: update to config file version 3
fossa_cmd_line.extend(["--project", fossa_yml_data['cli']['project']])
thirdparty_dir = get_thirdparty_dir()
fossa_modules_path = os.path.join(thirdparty_dir, 'fossa_modules.yml')
seen_urls = set()
start_time_sec = time.time()
if os.path.exists(fossa_modules_path):
thirdparty_fossa_modules_data = load_yaml_file(fossa_modules_path)
for thirdparty_module_data in thirdparty_fossa_modules_data:
fossa_module_data = thirdparty_module_data['fossa_module']
module_name = fossa_module_data['name']
if not should_include_fossa_module(module_name):
continue
fossa_module_yb_metadata = thirdparty_module_data['yb_metadata']
expected_sha256 = fossa_module_yb_metadata['sha256sum']
url = fossa_module_yb_metadata['url']
if url in seen_urls:
# Due to a bug in some versions of yugabyte-db-thirdparty scripts, as of 04/20/2021
# we may include the same dependency twice in the fossa_modules.yml file. We just
# skip the duplicates here.
continue
seen_urls.add(url)
logging.info(f"Adding module from {url}")
downloaded_path = downloader.download_url(
url,
download_parent_dir_path=None, # Download to cache directly.
verify_checksum=True,
expected_sha256=expected_sha256
)
fossa_module_data['target'] = downloaded_path
modules.append(fossa_module_data)
# TODO: Once we move to v2 fossa, we may want to use fossa-dep.yml file instead of
# re-writing the main file.
effective_fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa.yml')
write_yaml_file(fossa_yml_data, effective_fossa_yml_path)
logging.info(f"Wrote the expanded FOSSA file to {effective_fossa_yml_path}")
else:
logging.warning(
f"File {fossa_modules_path} does not exist. Some C/C++ dependencies will be missing "
f"from FOSSA analysis.")
effective_fossa_yml_path = fossa_yml_path
elapsed_time_sec = time.time() - start_time_sec
logging.info("Generated the effective FOSSA configuration file in %.1f sec", elapsed_time_sec)
logging.info(f"Running command: {shlex_join(fossa_cmd_line)})")
subprocess.check_call(fossa_cmd_line)
# and because some libraries might also have system library directories on their rpath.
run_patchelf('--remove-rpath', installed_binary)
post_install_path = os.path.join(dest_bin_dir, 'post_install.sh')
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
binary_names_to_patch = ' '.join(
['"{}"'.format(name) for name in elf_names_to_set_interpreter])
new_post_install_script = post_install_script.replace(
'${elf_names_to_set_interpreter}', binary_names_to_patch)
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)
if __name__ == '__main__':
if not os.path.isdir(get_thirdparty_dir()):
raise RuntimeError(
"Third-party dependency directory '{}' does not exist".format(
get_thirdparty_dir()))
parser = argparse.ArgumentParser(description=LibraryPackager.__doc__)
parser.add_argument(
'--build-dir',
help='Build directory to pick up executables/libraries from.',
required=True)
parser.add_argument(
'--dest-dir',
help='Destination directory to save the self-sufficient directory tree '
'of executables and libraries at.',
required=True)
parser.add_argument(
def package_binaries(self):
src = self.build_dir
dst = self.dest_dir
dst_bin_dir = os.path.join(dst, 'bin')
dst_lib_dir = os.path.join(dst, 'lib')
try:
os.makedirs(dst_bin_dir)
except OSError as e:
raise RuntimeError('Unable to create directory %s', dst)
logging.debug('Created directory %s', dst)
bin_dir_files = []
for seed_executable_glob in self.seed_executable_patterns:
if seed_executable_glob.find('postgres/bin/') >= 0:
# Skip postgres binaries since they are copied with the postgres root directory
# which is handled below.
continue
if seed_executable_glob.startswith('bin/'):
bin_dir_files.append(os.path.basename(seed_executable_glob))
logging.debug("Adding file '%s' to bash_scripts",
seed_executable_glob)
updated_glob = seed_executable_glob.replace(
'$BUILD_ROOT', self.build_dir)
if updated_glob != seed_executable_glob:
logging.info('Substituting: {} -> {}'.format(
seed_executable_glob, updated_glob))
seed_executable_glob = updated_glob
glob_results = glob.glob(seed_executable_glob)
if not glob_results:
raise RuntimeError(
"No files found matching the pattern '{}'".format(
seed_executable_glob))
for executable in glob_results:
shutil.copy(executable, dst_bin_dir)
src_lib_dir = os.path.join(src, 'lib')
yb_lib_file_for_postgres = os.path.join(src_lib_dir,
'libyb_pggate.dylib')
libedit_file_for_postgres = os.path.join(
get_thirdparty_dir(), 'installed/common/lib/libedit.dylib')
processed_libs = []
for bin_file in os.listdir(dst_bin_dir):
if bin_file.endswith('.sh') or bin_file in bin_dir_files:
logging.info(
"Not modifying rpath for file '%s' because it's not a binary file",
bin_file)
continue
logging.debug('Processing binary file: %s', bin_file)
libs = []
os.makedirs(os.path.join(dst, 'lib', bin_file))
libs = self.fix_load_paths(
os.path.join(dst_bin_dir, bin_file),
os.path.join(dst_lib_dir, bin_file),
os.path.join('@loader_path/../lib/', bin_file))
# Elements in libs are absolute paths.
logging.debug('library dependencies for file %s: %s', bin_file,
libs)
# Treat this as a special case for now (10/14/18).
libs.append(yb_lib_file_for_postgres)
libs.append(libedit_file_for_postgres)
for lib in libs:
if lib in processed_libs:
continue
# For each library dependency, check whether it already has its own directory (if it
# does, a physical copy of this library must exist there). If it doesn't, create it
# and copy the physical file there.
logging.debug('Processing library: %s', lib)
libname = os.path.basename(lib)
lib_dir_path = os.path.join(dst, 'lib', libname)
if os.path.exists(lib_dir_path):
continue
os.mkdir(lib_dir_path)
shutil.copy(lib, lib_dir_path)
lib_file_path = os.path.join(lib_dir_path, libname)
new_libs = self.fix_load_paths(lib_file_path, lib_dir_path,
'@loader_path')
for new_lib in new_libs:
if new_lib not in processed_libs and new_lib not in libs:
logging.info('Adding dependency %s for library %s',
new_lib, lib_file_path)
libs.append(new_lib)
processed_libs.append(lib)
# Handle postgres as a special case for now (10/14/18).
postgres_src = os.path.join(src, 'postgres')
postgres_dst = os.path.join(dst, 'postgres')
shutil.copytree(postgres_src, postgres_dst, symlinks=True)
postgres_bin = os.path.join(postgres_dst, 'bin')
postgres_lib = os.path.join(postgres_dst, 'lib')
for bin_file in os.listdir(postgres_bin):
self.fix_postgres_load_paths(os.path.join(postgres_bin, bin_file),
dst)
for lib_file in os.listdir(postgres_lib):
if os.path.isdir(os.path.join(postgres_lib, lib_file)):
continue
logging.debug("Processing postgres library %s", lib_file)
self.fix_postgres_load_paths(os.path.join(postgres_lib, lib_file),
dst)
# Remove rpath (we will set it appropriately in post_install.sh).
run_patchelf('--remove-rpath', installed_binary)
post_install_path = os.path.join(dest_bin_dir, 'post_install.sh')
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
binary_names_to_patch = ' '.join([
'"{}"'.format(name) for name in elf_names_to_set_interpreter])
new_post_install_script = post_install_script.replace('${elf_names_to_set_interpreter}',
binary_names_to_patch)
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)
if __name__ == '__main__':
if not os.path.isdir(get_thirdparty_dir()):
raise RuntimeError("Third-party dependency directory '{}' does not exist".format(
get_thirdparty_dir()))
parser = argparse.ArgumentParser(description=LibraryPackager.__doc__)
parser.add_argument('--build-dir',
help='Build directory to pick up executables/libraries from.',
required=True)
parser.add_argument('--dest-dir',
help='Destination directory to save the self-sufficient directory tree '
'of executables and libraries at.',
required=True)
parser.add_argument('--clean-dest',
help='Remove the destination directory if it already exists. Only works '
'with directories under /tmp/... for safety.',
action='store_true')
