def _lookup_in_system_whitelists(errors, whitelist, needed_dso, sysroots, msg_prelude, info_prelude,
sysroot_prefix, sysroot_substitution, verbose):
# A system or ignored dependency. We should be able to find it in one of the CDT o
# compiler packages on linux or at in a sysroot folder on other OSes. These usually
# start with '$RPATH/' which indicates pyldd did not find them, so remove that now.
if needed_dso.startswith(sysroot_substitution):
replacements = [sysroot_substitution] + sysroots
else:
replacements = [needed_dso]
in_whitelist = False
for replacement in replacements:
needed_dso_w = needed_dso.replace(sysroot_substitution, replacement)
in_whitelist = any([glob2.fnmatch.fnmatch(needed_dso_w, w) for w in whitelist])
if in_whitelist:
n_dso_p = "Needed DSO {}".format(needed_dso_w)
_print_msg(errors, '{}: {} found in the whitelist'.
format(info_prelude, n_dso_p), verbose=verbose)
break
if not in_whitelist and len(sysroots):
# Check if we have a CDT package.
dso_fname = os.path.basename(needed_dso)
sysroot_files = []
dirs_to_glob = [] # Optimization, ideally we'll not glob at all as it's slooow.
for sysroot in sysroots:
sysroot_os = sysroot.replace('/', os.sep)
if needed_dso.startswith(sysroot_substitution):
# Do we want to do this replace?
sysroot_files.append(needed_dso.replace(sysroot_substitution, sysroot_os))
else:
dirs_to_glob.extend((os.path.join(sysroot_os, '**', dso_fname)))
for dir_to_glob in dirs_to_glob:
sysroot_files.extend(glob(dir_to_glob))
if len(sysroot_files):
# Removing sysroot_prefix is only *really* for Linux, though we could
# use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about
# /opt/X11 too.
# Find the longest suffix match.
rev_needed_dso = needed_dso[::-1]
match_lens = [len(os.path.commonprefix([s[::-1], rev_needed_dso]))
for s in sysroot_files]
idx = max(range(len(match_lens)), key=match_lens.__getitem__)
in_prefix_dso = os.path.normpath(sysroot_files[idx].replace(
sysroot_prefix + os.sep, ''))
n_dso_p = "Needed DSO {}".format(in_prefix_dso)
pkgs = list(which_package(in_prefix_dso, sysroot_prefix))
if len(pkgs):
_print_msg(errors, '{}: {} found in CDT/compiler package {}'.
format(info_prelude, n_dso_p, pkgs[0]), verbose=verbose)
else:
_print_msg(errors, '{}: {} not found in any CDT/compiler package,'
' nor the whitelist?!'.
format(msg_prelude, n_dso_p), verbose=verbose)
else:
_print_msg(errors, "{}: {} not found in sysroot, is this binary repackaging?"
" .. do you need to use install_name_tool/patchelf?".
format(msg_prelude, needed_dso), verbose=verbose)
elif not in_whitelist:
_print_msg(errors, "{}: did not find - or even know where to look for: {}".
format(msg_prelude, needed_dso), verbose=verbose)
def _map_file_to_package(files, run_prefix, build_prefix, all_needed_dsos, pkg_vendored_dist, ignore_list_syms, sysroot_substitution):
# Form a mapping of file => package
prefix_owners = {}
contains_dsos = {}
contains_static_libs = {}
# Used for both dsos and static_libs
all_lib_exports = {}
for prefix in (run_prefix, build_prefix):
for subdir2, _, filez in os.walk(prefix):
for file in filez:
fp = os.path.join(subdir2, file)
dynamic_lib = any(glob2.fnmatch.fnmatch(fp, ext) for ext in ('*.so*', '*.dylib*', '*.dll')) and \
codefile_type(fp, skip_symlinks=False) is not None
static_lib = any(glob2.fnmatch.fnmatch(fp, ext) for ext in ('*.a', '*.lib'))
# Looking at all the files is very slow.
if not dynamic_lib and not static_lib:
continue
rp = os.path.relpath(fp, prefix)
if dynamic_lib and rp not in all_needed_dsos:
continue
if rp in all_lib_exports:
continue
owners = prefix_owners[rp] if rp in prefix_owners else []
# Self-vendoring, not such a big deal but may as well report it?
if not len(owners):
if rp in files:
owners.append(pkg_vendored_dist)
new_pkgs = list(which_package(rp, prefix))
# Cannot filter here as this means the DSO (eg libomp.dylib) will not be found in any package
# [owners.append(new_pkg) for new_pkg in new_pkgs if new_pkg not in owners
# and not any([glob2.fnmatch.fnmatch(new_pkg.name, i) for i in ignore_for_statics])]
for new_pkg in new_pkgs:
if new_pkg not in owners:
owners.append(new_pkg)
prefix_owners[rp] = owners
if len(prefix_owners[rp]):
exports = set(e for e in get_exports_memoized(fp) if not
any(glob2.fnmatch.fnmatch(e, pattern) for pattern in ignore_list_syms))
all_lib_exports[rp] = exports
# Check codefile_type to filter out linker scripts.
if dynamic_lib:
contains_dsos[prefix_owners[rp][0]] = True
elif static_lib:
if sysroot_substitution in fp:
if (prefix_owners[rp][0].name.startswith('gcc_impl_linux') or
prefix_owners[rp][0].name == 'llvm'):
continue
print("sysroot in {}, owner is {}".format(fp, prefix_owners[rp][0]))
contains_static_libs[prefix_owners[rp][0]] = True
return prefix_owners, contains_dsos, contains_static_libs, all_lib_exports
def _find_needed_dso_in_system(m, needed_dso, errors, sysroots, msg_prelude,
info_prelude, warn_prelude):
# A system dependency then. We should be able to find it in one of the CDT o
# compiler packages on linux or at in a sysroot folder on other OSes.
#
if m.config.verbose and len(sysroots):
# Check id we have a CDT package.
dso_fname = os.path.basename(needed_dso)
sysroot_files = []
for sysroot in sysroots:
sysroot_files.extend(glob(os.path.join(sysroot, '**', dso_fname)))
if len(sysroot_files):
# Removing config.build_prefix is only *really* for Linux, though we could
# use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about
# /opt/X11 too.
in_prefix_dso = os.path.normpath(sysroot_files[0].replace(
m.config.build_prefix + '/', ''))
n_dso_p = "Needed DSO {}".format(in_prefix_dso)
pkgs = list(which_package(in_prefix_dso, m.config.build_prefix))
if len(pkgs):
print_msg(
errors, '{}: {} found in CDT/compiler package {}'.format(
info_prelude, n_dso_p, pkgs[0]))
else:
print_msg(
errors,
'{}: {} not found in any CDT/compiler package?!'.format(
info_prelude, n_dso_p))
else:
prelude = warn_prelude if needed_dso.startswith(
'$RPATH') else msg_prelude
print_msg(
errors,
"{}: {} not found in sysroot, is this binary repackaging?"
" .. do you need to use install_name_tool/patchelf?".format(
prelude, needed_dso))
else:
# When a needed_dso begins with $RPATH it means we are making a CDT package
# (in any other case this would be a problem), but I should verify it is ok
# for CDT packages too.
if needed_dso.startswith('$RPATH'):
print_msg(
errors, "{}: {} returned by pyldd. A CDT package?".format(
warn_prelude, needed_dso))
else:
print_msg(
errors,
"{}: did not find - or even know where to look for: {}".format(
msg_prelude, needed_dso))
def _lookup_in_prefix_packages(errors, needed_dso, files, run_prefix, whitelist, info_prelude, msg_prelude,
warn_prelude, verbose, requirements_run, lib_packages, lib_packages_used):
in_prefix_dso = needed_dso
if in_prefix_dso == '/':
print('debug')
# os.path.normpath(needed_dso.replace(run_prefix + os.sep, ''))
n_dso_p = "Needed DSO {}".format(in_prefix_dso)
and_also = " (and also in this package)" if in_prefix_dso in files else ""
pkgs = list(which_package(in_prefix_dso, run_prefix))
in_pkgs_in_run_reqs = [pkg for pkg in pkgs if pkg.quad[0] in requirements_run]
# TODO :: metadata build/inherit_child_run_exports (for vc, mro-base-impl).
for pkg in in_pkgs_in_run_reqs:
if pkg in lib_packages:
lib_packages_used.add(pkg)
in_whitelist = any([glob2.fnmatch.fnmatch(in_prefix_dso, w) for w in whitelist])
if len(in_pkgs_in_run_reqs) == 1:
_print_msg(errors, '{}: {} found in {}{}'.format(info_prelude,
n_dso_p,
in_pkgs_in_run_reqs[0],
and_also), verbose=verbose)
elif in_whitelist:
_print_msg(errors, '{}: {} found in the whitelist'.
format(info_prelude, n_dso_p), verbose=verbose)
elif len(in_pkgs_in_run_reqs) == 0 and len(pkgs) > 0:
_print_msg(errors, '{}: {} found in {}{}'.format(msg_prelude,
n_dso_p,
[p.quad[0] for p in pkgs],
and_also), verbose=verbose)
_print_msg(errors, '{}: .. but {} not in reqs/run, (i.e. it is overlinking)'
' (likely) or a missing dependency (less likely)'.
format(msg_prelude, [p.quad[0] for p in pkgs]), verbose=verbose)
elif len(in_pkgs_in_run_reqs) > 1:
_print_msg(errors, '{}: {} found in multiple packages in run/reqs: {}{}'
.format(warn_prelude,
in_prefix_dso,
in_pkgs_in_run_reqs,
and_also), verbose=verbose)
else:
if in_prefix_dso not in files:
_print_msg(errors, '{}: {} not found in any packages'.format(msg_prelude,
in_prefix_dso), verbose=verbose)
elif verbose:
_print_msg(errors, '{}: {} found in this package'.format(info_prelude,
in_prefix_dso), verbose=verbose)
def _find_needed_dso_in_prefix(m, needed_dso, f, files, errors, run_reqs,
host_reqs, msg_prelude, info_prelude):
in_prefix_dso = os.path.normpath(
needed_dso.replace(m.config.host_prefix + '/', ''))
n_dso_p = "Needed DSO {} in {}".format(in_prefix_dso, f)
and_also = " (and also in this package)" if in_prefix_dso in files else ""
pkgs = list(which_package(in_prefix_dso, m.config.host_prefix))
if len(pkgs) == 1:
if pkgs[0].quad[0] not in run_reqs:
print_msg(
errors, '{}: {} found in {}{}'.format(msg_prelude, n_dso_p,
pkgs[0], and_also))
print_msg(
errors, '{}: .. but {} not in reqs/run, i.e. it is overlinked'
' (likely) or a missing dependency (less likely)'.format(
msg_prelude, pkgs[0].quad[0]))
elif m.config.verbose:
print_msg(
errors, '{}: {} found in {}{}'.format(info_prelude, n_dso_p,
pkgs[0], and_also))
elif len(pkgs) > 1:
print_msg(
errors, '{}: {} found in multiple packages:{}'.format(
msg_prelude, in_prefix_dso, and_also))
for pkg in pkgs:
print_msg(errors, '{}: {}'.format(msg_prelude, pkg))
if pkg.dist_name not in host_reqs:
print_msg(
errors,
'{}: .. but {} not in reqs/host (is transitive)'.format(
msg_prelude, pkg.dist_name))
else:
if in_prefix_dso not in files:
print_msg(
errors, '{}: {} not found in any packages'.format(
msg_prelude, in_prefix_dso))
elif m.config.verbose:
print_msg(
errors,
'{}: {} found in this package'.format(info_prelude,
in_prefix_dso))
def check_overlinking(m, files):
pkg_name = m.get_value('package/name')
errors = []
run_reqs = [req.split(' ')[0] for req in m.meta.get('requirements', {}).get('run', [])]
# sysroots and whitelists are similar, but the subtle distinctions are important.
sysroots = glob(os.path.join(m.config.build_prefix, '**', 'sysroot'))
whitelist = []
if 'target_platform' in m.config.variant and m.config.variant['target_platform'] == 'osx-64':
if not len(sysroots):
sysroots = ['/usr/lib', '/opt/X11', '/System/Library/Frameworks']
whitelist = ['/opt/X11/',
'/usr/lib/libSystem.B.dylib',
'/usr/lib/libcrypto.0.9.8.dylib',
'/usr/lib/libobjc.A.dylib',
'/System/Library/Frameworks/Accelerate.framework/*',
'/System/Library/Frameworks/AGL.framework/*',
'/System/Library/Frameworks/AppKit.framework/*',
'/System/Library/Frameworks/ApplicationServices.framework/*',
'/System/Library/Frameworks/AudioToolbox.framework/*',
'/System/Library/Frameworks/AudioUnit.framework/*',
'/System/Library/Frameworks/AVFoundation.framework/*',
'/System/Library/Frameworks/CFNetwork.framework/*',
'/System/Library/Frameworks/Carbon.framework/*',
'/System/Library/Frameworks/Cocoa.framework/*',
'/System/Library/Frameworks/CoreAudio.framework/*',
'/System/Library/Frameworks/CoreFoundation.framework/*',
'/System/Library/Frameworks/CoreGraphics.framework/*',
'/System/Library/Frameworks/CoreMedia.framework/*',
'/System/Library/Frameworks/CoreBluetooth.framework/*',
'/System/Library/Frameworks/CoreMIDI.framework/*',
'/System/Library/Frameworks/CoreMedia.framework/*',
'/System/Library/Frameworks/CoreServices.framework/*',
'/System/Library/Frameworks/CoreText.framework/*',
'/System/Library/Frameworks/CoreVideo.framework/*',
'/System/Library/Frameworks/CoreWLAN.framework/*',
'/System/Library/Frameworks/DiskArbitration.framework/*',
'/System/Library/Frameworks/Foundation.framework/*',
'/System/Library/Frameworks/GameController.framework/*',
'/System/Library/Frameworks/GLKit.framework/*',
'/System/Library/Frameworks/ImageIO.framework/*',
'/System/Library/Frameworks/IOBluetooth.framework/*',
'/System/Library/Frameworks/IOKit.framework/*',
'/System/Library/Frameworks/IOSurface.framework/*',
'/System/Library/Frameworks/OpenAL.framework/*',
'/System/Library/Frameworks/OpenGL.framework/*',
'/System/Library/Frameworks/Quartz.framework/*',
'/System/Library/Frameworks/QuartzCore.framework/*',
'/System/Library/Frameworks/Security.framework/*',
'/System/Library/Frameworks/StoreKit.framework/*',
'/System/Library/Frameworks/SystemConfiguration.framework/*',
'/System/Library/Frameworks/WebKit.framework/*']
whitelist += m.meta.get('build', {}).get('missing_dso_whitelist', [])
for f in files:
path = os.path.join(m.config.host_prefix, f)
if not codefile_type(path):
continue
warn_prelude = "WARNING ({},{})".format(pkg_name, f)
err_prelude = " ERROR ({},{})".format(pkg_name, f)
info_prelude = " INFO ({},{})".format(pkg_name, f)
msg_prelude = err_prelude if m.config.error_overlinking else warn_prelude
runpaths = get_runpaths(path)
if len(runpaths):
print_msg(errors, '{}: runpaths {} found in {}'.format(msg_prelude,
runpaths,
path))
needed = inspect_linkages(path, resolve_filenames=True, recurse=False)
for needed_dso in needed:
if needed_dso.startswith(m.config.host_prefix):
in_prefix_dso = os.path.normpath(needed_dso.replace(m.config.host_prefix + '/', ''))
n_dso_p = "Needed DSO {}".format(in_prefix_dso)
and_also = " (and also in this package)" if in_prefix_dso in files else ""
pkgs = list(which_package(in_prefix_dso, m.config.host_prefix))
in_pkgs_in_run_reqs = [pkg for pkg in pkgs if pkg.quad[0] in run_reqs]
in_whitelist = any([glob2.fnmatch.fnmatch(in_prefix_dso, w) for w in whitelist])
if in_whitelist:
print_msg(errors, '{}: {} found in the whitelist'.
format(info_prelude, n_dso_p))
elif len(in_pkgs_in_run_reqs) == 1 and m.config.verbose:
print_msg(errors, '{}: {} found in {}{}'.format(info_prelude,
n_dso_p,
in_pkgs_in_run_reqs[0],
and_also))
elif len(in_pkgs_in_run_reqs) == 0 and len(pkgs) > 0:
print_msg(errors, '{}: {} found in {}{}'.format(msg_prelude,
n_dso_p,
[p.quad[0] for p in pkgs],
and_also))
print_msg(errors, '{}: .. but {} not in reqs/run, i.e. it is overlinked'
' (likely) or a missing dependency (less likely)'.
format(msg_prelude, [p.quad[0] for p in pkgs]))
elif len(in_pkgs_in_run_reqs) > 1:
print_msg(errors, '{}: {} found in multiple packages in run/reqs: {}{}'
.format(warn_prelude,
in_prefix_dso,
[p.quad[0] for p in in_pkgs_in_run_reqs],
and_also))
else:
if in_prefix_dso not in files:
print_msg(errors, '{}: {} not found in any packages'.format(msg_prelude,
in_prefix_dso))
elif m.config.verbose:
print_msg(errors, '{}: {} found in this package'.format(info_prelude,
in_prefix_dso))
elif needed_dso.startswith(m.config.build_prefix):
print_msg(errors, "ERROR: {} found in build prefix; should never happen".format(
needed_dso))
else:
# A system or ignored dependency. We should be able to find it in one of the CDT o
# compiler packages on linux or at in a sysroot folder on other OSes. These usually
# start with '$RPATH/' which indicates pyldd did not find them, so remove that now.
if needed_dso.startswith('$RPATH/'):
needed_dso = needed_dso.replace('$RPATH/', '')
in_whitelist = any([glob2.fnmatch.fnmatch(needed_dso, w) for w in whitelist])
if in_whitelist:
n_dso_p = "Needed DSO {}".format(needed_dso)
print_msg(errors, '{}: {} found in the whitelist'.
format(info_prelude, n_dso_p))
elif m.config.verbose and len(sysroots):
# Check id we have a CDT package.
dso_fname = os.path.basename(needed_dso)
sysroot_files = []
for sysroot in sysroots:
sysroot_files.extend(glob(os.path.join(sysroot, '**', dso_fname)))
if len(sysroot_files):
# Removing config.build_prefix is only *really* for Linux, though we could
# use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about
# /opt/X11 too.
# Find the longest suffix match.
rev_needed_dso = needed_dso[::-1]
match_lens = [len(os.path.commonprefix([s[::-1], rev_needed_dso]))
for s in sysroot_files]
idx = max(range(len(match_lens)), key=match_lens.__getitem__)
in_prefix_dso = os.path.normpath(sysroot_files[idx].replace(
m.config.build_prefix + '/', ''))
n_dso_p = "Needed DSO {}".format(in_prefix_dso)
pkgs = list(which_package(in_prefix_dso, m.config.build_prefix))
if len(pkgs):
print_msg(errors, '{}: {} found in CDT/compiler package {}'.
format(info_prelude, n_dso_p, pkgs[0]))
else:
print_msg(errors, '{}: {} not found in any CDT/compiler package,'
' nor the whitelist?!'.
format(msg_prelude, n_dso_p))
else:
print_msg(errors, "{}: {} not found in sysroot, is this binary repackaging?"
" .. do you need to use install_name_tool/patchelf?".
format(msg_prelude, needed_dso))
else:
print_msg(errors, "{}: did not find - or even know where to look for: {}".
format(msg_prelude, needed_dso))
if len(errors):
sys.exit(1)
