def get_linkages(obj_files, prefix, sysroot):
res = {}
for f in obj_files:
path = join(prefix, f)
# ldd quite often fails on foreign architectures.
ldd_failed = False
try:
if sys.platform.startswith('linux'):
res[f] = ldd(path)
elif sys.platform.startswith('darwin'):
links = otool(path)
res[f] = [(basename(l['name']), l['name']) for l in links]
except:
ldd_failed = True
finally:
res_py = inspect_linkages(path, sysroot=sysroot)
res_py = [(basename(lp), lp) for lp in res_py]
print("set(res_py) {}".format(set(res_py)))
if ldd_failed:
res[f] = res_py
else:
print("set(res[f]) = {}".format(set(res[f])))
if set(res[f]) != set(res_py):
print("WARNING: pyldd disagrees with ldd/otool. This will not cause any")
print("WARNING: problems for this build, but please file a bug at:")
print("WARNING: https://github.com/conda/conda-build")
print("WARNING: and (if possible) attach file {}".format(path))
print("WARNING: ldd/tool gives {}, pyldd gives {}"
.format(set(res[f]), set(res_py)))
return res
def get_linkages(obj_files, prefix, sysroot):
res = {}
for f in obj_files:
path = join(prefix, f)
# ldd quite often fails on foreign architectures.
ldd_failed = False
# Detect the filetype to emulate what the system-native tool does.
klass = codefile_class(path)
if klass == machofile:
resolve_filenames = False
recurse = False
else:
resolve_filenames = True
recurse = True
try:
if sys.platform.startswith('linux'):
res[f] = ldd(path)
elif sys.platform.startswith('darwin'):
links = otool(path)
res[f] = [(basename(line['name']), line['name'])
for line in links]
except:
ldd_failed = True
finally:
res_py = inspect_linkages(path,
resolve_filenames=resolve_filenames,
sysroot=sysroot,
recurse=recurse)
res_py = [(basename(lp), lp) for lp in res_py]
if ldd_failed:
res[f] = res_py
else:
if set(res[f]) != set(res_py):
print(
"WARNING: pyldd disagrees with ldd/otool. This will not cause any"
)
print(
"WARNING: problems for this build, but please file a bug at:"
)
print("WARNING: https://github.com/conda/conda-build")
print("WARNING: and (if possible) attach file {}".format(
path))
print(
"WARNING: \nldd/otool gives:\n{}\npyldd gives:\n{}\n".
format("\n".join(str(e) for e in res[f]),
"\n".join(str(e) for e in res_py)))
print("Diffs\n{}".format(set(res[f]) - set(res_py)))
print("Diffs\n{}".format(set(res_py) - set(res[f])))
return res
def _inspect_file_linking(m, path, files, errors, pkg_name, run_reqs, host_reqs, sysroots):
f = os.path.basename(path)
warn_prelude = "WARNING ({},{})".format(pkg_name, f)
err_prelude = " ERROR ({},{})".format(pkg_name, f)
info_prelude = " INFO ({},{})".format(pkg_name, f)
msg_prelude = err_prelude if m.config.error_overlinking else warn_prelude
needed = inspect_linkages(path, resolve_filenames=True, recurse=False)
for needed_dso in needed:
if needed_dso.startswith(m.config.host_prefix):
_find_needed_dso_in_prefix(m, needed_dso, f, files, errors, run_reqs, host_reqs,
msg_prelude, info_prelude)
elif needed_dso.startswith(m.config.build_prefix):
print_msg(errors, "ERROR: {} found in build prefix; should never happen".format(
needed_dso))
else:
_find_needed_dso_in_system(m, needed_dso, errors, sysroots, msg_prelude, info_prelude,
warn_prelude)
def get_linkages(obj_files, prefix, sysroot):
res = {}
for f in obj_files:
path = join(prefix, f)
# ldd quite often fails on foreign architectures.
ldd_failed = False
# Detect the filetype to emulate what the system-native tool does.
klass = codefile_class(path)
if klass == machofile:
resolve_filenames = False
recurse = False
else:
resolve_filenames = True
recurse = True
try:
if sys.platform.startswith('linux'):
res[f] = ldd(path)
elif sys.platform.startswith('darwin'):
links = otool(path)
res[f] = [(basename(l['name']), l['name']) for l in links]
except:
ldd_failed = True
finally:
res_py = inspect_linkages(path, resolve_filenames=resolve_filenames,
sysroot=sysroot, recurse=recurse)
res_py = [(basename(lp), lp) for lp in res_py]
if ldd_failed:
res[f] = res_py
else:
if set(res[f]) != set(res_py):
print("WARNING: pyldd disagrees with ldd/otool. This will not cause any")
print("WARNING: problems for this build, but please file a bug at:")
print("WARNING: https://github.com/conda/conda-build")
print("WARNING: and (if possible) attach file {}".format(path))
print("WARNING: \nldd/otool gives:\n{}\npyldd gives:\n{}\n"
.format("\n".join(str(e) for e in res[f]), "\n".join(str(e)
for e in res_py)))
print("Diffs\n{}".format(set(res[f]) - set(res_py)))
print("Diffs\n{}".format(set(res_py) - set(res[f])))
return res
def check_overlinking(m, files):
pkg_name = m.get_value('package/name')
errors = []
run_reqs = [req.split(' ')[0] for req in m.meta.get('requirements', {}).get('run', [])]
# sysroots and whitelists are similar, but the subtle distinctions are important.
sysroots = glob(os.path.join(m.config.build_prefix, '**', 'sysroot'))
whitelist = []
if 'target_platform' in m.config.variant and m.config.variant['target_platform'] == 'osx-64':
if not len(sysroots):
sysroots = ['/usr/lib', '/opt/X11', '/System/Library/Frameworks']
whitelist = ['/opt/X11/',
'/usr/lib/libSystem.B.dylib',
'/usr/lib/libcrypto.0.9.8.dylib',
'/usr/lib/libobjc.A.dylib',
'/System/Library/Frameworks/Accelerate.framework/*',
'/System/Library/Frameworks/AGL.framework/*',
'/System/Library/Frameworks/AppKit.framework/*',
'/System/Library/Frameworks/ApplicationServices.framework/*',
'/System/Library/Frameworks/AudioToolbox.framework/*',
'/System/Library/Frameworks/AudioUnit.framework/*',
'/System/Library/Frameworks/AVFoundation.framework/*',
'/System/Library/Frameworks/CFNetwork.framework/*',
'/System/Library/Frameworks/Carbon.framework/*',
'/System/Library/Frameworks/Cocoa.framework/*',
'/System/Library/Frameworks/CoreAudio.framework/*',
'/System/Library/Frameworks/CoreFoundation.framework/*',
'/System/Library/Frameworks/CoreGraphics.framework/*',
'/System/Library/Frameworks/CoreMedia.framework/*',
'/System/Library/Frameworks/CoreBluetooth.framework/*',
'/System/Library/Frameworks/CoreMIDI.framework/*',
'/System/Library/Frameworks/CoreMedia.framework/*',
'/System/Library/Frameworks/CoreServices.framework/*',
'/System/Library/Frameworks/CoreText.framework/*',
'/System/Library/Frameworks/CoreVideo.framework/*',
'/System/Library/Frameworks/CoreWLAN.framework/*',
'/System/Library/Frameworks/DiskArbitration.framework/*',
'/System/Library/Frameworks/Foundation.framework/*',
'/System/Library/Frameworks/GameController.framework/*',
'/System/Library/Frameworks/GLKit.framework/*',
'/System/Library/Frameworks/ImageIO.framework/*',
'/System/Library/Frameworks/IOBluetooth.framework/*',
'/System/Library/Frameworks/IOKit.framework/*',
'/System/Library/Frameworks/IOSurface.framework/*',
'/System/Library/Frameworks/OpenAL.framework/*',
'/System/Library/Frameworks/OpenGL.framework/*',
'/System/Library/Frameworks/Quartz.framework/*',
'/System/Library/Frameworks/QuartzCore.framework/*',
'/System/Library/Frameworks/Security.framework/*',
'/System/Library/Frameworks/StoreKit.framework/*',
'/System/Library/Frameworks/SystemConfiguration.framework/*',
'/System/Library/Frameworks/WebKit.framework/*']
whitelist += m.meta.get('build', {}).get('missing_dso_whitelist', [])
for f in files:
path = os.path.join(m.config.host_prefix, f)
if not codefile_type(path):
continue
warn_prelude = "WARNING ({},{})".format(pkg_name, f)
err_prelude = " ERROR ({},{})".format(pkg_name, f)
info_prelude = " INFO ({},{})".format(pkg_name, f)
msg_prelude = err_prelude if m.config.error_overlinking else warn_prelude
runpaths = get_runpaths(path)
if len(runpaths):
print_msg(errors, '{}: runpaths {} found in {}'.format(msg_prelude,
runpaths,
path))
needed = inspect_linkages(path, resolve_filenames=True, recurse=False)
for needed_dso in needed:
if needed_dso.startswith(m.config.host_prefix):
in_prefix_dso = os.path.normpath(needed_dso.replace(m.config.host_prefix + '/', ''))
n_dso_p = "Needed DSO {}".format(in_prefix_dso)
and_also = " (and also in this package)" if in_prefix_dso in files else ""
pkgs = list(which_package(in_prefix_dso, m.config.host_prefix))
in_pkgs_in_run_reqs = [pkg for pkg in pkgs if pkg.quad[0] in run_reqs]
in_whitelist = any([glob2.fnmatch.fnmatch(in_prefix_dso, w) for w in whitelist])
if in_whitelist:
print_msg(errors, '{}: {} found in the whitelist'.
format(info_prelude, n_dso_p))
elif len(in_pkgs_in_run_reqs) == 1 and m.config.verbose:
print_msg(errors, '{}: {} found in {}{}'.format(info_prelude,
n_dso_p,
in_pkgs_in_run_reqs[0],
and_also))
elif len(in_pkgs_in_run_reqs) == 0 and len(pkgs) > 0:
print_msg(errors, '{}: {} found in {}{}'.format(msg_prelude,
n_dso_p,
[p.quad[0] for p in pkgs],
and_also))
print_msg(errors, '{}: .. but {} not in reqs/run, i.e. it is overlinked'
' (likely) or a missing dependency (less likely)'.
format(msg_prelude, [p.quad[0] for p in pkgs]))
elif len(in_pkgs_in_run_reqs) > 1:
print_msg(errors, '{}: {} found in multiple packages in run/reqs: {}{}'
.format(warn_prelude,
in_prefix_dso,
[p.quad[0] for p in in_pkgs_in_run_reqs],
and_also))
else:
if in_prefix_dso not in files:
print_msg(errors, '{}: {} not found in any packages'.format(msg_prelude,
in_prefix_dso))
elif m.config.verbose:
print_msg(errors, '{}: {} found in this package'.format(info_prelude,
in_prefix_dso))
elif needed_dso.startswith(m.config.build_prefix):
print_msg(errors, "ERROR: {} found in build prefix; should never happen".format(
needed_dso))
else:
# A system or ignored dependency. We should be able to find it in one of the CDT o
# compiler packages on linux or at in a sysroot folder on other OSes. These usually
# start with '$RPATH/' which indicates pyldd did not find them, so remove that now.
if needed_dso.startswith('$RPATH/'):
needed_dso = needed_dso.replace('$RPATH/', '')
in_whitelist = any([glob2.fnmatch.fnmatch(needed_dso, w) for w in whitelist])
if in_whitelist:
n_dso_p = "Needed DSO {}".format(needed_dso)
print_msg(errors, '{}: {} found in the whitelist'.
format(info_prelude, n_dso_p))
elif m.config.verbose and len(sysroots):
# Check id we have a CDT package.
dso_fname = os.path.basename(needed_dso)
sysroot_files = []
for sysroot in sysroots:
sysroot_files.extend(glob(os.path.join(sysroot, '**', dso_fname)))
if len(sysroot_files):
# Removing config.build_prefix is only *really* for Linux, though we could
# use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about
# /opt/X11 too.
# Find the longest suffix match.
rev_needed_dso = needed_dso[::-1]
match_lens = [len(os.path.commonprefix([s[::-1], rev_needed_dso]))
for s in sysroot_files]
idx = max(range(len(match_lens)), key=match_lens.__getitem__)
in_prefix_dso = os.path.normpath(sysroot_files[idx].replace(
m.config.build_prefix + '/', ''))
n_dso_p = "Needed DSO {}".format(in_prefix_dso)
pkgs = list(which_package(in_prefix_dso, m.config.build_prefix))
if len(pkgs):
print_msg(errors, '{}: {} found in CDT/compiler package {}'.
format(info_prelude, n_dso_p, pkgs[0]))
else:
print_msg(errors, '{}: {} not found in any CDT/compiler package,'
' nor the whitelist?!'.
format(msg_prelude, n_dso_p))
else:
print_msg(errors, "{}: {} not found in sysroot, is this binary repackaging?"
" .. do you need to use install_name_tool/patchelf?".
format(msg_prelude, needed_dso))
else:
print_msg(errors, "{}: did not find - or even know where to look for: {}".
format(msg_prelude, needed_dso))
if len(errors):
sys.exit(1)
