def get_linkages(obj_files, prefix, sysroot): res = {} for f in obj_files: path = join(prefix, f) # ldd quite often fails on foreign architectures. ldd_failed = False try: if sys.platform.startswith('linux'): res[f] = ldd(path) elif sys.platform.startswith('darwin'): links = otool(path) res[f] = [(basename(l['name']), l['name']) for l in links] except: ldd_failed = True finally: res_py = inspect_linkages(path, sysroot=sysroot) res_py = [(basename(lp), lp) for lp in res_py] print("set(res_py) {}".format(set(res_py))) if ldd_failed: res[f] = res_py else: print("set(res[f]) = {}".format(set(res[f]))) if set(res[f]) != set(res_py): print("WARNING: pyldd disagrees with ldd/otool. This will not cause any") print("WARNING: problems for this build, but please file a bug at:") print("WARNING: https://github.com/conda/conda-build") print("WARNING: and (if possible) attach file {}".format(path)) print("WARNING: ldd/tool gives {}, pyldd gives {}" .format(set(res[f]), set(res_py))) return res
def get_linkages(obj_files, prefix, sysroot): res = {} for f in obj_files: path = join(prefix, f) # ldd quite often fails on foreign architectures. ldd_failed = False # Detect the filetype to emulate what the system-native tool does. klass = codefile_class(path) if klass == machofile: resolve_filenames = False recurse = False else: resolve_filenames = True recurse = True try: if sys.platform.startswith('linux'): res[f] = ldd(path) elif sys.platform.startswith('darwin'): links = otool(path) res[f] = [(basename(line['name']), line['name']) for line in links] except: ldd_failed = True finally: res_py = inspect_linkages(path, resolve_filenames=resolve_filenames, sysroot=sysroot, recurse=recurse) res_py = [(basename(lp), lp) for lp in res_py] if ldd_failed: res[f] = res_py else: if set(res[f]) != set(res_py): print( "WARNING: pyldd disagrees with ldd/otool. This will not cause any" ) print( "WARNING: problems for this build, but please file a bug at:" ) print("WARNING: https://github.com/conda/conda-build") print("WARNING: and (if possible) attach file {}".format( path)) print( "WARNING: \nldd/otool gives:\n{}\npyldd gives:\n{}\n". format("\n".join(str(e) for e in res[f]), "\n".join(str(e) for e in res_py))) print("Diffs\n{}".format(set(res[f]) - set(res_py))) print("Diffs\n{}".format(set(res_py) - set(res[f]))) return res
def _inspect_file_linking(m, path, files, errors, pkg_name, run_reqs, host_reqs, sysroots): f = os.path.basename(path) warn_prelude = "WARNING ({},{})".format(pkg_name, f) err_prelude = " ERROR ({},{})".format(pkg_name, f) info_prelude = " INFO ({},{})".format(pkg_name, f) msg_prelude = err_prelude if m.config.error_overlinking else warn_prelude needed = inspect_linkages(path, resolve_filenames=True, recurse=False) for needed_dso in needed: if needed_dso.startswith(m.config.host_prefix): _find_needed_dso_in_prefix(m, needed_dso, f, files, errors, run_reqs, host_reqs, msg_prelude, info_prelude) elif needed_dso.startswith(m.config.build_prefix): print_msg(errors, "ERROR: {} found in build prefix; should never happen".format( needed_dso)) else: _find_needed_dso_in_system(m, needed_dso, errors, sysroots, msg_prelude, info_prelude, warn_prelude)
def get_linkages(obj_files, prefix, sysroot): res = {} for f in obj_files: path = join(prefix, f) # ldd quite often fails on foreign architectures. ldd_failed = False # Detect the filetype to emulate what the system-native tool does. klass = codefile_class(path) if klass == machofile: resolve_filenames = False recurse = False else: resolve_filenames = True recurse = True try: if sys.platform.startswith('linux'): res[f] = ldd(path) elif sys.platform.startswith('darwin'): links = otool(path) res[f] = [(basename(l['name']), l['name']) for l in links] except: ldd_failed = True finally: res_py = inspect_linkages(path, resolve_filenames=resolve_filenames, sysroot=sysroot, recurse=recurse) res_py = [(basename(lp), lp) for lp in res_py] if ldd_failed: res[f] = res_py else: if set(res[f]) != set(res_py): print("WARNING: pyldd disagrees with ldd/otool. This will not cause any") print("WARNING: problems for this build, but please file a bug at:") print("WARNING: https://github.com/conda/conda-build") print("WARNING: and (if possible) attach file {}".format(path)) print("WARNING: \nldd/otool gives:\n{}\npyldd gives:\n{}\n" .format("\n".join(str(e) for e in res[f]), "\n".join(str(e) for e in res_py))) print("Diffs\n{}".format(set(res[f]) - set(res_py))) print("Diffs\n{}".format(set(res_py) - set(res[f]))) return res
def check_overlinking(m, files): pkg_name = m.get_value('package/name') errors = [] run_reqs = [req.split(' ')[0] for req in m.meta.get('requirements', {}).get('run', [])] # sysroots and whitelists are similar, but the subtle distinctions are important. sysroots = glob(os.path.join(m.config.build_prefix, '**', 'sysroot')) whitelist = [] if 'target_platform' in m.config.variant and m.config.variant['target_platform'] == 'osx-64': if not len(sysroots): sysroots = ['/usr/lib', '/opt/X11', '/System/Library/Frameworks'] whitelist = ['/opt/X11/', '/usr/lib/libSystem.B.dylib', '/usr/lib/libcrypto.0.9.8.dylib', '/usr/lib/libobjc.A.dylib', '/System/Library/Frameworks/Accelerate.framework/*', '/System/Library/Frameworks/AGL.framework/*', '/System/Library/Frameworks/AppKit.framework/*', '/System/Library/Frameworks/ApplicationServices.framework/*', '/System/Library/Frameworks/AudioToolbox.framework/*', '/System/Library/Frameworks/AudioUnit.framework/*', '/System/Library/Frameworks/AVFoundation.framework/*', '/System/Library/Frameworks/CFNetwork.framework/*', '/System/Library/Frameworks/Carbon.framework/*', '/System/Library/Frameworks/Cocoa.framework/*', '/System/Library/Frameworks/CoreAudio.framework/*', '/System/Library/Frameworks/CoreFoundation.framework/*', '/System/Library/Frameworks/CoreGraphics.framework/*', '/System/Library/Frameworks/CoreMedia.framework/*', '/System/Library/Frameworks/CoreBluetooth.framework/*', '/System/Library/Frameworks/CoreMIDI.framework/*', '/System/Library/Frameworks/CoreMedia.framework/*', '/System/Library/Frameworks/CoreServices.framework/*', '/System/Library/Frameworks/CoreText.framework/*', '/System/Library/Frameworks/CoreVideo.framework/*', '/System/Library/Frameworks/CoreWLAN.framework/*', '/System/Library/Frameworks/DiskArbitration.framework/*', '/System/Library/Frameworks/Foundation.framework/*', '/System/Library/Frameworks/GameController.framework/*', '/System/Library/Frameworks/GLKit.framework/*', '/System/Library/Frameworks/ImageIO.framework/*', '/System/Library/Frameworks/IOBluetooth.framework/*', '/System/Library/Frameworks/IOKit.framework/*', '/System/Library/Frameworks/IOSurface.framework/*', '/System/Library/Frameworks/OpenAL.framework/*', '/System/Library/Frameworks/OpenGL.framework/*', '/System/Library/Frameworks/Quartz.framework/*', '/System/Library/Frameworks/QuartzCore.framework/*', '/System/Library/Frameworks/Security.framework/*', '/System/Library/Frameworks/StoreKit.framework/*', '/System/Library/Frameworks/SystemConfiguration.framework/*', '/System/Library/Frameworks/WebKit.framework/*'] whitelist += m.meta.get('build', {}).get('missing_dso_whitelist', []) for f in files: path = os.path.join(m.config.host_prefix, f) if not codefile_type(path): continue warn_prelude = "WARNING ({},{})".format(pkg_name, f) err_prelude = " ERROR ({},{})".format(pkg_name, f) info_prelude = " INFO ({},{})".format(pkg_name, f) msg_prelude = err_prelude if m.config.error_overlinking else warn_prelude runpaths = get_runpaths(path) if len(runpaths): print_msg(errors, '{}: runpaths {} found in {}'.format(msg_prelude, runpaths, path)) needed = inspect_linkages(path, resolve_filenames=True, recurse=False) for needed_dso in needed: if needed_dso.startswith(m.config.host_prefix): in_prefix_dso = os.path.normpath(needed_dso.replace(m.config.host_prefix + '/', '')) n_dso_p = "Needed DSO {}".format(in_prefix_dso) and_also = " (and also in this package)" if in_prefix_dso in files else "" pkgs = list(which_package(in_prefix_dso, m.config.host_prefix)) in_pkgs_in_run_reqs = [pkg for pkg in pkgs if pkg.quad[0] in run_reqs] in_whitelist = any([glob2.fnmatch.fnmatch(in_prefix_dso, w) for w in whitelist]) if in_whitelist: print_msg(errors, '{}: {} found in the whitelist'. format(info_prelude, n_dso_p)) elif len(in_pkgs_in_run_reqs) == 1 and m.config.verbose: print_msg(errors, '{}: {} found in {}{}'.format(info_prelude, n_dso_p, in_pkgs_in_run_reqs[0], and_also)) elif len(in_pkgs_in_run_reqs) == 0 and len(pkgs) > 0: print_msg(errors, '{}: {} found in {}{}'.format(msg_prelude, n_dso_p, [p.quad[0] for p in pkgs], and_also)) print_msg(errors, '{}: .. but {} not in reqs/run, i.e. it is overlinked' ' (likely) or a missing dependency (less likely)'. format(msg_prelude, [p.quad[0] for p in pkgs])) elif len(in_pkgs_in_run_reqs) > 1: print_msg(errors, '{}: {} found in multiple packages in run/reqs: {}{}' .format(warn_prelude, in_prefix_dso, [p.quad[0] for p in in_pkgs_in_run_reqs], and_also)) else: if in_prefix_dso not in files: print_msg(errors, '{}: {} not found in any packages'.format(msg_prelude, in_prefix_dso)) elif m.config.verbose: print_msg(errors, '{}: {} found in this package'.format(info_prelude, in_prefix_dso)) elif needed_dso.startswith(m.config.build_prefix): print_msg(errors, "ERROR: {} found in build prefix; should never happen".format( needed_dso)) else: # A system or ignored dependency. We should be able to find it in one of the CDT o # compiler packages on linux or at in a sysroot folder on other OSes. These usually # start with '$RPATH/' which indicates pyldd did not find them, so remove that now. if needed_dso.startswith('$RPATH/'): needed_dso = needed_dso.replace('$RPATH/', '') in_whitelist = any([glob2.fnmatch.fnmatch(needed_dso, w) for w in whitelist]) if in_whitelist: n_dso_p = "Needed DSO {}".format(needed_dso) print_msg(errors, '{}: {} found in the whitelist'. format(info_prelude, n_dso_p)) elif m.config.verbose and len(sysroots): # Check id we have a CDT package. dso_fname = os.path.basename(needed_dso) sysroot_files = [] for sysroot in sysroots: sysroot_files.extend(glob(os.path.join(sysroot, '**', dso_fname))) if len(sysroot_files): # Removing config.build_prefix is only *really* for Linux, though we could # use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about # /opt/X11 too. # Find the longest suffix match. rev_needed_dso = needed_dso[::-1] match_lens = [len(os.path.commonprefix([s[::-1], rev_needed_dso])) for s in sysroot_files] idx = max(range(len(match_lens)), key=match_lens.__getitem__) in_prefix_dso = os.path.normpath(sysroot_files[idx].replace( m.config.build_prefix + '/', '')) n_dso_p = "Needed DSO {}".format(in_prefix_dso) pkgs = list(which_package(in_prefix_dso, m.config.build_prefix)) if len(pkgs): print_msg(errors, '{}: {} found in CDT/compiler package {}'. format(info_prelude, n_dso_p, pkgs[0])) else: print_msg(errors, '{}: {} not found in any CDT/compiler package,' ' nor the whitelist?!'. format(msg_prelude, n_dso_p)) else: print_msg(errors, "{}: {} not found in sysroot, is this binary repackaging?" " .. do you need to use install_name_tool/patchelf?". format(msg_prelude, needed_dso)) else: print_msg(errors, "{}: did not find - or even know where to look for: {}". format(msg_prelude, needed_dso)) if len(errors): sys.exit(1)