def _lookup_in_system_whitelists(errors, whitelist, needed_dso, sysroots, msg_prelude, info_prelude, sysroot_prefix, sysroot_substitution, verbose): # A system or ignored dependency. We should be able to find it in one of the CDT o # compiler packages on linux or at in a sysroot folder on other OSes. These usually # start with '$RPATH/' which indicates pyldd did not find them, so remove that now. if needed_dso.startswith(sysroot_substitution): replacements = [sysroot_substitution] + sysroots else: replacements = [needed_dso] in_whitelist = False for replacement in replacements: needed_dso_w = needed_dso.replace(sysroot_substitution, replacement) in_whitelist = any([glob2.fnmatch.fnmatch(needed_dso_w, w) for w in whitelist]) if in_whitelist: n_dso_p = "Needed DSO {}".format(needed_dso_w) _print_msg(errors, '{}: {} found in the whitelist'. format(info_prelude, n_dso_p), verbose=verbose) break if not in_whitelist and len(sysroots): # Check if we have a CDT package. dso_fname = os.path.basename(needed_dso) sysroot_files = [] dirs_to_glob = [] # Optimization, ideally we'll not glob at all as it's slooow. for sysroot in sysroots: sysroot_os = sysroot.replace('/', os.sep) if needed_dso.startswith(sysroot_substitution): # Do we want to do this replace? sysroot_files.append(needed_dso.replace(sysroot_substitution, sysroot_os)) else: dirs_to_glob.extend((os.path.join(sysroot_os, '**', dso_fname))) for dir_to_glob in dirs_to_glob: sysroot_files.extend(glob(dir_to_glob)) if len(sysroot_files): # Removing sysroot_prefix is only *really* for Linux, though we could # use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about # /opt/X11 too. # Find the longest suffix match. rev_needed_dso = needed_dso[::-1] match_lens = [len(os.path.commonprefix([s[::-1], rev_needed_dso])) for s in sysroot_files] idx = max(range(len(match_lens)), key=match_lens.__getitem__) in_prefix_dso = os.path.normpath(sysroot_files[idx].replace( sysroot_prefix + os.sep, '')) n_dso_p = "Needed DSO {}".format(in_prefix_dso) pkgs = list(which_package(in_prefix_dso, sysroot_prefix)) if len(pkgs): _print_msg(errors, '{}: {} found in CDT/compiler package {}'. format(info_prelude, n_dso_p, pkgs[0]), verbose=verbose) else: _print_msg(errors, '{}: {} not found in any CDT/compiler package,' ' nor the whitelist?!'. format(msg_prelude, n_dso_p), verbose=verbose) else: _print_msg(errors, "{}: {} not found in sysroot, is this binary repackaging?" " .. do you need to use install_name_tool/patchelf?". format(msg_prelude, needed_dso), verbose=verbose) elif not in_whitelist: _print_msg(errors, "{}: did not find - or even know where to look for: {}". format(msg_prelude, needed_dso), verbose=verbose)
def _map_file_to_package(files, run_prefix, build_prefix, all_needed_dsos, pkg_vendored_dist, ignore_list_syms, sysroot_substitution): # Form a mapping of file => package prefix_owners = {} contains_dsos = {} contains_static_libs = {} # Used for both dsos and static_libs all_lib_exports = {} for prefix in (run_prefix, build_prefix): for subdir2, _, filez in os.walk(prefix): for file in filez: fp = os.path.join(subdir2, file) dynamic_lib = any(glob2.fnmatch.fnmatch(fp, ext) for ext in ('*.so*', '*.dylib*', '*.dll')) and \ codefile_type(fp, skip_symlinks=False) is not None static_lib = any(glob2.fnmatch.fnmatch(fp, ext) for ext in ('*.a', '*.lib')) # Looking at all the files is very slow. if not dynamic_lib and not static_lib: continue rp = os.path.relpath(fp, prefix) if dynamic_lib and rp not in all_needed_dsos: continue if rp in all_lib_exports: continue owners = prefix_owners[rp] if rp in prefix_owners else [] # Self-vendoring, not such a big deal but may as well report it? if not len(owners): if rp in files: owners.append(pkg_vendored_dist) new_pkgs = list(which_package(rp, prefix)) # Cannot filter here as this means the DSO (eg libomp.dylib) will not be found in any package # [owners.append(new_pkg) for new_pkg in new_pkgs if new_pkg not in owners # and not any([glob2.fnmatch.fnmatch(new_pkg.name, i) for i in ignore_for_statics])] for new_pkg in new_pkgs: if new_pkg not in owners: owners.append(new_pkg) prefix_owners[rp] = owners if len(prefix_owners[rp]): exports = set(e for e in get_exports_memoized(fp) if not any(glob2.fnmatch.fnmatch(e, pattern) for pattern in ignore_list_syms)) all_lib_exports[rp] = exports # Check codefile_type to filter out linker scripts. if dynamic_lib: contains_dsos[prefix_owners[rp][0]] = True elif static_lib: if sysroot_substitution in fp: if (prefix_owners[rp][0].name.startswith('gcc_impl_linux') or prefix_owners[rp][0].name == 'llvm'): continue print("sysroot in {}, owner is {}".format(fp, prefix_owners[rp][0])) contains_static_libs[prefix_owners[rp][0]] = True return prefix_owners, contains_dsos, contains_static_libs, all_lib_exports
def _find_needed_dso_in_system(m, needed_dso, errors, sysroots, msg_prelude, info_prelude, warn_prelude): # A system dependency then. We should be able to find it in one of the CDT o # compiler packages on linux or at in a sysroot folder on other OSes. # if m.config.verbose and len(sysroots): # Check id we have a CDT package. dso_fname = os.path.basename(needed_dso) sysroot_files = [] for sysroot in sysroots: sysroot_files.extend(glob(os.path.join(sysroot, '**', dso_fname))) if len(sysroot_files): # Removing config.build_prefix is only *really* for Linux, though we could # use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about # /opt/X11 too. in_prefix_dso = os.path.normpath(sysroot_files[0].replace( m.config.build_prefix + '/', '')) n_dso_p = "Needed DSO {}".format(in_prefix_dso) pkgs = list(which_package(in_prefix_dso, m.config.build_prefix)) if len(pkgs): print_msg( errors, '{}: {} found in CDT/compiler package {}'.format( info_prelude, n_dso_p, pkgs[0])) else: print_msg( errors, '{}: {} not found in any CDT/compiler package?!'.format( info_prelude, n_dso_p)) else: prelude = warn_prelude if needed_dso.startswith( '$RPATH') else msg_prelude print_msg( errors, "{}: {} not found in sysroot, is this binary repackaging?" " .. do you need to use install_name_tool/patchelf?".format( prelude, needed_dso)) else: # When a needed_dso begins with $RPATH it means we are making a CDT package # (in any other case this would be a problem), but I should verify it is ok # for CDT packages too. if needed_dso.startswith('$RPATH'): print_msg( errors, "{}: {} returned by pyldd. A CDT package?".format( warn_prelude, needed_dso)) else: print_msg( errors, "{}: did not find - or even know where to look for: {}".format( msg_prelude, needed_dso))
def _lookup_in_prefix_packages(errors, needed_dso, files, run_prefix, whitelist, info_prelude, msg_prelude, warn_prelude, verbose, requirements_run, lib_packages, lib_packages_used): in_prefix_dso = needed_dso if in_prefix_dso == '/': print('debug') # os.path.normpath(needed_dso.replace(run_prefix + os.sep, '')) n_dso_p = "Needed DSO {}".format(in_prefix_dso) and_also = " (and also in this package)" if in_prefix_dso in files else "" pkgs = list(which_package(in_prefix_dso, run_prefix)) in_pkgs_in_run_reqs = [pkg for pkg in pkgs if pkg.quad[0] in requirements_run] # TODO :: metadata build/inherit_child_run_exports (for vc, mro-base-impl). for pkg in in_pkgs_in_run_reqs: if pkg in lib_packages: lib_packages_used.add(pkg) in_whitelist = any([glob2.fnmatch.fnmatch(in_prefix_dso, w) for w in whitelist]) if len(in_pkgs_in_run_reqs) == 1: _print_msg(errors, '{}: {} found in {}{}'.format(info_prelude, n_dso_p, in_pkgs_in_run_reqs[0], and_also), verbose=verbose) elif in_whitelist: _print_msg(errors, '{}: {} found in the whitelist'. format(info_prelude, n_dso_p), verbose=verbose) elif len(in_pkgs_in_run_reqs) == 0 and len(pkgs) > 0: _print_msg(errors, '{}: {} found in {}{}'.format(msg_prelude, n_dso_p, [p.quad[0] for p in pkgs], and_also), verbose=verbose) _print_msg(errors, '{}: .. but {} not in reqs/run, (i.e. it is overlinking)' ' (likely) or a missing dependency (less likely)'. format(msg_prelude, [p.quad[0] for p in pkgs]), verbose=verbose) elif len(in_pkgs_in_run_reqs) > 1: _print_msg(errors, '{}: {} found in multiple packages in run/reqs: {}{}' .format(warn_prelude, in_prefix_dso, in_pkgs_in_run_reqs, and_also), verbose=verbose) else: if in_prefix_dso not in files: _print_msg(errors, '{}: {} not found in any packages'.format(msg_prelude, in_prefix_dso), verbose=verbose) elif verbose: _print_msg(errors, '{}: {} found in this package'.format(info_prelude, in_prefix_dso), verbose=verbose)
def _find_needed_dso_in_prefix(m, needed_dso, f, files, errors, run_reqs, host_reqs, msg_prelude, info_prelude): in_prefix_dso = os.path.normpath( needed_dso.replace(m.config.host_prefix + '/', '')) n_dso_p = "Needed DSO {} in {}".format(in_prefix_dso, f) and_also = " (and also in this package)" if in_prefix_dso in files else "" pkgs = list(which_package(in_prefix_dso, m.config.host_prefix)) if len(pkgs) == 1: if pkgs[0].quad[0] not in run_reqs: print_msg( errors, '{}: {} found in {}{}'.format(msg_prelude, n_dso_p, pkgs[0], and_also)) print_msg( errors, '{}: .. but {} not in reqs/run, i.e. it is overlinked' ' (likely) or a missing dependency (less likely)'.format( msg_prelude, pkgs[0].quad[0])) elif m.config.verbose: print_msg( errors, '{}: {} found in {}{}'.format(info_prelude, n_dso_p, pkgs[0], and_also)) elif len(pkgs) > 1: print_msg( errors, '{}: {} found in multiple packages:{}'.format( msg_prelude, in_prefix_dso, and_also)) for pkg in pkgs: print_msg(errors, '{}: {}'.format(msg_prelude, pkg)) if pkg.dist_name not in host_reqs: print_msg( errors, '{}: .. but {} not in reqs/host (is transitive)'.format( msg_prelude, pkg.dist_name)) else: if in_prefix_dso not in files: print_msg( errors, '{}: {} not found in any packages'.format( msg_prelude, in_prefix_dso)) elif m.config.verbose: print_msg( errors, '{}: {} found in this package'.format(info_prelude, in_prefix_dso))
def check_overlinking(m, files): pkg_name = m.get_value('package/name') errors = [] run_reqs = [req.split(' ')[0] for req in m.meta.get('requirements', {}).get('run', [])] # sysroots and whitelists are similar, but the subtle distinctions are important. sysroots = glob(os.path.join(m.config.build_prefix, '**', 'sysroot')) whitelist = [] if 'target_platform' in m.config.variant and m.config.variant['target_platform'] == 'osx-64': if not len(sysroots): sysroots = ['/usr/lib', '/opt/X11', '/System/Library/Frameworks'] whitelist = ['/opt/X11/', '/usr/lib/libSystem.B.dylib', '/usr/lib/libcrypto.0.9.8.dylib', '/usr/lib/libobjc.A.dylib', '/System/Library/Frameworks/Accelerate.framework/*', '/System/Library/Frameworks/AGL.framework/*', '/System/Library/Frameworks/AppKit.framework/*', '/System/Library/Frameworks/ApplicationServices.framework/*', '/System/Library/Frameworks/AudioToolbox.framework/*', '/System/Library/Frameworks/AudioUnit.framework/*', '/System/Library/Frameworks/AVFoundation.framework/*', '/System/Library/Frameworks/CFNetwork.framework/*', '/System/Library/Frameworks/Carbon.framework/*', '/System/Library/Frameworks/Cocoa.framework/*', '/System/Library/Frameworks/CoreAudio.framework/*', '/System/Library/Frameworks/CoreFoundation.framework/*', '/System/Library/Frameworks/CoreGraphics.framework/*', '/System/Library/Frameworks/CoreMedia.framework/*', '/System/Library/Frameworks/CoreBluetooth.framework/*', '/System/Library/Frameworks/CoreMIDI.framework/*', '/System/Library/Frameworks/CoreMedia.framework/*', '/System/Library/Frameworks/CoreServices.framework/*', '/System/Library/Frameworks/CoreText.framework/*', '/System/Library/Frameworks/CoreVideo.framework/*', '/System/Library/Frameworks/CoreWLAN.framework/*', '/System/Library/Frameworks/DiskArbitration.framework/*', '/System/Library/Frameworks/Foundation.framework/*', '/System/Library/Frameworks/GameController.framework/*', '/System/Library/Frameworks/GLKit.framework/*', '/System/Library/Frameworks/ImageIO.framework/*', '/System/Library/Frameworks/IOBluetooth.framework/*', '/System/Library/Frameworks/IOKit.framework/*', '/System/Library/Frameworks/IOSurface.framework/*', '/System/Library/Frameworks/OpenAL.framework/*', '/System/Library/Frameworks/OpenGL.framework/*', '/System/Library/Frameworks/Quartz.framework/*', '/System/Library/Frameworks/QuartzCore.framework/*', '/System/Library/Frameworks/Security.framework/*', '/System/Library/Frameworks/StoreKit.framework/*', '/System/Library/Frameworks/SystemConfiguration.framework/*', '/System/Library/Frameworks/WebKit.framework/*'] whitelist += m.meta.get('build', {}).get('missing_dso_whitelist', []) for f in files: path = os.path.join(m.config.host_prefix, f) if not codefile_type(path): continue warn_prelude = "WARNING ({},{})".format(pkg_name, f) err_prelude = " ERROR ({},{})".format(pkg_name, f) info_prelude = " INFO ({},{})".format(pkg_name, f) msg_prelude = err_prelude if m.config.error_overlinking else warn_prelude runpaths = get_runpaths(path) if len(runpaths): print_msg(errors, '{}: runpaths {} found in {}'.format(msg_prelude, runpaths, path)) needed = inspect_linkages(path, resolve_filenames=True, recurse=False) for needed_dso in needed: if needed_dso.startswith(m.config.host_prefix): in_prefix_dso = os.path.normpath(needed_dso.replace(m.config.host_prefix + '/', '')) n_dso_p = "Needed DSO {}".format(in_prefix_dso) and_also = " (and also in this package)" if in_prefix_dso in files else "" pkgs = list(which_package(in_prefix_dso, m.config.host_prefix)) in_pkgs_in_run_reqs = [pkg for pkg in pkgs if pkg.quad[0] in run_reqs] in_whitelist = any([glob2.fnmatch.fnmatch(in_prefix_dso, w) for w in whitelist]) if in_whitelist: print_msg(errors, '{}: {} found in the whitelist'. format(info_prelude, n_dso_p)) elif len(in_pkgs_in_run_reqs) == 1 and m.config.verbose: print_msg(errors, '{}: {} found in {}{}'.format(info_prelude, n_dso_p, in_pkgs_in_run_reqs[0], and_also)) elif len(in_pkgs_in_run_reqs) == 0 and len(pkgs) > 0: print_msg(errors, '{}: {} found in {}{}'.format(msg_prelude, n_dso_p, [p.quad[0] for p in pkgs], and_also)) print_msg(errors, '{}: .. but {} not in reqs/run, i.e. it is overlinked' ' (likely) or a missing dependency (less likely)'. format(msg_prelude, [p.quad[0] for p in pkgs])) elif len(in_pkgs_in_run_reqs) > 1: print_msg(errors, '{}: {} found in multiple packages in run/reqs: {}{}' .format(warn_prelude, in_prefix_dso, [p.quad[0] for p in in_pkgs_in_run_reqs], and_also)) else: if in_prefix_dso not in files: print_msg(errors, '{}: {} not found in any packages'.format(msg_prelude, in_prefix_dso)) elif m.config.verbose: print_msg(errors, '{}: {} found in this package'.format(info_prelude, in_prefix_dso)) elif needed_dso.startswith(m.config.build_prefix): print_msg(errors, "ERROR: {} found in build prefix; should never happen".format( needed_dso)) else: # A system or ignored dependency. We should be able to find it in one of the CDT o # compiler packages on linux or at in a sysroot folder on other OSes. These usually # start with '$RPATH/' which indicates pyldd did not find them, so remove that now. if needed_dso.startswith('$RPATH/'): needed_dso = needed_dso.replace('$RPATH/', '') in_whitelist = any([glob2.fnmatch.fnmatch(needed_dso, w) for w in whitelist]) if in_whitelist: n_dso_p = "Needed DSO {}".format(needed_dso) print_msg(errors, '{}: {} found in the whitelist'. format(info_prelude, n_dso_p)) elif m.config.verbose and len(sysroots): # Check id we have a CDT package. dso_fname = os.path.basename(needed_dso) sysroot_files = [] for sysroot in sysroots: sysroot_files.extend(glob(os.path.join(sysroot, '**', dso_fname))) if len(sysroot_files): # Removing config.build_prefix is only *really* for Linux, though we could # use CONDA_BUILD_SYSROOT for macOS. We should figure out what to do about # /opt/X11 too. # Find the longest suffix match. rev_needed_dso = needed_dso[::-1] match_lens = [len(os.path.commonprefix([s[::-1], rev_needed_dso])) for s in sysroot_files] idx = max(range(len(match_lens)), key=match_lens.__getitem__) in_prefix_dso = os.path.normpath(sysroot_files[idx].replace( m.config.build_prefix + '/', '')) n_dso_p = "Needed DSO {}".format(in_prefix_dso) pkgs = list(which_package(in_prefix_dso, m.config.build_prefix)) if len(pkgs): print_msg(errors, '{}: {} found in CDT/compiler package {}'. format(info_prelude, n_dso_p, pkgs[0])) else: print_msg(errors, '{}: {} not found in any CDT/compiler package,' ' nor the whitelist?!'. format(msg_prelude, n_dso_p)) else: print_msg(errors, "{}: {} not found in sysroot, is this binary repackaging?" " .. do you need to use install_name_tool/patchelf?". format(msg_prelude, needed_dso)) else: print_msg(errors, "{}: did not find - or even know where to look for: {}". format(msg_prelude, needed_dso)) if len(errors): sys.exit(1)