def getNumpyCoreBinaries(module):
"""Return any binaries in numpy/core and/or numpy/.libs.
Notes:
This covers the special cases like MKL binaries.
Returns:
tuple of abspaths of binaries.
"""
numpy_dir = module.getCompileTimeDirectory()
numpy_core_dir = os.path.join(numpy_dir, "core")
base_prefix = getSystemPrefixPath()
binaries = []
# first look in numpy/.libs for binaries
libdir = os.path.join(numpy_dir,
".libs" if getOS() != "Darwin" else ".dylibs")
suffix_start = len(libdir) + 1
if os.path.isdir(libdir):
dlls_pkg = os.listdir(libdir)
binaries += [[os.path.join(libdir, f), suffix_start] for f in dlls_pkg]
# then look for libraries in numpy.core package path
# should already return the MKL files in ordinary cases
re_anylib = re.compile(r"\w+\.(?:dll|so|dylib)", re.IGNORECASE)
dlls_pkg = [f for f in os.listdir(numpy_core_dir) if re_anylib.match(f)]
binaries += [[os.path.join(numpy_core_dir, f), suffix_start]
for f in dlls_pkg]
# Also look for MKL libraries in folder "above" numpy.
# This should meet the layout of Anaconda installs.
if isWin32Windows():
lib_dir = os.path.join(base_prefix, "Library", "bin")
suffix_start = len(lib_dir) + 1
else:
lib_dir = os.path.join(base_prefix, "lib")
suffix_start = len(lib_dir) + 1
if not os.path.isdir(lib_dir):
return binaries
re_mkllib = re.compile(r"^(?:lib)?mkl\w+\.(?:dll|so|dylib)", re.IGNORECASE)
for f in os.listdir(lib_dir):
if isWin32Windows():
if not (f.startswith(
("libi", "libm", "mkl")) and f.endswith(".dll")):
continue
else:
if not re_mkllib.match(f):
continue
binaries.append([os.path.join(lib_dir, f), suffix_start])
return binaries
def _getNumpyCoreBinaries(numpy_dir):
"""Return any binaries in numpy package.
Notes:
This covers the special cases like MKL binaries.
Returns:
tuple of abspaths of binaries.
"""
numpy_core_dir = os.path.join(numpy_dir, "core")
# first look in numpy/.libs for binaries
libdir = os.path.join(numpy_dir, ".libs" if not isMacOS() else ".dylibs")
if os.path.isdir(libdir):
for full_path, filename in listDir(libdir):
yield full_path, filename
# Then look for libraries in numpy.core package path
# should already return the MKL files in ordinary cases
re_anylib = re.compile(r"\w+\.(?:dll|so|dylib)", re.IGNORECASE)
for full_path, filename in listDir(numpy_core_dir):
if not re_anylib.match(filename):
continue
yield full_path, filename
# Also look for MKL libraries in folder "above" numpy.
# This should meet the layout of Anaconda installs.
base_prefix = getSystemPrefixPath()
if isWin32Windows():
lib_dir = os.path.join(base_prefix, "Library", "bin")
else:
lib_dir = os.path.join(base_prefix, "lib")
if os.path.isdir(lib_dir):
re_mkllib = re.compile(r"^(?:lib)?mkl\w+\.(?:dll|so|dylib)", re.IGNORECASE)
for full_path, filename in listDir(lib_dir):
if isWin32Windows():
if not (
filename.startswith(("libi", "libm", "mkl"))
and filename.endswith(".dll")
):
continue
else:
if not re_mkllib.match(filename):
continue
yield full_path, filename
def _getSystemStaticLibPythonPath():
# Return driven function with many cases, pylint: disable=too-many-branches,too-many-return-statements
sys_prefix = getSystemPrefixPath()
python_abi_version = python_version_str + getPythonABI()
if isNuitkaPython():
# Nuitka Python has this.
if isWin32Windows():
return os.path.join(
sys_prefix,
"libs",
"python" + python_abi_version.replace(".", "") + ".lib",
)
else:
return os.path.join(
sys_prefix,
"lib",
"libpython" + python_abi_version + ".a",
)
if isWin32Windows():
candidates = [
# Anaconda has this.
os.path.join(
sys_prefix,
"libs",
"libpython" + python_abi_version.replace(".", "") + ".dll.a",
),
# MSYS2 mingw64 Python has this.
os.path.join(
sys_prefix,
"lib",
"libpython" + python_abi_version + ".dll.a",
),
]
for candidate in candidates:
if os.path.exists(candidate):
return candidate
else:
candidate = os.path.join(
sys_prefix, "lib", "libpython" + python_abi_version + ".a"
)
if os.path.exists(candidate):
return candidate
# For Python2 this works. TODO: Figure out Debian and Python3.
if (
python_version < 0x300
and isDebianPackagePython()
and isDebianSuitableForStaticLinking()
):
candidate = locateStaticLinkLibrary("python" + python_abi_version)
else:
candidate = None
if candidate is not None and os.path.exists(candidate):
# Also check libz, can be missing
if not locateStaticLinkLibrary("z"):
general.warning(
"Error, missing libz-dev installation needed for static lib-python."
)
return candidate
# This is not necessarily only for Python3 on Debian, but maybe others as well,
# but that's what's been tested.
if python_version >= 0x300 and isDebianPackagePython() and isDebianBasedLinux():
try:
import sysconfig
candidate = os.path.join(
sysconfig.get_config_var("LIBPL"),
"libpython" + python_abi_version + "-pic.a",
)
if os.path.exists(candidate):
return candidate
except ImportError:
# Cannot detect this properly for Python 2.6, but we don't care much
# about that anyway.
pass
return None
def runSconsBackend(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=False),
"source_dir": OutputDirectories.getSourceDirectoryPath(),
"nuitka_python": asBoolStr(isNuitkaPython()),
"debug_mode": asBoolStr(Options.is_debug),
"python_debug": asBoolStr(Options.isPythonDebug()),
"module_mode": asBoolStr(Options.shallMakeModule()),
"full_compat": asBoolStr(Options.is_fullcompat),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"python_version": python_version_str,
"target_arch": getArchitecture(),
"python_prefix": getDirectoryRealPath(getSystemPrefixPath()),
"nuitka_src": SconsInterface.getSconsDataPath(),
"module_count": "%d"
% (
1
+ len(ModuleRegistry.getDoneModules())
+ len(ModuleRegistry.getUncompiledNonTechnicalModules())
),
}
if Options.isLowMemory():
options["low_memory"] = asBoolStr(True)
if not Options.shallMakeModule():
options["result_exe"] = OutputDirectories.getResultFullpath(onefile=False)
main_module = ModuleRegistry.getRootTopModule()
assert main_module.isMainModule()
main_module_name = main_module.getFullName()
if main_module_name != "__main__":
options["main_module_name"] = main_module_name
if Options.shallUseStaticLibPython():
options["static_libpython"] = getSystemStaticLibPythonPath()
if isDebianPackagePython():
options["debian_python"] = asBoolStr(True)
if isMSYS2MingwPython():
options["msys2_mingw_python"] = asBoolStr(True)
if isAnacondaPython():
options["anaconda_python"] = asBoolStr(True)
if isApplePython():
options["apple_python"] = asBoolStr(True)
if isPyenvPython():
options["pyenv_python"] = asBoolStr(True)
if Options.isStandaloneMode():
options["standalone_mode"] = asBoolStr(True)
if Options.isOnefileMode():
options["onefile_mode"] = asBoolStr(True)
if Options.isOnefileTempDirMode():
options["onefile_temp_mode"] = asBoolStr(True)
if Options.getForcedStdoutPath():
options["forced_stdout_path"] = Options.getForcedStdoutPath()
if Options.getForcedStderrPath():
options["forced_stderr_path"] = Options.getForcedStderrPath()
if Options.shallTreatUninstalledPython():
options["uninstalled_python"] = asBoolStr(True)
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules())
)
if Options.isProfile():
options["profile_mode"] = asBoolStr(True)
if hasPythonFlagNoWarnings():
options["no_python_warnings"] = asBoolStr(True)
if hasPythonFlagNoAsserts():
options["python_sysflag_optimize"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = asBoolStr(True)
if python_version < 0x300 and (
sys.flags.division_warning or sys.flags.py3k_warning
):
options["python_sysflag_division_warning"] = asBoolStr(True)
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = asBoolStr(True)
if int(os.environ.get("NUITKA_NOSITE_FLAG", Options.hasPythonFlagNoSite())):
options["python_sysflag_no_site"] = asBoolStr(True)
if Options.hasPythonFlagTraceImports():
options["python_sysflag_verbose"] = asBoolStr(True)
if Options.hasPythonFlagNoRandomization():
options["python_sysflag_no_randomization"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.unicode:
options["python_sysflag_unicode"] = asBoolStr(True)
if python_version >= 0x370 and sys.flags.utf8_mode:
options["python_sysflag_utf8"] = asBoolStr(True)
if hasPythonFlagUnbuffered():
options["python_sysflag_unbuffered"] = asBoolStr(True)
abiflags = getPythonABI()
if abiflags:
options["abiflags"] = abiflags
if Options.shallMakeModule():
options["module_suffix"] = getSharedLibrarySuffix(preferred=True)
SconsInterface.setCommonOptions(options)
if Options.shallCreatePgoInput():
options["pgo_mode"] = "python"
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
)
if not result:
return result, options
# Need to make it usable before executing it.
executePostProcessing()
_runPythonPgoBinary()
return True, options
# Need to restart compilation from scratch here.
if Options.isPgoMode():
# For C level PGO, we have a 2 pass system. TODO: Make it more global for onefile
# and standalone mode proper support, which might need data files to be
# there, which currently are not yet there, so it won't run.
if Options.isPgoMode():
options["pgo_mode"] = "generate"
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
)
if not result:
return result, options
# Need to make it usable before executing it.
executePostProcessing()
_runCPgoBinary()
options["pgo_mode"] = "use"
result = (
SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
),
options,
)
if options.get("pgo_mode") == "use" and _wasMsvcMode():
_deleteMsvcPGOFiles(pgo_mode="use")
return result
def runSconsBackend(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name":
OutputDirectories.getResultBasepath(onefile=False),
"source_dir":
OutputDirectories.getSourceDirectoryPath(),
"debug_mode":
asBoolStr(Options.is_debug),
"python_debug":
asBoolStr(Options.isPythonDebug()),
"unstripped_mode":
asBoolStr(Options.isUnstripped()),
"module_mode":
asBoolStr(Options.shallMakeModule()),
"full_compat":
asBoolStr(Options.is_fullcompat),
"experimental":
",".join(Options.getExperimentalIndications()),
"trace_mode":
asBoolStr(Options.shallTraceExecution()),
"python_version":
python_version_str,
"target_arch":
Utils.getArchitecture(),
"python_prefix":
getDirectoryRealPath(getSystemPrefixPath()),
"nuitka_src":
SconsInterface.getSconsDataPath(),
"module_count":
"%d" % (1 + len(ModuleRegistry.getDoneModules()) +
len(ModuleRegistry.getUncompiledNonTechnicalModules())),
}
if not Options.shallMakeModule():
options["result_exe"] = OutputDirectories.getResultFullpath(
onefile=False)
if Options.shallUseStaticLibPython() and getSystemStaticLibPythonPath(
) is not None:
options["static_libpython"] = getSystemStaticLibPythonPath()
if Options.isStandaloneMode():
options["standalone_mode"] = asBoolStr(True)
if Options.isOnefileMode():
options["onefile_mode"] = asBoolStr(True)
if Options.isWindowsOnefileTempDirMode():
options["onefile_temp_mode"] = asBoolStr(True)
if Options.getForcedStdoutPath():
options["forced_stdout_path"] = Options.getForcedStdoutPath()
if Options.getForcedStderrPath():
options["forced_stderr_path"] = Options.getForcedStderrPath()
if Options.shallTreatUninstalledPython():
options["uninstalled_python"] = asBoolStr(True)
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules()))
if Utils.getOS() == "Windows":
options["noelf_mode"] = asBoolStr(True)
if Options.isProfile():
options["profile_mode"] = asBoolStr(True)
if "no_warnings" in getPythonFlags():
options["no_python_warnings"] = asBoolStr(True)
if "no_asserts" in getPythonFlags():
options["python_sysflag_optimize"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = asBoolStr(True)
if python_version < 0x300 and (sys.flags.division_warning
or sys.flags.py3k_warning):
options["python_sysflag_division_warning"] = asBoolStr(True)
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = asBoolStr(True)
if int(os.environ.get("NUITKA_SITE_FLAG", Options.hasPythonFlagNoSite())):
options["python_sysflag_no_site"] = asBoolStr(True)
if "trace_imports" in Options.getPythonFlags():
options["python_sysflag_verbose"] = asBoolStr(True)
if "no_randomization" in Options.getPythonFlags():
options["python_sysflag_no_randomization"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.unicode:
options["python_sysflag_unicode"] = asBoolStr(True)
if python_version >= 0x370 and sys.flags.utf8_mode:
options["python_sysflag_utf8"] = asBoolStr(True)
abiflags = getPythonABI()
if abiflags:
options["abiflags"] = abiflags
if Options.shallMakeModule():
options["module_suffix"] = getSharedLibrarySuffix(preferred=True)
SconsInterface.setCommonOptions(options)
return (
SconsInterface.runScons(options=options,
quiet=quiet,
scons_filename="Backend.scons"),
options,
)