def enableClcache(the_compiler, env, source_dir): importFromInlineCopy("atomicwrites", must_exist=True) importFromInlineCopy("clcache", must_exist=True) cl_binary = getExecutablePath(the_compiler, env) # The compiler is passed via environment. setEnvironmentVariable(env, "CLCACHE_CL", cl_binary) env["CXX"] = env["CC"] = "<clcache>" setEnvironmentVariable(env, "CLCACHE_HIDE_OUTPUTS", "1") # The clcache stats filename needs absolute path, otherwise it will not work. clcache_stats_filename = os.path.abspath( os.path.join(source_dir, "clcache-stats.%d.txt" % os.getpid())) setEnvironmentVariable(env, "CLCACHE_STATS", clcache_stats_filename) env["CLCACHE_STATS"] = clcache_stats_filename # Unless asked to do otherwise, store ccache files in our own directory. if "CLCACHE_DIR" not in os.environ: clcache_dir = os.path.join(getCacheDir(), "clcache") makePath(clcache_dir) clcache_dir = getExternalUsePath(clcache_dir) setEnvironmentVariable(env, "CLCACHE_DIR", clcache_dir) env["CLCACHE_DIR"] = clcache_dir scons_details_logger.info( "Using inline copy of clcache with %r cl binary." % cl_binary) # Do not consider scons cache anymore. return True
def _unpackPathElement(path_entry): if not path_entry: return "." # empty means current directory if os.path.isfile(path_entry) and path_entry.lower().endswith(".egg"): if path_entry not in _egg_files: with open(path_entry, "rb") as f: checksum = hashlib.md5(f.read()).hexdigest() target_dir = os.path.join(getCacheDir(), "egg-content", checksum) if not os.path.exists(target_dir): try: # Not all Python versions allow using with here, pylint: disable=consider-using-with zip_ref = zipfile.ZipFile(path_entry, "r") zip_ref.extractall(target_dir) zip_ref.close() except BaseException: removeDirectory(target_dir, ignore_errors=True) raise _egg_files[path_entry] = target_dir return _egg_files[path_entry] return path_entry
def _getCacheFilename(dependency_tool, is_main_executable, source_dir, original_dir, binary_filename): original_filename = os.path.join(original_dir, os.path.basename(binary_filename)) original_filename = os.path.normcase(original_filename) if is_main_executable: # Normalize main program name for caching as well, but need to use the # scons information to distinguish different compilers, so we use # different libs there. hashed_value = getFileContents( os.path.join(source_dir, "scons-report.txt")) else: hashed_value = original_filename # Have different values for different Python major versions. hashed_value += sys.version + sys.executable if str is not bytes: hashed_value = hashed_value.encode("utf8") cache_dir = os.path.join(getCacheDir(), "library_deps", dependency_tool) makePath(cache_dir) return os.path.join(cache_dir, hashlib.md5(hashed_value).hexdigest())
def enableClcache(the_compiler, env, source_dir): importFromInlineCopy("atomicwrites", must_exist=True) importFromInlineCopy("clcache", must_exist=True) # Avoid importing this in threads, triggers CPython 3.9 importing bugs at least, # do it now, so it's not a race issue. import concurrent.futures.thread # pylint: disable=I0021,unused-import,unused-variable cl_binary = getExecutablePath(the_compiler, env) # The compiler is passed via environment. setEnvironmentVariable(env, "CLCACHE_CL", cl_binary) env["CXX"] = env["CC"] = "<clcache>" setEnvironmentVariable(env, "CLCACHE_HIDE_OUTPUTS", "1") # The clcache stats filename needs absolute path, otherwise it will not work. clcache_stats_filename = os.path.abspath( os.path.join(source_dir, "clcache-stats.%d.txt" % os.getpid()) ) setEnvironmentVariable(env, "CLCACHE_STATS", clcache_stats_filename) env["CLCACHE_STATS"] = clcache_stats_filename # Unless asked to do otherwise, store ccache files in our own directory. if "CLCACHE_DIR" not in os.environ: clcache_dir = os.path.join(getCacheDir(), "clcache") makePath(clcache_dir) clcache_dir = getExternalUsePath(clcache_dir) setEnvironmentVariable(env, "CLCACHE_DIR", clcache_dir) env["CLCACHE_DIR"] = clcache_dir scons_details_logger.info( "Using inline copy of clcache with %r cl binary." % cl_binary )
def _getCacheFilename(dependency_tool, is_main_executable, source_dir, original_dir, binary_filename): original_filename = os.path.join(original_dir, os.path.basename(binary_filename)) original_filename = os.path.normcase(original_filename) if is_main_executable: # Normalize main program name for caching as well, but need to use the # scons information to distinguish different compilers, so we use # different libs there. # Ignore values, that are variable per compilation. hashed_value = "".join( key + value for key, value in iterItems(readSconsReport(source_dir=source_dir)) if key not in ("CLCACHE_STATS", )) else: hashed_value = original_filename # Have different values for different Python major versions. hashed_value += sys.version + sys.executable if str is not bytes: hashed_value = hashed_value.encode("utf8") cache_dir = os.path.join(getCacheDir(), "library_dependencies", dependency_tool) makePath(cache_dir) return os.path.join(cache_dir, hashlib.md5(hashed_value).hexdigest())
def getTestingCPythonOutputsCacheDir(): cache_dir = getCacheDir() result = os.path.join(cache_dir, "cpython_outputs", os.environ.get("NUITKA_TEST_SUITE", "")) makePath(result) return result
def _compressFile(self, filename, use_cache): upx_options = ["-q", "--no-progress"] if os.path.basename(filename).startswith("vcruntime140"): return if use_cache: if self.upx_binary_hash is None: self.upx_binary_hash = getFileContentsHash(self.upx_binary, as_string=False) upx_hash = Hash() upx_hash.updateFromBytes(self.upx_binary_hash) upx_hash.updateFromValues(*upx_options) upx_hash.updateFromFile(filename) # TODO: Repeating pattern upx_cache_dir = os.path.join(getCacheDir(), "upx") makePath(upx_cache_dir) upx_cache_filename = os.path.join(upx_cache_dir, upx_hash.asHexDigest() + ".bin") if os.path.exists(upx_cache_filename): copyFile(upx_cache_filename, filename) return if use_cache: self.info("Uncached file, compressing '%s' may take a while." % os.path.basename(filename)) else: self.info("Compressing '%s'." % filename) command = [self.upx_binary] + upx_options + [filename] executeToolChecked( logger=self, command=command, absence_message=None, stderr_filter=self._filterUpxError, ) if use_cache: copyFile(filename, upx_cache_filename)
def enableCcache( the_compiler, env, source_dir, python_prefix, show_scons_mode, assume_yes_for_downloads, ): # The ccache needs absolute path, otherwise it will not work. ccache_logfile = os.path.abspath( os.path.join(source_dir, "ccache-%d.txt" % os.getpid())) setEnvironmentVariable(env, "CCACHE_LOGFILE", ccache_logfile) env["CCACHE_LOGFILE"] = ccache_logfile # Unless asked to do otherwise, store ccache files in our own directory. if "CCACHE_DIR" not in os.environ: ccache_dir = os.path.join(getCacheDir(), "ccache") makePath(ccache_dir) ccache_dir = getExternalUsePath(ccache_dir) setEnvironmentVariable(env, "CCACHE_DIR", ccache_dir) env["CCACHE_DIR"] = ccache_dir # First check if it's not already supposed to be a ccache, then do nothing. cc_path = getExecutablePath(the_compiler, env=env) cc_is_link, cc_link_path = getLinkTarget(cc_path) if cc_is_link and os.path.basename(cc_link_path) == "ccache": if show_scons_mode: scons_logger.info( "Chosen compiler %s is pointing to ccache %s already." % (cc_path, cc_link_path)) return True return _injectCcache( the_compiler=the_compiler, cc_path=cc_path, env=env, python_prefix=python_prefix, show_scons_mode=show_scons_mode, assume_yes_for_downloads=assume_yes_for_downloads, )
def _unpackPathElement(path_entry): if not path_entry: return '.' # empty means current directory if os.path.isfile(path_entry) and path_entry.lower().endswith(".egg"): if path_entry not in _egg_files: checksum = hashlib.md5(open(path_entry, "rb").read()).hexdigest() target_dir = os.path.join(getCacheDir(), "egg-content", checksum) zip_ref = zipfile.ZipFile(path_entry, 'r') zip_ref.extractall(target_dir) zip_ref.close() _egg_files[path_entry] = target_dir return _egg_files[path_entry] return path_entry
def _unpackPathElement(path_entry): if not path_entry: return "." # empty means current directory if os.path.isfile(path_entry) and path_entry.lower().endswith(".egg"): if path_entry not in _egg_files: with open(path_entry, "rb") as f: checksum = hashlib.md5(f.read()).hexdigest() target_dir = os.path.join(getCacheDir(), "egg-content", checksum) zip_ref = zipfile.ZipFile(path_entry, "r") zip_ref.extractall(target_dir) zip_ref.close() _egg_files[path_entry] = target_dir return _egg_files[path_entry] return path_entry
def _compressFile(self, filename, use_cache): upx_options = ["-q", "--no-progress"] if use_cache: if self.upx_binary_hash is None: self.upx_binary_hash = getFileContentsHash(self.upx_binary, as_string=False) upx_hash = Hash() upx_hash.updateFromBytes(self.upx_binary_hash) upx_hash.updateFromValues(*upx_options) upx_hash.updateFromFile(filename) # TODO: Repeating pattern upx_cache_dir = os.path.join(getCacheDir(), "upx") makePath(upx_cache_dir) upx_cache_filename = os.path.join(upx_cache_dir, upx_hash.asHexDigest() + ".bin") if os.path.exists(upx_cache_filename): copyFile(upx_cache_filename, filename) return if use_cache: self.info("Uncached file, compressing '%s' may take a while." % os.path.basename(filename)) else: self.info("Compressing '%s'." % filename) check_call( [self.upx_binary] + upx_options + [filename], stdout=getNullOutput(), shell=False, ) if use_cache: copyFile(filename, upx_cache_filename)
def main(): # pylint: disable=broad-except,too-many-branches,too-many-locals,too-many-statements setup() # cache_dir is where the git clones are cached cache_dir = os.path.join(getCacheDir(), "pypi-git-clones") base_dir = os.getcwd() if not os.path.isdir(cache_dir): os.mkdir(cache_dir) search_mode = createSearchMode() results = [] # load json with open("packages.json", "r") as f: packages = json.load(f) for package_name, details in sorted(packages.items()): active = search_mode.consider(dirname=None, filename=package_name) if not active: continue if str is not bytes: # running on python3 if package_name in ("futures", "future"): reportSkip("Does not run on Python3", ".", package_name) if search_mode.abortIfExecuted(): break continue if os.name == "nt": if package_name in ("cryptography",): reportSkip("Not working on Windows", ".", package_name) if search_mode.abortIfExecuted(): break continue if package_name == "pyyaml": reportSkip("Not yet supported, see Issue #476", ".", package_name) if search_mode.abortIfExecuted(): break continue if package_name in ("pycparser", "numpy"): reportSkip("Not yet supported, see Issue #477", ".", package_name) if search_mode.abortIfExecuted(): break continue if package_name in ( "google-auth", # bdist_nuitka fails AttributeError: single_version_externally_managed "jinja2", # ModuleNotFoundError: No module named 'jinja2.tests' "pandas", # ModuleNotFoundError: No module named 'Cython' "pytz", # need to 'make build' "rsa", # Now uses Poetry (no setup.py) ): if search_mode.abortIfExecuted(): break continue package_dir = os.path.join(cache_dir, package_name) try: gitClone(package_name, details["url"], cache_dir) os.chdir(base_dir) with withVirtualenv( "venv_%s" % package_name, delete=False, style="blue" ) as venv: dist_dir = os.path.join(package_dir, "dist") # delete ignored tests if any if details["ignored_tests"]: for test in details["ignored_tests"]: venv.runCommand("rm -rf %s" % os.path.join(package_dir, test)) # setup for pytest cmds = [ "python -m pip install pytest", "cd %s" % os.path.join(os.path.dirname(nuitka.__file__), ".."), "python setup.py develop", "cd %s" % package_dir, ] if details["requirements_file"]: cmds.append( "python -m pip install -r %s" % details["requirements_file"] ) if details.get("extra_commands"): cmds += details["extra_commands"] # build uncompiled .whl cmds.append("python setup.py bdist_wheel") venv.runCommand(commands=cmds) # install and print out if the active .whl is compiled or not venv.runCommand( commands=[ "python -m pip install -U %s" % os.path.join(dist_dir, os.listdir(dist_dir)[0]), # use triple quotes for linux """python -c "print(getattr(__import__('%s'),'__compiled__','__uncompiled_version__'))" """ % details.get("package_name", package_name), ] ) # get uncompiled pytest results uncompiled_stdout, uncompiled_stderr = venv.runCommandWithOutput( commands=[ "cd %s" % package_dir, "python -m pytest --disable-warnings", ], style="blue", ) # clean up before building compiled .whl cmds = ["cd %s" % package_dir, "git clean -dfx"] if details.get("extra_commands"): cmds += details["extra_commands"] # build nuitka compiled .whl cmds.append("python setup.py bdist_nuitka") venv.runCommand(commands=cmds) # install and print out if the active .whl is compiled or not venv.runCommand( commands=[ "python -m pip install -U %s" % os.path.join(dist_dir, os.listdir(dist_dir)[0]), # use triple quotes for linux """python -c "print(getattr(__import__('%s'),'__compiled__','__uncompiled_version__'))" """ % details.get("package_name", package_name), ] ) # get compiled pytest results compiled_stdout, compiled_stderr = venv.runCommandWithOutput( commands=[ "cd %s" % package_dir, "python -m pytest --disable-warnings", ], style="blue", ) venv.runCommand(commands=["cd %s" % package_dir, "git clean -q -dfx"]) except Exception as e: my_print( "Package", package_name, "ran into an exception during execution, traceback: ", ) my_print(e) results.append((package_name, "ERROR", "ERROR")) if search_mode.abortIfExecuted(): break continue # compare outputs stdout_diff = compareOutput( "stdout", uncompiled_stdout, compiled_stdout, ignore_warnings=True, syntax_errors=True, ) stderr_diff = compareOutput( "stderr", uncompiled_stderr, compiled_stderr, ignore_warnings=True, syntax_errors=True, ) results.append((package_name, stdout_diff, stderr_diff)) exit_code = stdout_diff or stderr_diff my_print( "\n=================================================================================", "\n--- %s ---" % package_name, "exit_stdout:", stdout_diff, "exit_stderr:", stderr_diff, "\nError, outputs differed for package %s." % package_name if exit_code else "\nNo differences found for package %s." % package_name, "\n=================================================================================\n", style="red" if exit_code else "green", ) if exit_code != 0 and search_mode.abortOnFinding( dirname=None, filename=package_name ): break if search_mode.abortIfExecuted(): break search_mode.finish() # give a summary of all packages my_print( "\n\n=====================================SUMMARY=====================================", style="yellow", ) for package_name, stdout_diff, stderr_diff in results: my_print( package_name, "-", end=" ", style="red" if (stdout_diff or stderr_diff) else "green", ) my_print( "stdout:", stdout_diff, end=" ", style="red" if stdout_diff else "green" ) my_print( "stderr:", stderr_diff, end="", style="red" if stderr_diff else "green" ) my_print( "\n---------------------------------------------------------------------------------" ) my_print("TOTAL NUMBER OF PACKAGES TESTED: %s" % len(results), style="yellow") num_failed = 0 num_errors = 0 # tally the number of errors and failed for _, y, z in results: if type(y) is str: # this means the package ran into an exception num_errors += 1 elif y or z: num_failed += 1 my_print( "TOTAL PASSED: %s" % (len(results) - num_failed - num_errors), style="green" ) my_print("TOTAL FAILED (differences): %s" % num_failed, style="red") my_print("TOTAL ERRORS (exceptions): %s" % num_errors, style="red")
def runScons(main_module, quiet): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches,too-many-statements options = { "name" : os.path.basename( getTreeFilenameWithSuffix(main_module, "") ), "result_name" : getResultBasepath(main_module), "source_dir" : getSourceDirectoryPath(main_module), "debug_mode" : _asBoolStr(Options.isDebug()), "python_debug" : _asBoolStr(Options.isPythonDebug()), "unstripped_mode" : _asBoolStr(Options.isUnstripped()), "module_mode" : _asBoolStr(Options.shallMakeModule()), "full_compat" : _asBoolStr(Options.isFullCompat()), "experimental" : ','.join(Options.getExperimentalIndications()), "trace_mode" : _asBoolStr(Options.shallTraceExecution()), "python_version" : python_version_str, "target_arch" : Utils.getArchitecture(), "python_prefix" : sys.prefix, "nuitka_src" : SconsInterface.getSconsDataPath(), "nuitka_cache" : getCacheDir(), "module_count" : "%d" % ( 1 + \ len(ModuleRegistry.getDoneUserModules()) + \ len(ModuleRegistry.getUncompiledNonTechnicalModules()) ) } if not Options.shallMakeModule(): options["result_exe"] = getResultFullpath(main_module) # Ask Scons to cache on Windows, except where the directory is thrown # away. On non-Windows you can should use ccache instead. if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows": options["cache_mode"] = "true" if Options.isLto(): options["lto_mode"] = "true" if Options.shallDisableConsoleWindow(): options["win_disable_console"] = "true" if Options.isStandaloneMode(): options["standalone_mode"] = "true" if not Options.isStandaloneMode() and \ not Options.shallMakeModule() and \ isUninstalledPython(): options["uninstalled_python"] = "true" if ModuleRegistry.getUncompiledTechnicalModules(): options["frozen_modules"] = str( len(ModuleRegistry.getUncompiledTechnicalModules()) ) if Options.isShowScons(): options["show_scons"] = "true" if Options.isMingw64(): options["mingw_mode"] = "true" if Options.getMsvcVersion(): msvc_version = Options.getMsvcVersion() msvc_version = msvc_version.replace("exp", "Exp") if '.' not in msvc_version: msvc_version += ".0" options["msvc_version"] = msvc_version if Options.isClang(): options["clang_mode"] = "true" if Options.getIconPath(): options["icon_path"] = Options.getIconPath() if Options.isProfile(): options["profile_mode"] = "true" if "no_warnings" in getPythonFlags(): options["no_python_warnings"] = "true" if python_version < 300 and sys.flags.py3k_warning: options["python_sysflag_py3k_warning"] = "true" if python_version < 300 and (sys.flags.division_warning or sys.flags.py3k_warning): options["python_sysflag_division_warning"] = "true" if sys.flags.bytes_warning: options["python_sysflag_bytes_warning"] = "true" if int(os.environ.get("NUITKA_SITE_FLAG", "no_site" in Options.getPythonFlags())): options["python_sysflag_no_site"] = "true" if "trace_imports" in Options.getPythonFlags(): options["python_sysflag_verbose"] = "true" if python_version < 300 and sys.flags.unicode: options["python_sysflag_unicode"] = "true" if python_version >= 370 and sys.flags.utf8_mode: options["python_sysflag_utf8"] = "true" abiflags = getPythonABI() if abiflags: options["abiflags"] = abiflags return SconsInterface.runScons(options, quiet), options
def getTestingCacheDir(): cache_dir = getCacheDir() result = os.path.join(cache_dir, "tests_state") makePath(result) return result
def runScons(main_module, quiet): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches,too-many-statements options = { "name": os.path.basename(getTreeFilenameWithSuffix(main_module, "")), "result_name": getResultBasepath(main_module), "source_dir": getSourceDirectoryPath(main_module), "debug_mode": _asBoolStr(Options.isDebug()), "python_debug": _asBoolStr(Options.isPythonDebug()), "unstripped_mode": _asBoolStr(Options.isUnstripped()), "module_mode": _asBoolStr(Options.shallMakeModule()), "full_compat": _asBoolStr(Options.isFullCompat()), "experimental": ",".join(Options.getExperimentalIndications()), "trace_mode": _asBoolStr(Options.shallTraceExecution()), "python_version": python_version_str, "target_arch": Utils.getArchitecture(), "python_prefix": sys.prefix, "nuitka_src": SconsInterface.getSconsDataPath(), "nuitka_cache": getCacheDir(), "module_count": "%d" % ( 1 + len(ModuleRegistry.getDoneUserModules()) + len(ModuleRegistry.getUncompiledNonTechnicalModules()) ), } if not Options.shallMakeModule(): options["result_exe"] = getResultFullpath(main_module) # Ask Scons to cache on Windows, except where the directory is thrown # away. On non-Windows you can should use ccache instead. if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows": options["cache_mode"] = "true" if Options.isLto(): options["lto_mode"] = "true" # For AnaConda default to trying static lib python library, which # normally is just not available or if it is even unusable. if "Anaconda" in sys.version: options["static_libpython"] = "true" if Options.shallDisableConsoleWindow(): options["win_disable_console"] = "true" if Options.isStandaloneMode(): options["standalone_mode"] = "true" if ( not Options.isStandaloneMode() and not Options.shallMakeModule() and isUninstalledPython() ): options["uninstalled_python"] = "true" if ModuleRegistry.getUncompiledTechnicalModules(): options["frozen_modules"] = str( len(ModuleRegistry.getUncompiledTechnicalModules()) ) if Options.isShowScons(): options["show_scons"] = "true" if Options.isMingw64(): options["mingw_mode"] = "true" if Options.getMsvcVersion(): msvc_version = Options.getMsvcVersion() msvc_version = msvc_version.replace("exp", "Exp") if "." not in msvc_version: msvc_version += ".0" options["msvc_version"] = msvc_version if Options.isClang(): options["clang_mode"] = "true" if Options.getIconPath(): options["icon_path"] = Options.getIconPath() if Options.isProfile(): options["profile_mode"] = "true" if "no_warnings" in getPythonFlags(): options["no_python_warnings"] = "true" if "no_asserts" in getPythonFlags(): options["python_sysflag_optimize"] = "true" if python_version < 300 and sys.flags.py3k_warning: options["python_sysflag_py3k_warning"] = "true" if python_version < 300 and (sys.flags.division_warning or sys.flags.py3k_warning): options["python_sysflag_division_warning"] = "true" if sys.flags.bytes_warning: options["python_sysflag_bytes_warning"] = "true" if int(os.environ.get("NUITKA_SITE_FLAG", "no_site" in Options.getPythonFlags())): options["python_sysflag_no_site"] = "true" if "trace_imports" in Options.getPythonFlags(): options["python_sysflag_verbose"] = "true" if python_version < 300 and sys.flags.unicode: options["python_sysflag_unicode"] = "true" if python_version >= 370 and sys.flags.utf8_mode: options["python_sysflag_utf8"] = "true" abiflags = getPythonABI() if abiflags: options["abiflags"] = abiflags return SconsInterface.runScons(options, quiet), options
def _getCacheDir(): module_cache_dir = os.path.join(getCacheDir(), "module-cache") makePath(module_cache_dir) return module_cache_dir
def getTestingCPythonOutputsCacheDir(): cache_dir = getCacheDir() result = os.path.join(cache_dir, "cpython_outputs") makePath(result) return result
def _getCacheFilename(module): module_cache_dir = os.path.join(getCacheDir(), "module-cache") makePath(module_cache_dir) return os.path.join(module_cache_dir, module.getFullName().asString() + ".xml")