def copyFile(source_path, dest_path): """Improved version of shutil.copy This handles errors with a chance to correct them, e.g. on Windows, files might be locked by running program or virus checkers. """ while 1: try: shutil.copyfile(source_path, dest_path) except PermissionError as e: if e.errno != errno.EACCES: raise general.warning("Problem copying file %s:" % e) try: reply = raw_input("Retry? (YES/no) ") or "yes" except EOFError: reply = "no" if reply.upper() == "YES": continue raise break
def addDLLInfo(count, source_dir, original_filename, binary_filename, package_name): used_dlls = _detectBinaryDLLs( is_main_executable=count == 0, source_dir=source_dir, original_filename=original_filename, binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) # Allow plugins to prevent inclusion, this may discard things from used_dlls. Plugins.removeDllDependencies(dll_filename=binary_filename, dll_filenames=used_dlls) for dll_filename in sorted(tuple(used_dlls)): if not os.path.isfile(dll_filename): if _not_found_dlls: general.warning("""\ Dependency '%s' could not be found, expect runtime issues. If this is working with Python, report a Nuitka bug.""" % dll_filename) _not_found_dlls.add(dll_filename) used_dlls.remove(dll_filename) reportProgressBar(binary_filename) return binary_filename, package_name, used_dlls
def getFunctionCode( context, function_identifier, parameters, closure_variables, user_variables, outline_variables, temp_variables, function_doc, file_scope, needs_exception_exit, ): try: return _getFunctionCode( context=context, function_identifier=function_identifier, parameters=parameters, closure_variables=closure_variables, user_variables=user_variables, outline_variables=outline_variables, temp_variables=temp_variables, function_doc=function_doc, file_scope=file_scope, needs_exception_exit=needs_exception_exit, ) except Exception: general.warning("Problem creating function code %r." % function_identifier) raise
def addDLLInfo(count, source_dir, original_filename, binary_filename, package_name): used_dlls = detectBinaryDLLs( is_main_executable=count == 0, source_dir=source_dir, original_filename=original_filename, binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) # Allow plugins to prevent inclusion, this may discard things from used_dlls. Plugins.removeDllDependencies(dll_filename=binary_filename, dll_filenames=used_dlls) for dll_filename in sorted(tuple(used_dlls)): if not os.path.isfile(dll_filename): if _unfound_dlls: general.warning( "Dependency '%s' could not be found, you might need to copy it manually." % dll_filename) _unfound_dlls.add(dll_filename) used_dlls.remove(dll_filename) return binary_filename, used_dlls
def packDistFolderToOnefile(dist_dir, binary_filename): """Pack distribution to onefile, i.e. a single file that is directly executable.""" if getOS() == "Linux": packDistFolderToOnefileLinux(dist_dir, binary_filename) elif getOS() == "Windows": packDistFolderToOnefileWindows(dist_dir) else: general.warning("Onefile mode is not yet available on '%s'." % getOS())
def _pickCompressor(): if Options.isExperimental("zstd"): try: from zstd import ZSTD_compress # pylint: disable=I0021,import-error except ImportError: general.warning( "Onefile mode cannot compress without 'zstd' module on '%s'." % getOS() ) else: return b"Y", lambda data: ZSTD_compress(data, 9, getJobLimit()) else: return b"X", lambda data: data
def packDistFolderToOnefileWindows(dist_dir): general.warning("Onefile mode is experimental on '%s'." % getOS()) postprocessing_logger.info( "Creating single file from dist folder, this may take a while.") onefile_output_filename = getResultFullpath(onefile=True) # First need to create the bootstrap binary for unpacking. _runOnefileScons(quiet=not Options.isShowScons()) # Make sure to copy the resources from the created binary to the bootstrap binary, these # are icons and version information. copyResourcesFromFileToFile( source_filename=getResultFullpath(onefile=False), target_filename=onefile_output_filename, resource_kinds=(RT_ICON, RT_GROUP_ICON, RT_VERSION), ) # Now need to append to payload it, potentially compressing it. compression_indicator, compressor = _pickCompressor() with open(onefile_output_filename, "ab") as output_file: # Seeking to end of file seems necessary on Python2 at least, maybe it's # just that tell reports wrong value initially. output_file.seek(0, 2) start_pos = output_file.tell() output_file.write(b"KA" + compression_indicator) # Move the binary to start immediately to the start position start_binary = getResultFullpath(onefile=False) file_list = getFileList(dist_dir) file_list.remove(start_binary) file_list.insert(0, start_binary) for filename_full in file_list: filename_relative = os.path.relpath(filename_full, dist_dir) filename_encoded = filename_relative.encode("utf-16le") + b"\0\0" output_file.write(filename_encoded) with open(filename_full, "rb") as input_file: compressed = compressor(input_file.read()) output_file.write(struct.pack("Q", len(compressed))) output_file.write(compressed) # Using empty filename as a terminator. output_file.write(b"\0\0") output_file.write(struct.pack("Q", start_pos))
def getLinuxDistribution(): """Name of the Linux distribution. We should usually avoid this, and rather test for the feature, but in some cases it's hard to manage that. """ if getOS() != "Linux": return None, None, None # singleton, pylint: disable=global-statement global _linux_distribution_info if _linux_distribution_info is None: result = None base = None version = None if os.path.exists("/etc/os-release"): result, base, version = _parseOsReleaseFileContents( "/etc/os-release") elif os.path.exists("/etc/SuSE-release"): result, base, version = _parseOsReleaseFileContents( "/etc/SuSE-release") elif os.path.exists("/etc/issue"): result, base, version = _parseOsReleaseFileContents("/etc/issue") if result is None: from .Execution import check_output try: result = check_output(["lsb_release", "-i", "-s"], shell=False) if str is not bytes: result = result.decode("utf8") except OSError: pass if result is None: from nuitka.Tracing import general general.warning( "Cannot detect Linux distribution, this may prevent optimization." ) result = "Unknown" # Change e.g. "11 (Bullseye)"" to "11". if version is not None and version.strip(): version = version.split()[0] _linux_distribution_info = result.title(), base, version return _linux_distribution_info
def reportProgressBar(item, total=None, update=True): if Tracing.progress is not None: try: if total is not None: Tracing.progress.updateTotal(total) Tracing.progress.setCurrent(item) if update: Tracing.progress.update() except Exception: # Catch all the things, pylint: disable=broad-except # We disable the progress bar now, because it's causing issues. general.warning("Progress bar disabled due to bug") closeProgressBar()
def _cleanupClangFormat(filename): """Call clang-format on a given filename to format C code. Args: filename: What file to re-format. """ # Using global here, as this is really a singleton, in # the form of a module, pylint: disable=global-statement global warned_clang_format clang_format_path = ( getExecutablePath("clang-format-10") or getExecutablePath("clang-format-9") or getExecutablePath("clang-format-8") or getExecutablePath("clang-format-7") ) # Extra ball on Windows, check default installations paths in MSVC and LLVM too. if not clang_format_path and getOS() == "Windows": with withEnvironmentPathAdded( "PATH", r"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\Llvm\bin", r"C:\Program Files\LLVM\bin", ): clang_format_path = getExecutablePath("clang-format") if clang_format_path: subprocess.call( [ clang_format_path, "-i", "-style={BasedOnStyle: llvm, IndentWidth: 4, ColumnLimit: 120}", filename, ] ) else: if not warned_clang_format: general.warning("Need to install LLVM for C files format.") warned_clang_format = True
def attemptToMarshal(constant_identifier, constant_value, emit): """ Try and marshal a value, if so decided. Indicate with return value. See above for why marshal is only used in problematic cases. """ if not isMarshalConstant(constant_value): return False marshal_value = marshal.dumps(constant_value) restored = marshal.loads(marshal_value) # TODO: The check in isMarshalConstant is currently preventing this from # happening. if not compareConstants(constant_value, restored): general.warning("Problem with marshal of constant %r", constant_value) return False emit("%s = PyMarshal_ReadObjectFromString((char *)%s);" % (constant_identifier, stream_data.getStreamDataCode(marshal_value))) return True
def makeSourceDirectory(main_module): """ Get the full list of modules imported, create code for all of them. """ # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=too-many-branches,too-many-locals,too-many-statements assert main_module.isCompiledPythonModule() # The global context used to generate code. global_context = CodeGeneration.makeGlobalContext() # assert main_module in ModuleRegistry.getDoneModules() # We might have chosen to include it as bytecode, and only compiled it for # fun, and to find its imports. In this case, now we just can drop it. Or # a module may shadow a frozen module, but be a different one, then we can # drop the frozen one. # TODO: This really should be done when the compiled module comes into # existence. for module in ModuleRegistry.getDoneUserModules(): if module.isCompiledPythonModule(): uncompiled_module = ModuleRegistry.getUncompiledModule( module_name=module.getFullName(), module_filename=module.getCompileTimeFilename(), ) if uncompiled_module is not None: # We now need to decide which one to keep, compiled or uncompiled # module. Some uncompiled modules may have been asked by the user # or technically required. By default, frozen code if it exists # is preferred, as it will be from standalone mode adding it. if (uncompiled_module.isUserProvided() or uncompiled_module.isTechnical()): ModuleRegistry.removeDoneModule(module) else: ModuleRegistry.removeUncompiledModule(uncompiled_module) # Lets check if the recurse-to modules are actually present, and warn the # user if one of those was not found. for any_case_module in Options.getShallFollowModules(): if "*" in any_case_module or "{" in any_case_module: continue for module in ModuleRegistry.getDoneUserModules(): if module.getFullName() == any_case_module: break else: general.warning("Not recursing to unused '%s'." % any_case_module) # Prepare code generation, i.e. execute finalization for it. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): Finalization.prepareCodeGeneration(module) # Pick filenames. source_dir = OutputDirectories.getSourceDirectoryPath() module_filenames = pickSourceFilenames( source_dir=source_dir, modules=ModuleRegistry.getDoneModules()) # First pass, generate code and use constants doing so, but prepare the # final code generation only, because constants code will be added at the # end only. prepared_modules = {} for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): c_filename = module_filenames[module] try: prepared_modules[ c_filename] = CodeGeneration.prepareModuleCode( global_context=global_context, module=module, module_name=module.getFullName(), ) except Exception: general.warning("Problem creating code for module %r." % module) raise # Main code constants need to be allocated already too. if module is main_module and not Options.shallMakeModule(): prepared_modules[c_filename][1].getConstantCode(0) # Second pass, generate the actual module code into the files. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): c_filename = module_filenames[module] template_values, module_context = prepared_modules[c_filename] source_code = CodeGeneration.generateModuleCode( module_context=module_context, template_values=template_values) writeSourceCode(filename=c_filename, source_code=source_code) if Options.isShowInclusion(): general.info("Included compiled module '%s'." % module.getFullName()) elif module.isPythonShlibModule(): target_filename = os.path.join( OutputDirectories.getStandaloneDirectoryPath(), *module.getFullName().split(".")) target_filename += Utils.getSharedLibrarySuffix() target_dir = os.path.dirname(target_filename) if not os.path.isdir(target_dir): makePath(target_dir) shutil.copyfile(module.getFilename(), target_filename) standalone_entry_points.append(( module.getFilename(), target_filename, module.getFullName().getPackageName(), )) elif module.isUncompiledPythonModule(): if Options.isShowInclusion(): general.info("Included uncompiled module '%s'." % module.getFullName()) else: assert False, module writeSourceCode( filename=os.path.join(source_dir, "__constants.c"), source_code=ConstantCodes.getConstantsDefinitionCode( context=global_context), ) helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode( ModuleRegistry.getDoneUserModules()) writeSourceCode(filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code) writeSourceCode(filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code) for filename, source_code in Plugins.getExtraCodeFiles().items(): target_dir = os.path.join(source_dir, "plugins") if not os.path.isdir(target_dir): makePath(target_dir) writeSourceCode(filename=os.path.join(target_dir, filename), source_code=source_code)
def _detectImports(command, user_provided, technical): # This is pretty complicated stuff, with variants to deal with. # pylint: disable=too-many-branches,too-many-locals,too-many-statements # Print statements for stuff to show, the modules loaded. if python_version >= 0x300: command += """ print("\\n".join(sorted( "import %s # sourcefile %s" % (module.__name__, module.__file__) for module in sys.modules.values() if getattr(module, "__file__", None) not in (None, "<frozen>" ))), file = sys.stderr)""" reduced_path = [ path_element for path_element in sys.path if not areSamePaths(path_element, ".") if not areSamePaths( path_element, os.path.dirname(sys.modules["__main__"].__file__)) ] # Make sure the right import path (the one Nuitka binary is running with) # is used. command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" % repr(reduced_path)) + command import tempfile tmp_file, tmp_filename = tempfile.mkstemp() try: if python_version >= 0x300: command = command.encode("utf8") os.write(tmp_file, command) os.close(tmp_file) process = subprocess.Popen( args=[sys.executable, "-s", "-S", "-v", tmp_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=dict(os.environ, PYTHONIOENCODING="utf_8"), ) _stdout, stderr = process.communicate() finally: os.unlink(tmp_filename) # Don't let errors here go unnoticed. if process.returncode != 0: general.warning( "There is a problem with detecting imports, CPython said:") for line in stderr.split(b"\n"): printError(line) general.sysexit("Error, please report the issue with above output.") result = [] detections = [] for line in stderr.replace(b"\r", b"").split(b"\n"): if line.startswith(b"import "): # print(line) parts = line.split(b" # ", 2) module_name = parts[0].split(b" ", 2)[1] origin = parts[1].split()[0] if python_version >= 0x300: module_name = module_name.decode("utf-8") module_name = ModuleName(module_name) if origin == b"precompiled": # This is a ".pyc" file that was imported, even before we have a # chance to do anything, we need to preserve it. filename = parts[1][len(b"precompiled from "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 3, "precompiled", filename)) elif origin == b"sourcefile": filename = parts[1][len(b"sourcefile "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) elif filename.endswith(".pyc"): detections.append( (module_name, 3, "precompiled", filename)) elif not filename.endswith("<frozen>"): # Python3 started lying in "__name__" for the "_decimal" # calls itself "decimal", which then is wrong and also # clashes with "decimal" proper if python_version >= 0x300: if module_name == "decimal": module_name = ModuleName("_decimal") detections.append((module_name, 2, "shlib", filename)) elif origin == b"dynamically": # Shared library in early load, happens on RPM based systems and # or self compiled Python installations. filename = parts[1][len(b"dynamically loaded from "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 1, "shlib", filename)) for module_name, _prio, kind, filename in sorted(detections): if kind == "precompiled": _detectedPrecompiledFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "sourcefile": _detectedSourceFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "shlib": _detectedShlibFile(filename=filename, module_name=module_name) else: assert False, kind return result
def packDistFolderToOnefileLinux(dist_dir, binary_filename): """Pack to onefile binary on Linux. Notes: This is mostly a wrapper around AppImage, which does all the heavy lifting. """ # This might be possible to avoid being done with --runtime-file. apprun_filename = os.path.join(dist_dir, "AppRun") with open(apprun_filename, "w") as output_file: output_file.write("""\ #!/bin/sh exec $APPDIR/%s $@""" % os.path.basename(binary_filename)) addFileExecutablePermission(apprun_filename) binary_basename = os.path.basename(getResultBasepath()) icon_paths = getIconPaths() if not icon_paths: if os.path.exists("/usr/share/pixmaps/python.xpm"): icon_paths.append("/usr/share/pixmaps/python.xpm") if icon_paths: extension = os.path.splitext(icon_paths[0])[1].lower() shutil.copyfile(icon_paths[0], getResultBasepath() + extension) else: general.warning( "Cannot apply onefile unless icon file is specified. Yes, crazy.") return with open(getResultBasepath() + ".desktop", "w") as output_file: output_file.write( """\ [Desktop Entry] Name=%(binary_basename)s Exec=%(binary_filename)s Icon=%(binary_basename)s Type=Application Categories=Utility;""" % { "binary_basename": binary_basename, "binary_filename": os.path.basename(binary_filename), }) postprocessing_logger.info( "Creating single file from dist folder, this may take a while.") onefile_output_filename = getResultFullpath(onefile=True) # Starting the process while locked, so file handles are not duplicated. appimagetool_process = subprocess.Popen( ( getAppImageToolPath(), dist_dir, "--comp", "xz", "-n", onefile_output_filename, ), shell=False, stderr=getNullOutput(), stdout=getNullOutput(), ) # TODO: Exit code should be checked. result = appimagetool_process.wait() if not os.path.exists(onefile_output_filename): sys.exit("Error, expected output file %s not created by AppImage." % onefile_output_filename) postprocessing_logger.info("Completed onefile execution.") assert result == 0, result
def _getSystemStaticLibPythonPath(): # Return driven function with many cases, pylint: disable=too-many-branches,too-many-return-statements sys_prefix = getSystemPrefixPath() python_abi_version = python_version_str + getPythonABI() if isNuitkaPython(): # Nuitka Python has this. if isWin32Windows(): return os.path.join( sys_prefix, "libs", "python" + python_abi_version.replace(".", "") + ".lib", ) else: return os.path.join( sys_prefix, "lib", "libpython" + python_abi_version + ".a", ) if isWin32Windows(): candidates = [ # Anaconda has this. os.path.join( sys_prefix, "libs", "libpython" + python_abi_version.replace(".", "") + ".dll.a", ), # MSYS2 mingw64 Python has this. os.path.join( sys_prefix, "lib", "libpython" + python_abi_version + ".dll.a", ), ] for candidate in candidates: if os.path.exists(candidate): return candidate else: candidate = os.path.join( sys_prefix, "lib", "libpython" + python_abi_version + ".a" ) if os.path.exists(candidate): return candidate # For Python2 this works. TODO: Figure out Debian and Python3. if ( python_version < 0x300 and isDebianPackagePython() and isDebianSuitableForStaticLinking() ): candidate = locateStaticLinkLibrary("python" + python_abi_version) else: candidate = None if candidate is not None and os.path.exists(candidate): # Also check libz, can be missing if not locateStaticLinkLibrary("z"): general.warning( "Error, missing libz-dev installation needed for static lib-python." ) return candidate # This is not necessarily only for Python3 on Debian, but maybe others as well, # but that's what's been tested. if python_version >= 0x300 and isDebianPackagePython() and isDebianBasedLinux(): try: import sysconfig candidate = os.path.join( sysconfig.get_config_var("LIBPL"), "libpython" + python_abi_version + "-pic.a", ) if os.path.exists(candidate): return candidate except ImportError: # Cannot detect this properly for Python 2.6, but we don't care much # about that anyway. pass return None
def makeSourceDirectory(): """Get the full list of modules imported, create code for all of them.""" # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=too-many-branches # assert main_module in ModuleRegistry.getDoneModules() # We might have chosen to include it as bytecode, and only compiled it for # fun, and to find its imports. In this case, now we just can drop it. Or # a module may shadow a frozen module, but be a different one, then we can # drop the frozen one. # TODO: This really should be done when the compiled module comes into # existence. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): uncompiled_module = ModuleRegistry.getUncompiledModule( module_name=module.getFullName(), module_filename=module.getCompileTimeFilename(), ) if uncompiled_module is not None: # We now need to decide which one to keep, compiled or uncompiled # module. Some uncompiled modules may have been asked by the user # or technically required. By default, frozen code if it exists # is preferred, as it will be from standalone mode adding it. if ( uncompiled_module.isUserProvided() or uncompiled_module.isTechnical() ): ModuleRegistry.removeDoneModule(module) else: ModuleRegistry.removeUncompiledModule(uncompiled_module) # Lets check if the asked modules are actually present, and warn the # user if one of those was not found. for any_case_module in Options.getShallFollowModules(): if "*" in any_case_module or "{" in any_case_module: continue for module in ModuleRegistry.getDoneModules(): if module.getFullName() == any_case_module: break else: general.warning( "Did not follow import to unused '%s', consider include options." % any_case_module ) # Prepare code generation, i.e. execute finalization for it. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): Finalization.prepareCodeGeneration(module) # Do some reporting and determine compiled module to work on compiled_modules = [] for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): compiled_modules.append(module) if Options.isShowInclusion(): inclusion_logger.info( "Included compiled module '%s'." % module.getFullName() ) elif module.isPythonExtensionModule(): addExtensionModuleEntryPoint(module) if Options.isShowInclusion(): inclusion_logger.info( "Included extension module '%s'." % module.getFullName() ) elif module.isUncompiledPythonModule(): if Options.isShowInclusion(): inclusion_logger.info( "Included uncompiled module '%s'." % module.getFullName() ) else: assert False, module # Pick filenames. source_dir = OutputDirectories.getSourceDirectoryPath() module_filenames = pickSourceFilenames( source_dir=source_dir, modules=compiled_modules ) setupProgressBar( stage="C Source Generation", unit="module", total=len(compiled_modules), ) # Generate code for compiled modules, this can be slow, so do it separately # with a progress bar. for module in compiled_modules: c_filename = module_filenames[module] reportProgressBar( item=module.getFullName(), ) source_code = CodeGeneration.generateModuleCode( module=module, data_filename=os.path.basename(c_filename + "onst"), # Really .const ) writeSourceCode(filename=c_filename, source_code=source_code) closeProgressBar() ( helper_decl_code, helper_impl_code, constants_header_code, constants_body_code, ) = CodeGeneration.generateHelpersCode() writeSourceCode( filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code ) writeSourceCode( filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code ) writeSourceCode( filename=os.path.join(source_dir, "__constants.h"), source_code=constants_header_code, ) writeSourceCode( filename=os.path.join(source_dir, "__constants.c"), source_code=constants_body_code, )
def makeSourceDirectory(main_module): """Get the full list of modules imported, create code for all of them.""" # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=too-many-branches assert main_module.isCompiledPythonModule() # assert main_module in ModuleRegistry.getDoneModules() # We might have chosen to include it as bytecode, and only compiled it for # fun, and to find its imports. In this case, now we just can drop it. Or # a module may shadow a frozen module, but be a different one, then we can # drop the frozen one. # TODO: This really should be done when the compiled module comes into # existence. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): uncompiled_module = ModuleRegistry.getUncompiledModule( module_name=module.getFullName(), module_filename=module.getCompileTimeFilename(), ) if uncompiled_module is not None: # We now need to decide which one to keep, compiled or uncompiled # module. Some uncompiled modules may have been asked by the user # or technically required. By default, frozen code if it exists # is preferred, as it will be from standalone mode adding it. if ( uncompiled_module.isUserProvided() or uncompiled_module.isTechnical() ): ModuleRegistry.removeDoneModule(module) else: ModuleRegistry.removeUncompiledModule(uncompiled_module) # Lets check if the recurse-to modules are actually present, and warn the # user if one of those was not found. for any_case_module in Options.getShallFollowModules(): if "*" in any_case_module or "{" in any_case_module: continue for module in ModuleRegistry.getDoneModules(): if module.getFullName() == any_case_module: break else: general.warning("Not recursing to unused '%s'." % any_case_module) # Prepare code generation, i.e. execute finalization for it. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): Finalization.prepareCodeGeneration(module) # Pick filenames. source_dir = OutputDirectories.getSourceDirectoryPath() module_filenames = pickSourceFilenames( source_dir=source_dir, modules=ModuleRegistry.getDoneModules() ) # Generate code for modules. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): c_filename = module_filenames[module] source_code = CodeGeneration.generateModuleCode( module=module, data_filename=os.path.basename(c_filename + "onst"), # Really .const ) writeSourceCode(filename=c_filename, source_code=source_code) if Options.isShowInclusion(): inclusion_logger.info( "Included compiled module '%s'." % module.getFullName() ) elif module.isPythonShlibModule(): addShlibEntryPoint(module) if Options.isShowInclusion(): inclusion_logger.info( "Included extension module '%s'." % module.getFullName() ) elif module.isUncompiledPythonModule(): if Options.isShowInclusion(): inclusion_logger.info( "Included uncompiled module '%s'." % module.getFullName() ) else: assert False, module ( helper_decl_code, helper_impl_code, constants_header_code, constants_body_code, ) = CodeGeneration.generateHelpersCode() writeSourceCode( filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code ) writeSourceCode( filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code ) writeSourceCode( filename=os.path.join(source_dir, "__constants.h"), source_code=constants_header_code, ) writeSourceCode( filename=os.path.join(source_dir, "__constants.c"), source_code=constants_body_code, ) for filename, source_code in Plugins.getExtraCodeFiles().items(): target_dir = os.path.join(source_dir, "plugins") if not os.path.isdir(target_dir): makePath(target_dir) writeSourceCode( filename=os.path.join(target_dir, filename), source_code=source_code )
def namifyConstant(constant): # Many branches, statements and every case has a return, this is a huge case # statement, that encodes the naming policy of constants, with often complex # conditions, pylint: disable=too-many-branches,too-many-return-statements,too-many-statements if type(constant) is int: if constant == 0: result = "int_0" elif constant > 0: result = "int_pos_%d" % constant else: result = "int_neg_%d" % abs(constant) if len(result) > 32: result = _digest(result) return result elif type(constant) is long: if constant == 0: result = "long_0" elif constant > 0: result = "long_pos_%d" % constant else: result = "long_neg_%d" % abs(constant) if len(result) > 32: result = _digest(result) return result elif constant is None: return "none" elif constant is True: return "true" elif constant is False: return "false" elif constant is Ellipsis: return "ellipsis" elif type(constant) is str: return "str_" + _namifyString(constant) elif type(constant) is bytes: return "bytes_" + _namifyString(constant) elif type(constant) is unicode: if _isAscii(constant): return "unicode_" + _namifyString(str(constant)) else: # Others are better digested to not cause compiler trouble return "unicode_digest_" + _digest(repr(constant)) elif type(constant) is float: if math.isnan(constant): return "float_%s_nan" % ("minus" if math.copysign(1, constant) < 0 else "plus") return "float_%s" % repr(constant).replace(".", "_").replace( "-", "minus_").replace("+", "") elif type(constant) is complex: value = "%s__%s" % (constant.real, constant.imag) value = value.replace("+", "p").replace("-", "m").replace(".", "_") if value.startswith("(") and value.endswith(")"): value = value[1:-1] return "complex_%s" % value elif type(constant) is dict: if constant == {}: return "dict_empty" else: return "dict_" + _digest(repr(constant)) elif type(constant) is set: if constant == set(): return "set_empty" else: return "set_" + _digest(repr(constant)) elif type(constant) is frozenset: if constant == frozenset(): return "frozenset_empty" else: return "frozenset_" + _digest(repr(constant)) elif type(constant) is tuple: if constant == (): return "tuple_empty" else: try: result = "_".join(namifyConstant(value) for value in constant) if len(result) > 60: result = _digest(repr(constant)) return "tuple_" + result + "_tuple" except ExceptionCannotNamify: general.warning("Couldn't namify '%r'" % (constant, )) return "tuple_" + _digest(repr(constant)) elif type(constant) is list: if constant == []: return "list_empty" else: try: result = "_".join(namifyConstant(value) for value in constant) if len(result) > 60: result = _digest(repr(constant)) return "list_" + result + "_list" except ExceptionCannotNamify: general.warning("Couldn't namify '%r'" % value) return "list_" + _digest(repr(constant)) elif type(constant) is bytearray: return "bytearray_" + _digest(repr(constant)) elif type(constant) is xrange: return "xrange_%s" % ( str(constant)[7 if str is bytes else 6:-1].replace( " ", "").replace(",", "_").replace("-", "neg")) elif type(constant) is slice: return "slice_%s_%s_%s" % ( namifyConstant(constant.start), namifyConstant(constant.stop), namifyConstant(constant.step), ) elif constant in builtin_anon_values: return "anon_%s" % builtin_anon_values[constant] elif type(constant) is type: return "type_%s" % constant.__name__ elif type(constant) is BuiltinFunctionType: assert constant in builtin_named_values_list return "builtin_%s" % constant.__name__ elif constant is NotImplemented: return "type_notimplemented" else: raise ExceptionCannotNamify("%r" % constant, type(constant))
def _detectImports(command, user_provided, technical): # This is pretty complicated stuff, with variants to deal with. # pylint: disable=too-many-branches,too-many-locals,too-many-statements # Print statements for stuff to show, the modules loaded. if python_version >= 0x300: command += """ print("\\n".join(sorted( "import %s # sourcefile %s" % (module.__name__, module.__file__) for module in sys.modules.values() if getattr(module, "__file__", None) not in (None, "<frozen>" ))), file = sys.stderr)""" reduced_path = [ path_element for path_element in sys.path if not areSamePaths(path_element, ".") if not areSamePaths( path_element, os.path.dirname(sys.modules["__main__"].__file__)) ] # Make sure the right import path (the one Nuitka binary is running with) # is used. command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" % repr(reduced_path)) + command if str is not bytes: command = command.encode("utf8") _stdout, stderr, exit_code = executeProcess( command=( sys.executable, "-s", "-S", "-v", "-c", "import sys;exec(sys.stdin.read())", ), stdin=command, env=dict(os.environ, PYTHONIOENCODING="utf-8"), ) assert type(stderr) is bytes # Don't let errors here go unnoticed. if exit_code != 0: # An error by the user pressing CTRL-C should not lead to the below output. if b"KeyboardInterrupt" in stderr: general.sysexit("Pressed CTRL-C while detecting early imports.") general.warning( "There is a problem with detecting imports, CPython said:") for line in stderr.split(b"\n"): printError(line) general.sysexit("Error, please report the issue with above output.") result = [] detections = [] for line in stderr.replace(b"\r", b"").split(b"\n"): if line.startswith(b"import "): parts = line.split(b" # ", 2) module_name = parts[0].split(b" ", 2)[1] origin = parts[1].split()[0] if python_version >= 0x300: module_name = module_name.decode("utf8") module_name = ModuleName(module_name) if origin == b"precompiled": # This is a ".pyc" file that was imported, even before we have a # chance to do anything, we need to preserve it. filename = parts[1][len(b"precompiled from "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 3, "precompiled", filename)) elif origin == b"from" and python_version < 0x300: filename = parts[1][len(b"from "):] if str is not bytes: # For consistency, and maybe later reuse filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) else: assert False elif origin == b"sourcefile": filename = parts[1][len(b"sourcefile "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) elif filename.endswith(".pyc"): detections.append( (module_name, 3, "precompiled", filename)) elif not filename.endswith("<frozen>"): # Python3 started lying in "__name__" for the "_decimal" # calls itself "decimal", which then is wrong and also # clashes with "decimal" proper if python_version >= 0x300 and module_name == "decimal": module_name = ModuleName("_decimal") detections.append((module_name, 2, "extension", filename)) elif origin == b"dynamically": # Shared library in early load, happens on RPM based systems and # or self compiled Python installations. filename = parts[1][len(b"dynamically loaded from "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 1, "extension", filename)) for module_name, _priority, kind, filename in sorted(detections): if isStandardLibraryNoAutoInclusionModule(module_name): continue if kind == "extension": _detectedExtensionModule( filename=filename, module_name=module_name, result=result, technical=technical, ) elif kind == "precompiled": _detectedPrecompiledFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "sourcefile": _detectedSourceFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) else: assert False, kind return result