def __init__( self, noinclude_setuptools_mode, noinclude_pytest_mode, noinclude_ipython_mode, noinclude_default_mode, custom_choices, ): # Default manually to default argument value: if noinclude_setuptools_mode is None: noinclude_setuptools_mode = noinclude_default_mode if noinclude_pytest_mode is None: noinclude_pytest_mode = noinclude_default_mode if noinclude_ipython_mode is None: noinclude_ipython_mode = noinclude_default_mode self.config = parsePackageYaml(__package__, "anti-bloat.yml") self.handled_modules = OrderedDict() # These should be checked, to allow disabling anti-bloat contents. self.control_tags = set() if noinclude_setuptools_mode != "allow": self.handled_modules["setuptools"] = noinclude_setuptools_mode else: self.control_tags.add("allow_setuptools") if noinclude_pytest_mode != "allow": self.handled_modules["pytest"] = noinclude_pytest_mode else: self.control_tags.add("allow_pytest") if noinclude_ipython_mode != "allow": self.handled_modules["IPython"] = noinclude_ipython_mode else: self.control_tags.add("allow_ipython") for custom_choice in custom_choices: if ":" not in custom_choice: self.sysexit( "Error, malformed value '%s' for '--noinclude-custom-mode' used." % custom_choice ) module_name, mode = custom_choice.rsplit(":", 1) if mode not in ("error", "warning", "nofollow", "allow", "bytecode"): self.sysexit( "Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'" % (mode, custom_choice) ) self.handled_modules[ModuleName(module_name)] = mode
def _getAnonBuiltins(): # We use the order when encoding in the constants blob. Therefore it is imported # to not reorder these values, the C side uses the absolute indexes. anon_names = OrderedDict() anon_codes = OrderedDict() # Strangely these are not in Python3 types module anon_names["NoneType"] = type(None) anon_codes["NoneType"] = "Py_TYPE(Py_None)" anon_names["ellipsis"] = type(Ellipsis) # see above anon_codes["ellipsis"] = "&PyEllipsis_Type" anon_names["NotImplementedType"] = type(NotImplemented) anon_codes["NotImplementedType"] = "Py_TYPE(Py_NotImplemented)" anon_names["function"] = FunctionType anon_codes["function"] = "&PyFunction_Type" anon_names["generator"] = GeneratorType anon_codes["generator"] = "&PyGenerator_Type" anon_names["builtin_function_or_method"] = BuiltinFunctionType anon_codes["builtin_function_or_method"] = "&PyCFunction_Type" # Can't really have it until we have __nuitka__ # "compiled_function" : BuiltinFunctionType, # "compiled_generator" : GeneratorType, # see above # anon_codes["compiled_function"] = "&Nuitka_Function_Type" # anon_codes["compiled_generator"] = "&Nuitka_Generator_Type" anon_names["code"] = type(_getAnonBuiltins.__code__) anon_codes["code"] = "&PyCode_Type" if python_version < 0x300: # There are only there for Python2, # pylint: disable=I0021,no-name-in-module from types import ClassType, InstanceType, MethodType with open(sys.executable) as any_file: anon_names["file"] = type(any_file) anon_codes["file"] = "&PyFile_Type" anon_names["classobj"] = ClassType anon_codes["classobj"] = "&PyClass_Type" anon_names["instance"] = InstanceType anon_codes["instance"] = "&PyInstance_Type" anon_names["instancemethod"] = MethodType anon_codes["instancemethod"] = "&PyMethod_Type" return anon_names, anon_codes
def __init__(self, name, code_prefix, source_ref): CodeNodeBase.__init__( self, name = name, code_prefix = code_prefix, source_ref = source_ref ) self.providing = OrderedDict() self.keeper_variables = OrderedSet() self.temp_variables = OrderedDict() self.temp_scopes = OrderedDict()
def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None
def getPreprocessorSymbols(cls): """Let plugins provide C defines to be used in compilation. Notes: The plugins can each contribute, but are hopefully using a namespace for their defines. Returns: OrderedDict(), where None value indicates no define value, i.e. "-Dkey=value" vs. "-Dkey" """ if cls.preprocessor_symbols is None: cls.preprocessor_symbols = OrderedDict() for plugin in getActivePlugins(): value = plugin.getPreprocessorSymbols() if value is not None: assert type(value) is dict, value # We order per plugin, but from the plugins, lets just take a dict # and achieve determism by ordering the defines by name. for key, value in sorted(value.items()): # False alarm, pylint: disable=I0021,unsupported-assignment-operation cls.preprocessor_symbols[key] = value return cls.preprocessor_symbols
def detectUsedDLLs(source_dir, standalone_entry_points): def addDLLInfo(count, source_dir, original_filename, binary_filename, package_name): used_dlls = detectBinaryDLLs(is_main_executable=count == 0, source_dir=source_dir, original_filename=original_filename, binary_filename=binary_filename, package_name=package_name) return binary_filename, used_dlls result = OrderedDict() with ThreadPoolExecutor(max_workers=Utils.getCoreCount() * 3) as worker_pool: workers = [] for count, (original_filename, binary_filename, package_name) in enumerate(standalone_entry_points): workers.append( worker_pool.submit(addDLLInfo, count, source_dir, original_filename, binary_filename, package_name)) for binary_filename, used_dlls in waitWorkers(workers): for dll_filename in used_dlls: # We want these to be absolute paths. Solve that in the parts # where detectBinaryDLLs is platform specific. assert os.path.isabs(dll_filename), dll_filename if dll_filename not in result: result[dll_filename] = [] result[dll_filename].append(binary_filename) return result
def getPreprocessorSymbols(): """ Let plugins provide C defines to be used in compilation. Notes: The plugins can each contribute, but are hopefully using a namespace for their defines. Returns: OrderedDict(), where None value indicates no define value, i.e. "-Dkey=value" vs. "-Dkey" """ result = OrderedDict() for plugin in getActivePlugins(): value = plugin.getPreprocessorSymbols() if value is not None: assert type(value) is dict # We order per plugin, but from the plugins, lets just take a dict # and achieve determism by ordering the defines by name. for key, value in sorted(value.items()): result[key] = value return result
def detectUsedDLLs(source_dir, standalone_entry_points, use_cache, update_cache): def addDLLInfo(count, source_dir, original_filename, binary_filename, package_name): used_dlls = detectBinaryDLLs( is_main_executable=count == 0, source_dir=source_dir, original_filename=original_filename, binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) # Allow plugins to prevent inclusion, this may discard things from used_dlls. Plugins.removeDllDependencies(dll_filename=binary_filename, dll_filenames=used_dlls) for dll_filename in sorted(tuple(used_dlls)): if not os.path.isfile(dll_filename): if _unfound_dlls: general.warning( "Dependency '%s' could not be found, you might need to copy it manually." % dll_filename) _unfound_dlls.add(dll_filename) used_dlls.remove(dll_filename) return binary_filename, used_dlls result = OrderedDict() with ThreadPoolExecutor(max_workers=Utils.getCoreCount() * 3) as worker_pool: workers = [] for count, standalone_entry_point in enumerate( standalone_entry_points): workers.append( worker_pool.submit( addDLLInfo, count, source_dir, standalone_entry_point.source_path, standalone_entry_point.dest_path, standalone_entry_point.package_name, )) for binary_filename, used_dlls in waitWorkers(workers): for dll_filename in used_dlls: # We want these to be absolute paths. Solve that in the parts # where detectBinaryDLLs is platform specific. assert os.path.isabs(dll_filename), dll_filename if dll_filename not in result: result[dll_filename] = [] result[dll_filename].append(binary_filename) return result
def _detectBinaryPathDLLsMacOS(original_dir, binary_filename, package_name, keep_unresolved, recursive): assert os.path.exists(binary_filename), binary_filename package_specific_dirs = _getLdLibraryPath(package_name=package_name, python_rpath=None, original_dir=original_dir) # This is recursive potentially and might add more and more. stdout = getOtoolDependencyOutput( filename=binary_filename, package_specific_dirs=_getLdLibraryPath(package_name=package_name, python_rpath=None, original_dir=original_dir), ) paths = _parseOtoolListingOutput(stdout) had_self, resolved_result = _resolveBinaryPathDLLsMacOS( original_dir=original_dir, binary_filename=binary_filename, paths=paths, package_specific_dirs=package_specific_dirs, ) if recursive: merged_result = OrderedDict(resolved_result) for sub_dll_filename in resolved_result: _, sub_result = _detectBinaryPathDLLsMacOS( original_dir=os.path.dirname(sub_dll_filename), binary_filename=sub_dll_filename, package_name=package_name, recursive=True, keep_unresolved=True, ) merged_result.update(sub_result) resolved_result = merged_result if keep_unresolved: return had_self, resolved_result else: return OrderedSet(resolved_result)
def _getBuiltinExceptionNames(): def isExceptionName(builtin_name): if builtin_name.endswith("Error") or builtin_name.endswith( "Exception"): return True elif builtin_name in ( "StopIteration", "GeneratorExit", "SystemExit", "NotImplemented", "KeyboardInterrupt", "StopAsyncIteration", ): return True else: return False exceptions = OrderedDict() # Hide Python3 changes for built-in exception names if python_version < 0x300: import exceptions as builtin_exceptions for key in sorted(dir(builtin_exceptions)): name = str(key) if isExceptionName(name): exceptions[name] = getattr(builtin_exceptions, key) for key in sorted(dir(builtins)): name = str(key) if isExceptionName(name): exceptions[name] = getattr(builtins, key) else: for key in sorted(dir(builtins)): if isExceptionName(key): exceptions[key] = getattr(builtins, key) return list(exceptions.keys()), exceptions
def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name) return self.propagation[variable_name]
def __init__(self, name, code_prefix): CodeNodeMixin.__init__(self, name=name, code_prefix=code_prefix) # TODO: Only Python3 classes need this to be an ordered dict, the order # of it should come from elsewhere though. self.providing = OrderedDict() self.temp_variables = {} self.temp_scopes = {} self.preserver_id = 0
def __init__(self, setuptools_mode, custom_choices): self.handled_modules = OrderedDict() if setuptools_mode != "allow": self.handled_modules["setuptools"] = setuptools_mode for custom_choice in custom_choices: if ":" not in custom_choice: self.sysexit( "Error, malformed value '%s' for '--noinclude-custom-mode' used." % custom_choice ) module_name, mode = custom_choice.rsplit(":", 1) if mode not in ("error", "warning", "nofollow", "allow"): self.sysexit( "Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'" % (mode, custom_choice) ) self.handled_modules[ModuleName(module_name)] = mode
def __init__(self, name, code_prefix): CodeNodeMixin.__init__(self, name=name, code_prefix=code_prefix) self.providing = OrderedDict() self.temp_variables = OrderedDict() self.temp_scopes = OrderedDict() self.preserver_id = 0
def getExtraCodeFiles(): result = OrderedDict() for plugin in getActivePlugins(): value = plugin.getExtraCodeFiles() if value is not None: assert type(value) is dict # We order per plugin, but from the plugins, lets just take a dict # and achieve determism by ordering the files by name. for key, value in sorted(value.items()): assert key not in result, key result["plugin." + plugin.plugin_name + "." + key] = value return result
def detectUsedDLLs(standalone_entry_points): result = OrderedDict() for original_filename, binary_filename, package_name in standalone_entry_points: used_dlls = detectBinaryDLLs(original_filename=original_filename, binary_filename=binary_filename, package_name=package_name) for dll_filename in used_dlls: # We want these to be absolute paths. assert os.path.isabs(dll_filename), dll_filename if dll_filename not in result: result[dll_filename] = [] result[dll_filename].append(binary_filename) return result
def _resolveBinaryPathDLLsMacOS(original_dir, binary_filename, paths, package_specific_dirs): had_self = False result = OrderedDict() rpaths = _detectBinaryRPathsMacOS(original_dir, binary_filename) rpaths.update(package_specific_dirs) for path in paths: if path.startswith("@rpath/"): # Resolve rpath to just the ones given, first match. for rpath in rpaths: if os.path.exists(os.path.join(rpath, path[7:])): resolved_path = os.path.normpath( os.path.join(rpath, path[7:])) break else: # This is only a guess, might be missing package specific directories. resolved_path = os.path.join(original_dir, path[7:]) elif path.startswith("@loader_path/"): resolved_path = os.path.join(original_dir, path[13:]) elif os.path.basename(path) == os.path.basename(binary_filename): # We ignore the references to itself coming from the library id. continue else: resolved_path = path if not os.path.exists(resolved_path): inclusion_logger.sysexit( "Error, failed to resolve DLL path %s (for %s), please report the bug." % (path, binary_filename)) # Some libraries depend on themselves. if areSamePaths(binary_filename, resolved_path): had_self = True continue result[resolved_path] = path return had_self, result
def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation
def detectUsedDLLs(source_dir, standalone_entry_points): result = OrderedDict() for count, (original_filename, binary_filename, _package_name) in enumerate(standalone_entry_points): used_dlls = detectBinaryDLLs(is_main_executable=count == 0, source_dir=source_dir, original_filename=original_filename, binary_filename=binary_filename) for dll_filename in used_dlls: # We want these to be absolute paths. Solve that in the parts # where detectBinaryDLLs is platform specific. assert os.path.isabs(dll_filename), dll_filename if dll_filename not in result: result[dll_filename] = [] result[dll_filename].append(binary_filename) return result
def __init__(self, hinted_json_file): """ Read the JSON file and enable any standard plugins. Notes: Read the JSON file produced during the get-hints step. It will contain a list of imported items ("calls") and a list of modules / packages ("files") to be loaded and recursed into. Depending on the items in 'files', we will trigger loading standard plugins. """ # start a timer self.timer = StopWatch() self.timer.start() self.implicit_imports = OrderedSet() # speed up repeated lookups self.ignored_modules = OrderedSet() # speed up repeated lookups options = Options.options # Load json file contents from --hinted-json-file= argument filename = hinted_json_file try: # read it and extract the two lists import_info = json.loads(getFileContents(filename)) except (ValueError, FileNotFoundError): raise FileNotFoundError('Cannot load json file %s' % filename) self.import_calls = import_info["calls"] self.import_files = import_info["files"] self.msg_count = dict() # to limit keep messages self.msg_limit = 21 # suppress pytest / _pytest / unittest? # TODO: disabled because self.getPluginOptionBool does not exist anymore #self.accept_test = self.getPluginOptionBool("test", False) self.accept_test = False """ Check if we should enable any (optional) standard plugins. This code must be modified whenever more standard plugin become available. """ show_msg = False # only show info if one ore more detected # indicators for found packages tk = np = qt = scipy = mp = pmw = torch = sklearn = False eventlet = tflow = gevent = mpl = trio = dill = False msg = "'%s' is adding the following options:" % os.path.basename( self.plugin_name) # we need matplotlib-specific cleanup to happen first: # if no mpl backend is used, reference to matplotlib is removed alltogether if "matplotlib.backends" not in self.import_files: temp = [ f for f in self.import_calls if not f.startswith(("matplotlib", "mpl_toolkits")) ] self.import_calls = temp temp = [ f for f in self.import_files if not f.startswith(("matplotlib", "mpl_toolkits")) ] self.import_files = temp # detect required standard plugins and request enabling them for m in self.import_calls: # scan thru called items if m in ("numpy", "numpy.*"): np = True show_msg = True if m in ("matplotlib", "matplotlib.*"): mpl = True show_msg = True elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"): tk = True show_msg = True elif m.startswith(("PyQt", "PySide")): qt = True show_msg = True elif m in ("scipy", "scipy.*"): scipy = True show_msg = True elif m in ("multiprocessing", "multiprocessing.*") and getOS() == "Windows": mp = True show_msg = True elif m in ("Pmw", "Pmw.*"): pmw = True show_msg = True elif m == "torch": torch = True show_msg = True elif m in ("sklearn", "sklearn.*"): sklearn = True show_msg = True elif m in ("tensorflow", "tensorflow.*"): tflow = True show_msg = True elif m in ("gevent", "gevent.*"): gevent = True show_msg = True elif m in ("eventlet", "eventlet.*"): eventlet = True show_msg = True elif m in ("dill", "dill.*"): dill = True show_msg = True # elif m in ("trio", "trio.*"): # trio = True # show_msg = True if show_msg is True: self.info(msg) to_enable = OrderedDict() if np: to_enable["numpy"] = { "matplotlib": mpl, "scipy": scipy, # TODO: Numpy plugin didn't use this, work in progress or not needed? # "sklearn" : sklearn } if tk: to_enable["tk-inter"] = {} if qt: # TODO more scrutiny for the qt options! to_enable["qt-plugins"] = {} if mp: to_enable["multiprocessing"] = {} if pmw: to_enable["pmw-freezer"] = {} if torch: to_enable["torch"] = {} if tflow: to_enable["tensorflow"] = {} if gevent: to_enable["gevent"] = {} if eventlet: to_enable["eventlet"] = {} if dill: to_enable["dill-compat"] = {} # if trio: # to_enable["trio"] = {} recurse_count = 0 for f in self.import_files: # request recursion to called modules if self.accept_test is False and f.startswith( ("pytest", "_pytest", "unittest")): continue options.recurse_modules.append(f) recurse_count += 1 # no plugin detected, but recursing to modules? if not show_msg and recurse_count > 0: self.info(msg) for plugin_name, option_values in to_enable.items(): self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name) # No the values could be set. lateActivatePlugin(plugin_name, option_values) if len(self.import_files) > 0: msg = "--recurse-to=%s and %i more modules" % ( self.import_files[-1], recurse_count - 1, ) self.info(msg) self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
from nuitka import Options from nuitka.__past__ import basestring # pylint: disable=I0021,redefined-builtin from nuitka.build.DataComposerInterface import deriveModuleConstantsBlobName from nuitka.containers.odict import OrderedDict from nuitka.containers.oset import OrderedSet from nuitka.Errors import NuitkaPluginError from nuitka.freezer.IncludedEntryPoints import makeDllEntryPointOld from nuitka.ModuleRegistry import addUsedModule from nuitka.Tracing import plugins_logger, printLine from nuitka.utils.FileOperations import makePath, relpath from nuitka.utils.Importing import importFileAsModule from nuitka.utils.ModuleNames import ModuleName from .PluginBase import NuitkaPluginBase, post_modules, pre_modules active_plugins = OrderedDict() plugin_name2plugin_classes = {} plugin_options = {} plugin_values = {} user_plugins = OrderedSet() def _addActivePlugin(plugin_class, args, force=False): plugin_name = plugin_class.plugin_name # No duplicates please. if not force: assert plugin_name not in active_plugins.keys(), ( plugin_name, active_plugins[plugin_name], )
class LocalsDictHandleBase(object): # TODO: Might remove some of these later, pylint: disable=too-many-instance-attributes __slots__ = ( "locals_name", # TODO: Specialize what the kinds really use. "variables", "local_variables", "providing", "mark_for_propagation", "propagation", "owner", "complete", ) @counted_init def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None self.complete = False if isCountingInstances(): __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name @staticmethod def isModuleScope(): return False @staticmethod def isClassScope(): return False @staticmethod def isFunctionScope(): return False def getProvidedVariables(self): return self.providing.values() def registerProvidedVariable(self, variable): variable_name = variable.getName() self.providing[variable_name] = variable def unregisterProvidedVariable(self, variable): """Remove provided variable, e.g. because it became unused.""" variable_name = variable.getName() if variable_name in self.providing: del self.providing[variable_name] registerClosureVariable = registerProvidedVariable unregisterClosureVariable = unregisterProvidedVariable def hasProvidedVariable(self, variable_name): """Test if a variable is provided.""" return variable_name in self.providing def getProvidedVariable(self, variable_name): """Test if a variable is provided.""" return self.providing[variable_name] def getLocalsRelevantVariables(self): """The variables relevant to locals.""" return self.providing.values() def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable(owner=self, variable_name=variable_name) self.variables[variable_name] = result return self.variables[variable_name] # TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here. def getLocalVariable(self, owner, variable_name): if variable_name not in self.local_variables: result = Variables.LocalVariable(owner=owner, variable_name=variable_name) self.local_variables[variable_name] = result return self.local_variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): # Make it unusable when it's become empty, not used. self.owner.locals_scope = None del self.owner del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() for variable in self.local_variables.values(): variable.finalize() del self.variables del self.providing def markAsComplete(self, trace_collection): self.complete = True self._considerUnusedUserLocalVariables(trace_collection) self._considerPropagation(trace_collection) # TODO: Limited to Python2 classes for now, more overloads need to be added, this # ought to be abstract and have variants with TODOs for each of them. @staticmethod def _considerPropagation(trace_collection): """For overload by scope type. Check if this can be replaced.""" def _considerUnusedUserLocalVariables(self, trace_collection): """Check scope for unused variables.""" provided = self.getProvidedVariables() removals = [] for variable in provided: if (variable.isLocalVariable() and not variable.isParameterVariable() and variable.getOwner() is self.owner): empty = trace_collection.hasEmptyTraces(variable) if empty: removals.append(variable) for variable in removals: self.unregisterProvidedVariable(variable) trace_collection.signalChange( "var_usage", self.owner.getSourceReference(), message="Remove unused local variable '%s'." % variable.getName(), )
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Reports about code generation. Initially this is about missing optimization only, but it should expand into real stuff. """ from nuitka.containers.odict import OrderedDict from nuitka.containers.oset import OrderedSet from nuitka.Tracing import codegen_logger, optimization_logger _missing_helpers = OrderedDict() _missing_operations = OrderedSet() _error_for_missing = False # _error_for_missing = True def doMissingOptimizationReport(): for helper, source_refs in _missing_helpers.items(): message = "Missing C helper code variant, used fallback: %s at %s" % ( ",".join(source_ref.getAsString() for source_ref in source_refs), helper, ) if _error_for_missing:
def _getAnonBuiltins(): # We use the order when encoding in the constants blob. Therefore it is imported # to not reorder these values, the C side uses the absolute indexes. anon_names = OrderedDict() anon_codes = OrderedDict() # Strangely these are not in Python3 types module anon_names["NoneType"] = type(None) anon_codes["NoneType"] = "Py_TYPE(Py_None)" anon_names["ellipsis"] = type(Ellipsis) # see above anon_codes["ellipsis"] = "&PyEllipsis_Type" anon_names["NotImplementedType"] = type(NotImplemented) anon_codes["NotImplementedType"] = "Py_TYPE(Py_NotImplemented)" anon_names["function"] = FunctionType anon_codes["function"] = "&PyFunction_Type" anon_names["generator"] = GeneratorType anon_codes["generator"] = "&PyGenerator_Type" anon_names["builtin_function_or_method"] = BuiltinFunctionType anon_codes["builtin_function_or_method"] = "&PyCFunction_Type" # Can't really have it until we have __nuitka__ # "compiled_function" : BuiltinFunctionType, # "compiled_generator" : GeneratorType, # see above # anon_codes["compiled_function"] = "&Nuitka_Function_Type" # anon_codes["compiled_generator"] = "&Nuitka_Generator_Type" anon_names["code"] = type(_getAnonBuiltins.__code__) anon_codes["code"] = "&PyCode_Type" if python_version < 0x300: # There are only there for Python2, # pylint: disable=I0021,no-name-in-module from types import ClassType, InstanceType, MethodType # We can to be sure of the type here, so use open without encoding, pylint: disable=unspecified-encoding with open(sys.executable) as any_file: anon_names["file"] = type(any_file) anon_codes["file"] = "&PyFile_Type" anon_names["classobj"] = ClassType anon_codes["classobj"] = "&PyClass_Type" anon_names["instance"] = InstanceType anon_codes["instance"] = "&PyInstance_Type" anon_names["instancemethod"] = MethodType anon_codes["instancemethod"] = "&PyMethod_Type" if python_version >= 0x270: anon_names["version_info"] = type(sys.version_info) anon_codes[ "version_info"] = 'Py_TYPE(Nuitka_SysGetObject("version_info"))' if python_version >= 0x3A0: # 3.10 only code, pylint: disable=I0021,unsupported-binary-operation anon_names["UnionType"] = type(int | str) anon_codes["UnionType"] = "Nuitka_PyUnion_Type" return anon_names, anon_codes
class NuitkaPluginAntiBloat(NuitkaPluginBase): plugin_name = "anti-bloat" plugin_desc = "Patch stupid imports out of common library modules source code." def __init__(self, setuptools_mode, custom_choices): self.handled_modules = OrderedDict() if setuptools_mode != "allow": self.handled_modules["setuptools"] = setuptools_mode for custom_choice in custom_choices: if ":" not in custom_choice: self.sysexit( "Error, malformed value '%s' for '--noinclude-custom-mode' used." % custom_choice ) module_name, mode = custom_choice.rsplit(":", 1) if mode not in ("error", "warning", "nofollow", "allow"): self.sysexit( "Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'" % (mode, custom_choice) ) self.handled_modules[ModuleName(module_name)] = mode @classmethod def addPluginCommandLineOptions(cls, group): group.add_option( "--noinclude-setuptools-mode", action="store", dest="setuptools_mode", choices=("error", "warning", "nofollow", "allow"), default="allow", help="""\ What to do if a setuptools import is encountered. This can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-custom-mode", action="append", dest="custom_choices", default=[], help="""\ What to do if a specific import is encountered. Format is module name, which can and should be a top level package and then one choice, "error", "warning", "nofollow", e.g. PyQt5:error.""", ) def onModuleEncounter(self, module_filename, module_name, module_kind): for handled_module_name, mode in self.handled_modules.items(): if module_name.hasNamespace(handled_module_name): # Make sure the compilation abrts. if mode == "error": raise NuitkaForbiddenImportEncounter(module_name) # Either issue a warning, or pretend the module doesn't exist for standalone or # at least will not be included. if mode == "warning": self.warning("Forbidden import of '%s' encountered." % module_name) elif mode == "nofollow": self.info( "Forcing import of '%s' to not be followed." % module_name ) return ( False, "user requested to not follow '%s' import" % module_name, ) # Do not provide an opinion about it. return None
anon_codes["code"] = "&PyCode_Type" if python_version < 0x300: # There are only there for Python2, # pylint: disable=I0021,no-name-in-module from types import ClassType, InstanceType, MethodType with open(sys.executable) as any_file: anon_names["file"] = type(any_file) anon_codes["file"] = "&PyFile_Type" anon_names["classobj"] = ClassType anon_codes["classobj"] = "&PyClass_Type" anon_names["instance"] = InstanceType anon_codes["instance"] = "&PyInstance_Type" anon_names["instancemethod"] = MethodType anon_codes["instancemethod"] = "&PyMethod_Type" return anon_names, anon_codes builtin_anon_names, builtin_anon_codes = _getAnonBuiltins() builtin_anon_values = OrderedDict( (b, a) for a, b in builtin_anon_names.items()) # For being able to check if it's not hashable, we need something not using # a hash. builtin_anon_value_list = tuple(builtin_anon_values)
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Reports about code generation. Initially this is about missing optimization only, but it should expand into real stuff. """ from nuitka.containers.odict import OrderedDict from nuitka.containers.oset import OrderedSet from nuitka.Tracing import codegen_logger, optimization_logger _missing_helpers = OrderedDict() _missing_operations = OrderedSet() _missing_trust = OrderedDict() _error_for_missing = False # _error_for_missing = True def doMissingOptimizationReport(): for helper, source_refs in _missing_helpers.items(): message = "Missing C helper code variant, used fallback: %s at %s" % ( ",".join(source_ref.getAsString() for source_ref in source_refs), helper, )
class NuitkaPluginAntiBloat(NuitkaPluginBase): plugin_name = "anti-bloat" plugin_desc = ( "Patch stupid imports out of widely used library modules source codes." ) @staticmethod def isAlwaysEnabled(): return True def __init__( self, noinclude_setuptools_mode, noinclude_pytest_mode, noinclude_ipython_mode, noinclude_default_mode, custom_choices, ): # Default manually to default argument value: if noinclude_setuptools_mode is None: noinclude_setuptools_mode = noinclude_default_mode if noinclude_pytest_mode is None: noinclude_pytest_mode = noinclude_default_mode if noinclude_ipython_mode is None: noinclude_ipython_mode = noinclude_default_mode self.config = parsePackageYaml(__package__, "anti-bloat.yml") self.handled_modules = OrderedDict() # These should be checked, to allow disabling anti-bloat contents. self.control_tags = set() if noinclude_setuptools_mode != "allow": self.handled_modules["setuptools"] = noinclude_setuptools_mode else: self.control_tags.add("allow_setuptools") if noinclude_pytest_mode != "allow": self.handled_modules["pytest"] = noinclude_pytest_mode else: self.control_tags.add("allow_pytest") if noinclude_ipython_mode != "allow": self.handled_modules["IPython"] = noinclude_ipython_mode else: self.control_tags.add("allow_ipython") for custom_choice in custom_choices: if ":" not in custom_choice: self.sysexit( "Error, malformed value '%s' for '--noinclude-custom-mode' used." % custom_choice) module_name, mode = custom_choice.rsplit(":", 1) if mode not in ("error", "warning", "nofollow", "allow", "bytecode"): self.sysexit( "Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'" % (mode, custom_choice)) self.handled_modules[ModuleName(module_name)] = mode @classmethod def addPluginCommandLineOptions(cls, group): group.add_option( "--noinclude-setuptools-mode", action="store", dest="noinclude_setuptools_mode", choices=("error", "warning", "nofollow", "allow"), default=None, help="""\ What to do if a setuptools import is encountered. This package can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-pytest-mode", action="store", dest="noinclude_pytest_mode", choices=("error", "warning", "nofollow", "allow"), default=None, help="""\ What to do if a pytest import is encountered. This package can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-IPython-mode", action="store", dest="noinclude_ipython_mode", choices=("error", "warning", "nofollow", "allow"), default=None, help="""\ What to do if a IPython import is encountered. This package can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-default-mode", action="store", dest="noinclude_default_mode", choices=("error", "warning", "nofollow", "allow"), default="warning", help="""\ This actually provides the default "warning" value for above options, and can be used to turn all of these on.""", ) group.add_option( "--noinclude-custom-mode", action="append", dest="custom_choices", default=[], help="""\ What to do if a specific import is encountered. Format is module name, which can and should be a top level package and then one choice, "error", "warning", "nofollow", e.g. PyQt5:error.""", ) def onModuleSourceCode(self, module_name, source_code): # Complex dealing with many cases, pylint: disable=too-many-branches,too-many-locals,too-many-statements config = self.config.get(module_name) if not config: return source_code # Allow disabling config for a module with matching control tags. for control_tag in config.get("control_tags", ()): if control_tag in self.control_tags: return source_code description = config.get("description", "description not given") # To allow detection if it did anything. change_count = 0 context = {} context_code = config.get("context", "") if type(context_code) in (tuple, list): context_code = "\n".join(context_code) # We trust the yaml files, pylint: disable=eval-used,exec-used context_ready = not bool(context_code) for replace_src, replace_code in config.get("replacements", {}).items(): # Avoid the eval, if the replace doesn't hit. if replace_src not in source_code: continue if replace_code: if not context_ready: try: exec(context_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot execute context code '%s' due to: %s" % (context_code, e)) context_ready = True try: replace_dst = eval(replace_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot evaluate code '%s' in '%s' due to: %s" % (replace_code, context_code, e)) else: replace_dst = "" if type(replace_dst) is not str: self.sysexit( "Error, expression needs to generate string, not %s" % type(replace_dst)) old = source_code source_code = source_code.replace(replace_src, replace_dst) if old != source_code: change_count += 1 append_code = config.get("append_result", "") if type(append_code) in (tuple, list): append_code = "\n".join(append_code) if append_code: if not context_ready: exec(context_code, context) context_ready = True try: append_result = eval(append_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot evaluate code '%s' in '%s' due to: %s" % (append_code, context_code, e)) source_code += "\n" + append_result change_count += 1 if change_count > 0: self.info("Handling module '%s' with %d change(s) for: %s." % (module_name.asString(), change_count, description)) module_code = config.get("module_code", None) if module_code is not None: assert not change_count self.info("Handling module '%s' with full replacement : %s." % (module_name.asString(), description)) source_code = module_code return source_code def onFunctionBodyParsing(self, module_name, function_name, body): config = self.config.get(module_name) if not config: return context = {} context_code = config.get("context", "") if type(context_code) in (tuple, list): context_code = "\n".join(context_code) # We trust the yaml files, pylint: disable=eval-used,exec-used context_ready = not bool(context_code) for change_function_name, replace_code in config.get( "change_function", {}).items(): if function_name != change_function_name: continue if not context_ready: exec(context_code, context) context_ready = True try: replacement = eval(replace_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot evaluate code '%s' in '%s' due to: %s" % (replace_code, context_code, e)) # Single node is required, extrace the generated module body with # single expression only statement value or a function body. replacement = ast.parse(replacement).body[0] if type(replacement) is ast.Expr: body[:] = [ast.Return(replacement.value)] else: body[:] = replacement.body self.info("Updated module '%s' function '%s'." % (module_name.asString(), function_name)) def onModuleEncounter(self, module_filename, module_name, module_kind): for handled_module_name, mode in self.handled_modules.items(): if module_name.hasNamespace(handled_module_name): # Make sure the compilation abrts. if mode == "error": raise NuitkaForbiddenImportEncounter(module_name) # Either issue a warning, or pretend the module doesn't exist for standalone or # at least will not be included. if mode == "warning": self.warning("Unwanted import of '%s' encountered." % module_name) elif mode == "nofollow": self.info("Forcing import of '%s' to not be followed." % module_name) return ( False, "user requested to not follow '%s' import" % module_name, ) # Do not provide an opinion about it. return None def decideCompilation(self, module_name): for handled_module_name, mode in self.handled_modules.items(): if mode != "bytecode": continue if module_name.hasNamespace(handled_module_name): return "bytecode"
class ClosureGiverNodeBase(CodeNodeBase): """ Mixin for nodes that provide variables for closure takers. """ def __init__(self, name, code_prefix, source_ref): CodeNodeBase.__init__( self, name = name, code_prefix = code_prefix, source_ref = source_ref ) self.providing = OrderedDict() self.keeper_variables = OrderedSet() self.temp_variables = OrderedDict() self.temp_scopes = OrderedDict() def hasProvidedVariable(self, variable_name): return variable_name in self.providing def getProvidedVariable(self, variable_name): if variable_name not in self.providing: self.providing[variable_name] = self.createProvidedVariable( variable_name = variable_name ) return self.providing[variable_name] def createProvidedVariable(self, variable_name): # Virtual method, pylint: disable=R0201 assert type(variable_name) is str return None def registerProvidedVariables(self, *variables): for variable in variables: self.registerProvidedVariable(variable) def registerProvidedVariable(self, variable): assert variable is not None self.providing[variable.getName()] = variable def getProvidedVariables(self): return self.providing.values() def allocateTempScope(self, name, allow_closure = False): self.temp_scopes[name] = self.temp_scopes.get(name, 0) + 1 # TODO: Instead of using overly long code name, could just visit parents # and make sure to allocate the scope at the top. if allow_closure: return "%s_%s_%d" % ( self.getCodeName(), name, self.temp_scopes[name] ) else: return "%s_%d" % ( name, self.temp_scopes[name] ) def allocateTempVariable(self, temp_scope, name): if temp_scope is not None: full_name = "%s__%s" % ( temp_scope, name ) else: assert name != "result" full_name = name del name assert full_name not in self.temp_variables, full_name result = Variables.TempVariable( owner = self, variable_name = full_name ) self.temp_variables[full_name] = result addVariableUsage(result, self) return result def getTempVariable(self, temp_scope, name): if temp_scope is not None: full_name = "%s__%s" % (temp_scope, name) else: full_name = name return self.temp_variables[full_name] def getTempVariables(self): return tuple(self.temp_variables.values()) def removeTempVariable(self, variable): del self.temp_variables[variable.getName()]
class LocalsDictHandleBase(object): __slots__ = ( "locals_name", # TODO: Specialize what the kinds really use. "variables", "local_variables", "providing", "mark_for_propagation", "propagation", "owner", ) @counted_init def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name @staticmethod def isModuleScope(): return False @staticmethod def isClassScope(): return False @staticmethod def isFunctionScope(): return False def getProvidedVariables(self): return self.providing.values() def registerProvidedVariable(self, variable): variable_name = variable.getName() self.providing[variable_name] = variable def unregisterProvidedVariable(self, variable): """ Remove provided variable, e.g. because it became unused. """ variable_name = variable.getName() if variable_name in self.providing: del self.providing[variable_name] registerClosureVariable = registerProvidedVariable unregisterClosureVariable = unregisterProvidedVariable def hasProvidedVariable(self, variable_name): """ Test if a variable is provided. """ return variable_name in self.providing def getProvidedVariable(self, variable_name): """ Test if a variable is provided. """ return self.providing[variable_name] def getLocalsRelevantVariables(self): """ The variables relevant to locals. """ return self.providing.values() def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable( owner=self, variable_name=variable_name ) self.variables[variable_name] = result return self.variables[variable_name] # TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here. def getLocalVariable(self, owner, variable_name): if variable_name not in self.local_variables: result = Variables.LocalVariable(owner=owner, variable_name=variable_name) self.local_variables[variable_name] = result return self.local_variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name ) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): # Make it unusable when it's become empty, not used. self.owner.locals_scope = None del self.owner del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() for variable in self.local_variables.values(): variable.finalize() del self.variables del self.providing
class ClosureGiverNodeMixin(CodeNodeMixin): """ Blass class for nodes that provide variables for closure takers. """ def __init__(self, name, code_prefix): CodeNodeMixin.__init__(self, name=name, code_prefix=code_prefix) self.providing = OrderedDict() self.temp_variables = OrderedDict() self.temp_scopes = OrderedDict() self.preserver_id = 0 def hasProvidedVariable(self, variable_name): return variable_name in self.providing def getProvidedVariable(self, variable_name): if variable_name not in self.providing: self.providing[variable_name] = self.createProvidedVariable( variable_name=variable_name) return self.providing[variable_name] def createProvidedVariable(self, variable_name): # Virtual method, pylint: disable=no-self-use assert type(variable_name) is str return None def registerProvidedVariable(self, variable): assert variable is not None self.providing[variable.getName()] = variable def getProvidedVariables(self): return self.providing.values() def allocateTempScope(self, name, allow_closure=False): self.temp_scopes[name] = self.temp_scopes.get(name, 0) + 1 # TODO: Instead of using overly long code name, could just visit parents # and make sure to allocate the scope at the top. if allow_closure: return "%s_%s_%d" % (self.getCodeName(), name, self.temp_scopes[name]) else: return "%s_%d" % (name, self.temp_scopes[name]) def allocateTempVariable(self, temp_scope, name): if temp_scope is not None: full_name = "%s__%s" % (temp_scope, name) else: assert name != "result" full_name = name # No duplicates please. assert full_name not in self.temp_variables, full_name result = self.createTempVariable(temp_name=full_name) return result def createTempVariable(self, temp_name): if temp_name in self.temp_variables: return self.temp_variables[temp_name] result = Variables.TempVariable(owner=self, variable_name=temp_name) self.temp_variables[temp_name] = result return result def getTempVariable(self, temp_scope, name): if temp_scope is not None: full_name = "%s__%s" % (temp_scope, name) else: full_name = name return self.temp_variables[full_name] def getTempVariables(self): return tuple(self.temp_variables.values()) def removeTempVariable(self, variable): del self.temp_variables[variable.getName()] def allocatePreserverId(self): if python_version >= 300: self.preserver_id += 1 return self.preserver_id