class LocalsDictHandleBase(object): # TODO: Might remove some of these later, pylint: disable=too-many-instance-attributes __slots__ = ( "locals_name", # TODO: Specialize what the kinds really use. "variables", "local_variables", "providing", "mark_for_propagation", "propagation", "owner", "complete", ) @counted_init def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None self.complete = False if isCountingInstances(): __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name @staticmethod def isModuleScope(): return False @staticmethod def isClassScope(): return False @staticmethod def isFunctionScope(): return False def getProvidedVariables(self): return self.providing.values() def registerProvidedVariable(self, variable): variable_name = variable.getName() self.providing[variable_name] = variable def unregisterProvidedVariable(self, variable): """Remove provided variable, e.g. because it became unused.""" variable_name = variable.getName() if variable_name in self.providing: del self.providing[variable_name] registerClosureVariable = registerProvidedVariable unregisterClosureVariable = unregisterProvidedVariable def hasProvidedVariable(self, variable_name): """Test if a variable is provided.""" return variable_name in self.providing def getProvidedVariable(self, variable_name): """Test if a variable is provided.""" return self.providing[variable_name] def getLocalsRelevantVariables(self): """The variables relevant to locals.""" return self.providing.values() def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable(owner=self, variable_name=variable_name) self.variables[variable_name] = result return self.variables[variable_name] # TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here. def getLocalVariable(self, owner, variable_name): if variable_name not in self.local_variables: result = Variables.LocalVariable(owner=owner, variable_name=variable_name) self.local_variables[variable_name] = result return self.local_variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): # Make it unusable when it's become empty, not used. self.owner.locals_scope = None del self.owner del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() for variable in self.local_variables.values(): variable.finalize() del self.variables del self.providing def markAsComplete(self, trace_collection): self.complete = True self._considerUnusedUserLocalVariables(trace_collection) self._considerPropagation(trace_collection) # TODO: Limited to Python2 classes for now, more overloads need to be added, this # ought to be abstract and have variants with TODOs for each of them. @staticmethod def _considerPropagation(trace_collection): """For overload by scope type. Check if this can be replaced.""" def _considerUnusedUserLocalVariables(self, trace_collection): """Check scope for unused variables.""" provided = self.getProvidedVariables() removals = [] for variable in provided: if (variable.isLocalVariable() and not variable.isParameterVariable() and variable.getOwner() is self.owner): empty = trace_collection.hasEmptyTraces(variable) if empty: removals.append(variable) for variable in removals: self.unregisterProvidedVariable(variable) trace_collection.signalChange( "var_usage", self.owner.getSourceReference(), message="Remove unused local variable '%s'." % variable.getName(), )
def __init__(self, hinted_json_file): """ Read the JSON file and enable any standard plugins. Notes: Read the JSON file produced during the get-hints step. It will contain a list of imported items ("calls") and a list of modules / packages ("files") to be loaded and recursed into. Depending on the items in 'files', we will trigger loading standard plugins. """ # start a timer self.timer = StopWatch() self.timer.start() self.implicit_imports = OrderedSet() # speed up repeated lookups self.ignored_modules = OrderedSet() # speed up repeated lookups options = Options.options # Load json file contents from --hinted-json-file= argument filename = hinted_json_file try: # read it and extract the two lists import_info = json.loads(getFileContents(filename)) except (ValueError, FileNotFoundError): raise FileNotFoundError('Cannot load json file %s' % filename) self.import_calls = import_info["calls"] self.import_files = import_info["files"] self.msg_count = dict() # to limit keep messages self.msg_limit = 21 # suppress pytest / _pytest / unittest? # TODO: disabled because self.getPluginOptionBool does not exist anymore #self.accept_test = self.getPluginOptionBool("test", False) self.accept_test = False """ Check if we should enable any (optional) standard plugins. This code must be modified whenever more standard plugin become available. """ show_msg = False # only show info if one ore more detected # indicators for found packages tk = np = qt = scipy = mp = pmw = torch = sklearn = False eventlet = tflow = gevent = mpl = trio = dill = False msg = "'%s' is adding the following options:" % os.path.basename( self.plugin_name) # we need matplotlib-specific cleanup to happen first: # if no mpl backend is used, reference to matplotlib is removed alltogether if "matplotlib.backends" not in self.import_files: temp = [ f for f in self.import_calls if not f.startswith(("matplotlib", "mpl_toolkits")) ] self.import_calls = temp temp = [ f for f in self.import_files if not f.startswith(("matplotlib", "mpl_toolkits")) ] self.import_files = temp # detect required standard plugins and request enabling them for m in self.import_calls: # scan thru called items if m in ("numpy", "numpy.*"): np = True show_msg = True if m in ("matplotlib", "matplotlib.*"): mpl = True show_msg = True elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"): tk = True show_msg = True elif m.startswith(("PyQt", "PySide")): qt = True show_msg = True elif m in ("scipy", "scipy.*"): scipy = True show_msg = True elif m in ("multiprocessing", "multiprocessing.*") and getOS() == "Windows": mp = True show_msg = True elif m in ("Pmw", "Pmw.*"): pmw = True show_msg = True elif m == "torch": torch = True show_msg = True elif m in ("sklearn", "sklearn.*"): sklearn = True show_msg = True elif m in ("tensorflow", "tensorflow.*"): tflow = True show_msg = True elif m in ("gevent", "gevent.*"): gevent = True show_msg = True elif m in ("eventlet", "eventlet.*"): eventlet = True show_msg = True elif m in ("dill", "dill.*"): dill = True show_msg = True # elif m in ("trio", "trio.*"): # trio = True # show_msg = True if show_msg is True: self.info(msg) to_enable = OrderedDict() if np: to_enable["numpy"] = { "matplotlib": mpl, "scipy": scipy, # TODO: Numpy plugin didn't use this, work in progress or not needed? # "sklearn" : sklearn } if tk: to_enable["tk-inter"] = {} if qt: # TODO more scrutiny for the qt options! to_enable["qt-plugins"] = {} if mp: to_enable["multiprocessing"] = {} if pmw: to_enable["pmw-freezer"] = {} if torch: to_enable["torch"] = {} if tflow: to_enable["tensorflow"] = {} if gevent: to_enable["gevent"] = {} if eventlet: to_enable["eventlet"] = {} if dill: to_enable["dill-compat"] = {} # if trio: # to_enable["trio"] = {} recurse_count = 0 for f in self.import_files: # request recursion to called modules if self.accept_test is False and f.startswith( ("pytest", "_pytest", "unittest")): continue options.recurse_modules.append(f) recurse_count += 1 # no plugin detected, but recursing to modules? if not show_msg and recurse_count > 0: self.info(msg) for plugin_name, option_values in to_enable.items(): self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name) # No the values could be set. lateActivatePlugin(plugin_name, option_values) if len(self.import_files) > 0: msg = "--recurse-to=%s and %i more modules" % ( self.import_files[-1], recurse_count - 1, ) self.info(msg) self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
class LocalsDictHandleBase(object): __slots__ = ( "locals_name", # TODO: Specialize what the kinds really use. "variables", "local_variables", "providing", "mark_for_propagation", "propagation", "owner", ) @counted_init def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name @staticmethod def isModuleScope(): return False @staticmethod def isClassScope(): return False @staticmethod def isFunctionScope(): return False def getProvidedVariables(self): return self.providing.values() def registerProvidedVariable(self, variable): variable_name = variable.getName() self.providing[variable_name] = variable def unregisterProvidedVariable(self, variable): """ Remove provided variable, e.g. because it became unused. """ variable_name = variable.getName() if variable_name in self.providing: del self.providing[variable_name] registerClosureVariable = registerProvidedVariable unregisterClosureVariable = unregisterProvidedVariable def hasProvidedVariable(self, variable_name): """ Test if a variable is provided. """ return variable_name in self.providing def getProvidedVariable(self, variable_name): """ Test if a variable is provided. """ return self.providing[variable_name] def getLocalsRelevantVariables(self): """ The variables relevant to locals. """ return self.providing.values() def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable( owner=self, variable_name=variable_name ) self.variables[variable_name] = result return self.variables[variable_name] # TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here. def getLocalVariable(self, owner, variable_name): if variable_name not in self.local_variables: result = Variables.LocalVariable(owner=owner, variable_name=variable_name) self.local_variables[variable_name] = result return self.local_variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name ) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): # Make it unusable when it's become empty, not used. self.owner.locals_scope = None del self.owner del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() for variable in self.local_variables.values(): variable.finalize() del self.variables del self.providing
class NuitkaPluginAntiBloat(NuitkaPluginBase): plugin_name = "anti-bloat" plugin_desc = ( "Patch stupid imports out of widely used library modules source codes." ) @staticmethod def isAlwaysEnabled(): return True def __init__( self, noinclude_setuptools_mode, noinclude_pytest_mode, noinclude_ipython_mode, noinclude_default_mode, custom_choices, ): # Default manually to default argument value: if noinclude_setuptools_mode is None: noinclude_setuptools_mode = noinclude_default_mode if noinclude_pytest_mode is None: noinclude_pytest_mode = noinclude_default_mode if noinclude_ipython_mode is None: noinclude_ipython_mode = noinclude_default_mode self.config = parsePackageYaml(__package__, "anti-bloat.yml") self.handled_modules = OrderedDict() # These should be checked, to allow disabling anti-bloat contents. self.control_tags = set() if noinclude_setuptools_mode != "allow": self.handled_modules["setuptools"] = noinclude_setuptools_mode else: self.control_tags.add("allow_setuptools") if noinclude_pytest_mode != "allow": self.handled_modules["pytest"] = noinclude_pytest_mode else: self.control_tags.add("allow_pytest") if noinclude_ipython_mode != "allow": self.handled_modules["IPython"] = noinclude_ipython_mode else: self.control_tags.add("allow_ipython") for custom_choice in custom_choices: if ":" not in custom_choice: self.sysexit( "Error, malformed value '%s' for '--noinclude-custom-mode' used." % custom_choice) module_name, mode = custom_choice.rsplit(":", 1) if mode not in ("error", "warning", "nofollow", "allow", "bytecode"): self.sysexit( "Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'" % (mode, custom_choice)) self.handled_modules[ModuleName(module_name)] = mode @classmethod def addPluginCommandLineOptions(cls, group): group.add_option( "--noinclude-setuptools-mode", action="store", dest="noinclude_setuptools_mode", choices=("error", "warning", "nofollow", "allow"), default=None, help="""\ What to do if a setuptools import is encountered. This package can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-pytest-mode", action="store", dest="noinclude_pytest_mode", choices=("error", "warning", "nofollow", "allow"), default=None, help="""\ What to do if a pytest import is encountered. This package can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-IPython-mode", action="store", dest="noinclude_ipython_mode", choices=("error", "warning", "nofollow", "allow"), default=None, help="""\ What to do if a IPython import is encountered. This package can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-default-mode", action="store", dest="noinclude_default_mode", choices=("error", "warning", "nofollow", "allow"), default="warning", help="""\ This actually provides the default "warning" value for above options, and can be used to turn all of these on.""", ) group.add_option( "--noinclude-custom-mode", action="append", dest="custom_choices", default=[], help="""\ What to do if a specific import is encountered. Format is module name, which can and should be a top level package and then one choice, "error", "warning", "nofollow", e.g. PyQt5:error.""", ) def onModuleSourceCode(self, module_name, source_code): # Complex dealing with many cases, pylint: disable=too-many-branches,too-many-locals,too-many-statements config = self.config.get(module_name) if not config: return source_code # Allow disabling config for a module with matching control tags. for control_tag in config.get("control_tags", ()): if control_tag in self.control_tags: return source_code description = config.get("description", "description not given") # To allow detection if it did anything. change_count = 0 context = {} context_code = config.get("context", "") if type(context_code) in (tuple, list): context_code = "\n".join(context_code) # We trust the yaml files, pylint: disable=eval-used,exec-used context_ready = not bool(context_code) for replace_src, replace_code in config.get("replacements", {}).items(): # Avoid the eval, if the replace doesn't hit. if replace_src not in source_code: continue if replace_code: if not context_ready: try: exec(context_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot execute context code '%s' due to: %s" % (context_code, e)) context_ready = True try: replace_dst = eval(replace_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot evaluate code '%s' in '%s' due to: %s" % (replace_code, context_code, e)) else: replace_dst = "" if type(replace_dst) is not str: self.sysexit( "Error, expression needs to generate string, not %s" % type(replace_dst)) old = source_code source_code = source_code.replace(replace_src, replace_dst) if old != source_code: change_count += 1 append_code = config.get("append_result", "") if type(append_code) in (tuple, list): append_code = "\n".join(append_code) if append_code: if not context_ready: exec(context_code, context) context_ready = True try: append_result = eval(append_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot evaluate code '%s' in '%s' due to: %s" % (append_code, context_code, e)) source_code += "\n" + append_result change_count += 1 if change_count > 0: self.info("Handling module '%s' with %d change(s) for: %s." % (module_name.asString(), change_count, description)) module_code = config.get("module_code", None) if module_code is not None: assert not change_count self.info("Handling module '%s' with full replacement : %s." % (module_name.asString(), description)) source_code = module_code return source_code def onFunctionBodyParsing(self, module_name, function_name, body): config = self.config.get(module_name) if not config: return context = {} context_code = config.get("context", "") if type(context_code) in (tuple, list): context_code = "\n".join(context_code) # We trust the yaml files, pylint: disable=eval-used,exec-used context_ready = not bool(context_code) for change_function_name, replace_code in config.get( "change_function", {}).items(): if function_name != change_function_name: continue if not context_ready: exec(context_code, context) context_ready = True try: replacement = eval(replace_code, context) except Exception as e: # pylint: disable=broad-except self.sysexit( "Error, cannot evaluate code '%s' in '%s' due to: %s" % (replace_code, context_code, e)) # Single node is required, extrace the generated module body with # single expression only statement value or a function body. replacement = ast.parse(replacement).body[0] if type(replacement) is ast.Expr: body[:] = [ast.Return(replacement.value)] else: body[:] = replacement.body self.info("Updated module '%s' function '%s'." % (module_name.asString(), function_name)) def onModuleEncounter(self, module_filename, module_name, module_kind): for handled_module_name, mode in self.handled_modules.items(): if module_name.hasNamespace(handled_module_name): # Make sure the compilation abrts. if mode == "error": raise NuitkaForbiddenImportEncounter(module_name) # Either issue a warning, or pretend the module doesn't exist for standalone or # at least will not be included. if mode == "warning": self.warning("Unwanted import of '%s' encountered." % module_name) elif mode == "nofollow": self.info("Forcing import of '%s' to not be followed." % module_name) return ( False, "user requested to not follow '%s' import" % module_name, ) # Do not provide an opinion about it. return None def decideCompilation(self, module_name): for handled_module_name, mode in self.handled_modules.items(): if mode != "bytecode": continue if module_name.hasNamespace(handled_module_name): return "bytecode"
class NuitkaPluginAntiBloat(NuitkaPluginBase): plugin_name = "anti-bloat" plugin_desc = "Patch stupid imports out of common library modules source code." def __init__(self, setuptools_mode, custom_choices): self.handled_modules = OrderedDict() if setuptools_mode != "allow": self.handled_modules["setuptools"] = setuptools_mode for custom_choice in custom_choices: if ":" not in custom_choice: self.sysexit( "Error, malformed value '%s' for '--noinclude-custom-mode' used." % custom_choice ) module_name, mode = custom_choice.rsplit(":", 1) if mode not in ("error", "warning", "nofollow", "allow"): self.sysexit( "Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'" % (mode, custom_choice) ) self.handled_modules[ModuleName(module_name)] = mode @classmethod def addPluginCommandLineOptions(cls, group): group.add_option( "--noinclude-setuptools-mode", action="store", dest="setuptools_mode", choices=("error", "warning", "nofollow", "allow"), default="allow", help="""\ What to do if a setuptools import is encountered. This can be big with dependencies, and should definitely be avoided.""", ) group.add_option( "--noinclude-custom-mode", action="append", dest="custom_choices", default=[], help="""\ What to do if a specific import is encountered. Format is module name, which can and should be a top level package and then one choice, "error", "warning", "nofollow", e.g. PyQt5:error.""", ) def onModuleEncounter(self, module_filename, module_name, module_kind): for handled_module_name, mode in self.handled_modules.items(): if module_name.hasNamespace(handled_module_name): # Make sure the compilation abrts. if mode == "error": raise NuitkaForbiddenImportEncounter(module_name) # Either issue a warning, or pretend the module doesn't exist for standalone or # at least will not be included. if mode == "warning": self.warning("Forbidden import of '%s' encountered." % module_name) elif mode == "nofollow": self.info( "Forcing import of '%s' to not be followed." % module_name ) return ( False, "user requested to not follow '%s' import" % module_name, ) # Do not provide an opinion about it. return None