class LocalsDictHandleBase(object):
# TODO: Might remove some of these later, pylint: disable=too-many-instance-attributes
__slots__ = (
"locals_name",
# TODO: Specialize what the kinds really use.
"variables",
"local_variables",
"providing",
"mark_for_propagation",
"propagation",
"owner",
"complete",
)
@counted_init
def __init__(self, locals_name, owner):
self.locals_name = locals_name
self.owner = owner
# For locals dict variables in this scope.
self.variables = {}
# For local variables in this scope.
self.local_variables = {}
self.providing = OrderedDict()
# Can this be eliminated through replacement of temporary variables
self.mark_for_propagation = False
self.propagation = None
self.complete = False
if isCountingInstances():
__del__ = counted_del()
def __repr__(self):
return "<%s of %s>" % (self.__class__.__name__, self.locals_name)
def getName(self):
return self.locals_name
def makeClone(self, new_owner):
count = 1
# Make it unique.
while 1:
locals_name = self.locals_name + "_inline_%d" % count
if locals_name not in locals_dict_handles:
break
count += 1
result = self.__class__(locals_name=locals_name, owner=new_owner)
variable_translation = {}
# Clone variables as well.
for variable_name, variable in self.variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.variables[variable_name] = new_variable
for variable_name, variable in self.local_variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.local_variables[variable_name] = new_variable
result.providing = OrderedDict()
for variable_name, variable in self.providing.items():
if variable in variable_translation:
new_variable = variable_translation[variable]
else:
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.providing[variable_name] = new_variable
return result, variable_translation
@staticmethod
def getTypeShape():
return tshape_dict
def getCodeName(self):
return self.locals_name
@staticmethod
def isModuleScope():
return False
@staticmethod
def isClassScope():
return False
@staticmethod
def isFunctionScope():
return False
def getProvidedVariables(self):
return self.providing.values()
def registerProvidedVariable(self, variable):
variable_name = variable.getName()
self.providing[variable_name] = variable
def unregisterProvidedVariable(self, variable):
"""Remove provided variable, e.g. because it became unused."""
variable_name = variable.getName()
if variable_name in self.providing:
del self.providing[variable_name]
registerClosureVariable = registerProvidedVariable
unregisterClosureVariable = unregisterProvidedVariable
def hasProvidedVariable(self, variable_name):
"""Test if a variable is provided."""
return variable_name in self.providing
def getProvidedVariable(self, variable_name):
"""Test if a variable is provided."""
return self.providing[variable_name]
def getLocalsRelevantVariables(self):
"""The variables relevant to locals."""
return self.providing.values()
def getLocalsDictVariable(self, variable_name):
if variable_name not in self.variables:
result = Variables.LocalsDictVariable(owner=self,
variable_name=variable_name)
self.variables[variable_name] = result
return self.variables[variable_name]
# TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here.
def getLocalVariable(self, owner, variable_name):
if variable_name not in self.local_variables:
result = Variables.LocalVariable(owner=owner,
variable_name=variable_name)
self.local_variables[variable_name] = result
return self.local_variables[variable_name]
def markForLocalsDictPropagation(self):
self.mark_for_propagation = True
def isMarkedForPropagation(self):
return self.mark_for_propagation
def allocateTempReplacementVariable(self, trace_collection, variable_name):
if self.propagation is None:
self.propagation = OrderedDict()
if variable_name not in self.propagation:
provider = trace_collection.getOwner()
self.propagation[variable_name] = provider.allocateTempVariable(
temp_scope=None,
name=self.getCodeName() + "_key_" + variable_name)
return self.propagation[variable_name]
def getPropagationVariables(self):
if self.propagation is None:
return ()
return self.propagation
def finalize(self):
# Make it unusable when it's become empty, not used.
self.owner.locals_scope = None
del self.owner
del self.propagation
del self.mark_for_propagation
for variable in self.variables.values():
variable.finalize()
for variable in self.local_variables.values():
variable.finalize()
del self.variables
del self.providing
def markAsComplete(self, trace_collection):
self.complete = True
self._considerUnusedUserLocalVariables(trace_collection)
self._considerPropagation(trace_collection)
# TODO: Limited to Python2 classes for now, more overloads need to be added, this
# ought to be abstract and have variants with TODOs for each of them.
@staticmethod
def _considerPropagation(trace_collection):
"""For overload by scope type. Check if this can be replaced."""
def _considerUnusedUserLocalVariables(self, trace_collection):
"""Check scope for unused variables."""
provided = self.getProvidedVariables()
removals = []
for variable in provided:
if (variable.isLocalVariable()
and not variable.isParameterVariable()
and variable.getOwner() is self.owner):
empty = trace_collection.hasEmptyTraces(variable)
if empty:
removals.append(variable)
for variable in removals:
self.unregisterProvidedVariable(variable)
trace_collection.signalChange(
"var_usage",
self.owner.getSourceReference(),
message="Remove unused local variable '%s'." %
variable.getName(),
)
def __init__(self, hinted_json_file):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
# Load json file contents from --hinted-json-file= argument
filename = hinted_json_file
try:
# read it and extract the two lists
import_info = json.loads(getFileContents(filename))
except (ValueError, FileNotFoundError):
raise FileNotFoundError('Cannot load json file %s' % filename)
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
# TODO: disabled because self.getPluginOptionBool does not exist anymore
#self.accept_test = self.getPluginOptionBool("test", False)
self.accept_test = False
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = scipy = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
scipy = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
self.info(msg)
to_enable = OrderedDict()
if np:
to_enable["numpy"] = {
"matplotlib": mpl,
"scipy": scipy,
# TODO: Numpy plugin didn't use this, work in progress or not needed?
# "sklearn" : sklearn
}
if tk:
to_enable["tk-inter"] = {}
if qt:
# TODO more scrutiny for the qt options!
to_enable["qt-plugins"] = {}
if mp:
to_enable["multiprocessing"] = {}
if pmw:
to_enable["pmw-freezer"] = {}
if torch:
to_enable["torch"] = {}
if tflow:
to_enable["tensorflow"] = {}
if gevent:
to_enable["gevent"] = {}
if eventlet:
to_enable["eventlet"] = {}
if dill:
to_enable["dill-compat"] = {}
# if trio:
# to_enable["trio"] = {}
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if not show_msg and recurse_count > 0:
self.info(msg)
for plugin_name, option_values in to_enable.items():
self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name)
# No the values could be set.
lateActivatePlugin(plugin_name, option_values)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
self.info(msg)
self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
class LocalsDictHandleBase(object):
__slots__ = (
"locals_name",
# TODO: Specialize what the kinds really use.
"variables",
"local_variables",
"providing",
"mark_for_propagation",
"propagation",
"owner",
)
@counted_init
def __init__(self, locals_name, owner):
self.locals_name = locals_name
self.owner = owner
# For locals dict variables in this scope.
self.variables = {}
# For local variables in this scope.
self.local_variables = {}
self.providing = OrderedDict()
# Can this be eliminated through replacement of temporary variables
self.mark_for_propagation = False
self.propagation = None
__del__ = counted_del()
def __repr__(self):
return "<%s of %s>" % (self.__class__.__name__, self.locals_name)
def getName(self):
return self.locals_name
def makeClone(self, new_owner):
count = 1
# Make it unique.
while 1:
locals_name = self.locals_name + "_inline_%d" % count
if locals_name not in locals_dict_handles:
break
count += 1
result = self.__class__(locals_name=locals_name, owner=new_owner)
variable_translation = {}
# Clone variables as well.
for variable_name, variable in self.variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.variables[variable_name] = new_variable
for variable_name, variable in self.local_variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.local_variables[variable_name] = new_variable
result.providing = OrderedDict()
for variable_name, variable in self.providing.items():
if variable in variable_translation:
new_variable = variable_translation[variable]
else:
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.providing[variable_name] = new_variable
return result, variable_translation
@staticmethod
def getTypeShape():
return tshape_dict
def getCodeName(self):
return self.locals_name
@staticmethod
def isModuleScope():
return False
@staticmethod
def isClassScope():
return False
@staticmethod
def isFunctionScope():
return False
def getProvidedVariables(self):
return self.providing.values()
def registerProvidedVariable(self, variable):
variable_name = variable.getName()
self.providing[variable_name] = variable
def unregisterProvidedVariable(self, variable):
""" Remove provided variable, e.g. because it became unused. """
variable_name = variable.getName()
if variable_name in self.providing:
del self.providing[variable_name]
registerClosureVariable = registerProvidedVariable
unregisterClosureVariable = unregisterProvidedVariable
def hasProvidedVariable(self, variable_name):
""" Test if a variable is provided. """
return variable_name in self.providing
def getProvidedVariable(self, variable_name):
""" Test if a variable is provided. """
return self.providing[variable_name]
def getLocalsRelevantVariables(self):
""" The variables relevant to locals. """
return self.providing.values()
def getLocalsDictVariable(self, variable_name):
if variable_name not in self.variables:
result = Variables.LocalsDictVariable(
owner=self, variable_name=variable_name
)
self.variables[variable_name] = result
return self.variables[variable_name]
# TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here.
def getLocalVariable(self, owner, variable_name):
if variable_name not in self.local_variables:
result = Variables.LocalVariable(owner=owner, variable_name=variable_name)
self.local_variables[variable_name] = result
return self.local_variables[variable_name]
def markForLocalsDictPropagation(self):
self.mark_for_propagation = True
def isMarkedForPropagation(self):
return self.mark_for_propagation
def allocateTempReplacementVariable(self, trace_collection, variable_name):
if self.propagation is None:
self.propagation = OrderedDict()
if variable_name not in self.propagation:
provider = trace_collection.getOwner()
self.propagation[variable_name] = provider.allocateTempVariable(
temp_scope=None, name=self.getCodeName() + "_key_" + variable_name
)
return self.propagation[variable_name]
def getPropagationVariables(self):
if self.propagation is None:
return ()
return self.propagation
def finalize(self):
# Make it unusable when it's become empty, not used.
self.owner.locals_scope = None
del self.owner
del self.propagation
del self.mark_for_propagation
for variable in self.variables.values():
variable.finalize()
for variable in self.local_variables.values():
variable.finalize()
del self.variables
del self.providing
class NuitkaPluginAntiBloat(NuitkaPluginBase):
plugin_name = "anti-bloat"
plugin_desc = (
"Patch stupid imports out of widely used library modules source codes."
)
@staticmethod
def isAlwaysEnabled():
return True
def __init__(
self,
noinclude_setuptools_mode,
noinclude_pytest_mode,
noinclude_ipython_mode,
noinclude_default_mode,
custom_choices,
):
# Default manually to default argument value:
if noinclude_setuptools_mode is None:
noinclude_setuptools_mode = noinclude_default_mode
if noinclude_pytest_mode is None:
noinclude_pytest_mode = noinclude_default_mode
if noinclude_ipython_mode is None:
noinclude_ipython_mode = noinclude_default_mode
self.config = parsePackageYaml(__package__, "anti-bloat.yml")
self.handled_modules = OrderedDict()
# These should be checked, to allow disabling anti-bloat contents.
self.control_tags = set()
if noinclude_setuptools_mode != "allow":
self.handled_modules["setuptools"] = noinclude_setuptools_mode
else:
self.control_tags.add("allow_setuptools")
if noinclude_pytest_mode != "allow":
self.handled_modules["pytest"] = noinclude_pytest_mode
else:
self.control_tags.add("allow_pytest")
if noinclude_ipython_mode != "allow":
self.handled_modules["IPython"] = noinclude_ipython_mode
else:
self.control_tags.add("allow_ipython")
for custom_choice in custom_choices:
if ":" not in custom_choice:
self.sysexit(
"Error, malformed value '%s' for '--noinclude-custom-mode' used."
% custom_choice)
module_name, mode = custom_choice.rsplit(":", 1)
if mode not in ("error", "warning", "nofollow", "allow",
"bytecode"):
self.sysexit(
"Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'"
% (mode, custom_choice))
self.handled_modules[ModuleName(module_name)] = mode
@classmethod
def addPluginCommandLineOptions(cls, group):
group.add_option(
"--noinclude-setuptools-mode",
action="store",
dest="noinclude_setuptools_mode",
choices=("error", "warning", "nofollow", "allow"),
default=None,
help="""\
What to do if a setuptools import is encountered. This package can be big with
dependencies, and should definitely be avoided.""",
)
group.add_option(
"--noinclude-pytest-mode",
action="store",
dest="noinclude_pytest_mode",
choices=("error", "warning", "nofollow", "allow"),
default=None,
help="""\
What to do if a pytest import is encountered. This package can be big with
dependencies, and should definitely be avoided.""",
)
group.add_option(
"--noinclude-IPython-mode",
action="store",
dest="noinclude_ipython_mode",
choices=("error", "warning", "nofollow", "allow"),
default=None,
help="""\
What to do if a IPython import is encountered. This package can be big with
dependencies, and should definitely be avoided.""",
)
group.add_option(
"--noinclude-default-mode",
action="store",
dest="noinclude_default_mode",
choices=("error", "warning", "nofollow", "allow"),
default="warning",
help="""\
This actually provides the default "warning" value for above options, and
can be used to turn all of these on.""",
)
group.add_option(
"--noinclude-custom-mode",
action="append",
dest="custom_choices",
default=[],
help="""\
What to do if a specific import is encountered. Format is module name,
which can and should be a top level package and then one choice, "error",
"warning", "nofollow", e.g. PyQt5:error.""",
)
def onModuleSourceCode(self, module_name, source_code):
# Complex dealing with many cases, pylint: disable=too-many-branches,too-many-locals,too-many-statements
config = self.config.get(module_name)
if not config:
return source_code
# Allow disabling config for a module with matching control tags.
for control_tag in config.get("control_tags", ()):
if control_tag in self.control_tags:
return source_code
description = config.get("description", "description not given")
# To allow detection if it did anything.
change_count = 0
context = {}
context_code = config.get("context", "")
if type(context_code) in (tuple, list):
context_code = "\n".join(context_code)
# We trust the yaml files, pylint: disable=eval-used,exec-used
context_ready = not bool(context_code)
for replace_src, replace_code in config.get("replacements",
{}).items():
# Avoid the eval, if the replace doesn't hit.
if replace_src not in source_code:
continue
if replace_code:
if not context_ready:
try:
exec(context_code, context)
except Exception as e: # pylint: disable=broad-except
self.sysexit(
"Error, cannot execute context code '%s' due to: %s"
% (context_code, e))
context_ready = True
try:
replace_dst = eval(replace_code, context)
except Exception as e: # pylint: disable=broad-except
self.sysexit(
"Error, cannot evaluate code '%s' in '%s' due to: %s" %
(replace_code, context_code, e))
else:
replace_dst = ""
if type(replace_dst) is not str:
self.sysexit(
"Error, expression needs to generate string, not %s" %
type(replace_dst))
old = source_code
source_code = source_code.replace(replace_src, replace_dst)
if old != source_code:
change_count += 1
append_code = config.get("append_result", "")
if type(append_code) in (tuple, list):
append_code = "\n".join(append_code)
if append_code:
if not context_ready:
exec(context_code, context)
context_ready = True
try:
append_result = eval(append_code, context)
except Exception as e: # pylint: disable=broad-except
self.sysexit(
"Error, cannot evaluate code '%s' in '%s' due to: %s" %
(append_code, context_code, e))
source_code += "\n" + append_result
change_count += 1
if change_count > 0:
self.info("Handling module '%s' with %d change(s) for: %s." %
(module_name.asString(), change_count, description))
module_code = config.get("module_code", None)
if module_code is not None:
assert not change_count
self.info("Handling module '%s' with full replacement : %s." %
(module_name.asString(), description))
source_code = module_code
return source_code
def onFunctionBodyParsing(self, module_name, function_name, body):
config = self.config.get(module_name)
if not config:
return
context = {}
context_code = config.get("context", "")
if type(context_code) in (tuple, list):
context_code = "\n".join(context_code)
# We trust the yaml files, pylint: disable=eval-used,exec-used
context_ready = not bool(context_code)
for change_function_name, replace_code in config.get(
"change_function", {}).items():
if function_name != change_function_name:
continue
if not context_ready:
exec(context_code, context)
context_ready = True
try:
replacement = eval(replace_code, context)
except Exception as e: # pylint: disable=broad-except
self.sysexit(
"Error, cannot evaluate code '%s' in '%s' due to: %s" %
(replace_code, context_code, e))
# Single node is required, extrace the generated module body with
# single expression only statement value or a function body.
replacement = ast.parse(replacement).body[0]
if type(replacement) is ast.Expr:
body[:] = [ast.Return(replacement.value)]
else:
body[:] = replacement.body
self.info("Updated module '%s' function '%s'." %
(module_name.asString(), function_name))
def onModuleEncounter(self, module_filename, module_name, module_kind):
for handled_module_name, mode in self.handled_modules.items():
if module_name.hasNamespace(handled_module_name):
# Make sure the compilation abrts.
if mode == "error":
raise NuitkaForbiddenImportEncounter(module_name)
# Either issue a warning, or pretend the module doesn't exist for standalone or
# at least will not be included.
if mode == "warning":
self.warning("Unwanted import of '%s' encountered." %
module_name)
elif mode == "nofollow":
self.info("Forcing import of '%s' to not be followed." %
module_name)
return (
False,
"user requested to not follow '%s' import" %
module_name,
)
# Do not provide an opinion about it.
return None
def decideCompilation(self, module_name):
for handled_module_name, mode in self.handled_modules.items():
if mode != "bytecode":
continue
if module_name.hasNamespace(handled_module_name):
return "bytecode"
class NuitkaPluginAntiBloat(NuitkaPluginBase):
plugin_name = "anti-bloat"
plugin_desc = "Patch stupid imports out of common library modules source code."
def __init__(self, setuptools_mode, custom_choices):
self.handled_modules = OrderedDict()
if setuptools_mode != "allow":
self.handled_modules["setuptools"] = setuptools_mode
for custom_choice in custom_choices:
if ":" not in custom_choice:
self.sysexit(
"Error, malformed value '%s' for '--noinclude-custom-mode' used."
% custom_choice
)
module_name, mode = custom_choice.rsplit(":", 1)
if mode not in ("error", "warning", "nofollow", "allow"):
self.sysexit(
"Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'"
% (mode, custom_choice)
)
self.handled_modules[ModuleName(module_name)] = mode
@classmethod
def addPluginCommandLineOptions(cls, group):
group.add_option(
"--noinclude-setuptools-mode",
action="store",
dest="setuptools_mode",
choices=("error", "warning", "nofollow", "allow"),
default="allow",
help="""\
What to do if a setuptools import is encountered. This can be big with
dependencies, and should definitely be avoided.""",
)
group.add_option(
"--noinclude-custom-mode",
action="append",
dest="custom_choices",
default=[],
help="""\
What to do if a specific import is encountered. Format is module name,
which can and should be a top level package and then one choice, "error",
"warning", "nofollow", e.g. PyQt5:error.""",
)
def onModuleEncounter(self, module_filename, module_name, module_kind):
for handled_module_name, mode in self.handled_modules.items():
if module_name.hasNamespace(handled_module_name):
# Make sure the compilation abrts.
if mode == "error":
raise NuitkaForbiddenImportEncounter(module_name)
# Either issue a warning, or pretend the module doesn't exist for standalone or
# at least will not be included.
if mode == "warning":
self.warning("Forbidden import of '%s' encountered." % module_name)
elif mode == "nofollow":
self.info(
"Forcing import of '%s' to not be followed." % module_name
)
return (
False,
"user requested to not follow '%s' import" % module_name,
)
# Do not provide an opinion about it.
return None
