class CompiledPythonModule(
ChildrenHavingMixin,
ClosureGiverNodeMixin,
MarkNeedsAnnotationsMixin,
EntryPointMixin,
PythonModuleBase,
):
""" Compiled Python Module
"""
# This one has a few indicators, pylint: disable=too-many-instance-attributes
kind = "COMPILED_PYTHON_MODULE"
named_children = ("body", "functions")
checkers = {"body": checkStatementsSequenceOrNone}
def __init__(self, name, package_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(
self, name=name, package_name=package_name, source_ref=source_ref
)
ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module")
ChildrenHavingMixin.__init__(
self, values={"body": None, "functions": ()} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
self.module_dict_name = "globals_%s" % (self.getCodeName(),)
setLocalsDictType(self.module_dict_name, "module_dict")
def getDetails(self):
return {
"filename": self.source_ref.getFilename(),
"package": self.package_name,
"name": self.name,
}
def getDetailsForDisplay(self):
result = self.getDetails()
if self.future_spec is not None:
result["code_flags"] = ",".join(self.future_spec.asFlags())
return result
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider, must not be used,
assert False
def getFutureSpec(self):
return self.future_spec
def isTopModule(self):
return self.is_top
def asGraph(self, graph, desc):
graph = graph.add_subgraph(
name="cluster_%s" % desc, comment="Graph for %s" % self.getName()
)
# graph.body.append("style=filled")
# graph.body.append("color=lightgrey")
# graph.body.append("label=Iteration_%d" % desc)
def makeTraceNodeName(variable, version, variable_trace):
return "%s/ %s %s %s" % (
desc,
variable.getName(),
version,
variable_trace.__class__.__name__,
)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
node_names = {}
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = makeTraceNodeName(variable, version, variable_trace)
node_names[variable_trace] = node_name
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = node_names[variable_trace]
previous = variable_trace.getPrevious()
attrs = {"style": "filled"}
if variable_trace.hasDefiniteUsages():
attrs["color"] = "blue"
elif variable_trace.hasPotentialUsages():
attrs["color"] = "yellow"
else:
attrs["color"] = "red"
graph.add_node(node_name, **attrs)
if type(previous) is tuple:
for prev_trace in previous:
graph.add_edge(node_names[prev_trace], node_name)
assert prev_trace is not variable_trace
elif previous is not None:
assert previous is not variable_trace
graph.add_edge(node_names[previous], node_name)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
getFunctions = ChildrenHavingMixin.childGetter("functions")
setFunctions = ChildrenHavingMixin.childSetter("functions")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
# We have never have a parent
return None
def getParentVariableProvider(self):
# We have never have a provider
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(variable_name=variable_name)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(module=self, variable_name=variable_name)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
@staticmethod
def isEarlyClosure():
# Modules should immediately closure variables on use.
return True
def getEntryPoint(self):
return self
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.getFunctions()
assert function_body not in functions
functions += (function_body,)
self.setFunctions(functions)
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.getFunctions()
assert (
function_body.isExpressionFunctionBody()
or function_body.isExpressionClassBody()
or function_body.isExpressionGeneratorObjectBody()
or function_body.isExpressionCoroutineObjectBody()
or function_body.isExpressionAsyncgenObjectBody()
)
result = function_body not in self.active_functions
if result:
self.active_functions.add(function_body)
return result
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.getFunctions():
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.getFunctions():
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
elif main_filename.endswith(".pyw"):
result = main_filename[:-4]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(")", "").replace("(", "")
def computeModule(self):
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection=self.trace_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref=new_module.getSourceReference(),
tags="new_code",
message="Recursed to module package.",
)
self.trace_collection.updateVariablesFromCollection(old_collection)
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def isUnoptimized(self):
# Modules don't do this, pylint: disable=no-self-use
return False
def getLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getUserLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getOutlineLocalVariables(self):
outlines = self.getTraceCollection().getOutlineFunctions()
if outlines is None:
return ()
result = []
for outline in outlines:
result.extend(outline.getUserLocalVariables())
return tuple(result)
def hasClosureVariable(self, variable):
# Modules don't do this, pylint: disable=no-self-use,unused-argument
return False
def removeUserVariable(self, variable):
outlines = self.getTraceCollection().getOutlineFunctions()
for outline in outlines:
user_locals = outline.getUserLocalVariables()
if variable in user_locals:
outline.removeUserVariable(variable)
break
@staticmethod
def getFunctionLocalsScope():
""" Modules have no locals scope. """
return None
def getModuleDictScope(self):
return getLocalsDictHandle(self.module_dict_name)
class CompiledPythonModule(
ChildrenHavingMixin,
ClosureGiverNodeMixin,
MarkNeedsAnnotationsMixin,
EntryPointMixin,
PythonModuleBase,
):
"""Compiled Python Module"""
# This one has a few indicators, pylint: disable=too-many-instance-attributes
kind = "COMPILED_PYTHON_MODULE"
__slots__ = (
"is_top",
"name",
"code_prefix",
"code_name",
"uids",
"temp_variables",
"temp_scopes",
"preserver_id",
"needs_annotations_dict",
"trace_collection",
"mode",
"variables",
"active_functions",
"visited_functions",
"cross_used_functions",
"used_modules",
"future_spec",
"source_code",
"module_dict_name",
"locals_scope",
)
named_children = ("body", "functions")
checkers = {"body": checkStatementsSequenceOrNone}
def __init__(self, module_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(self,
module_name=module_name,
source_ref=source_ref)
ClosureGiverNodeMixin.__init__(self,
name=module_name.getBasename(),
code_prefix="module")
ChildrenHavingMixin.__init__(
self,
values={
"body": None,
"functions": ()
} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
# Functions that have been used.
self.active_functions = OrderedSet()
# Functions that should be visited again.
self.visited_functions = set()
self.cross_used_functions = OrderedSet()
self.used_modules = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
# The source code of the module if changed or not from disk.
self.source_code = None
self.module_dict_name = "globals_%s" % (self.getCodeName(), )
self.locals_scope = getLocalsDictHandle(self.module_dict_name,
"module_dict", self)
@staticmethod
def isCompiledPythonModule():
return True
def getDetails(self):
return {
"filename": self.source_ref.getFilename(),
"module_name": self.module_name,
}
def getDetailsForDisplay(self):
result = self.getDetails()
if self.future_spec is not None:
result["code_flags"] = ",".join(self.future_spec.asFlags())
return result
def getCompilationMode(self):
return self.mode
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider, must not be used,
assert False
def getFutureSpec(self):
return self.future_spec
def setFutureSpec(self, future_spec):
self.future_spec = future_spec
def isTopModule(self):
return self.is_top
def asGraph(self, graph, desc):
graph = graph.add_subgraph(name="cluster_%s" % desc,
comment="Graph for %s" % self.getName())
# graph.body.append("style=filled")
# graph.body.append("color=lightgrey")
# graph.body.append("label=Iteration_%d" % desc)
def makeTraceNodeName(variable, version, variable_trace):
return "%s/ %s %s %s" % (
desc,
variable.getName(),
version,
variable_trace.__class__.__name__,
)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
node_names = {}
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = makeTraceNodeName(variable, version,
variable_trace)
node_names[variable_trace] = node_name
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = node_names[variable_trace]
previous = variable_trace.getPrevious()
attrs = {"style": "filled"}
if variable_trace.getUsageCount():
attrs["color"] = "blue"
else:
attrs["color"] = "red"
graph.add_node(node_name, **attrs)
if type(previous) is tuple:
for prev_trace in previous:
graph.add_edge(node_names[prev_trace], node_name)
assert prev_trace is not variable_trace
elif previous is not None:
assert previous is not variable_trace
graph.add_edge(node_names[previous], node_name)
return graph
def getSourceCode(self):
if self.source_code is not None:
return self.source_code
else:
return readSourceCodeFromFilename(
module_name=self.getFullName(),
source_filename=self.getCompileTimeFilename(),
)
def setSourceCode(self, code):
self.source_code = code
def getParent(self):
# We have never have a parent
return None
def getParentVariableProvider(self):
# We have never have a provider
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getProvidedVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(variable_name=variable_name)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(module=self,
variable_name=variable_name)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
@staticmethod
def isEarlyClosure():
# Modules should immediately closure variables on use.
return True
def getEntryPoint(self):
return self
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.subnode_functions
assert function_body not in functions
functions += (function_body, )
self.setChild("functions", functions)
def startTraversal(self):
self.used_modules = OrderedSet()
self.active_functions = OrderedSet()
def restartTraversal(self):
self.visited_functions = set()
def addUsedModule(self, key):
self.used_modules.add(key)
def getUsedModules(self):
return self.used_modules
def addUsedFunction(self, function_body):
assert function_body in self.subnode_functions, function_body
assert (function_body.isExpressionFunctionBody()
or function_body.isExpressionClassBody()
or function_body.isExpressionGeneratorObjectBody()
or function_body.isExpressionCoroutineObjectBody()
or function_body.isExpressionAsyncgenObjectBody())
self.active_functions.add(function_body)
result = function_body not in self.visited_functions
self.visited_functions.add(function_body)
return result
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.subnode_functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.subnode_functions:
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
elif main_filename.endswith(".pyw"):
result = main_filename[:-4]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(")", "").replace("(", "")
def computeModule(self):
self.restartTraversal()
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.subnode_body
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection=self.trace_collection)
if result is not module_body:
self.setChild("body", result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref=new_module.getSourceReference(),
tags="new_code",
message="Recursed to module package.",
)
# Finalize locals scopes previously determined for removal in last pass.
self.trace_collection.updateVariablesFromCollection(
old_collection, self.source_ref)
# Indicate if this is pass 1 for the module as return value.
was_complete = not self.locals_scope.complete
def markAsComplete(body, trace_collection):
if (body.locals_scope is not None
and body.locals_scope.isMarkedForPropagation()):
body.locals_scope = None
if body.locals_scope is not None:
body.locals_scope.markAsComplete(trace_collection)
def markEntryPointAsComplete(body):
markAsComplete(body, body.trace_collection)
outline_bodies = body.trace_collection.getOutlineFunctions()
if outline_bodies is not None:
for outline_body in outline_bodies:
markAsComplete(outline_body, body.trace_collection)
body.optimizeUnusedTempVariables()
markEntryPointAsComplete(self)
for function_body in self.getUsedFunctions():
markEntryPointAsComplete(function_body)
function_body.optimizeUnusedClosureVariables()
function_body.optimizeVariableReleases()
return was_complete
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def isUnoptimized(self):
# Modules don't do this, pylint: disable=no-self-use
return False
def getLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getUserLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
@staticmethod
def getFunctionVariablesWithAutoReleases():
"""Return the list of function variables that should be released at exit."""
return ()
def getOutlineLocalVariables(self):
outlines = self.getTraceCollection().getOutlineFunctions()
if outlines is None:
return ()
result = []
for outline in outlines:
result.extend(outline.getUserLocalVariables())
return result
def hasClosureVariable(self, variable):
# Modules don't do this, pylint: disable=no-self-use,unused-argument
return False
def removeUserVariable(self, variable):
outlines = self.getTraceCollection().getOutlineFunctions()
for outline in outlines:
user_locals = outline.getUserLocalVariables()
if variable in user_locals:
outline.removeUserVariable(variable)
break
def getLocalsScope(self):
return self.locals_scope
class CompiledPythonModule(ChildrenHavingMixin, ClosureGiverNodeMixin,
MarkNeedsAnnotationsMixin, EntryPointMixin,
PythonModuleBase):
""" Compiled Python Module
"""
kind = "COMPILED_PYTHON_MODULE"
named_children = ("body", "functions")
checkers = {"body": checkStatementsSequenceOrNone}
def __init__(self, name, package_name, mode, future_spec, source_ref):
PythonModuleBase.__init__(self,
name=name,
package_name=package_name,
source_ref=source_ref)
ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module")
ChildrenHavingMixin.__init__(
self,
values={
"body": None, # delayed
"functions": (),
},
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
def getDetails(self):
return {
"filename": self.source_ref.getFilename(),
"package": self.package_name,
"name": self.name
}
def getDetailsForDisplay(self):
result = self.getDetails()
if self.future_spec is not None:
result["code_flags"] = ','.join(self.future_spec.asFlags())
return result
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider, must not be used,
assert False
def getFutureSpec(self):
return self.future_spec
def asGraph(self, computation_counter):
from graphviz import Digraph # @UnresolvedImport pylint: disable=I0021,import-error
graph = Digraph("cluster_%d" % computation_counter,
comment="Graph for %s" % self.getName())
graph.body.append("style=filled")
graph.body.append("color=lightgrey")
graph.body.append("label=Iteration_%d" % computation_counter)
def makeTraceNodeName(variable_trace):
return "%d/ %s %s %s" % (
computation_counter, variable_trace.getVariable(),
variable_trace.getVersion(), variable_trace.__class__.__name__)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
for (_variable, _version
), variable_trace in trace_collection.getVariableTracesAll(
).items():
node = makeTraceNodeName(variable_trace)
previous = variable_trace.getPrevious()
if variable_trace.hasDefiniteUsages():
graph.attr("node", style="filled", color="blue")
elif variable_trace.hasPotentialUsages():
graph.attr("node", style="filled", color="yellow")
else:
graph.attr("node", style="filled", color="red")
graph.node(node)
if type(previous) is tuple:
for previous in previous:
graph.edge(makeTraceNodeName(previous), node)
elif previous is not None:
graph.edge(makeTraceNodeName(previous), node)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
getFunctions = ChildrenHavingMixin.childGetter("functions")
setFunctions = ChildrenHavingMixin.childSetter("functions")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
# We have never have a parent
return None
def getParentVariableProvider(self):
# We have never have a provider
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(variable_name=variable_name)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(module=self,
variable_name=variable_name)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
@staticmethod
def isEarlyClosure():
# Modules should immediately closure variables on use.
return True
def getEntryPoint(self):
return self
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.getFunctions()
assert function_body not in functions
functions += (function_body, )
self.setFunctions(functions)
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.getFunctions()
assert function_body.isExpressionFunctionBody() or \
function_body.isExpressionClassBody() or \
function_body.isExpressionGeneratorObjectBody() or \
function_body.isExpressionCoroutineObjectBody() or \
function_body.isExpressionAsyncgenObjectBody()
result = function_body not in self.active_functions
if result:
self.active_functions.add(function_body)
return result
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.getFunctions():
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.getFunctions():
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
elif main_filename.endswith(".pyw"):
result = main_filename[:-4]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')', "").replace('(', "")
def computeModule(self):
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection=self.trace_collection)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref=new_module.getSourceReference(),
tags="new_code",
message="Recursed to module package.")
self.trace_collection.updateVariablesFromCollection(old_collection)
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def isUnoptimized(self):
# Modules don't do this, pylint: disable=no-self-use
return False
def getLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getUserLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getOutlineLocalVariables(self):
outlines = self.getTraceCollection().getOutlineFunctions()
if outlines is None:
return ()
result = []
for outline in outlines:
result.extend(outline.getUserLocalVariables())
return tuple(result)
def hasClosureVariable(self, variable):
# Modules don't do this, pylint: disable=no-self-use,unused-argument
return False
def removeUserVariable(self, variable):
outlines = self.getTraceCollection().getOutlineFunctions()
for outline in outlines:
user_locals = outline.getUserLocalVariables()
if variable in user_locals:
outline.removeUserVariable(variable)
break