class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Compiled Python Module
"""
kind = "COMPILED_PYTHON_MODULE"
named_children = (
"body",
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = {}
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
self.code_object = CodeObjectSpec(
code_name = "<module>" if self.isMainModule() else self.getName(),
code_kind = "Module",
arg_names = (),
kw_only_count = 0,
has_stardict = False,
has_starlist = False,
)
def getCodeObject(self):
return self.code_object
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def asXml(self):
result = super(CompiledPythonModule, self).asXml()
for function_body in self.active_functions:
result.append(function_body.asXml())
return result
def asGraph(self, computation_counter):
from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021
graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName())
graph.body.append("style=filled")
graph.body.append("color=lightgrey")
graph.body.append("label=Iteration_%d" % computation_counter)
def makeTraceNodeName(variable_trace):
return "%d/ %s %s %s" % (
computation_counter,
variable_trace.getVariable(),
variable_trace.getVersion(),
variable_trace.__class__.__name__
)
for function_body in self.active_functions:
constraint_collection = function_body.constraint_collection
for (_variable, _version), variable_trace in constraint_collection.getVariableTracesAll().items():
node = makeTraceNodeName(variable_trace)
previous = variable_trace.getPrevious()
if variable_trace.hasDefiniteUsages():
graph.attr("node", style = "filled", color = "blue")
elif variable_trace.hasPotentialUsages():
graph.attr("node", style = "filled", color = "yellow")
else:
graph.attr("node", style = "filled", color = "red")
graph.node(node)
if type(previous) is tuple:
for previous in previous:
graph.edge(makeTraceNodeName(previous), node)
elif previous is not None:
graph.edge(makeTraceNodeName(previous), node)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r ,c)
for c in
self.getFullName()
)
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody() or \
function_body.isExpressionClassBody() or \
function_body.isExpressionGeneratorObjectBody() or \
function_body.isExpressionCoroutineObjectBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection = self.constraint_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection
class PythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Module
The module is the only possible root of a tree. When there are many
modules they form a forest.
"""
kind = "PYTHON_MODULE"
named_children = (
"body",
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = set()
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def asXml(self):
result = super(PythonModule, self).asXml()
for function_body in self.active_functions:
result.append(function_body.asXml())
return result
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def getVariables(self):
return self.variables
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
assert result not in self.variables
self.variables.add(result)
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r ,c)
for c in
self.getFullName()
)
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule()
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection = self.constraint_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection
def hasUnclearLocals(self):
for collection in self.getTraceCollections():
if collection.hasUnclearLocals():
return True
return False
class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Compiled Python Module
"""
kind = "COMPILED_PYTHON_MODULE"
named_children = (
"body",
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = {}
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
self.code_object = CodeObjectSpec(
code_name = "<module>" if self.isMainModule() else self.getName(),
code_kind = "Module",
arg_names = (),
kw_only_count = 0,
has_stardict = False,
has_starlist = False,
)
def getCodeObject(self):
return self.code_object
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def asXml(self):
result = super(CompiledPythonModule, self).asXml()
for function_body in self.active_functions:
result.append(function_body.asXml())
return result
def asGraph(self, computation_counter):
from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021
graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName())
graph.body.append("style=filled")
graph.body.append("color=lightgrey")
graph.body.append("label=Iteration_%d" % computation_counter)
def makeTraceNodeName(variable_trace):
return "%d/ %s %s %s" % (
computation_counter,
variable_trace.getVariable(),
variable_trace.getVersion(),
variable_trace.__class__.__name__
)
for function_body in self.active_functions:
constraint_collection = function_body.constraint_collection
for (_variable, _version), variable_trace in constraint_collection.getVariableTracesAll().items():
node = makeTraceNodeName(variable_trace)
previous = variable_trace.getPrevious()
if variable_trace.hasDefiniteUsages():
graph.attr("node", style = "filled", color = "blue")
elif variable_trace.hasPotentialUsages():
graph.attr("node", style = "filled", color = "yellow")
else:
graph.attr("node", style = "filled", color = "red")
graph.node(node)
if type(previous) is tuple:
for previous in previous:
graph.edge(makeTraceNodeName(previous), node)
elif previous is not None:
graph.edge(makeTraceNodeName(previous), node)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r ,c)
for c in
self.getFullName()
)
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody() or \
function_body.isExpressionClassBody() or \
function_body.isExpressionGeneratorObjectBody() or \
function_body.isExpressionCoroutineObjectBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection = self.constraint_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection
class PythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Module
The module is the only possible root of a tree. When there are many
modules they form a forest.
"""
kind = "PYTHON_MODULE"
named_children = ("body", )
checkers = {"body": checkStatementsSequenceOrNone}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(self,
name=name,
code_prefix="module",
source_ref=source_ref)
PythonModuleMixin.__init__(self, name=name, package_name=package_name)
ChildrenHavingMixin.__init__(
self,
values={
"body": None # delayed
},
)
self.variables = set()
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
def getDetails(self):
return {
"filename": self.source_ref.getFilename(),
"package": self.package_name,
"name": self.name
}
def asXml(self):
result = super(PythonModule, self).asXml()
for function_body in self.functions:
result.append(function_body.asXml())
return result
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def getVariables(self):
return self.variables
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(variable_name=variable_name)
def createProvidedVariable(self, variable_name):
result = Variables.ModuleVariable(module=self,
variable_name=variable_name)
assert result not in self.variables
self.variables.add(result)
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r, c) for c in self.getFullName())
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')', "").replace('(', "")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule()
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection=self.constraint_collection)
if result is not module_body:
self.setBody(result)
self.constraint_collection.makeVariableTraceOptimizations(self)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref=new_module.getSourceReference(),
tags="new_code",
message="Recursed to module package.")
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection
def hasUnclearLocals(self):
for collection in self.getTraceCollections():
if collection.hasUnclearLocals():
return True
return False