def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
VariableRegistry.considerCompletion()
finished = makeOptimizationPass(False)
# Demote to bytecode if now.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
# Second, endless pass.
if _progress:
info("PASS 2..:")
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def removeUserVariable(self, variable):
assert variable in self.providing.values(), (self.providing, variable)
del self.providing[variable.getName()]
assert not variable.isParameterVariable() or \
variable.getOwner() is not self
VariableRegistry.removeVariableUsage(variable, self)
def removeUserVariable(self, variable):
assert variable in self.providing.values(), (self.providing, variable)
del self.providing[variable.getName()]
assert not variable.isParameterVariable() or \
variable.getOwner() is not self
VariableRegistry.removeVariableUsage(variable, self)
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
printLine("Initial optimization pass.")
else:
printLine("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
for current_module in ModuleRegistry.getDoneModules():
optimizeVariables(current_module)
return finished
def optimizeUnusedClosureVariables(function_body):
for closure_variable in function_body.getClosureVariables():
# print "VAR", closure_variable
variable_traces = function_body.constraint_collection.getVariableTraces(
variable=closure_variable)
empty = areEmptyTraces(variable_traces)
if empty:
signalChange("var_usage",
function_body.getSourceReference(),
message="Remove unused closure variable.")
function_body.removeClosureVariable(closure_variable)
else:
read_only = areReadOnlyTraces(variable_traces)
if read_only:
global_trace = VariableRegistry.getGlobalVariableTrace(
closure_variable)
if global_trace is not None:
if not global_trace.hasWritesOutsideOf(function_body):
function_body.demoteClosureVariable(closure_variable)
signalChange(
"var_usage",
function_body.getSourceReference(),
message=
"Turn read-only usage of unassigned closure variable to local variable."
)
def optimizeUnusedClosureVariables(function_body):
for closure_variable in function_body.getClosureVariables():
# print "VAR", closure_variable
variable_traces = function_body.constraint_collection.getVariableTraces(
variable = closure_variable
)
empty = areEmptyTraces(variable_traces)
if empty:
signalChange(
"var_usage",
function_body.getSourceReference(),
message = "Remove unused closure variable."
)
function_body.removeClosureVariable(closure_variable)
else:
read_only = areReadOnlyTraces(variable_traces)
if read_only:
global_trace = VariableRegistry.getGlobalVariableTrace(closure_variable)
if global_trace is not None:
if not global_trace.hasWritesOutsideOf(function_body):
function_body.demoteClosureVariable(closure_variable)
signalChange(
"var_usage",
function_body.getSourceReference(),
message = "Turn read-only usage of unassigned closure variable to local variable."
)
def demoteClosureVariable(self, variable):
assert variable.isLocalVariable()
self.taken.remove(variable)
assert variable.getOwner() is not self
new_variable = Variables.LocalVariable(
owner=self, variable_name=variable.getName())
self.providing[variable.getName()] = new_variable
updateVariableUsage(provider=self,
old_variable=variable,
new_variable=new_variable)
VariableRegistry.addVariableUsage(new_variable, self)
def demoteClosureVariable(self, variable):
assert variable.isLocalVariable()
self.taken.remove(variable)
assert variable.getOwner() is not self
new_variable = Variables.LocalVariable(
owner = self,
variable_name = variable.getName()
)
self.providing[variable.getName()] = new_variable
updateVariableUsage(
provider = self,
old_variable = variable,
new_variable = new_variable
)
VariableRegistry.removeVariableUsage(variable, self)
VariableRegistry.addVariableUsage(new_variable, self)
def computeExpression(self, constraint_collection):
variable = self.variable
assert variable is not None
self.variable_trace = constraint_collection.getVariableCurrentTrace(
variable = variable
)
replacement = self.variable_trace.getReplacementNode(self)
if replacement is not None:
return (
replacement,
"new_expression",
"Value propagated for '%s' from '%s'." % (
self.variable.getName(),
replacement.getSourceReference().getAsString()
)
)
self.global_trace = VariableRegistry.getGlobalVariableTrace(variable)
# TODO: Maybe local variables are factored into this strangely.
if self.global_trace is None and variable.isModuleVariable():
constraint_collection.assumeUnclearLocals()
elif (variable.isModuleVariable() and not self.global_trace.hasDefiniteWrites() ) or \
variable.isMaybeLocalVariable():
if self.variable_name in Builtins.builtin_exception_names:
from .BuiltinRefNodes import ExpressionBuiltinExceptionRef
new_node = ExpressionBuiltinExceptionRef(
exception_name = self.variable_name,
source_ref = self.getSourceReference()
)
change_tags = "new_builtin_ref"
change_desc = """\
Module variable '%s' found to be built-in exception reference.""" % (
self.variable_name
)
elif self.variable_name in Builtins.builtin_names and \
self.variable_name != "pow":
from .BuiltinRefNodes import ExpressionBuiltinRef
new_node = ExpressionBuiltinRef(
builtin_name = self.variable_name,
source_ref = self.getSourceReference()
)
change_tags = "new_builtin_ref"
change_desc = """\
Module variable '%s' found to be built-in reference.""" % (
self.variable_name
)
elif self.variable_name == "__name__":
new_node = ExpressionConstantRef(
constant = variable.getOwner().getParentModule().\
getFullName(),
source_ref = self.getSourceReference()
)
change_tags = "new_constant"
change_desc = """\
Replaced read-only module attribute '__name__' with constant value."""
elif self.variable_name == "__package__":
new_node = ExpressionConstantRef(
constant = variable.getOwner().getPackage(),
source_ref = self.getSourceReference()
)
change_tags = "new_constant"
change_desc = """\
Replaced read-only module attribute '__package__' with constant value."""
else:
self.variable_trace.addUsage()
# Probably should give a warning once about it.
new_node = self
change_tags = None
change_desc = None
return new_node, change_tags, change_desc
self.variable_trace.addUsage()
return self, None, None
def updateFromCollection(self, old_collection):
VariableRegistry.updateFromCollection(old_collection, self)
def computeStatement(self, constraint_collection):
# This is very complex stuff, pylint: disable=R0912
# TODO: Way too ugly to have global trace kinds just here, and needs to
# be abstracted somehow. But for now we let it live here: pylint: disable=R0911,R0915
# Assignment source may re-compute here:
constraint_collection.onExpression(self.getAssignSource())
source = self.getAssignSource()
# No assignment will occur, if the assignment source raises, so strip it
# away.
if source.willRaiseException(BaseException):
result = makeStatementExpressionOnlyReplacementNode(expression=source, node=self)
return (
result,
"new_raise",
"""\
Assignment raises exception in assigned value, removed assignment.""",
)
variable_ref = self.getTargetVariableRef()
variable = variable_ref.getVariable()
# Not allowed anymore at this point.
assert variable is not None
# Assigning from and to the same variable, can be optimized away
# immediately, there is no point in doing it. Exceptions are of course
# module variables that collide with built-in names.
if not variable.isModuleVariable() and source.isExpressionVariableRef() and source.getVariable() == variable:
# A variable access that has a side effect, must be preserved,
# so it can e.g. raise an exception, otherwise we can be fully
# removed.
if source.mayHaveSideEffects():
result = makeStatementExpressionOnlyReplacementNode(expression=source, node=self)
return (
result,
"new_statements",
"""\
Reduced assignment of variable from itself to access of it.""",
)
else:
return (
None,
"new_statements",
"""\
Removed assignment of variable from itself which is known to be defined.""",
)
# If the assignment source has side effects, we can simply evaluate them
# beforehand, we have already visited and evaluated them before.
if source.isExpressionSideEffects():
statements = [
makeStatementExpressionOnlyReplacementNode(side_effect, self) for side_effect in source.getSideEffects()
]
statements.append(self)
parent = self.parent
result = makeStatementsSequenceReplacementNode(statements=statements, node=self)
result.parent = parent
# Need to update it.
self.setAssignSource(source.getExpression())
source = self.getAssignSource()
return (
result,
"new_statements",
"""\
Side effects of assignments promoted to statements.""",
)
# Set-up the trace to the trace collection, so future references will
# find this assignment.
self.variable_trace = constraint_collection.onVariableSet(assign_node=self)
global_trace = VariableRegistry.getGlobalVariableTrace(variable)
if global_trace is not None and Options.isExperimental():
last_trace = global_trace.getMatchingAssignTrace(self)
if last_trace is not None:
if variable.isLocalVariable() or variable.isTempVariable():
if source.isCompileTimeConstant():
# Can safely forward propagate only non-mutable constants.
if not source.isMutable():
provider = self.getParentVariableProvider()
if variable.isTempVariable() or (
not provider.isUnoptimized() and not provider.isClassDictCreation()
):
if last_trace.hasDefiniteUsages():
self.variable_trace.setReplacementNode(lambda usage: source.makeClone())
propagated = True
else:
propagated = False
if not last_trace.hasPotentialUsages() and not last_trace.hasNameUsages():
# This limitation may fall later.
if not variable.isSharedLogically():
if not last_trace.getPrevious().isUninitTrace():
# TODO: We could well decide, if that's even necessary.
result = StatementDelVariable(
variable_ref=self.getTargetVariableRef(),
tolerant=True,
source_ref=self.getSourceReference(),
)
else:
result = None
return (
result,
"new_statements",
"Dropped %s assignment statement to '%s'."
% (
"propagated" if propagated else "dead",
self.getTargetVariableRef().getVariableName(),
),
)
else:
# Something might be possible still.
pass
elif (
Options.isExperimental()
and source.isExpressionFunctionCreation()
and not source.getFunctionRef().getFunctionBody().isGenerator()
and not source.getFunctionRef().getFunctionBody().isClassDictCreation()
and not source.getDefaults()
and not source.getKwDefaults()
and not source.getAnnotations()
):
# TODO: These are very mutable, right?
provider = self.getParentVariableProvider()
if variable.isTempVariable() or (
not provider.isUnoptimized() and not provider.isClassDictCreation()
):
# This limitation may fall later.
if not variable.isSharedLogically():
if (
last_trace.getDefiniteUsages() <= 1
and not last_trace.hasPotentialUsages()
and not last_trace.hasNameUsages()
):
if last_trace.getDefiniteUsages() == 1:
self.variable_trace.setReplacementNode(lambda usage: source.makeClone())
propagated = True
else:
propagated = False
if not last_trace.getPrevious().isUninitTrace():
# TODO: We could well decide, if that's even necessary.
result = StatementDelVariable(
variable_ref=self.getTargetVariableRef(),
tolerant=True,
source_ref=self.getSourceReference(),
)
else:
result = None
return (
result,
"new_statements",
"Dropped %s assignment statement to '%s'."
% (
"propagated" if propagated else "dead",
self.getTargetVariableRef().getVariableName(),
),
)
else:
# More cases thinkable.
pass
return self, None, None
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if VariableRegistry.considerCompletion():
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def updateFromCollection(self, old_collection):
VariableRegistry.updateFromCollection(old_collection, self)
def computeExpression(self, constraint_collection):
variable = self.variable
assert variable is not None
self.variable_trace = constraint_collection.getVariableCurrentTrace(
variable=variable)
replacement = self.variable_trace.getReplacementNode(self)
if replacement is not None:
constraint_collection.signalChange(
"new_expression", self.source_ref,
"Value propagated for '%s' from '%s'." %
(self.variable.getName(),
replacement.getSourceReference().getAsString()))
# Need to compute the replacement still.
return replacement.computeExpression(constraint_collection)
if not self.variable_trace.mustHaveValue():
# TODO: This could be way more specific surely.
constraint_collection.onExceptionRaiseExit(BaseException)
self.global_trace = VariableRegistry.getGlobalVariableTrace(variable)
# TODO: Maybe local variables are factored into this strangely.
if self.global_trace is None and variable.isModuleVariable():
constraint_collection.assumeUnclearLocals()
elif (variable.isModuleVariable() and not self.global_trace.hasDefiniteWrites() ) or \
variable.isMaybeLocalVariable():
if self.variable_name in Builtins.builtin_exception_names:
from .BuiltinRefNodes import ExpressionBuiltinExceptionRef
new_node = ExpressionBuiltinExceptionRef(
exception_name=self.variable_name,
source_ref=self.getSourceReference())
change_tags = "new_builtin_ref"
change_desc = """\
Module variable '%s' found to be built-in exception reference.""" % (
self.variable_name)
elif self.variable_name in Builtins.builtin_names and \
self.variable_name != "pow":
from .BuiltinRefNodes import ExpressionBuiltinRef
new_node = ExpressionBuiltinRef(
builtin_name=self.variable_name,
source_ref=self.getSourceReference())
change_tags = "new_builtin_ref"
change_desc = """\
Module variable '%s' found to be built-in reference.""" % (self.variable_name)
elif self.variable_name == "__name__":
new_node = ExpressionConstantRef(
constant = variable.getOwner().getParentModule().\
getFullName(),
source_ref = self.getSourceReference()
)
change_tags = "new_constant"
change_desc = """\
Replaced read-only module attribute '__name__' with constant value."""
elif self.variable_name == "__package__":
new_node = ExpressionConstantRef(
constant=variable.getOwner().getPackage(),
source_ref=self.getSourceReference())
change_tags = "new_constant"
change_desc = """\
Replaced read-only module attribute '__package__' with constant value."""
else:
self.variable_trace.addUsage()
# Probably should give a warning once about it.
new_node = self
change_tags = None
change_desc = None
return new_node, change_tags, change_desc
self.variable_trace.addUsage()
return self, None, None
def computeStatement(self, constraint_collection):
# Assignment source may re-compute here:
constraint_collection.onExpression(self.getAssignSource())
source = self.getAssignSource()
# No assignment will occur, if the assignment source raises, so strip it
# away.
if source.willRaiseException(BaseException):
result = makeStatementExpressionOnlyReplacementNode(
expression = source,
node = self
)
return result, "new_raise", """\
Assignment raises exception in assigned value, removed assignment."""
variable_ref = self.getTargetVariableRef()
variable = variable_ref.getVariable()
# Not allowed anymore at this point.
assert variable is not None
# Assigning from and to the same variable, can be optimized away
# immediately, there is no point in doing it. Exceptions are of course
# module variables that collide with built-in names.
if not variable.isModuleVariable() and \
source.isExpressionVariableRef() and \
source.getVariable() == variable:
# A variable access that has a side effect, must be preserved,
# so it can e.g. raise an exception, otherwise we can be fully
# removed.
if source.mayHaveSideEffects():
result = makeStatementExpressionOnlyReplacementNode(
expression = source,
node = self
)
return result, "new_statements", """\
Reduced assignment of variable from itself to access of it."""
else:
return None, "new_statements", """\
Removed assignment of variable from itself which is known to be defined."""
# If the assignment source has side effects, we can simply evaluate them
# beforehand, we have already visited and evaluated them before.
if source.isExpressionSideEffects():
statements = [
makeStatementExpressionOnlyReplacementNode(
side_effect,
self
)
for side_effect in
source.getSideEffects()
]
statements.append(self)
parent = self.parent
result = makeStatementsSequenceReplacementNode(
statements = statements,
node = self,
)
result.parent = parent
# Need to update it.
self.setAssignSource(source.getExpression())
source = self.getAssignSource()
result = result, "new_statements", """\
Side effects of assignments promoted to statements."""
else:
result = self, None, None
# Set-up the trace to the trace collection, so future references will
# find this assignment.
self.variable_trace = constraint_collection.onVariableSet(
assign_node = self
)
global_trace = VariableRegistry.getGlobalVariableTrace(variable)
if global_trace is not None:
if variable.isTempVariable():
if source.isCompileTimeConstant() and not source.isMutable():
self.variable_trace.setReplacementNode(source)
elif variable.isLocalVariable():
if source.isCompileTimeConstant() and not source.isMutable():
provider = self.getParentVariableProvider()
if provider.isPythonModule() or \
(not provider.isUnoptimized() and not provider.isClassDictCreation()):
self.variable_trace.setReplacementNode(source)
return result
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=Utils.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection=function.constraint_collection,
new_collection=None)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def computeExpression(self, constraint_collection):
variable = self.variable
assert variable is not None
self.variable_trace = constraint_collection.getVariableCurrentTrace(
variable = variable
)
global_trace = VariableRegistry.getGlobalVariableTrace(variable)
# TODO: Maybe local variables are factored into this strangely.
if global_trace is None and variable.isModuleVariable():
constraint_collection.assumeUnclearLocals()
elif (variable.isModuleVariable() and not global_trace.hasDefiniteWrites() ) or \
variable.isMaybeLocalVariable():
if self.variable_name in Builtins.builtin_exception_names:
from .BuiltinRefNodes import ExpressionBuiltinExceptionRef
new_node = ExpressionBuiltinExceptionRef(
exception_name = self.variable_name,
source_ref = self.getSourceReference()
)
change_tags = "new_builtin_ref"
change_desc = """\
Module variable '%s' found to be built-in exception reference.""" % (
self.variable_name
)
elif self.variable_name in Builtins.builtin_names and \
self.variable_name != "pow":
from .BuiltinRefNodes import ExpressionBuiltinRef
new_node = ExpressionBuiltinRef(
builtin_name = self.variable_name,
source_ref = self.getSourceReference()
)
change_tags = "new_builtin_ref"
change_desc = """\
Module variable '%s' found to be built-in reference.""" % (
self.variable_name
)
elif self.variable_name == "__name__":
new_node = ExpressionConstantRef(
constant = variable.getOwner().getParentModule().\
getFullName(),
source_ref = self.getSourceReference()
)
change_tags = "new_constant"
change_desc = """\
Replaced read-only module attribute '__name__' with constant value."""
elif self.variable_name == "__package__":
new_node = ExpressionConstantRef(
constant = variable.getOwner().getPackage(),
source_ref = self.getSourceReference()
)
change_tags = "new_constant"
change_desc = """\
Replaced read-only module attribute '__package__' with constant value."""
else:
# Probably should give a warning once about it.
new_node = self
change_tags = None
change_desc = None
return new_node, change_tags, change_desc
return self, None, None
