class GlobalVariableTrace:
@counted_init
def __init__(self):
self.traces = set()
__del__ = counted_del()
def add(self, variable_trace):
self.traces.add(variable_trace)
def remove(self, variable_trace):
self.traces.remove(variable_trace)
def hasDefiniteWrites(self):
for trace in self.traces:
if trace.isAssignTrace():
return True
return False
def getMatchingAssignTrace(self, assign_node):
for trace in self.traces:
if trace.isAssignTrace() and trace.getAssignNode() is assign_node:
return trace
return None
def hasWritesOutsideOf(self, provider):
for trace in self.traces:
if trace.isAssignTrace():
if trace.getAssignNode().getParentVariableProvider(
) is not provider:
return True
return False
class VariableInformation:
@counted_init
def __init__(self, variable):
self.variable = variable
self.users = set()
self.active_users = set()
def __repr__(self):
return "<%s object for %s>" % (
self.__class__.__name__,
self.variable
)
__del__ = counted_del()
def addUser(self, user):
self.users.add(user)
self.active_users.add(user)
def removeUser(self, user):
try:
self.active_users.remove(user)
except KeyError:
raise KeyError(self, user)
def getUsers(self):
return self.users
def getActiveUsers(self):
return self.active_users
def getTopOwner(self):
return self.variable.getOwner()
class SourceCodeReferenceInternal(SourceCodeReference):
__slots__ = ()
__del__ = counted_del()
@counted_init
def __init__(self):
SourceCodeReference.__init__(self)
@staticmethod
def isInternal():
return True
class GlobalVariableTrace:
@counted_init
def __init__(self):
self.traces = set()
self.writers = None
self.users = None
__del__ = counted_del()
def addTrace(self, variable_trace):
self.traces.add(variable_trace)
def removeTrace(self, variable_trace):
self.traces.remove(variable_trace)
def updateUsageState(self):
writers = set()
users = set()
for trace in self.traces:
owner = trace.owner
if trace.isAssignTrace():
writers.add(owner)
users.add(owner)
self.writers = writers
self.users = users
def hasDefiniteWrites(self):
return bool(self.writers)
def getMatchingAssignTrace(self, assign_node):
for trace in self.traces:
if trace.isAssignTrace() and trace.getAssignNode() is assign_node:
return trace
return None
def hasWritesOutsideOf(self, provider):
if provider in self.writers:
return len(self.writers) > 1
else:
return bool(self.writers)
def hasAccessesOutsideOf(self, provider):
if provider in self.users:
return len(self.users) > 1
else:
return bool(self.users)
class VariableInformation:
@counted_init
def __init__(self, variable):
self.variable = variable
self.users = set()
__del__ = counted_del()
def addUser(self, user):
self.users.add(user)
def getUsers(self):
return self.users
def getTopOwner(self):
return self.variable.getOwner()
class LocalsDictHandle(object):
__slots__ = ("locals_name", )
@counted_init
def __init__(self, locals_name):
self.locals_name = locals_name
__del__ = counted_del()
def __repr__(self):
return "<%s of %s>" % (self.__class__.__name__, self.locals_name)
@staticmethod
def getTypeShape():
return ShapeTypeDict
def getCodeName(self):
return self.locals_name
class LocalsDictHandleBase(object):
# TODO: Might remove some of these later, pylint: disable=too-many-instance-attributes
__slots__ = (
"locals_name",
# TODO: Specialize what the kinds really use.
"variables",
"local_variables",
"providing",
"mark_for_propagation",
"propagation",
"owner",
"complete",
)
@counted_init
def __init__(self, locals_name, owner):
self.locals_name = locals_name
self.owner = owner
# For locals dict variables in this scope.
self.variables = {}
# For local variables in this scope.
self.local_variables = {}
self.providing = OrderedDict()
# Can this be eliminated through replacement of temporary variables
self.mark_for_propagation = False
self.propagation = None
self.complete = False
if isCountingInstances():
__del__ = counted_del()
def __repr__(self):
return "<%s of %s>" % (self.__class__.__name__, self.locals_name)
def getName(self):
return self.locals_name
def makeClone(self, new_owner):
count = 1
# Make it unique.
while 1:
locals_name = self.locals_name + "_inline_%d" % count
if locals_name not in locals_dict_handles:
break
count += 1
result = self.__class__(locals_name=locals_name, owner=new_owner)
variable_translation = {}
# Clone variables as well.
for variable_name, variable in self.variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.variables[variable_name] = new_variable
for variable_name, variable in self.local_variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.local_variables[variable_name] = new_variable
result.providing = OrderedDict()
for variable_name, variable in self.providing.items():
if variable in variable_translation:
new_variable = variable_translation[variable]
else:
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.providing[variable_name] = new_variable
return result, variable_translation
@staticmethod
def getTypeShape():
return tshape_dict
def getCodeName(self):
return self.locals_name
@staticmethod
def isModuleScope():
return False
@staticmethod
def isClassScope():
return False
@staticmethod
def isFunctionScope():
return False
def getProvidedVariables(self):
return self.providing.values()
def registerProvidedVariable(self, variable):
variable_name = variable.getName()
self.providing[variable_name] = variable
def unregisterProvidedVariable(self, variable):
"""Remove provided variable, e.g. because it became unused."""
variable_name = variable.getName()
if variable_name in self.providing:
del self.providing[variable_name]
registerClosureVariable = registerProvidedVariable
unregisterClosureVariable = unregisterProvidedVariable
def hasProvidedVariable(self, variable_name):
"""Test if a variable is provided."""
return variable_name in self.providing
def getProvidedVariable(self, variable_name):
"""Test if a variable is provided."""
return self.providing[variable_name]
def getLocalsRelevantVariables(self):
"""The variables relevant to locals."""
return self.providing.values()
def getLocalsDictVariable(self, variable_name):
if variable_name not in self.variables:
result = Variables.LocalsDictVariable(owner=self,
variable_name=variable_name)
self.variables[variable_name] = result
return self.variables[variable_name]
# TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here.
def getLocalVariable(self, owner, variable_name):
if variable_name not in self.local_variables:
result = Variables.LocalVariable(owner=owner,
variable_name=variable_name)
self.local_variables[variable_name] = result
return self.local_variables[variable_name]
def markForLocalsDictPropagation(self):
self.mark_for_propagation = True
def isMarkedForPropagation(self):
return self.mark_for_propagation
def allocateTempReplacementVariable(self, trace_collection, variable_name):
if self.propagation is None:
self.propagation = OrderedDict()
if variable_name not in self.propagation:
provider = trace_collection.getOwner()
self.propagation[variable_name] = provider.allocateTempVariable(
temp_scope=None,
name=self.getCodeName() + "_key_" + variable_name)
return self.propagation[variable_name]
def getPropagationVariables(self):
if self.propagation is None:
return ()
return self.propagation
def finalize(self):
# Make it unusable when it's become empty, not used.
self.owner.locals_scope = None
del self.owner
del self.propagation
del self.mark_for_propagation
for variable in self.variables.values():
variable.finalize()
for variable in self.local_variables.values():
variable.finalize()
del self.variables
del self.providing
def markAsComplete(self, trace_collection):
self.complete = True
self._considerUnusedUserLocalVariables(trace_collection)
self._considerPropagation(trace_collection)
# TODO: Limited to Python2 classes for now, more overloads need to be added, this
# ought to be abstract and have variants with TODOs for each of them.
@staticmethod
def _considerPropagation(trace_collection):
"""For overload by scope type. Check if this can be replaced."""
def _considerUnusedUserLocalVariables(self, trace_collection):
"""Check scope for unused variables."""
provided = self.getProvidedVariables()
removals = []
for variable in provided:
if (variable.isLocalVariable()
and not variable.isParameterVariable()
and variable.getOwner() is self.owner):
empty = trace_collection.hasEmptyTraces(variable)
if empty:
removals.append(variable)
for variable in removals:
self.unregisterProvidedVariable(variable)
trace_collection.signalChange(
"var_usage",
self.owner.getSourceReference(),
message="Remove unused local variable '%s'." %
variable.getName(),
)
class ParameterSpec(object):
# These got many attributes, in part duplicating name and instance of
# variables, pylint: disable=too-many-instance-attributes
__slots__ = (
"name",
"owner",
"normal_args",
"normal_variables",
"list_star_arg",
"dict_star_arg",
"list_star_variable",
"dict_star_variable",
"default_count",
"kw_only_args",
"kw_only_variables",
)
@counted_init
def __init__(
self,
ps_name,
ps_normal_args,
ps_kw_only_args,
ps_list_star_arg,
ps_dict_star_arg,
ps_default_count,
):
if type(ps_normal_args) is str:
if ps_normal_args == "":
ps_normal_args = ()
else:
ps_normal_args = ps_normal_args.split(",")
if type(ps_kw_only_args) is str:
if ps_kw_only_args == "":
ps_kw_only_args = ()
else:
ps_kw_only_args = ps_kw_only_args.split(",")
assert None not in ps_normal_args
self.owner = None
self.name = ps_name
self.normal_args = tuple(ps_normal_args)
self.normal_variables = None
assert (
ps_list_star_arg is None or type(ps_list_star_arg) is str
), ps_list_star_arg
assert (
ps_dict_star_arg is None or type(ps_dict_star_arg) is str
), ps_dict_star_arg
self.list_star_arg = ps_list_star_arg if ps_list_star_arg else None
self.dict_star_arg = ps_dict_star_arg if ps_dict_star_arg else None
self.list_star_variable = None
self.dict_star_variable = None
self.default_count = ps_default_count
self.kw_only_args = tuple(ps_kw_only_args)
self.kw_only_variables = None
__del__ = counted_del()
def makeClone(self):
return ParameterSpec(
ps_name=self.name,
ps_normal_args=self.normal_args,
ps_kw_only_args=self.kw_only_args,
ps_list_star_arg=self.list_star_arg,
ps_dict_star_arg=self.dict_star_arg,
ps_default_count=self.default_count,
)
def getDetails(self):
return {
"ps_name": self.name,
"ps_normal_args": ",".join(self.normal_args),
"ps_kw_only_args": ",".join(self.kw_only_args),
"ps_list_star_arg": self.list_star_arg
if self.list_star_arg is not None
else "",
"ps_dict_star_arg": self.dict_star_arg
if self.dict_star_arg is not None
else "",
"ps_default_count": self.default_count,
}
def checkParametersValid(self):
arg_names = self.getParameterNames()
# Check for duplicate arguments, could happen.
for arg_name in arg_names:
if arg_names.count(arg_name) != 1:
return "duplicate argument '%s' in function definition" % arg_name
return None
def __repr__(self):
parts = [str(normal_arg) for normal_arg in self.normal_args]
if self.list_star_arg is not None:
parts.append("*%s" % self.list_star_arg)
if self.dict_star_variable is not None:
parts.append("**%s" % self.dict_star_variable)
if parts:
return "<ParameterSpec '%s'>" % ",".join(parts)
else:
return "<NoParameters>"
def getArgumentCount(self):
return len(self.normal_args)
def setOwner(self, owner):
if self.owner is not None:
return
self.owner = owner
self.normal_variables = []
for normal_arg in self.normal_args:
if type(normal_arg) is str:
normal_variable = Variables.ParameterVariable(
owner=self.owner, parameter_name=normal_arg
)
else:
assert False, normal_arg
self.normal_variables.append(normal_variable)
if self.list_star_arg:
self.list_star_variable = Variables.ParameterVariable(
owner=owner, parameter_name=self.list_star_arg
)
else:
self.list_star_variable = None
if self.dict_star_arg:
self.dict_star_variable = Variables.ParameterVariable(
owner=owner, parameter_name=self.dict_star_arg
)
else:
self.dict_star_variable = None
self.kw_only_variables = [
Variables.ParameterVariable(owner=self.owner, parameter_name=kw_only_arg)
for kw_only_arg in self.kw_only_args
]
def getDefaultCount(self):
return self.default_count
def getPositionalOnlyCount(self):
# Virtual method, pylint: disable=no-self-use
return 0
def hasDefaultParameters(self):
return self.getDefaultCount() > 0
def getTopLevelVariables(self):
return self.normal_variables + self.kw_only_variables
def getAllVariables(self):
result = list(self.normal_variables)
result += self.kw_only_variables
if self.list_star_variable is not None:
result.append(self.list_star_variable)
if self.dict_star_variable is not None:
result.append(self.dict_star_variable)
return result
def getParameterNames(self):
result = list(self.normal_args)
result += self.kw_only_args
if self.list_star_arg is not None:
result.append(self.list_star_arg)
if self.dict_star_arg is not None:
result.append(self.dict_star_arg)
return result
def getStarListArgumentName(self):
return self.list_star_arg
def getListStarArgVariable(self):
return self.list_star_variable
def getStarDictArgumentName(self):
return self.dict_star_arg
def getDictStarArgVariable(self):
return self.dict_star_variable
def getKwOnlyVariables(self):
return self.kw_only_variables
def allowsKeywords(self):
# Abstract method, pylint: disable=no-self-use
return True
def getKeywordRefusalText(self):
return "%s() takes no keyword arguments" % self.name
def getArgumentNames(self):
return self.normal_args
def getKwOnlyParameterNames(self):
return self.kw_only_args
def getKwOnlyParameterCount(self):
return len(self.kw_only_args)
class SourceCodeReference(object):
__slots__ = ["filename", "line", "column"]
@classmethod
def fromFilenameAndLine(cls, filename, line):
result = cls()
result.filename = filename
result.line = line
return result
if isCountingInstances():
__del__ = counted_del()
@counted_init
def __init__(self):
self.filename = None
self.line = None
self.column = None
def __repr__(self):
return "<%s to %s:%s>" % (self.__class__.__name__, self.filename,
self.line)
def __hash__(self):
return hash((self.filename, self.line, self.column))
def __lt__(self, other):
# Many cases decide early, pylint: disable=too-many-return-statements
if other is None:
return True
if other is self:
return False
assert isinstance(other, SourceCodeReference), other
if self.filename < other.filename:
return True
elif self.filename > other.filename:
return False
else:
if self.line < other.line:
return True
elif self.line > other.line:
return False
else:
if self.column < other.column:
return True
elif self.column > other.column:
return False
else:
return self.isInternal() < other.isInternal()
def __eq__(self, other):
if other is None:
return False
if other is self:
return True
assert isinstance(other, SourceCodeReference), other
if self.filename != other.filename:
return False
if self.line != other.line:
return False
if self.column != other.column:
return False
return self.isInternal() is other.isInternal()
def _clone(self, line):
"""Make a copy it itself."""
return self.fromFilenameAndLine(filename=self.filename, line=line)
def atInternal(self):
"""Make a copy it itself but mark as internal code.
Avoids useless copies, by returning an internal object again if
it is already internal.
"""
if not self.isInternal():
result = self._clone(self.line)
return result
else:
return self
def atLineNumber(self, line):
"""Make a reference to the same file, but different line.
Avoids useless copies, by returning same object if the line is
the same.
"""
assert type(line) is int, line
if self.line != line:
return self._clone(line)
else:
return self
def atColumnNumber(self, column):
assert type(column) is int, column
if self.column != column:
result = self._clone(self.line)
result.column = column
return result
else:
return self
def getLineNumber(self):
return self.line
def getColumnNumber(self):
return self.column
def getFilename(self):
return self.filename
def getAsString(self):
return "%s:%s" % (self.filename, self.line)
@staticmethod
def isInternal():
return False
class LocalsDictHandle(object):
__slots__ = ("locals_name", "variables", "mark_for_propagation",
"propagation")
@counted_init
def __init__(self, locals_name):
self.locals_name = locals_name
# For locals dict variables in this scope.
self.variables = {}
# Can this be eliminated through replacement of temporary variables
self.mark_for_propagation = False
self.propagation = None
__del__ = counted_del()
def __repr__(self):
return "<%s of %s>" % (self.__class__.__name__, self.locals_name)
def getName(self):
return self.locals_name
@staticmethod
def getTypeShape():
return tshape_dict
def getCodeName(self):
return self.locals_name
def getLocalsDictVariable(self, variable_name):
if variable_name not in self.variables:
result = Variables.LocalsDictVariable(owner=self,
variable_name=variable_name)
self.variables[variable_name] = result
return self.variables[variable_name]
def markForLocalsDictPropagation(self):
self.mark_for_propagation = True
def isMarkedForPropagation(self):
return self.mark_for_propagation
def allocateTempReplacementVariable(self, trace_collection, variable_name):
if self.propagation is None:
self.propagation = OrderedDict()
if variable_name not in self.propagation:
provider = trace_collection.getOwner()
self.propagation[variable_name] = provider.allocateTempVariable(
temp_scope=None,
name=self.getCodeName() + "_key_" + variable_name)
return self.propagation[variable_name]
def getPropagationVariables(self):
if self.propagation is None:
return ()
return self.propagation
def finalize(self):
del self.propagation
del self.mark_for_propagation
for variable in self.variables.values():
variable.finalize()
del self.variables
class CodeObjectSpec(object):
# One attribute for each code object aspect, and even flags,
# pylint: disable=too-many-arguments,too-many-instance-attributes
__slots__ = ("co_name", "co_kind", "co_varnames", "co_argcount",
"co_kwonlyargcount", "co_has_starlist", "co_has_stardict",
"filename", "line_number", "future_spec", "new_locals",
"has_closure", "is_optimized")
@counted_init
def __init__(self,
co_name,
co_kind,
co_varnames,
co_argcount,
co_kwonlyargcount,
co_has_starlist,
co_has_stardict,
co_filename,
co_lineno,
future_spec,
co_new_locals=None,
co_has_closure=None,
co_is_optimized=None):
self.co_name = co_name
self.co_kind = co_kind
self.future_spec = future_spec
assert future_spec
# Strings happens from XML parsing, make sure to convert them.
if type(co_varnames) is str:
if co_varnames == "":
co_varnames = ()
else:
co_varnames = co_varnames.split(',')
if type(co_has_starlist) is not bool:
co_has_starlist = co_has_starlist != "False"
if type(co_has_stardict) is not bool:
co_has_stardict = co_has_stardict != "False"
self.co_varnames = tuple(co_varnames)
self.co_argcount = int(co_argcount)
self.co_kwonlyargcount = int(co_kwonlyargcount)
self.co_has_starlist = co_has_starlist
self.co_has_stardict = co_has_stardict
self.filename = co_filename
self.line_number = int(co_lineno)
if type(co_has_starlist) is not bool:
co_new_locals = co_new_locals != "False"
if type(co_has_starlist) is not bool:
co_has_closure = co_has_closure != "False"
if type(co_has_starlist) is not bool:
co_is_optimized = co_is_optimized != "False"
self.new_locals = co_new_locals
self.has_closure = co_has_closure
self.is_optimized = co_is_optimized
__del__ = counted_del()
def __repr__(self):
return """\
<CodeObjectSpec %(co_kind)s '%(co_name)s' with %(co_varnames)r>""" % self.getDetails(
)
def getDetails(self):
return {
"co_name": self.co_name,
"co_kind": self.co_kind,
"co_varnames": ','.join(self.co_varnames),
"co_argcount": self.co_argcount,
"co_kwonlyargcount": self.co_kwonlyargcount,
"co_has_starlist": self.co_has_starlist,
"co_has_stardict": self.co_has_stardict,
"co_filename": self.filename,
"co_lineno": self.line_number,
"co_new_locals": self.new_locals,
"co_has_closure": self.has_closure,
"co_is_optimized": self.is_optimized,
"code_flags": ','.join(self.future_spec.asFlags())
}
def getCodeObjectKind(self):
return self.co_kind
def updateLocalNames(self, local_names):
""" Add detected local variables after closure has been decided.
"""
self.co_varnames += tuple(local_name for local_name in local_names
if local_name not in self.co_varnames)
def setFlagIsOptimizedValue(self, value):
self.is_optimized = value
def getFlagIsOptimizedValue(self):
return self.is_optimized
def setFlagNewLocalsValue(self, value):
self.new_locals = value
def getFlagNewLocalsValue(self):
return self.new_locals
def setFlagHasClosureValue(self, value):
self.has_closure = value
def getFlagHasClosureValue(self):
return self.has_closure
def getFutureSpec(self):
return self.future_spec
def getVarNames(self):
return self.co_varnames
def getArgumentCount(self):
return self.co_argcount
def getKwOnlyParameterCount(self):
return self.co_kwonlyargcount
def getCodeObjectName(self):
return self.co_name
def hasStarListArg(self):
return self.co_has_starlist
def hasStarDictArg(self):
return self.co_has_stardict
def getFilename(self):
return self.filename
def getLineNumber(self):
return self.line_number
class TraceCollectionBase(CollectionTracingMixin):
__del__ = counted_del()
@counted_init
def __init__(self, owner, name, parent):
CollectionTracingMixin.__init__(self)
self.owner = owner
self.parent = parent
self.name = name
# Value state extra information per node.
self.value_states = {}
def __repr__(self):
return "<%s for %s %d>" % (
self.__class__.__name__,
self.name,
id(self)
)
@staticmethod
def signalChange(tags, source_ref, message):
# This is monkey patched from another module, pylint: disable=E1102
signalChange(tags, source_ref, message)
def onUsedModule(self, module):
return self.parent.onUsedModule(module)
@staticmethod
def mustAlias(a, b):
if a.isExpressionVariableRef() and b.isExpressionVariableRef():
return a.getVariable() is b.getVariable()
return False
@staticmethod
def mustNotAlias(a, b):
# TODO: not yet really implemented
if a.isExpressionConstantDictRef() and b.isExpressionConstantRef():
return True
return False
def removeKnowledge(self, node):
pass
def onControlFlowEscape(self, node):
# TODO: One day, we should trace which nodes exactly cause a variable
# to be considered escaped, pylint: disable=W0613
for variable in self.getActiveVariables():
if variable.isModuleVariable():
# print variable
self.markActiveVariableAsUnknown(variable)
elif python_version >= 300 or variable.isSharedTechnically() is not False:
# print variable
# TODO: Could be limited to shared variables that are actually
# written to. Most of the time, that won't be the case.
self.markActiveVariableAsUnknown(variable)
def removeAllKnowledge(self):
self.markActiveVariablesAsUnknown()
def getVariableTrace(self, variable, version):
return self.parent.getVariableTrace(variable, version)
def hasVariableTrace(self, variable, version):
return self.parent.hasVariableTrace(variable, version)
def addVariableTrace(self, variable, version, trace):
self.parent.addVariableTrace(variable, version, trace)
def addVariableMergeMultipleTrace(self, variable, traces):
return self.parent.addVariableMergeMultipleTrace(variable, traces)
def onVariableSet(self, assign_node):
variable_ref = assign_node.getTargetVariableRef()
version = variable_ref.getVariableVersion()
variable = variable_ref.getVariable()
variable_trace = VariableTraceAssign(
owner = self.owner,
assign_node = assign_node,
variable = variable,
version = version,
previous = self.getVariableCurrentTrace(
variable = variable
)
)
self.addVariableTrace(
variable = variable,
version = version,
trace = variable_trace
)
# Make references point to it.
self.markCurrentVariableTrace(variable, version)
return variable_trace
def onVariableDel(self, variable_ref):
# Add a new trace, allocating a new version for the variable, and
# remember the delete of the current
variable = variable_ref.getVariable()
version = variable_ref.getVariableVersion()
old_trace = self.getVariableCurrentTrace(variable)
variable_trace = VariableTraceUninit(
owner = self.owner,
variable = variable,
version = version,
previous = old_trace
)
# Assign to not initialized again.
self.addVariableTrace(
variable = variable,
version = version,
trace = variable_trace
)
# Make references point to it.
self.markCurrentVariableTrace(variable, version)
def onLocalsUsage(self):
for variable in self.getActiveVariables():
# TODO: Currently this is a bit difficult to express in a positive
# way, but we want to have only local variables.
if not variable.isTempVariable() and \
not variable.isModuleVariable():
variable_trace = self.getVariableCurrentTrace(
variable
)
variable_trace.addNameUsage()
def onVariableRelease(self, variable):
current = self.getVariableCurrentTrace(variable)
# Annotate that releases to the trace, it may be important knowledge.
current.addRelease()
return current
def onVariableContentEscapes(self, variable):
self.getVariableCurrentTrace(variable).onValueEscape()
def onExpression(self, expression, allow_none = False):
if expression is None and allow_none:
return None
assert expression.isExpression(), expression
assert expression.parent, expression
# Now compute this expression, allowing it to replace itself with
# something else as part of a local peep hole optimization.
r = expression.computeExpressionRaw(
trace_collection = self
)
assert type(r) is tuple, expression
new_node, change_tags, change_desc = r
if change_tags is not None:
# This is mostly for tracing and indication that a change occurred
# and it may be interesting to look again.
self.signalChange(
change_tags,
expression.getSourceReference(),
change_desc
)
if new_node is not expression:
expression.replaceWith(new_node)
if new_node.isExpressionVariableRef() or \
new_node.isExpressionTempVariableRef():
# Remember the reference for constraint collection.
assert new_node.variable_trace.hasDefiniteUsages()
# We add variable reference nodes late to their traces, only after they
# are actually produced, and not resolved to something else, so we do
# not have them dangling, and the code complexity inside of their own
# "computeExpression" functions.
if new_node.isExpressionVariableRef() or \
new_node.isExpressionTempVariableRef():
if new_node.getVariable().isMaybeLocalVariable():
variable_trace = self.getVariableCurrentTrace(
variable = new_node.getVariable().getMaybeVariable()
)
variable_trace.addUsage()
return new_node
def onStatement(self, statement):
try:
assert statement.isStatement(), statement
new_statement, change_tags, change_desc = \
statement.computeStatement(self)
# print new_statement, change_tags, change_desc
if new_statement is not statement:
self.signalChange(
change_tags,
statement.getSourceReference(),
change_desc
)
return new_statement
except Exception:
Tracing.printError(
"Problem with statement at %s:" %
statement.getSourceReference().getAsString()
)
raise
def mergeBranches(self, collection_yes, collection_no):
""" Merge two alternative branches into this trace.
This is mostly for merging conditional branches, or other ways
of having alternative control flow. This deals with up to two
alternative branches to both change this collection.
"""
# Refuse to do stupid work
if collection_yes is None and collection_no is None:
pass
elif collection_yes is None or collection_no is None:
# Handle one branch case, we need to merge versions backwards as
# they may make themselves obsolete.
self.mergeMultipleBranches(
collections = (self, collection_yes or collection_no)
)
else:
self.mergeMultipleBranches(
collections = (collection_yes,collection_no)
)
def mergeMultipleBranches(self, collections):
assert len(collections) > 0
# Optimize for length 1, which is trivial merge and needs not a
# lot of work.
if len(collections) == 1:
self.replaceBranch(collections[0])
return
variable_versions = {}
for collection in collections:
for variable, version in iterItems(collection.variable_actives):
if variable not in variable_versions:
variable_versions[variable] = set([version])
else:
variable_versions[variable].add(version)
for collection in collections:
for variable, versions in iterItems(variable_versions):
if variable not in collection.variable_actives:
versions.add(0)
self.variable_actives = {}
for variable, versions in iterItems(variable_versions):
if len(versions) == 1:
version, = versions
else:
version = self.addVariableMergeMultipleTrace(
variable = variable,
traces = [
self.getVariableTrace(variable, version)
for version in
versions
]
)
self.markCurrentVariableTrace(variable, version)
def replaceBranch(self, collection_replace):
self.variable_actives.update(collection_replace.variable_actives)
collection_replace.variable_actives = None
def onLoopBreak(self, collection = None):
if collection is None:
collection = self
return self.parent.onLoopBreak(collection)
def onLoopContinue(self, collection = None):
if collection is None:
collection = self
return self.parent.onLoopContinue(collection)
def onFunctionReturn(self, collection = None):
if collection is None:
collection = self
return self.parent.onFunctionReturn(collection)
def onExceptionRaiseExit(self, raisable_exceptions, collection = None):
if collection is None:
collection = self
return self.parent.onExceptionRaiseExit(raisable_exceptions, collection)
def getLoopBreakCollections(self):
return self.parent.getLoopBreakCollections()
def getLoopContinueCollections(self):
return self.parent.getLoopContinueCollections()
def getFunctionReturnCollections(self):
return self.parent.getFunctionReturnCollections()
def getExceptionRaiseCollections(self):
return self.parent.getExceptionRaiseCollections()
def makeAbortStackContext(self, catch_breaks, catch_continues,
catch_returns, catch_exceptions):
return self.parent.makeAbortStackContext(
catch_breaks = catch_breaks,
catch_continues = catch_continues,
catch_returns = catch_returns,
catch_exceptions = catch_exceptions
)
def getCompileTimeComputationResult(self, node, computation, description):
new_node, change_tags, message = getComputationResult(
node = node,
computation = computation,
description = description
)
if change_tags == "new_raise":
self.onExceptionRaiseExit(BaseException)
return new_node, change_tags, message
def getIteratorNextCount(self, iter_node):
return self.value_states.get(iter_node, None)
def initIteratorValue(self, iter_node):
# TODO: More complex state information will be needed eventually.
self.value_states[iter_node] = 0
def onIteratorNext(self, iter_node):
if iter_node in self.value_states:
self.value_states[iter_node] += 1
def resetValueStates(self):
for key in self.value_states:
self.value_states[key] = None
class ValueTraceBase(object):
# We are going to have many instance attributes, but should strive to minimize, as
# there is going to be a lot of fluctuation in these objects.
__slots__ = (
"owner",
"usage_count",
"name_usage_count",
"merge_usage_count",
"closure_usages",
"previous",
)
@counted_init
def __init__(self, owner, previous):
self.owner = owner
# Definite usage indicator.
self.usage_count = 0
# If 0, this indicates, the variable name needs to be assigned as name.
self.name_usage_count = 0
# If 0, this indicates no value merges happened on the value.
self.merge_usage_count = 0
self.closure_usages = False
# Previous trace this is replacing.
self.previous = previous
if isCountingInstances():
__del__ = counted_del()
def __repr__(self):
return "<%s of %s>" % (self.__class__.__name__,
self.owner.getCodeName())
def getOwner(self):
return self.owner
@staticmethod
def isLoopTrace():
return False
def addUsage(self):
self.usage_count += 1
def addNameUsage(self):
self.usage_count += 1
self.name_usage_count += 1
if self.name_usage_count <= 2 and self.previous is not None:
self.previous.addNameUsage()
def addMergeUsage(self):
self.usage_count += 1
self.merge_usage_count += 1
def getUsageCount(self):
return self.usage_count
def getNameUsageCount(self):
return self.name_usage_count
def getMergeUsageCount(self):
return self.merge_usage_count
def getMergeOrNameUsageCount(self):
return self.merge_usage_count + self.name_usage_count
def getPrevious(self):
return self.previous
@staticmethod
def isAssignTrace():
return False
@staticmethod
def isUnassignedTrace():
return False
@staticmethod
def isDeletedTrace():
return False
@staticmethod
def isUninitTrace():
return False
@staticmethod
def isInitTrace():
return False
@staticmethod
def isUnknownTrace():
return False
@staticmethod
def isMergeTrace():
return False
def mustHaveValue(self):
"""Will this definitely have a value.
Every trace has this overloaded.
"""
assert False, self
def mustNotHaveValue(self):
"""Will this definitely have a value.
Every trace has this overloaded.
"""
assert False, self
def getReplacementNode(self, usage):
# Virtual method, pylint: disable=no-self-use,unused-argument
return None
@staticmethod
def hasShapeDictionaryExact():
return False
@staticmethod
def getTruthValue():
return None
class CodeObjectSpec(object):
# One attribute for each code object aspect, and even flags,
# pylint: disable=too-many-arguments,too-many-instance-attributes
__slots__ = (
"co_name",
"co_kind",
"co_varnames",
"co_argcount",
"co_freevars",
"co_posonlyargcount",
"co_kwonlyargcount",
"co_has_starlist",
"co_has_stardict",
"filename",
"line_number",
"future_spec",
"new_locals",
"is_optimized",
)
@counted_init
def __init__(
self,
co_name,
co_kind,
co_varnames,
co_freevars,
co_argcount,
co_posonlyargcount,
co_kwonlyargcount,
co_has_starlist,
co_has_stardict,
co_filename,
co_lineno,
future_spec,
co_new_locals=None,
co_is_optimized=None,
):
# pylint: disable=I0021,too-many-locals
self.co_name = co_name
self.co_kind = co_kind
self.future_spec = future_spec
assert future_spec
# Strings happens from XML parsing, make sure to convert them.
if type(co_varnames) is str:
if co_varnames == "":
co_varnames = ()
else:
co_varnames = co_varnames.split(",")
if type(co_freevars) is str:
if co_freevars == "":
co_freevars = ()
else:
co_freevars = co_freevars.split(",")
if type(co_has_starlist) is not bool:
co_has_starlist = co_has_starlist != "False"
if type(co_has_stardict) is not bool:
co_has_stardict = co_has_stardict != "False"
self.co_varnames = tuple(co_varnames)
self.co_freevars = tuple(co_freevars)
self.co_argcount = int(co_argcount)
self.co_posonlyargcount = int(co_posonlyargcount)
self.co_kwonlyargcount = int(co_kwonlyargcount)
self.co_has_starlist = co_has_starlist
self.co_has_stardict = co_has_stardict
self.filename = co_filename
self.line_number = int(co_lineno)
if type(co_has_starlist) is not bool:
co_new_locals = co_new_locals != "False"
if type(co_has_starlist) is not bool:
co_is_optimized = co_is_optimized != "False"
self.new_locals = co_new_locals
self.is_optimized = co_is_optimized
__del__ = counted_del()
def __repr__(self):
return ("""\
<CodeObjectSpec %(co_kind)s '%(co_name)s' with %(co_varnames)r>""" %
self.getDetails())
def getDetails(self):
return {
"co_name": self.co_name,
"co_kind": self.co_kind,
"co_varnames": ",".join(self.co_varnames),
"co_freevars": ",".join(self.co_freevars),
"co_argcount": self.co_argcount,
"co_posonlyargcount": self.co_posonlyargcount,
"co_kwonlyargcount": self.co_kwonlyargcount,
"co_has_starlist": self.co_has_starlist,
"co_has_stardict": self.co_has_stardict,
"co_filename": self.filename,
"co_lineno": self.line_number,
"co_new_locals": self.new_locals,
"co_is_optimized": self.is_optimized,
"code_flags": ",".join(self.future_spec.asFlags()),
}
def getCodeObjectKind(self):
return self.co_kind
def updateLocalNames(self, local_names, freevar_names):
"""Move detected local variables after closure has been decided."""
self.co_varnames += tuple(
local_name for local_name in local_names
if local_name not in self.co_varnames
# TODO: This is actually a bug, but we have a hard time without it to know
# frame locals easily. We use this in compiled function run time, that all
# variables, including closure variables are found there. This would have to
# be cleaned up, for potentially little gain.
# if local_name not in freevar_names
)
self.co_freevars = tuple(freevar_names)
def removeFreeVarname(self, freevar_name):
self.co_freevars = tuple(var_name for var_name in self.co_freevars
if var_name != freevar_name)
def setFlagIsOptimizedValue(self, value):
self.is_optimized = value
def getFlagIsOptimizedValue(self):
return self.is_optimized
def setFlagNewLocalsValue(self, value):
self.new_locals = value
def getFlagNewLocalsValue(self):
return self.new_locals
def getFutureSpec(self):
return self.future_spec
def getVarNames(self):
return self.co_varnames
def getFreeVarNames(self):
return self.co_freevars
def getArgumentCount(self):
return self.co_argcount
def getPosOnlyParameterCount(self):
return self.co_posonlyargcount
def getKwOnlyParameterCount(self):
return self.co_kwonlyargcount
def getCodeObjectName(self):
return self.co_name
def hasStarListArg(self):
return self.co_has_starlist
def hasStarDictArg(self):
return self.co_has_stardict
def getFilename(self):
return self.filename
def getLineNumber(self):
return self.line_number
class PythonContextBase(getMetaClassBase("Context")):
@counted_init
def __init__(self):
self.source_ref = None
self.current_source_ref = None
if isCountingInstances():
__del__ = counted_del()
def getCurrentSourceCodeReference(self):
return self.current_source_ref
def setCurrentSourceCodeReference(self, value):
result = self.current_source_ref
self.current_source_ref = value
return result
@contextmanager
def withCurrentSourceCodeReference(self, value):
old_source_ref = self.setCurrentSourceCodeReference(value)
yield old_source_ref
self.setCurrentSourceCodeReference(value)
def getInplaceLeftName(self):
return self.allocateTempName("inplace_orig", "PyObject *", True)
@abstractmethod
def getConstantCode(self, constant, deep_check=False):
pass
@abstractmethod
def getModuleCodeName(self):
pass
@abstractmethod
def getModuleName(self):
pass
@abstractmethod
def addHelperCode(self, key, code):
pass
@abstractmethod
def hasHelperCode(self, key):
pass
@abstractmethod
def addDeclaration(self, key, code):
pass
@abstractmethod
def pushFrameVariables(self, frame_variables):
pass
@abstractmethod
def popFrameVariables(self):
pass
@abstractmethod
def getFrameVariableTypeDescriptions(self):
pass
@abstractmethod
def getFrameVariableTypeDescription(self):
pass
@abstractmethod
def getFrameTypeDescriptionDeclaration(self):
pass
@abstractmethod
def getFrameVariableCodeNames(self):
pass
@abstractmethod
def allocateTempName(self, base_name, type_name="PyObject *", unique=False):
pass
@abstractmethod
def skipTempName(self, base_name):
pass
@abstractmethod
def getIntResName(self):
pass
@abstractmethod
def getBoolResName(self):
pass
@abstractmethod
def hasTempName(self, base_name):
pass
@abstractmethod
def getExceptionEscape(self):
pass
@abstractmethod
def setExceptionEscape(self, label):
pass
@abstractmethod
def getLoopBreakTarget(self):
pass
@abstractmethod
def setLoopBreakTarget(self, label):
pass
@abstractmethod
def getLoopContinueTarget(self):
pass
@abstractmethod
def setLoopContinueTarget(self, label):
pass
@abstractmethod
def allocateLabel(self, label):
pass
@abstractmethod
def allocateExceptionKeeperVariables(self):
pass
@abstractmethod
def getExceptionKeeperVariables(self):
pass
@abstractmethod
def setExceptionKeeperVariables(self, keeper_vars):
pass
@abstractmethod
def addExceptionPreserverVariables(self, count):
pass
@abstractmethod
def getTrueBranchTarget(self):
pass
@abstractmethod
def getFalseBranchTarget(self):
pass
@abstractmethod
def setTrueBranchTarget(self, label):
pass
@abstractmethod
def setFalseBranchTarget(self, label):
pass
@abstractmethod
def getCleanupTempnames(self):
pass
@abstractmethod
def addCleanupTempName(self, tmp_name):
pass
@abstractmethod
def removeCleanupTempName(self, tmp_name):
pass
@abstractmethod
def needsCleanup(self, tmp_name):
pass
@abstractmethod
def pushCleanupScope(self):
pass
@abstractmethod
def popCleanupScope(self):
pass
class Variable(getMetaClassBase("Variable")):
# We will need all of these attributes, since we track the global
# state and cache some decisions as attributes. TODO: But in some
# cases, part of the these might be moved to the outside.
__slots__ = (
"variable_name",
"owner",
"version_number",
"shared_users",
"traces",
"users",
"writers",
)
@counted_init
def __init__(self, owner, variable_name):
assert type(variable_name) is str, variable_name
assert type(owner) not in (tuple, list), owner
self.variable_name = variable_name
self.owner = owner
self.version_number = 0
self.shared_users = False
self.traces = set()
# Derived from all traces.
self.users = None
self.writers = None
if isCountingInstances():
__del__ = counted_del()
def finalize(self):
del self.users
del self.writers
del self.traces
del self.owner
def __repr__(self):
return "<%s '%s' of '%s'>" % (
self.__class__.__name__,
self.variable_name,
self.owner.getName(),
)
@abstractmethod
def getVariableType(self):
pass
def getDescription(self):
return "variable '%s'" % self.variable_name
def getName(self):
return self.variable_name
def getOwner(self):
return self.owner
def getEntryPoint(self):
return self.owner.getEntryPoint()
def getCodeName(self):
var_name = self.variable_name
var_name = var_name.replace(".", "$")
var_name = Utils.encodeNonAscii(var_name)
return var_name
def allocateTargetNumber(self):
self.version_number += 1
return self.version_number
@staticmethod
def isLocalVariable():
return False
@staticmethod
def isParameterVariable():
return False
@staticmethod
def isModuleVariable():
return False
@staticmethod
def isIncompleteModuleVariable():
return False
@staticmethod
def isTempVariable():
return False
@staticmethod
def isTempVariableBool():
return False
@staticmethod
def isLocalsDictVariable():
return False
def addVariableUser(self, user):
# Update the shared scopes flag.
if user is not self.owner:
self.shared_users = True
# These are not really scopes, just shared uses.
if (user.isExpressionGeneratorObjectBody()
or user.isExpressionCoroutineObjectBody()
or user.isExpressionAsyncgenObjectBody()):
if self.owner is user.getParentVariableProvider():
return
_variables_in_shared_scopes.add(self)
def isSharedTechnically(self):
if not self.shared_users:
return False
if not self.users:
return False
owner = self.owner.getEntryPoint()
for user in self.users:
user = user.getEntryPoint()
while user is not owner and (
(user.isExpressionFunctionBody() and not user.needsCreation())
or user.isExpressionClassBody()):
user = user.getParentVariableProvider()
if user is not owner:
return True
return False
def addTrace(self, variable_trace):
self.traces.add(variable_trace)
def removeTrace(self, variable_trace):
self.traces.remove(variable_trace)
def updateUsageState(self):
writers = set()
users = set()
for trace in self.traces:
owner = trace.owner
users.add(owner)
if trace.isAssignTrace():
writers.add(owner)
elif trace.isDeletedTrace() and owner is not self.owner:
writers.add(owner)
self.writers = writers
self.users = users
def hasAccessesOutsideOf(self, provider):
if not self.owner.locals_scope.complete:
return None
elif self.users is None:
return False
elif provider in self.users:
return len(self.users) > 1
else:
return bool(self.users)
def getMatchingAssignTrace(self, assign_node):
for trace in self.traces:
if trace.isAssignTrace() and trace.getAssignNode() is assign_node:
return trace
return None
def getMatchingDelTrace(self, del_node):
for trace in self.traces:
if trace.isDeletedTrace() and trace.getDelNode() is del_node:
return trace
return None
def getTypeShapes(self):
result = set()
for trace in self.traces:
if trace.isAssignTrace():
result.add(trace.getAssignNode().getTypeShape())
elif trace.isUnknownTrace():
result.add(tshape_unknown)
elif trace.isInitTrace():
result.add(tshape_unknown)
elif trace.isUnassignedTrace():
pass
elif trace.isMergeTrace():
pass
# TODO: Remove this and be not unknown.
elif trace.isLoopTrace():
trace.getTypeShape().emitAlternatives(result.add)
else:
assert False, trace
return result
class NodeBase(NodeMetaClassBase):
__slots__ = "parent", "source_ref", "effective_source_ref"
# String to identify the node class, to be consistent with its name.
kind = None
@counted_init
def __init__(self, source_ref):
# The base class has no __init__ worth calling.
# Check source reference to meet basic standards, so we note errors
# when they occur.
assert source_ref is not None
assert source_ref.line is not None
self.parent = None
self.source_ref = source_ref
__del__ = counted_del()
def __repr__(self):
return "<Node %s>" % (self.getDescription())
def getDescription(self):
""" Description of the node, intended for use in __repr__ and
graphical display.
"""
details = self.getDetails()
if details:
return "'%s' with %s" % (self.kind, self.getDetails())
else:
return "'%s'" % self.kind
def getDetails(self):
""" Details of the node, intended for re-creation.
We are not using the pickle mechanisms, but this is basically
part of what the constructor call needs. Real children will
also be added.
"""
# Virtual method, pylint: disable=no-self-use
return {}
def getDetailsForDisplay(self):
""" Details of the node, intended for use in __repr__ and dumps.
This is also used for XML.
"""
return self.getDetails()
def getDetail(self):
""" Details of the node, intended for use in __repr__ and graphical
display.
"""
return str(self.getDetails())[1:-1]
def makeClone(self):
try:
# Using star dictionary arguments here for generic use.
result = self.__class__(source_ref=self.source_ref,
**self.getDetails())
except TypeError:
print("Problem cloning", self.__class__)
raise
effective_source_ref = self.getCompatibleSourceReference()
if effective_source_ref is not self.source_ref:
result.setCompatibleSourceReference(effective_source_ref)
return result
def makeCloneAt(self, source_ref):
result = self.makeClone()
result.source_ref = source_ref
return result
def getParent(self):
""" Parent of the node. Every node except modules have to have a parent.
"""
if self.parent is None and not self.isCompiledPythonModule():
# print self.getVisitableNodesNamed()
assert False, (self, self.source_ref)
return self.parent
def getChildName(self):
""" Return the role in the current parent, subject to changes.
"""
parent = self.getParent()
for key in parent.named_children:
value = parent.getChild(key)
if self is value:
return key
if type(value) is tuple:
if self in value:
return key, value.index(self)
# TODO: Not checking tuples yet
return None
def getChildNameNice(self):
child_name = self.getChildName()
if hasattr(self.parent, "nice_children"):
return self.parent.nice_children[self.parent.named_children.index(
child_name)]
else:
return child_name
def getParentFunction(self):
""" Return the parent that is a function.
"""
parent = self.getParent()
while parent is not None and not parent.isExpressionFunctionBodyBase():
parent = parent.getParent()
return parent
def getParentModule(self):
""" Return the parent that is module.
"""
parent = self
while not parent.isCompiledPythonModule():
if hasattr(parent, "provider"):
# After we checked, we can use it, will be much faster route
# to take.
parent = parent.provider
else:
parent = parent.getParent()
return parent
def isParentVariableProvider(self):
# Check if it's a closure giver, in which cases it can provide variables,
return isinstance(self, ClosureGiverNodeMixin)
def getParentVariableProvider(self):
parent = self.getParent()
while not parent.isParentVariableProvider():
parent = parent.getParent()
return parent
def getParentReturnConsumer(self):
parent = self.getParent()
while not parent.isParentVariableProvider() and \
not parent.isExpressionOutlineBody():
parent = parent.getParent()
return parent
def getParentStatementsFrame(self):
current = self.getParent()
while True:
if current.isStatementsFrame():
return current
if current.isParentVariableProvider():
return None
if current.isExpressionOutlineBody():
return None
current = current.getParent()
def getSourceReference(self):
return self.source_ref
def setCompatibleSourceReference(self, source_ref):
""" Bug compatible line numbers information.
As CPython outputs the last bit of bytecode executed, and not the
line of the operation. For example calls, output the line of the
last argument, as opposed to the line of the operation start.
For tests, we wants to be compatible. In improved more, we are
not being fully compatible, and just drop it altogether.
"""
# Getting the same source reference can be dealt with quickly, so do
# this first.
if self.source_ref is not source_ref and \
Options.isFullCompat() and \
self.source_ref != source_ref:
# An attribute outside of "__init__", so we save one memory for the
# most cases. Very few cases involve splitting across lines.
# pylint: disable=W0201
self.effective_source_ref = source_ref
def getCompatibleSourceReference(self):
""" Bug compatible line numbers information.
See above.
"""
return getattr(self, "effective_source_ref", self.source_ref)
def asXml(self):
line = self.getSourceReference().getLineNumber()
result = TreeXML.Element("node",
kind=self.__class__.__name__,
line="%s" % line)
compat_line = self.getCompatibleSourceReference().getLineNumber()
if compat_line != line:
result.attrib["compat_line"] = str(compat_line)
for key, value in iterItems(self.getDetailsForDisplay()):
result.set(key, str(value))
for name, children in self.getVisitableNodesNamed():
role = TreeXML.Element("role", name=name)
result.append(role)
if children is None:
role.attrib["type"] = "none"
elif type(children) not in (list, tuple):
role.append(children.asXml())
else:
role.attrib["type"] = "list"
for child in children:
role.append(child.asXml())
return result
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Only some things need a provider, pylint: disable=unused-argument
return cls(source_ref=source_ref, **args)
def asXmlText(self):
xml = self.asXml()
return TreeXML.toString(xml)
def dump(self, level=0):
Tracing.printIndented(level, self)
Tracing.printSeparator(level)
for visitable in self.getVisitableNodes():
visitable.dump(level + 1)
Tracing.printSeparator(level)
@staticmethod
def isStatementsFrame():
return False
@staticmethod
def isCompiledPythonModule():
# For overload by module nodes
return False
def isExpression(self):
return self.kind.startswith("EXPRESSION_")
def isStatement(self):
return self.kind.startswith("STATEMENT_")
def isExpressionBuiltin(self):
return self.kind.startswith("EXPRESSION_BUILTIN_")
@staticmethod
def isExpressionConstantRef():
return False
@staticmethod
def isExpressionOperationBinary():
return False
def isExpressionSideEffects(self):
# Virtual method, pylint: disable=no-self-use
# We need to provide this, as these node kinds are only imported if
# necessary, but we test against them.
return False
def isStatementReraiseException(self):
# Virtual method, pylint: disable=no-self-use
return False
def isExpressionMakeSequence(self):
# Virtual method, pylint: disable=no-self-use
return False
def isNumberConstant(self):
# Virtual method, pylint: disable=no-self-use
return False
def isExpressionCall(self):
# Virtual method, pylint: disable=no-self-use
return False
def visit(self, context, visitor):
visitor(self)
for visitable in self.getVisitableNodes():
visitable.visit(context, visitor)
def getVisitableNodes(self):
# Virtual method, pylint: disable=no-self-use
return ()
def getVisitableNodesNamed(self):
""" Named children dictionary.
For use in debugging and XML output.
"""
# Virtual method, pylint: disable=no-self-use
return ()
def replaceWith(self, new_node):
self.parent.replaceChild(old_node=self, new_node=new_node)
def getName(self):
# Virtual method, pylint: disable=no-self-use
return None
def mayHaveSideEffects(self):
""" Unless we are told otherwise, everything may have a side effect. """
# Virtual method, pylint: disable=no-self-use
return True
def isOrderRelevant(self):
return self.mayHaveSideEffects()
def extractSideEffects(self):
""" Unless defined otherwise, the expression is the side effect. """
return (self, )
def mayRaiseException(self, exception_type):
""" Unless we are told otherwise, everything may raise everything. """
# Virtual method, pylint: disable=no-self-use,unused-argument
return True
def mayReturn(self):
return "_RETURN" in self.kind
def mayBreak(self):
# For overload, pylint: disable=no-self-use
return False
def mayContinue(self):
# For overload, pylint: disable=no-self-use
return False
def needsFrame(self):
""" Unless we are tolder otherwise, this depends on exception raise. """
return self.mayRaiseException(BaseException)
def willRaiseException(self, exception_type):
""" Unless we are told otherwise, nothing may raise anything. """
# Virtual method, pylint: disable=no-self-use,unused-argument
return False
def isStatementAborting(self):
""" Is the node aborting, control flow doesn't continue after this node. """
assert self.isStatement(), self.kind
return False
def needsLocalsDict(self):
""" Node requires a locals dictionary by provider. """
# Virtual method, pylint: disable=no-self-use
return False
class LocalsDictHandleBase(object):
__slots__ = (
"locals_name",
# TODO: Specialize what the kinds really use.
"variables",
"local_variables",
"providing",
"mark_for_propagation",
"propagation",
"owner",
)
@counted_init
def __init__(self, locals_name, owner):
self.locals_name = locals_name
self.owner = owner
# For locals dict variables in this scope.
self.variables = {}
# For local variables in this scope.
self.local_variables = {}
self.providing = OrderedDict()
# Can this be eliminated through replacement of temporary variables
self.mark_for_propagation = False
self.propagation = None
__del__ = counted_del()
def __repr__(self):
return "<%s of %s>" % (self.__class__.__name__, self.locals_name)
def getName(self):
return self.locals_name
def makeClone(self, new_owner):
count = 1
# Make it unique.
while 1:
locals_name = self.locals_name + "_inline_%d" % count
if locals_name not in locals_dict_handles:
break
count += 1
result = self.__class__(locals_name=locals_name, owner=new_owner)
variable_translation = {}
# Clone variables as well.
for variable_name, variable in self.variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.variables[variable_name] = new_variable
for variable_name, variable in self.local_variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.local_variables[variable_name] = new_variable
result.providing = OrderedDict()
for variable_name, variable in self.providing.items():
if variable in variable_translation:
new_variable = variable_translation[variable]
else:
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.providing[variable_name] = new_variable
return result, variable_translation
@staticmethod
def getTypeShape():
return tshape_dict
def getCodeName(self):
return self.locals_name
@staticmethod
def isModuleScope():
return False
@staticmethod
def isClassScope():
return False
@staticmethod
def isFunctionScope():
return False
def getProvidedVariables(self):
return self.providing.values()
def registerProvidedVariable(self, variable):
variable_name = variable.getName()
self.providing[variable_name] = variable
def unregisterProvidedVariable(self, variable):
""" Remove provided variable, e.g. because it became unused. """
variable_name = variable.getName()
if variable_name in self.providing:
del self.providing[variable_name]
registerClosureVariable = registerProvidedVariable
unregisterClosureVariable = unregisterProvidedVariable
def hasProvidedVariable(self, variable_name):
""" Test if a variable is provided. """
return variable_name in self.providing
def getProvidedVariable(self, variable_name):
""" Test if a variable is provided. """
return self.providing[variable_name]
def getLocalsRelevantVariables(self):
""" The variables relevant to locals. """
return self.providing.values()
def getLocalsDictVariable(self, variable_name):
if variable_name not in self.variables:
result = Variables.LocalsDictVariable(
owner=self, variable_name=variable_name
)
self.variables[variable_name] = result
return self.variables[variable_name]
# TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here.
def getLocalVariable(self, owner, variable_name):
if variable_name not in self.local_variables:
result = Variables.LocalVariable(owner=owner, variable_name=variable_name)
self.local_variables[variable_name] = result
return self.local_variables[variable_name]
def markForLocalsDictPropagation(self):
self.mark_for_propagation = True
def isMarkedForPropagation(self):
return self.mark_for_propagation
def allocateTempReplacementVariable(self, trace_collection, variable_name):
if self.propagation is None:
self.propagation = OrderedDict()
if variable_name not in self.propagation:
provider = trace_collection.getOwner()
self.propagation[variable_name] = provider.allocateTempVariable(
temp_scope=None, name=self.getCodeName() + "_key_" + variable_name
)
return self.propagation[variable_name]
def getPropagationVariables(self):
if self.propagation is None:
return ()
return self.propagation
def finalize(self):
# Make it unusable when it's become empty, not used.
self.owner.locals_scope = None
del self.owner
del self.propagation
del self.mark_for_propagation
for variable in self.variables.values():
variable.finalize()
for variable in self.local_variables.values():
variable.finalize()
del self.variables
del self.providing
class PythonContextBase(ContextMetaClassBase):
@counted_init
def __init__(self):
self.source_ref = None
self.current_source_ref = None
self.last_source_ref = None
__del__ = counted_del()
def getCurrentSourceCodeReference(self):
return self.current_source_ref
def setCurrentSourceCodeReference(self, value):
result = self.current_source_ref
self.current_source_ref = value
if value is not None:
self.last_source_ref = result
return result
def getLastSourceCodeReference(self):
result = self.last_source_ref
# self.last_source_ref = None
return result
def isUsed(self, tmp_name):
if tmp_name.startswith("tmp_unused_"):
return False
else:
return True
@abstractmethod
def getConstantCode(self, constant):
pass
@abstractmethod
def getModuleCodeName(self):
pass
@abstractmethod
def getModuleName(self):
pass
@abstractmethod
def addHelperCode(self, key, code):
pass
@abstractmethod
def hasHelperCode(self, key):
pass
@abstractmethod
def addDeclaration(self, key, code):
pass
@abstractmethod
def pushFrameVariables(self, frame_variables):
pass
@abstractmethod
def popFrameVariables(self):
pass
@abstractmethod
def getFrameVariableTypeDescriptions(self):
pass
@abstractmethod
def getFrameVariableTypeDescription(self):
pass
@abstractmethod
def getFrameVariableTypeDescriptionName(self):
pass
@abstractmethod
def getFrameVariableCodeNames(self):
pass
@abstractmethod
def getLocalsDictName(self):
pass
@abstractmethod
def allocateLocalsDictName(self):
pass
@abstractmethod
def endLocalsDictName(self):
pass
@abstractmethod
def allocateTempName(self,
base_name,
type_name="PyObject *",
unique=False):
pass
@abstractmethod
def getIntResName(self):
pass
@abstractmethod
def getBoolResName(self):
pass
@abstractmethod
def hasTempName(self, base_name):
pass
@abstractmethod
def forgetTempName(self, tmp_name):
pass
@abstractmethod
def getTempNameInfos(self):
pass
@abstractmethod
def getExceptionEscape(self):
pass
@abstractmethod
def setExceptionEscape(self, label):
pass
@abstractmethod
def getLoopBreakTarget(self):
pass
@abstractmethod
def setLoopBreakTarget(self, label):
pass
@abstractmethod
def getLoopContinueTarget(self):
pass
@abstractmethod
def setLoopContinueTarget(self, label):
pass
@abstractmethod
def allocateLabel(self, label):
pass
@abstractmethod
def needsExceptionVariables(self):
pass
@abstractmethod
def markAsNeedsExceptionVariables(self):
pass
@abstractmethod
def allocateExceptionKeeperVariables(self):
pass
@abstractmethod
def getExceptionKeeperVariables(self):
pass
@abstractmethod
def setExceptionKeeperVariables(self, keeper_vars):
pass
@abstractmethod
def addExceptionPreserverVariables(self, count):
pass
@abstractmethod
def getTrueBranchTarget(self):
pass
@abstractmethod
def getFalseBranchTarget(self):
pass
@abstractmethod
def setTrueBranchTarget(self, label):
pass
@abstractmethod
def setFalseBranchTarget(self, label):
pass
@abstractmethod
def getCleanupTempnames(self):
pass
@abstractmethod
def addCleanupTempName(self, tmp_name):
pass
@abstractmethod
def removeCleanupTempName(self, tmp_name):
pass
@abstractmethod
def needsCleanup(self, tmp_name):
pass
@abstractmethod
def pushCleanupScope(self):
pass
@abstractmethod
def popCleanupScope(self):
pass
class FutureSpec:
@counted_init
def __init__(self):
self.future_division = _future_division_default
self.unicode_literals = False
self.absolute_import = _future_absolute_import_default
self.future_print = False
self.barry_bdfl = False
self.generator_stop = _future_generator_stop_default
__del__ = counted_del()
def clone(self):
result = FutureSpec()
result.future_division = self.future_division
result.unicode_literals = self.unicode_literals
result.absolute_import = self.absolute_import
result.future_print = self.future_print
result.barry_bdfl = self.barry_bdfl
return result
def isFutureDivision(self):
return self.future_division
def enableFutureDivision(self):
self.future_division = True
def enableFuturePrint(self):
self.future_print = True
def enableUnicodeLiterals(self):
self.unicode_literals = True
def enableAbsoluteImport(self):
self.absolute_import = True
def enableBarry(self):
self.barry_bdfl = True
def enableGeneratorStop(self):
self.generator_stop = True
def isAbsoluteImport(self):
return self.absolute_import
def isGeneratorStop(self):
return self.generator_stop
def asFlags(self):
""" Create a list of C identifiers to represent the flag values.
This is for use in code generation only.
"""
result = []
if self.future_division and python_version < 300:
result.append("CO_FUTURE_DIVISION")
if self.unicode_literals:
result.append("CO_FUTURE_UNICODE_LITERALS")
if self.absolute_import and python_version < 300:
result.append("CO_FUTURE_ABSOLUTE_IMPORT")
if self.future_print and python_version < 300:
result.append("CO_FUTURE_PRINT_FUNCTION")
if self.barry_bdfl and python_version >= 300:
result.append("CO_FUTURE_BARRY_AS_BDFL")
if self.generator_stop and python_version >= 350:
result.append("CO_FUTURE_GENERATOR_STOP")
return tuple(result)
class FutureSpec(object):
__slots__ = (
"future_division",
"unicode_literals",
"absolute_import",
"future_print",
"barry_bdfl",
"generator_stop",
"future_annotations",
)
@counted_init
def __init__(self):
self.future_division = _future_division_default
self.unicode_literals = False
self.absolute_import = _future_absolute_import_default
self.future_print = False
self.barry_bdfl = False
self.generator_stop = _future_generator_stop_default
self.future_annotations = _future_annotations_default
__del__ = counted_del()
def __repr__(self):
return "<FutureSpec %s>" % ",".join(self.asFlags())
def clone(self):
result = FutureSpec()
result.future_division = self.future_division
result.unicode_literals = self.unicode_literals
result.absolute_import = self.absolute_import
result.future_print = self.future_print
result.barry_bdfl = self.barry_bdfl
result.generator_stop = self.generator_stop
result.future_annotations = result.future_annotations
return result
def isFutureDivision(self):
return self.future_division
def enableFutureDivision(self):
self.future_division = True
def isFuturePrint(self):
return self.future_print
def enableFuturePrint(self):
self.future_print = True
def enableUnicodeLiterals(self):
self.unicode_literals = True
def enableAbsoluteImport(self):
self.absolute_import = True
def enableBarry(self):
self.barry_bdfl = True
def enableGeneratorStop(self):
self.generator_stop = True
def isAbsoluteImport(self):
return self.absolute_import
def isGeneratorStop(self):
return self.generator_stop
def enableFutureAnnotations(self):
self.future_annotations = True
def isFutureAnnotations(self):
return self.future_annotations
def asFlags(self):
""" Create a list of C identifiers to represent the flag values.
This is for use in code generation and to restore from
saved modules.
"""
result = []
if python_version < 300 and self.future_division:
result.append("CO_FUTURE_DIVISION")
if self.unicode_literals:
result.append("CO_FUTURE_UNICODE_LITERALS")
if python_version < 300 and self.absolute_import:
result.append("CO_FUTURE_ABSOLUTE_IMPORT")
if python_version < 300 and self.future_print:
result.append("CO_FUTURE_PRINT_FUNCTION")
if python_version >= 300 and self.barry_bdfl:
result.append("CO_FUTURE_BARRY_AS_BDFL")
if 350 <= python_version < 370 and self.generator_stop:
result.append("CO_FUTURE_GENERATOR_STOP")
if python_version >= 370 and self.future_annotations:
result.append("CO_FUTURE_ANNOTATIONS")
return tuple(result)
class SourceCodeReference(object):
# TODO: Measure the access speed impact of slots. The memory savings is
# not worth it (only a few percent).
__slots__ = ["filename", "line", "future_spec", "internal"]
@classmethod
def fromFilenameAndLine(cls, filename, line, future_spec):
result = cls()
result.filename = filename
result.line = line
result.future_spec = future_spec
return result
__del__ = counted_del()
@counted_init
def __init__(self):
self.line = None
self.filename = None
self.future_spec = None
self.internal = False
def __repr__(self):
return "<%s to %s:%s>" % (self.__class__.__name__, self.filename, self.line)
def __cmp__(self, other):
if other is None:
return -1
assert isinstance(other, SourceCodeReference), other
result = cmp(self.filename, other.filename)
if result == 0:
result = cmp(self.line, other.line)
if result == 0:
result = cmp(self.internal, other.internal)
return result
def _clone(self, line):
""" Make a copy it itself.
"""
result = SourceCodeReference.fromFilenameAndLine(
filename = self.filename,
line = line,
future_spec = self.future_spec
)
result.internal = self.internal
return result
def atInternal(self):
""" Make a copy it itself but mark as internal code.
Avoids useless copies, by returning an internal object again if
it is already internal.
"""
if not self.internal:
result = self._clone(self.line)
result.internal = True
return result
else:
return self
def atLineNumber(self, line):
""" Make a reference to the same file, but different line.
Avoids useless copies, by returning same object if the line is
the same.
"""
assert type(line) is int, line
if self.line != line:
return self._clone(line)
else:
return self
def getLineNumber(self):
return self.line
def getFilename(self):
return self.filename
def getFutureSpec(self):
return self.future_spec
def getAsString(self):
return "%s:%s" % (self.filename, self.line)
def isInternal(self):
return self.internal
class TraceCollectionBase(object):
"""This contains for logic for maintaining active traces.
They are kept for "variable" and versions.
"""
__slots__ = ("owner", "parent", "name", "value_states", "variable_actives")
if isCountingInstances():
__del__ = counted_del()
@counted_init
def __init__(self, owner, name, parent):
self.owner = owner
self.parent = parent
self.name = name
# Value state extra information per node.
self.value_states = {}
# Currently active values in the tracing.
self.variable_actives = {}
def __repr__(self):
return "<%s for %s at 0x%x>" % (self.__class__.__name__, self.name, id(self))
def getOwner(self):
return self.owner
def getVariableCurrentTrace(self, variable):
"""Get the current value trace associated to this variable
It is also created on the fly if necessary. We create them
lazy so to keep the tracing branches minimal where possible.
"""
return self.getVariableTrace(
variable=variable, version=self._getCurrentVariableVersion(variable)
)
def markCurrentVariableTrace(self, variable, version):
self.variable_actives[variable] = version
def _getCurrentVariableVersion(self, variable):
try:
return self.variable_actives[variable]
except KeyError:
# Initialize variables on the fly.
if not self.hasVariableTrace(variable, 0):
self.initVariable(variable)
self.markCurrentVariableTrace(variable, 0)
return self.variable_actives[variable]
def getActiveVariables(self):
return self.variable_actives.keys()
def markActiveVariableAsEscaped(self, variable):
current = self.getVariableCurrentTrace(variable=variable)
if not current.isUnknownTrace():
version = variable.allocateTargetNumber()
self.addVariableTrace(
variable=variable,
version=version,
trace=ValueTraceEscaped(owner=self.owner, previous=current),
)
self.markCurrentVariableTrace(variable, version)
def markActiveVariableAsLoopMerge(
self, loop_node, current, variable, shapes, incomplete
):
if incomplete:
result = ValueTraceLoopIncomplete(loop_node, current, shapes)
else:
# TODO: Empty is a missing optimization somewhere, but it also happens that
# a variable is getting released in a loop.
# assert shapes, (variable, current)
if not shapes:
shapes.add(tshape_uninit)
result = ValueTraceLoopComplete(loop_node, current, shapes)
version = variable.allocateTargetNumber()
self.addVariableTrace(variable=variable, version=version, trace=result)
self.markCurrentVariableTrace(variable, version)
return result
def markActiveVariablesAsUnknown(self):
for variable in self.getActiveVariables():
if variable.isTempVariable():
continue
self.markActiveVariableAsEscaped(variable)
@staticmethod
def signalChange(tags, source_ref, message):
# This is monkey patched from another module. pylint: disable=I0021,not-callable
signalChange(tags, source_ref, message)
def onUsedModule(self, module_name, module_relpath):
return self.parent.onUsedModule(module_name, module_relpath)
def onUsedFunction(self, function_body):
owning_module = function_body.getParentModule()
# Make sure the owning module is added to the used set. This is most
# important for helper functions, or modules, which otherwise have
# become unused.
addUsedModule(owning_module)
needs_visit = owning_module.addUsedFunction(function_body)
if needs_visit:
function_body.computeFunctionRaw(self)
@staticmethod
def mustAlias(a, b):
if a.isExpressionVariableRef() and b.isExpressionVariableRef():
return a.getVariable() is b.getVariable()
return False
@staticmethod
def mustNotAlias(a, b):
# TODO: not yet really implemented
if a.isExpressionConstantRef() and b.isExpressionConstantRef():
if a.isMutable() or b.isMutable():
return True
return False
def onControlFlowEscape(self, node):
# TODO: One day, we should trace which nodes exactly cause a variable
# to be considered escaped, pylint: disable=unused-argument
for variable in self.getActiveVariables():
if variable.isModuleVariable():
# print variable
self.markActiveVariableAsEscaped(variable)
elif variable.isLocalVariable():
if variable.hasAccessesOutsideOf(self.owner) is not False:
self.markActiveVariableAsEscaped(variable)
def removeKnowledge(self, node):
if node.isExpressionVariableRef():
self.markActiveVariableAsEscaped(node.variable)
def onValueEscapeStr(self, node):
# TODO: We can ignore these for now.
pass
def removeAllKnowledge(self):
self.markActiveVariablesAsUnknown()
def getVariableTrace(self, variable, version):
return self.parent.getVariableTrace(variable, version)
def hasVariableTrace(self, variable, version):
return self.parent.hasVariableTrace(variable, version)
def addVariableTrace(self, variable, version, trace):
self.parent.addVariableTrace(variable, version, trace)
def addVariableMergeMultipleTrace(self, variable, traces):
return self.parent.addVariableMergeMultipleTrace(variable, traces)
def onVariableSet(self, variable, version, assign_node):
variable_trace = ValueTraceAssign(
owner=self.owner,
assign_node=assign_node,
previous=self.getVariableCurrentTrace(variable=variable),
)
self.addVariableTrace(variable=variable, version=version, trace=variable_trace)
# Make references point to it.
self.markCurrentVariableTrace(variable, version)
return variable_trace
def onVariableDel(self, variable, version, del_node):
# Add a new trace, allocating a new version for the variable, and
# remember the delete of the current
old_trace = self.getVariableCurrentTrace(variable)
variable_trace = ValueTraceDeleted(
owner=self.owner, del_node=del_node, previous=old_trace
)
# Assign to not initialized again.
self.addVariableTrace(variable=variable, version=version, trace=variable_trace)
# Make references point to it.
self.markCurrentVariableTrace(variable, version)
return variable_trace
def onLocalsUsage(self, locals_scope):
self.onLocalsDictEscaped(locals_scope)
result = []
scope_locals_variables = locals_scope.getLocalsRelevantVariables()
for variable in self.getActiveVariables():
if variable.isLocalVariable() and variable in scope_locals_variables:
variable_trace = self.getVariableCurrentTrace(variable)
variable_trace.addNameUsage()
result.append((variable, variable_trace))
return result
def onVariableContentEscapes(self, variable):
self.markActiveVariableAsEscaped(variable)
def onExpression(self, expression, allow_none=False):
if expression is None and allow_none:
return None
assert expression.isExpression(), expression
parent = expression.parent
assert parent, expression
# Now compute this expression, allowing it to replace itself with
# something else as part of a local peep hole optimization.
r = expression.computeExpressionRaw(trace_collection=self)
assert type(r) is tuple, (expression, expression.getVisitableNodes(), r)
new_node, change_tags, change_desc = r
if change_tags is not None:
# This is mostly for tracing and indication that a change occurred
# and it may be interesting to look again.
self.signalChange(change_tags, expression.getSourceReference(), change_desc)
if new_node is not expression:
parent.replaceChild(expression, new_node)
return new_node
def onStatement(self, statement):
try:
assert statement.isStatement(), statement
new_statement, change_tags, change_desc = statement.computeStatement(self)
# print new_statement, change_tags, change_desc
if new_statement is not statement:
self.signalChange(
change_tags, statement.getSourceReference(), change_desc
)
return new_statement
except Exception:
Tracing.printError(
"Problem with statement at %s:\n-> %s"
% (
statement.source_ref.getAsString(),
readSourceLine(statement.source_ref),
)
)
raise
def computedStatementResult(self, statement, change_tags, change_desc):
"""Make sure the replacement statement is computed.
Use this when a replacement expression needs to be seen by the trace
collection and be computed, without causing any duplication, but where
otherwise there would be loss of annotated effects.
This may e.g. be true for nodes that need an initial run to know their
exception result and type shape.
"""
# Need to compute the replacement still.
new_statement = statement.computeStatement(self)
if new_statement[0] is not statement:
# Signal intermediate result as well.
self.signalChange(change_tags, statement.getSourceReference(), change_desc)
return new_statement
else:
return statement, change_tags, change_desc
def computedExpressionResult(self, expression, change_tags, change_desc):
"""Make sure the replacement expression is computed.
Use this when a replacement expression needs to be seen by the trace
collection and be computed, without causing any duplication, but where
otherwise there would be loss of annotated effects.
This may e.g. be true for nodes that need an initial run to know their
exception result and type shape.
"""
# Need to compute the replacement still.
new_expression = expression.computeExpression(self)
if new_expression[0] is not expression:
# Signal intermediate result as well.
self.signalChange(change_tags, expression.getSourceReference(), change_desc)
return new_expression
else:
return expression, change_tags, change_desc
def mergeBranches(self, collection_yes, collection_no):
"""Merge two alternative branches into this trace.
This is mostly for merging conditional branches, or other ways
of having alternative control flow. This deals with up to two
alternative branches to both change this collection.
"""
# Many branches due to inlining the actual merge and preparing it
# pylint: disable=too-many-branches
if collection_yes is None:
if collection_no is not None:
# Handle one branch case, we need to merge versions backwards as
# they may make themselves obsolete.
collection1 = self
collection2 = collection_no
else:
# Refuse to do stupid work
return
elif collection_no is None:
# Handle one branch case, we need to merge versions backwards as
# they may make themselves obsolete.
collection1 = self
collection2 = collection_yes
else:
# Handle two branch case, they may or may not do the same things.
collection1 = collection_yes
collection2 = collection_no
variable_versions = {}
for variable, version in iterItems(collection1.variable_actives):
variable_versions[variable] = version
for variable, version in iterItems(collection2.variable_actives):
if variable not in variable_versions:
variable_versions[variable] = 0, version
else:
other = variable_versions[variable]
if other != version:
variable_versions[variable] = other, version
else:
variable_versions[variable] = other
for variable in variable_versions:
if variable not in collection2.variable_actives:
variable_versions[variable] = variable_versions[variable], 0
self.variable_actives = {}
for variable, versions in iterItems(variable_versions):
if type(versions) is tuple:
version = self.addVariableMergeMultipleTrace(
variable=variable,
traces=(
self.getVariableTrace(variable, versions[0]),
self.getVariableTrace(variable, versions[1]),
),
)
else:
version = versions
self.markCurrentVariableTrace(variable, version)
def mergeMultipleBranches(self, collections):
assert collections
# Optimize for length 1, which is trivial merge and needs not a
# lot of work.
if len(collections) == 1:
self.replaceBranch(collections[0])
return None
variable_versions = {}
for collection in collections:
for variable, version in iterItems(collection.variable_actives):
if variable not in variable_versions:
variable_versions[variable] = OrderedSet((version,))
else:
variable_versions[variable].add(version)
for collection in collections:
for variable, versions in iterItems(variable_versions):
if variable not in collection.variable_actives:
versions.add(0)
self.variable_actives = {}
for variable, versions in iterItems(variable_versions):
if len(versions) == 1:
(version,) = versions
else:
version = self.addVariableMergeMultipleTrace(
variable=variable,
traces=tuple(
self.getVariableTrace(variable, version) for version in versions
),
)
self.markCurrentVariableTrace(variable, version)
def replaceBranch(self, collection_replace):
self.variable_actives.update(collection_replace.variable_actives)
collection_replace.variable_actives = None
def onLoopBreak(self, collection=None):
if collection is None:
collection = self
return self.parent.onLoopBreak(collection)
def onLoopContinue(self, collection=None):
if collection is None:
collection = self
return self.parent.onLoopContinue(collection)
def onFunctionReturn(self, collection=None):
if collection is None:
collection = self
return self.parent.onFunctionReturn(collection)
def onExceptionRaiseExit(self, raisable_exceptions, collection=None):
if collection is None:
collection = self
return self.parent.onExceptionRaiseExit(raisable_exceptions, collection)
def getLoopBreakCollections(self):
return self.parent.getLoopBreakCollections()
def getLoopContinueCollections(self):
return self.parent.getLoopContinueCollections()
def getFunctionReturnCollections(self):
return self.parent.getFunctionReturnCollections()
def getExceptionRaiseCollections(self):
return self.parent.getExceptionRaiseCollections()
def makeAbortStackContext(
self, catch_breaks, catch_continues, catch_returns, catch_exceptions
):
return self.parent.makeAbortStackContext(
catch_breaks=catch_breaks,
catch_continues=catch_continues,
catch_returns=catch_returns,
catch_exceptions=catch_exceptions,
)
def onLocalsDictEscaped(self, locals_scope):
self.parent.onLocalsDictEscaped(locals_scope)
def getCompileTimeComputationResult(self, node, computation, description):
new_node, change_tags, message = getComputationResult(
node=node,
computation=computation,
description=description,
user_provided=False,
)
if change_tags == "new_raise":
self.onExceptionRaiseExit(BaseException)
return new_node, change_tags, message
def getIteratorNextCount(self, iter_node):
return self.value_states.get(iter_node)
def initIteratorValue(self, iter_node):
# TODO: More complex state information will be needed eventually.
self.value_states[iter_node] = 0
def onIteratorNext(self, iter_node):
if iter_node in self.value_states:
self.value_states[iter_node] += 1
def resetValueStates(self):
for key in self.value_states:
self.value_states[key] = None
def addOutlineFunction(self, outline):
self.parent.addOutlineFunction(outline)
class SourceCodeReference(object):
# TODO: Measure the access speed impact of slots. The memory savings is
# not worth it (only a few percent).
__slots__ = ["filename", "line", "future_spec", "set_line"]
@classmethod
def fromFilenameAndLine(cls, filename, line, future_spec):
result = cls()
result.filename = filename
result.line = line
result.future_spec = future_spec
return result
__del__ = counted_del()
@counted_init
def __init__(self):
self.line = None
self.filename = None
self.future_spec = None
self.set_line = True
def __repr__(self):
return "<%s to %s:%s>" % (self.__class__.__name__, self.filename,
self.line)
def clone(self, line):
result = SourceCodeReference.fromFilenameAndLine(
filename=self.filename, line=line, future_spec=self.future_spec)
result.set_line = self.set_line
return result
def atLineNumber(self, line):
assert type(line) is int, line
if self.line != line:
return self.clone(line)
else:
return self
def getLineNumber(self):
return self.line
def getFilename(self):
return self.filename
def getFutureSpec(self):
return self.future_spec
def getAsString(self):
return "%s:%s" % (self.filename, self.line)
def shallSetCurrentLine(self):
return self.set_line
def __cmp__(self, other):
if other is None:
return -1
assert isinstance(other, SourceCodeReference), other
result = cmp(self.filename, other.filename)
if result == 0:
result = cmp(self.line, other.line)
return result
def atInternal(self):
if self.set_line:
result = self.clone(self.line)
result.set_line = False
return result
else:
return self
class TraceCollectionBase(CollectionTracingMixin):
__del__ = counted_del()
@counted_init
def __init__(self, owner, name, parent):
CollectionTracingMixin.__init__(self)
self.owner = owner
self.parent = parent
self.name = name
# Value state extra information per node.
self.value_states = {}
def __repr__(self):
return "<%s for %s at 0x%x>" % (self.__class__.__name__, self.name,
id(self))
def getOwner(self):
return self.owner
@staticmethod
def signalChange(tags, source_ref, message):
# This is monkey patched from another module.
signalChange(tags, source_ref, message)
def onUsedModule(self, module_name, module_relpath):
return self.parent.onUsedModule(module_name, module_relpath)
@staticmethod
def mustAlias(a, b):
if a.isExpressionVariableRef() and b.isExpressionVariableRef():
return a.getVariable() is b.getVariable()
return False
@staticmethod
def mustNotAlias(a, b):
# TODO: not yet really implemented
if a.isExpressionConstantRef() and b.isExpressionConstantRef():
if a.isMutable() or b.isMutable():
return True
return False
def removeKnowledge(self, node):
pass
def onControlFlowEscape(self, node):
# TODO: One day, we should trace which nodes exactly cause a variable
# to be considered escaped, pylint: disable=unused-argument
for variable in self.getActiveVariables():
if variable.isModuleVariable():
# print variable
self.markActiveVariableAsUnknown(variable)
elif variable.isLocalVariable():
if (python_version >= 300 and variable.hasWritesOutsideOf(
self.owner) is not False):
self.markActiveVariableAsUnknown(variable)
def removeAllKnowledge(self):
self.markActiveVariablesAsUnknown()
def getVariableTrace(self, variable, version):
return self.parent.getVariableTrace(variable, version)
def hasVariableTrace(self, variable, version):
return self.parent.hasVariableTrace(variable, version)
def addVariableTrace(self, variable, version, trace):
self.parent.addVariableTrace(variable, version, trace)
def addVariableMergeMultipleTrace(self, variable, traces):
return self.parent.addVariableMergeMultipleTrace(variable, traces)
def onVariableSet(self, variable, version, assign_node):
variable_trace = ValueTraceAssign(
owner=self.owner,
assign_node=assign_node,
previous=self.getVariableCurrentTrace(variable=variable),
)
self.addVariableTrace(variable=variable,
version=version,
trace=variable_trace)
# Make references point to it.
self.markCurrentVariableTrace(variable, version)
return variable_trace
def onVariableDel(self, variable, version):
# Add a new trace, allocating a new version for the variable, and
# remember the delete of the current
old_trace = self.getVariableCurrentTrace(variable)
variable_trace = ValueTraceUninit(owner=self.owner, previous=old_trace)
# Assign to not initialized again.
self.addVariableTrace(variable=variable,
version=version,
trace=variable_trace)
# Make references point to it.
self.markCurrentVariableTrace(variable, version)
return variable_trace
def onLocalsUsage(self, locals_owner):
self.onLocalsDictEscaped(locals_owner.getFunctionLocalsScope())
result = []
include_closure = (locals_owner.isExpressionFunctionBody()
and not locals_owner.isUnoptimized())
for variable in self.getActiveVariables():
if (variable.isLocalVariable()
and (variable.getOwner() is locals_owner or include_closure
and locals_owner.hasClosureVariable(variable))
and variable.getName() != ".0"):
variable_trace = self.getVariableCurrentTrace(variable)
variable_trace.addNameUsage()
result.append((variable, variable_trace))
return result
def onVariableContentEscapes(self, variable):
self.getVariableCurrentTrace(variable).onValueEscape()
def onExpression(self, expression, allow_none=False):
if expression is None and allow_none:
return None
assert expression.isExpression(), expression
parent = expression.parent
assert parent, expression
# Now compute this expression, allowing it to replace itself with
# something else as part of a local peep hole optimization.
r = expression.computeExpressionRaw(trace_collection=self)
assert type(r) is tuple, expression
new_node, change_tags, change_desc = r
if change_tags is not None:
# This is mostly for tracing and indication that a change occurred
# and it may be interesting to look again.
self.signalChange(change_tags, expression.getSourceReference(),
change_desc)
if new_node is not expression:
parent.replaceChild(expression, new_node)
return new_node
def onStatement(self, statement):
try:
assert statement.isStatement(), statement
new_statement, change_tags, change_desc = statement.computeStatement(
self)
# print new_statement, change_tags, change_desc
if new_statement is not statement:
self.signalChange(change_tags, statement.getSourceReference(),
change_desc)
return new_statement
except Exception:
Tracing.printError("Problem with statement at %s:\n-> %s" % (
statement.source_ref.getAsString(),
readSourceLine(statement.source_ref),
))
raise
def mergeBranches(self, collection_yes, collection_no):
""" Merge two alternative branches into this trace.
This is mostly for merging conditional branches, or other ways
of having alternative control flow. This deals with up to two
alternative branches to both change this collection.
"""
# Refuse to do stupid work
if collection_yes is None and collection_no is None:
return None
elif collection_yes is None or collection_no is None:
# Handle one branch case, we need to merge versions backwards as
# they may make themselves obsolete.
return self.mergeMultipleBranches(
collections=(self, collection_yes or collection_no))
else:
return self.mergeMultipleBranches(collections=(collection_yes,
collection_no))
def mergeMultipleBranches(self, collections):
assert collections
# Optimize for length 1, which is trivial merge and needs not a
# lot of work.
if len(collections) == 1:
self.replaceBranch(collections[0])
return None
variable_versions = {}
for collection in collections:
for variable, version in iterItems(collection.variable_actives):
if variable not in variable_versions:
variable_versions[variable] = set([version])
else:
variable_versions[variable].add(version)
for collection in collections:
for variable, versions in iterItems(variable_versions):
if variable not in collection.variable_actives:
versions.add(0)
self.variable_actives = {}
# merge_traces = None
for variable, versions in iterItems(variable_versions):
if len(versions) == 1:
version, = versions
else:
version = self.addVariableMergeMultipleTrace(
variable=variable,
traces=[
self.getVariableTrace(variable, version)
for version in versions
],
)
# if merge_traces is None:
# merge_traces = [trace_merge]
# else:
# merge_traces.append(trace_merge)
self.markCurrentVariableTrace(variable, version)
# Return "None", or turn the list into a tuple for memory savings.
# return merge_traces and tuple(merge_traces)
def replaceBranch(self, collection_replace):
self.variable_actives.update(collection_replace.variable_actives)
collection_replace.variable_actives = None
def onLoopBreak(self, collection=None):
if collection is None:
collection = self
return self.parent.onLoopBreak(collection)
def onLoopContinue(self, collection=None):
if collection is None:
collection = self
return self.parent.onLoopContinue(collection)
def onFunctionReturn(self, collection=None):
if collection is None:
collection = self
return self.parent.onFunctionReturn(collection)
def onExceptionRaiseExit(self, raisable_exceptions, collection=None):
if collection is None:
collection = self
return self.parent.onExceptionRaiseExit(raisable_exceptions,
collection)
def getLoopBreakCollections(self):
return self.parent.getLoopBreakCollections()
def getLoopContinueCollections(self):
return self.parent.getLoopContinueCollections()
def getFunctionReturnCollections(self):
return self.parent.getFunctionReturnCollections()
def getExceptionRaiseCollections(self):
return self.parent.getExceptionRaiseCollections()
def makeAbortStackContext(self, catch_breaks, catch_continues,
catch_returns, catch_exceptions):
return self.parent.makeAbortStackContext(
catch_breaks=catch_breaks,
catch_continues=catch_continues,
catch_returns=catch_returns,
catch_exceptions=catch_exceptions,
)
def onLocalsDictEscaped(self, locals_scope):
self.parent.onLocalsDictEscaped(locals_scope)
def getCompileTimeComputationResult(self, node, computation, description):
new_node, change_tags, message = getComputationResult(
node=node, computation=computation, description=description)
if change_tags == "new_raise":
self.onExceptionRaiseExit(BaseException)
return new_node, change_tags, message
def getIteratorNextCount(self, iter_node):
return self.value_states.get(iter_node, None)
def initIteratorValue(self, iter_node):
# TODO: More complex state information will be needed eventually.
self.value_states[iter_node] = 0
def onIteratorNext(self, iter_node):
if iter_node in self.value_states:
self.value_states[iter_node] += 1
def resetValueStates(self):
for key in self.value_states:
self.value_states[key] = None
def addOutlineFunction(self, outline):
self.parent.addOutlineFunction(outline)
class NodeBase(NodeMetaClassBase):
kind = None
@counted_init
def __init__(self, source_ref):
# The base class has no __init__ worth calling.
# Check source reference to meet basic standards, so we note errors
# when they occur.
assert source_ref is not None
assert source_ref.line is not None
self.parent = None
self.source_ref = source_ref
__del__ = counted_del()
def __repr__(self):
# This is to avoid crashes, because of bugs in detail.
# pylint: disable=W0703
try:
detail = self.getDetail()
except Exception as e:
detail = "detail raises exception %s" % e
if not detail:
return "<Node %s>" % self.getDescription()
else:
return "<Node %s %s>" % (self.getDescription(), detail)
def getDescription(self):
""" Description of the node, intented for use in __repr__ and
graphical display.
"""
return "%s at %s" % (self.kind, self.source_ref.getAsString())
def getDetails(self):
""" Details of the node, intended for use in __repr__ and dumps.
"""
# Virtual method, pylint: disable=R0201
return {}
def getDetail(self):
""" Details of the node, intended for use in __repr__ and graphical
display.
"""
return str(self.getDetails())[1:-1]
def getParent(self):
""" Parent of the node. Every node except modules have to have a parent.
"""
if self.parent is None and not self.isPythonModule():
assert False, (self, self.source_ref)
return self.parent
def getParents(self):
""" Parents of the node. Up to module level.
"""
result = []
current = self
while True:
current = current.getParent()
result.append(current)
if current.isPythonModule() or current.isExpressionFunctionBody():
break
assert None not in result, self
result.reverse()
return result
def getChildName(self):
""" Return the role in the current parent, subject to changes.
"""
parent = self.getParent()
for key, value in parent.child_values.items():
if self is value:
return key
# TODO: Not checking tuples yet
return None
def getParentFunction(self):
""" Return the parent that is a function.
"""
parent = self.getParent()
while parent is not None and not parent.isExpressionFunctionBody():
parent = parent.getParent()
return parent
def getParentModule(self):
""" Return the parent that is module.
"""
parent = self
while not parent.isPythonModule():
if hasattr(parent, "provider"):
# After we checked, we can use it, will be much faster route
# to take.
parent = parent.provider
else:
parent = parent.getParent()
return parent
def isParentVariableProvider(self):
# Check if it's a closure giver, in which cases it can provide variables,
return isinstance(self, ClosureGiverNodeBase)
def getParentVariableProvider(self):
parent = self.getParent()
while not parent.isParentVariableProvider():
parent = parent.getParent()
return parent
def getParentStatementsFrame(self):
current = self.getParent()
while True:
if current.isStatementsFrame():
return current
if current.isParentVariableProvider():
return None
current = current.getParent()
def getSourceReference(self):
return self.source_ref
def setCompatibleSourceReference(self, source_ref):
""" Bug compatible line numbers information.
As CPython outputs the last bit of bytecode executed, and not the
line of the operation. For example calls, output the line of the
last argument, as opposed to the line of the operation start.
For tests, we wants to be compatible. In improved more, we are
not being fully compatible, and just drop it altogether.
"""
# Getting the same source reference can be dealt with quickly, so do
# this first.
if self.source_ref is not source_ref and \
Options.isFullCompat() and \
self.source_ref != source_ref:
# An attribute outside of "__init__", so we save one memory for the
# most cases. Very few cases involve splitting across lines.
# pylint: disable=W0201
self.effective_source_ref = source_ref
def getCompatibleSourceReference(self):
""" Bug compatible line numbers information.
See above.
"""
return getattr(self, "effective_source_ref", self.source_ref)
def asXml(self):
line = self.getSourceReference().getLineNumber()
result = TreeXML.Element(
"node",
kind = self.__class__.__name__,
line = "%s" % line
)
compat_line = self.getCompatibleSourceReference().getLineNumber()
if compat_line != line:
result.attrib["compat_line"] = str(compat_line)
for key, value in iterItems(self.getDetails()):
value = str(value)
if value.startswith('<') and value.endswith('>'):
value = value[1:-1]
result.set(key, str(value))
for name, children in self.getVisitableNodesNamed():
if type(children) not in (list, tuple):
children = (children,)
role = TreeXML.Element(
"role",
name = name
)
result.append(role)
for child in children:
if child is not None:
role.append(
child.asXml()
)
return result
def dump(self, level = 0):
Tracing.printIndented(level, self)
Tracing.printSeparator(level)
for visitable in self.getVisitableNodes():
visitable.dump(level + 1)
Tracing.printSeparator(level)
@staticmethod
def isPythonModule():
# For overload by module nodes
return False
def isExpression(self):
return self.kind.startswith("EXPRESSION_")
def isStatement(self):
return self.kind.startswith("STATEMENT_")
def isExpressionBuiltin(self):
return self.kind.startswith("EXPRESSION_BUILTIN_")
def isOperation(self):
return self.kind.startswith("EXPRESSION_OPERATION_")
def isStatementReraiseException(self):
# Virtual method, pylint: disable=R0201
return False
def isExpressionMakeSequence(self):
# Virtual method, pylint: disable=R0201
return False
def isIteratorMaking(self):
# Virtual method, pylint: disable=R0201
return False
def isNumberConstant(self):
# Virtual method, pylint: disable=R0201
return False
def isExpressionCall(self):
# Virtual method, pylint: disable=R0201
return False
def visit(self, context, visitor):
visitor(self)
for visitable in self.getVisitableNodes():
visitable.visit(context, visitor)
def getVisitableNodes(self):
# Virtual method, pylint: disable=R0201
return ()
def getVisitableNodesNamed(self):
# Virtual method, pylint: disable=R0201
return ()
def replaceWith(self, new_node):
self.parent.replaceChild(
old_node = self,
new_node = new_node
)
def getName(self):
# Virtual method, pylint: disable=R0201
return None
def mayHaveSideEffects(self):
""" Unless we are told otherwise, everything may have a side effect. """
# Virtual method, pylint: disable=R0201
return True
def isOrderRelevant(self):
return self.mayHaveSideEffects()
def mayHaveSideEffectsBool(self):
""" Unless we are told otherwise, everything may have a side effect. """
# Virtual method, pylint: disable=R0201
return True
def extractSideEffects(self):
""" Unless defined otherwise, the expression is the side effect. """
return (self,)
def mayRaiseException(self, exception_type):
""" Unless we are told otherwise, everything may raise everything. """
# Virtual method, pylint: disable=R0201,W0613
return True
def mayRaiseExceptionBool(self, exception_type):
""" Unless we are told otherwise, everything may raise being checked. """
# Virtual method, pylint: disable=R0201,W0613
return True
def mayReturn(self):
return "_RETURN" in self.kind
def mayBreak(self):
# For overload, pylint: disable=R0201
return False
def mayContinue(self):
# For overload, pylint: disable=R0201
return False
def needsFrame(self):
""" Unless we are tolder otherwise, this depends on exception raise. """
return self.mayRaiseException(BaseException)
def willRaiseException(self, exception_type):
""" Unless we are told otherwise, nothing may raise anything. """
# Virtual method, pylint: disable=R0201,W0613
return False
def isIndexable(self):
""" Unless we are told otherwise, it's not indexable. """
# Virtual method, pylint: disable=R0201
return False
def isStatementAborting(self):
""" Is the node aborting, control flow doesn't continue after this node. """
assert self.isStatement(), self.kind
return False
def needsLocalsDict(self):
""" Node requires a locals dictionary by provider. """
# Virtual method, pylint: disable=R0201
return False
def getIntegerValue(self):
""" Node as integer value, if possible."""
# Virtual method, pylint: disable=R0201
return None
