class GlobalVariableTrace: @counted_init def __init__(self): self.traces = set() __del__ = counted_del() def add(self, variable_trace): self.traces.add(variable_trace) def remove(self, variable_trace): self.traces.remove(variable_trace) def hasDefiniteWrites(self): for trace in self.traces: if trace.isAssignTrace(): return True return False def getMatchingAssignTrace(self, assign_node): for trace in self.traces: if trace.isAssignTrace() and trace.getAssignNode() is assign_node: return trace return None def hasWritesOutsideOf(self, provider): for trace in self.traces: if trace.isAssignTrace(): if trace.getAssignNode().getParentVariableProvider( ) is not provider: return True return False
class VariableInformation: @counted_init def __init__(self, variable): self.variable = variable self.users = set() self.active_users = set() def __repr__(self): return "<%s object for %s>" % ( self.__class__.__name__, self.variable ) __del__ = counted_del() def addUser(self, user): self.users.add(user) self.active_users.add(user) def removeUser(self, user): try: self.active_users.remove(user) except KeyError: raise KeyError(self, user) def getUsers(self): return self.users def getActiveUsers(self): return self.active_users def getTopOwner(self): return self.variable.getOwner()
class SourceCodeReferenceInternal(SourceCodeReference): __slots__ = () __del__ = counted_del() @counted_init def __init__(self): SourceCodeReference.__init__(self) @staticmethod def isInternal(): return True
class GlobalVariableTrace: @counted_init def __init__(self): self.traces = set() self.writers = None self.users = None __del__ = counted_del() def addTrace(self, variable_trace): self.traces.add(variable_trace) def removeTrace(self, variable_trace): self.traces.remove(variable_trace) def updateUsageState(self): writers = set() users = set() for trace in self.traces: owner = trace.owner if trace.isAssignTrace(): writers.add(owner) users.add(owner) self.writers = writers self.users = users def hasDefiniteWrites(self): return bool(self.writers) def getMatchingAssignTrace(self, assign_node): for trace in self.traces: if trace.isAssignTrace() and trace.getAssignNode() is assign_node: return trace return None def hasWritesOutsideOf(self, provider): if provider in self.writers: return len(self.writers) > 1 else: return bool(self.writers) def hasAccessesOutsideOf(self, provider): if provider in self.users: return len(self.users) > 1 else: return bool(self.users)
class VariableInformation: @counted_init def __init__(self, variable): self.variable = variable self.users = set() __del__ = counted_del() def addUser(self, user): self.users.add(user) def getUsers(self): return self.users def getTopOwner(self): return self.variable.getOwner()
class LocalsDictHandle(object): __slots__ = ("locals_name", ) @counted_init def __init__(self, locals_name): self.locals_name = locals_name __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) @staticmethod def getTypeShape(): return ShapeTypeDict def getCodeName(self): return self.locals_name
class LocalsDictHandleBase(object): # TODO: Might remove some of these later, pylint: disable=too-many-instance-attributes __slots__ = ( "locals_name", # TODO: Specialize what the kinds really use. "variables", "local_variables", "providing", "mark_for_propagation", "propagation", "owner", "complete", ) @counted_init def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None self.complete = False if isCountingInstances(): __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name @staticmethod def isModuleScope(): return False @staticmethod def isClassScope(): return False @staticmethod def isFunctionScope(): return False def getProvidedVariables(self): return self.providing.values() def registerProvidedVariable(self, variable): variable_name = variable.getName() self.providing[variable_name] = variable def unregisterProvidedVariable(self, variable): """Remove provided variable, e.g. because it became unused.""" variable_name = variable.getName() if variable_name in self.providing: del self.providing[variable_name] registerClosureVariable = registerProvidedVariable unregisterClosureVariable = unregisterProvidedVariable def hasProvidedVariable(self, variable_name): """Test if a variable is provided.""" return variable_name in self.providing def getProvidedVariable(self, variable_name): """Test if a variable is provided.""" return self.providing[variable_name] def getLocalsRelevantVariables(self): """The variables relevant to locals.""" return self.providing.values() def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable(owner=self, variable_name=variable_name) self.variables[variable_name] = result return self.variables[variable_name] # TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here. def getLocalVariable(self, owner, variable_name): if variable_name not in self.local_variables: result = Variables.LocalVariable(owner=owner, variable_name=variable_name) self.local_variables[variable_name] = result return self.local_variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): # Make it unusable when it's become empty, not used. self.owner.locals_scope = None del self.owner del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() for variable in self.local_variables.values(): variable.finalize() del self.variables del self.providing def markAsComplete(self, trace_collection): self.complete = True self._considerUnusedUserLocalVariables(trace_collection) self._considerPropagation(trace_collection) # TODO: Limited to Python2 classes for now, more overloads need to be added, this # ought to be abstract and have variants with TODOs for each of them. @staticmethod def _considerPropagation(trace_collection): """For overload by scope type. Check if this can be replaced.""" def _considerUnusedUserLocalVariables(self, trace_collection): """Check scope for unused variables.""" provided = self.getProvidedVariables() removals = [] for variable in provided: if (variable.isLocalVariable() and not variable.isParameterVariable() and variable.getOwner() is self.owner): empty = trace_collection.hasEmptyTraces(variable) if empty: removals.append(variable) for variable in removals: self.unregisterProvidedVariable(variable) trace_collection.signalChange( "var_usage", self.owner.getSourceReference(), message="Remove unused local variable '%s'." % variable.getName(), )
class ParameterSpec(object): # These got many attributes, in part duplicating name and instance of # variables, pylint: disable=too-many-instance-attributes __slots__ = ( "name", "owner", "normal_args", "normal_variables", "list_star_arg", "dict_star_arg", "list_star_variable", "dict_star_variable", "default_count", "kw_only_args", "kw_only_variables", ) @counted_init def __init__( self, ps_name, ps_normal_args, ps_kw_only_args, ps_list_star_arg, ps_dict_star_arg, ps_default_count, ): if type(ps_normal_args) is str: if ps_normal_args == "": ps_normal_args = () else: ps_normal_args = ps_normal_args.split(",") if type(ps_kw_only_args) is str: if ps_kw_only_args == "": ps_kw_only_args = () else: ps_kw_only_args = ps_kw_only_args.split(",") assert None not in ps_normal_args self.owner = None self.name = ps_name self.normal_args = tuple(ps_normal_args) self.normal_variables = None assert ( ps_list_star_arg is None or type(ps_list_star_arg) is str ), ps_list_star_arg assert ( ps_dict_star_arg is None or type(ps_dict_star_arg) is str ), ps_dict_star_arg self.list_star_arg = ps_list_star_arg if ps_list_star_arg else None self.dict_star_arg = ps_dict_star_arg if ps_dict_star_arg else None self.list_star_variable = None self.dict_star_variable = None self.default_count = ps_default_count self.kw_only_args = tuple(ps_kw_only_args) self.kw_only_variables = None __del__ = counted_del() def makeClone(self): return ParameterSpec( ps_name=self.name, ps_normal_args=self.normal_args, ps_kw_only_args=self.kw_only_args, ps_list_star_arg=self.list_star_arg, ps_dict_star_arg=self.dict_star_arg, ps_default_count=self.default_count, ) def getDetails(self): return { "ps_name": self.name, "ps_normal_args": ",".join(self.normal_args), "ps_kw_only_args": ",".join(self.kw_only_args), "ps_list_star_arg": self.list_star_arg if self.list_star_arg is not None else "", "ps_dict_star_arg": self.dict_star_arg if self.dict_star_arg is not None else "", "ps_default_count": self.default_count, } def checkParametersValid(self): arg_names = self.getParameterNames() # Check for duplicate arguments, could happen. for arg_name in arg_names: if arg_names.count(arg_name) != 1: return "duplicate argument '%s' in function definition" % arg_name return None def __repr__(self): parts = [str(normal_arg) for normal_arg in self.normal_args] if self.list_star_arg is not None: parts.append("*%s" % self.list_star_arg) if self.dict_star_variable is not None: parts.append("**%s" % self.dict_star_variable) if parts: return "<ParameterSpec '%s'>" % ",".join(parts) else: return "<NoParameters>" def getArgumentCount(self): return len(self.normal_args) def setOwner(self, owner): if self.owner is not None: return self.owner = owner self.normal_variables = [] for normal_arg in self.normal_args: if type(normal_arg) is str: normal_variable = Variables.ParameterVariable( owner=self.owner, parameter_name=normal_arg ) else: assert False, normal_arg self.normal_variables.append(normal_variable) if self.list_star_arg: self.list_star_variable = Variables.ParameterVariable( owner=owner, parameter_name=self.list_star_arg ) else: self.list_star_variable = None if self.dict_star_arg: self.dict_star_variable = Variables.ParameterVariable( owner=owner, parameter_name=self.dict_star_arg ) else: self.dict_star_variable = None self.kw_only_variables = [ Variables.ParameterVariable(owner=self.owner, parameter_name=kw_only_arg) for kw_only_arg in self.kw_only_args ] def getDefaultCount(self): return self.default_count def getPositionalOnlyCount(self): # Virtual method, pylint: disable=no-self-use return 0 def hasDefaultParameters(self): return self.getDefaultCount() > 0 def getTopLevelVariables(self): return self.normal_variables + self.kw_only_variables def getAllVariables(self): result = list(self.normal_variables) result += self.kw_only_variables if self.list_star_variable is not None: result.append(self.list_star_variable) if self.dict_star_variable is not None: result.append(self.dict_star_variable) return result def getParameterNames(self): result = list(self.normal_args) result += self.kw_only_args if self.list_star_arg is not None: result.append(self.list_star_arg) if self.dict_star_arg is not None: result.append(self.dict_star_arg) return result def getStarListArgumentName(self): return self.list_star_arg def getListStarArgVariable(self): return self.list_star_variable def getStarDictArgumentName(self): return self.dict_star_arg def getDictStarArgVariable(self): return self.dict_star_variable def getKwOnlyVariables(self): return self.kw_only_variables def allowsKeywords(self): # Abstract method, pylint: disable=no-self-use return True def getKeywordRefusalText(self): return "%s() takes no keyword arguments" % self.name def getArgumentNames(self): return self.normal_args def getKwOnlyParameterNames(self): return self.kw_only_args def getKwOnlyParameterCount(self): return len(self.kw_only_args)
class SourceCodeReference(object): __slots__ = ["filename", "line", "column"] @classmethod def fromFilenameAndLine(cls, filename, line): result = cls() result.filename = filename result.line = line return result if isCountingInstances(): __del__ = counted_del() @counted_init def __init__(self): self.filename = None self.line = None self.column = None def __repr__(self): return "<%s to %s:%s>" % (self.__class__.__name__, self.filename, self.line) def __hash__(self): return hash((self.filename, self.line, self.column)) def __lt__(self, other): # Many cases decide early, pylint: disable=too-many-return-statements if other is None: return True if other is self: return False assert isinstance(other, SourceCodeReference), other if self.filename < other.filename: return True elif self.filename > other.filename: return False else: if self.line < other.line: return True elif self.line > other.line: return False else: if self.column < other.column: return True elif self.column > other.column: return False else: return self.isInternal() < other.isInternal() def __eq__(self, other): if other is None: return False if other is self: return True assert isinstance(other, SourceCodeReference), other if self.filename != other.filename: return False if self.line != other.line: return False if self.column != other.column: return False return self.isInternal() is other.isInternal() def _clone(self, line): """Make a copy it itself.""" return self.fromFilenameAndLine(filename=self.filename, line=line) def atInternal(self): """Make a copy it itself but mark as internal code. Avoids useless copies, by returning an internal object again if it is already internal. """ if not self.isInternal(): result = self._clone(self.line) return result else: return self def atLineNumber(self, line): """Make a reference to the same file, but different line. Avoids useless copies, by returning same object if the line is the same. """ assert type(line) is int, line if self.line != line: return self._clone(line) else: return self def atColumnNumber(self, column): assert type(column) is int, column if self.column != column: result = self._clone(self.line) result.column = column return result else: return self def getLineNumber(self): return self.line def getColumnNumber(self): return self.column def getFilename(self): return self.filename def getAsString(self): return "%s:%s" % (self.filename, self.line) @staticmethod def isInternal(): return False
class LocalsDictHandle(object): __slots__ = ("locals_name", "variables", "mark_for_propagation", "propagation") @counted_init def __init__(self, locals_name): self.locals_name = locals_name # For locals dict variables in this scope. self.variables = {} # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable(owner=self, variable_name=variable_name) self.variables[variable_name] = result return self.variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() del self.variables
class CodeObjectSpec(object): # One attribute for each code object aspect, and even flags, # pylint: disable=too-many-arguments,too-many-instance-attributes __slots__ = ("co_name", "co_kind", "co_varnames", "co_argcount", "co_kwonlyargcount", "co_has_starlist", "co_has_stardict", "filename", "line_number", "future_spec", "new_locals", "has_closure", "is_optimized") @counted_init def __init__(self, co_name, co_kind, co_varnames, co_argcount, co_kwonlyargcount, co_has_starlist, co_has_stardict, co_filename, co_lineno, future_spec, co_new_locals=None, co_has_closure=None, co_is_optimized=None): self.co_name = co_name self.co_kind = co_kind self.future_spec = future_spec assert future_spec # Strings happens from XML parsing, make sure to convert them. if type(co_varnames) is str: if co_varnames == "": co_varnames = () else: co_varnames = co_varnames.split(',') if type(co_has_starlist) is not bool: co_has_starlist = co_has_starlist != "False" if type(co_has_stardict) is not bool: co_has_stardict = co_has_stardict != "False" self.co_varnames = tuple(co_varnames) self.co_argcount = int(co_argcount) self.co_kwonlyargcount = int(co_kwonlyargcount) self.co_has_starlist = co_has_starlist self.co_has_stardict = co_has_stardict self.filename = co_filename self.line_number = int(co_lineno) if type(co_has_starlist) is not bool: co_new_locals = co_new_locals != "False" if type(co_has_starlist) is not bool: co_has_closure = co_has_closure != "False" if type(co_has_starlist) is not bool: co_is_optimized = co_is_optimized != "False" self.new_locals = co_new_locals self.has_closure = co_has_closure self.is_optimized = co_is_optimized __del__ = counted_del() def __repr__(self): return """\ <CodeObjectSpec %(co_kind)s '%(co_name)s' with %(co_varnames)r>""" % self.getDetails( ) def getDetails(self): return { "co_name": self.co_name, "co_kind": self.co_kind, "co_varnames": ','.join(self.co_varnames), "co_argcount": self.co_argcount, "co_kwonlyargcount": self.co_kwonlyargcount, "co_has_starlist": self.co_has_starlist, "co_has_stardict": self.co_has_stardict, "co_filename": self.filename, "co_lineno": self.line_number, "co_new_locals": self.new_locals, "co_has_closure": self.has_closure, "co_is_optimized": self.is_optimized, "code_flags": ','.join(self.future_spec.asFlags()) } def getCodeObjectKind(self): return self.co_kind def updateLocalNames(self, local_names): """ Add detected local variables after closure has been decided. """ self.co_varnames += tuple(local_name for local_name in local_names if local_name not in self.co_varnames) def setFlagIsOptimizedValue(self, value): self.is_optimized = value def getFlagIsOptimizedValue(self): return self.is_optimized def setFlagNewLocalsValue(self, value): self.new_locals = value def getFlagNewLocalsValue(self): return self.new_locals def setFlagHasClosureValue(self, value): self.has_closure = value def getFlagHasClosureValue(self): return self.has_closure def getFutureSpec(self): return self.future_spec def getVarNames(self): return self.co_varnames def getArgumentCount(self): return self.co_argcount def getKwOnlyParameterCount(self): return self.co_kwonlyargcount def getCodeObjectName(self): return self.co_name def hasStarListArg(self): return self.co_has_starlist def hasStarDictArg(self): return self.co_has_stardict def getFilename(self): return self.filename def getLineNumber(self): return self.line_number
class TraceCollectionBase(CollectionTracingMixin): __del__ = counted_del() @counted_init def __init__(self, owner, name, parent): CollectionTracingMixin.__init__(self) self.owner = owner self.parent = parent self.name = name # Value state extra information per node. self.value_states = {} def __repr__(self): return "<%s for %s %d>" % ( self.__class__.__name__, self.name, id(self) ) @staticmethod def signalChange(tags, source_ref, message): # This is monkey patched from another module, pylint: disable=E1102 signalChange(tags, source_ref, message) def onUsedModule(self, module): return self.parent.onUsedModule(module) @staticmethod def mustAlias(a, b): if a.isExpressionVariableRef() and b.isExpressionVariableRef(): return a.getVariable() is b.getVariable() return False @staticmethod def mustNotAlias(a, b): # TODO: not yet really implemented if a.isExpressionConstantDictRef() and b.isExpressionConstantRef(): return True return False def removeKnowledge(self, node): pass def onControlFlowEscape(self, node): # TODO: One day, we should trace which nodes exactly cause a variable # to be considered escaped, pylint: disable=W0613 for variable in self.getActiveVariables(): if variable.isModuleVariable(): # print variable self.markActiveVariableAsUnknown(variable) elif python_version >= 300 or variable.isSharedTechnically() is not False: # print variable # TODO: Could be limited to shared variables that are actually # written to. Most of the time, that won't be the case. self.markActiveVariableAsUnknown(variable) def removeAllKnowledge(self): self.markActiveVariablesAsUnknown() def getVariableTrace(self, variable, version): return self.parent.getVariableTrace(variable, version) def hasVariableTrace(self, variable, version): return self.parent.hasVariableTrace(variable, version) def addVariableTrace(self, variable, version, trace): self.parent.addVariableTrace(variable, version, trace) def addVariableMergeMultipleTrace(self, variable, traces): return self.parent.addVariableMergeMultipleTrace(variable, traces) def onVariableSet(self, assign_node): variable_ref = assign_node.getTargetVariableRef() version = variable_ref.getVariableVersion() variable = variable_ref.getVariable() variable_trace = VariableTraceAssign( owner = self.owner, assign_node = assign_node, variable = variable, version = version, previous = self.getVariableCurrentTrace( variable = variable ) ) self.addVariableTrace( variable = variable, version = version, trace = variable_trace ) # Make references point to it. self.markCurrentVariableTrace(variable, version) return variable_trace def onVariableDel(self, variable_ref): # Add a new trace, allocating a new version for the variable, and # remember the delete of the current variable = variable_ref.getVariable() version = variable_ref.getVariableVersion() old_trace = self.getVariableCurrentTrace(variable) variable_trace = VariableTraceUninit( owner = self.owner, variable = variable, version = version, previous = old_trace ) # Assign to not initialized again. self.addVariableTrace( variable = variable, version = version, trace = variable_trace ) # Make references point to it. self.markCurrentVariableTrace(variable, version) def onLocalsUsage(self): for variable in self.getActiveVariables(): # TODO: Currently this is a bit difficult to express in a positive # way, but we want to have only local variables. if not variable.isTempVariable() and \ not variable.isModuleVariable(): variable_trace = self.getVariableCurrentTrace( variable ) variable_trace.addNameUsage() def onVariableRelease(self, variable): current = self.getVariableCurrentTrace(variable) # Annotate that releases to the trace, it may be important knowledge. current.addRelease() return current def onVariableContentEscapes(self, variable): self.getVariableCurrentTrace(variable).onValueEscape() def onExpression(self, expression, allow_none = False): if expression is None and allow_none: return None assert expression.isExpression(), expression assert expression.parent, expression # Now compute this expression, allowing it to replace itself with # something else as part of a local peep hole optimization. r = expression.computeExpressionRaw( trace_collection = self ) assert type(r) is tuple, expression new_node, change_tags, change_desc = r if change_tags is not None: # This is mostly for tracing and indication that a change occurred # and it may be interesting to look again. self.signalChange( change_tags, expression.getSourceReference(), change_desc ) if new_node is not expression: expression.replaceWith(new_node) if new_node.isExpressionVariableRef() or \ new_node.isExpressionTempVariableRef(): # Remember the reference for constraint collection. assert new_node.variable_trace.hasDefiniteUsages() # We add variable reference nodes late to their traces, only after they # are actually produced, and not resolved to something else, so we do # not have them dangling, and the code complexity inside of their own # "computeExpression" functions. if new_node.isExpressionVariableRef() or \ new_node.isExpressionTempVariableRef(): if new_node.getVariable().isMaybeLocalVariable(): variable_trace = self.getVariableCurrentTrace( variable = new_node.getVariable().getMaybeVariable() ) variable_trace.addUsage() return new_node def onStatement(self, statement): try: assert statement.isStatement(), statement new_statement, change_tags, change_desc = \ statement.computeStatement(self) # print new_statement, change_tags, change_desc if new_statement is not statement: self.signalChange( change_tags, statement.getSourceReference(), change_desc ) return new_statement except Exception: Tracing.printError( "Problem with statement at %s:" % statement.getSourceReference().getAsString() ) raise def mergeBranches(self, collection_yes, collection_no): """ Merge two alternative branches into this trace. This is mostly for merging conditional branches, or other ways of having alternative control flow. This deals with up to two alternative branches to both change this collection. """ # Refuse to do stupid work if collection_yes is None and collection_no is None: pass elif collection_yes is None or collection_no is None: # Handle one branch case, we need to merge versions backwards as # they may make themselves obsolete. self.mergeMultipleBranches( collections = (self, collection_yes or collection_no) ) else: self.mergeMultipleBranches( collections = (collection_yes,collection_no) ) def mergeMultipleBranches(self, collections): assert len(collections) > 0 # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return variable_versions = {} for collection in collections: for variable, version in iterItems(collection.variable_actives): if variable not in variable_versions: variable_versions[variable] = set([version]) else: variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} for variable, versions in iterItems(variable_versions): if len(versions) == 1: version, = versions else: version = self.addVariableMergeMultipleTrace( variable = variable, traces = [ self.getVariableTrace(variable, version) for version in versions ] ) self.markCurrentVariableTrace(variable, version) def replaceBranch(self, collection_replace): self.variable_actives.update(collection_replace.variable_actives) collection_replace.variable_actives = None def onLoopBreak(self, collection = None): if collection is None: collection = self return self.parent.onLoopBreak(collection) def onLoopContinue(self, collection = None): if collection is None: collection = self return self.parent.onLoopContinue(collection) def onFunctionReturn(self, collection = None): if collection is None: collection = self return self.parent.onFunctionReturn(collection) def onExceptionRaiseExit(self, raisable_exceptions, collection = None): if collection is None: collection = self return self.parent.onExceptionRaiseExit(raisable_exceptions, collection) def getLoopBreakCollections(self): return self.parent.getLoopBreakCollections() def getLoopContinueCollections(self): return self.parent.getLoopContinueCollections() def getFunctionReturnCollections(self): return self.parent.getFunctionReturnCollections() def getExceptionRaiseCollections(self): return self.parent.getExceptionRaiseCollections() def makeAbortStackContext(self, catch_breaks, catch_continues, catch_returns, catch_exceptions): return self.parent.makeAbortStackContext( catch_breaks = catch_breaks, catch_continues = catch_continues, catch_returns = catch_returns, catch_exceptions = catch_exceptions ) def getCompileTimeComputationResult(self, node, computation, description): new_node, change_tags, message = getComputationResult( node = node, computation = computation, description = description ) if change_tags == "new_raise": self.onExceptionRaiseExit(BaseException) return new_node, change_tags, message def getIteratorNextCount(self, iter_node): return self.value_states.get(iter_node, None) def initIteratorValue(self, iter_node): # TODO: More complex state information will be needed eventually. self.value_states[iter_node] = 0 def onIteratorNext(self, iter_node): if iter_node in self.value_states: self.value_states[iter_node] += 1 def resetValueStates(self): for key in self.value_states: self.value_states[key] = None
class ValueTraceBase(object): # We are going to have many instance attributes, but should strive to minimize, as # there is going to be a lot of fluctuation in these objects. __slots__ = ( "owner", "usage_count", "name_usage_count", "merge_usage_count", "closure_usages", "previous", ) @counted_init def __init__(self, owner, previous): self.owner = owner # Definite usage indicator. self.usage_count = 0 # If 0, this indicates, the variable name needs to be assigned as name. self.name_usage_count = 0 # If 0, this indicates no value merges happened on the value. self.merge_usage_count = 0 self.closure_usages = False # Previous trace this is replacing. self.previous = previous if isCountingInstances(): __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.owner.getCodeName()) def getOwner(self): return self.owner @staticmethod def isLoopTrace(): return False def addUsage(self): self.usage_count += 1 def addNameUsage(self): self.usage_count += 1 self.name_usage_count += 1 if self.name_usage_count <= 2 and self.previous is not None: self.previous.addNameUsage() def addMergeUsage(self): self.usage_count += 1 self.merge_usage_count += 1 def getUsageCount(self): return self.usage_count def getNameUsageCount(self): return self.name_usage_count def getMergeUsageCount(self): return self.merge_usage_count def getMergeOrNameUsageCount(self): return self.merge_usage_count + self.name_usage_count def getPrevious(self): return self.previous @staticmethod def isAssignTrace(): return False @staticmethod def isUnassignedTrace(): return False @staticmethod def isDeletedTrace(): return False @staticmethod def isUninitTrace(): return False @staticmethod def isInitTrace(): return False @staticmethod def isUnknownTrace(): return False @staticmethod def isMergeTrace(): return False def mustHaveValue(self): """Will this definitely have a value. Every trace has this overloaded. """ assert False, self def mustNotHaveValue(self): """Will this definitely have a value. Every trace has this overloaded. """ assert False, self def getReplacementNode(self, usage): # Virtual method, pylint: disable=no-self-use,unused-argument return None @staticmethod def hasShapeDictionaryExact(): return False @staticmethod def getTruthValue(): return None
class CodeObjectSpec(object): # One attribute for each code object aspect, and even flags, # pylint: disable=too-many-arguments,too-many-instance-attributes __slots__ = ( "co_name", "co_kind", "co_varnames", "co_argcount", "co_freevars", "co_posonlyargcount", "co_kwonlyargcount", "co_has_starlist", "co_has_stardict", "filename", "line_number", "future_spec", "new_locals", "is_optimized", ) @counted_init def __init__( self, co_name, co_kind, co_varnames, co_freevars, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_has_starlist, co_has_stardict, co_filename, co_lineno, future_spec, co_new_locals=None, co_is_optimized=None, ): # pylint: disable=I0021,too-many-locals self.co_name = co_name self.co_kind = co_kind self.future_spec = future_spec assert future_spec # Strings happens from XML parsing, make sure to convert them. if type(co_varnames) is str: if co_varnames == "": co_varnames = () else: co_varnames = co_varnames.split(",") if type(co_freevars) is str: if co_freevars == "": co_freevars = () else: co_freevars = co_freevars.split(",") if type(co_has_starlist) is not bool: co_has_starlist = co_has_starlist != "False" if type(co_has_stardict) is not bool: co_has_stardict = co_has_stardict != "False" self.co_varnames = tuple(co_varnames) self.co_freevars = tuple(co_freevars) self.co_argcount = int(co_argcount) self.co_posonlyargcount = int(co_posonlyargcount) self.co_kwonlyargcount = int(co_kwonlyargcount) self.co_has_starlist = co_has_starlist self.co_has_stardict = co_has_stardict self.filename = co_filename self.line_number = int(co_lineno) if type(co_has_starlist) is not bool: co_new_locals = co_new_locals != "False" if type(co_has_starlist) is not bool: co_is_optimized = co_is_optimized != "False" self.new_locals = co_new_locals self.is_optimized = co_is_optimized __del__ = counted_del() def __repr__(self): return ("""\ <CodeObjectSpec %(co_kind)s '%(co_name)s' with %(co_varnames)r>""" % self.getDetails()) def getDetails(self): return { "co_name": self.co_name, "co_kind": self.co_kind, "co_varnames": ",".join(self.co_varnames), "co_freevars": ",".join(self.co_freevars), "co_argcount": self.co_argcount, "co_posonlyargcount": self.co_posonlyargcount, "co_kwonlyargcount": self.co_kwonlyargcount, "co_has_starlist": self.co_has_starlist, "co_has_stardict": self.co_has_stardict, "co_filename": self.filename, "co_lineno": self.line_number, "co_new_locals": self.new_locals, "co_is_optimized": self.is_optimized, "code_flags": ",".join(self.future_spec.asFlags()), } def getCodeObjectKind(self): return self.co_kind def updateLocalNames(self, local_names, freevar_names): """Move detected local variables after closure has been decided.""" self.co_varnames += tuple( local_name for local_name in local_names if local_name not in self.co_varnames # TODO: This is actually a bug, but we have a hard time without it to know # frame locals easily. We use this in compiled function run time, that all # variables, including closure variables are found there. This would have to # be cleaned up, for potentially little gain. # if local_name not in freevar_names ) self.co_freevars = tuple(freevar_names) def removeFreeVarname(self, freevar_name): self.co_freevars = tuple(var_name for var_name in self.co_freevars if var_name != freevar_name) def setFlagIsOptimizedValue(self, value): self.is_optimized = value def getFlagIsOptimizedValue(self): return self.is_optimized def setFlagNewLocalsValue(self, value): self.new_locals = value def getFlagNewLocalsValue(self): return self.new_locals def getFutureSpec(self): return self.future_spec def getVarNames(self): return self.co_varnames def getFreeVarNames(self): return self.co_freevars def getArgumentCount(self): return self.co_argcount def getPosOnlyParameterCount(self): return self.co_posonlyargcount def getKwOnlyParameterCount(self): return self.co_kwonlyargcount def getCodeObjectName(self): return self.co_name def hasStarListArg(self): return self.co_has_starlist def hasStarDictArg(self): return self.co_has_stardict def getFilename(self): return self.filename def getLineNumber(self): return self.line_number
class PythonContextBase(getMetaClassBase("Context")): @counted_init def __init__(self): self.source_ref = None self.current_source_ref = None if isCountingInstances(): __del__ = counted_del() def getCurrentSourceCodeReference(self): return self.current_source_ref def setCurrentSourceCodeReference(self, value): result = self.current_source_ref self.current_source_ref = value return result @contextmanager def withCurrentSourceCodeReference(self, value): old_source_ref = self.setCurrentSourceCodeReference(value) yield old_source_ref self.setCurrentSourceCodeReference(value) def getInplaceLeftName(self): return self.allocateTempName("inplace_orig", "PyObject *", True) @abstractmethod def getConstantCode(self, constant, deep_check=False): pass @abstractmethod def getModuleCodeName(self): pass @abstractmethod def getModuleName(self): pass @abstractmethod def addHelperCode(self, key, code): pass @abstractmethod def hasHelperCode(self, key): pass @abstractmethod def addDeclaration(self, key, code): pass @abstractmethod def pushFrameVariables(self, frame_variables): pass @abstractmethod def popFrameVariables(self): pass @abstractmethod def getFrameVariableTypeDescriptions(self): pass @abstractmethod def getFrameVariableTypeDescription(self): pass @abstractmethod def getFrameTypeDescriptionDeclaration(self): pass @abstractmethod def getFrameVariableCodeNames(self): pass @abstractmethod def allocateTempName(self, base_name, type_name="PyObject *", unique=False): pass @abstractmethod def skipTempName(self, base_name): pass @abstractmethod def getIntResName(self): pass @abstractmethod def getBoolResName(self): pass @abstractmethod def hasTempName(self, base_name): pass @abstractmethod def getExceptionEscape(self): pass @abstractmethod def setExceptionEscape(self, label): pass @abstractmethod def getLoopBreakTarget(self): pass @abstractmethod def setLoopBreakTarget(self, label): pass @abstractmethod def getLoopContinueTarget(self): pass @abstractmethod def setLoopContinueTarget(self, label): pass @abstractmethod def allocateLabel(self, label): pass @abstractmethod def allocateExceptionKeeperVariables(self): pass @abstractmethod def getExceptionKeeperVariables(self): pass @abstractmethod def setExceptionKeeperVariables(self, keeper_vars): pass @abstractmethod def addExceptionPreserverVariables(self, count): pass @abstractmethod def getTrueBranchTarget(self): pass @abstractmethod def getFalseBranchTarget(self): pass @abstractmethod def setTrueBranchTarget(self, label): pass @abstractmethod def setFalseBranchTarget(self, label): pass @abstractmethod def getCleanupTempnames(self): pass @abstractmethod def addCleanupTempName(self, tmp_name): pass @abstractmethod def removeCleanupTempName(self, tmp_name): pass @abstractmethod def needsCleanup(self, tmp_name): pass @abstractmethod def pushCleanupScope(self): pass @abstractmethod def popCleanupScope(self): pass
class Variable(getMetaClassBase("Variable")): # We will need all of these attributes, since we track the global # state and cache some decisions as attributes. TODO: But in some # cases, part of the these might be moved to the outside. __slots__ = ( "variable_name", "owner", "version_number", "shared_users", "traces", "users", "writers", ) @counted_init def __init__(self, owner, variable_name): assert type(variable_name) is str, variable_name assert type(owner) not in (tuple, list), owner self.variable_name = variable_name self.owner = owner self.version_number = 0 self.shared_users = False self.traces = set() # Derived from all traces. self.users = None self.writers = None if isCountingInstances(): __del__ = counted_del() def finalize(self): del self.users del self.writers del self.traces del self.owner def __repr__(self): return "<%s '%s' of '%s'>" % ( self.__class__.__name__, self.variable_name, self.owner.getName(), ) @abstractmethod def getVariableType(self): pass def getDescription(self): return "variable '%s'" % self.variable_name def getName(self): return self.variable_name def getOwner(self): return self.owner def getEntryPoint(self): return self.owner.getEntryPoint() def getCodeName(self): var_name = self.variable_name var_name = var_name.replace(".", "$") var_name = Utils.encodeNonAscii(var_name) return var_name def allocateTargetNumber(self): self.version_number += 1 return self.version_number @staticmethod def isLocalVariable(): return False @staticmethod def isParameterVariable(): return False @staticmethod def isModuleVariable(): return False @staticmethod def isIncompleteModuleVariable(): return False @staticmethod def isTempVariable(): return False @staticmethod def isTempVariableBool(): return False @staticmethod def isLocalsDictVariable(): return False def addVariableUser(self, user): # Update the shared scopes flag. if user is not self.owner: self.shared_users = True # These are not really scopes, just shared uses. if (user.isExpressionGeneratorObjectBody() or user.isExpressionCoroutineObjectBody() or user.isExpressionAsyncgenObjectBody()): if self.owner is user.getParentVariableProvider(): return _variables_in_shared_scopes.add(self) def isSharedTechnically(self): if not self.shared_users: return False if not self.users: return False owner = self.owner.getEntryPoint() for user in self.users: user = user.getEntryPoint() while user is not owner and ( (user.isExpressionFunctionBody() and not user.needsCreation()) or user.isExpressionClassBody()): user = user.getParentVariableProvider() if user is not owner: return True return False def addTrace(self, variable_trace): self.traces.add(variable_trace) def removeTrace(self, variable_trace): self.traces.remove(variable_trace) def updateUsageState(self): writers = set() users = set() for trace in self.traces: owner = trace.owner users.add(owner) if trace.isAssignTrace(): writers.add(owner) elif trace.isDeletedTrace() and owner is not self.owner: writers.add(owner) self.writers = writers self.users = users def hasAccessesOutsideOf(self, provider): if not self.owner.locals_scope.complete: return None elif self.users is None: return False elif provider in self.users: return len(self.users) > 1 else: return bool(self.users) def getMatchingAssignTrace(self, assign_node): for trace in self.traces: if trace.isAssignTrace() and trace.getAssignNode() is assign_node: return trace return None def getMatchingDelTrace(self, del_node): for trace in self.traces: if trace.isDeletedTrace() and trace.getDelNode() is del_node: return trace return None def getTypeShapes(self): result = set() for trace in self.traces: if trace.isAssignTrace(): result.add(trace.getAssignNode().getTypeShape()) elif trace.isUnknownTrace(): result.add(tshape_unknown) elif trace.isInitTrace(): result.add(tshape_unknown) elif trace.isUnassignedTrace(): pass elif trace.isMergeTrace(): pass # TODO: Remove this and be not unknown. elif trace.isLoopTrace(): trace.getTypeShape().emitAlternatives(result.add) else: assert False, trace return result
class NodeBase(NodeMetaClassBase): __slots__ = "parent", "source_ref", "effective_source_ref" # String to identify the node class, to be consistent with its name. kind = None @counted_init def __init__(self, source_ref): # The base class has no __init__ worth calling. # Check source reference to meet basic standards, so we note errors # when they occur. assert source_ref is not None assert source_ref.line is not None self.parent = None self.source_ref = source_ref __del__ = counted_del() def __repr__(self): return "<Node %s>" % (self.getDescription()) def getDescription(self): """ Description of the node, intended for use in __repr__ and graphical display. """ details = self.getDetails() if details: return "'%s' with %s" % (self.kind, self.getDetails()) else: return "'%s'" % self.kind def getDetails(self): """ Details of the node, intended for re-creation. We are not using the pickle mechanisms, but this is basically part of what the constructor call needs. Real children will also be added. """ # Virtual method, pylint: disable=no-self-use return {} def getDetailsForDisplay(self): """ Details of the node, intended for use in __repr__ and dumps. This is also used for XML. """ return self.getDetails() def getDetail(self): """ Details of the node, intended for use in __repr__ and graphical display. """ return str(self.getDetails())[1:-1] def makeClone(self): try: # Using star dictionary arguments here for generic use. result = self.__class__(source_ref=self.source_ref, **self.getDetails()) except TypeError: print("Problem cloning", self.__class__) raise effective_source_ref = self.getCompatibleSourceReference() if effective_source_ref is not self.source_ref: result.setCompatibleSourceReference(effective_source_ref) return result def makeCloneAt(self, source_ref): result = self.makeClone() result.source_ref = source_ref return result def getParent(self): """ Parent of the node. Every node except modules have to have a parent. """ if self.parent is None and not self.isCompiledPythonModule(): # print self.getVisitableNodesNamed() assert False, (self, self.source_ref) return self.parent def getChildName(self): """ Return the role in the current parent, subject to changes. """ parent = self.getParent() for key in parent.named_children: value = parent.getChild(key) if self is value: return key if type(value) is tuple: if self in value: return key, value.index(self) # TODO: Not checking tuples yet return None def getChildNameNice(self): child_name = self.getChildName() if hasattr(self.parent, "nice_children"): return self.parent.nice_children[self.parent.named_children.index( child_name)] else: return child_name def getParentFunction(self): """ Return the parent that is a function. """ parent = self.getParent() while parent is not None and not parent.isExpressionFunctionBodyBase(): parent = parent.getParent() return parent def getParentModule(self): """ Return the parent that is module. """ parent = self while not parent.isCompiledPythonModule(): if hasattr(parent, "provider"): # After we checked, we can use it, will be much faster route # to take. parent = parent.provider else: parent = parent.getParent() return parent def isParentVariableProvider(self): # Check if it's a closure giver, in which cases it can provide variables, return isinstance(self, ClosureGiverNodeMixin) def getParentVariableProvider(self): parent = self.getParent() while not parent.isParentVariableProvider(): parent = parent.getParent() return parent def getParentReturnConsumer(self): parent = self.getParent() while not parent.isParentVariableProvider() and \ not parent.isExpressionOutlineBody(): parent = parent.getParent() return parent def getParentStatementsFrame(self): current = self.getParent() while True: if current.isStatementsFrame(): return current if current.isParentVariableProvider(): return None if current.isExpressionOutlineBody(): return None current = current.getParent() def getSourceReference(self): return self.source_ref def setCompatibleSourceReference(self, source_ref): """ Bug compatible line numbers information. As CPython outputs the last bit of bytecode executed, and not the line of the operation. For example calls, output the line of the last argument, as opposed to the line of the operation start. For tests, we wants to be compatible. In improved more, we are not being fully compatible, and just drop it altogether. """ # Getting the same source reference can be dealt with quickly, so do # this first. if self.source_ref is not source_ref and \ Options.isFullCompat() and \ self.source_ref != source_ref: # An attribute outside of "__init__", so we save one memory for the # most cases. Very few cases involve splitting across lines. # pylint: disable=W0201 self.effective_source_ref = source_ref def getCompatibleSourceReference(self): """ Bug compatible line numbers information. See above. """ return getattr(self, "effective_source_ref", self.source_ref) def asXml(self): line = self.getSourceReference().getLineNumber() result = TreeXML.Element("node", kind=self.__class__.__name__, line="%s" % line) compat_line = self.getCompatibleSourceReference().getLineNumber() if compat_line != line: result.attrib["compat_line"] = str(compat_line) for key, value in iterItems(self.getDetailsForDisplay()): result.set(key, str(value)) for name, children in self.getVisitableNodesNamed(): role = TreeXML.Element("role", name=name) result.append(role) if children is None: role.attrib["type"] = "none" elif type(children) not in (list, tuple): role.append(children.asXml()) else: role.attrib["type"] = "list" for child in children: role.append(child.asXml()) return result @classmethod def fromXML(cls, provider, source_ref, **args): # Only some things need a provider, pylint: disable=unused-argument return cls(source_ref=source_ref, **args) def asXmlText(self): xml = self.asXml() return TreeXML.toString(xml) def dump(self, level=0): Tracing.printIndented(level, self) Tracing.printSeparator(level) for visitable in self.getVisitableNodes(): visitable.dump(level + 1) Tracing.printSeparator(level) @staticmethod def isStatementsFrame(): return False @staticmethod def isCompiledPythonModule(): # For overload by module nodes return False def isExpression(self): return self.kind.startswith("EXPRESSION_") def isStatement(self): return self.kind.startswith("STATEMENT_") def isExpressionBuiltin(self): return self.kind.startswith("EXPRESSION_BUILTIN_") @staticmethod def isExpressionConstantRef(): return False @staticmethod def isExpressionOperationBinary(): return False def isExpressionSideEffects(self): # Virtual method, pylint: disable=no-self-use # We need to provide this, as these node kinds are only imported if # necessary, but we test against them. return False def isStatementReraiseException(self): # Virtual method, pylint: disable=no-self-use return False def isExpressionMakeSequence(self): # Virtual method, pylint: disable=no-self-use return False def isNumberConstant(self): # Virtual method, pylint: disable=no-self-use return False def isExpressionCall(self): # Virtual method, pylint: disable=no-self-use return False def visit(self, context, visitor): visitor(self) for visitable in self.getVisitableNodes(): visitable.visit(context, visitor) def getVisitableNodes(self): # Virtual method, pylint: disable=no-self-use return () def getVisitableNodesNamed(self): """ Named children dictionary. For use in debugging and XML output. """ # Virtual method, pylint: disable=no-self-use return () def replaceWith(self, new_node): self.parent.replaceChild(old_node=self, new_node=new_node) def getName(self): # Virtual method, pylint: disable=no-self-use return None def mayHaveSideEffects(self): """ Unless we are told otherwise, everything may have a side effect. """ # Virtual method, pylint: disable=no-self-use return True def isOrderRelevant(self): return self.mayHaveSideEffects() def extractSideEffects(self): """ Unless defined otherwise, the expression is the side effect. """ return (self, ) def mayRaiseException(self, exception_type): """ Unless we are told otherwise, everything may raise everything. """ # Virtual method, pylint: disable=no-self-use,unused-argument return True def mayReturn(self): return "_RETURN" in self.kind def mayBreak(self): # For overload, pylint: disable=no-self-use return False def mayContinue(self): # For overload, pylint: disable=no-self-use return False def needsFrame(self): """ Unless we are tolder otherwise, this depends on exception raise. """ return self.mayRaiseException(BaseException) def willRaiseException(self, exception_type): """ Unless we are told otherwise, nothing may raise anything. """ # Virtual method, pylint: disable=no-self-use,unused-argument return False def isStatementAborting(self): """ Is the node aborting, control flow doesn't continue after this node. """ assert self.isStatement(), self.kind return False def needsLocalsDict(self): """ Node requires a locals dictionary by provider. """ # Virtual method, pylint: disable=no-self-use return False
class LocalsDictHandleBase(object): __slots__ = ( "locals_name", # TODO: Specialize what the kinds really use. "variables", "local_variables", "providing", "mark_for_propagation", "propagation", "owner", ) @counted_init def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name @staticmethod def isModuleScope(): return False @staticmethod def isClassScope(): return False @staticmethod def isFunctionScope(): return False def getProvidedVariables(self): return self.providing.values() def registerProvidedVariable(self, variable): variable_name = variable.getName() self.providing[variable_name] = variable def unregisterProvidedVariable(self, variable): """ Remove provided variable, e.g. because it became unused. """ variable_name = variable.getName() if variable_name in self.providing: del self.providing[variable_name] registerClosureVariable = registerProvidedVariable unregisterClosureVariable = unregisterProvidedVariable def hasProvidedVariable(self, variable_name): """ Test if a variable is provided. """ return variable_name in self.providing def getProvidedVariable(self, variable_name): """ Test if a variable is provided. """ return self.providing[variable_name] def getLocalsRelevantVariables(self): """ The variables relevant to locals. """ return self.providing.values() def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable( owner=self, variable_name=variable_name ) self.variables[variable_name] = result return self.variables[variable_name] # TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here. def getLocalVariable(self, owner, variable_name): if variable_name not in self.local_variables: result = Variables.LocalVariable(owner=owner, variable_name=variable_name) self.local_variables[variable_name] = result return self.local_variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name ) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): # Make it unusable when it's become empty, not used. self.owner.locals_scope = None del self.owner del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() for variable in self.local_variables.values(): variable.finalize() del self.variables del self.providing
class PythonContextBase(ContextMetaClassBase): @counted_init def __init__(self): self.source_ref = None self.current_source_ref = None self.last_source_ref = None __del__ = counted_del() def getCurrentSourceCodeReference(self): return self.current_source_ref def setCurrentSourceCodeReference(self, value): result = self.current_source_ref self.current_source_ref = value if value is not None: self.last_source_ref = result return result def getLastSourceCodeReference(self): result = self.last_source_ref # self.last_source_ref = None return result def isUsed(self, tmp_name): if tmp_name.startswith("tmp_unused_"): return False else: return True @abstractmethod def getConstantCode(self, constant): pass @abstractmethod def getModuleCodeName(self): pass @abstractmethod def getModuleName(self): pass @abstractmethod def addHelperCode(self, key, code): pass @abstractmethod def hasHelperCode(self, key): pass @abstractmethod def addDeclaration(self, key, code): pass @abstractmethod def pushFrameVariables(self, frame_variables): pass @abstractmethod def popFrameVariables(self): pass @abstractmethod def getFrameVariableTypeDescriptions(self): pass @abstractmethod def getFrameVariableTypeDescription(self): pass @abstractmethod def getFrameVariableTypeDescriptionName(self): pass @abstractmethod def getFrameVariableCodeNames(self): pass @abstractmethod def getLocalsDictName(self): pass @abstractmethod def allocateLocalsDictName(self): pass @abstractmethod def endLocalsDictName(self): pass @abstractmethod def allocateTempName(self, base_name, type_name="PyObject *", unique=False): pass @abstractmethod def getIntResName(self): pass @abstractmethod def getBoolResName(self): pass @abstractmethod def hasTempName(self, base_name): pass @abstractmethod def forgetTempName(self, tmp_name): pass @abstractmethod def getTempNameInfos(self): pass @abstractmethod def getExceptionEscape(self): pass @abstractmethod def setExceptionEscape(self, label): pass @abstractmethod def getLoopBreakTarget(self): pass @abstractmethod def setLoopBreakTarget(self, label): pass @abstractmethod def getLoopContinueTarget(self): pass @abstractmethod def setLoopContinueTarget(self, label): pass @abstractmethod def allocateLabel(self, label): pass @abstractmethod def needsExceptionVariables(self): pass @abstractmethod def markAsNeedsExceptionVariables(self): pass @abstractmethod def allocateExceptionKeeperVariables(self): pass @abstractmethod def getExceptionKeeperVariables(self): pass @abstractmethod def setExceptionKeeperVariables(self, keeper_vars): pass @abstractmethod def addExceptionPreserverVariables(self, count): pass @abstractmethod def getTrueBranchTarget(self): pass @abstractmethod def getFalseBranchTarget(self): pass @abstractmethod def setTrueBranchTarget(self, label): pass @abstractmethod def setFalseBranchTarget(self, label): pass @abstractmethod def getCleanupTempnames(self): pass @abstractmethod def addCleanupTempName(self, tmp_name): pass @abstractmethod def removeCleanupTempName(self, tmp_name): pass @abstractmethod def needsCleanup(self, tmp_name): pass @abstractmethod def pushCleanupScope(self): pass @abstractmethod def popCleanupScope(self): pass
class FutureSpec: @counted_init def __init__(self): self.future_division = _future_division_default self.unicode_literals = False self.absolute_import = _future_absolute_import_default self.future_print = False self.barry_bdfl = False self.generator_stop = _future_generator_stop_default __del__ = counted_del() def clone(self): result = FutureSpec() result.future_division = self.future_division result.unicode_literals = self.unicode_literals result.absolute_import = self.absolute_import result.future_print = self.future_print result.barry_bdfl = self.barry_bdfl return result def isFutureDivision(self): return self.future_division def enableFutureDivision(self): self.future_division = True def enableFuturePrint(self): self.future_print = True def enableUnicodeLiterals(self): self.unicode_literals = True def enableAbsoluteImport(self): self.absolute_import = True def enableBarry(self): self.barry_bdfl = True def enableGeneratorStop(self): self.generator_stop = True def isAbsoluteImport(self): return self.absolute_import def isGeneratorStop(self): return self.generator_stop def asFlags(self): """ Create a list of C identifiers to represent the flag values. This is for use in code generation only. """ result = [] if self.future_division and python_version < 300: result.append("CO_FUTURE_DIVISION") if self.unicode_literals: result.append("CO_FUTURE_UNICODE_LITERALS") if self.absolute_import and python_version < 300: result.append("CO_FUTURE_ABSOLUTE_IMPORT") if self.future_print and python_version < 300: result.append("CO_FUTURE_PRINT_FUNCTION") if self.barry_bdfl and python_version >= 300: result.append("CO_FUTURE_BARRY_AS_BDFL") if self.generator_stop and python_version >= 350: result.append("CO_FUTURE_GENERATOR_STOP") return tuple(result)
class FutureSpec(object): __slots__ = ( "future_division", "unicode_literals", "absolute_import", "future_print", "barry_bdfl", "generator_stop", "future_annotations", ) @counted_init def __init__(self): self.future_division = _future_division_default self.unicode_literals = False self.absolute_import = _future_absolute_import_default self.future_print = False self.barry_bdfl = False self.generator_stop = _future_generator_stop_default self.future_annotations = _future_annotations_default __del__ = counted_del() def __repr__(self): return "<FutureSpec %s>" % ",".join(self.asFlags()) def clone(self): result = FutureSpec() result.future_division = self.future_division result.unicode_literals = self.unicode_literals result.absolute_import = self.absolute_import result.future_print = self.future_print result.barry_bdfl = self.barry_bdfl result.generator_stop = self.generator_stop result.future_annotations = result.future_annotations return result def isFutureDivision(self): return self.future_division def enableFutureDivision(self): self.future_division = True def isFuturePrint(self): return self.future_print def enableFuturePrint(self): self.future_print = True def enableUnicodeLiterals(self): self.unicode_literals = True def enableAbsoluteImport(self): self.absolute_import = True def enableBarry(self): self.barry_bdfl = True def enableGeneratorStop(self): self.generator_stop = True def isAbsoluteImport(self): return self.absolute_import def isGeneratorStop(self): return self.generator_stop def enableFutureAnnotations(self): self.future_annotations = True def isFutureAnnotations(self): return self.future_annotations def asFlags(self): """ Create a list of C identifiers to represent the flag values. This is for use in code generation and to restore from saved modules. """ result = [] if python_version < 300 and self.future_division: result.append("CO_FUTURE_DIVISION") if self.unicode_literals: result.append("CO_FUTURE_UNICODE_LITERALS") if python_version < 300 and self.absolute_import: result.append("CO_FUTURE_ABSOLUTE_IMPORT") if python_version < 300 and self.future_print: result.append("CO_FUTURE_PRINT_FUNCTION") if python_version >= 300 and self.barry_bdfl: result.append("CO_FUTURE_BARRY_AS_BDFL") if 350 <= python_version < 370 and self.generator_stop: result.append("CO_FUTURE_GENERATOR_STOP") if python_version >= 370 and self.future_annotations: result.append("CO_FUTURE_ANNOTATIONS") return tuple(result)
class SourceCodeReference(object): # TODO: Measure the access speed impact of slots. The memory savings is # not worth it (only a few percent). __slots__ = ["filename", "line", "future_spec", "internal"] @classmethod def fromFilenameAndLine(cls, filename, line, future_spec): result = cls() result.filename = filename result.line = line result.future_spec = future_spec return result __del__ = counted_del() @counted_init def __init__(self): self.line = None self.filename = None self.future_spec = None self.internal = False def __repr__(self): return "<%s to %s:%s>" % (self.__class__.__name__, self.filename, self.line) def __cmp__(self, other): if other is None: return -1 assert isinstance(other, SourceCodeReference), other result = cmp(self.filename, other.filename) if result == 0: result = cmp(self.line, other.line) if result == 0: result = cmp(self.internal, other.internal) return result def _clone(self, line): """ Make a copy it itself. """ result = SourceCodeReference.fromFilenameAndLine( filename = self.filename, line = line, future_spec = self.future_spec ) result.internal = self.internal return result def atInternal(self): """ Make a copy it itself but mark as internal code. Avoids useless copies, by returning an internal object again if it is already internal. """ if not self.internal: result = self._clone(self.line) result.internal = True return result else: return self def atLineNumber(self, line): """ Make a reference to the same file, but different line. Avoids useless copies, by returning same object if the line is the same. """ assert type(line) is int, line if self.line != line: return self._clone(line) else: return self def getLineNumber(self): return self.line def getFilename(self): return self.filename def getFutureSpec(self): return self.future_spec def getAsString(self): return "%s:%s" % (self.filename, self.line) def isInternal(self): return self.internal
class TraceCollectionBase(object): """This contains for logic for maintaining active traces. They are kept for "variable" and versions. """ __slots__ = ("owner", "parent", "name", "value_states", "variable_actives") if isCountingInstances(): __del__ = counted_del() @counted_init def __init__(self, owner, name, parent): self.owner = owner self.parent = parent self.name = name # Value state extra information per node. self.value_states = {} # Currently active values in the tracing. self.variable_actives = {} def __repr__(self): return "<%s for %s at 0x%x>" % (self.__class__.__name__, self.name, id(self)) def getOwner(self): return self.owner def getVariableCurrentTrace(self, variable): """Get the current value trace associated to this variable It is also created on the fly if necessary. We create them lazy so to keep the tracing branches minimal where possible. """ return self.getVariableTrace( variable=variable, version=self._getCurrentVariableVersion(variable) ) def markCurrentVariableTrace(self, variable, version): self.variable_actives[variable] = version def _getCurrentVariableVersion(self, variable): try: return self.variable_actives[variable] except KeyError: # Initialize variables on the fly. if not self.hasVariableTrace(variable, 0): self.initVariable(variable) self.markCurrentVariableTrace(variable, 0) return self.variable_actives[variable] def getActiveVariables(self): return self.variable_actives.keys() def markActiveVariableAsEscaped(self, variable): current = self.getVariableCurrentTrace(variable=variable) if not current.isUnknownTrace(): version = variable.allocateTargetNumber() self.addVariableTrace( variable=variable, version=version, trace=ValueTraceEscaped(owner=self.owner, previous=current), ) self.markCurrentVariableTrace(variable, version) def markActiveVariableAsLoopMerge( self, loop_node, current, variable, shapes, incomplete ): if incomplete: result = ValueTraceLoopIncomplete(loop_node, current, shapes) else: # TODO: Empty is a missing optimization somewhere, but it also happens that # a variable is getting released in a loop. # assert shapes, (variable, current) if not shapes: shapes.add(tshape_uninit) result = ValueTraceLoopComplete(loop_node, current, shapes) version = variable.allocateTargetNumber() self.addVariableTrace(variable=variable, version=version, trace=result) self.markCurrentVariableTrace(variable, version) return result def markActiveVariablesAsUnknown(self): for variable in self.getActiveVariables(): if variable.isTempVariable(): continue self.markActiveVariableAsEscaped(variable) @staticmethod def signalChange(tags, source_ref, message): # This is monkey patched from another module. pylint: disable=I0021,not-callable signalChange(tags, source_ref, message) def onUsedModule(self, module_name, module_relpath): return self.parent.onUsedModule(module_name, module_relpath) def onUsedFunction(self, function_body): owning_module = function_body.getParentModule() # Make sure the owning module is added to the used set. This is most # important for helper functions, or modules, which otherwise have # become unused. addUsedModule(owning_module) needs_visit = owning_module.addUsedFunction(function_body) if needs_visit: function_body.computeFunctionRaw(self) @staticmethod def mustAlias(a, b): if a.isExpressionVariableRef() and b.isExpressionVariableRef(): return a.getVariable() is b.getVariable() return False @staticmethod def mustNotAlias(a, b): # TODO: not yet really implemented if a.isExpressionConstantRef() and b.isExpressionConstantRef(): if a.isMutable() or b.isMutable(): return True return False def onControlFlowEscape(self, node): # TODO: One day, we should trace which nodes exactly cause a variable # to be considered escaped, pylint: disable=unused-argument for variable in self.getActiveVariables(): if variable.isModuleVariable(): # print variable self.markActiveVariableAsEscaped(variable) elif variable.isLocalVariable(): if variable.hasAccessesOutsideOf(self.owner) is not False: self.markActiveVariableAsEscaped(variable) def removeKnowledge(self, node): if node.isExpressionVariableRef(): self.markActiveVariableAsEscaped(node.variable) def onValueEscapeStr(self, node): # TODO: We can ignore these for now. pass def removeAllKnowledge(self): self.markActiveVariablesAsUnknown() def getVariableTrace(self, variable, version): return self.parent.getVariableTrace(variable, version) def hasVariableTrace(self, variable, version): return self.parent.hasVariableTrace(variable, version) def addVariableTrace(self, variable, version, trace): self.parent.addVariableTrace(variable, version, trace) def addVariableMergeMultipleTrace(self, variable, traces): return self.parent.addVariableMergeMultipleTrace(variable, traces) def onVariableSet(self, variable, version, assign_node): variable_trace = ValueTraceAssign( owner=self.owner, assign_node=assign_node, previous=self.getVariableCurrentTrace(variable=variable), ) self.addVariableTrace(variable=variable, version=version, trace=variable_trace) # Make references point to it. self.markCurrentVariableTrace(variable, version) return variable_trace def onVariableDel(self, variable, version, del_node): # Add a new trace, allocating a new version for the variable, and # remember the delete of the current old_trace = self.getVariableCurrentTrace(variable) variable_trace = ValueTraceDeleted( owner=self.owner, del_node=del_node, previous=old_trace ) # Assign to not initialized again. self.addVariableTrace(variable=variable, version=version, trace=variable_trace) # Make references point to it. self.markCurrentVariableTrace(variable, version) return variable_trace def onLocalsUsage(self, locals_scope): self.onLocalsDictEscaped(locals_scope) result = [] scope_locals_variables = locals_scope.getLocalsRelevantVariables() for variable in self.getActiveVariables(): if variable.isLocalVariable() and variable in scope_locals_variables: variable_trace = self.getVariableCurrentTrace(variable) variable_trace.addNameUsage() result.append((variable, variable_trace)) return result def onVariableContentEscapes(self, variable): self.markActiveVariableAsEscaped(variable) def onExpression(self, expression, allow_none=False): if expression is None and allow_none: return None assert expression.isExpression(), expression parent = expression.parent assert parent, expression # Now compute this expression, allowing it to replace itself with # something else as part of a local peep hole optimization. r = expression.computeExpressionRaw(trace_collection=self) assert type(r) is tuple, (expression, expression.getVisitableNodes(), r) new_node, change_tags, change_desc = r if change_tags is not None: # This is mostly for tracing and indication that a change occurred # and it may be interesting to look again. self.signalChange(change_tags, expression.getSourceReference(), change_desc) if new_node is not expression: parent.replaceChild(expression, new_node) return new_node def onStatement(self, statement): try: assert statement.isStatement(), statement new_statement, change_tags, change_desc = statement.computeStatement(self) # print new_statement, change_tags, change_desc if new_statement is not statement: self.signalChange( change_tags, statement.getSourceReference(), change_desc ) return new_statement except Exception: Tracing.printError( "Problem with statement at %s:\n-> %s" % ( statement.source_ref.getAsString(), readSourceLine(statement.source_ref), ) ) raise def computedStatementResult(self, statement, change_tags, change_desc): """Make sure the replacement statement is computed. Use this when a replacement expression needs to be seen by the trace collection and be computed, without causing any duplication, but where otherwise there would be loss of annotated effects. This may e.g. be true for nodes that need an initial run to know their exception result and type shape. """ # Need to compute the replacement still. new_statement = statement.computeStatement(self) if new_statement[0] is not statement: # Signal intermediate result as well. self.signalChange(change_tags, statement.getSourceReference(), change_desc) return new_statement else: return statement, change_tags, change_desc def computedExpressionResult(self, expression, change_tags, change_desc): """Make sure the replacement expression is computed. Use this when a replacement expression needs to be seen by the trace collection and be computed, without causing any duplication, but where otherwise there would be loss of annotated effects. This may e.g. be true for nodes that need an initial run to know their exception result and type shape. """ # Need to compute the replacement still. new_expression = expression.computeExpression(self) if new_expression[0] is not expression: # Signal intermediate result as well. self.signalChange(change_tags, expression.getSourceReference(), change_desc) return new_expression else: return expression, change_tags, change_desc def mergeBranches(self, collection_yes, collection_no): """Merge two alternative branches into this trace. This is mostly for merging conditional branches, or other ways of having alternative control flow. This deals with up to two alternative branches to both change this collection. """ # Many branches due to inlining the actual merge and preparing it # pylint: disable=too-many-branches if collection_yes is None: if collection_no is not None: # Handle one branch case, we need to merge versions backwards as # they may make themselves obsolete. collection1 = self collection2 = collection_no else: # Refuse to do stupid work return elif collection_no is None: # Handle one branch case, we need to merge versions backwards as # they may make themselves obsolete. collection1 = self collection2 = collection_yes else: # Handle two branch case, they may or may not do the same things. collection1 = collection_yes collection2 = collection_no variable_versions = {} for variable, version in iterItems(collection1.variable_actives): variable_versions[variable] = version for variable, version in iterItems(collection2.variable_actives): if variable not in variable_versions: variable_versions[variable] = 0, version else: other = variable_versions[variable] if other != version: variable_versions[variable] = other, version else: variable_versions[variable] = other for variable in variable_versions: if variable not in collection2.variable_actives: variable_versions[variable] = variable_versions[variable], 0 self.variable_actives = {} for variable, versions in iterItems(variable_versions): if type(versions) is tuple: version = self.addVariableMergeMultipleTrace( variable=variable, traces=( self.getVariableTrace(variable, versions[0]), self.getVariableTrace(variable, versions[1]), ), ) else: version = versions self.markCurrentVariableTrace(variable, version) def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None variable_versions = {} for collection in collections: for variable, version in iterItems(collection.variable_actives): if variable not in variable_versions: variable_versions[variable] = OrderedSet((version,)) else: variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} for variable, versions in iterItems(variable_versions): if len(versions) == 1: (version,) = versions else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=tuple( self.getVariableTrace(variable, version) for version in versions ), ) self.markCurrentVariableTrace(variable, version) def replaceBranch(self, collection_replace): self.variable_actives.update(collection_replace.variable_actives) collection_replace.variable_actives = None def onLoopBreak(self, collection=None): if collection is None: collection = self return self.parent.onLoopBreak(collection) def onLoopContinue(self, collection=None): if collection is None: collection = self return self.parent.onLoopContinue(collection) def onFunctionReturn(self, collection=None): if collection is None: collection = self return self.parent.onFunctionReturn(collection) def onExceptionRaiseExit(self, raisable_exceptions, collection=None): if collection is None: collection = self return self.parent.onExceptionRaiseExit(raisable_exceptions, collection) def getLoopBreakCollections(self): return self.parent.getLoopBreakCollections() def getLoopContinueCollections(self): return self.parent.getLoopContinueCollections() def getFunctionReturnCollections(self): return self.parent.getFunctionReturnCollections() def getExceptionRaiseCollections(self): return self.parent.getExceptionRaiseCollections() def makeAbortStackContext( self, catch_breaks, catch_continues, catch_returns, catch_exceptions ): return self.parent.makeAbortStackContext( catch_breaks=catch_breaks, catch_continues=catch_continues, catch_returns=catch_returns, catch_exceptions=catch_exceptions, ) def onLocalsDictEscaped(self, locals_scope): self.parent.onLocalsDictEscaped(locals_scope) def getCompileTimeComputationResult(self, node, computation, description): new_node, change_tags, message = getComputationResult( node=node, computation=computation, description=description, user_provided=False, ) if change_tags == "new_raise": self.onExceptionRaiseExit(BaseException) return new_node, change_tags, message def getIteratorNextCount(self, iter_node): return self.value_states.get(iter_node) def initIteratorValue(self, iter_node): # TODO: More complex state information will be needed eventually. self.value_states[iter_node] = 0 def onIteratorNext(self, iter_node): if iter_node in self.value_states: self.value_states[iter_node] += 1 def resetValueStates(self): for key in self.value_states: self.value_states[key] = None def addOutlineFunction(self, outline): self.parent.addOutlineFunction(outline)
class SourceCodeReference(object): # TODO: Measure the access speed impact of slots. The memory savings is # not worth it (only a few percent). __slots__ = ["filename", "line", "future_spec", "set_line"] @classmethod def fromFilenameAndLine(cls, filename, line, future_spec): result = cls() result.filename = filename result.line = line result.future_spec = future_spec return result __del__ = counted_del() @counted_init def __init__(self): self.line = None self.filename = None self.future_spec = None self.set_line = True def __repr__(self): return "<%s to %s:%s>" % (self.__class__.__name__, self.filename, self.line) def clone(self, line): result = SourceCodeReference.fromFilenameAndLine( filename=self.filename, line=line, future_spec=self.future_spec) result.set_line = self.set_line return result def atLineNumber(self, line): assert type(line) is int, line if self.line != line: return self.clone(line) else: return self def getLineNumber(self): return self.line def getFilename(self): return self.filename def getFutureSpec(self): return self.future_spec def getAsString(self): return "%s:%s" % (self.filename, self.line) def shallSetCurrentLine(self): return self.set_line def __cmp__(self, other): if other is None: return -1 assert isinstance(other, SourceCodeReference), other result = cmp(self.filename, other.filename) if result == 0: result = cmp(self.line, other.line) return result def atInternal(self): if self.set_line: result = self.clone(self.line) result.set_line = False return result else: return self
class TraceCollectionBase(CollectionTracingMixin): __del__ = counted_del() @counted_init def __init__(self, owner, name, parent): CollectionTracingMixin.__init__(self) self.owner = owner self.parent = parent self.name = name # Value state extra information per node. self.value_states = {} def __repr__(self): return "<%s for %s at 0x%x>" % (self.__class__.__name__, self.name, id(self)) def getOwner(self): return self.owner @staticmethod def signalChange(tags, source_ref, message): # This is monkey patched from another module. signalChange(tags, source_ref, message) def onUsedModule(self, module_name, module_relpath): return self.parent.onUsedModule(module_name, module_relpath) @staticmethod def mustAlias(a, b): if a.isExpressionVariableRef() and b.isExpressionVariableRef(): return a.getVariable() is b.getVariable() return False @staticmethod def mustNotAlias(a, b): # TODO: not yet really implemented if a.isExpressionConstantRef() and b.isExpressionConstantRef(): if a.isMutable() or b.isMutable(): return True return False def removeKnowledge(self, node): pass def onControlFlowEscape(self, node): # TODO: One day, we should trace which nodes exactly cause a variable # to be considered escaped, pylint: disable=unused-argument for variable in self.getActiveVariables(): if variable.isModuleVariable(): # print variable self.markActiveVariableAsUnknown(variable) elif variable.isLocalVariable(): if (python_version >= 300 and variable.hasWritesOutsideOf( self.owner) is not False): self.markActiveVariableAsUnknown(variable) def removeAllKnowledge(self): self.markActiveVariablesAsUnknown() def getVariableTrace(self, variable, version): return self.parent.getVariableTrace(variable, version) def hasVariableTrace(self, variable, version): return self.parent.hasVariableTrace(variable, version) def addVariableTrace(self, variable, version, trace): self.parent.addVariableTrace(variable, version, trace) def addVariableMergeMultipleTrace(self, variable, traces): return self.parent.addVariableMergeMultipleTrace(variable, traces) def onVariableSet(self, variable, version, assign_node): variable_trace = ValueTraceAssign( owner=self.owner, assign_node=assign_node, previous=self.getVariableCurrentTrace(variable=variable), ) self.addVariableTrace(variable=variable, version=version, trace=variable_trace) # Make references point to it. self.markCurrentVariableTrace(variable, version) return variable_trace def onVariableDel(self, variable, version): # Add a new trace, allocating a new version for the variable, and # remember the delete of the current old_trace = self.getVariableCurrentTrace(variable) variable_trace = ValueTraceUninit(owner=self.owner, previous=old_trace) # Assign to not initialized again. self.addVariableTrace(variable=variable, version=version, trace=variable_trace) # Make references point to it. self.markCurrentVariableTrace(variable, version) return variable_trace def onLocalsUsage(self, locals_owner): self.onLocalsDictEscaped(locals_owner.getFunctionLocalsScope()) result = [] include_closure = (locals_owner.isExpressionFunctionBody() and not locals_owner.isUnoptimized()) for variable in self.getActiveVariables(): if (variable.isLocalVariable() and (variable.getOwner() is locals_owner or include_closure and locals_owner.hasClosureVariable(variable)) and variable.getName() != ".0"): variable_trace = self.getVariableCurrentTrace(variable) variable_trace.addNameUsage() result.append((variable, variable_trace)) return result def onVariableContentEscapes(self, variable): self.getVariableCurrentTrace(variable).onValueEscape() def onExpression(self, expression, allow_none=False): if expression is None and allow_none: return None assert expression.isExpression(), expression parent = expression.parent assert parent, expression # Now compute this expression, allowing it to replace itself with # something else as part of a local peep hole optimization. r = expression.computeExpressionRaw(trace_collection=self) assert type(r) is tuple, expression new_node, change_tags, change_desc = r if change_tags is not None: # This is mostly for tracing and indication that a change occurred # and it may be interesting to look again. self.signalChange(change_tags, expression.getSourceReference(), change_desc) if new_node is not expression: parent.replaceChild(expression, new_node) return new_node def onStatement(self, statement): try: assert statement.isStatement(), statement new_statement, change_tags, change_desc = statement.computeStatement( self) # print new_statement, change_tags, change_desc if new_statement is not statement: self.signalChange(change_tags, statement.getSourceReference(), change_desc) return new_statement except Exception: Tracing.printError("Problem with statement at %s:\n-> %s" % ( statement.source_ref.getAsString(), readSourceLine(statement.source_ref), )) raise def mergeBranches(self, collection_yes, collection_no): """ Merge two alternative branches into this trace. This is mostly for merging conditional branches, or other ways of having alternative control flow. This deals with up to two alternative branches to both change this collection. """ # Refuse to do stupid work if collection_yes is None and collection_no is None: return None elif collection_yes is None or collection_no is None: # Handle one branch case, we need to merge versions backwards as # they may make themselves obsolete. return self.mergeMultipleBranches( collections=(self, collection_yes or collection_no)) else: return self.mergeMultipleBranches(collections=(collection_yes, collection_no)) def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None variable_versions = {} for collection in collections: for variable, version in iterItems(collection.variable_actives): if variable not in variable_versions: variable_versions[variable] = set([version]) else: variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} # merge_traces = None for variable, versions in iterItems(variable_versions): if len(versions) == 1: version, = versions else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=[ self.getVariableTrace(variable, version) for version in versions ], ) # if merge_traces is None: # merge_traces = [trace_merge] # else: # merge_traces.append(trace_merge) self.markCurrentVariableTrace(variable, version) # Return "None", or turn the list into a tuple for memory savings. # return merge_traces and tuple(merge_traces) def replaceBranch(self, collection_replace): self.variable_actives.update(collection_replace.variable_actives) collection_replace.variable_actives = None def onLoopBreak(self, collection=None): if collection is None: collection = self return self.parent.onLoopBreak(collection) def onLoopContinue(self, collection=None): if collection is None: collection = self return self.parent.onLoopContinue(collection) def onFunctionReturn(self, collection=None): if collection is None: collection = self return self.parent.onFunctionReturn(collection) def onExceptionRaiseExit(self, raisable_exceptions, collection=None): if collection is None: collection = self return self.parent.onExceptionRaiseExit(raisable_exceptions, collection) def getLoopBreakCollections(self): return self.parent.getLoopBreakCollections() def getLoopContinueCollections(self): return self.parent.getLoopContinueCollections() def getFunctionReturnCollections(self): return self.parent.getFunctionReturnCollections() def getExceptionRaiseCollections(self): return self.parent.getExceptionRaiseCollections() def makeAbortStackContext(self, catch_breaks, catch_continues, catch_returns, catch_exceptions): return self.parent.makeAbortStackContext( catch_breaks=catch_breaks, catch_continues=catch_continues, catch_returns=catch_returns, catch_exceptions=catch_exceptions, ) def onLocalsDictEscaped(self, locals_scope): self.parent.onLocalsDictEscaped(locals_scope) def getCompileTimeComputationResult(self, node, computation, description): new_node, change_tags, message = getComputationResult( node=node, computation=computation, description=description) if change_tags == "new_raise": self.onExceptionRaiseExit(BaseException) return new_node, change_tags, message def getIteratorNextCount(self, iter_node): return self.value_states.get(iter_node, None) def initIteratorValue(self, iter_node): # TODO: More complex state information will be needed eventually. self.value_states[iter_node] = 0 def onIteratorNext(self, iter_node): if iter_node in self.value_states: self.value_states[iter_node] += 1 def resetValueStates(self): for key in self.value_states: self.value_states[key] = None def addOutlineFunction(self, outline): self.parent.addOutlineFunction(outline)
class NodeBase(NodeMetaClassBase): kind = None @counted_init def __init__(self, source_ref): # The base class has no __init__ worth calling. # Check source reference to meet basic standards, so we note errors # when they occur. assert source_ref is not None assert source_ref.line is not None self.parent = None self.source_ref = source_ref __del__ = counted_del() def __repr__(self): # This is to avoid crashes, because of bugs in detail. # pylint: disable=W0703 try: detail = self.getDetail() except Exception as e: detail = "detail raises exception %s" % e if not detail: return "<Node %s>" % self.getDescription() else: return "<Node %s %s>" % (self.getDescription(), detail) def getDescription(self): """ Description of the node, intented for use in __repr__ and graphical display. """ return "%s at %s" % (self.kind, self.source_ref.getAsString()) def getDetails(self): """ Details of the node, intended for use in __repr__ and dumps. """ # Virtual method, pylint: disable=R0201 return {} def getDetail(self): """ Details of the node, intended for use in __repr__ and graphical display. """ return str(self.getDetails())[1:-1] def getParent(self): """ Parent of the node. Every node except modules have to have a parent. """ if self.parent is None and not self.isPythonModule(): assert False, (self, self.source_ref) return self.parent def getParents(self): """ Parents of the node. Up to module level. """ result = [] current = self while True: current = current.getParent() result.append(current) if current.isPythonModule() or current.isExpressionFunctionBody(): break assert None not in result, self result.reverse() return result def getChildName(self): """ Return the role in the current parent, subject to changes. """ parent = self.getParent() for key, value in parent.child_values.items(): if self is value: return key # TODO: Not checking tuples yet return None def getParentFunction(self): """ Return the parent that is a function. """ parent = self.getParent() while parent is not None and not parent.isExpressionFunctionBody(): parent = parent.getParent() return parent def getParentModule(self): """ Return the parent that is module. """ parent = self while not parent.isPythonModule(): if hasattr(parent, "provider"): # After we checked, we can use it, will be much faster route # to take. parent = parent.provider else: parent = parent.getParent() return parent def isParentVariableProvider(self): # Check if it's a closure giver, in which cases it can provide variables, return isinstance(self, ClosureGiverNodeBase) def getParentVariableProvider(self): parent = self.getParent() while not parent.isParentVariableProvider(): parent = parent.getParent() return parent def getParentStatementsFrame(self): current = self.getParent() while True: if current.isStatementsFrame(): return current if current.isParentVariableProvider(): return None current = current.getParent() def getSourceReference(self): return self.source_ref def setCompatibleSourceReference(self, source_ref): """ Bug compatible line numbers information. As CPython outputs the last bit of bytecode executed, and not the line of the operation. For example calls, output the line of the last argument, as opposed to the line of the operation start. For tests, we wants to be compatible. In improved more, we are not being fully compatible, and just drop it altogether. """ # Getting the same source reference can be dealt with quickly, so do # this first. if self.source_ref is not source_ref and \ Options.isFullCompat() and \ self.source_ref != source_ref: # An attribute outside of "__init__", so we save one memory for the # most cases. Very few cases involve splitting across lines. # pylint: disable=W0201 self.effective_source_ref = source_ref def getCompatibleSourceReference(self): """ Bug compatible line numbers information. See above. """ return getattr(self, "effective_source_ref", self.source_ref) def asXml(self): line = self.getSourceReference().getLineNumber() result = TreeXML.Element( "node", kind = self.__class__.__name__, line = "%s" % line ) compat_line = self.getCompatibleSourceReference().getLineNumber() if compat_line != line: result.attrib["compat_line"] = str(compat_line) for key, value in iterItems(self.getDetails()): value = str(value) if value.startswith('<') and value.endswith('>'): value = value[1:-1] result.set(key, str(value)) for name, children in self.getVisitableNodesNamed(): if type(children) not in (list, tuple): children = (children,) role = TreeXML.Element( "role", name = name ) result.append(role) for child in children: if child is not None: role.append( child.asXml() ) return result def dump(self, level = 0): Tracing.printIndented(level, self) Tracing.printSeparator(level) for visitable in self.getVisitableNodes(): visitable.dump(level + 1) Tracing.printSeparator(level) @staticmethod def isPythonModule(): # For overload by module nodes return False def isExpression(self): return self.kind.startswith("EXPRESSION_") def isStatement(self): return self.kind.startswith("STATEMENT_") def isExpressionBuiltin(self): return self.kind.startswith("EXPRESSION_BUILTIN_") def isOperation(self): return self.kind.startswith("EXPRESSION_OPERATION_") def isStatementReraiseException(self): # Virtual method, pylint: disable=R0201 return False def isExpressionMakeSequence(self): # Virtual method, pylint: disable=R0201 return False def isIteratorMaking(self): # Virtual method, pylint: disable=R0201 return False def isNumberConstant(self): # Virtual method, pylint: disable=R0201 return False def isExpressionCall(self): # Virtual method, pylint: disable=R0201 return False def visit(self, context, visitor): visitor(self) for visitable in self.getVisitableNodes(): visitable.visit(context, visitor) def getVisitableNodes(self): # Virtual method, pylint: disable=R0201 return () def getVisitableNodesNamed(self): # Virtual method, pylint: disable=R0201 return () def replaceWith(self, new_node): self.parent.replaceChild( old_node = self, new_node = new_node ) def getName(self): # Virtual method, pylint: disable=R0201 return None def mayHaveSideEffects(self): """ Unless we are told otherwise, everything may have a side effect. """ # Virtual method, pylint: disable=R0201 return True def isOrderRelevant(self): return self.mayHaveSideEffects() def mayHaveSideEffectsBool(self): """ Unless we are told otherwise, everything may have a side effect. """ # Virtual method, pylint: disable=R0201 return True def extractSideEffects(self): """ Unless defined otherwise, the expression is the side effect. """ return (self,) def mayRaiseException(self, exception_type): """ Unless we are told otherwise, everything may raise everything. """ # Virtual method, pylint: disable=R0201,W0613 return True def mayRaiseExceptionBool(self, exception_type): """ Unless we are told otherwise, everything may raise being checked. """ # Virtual method, pylint: disable=R0201,W0613 return True def mayReturn(self): return "_RETURN" in self.kind def mayBreak(self): # For overload, pylint: disable=R0201 return False def mayContinue(self): # For overload, pylint: disable=R0201 return False def needsFrame(self): """ Unless we are tolder otherwise, this depends on exception raise. """ return self.mayRaiseException(BaseException) def willRaiseException(self, exception_type): """ Unless we are told otherwise, nothing may raise anything. """ # Virtual method, pylint: disable=R0201,W0613 return False def isIndexable(self): """ Unless we are told otherwise, it's not indexable. """ # Virtual method, pylint: disable=R0201 return False def isStatementAborting(self): """ Is the node aborting, control flow doesn't continue after this node. """ assert self.isStatement(), self.kind return False def needsLocalsDict(self): """ Node requires a locals dictionary by provider. """ # Virtual method, pylint: disable=R0201 return False def getIntegerValue(self): """ Node as integer value, if possible.""" # Virtual method, pylint: disable=R0201 return None