class TraceCollectionBase(object): """This contains for logic for maintaining active traces. They are kept for "variable" and versions. """ __slots__ = ("owner", "parent", "name", "value_states", "variable_actives") if isCountingInstances(): __del__ = counted_del() @counted_init def __init__(self, owner, name, parent): self.owner = owner self.parent = parent self.name = name # Value state extra information per node. self.value_states = {} # Currently active values in the tracing. self.variable_actives = {} def __repr__(self): return "<%s for %s at 0x%x>" % (self.__class__.__name__, self.name, id(self)) def getOwner(self): return self.owner def getVariableCurrentTrace(self, variable): """Get the current value trace associated to this variable It is also created on the fly if necessary. We create them lazy so to keep the tracing branches minimal where possible. """ return self.getVariableTrace( variable=variable, version=self._getCurrentVariableVersion(variable) ) def markCurrentVariableTrace(self, variable, version): self.variable_actives[variable] = version def _getCurrentVariableVersion(self, variable): try: return self.variable_actives[variable] except KeyError: # Initialize variables on the fly. if not self.hasVariableTrace(variable, 0): self.initVariable(variable) self.markCurrentVariableTrace(variable, 0) return self.variable_actives[variable] def getActiveVariables(self): return self.variable_actives.keys() def markActiveVariableAsEscaped(self, variable): current = self.getVariableCurrentTrace(variable=variable) if not current.isUnknownTrace(): version = variable.allocateTargetNumber() self.addVariableTrace( variable=variable, version=version, trace=ValueTraceEscaped(owner=self.owner, previous=current), ) self.markCurrentVariableTrace(variable, version) def markActiveVariableAsLoopMerge( self, loop_node, current, variable, shapes, incomplete ): if incomplete: result = ValueTraceLoopIncomplete(loop_node, current, shapes) else: # TODO: Empty is a missing optimization somewhere, but it also happens that # a variable is getting released in a loop. # assert shapes, (variable, current) if not shapes: shapes.add(tshape_uninit) result = ValueTraceLoopComplete(loop_node, current, shapes) version = variable.allocateTargetNumber() self.addVariableTrace(variable=variable, version=version, trace=result) self.markCurrentVariableTrace(variable, version) return result def markActiveVariablesAsUnknown(self): for variable in self.getActiveVariables(): if variable.isTempVariable(): continue self.markActiveVariableAsEscaped(variable) @staticmethod def signalChange(tags, source_ref, message): # This is monkey patched from another module. pylint: disable=I0021,not-callable signalChange(tags, source_ref, message) def onUsedModule(self, module_name, module_relpath): return self.parent.onUsedModule(module_name, module_relpath) def onUsedFunction(self, function_body): owning_module = function_body.getParentModule() # Make sure the owning module is added to the used set. This is most # important for helper functions, or modules, which otherwise have # become unused. addUsedModule(owning_module) needs_visit = owning_module.addUsedFunction(function_body) if needs_visit: function_body.computeFunctionRaw(self) @staticmethod def mustAlias(a, b): if a.isExpressionVariableRef() and b.isExpressionVariableRef(): return a.getVariable() is b.getVariable() return False @staticmethod def mustNotAlias(a, b): # TODO: not yet really implemented if a.isExpressionConstantRef() and b.isExpressionConstantRef(): if a.isMutable() or b.isMutable(): return True return False def onControlFlowEscape(self, node): # TODO: One day, we should trace which nodes exactly cause a variable # to be considered escaped, pylint: disable=unused-argument for variable in self.getActiveVariables(): if variable.isModuleVariable(): # print variable self.markActiveVariableAsEscaped(variable) elif variable.isLocalVariable(): if variable.hasAccessesOutsideOf(self.owner) is not False: self.markActiveVariableAsEscaped(variable) def removeKnowledge(self, node): if node.isExpressionVariableRef(): self.markActiveVariableAsEscaped(node.variable) def onValueEscapeStr(self, node): # TODO: We can ignore these for now. pass def removeAllKnowledge(self): self.markActiveVariablesAsUnknown() def getVariableTrace(self, variable, version): return self.parent.getVariableTrace(variable, version) def hasVariableTrace(self, variable, version): return self.parent.hasVariableTrace(variable, version) def addVariableTrace(self, variable, version, trace): self.parent.addVariableTrace(variable, version, trace) def addVariableMergeMultipleTrace(self, variable, traces): return self.parent.addVariableMergeMultipleTrace(variable, traces) def onVariableSet(self, variable, version, assign_node): variable_trace = ValueTraceAssign( owner=self.owner, assign_node=assign_node, previous=self.getVariableCurrentTrace(variable=variable), ) self.addVariableTrace(variable=variable, version=version, trace=variable_trace) # Make references point to it. self.markCurrentVariableTrace(variable, version) return variable_trace def onVariableDel(self, variable, version, del_node): # Add a new trace, allocating a new version for the variable, and # remember the delete of the current old_trace = self.getVariableCurrentTrace(variable) variable_trace = ValueTraceDeleted( owner=self.owner, del_node=del_node, previous=old_trace ) # Assign to not initialized again. self.addVariableTrace(variable=variable, version=version, trace=variable_trace) # Make references point to it. self.markCurrentVariableTrace(variable, version) return variable_trace def onLocalsUsage(self, locals_scope): self.onLocalsDictEscaped(locals_scope) result = [] scope_locals_variables = locals_scope.getLocalsRelevantVariables() for variable in self.getActiveVariables(): if variable.isLocalVariable() and variable in scope_locals_variables: variable_trace = self.getVariableCurrentTrace(variable) variable_trace.addNameUsage() result.append((variable, variable_trace)) return result def onVariableContentEscapes(self, variable): self.markActiveVariableAsEscaped(variable) def onExpression(self, expression, allow_none=False): if expression is None and allow_none: return None assert expression.isExpression(), expression parent = expression.parent assert parent, expression # Now compute this expression, allowing it to replace itself with # something else as part of a local peep hole optimization. r = expression.computeExpressionRaw(trace_collection=self) assert type(r) is tuple, (expression, expression.getVisitableNodes(), r) new_node, change_tags, change_desc = r if change_tags is not None: # This is mostly for tracing and indication that a change occurred # and it may be interesting to look again. self.signalChange(change_tags, expression.getSourceReference(), change_desc) if new_node is not expression: parent.replaceChild(expression, new_node) return new_node def onStatement(self, statement): try: assert statement.isStatement(), statement new_statement, change_tags, change_desc = statement.computeStatement(self) # print new_statement, change_tags, change_desc if new_statement is not statement: self.signalChange( change_tags, statement.getSourceReference(), change_desc ) return new_statement except Exception: Tracing.printError( "Problem with statement at %s:\n-> %s" % ( statement.source_ref.getAsString(), readSourceLine(statement.source_ref), ) ) raise def computedStatementResult(self, statement, change_tags, change_desc): """Make sure the replacement statement is computed. Use this when a replacement expression needs to be seen by the trace collection and be computed, without causing any duplication, but where otherwise there would be loss of annotated effects. This may e.g. be true for nodes that need an initial run to know their exception result and type shape. """ # Need to compute the replacement still. new_statement = statement.computeStatement(self) if new_statement[0] is not statement: # Signal intermediate result as well. self.signalChange(change_tags, statement.getSourceReference(), change_desc) return new_statement else: return statement, change_tags, change_desc def computedExpressionResult(self, expression, change_tags, change_desc): """Make sure the replacement expression is computed. Use this when a replacement expression needs to be seen by the trace collection and be computed, without causing any duplication, but where otherwise there would be loss of annotated effects. This may e.g. be true for nodes that need an initial run to know their exception result and type shape. """ # Need to compute the replacement still. new_expression = expression.computeExpression(self) if new_expression[0] is not expression: # Signal intermediate result as well. self.signalChange(change_tags, expression.getSourceReference(), change_desc) return new_expression else: return expression, change_tags, change_desc def mergeBranches(self, collection_yes, collection_no): """Merge two alternative branches into this trace. This is mostly for merging conditional branches, or other ways of having alternative control flow. This deals with up to two alternative branches to both change this collection. """ # Many branches due to inlining the actual merge and preparing it # pylint: disable=too-many-branches if collection_yes is None: if collection_no is not None: # Handle one branch case, we need to merge versions backwards as # they may make themselves obsolete. collection1 = self collection2 = collection_no else: # Refuse to do stupid work return elif collection_no is None: # Handle one branch case, we need to merge versions backwards as # they may make themselves obsolete. collection1 = self collection2 = collection_yes else: # Handle two branch case, they may or may not do the same things. collection1 = collection_yes collection2 = collection_no variable_versions = {} for variable, version in iterItems(collection1.variable_actives): variable_versions[variable] = version for variable, version in iterItems(collection2.variable_actives): if variable not in variable_versions: variable_versions[variable] = 0, version else: other = variable_versions[variable] if other != version: variable_versions[variable] = other, version else: variable_versions[variable] = other for variable in variable_versions: if variable not in collection2.variable_actives: variable_versions[variable] = variable_versions[variable], 0 self.variable_actives = {} for variable, versions in iterItems(variable_versions): if type(versions) is tuple: version = self.addVariableMergeMultipleTrace( variable=variable, traces=( self.getVariableTrace(variable, versions[0]), self.getVariableTrace(variable, versions[1]), ), ) else: version = versions self.markCurrentVariableTrace(variable, version) def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None variable_versions = {} for collection in collections: for variable, version in iterItems(collection.variable_actives): if variable not in variable_versions: variable_versions[variable] = OrderedSet((version,)) else: variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} for variable, versions in iterItems(variable_versions): if len(versions) == 1: (version,) = versions else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=tuple( self.getVariableTrace(variable, version) for version in versions ), ) self.markCurrentVariableTrace(variable, version) def replaceBranch(self, collection_replace): self.variable_actives.update(collection_replace.variable_actives) collection_replace.variable_actives = None def onLoopBreak(self, collection=None): if collection is None: collection = self return self.parent.onLoopBreak(collection) def onLoopContinue(self, collection=None): if collection is None: collection = self return self.parent.onLoopContinue(collection) def onFunctionReturn(self, collection=None): if collection is None: collection = self return self.parent.onFunctionReturn(collection) def onExceptionRaiseExit(self, raisable_exceptions, collection=None): if collection is None: collection = self return self.parent.onExceptionRaiseExit(raisable_exceptions, collection) def getLoopBreakCollections(self): return self.parent.getLoopBreakCollections() def getLoopContinueCollections(self): return self.parent.getLoopContinueCollections() def getFunctionReturnCollections(self): return self.parent.getFunctionReturnCollections() def getExceptionRaiseCollections(self): return self.parent.getExceptionRaiseCollections() def makeAbortStackContext( self, catch_breaks, catch_continues, catch_returns, catch_exceptions ): return self.parent.makeAbortStackContext( catch_breaks=catch_breaks, catch_continues=catch_continues, catch_returns=catch_returns, catch_exceptions=catch_exceptions, ) def onLocalsDictEscaped(self, locals_scope): self.parent.onLocalsDictEscaped(locals_scope) def getCompileTimeComputationResult(self, node, computation, description): new_node, change_tags, message = getComputationResult( node=node, computation=computation, description=description, user_provided=False, ) if change_tags == "new_raise": self.onExceptionRaiseExit(BaseException) return new_node, change_tags, message def getIteratorNextCount(self, iter_node): return self.value_states.get(iter_node) def initIteratorValue(self, iter_node): # TODO: More complex state information will be needed eventually. self.value_states[iter_node] = 0 def onIteratorNext(self, iter_node): if iter_node in self.value_states: self.value_states[iter_node] += 1 def resetValueStates(self): for key in self.value_states: self.value_states[key] = None def addOutlineFunction(self, outline): self.parent.addOutlineFunction(outline)
class SourceCodeReference(object): __slots__ = ["filename", "line", "column"] @classmethod def fromFilenameAndLine(cls, filename, line): result = cls() result.filename = filename result.line = line return result if isCountingInstances(): __del__ = counted_del() @counted_init def __init__(self): self.filename = None self.line = None self.column = None def __repr__(self): return "<%s to %s:%s>" % (self.__class__.__name__, self.filename, self.line) def __hash__(self): return hash((self.filename, self.line, self.column)) def __lt__(self, other): # Many cases decide early, pylint: disable=too-many-return-statements if other is None: return True if other is self: return False assert isinstance(other, SourceCodeReference), other if self.filename < other.filename: return True elif self.filename > other.filename: return False else: if self.line < other.line: return True elif self.line > other.line: return False else: if self.column < other.column: return True elif self.column > other.column: return False else: return self.isInternal() < other.isInternal() def __eq__(self, other): if other is None: return False if other is self: return True assert isinstance(other, SourceCodeReference), other if self.filename != other.filename: return False if self.line != other.line: return False if self.column != other.column: return False return self.isInternal() is other.isInternal() def _clone(self, line): """Make a copy it itself.""" return self.fromFilenameAndLine(filename=self.filename, line=line) def atInternal(self): """Make a copy it itself but mark as internal code. Avoids useless copies, by returning an internal object again if it is already internal. """ if not self.isInternal(): result = self._clone(self.line) return result else: return self def atLineNumber(self, line): """Make a reference to the same file, but different line. Avoids useless copies, by returning same object if the line is the same. """ assert type(line) is int, line if self.line != line: return self._clone(line) else: return self def atColumnNumber(self, column): assert type(column) is int, column if self.column != column: result = self._clone(self.line) result.column = column return result else: return self def getLineNumber(self): return self.line def getColumnNumber(self): return self.column def getFilename(self): return self.filename def getAsString(self): return "%s:%s" % (self.filename, self.line) @staticmethod def isInternal(): return False
class LocalsDictHandleBase(object): # TODO: Might remove some of these later, pylint: disable=too-many-instance-attributes __slots__ = ( "locals_name", # TODO: Specialize what the kinds really use. "variables", "local_variables", "providing", "mark_for_propagation", "propagation", "owner", "complete", ) @counted_init def __init__(self, locals_name, owner): self.locals_name = locals_name self.owner = owner # For locals dict variables in this scope. self.variables = {} # For local variables in this scope. self.local_variables = {} self.providing = OrderedDict() # Can this be eliminated through replacement of temporary variables self.mark_for_propagation = False self.propagation = None self.complete = False if isCountingInstances(): __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.locals_name) def getName(self): return self.locals_name def makeClone(self, new_owner): count = 1 # Make it unique. while 1: locals_name = self.locals_name + "_inline_%d" % count if locals_name not in locals_dict_handles: break count += 1 result = self.__class__(locals_name=locals_name, owner=new_owner) variable_translation = {} # Clone variables as well. for variable_name, variable in self.variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.variables[variable_name] = new_variable for variable_name, variable in self.local_variables.items(): new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.local_variables[variable_name] = new_variable result.providing = OrderedDict() for variable_name, variable in self.providing.items(): if variable in variable_translation: new_variable = variable_translation[variable] else: new_variable = variable.makeClone(new_owner=new_owner) variable_translation[variable] = new_variable result.providing[variable_name] = new_variable return result, variable_translation @staticmethod def getTypeShape(): return tshape_dict def getCodeName(self): return self.locals_name @staticmethod def isModuleScope(): return False @staticmethod def isClassScope(): return False @staticmethod def isFunctionScope(): return False def getProvidedVariables(self): return self.providing.values() def registerProvidedVariable(self, variable): variable_name = variable.getName() self.providing[variable_name] = variable def unregisterProvidedVariable(self, variable): """Remove provided variable, e.g. because it became unused.""" variable_name = variable.getName() if variable_name in self.providing: del self.providing[variable_name] registerClosureVariable = registerProvidedVariable unregisterClosureVariable = unregisterProvidedVariable def hasProvidedVariable(self, variable_name): """Test if a variable is provided.""" return variable_name in self.providing def getProvidedVariable(self, variable_name): """Test if a variable is provided.""" return self.providing[variable_name] def getLocalsRelevantVariables(self): """The variables relevant to locals.""" return self.providing.values() def getLocalsDictVariable(self, variable_name): if variable_name not in self.variables: result = Variables.LocalsDictVariable(owner=self, variable_name=variable_name) self.variables[variable_name] = result return self.variables[variable_name] # TODO: Have variable ownership moved to the locals scope, so owner becomes not needed here. def getLocalVariable(self, owner, variable_name): if variable_name not in self.local_variables: result = Variables.LocalVariable(owner=owner, variable_name=variable_name) self.local_variables[variable_name] = result return self.local_variables[variable_name] def markForLocalsDictPropagation(self): self.mark_for_propagation = True def isMarkedForPropagation(self): return self.mark_for_propagation def allocateTempReplacementVariable(self, trace_collection, variable_name): if self.propagation is None: self.propagation = OrderedDict() if variable_name not in self.propagation: provider = trace_collection.getOwner() self.propagation[variable_name] = provider.allocateTempVariable( temp_scope=None, name=self.getCodeName() + "_key_" + variable_name) return self.propagation[variable_name] def getPropagationVariables(self): if self.propagation is None: return () return self.propagation def finalize(self): # Make it unusable when it's become empty, not used. self.owner.locals_scope = None del self.owner del self.propagation del self.mark_for_propagation for variable in self.variables.values(): variable.finalize() for variable in self.local_variables.values(): variable.finalize() del self.variables del self.providing def markAsComplete(self, trace_collection): self.complete = True self._considerUnusedUserLocalVariables(trace_collection) self._considerPropagation(trace_collection) # TODO: Limited to Python2 classes for now, more overloads need to be added, this # ought to be abstract and have variants with TODOs for each of them. @staticmethod def _considerPropagation(trace_collection): """For overload by scope type. Check if this can be replaced.""" def _considerUnusedUserLocalVariables(self, trace_collection): """Check scope for unused variables.""" provided = self.getProvidedVariables() removals = [] for variable in provided: if (variable.isLocalVariable() and not variable.isParameterVariable() and variable.getOwner() is self.owner): empty = trace_collection.hasEmptyTraces(variable) if empty: removals.append(variable) for variable in removals: self.unregisterProvidedVariable(variable) trace_collection.signalChange( "var_usage", self.owner.getSourceReference(), message="Remove unused local variable '%s'." % variable.getName(), )
class Variable(getMetaClassBase("Variable")): # We will need all of these attributes, since we track the global # state and cache some decisions as attributes. TODO: But in some # cases, part of the these might be moved to the outside. __slots__ = ( "variable_name", "owner", "version_number", "shared_users", "traces", "users", "writers", ) @counted_init def __init__(self, owner, variable_name): assert type(variable_name) is str, variable_name assert type(owner) not in (tuple, list), owner self.variable_name = variable_name self.owner = owner self.version_number = 0 self.shared_users = False self.traces = set() # Derived from all traces. self.users = None self.writers = None if isCountingInstances(): __del__ = counted_del() def finalize(self): del self.users del self.writers del self.traces del self.owner def __repr__(self): return "<%s '%s' of '%s'>" % ( self.__class__.__name__, self.variable_name, self.owner.getName(), ) @abstractmethod def getVariableType(self): pass def getDescription(self): return "variable '%s'" % self.variable_name def getName(self): return self.variable_name def getOwner(self): return self.owner def getEntryPoint(self): return self.owner.getEntryPoint() def getCodeName(self): var_name = self.variable_name var_name = var_name.replace(".", "$") var_name = Utils.encodeNonAscii(var_name) return var_name def allocateTargetNumber(self): self.version_number += 1 return self.version_number @staticmethod def isLocalVariable(): return False @staticmethod def isParameterVariable(): return False @staticmethod def isModuleVariable(): return False @staticmethod def isIncompleteModuleVariable(): return False @staticmethod def isTempVariable(): return False @staticmethod def isTempVariableBool(): return False @staticmethod def isLocalsDictVariable(): return False def addVariableUser(self, user): # Update the shared scopes flag. if user is not self.owner: self.shared_users = True # These are not really scopes, just shared uses. if (user.isExpressionGeneratorObjectBody() or user.isExpressionCoroutineObjectBody() or user.isExpressionAsyncgenObjectBody()): if self.owner is user.getParentVariableProvider(): return _variables_in_shared_scopes.add(self) def isSharedTechnically(self): if not self.shared_users: return False if not self.users: return False owner = self.owner.getEntryPoint() for user in self.users: user = user.getEntryPoint() while user is not owner and ( (user.isExpressionFunctionBody() and not user.needsCreation()) or user.isExpressionClassBody()): user = user.getParentVariableProvider() if user is not owner: return True return False def addTrace(self, variable_trace): self.traces.add(variable_trace) def removeTrace(self, variable_trace): self.traces.remove(variable_trace) def updateUsageState(self): writers = set() users = set() for trace in self.traces: owner = trace.owner users.add(owner) if trace.isAssignTrace(): writers.add(owner) elif trace.isDeletedTrace() and owner is not self.owner: writers.add(owner) self.writers = writers self.users = users def hasAccessesOutsideOf(self, provider): if not self.owner.locals_scope.complete: return None elif self.users is None: return False elif provider in self.users: return len(self.users) > 1 else: return bool(self.users) def getMatchingAssignTrace(self, assign_node): for trace in self.traces: if trace.isAssignTrace() and trace.getAssignNode() is assign_node: return trace return None def getMatchingDelTrace(self, del_node): for trace in self.traces: if trace.isDeletedTrace() and trace.getDelNode() is del_node: return trace return None def getTypeShapes(self): result = set() for trace in self.traces: if trace.isAssignTrace(): result.add(trace.getAssignNode().getTypeShape()) elif trace.isUnknownTrace(): result.add(tshape_unknown) elif trace.isInitTrace(): result.add(tshape_unknown) elif trace.isUnassignedTrace(): pass elif trace.isMergeTrace(): pass # TODO: Remove this and be not unknown. elif trace.isLoopTrace(): trace.getTypeShape().emitAlternatives(result.add) else: assert False, trace return result
class PythonContextBase(getMetaClassBase("Context")): @counted_init def __init__(self): self.source_ref = None self.current_source_ref = None if isCountingInstances(): __del__ = counted_del() def getCurrentSourceCodeReference(self): return self.current_source_ref def setCurrentSourceCodeReference(self, value): result = self.current_source_ref self.current_source_ref = value return result @contextmanager def withCurrentSourceCodeReference(self, value): old_source_ref = self.setCurrentSourceCodeReference(value) yield old_source_ref self.setCurrentSourceCodeReference(value) def getInplaceLeftName(self): return self.allocateTempName("inplace_orig", "PyObject *", True) @abstractmethod def getConstantCode(self, constant, deep_check=False): pass @abstractmethod def getModuleCodeName(self): pass @abstractmethod def getModuleName(self): pass @abstractmethod def addHelperCode(self, key, code): pass @abstractmethod def hasHelperCode(self, key): pass @abstractmethod def addDeclaration(self, key, code): pass @abstractmethod def pushFrameVariables(self, frame_variables): pass @abstractmethod def popFrameVariables(self): pass @abstractmethod def getFrameVariableTypeDescriptions(self): pass @abstractmethod def getFrameVariableTypeDescription(self): pass @abstractmethod def getFrameTypeDescriptionDeclaration(self): pass @abstractmethod def getFrameVariableCodeNames(self): pass @abstractmethod def allocateTempName(self, base_name, type_name="PyObject *", unique=False): pass @abstractmethod def skipTempName(self, base_name): pass @abstractmethod def getIntResName(self): pass @abstractmethod def getBoolResName(self): pass @abstractmethod def hasTempName(self, base_name): pass @abstractmethod def getExceptionEscape(self): pass @abstractmethod def setExceptionEscape(self, label): pass @abstractmethod def getLoopBreakTarget(self): pass @abstractmethod def setLoopBreakTarget(self, label): pass @abstractmethod def getLoopContinueTarget(self): pass @abstractmethod def setLoopContinueTarget(self, label): pass @abstractmethod def allocateLabel(self, label): pass @abstractmethod def allocateExceptionKeeperVariables(self): pass @abstractmethod def getExceptionKeeperVariables(self): pass @abstractmethod def setExceptionKeeperVariables(self, keeper_vars): pass @abstractmethod def addExceptionPreserverVariables(self, count): pass @abstractmethod def getTrueBranchTarget(self): pass @abstractmethod def getFalseBranchTarget(self): pass @abstractmethod def setTrueBranchTarget(self, label): pass @abstractmethod def setFalseBranchTarget(self, label): pass @abstractmethod def getCleanupTempnames(self): pass @abstractmethod def addCleanupTempName(self, tmp_name): pass @abstractmethod def removeCleanupTempName(self, tmp_name): pass @abstractmethod def needsCleanup(self, tmp_name): pass @abstractmethod def pushCleanupScope(self): pass @abstractmethod def popCleanupScope(self): pass
class NodeBase(NodeMetaClassBase): __slots__ = "parent", "source_ref" # This can trigger if this is included to early. assert Options.is_fullcompat is not None # Avoid the attribute unless it's really necessary. if Options.is_fullcompat: __slots__ += ("effective_source_ref", ) # String to identify the node class, to be consistent with its name. kind = None @counted_init def __init__(self, source_ref): # The base class has no __init__ worth calling. # Check source reference to meet basic standards, so we note errors # when they occur. assert source_ref is not None assert source_ref.line is not None self.parent = None self.source_ref = source_ref if isCountingInstances(): __del__ = counted_del() @abstractmethod def finalize(self): pass def __repr__(self): return "<Node %s>" % (self.getDescription()) def getDescription(self): """Description of the node, intended for use in __repr__ and graphical display. """ details = self.getDetailsForDisplay() if details: return "'%s' with %s" % (self.kind, details) else: return "'%s'" % self.kind def getDetails(self): """Details of the node, intended for re-creation. We are not using the pickle mechanisms, but this is basically part of what the constructor call needs. Real children will also be added. """ # Virtual method, pylint: disable=no-self-use return {} def getDetailsForDisplay(self): """Details of the node, intended for use in __repr__ and dumps. This is also used for XML. """ return self.getDetails() def getCloneArgs(self): return self.getDetails() def makeClone(self): try: # Using star dictionary arguments here for generic use. result = self.__class__(source_ref=self.source_ref, **self.getCloneArgs()) except TypeError as e: raise NuitkaNodeError("Problem cloning node", self, e) effective_source_ref = self.getCompatibleSourceReference() if effective_source_ref is not self.source_ref: result.setCompatibleSourceReference(effective_source_ref) return result def makeCloneShallow(self): args = self.getDetails() args.update(self.getVisitableNodesNamed()) try: # Using star dictionary arguments here for generic use. result = self.__class__(source_ref=self.source_ref, **args) except TypeError as e: raise NuitkaNodeError("Problem cloning node", self, e) effective_source_ref = self.getCompatibleSourceReference() if effective_source_ref is not self.source_ref: result.setCompatibleSourceReference(effective_source_ref) return result def getParent(self): """Parent of the node. Every node except modules has to have a parent.""" if self.parent is None and not self.isCompiledPythonModule(): assert False, (self, self.source_ref) return self.parent def getChildName(self): """Return the role in the current parent, subject to changes.""" parent = self.getParent() for key, value in parent.getVisitableNodesNamed(): if self is value: return key if type(value) is tuple: if self in value: return key, value.index(self) return None def getChildNameNice(self): child_name = self.getChildName() if hasattr(self.parent, "nice_children"): return self.parent.nice_children[self.parent.named_children.index( child_name)] elif hasattr(self.parent, "nice_child"): return self.parent.nice_child else: return child_name def getParentFunction(self): """Return the parent that is a function.""" parent = self.getParent() while parent is not None and not parent.isExpressionFunctionBodyBase(): parent = parent.getParent() return parent def getParentModule(self): """Return the parent that is module.""" parent = self while not parent.isCompiledPythonModule(): if hasattr(parent, "provider"): # After we checked, we can use it, will be much faster route # to take. parent = parent.provider else: parent = parent.getParent() return parent def isParentVariableProvider(self): # Check if it's a closure giver, in which cases it can provide variables, return isinstance(self, ClosureGiverNodeMixin) def getParentVariableProvider(self): parent = self.getParent() while not parent.isParentVariableProvider(): parent = parent.getParent() return parent def getParentReturnConsumer(self): parent = self.getParent() while (not parent.isParentVariableProvider() and not parent.isExpressionOutlineBody()): parent = parent.getParent() return parent def getParentStatementsFrame(self): current = self.getParent() while True: if current.isStatementsFrame(): return current if current.isParentVariableProvider(): return None if current.isExpressionOutlineBody(): return None current = current.getParent() def getSourceReference(self): return self.source_ref def setCompatibleSourceReference(self, source_ref): """Bug compatible line numbers information. As CPython outputs the last bit of bytecode executed, and not the line of the operation. For example calls, output the line of the last argument, as opposed to the line of the operation start. For tests, we wants to be compatible. In improved more, we are not being fully compatible, and just drop it altogether. """ # Getting the same source reference can be dealt with quickly, so do # this first. if (self.source_ref is not source_ref and Options.is_fullcompat and self.source_ref != source_ref): # An attribute outside of "__init__", so we save one memory for the # most cases. Very few cases involve splitting across lines. # false alarm for non-slot: # pylint: disable=I0021,assigning-non-slot,attribute-defined-outside-init self.effective_source_ref = source_ref def getCompatibleSourceReference(self): """Bug compatible line numbers information. See above. """ return getattr(self, "effective_source_ref", self.source_ref) def asXml(self): line = self.source_ref.getLineNumber() result = TreeXML.Element("node", kind=self.__class__.__name__, line=str(line)) compat_line = self.getCompatibleSourceReference().getLineNumber() if compat_line != line: result.attrib["compat_line"] = str(compat_line) for key, value in iterItems(self.getDetailsForDisplay()): result.set(key, str(value)) for name, children in self.getVisitableNodesNamed(): role = TreeXML.Element("role", name=name) result.append(role) if children is None: role.attrib["type"] = "none" elif type(children) not in (list, tuple): role.append(children.asXml()) else: role.attrib["type"] = "list" for child in children: role.append(child.asXml()) return result @classmethod def fromXML(cls, provider, source_ref, **args): # Only some things need a provider, pylint: disable=unused-argument return cls(source_ref=source_ref, **args) def asXmlText(self): xml = self.asXml() return TreeXML.toString(xml) def dump(self, level=0): Tracing.printIndented(level, self) Tracing.printSeparator(level) for visitable in self.getVisitableNodes(): visitable.dump(level + 1) Tracing.printSeparator(level) @staticmethod def isStatementsFrame(): return False @staticmethod def isCompiledPythonModule(): # For overload by module nodes return False def isExpression(self): return self.kind.startswith("EXPRESSION_") def isStatement(self): return self.kind.startswith("STATEMENT_") def isExpressionBuiltin(self): return self.kind.startswith("EXPRESSION_BUILTIN_") @staticmethod def isExpressionConstantRef(): return False @staticmethod def isExpressionConstantBoolRef(): return False @staticmethod def isExpressionOperationUnary(): return False @staticmethod def isExpressionOperationBinary(): return False @staticmethod def isExpressionOperationInplace(): return False @staticmethod def isExpressionComparison(): return False @staticmethod def isExpressionSideEffects(): return False @staticmethod def isExpressionMakeSequence(): return False @staticmethod def isNumberConstant(): return False @staticmethod def isExpressionCall(): return False @staticmethod def isExpressionFunctionBodyBase(): return False @staticmethod def isExpressionOutlineFunctionBase(): return False def visit(self, context, visitor): visitor(self) for visitable in self.getVisitableNodes(): visitable.visit(context, visitor) @staticmethod def getVisitableNodes(): return () @staticmethod def getVisitableNodesNamed(): """Named children dictionary. For use in debugging and XML output. """ return () @staticmethod def getName(): """Name of the node if any.""" return None @staticmethod def mayHaveSideEffects(): """Unless we are told otherwise, everything may have a side effect.""" return True def isOrderRelevant(self): return self.mayHaveSideEffects() def extractSideEffects(self): """Unless defined otherwise, the expression is the side effect.""" return (self, ) @staticmethod def mayRaiseException(exception_type): """Unless we are told otherwise, everything may raise everything.""" # Virtual method, pylint: disable=unused-argument return True @staticmethod def mayReturn(): """May this node do a return exit, to be overloaded for things that might.""" return False @staticmethod def mayBreak(): return False @staticmethod def mayContinue(): return False def needsFrame(self): """Unless we are tolder otherwise, this depends on exception raise.""" return self.mayRaiseException(BaseException) @staticmethod def willRaiseException(exception_type): """Unless we are told otherwise, nothing may raise anything.""" # Virtual method, pylint: disable=unused-argument return False @staticmethod def isStatementAborting(): """Is the node aborting, control flow doesn't continue after this node.""" return False
class FutureSpec(object): __slots__ = ( "future_division", "unicode_literals", "absolute_import", "future_print", "barry_bdfl", "generator_stop", "future_annotations", ) @counted_init def __init__(self): self.future_division = _future_division_default self.unicode_literals = False self.absolute_import = _future_absolute_import_default self.future_print = False self.barry_bdfl = False self.generator_stop = _future_generator_stop_default self.future_annotations = _future_annotations_default if isCountingInstances(): __del__ = counted_del() def __repr__(self): return "<FutureSpec %s>" % ",".join(self.asFlags()) def clone(self): result = FutureSpec() result.future_division = self.future_division result.unicode_literals = self.unicode_literals result.absolute_import = self.absolute_import result.future_print = self.future_print result.barry_bdfl = self.barry_bdfl result.generator_stop = self.generator_stop result.future_annotations = result.future_annotations return result def isFutureDivision(self): return self.future_division def enableFutureDivision(self): self.future_division = True def isFuturePrint(self): return self.future_print def enableFuturePrint(self): self.future_print = True def enableUnicodeLiterals(self): self.unicode_literals = True def enableAbsoluteImport(self): self.absolute_import = True def enableBarry(self): self.barry_bdfl = True def enableGeneratorStop(self): self.generator_stop = True def isAbsoluteImport(self): return self.absolute_import def isGeneratorStop(self): return self.generator_stop def enableFutureAnnotations(self): self.future_annotations = True def isFutureAnnotations(self): return self.future_annotations def asFlags(self): """Create a list of C identifiers to represent the flag values. This is for use in code generation and to restore from saved modules. """ result = [] if python_version < 0x300 and self.future_division: result.append("CO_FUTURE_DIVISION") if self.unicode_literals: result.append("CO_FUTURE_UNICODE_LITERALS") if python_version < 0x300 and self.absolute_import: result.append("CO_FUTURE_ABSOLUTE_IMPORT") if python_version < 0x300 and self.future_print: result.append("CO_FUTURE_PRINT_FUNCTION") if python_version >= 0x300 and self.barry_bdfl: result.append("CO_FUTURE_BARRY_AS_BDFL") if 0x350 <= python_version < 0x370 and self.generator_stop: result.append("CO_FUTURE_GENERATOR_STOP") if python_version >= 0x370 and self.future_annotations: result.append("CO_FUTURE_ANNOTATIONS") return tuple(result)
class ValueTraceBase(object): # We are going to have many instance attributes, but should strive to minimize, as # there is going to be a lot of fluctuation in these objects. __slots__ = ( "owner", "usage_count", "name_usage_count", "merge_usage_count", "closure_usages", "previous", ) @counted_init def __init__(self, owner, previous): self.owner = owner # Definite usage indicator. self.usage_count = 0 # If 0, this indicates, the variable name needs to be assigned as name. self.name_usage_count = 0 # If 0, this indicates no value merges happened on the value. self.merge_usage_count = 0 self.closure_usages = False # Previous trace this is replacing. self.previous = previous if isCountingInstances(): __del__ = counted_del() def __repr__(self): return "<%s of %s>" % (self.__class__.__name__, self.owner.getCodeName()) def getOwner(self): return self.owner @staticmethod def isLoopTrace(): return False def addUsage(self): self.usage_count += 1 def addNameUsage(self): self.usage_count += 1 self.name_usage_count += 1 if self.name_usage_count <= 2 and self.previous is not None: self.previous.addNameUsage() def addMergeUsage(self): self.usage_count += 1 self.merge_usage_count += 1 def getUsageCount(self): return self.usage_count def getNameUsageCount(self): return self.name_usage_count def getMergeUsageCount(self): return self.merge_usage_count def getMergeOrNameUsageCount(self): return self.merge_usage_count + self.name_usage_count def getPrevious(self): return self.previous @staticmethod def isAssignTrace(): return False @staticmethod def isUnassignedTrace(): return False @staticmethod def isDeletedTrace(): return False @staticmethod def isUninitTrace(): return False @staticmethod def isInitTrace(): return False @staticmethod def isUnknownTrace(): return False @staticmethod def isMergeTrace(): return False def mustHaveValue(self): """Will this definitely have a value. Every trace has this overloaded. """ assert False, self def mustNotHaveValue(self): """Will this definitely have a value. Every trace has this overloaded. """ assert False, self def getReplacementNode(self, usage): # Virtual method, pylint: disable=no-self-use,unused-argument return None @staticmethod def hasShapeDictionaryExact(): return False @staticmethod def getTruthValue(): return None
class ParameterSpec(object): # These got many attributes, in part duplicating name and instance of # variables, pylint: disable=too-many-instance-attributes __slots__ = ( "name", "owner", "normal_args", "normal_variables", "list_star_arg", "dict_star_arg", "list_star_variable", "dict_star_variable", "default_count", "kw_only_args", "kw_only_variables", "pos_only_args", "pos_only_variables", ) @counted_init def __init__( self, ps_name, ps_normal_args, ps_pos_only_args, ps_kw_only_args, ps_list_star_arg, ps_dict_star_arg, ps_default_count, ): if type(ps_normal_args) is str: if ps_normal_args == "": ps_normal_args = () else: ps_normal_args = ps_normal_args.split(",") if type(ps_kw_only_args) is str: if ps_kw_only_args == "": ps_kw_only_args = () else: ps_kw_only_args = ps_kw_only_args.split(",") assert None not in ps_normal_args self.owner = None self.name = ps_name self.normal_args = tuple(ps_normal_args) self.normal_variables = None assert (ps_list_star_arg is None or type(ps_list_star_arg) is str), ps_list_star_arg assert (ps_dict_star_arg is None or type(ps_dict_star_arg) is str), ps_dict_star_arg self.list_star_arg = ps_list_star_arg if ps_list_star_arg else None self.dict_star_arg = ps_dict_star_arg if ps_dict_star_arg else None self.list_star_variable = None self.dict_star_variable = None self.default_count = ps_default_count self.kw_only_args = tuple(ps_kw_only_args) self.kw_only_variables = None self.pos_only_args = tuple(ps_pos_only_args) self.pos_only_variables = None if isCountingInstances(): __del__ = counted_del() def makeClone(self): return ParameterSpec( ps_name=self.name, ps_normal_args=self.normal_args, ps_pos_only_args=self.pos_only_args, ps_kw_only_args=self.kw_only_args, ps_list_star_arg=self.list_star_arg, ps_dict_star_arg=self.dict_star_arg, ps_default_count=self.default_count, ) def getDetails(self): return { "ps_name": self.name, "ps_normal_args": ",".join(self.normal_args), "ps_pos_only_args": self.pos_only_args, "ps_kw_only_args": ",".join(self.kw_only_args), "ps_list_star_arg": self.list_star_arg if self.list_star_arg is not None else "", "ps_dict_star_arg": self.dict_star_arg if self.dict_star_arg is not None else "", "ps_default_count": self.default_count, } def checkParametersValid(self): arg_names = self.getParameterNames() # Check for duplicate arguments, could happen. for arg_name in arg_names: if arg_names.count(arg_name) != 1: return "duplicate argument '%s' in function definition" % arg_name return None def __repr__(self): parts = [str(normal_arg) for normal_arg in self.pos_only_args] if parts: parts.append("/") parts += [str(normal_arg) for normal_arg in self.normal_args] if self.list_star_arg is not None: parts.append("*%s" % self.list_star_arg) if self.dict_star_variable is not None: parts.append("**%s" % self.dict_star_variable) if parts: return "<ParameterSpec '%s'>" % ",".join(parts) else: return "<NoParameters>" def setOwner(self, owner): if self.owner is not None: return self.owner = owner self.normal_variables = [] for normal_arg in self.normal_args: if type(normal_arg) is str: normal_variable = Variables.ParameterVariable( owner=self.owner, parameter_name=normal_arg) else: assert False, normal_arg self.normal_variables.append(normal_variable) if self.list_star_arg: self.list_star_variable = Variables.ParameterVariable( owner=owner, parameter_name=self.list_star_arg) else: self.list_star_variable = None if self.dict_star_arg: self.dict_star_variable = Variables.ParameterVariable( owner=owner, parameter_name=self.dict_star_arg) else: self.dict_star_variable = None self.kw_only_variables = [ Variables.ParameterVariable(owner=self.owner, parameter_name=kw_only_arg) for kw_only_arg in self.kw_only_args ] self.pos_only_variables = [ Variables.ParameterVariable(owner=self.owner, parameter_name=pos_only_arg) for pos_only_arg in self.pos_only_args ] def getDefaultCount(self): return self.default_count def hasDefaultParameters(self): return self.getDefaultCount() > 0 def getTopLevelVariables(self): return self.pos_only_variables + self.normal_variables + self.kw_only_variables def getAllVariables(self): result = list(self.pos_only_variables) result += self.normal_variables result += self.kw_only_variables if self.list_star_variable is not None: result.append(self.list_star_variable) if self.dict_star_variable is not None: result.append(self.dict_star_variable) return result def getParameterNames(self): result = list(self.pos_only_args + self.normal_args) result += self.kw_only_args if self.list_star_arg is not None: result.append(self.list_star_arg) if self.dict_star_arg is not None: result.append(self.dict_star_arg) return result def getStarListArgumentName(self): return self.list_star_arg def getListStarArgVariable(self): return self.list_star_variable def getStarDictArgumentName(self): return self.dict_star_arg def getDictStarArgVariable(self): return self.dict_star_variable def getKwOnlyVariables(self): return self.kw_only_variables def allowsKeywords(self): # Abstract method, pylint: disable=no-self-use return True def getKeywordRefusalText(self): return "%s() takes no keyword arguments" % self.name def getArgumentNames(self): return self.pos_only_args + self.normal_args def getArgumentCount(self): return len(self.normal_args) + len(self.pos_only_args) def getKwOnlyParameterNames(self): return self.kw_only_args def getKwOnlyParameterCount(self): return len(self.kw_only_args) def getPosOnlyParameterCount(self): return len(self.pos_only_args)
class CodeObjectSpec(object): # One attribute for each code object aspect, and even flags, # pylint: disable=too-many-arguments,too-many-instance-attributes __slots__ = ( "co_name", "co_kind", "co_varnames", "co_argcount", "co_freevars", "co_posonlyargcount", "co_kwonlyargcount", "co_has_starlist", "co_has_stardict", "filename", "line_number", "future_spec", "new_locals", "is_optimized", ) @counted_init def __init__( self, co_name, co_kind, co_varnames, co_freevars, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_has_starlist, co_has_stardict, co_filename, co_lineno, future_spec, co_new_locals=None, co_is_optimized=None, ): # pylint: disable=I0021,too-many-locals self.co_name = co_name self.co_kind = co_kind self.future_spec = future_spec assert future_spec # Strings happens from XML parsing, make sure to convert them. if type(co_varnames) is str: if co_varnames == "": co_varnames = () else: co_varnames = co_varnames.split(",") if type(co_freevars) is str: if co_freevars == "": co_freevars = () else: co_freevars = co_freevars.split(",") if type(co_has_starlist) is not bool: co_has_starlist = co_has_starlist != "False" if type(co_has_stardict) is not bool: co_has_stardict = co_has_stardict != "False" self.co_varnames = tuple(co_varnames) self.co_freevars = tuple(co_freevars) self.co_argcount = int(co_argcount) self.co_posonlyargcount = int(co_posonlyargcount) self.co_kwonlyargcount = int(co_kwonlyargcount) self.co_has_starlist = co_has_starlist self.co_has_stardict = co_has_stardict self.filename = co_filename self.line_number = int(co_lineno) if type(co_has_starlist) is not bool: co_new_locals = co_new_locals != "False" if type(co_has_starlist) is not bool: co_is_optimized = co_is_optimized != "False" self.new_locals = co_new_locals self.is_optimized = co_is_optimized if isCountingInstances(): __del__ = counted_del() def __repr__(self): return ("""\ <CodeObjectSpec %(co_kind)s '%(co_name)s' with %(co_varnames)r>""" % self.getDetails()) def getDetails(self): return { "co_name": self.co_name, "co_kind": self.co_kind, "co_varnames": ",".join(self.co_varnames), "co_freevars": ",".join(self.co_freevars), "co_argcount": self.co_argcount, "co_posonlyargcount": self.co_posonlyargcount, "co_kwonlyargcount": self.co_kwonlyargcount, "co_has_starlist": self.co_has_starlist, "co_has_stardict": self.co_has_stardict, "co_filename": self.filename, "co_lineno": self.line_number, "co_new_locals": self.new_locals, "co_is_optimized": self.is_optimized, "code_flags": ",".join(self.future_spec.asFlags()), } def getCodeObjectKind(self): return self.co_kind def updateLocalNames(self, local_names, freevar_names): """Move detected local variables after closure has been decided.""" self.co_varnames += tuple( local_name for local_name in local_names if local_name not in self.co_varnames # TODO: This is actually a bug, but we have a hard time without it to know # frame locals easily. We use this in compiled function run time, that all # variables, including closure variables are found there. This would have to # be cleaned up, for potentially little gain. # if local_name not in freevar_names ) self.co_freevars = tuple(freevar_names) def removeFreeVarname(self, freevar_name): self.co_freevars = tuple(var_name for var_name in self.co_freevars if var_name != freevar_name) def setFlagIsOptimizedValue(self, value): self.is_optimized = value def getFlagIsOptimizedValue(self): return self.is_optimized def setFlagNewLocalsValue(self, value): self.new_locals = value def getFlagNewLocalsValue(self): return self.new_locals def getFutureSpec(self): return self.future_spec def getVarNames(self): return self.co_varnames def getFreeVarNames(self): return self.co_freevars def getArgumentCount(self): return self.co_argcount def getPosOnlyParameterCount(self): return self.co_posonlyargcount def getKwOnlyParameterCount(self): return self.co_kwonlyargcount def getCodeObjectName(self): return self.co_name def hasStarListArg(self): return self.co_has_starlist def hasStarDictArg(self): return self.co_has_stardict def getFilename(self): return self.filename def getLineNumber(self): return self.line_number