def getConstantsDeclCode(context): # There are many cases for constants of different types. # pylint: disable=R0912 statements = [] sorted_constants = sorted( iterItems(context.getConstants()), key = lambda k: (len(k), k ) ) for constant_identifier, constant_value in sorted_constants: # Need not declare built-in types. if constant_value in constant_builtin_types: continue if constant_value is None: continue if constant_value is False: continue if constant_value is True: continue if constant_value is Ellipsis: continue if context.getConstantUseCount(constant_identifier) != 1: statements.append("PyObject *%s;" % constant_identifier) return statements
def _getCacheFilename(dependency_tool, is_main_executable, source_dir, original_dir, binary_filename): original_filename = os.path.join(original_dir, os.path.basename(binary_filename)) original_filename = os.path.normcase(original_filename) if is_main_executable: # Normalize main program name for caching as well, but need to use the # scons information to distinguish different compilers, so we use # different libs there. # Ignore values, that are variable per compilation. hashed_value = "".join( key + value for key, value in iterItems(readSconsReport(source_dir=source_dir)) if key not in ("CLCACHE_STATS", )) else: hashed_value = original_filename # Have different values for different Python major versions. hashed_value += sys.version + sys.executable if str is not bytes: hashed_value = hashed_value.encode("utf8") cache_dir = os.path.join(getCacheDir(), "library_dependencies", dependency_tool) makePath(cache_dir) return os.path.join(cache_dir, hashlib.md5(hashed_value).hexdigest())
def getConstantsDeclCode(context): statements = [] # Sort items by length and name, so we are deterministic and pretty. sorted_constants = sorted(iterItems(context.getConstants()), key=lambda k: (len(k[0]), k[0])) for constant_identifier, constant_value in sorted_constants: # Need not declare built-in types. if constant_value in constant_builtin_types: continue if constant_value is None: continue if constant_value is False: continue if constant_value is True: continue if constant_value is Ellipsis: continue if context.getConstantUseCount(constant_identifier) != 1: statements.append("PyObject *%s;" % constant_identifier) if Options.isDebug(): statements.append("Py_hash_t hash_%s;" % constant_identifier) return statements
def getTempNameInfos(self): result = [] for base_name, count in sorted(iterItems(self.tmp_names)): if count is not None: for number in range(1,count+1): tmp_name = self.formatTempName( base_name = base_name, number = number ) if tmp_name not in self.forgotten_names: result.append( ( tmp_name, self.tmp_types[base_name] ) ) else: tmp_name = self.formatTempName( base_name = base_name, number = None ) if tmp_name not in self.forgotten_names: result.append( ( tmp_name, self.tmp_types[base_name] ) ) return result
def decideMarshal(constant_value): """ Decide of a constant can be created using "marshal" module methods. This is not the case for everything. A prominent exception is types, they are constants, but the "marshal" module refuses to work with them. """ constant_type = type(constant_value) if constant_type is type: # Types cannot be marshaled, there is no choice about it. return False elif constant_type is dict: # Look at all the keys an values, if one of it cannot be marshaled, # or should not, that is it. for key, value in iterItems(constant_value): if not decideMarshal(key): return False if not decideMarshal(value): return False elif constant_type in (tuple, list, set, frozenset): for element_value in constant_value: if not decideMarshal(element_value): return False return True
def asXml(self): line = self.source_ref.getLineNumber() result = TreeXML.Element("node", kind=self.__class__.__name__, line="%s" % line) compat_line = self.getCompatibleSourceReference().getLineNumber() if compat_line != line: result.attrib["compat_line"] = str(compat_line) for key, value in iterItems(self.getDetailsForDisplay()): result.set(key, str(value)) for name, children in self.getVisitableNodesNamed(): role = TreeXML.Element("role", name=name) result.append(role) if children is None: role.attrib["type"] = "none" elif type(children) not in (list, tuple): role.append(children.asXml()) else: role.attrib["type"] = "list" for child in children: role.append(child.asXml()) return result
def dumpTraces(self): debug("Constraint collection state: %s", self) for _variable_desc, variable_trace in sorted( iterItems(self.variable_traces)): # debug( "%r: %r", variable_trace ) variable_trace.dump()
def considerForDeferral(constant_value): module_context.getConstantCode(constant_value) if isMarshalConstant(constant_value): return constant_type = type(constant_value) if constant_type in (tuple, list, set, frozenset): for element in constant_value: considerForDeferral(element) elif constant_type is dict: for key, value in iterItems(constant_value): considerForDeferral(key) considerForDeferral(value) elif constant_type is slice: considerForDeferral(constant_value.start) considerForDeferral(constant_value.step) considerForDeferral(constant_value.stop) elif constant_type is xrange: if xrange is range: # For Python2 ranges, we use C long values directly. considerForDeferral(constant_value.start) considerForDeferral(constant_value.step) considerForDeferral(constant_value.stop) elif constant_value in builtin_named_values_list: considerForDeferral(builtin_named_values[constant_value])
def _copyDllsUsed(dist_dir, used_dlls): setupProgressBar( stage="Copying used DLLs", unit="DLL", total=len(used_dlls), ) dll_map = [] for dll_filename, (package_name, sources) in iterItems(used_dlls): dll_name = os.path.basename(dll_filename) target_path = os.path.join(dist_dir, dll_name) reportProgressBar(target_path) # Sometimes DLL dependencies were copied there already. TODO: That should # actually become disallowed with plugins no longer seeing that folder. if not os.path.exists(target_path): copyDllFile(source_path=dll_filename, dest_path=target_path) dll_map.append((dll_filename, package_name, dll_name)) if Options.isShowInclusion(): inclusion_logger.info( "Included used shared library '%s' (used by %s)." % (dll_filename, ", ".join(sources))) closeProgressBar() return dll_map
def asXml(self): result = TreeXML.Element( "node", kind = self.__class__.__name__, line = "%s" % self.getSourceReference().getLineNumber() ) for key, value in iterItems(self.getDetails()): value = str(value) if value.startswith("<") and value.endswith(">"): value = value[1:-1] result.set(key, str(value)) for name, children in self.getVisitableNodesNamed(): if type(children) not in (list, tuple): children = (children,) role = TreeXML.Element( "role", name = name ) result.append(role) for child in children: if child is not None: role.append( child.asXml() ) return result
def computeExpressionCall(self, call_node, constraint_collection): # TODO: Until we have something to re-order the arguments, we need to # skip this. For the immediate need, we avoid this complexity, as a # re-ordering will be needed. if call_node.getNamedArgumentPairs(): return call_node, None, None call_spec = self.getParameters() try: args_dict = matchCall( func_name=self.getName(), args=call_spec.getArgumentNames(), star_list_arg=call_spec.getStarListArgumentName(), star_dict_arg=call_spec.getStarDictArgumentName(), num_defaults=call_spec.getDefaultCount(), positional=call_node.getPositionalArguments(), pairs=(), ) values = [] for positional_arg in call_node.getPositionalArguments(): for _arg_name, arg_value in iterItems(args_dict): if arg_value is positional_arg: values.append(arg_value) result = ExpressionFunctionCall( function_body=self, values=values, source_ref=call_node.getSourceReference() ) return ( result, "new_statements", # TODO: More appropiate tag maybe. """Replaced call to created function body '%s' with direct \ function call""" % self.getName(), ) except TooManyArguments as e: from .NodeMakingHelpers import ( makeRaiseExceptionReplacementExpressionFromInstance, wrapExpressionWithSideEffects, ) result = wrapExpressionWithSideEffects( new_node=makeRaiseExceptionReplacementExpressionFromInstance( expression=call_node, exception=e.getRealException() ), old_node=call_node, side_effects=call_node.extractPreCallSideEffects(), ) return ( result, "new_statements,new_raise", # TODO: More appropiate tag maybe. """Replaced call to created function body '%s' to argument \ error""" % self.getName(), )
def getConstantsDeclCode(context): statements = [] # Sort items by length and name, so we are deterministic and pretty. sorted_constants = sorted( iterItems(context.getConstants()), key = lambda k: (len(k[0]), k[0]) ) for constant_identifier, constant_value in sorted_constants: # Need not declare built-in types. if constant_value in constant_builtin_types: continue if constant_value is None: continue if constant_value is False: continue if constant_value is True: continue if constant_value is Ellipsis: continue if context.getConstantUseCount(constant_identifier) != 1: statements.append("PyObject *%s;" % constant_identifier) if Options.isDebug(): statements.append("Py_hash_t hash_%s;" % constant_identifier) return statements
def setIndications(self): for variable, usage in iterItems( self.variable_usages ): if variable.isTempKeeperVariable(): variable.setNeedsFree( usage.getNeedsFree() ) if usage.isWriteOnly(): variable.setWriteOnly()
def computeExpressionCall(self, call_node, constraint_collection): call_kw = call_node.getCallKw() # TODO: Until we have something to re-order the arguments, we need to # skip this. For the immediate need, we avoid this complexity, as a # re-ordering will be needed. if call_kw: return call_node, None, None # TODO: Actually the above disables it entirely, as it is at least # the empty dictionary node in any case. We will need some enhanced # interfaces for "matchCall" to work on. call_spec = self.getFunctionRef().getFunctionBody().getParameters() try: args_dict = matchCall( func_name=self.getName(), args=call_spec.getArgumentNames(), star_list_arg=call_spec.getStarListArgumentName(), star_dict_arg=call_spec.getStarDictArgumentName(), num_defaults=call_spec.getDefaultCount(), positional=call_node.getCallArgsTuple(), pairs=()) values = [] for positional_arg in call_node.getCallArgs(): for _arg_name, arg_value in iterItems(args_dict): if arg_value is positional_arg: values.append(arg_value) result = ExpressionFunctionCall( function=self, values=values, source_ref=call_node.getSourceReference()) return ( result, "new_statements", # TODO: More appropriate tag maybe. """Replaced call to created function body '%s' with direct \ function call""" % self.getName()) except TooManyArguments as e: from .NodeMakingHelpers import ( makeRaiseExceptionReplacementExpressionFromInstance, wrapExpressionWithSideEffects) result = wrapExpressionWithSideEffects( new_node=makeRaiseExceptionReplacementExpressionFromInstance( expression=call_node, exception=e.getRealException()), old_node=call_node, side_effects=call_node.extractPreCallSideEffects()) return ( result, "new_statements,new_raise", # TODO: More appropriate tag maybe. """Replaced call to created function body '%s' to argument \ error""" % self.getName())
def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None variable_versions = {} for collection in collections: for variable, version in iterItems(collection.variable_actives): if variable not in variable_versions: variable_versions[variable] = set([version]) else: variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} # merge_traces = None for variable, versions in iterItems(variable_versions): if len(versions) == 1: version, = versions else: version = self.addVariableMergeMultipleTrace( variable = variable, traces = [ self.getVariableTrace(variable, version) for version in versions ] ) # if merge_traces is None: # merge_traces = [trace_merge] # else: # merge_traces.append(trace_merge) self.markCurrentVariableTrace(variable, version)
def setIndications(self): for variable, usage in iterItems(self.variable_usages): if variable.isTempVariable(): variable.setNeedsFree(usage.getNeedsFree()) if variable.isTempKeeperVariable(): if usage.isWriteOnly(): variable.setWriteOnly()
def computeExpressionCall( self, call_node, constraint_collection ): # TODO: Until we have something to re-order the arguments, we need to skip this. For # the immediate need, we avoid this complexity, as a re-ordering will be needed. if call_node.getNamedArgumentPairs(): return call_node, None, None call_spec = self.getParameters() try: args_dict = matchCall( func_name = self.getName(), args = call_spec.getArgumentNames(), star_list_arg = call_spec.getStarListArgumentName(), star_dict_arg = call_spec.getStarDictArgumentName(), num_defaults = call_spec.getDefaultCount(), positional = call_node.getPositionalArguments(), pairs = () ) values = [] for positional_arg in call_node.getPositionalArguments(): for _arg_name, arg_value in iterItems( args_dict ): if arg_value is positional_arg: values.append( arg_value ) result = ExpressionFunctionCall( function_body = self, values = values, source_ref = call_node.getSourceReference() ) return ( result, "new_statements", # TODO: More appropiate tag maybe. "Replaced call to created function body '%s' with direct function call" % self.getName() ) except TooManyArguments as e: from nuitka.nodes.NodeMakingHelpers import ( makeRaiseExceptionReplacementExpressionFromInstance, wrapExpressionWithSideEffects ) result = wrapExpressionWithSideEffects( new_node = makeRaiseExceptionReplacementExpressionFromInstance( expression = call_node, exception = e.getRealException() ), old_node = call_node, side_effects = call_node.extractPreCallSideEffects() ) return ( result, "new_statements,new_raise", # TODO: More appropiate tag maybe. "Replaced call to created function body '%s' to argument error" % self.getName() )
def removeKnowledge( self, value_friend ): to_remove = [] for variable, value in iterItems( self.variables ): if value is value_friend: to_remove.append( variable ) for remove in to_remove: del self.variables[ remove ]
def removeKnowledge(self, value_friend): to_remove = [] for variable, value in iterItems(self.variables): if value is value_friend: to_remove.append(variable) for remove in to_remove: del self.variables[remove]
def getMappingStringKeyPairs(self): pairs = [] for key, value in iterItems(self.constant): pairs.append((key, makeConstantRefNode(constant=value, source_ref=self.source_ref))) return pairs
def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None variable_versions = {} for collection in collections: for variable, version in iterItems(collection.variable_actives): if variable not in variable_versions: variable_versions[variable] = set([version]) else: variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} # merge_traces = None for variable, versions in iterItems(variable_versions): if len(versions) == 1: version, = versions else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=[ self.getVariableTrace(variable, version) for version in versions ], ) # if merge_traces is None: # merge_traces = [trace_merge] # else: # merge_traces.append(trace_merge) self.markCurrentVariableTrace(variable, version)
def dumpTrace(self): for i in range(5): print("*" * 80) for variable, traces in iterItems(self.variable_targets): print(variable) print("*" * 80) for trace in traces: print(trace) print("*" * 80)
def wrapper(*args, **kw): new_kw = {} for key, value in iterItems(kw): if key in builtin_all_names: key = key + "_arg" new_kw[key] = value return f(*args, **new_kw)
def getVariableTraces(self, variable): result = [] for key, variable_trace in iterItems(self.variable_traces): candidate = key[0] if variable is candidate: result.append(variable_trace) return result
def dumpTrace( self ): for i in range(5): print( "*" * 80 ) for variable, traces in iterItems( self.variable_targets ): print( variable ) print( "*" * 80 ) for trace in traces: print( trace ) print( "*" * 80 )
def getMappingStringKeyPairs(self): assert self.isMapping() pairs = [] source_ref = self.getSourceReference() for key, value in iterItems(self.constant): pairs.append((key, ExpressionConstantRef(constant=value, source_ref=source_ref))) return pairs
def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None # print("Enter mergeMultipleBranches", len(collections)) with TimerReport( message="Running merge for %s took %%.2f seconds" % collections, decider=lambda: 0, ): variable_versions = defaultdict(OrderedSet) for collection in collections: for variable, version in iterItems( collection.variable_actives): variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} for variable, versions in iterItems(variable_versions): if len(versions) == 1: (version, ) = versions else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=tuple( self.getVariableTrace(variable, version) for version in versions), ) self.markCurrentVariableTrace(variable, version)
def updateVariablesFromCollection(old_collection, new_collection, source_ref): # After removing/adding traces, we need to pre-compute the users state # information. touched_variables = set() loop_trace_removal = set() if old_collection is not None: for (variable, _version), variable_trace in iterItems( old_collection.getVariableTracesAll()): variable.removeTrace(variable_trace) touched_variables.add(variable) if variable_trace.isLoopTrace(): loop_trace_removal.add(variable) if new_collection is not None: for (variable, _version), variable_trace in iterItems( new_collection.getVariableTracesAll()): variable.addTrace(variable_trace) touched_variables.add(variable) if variable_trace.isLoopTrace(): if variable in loop_trace_removal: loop_trace_removal.remove(variable) # Release the memory, and prevent the "active" state from being ever # inspected, it's useless now. new_collection.variable_actives.clear() del new_collection.variable_actives for variable in touched_variables: variable.updateUsageState() if loop_trace_removal: if new_collection is not None: new_collection.signalChange( "var_usage", source_ref, lambda: "Loop variable '%s' usage ceased." % ",".join( variable.getName() for variable in loop_trace_removal), )
def getMappingPairs(self): assert self.isMapping() pairs = [] source_ref = self.getSourceReference() for key, value in iterItems(self.constant): pairs.append( ExpressionConstantRef(constant=key, source_ref=source_ref), ExpressionConstantRef(constant=value, source_ref=source_ref)) return pairs
def getMappingStringKeyPairs(self): assert self.isMapping() pairs = [] source_ref = self.getSourceReference() for key, value in iterItems(self.constant): pairs.append( (key, makeConstantRefNode(constant=value, source_ref=source_ref))) return pairs
def mergeBranchVariables(a, b): result = {} for variable, value in iterItems(a): if variable in b: merged = ValueFriends.mergeBranchFriendValues( value, b[variable]) if merged is not None: result[variable] = merged else: pass return result
def getMappingPairs(self): assert self.isMapping() pairs = [] source_ref = self.getSourceReference() for key, value in iterItems(self.constant): pairs.append( makeConstantRefNode(constant=key, source_ref=source_ref), makeConstantRefNode(constant=value, source_ref=source_ref), ) return pairs
def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None variable_versions = {} for collection in collections: for variable, version in iterItems(collection.variable_actives): if variable not in variable_versions: variable_versions[variable] = OrderedSet((version,)) else: variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} for variable, versions in iterItems(variable_versions): if len(versions) == 1: (version,) = versions else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=tuple( self.getVariableTrace(variable, version) for version in versions ), ) self.markCurrentVariableTrace(variable, version)
def considerForDeferral(constant_value): if isMarshalConstant(constant_value): return module_context.getConstantCode(constant_value) constant_type = type(constant_value) if constant_type in (tuple, list, set, frozenset): for element in constant_value: considerForDeferral(element) elif constant_type is dict: for key, value in iterItems(constant_value): considerForDeferral(key) considerForDeferral(value)
def getConstantsInitCode(context): emit = SourceCodeCollector() sorted_constants = sorted(iterItems(context.getConstants()), key=lambda k: (len(k), k)) for constant_identifier, constant_value in sorted_constants: _addConstantInitCode(emit=emit, constant_type=type(constant_value), constant_value=constant_value, constant_identifier=constant_identifier, module_level=False, context=context) return emit.codes
def updateVariablesFromCollection(old_collection, new_collection): # After removing/adding traces, we need to pre-compute the users state # information. touched_variables = set() if old_collection is not None: for (variable, _version), variable_trace in iterItems( old_collection.getVariableTracesAll()): variable.removeTrace(variable_trace) touched_variables.add(variable) if new_collection is not None: for (variable, _version), variable_trace in iterItems( new_collection.getVariableTracesAll()): variable.addTrace(variable_trace) touched_variables.add(variable) # Release the memory, and prevent the "active" state from being ever # inspected, it's useless now. new_collection.variable_actives.clear() del new_collection.variable_actives for variable in touched_variables: variable.updateUsageState()
def enableDebug(globals_dict): templates = dict(globals_dict) class TemplateWrapper: """ Wrapper around templates. To better trace and control template usage. """ def __init__(self, name, value): self.name = name self.value = value def __str__(self): return self.value def __mod__(self, other): assert type(other) is dict, self.name for key in other.keys(): if "%%(%s)" % key not in self.value: from logging import warning warning( "Extra value '%s' provided to template '%s'.", key, self.name ) try: return self.value % other except KeyError as e: raise KeyError(self.name, *e.args) def split(self, sep): return self.value.split(sep) for template_name, template_value in iterItems(templates): # Ignore internal attribute like "__name__" that the module will also # have of course. if template_name.startswith('_'): continue if type(template_value) is str: globals_dict[template_name] = TemplateWrapper( template_name, template_value )
def getConstantsInitCode(context): emit = SourceCodeCollector() # Sort items by length and name, so we are determistic and pretty. sorted_constants = sorted(iterItems(context.getConstants()), key=lambda k: (len(k), k)) for constant_identifier, constant_value in sorted_constants: _addConstantInitCode(emit=emit, constant_type=type(constant_value), constant_value=constant_value, constant_identifier=constant_identifier, module_level=False, context=context) return emit.codes
def mergeBranchVariables( a, b ): result = {} for variable, value in iterItems( a ): if variable in b: merged = ValueFriends.mergeBranchFriendValues( value, b[ variable ] ) if merged is not None: result[ variable ] = merged else: pass return result
def getConstantsInitCode(context): # There are many cases for constants to be created in the most efficient # way, pylint: disable=R0912 emit = SourceCodeCollector() sorted_constants = sorted(iterItems(context.getConstants()), key=lambda k: (len(k), k)) for constant_identifier, constant_value in sorted_constants: _addConstantInitCode(emit=emit, constant_type=type(constant_value), constant_value=constant_value, constant_identifier=constant_identifier, module_level=False, context=context) return emit.codes
def computeStatement( self, constraint_collection ): old_body = self.getBody() result = constraint_collection.onStatementsSequence( old_body ) if result is not old_body: self.setBody( result ) # TODO: That should be a method of the constraint_collection for variable, friend in iterItems( dict( constraint_collection.variables ) ): if variable.getOwner() is self: del constraint_collection.variables[ variable ] # TODO: Back propagate now. friend.onRelease( self ) if result is None: return None, "new_statements", "Removed empty temporary block" return self, None, None
def getConstantsInitCode(context): emit = SourceCodeCollector() sorted_constants = sorted( iterItems(context.getConstants()), key = lambda k: (len(k), k ) ) for constant_identifier, constant_value in sorted_constants: _addConstantInitCode( emit = emit, constant_type = type(constant_value), constant_value = constant_value, constant_identifier = constant_identifier, module_level = False, context = context ) return emit.codes
def decideMarshal(constant_value): constant_type = type(constant_value) if constant_type is type: # Types cannot be marshalled, there is no choice about it. return False elif constant_type is dict: # Look at all the keys an values, if one of it cannot be marshalled, # or should not, that is it. for key, value in iterItems(constant_value): if not decideMarshal(key): return False if not decideMarshal(value): return False elif constant_type in (tuple, list, set, frozenset): for element_value in constant_value: if not decideMarshal(element_value): return False return True
def getConstantsInitCode(context): emit = SourceCodeCollector() check = SourceCodeCollector() # Sort items by length and name, so we are deterministic and pretty. sorted_constants = sorted(iterItems(context.getConstants()), key=lambda k: (len(k[0]), k[0])) for constant_identifier, constant_value in sorted_constants: _addConstantInitCode( emit=emit, check=check, constant_type=type(constant_value), constant_value=constant_value, constant_identifier=constant_identifier, module_level=False, context=context, ) return emit.codes, check.codes
def getConstantsInitCode(context): # There are many cases for constants to be created in the most efficient # way, pylint: disable=R0912 emit = SourceCodeCollector() sorted_constants = sorted( iterItems(context.getConstants()), key = lambda k: (len(k), k ) ) for constant_identifier, constant_value in sorted_constants: _addConstantInitCode( emit = emit, constant_type = type(constant_value), constant_value = constant_value, constant_identifier = constant_identifier, module_level = False, context = context ) return emit.codes
def enableDebug(): templates = dict( globals() ) class TemplateWrapper: """ Wrapper around templates, to better trace and control template usage. """ def __init__( self, name, value ): self.name = name self.value = value def __str__( self ): return self.value def __mod__( self, other ): assert type( other ) is dict, self.name for key in other.keys(): if "%%(%s)" % key not in self.value: from logging import warning warning( "Extra value '%s' provided to template '%s'.", key, self.name ) return self.value % other def split( self, sep ): return self.value.split( sep ) from nuitka.__past__ import iterItems for template_name, template_value in iterItems( templates ): if template_name.startswith( "_" ): continue if type( template_value ) is str: globals()[ template_name ] = TemplateWrapper( template_name, template_value )
def considerForDeferral(constant_value): module_context.getConstantCode(constant_value) if isMarshalConstant(constant_value): return constant_type = type(constant_value) if constant_type in (tuple, list, set, frozenset): for element in constant_value: considerForDeferral(element) elif constant_type is dict: for key, value in iterItems(constant_value): considerForDeferral(key) considerForDeferral(value) elif constant_type is slice: considerForDeferral(constant_value.start) considerForDeferral(constant_value.step) considerForDeferral(constant_value.stop) elif constant_type is xrange: if xrange is range: if python_version >= 330: # For Python2 ranges, we use C long values directly. considerForDeferral(constant_value.start) considerForDeferral(constant_value.step) considerForDeferral(constant_value.stop) else: parts = [int(value) for value in str(constant_value)[6:-1].split(",")] if len(parts) <= 1: parts.append(0) if len(parts) <= 2: parts.append(1) for value in parts: considerForDeferral(value)
def getWrittenVariables( self ): return [ variable for variable, usage in iterItems( self.variable_usages ) if not usage.isReadOnly() ]
def dump( self ): debug( "Constraint collection state:" ) for variable_name, variable_info in sorted( iterItems( self.variables ) ): debug( "%r: %r", variable_name, variable_info )
def getConstantAccess(to_name, constant, emit, context): # Many cases, because for each type, we may copy or optimize by creating # empty. pylint: disable=R0912,R0915 if type(constant) is dict: if constant: for key, value in iterItems(constant): # key cannot be mutable. assert not isMutable(key) if isMutable(value): needs_deep = True break else: needs_deep = False if needs_deep: code = "DEEP_COPY( %s )" % getConstantCode( constant = constant, context = context ) else: code = "PyDict_Copy( %s )" % getConstantCode( constant = constant, context = context ) else: code = "PyDict_New()" ref_count = 1 elif type(constant) is set: if constant: code = "PySet_New( %s )" % getConstantCode( constant = constant, context = context ) else: code = "PySet_New( NULL )" ref_count = 1 elif type(constant) is list: if constant: for value in constant: if isMutable(value): needs_deep = True break else: needs_deep = False if needs_deep: code = "DEEP_COPY( %s )" % getConstantCode( constant = constant, context = context ) else: code = "LIST_COPY( %s )" % getConstantCode( constant = constant, context = context ) else: code = "PyList_New( 0 )" ref_count = 1 elif type(constant) is tuple: for value in constant: if isMutable(value): needs_deep = True break else: needs_deep = False if needs_deep: code = "DEEP_COPY( %s )" % getConstantCode( constant = constant, context = context ) ref_count = 1 else: code = getConstantCode( context = context, constant = constant ) ref_count = 0 else: code = getConstantCode( context = context, constant = constant ) ref_count = 0 emit( "%s = %s;" % ( to_name, code, ) ) if ref_count: context.addCleanupTempName(to_name)
def __addConstantInitCode(context, emit, check, constant_type, constant_value, constant_identifier, module_level): """ Emit code for a specific constant to be prepared during init. This may be module or global init. Code makes sure that nested constants belong into the same scope. """ # This has many cases, that all return, and do a lot. # pylint: disable=R0911,R0912,R0915,R0914 # For the module level, we only mean to create constants that are used only # inside of it. For the global level, it must must be single use. if module_level: if context.global_context.getConstantUseCount(constant_identifier) != 1: return else: if context.getConstantUseCount(constant_identifier) == 1: return # Adding it to "done". We cannot have recursive constants, so this is OK # to be done now. done.add(constant_identifier) # Use shortest code for ints and longs. if constant_type is long: # See above, same for long values. Note: These are of course not # existent with Python3 which would have covered it before. if constant_value >= 0 and constant_value <= max_unsigned_long: emit ( "%s = PyLong_FromUnsignedLong( %sul );" % ( constant_identifier, constant_value ) ) return elif constant_value < 0 and constant_value >= min_signed_long: emit ( "%s = PyLong_FromLong( %sl );" % ( constant_identifier, constant_value ) ) return elif constant_value == min_signed_long-1: # There are compilers out there, that give warnings for the literal # MININT when used. We work around that warning here. emit( """\ %s = PyLong_FromLong( %sl ); // To be corrected with -1 in-place next lines. CHECK_OBJECT( const_int_pos_1 ); %s = PyNumber_InPlaceSubtract( %s, const_int_pos_1 );""" % ( constant_identifier, min_signed_long, constant_identifier, constant_identifier ) ) return else: # Note, other longs cannot be handled like that yet. We might create # code that does it better in the future, abusing e.g. internal # representation of "long" integer values. pass elif constant_type is int: if constant_value >= min_signed_long: emit( "%s = PyInt_FromLong( %sl );" % ( constant_identifier, constant_value ) ) return else: # There are compilers out there, that give warnings for the literal # MININT when used. We work around that warning here. assert constant_value == min_signed_long-1 emit( """\ %s = PyInt_FromLong( %sl ); // To be corrected in next line. %s = PyNumber_InPlaceSubtract( %s, const_int_pos_1 );""" % ( constant_identifier, min_signed_long, constant_identifier, constant_identifier ) ) return if constant_type is unicode: # Attempting to marshal is OK, but esp. Python2 cannot do it for all # "unicode" values. if attemptToMarshal(constant_identifier, constant_value, emit): return try: encoded = constant_value.encode("utf-8") if str is not unicode: emit( "%s = UNSTREAM_UNICODE( %s );" % ( constant_identifier, stream_data.getStreamDataCode(encoded) ) ) else: emit( "%s = UNSTREAM_STRING( %s, %d );" % ( constant_identifier, stream_data.getStreamDataCode(encoded), 1 if _isAttributeName(constant_value) else 0 ) ) return except UnicodeEncodeError: # So fall back to below code, which will unstream it then. pass elif constant_type is str: # Python3: Strings that can be encoded as UTF-8 are done more or less # directly. When they cannot be expressed as UTF-8, that is rare not we # can indeed use pickling. assert str is not unicode if len(constant_value) == 1: emit( "%s = UNSTREAM_CHAR( %d, %d );" % ( constant_identifier, ord(constant_value[0]), 1 if _isAttributeName(constant_value) else 0 ) ) else: emit( "%s = UNSTREAM_STRING( %s, %d );" % ( constant_identifier, stream_data.getStreamDataCode(constant_value), 1 if _isAttributeName(constant_value) else 0 ) ) return elif constant_type is bytes: # Python3 only, for Python2, bytes do not happen. assert str is unicode emit( "%s = UNSTREAM_BYTES( %s );" % ( constant_identifier, stream_data.getStreamDataCode(constant_value) ) ) return if constant_type is float: emit( "%s = UNSTREAM_FLOAT( %s );" % ( constant_identifier, stream_data.getStreamDataCode( value = struct.pack("<d", constant_value), fixed_size = True ) ) ) return if constant_type is dict: # Not all dictionaries can or should be marshaled. For small ones, # or ones with strange values, like "{1:type}", we have to do it. if attemptToMarshal(constant_identifier, constant_value, emit): return emit( "%s = _PyDict_NewPresized( %d );" % ( constant_identifier, len(constant_value) ) ) for key, value in iterItems(constant_value): key_name = getConstantCodeName(context, key) _addConstantInitCode( emit = emit, check = check, constant_type = type(key), constant_value = key, constant_identifier = key_name, module_level = module_level, context = context ) value_name = getConstantCodeName(context, value) _addConstantInitCode( emit = emit, check = check, constant_type = type(value), constant_value = value, constant_identifier = value_name, module_level = module_level, context = context ) # TODO: Error checking for debug. emit( "PyDict_SetItem( %s, %s, %s );" % ( constant_identifier, key_name, value_name ) ) emit( "assert( PyDict_Size( %s ) == %d );" % ( constant_identifier, len(constant_value) ) ) return if constant_type is tuple: # Not all tuples can or should be marshaled. For small ones, # or ones with strange values, like "(type,)", we have to do it. if attemptToMarshal(constant_identifier, constant_value, emit): return emit( "%s = PyTuple_New( %d );" % ( constant_identifier, len(constant_value) ) ) for count, element_value in enumerate(constant_value): element_name = getConstantCodeName( context = context, constant = element_value ) _addConstantInitCode( emit = emit, check = check, constant_type = type(element_value), constant_value = element_value, constant_identifier = getConstantCodeName( context = context, constant = element_value ), module_level = module_level, context = context ) # Do not take references, these won't be deleted ever. emit( "PyTuple_SET_ITEM( %s, %d, %s ); Py_INCREF( %s );" % ( constant_identifier, count, element_name, element_name ) ) return if constant_type is list: # Not all lists can or should be marshaled. For small ones, # or ones with strange values, like "[type]", we have to do it. if attemptToMarshal(constant_identifier, constant_value, emit): return emit( "%s = PyList_New( %d );" % ( constant_identifier, len(constant_value) ) ) for count, element_value in enumerate(constant_value): element_name = getConstantCodeName( context = context, constant = element_value ) _addConstantInitCode( emit = emit, check = check, constant_type = type(element_value), constant_value = element_value, constant_identifier = element_name, module_level = module_level, context = context ) # Do not take references, these won't be deleted ever. emit( "PyList_SET_ITEM( %s, %d, %s ); Py_INCREF( %s );" % ( constant_identifier, count, element_name, element_name ) ) return if constant_type is set: # Not all sets can or should be marshaled. For small ones, # or ones with strange values, like "{type}", we have to do it. if attemptToMarshal(constant_identifier, constant_value, emit): return # TODO: Hinting size is really not possible? emit( "%s = PySet_New( NULL );" % constant_identifier ) for element_value in constant_value: element_name = getConstantCodeName( context = context, constant = element_value ) _addConstantInitCode( emit = emit, check = check, constant_type = type(element_value), constant_value = element_value, constant_identifier = element_name, module_level = module_level, context = context ) emit( "PySet_Add( %s, %s );" % ( constant_identifier, element_name ) ) emit( "assert( PySet_Size( %s ) == %d );" % ( constant_identifier, len(constant_value) ) ) return if constant_type is slice: slice1_name = getConstantCodeName(context, constant_value.start) _addConstantInitCode( emit = emit, check = check, constant_type = type(constant_value.start), constant_value = constant_value.start, constant_identifier = slice1_name, module_level = module_level, context = context ) slice2_name = getConstantCodeName(context, constant_value.stop) _addConstantInitCode( emit = emit, check = check, constant_type = type(constant_value.stop), constant_value = constant_value.stop, constant_identifier = slice2_name, module_level = module_level, context = context ) slice3_name = getConstantCodeName(context, constant_value.step) _addConstantInitCode( emit = emit, check = check, constant_type = type(constant_value.step), constant_value = constant_value.step, constant_identifier = slice3_name, module_level = module_level, context = context ) emit( "%s = PySlice_New( %s, %s, %s );" % ( constant_identifier, slice1_name, slice2_name, slice3_name ) ) return # TODO: Ranges could very well be created for Python3. And "frozenset" and # set, are to be examined. if constant_type in (frozenset, complex, unicode, long, range): # Lets attempt marshal these. if attemptToMarshal(constant_identifier, constant_value, emit): return emit( _getUnstreamCode(constant_value, constant_identifier) ) return # Must not reach this, if we did, it's in error, and we need to know. assert False, (type(constant_value), constant_value, constant_identifier)
def getCodeObjects(self): return sorted(iterItems(self.code_objects))