def getConstantsDeclCode(context):
# There are many cases for constants of different types.
# pylint: disable=R0912
statements = []
sorted_constants = sorted(
iterItems(context.getConstants()),
key = lambda k: (len(k), k )
)
for constant_identifier, constant_value in sorted_constants:
# Need not declare built-in types.
if constant_value in constant_builtin_types:
continue
if constant_value is None:
continue
if constant_value is False:
continue
if constant_value is True:
continue
if constant_value is Ellipsis:
continue
if context.getConstantUseCount(constant_identifier) != 1:
statements.append("PyObject *%s;" % constant_identifier)
return statements
def _getCacheFilename(dependency_tool, is_main_executable, source_dir,
original_dir, binary_filename):
original_filename = os.path.join(original_dir,
os.path.basename(binary_filename))
original_filename = os.path.normcase(original_filename)
if is_main_executable:
# Normalize main program name for caching as well, but need to use the
# scons information to distinguish different compilers, so we use
# different libs there.
# Ignore values, that are variable per compilation.
hashed_value = "".join(
key + value
for key, value in iterItems(readSconsReport(source_dir=source_dir))
if key not in ("CLCACHE_STATS", ))
else:
hashed_value = original_filename
# Have different values for different Python major versions.
hashed_value += sys.version + sys.executable
if str is not bytes:
hashed_value = hashed_value.encode("utf8")
cache_dir = os.path.join(getCacheDir(), "library_dependencies",
dependency_tool)
makePath(cache_dir)
return os.path.join(cache_dir, hashlib.md5(hashed_value).hexdigest())
def getConstantsDeclCode(context):
statements = []
# Sort items by length and name, so we are deterministic and pretty.
sorted_constants = sorted(iterItems(context.getConstants()),
key=lambda k: (len(k[0]), k[0]))
for constant_identifier, constant_value in sorted_constants:
# Need not declare built-in types.
if constant_value in constant_builtin_types:
continue
if constant_value is None:
continue
if constant_value is False:
continue
if constant_value is True:
continue
if constant_value is Ellipsis:
continue
if context.getConstantUseCount(constant_identifier) != 1:
statements.append("PyObject *%s;" % constant_identifier)
if Options.isDebug():
statements.append("Py_hash_t hash_%s;" % constant_identifier)
return statements
def getTempNameInfos(self):
result = []
for base_name, count in sorted(iterItems(self.tmp_names)):
if count is not None:
for number in range(1,count+1):
tmp_name = self.formatTempName(
base_name = base_name,
number = number
)
if tmp_name not in self.forgotten_names:
result.append(
(
tmp_name,
self.tmp_types[base_name]
)
)
else:
tmp_name = self.formatTempName(
base_name = base_name,
number = None
)
if tmp_name not in self.forgotten_names:
result.append(
(
tmp_name,
self.tmp_types[base_name]
)
)
return result
def decideMarshal(constant_value):
""" Decide of a constant can be created using "marshal" module methods.
This is not the case for everything. A prominent exception is types,
they are constants, but the "marshal" module refuses to work with
them.
"""
constant_type = type(constant_value)
if constant_type is type:
# Types cannot be marshaled, there is no choice about it.
return False
elif constant_type is dict:
# Look at all the keys an values, if one of it cannot be marshaled,
# or should not, that is it.
for key, value in iterItems(constant_value):
if not decideMarshal(key):
return False
if not decideMarshal(value):
return False
elif constant_type in (tuple, list, set, frozenset):
for element_value in constant_value:
if not decideMarshal(element_value):
return False
return True
def asXml(self):
line = self.source_ref.getLineNumber()
result = TreeXML.Element("node",
kind=self.__class__.__name__,
line="%s" % line)
compat_line = self.getCompatibleSourceReference().getLineNumber()
if compat_line != line:
result.attrib["compat_line"] = str(compat_line)
for key, value in iterItems(self.getDetailsForDisplay()):
result.set(key, str(value))
for name, children in self.getVisitableNodesNamed():
role = TreeXML.Element("role", name=name)
result.append(role)
if children is None:
role.attrib["type"] = "none"
elif type(children) not in (list, tuple):
role.append(children.asXml())
else:
role.attrib["type"] = "list"
for child in children:
role.append(child.asXml())
return result
def dumpTraces(self):
debug("Constraint collection state: %s", self)
for _variable_desc, variable_trace in sorted(
iterItems(self.variable_traces)):
# debug( "%r: %r", variable_trace )
variable_trace.dump()
def considerForDeferral(constant_value):
module_context.getConstantCode(constant_value)
if isMarshalConstant(constant_value):
return
constant_type = type(constant_value)
if constant_type in (tuple, list, set, frozenset):
for element in constant_value:
considerForDeferral(element)
elif constant_type is dict:
for key, value in iterItems(constant_value):
considerForDeferral(key)
considerForDeferral(value)
elif constant_type is slice:
considerForDeferral(constant_value.start)
considerForDeferral(constant_value.step)
considerForDeferral(constant_value.stop)
elif constant_type is xrange:
if xrange is range:
# For Python2 ranges, we use C long values directly.
considerForDeferral(constant_value.start)
considerForDeferral(constant_value.step)
considerForDeferral(constant_value.stop)
elif constant_value in builtin_named_values_list:
considerForDeferral(builtin_named_values[constant_value])
def decideMarshal(constant_value):
""" Decide of a constant can be created using "marshal" module methods.
This is not the case for everything. A prominent exception is types,
they are constants, but the "marshal" module refuses to work with
them.
"""
constant_type = type(constant_value)
if constant_type is type:
# Types cannot be marshaled, there is no choice about it.
return False
elif constant_type is dict:
# Look at all the keys an values, if one of it cannot be marshaled,
# or should not, that is it.
for key, value in iterItems(constant_value):
if not decideMarshal(key):
return False
if not decideMarshal(value):
return False
elif constant_type in (tuple, list, set, frozenset):
for element_value in constant_value:
if not decideMarshal(element_value):
return False
return True
def getTempNameInfos(self):
result = []
for base_name, count in sorted(iterItems(self.tmp_names)):
if count is not None:
for number in range(1,count+1):
tmp_name = self.formatTempName(
base_name = base_name,
number = number
)
if tmp_name not in self.forgotten_names:
result.append(
(
tmp_name,
self.tmp_types[base_name]
)
)
else:
tmp_name = self.formatTempName(
base_name = base_name,
number = None
)
if tmp_name not in self.forgotten_names:
result.append(
(
tmp_name,
self.tmp_types[base_name]
)
)
return result
def _copyDllsUsed(dist_dir, used_dlls):
setupProgressBar(
stage="Copying used DLLs",
unit="DLL",
total=len(used_dlls),
)
dll_map = []
for dll_filename, (package_name, sources) in iterItems(used_dlls):
dll_name = os.path.basename(dll_filename)
target_path = os.path.join(dist_dir, dll_name)
reportProgressBar(target_path)
# Sometimes DLL dependencies were copied there already. TODO: That should
# actually become disallowed with plugins no longer seeing that folder.
if not os.path.exists(target_path):
copyDllFile(source_path=dll_filename, dest_path=target_path)
dll_map.append((dll_filename, package_name, dll_name))
if Options.isShowInclusion():
inclusion_logger.info(
"Included used shared library '%s' (used by %s)." %
(dll_filename, ", ".join(sources)))
closeProgressBar()
return dll_map
def asXml(self):
result = TreeXML.Element(
"node",
kind = self.__class__.__name__,
line = "%s" % self.getSourceReference().getLineNumber()
)
for key, value in iterItems(self.getDetails()):
value = str(value)
if value.startswith("<") and value.endswith(">"):
value = value[1:-1]
result.set(key, str(value))
for name, children in self.getVisitableNodesNamed():
if type(children) not in (list, tuple):
children = (children,)
role = TreeXML.Element(
"role",
name = name
)
result.append(role)
for child in children:
if child is not None:
role.append(
child.asXml()
)
return result
def computeExpressionCall(self, call_node, constraint_collection):
# TODO: Until we have something to re-order the arguments, we need to
# skip this. For the immediate need, we avoid this complexity, as a
# re-ordering will be needed.
if call_node.getNamedArgumentPairs():
return call_node, None, None
call_spec = self.getParameters()
try:
args_dict = matchCall(
func_name=self.getName(),
args=call_spec.getArgumentNames(),
star_list_arg=call_spec.getStarListArgumentName(),
star_dict_arg=call_spec.getStarDictArgumentName(),
num_defaults=call_spec.getDefaultCount(),
positional=call_node.getPositionalArguments(),
pairs=(),
)
values = []
for positional_arg in call_node.getPositionalArguments():
for _arg_name, arg_value in iterItems(args_dict):
if arg_value is positional_arg:
values.append(arg_value)
result = ExpressionFunctionCall(
function_body=self, values=values, source_ref=call_node.getSourceReference()
)
return (
result,
"new_statements", # TODO: More appropiate tag maybe.
"""Replaced call to created function body '%s' with direct \
function call"""
% self.getName(),
)
except TooManyArguments as e:
from .NodeMakingHelpers import (
makeRaiseExceptionReplacementExpressionFromInstance,
wrapExpressionWithSideEffects,
)
result = wrapExpressionWithSideEffects(
new_node=makeRaiseExceptionReplacementExpressionFromInstance(
expression=call_node, exception=e.getRealException()
),
old_node=call_node,
side_effects=call_node.extractPreCallSideEffects(),
)
return (
result,
"new_statements,new_raise", # TODO: More appropiate tag maybe.
"""Replaced call to created function body '%s' to argument \
error"""
% self.getName(),
)
def getConstantsDeclCode(context):
statements = []
# Sort items by length and name, so we are deterministic and pretty.
sorted_constants = sorted(
iterItems(context.getConstants()),
key = lambda k: (len(k[0]), k[0])
)
for constant_identifier, constant_value in sorted_constants:
# Need not declare built-in types.
if constant_value in constant_builtin_types:
continue
if constant_value is None:
continue
if constant_value is False:
continue
if constant_value is True:
continue
if constant_value is Ellipsis:
continue
if context.getConstantUseCount(constant_identifier) != 1:
statements.append("PyObject *%s;" % constant_identifier)
if Options.isDebug():
statements.append("Py_hash_t hash_%s;" % constant_identifier)
return statements
def setIndications(self):
for variable, usage in iterItems( self.variable_usages ):
if variable.isTempKeeperVariable():
variable.setNeedsFree( usage.getNeedsFree() )
if usage.isWriteOnly():
variable.setWriteOnly()
def computeExpressionCall(self, call_node, constraint_collection):
call_kw = call_node.getCallKw()
# TODO: Until we have something to re-order the arguments, we need to
# skip this. For the immediate need, we avoid this complexity, as a
# re-ordering will be needed.
if call_kw:
return call_node, None, None
# TODO: Actually the above disables it entirely, as it is at least
# the empty dictionary node in any case. We will need some enhanced
# interfaces for "matchCall" to work on.
call_spec = self.getFunctionRef().getFunctionBody().getParameters()
try:
args_dict = matchCall(
func_name=self.getName(),
args=call_spec.getArgumentNames(),
star_list_arg=call_spec.getStarListArgumentName(),
star_dict_arg=call_spec.getStarDictArgumentName(),
num_defaults=call_spec.getDefaultCount(),
positional=call_node.getCallArgsTuple(),
pairs=())
values = []
for positional_arg in call_node.getCallArgs():
for _arg_name, arg_value in iterItems(args_dict):
if arg_value is positional_arg:
values.append(arg_value)
result = ExpressionFunctionCall(
function=self,
values=values,
source_ref=call_node.getSourceReference())
return (
result,
"new_statements", # TODO: More appropriate tag maybe.
"""Replaced call to created function body '%s' with direct \
function call""" % self.getName())
except TooManyArguments as e:
from .NodeMakingHelpers import (
makeRaiseExceptionReplacementExpressionFromInstance,
wrapExpressionWithSideEffects)
result = wrapExpressionWithSideEffects(
new_node=makeRaiseExceptionReplacementExpressionFromInstance(
expression=call_node, exception=e.getRealException()),
old_node=call_node,
side_effects=call_node.extractPreCallSideEffects())
return (
result,
"new_statements,new_raise", # TODO: More appropriate tag maybe.
"""Replaced call to created function body '%s' to argument \
error""" % self.getName())
def mergeMultipleBranches(self, collections):
assert collections
# Optimize for length 1, which is trivial merge and needs not a
# lot of work.
if len(collections) == 1:
self.replaceBranch(collections[0])
return None
variable_versions = {}
for collection in collections:
for variable, version in iterItems(collection.variable_actives):
if variable not in variable_versions:
variable_versions[variable] = set([version])
else:
variable_versions[variable].add(version)
for collection in collections:
for variable, versions in iterItems(variable_versions):
if variable not in collection.variable_actives:
versions.add(0)
self.variable_actives = {}
# merge_traces = None
for variable, versions in iterItems(variable_versions):
if len(versions) == 1:
version, = versions
else:
version = self.addVariableMergeMultipleTrace(
variable = variable,
traces = [
self.getVariableTrace(variable, version)
for version in
versions
]
)
# if merge_traces is None:
# merge_traces = [trace_merge]
# else:
# merge_traces.append(trace_merge)
self.markCurrentVariableTrace(variable, version)
def setIndications(self):
for variable, usage in iterItems(self.variable_usages):
if variable.isTempVariable():
variable.setNeedsFree(usage.getNeedsFree())
if variable.isTempKeeperVariable():
if usage.isWriteOnly():
variable.setWriteOnly()
def computeExpressionCall( self, call_node, constraint_collection ):
# TODO: Until we have something to re-order the arguments, we need to skip this. For
# the immediate need, we avoid this complexity, as a re-ordering will be needed.
if call_node.getNamedArgumentPairs():
return call_node, None, None
call_spec = self.getParameters()
try:
args_dict = matchCall(
func_name = self.getName(),
args = call_spec.getArgumentNames(),
star_list_arg = call_spec.getStarListArgumentName(),
star_dict_arg = call_spec.getStarDictArgumentName(),
num_defaults = call_spec.getDefaultCount(),
positional = call_node.getPositionalArguments(),
pairs = ()
)
values = []
for positional_arg in call_node.getPositionalArguments():
for _arg_name, arg_value in iterItems( args_dict ):
if arg_value is positional_arg:
values.append( arg_value )
result = ExpressionFunctionCall(
function_body = self,
values = values,
source_ref = call_node.getSourceReference()
)
return (
result,
"new_statements", # TODO: More appropiate tag maybe.
"Replaced call to created function body '%s' with direct function call" % self.getName()
)
except TooManyArguments as e:
from nuitka.nodes.NodeMakingHelpers import (
makeRaiseExceptionReplacementExpressionFromInstance,
wrapExpressionWithSideEffects
)
result = wrapExpressionWithSideEffects(
new_node = makeRaiseExceptionReplacementExpressionFromInstance(
expression = call_node,
exception = e.getRealException()
),
old_node = call_node,
side_effects = call_node.extractPreCallSideEffects()
)
return (
result,
"new_statements,new_raise", # TODO: More appropiate tag maybe.
"Replaced call to created function body '%s' to argument error" % self.getName()
)
def removeKnowledge( self, value_friend ):
to_remove = []
for variable, value in iterItems( self.variables ):
if value is value_friend:
to_remove.append( variable )
for remove in to_remove:
del self.variables[ remove ]
def removeKnowledge(self, value_friend):
to_remove = []
for variable, value in iterItems(self.variables):
if value is value_friend:
to_remove.append(variable)
for remove in to_remove:
del self.variables[remove]
def getMappingStringKeyPairs(self):
pairs = []
for key, value in iterItems(self.constant):
pairs.append((key,
makeConstantRefNode(constant=value,
source_ref=self.source_ref)))
return pairs
def mergeMultipleBranches(self, collections):
assert collections
# Optimize for length 1, which is trivial merge and needs not a
# lot of work.
if len(collections) == 1:
self.replaceBranch(collections[0])
return None
variable_versions = {}
for collection in collections:
for variable, version in iterItems(collection.variable_actives):
if variable not in variable_versions:
variable_versions[variable] = set([version])
else:
variable_versions[variable].add(version)
for collection in collections:
for variable, versions in iterItems(variable_versions):
if variable not in collection.variable_actives:
versions.add(0)
self.variable_actives = {}
# merge_traces = None
for variable, versions in iterItems(variable_versions):
if len(versions) == 1:
version, = versions
else:
version = self.addVariableMergeMultipleTrace(
variable=variable,
traces=[
self.getVariableTrace(variable, version) for version in versions
],
)
# if merge_traces is None:
# merge_traces = [trace_merge]
# else:
# merge_traces.append(trace_merge)
self.markCurrentVariableTrace(variable, version)
def dumpTrace(self):
for i in range(5):
print("*" * 80)
for variable, traces in iterItems(self.variable_targets):
print(variable)
print("*" * 80)
for trace in traces:
print(trace)
print("*" * 80)
def wrapper(*args, **kw):
new_kw = {}
for key, value in iterItems(kw):
if key in builtin_all_names:
key = key + "_arg"
new_kw[key] = value
return f(*args, **new_kw)
def wrapper(*args, **kw):
new_kw = {}
for key, value in iterItems(kw):
if key in builtin_all_names:
key = key + "_arg"
new_kw[key] = value
return f(*args, **new_kw)
def getVariableTraces(self, variable):
result = []
for key, variable_trace in iterItems(self.variable_traces):
candidate = key[0]
if variable is candidate:
result.append(variable_trace)
return result
def dumpTrace( self ):
for i in range(5):
print( "*" * 80 )
for variable, traces in iterItems( self.variable_targets ):
print( variable )
print( "*" * 80 )
for trace in traces:
print( trace )
print( "*" * 80 )
def getVariableTraces(self, variable):
result = []
for key, variable_trace in iterItems(self.variable_traces):
candidate = key[0]
if variable is candidate:
result.append(variable_trace)
return result
def getMappingStringKeyPairs(self):
assert self.isMapping()
pairs = []
source_ref = self.getSourceReference()
for key, value in iterItems(self.constant):
pairs.append((key, ExpressionConstantRef(constant=value, source_ref=source_ref)))
return pairs
def mergeMultipleBranches(self, collections):
assert collections
# Optimize for length 1, which is trivial merge and needs not a
# lot of work.
if len(collections) == 1:
self.replaceBranch(collections[0])
return None
# print("Enter mergeMultipleBranches", len(collections))
with TimerReport(
message="Running merge for %s took %%.2f seconds" %
collections,
decider=lambda: 0,
):
variable_versions = defaultdict(OrderedSet)
for collection in collections:
for variable, version in iterItems(
collection.variable_actives):
variable_versions[variable].add(version)
for collection in collections:
for variable, versions in iterItems(variable_versions):
if variable not in collection.variable_actives:
versions.add(0)
self.variable_actives = {}
for variable, versions in iterItems(variable_versions):
if len(versions) == 1:
(version, ) = versions
else:
version = self.addVariableMergeMultipleTrace(
variable=variable,
traces=tuple(
self.getVariableTrace(variable, version)
for version in versions),
)
self.markCurrentVariableTrace(variable, version)
def updateVariablesFromCollection(old_collection, new_collection, source_ref):
# After removing/adding traces, we need to pre-compute the users state
# information.
touched_variables = set()
loop_trace_removal = set()
if old_collection is not None:
for (variable, _version), variable_trace in iterItems(
old_collection.getVariableTracesAll()):
variable.removeTrace(variable_trace)
touched_variables.add(variable)
if variable_trace.isLoopTrace():
loop_trace_removal.add(variable)
if new_collection is not None:
for (variable, _version), variable_trace in iterItems(
new_collection.getVariableTracesAll()):
variable.addTrace(variable_trace)
touched_variables.add(variable)
if variable_trace.isLoopTrace():
if variable in loop_trace_removal:
loop_trace_removal.remove(variable)
# Release the memory, and prevent the "active" state from being ever
# inspected, it's useless now.
new_collection.variable_actives.clear()
del new_collection.variable_actives
for variable in touched_variables:
variable.updateUsageState()
if loop_trace_removal:
if new_collection is not None:
new_collection.signalChange(
"var_usage",
source_ref,
lambda: "Loop variable '%s' usage ceased." % ",".join(
variable.getName() for variable in loop_trace_removal),
)
def getMappingPairs(self):
assert self.isMapping()
pairs = []
source_ref = self.getSourceReference()
for key, value in iterItems(self.constant):
pairs.append(
ExpressionConstantRef(constant=key, source_ref=source_ref),
ExpressionConstantRef(constant=value, source_ref=source_ref))
return pairs
def getMappingStringKeyPairs(self):
assert self.isMapping()
pairs = []
source_ref = self.getSourceReference()
for key, value in iterItems(self.constant):
pairs.append(
(key, makeConstantRefNode(constant=value,
source_ref=source_ref)))
return pairs
def mergeBranchVariables(a, b):
result = {}
for variable, value in iterItems(a):
if variable in b:
merged = ValueFriends.mergeBranchFriendValues(
value, b[variable])
if merged is not None:
result[variable] = merged
else:
pass
return result
def getMappingPairs(self):
assert self.isMapping()
pairs = []
source_ref = self.getSourceReference()
for key, value in iterItems(self.constant):
pairs.append(
makeConstantRefNode(constant=key, source_ref=source_ref),
makeConstantRefNode(constant=value, source_ref=source_ref),
)
return pairs
def mergeMultipleBranches(self, collections):
assert collections
# Optimize for length 1, which is trivial merge and needs not a
# lot of work.
if len(collections) == 1:
self.replaceBranch(collections[0])
return None
variable_versions = {}
for collection in collections:
for variable, version in iterItems(collection.variable_actives):
if variable not in variable_versions:
variable_versions[variable] = OrderedSet((version,))
else:
variable_versions[variable].add(version)
for collection in collections:
for variable, versions in iterItems(variable_versions):
if variable not in collection.variable_actives:
versions.add(0)
self.variable_actives = {}
for variable, versions in iterItems(variable_versions):
if len(versions) == 1:
(version,) = versions
else:
version = self.addVariableMergeMultipleTrace(
variable=variable,
traces=tuple(
self.getVariableTrace(variable, version) for version in versions
),
)
self.markCurrentVariableTrace(variable, version)
def considerForDeferral(constant_value):
if isMarshalConstant(constant_value):
return
module_context.getConstantCode(constant_value)
constant_type = type(constant_value)
if constant_type in (tuple, list, set, frozenset):
for element in constant_value:
considerForDeferral(element)
elif constant_type is dict:
for key, value in iterItems(constant_value):
considerForDeferral(key)
considerForDeferral(value)
def considerForDeferral(constant_value):
if isMarshalConstant(constant_value):
return
module_context.getConstantCode(constant_value)
constant_type = type(constant_value)
if constant_type in (tuple, list, set, frozenset):
for element in constant_value:
considerForDeferral(element)
elif constant_type is dict:
for key, value in iterItems(constant_value):
considerForDeferral(key)
considerForDeferral(value)
def getConstantsInitCode(context):
emit = SourceCodeCollector()
sorted_constants = sorted(iterItems(context.getConstants()),
key=lambda k: (len(k), k))
for constant_identifier, constant_value in sorted_constants:
_addConstantInitCode(emit=emit,
constant_type=type(constant_value),
constant_value=constant_value,
constant_identifier=constant_identifier,
module_level=False,
context=context)
return emit.codes
def updateVariablesFromCollection(old_collection, new_collection):
# After removing/adding traces, we need to pre-compute the users state
# information.
touched_variables = set()
if old_collection is not None:
for (variable, _version), variable_trace in iterItems(
old_collection.getVariableTracesAll()):
variable.removeTrace(variable_trace)
touched_variables.add(variable)
if new_collection is not None:
for (variable, _version), variable_trace in iterItems(
new_collection.getVariableTracesAll()):
variable.addTrace(variable_trace)
touched_variables.add(variable)
# Release the memory, and prevent the "active" state from being ever
# inspected, it's useless now.
new_collection.variable_actives.clear()
del new_collection.variable_actives
for variable in touched_variables:
variable.updateUsageState()
def enableDebug(globals_dict):
templates = dict(globals_dict)
class TemplateWrapper:
""" Wrapper around templates.
To better trace and control template usage.
"""
def __init__(self, name, value):
self.name = name
self.value = value
def __str__(self):
return self.value
def __mod__(self, other):
assert type(other) is dict, self.name
for key in other.keys():
if "%%(%s)" % key not in self.value:
from logging import warning
warning(
"Extra value '%s' provided to template '%s'.",
key,
self.name
)
try:
return self.value % other
except KeyError as e:
raise KeyError(self.name, *e.args)
def split(self, sep):
return self.value.split(sep)
for template_name, template_value in iterItems(templates):
# Ignore internal attribute like "__name__" that the module will also
# have of course.
if template_name.startswith('_'):
continue
if type(template_value) is str:
globals_dict[template_name] = TemplateWrapper(
template_name,
template_value
)
def enableDebug(globals_dict):
templates = dict(globals_dict)
class TemplateWrapper:
""" Wrapper around templates.
To better trace and control template usage.
"""
def __init__(self, name, value):
self.name = name
self.value = value
def __str__(self):
return self.value
def __mod__(self, other):
assert type(other) is dict, self.name
for key in other.keys():
if "%%(%s)" % key not in self.value:
from logging import warning
warning(
"Extra value '%s' provided to template '%s'.",
key,
self.name
)
try:
return self.value % other
except KeyError as e:
raise KeyError(self.name, *e.args)
def split(self, sep):
return self.value.split(sep)
for template_name, template_value in iterItems(templates):
# Ignore internal attribute like "__name__" that the module will also
# have of course.
if template_name.startswith('_'):
continue
if type(template_value) is str:
globals_dict[template_name] = TemplateWrapper(
template_name,
template_value
)
def getConstantsInitCode(context):
emit = SourceCodeCollector()
# Sort items by length and name, so we are determistic and pretty.
sorted_constants = sorted(iterItems(context.getConstants()),
key=lambda k: (len(k), k))
for constant_identifier, constant_value in sorted_constants:
_addConstantInitCode(emit=emit,
constant_type=type(constant_value),
constant_value=constant_value,
constant_identifier=constant_identifier,
module_level=False,
context=context)
return emit.codes
def mergeBranchVariables( a, b ):
result = {}
for variable, value in iterItems( a ):
if variable in b:
merged = ValueFriends.mergeBranchFriendValues(
value,
b[ variable ]
)
if merged is not None:
result[ variable ] = merged
else:
pass
return result
def getConstantsInitCode(context):
# There are many cases for constants to be created in the most efficient
# way, pylint: disable=R0912
emit = SourceCodeCollector()
sorted_constants = sorted(iterItems(context.getConstants()),
key=lambda k: (len(k), k))
for constant_identifier, constant_value in sorted_constants:
_addConstantInitCode(emit=emit,
constant_type=type(constant_value),
constant_value=constant_value,
constant_identifier=constant_identifier,
module_level=False,
context=context)
return emit.codes
def computeStatement( self, constraint_collection ):
old_body = self.getBody()
result = constraint_collection.onStatementsSequence( old_body )
if result is not old_body:
self.setBody( result )
# TODO: That should be a method of the constraint_collection
for variable, friend in iterItems( dict( constraint_collection.variables ) ):
if variable.getOwner() is self:
del constraint_collection.variables[ variable ]
# TODO: Back propagate now.
friend.onRelease( self )
if result is None:
return None, "new_statements", "Removed empty temporary block"
return self, None, None
def getConstantsInitCode(context):
emit = SourceCodeCollector()
sorted_constants = sorted(
iterItems(context.getConstants()),
key = lambda k: (len(k), k )
)
for constant_identifier, constant_value in sorted_constants:
_addConstantInitCode(
emit = emit,
constant_type = type(constant_value),
constant_value = constant_value,
constant_identifier = constant_identifier,
module_level = False,
context = context
)
return emit.codes
def decideMarshal(constant_value):
constant_type = type(constant_value)
if constant_type is type:
# Types cannot be marshalled, there is no choice about it.
return False
elif constant_type is dict:
# Look at all the keys an values, if one of it cannot be marshalled,
# or should not, that is it.
for key, value in iterItems(constant_value):
if not decideMarshal(key):
return False
if not decideMarshal(value):
return False
elif constant_type in (tuple, list, set, frozenset):
for element_value in constant_value:
if not decideMarshal(element_value):
return False
return True
def decideMarshal(constant_value):
constant_type = type(constant_value)
if constant_type is type:
# Types cannot be marshalled, there is no choice about it.
return False
elif constant_type is dict:
# Look at all the keys an values, if one of it cannot be marshalled,
# or should not, that is it.
for key, value in iterItems(constant_value):
if not decideMarshal(key):
return False
if not decideMarshal(value):
return False
elif constant_type in (tuple, list, set, frozenset):
for element_value in constant_value:
if not decideMarshal(element_value):
return False
return True
def getConstantsInitCode(context):
emit = SourceCodeCollector()
check = SourceCodeCollector()
# Sort items by length and name, so we are deterministic and pretty.
sorted_constants = sorted(iterItems(context.getConstants()), key=lambda k: (len(k[0]), k[0]))
for constant_identifier, constant_value in sorted_constants:
_addConstantInitCode(
emit=emit,
check=check,
constant_type=type(constant_value),
constant_value=constant_value,
constant_identifier=constant_identifier,
module_level=False,
context=context,
)
return emit.codes, check.codes
def getConstantsInitCode(context):
# There are many cases for constants to be created in the most efficient
# way, pylint: disable=R0912
emit = SourceCodeCollector()
sorted_constants = sorted(
iterItems(context.getConstants()),
key = lambda k: (len(k), k )
)
for constant_identifier, constant_value in sorted_constants:
_addConstantInitCode(
emit = emit,
constant_type = type(constant_value),
constant_value = constant_value,
constant_identifier = constant_identifier,
module_level = False,
context = context
)
return emit.codes
def enableDebug():
templates = dict( globals() )
class TemplateWrapper:
""" Wrapper around templates, to better trace and control template usage. """
def __init__( self, name, value ):
self.name = name
self.value = value
def __str__( self ):
return self.value
def __mod__( self, other ):
assert type( other ) is dict, self.name
for key in other.keys():
if "%%(%s)" % key not in self.value:
from logging import warning
warning( "Extra value '%s' provided to template '%s'.", key, self.name )
return self.value % other
def split( self, sep ):
return self.value.split( sep )
from nuitka.__past__ import iterItems
for template_name, template_value in iterItems( templates ):
if template_name.startswith( "_" ):
continue
if type( template_value ) is str:
globals()[ template_name ] = TemplateWrapper(
template_name,
template_value
)
def considerForDeferral(constant_value):
module_context.getConstantCode(constant_value)
if isMarshalConstant(constant_value):
return
constant_type = type(constant_value)
if constant_type in (tuple, list, set, frozenset):
for element in constant_value:
considerForDeferral(element)
elif constant_type is dict:
for key, value in iterItems(constant_value):
considerForDeferral(key)
considerForDeferral(value)
elif constant_type is slice:
considerForDeferral(constant_value.start)
considerForDeferral(constant_value.step)
considerForDeferral(constant_value.stop)
elif constant_type is xrange:
if xrange is range:
if python_version >= 330:
# For Python2 ranges, we use C long values directly.
considerForDeferral(constant_value.start)
considerForDeferral(constant_value.step)
considerForDeferral(constant_value.stop)
else:
parts = [int(value) for value in str(constant_value)[6:-1].split(",")]
if len(parts) <= 1:
parts.append(0)
if len(parts) <= 2:
parts.append(1)
for value in parts:
considerForDeferral(value)
def getWrittenVariables( self ):
return [
variable
for variable, usage in iterItems( self.variable_usages )
if not usage.isReadOnly()
]
def dump( self ):
debug( "Constraint collection state:" )
for variable_name, variable_info in sorted( iterItems( self.variables ) ):
debug( "%r: %r", variable_name, variable_info )
def getConstantAccess(to_name, constant, emit, context):
# Many cases, because for each type, we may copy or optimize by creating
# empty. pylint: disable=R0912,R0915
if type(constant) is dict:
if constant:
for key, value in iterItems(constant):
# key cannot be mutable.
assert not isMutable(key)
if isMutable(value):
needs_deep = True
break
else:
needs_deep = False
if needs_deep:
code = "DEEP_COPY( %s )" % getConstantCode(
constant = constant,
context = context
)
else:
code = "PyDict_Copy( %s )" % getConstantCode(
constant = constant,
context = context
)
else:
code = "PyDict_New()"
ref_count = 1
elif type(constant) is set:
if constant:
code = "PySet_New( %s )" % getConstantCode(
constant = constant,
context = context
)
else:
code = "PySet_New( NULL )"
ref_count = 1
elif type(constant) is list:
if constant:
for value in constant:
if isMutable(value):
needs_deep = True
break
else:
needs_deep = False
if needs_deep:
code = "DEEP_COPY( %s )" % getConstantCode(
constant = constant,
context = context
)
else:
code = "LIST_COPY( %s )" % getConstantCode(
constant = constant,
context = context
)
else:
code = "PyList_New( 0 )"
ref_count = 1
elif type(constant) is tuple:
for value in constant:
if isMutable(value):
needs_deep = True
break
else:
needs_deep = False
if needs_deep:
code = "DEEP_COPY( %s )" % getConstantCode(
constant = constant,
context = context
)
ref_count = 1
else:
code = getConstantCode(
context = context,
constant = constant
)
ref_count = 0
else:
code = getConstantCode(
context = context,
constant = constant
)
ref_count = 0
emit(
"%s = %s;" % (
to_name,
code,
)
)
if ref_count:
context.addCleanupTempName(to_name)
def __addConstantInitCode(context, emit, check, constant_type, constant_value,
constant_identifier, module_level):
""" Emit code for a specific constant to be prepared during init.
This may be module or global init. Code makes sure that nested
constants belong into the same scope.
"""
# This has many cases, that all return, and do a lot.
# pylint: disable=R0911,R0912,R0915,R0914
# For the module level, we only mean to create constants that are used only
# inside of it. For the global level, it must must be single use.
if module_level:
if context.global_context.getConstantUseCount(constant_identifier) != 1:
return
else:
if context.getConstantUseCount(constant_identifier) == 1:
return
# Adding it to "done". We cannot have recursive constants, so this is OK
# to be done now.
done.add(constant_identifier)
# Use shortest code for ints and longs.
if constant_type is long:
# See above, same for long values. Note: These are of course not
# existent with Python3 which would have covered it before.
if constant_value >= 0 and constant_value <= max_unsigned_long:
emit (
"%s = PyLong_FromUnsignedLong( %sul );" % (
constant_identifier,
constant_value
)
)
return
elif constant_value < 0 and constant_value >= min_signed_long:
emit (
"%s = PyLong_FromLong( %sl );" % (
constant_identifier,
constant_value
)
)
return
elif constant_value == min_signed_long-1:
# There are compilers out there, that give warnings for the literal
# MININT when used. We work around that warning here.
emit(
"""\
%s = PyLong_FromLong( %sl ); // To be corrected with -1 in-place next lines.
CHECK_OBJECT( const_int_pos_1 );
%s = PyNumber_InPlaceSubtract( %s, const_int_pos_1 );""" % (
constant_identifier,
min_signed_long,
constant_identifier,
constant_identifier
)
)
return
else:
# Note, other longs cannot be handled like that yet. We might create
# code that does it better in the future, abusing e.g. internal
# representation of "long" integer values.
pass
elif constant_type is int:
if constant_value >= min_signed_long:
emit(
"%s = PyInt_FromLong( %sl );" % (
constant_identifier,
constant_value
)
)
return
else:
# There are compilers out there, that give warnings for the literal
# MININT when used. We work around that warning here.
assert constant_value == min_signed_long-1
emit(
"""\
%s = PyInt_FromLong( %sl ); // To be corrected in next line.
%s = PyNumber_InPlaceSubtract( %s, const_int_pos_1 );""" % (
constant_identifier,
min_signed_long,
constant_identifier,
constant_identifier
)
)
return
if constant_type is unicode:
# Attempting to marshal is OK, but esp. Python2 cannot do it for all
# "unicode" values.
if attemptToMarshal(constant_identifier, constant_value, emit):
return
try:
encoded = constant_value.encode("utf-8")
if str is not unicode:
emit(
"%s = UNSTREAM_UNICODE( %s );" % (
constant_identifier,
stream_data.getStreamDataCode(encoded)
)
)
else:
emit(
"%s = UNSTREAM_STRING( %s, %d );" % (
constant_identifier,
stream_data.getStreamDataCode(encoded),
1 if _isAttributeName(constant_value) else 0
)
)
return
except UnicodeEncodeError:
# So fall back to below code, which will unstream it then.
pass
elif constant_type is str:
# Python3: Strings that can be encoded as UTF-8 are done more or less
# directly. When they cannot be expressed as UTF-8, that is rare not we
# can indeed use pickling.
assert str is not unicode
if len(constant_value) == 1:
emit(
"%s = UNSTREAM_CHAR( %d, %d );" % (
constant_identifier,
ord(constant_value[0]),
1 if _isAttributeName(constant_value) else 0
)
)
else:
emit(
"%s = UNSTREAM_STRING( %s, %d );" % (
constant_identifier,
stream_data.getStreamDataCode(constant_value),
1 if _isAttributeName(constant_value) else 0
)
)
return
elif constant_type is bytes:
# Python3 only, for Python2, bytes do not happen.
assert str is unicode
emit(
"%s = UNSTREAM_BYTES( %s );" % (
constant_identifier,
stream_data.getStreamDataCode(constant_value)
)
)
return
if constant_type is float:
emit(
"%s = UNSTREAM_FLOAT( %s );" % (
constant_identifier,
stream_data.getStreamDataCode(
value = struct.pack("<d", constant_value),
fixed_size = True
)
)
)
return
if constant_type is dict:
# Not all dictionaries can or should be marshaled. For small ones,
# or ones with strange values, like "{1:type}", we have to do it.
if attemptToMarshal(constant_identifier, constant_value, emit):
return
emit(
"%s = _PyDict_NewPresized( %d );" % (
constant_identifier,
len(constant_value)
)
)
for key, value in iterItems(constant_value):
key_name = getConstantCodeName(context, key)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(key),
constant_value = key,
constant_identifier = key_name,
module_level = module_level,
context = context
)
value_name = getConstantCodeName(context, value)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(value),
constant_value = value,
constant_identifier = value_name,
module_level = module_level,
context = context
)
# TODO: Error checking for debug.
emit(
"PyDict_SetItem( %s, %s, %s );" % (
constant_identifier,
key_name,
value_name
)
)
emit(
"assert( PyDict_Size( %s ) == %d );" % (
constant_identifier,
len(constant_value)
)
)
return
if constant_type is tuple:
# Not all tuples can or should be marshaled. For small ones,
# or ones with strange values, like "(type,)", we have to do it.
if attemptToMarshal(constant_identifier, constant_value, emit):
return
emit(
"%s = PyTuple_New( %d );" % (
constant_identifier,
len(constant_value)
)
)
for count, element_value in enumerate(constant_value):
element_name = getConstantCodeName(
context = context,
constant = element_value
)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(element_value),
constant_value = element_value,
constant_identifier = getConstantCodeName(
context = context,
constant = element_value
),
module_level = module_level,
context = context
)
# Do not take references, these won't be deleted ever.
emit(
"PyTuple_SET_ITEM( %s, %d, %s ); Py_INCREF( %s );" % (
constant_identifier,
count,
element_name,
element_name
)
)
return
if constant_type is list:
# Not all lists can or should be marshaled. For small ones,
# or ones with strange values, like "[type]", we have to do it.
if attemptToMarshal(constant_identifier, constant_value, emit):
return
emit(
"%s = PyList_New( %d );" % (
constant_identifier,
len(constant_value)
)
)
for count, element_value in enumerate(constant_value):
element_name = getConstantCodeName(
context = context,
constant = element_value
)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(element_value),
constant_value = element_value,
constant_identifier = element_name,
module_level = module_level,
context = context
)
# Do not take references, these won't be deleted ever.
emit(
"PyList_SET_ITEM( %s, %d, %s ); Py_INCREF( %s );" % (
constant_identifier,
count,
element_name,
element_name
)
)
return
if constant_type is set:
# Not all sets can or should be marshaled. For small ones,
# or ones with strange values, like "{type}", we have to do it.
if attemptToMarshal(constant_identifier, constant_value, emit):
return
# TODO: Hinting size is really not possible?
emit(
"%s = PySet_New( NULL );" % constant_identifier
)
for element_value in constant_value:
element_name = getConstantCodeName(
context = context,
constant = element_value
)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(element_value),
constant_value = element_value,
constant_identifier = element_name,
module_level = module_level,
context = context
)
emit(
"PySet_Add( %s, %s );" % (
constant_identifier,
element_name
)
)
emit(
"assert( PySet_Size( %s ) == %d );" % (
constant_identifier,
len(constant_value)
)
)
return
if constant_type is slice:
slice1_name = getConstantCodeName(context, constant_value.start)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(constant_value.start),
constant_value = constant_value.start,
constant_identifier = slice1_name,
module_level = module_level,
context = context
)
slice2_name = getConstantCodeName(context, constant_value.stop)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(constant_value.stop),
constant_value = constant_value.stop,
constant_identifier = slice2_name,
module_level = module_level,
context = context
)
slice3_name = getConstantCodeName(context, constant_value.step)
_addConstantInitCode(
emit = emit,
check = check,
constant_type = type(constant_value.step),
constant_value = constant_value.step,
constant_identifier = slice3_name,
module_level = module_level,
context = context
)
emit(
"%s = PySlice_New( %s, %s, %s );" % (
constant_identifier,
slice1_name,
slice2_name,
slice3_name
)
)
return
# TODO: Ranges could very well be created for Python3. And "frozenset" and
# set, are to be examined.
if constant_type in (frozenset, complex, unicode, long, range):
# Lets attempt marshal these.
if attemptToMarshal(constant_identifier, constant_value, emit):
return
emit(
_getUnstreamCode(constant_value, constant_identifier)
)
return
# Must not reach this, if we did, it's in error, and we need to know.
assert False, (type(constant_value), constant_value, constant_identifier)
def getCodeObjects(self):
return sorted(iterItems(self.code_objects))
