def __init__(self, name, code_prefix):
CodeNodeMixin.__init__(self, name=name, code_prefix=code_prefix)
self.providing = OrderedDict()
self.temp_variables = OrderedDict()
self.temp_scopes = OrderedDict()
self.preserver_id = 0
def _getAnonBuiltins():
# We use the order when encoding in the constants blob. Therefore it is imported
# to not reorder these values, the C side uses the absolute indexes.
anon_names = OrderedDict()
anon_codes = OrderedDict()
# Strangely these are not in Python3 types module
anon_names["NoneType"] = type(None)
anon_codes["NoneType"] = "Py_TYPE(Py_None)"
anon_names["ellipsis"] = type(Ellipsis) # see above
anon_codes["ellipsis"] = "&PyEllipsis_Type"
anon_names["NotImplementedType"] = type(NotImplemented)
anon_codes["NotImplementedType"] = "Py_TYPE(Py_NotImplemented)"
anon_names["function"] = FunctionType
anon_codes["function"] = "&PyFunction_Type"
anon_names["generator"] = GeneratorType
anon_codes["generator"] = "&PyGenerator_Type"
anon_names["builtin_function_or_method"] = BuiltinFunctionType
anon_codes["builtin_function_or_method"] = "&PyCFunction_Type"
# Can't really have it until we have __nuitka__
# "compiled_function" : BuiltinFunctionType,
# "compiled_generator" : GeneratorType, # see above
# anon_codes["compiled_function"] = "&Nuitka_Function_Type"
# anon_codes["compiled_generator"] = "&Nuitka_Generator_Type"
anon_names["code"] = type(_getAnonBuiltins.__code__)
anon_codes["code"] = "&PyCode_Type"
if python_version < 0x300:
# There are only there for Python2,
# pylint: disable=I0021,no-name-in-module
from types import ClassType, InstanceType, MethodType
with open(sys.executable) as any_file:
anon_names["file"] = type(any_file)
anon_codes["file"] = "&PyFile_Type"
anon_names["classobj"] = ClassType
anon_codes["classobj"] = "&PyClass_Type"
anon_names["instance"] = InstanceType
anon_codes["instance"] = "&PyInstance_Type"
anon_names["instancemethod"] = MethodType
anon_codes["instancemethod"] = "&PyMethod_Type"
return anon_names, anon_codes
def __init__(self, name, code_prefix, source_ref):
CodeNodeBase.__init__(
self,
name = name,
code_prefix = code_prefix,
source_ref = source_ref
)
self.providing = OrderedDict()
self.keeper_variables = OrderedSet()
self.temp_variables = OrderedDict()
self.temp_scopes = OrderedDict()
def getPreprocessorSymbols():
""" Let plugins provide C defines to be used in compilation.
Notes:
The plugins can each contribute, but are hopefully using
a namespace for their defines.
Returns:
OrderedDict(), where None value indicates no define value,
i.e. "-Dkey=value" vs. "-Dkey"
"""
result = OrderedDict()
for plugin in getActivePlugins():
value = plugin.getPreprocessorSymbols()
if value is not None:
assert type(value) is dict
# We order per plugin, but from the plugins, lets just take a dict
# and achieve determism by ordering the defines by name.
for key, value in sorted(value.items()):
result[key] = value
return result
def getPreprocessorSymbols(cls):
"""Let plugins provide C defines to be used in compilation.
Notes:
The plugins can each contribute, but are hopefully using
a namespace for their defines.
Returns:
OrderedDict(), where None value indicates no define value,
i.e. "-Dkey=value" vs. "-Dkey"
"""
if cls.preprocessor_symbols is None:
cls.preprocessor_symbols = OrderedDict()
for plugin in getActivePlugins():
value = plugin.getPreprocessorSymbols()
if value is not None:
assert type(value) is dict, value
# We order per plugin, but from the plugins, lets just take a dict
# and achieve determism by ordering the defines by name.
for key, value in sorted(value.items()):
# False alarm, pylint: disable=I0021,unsupported-assignment-operation
cls.preprocessor_symbols[key] = value
return cls.preprocessor_symbols
def detectUsedDLLs(source_dir, standalone_entry_points):
def addDLLInfo(count, source_dir, original_filename, binary_filename,
package_name):
used_dlls = detectBinaryDLLs(is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name)
return binary_filename, used_dlls
result = OrderedDict()
with ThreadPoolExecutor(max_workers=Utils.getCoreCount() *
3) as worker_pool:
workers = []
for count, (original_filename, binary_filename,
package_name) in enumerate(standalone_entry_points):
workers.append(
worker_pool.submit(addDLLInfo, count, source_dir,
original_filename, binary_filename,
package_name))
for binary_filename, used_dlls in waitWorkers(workers):
for dll_filename in used_dlls:
# We want these to be absolute paths. Solve that in the parts
# where detectBinaryDLLs is platform specific.
assert os.path.isabs(dll_filename), dll_filename
if dll_filename not in result:
result[dll_filename] = []
result[dll_filename].append(binary_filename)
return result
def detectUsedDLLs(source_dir, standalone_entry_points, use_cache,
update_cache):
def addDLLInfo(count, source_dir, original_filename, binary_filename,
package_name):
used_dlls = detectBinaryDLLs(
is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name,
use_cache=use_cache,
update_cache=update_cache,
)
# Allow plugins to prevent inclusion, this may discard things from used_dlls.
Plugins.removeDllDependencies(dll_filename=binary_filename,
dll_filenames=used_dlls)
for dll_filename in sorted(tuple(used_dlls)):
if not os.path.isfile(dll_filename):
if _unfound_dlls:
general.warning(
"Dependency '%s' could not be found, you might need to copy it manually."
% dll_filename)
_unfound_dlls.add(dll_filename)
used_dlls.remove(dll_filename)
return binary_filename, used_dlls
result = OrderedDict()
with ThreadPoolExecutor(max_workers=Utils.getCoreCount() *
3) as worker_pool:
workers = []
for count, standalone_entry_point in enumerate(
standalone_entry_points):
workers.append(
worker_pool.submit(
addDLLInfo,
count,
source_dir,
standalone_entry_point.source_path,
standalone_entry_point.dest_path,
standalone_entry_point.package_name,
))
for binary_filename, used_dlls in waitWorkers(workers):
for dll_filename in used_dlls:
# We want these to be absolute paths. Solve that in the parts
# where detectBinaryDLLs is platform specific.
assert os.path.isabs(dll_filename), dll_filename
if dll_filename not in result:
result[dll_filename] = []
result[dll_filename].append(binary_filename)
return result
def __init__(
self,
noinclude_setuptools_mode,
noinclude_pytest_mode,
noinclude_ipython_mode,
noinclude_default_mode,
custom_choices,
):
# Default manually to default argument value:
if noinclude_setuptools_mode is None:
noinclude_setuptools_mode = noinclude_default_mode
if noinclude_pytest_mode is None:
noinclude_pytest_mode = noinclude_default_mode
if noinclude_ipython_mode is None:
noinclude_ipython_mode = noinclude_default_mode
self.config = parsePackageYaml(__package__, "anti-bloat.yml")
self.handled_modules = OrderedDict()
# These should be checked, to allow disabling anti-bloat contents.
self.control_tags = set()
if noinclude_setuptools_mode != "allow":
self.handled_modules["setuptools"] = noinclude_setuptools_mode
else:
self.control_tags.add("allow_setuptools")
if noinclude_pytest_mode != "allow":
self.handled_modules["pytest"] = noinclude_pytest_mode
else:
self.control_tags.add("allow_pytest")
if noinclude_ipython_mode != "allow":
self.handled_modules["IPython"] = noinclude_ipython_mode
else:
self.control_tags.add("allow_ipython")
for custom_choice in custom_choices:
if ":" not in custom_choice:
self.sysexit(
"Error, malformed value '%s' for '--noinclude-custom-mode' used."
% custom_choice
)
module_name, mode = custom_choice.rsplit(":", 1)
if mode not in ("error", "warning", "nofollow", "allow", "bytecode"):
self.sysexit(
"Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'"
% (mode, custom_choice)
)
self.handled_modules[ModuleName(module_name)] = mode
def allocateTempReplacementVariable(self, trace_collection, variable_name):
if self.propagation is None:
self.propagation = OrderedDict()
if variable_name not in self.propagation:
provider = trace_collection.getOwner()
self.propagation[variable_name] = provider.allocateTempVariable(
temp_scope=None,
name=self.getCodeName() + "_key_" + variable_name)
return self.propagation[variable_name]
def __init__(self, name, code_prefix):
CodeNodeMixin.__init__(self, name=name, code_prefix=code_prefix)
# TODO: Only Python3 classes need this to be an ordered dict, the order
# of it should come from elsewhere though.
self.providing = OrderedDict()
self.temp_variables = {}
self.temp_scopes = {}
self.preserver_id = 0
def __init__(self, locals_name, owner):
self.locals_name = locals_name
self.owner = owner
# For locals dict variables in this scope.
self.variables = {}
# For local variables in this scope.
self.local_variables = {}
self.providing = OrderedDict()
# Can this be eliminated through replacement of temporary variables
self.mark_for_propagation = False
self.propagation = None
def getExtraCodeFiles():
result = OrderedDict()
for plugin in getActivePlugins():
value = plugin.getExtraCodeFiles()
if value is not None:
assert type(value) is dict
# We order per plugin, but from the plugins, lets just take a dict
# and achieve determism by ordering the files by name.
for key, value in sorted(value.items()):
assert key not in result, key
result["plugin." + plugin.plugin_name + "." + key] = value
return result
def detectUsedDLLs(standalone_entry_points):
result = OrderedDict()
for original_filename, binary_filename, package_name in standalone_entry_points:
used_dlls = detectBinaryDLLs(original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name)
for dll_filename in used_dlls:
# We want these to be absolute paths.
assert os.path.isabs(dll_filename), dll_filename
if dll_filename not in result:
result[dll_filename] = []
result[dll_filename].append(binary_filename)
return result
def _detectBinaryPathDLLsMacOS(original_dir, binary_filename, package_name,
keep_unresolved, recursive):
assert os.path.exists(binary_filename), binary_filename
package_specific_dirs = _getLdLibraryPath(package_name=package_name,
python_rpath=None,
original_dir=original_dir)
# This is recursive potentially and might add more and more.
stdout = getOtoolDependencyOutput(
filename=binary_filename,
package_specific_dirs=_getLdLibraryPath(package_name=package_name,
python_rpath=None,
original_dir=original_dir),
)
paths = _parseOtoolListingOutput(stdout)
had_self, resolved_result = _resolveBinaryPathDLLsMacOS(
original_dir=original_dir,
binary_filename=binary_filename,
paths=paths,
package_specific_dirs=package_specific_dirs,
)
if recursive:
merged_result = OrderedDict(resolved_result)
for sub_dll_filename in resolved_result:
_, sub_result = _detectBinaryPathDLLsMacOS(
original_dir=os.path.dirname(sub_dll_filename),
binary_filename=sub_dll_filename,
package_name=package_name,
recursive=True,
keep_unresolved=True,
)
merged_result.update(sub_result)
resolved_result = merged_result
if keep_unresolved:
return had_self, resolved_result
else:
return OrderedSet(resolved_result)
def makeClone(self, new_owner):
count = 1
# Make it unique.
while 1:
locals_name = self.locals_name + "_inline_%d" % count
if locals_name not in locals_dict_handles:
break
count += 1
result = self.__class__(locals_name=locals_name, owner=new_owner)
variable_translation = {}
# Clone variables as well.
for variable_name, variable in self.variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.variables[variable_name] = new_variable
for variable_name, variable in self.local_variables.items():
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.local_variables[variable_name] = new_variable
result.providing = OrderedDict()
for variable_name, variable in self.providing.items():
if variable in variable_translation:
new_variable = variable_translation[variable]
else:
new_variable = variable.makeClone(new_owner=new_owner)
variable_translation[variable] = new_variable
result.providing[variable_name] = new_variable
return result, variable_translation
def _getBuiltinExceptionNames():
def isExceptionName(builtin_name):
if builtin_name.endswith("Error") or builtin_name.endswith(
"Exception"):
return True
elif builtin_name in (
"StopIteration",
"GeneratorExit",
"SystemExit",
"NotImplemented",
"KeyboardInterrupt",
"StopAsyncIteration",
):
return True
else:
return False
exceptions = OrderedDict()
# Hide Python3 changes for built-in exception names
if python_version < 0x300:
import exceptions as builtin_exceptions
for key in sorted(dir(builtin_exceptions)):
name = str(key)
if isExceptionName(name):
exceptions[name] = getattr(builtin_exceptions, key)
for key in sorted(dir(builtins)):
name = str(key)
if isExceptionName(name):
exceptions[name] = getattr(builtins, key)
else:
for key in sorted(dir(builtins)):
if isExceptionName(key):
exceptions[key] = getattr(builtins, key)
return list(exceptions.keys()), exceptions
def _resolveBinaryPathDLLsMacOS(original_dir, binary_filename, paths,
package_specific_dirs):
had_self = False
result = OrderedDict()
rpaths = _detectBinaryRPathsMacOS(original_dir, binary_filename)
rpaths.update(package_specific_dirs)
for path in paths:
if path.startswith("@rpath/"):
# Resolve rpath to just the ones given, first match.
for rpath in rpaths:
if os.path.exists(os.path.join(rpath, path[7:])):
resolved_path = os.path.normpath(
os.path.join(rpath, path[7:]))
break
else:
# This is only a guess, might be missing package specific directories.
resolved_path = os.path.join(original_dir, path[7:])
elif path.startswith("@loader_path/"):
resolved_path = os.path.join(original_dir, path[13:])
elif os.path.basename(path) == os.path.basename(binary_filename):
# We ignore the references to itself coming from the library id.
continue
else:
resolved_path = path
if not os.path.exists(resolved_path):
inclusion_logger.sysexit(
"Error, failed to resolve DLL path %s (for %s), please report the bug."
% (path, binary_filename))
# Some libraries depend on themselves.
if areSamePaths(binary_filename, resolved_path):
had_self = True
continue
result[resolved_path] = path
return had_self, result
def detectUsedDLLs(source_dir, standalone_entry_points):
result = OrderedDict()
for count, (original_filename, binary_filename,
_package_name) in enumerate(standalone_entry_points):
used_dlls = detectBinaryDLLs(is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename)
for dll_filename in used_dlls:
# We want these to be absolute paths. Solve that in the parts
# where detectBinaryDLLs is platform specific.
assert os.path.isabs(dll_filename), dll_filename
if dll_filename not in result:
result[dll_filename] = []
result[dll_filename].append(binary_filename)
return result
def __init__(self, setuptools_mode, custom_choices):
self.handled_modules = OrderedDict()
if setuptools_mode != "allow":
self.handled_modules["setuptools"] = setuptools_mode
for custom_choice in custom_choices:
if ":" not in custom_choice:
self.sysexit(
"Error, malformed value '%s' for '--noinclude-custom-mode' used."
% custom_choice
)
module_name, mode = custom_choice.rsplit(":", 1)
if mode not in ("error", "warning", "nofollow", "allow"):
self.sysexit(
"Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'"
% (mode, custom_choice)
)
self.handled_modules[ModuleName(module_name)] = mode
anon_codes["code"] = "&PyCode_Type"
if python_version < 0x300:
# There are only there for Python2,
# pylint: disable=I0021,no-name-in-module
from types import ClassType, InstanceType, MethodType
with open(sys.executable) as any_file:
anon_names["file"] = type(any_file)
anon_codes["file"] = "&PyFile_Type"
anon_names["classobj"] = ClassType
anon_codes["classobj"] = "&PyClass_Type"
anon_names["instance"] = InstanceType
anon_codes["instance"] = "&PyInstance_Type"
anon_names["instancemethod"] = MethodType
anon_codes["instancemethod"] = "&PyMethod_Type"
return anon_names, anon_codes
builtin_anon_names, builtin_anon_codes = _getAnonBuiltins()
builtin_anon_values = OrderedDict(
(b, a) for a, b in builtin_anon_names.items())
# For being able to check if it's not hashable, we need something not using
# a hash.
builtin_anon_value_list = tuple(builtin_anon_values)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
""" Reports about code generation.
Initially this is about missing optimization only, but it should expand into
real stuff.
"""
from nuitka.containers.odict import OrderedDict
from nuitka.containers.oset import OrderedSet
from nuitka.Tracing import codegen_logger, optimization_logger
_missing_helpers = OrderedDict()
_missing_operations = OrderedSet()
_missing_trust = OrderedDict()
_error_for_missing = False
# _error_for_missing = True
def doMissingOptimizationReport():
for helper, source_refs in _missing_helpers.items():
message = "Missing C helper code variant, used fallback: %s at %s" % (
",".join(source_ref.getAsString() for source_ref in source_refs),
helper,
)
def createPlistInfoFile(logger, onefile):
# Many details, pylint: disable=too-many-locals
import plistlib
if Options.isStandaloneMode():
bundle_dir = os.path.dirname(
OutputDirectories.getStandaloneDirectoryPath())
else:
bundle_dir = os.path.dirname(
OutputDirectories.getResultRunFilename(onefile=onefile))
result_filename = OutputDirectories.getResultFullpath(onefile=onefile)
app_name = Options.getMacOSAppName() or os.path.basename(result_filename)
signed_app_name = Options.getMacOSSignedAppName() or app_name
app_version = Options.getMacOSAppVersion() or "1.0"
# TODO: We want an OrderedDict probably for stability.
infos = OrderedDict([
("CFBundleDisplayName", app_name),
("CFBundleName", app_name),
("CFBundleIdentifier", signed_app_name),
("CFBundleExecutable", app_name),
("CFBundleInfoDictionaryVersion", "6.0"),
("CFBundlePackageType", "APPL"),
("CFBundleShortVersionString", app_version),
])
icon_paths = Options.getIconPaths()
if icon_paths:
assert len(icon_paths) == 1
icon_path = icon_paths[0]
# Convert to single macOS .icns file if necessary
if not icon_path.endswith(".icns"):
logger.info(
"File '%s' is not in macOS icon format, converting to it." %
icon_path)
icon_build_path = os.path.join(
OutputDirectories.getSourceDirectoryPath(onefile=onefile),
"icons",
)
makePath(icon_build_path)
converted_icon_path = os.path.join(
icon_build_path,
"Icons.icns",
)
convertImageToIconFormat(
logger=logger,
image_filename=icon_path,
icon_filename=converted_icon_path,
)
icon_path = converted_icon_path
icon_name = os.path.basename(icon_path)
resources_dir = os.path.join(bundle_dir, "Resources")
makePath(resources_dir)
copyFile(icon_path, os.path.join(resources_dir, icon_name))
infos["CFBundleIconFile"] = icon_name
# Console mode, which is why we have to use bundle in the first place typically.
if Options.shallDisableConsoleWindow():
infos["NSHighResolutionCapable"] = True
else:
infos["LSBackgroundOnly"] = True
for resource_name, resource_desc in Options.getMacOSAppProtectedResourcesAccesses(
):
if resource_name in infos:
logger.sysexit("Duplicate value for '%s' is not allowed." %
resource_name)
infos[resource_name] = resource_desc
filename = os.path.join(bundle_dir, "Info.plist")
if str is bytes:
plist_contents = plistlib.writePlistToString(infos)
else:
plist_contents = plistlib.dumps(infos)
with openTextFile(filename=filename, mode="wb") as plist_file:
plist_file.write(plist_contents)
def __init__(self, hinted_json_file):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
# Load json file contents from --hinted-json-file= argument
filename = hinted_json_file
try:
# read it and extract the two lists
import_info = json.loads(getFileContents(filename))
except (ValueError, FileNotFoundError):
raise FileNotFoundError('Cannot load json file %s' % filename)
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
# TODO: disabled because self.getPluginOptionBool does not exist anymore
#self.accept_test = self.getPluginOptionBool("test", False)
self.accept_test = False
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = scipy = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
scipy = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
self.info(msg)
to_enable = OrderedDict()
if np:
to_enable["numpy"] = {
"matplotlib": mpl,
"scipy": scipy,
# TODO: Numpy plugin didn't use this, work in progress or not needed?
# "sklearn" : sklearn
}
if tk:
to_enable["tk-inter"] = {}
if qt:
# TODO more scrutiny for the qt options!
to_enable["qt-plugins"] = {}
if mp:
to_enable["multiprocessing"] = {}
if pmw:
to_enable["pmw-freezer"] = {}
if torch:
to_enable["torch"] = {}
if tflow:
to_enable["tensorflow"] = {}
if gevent:
to_enable["gevent"] = {}
if eventlet:
to_enable["eventlet"] = {}
if dill:
to_enable["dill-compat"] = {}
# if trio:
# to_enable["trio"] = {}
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if not show_msg and recurse_count > 0:
self.info(msg)
for plugin_name, option_values in to_enable.items():
self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name)
# No the values could be set.
lateActivatePlugin(plugin_name, option_values)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
self.info(msg)
self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
""" Reports about code generation.
Initially this is about missing optimization only, but it should expand into
real stuff.
"""
from nuitka.containers.odict import OrderedDict
from nuitka.containers.oset import OrderedSet
from nuitka.Tracing import codegen_logger, optimization_logger
_missing_helpers = OrderedDict()
_missing_operations = OrderedSet()
_error_for_missing = False
# _error_for_missing = True
def doMissingOptimizationReport():
for helper, source_refs in _missing_helpers.items():
message = "Missing C helper code variant, used fallback: %s at %s" % (
",".join(source_ref.getAsString() for source_ref in source_refs),
helper,
)
if _error_for_missing:
from nuitka import Options
from nuitka.__past__ import basestring # pylint: disable=I0021,redefined-builtin
from nuitka.build.DataComposerInterface import deriveModuleConstantsBlobName
from nuitka.containers.odict import OrderedDict
from nuitka.containers.oset import OrderedSet
from nuitka.Errors import NuitkaPluginError
from nuitka.freezer.IncludedEntryPoints import makeDllEntryPointOld
from nuitka.ModuleRegistry import addUsedModule
from nuitka.Tracing import plugins_logger, printLine
from nuitka.utils.FileOperations import makePath, relpath
from nuitka.utils.Importing import importFileAsModule
from nuitka.utils.ModuleNames import ModuleName
from .PluginBase import NuitkaPluginBase, post_modules, pre_modules
active_plugins = OrderedDict()
plugin_name2plugin_classes = {}
plugin_options = {}
plugin_values = {}
user_plugins = OrderedSet()
def _addActivePlugin(plugin_class, args, force=False):
plugin_name = plugin_class.plugin_name
# No duplicates please.
if not force:
assert plugin_name not in active_plugins.keys(), (
plugin_name,
active_plugins[plugin_name],
)
def _getAnonBuiltins():
# We use the order when encoding in the constants blob. Therefore it is imported
# to not reorder these values, the C side uses the absolute indexes.
anon_names = OrderedDict()
anon_codes = OrderedDict()
# Strangely these are not in Python3 types module
anon_names["NoneType"] = type(None)
anon_codes["NoneType"] = "Py_TYPE(Py_None)"
anon_names["ellipsis"] = type(Ellipsis) # see above
anon_codes["ellipsis"] = "&PyEllipsis_Type"
anon_names["NotImplementedType"] = type(NotImplemented)
anon_codes["NotImplementedType"] = "Py_TYPE(Py_NotImplemented)"
anon_names["function"] = FunctionType
anon_codes["function"] = "&PyFunction_Type"
anon_names["generator"] = GeneratorType
anon_codes["generator"] = "&PyGenerator_Type"
anon_names["builtin_function_or_method"] = BuiltinFunctionType
anon_codes["builtin_function_or_method"] = "&PyCFunction_Type"
# Can't really have it until we have __nuitka__
# "compiled_function" : BuiltinFunctionType,
# "compiled_generator" : GeneratorType, # see above
# anon_codes["compiled_function"] = "&Nuitka_Function_Type"
# anon_codes["compiled_generator"] = "&Nuitka_Generator_Type"
anon_names["code"] = type(_getAnonBuiltins.__code__)
anon_codes["code"] = "&PyCode_Type"
if python_version < 0x300:
# There are only there for Python2,
# pylint: disable=I0021,no-name-in-module
from types import ClassType, InstanceType, MethodType
# We can to be sure of the type here, so use open without encoding, pylint: disable=unspecified-encoding
with open(sys.executable) as any_file:
anon_names["file"] = type(any_file)
anon_codes["file"] = "&PyFile_Type"
anon_names["classobj"] = ClassType
anon_codes["classobj"] = "&PyClass_Type"
anon_names["instance"] = InstanceType
anon_codes["instance"] = "&PyInstance_Type"
anon_names["instancemethod"] = MethodType
anon_codes["instancemethod"] = "&PyMethod_Type"
if python_version >= 0x270:
anon_names["version_info"] = type(sys.version_info)
anon_codes[
"version_info"] = 'Py_TYPE(Nuitka_SysGetObject("version_info"))'
if python_version >= 0x3A0:
# 3.10 only code, pylint: disable=I0021,unsupported-binary-operation
anon_names["UnionType"] = type(int | str)
anon_codes["UnionType"] = "Nuitka_PyUnion_Type"
return anon_names, anon_codes
def _detectUsedDLLs(source_dir, standalone_entry_points, use_cache,
update_cache):
setupProgressBar(
stage="Detecting used DLLs",
unit="DLL",
total=len(standalone_entry_points),
)
def addDLLInfo(count, source_dir, original_filename, binary_filename,
package_name):
used_dlls = _detectBinaryDLLs(
is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name,
use_cache=use_cache,
update_cache=update_cache,
)
# Allow plugins to prevent inclusion, this may discard things from used_dlls.
Plugins.removeDllDependencies(dll_filename=binary_filename,
dll_filenames=used_dlls)
for dll_filename in sorted(tuple(used_dlls)):
if not os.path.isfile(dll_filename):
if _not_found_dlls:
general.warning("""\
Dependency '%s' could not be found, expect runtime issues. If this is
working with Python, report a Nuitka bug.""" % dll_filename)
_not_found_dlls.add(dll_filename)
used_dlls.remove(dll_filename)
reportProgressBar(binary_filename)
return binary_filename, package_name, used_dlls
result = OrderedDict()
with ThreadPoolExecutor(max_workers=Utils.getCoreCount() *
3) as worker_pool:
workers = []
for count, standalone_entry_point in enumerate(
standalone_entry_points):
workers.append(
worker_pool.submit(
addDLLInfo,
count,
source_dir,
standalone_entry_point.source_path,
standalone_entry_point.dest_path,
standalone_entry_point.package_name,
))
for binary_filename, package_name, used_dlls in waitWorkers(workers):
for dll_filename in used_dlls:
# We want these to be absolute paths. Solve that in the parts
# where _detectBinaryDLLs is platform specific.
assert os.path.isabs(dll_filename), dll_filename
if dll_filename not in result:
result[dll_filename] = (package_name, [])
result[dll_filename][1].append(binary_filename)
closeProgressBar()
return result
