def _detectedSourceFile(filename, module_name, result, user_provided):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(filename=filename,
module_name="_collections_abc",
result=result,
user_provided=user_provided)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = '\n' + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep, Utils.basename(filename), source_code)
debug("Freezing module '%s' (from '%s').", module_name, filename)
result.append(
makeUncompiledPythonModule(
module_name=module_name,
bytecode=marshal.dumps(compile(source_code, filename, "exec")),
is_package=Utils.basename(filename) == "__init__.py",
filename=filename,
user_provided=user_provided))
module_names.add(module_name)
def _detectedPrecompiledFile(filename, module_name, result, user_provided,
technical):
if filename.endswith(".pyc"):
if os.path.isfile(filename[:-1]):
return _detectedSourceFile(
filename=filename[:-1],
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
if module_name in module_names:
return
if Options.isShowInclusion():
inclusion_logger.info("Freezing module '%s' (from '%s')." %
(module_name, filename))
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=loadCodeObjectData(precompiled_filename=filename),
is_package="__init__" in filename,
filename=filename,
user_provided=user_provided,
technical=technical,
)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def _detectedPrecompiledFile(filename, module_name, result, user_provided,
technical):
if filename.endswith(".pyc"):
if Utils.isFile(filename[:-1]):
return _detectedSourceFile(
filename = filename[:-1],
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
if module_name in module_names:
return
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
result.append(
makeUncompiledPythonModule(
module_name = module_name,
bytecode = loadCodeObjectData(
precompiled_filename = filename
),
is_package = "__init__" in filename,
filename = filename,
user_provided = user_provided,
technical = technical
)
)
module_names.add(module_name)
def _detectedPrecompiledFile(filename, module_name, result, user_provided,
technical):
if filename.endswith(".pyc"):
if Utils.isFile(filename[:-1]):
return _detectedSourceFile(
filename = filename[:-1],
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
if module_name in module_names:
return
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
result.append(
makeUncompiledPythonModule(
module_name = module_name,
bytecode = loadCodeObjectData(
precompiled_filename = filename
),
is_package = "__init__" in filename,
filename = filename,
user_provided = user_provided,
technical = technical
)
)
module_names.add(module_name)
def demoteCompiledModuleToBytecode(module):
""" Demote a compiled module to uncompiled (bytecode).
"""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
source_code = readSourceCodeFromFilename(full_name, filename)
source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name,
is_package=False,
source_code=source_code)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name,
is_package=False,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=filename,
bytecode=marshal.dumps(bytecode),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False)
replaceImportedModule(old=module, new=uncompiled_module)
replaceRootModule(old=module, new=uncompiled_module)
assert module.constraint_collection is not None
uncompiled_module.setUsedModules(
module.constraint_collection.getUsedModules())
def _detectedSourceFile(filename, module_name, result, user_provided,
technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(filename=filename,
module_name="_collections_abc",
result=result,
user_provided=user_provided,
technical=technical)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = '\n' + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep, os.path.basename(filename), source_code)
# Debian stretch site.py
source_code = source_code.replace(
"PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")
# Anaconda3 4.1.2 site.py
source_code = source_code.replace(
"def main():",
"def main():return\n\nif 0:\n def _unused():",
)
debug("Freezing module '%s' (from '%s').", module_name, filename)
is_package = os.path.basename(filename) == "__init__.py"
source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
is_package=is_package,
source_code=source_code)
bytecode = compile(source_code, filename, "exec", dont_inherit=True)
bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
is_package=is_package,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=marshal.dumps(bytecode),
is_package=is_package,
filename=filename,
user_provided=user_provided,
technical=technical)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def demoteCompiledModuleToBytecode(module):
""" Demote a compiled module to uncompiled (bytecode).
"""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
if Options.isShowProgress():
info("Demoting module '%s' to bytecode from '%s'." %
(full_name, filename))
source_code = module.getSourceCode()
# Second chance for plugins to modify source code just before turning it
# to bytecode.
source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name,
is_package=False,
source_code=source_code)
bytecode = compile(source_code, filename, "exec", dont_inherit=True)
bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name,
is_package=False,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=full_name,
filename=filename,
bytecode=marshal.dumps(bytecode),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
uncompiled_module.setUsedModules(module.getUsedModules())
module.finalize()
if isImportedModuleByName(full_name):
replaceImportedModule(old=module, new=uncompiled_module)
replaceRootModule(old=module, new=uncompiled_module)
from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule
if isTriggerModule(module):
replaceTriggerModule(old=module, new=uncompiled_module)
def _detectedSourceFile(filename, module_name, result, user_provided):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename = filename,
module_name = "_collections_abc",
result = result,
user_provided = user_provided
)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = '\n' + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
Utils.basename(filename),
source_code
)
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
result.append(
makeUncompiledPythonModule(
module_name = module_name,
bytecode = marshal.dumps(
compile(source_code, filename, "exec")
),
is_package = Utils.basename(filename) == "__init__.py",
filename = filename,
user_provided = user_provided
)
)
module_names.add(module_name)
def demoteCompiledModuleToBytecode(module):
""" Demote a compiled module to uncompiled (bytecode).
"""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
source_code = readSourceCodeFromFilename(full_name, filename)
source_code = Plugins.onFrozenModuleSourceCode(
module_name = full_name,
is_package = False,
source_code = source_code
)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = full_name,
is_package = False,
bytecode = bytecode
)
uncompiled_module = makeUncompiledPythonModule(
module_name = module.getFullName(),
filename = filename,
bytecode = marshal.dumps(bytecode),
is_package = module.isCompiledPythonPackage(),
user_provided = True,
technical = False
)
replaceImportedModule(
old = module,
new = uncompiled_module
)
replaceRootModule(
old = module,
new = uncompiled_module
)
assert module.constraint_collection is not None
uncompiled_module.setUsedModules(module.constraint_collection.getUsedModules())
def demoteCompiledModuleToBytecode(module):
"""Demote a compiled module to uncompiled (bytecode)."""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
if Options.isShowProgress():
inclusion_logger.info("Demoting module %r to bytecode from %r." %
(full_name.asString(), filename))
source_code = module.getSourceCode()
bytecode = demoteSourceCodeToBytecode(module_name=full_name,
source_code=source_code,
filename=filename)
uncompiled_module = makeUncompiledPythonModule(
module_name=full_name,
filename=filename,
bytecode=bytecode,
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
used_modules = module.getUsedModules()
uncompiled_module.setUsedModules(used_modules)
module.finalize()
if isImportedModuleByName(full_name):
replaceImportedModule(old=module, new=uncompiled_module)
replaceRootModule(old=module, new=uncompiled_module)
from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule
if isTriggerModule(module):
replaceTriggerModule(old=module, new=uncompiled_module)
writeImportedModulesNamesToCache(module_name=full_name,
source_code=source_code,
used_modules=used_modules)
def _loadUncompiledModuleFromCache(module_name, is_package, source_code,
source_ref):
result = makeUncompiledPythonModule(
module_name=module_name,
filename=source_ref.getFilename(),
bytecode=demoteSourceCodeToBytecode(
module_name=module_name,
source_code=source_code,
filename=source_ref.getFilename(),
),
user_provided=False,
technical=False,
is_package=is_package,
)
used_modules = OrderedSet()
for used_module_name, line_number in getCachedImportedModulesNames(
module_name=module_name, source_code=source_code):
_module_name, module_filename, finding = Importing.locateModule(
module_name=used_module_name,
parent_package=None,
level=0,
)
assert _module_name == used_module_name
used_modules.add((
used_module_name,
module_filename,
finding,
0,
source_ref.atLineNumber(line_number),
))
# assert not is_package, (module_name, used_modules, result, result.getCompileTimeFilename())
result.setUsedModules(used_modules)
return result
def _makeModuleBodyTooComplex(module_name, module_filename, source_code,
is_package):
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning("""\
Cannot follow import to import module '%r' ('%r') because code is too complex."""
% (
module_name,
module_filename,
))
module = makeUncompiledPythonModule(
module_name=module_name,
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code, module_filename, "exec", dont_inherit=True)),
is_package=is_package,
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
def recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
if not ImportCache.isImportedModuleByPath(module_relpath):
module, source_ref, source_filename = Building.decideModuleTree(
filename = module_filename,
package = module_package,
is_top = False,
is_main = False,
is_shlib = module_kind == "shlib"
)
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
debug(
"Recurse to import '%s' from %s. (%s)",
module.getFullName(),
module_relpath,
reason
)
if module_kind == "py" and source_filename is not None:
try:
Building.createModuleTree(
module = module,
source_ref = source_ref,
source_code = readSourceCodeFromFilename(
module_name = module.getFullName(),
source_filename = source_filename
),
is_main = False
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__
)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name = module.getFullName(),
filename = module_filename,
bytecode = marshal.dumps(
compile(
readSourceCodeFromFilename(module.getFullName(), module_filename),
module_filename,
"exec"
)
),
is_package = module.isCompiledPythonPackage(),
user_provided = True,
technical = False
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
is_added = True
else:
module = ImportCache.getImportedModuleByName(
module.getFullName()
)
is_added = False
assert not module_relpath.endswith("/__init__.py"), module
return module, is_added
else:
return ImportCache.getImportedModuleByPath(module_relpath), False
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
from nuitka.tree import Building
module, source_ref, source_code = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib",
)
if Options.isShowInclusion():
recursion_logger.info("Recurse to import '%s' from '%s'. (%s)" %
(module.getFullName(), module_relpath, reason))
if source_code is not None:
try:
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False,
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning(
"""\
Cannot follow import to module %r (%r) because of %r""" %
(module_relpath, module_filename, e.__class__.__name__))
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning("""\
Cannot recurse to import module %r (%r) because code is too complex.""" % (
module_relpath,
module_filename,
))
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code,
module_filename,
"exec",
dont_inherit=True)),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
return module, True
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
module, source_ref, source_filename = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib")
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
logRecursion("Recurse to import '%s' from '%s'. (%s)",
module.getFullName(), module_relpath, reason)
if module_kind == "py" and source_filename is not None:
try:
source_code = readSourceCodeFromFilename(
module_name=module.getFullName(),
source_filename=source_filename)
Building.createModuleTree(module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath,
module_filename, e.__class__.__name__)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code,
module_filename,
"exec",
dont_inherit=True)),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
is_added = True
else:
module = ImportCache.getImportedModuleByName(module.getFullName())
is_added = False
return module, is_added
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
module, source_ref, source_filename = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib",
)
logRecursion(
"Recurse to import '%s' from '%s'. (%s)",
module.getFullName(),
module_relpath,
reason,
)
if module_kind == "py" and source_filename is not None:
try:
source_code = readSourceCodeFromFilename(
module_name=module.getFullName(), source_filename=source_filename
)
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False,
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__,
)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(
source_code, module_filename, "exec", dont_inherit=True
)
),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
return module, True
def _detectedSourceFile(filename, module_name, result, user_provided,
technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename=filename,
module_name=ModuleName("_collections_abc"),
result=result,
user_provided=user_provided,
technical=technical,
)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = "\n" + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
os.path.basename(filename),
source_code,
)
# Debian stretch site.py
source_code = source_code.replace(
"PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")
if Options.isShowInclusion():
inclusion_logger.info("Freezing module '%s' (from '%s')." %
(module_name, filename))
is_package = os.path.basename(filename) == "__init__.py"
# Plugins can modify source code:
source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
is_package=is_package,
source_code=source_code)
bytecode = compileSourceToBytecode(
source_code=source_code,
filename=module_name.replace(".", os.path.sep) + ".py",
)
# Plugins can modify bytecode code:
bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
is_package=is_package,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=marshal.dumps(bytecode),
is_package=is_package,
filename=filename,
user_provided=user_provided,
technical=technical,
)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
