def _addSlaveMainModule(root_module):
from nuitka.tree.Building import CompiledPythonModule, readSourceCodeFromFilename, createModuleTree
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name=module_name,
package_name=None,
mode=mode,
source_ref=root_module.getSourceReference())
source_code = readSourceCodeFromFilename("__parents_main__",
root_module.getFilename())
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace.
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.main()"""
createModuleTree(module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def _createTriggerLoadedModule(module, trigger_name, code):
from nuitka.tree.Building import createModuleTree
from nuitka.nodes.ModuleNodes import CompiledPythonModule
from nuitka.plugins.Plugins import Plugins
module_name = module.getName() + trigger_name
source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name)
mode = Plugins.decideCompilation(module_name, source_ref)
trigger_module = CompiledPythonModule(
name = module_name,
package_name = module.getPackage(),
mode = mode,
source_ref = source_ref
)
createModuleTree(
module = trigger_module,
source_ref = module.getSourceReference(),
source_code = code,
is_main = False
)
return trigger_module
def decideCompilationMode(is_top, module_name, source_ref):
result = Plugins.decideCompilation(module_name, source_ref)
if result == "bytecode" and is_top:
plugins_logger.warning("""\
Ignoring plugin decision to compile top level package '%s'
as bytecode, the extension module entry point is technically
required to compiled.""" % module_name)
result = "compiled"
return result
def onModuleInitialSet(self):
from nuitka.importing.ImportCache import addImportedModule
from nuitka.ModuleRegistry import getRootTopModule
from nuitka.plugins.Plugins import Plugins
from nuitka.tree.Building import (
CompiledPythonModule,
createModuleTree,
readSourceCodeFromFilename,
)
# First, build the module node and then read again from the
# source code.
root_module = getRootTopModule()
module_name = ModuleName("__parents_main__")
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
module_name=module_name,
is_top=False,
mode=mode,
future_spec=None,
source_ref=root_module.getSourceReference(),
)
source_code = readSourceCodeFromFilename(module_name,
root_module.getFilename())
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace. The forking module was split up
# into multiple modules in Python 3.4.
if python_version >= 0x340:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.spawn").spawn.freeze_support()"""
else:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""
createModuleTree(
module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False,
)
addImportedModule(imported_module=slave_main_module)
yield slave_main_module
def _addSlaveMainModule(root_module):
from nuitka.tree.Building import (
CompiledPythonModule,
readSourceCodeFromFilename,
createModuleTree,
)
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
from sys import hexversion
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name=module_name,
package_name=None,
is_top=False,
mode=mode,
future_spec=None,
source_ref=root_module.getSourceReference(),
)
source_code = readSourceCodeFromFilename(
"__parents_main__", root_module.getFilename()
)
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace. The forking module was split up
# into multiple modules in Python 3.4.0.a2
if hexversion >= 0x030400A2:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.spawn").spawn.freeze_support()"""
else:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""
createModuleTree(
module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False,
)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def _addSlaveMainModule(root_module):
from nuitka.tree.Building import (
CompiledPythonModule,
readSourceCodeFromFilename,
createModuleTree,
)
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
from sys import hexversion
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name=module_name,
package_name=None,
is_top=False,
mode=mode,
future_spec=None,
source_ref=root_module.getSourceReference(),
)
source_code = readSourceCodeFromFilename("__parents_main__",
root_module.getFilename())
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace. The forking module was split up
# into multiple modules in Python 3.4.0.a2
if hexversion >= 0x030400A2:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.spawn").spawn.freeze_support()"""
else:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""
createModuleTree(
module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False,
)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def _createTriggerLoadedModule(module, trigger_name, code):
from nuitka.tree.Building import createModuleTree
from nuitka.nodes.ModuleNodes import CompiledPythonModule
from nuitka.plugins.Plugins import Plugins
module_name = module.getName() + trigger_name
source_ref = fromFilename(module.getCompileTimeFilename() +
trigger_name)
mode = Plugins.decideCompilation(module_name, source_ref)
trigger_module = CompiledPythonModule(name=module_name,
package_name=module.getPackage(),
mode=mode,
source_ref=source_ref)
createModuleTree(module=trigger_module,
source_ref=module.getSourceReference(),
source_code=code,
is_main=False)
return trigger_module
def _addSlaveMainModule(root_module):
from nuitka.tree.Building import CompiledPythonModule, readSourceCodeFromFilename, createModuleTree
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name = module_name,
package_name = None,
mode = mode,
source_ref = root_module.getSourceReference()
)
source_code = readSourceCodeFromFilename(
"__parents_main__",
root_module.getFilename()
)
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace.
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.main()"""
createModuleTree(
module = slave_main_module,
source_ref = root_module.getSourceReference(),
source_code = source_code,
is_main = False
)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def decideCompilationMode(is_top, module_name, for_pgo):
"""Decide the compilation mode for a module.
module_name - The module to decide compilation mode for.
for_pgo - consider PGO information or not
"""
result = Plugins.decideCompilation(module_name)
# Cannot change mode of __main__ to bytecode, that is not going
# to work currently.
if result == "bytecode" and is_top:
plugins_logger.warning("""\
Ignoring plugin decision to compile top level package '%s'
as bytecode, the extension module entry point is technically
required to compiled.""" % module_name)
result = "compiled"
# Include all of standard library as bytecode, for now. We need to identify
# which ones really need that.
if not is_top:
module_filename = Importing.locateModule(module_name=module_name,
parent_package=None,
level=0)[1]
if module_filename is not None and isStandardLibraryPath(
module_filename):
result = "bytecode"
# Plugins need to win over PGO, as they might know it better
if result is None and not for_pgo:
result = decideCompilationFromPGO(module_name=module_name)
# Default if neither plugins nor PGO have expressed an opinion
if result is None:
result = "compiled"
return result
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
# Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements
assert package is None or type(package) is str
assert filename is not None
if is_main and os.path.isdir(filename):
source_filename = os.path.join(filename, "__main__.py")
if not os.path.isfile(source_filename):
sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
(os.path.basename(sys.argv[0]), filename))
sys.exit(2)
filename = source_filename
main_added = True
else:
main_added = False
if os.path.isfile(filename):
source_filename = filename
source_ref = SourceCodeReferences.fromFilename(filename=filename, )
if is_main:
module_name = "__main__"
else:
module_name = os.path.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
if is_shlib:
module_name = module_name.split('.')[0]
if '.' in module_name:
sys.stderr.write(
"Error, '%s' is not a proper python module name.\n" %
(module_name))
sys.exit(2)
if is_shlib:
result = PythonShlibModule(name=module_name,
package_name=package,
source_ref=source_ref)
elif is_main:
result = PythonMainModule(main_added=main_added,
mode=Plugins.decideCompilation(
"__main__", source_ref),
future_spec=None,
source_ref=source_ref)
else:
if package is not None:
full_name = package + '.' + module_name
else:
full_name = module_name
result = CompiledPythonModule(name=module_name,
package_name=package,
mode=Plugins.decideCompilation(
full_name, source_ref),
future_spec=None,
source_ref=source_ref)
elif Importing.isPackageDir(filename):
if is_top:
module_name = splitPath(filename)[-1]
else:
module_name = os.path.basename(filename)
source_filename = os.path.join(filename, "__init__.py")
if not os.path.isfile(source_filename):
source_ref, result = createNamespacePackage(
module_name=module_name,
package_name=package,
module_relpath=filename)
source_filename = None
else:
source_ref = SourceCodeReferences.fromFilename(
filename=os.path.abspath(source_filename), )
if package is not None:
full_name = package + '.' + module_name
else:
full_name = module_name
result = CompiledPythonPackage(name=module_name,
package_name=package,
mode=Plugins.decideCompilation(
full_name, source_ref),
future_spec=None,
source_ref=source_ref)
assert result.getFullName() == full_name, result
else:
sys.stderr.write("%s: can't open file '%s'.\n" %
(os.path.basename(sys.argv[0]), filename))
sys.exit(2)
if not Options.shallHaveStatementLines():
source_ref = source_ref.atInternal()
return result, source_ref, source_filename
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
# Many variables, branches, due to the many cases, pylint: disable=R0912,R0915
assert package is None or type(package) is str
assert filename is not None
if is_main and Utils.isDir(filename):
source_filename = Utils.joinpath(filename, "__main__.py")
if not Utils.isFile(source_filename):
sys.stderr.write(
"%s: can't find '__main__' module in '%s'\n" % (
Utils.basename(sys.argv[0]),
filename
)
)
sys.exit(2)
filename = source_filename
main_added = True
else:
main_added = False
if Utils.isFile(filename):
source_filename = filename
source_ref = SourceCodeReferences.fromFilename(
filename = filename,
)
if is_main:
module_name = "__main__"
else:
module_name = Utils.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
if is_shlib:
module_name = module_name.split('.')[0]
if '.' in module_name:
sys.stderr.write(
"Error, '%s' is not a proper python module name.\n" % (
module_name
)
)
sys.exit(2)
if is_shlib:
result = PythonShlibModule(
name = module_name,
package_name = package,
source_ref = source_ref
)
elif is_main:
result = PythonMainModule(
main_added = main_added,
source_ref = source_ref
)
else:
if package is not None:
full_name = package + "." + module_name
else:
full_name = module_name
decision = Plugins.decideCompilation(full_name, source_ref)
if decision == "compiled":
result = CompiledPythonModule(
name = module_name,
package_name = package,
source_ref = source_ref
)
else:
source_code = readSourceCodeFromFilename(module_name, filename)
source_code = Plugins.onFrozenModuleSourceCode(
module_name = full_name,
is_package = False,
source_code = source_code
)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = module_name,
is_package = False,
bytecode = bytecode
)
result = UncompiledPythonModule(
name = module_name,
package_name = package,
bytecode = bytecode,
filename = filename,
user_provided = True,
technical = False,
source_ref = source_ref
)
# Don't read it anymore.
source_filename = None
elif Importing.isPackageDir(filename):
if is_top:
package_name = Utils.splitpath(filename)[-1]
else:
package_name = Utils.basename(filename)
source_filename = Utils.joinpath(filename, "__init__.py")
if not Utils.isFile(source_filename):
source_ref, result = createNamespacePackage(
package_name = package_name,
module_relpath = filename
)
source_filename = None
else:
source_ref = SourceCodeReferences.fromFilename(
filename = Utils.abspath(source_filename),
)
if package is not None:
full_name = package + "." + package_name
else:
full_name = package_name
decision = Plugins.decideCompilation(full_name, source_ref)
if decision == "compiled":
result = CompiledPythonPackage(
name = package_name,
package_name = package,
source_ref = source_ref
)
else:
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = module_name,
is_package = False,
bytecode = bytecode
)
result = UncompiledPythonPackage(
name = module_name,
package_name = package,
bytecode = bytecode,
filename = filename,
user_provided = True,
technical = False,
source_ref = source_ref
)
# Don't read it anymore.
source_filename = None
else:
sys.stderr.write(
"%s: can't open file '%s'.\n" % (
Utils.basename(sys.argv[0]),
filename
)
)
sys.exit(2)
if not Options.shallHaveStatementLines():
source_ref = source_ref.atInternal()
return result, source_ref, source_filename
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
# Many variables, branches, due to the many cases, pylint: disable=R0912,R0915
assert package is None or type(package) is str
assert filename is not None
if is_main and Utils.isDir(filename):
source_filename = Utils.joinpath(filename, "__main__.py")
if not Utils.isFile(source_filename):
sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
(Utils.basename(sys.argv[0]), filename))
sys.exit(2)
filename = source_filename
main_added = True
else:
main_added = False
if Utils.isFile(filename):
source_filename = filename
source_ref = SourceCodeReferences.fromFilename(filename=filename, )
if is_main:
module_name = "__main__"
else:
module_name = Utils.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
if is_shlib:
module_name = module_name.split('.')[0]
if '.' in module_name:
sys.stderr.write(
"Error, '%s' is not a proper python module name.\n" %
(module_name))
sys.exit(2)
if is_shlib:
result = PythonShlibModule(name=module_name,
package_name=package,
source_ref=source_ref)
elif is_main:
result = PythonMainModule(main_added=main_added,
source_ref=source_ref)
else:
if package is not None:
full_name = package + "." + module_name
else:
full_name = module_name
decision = Plugins.decideCompilation(full_name, source_ref)
if decision == "compiled":
result = CompiledPythonModule(name=module_name,
package_name=package,
source_ref=source_ref)
else:
source_code = readSourceCodeFromFilename(module_name, filename)
source_code = Plugins.onFrozenModuleSourceCode(
module_name=full_name,
is_package=False,
source_code=source_code)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name=module_name,
is_package=False,
bytecode=bytecode)
result = UncompiledPythonModule(name=module_name,
package_name=package,
bytecode=bytecode,
filename=filename,
user_provided=True,
technical=False,
source_ref=source_ref)
# Don't read it anymore.
source_filename = None
elif Importing.isPackageDir(filename):
if is_top:
package_name = Utils.splitpath(filename)[-1]
else:
package_name = Utils.basename(filename)
source_filename = Utils.joinpath(filename, "__init__.py")
if not Utils.isFile(source_filename):
source_ref, result = createNamespacePackage(
package_name=package_name, module_relpath=filename)
source_filename = None
else:
source_ref = SourceCodeReferences.fromFilename(
filename=Utils.abspath(source_filename), )
if package is not None:
full_name = package + "." + package_name
else:
full_name = package_name
decision = Plugins.decideCompilation(full_name, source_ref)
if decision == "compiled":
result = CompiledPythonPackage(name=package_name,
package_name=package,
source_ref=source_ref)
else:
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name=module_name,
is_package=False,
bytecode=bytecode)
result = UncompiledPythonPackage(name=module_name,
package_name=package,
bytecode=bytecode,
filename=filename,
user_provided=True,
technical=False,
source_ref=source_ref)
# Don't read it anymore.
source_filename = None
else:
sys.stderr.write("%s: can't open file '%s'.\n" %
(Utils.basename(sys.argv[0]), filename))
sys.exit(2)
if not Options.shallHaveStatementLines():
source_ref = source_ref.atInternal()
return result, source_ref, source_filename
