def _getCPythonResults(cpython_cmd):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(
args=cpython_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython
):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def _getCPythonResults(cpython_cmd, send_kill):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(args=cpython_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if send_kill:
# Doing it per loop iteration hopefully, pylint: disable=cell-var-from-loop
executeAfterTimePassed(
1.0,
lambda: killProcess("Uncompiled Python program", process.pid))
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def __init__(self):
options = Options.options
self.sep_line1 = "=" * 80
self.sep_line2 = "-" * 80
self.excludes = []
info(self.sep_line1)
if not Options.isStandaloneMode():
info(" can only run in standalone mode")
info(self.sep_line1)
raise SystemExit()
# start a timer
self.timer = StopWatch()
self.timer.start()
# get the list of options
self.myoptions = self.getPluginOptions()
self.tk = self.getPluginOptionBool("tk", None)
self.qt = self.getPluginOptionBool("qt", None)
self.numpy = self.getPluginOptionBool("np", None)
# check for post processors
self.onefile = 1 if self.getPluginOptionBool("onefile", False) else 0
self.onedir = 1 if self.getPluginOptionBool("onedir", False) else 0
self.upx = 1 if self.getPluginOptionBool("upx", False) else 0
if self.onefile + self.onedir + self.upx > 1:
raise SystemExit("only 1 post-processor can be chosen")
# announce how we will execute
msg = " '%s' established the following configuration" % self.plugin_name
info(msg)
info(self.sep_line2)
if self.numpy is False:
options.recurse_not_modules.append("numpy")
info(" --recurse-not-to=numpy")
options.plugins_disabled.append("numpy")
info(" --disable-plugin=numpy")
elif self.numpy is True:
options.plugins_enabled.append("numpy")
info(" --enable-plugin=numpy")
if self.qt is False:
options.recurse_not_modules.append("PIL.ImageQt")
info(" --recurse-not-to=PIL.ImageQt")
options.plugins_disabled.append("qt-plugins")
info(" --disable-plugin=qt-plugins")
elif self.qt is True:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if self.tk is False:
options.recurse_not_modules.append("PIL.ImageTk")
info(" --recurse-not-to=PIL.ImageTk")
options.plugins_disabled.append("tk-inter")
info(" --disable-plugin=tk-inter")
elif self.tk is True:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
info(self.sep_line2)
class MyExit(NuitkaPluginBase):
""" User plugin supporting post-processing in standalone mode.
Notes:
Upon initialization, this plugin establishes a few options.
Among these are enabling or disabling standard plugins.
Compilation post-processing can be either
(a) option "upx" for invoking UPX compression, or
(b) option "onefile" for creating a distribution file in OneFile format, or
(c) TODO: option "onedir" for creating a distribution file in OneDir format.
The options for the standard plugins tk-plugin (code "tk"), numpy-plugin
("np") and qt-plugins ("qt") can also be added as options to this plugin.
This will enable the respective plugin. If the code is prefixed with "no",
then the plugin will be disabled and certain modules will not be recursed
to. For example:
- "tk" will generate "--enable-plugin=tk-plugin"
- "notk" will generate "--disable-plugin=tk-plugin" and
"--recurse-not-to=PIL.ImageTk".
"""
plugin_name = __file__
def __init__(self):
options = Options.options
self.sep_line1 = "=" * 80
self.sep_line2 = "-" * 80
self.excludes = []
info(self.sep_line1)
if not Options.isStandaloneMode():
info(" can only run in standalone mode")
info(self.sep_line1)
raise SystemExit()
# start a timer
self.timer = StopWatch()
self.timer.start()
# get the list of options
self.myoptions = self.getPluginOptions()
self.tk = self.getPluginOptionBool("tk", None)
self.qt = self.getPluginOptionBool("qt", None)
self.numpy = self.getPluginOptionBool("np", None)
# check for post processors
self.onefile = 1 if self.getPluginOptionBool("onefile", False) else 0
self.onedir = 1 if self.getPluginOptionBool("onedir", False) else 0
self.upx = 1 if self.getPluginOptionBool("upx", False) else 0
if self.onefile + self.onedir + self.upx > 1:
raise SystemExit("only 1 post-processor can be chosen")
# announce how we will execute
msg = " '%s' established the following configuration" % self.plugin_name
info(msg)
info(self.sep_line2)
if self.numpy is False:
options.recurse_not_modules.append("numpy")
info(" --recurse-not-to=numpy")
options.plugins_disabled.append("numpy")
info(" --disable-plugin=numpy")
elif self.numpy is True:
options.plugins_enabled.append("numpy")
info(" --enable-plugin=numpy")
if self.qt is False:
options.recurse_not_modules.append("PIL.ImageQt")
info(" --recurse-not-to=PIL.ImageQt")
options.plugins_disabled.append("qt-plugins")
info(" --disable-plugin=qt-plugins")
elif self.qt is True:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if self.tk is False:
options.recurse_not_modules.append("PIL.ImageTk")
info(" --recurse-not-to=PIL.ImageTk")
options.plugins_disabled.append("tk-inter")
info(" --disable-plugin=tk-inter")
elif self.tk is True:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
info(self.sep_line2)
def removeDllDependencies(self, dll_filename, dll_filenames):
if self.tk is False:
basename = os.path.basename(dll_filename)
if basename.startswith(("tk", "tcl")):
info(" exluding " + basename)
self.excludes.append(basename)
yield ()
if "qt" in dll_filename.lower():
print(dll_filename)
yield ()
def onStandaloneDistributionFinished(self, dist_dir):
""" Post-process the distribution folder.
Notes:
Except just exiting, other options are available:
* Create an installation file with the OneFile option
* TODO: create a normal installation file
* Compress the folder using UPX
Args:
dist_dir: name of the distribution folder
Returns:
None
"""
self.timer.end()
t = int(round(self.timer.delta()))
info(" Compilation ended in %i seconds." % t)
for f in self.excludes:
fname = os.path.join(dist_dir, f)
if os.path.exists(fname):
os.remove(fname)
if self.onefile:
info(" Now starting OneFile maker")
info(self.sep_line1)
if "linux" in platform.system():
subprocess.call("python onefile-maker-linux.py " + dist_dir)
elif "win32" in platform.system():
subprocess.call("python onefile-maker-windows.py " + dist_dir)
else:
raise SystemError("Platform not supported")
return None
if self.onedir:
info(" Now starting OneDir maker")
info(self.sep_line1)
subprocess.call("python onedir-maker.py " + dist_dir)
return None
if self.upx:
info(" Now starting UPX packer")
info(self.sep_line1)
subprocess.call("python upx-packer.py " + dist_dir)
return None
info(self.sep_line1)
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
if not getNuitkaVersion() >= "0.6.6":
sys.exit("Need Nuitka v0.6.6+ for hinted compilation.")
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
import_info = json.loads(
getFileContents(fin_name)) # read it and extract the two lists
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
self.accept_test = self.getPluginOptionBool("test", False)
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = sc = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = False
msg = "'%s' is adding the following options:" % self.plugin_name
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
sc = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
info(msg)
if np:
o = ["numpy="]
if mpl:
o.append("matplotlib")
if sc:
o.append("scipy")
if sklearn:
o.append("sklearn")
o = ",".join(o).replace("=,", "=")
if o.endswith("="):
o = o[:-1]
options.plugins_enabled.append(o) # enable numpy
info("--enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter") # enable tk-inter
info("--enable-plugin=tk-inter")
if qt:
# TODO more scrutiny for the qt options!
options.plugins_enabled.append("qt-plugins=sensible")
info("--enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info("--enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info("--enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info("--enable-plugin=torch")
if tflow:
options.plugins_enabled.append("tensorflow")
info("--enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info("--enable-plugin=gevent")
if eventlet:
options.plugins_enabled.append("eventlet")
info("--enable-plugin=eventlet")
# if trio:
# options.plugins_enabled.append("trio")
# info("--enable-plugin=trio")
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if show_msg is False and recurse_count > 0:
info(msg)
msg = "--recurse-to for %i imported modules." % recurse_count
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin object
return None
class UserPlugin(NuitkaPluginBase):
plugin_name = __file__
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
if not getNuitkaVersion() >= "0.6.6":
sys.exit("Need Nuitka v0.6.6+ for hinted compilation.")
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
import_info = json.loads(
getFileContents(fin_name)) # read it and extract the two lists
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
self.accept_test = self.getPluginOptionBool("test", False)
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = sc = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = False
msg = "'%s' is adding the following options:" % self.plugin_name
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
sc = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
info(msg)
if np:
o = ["numpy="]
if mpl:
o.append("matplotlib")
if sc:
o.append("scipy")
if sklearn:
o.append("sklearn")
o = ",".join(o).replace("=,", "=")
if o.endswith("="):
o = o[:-1]
options.plugins_enabled.append(o) # enable numpy
info("--enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter") # enable tk-inter
info("--enable-plugin=tk-inter")
if qt:
# TODO more scrutiny for the qt options!
options.plugins_enabled.append("qt-plugins=sensible")
info("--enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info("--enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info("--enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info("--enable-plugin=torch")
if tflow:
options.plugins_enabled.append("tensorflow")
info("--enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info("--enable-plugin=gevent")
if eventlet:
options.plugins_enabled.append("eventlet")
info("--enable-plugin=eventlet")
# if trio:
# options.plugins_enabled.append("trio")
# info("--enable-plugin=trio")
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if show_msg is False and recurse_count > 0:
info(msg)
msg = "--recurse-to for %i imported modules." % recurse_count
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin object
return None
def onModuleEncounter(self, module_filename, module_name, module_kind):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: path of the module
module_name: module name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
full_name = module_name
elements = full_name.split(".")
package = module_name.getPackageName()
package_dir = remove_suffix(module_filename, elements[0])
# fall through for easy cases
if elements[0] == "pkg_resources":
return None
if (full_name in self.ignored_modules
or elements[0] in self.ignored_modules): # known to be ignored
return False, "module is not used"
if self.accept_test is False and elements[0] in (
"pytest",
"_pytest",
"unittest",
):
info(drop_msg(full_name, package))
self.ignored_modules.add(full_name)
return False, "suppress testing components"
if full_name in self.implicit_imports: # known implicit import
return True, "module is an implicit import" # ok
# check if other plugins would accept this
for plugin in active_plugin_list:
if plugin.plugin_name == self.plugin_name:
continue # skip myself of course
rc = plugin.onModuleEncounter(module_filename, module_name,
module_kind)
if rc is not None:
if rc[0] is True: # plugin wants to keep this
self.implicit_imports.add(full_name)
keep_msg = "keep %s (plugin '%s')" % (full_name,
plugin.plugin_name)
count = self.msg_count.get(plugin.plugin_name, 0)
if count < self.msg_limit:
info(keep_msg)
self.msg_count[plugin.plugin_name] = count + 1
if count == self.msg_limit:
info("... 'keep' msg limit exceeded for '%s'." %
plugin.plugin_name)
return True, "module is imported" # ok
# plugin wants to drop this
self.ignored_modules.add(full_name)
ignore_msg = "drop %s (plugin '%s')" % (full_name,
plugin.plugin_name)
info(ignore_msg)
return False, "dropped by plugin " + plugin.plugin_name
if full_name == "cv2":
return True, "needed by OpenCV"
if full_name.startswith('pywin'):
return True, "needed by pywin32"
checklist = get_checklist(full_name)
for m in self.import_calls: # loop thru the called items
if m in checklist:
return True, "module is hinted to" # ok
if check_dependents(full_name, self.import_files) is True:
return True, "parent of recursed-to module"
# next we ask if implicit imports knows our candidate
if self.ImplicitImports is None: # the plugin is not yet loaded
for plugin in active_plugin_list:
if plugin.plugin_name == "implicit-imports":
self.ImplicitImports = plugin
break
if self.ImplicitImports is None:
sys.exit("could not find 'implicit-imports' plugin")
# ask the 'implicit-imports' plugin whether it knows this guy
if package is not None:
import_set = self.ImplicitImports.getImportsByFullname(
package, package_dir)
import_list0 = [item[0] for item in import_set] # only the names
if full_name in import_list0: # found!
for item in import_list0: # store everything in that list
self.implicit_imports.add(item)
return True, "module is an implicit import" # ok
# not known by anyone: kick it out!
info(drop_msg(full_name, package)) # issue ignore message
# faster decision next time
self.ignored_modules.add(full_name)
return False, "module is not used"
def onStandaloneDistributionFinished(self, dist_dir):
""" Only used to output the compilation time.
"""
self.timer.end()
t = int(round(self.timer.delta()))
if t > 240:
unit = "minutes"
if t >= 600:
t = int(round(t / 60.0))
else:
t = round(t / 60, 1)
else:
unit = "seconds"
info("Compiled '%s' in %g %s." % (sys.argv[-1], t, unit))
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("module_mode")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = (
hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0"
)
remove_output = hasArg("remove_output")
standalone_mode = hasArg("--standalone")
onefile_mode = hasArg("--onefile")
no_site = hasArg("no_site")
recurse_none = hasArg("recurse_none")
recurse_all = hasArg("recurse_all")
timing = hasArg("timing")
coverage_mode = hasArg("coverage")
original_file = hasArg("original_file")
runtime_file = hasArg("runtime_file")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
cpython_cached = hasArg("cpython_cache")
syntax_errors = hasArg("syntax_errors")
noprefer_source = hasArg("noprefer_source")
noverbose_log = hasArg("noverbose_log")
noinclusion_log = hasArg("noinclusion_log")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:") :])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:") :])
del args[count]
user_plugins = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("user_plugin:"):
user_plugins.append(arg[len("user_plugin:") :])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:") :])
del args[count]
recurse_to = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_to:"):
recurse_to.append(arg[len("recurse_to:") :])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ",".join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
assert not standalone_mode or not module_mode
assert not recurse_all or not recurse_none
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print(
"""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
else:
my_print(
"""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("CPython:")
my_print("*" * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
cpython_cmd = [
os.environ["PYTHON"],
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
if no_warnings:
cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename]
else:
cpython_cmd = [os.environ["PYTHON"], filename]
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if runtime_file:
extra_options.append("--file-reference-choice=runtime")
if full_compat:
extra_options.append("--full-compat")
if noprefer_source:
extra_options.append("--no-prefer-source")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output(
[
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
]
)
if sys.version_info >= (3,):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
addToPythonPath(os.path.dirname(os.path.abspath(filename)))
if keep_python_path or binary_python_path:
extra_options.append("--execute-with-pythonpath")
if recurse_none:
extra_options.append("--nofollow-imports")
if recurse_all:
extra_options.append("--follow-imports")
if recurse_not:
extra_options.extend("--nofollow-import-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--plugin-enable=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--plugin-disable=" + plugin_disabled)
for user_plugin in user_plugins:
extra_options.append("--user-plugin=" + user_plugin)
if not noverbose_log:
extra_options.append("--verbose-output=%s.optimization.log" % filename)
if not noinclusion_log:
extra_options.append("--show-modules-output=%s.inclusion.log" % filename)
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename]
elif onefile_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename]
elif standalone_mode:
nuitka_cmd = (
nuitka_call + extra_options + ["--run", "--standalone", filename]
)
else:
nuitka_cmd = nuitka_call + extra_options + ["--run", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = (
nuitka_call + extra_options + ["--module", os.path.abspath(filename)]
)
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename]
else:
nuitka_cmd1 = nuitka_call + extra_options + [filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = "."
if module_mode:
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import %s" % os.path.basename(filename),
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(")", "").replace("(", "")
exe_filename += ".exe" if os.name == "nt" else ".bin"
nuitka_cmd2 = [os.path.join(output_dir, exe_filename)]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults(
cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False
)
if not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("Nuitka:")
my_print("*" * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
stop_watch = StopWatch()
stop_watch.start()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
# Try a couple of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka1, stderr_nuitka1 = process.communicate()
exit_nuitka1 = process.returncode
if exit_nuitka1 != 0:
if (
not expect_failure
and not comparison_mode
and not os.path.exists(".coverage")
):
sys.exit(
"""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
"""
% (os.environ["PYTHON"], stderr_nuitka1)
)
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdin=subprocess.PIPE
)
process.communicate()
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
stop_watch.stop()
nuitka_time = stop_watch.getDelta()
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka
)
)
return exit_code_stdout, exit_code_stderr, exit_code_return
if cpython_cached:
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=False
)
if exit_code_stdout or exit_code_stderr or exit_code_return:
old_stdout_cpython = stdout_cpython
old_stderr_cpython = stderr_cpython
old_exit_cpython = exit_cpython
my_print(
"Updating CPython cache by force due to non-matching comparison results.",
style="yellow",
)
(
cpython_time,
stdout_cpython,
stderr_cpython,
exit_cpython,
) = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=True,
)
if not silent_mode:
if (
old_stdout_cpython != stdout_cpython
or old_stderr_cpython != stderr_cpython
or old_exit_cpython != exit_cpython
):
displayOutput(stdout_cpython, stderr_cpython)
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=True
)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if (
exit_code_return
and exit_nuitka in (-11, -6)
and sys.platform != "nt"
and not module_mode
and not two_step_execution
):
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE)
process.communicate()
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
problems = []
if exit_code_stdout:
problems.append("stdout")
if exit_code_stderr:
problems.append("stderr")
if exit_code_return:
problems.append("exit_code")
sys.exit("Error, results differed (%s)." % ",".join(problems))
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]):
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
if os.path.exists(nuitka_cmd2[0] + ".away"):
os.unlink(nuitka_cmd2[0] + ".away")
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(2)
continue
else:
break
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
module_filename = os.path.basename(filename) + getSharedLibrarySuffix(
preferred=True
)
if os.path.exists(module_filename):
os.unlink(module_filename)
if comparison_mode and timing:
my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time))
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
def __init__(self, hinted_json_file):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
# Load json file contents from --hinted-json-file= argument
filename = hinted_json_file
try:
# read it and extract the two lists
import_info = json.loads(getFileContents(filename))
except (ValueError, FileNotFoundError):
raise FileNotFoundError('Cannot load json file %s' % filename)
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
# TODO: disabled because self.getPluginOptionBool does not exist anymore
#self.accept_test = self.getPluginOptionBool("test", False)
self.accept_test = False
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = scipy = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
scipy = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
self.info(msg)
to_enable = OrderedDict()
if np:
to_enable["numpy"] = {
"matplotlib": mpl,
"scipy": scipy,
# TODO: Numpy plugin didn't use this, work in progress or not needed?
# "sklearn" : sklearn
}
if tk:
to_enable["tk-inter"] = {}
if qt:
# TODO more scrutiny for the qt options!
to_enable["qt-plugins"] = {}
if mp:
to_enable["multiprocessing"] = {}
if pmw:
to_enable["pmw-freezer"] = {}
if torch:
to_enable["torch"] = {}
if tflow:
to_enable["tensorflow"] = {}
if gevent:
to_enable["gevent"] = {}
if eventlet:
to_enable["eventlet"] = {}
if dill:
to_enable["dill-compat"] = {}
# if trio:
# to_enable["trio"] = {}
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if not show_msg and recurse_count > 0:
self.info(msg)
for plugin_name, option_values in to_enable.items():
self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name)
# No the values could be set.
lateActivatePlugin(plugin_name, option_values)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
self.info(msg)
self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
class HintedModsPlugin(NuitkaPluginBase):
# Derive from filename, but can and should also be explicit.
plugin_name = __name__.split(".")[-1]
def __init__(self, hinted_json_file):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
# Load json file contents from --hinted-json-file= argument
filename = hinted_json_file
try:
# read it and extract the two lists
import_info = json.loads(getFileContents(filename))
except (ValueError, FileNotFoundError):
raise FileNotFoundError('Cannot load json file %s' % filename)
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
# TODO: disabled because self.getPluginOptionBool does not exist anymore
#self.accept_test = self.getPluginOptionBool("test", False)
self.accept_test = False
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = scipy = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
scipy = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
self.info(msg)
to_enable = OrderedDict()
if np:
to_enable["numpy"] = {
"matplotlib": mpl,
"scipy": scipy,
# TODO: Numpy plugin didn't use this, work in progress or not needed?
# "sklearn" : sklearn
}
if tk:
to_enable["tk-inter"] = {}
if qt:
# TODO more scrutiny for the qt options!
to_enable["qt-plugins"] = {}
if mp:
to_enable["multiprocessing"] = {}
if pmw:
to_enable["pmw-freezer"] = {}
if torch:
to_enable["torch"] = {}
if tflow:
to_enable["tensorflow"] = {}
if gevent:
to_enable["gevent"] = {}
if eventlet:
to_enable["eventlet"] = {}
if dill:
to_enable["dill-compat"] = {}
# if trio:
# to_enable["trio"] = {}
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if not show_msg and recurse_count > 0:
self.info(msg)
for plugin_name, option_values in to_enable.items():
self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name)
# No the values could be set.
lateActivatePlugin(plugin_name, option_values)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
self.info(msg)
self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
@classmethod
def addPluginCommandLineOptions(cls, group):
group.add_option(
"--hinted-json-file",
action="store",
dest="hinted_json_file",
default=None,
help="[REQUIRED] Path to the json file produced by get-hints.")
def onModuleEncounter(self, module_filename, module_name, module_kind):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: path of the module
module_name: module name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
full_name = module_name
elements = full_name.split(".")
package = module_name.getPackageName()
package_dir = remove_suffix(module_filename, elements[0])
# fall through for easy cases
if elements[0] == "pkg_resources":
return None
if (full_name in self.ignored_modules
or elements[0] in self.ignored_modules): # known to be ignored
return False, "module is not used"
if self.accept_test is False and elements[0] in (
"pytest",
"_pytest",
"unittest",
):
self.info(drop_msg(full_name, package))
self.ignored_modules.add(full_name)
return False, "suppress testing components"
if full_name in self.implicit_imports: # known implicit import
return True, "module is an implicit import" # ok
# check if other plugins would accept this
for plugin in getActivePlugins():
if plugin.plugin_name == self.plugin_name:
continue # skip myself of course
rc = plugin.onModuleEncounter(module_filename, module_name,
module_kind)
if rc is not None:
if rc[0] is True: # plugin wants to keep this
self.implicit_imports.add(full_name)
keep_msg = "keep %s (plugin '%s')" % (full_name,
plugin.plugin_name)
count = self.msg_count.get(plugin.plugin_name, 0)
if count < self.msg_limit:
self.info(keep_msg)
self.msg_count[plugin.plugin_name] = count + 1
if count == self.msg_limit:
self.info("... 'keep' msg limit exceeded for '%s'." %
plugin.plugin_name)
return True, "module is imported" # ok
# plugin wants to drop this
self.ignored_modules.add(full_name)
ignore_msg = "drop %s (plugin '%s')" % (full_name,
plugin.plugin_name)
self.info(ignore_msg)
return False, "dropped by plugin " + plugin.plugin_name
if full_name == "cv2":
return True, "needed by OpenCV"
if full_name.startswith("pywin"):
return True, "needed by pywin32"
checklist = get_checklist(full_name)
for m in self.import_calls: # loop thru the called items
if m in checklist:
return True, "module is hinted to" # ok
if check_dependents(full_name, self.import_files) is True:
return True, "parent of recursed-to module"
# next we ask if implicit imports knows our candidate
if self.implicit_imports_plugin is None: # the plugin is not yet loaded
for plugin in getActivePlugins():
if plugin.plugin_name == "implicit-imports":
self.implicit_imports_plugin = plugin
break
if self.implicit_imports_plugin is None:
sys.exit("could not find 'implicit-imports' plugin")
# ask the 'implicit-imports' plugin whether it knows this guy
if package is not None:
try:
import_set = self.implicit_imports_plugin.getImportsByFullname(
package, package_dir)
except TypeError:
sys.exit(
"versions of hinted-mods.py and ImplicitImports.py are incompatible"
)
import_list0 = [item[0] for item in import_set] # only the names
if full_name in import_list0: # found!
for item in import_list0: # store everything in that list
self.implicit_imports.add(item)
return True, "module is an implicit import" # ok
# not known by anyone: kick it out!
self.info(drop_msg(full_name, package)) # issue ignore message
# faster decision next time
self.ignored_modules.add(full_name)
return False, "module is not used"
def getImplicitImports(self, module):
"""Declare all matplotlib.backends modules as implicit imports."""
full_name = module.getFullName()
if full_name == "__main__": # need to make sure that backends are used
for f in Options.options.recurse_modules:
if f.startswith("matplotlib.backends"):
yield f, False
def onStandaloneDistributionFinished(self, dist_dir):
""" Only used to output the compilation time."""
self.timer.end()
t = int(round(self.timer.getDelta()))
if t > 240:
unit = "minutes"
if t >= 600:
t = int(round(t / 60.0))
else:
t = round(t / 60, 1)
else:
unit = "seconds"
self.info("Compiled '%s' in %g %s." % (sys.argv[-1], t, unit))
def __init__(self):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
import_info = json.loads(
getFileContents(fin_name)) # read it and extract the two lists
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
self.accept_test = self.getPluginOptionBool("test", False)
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = sc = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
sc = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
info(msg)
if np:
o = ["numpy="]
if mpl:
o.append("matplotlib")
if sc:
o.append("scipy")
if sklearn:
o.append("sklearn")
o = ",".join(o).replace("=,", "=")
if o.endswith("="):
o = o[:-1]
options.plugins_enabled.append(o) # enable numpy
info("--enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter") # enable tk-inter
info("--enable-plugin=tk-inter")
if qt:
# TODO more scrutiny for the qt options!
options.plugins_enabled.append("qt-plugins=sensible")
info("--enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info("--enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info("--enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info("--enable-plugin=torch")
if tflow:
options.plugins_enabled.append("tensorflow")
info("--enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info("--enable-plugin=gevent")
if eventlet:
options.plugins_enabled.append("eventlet")
info("--enable-plugin=eventlet")
if dill:
options.plugins_enabled.append("dill-compat")
info("--enable-plugin=dill-compat")
# if trio:
# options.plugins_enabled.append("trio")
# info("--enable-plugin=trio")
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if show_msg is False and recurse_count > 0:
info(msg)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin object
return None
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = [] # speed up repeated lookups
self.ignored_modules = [] # speed up repeated lookups
self.nuitka_modules = False # switch when checking Nuitka modules
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.modules = json.loads(fin.read()) # read it and make an array
fin.close()
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
tk = np = qt = sc = mp = pmw = False
msg = " Enabling the following plugins:"
for m in self.modules: # scan thru called items
if m == "numpy":
np = True
elif m == "_tkinter": # valid indicator for PY2 and PY3
tk = True
elif m.startswith(("PyQt", "PySide")):
qt = True
elif m == "scipy":
sc = True
elif m == "multiprocessing":
mp = True
elif m == "Pmw":
pmw = True
if any((tk, np, sc, qt, mp)):
info(msg)
if np:
o = "numpy" if not sc else "numpy=scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
return None
class Usr_Plugin(UserPluginBase):
plugin_name = __file__
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = [] # speed up repeated lookups
self.ignored_modules = [] # speed up repeated lookups
self.nuitka_modules = False # switch when checking Nuitka modules
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.modules = json.loads(fin.read()) # read it and make an array
fin.close()
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
tk = np = qt = sc = mp = pmw = False
msg = " Enabling the following plugins:"
for m in self.modules: # scan thru called items
if m == "numpy":
np = True
elif m == "_tkinter": # valid indicator for PY2 and PY3
tk = True
elif m.startswith(("PyQt", "PySide")):
qt = True
elif m == "scipy":
sc = True
elif m == "multiprocessing":
mp = True
elif m == "Pmw":
pmw = True
if any((tk, np, sc, qt, mp)):
info(msg)
if np:
o = "numpy" if not sc else "numpy=scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
return None
def onModuleEncounter(
self, module_filename, module_name, module_package, module_kind
):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: filename (not used here)
module_name: module name
module_package: package name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
if module_package:
# the standard case:
full_name = module_package + "." + module_name
# also happens: module_name = package.module
# then use module_name as the full_name
if module_name.startswith(module_package):
t = module_name[len(module_package) :]
if t.startswith("."):
full_name = module_name
else:
full_name = module_name
if full_name in self.ignored_modules: # known to be ignored
return False, "module is not used"
if full_name in self.implicit_imports: # known implicit import
return None
for m in self.modules: # loop thru the called items
if m == full_name: # full name found
return None # ok
if m == full_name + ".*": # is a '*'-import
return None # ok
if module_package and m == module_package + ".*":
# is part of a package
return None # ok
"""
We have a dubious case here:
Check if full_name is one of the implicit imports.
Expensive logic, but can only happen once per module.
Scan through all modules identified by Nuitka and ask each active
plugin, if full_name is an implicit import of any of them.
"""
if not self.nuitka_modules: # first time here?
# make our copy of implicit import names known to Nuitka modules.
modules = []
for m in getNuitkaModules():
for plugin in active_plugin_list:
for im in plugin.getImplicitImports(m):
modules.append(im[0])
self.implicit_imports = sorted(list(set(modules)))
self.nuitka_modules = True
if full_name not in self.implicit_imports:
# check if other plugins would accept this
for plugin in active_plugin_list:
if plugin.plugin_name == self.plugin_name:
continue
rc = plugin.onModuleEncounter(
module_filename, module_name, module_package, module_kind
)
if rc is not None and rc[0] is True:
self.implicit_imports.append(full_name)
if full_name in self.implicit_imports:
# full_name accepted by someone else
info(" implicit: " + full_name)
return None # ok
if module_package is not None:
ignore_msg = " ignoring %s (%s)" % (module_name, module_package)
else:
ignore_msg = " ignoring %s" % module_name
info(ignore_msg) # issue ignore message
self.ignored_modules.append(full_name) # faster decision next time
return False, "module is not used"
def onStandaloneDistributionFinished(self, dist_dir):
self.timer.end()
t = int(round(self.timer.delta()))
info(" Compile time %i seconds." % t)
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = set() # speed up repeated lookups
self.ignored_modules = set() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.import_info = json.loads(fin.read()) # read it and make an array
fin.close()
self.import_calls = self.import_info["calls"]
self.import_files = self.import_info["files"]
self.msg_count = dict()
self.msg_limit = 21
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
show_msg = False # only show info message if parameters are generated
tk = np = qt = sc = mp = pmw = torch = sklearn = False
tflow = gevent = mpl = False
msg = " '%s' is adding the following options:" % self.plugin_name
# detect required standard plugin in order to enable them
for mod in self.import_calls: # scan thru called items
m = mod[0]
if m == "numpy":
np = True
show_msg = True
if m == "matplotlib":
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m == "scipy":
sc = True
show_msg = True
elif m == "multiprocessing" and getOS() == "Windows":
mp = True
show_msg = True
elif m == "Pmw":
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m == "sklearn":
sklearn = True
show_msg = True
elif m == "tensorflow":
tflow = True
show_msg = True
elif m == "gevent":
gevent = True
show_msg = True
if show_msg is True:
info(msg)
if np:
o = "numpy="
if mpl:
o += "matplotlib"
if sc:
o += "scipy" if o.endswith("=") else ",scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins=sensible")
info(" --enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info(" --enable-plugin=torch")
if sklearn:
options.plugins_enabled.append("sklearn")
info(" --enable-plugin=sklearn")
if tflow:
options.plugins_enabled.append("tensorflow")
info(" --enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info(" --enable-plugin=gevent")
for f in self.import_files:
options.recurse_modules.append(f)
# no plugin detected, but recursing to modules?
if show_msg is False and len(self.import_files) > 0:
info(msg)
msg = " --recurse-to for %i imported modules." % len(self.import_files)
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin goes here
return None
class UserPlugin(NuitkaPluginBase):
plugin_name = __file__
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = set() # speed up repeated lookups
self.ignored_modules = set() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.import_info = json.loads(fin.read()) # read it and make an array
fin.close()
self.import_calls = self.import_info["calls"]
self.import_files = self.import_info["files"]
self.msg_count = dict()
self.msg_limit = 21
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
show_msg = False # only show info message if parameters are generated
tk = np = qt = sc = mp = pmw = torch = sklearn = False
tflow = gevent = mpl = False
msg = " '%s' is adding the following options:" % self.plugin_name
# detect required standard plugin in order to enable them
for mod in self.import_calls: # scan thru called items
m = mod[0]
if m == "numpy":
np = True
show_msg = True
if m == "matplotlib":
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m == "scipy":
sc = True
show_msg = True
elif m == "multiprocessing" and getOS() == "Windows":
mp = True
show_msg = True
elif m == "Pmw":
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m == "sklearn":
sklearn = True
show_msg = True
elif m == "tensorflow":
tflow = True
show_msg = True
elif m == "gevent":
gevent = True
show_msg = True
if show_msg is True:
info(msg)
if np:
o = "numpy="
if mpl:
o += "matplotlib"
if sc:
o += "scipy" if o.endswith("=") else ",scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins=sensible")
info(" --enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info(" --enable-plugin=torch")
if sklearn:
options.plugins_enabled.append("sklearn")
info(" --enable-plugin=sklearn")
if tflow:
options.plugins_enabled.append("tensorflow")
info(" --enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info(" --enable-plugin=gevent")
for f in self.import_files:
options.recurse_modules.append(f)
# no plugin detected, but recursing to modules?
if show_msg is False and len(self.import_files) > 0:
info(msg)
msg = " --recurse-to for %i imported modules." % len(self.import_files)
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin goes here
return None
def onModuleEncounter(
self, module_filename, module_name, module_package, module_kind
):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: filename (not used here)
module_name: module name
module_package: package name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
if module_package:
# the standard case:
full_name = module_package + "." + module_name
# also happens: module_name = package.module
# then use module_name as the full_name
if module_name.startswith(module_package):
t = module_name[len(module_package) :]
if t.startswith("."):
full_name = module_name
# also happens: package = a.b.c.module
# then use package as full_name
elif module_package.endswith(module_name):
full_name = module_package
else:
full_name = module_name
# fall through for easy cases
if full_name in self.ignored_modules: # known to be ignored
return False, "module is not used"
if full_name in self.implicit_imports: # known implicit import
return True, "module is an implicit import" # ok
# check if other plugins would accept this
for plugin in active_plugin_list:
if plugin.plugin_name == self.plugin_name:
continue # skip myself of course
rc = plugin.onModuleEncounter(
module_filename, module_name, module_package, module_kind
)
if rc is not None:
if rc[0] is True: # plugin wants to keep this
self.implicit_imports.add(full_name)
keep_msg = " keep %s (plugin '%s')" % (
full_name,
plugin.plugin_name,
)
count = self.msg_count.get(plugin.plugin_name, 0)
if count < self.msg_limit:
info(keep_msg)
self.msg_count[plugin.plugin_name] = count + 1
if count == self.msg_limit:
info(
" ... 'keep' msg limit exceeded for '%s'."
% plugin.plugin_name
)
return True, "module is imported" # ok
# plugin wants to drop this
self.ignored_modules.add(full_name)
ignore_msg = " drop %s (plugin '%s')" % (full_name, plugin.plugin_name)
info(ignore_msg)
return False, "dropped by plugin " + plugin.plugin_name
if full_name == "cv2":
return True, "needed by OpenCV"
checklist = get_checklist(full_name)
for mod in self.import_calls: # loop thru the called items
m = mod[0]
if m in checklist:
return True, "module is hinted to" # ok
# next we ask if implicit imports knows our candidate
if self.ImplicitImports is None: # the plugin is not yet loaded
for plugin in active_plugin_list:
if plugin.plugin_name == "implicit-imports":
self.ImplicitImports = plugin
break
if self.ImplicitImports is None:
sys.exit("could not find 'implicit-imports' plugin")
# ask the 'implicit-imports' plugin whether it knows this guy
if module_package is not None:
import_set = self.ImplicitImports.getImportsByFullname(module_package)
import_list0 = [item[0] for item in import_set] # only the names
if full_name in import_list0: # found!
for item in import_list0: # store everything in that list
self.implicit_imports.add(item)
return True, "module is an implicit imported" # ok
# not known by anyone: kick it out!
if module_package is not None:
ignore_msg = " drop %s (in %s)" % (module_name, module_package)
else:
ignore_msg = " drop %s" % module_name
info(ignore_msg) # issue ignore message
# faster decision next time
self.ignored_modules.add(full_name)
return False, "module is not used"
def onStandaloneDistributionFinished(self, dist_dir):
self.timer.end()
t = int(round(self.timer.delta()))
if t > 300:
t = int(round(t / 60.0))
unit = "minutes"
else:
unit = "seconds"
info(" Compile time %i %s." % (t, unit))