def _getCPythonResults(cpython_cmd, send_kill):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(args=cpython_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if send_kill:
# Doing it per loop iteration hopefully, pylint: disable=cell-var-from-loop
executeAfterTimePassed(
1.0,
lambda: killProcess("Uncompiled Python program", process.pid))
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def _getCPythonResults(cpython_cmd):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(
args=cpython_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython
):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def __init__(self):
options = Options.options
self.sep_line1 = "=" * 80
self.sep_line2 = "-" * 80
self.excludes = []
info(self.sep_line1)
if not Options.isStandaloneMode():
info(" can only run in standalone mode")
info(self.sep_line1)
raise SystemExit()
# start a timer
self.timer = StopWatch()
self.timer.start()
# get the list of options
self.myoptions = self.getPluginOptions()
self.tk = self.getPluginOptionBool("tk", None)
self.qt = self.getPluginOptionBool("qt", None)
self.numpy = self.getPluginOptionBool("np", None)
# check for post processors
self.onefile = 1 if self.getPluginOptionBool("onefile", False) else 0
self.onedir = 1 if self.getPluginOptionBool("onedir", False) else 0
self.upx = 1 if self.getPluginOptionBool("upx", False) else 0
if self.onefile + self.onedir + self.upx > 1:
raise SystemExit("only 1 post-processor can be chosen")
# announce how we will execute
msg = " '%s' established the following configuration" % self.plugin_name
info(msg)
info(self.sep_line2)
if self.numpy is False:
options.recurse_not_modules.append("numpy")
info(" --recurse-not-to=numpy")
options.plugins_disabled.append("numpy")
info(" --disable-plugin=numpy")
elif self.numpy is True:
options.plugins_enabled.append("numpy")
info(" --enable-plugin=numpy")
if self.qt is False:
options.recurse_not_modules.append("PIL.ImageQt")
info(" --recurse-not-to=PIL.ImageQt")
options.plugins_disabled.append("qt-plugins")
info(" --disable-plugin=qt-plugins")
elif self.qt is True:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if self.tk is False:
options.recurse_not_modules.append("PIL.ImageTk")
info(" --recurse-not-to=PIL.ImageTk")
options.plugins_disabled.append("tk-inter")
info(" --disable-plugin=tk-inter")
elif self.tk is True:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
info(self.sep_line2)
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
if not getNuitkaVersion() >= "0.6.6":
sys.exit("Need Nuitka v0.6.6+ for hinted compilation.")
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
import_info = json.loads(
getFileContents(fin_name)) # read it and extract the two lists
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
self.accept_test = self.getPluginOptionBool("test", False)
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = sc = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = False
msg = "'%s' is adding the following options:" % self.plugin_name
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
sc = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
info(msg)
if np:
o = ["numpy="]
if mpl:
o.append("matplotlib")
if sc:
o.append("scipy")
if sklearn:
o.append("sklearn")
o = ",".join(o).replace("=,", "=")
if o.endswith("="):
o = o[:-1]
options.plugins_enabled.append(o) # enable numpy
info("--enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter") # enable tk-inter
info("--enable-plugin=tk-inter")
if qt:
# TODO more scrutiny for the qt options!
options.plugins_enabled.append("qt-plugins=sensible")
info("--enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info("--enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info("--enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info("--enable-plugin=torch")
if tflow:
options.plugins_enabled.append("tensorflow")
info("--enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info("--enable-plugin=gevent")
if eventlet:
options.plugins_enabled.append("eventlet")
info("--enable-plugin=eventlet")
# if trio:
# options.plugins_enabled.append("trio")
# info("--enable-plugin=trio")
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if show_msg is False and recurse_count > 0:
info(msg)
msg = "--recurse-to for %i imported modules." % recurse_count
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin object
return None
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("module_mode")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = (
hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0"
)
remove_output = hasArg("remove_output")
standalone_mode = hasArg("--standalone")
onefile_mode = hasArg("--onefile")
no_site = hasArg("no_site")
recurse_none = hasArg("recurse_none")
recurse_all = hasArg("recurse_all")
timing = hasArg("timing")
coverage_mode = hasArg("coverage")
original_file = hasArg("original_file")
runtime_file = hasArg("runtime_file")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
cpython_cached = hasArg("cpython_cache")
syntax_errors = hasArg("syntax_errors")
noprefer_source = hasArg("noprefer_source")
noverbose_log = hasArg("noverbose_log")
noinclusion_log = hasArg("noinclusion_log")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:") :])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:") :])
del args[count]
user_plugins = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("user_plugin:"):
user_plugins.append(arg[len("user_plugin:") :])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:") :])
del args[count]
recurse_to = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_to:"):
recurse_to.append(arg[len("recurse_to:") :])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ",".join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
assert not standalone_mode or not module_mode
assert not recurse_all or not recurse_none
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print(
"""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
else:
my_print(
"""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("CPython:")
my_print("*" * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
cpython_cmd = [
os.environ["PYTHON"],
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
if no_warnings:
cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename]
else:
cpython_cmd = [os.environ["PYTHON"], filename]
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if runtime_file:
extra_options.append("--file-reference-choice=runtime")
if full_compat:
extra_options.append("--full-compat")
if noprefer_source:
extra_options.append("--no-prefer-source")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output(
[
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
]
)
if sys.version_info >= (3,):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
addToPythonPath(os.path.dirname(os.path.abspath(filename)))
if keep_python_path or binary_python_path:
extra_options.append("--execute-with-pythonpath")
if recurse_none:
extra_options.append("--nofollow-imports")
if recurse_all:
extra_options.append("--follow-imports")
if recurse_not:
extra_options.extend("--nofollow-import-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--plugin-enable=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--plugin-disable=" + plugin_disabled)
for user_plugin in user_plugins:
extra_options.append("--user-plugin=" + user_plugin)
if not noverbose_log:
extra_options.append("--verbose-output=%s.optimization.log" % filename)
if not noinclusion_log:
extra_options.append("--show-modules-output=%s.inclusion.log" % filename)
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename]
elif onefile_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename]
elif standalone_mode:
nuitka_cmd = (
nuitka_call + extra_options + ["--run", "--standalone", filename]
)
else:
nuitka_cmd = nuitka_call + extra_options + ["--run", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = (
nuitka_call + extra_options + ["--module", os.path.abspath(filename)]
)
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename]
else:
nuitka_cmd1 = nuitka_call + extra_options + [filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = "."
if module_mode:
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import %s" % os.path.basename(filename),
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(")", "").replace("(", "")
exe_filename += ".exe" if os.name == "nt" else ".bin"
nuitka_cmd2 = [os.path.join(output_dir, exe_filename)]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults(
cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False
)
if not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("Nuitka:")
my_print("*" * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
stop_watch = StopWatch()
stop_watch.start()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
# Try a couple of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka1, stderr_nuitka1 = process.communicate()
exit_nuitka1 = process.returncode
if exit_nuitka1 != 0:
if (
not expect_failure
and not comparison_mode
and not os.path.exists(".coverage")
):
sys.exit(
"""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
"""
% (os.environ["PYTHON"], stderr_nuitka1)
)
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdin=subprocess.PIPE
)
process.communicate()
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
stop_watch.stop()
nuitka_time = stop_watch.getDelta()
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka
)
)
return exit_code_stdout, exit_code_stderr, exit_code_return
if cpython_cached:
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=False
)
if exit_code_stdout or exit_code_stderr or exit_code_return:
old_stdout_cpython = stdout_cpython
old_stderr_cpython = stderr_cpython
old_exit_cpython = exit_cpython
my_print(
"Updating CPython cache by force due to non-matching comparison results.",
style="yellow",
)
(
cpython_time,
stdout_cpython,
stderr_cpython,
exit_cpython,
) = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=True,
)
if not silent_mode:
if (
old_stdout_cpython != stdout_cpython
or old_stderr_cpython != stderr_cpython
or old_exit_cpython != exit_cpython
):
displayOutput(stdout_cpython, stderr_cpython)
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=True
)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if (
exit_code_return
and exit_nuitka in (-11, -6)
and sys.platform != "nt"
and not module_mode
and not two_step_execution
):
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE)
process.communicate()
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
problems = []
if exit_code_stdout:
problems.append("stdout")
if exit_code_stderr:
problems.append("stderr")
if exit_code_return:
problems.append("exit_code")
sys.exit("Error, results differed (%s)." % ",".join(problems))
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]):
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
if os.path.exists(nuitka_cmd2[0] + ".away"):
os.unlink(nuitka_cmd2[0] + ".away")
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(2)
continue
else:
break
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
module_filename = os.path.basename(filename) + getSharedLibrarySuffix(
preferred=True
)
if os.path.exists(module_filename):
os.unlink(module_filename)
if comparison_mode and timing:
my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time))
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
def __init__(self, hinted_json_file):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
# Load json file contents from --hinted-json-file= argument
filename = hinted_json_file
try:
# read it and extract the two lists
import_info = json.loads(getFileContents(filename))
except (ValueError, FileNotFoundError):
raise FileNotFoundError('Cannot load json file %s' % filename)
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
# TODO: disabled because self.getPluginOptionBool does not exist anymore
#self.accept_test = self.getPluginOptionBool("test", False)
self.accept_test = False
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = scipy = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
scipy = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
self.info(msg)
to_enable = OrderedDict()
if np:
to_enable["numpy"] = {
"matplotlib": mpl,
"scipy": scipy,
# TODO: Numpy plugin didn't use this, work in progress or not needed?
# "sklearn" : sklearn
}
if tk:
to_enable["tk-inter"] = {}
if qt:
# TODO more scrutiny for the qt options!
to_enable["qt-plugins"] = {}
if mp:
to_enable["multiprocessing"] = {}
if pmw:
to_enable["pmw-freezer"] = {}
if torch:
to_enable["torch"] = {}
if tflow:
to_enable["tensorflow"] = {}
if gevent:
to_enable["gevent"] = {}
if eventlet:
to_enable["eventlet"] = {}
if dill:
to_enable["dill-compat"] = {}
# if trio:
# to_enable["trio"] = {}
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if not show_msg and recurse_count > 0:
self.info(msg)
for plugin_name, option_values in to_enable.items():
self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name)
# No the values could be set.
lateActivatePlugin(plugin_name, option_values)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
self.info(msg)
self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
def __init__(self):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
import_info = json.loads(
getFileContents(fin_name)) # read it and extract the two lists
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
self.accept_test = self.getPluginOptionBool("test", False)
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = sc = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
sc = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
info(msg)
if np:
o = ["numpy="]
if mpl:
o.append("matplotlib")
if sc:
o.append("scipy")
if sklearn:
o.append("sklearn")
o = ",".join(o).replace("=,", "=")
if o.endswith("="):
o = o[:-1]
options.plugins_enabled.append(o) # enable numpy
info("--enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter") # enable tk-inter
info("--enable-plugin=tk-inter")
if qt:
# TODO more scrutiny for the qt options!
options.plugins_enabled.append("qt-plugins=sensible")
info("--enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info("--enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info("--enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info("--enable-plugin=torch")
if tflow:
options.plugins_enabled.append("tensorflow")
info("--enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info("--enable-plugin=gevent")
if eventlet:
options.plugins_enabled.append("eventlet")
info("--enable-plugin=eventlet")
if dill:
options.plugins_enabled.append("dill-compat")
info("--enable-plugin=dill-compat")
# if trio:
# options.plugins_enabled.append("trio")
# info("--enable-plugin=trio")
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if show_msg is False and recurse_count > 0:
info(msg)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin object
return None
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = [] # speed up repeated lookups
self.ignored_modules = [] # speed up repeated lookups
self.nuitka_modules = False # switch when checking Nuitka modules
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.modules = json.loads(fin.read()) # read it and make an array
fin.close()
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
tk = np = qt = sc = mp = pmw = False
msg = " Enabling the following plugins:"
for m in self.modules: # scan thru called items
if m == "numpy":
np = True
elif m == "_tkinter": # valid indicator for PY2 and PY3
tk = True
elif m.startswith(("PyQt", "PySide")):
qt = True
elif m == "scipy":
sc = True
elif m == "multiprocessing":
mp = True
elif m == "Pmw":
pmw = True
if any((tk, np, sc, qt, mp)):
info(msg)
if np:
o = "numpy" if not sc else "numpy=scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
return None
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = set() # speed up repeated lookups
self.ignored_modules = set() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.import_info = json.loads(fin.read()) # read it and make an array
fin.close()
self.import_calls = self.import_info["calls"]
self.import_files = self.import_info["files"]
self.msg_count = dict()
self.msg_limit = 21
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
show_msg = False # only show info message if parameters are generated
tk = np = qt = sc = mp = pmw = torch = sklearn = False
tflow = gevent = mpl = False
msg = " '%s' is adding the following options:" % self.plugin_name
# detect required standard plugin in order to enable them
for mod in self.import_calls: # scan thru called items
m = mod[0]
if m == "numpy":
np = True
show_msg = True
if m == "matplotlib":
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m == "scipy":
sc = True
show_msg = True
elif m == "multiprocessing" and getOS() == "Windows":
mp = True
show_msg = True
elif m == "Pmw":
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m == "sklearn":
sklearn = True
show_msg = True
elif m == "tensorflow":
tflow = True
show_msg = True
elif m == "gevent":
gevent = True
show_msg = True
if show_msg is True:
info(msg)
if np:
o = "numpy="
if mpl:
o += "matplotlib"
if sc:
o += "scipy" if o.endswith("=") else ",scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins=sensible")
info(" --enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info(" --enable-plugin=torch")
if sklearn:
options.plugins_enabled.append("sklearn")
info(" --enable-plugin=sklearn")
if tflow:
options.plugins_enabled.append("tensorflow")
info(" --enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info(" --enable-plugin=gevent")
for f in self.import_files:
options.recurse_modules.append(f)
# no plugin detected, but recursing to modules?
if show_msg is False and len(self.import_files) > 0:
info(msg)
msg = " --recurse-to for %i imported modules." % len(self.import_files)
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin goes here
return None