def _getCPythonResults(cpython_cmd):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(
args=cpython_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython
):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def _getCPythonResults(cpython_cmd, send_kill):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(args=cpython_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if send_kill:
# Doing it per loop iteration hopefully, pylint: disable=cell-var-from-loop
executeAfterTimePassed(
1.0,
lambda: killProcess("Uncompiled Python program", process.pid))
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("module_mode")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = (
hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0"
)
remove_output = hasArg("remove_output")
standalone_mode = hasArg("--standalone")
onefile_mode = hasArg("--onefile")
no_site = hasArg("no_site")
recurse_none = hasArg("recurse_none")
recurse_all = hasArg("recurse_all")
timing = hasArg("timing")
coverage_mode = hasArg("coverage")
original_file = hasArg("original_file")
runtime_file = hasArg("runtime_file")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
cpython_cached = hasArg("cpython_cache")
syntax_errors = hasArg("syntax_errors")
noprefer_source = hasArg("noprefer_source")
noverbose_log = hasArg("noverbose_log")
noinclusion_log = hasArg("noinclusion_log")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:") :])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:") :])
del args[count]
user_plugins = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("user_plugin:"):
user_plugins.append(arg[len("user_plugin:") :])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:") :])
del args[count]
recurse_to = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_to:"):
recurse_to.append(arg[len("recurse_to:") :])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ",".join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
assert not standalone_mode or not module_mode
assert not recurse_all or not recurse_none
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print(
"""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
else:
my_print(
"""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("CPython:")
my_print("*" * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
cpython_cmd = [
os.environ["PYTHON"],
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
if no_warnings:
cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename]
else:
cpython_cmd = [os.environ["PYTHON"], filename]
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if runtime_file:
extra_options.append("--file-reference-choice=runtime")
if full_compat:
extra_options.append("--full-compat")
if noprefer_source:
extra_options.append("--no-prefer-source")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output(
[
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
]
)
if sys.version_info >= (3,):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
addToPythonPath(os.path.dirname(os.path.abspath(filename)))
if keep_python_path or binary_python_path:
extra_options.append("--execute-with-pythonpath")
if recurse_none:
extra_options.append("--nofollow-imports")
if recurse_all:
extra_options.append("--follow-imports")
if recurse_not:
extra_options.extend("--nofollow-import-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--plugin-enable=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--plugin-disable=" + plugin_disabled)
for user_plugin in user_plugins:
extra_options.append("--user-plugin=" + user_plugin)
if not noverbose_log:
extra_options.append("--verbose-output=%s.optimization.log" % filename)
if not noinclusion_log:
extra_options.append("--show-modules-output=%s.inclusion.log" % filename)
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename]
elif onefile_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename]
elif standalone_mode:
nuitka_cmd = (
nuitka_call + extra_options + ["--run", "--standalone", filename]
)
else:
nuitka_cmd = nuitka_call + extra_options + ["--run", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = (
nuitka_call + extra_options + ["--module", os.path.abspath(filename)]
)
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename]
else:
nuitka_cmd1 = nuitka_call + extra_options + [filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = "."
if module_mode:
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import %s" % os.path.basename(filename),
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(")", "").replace("(", "")
exe_filename += ".exe" if os.name == "nt" else ".bin"
nuitka_cmd2 = [os.path.join(output_dir, exe_filename)]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults(
cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False
)
if not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("Nuitka:")
my_print("*" * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
stop_watch = StopWatch()
stop_watch.start()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
# Try a couple of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka1, stderr_nuitka1 = process.communicate()
exit_nuitka1 = process.returncode
if exit_nuitka1 != 0:
if (
not expect_failure
and not comparison_mode
and not os.path.exists(".coverage")
):
sys.exit(
"""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
"""
% (os.environ["PYTHON"], stderr_nuitka1)
)
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdin=subprocess.PIPE
)
process.communicate()
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
stop_watch.stop()
nuitka_time = stop_watch.getDelta()
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka
)
)
return exit_code_stdout, exit_code_stderr, exit_code_return
if cpython_cached:
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=False
)
if exit_code_stdout or exit_code_stderr or exit_code_return:
old_stdout_cpython = stdout_cpython
old_stderr_cpython = stderr_cpython
old_exit_cpython = exit_cpython
my_print(
"Updating CPython cache by force due to non-matching comparison results.",
style="yellow",
)
(
cpython_time,
stdout_cpython,
stderr_cpython,
exit_cpython,
) = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=True,
)
if not silent_mode:
if (
old_stdout_cpython != stdout_cpython
or old_stderr_cpython != stderr_cpython
or old_exit_cpython != exit_cpython
):
displayOutput(stdout_cpython, stderr_cpython)
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=True
)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if (
exit_code_return
and exit_nuitka in (-11, -6)
and sys.platform != "nt"
and not module_mode
and not two_step_execution
):
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE)
process.communicate()
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
problems = []
if exit_code_stdout:
problems.append("stdout")
if exit_code_stderr:
problems.append("stderr")
if exit_code_return:
problems.append("exit_code")
sys.exit("Error, results differed (%s)." % ",".join(problems))
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]):
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
if os.path.exists(nuitka_cmd2[0] + ".away"):
os.unlink(nuitka_cmd2[0] + ".away")
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(2)
continue
else:
break
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
module_filename = os.path.basename(filename) + getSharedLibrarySuffix(
preferred=True
)
if os.path.exists(module_filename):
os.unlink(module_filename)
if comparison_mode and timing:
my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time))
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
class HintedModsPlugin(NuitkaPluginBase):
# Derive from filename, but can and should also be explicit.
plugin_name = __name__.split(".")[-1]
def __init__(self, hinted_json_file):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
# Load json file contents from --hinted-json-file= argument
filename = hinted_json_file
try:
# read it and extract the two lists
import_info = json.loads(getFileContents(filename))
except (ValueError, FileNotFoundError):
raise FileNotFoundError('Cannot load json file %s' % filename)
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
# TODO: disabled because self.getPluginOptionBool does not exist anymore
#self.accept_test = self.getPluginOptionBool("test", False)
self.accept_test = False
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = scipy = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
scipy = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
self.info(msg)
to_enable = OrderedDict()
if np:
to_enable["numpy"] = {
"matplotlib": mpl,
"scipy": scipy,
# TODO: Numpy plugin didn't use this, work in progress or not needed?
# "sklearn" : sklearn
}
if tk:
to_enable["tk-inter"] = {}
if qt:
# TODO more scrutiny for the qt options!
to_enable["qt-plugins"] = {}
if mp:
to_enable["multiprocessing"] = {}
if pmw:
to_enable["pmw-freezer"] = {}
if torch:
to_enable["torch"] = {}
if tflow:
to_enable["tensorflow"] = {}
if gevent:
to_enable["gevent"] = {}
if eventlet:
to_enable["eventlet"] = {}
if dill:
to_enable["dill-compat"] = {}
# if trio:
# to_enable["trio"] = {}
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if not show_msg and recurse_count > 0:
self.info(msg)
for plugin_name, option_values in to_enable.items():
self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name)
# No the values could be set.
lateActivatePlugin(plugin_name, option_values)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
self.info(msg)
self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
@classmethod
def addPluginCommandLineOptions(cls, group):
group.add_option(
"--hinted-json-file",
action="store",
dest="hinted_json_file",
default=None,
help="[REQUIRED] Path to the json file produced by get-hints.")
def onModuleEncounter(self, module_filename, module_name, module_kind):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: path of the module
module_name: module name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
full_name = module_name
elements = full_name.split(".")
package = module_name.getPackageName()
package_dir = remove_suffix(module_filename, elements[0])
# fall through for easy cases
if elements[0] == "pkg_resources":
return None
if (full_name in self.ignored_modules
or elements[0] in self.ignored_modules): # known to be ignored
return False, "module is not used"
if self.accept_test is False and elements[0] in (
"pytest",
"_pytest",
"unittest",
):
self.info(drop_msg(full_name, package))
self.ignored_modules.add(full_name)
return False, "suppress testing components"
if full_name in self.implicit_imports: # known implicit import
return True, "module is an implicit import" # ok
# check if other plugins would accept this
for plugin in getActivePlugins():
if plugin.plugin_name == self.plugin_name:
continue # skip myself of course
rc = plugin.onModuleEncounter(module_filename, module_name,
module_kind)
if rc is not None:
if rc[0] is True: # plugin wants to keep this
self.implicit_imports.add(full_name)
keep_msg = "keep %s (plugin '%s')" % (full_name,
plugin.plugin_name)
count = self.msg_count.get(plugin.plugin_name, 0)
if count < self.msg_limit:
self.info(keep_msg)
self.msg_count[plugin.plugin_name] = count + 1
if count == self.msg_limit:
self.info("... 'keep' msg limit exceeded for '%s'." %
plugin.plugin_name)
return True, "module is imported" # ok
# plugin wants to drop this
self.ignored_modules.add(full_name)
ignore_msg = "drop %s (plugin '%s')" % (full_name,
plugin.plugin_name)
self.info(ignore_msg)
return False, "dropped by plugin " + plugin.plugin_name
if full_name == "cv2":
return True, "needed by OpenCV"
if full_name.startswith("pywin"):
return True, "needed by pywin32"
checklist = get_checklist(full_name)
for m in self.import_calls: # loop thru the called items
if m in checklist:
return True, "module is hinted to" # ok
if check_dependents(full_name, self.import_files) is True:
return True, "parent of recursed-to module"
# next we ask if implicit imports knows our candidate
if self.implicit_imports_plugin is None: # the plugin is not yet loaded
for plugin in getActivePlugins():
if plugin.plugin_name == "implicit-imports":
self.implicit_imports_plugin = plugin
break
if self.implicit_imports_plugin is None:
sys.exit("could not find 'implicit-imports' plugin")
# ask the 'implicit-imports' plugin whether it knows this guy
if package is not None:
try:
import_set = self.implicit_imports_plugin.getImportsByFullname(
package, package_dir)
except TypeError:
sys.exit(
"versions of hinted-mods.py and ImplicitImports.py are incompatible"
)
import_list0 = [item[0] for item in import_set] # only the names
if full_name in import_list0: # found!
for item in import_list0: # store everything in that list
self.implicit_imports.add(item)
return True, "module is an implicit import" # ok
# not known by anyone: kick it out!
self.info(drop_msg(full_name, package)) # issue ignore message
# faster decision next time
self.ignored_modules.add(full_name)
return False, "module is not used"
def getImplicitImports(self, module):
"""Declare all matplotlib.backends modules as implicit imports."""
full_name = module.getFullName()
if full_name == "__main__": # need to make sure that backends are used
for f in Options.options.recurse_modules:
if f.startswith("matplotlib.backends"):
yield f, False
def onStandaloneDistributionFinished(self, dist_dir):
""" Only used to output the compilation time."""
self.timer.end()
t = int(round(self.timer.getDelta()))
if t > 240:
unit = "minutes"
if t >= 600:
t = int(round(t / 60.0))
else:
t = round(t / 60, 1)
else:
unit = "seconds"
self.info("Compiled '%s' in %g %s." % (sys.argv[-1], t, unit))