class MyExit(NuitkaPluginBase):
""" User plugin supporting post-processing in standalone mode.
Notes:
Upon initialization, this plugin establishes a few options.
Among these are enabling or disabling standard plugins.
Compilation post-processing can be either
(a) option "upx" for invoking UPX compression, or
(b) option "onefile" for creating a distribution file in OneFile format, or
(c) TODO: option "onedir" for creating a distribution file in OneDir format.
The options for the standard plugins tk-plugin (code "tk"), numpy-plugin
("np") and qt-plugins ("qt") can also be added as options to this plugin.
This will enable the respective plugin. If the code is prefixed with "no",
then the plugin will be disabled and certain modules will not be recursed
to. For example:
- "tk" will generate "--enable-plugin=tk-plugin"
- "notk" will generate "--disable-plugin=tk-plugin" and
"--recurse-not-to=PIL.ImageTk".
"""
plugin_name = __file__
def __init__(self):
options = Options.options
self.sep_line1 = "=" * 80
self.sep_line2 = "-" * 80
self.excludes = []
info(self.sep_line1)
if not Options.isStandaloneMode():
info(" can only run in standalone mode")
info(self.sep_line1)
raise SystemExit()
# start a timer
self.timer = StopWatch()
self.timer.start()
# get the list of options
self.myoptions = self.getPluginOptions()
self.tk = self.getPluginOptionBool("tk", None)
self.qt = self.getPluginOptionBool("qt", None)
self.numpy = self.getPluginOptionBool("np", None)
# check for post processors
self.onefile = 1 if self.getPluginOptionBool("onefile", False) else 0
self.onedir = 1 if self.getPluginOptionBool("onedir", False) else 0
self.upx = 1 if self.getPluginOptionBool("upx", False) else 0
if self.onefile + self.onedir + self.upx > 1:
raise SystemExit("only 1 post-processor can be chosen")
# announce how we will execute
msg = " '%s' established the following configuration" % self.plugin_name
info(msg)
info(self.sep_line2)
if self.numpy is False:
options.recurse_not_modules.append("numpy")
info(" --recurse-not-to=numpy")
options.plugins_disabled.append("numpy")
info(" --disable-plugin=numpy")
elif self.numpy is True:
options.plugins_enabled.append("numpy")
info(" --enable-plugin=numpy")
if self.qt is False:
options.recurse_not_modules.append("PIL.ImageQt")
info(" --recurse-not-to=PIL.ImageQt")
options.plugins_disabled.append("qt-plugins")
info(" --disable-plugin=qt-plugins")
elif self.qt is True:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if self.tk is False:
options.recurse_not_modules.append("PIL.ImageTk")
info(" --recurse-not-to=PIL.ImageTk")
options.plugins_disabled.append("tk-inter")
info(" --disable-plugin=tk-inter")
elif self.tk is True:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
info(self.sep_line2)
def removeDllDependencies(self, dll_filename, dll_filenames):
if self.tk is False:
basename = os.path.basename(dll_filename)
if basename.startswith(("tk", "tcl")):
info(" exluding " + basename)
self.excludes.append(basename)
yield ()
if "qt" in dll_filename.lower():
print(dll_filename)
yield ()
def onStandaloneDistributionFinished(self, dist_dir):
""" Post-process the distribution folder.
Notes:
Except just exiting, other options are available:
* Create an installation file with the OneFile option
* TODO: create a normal installation file
* Compress the folder using UPX
Args:
dist_dir: name of the distribution folder
Returns:
None
"""
self.timer.end()
t = int(round(self.timer.delta()))
info(" Compilation ended in %i seconds." % t)
for f in self.excludes:
fname = os.path.join(dist_dir, f)
if os.path.exists(fname):
os.remove(fname)
if self.onefile:
info(" Now starting OneFile maker")
info(self.sep_line1)
if "linux" in platform.system():
subprocess.call("python onefile-maker-linux.py " + dist_dir)
elif "win32" in platform.system():
subprocess.call("python onefile-maker-windows.py " + dist_dir)
else:
raise SystemError("Platform not supported")
return None
if self.onedir:
info(" Now starting OneDir maker")
info(self.sep_line1)
subprocess.call("python onedir-maker.py " + dist_dir)
return None
if self.upx:
info(" Now starting UPX packer")
info(self.sep_line1)
subprocess.call("python upx-packer.py " + dist_dir)
return None
info(self.sep_line1)
class UserPlugin(NuitkaPluginBase):
plugin_name = __file__
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
if not getNuitkaVersion() >= "0.6.6":
sys.exit("Need Nuitka v0.6.6+ for hinted compilation.")
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
import_info = json.loads(
getFileContents(fin_name)) # read it and extract the two lists
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
self.accept_test = self.getPluginOptionBool("test", False)
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = sc = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = False
msg = "'%s' is adding the following options:" % self.plugin_name
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
sc = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
info(msg)
if np:
o = ["numpy="]
if mpl:
o.append("matplotlib")
if sc:
o.append("scipy")
if sklearn:
o.append("sklearn")
o = ",".join(o).replace("=,", "=")
if o.endswith("="):
o = o[:-1]
options.plugins_enabled.append(o) # enable numpy
info("--enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter") # enable tk-inter
info("--enable-plugin=tk-inter")
if qt:
# TODO more scrutiny for the qt options!
options.plugins_enabled.append("qt-plugins=sensible")
info("--enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info("--enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info("--enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info("--enable-plugin=torch")
if tflow:
options.plugins_enabled.append("tensorflow")
info("--enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info("--enable-plugin=gevent")
if eventlet:
options.plugins_enabled.append("eventlet")
info("--enable-plugin=eventlet")
# if trio:
# options.plugins_enabled.append("trio")
# info("--enable-plugin=trio")
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if show_msg is False and recurse_count > 0:
info(msg)
msg = "--recurse-to for %i imported modules." % recurse_count
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin object
return None
def onModuleEncounter(self, module_filename, module_name, module_kind):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: path of the module
module_name: module name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
full_name = module_name
elements = full_name.split(".")
package = module_name.getPackageName()
package_dir = remove_suffix(module_filename, elements[0])
# fall through for easy cases
if elements[0] == "pkg_resources":
return None
if (full_name in self.ignored_modules
or elements[0] in self.ignored_modules): # known to be ignored
return False, "module is not used"
if self.accept_test is False and elements[0] in (
"pytest",
"_pytest",
"unittest",
):
info(drop_msg(full_name, package))
self.ignored_modules.add(full_name)
return False, "suppress testing components"
if full_name in self.implicit_imports: # known implicit import
return True, "module is an implicit import" # ok
# check if other plugins would accept this
for plugin in active_plugin_list:
if plugin.plugin_name == self.plugin_name:
continue # skip myself of course
rc = plugin.onModuleEncounter(module_filename, module_name,
module_kind)
if rc is not None:
if rc[0] is True: # plugin wants to keep this
self.implicit_imports.add(full_name)
keep_msg = "keep %s (plugin '%s')" % (full_name,
plugin.plugin_name)
count = self.msg_count.get(plugin.plugin_name, 0)
if count < self.msg_limit:
info(keep_msg)
self.msg_count[plugin.plugin_name] = count + 1
if count == self.msg_limit:
info("... 'keep' msg limit exceeded for '%s'." %
plugin.plugin_name)
return True, "module is imported" # ok
# plugin wants to drop this
self.ignored_modules.add(full_name)
ignore_msg = "drop %s (plugin '%s')" % (full_name,
plugin.plugin_name)
info(ignore_msg)
return False, "dropped by plugin " + plugin.plugin_name
if full_name == "cv2":
return True, "needed by OpenCV"
if full_name.startswith('pywin'):
return True, "needed by pywin32"
checklist = get_checklist(full_name)
for m in self.import_calls: # loop thru the called items
if m in checklist:
return True, "module is hinted to" # ok
if check_dependents(full_name, self.import_files) is True:
return True, "parent of recursed-to module"
# next we ask if implicit imports knows our candidate
if self.ImplicitImports is None: # the plugin is not yet loaded
for plugin in active_plugin_list:
if plugin.plugin_name == "implicit-imports":
self.ImplicitImports = plugin
break
if self.ImplicitImports is None:
sys.exit("could not find 'implicit-imports' plugin")
# ask the 'implicit-imports' plugin whether it knows this guy
if package is not None:
import_set = self.ImplicitImports.getImportsByFullname(
package, package_dir)
import_list0 = [item[0] for item in import_set] # only the names
if full_name in import_list0: # found!
for item in import_list0: # store everything in that list
self.implicit_imports.add(item)
return True, "module is an implicit import" # ok
# not known by anyone: kick it out!
info(drop_msg(full_name, package)) # issue ignore message
# faster decision next time
self.ignored_modules.add(full_name)
return False, "module is not used"
def onStandaloneDistributionFinished(self, dist_dir):
""" Only used to output the compilation time.
"""
self.timer.end()
t = int(round(self.timer.delta()))
if t > 240:
unit = "minutes"
if t >= 600:
t = int(round(t / 60.0))
else:
t = round(t / 60, 1)
else:
unit = "seconds"
info("Compiled '%s' in %g %s." % (sys.argv[-1], t, unit))
class Usr_Plugin(UserPluginBase):
plugin_name = __file__
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = [] # speed up repeated lookups
self.ignored_modules = [] # speed up repeated lookups
self.nuitka_modules = False # switch when checking Nuitka modules
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.modules = json.loads(fin.read()) # read it and make an array
fin.close()
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
tk = np = qt = sc = mp = pmw = False
msg = " Enabling the following plugins:"
for m in self.modules: # scan thru called items
if m == "numpy":
np = True
elif m == "_tkinter": # valid indicator for PY2 and PY3
tk = True
elif m.startswith(("PyQt", "PySide")):
qt = True
elif m == "scipy":
sc = True
elif m == "multiprocessing":
mp = True
elif m == "Pmw":
pmw = True
if any((tk, np, sc, qt, mp)):
info(msg)
if np:
o = "numpy" if not sc else "numpy=scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins")
info(" --enable-plugin=qt-plugins")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
return None
def onModuleEncounter(
self, module_filename, module_name, module_package, module_kind
):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: filename (not used here)
module_name: module name
module_package: package name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
if module_package:
# the standard case:
full_name = module_package + "." + module_name
# also happens: module_name = package.module
# then use module_name as the full_name
if module_name.startswith(module_package):
t = module_name[len(module_package) :]
if t.startswith("."):
full_name = module_name
else:
full_name = module_name
if full_name in self.ignored_modules: # known to be ignored
return False, "module is not used"
if full_name in self.implicit_imports: # known implicit import
return None
for m in self.modules: # loop thru the called items
if m == full_name: # full name found
return None # ok
if m == full_name + ".*": # is a '*'-import
return None # ok
if module_package and m == module_package + ".*":
# is part of a package
return None # ok
"""
We have a dubious case here:
Check if full_name is one of the implicit imports.
Expensive logic, but can only happen once per module.
Scan through all modules identified by Nuitka and ask each active
plugin, if full_name is an implicit import of any of them.
"""
if not self.nuitka_modules: # first time here?
# make our copy of implicit import names known to Nuitka modules.
modules = []
for m in getNuitkaModules():
for plugin in active_plugin_list:
for im in plugin.getImplicitImports(m):
modules.append(im[0])
self.implicit_imports = sorted(list(set(modules)))
self.nuitka_modules = True
if full_name not in self.implicit_imports:
# check if other plugins would accept this
for plugin in active_plugin_list:
if plugin.plugin_name == self.plugin_name:
continue
rc = plugin.onModuleEncounter(
module_filename, module_name, module_package, module_kind
)
if rc is not None and rc[0] is True:
self.implicit_imports.append(full_name)
if full_name in self.implicit_imports:
# full_name accepted by someone else
info(" implicit: " + full_name)
return None # ok
if module_package is not None:
ignore_msg = " ignoring %s (%s)" % (module_name, module_package)
else:
ignore_msg = " ignoring %s" % module_name
info(ignore_msg) # issue ignore message
self.ignored_modules.append(full_name) # faster decision next time
return False, "module is not used"
def onStandaloneDistributionFinished(self, dist_dir):
self.timer.end()
t = int(round(self.timer.delta()))
info(" Compile time %i seconds." % t)
class HintedModsPlugin(NuitkaPluginBase):
# Derive from filename, but can and should also be explicit.
plugin_name = __name__.split(".")[-1]
def __init__(self, hinted_json_file):
""" Read the JSON file and enable any standard plugins.
Notes:
Read the JSON file produced during the get-hints step. It will
contain a list of imported items ("calls") and a list of modules /
packages ("files") to be loaded and recursed into.
Depending on the items in 'files', we will trigger loading standard
plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
# Load json file contents from --hinted-json-file= argument
filename = hinted_json_file
try:
# read it and extract the two lists
import_info = json.loads(getFileContents(filename))
except (ValueError, FileNotFoundError):
raise FileNotFoundError('Cannot load json file %s' % filename)
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
# TODO: disabled because self.getPluginOptionBool does not exist anymore
#self.accept_test = self.getPluginOptionBool("test", False)
self.accept_test = False
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = scipy = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = dill = False
msg = "'%s' is adding the following options:" % os.path.basename(
self.plugin_name)
# we need matplotlib-specific cleanup to happen first:
# if no mpl backend is used, reference to matplotlib is removed alltogether
if "matplotlib.backends" not in self.import_files:
temp = [
f for f in self.import_calls
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_calls = temp
temp = [
f for f in self.import_files
if not f.startswith(("matplotlib", "mpl_toolkits"))
]
self.import_files = temp
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
scipy = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
elif m in ("dill", "dill.*"):
dill = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
self.info(msg)
to_enable = OrderedDict()
if np:
to_enable["numpy"] = {
"matplotlib": mpl,
"scipy": scipy,
# TODO: Numpy plugin didn't use this, work in progress or not needed?
# "sklearn" : sklearn
}
if tk:
to_enable["tk-inter"] = {}
if qt:
# TODO more scrutiny for the qt options!
to_enable["qt-plugins"] = {}
if mp:
to_enable["multiprocessing"] = {}
if pmw:
to_enable["pmw-freezer"] = {}
if torch:
to_enable["torch"] = {}
if tflow:
to_enable["tensorflow"] = {}
if gevent:
to_enable["gevent"] = {}
if eventlet:
to_enable["eventlet"] = {}
if dill:
to_enable["dill-compat"] = {}
# if trio:
# to_enable["trio"] = {}
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if not show_msg and recurse_count > 0:
self.info(msg)
for plugin_name, option_values in to_enable.items():
self.info("Enabling Nuitka plugin '%s' as needed." % plugin_name)
# No the values could be set.
lateActivatePlugin(plugin_name, option_values)
if len(self.import_files) > 0:
msg = "--recurse-to=%s and %i more modules" % (
self.import_files[-1],
recurse_count - 1,
)
self.info(msg)
self.implicit_imports_plugin = None # the 'implicit-imports' plugin object
@classmethod
def addPluginCommandLineOptions(cls, group):
group.add_option(
"--hinted-json-file",
action="store",
dest="hinted_json_file",
default=None,
help="[REQUIRED] Path to the json file produced by get-hints.")
def onModuleEncounter(self, module_filename, module_name, module_kind):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: path of the module
module_name: module name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
full_name = module_name
elements = full_name.split(".")
package = module_name.getPackageName()
package_dir = remove_suffix(module_filename, elements[0])
# fall through for easy cases
if elements[0] == "pkg_resources":
return None
if (full_name in self.ignored_modules
or elements[0] in self.ignored_modules): # known to be ignored
return False, "module is not used"
if self.accept_test is False and elements[0] in (
"pytest",
"_pytest",
"unittest",
):
self.info(drop_msg(full_name, package))
self.ignored_modules.add(full_name)
return False, "suppress testing components"
if full_name in self.implicit_imports: # known implicit import
return True, "module is an implicit import" # ok
# check if other plugins would accept this
for plugin in getActivePlugins():
if plugin.plugin_name == self.plugin_name:
continue # skip myself of course
rc = plugin.onModuleEncounter(module_filename, module_name,
module_kind)
if rc is not None:
if rc[0] is True: # plugin wants to keep this
self.implicit_imports.add(full_name)
keep_msg = "keep %s (plugin '%s')" % (full_name,
plugin.plugin_name)
count = self.msg_count.get(plugin.plugin_name, 0)
if count < self.msg_limit:
self.info(keep_msg)
self.msg_count[plugin.plugin_name] = count + 1
if count == self.msg_limit:
self.info("... 'keep' msg limit exceeded for '%s'." %
plugin.plugin_name)
return True, "module is imported" # ok
# plugin wants to drop this
self.ignored_modules.add(full_name)
ignore_msg = "drop %s (plugin '%s')" % (full_name,
plugin.plugin_name)
self.info(ignore_msg)
return False, "dropped by plugin " + plugin.plugin_name
if full_name == "cv2":
return True, "needed by OpenCV"
if full_name.startswith("pywin"):
return True, "needed by pywin32"
checklist = get_checklist(full_name)
for m in self.import_calls: # loop thru the called items
if m in checklist:
return True, "module is hinted to" # ok
if check_dependents(full_name, self.import_files) is True:
return True, "parent of recursed-to module"
# next we ask if implicit imports knows our candidate
if self.implicit_imports_plugin is None: # the plugin is not yet loaded
for plugin in getActivePlugins():
if plugin.plugin_name == "implicit-imports":
self.implicit_imports_plugin = plugin
break
if self.implicit_imports_plugin is None:
sys.exit("could not find 'implicit-imports' plugin")
# ask the 'implicit-imports' plugin whether it knows this guy
if package is not None:
try:
import_set = self.implicit_imports_plugin.getImportsByFullname(
package, package_dir)
except TypeError:
sys.exit(
"versions of hinted-mods.py and ImplicitImports.py are incompatible"
)
import_list0 = [item[0] for item in import_set] # only the names
if full_name in import_list0: # found!
for item in import_list0: # store everything in that list
self.implicit_imports.add(item)
return True, "module is an implicit import" # ok
# not known by anyone: kick it out!
self.info(drop_msg(full_name, package)) # issue ignore message
# faster decision next time
self.ignored_modules.add(full_name)
return False, "module is not used"
def getImplicitImports(self, module):
"""Declare all matplotlib.backends modules as implicit imports."""
full_name = module.getFullName()
if full_name == "__main__": # need to make sure that backends are used
for f in Options.options.recurse_modules:
if f.startswith("matplotlib.backends"):
yield f, False
def onStandaloneDistributionFinished(self, dist_dir):
""" Only used to output the compilation time."""
self.timer.end()
t = int(round(self.timer.getDelta()))
if t > 240:
unit = "minutes"
if t >= 600:
t = int(round(t / 60.0))
else:
t = round(t / 60, 1)
else:
unit = "seconds"
self.info("Compiled '%s' in %g %s." % (sys.argv[-1], t, unit))
class UserPlugin(NuitkaPluginBase):
plugin_name = __file__
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = set() # speed up repeated lookups
self.ignored_modules = set() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
fin = open(fin_name)
self.import_info = json.loads(fin.read()) # read it and make an array
fin.close()
self.import_calls = self.import_info["calls"]
self.import_files = self.import_info["files"]
self.msg_count = dict()
self.msg_limit = 21
"""
Check if we should enable any standard plugins.
Currently supported: "tk-inter", "numpy", "multiprocessing" and
"qt-plugins". For "numpy", we also support the "scipy" option.
"""
show_msg = False # only show info message if parameters are generated
tk = np = qt = sc = mp = pmw = torch = sklearn = False
tflow = gevent = mpl = False
msg = " '%s' is adding the following options:" % self.plugin_name
# detect required standard plugin in order to enable them
for mod in self.import_calls: # scan thru called items
m = mod[0]
if m == "numpy":
np = True
show_msg = True
if m == "matplotlib":
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m == "scipy":
sc = True
show_msg = True
elif m == "multiprocessing" and getOS() == "Windows":
mp = True
show_msg = True
elif m == "Pmw":
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m == "sklearn":
sklearn = True
show_msg = True
elif m == "tensorflow":
tflow = True
show_msg = True
elif m == "gevent":
gevent = True
show_msg = True
if show_msg is True:
info(msg)
if np:
o = "numpy="
if mpl:
o += "matplotlib"
if sc:
o += "scipy" if o.endswith("=") else ",scipy"
options.plugins_enabled.append(o)
info(" --enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter")
info(" --enable-plugin=tk-inter")
if qt:
options.plugins_enabled.append("qt-plugins=sensible")
info(" --enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info(" --enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info(" --enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info(" --enable-plugin=torch")
if sklearn:
options.plugins_enabled.append("sklearn")
info(" --enable-plugin=sklearn")
if tflow:
options.plugins_enabled.append("tensorflow")
info(" --enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info(" --enable-plugin=gevent")
for f in self.import_files:
options.recurse_modules.append(f)
# no plugin detected, but recursing to modules?
if show_msg is False and len(self.import_files) > 0:
info(msg)
msg = " --recurse-to for %i imported modules." % len(self.import_files)
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin goes here
return None
def onModuleEncounter(
self, module_filename, module_name, module_package, module_kind
):
""" Help decide whether to include a module.
Notes:
Performance considerations: the calls array is rather long
(may be thousands of items). So we store ignored modules
separately and check that array first.
We also maintain an array for known implicit imports and early
check against them, too.
Args:
module_filename: filename (not used here)
module_name: module name
module_package: package name
module_kind: one of "py" or "shlib" (not used here)
Returns:
None, (True, 'text') or (False, 'text').
Example: (False, "because it is not called").
"""
if module_package:
# the standard case:
full_name = module_package + "." + module_name
# also happens: module_name = package.module
# then use module_name as the full_name
if module_name.startswith(module_package):
t = module_name[len(module_package) :]
if t.startswith("."):
full_name = module_name
# also happens: package = a.b.c.module
# then use package as full_name
elif module_package.endswith(module_name):
full_name = module_package
else:
full_name = module_name
# fall through for easy cases
if full_name in self.ignored_modules: # known to be ignored
return False, "module is not used"
if full_name in self.implicit_imports: # known implicit import
return True, "module is an implicit import" # ok
# check if other plugins would accept this
for plugin in active_plugin_list:
if plugin.plugin_name == self.plugin_name:
continue # skip myself of course
rc = plugin.onModuleEncounter(
module_filename, module_name, module_package, module_kind
)
if rc is not None:
if rc[0] is True: # plugin wants to keep this
self.implicit_imports.add(full_name)
keep_msg = " keep %s (plugin '%s')" % (
full_name,
plugin.plugin_name,
)
count = self.msg_count.get(plugin.plugin_name, 0)
if count < self.msg_limit:
info(keep_msg)
self.msg_count[plugin.plugin_name] = count + 1
if count == self.msg_limit:
info(
" ... 'keep' msg limit exceeded for '%s'."
% plugin.plugin_name
)
return True, "module is imported" # ok
# plugin wants to drop this
self.ignored_modules.add(full_name)
ignore_msg = " drop %s (plugin '%s')" % (full_name, plugin.plugin_name)
info(ignore_msg)
return False, "dropped by plugin " + plugin.plugin_name
if full_name == "cv2":
return True, "needed by OpenCV"
checklist = get_checklist(full_name)
for mod in self.import_calls: # loop thru the called items
m = mod[0]
if m in checklist:
return True, "module is hinted to" # ok
# next we ask if implicit imports knows our candidate
if self.ImplicitImports is None: # the plugin is not yet loaded
for plugin in active_plugin_list:
if plugin.plugin_name == "implicit-imports":
self.ImplicitImports = plugin
break
if self.ImplicitImports is None:
sys.exit("could not find 'implicit-imports' plugin")
# ask the 'implicit-imports' plugin whether it knows this guy
if module_package is not None:
import_set = self.ImplicitImports.getImportsByFullname(module_package)
import_list0 = [item[0] for item in import_set] # only the names
if full_name in import_list0: # found!
for item in import_list0: # store everything in that list
self.implicit_imports.add(item)
return True, "module is an implicit imported" # ok
# not known by anyone: kick it out!
if module_package is not None:
ignore_msg = " drop %s (in %s)" % (module_name, module_package)
else:
ignore_msg = " drop %s" % module_name
info(ignore_msg) # issue ignore message
# faster decision next time
self.ignored_modules.add(full_name)
return False, "module is not used"
def onStandaloneDistributionFinished(self, dist_dir):
self.timer.end()
t = int(round(self.timer.delta()))
if t > 300:
t = int(round(t / 60.0))
unit = "minutes"
else:
unit = "seconds"
info(" Compile time %i %s." % (t, unit))