def copyDataFiles(dist_dir):
""" Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
for _plugin_name, (source_desc, target_filename) in Plugins.considerDataFiles(
module
):
target_filename = os.path.join(dist_dir, target_filename)
assert isPathBelow(dist_dir, target_filename)
makePath(os.path.dirname(target_filename))
if inspect.isfunction(source_desc):
content = source_desc(target_filename)
if content is not None: # support creation of empty directories
with open(
target_filename, "wb" if type(content) is bytes else "w"
) as output:
output.write(content)
else:
shutil.copy2(source_desc, target_filename)
def enableClcache(the_compiler, env, source_dir):
importFromInlineCopy("atomicwrites", must_exist=True)
importFromInlineCopy("clcache", must_exist=True)
cl_binary = getExecutablePath(the_compiler, env)
# The compiler is passed via environment.
setEnvironmentVariable(env, "CLCACHE_CL", cl_binary)
env["CXX"] = env["CC"] = "<clcache>"
setEnvironmentVariable(env, "CLCACHE_HIDE_OUTPUTS", "1")
# The clcache stats filename needs absolute path, otherwise it will not work.
clcache_stats_filename = os.path.abspath(
os.path.join(source_dir, "clcache-stats.%d.txt" % os.getpid()))
setEnvironmentVariable(env, "CLCACHE_STATS", clcache_stats_filename)
env["CLCACHE_STATS"] = clcache_stats_filename
# Unless asked to do otherwise, store ccache files in our own directory.
if "CLCACHE_DIR" not in os.environ:
clcache_dir = os.path.join(getCacheDir(), "clcache")
makePath(clcache_dir)
clcache_dir = getExternalUsePath(clcache_dir)
setEnvironmentVariable(env, "CLCACHE_DIR", clcache_dir)
env["CLCACHE_DIR"] = clcache_dir
scons_details_logger.info(
"Using inline copy of clcache with %r cl binary." % cl_binary)
# Do not consider scons cache anymore.
return True
def enableClcache(the_compiler, env, source_dir):
importFromInlineCopy("atomicwrites", must_exist=True)
importFromInlineCopy("clcache", must_exist=True)
# Avoid importing this in threads, triggers CPython 3.9 importing bugs at least,
# do it now, so it's not a race issue.
import concurrent.futures.thread # pylint: disable=I0021,unused-import,unused-variable
cl_binary = getExecutablePath(the_compiler, env)
# The compiler is passed via environment.
setEnvironmentVariable(env, "CLCACHE_CL", cl_binary)
env["CXX"] = env["CC"] = "<clcache>"
setEnvironmentVariable(env, "CLCACHE_HIDE_OUTPUTS", "1")
# The clcache stats filename needs absolute path, otherwise it will not work.
clcache_stats_filename = os.path.abspath(
os.path.join(source_dir, "clcache-stats.%d.txt" % os.getpid())
)
setEnvironmentVariable(env, "CLCACHE_STATS", clcache_stats_filename)
env["CLCACHE_STATS"] = clcache_stats_filename
# Unless asked to do otherwise, store ccache files in our own directory.
if "CLCACHE_DIR" not in os.environ:
clcache_dir = os.path.join(getCacheDir(), "clcache")
makePath(clcache_dir)
clcache_dir = getExternalUsePath(clcache_dir)
setEnvironmentVariable(env, "CLCACHE_DIR", clcache_dir)
env["CLCACHE_DIR"] = clcache_dir
scons_details_logger.info(
"Using inline copy of clcache with %r cl binary." % cl_binary
)
def _getCacheFilename(dependency_tool, is_main_executable, source_dir,
original_dir, binary_filename):
original_filename = os.path.join(original_dir,
os.path.basename(binary_filename))
original_filename = os.path.normcase(original_filename)
if is_main_executable:
# Normalize main program name for caching as well, but need to use the
# scons information to distinguish different compilers, so we use
# different libs there.
# Ignore values, that are variable per compilation.
hashed_value = "".join(
key + value
for key, value in iterItems(readSconsReport(source_dir=source_dir))
if key not in ("CLCACHE_STATS", ))
else:
hashed_value = original_filename
# Have different values for different Python major versions.
hashed_value += sys.version + sys.executable
if str is not bytes:
hashed_value = hashed_value.encode("utf8")
cache_dir = os.path.join(getCacheDir(), "library_dependencies",
dependency_tool)
makePath(cache_dir)
return os.path.join(cache_dir, hashlib.md5(hashed_value).hexdigest())
def considerExtraDlls(dist_dir, module):
result = []
for plugin in getActivePlugins():
for extra_dll in plugin.considerExtraDlls(dist_dir, module):
# Backward compatibility with plugins not yet migrated to getExtraDlls usage.
if len(extra_dll) == 3:
extra_dll = makeDllEntryPointOld(
source_path=extra_dll[0],
dest_path=extra_dll[1],
package_name=extra_dll[2],
)
if not os.path.isfile(extra_dll.dest_path):
sys.exit(
"Error, copied filename %r for module %r that is not a file."
% (extra_dll.dest_path, module.getFullName()))
else:
if not os.path.isfile(extra_dll.source_path):
sys.exit(
"Error, attempting to copy plugin determined filename %r for module %r that is not a file."
% (extra_dll.source_path, module.getFullName()))
makePath(os.path.dirname(extra_dll.dest_path))
shutil.copyfile(extra_dll.source_path, extra_dll.dest_path)
result.append(extra_dll)
return result
def copyDataFiles(dist_dir, data_files):
""" Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
data_files:
Tuple of pairs describing (source, dest) or (func, dest) that
should be copied.
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
for source_desc, target_filename in data_files:
target_filename = os.path.join(dist_dir, target_filename)
assert isPathBelow(dist_dir, target_filename)
makePath(os.path.dirname(target_filename))
if inspect.isfunction(source_desc):
content = source_desc(target_filename)
if content is not None: # support creation of empty directories
with open(
target_filename, "wb" if type(content) is bytes else "w"
) as output:
output.write(content)
else:
shutil.copy2(source_desc, target_filename)
def _getCacheFilename(dependency_tool, is_main_executable, source_dir,
original_dir, binary_filename):
original_filename = os.path.join(original_dir,
os.path.basename(binary_filename))
original_filename = os.path.normcase(original_filename)
if is_main_executable:
# Normalize main program name for caching as well, but need to use the
# scons information to distinguish different compilers, so we use
# different libs there.
hashed_value = getFileContents(
os.path.join(source_dir, "scons-report.txt"))
else:
hashed_value = original_filename
# Have different values for different Python major versions.
hashed_value += sys.version + sys.executable
if str is not bytes:
hashed_value = hashed_value.encode("utf8")
cache_dir = os.path.join(getCacheDir(), "library_deps", dependency_tool)
makePath(cache_dir)
return os.path.join(cache_dir, hashlib.md5(hashed_value).hexdigest())
def getTestingCPythonOutputsCacheDir():
cache_dir = getCacheDir()
result = os.path.join(cache_dir, "cpython_outputs",
os.environ.get("NUITKA_TEST_SUITE", ""))
makePath(result)
return result
def considerExtraDlls(self, dist_dir, module):
""" Copy extra shared libraries or data for this installation.
Args:
dist_dir: the name of the program's dist folder
module: module object
Returns:
empty tuple
"""
full_name = module.getFullName()
elements = full_name.split(".")
if not self.numpy_copied and full_name == "numpy":
self.numpy_copied = True
binaries = getNumpyCoreBinaries(module)
for f in binaries:
bin_file, idx = f # (filename, pos. prefix + 1)
back_end = bin_file[idx:]
tar_file = os.path.join(dist_dir, back_end)
makePath( # create any missing intermediate folders
os.path.dirname(tar_file))
shutil.copyfile(bin_file, tar_file)
bin_total = len(binaries) # anything there at all?
if bin_total > 0:
msg = "Copied %i %s from 'numpy' installation." % (
bin_total,
"file" if bin_total < 2 else "files",
)
self.info(msg)
if not self.scipy_copied and full_name == "scipy":
self.scipy_copied = True
binaries = getScipyCoreBinaries(module)
for f in binaries:
bin_file, idx = f # (filename, pos. prefix + 1)
back_end = bin_file[idx:]
tar_file = os.path.join(dist_dir, back_end)
makePath( # create any missing intermediate folders
os.path.dirname(tar_file))
shutil.copyfile(bin_file, tar_file)
bin_total = len(binaries)
if bin_total > 0:
msg = "Copied %i %s from 'scipy' installation." % (
bin_total,
"file" if bin_total < 2 else "files",
)
self.info(msg)
if not self.mpl_data_copied and "matplotlib" in elements:
self.mpl_data_copied = True
copyMplDataFiles(module, dist_dir)
self.info("Copied 'matplotlib/mpl-data'.")
return ()
def getSourceDirectoryPath():
"""Return path inside the build directory."""
result = Options.getOutputPath(path=os.path.basename(
getTreeFilenameWithSuffix(_main_module, ".build")))
makePath(result)
return result
def considerExtraDlls(self, dist_dir, module):
"""Copy extra shared libraries or data for this installation.
Args:
dist_dir: the name of the program's dist folder
module: module object
Returns:
empty tuple
"""
full_name = module.getFullName()
if not self.numpy_copied and full_name == "numpy":
self.numpy_copied = True
binaries = getNumpyCoreBinaries(module)
for f in binaries:
bin_file, idx = f # (filename, pos. prefix + 1)
back_end = bin_file[idx:]
tar_file = os.path.join(dist_dir, back_end)
makePath( # create any missing intermediate folders
os.path.dirname(tar_file)
)
shutil.copyfile(bin_file, tar_file)
bin_total = len(binaries) # anything there at all?
if bin_total > 0:
msg = "Copied %i %s from 'numpy' installation." % (
bin_total,
"file" if bin_total < 2 else "files",
)
self.info(msg)
if os.name == "nt" and not self.scipy_copied and full_name == "scipy":
# TODO: We are not getting called twice, are we?
assert not self.scipy_copied
self.scipy_copied = True
bin_total = 0
for entry_point in self._getScipyCoreBinaries(
scipy_dir=module.getCompileTimeDirectory()
):
yield entry_point
bin_total += 1
if bin_total > 0:
msg = "Copied %i %s from 'scipy' installation." % (
bin_total,
"file" if bin_total < 2 else "files",
)
self.info(msg)
# TODO: Ouch, do not copy data files when asked to copy DLLs.
if self.matplotlib and full_name == "matplotlib" and not self.mpl_data_copied:
self.mpl_data_copied = True
self.copyMplDataFiles(dist_dir)
def cleanSourceDirectory(source_dir):
extensions = (".bin", ".c", ".cpp", ".exp", ".h", ".lib", ".manifest",
".o", ".obj", ".os", ".rc", ".res", ".S")
if os.path.isdir(source_dir):
for path, _filename in listDir(source_dir):
if hasFilenameExtension(path, extensions):
deleteFile(path, must_exist=True)
else:
makePath(source_dir)
def copyMplDataFiles(self, dist_dir):
""" Write matplotlib data files ('mpl-data')."""
matplotlib_info = self._getMatplotlibInfo()
if not os.path.isdir(matplotlib_info.data_path):
self.sysexit(
"mpl-data missing, matplotlib installation appears to be broken"
)
# Copy data files to dist folder
for fullname in getFileList(matplotlib_info.data_path):
filename = os.path.relpath(fullname, matplotlib_info.data_path)
if filename.endswith("matplotlibrc"): # handle config separately
continue
target_filename = os.path.join(dist_dir, "matplotlib", "mpl-data",
filename)
makePath(os.path.dirname(
target_filename)) # create intermediate folders
shutil.copyfile(fullname, target_filename)
old_lines = (open(
matplotlib_info.matplotlibrc_filename).read().splitlines()
) # old config file lines
new_lines = [] # new config file lines
found = False # checks whether backend definition encountered
for line in old_lines:
line = line.rstrip()
# omit meaningless lines
if line.startswith(
"#") and matplotlib_info.matplotlib_version < "3":
continue
new_lines.append(line)
if line.startswith(("backend ", "backend:")):
# old config file has a backend definition
found = True
if not found and matplotlib_info.matplotlib_version < "3":
# Set the backend, so even if it was run time determined, we now enforce it.
new_lines.append("backend: %s" % matplotlib_info.backend)
matplotlibrc_filename = os.path.join(dist_dir, "matplotlib",
"mpl-data", "matplotlibrc")
putTextFileContents(filename=matplotlibrc_filename, contents=new_lines)
self.info("Copied 'matplotlib/mpl-data'.")
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule())
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename,
module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize()
if Options.isExperimental():
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def considerExtraDlls(self, dist_dir, module):
""" Copy extra shared libraries or data for this installation.
Args:
dist_dir: the name of the program's dist folder
module: module object
Returns:
empty tuple
"""
full_name = module.getFullName()
if full_name == "numpy" and not self.numpy_copied:
self.numpy_copied = True
binaries = get_numpy_core_binaries(module)
for f in binaries:
bin_file, idx = f # (filename, pos. prefix + 1)
back_end = bin_file[idx:]
tar_file = os.path.join(dist_dir, back_end)
makePath( # create any missing intermediate folders
os.path.dirname(tar_file))
shutil.copyfile(bin_file, tar_file)
bin_total = len(binaries) # anything there at all?
if bin_total > 0:
msg = "Copied %i %s from 'numpy' installation." % (
bin_total,
"file" if bin_total < 2 else "files",
)
info(msg)
if full_name == "scipy" and not self.scipy_copied:
self.scipy_copied = True
binaries = get_scipy_core_binaries(module)
for f in binaries:
bin_file, idx = f # (filename, pos. prefix + 1)
back_end = bin_file[idx:]
tar_file = os.path.join(dist_dir, back_end)
makePath( # create any missing intermediate folders
os.path.dirname(tar_file))
shutil.copyfile(bin_file, tar_file)
bin_total = len(binaries)
if bin_total > 0:
msg = "Copied %i %s from 'scipy' installation." % (
bin_total,
"file" if bin_total < 2 else "files",
)
info(msg)
return ()
def getSourceDirectoryPath(onefile=False):
"""Return path inside the build directory."""
# Distinct build folders for oneline mode.
if onefile:
suffix = ".onefile-build"
else:
suffix = ".build"
result = Options.getOutputPath(
path=os.path.basename(getTreeFilenameWithSuffix(_main_module, suffix)))
makePath(result)
return result
def addShlibEntryPoint(module):
target_filename = os.path.join(getStandaloneDirectoryPath(),
*module.getFullName().split("."))
target_filename += getSharedLibrarySuffix(preferred=False)
target_dir = os.path.dirname(target_filename)
if not os.path.isdir(target_dir):
makePath(target_dir)
shutil.copyfile(module.getFilename(), target_filename)
standalone_entry_points.append(
makeExtensionModuleEntryPoint(
source_path=module.getFilename(),
dest_path=target_filename,
package_name=module.getFullName().getPackageName(),
))
def considerExtraDlls(self, dist_dir, module):
"""Provide a tuple of names of binaries to be included.
Args:
dist_dir: the distribution folder
module: the module object needing the binaries
Returns:
tuple
"""
# TODO: This should no longer be here, as this API is obsolete, pylint: disable=unused-argument
for included_entry_point in self.getExtraDlls(module):
# Copy to the dist directory, which normally should not be a plugin task, but is for now.
makePath(os.path.dirname(included_entry_point.dest_path))
copyFile(included_entry_point.source_path,
included_entry_point.dest_path)
yield included_entry_point
def _compressFile(self, filename, use_cache):
upx_options = ["-q", "--no-progress"]
if os.path.basename(filename).startswith("vcruntime140"):
return
if use_cache:
if self.upx_binary_hash is None:
self.upx_binary_hash = getFileContentsHash(self.upx_binary,
as_string=False)
upx_hash = Hash()
upx_hash.updateFromBytes(self.upx_binary_hash)
upx_hash.updateFromValues(*upx_options)
upx_hash.updateFromFile(filename)
# TODO: Repeating pattern
upx_cache_dir = os.path.join(getCacheDir(), "upx")
makePath(upx_cache_dir)
upx_cache_filename = os.path.join(upx_cache_dir,
upx_hash.asHexDigest() + ".bin")
if os.path.exists(upx_cache_filename):
copyFile(upx_cache_filename, filename)
return
if use_cache:
self.info("Uncached file, compressing '%s' may take a while." %
os.path.basename(filename))
else:
self.info("Compressing '%s'." % filename)
command = [self.upx_binary] + upx_options + [filename]
executeToolChecked(
logger=self,
command=command,
absence_message=None,
stderr_filter=self._filterUpxError,
)
if use_cache:
copyFile(filename, upx_cache_filename)
def considerExtraDlls(dist_dir, module):
"""Ask plugins to provide extra DLLs.
Notes:
These will be of type nuitka.freezer.IncludedEntryPoints.IncludedEntryPoint
and currently there is a backward compatibility for old style plugins that do
provide tuples of 3 elements. But plugins are really supposed to provide the
stuff created from factory functions for that type.
"""
result = []
for plugin in getActivePlugins():
for extra_dll in plugin.considerExtraDlls(dist_dir, module):
# Backward compatibility with plugins not yet migrated to getExtraDlls usage.
if len(extra_dll) == 3:
extra_dll = makeDllEntryPointOld(
source_path=extra_dll[0],
dest_path=extra_dll[1],
package_name=extra_dll[2],
)
if not os.path.isfile(extra_dll.dest_path):
plugin.sysexit(
"Error, copied filename %r for module %r that is not a file."
% (extra_dll.dest_path, module.getFullName()))
else:
if not os.path.isfile(extra_dll.source_path):
plugin.sysexit(
"Error, attempting to copy plugin determined filename %r for module %r that is not a file."
% (extra_dll.source_path, module.getFullName()))
makePath(os.path.dirname(extra_dll.dest_path))
copyFile(extra_dll.source_path, extra_dll.dest_path)
if extra_dll.executable:
addFileExecutablePermission(extra_dll.dest_path)
result.append(extra_dll)
return result
def enableCcache(
the_compiler,
env,
source_dir,
python_prefix,
show_scons_mode,
assume_yes_for_downloads,
):
# The ccache needs absolute path, otherwise it will not work.
ccache_logfile = os.path.abspath(
os.path.join(source_dir, "ccache-%d.txt" % os.getpid()))
setEnvironmentVariable(env, "CCACHE_LOGFILE", ccache_logfile)
env["CCACHE_LOGFILE"] = ccache_logfile
# Unless asked to do otherwise, store ccache files in our own directory.
if "CCACHE_DIR" not in os.environ:
ccache_dir = os.path.join(getCacheDir(), "ccache")
makePath(ccache_dir)
ccache_dir = getExternalUsePath(ccache_dir)
setEnvironmentVariable(env, "CCACHE_DIR", ccache_dir)
env["CCACHE_DIR"] = ccache_dir
# First check if it's not already supposed to be a ccache, then do nothing.
cc_path = getExecutablePath(the_compiler, env=env)
cc_is_link, cc_link_path = getLinkTarget(cc_path)
if cc_is_link and os.path.basename(cc_link_path) == "ccache":
if show_scons_mode:
scons_logger.info(
"Chosen compiler %s is pointing to ccache %s already." %
(cc_path, cc_link_path))
return True
return _injectCcache(
the_compiler=the_compiler,
cc_path=cc_path,
env=env,
python_prefix=python_prefix,
show_scons_mode=show_scons_mode,
assume_yes_for_downloads=assume_yes_for_downloads,
)
def _handleDataFile(dist_dir, tracer, included_datafile):
"""Handle a data file."""
if isinstance(included_datafile, IncludedDataFile):
if included_datafile.kind == "empty_dirs":
tracer.info("Included empty directories %s due to %s." % (
",".join(included_datafile.dest_path),
included_datafile.reason,
))
for sub_dir in included_datafile.dest_path:
makePath(os.path.join(dist_dir, sub_dir))
elif included_datafile.kind == "data_file":
dest_path = os.path.join(dist_dir, included_datafile.dest_path)
tracer.info("Included data file %r due to %s." % (
included_datafile.dest_path,
included_datafile.reason,
))
makePath(os.path.dirname(dest_path))
shutil.copyfile(included_datafile.source_path, dest_path)
elif included_datafile.kind == "data_dir":
dest_path = os.path.join(dist_dir, included_datafile.dest_path)
makePath(os.path.dirname(dest_path))
copied = copyTree(included_datafile.source_path, dest_path)
tracer.info("Included data dir %r with %d files due to %s." % (
included_datafile.dest_path,
len(copied),
included_datafile.reason,
))
else:
assert False, included_datafile
else:
# TODO: Goal is have this unused.
source_desc, target_filename = included_datafile
if not isPathBelow(dist_dir, target_filename):
target_filename = os.path.join(dist_dir, target_filename)
makePath(os.path.dirname(target_filename))
if inspect.isfunction(source_desc):
content = source_desc(target_filename)
if content is not None: # support creation of empty directories
with open(target_filename,
"wb" if type(content) is bytes else "w") as output:
output.write(content)
else:
copyFileWithPermissions(source_desc, target_filename)
def copyMplDataFiles(module, dist_dir):
""" Write matplotlib data files ('mpl-data')."""
data_dir = os.path.join(module.getCompileTimeDirectory(),
"mpl-data") # must exist
if not os.path.isdir(data_dir):
sys.exit("mpl-data missing: matplotlib installation is broken")
matplotlibrc, backend = getMatplotlibRc() # get matplotlibrc, backend
prefix = os.path.join("matplotlib", "mpl-data")
for item in getFileList(data_dir): # copy data files to dist folder
if item.endswith("matplotlibrc"): # handle config separately
continue
idx = item.find(
prefix) # need string starting with 'matplotlib/mpl-data'
tar_file = os.path.join(dist_dir, item[idx:])
makePath(os.path.dirname(tar_file)) # create intermediate folders
shutil.copyfile(item, tar_file)
old_lines = open(matplotlibrc).read().splitlines() # old config file lines
new_lines = ["# modified by Nuitka plugin 'numpy'"
] # new config file lines
found = False # checks whether backend definition encountered
for line in old_lines:
line = line.strip() # omit meaningless lines
if line.startswith("#") or line == "":
continue
new_lines.append(line)
if line.startswith(("backend ", "backend:")):
found = True # old config file has a backend definition
new_lines.append("# backend definition copied from installation")
if not found:
# get the string from interpreted mode and insert it in matplotlibrc
new_lines.append("backend: %s" % backend)
matplotlibrc = os.path.join(dist_dir, prefix, "matplotlibrc")
outfile = open(matplotlibrc, "w")
outfile.write("\n".join(new_lines))
outfile.close()
def _compressFile(self, filename, use_cache):
upx_options = ["-q", "--no-progress"]
if use_cache:
if self.upx_binary_hash is None:
self.upx_binary_hash = getFileContentsHash(self.upx_binary,
as_string=False)
upx_hash = Hash()
upx_hash.updateFromBytes(self.upx_binary_hash)
upx_hash.updateFromValues(*upx_options)
upx_hash.updateFromFile(filename)
# TODO: Repeating pattern
upx_cache_dir = os.path.join(getCacheDir(), "upx")
makePath(upx_cache_dir)
upx_cache_filename = os.path.join(upx_cache_dir,
upx_hash.asHexDigest() + ".bin")
if os.path.exists(upx_cache_filename):
copyFile(upx_cache_filename, filename)
return
if use_cache:
self.info("Uncached file, compressing '%s' may take a while." %
os.path.basename(filename))
else:
self.info("Compressing '%s'." % filename)
check_call(
[self.upx_binary] + upx_options + [filename],
stdout=getNullOutput(),
shell=False,
)
if use_cache:
copyFile(filename, upx_cache_filename)
def cleanSourceDirectory(source_dir):
extensions = (
".bin",
".c",
".cpp",
".exp",
".h",
".lib",
".manifest",
".o",
".obj",
".os",
".rc",
".res",
".S",
".txt",
)
if os.path.isdir(source_dir):
for path, _filename in listDir(source_dir):
if hasFilenameExtension(path, extensions):
deleteFile(path, must_exist=True)
else:
makePath(source_dir)
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
full_name = module.getFullName()
elements = full_name.split(".")
plugin_dirs = None
if elements[0] in ("PyQt4", "PyQt5"):
qt_version = int(elements[0][-1])
plugin_dirs = self.getPyQtPluginDirs(qt_version)
if full_name in ("PyQt4", "PyQt5"):
if not plugin_dirs:
sys.exit("Error, failed to detect %s plugin directories." % full_name)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = set(self.qt_plugins.split(","))
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
"platforms",
)
if self.hasPluginFamily(plugin_dirs, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
self.info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
for plugin_dir in plugin_dirs:
copyTree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for plugin_dir in plugin_dirs
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if isWin32Windows():
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_files = sum(
(
getFileList(qt_bin_dir)
for qt_bin_dir in self._getQtBinDirs(plugin_dirs)
),
[],
)
self.info("Copying OpenSSL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if "qml" in plugin_options or "all" in plugin_options:
for plugin_dir in plugin_dirs:
qml_plugin_dir = os.path.normpath(
os.path.join(plugin_dir, "..", "qml")
)
if os.path.exists(qml_plugin_dir):
break
else:
sys.exit("Error, no such Qt plugin family: qml")
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
self.info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
copyTree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
# Also copy required OpenGL DLLs on Windows
if isWin32Windows():
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
self.info("Copying OpenGL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
elif full_name == "PyQt5.QtNetwork":
if not isWin32Windows():
dll_path = locateDLL("crypto")
if dll_path is None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
dll_path = locateDLL("ssl")
if dll_path is not None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
elif (
full_name.startswith(("PyQt4.QtWebEngine", "PyQt5.QtWebEngine"))
and not self.webengine_done
):
self.webengine_done = True # prevent multiple copies
self.info("Copying QtWebEngine components")
plugin_parent = os.path.dirname(plugin_dirs[0])
if isWin32Windows():
bin_dir = os.path.join(plugin_parent, "bin")
else: # TODO verify this for non-Windows!
bin_dir = os.path.join(plugin_parent, "libexec")
target_bin_dir = os.path.join(dist_dir)
for f in os.listdir(bin_dir):
if f.startswith("QtWebEngineProcess"):
shutil.copy(os.path.join(bin_dir, f), target_bin_dir)
resources_dir = os.path.join(plugin_parent, "resources")
target_resources_dir = os.path.join(dist_dir)
for f in os.listdir(resources_dir):
shutil.copy(os.path.join(resources_dir, f), target_resources_dir)
translations_dir = os.path.join(plugin_parent, "translations")
pos = len(translations_dir) + 1
target_translations_dir = os.path.join(
dist_dir, elements[0], "Qt", "translations"
)
for f in getFileList(translations_dir):
tar_f = os.path.join(target_translations_dir, f[pos:])
makePath(os.path.dirname(tar_f))
shutil.copyfile(f, tar_f)
return ()
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir = os.path.dirname(os.path.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt"
)
if os.path.isfile(origin_prefix_filename):
data_files.append(
(filename, "orig-prefix.txt")
)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(
filename = filename
)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree(
main_module = main_module
)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
standalone_entry_points.insert(
0,
(binary_filename, binary_filename, None)
)
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
copyUsedDLLs(
source_dir = getSourceDirectoryPath(main_module),
dist_dir = dist_dir,
standalone_entry_points = standalone_entry_points
)
for module in ModuleRegistry.getDoneModules():
data_files.extend(
Plugins.considerDataFiles(module)
)
for source_filename, target_filename in data_files:
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module),
target_filename
)
makePath(os.path.dirname(target_filename))
shutil.copy2(
source_filename,
target_filename
)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(
path = getSourceDirectoryPath(main_module),
ignore_errors = False
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here. TODO: Move inside scons file and avoid subprocess call.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(
(
"chmod",
"-x",
getResultFullpath(main_module)
)
)
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = getResultBasepath(main_module) + ".pyi"
with open(pyi_filename, 'w') as pyi_file:
pyi_file.write(
"""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
""" % {
"imports" : '\n'.join(
"import %s" % module_name
for module_name in
getImportedNames()
)
}
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree = main_module,
clean_path = Options.shallClearPythonPathEnvironment()
)
else:
executeMain(
binary_filename = getResultFullpath(main_module),
clean_path = Options.shallClearPythonPathEnvironment()
)
def makeSourceDirectory(main_module):
""" Get the full list of modules imported, create code for all of them.
"""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches,too-many-locals,too-many-statements
assert main_module.isCompiledPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name = module.getFullName(),
module_filename = module.getCompileTimeFilename()
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if uncompiled_module.isUserProvided() or \
uncompiled_module.isTechnical():
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if '*' in any_case_module or '{' in any_case_module:
continue
for module in ModuleRegistry.getDoneUserModules():
if module.getFullName() == any_case_module:
break
else:
warning(
"Didn't recurse to '%s', apparently not used." % \
any_case_module
)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = getSourceDirectoryPath(main_module)
module_filenames = pickSourceFilenames(
source_dir = source_dir,
modules = ModuleRegistry.getDoneModules()
)
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
prepared_modules[c_filename] = CodeGeneration.prepareModuleCode(
global_context = global_context,
module = module,
module_name = module.getFullName(),
)
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[c_filename][1].getConstantCode(0)
# Second pass, generate the actual module code into the files.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
template_values, module_context = prepared_modules[c_filename]
source_code = CodeGeneration.generateModuleCode(
module_context = module_context,
template_values = template_values
)
writeSourceCode(
filename = c_filename,
source_code = source_code
)
if Options.isShowInclusion():
info("Included compiled module '%s'." % module.getFullName())
elif module.isPythonShlibModule():
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module),
*module.getFullName().split('.')
)
if Utils.getOS() == "Windows":
target_filename += ".pyd"
else:
target_filename += ".so"
target_dir = os.path.dirname(target_filename)
if not os.path.isdir(target_dir):
makePath(target_dir)
shutil.copyfile(
module.getFilename(),
target_filename
)
standalone_entry_points.append(
(
module.getFilename(),
target_filename,
module.getPackage()
)
)
elif module.isUncompiledPythonModule():
pass
else:
assert False, module
writeSourceCode(
filename = os.path.join(
source_dir,
"__constants.c"
),
source_code = ConstantCodes.getConstantsDefinitionCode(
context = global_context
)
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
ModuleRegistry.getDoneUserModules()
)
writeSourceCode(
filename = os.path.join(
source_dir,
"__helpers.h"
),
source_code = helper_decl_code
)
writeSourceCode(
filename = os.path.join(
source_dir,
"__helpers.c"
),
source_code = helper_impl_code
)
def getDependsExePath():
""" Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if Utils.getArchitecture() == "x86":
depends_url = "http://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "http://dependencywalker.com/depends22_x64.zip"
if "APPDATA" not in os.environ:
sys.exit("Error, standalone mode cannot find 'APPDATA' environment.")
nuitka_app_dir = os.path.join(os.environ["APPDATA"], "nuitka")
if not os.path.isdir(nuitka_app_dir):
makePath(nuitka_app_dir)
nuitka_depends_zip = os.path.join(nuitka_app_dir,
os.path.basename(depends_url))
if not os.path.isfile(nuitka_depends_zip):
Tracing.printLine("""\
Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool
to analyze the dependencies of Python extension modules. Is it OK to download
and put it in APPDATA (no installer needed, cached, one time question).""")
reply = raw_input("Proceed and download? [Yes]/No ")
if reply.lower() in ("no", 'n'):
sys.exit(
"Nuitka does not work in --standalone on Windows without.")
info("Downloading '%s'" % depends_url)
urlretrieve(depends_url, nuitka_depends_zip)
nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture())
if not os.path.isdir(nuitka_depends_dir):
os.makedirs(nuitka_depends_dir)
depends_exe = os.path.join(nuitka_depends_dir, "depends.exe")
if not os.path.isfile(depends_exe):
info("Extracting to '%s'" % depends_exe)
import zipfile
try:
depends_zip = zipfile.ZipFile(nuitka_depends_zip)
depends_zip.extractall(nuitka_depends_dir)
except Exception: # Catching anything zip throws, pylint:disable=W0703
info("Problem with the downloaded zip file, deleting it.")
deleteFile(depends_exe, must_exist=False)
deleteFile(nuitka_depends_zip, must_exist=True)
sys.exit("Error, need '%s' as extracted from '%s'." %
(depends_exe, depends_url))
assert os.path.isfile(depends_exe)
return depends_exe
def getTestingCacheDir():
cache_dir = getCacheDir()
result = os.path.join(cache_dir, "tests_state")
makePath(result)
return result
def makeSourceDirectory(main_module):
""" Get the full list of modules imported, create code for all of them.
"""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches,too-many-locals,too-many-statements
assert main_module.isCompiledPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (
uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()
):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneUserModules():
if module.getFullName() == any_case_module:
break
else:
warning("Didn't recurse to '%s', apparently not used." % any_case_module)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = getSourceDirectoryPath(main_module)
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=ModuleRegistry.getDoneModules()
)
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
try:
prepared_modules[c_filename] = CodeGeneration.prepareModuleCode(
global_context=global_context,
module=module,
module_name=module.getFullName(),
)
except Exception:
warning("Problem creating code for module %r." % module)
raise
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[c_filename][1].getConstantCode(0)
# Second pass, generate the actual module code into the files.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
template_values, module_context = prepared_modules[c_filename]
source_code = CodeGeneration.generateModuleCode(
module_context=module_context, template_values=template_values
)
writeSourceCode(filename=c_filename, source_code=source_code)
if Options.isShowInclusion():
info("Included compiled module '%s'." % module.getFullName())
elif module.isPythonShlibModule():
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module),
*module.getFullName().split(".")
)
target_filename += Utils.getSharedLibrarySuffix()
target_dir = os.path.dirname(target_filename)
if not os.path.isdir(target_dir):
makePath(target_dir)
shutil.copyfile(module.getFilename(), target_filename)
standalone_entry_points.append(
(module.getFilename(), target_filename, module.getPackage())
)
elif module.isUncompiledPythonModule():
pass
else:
assert False, module
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=ConstantCodes.getConstantsDefinitionCode(context=global_context),
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
ModuleRegistry.getDoneUserModules()
)
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code
)
def getDependsExePath():
""" Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if Utils.getArchitecture() == "x86":
depends_url = "http://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "http://dependencywalker.com/depends22_x64.zip"
nuitka_app_dir = getAppDir()
nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture())
nuitka_depends_zip = os.path.join(nuitka_depends_dir, os.path.basename(depends_url))
depends_exe = os.path.join(nuitka_depends_dir, "depends.exe")
makePath(nuitka_depends_dir)
if not os.path.isfile(nuitka_depends_zip) and not os.path.isfile(depends_exe):
if assumeYesForDownloads():
reply = "y"
else:
Tracing.printLine(
"""\
Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool
to analyze the dependencies of Python extension modules. Is it OK to download
and put it in "%s".
No installer needed, cached, one time question.
Proceed and download? [Yes]/No """
% (nuitka_app_dir)
)
Tracing.flushStdout()
reply = raw_input()
if reply.lower() in ("no", "n"):
sys.exit("Nuitka does not work in --standalone on Windows without.")
info("Downloading '%s'" % depends_url)
try:
urlretrieve(depends_url, nuitka_depends_zip)
except Exception: # Any kind of error, pylint: disable=broad-except
sys.exit(
"""Failed to download '%s'.\
Contents should manually be extracted to '%s'."""
% (depends_url, nuitka_depends_dir)
)
if not os.path.isfile(depends_exe):
info("Extracting to '%s'" % depends_exe)
import zipfile
try:
depends_zip = zipfile.ZipFile(nuitka_depends_zip)
depends_zip.extractall(nuitka_depends_dir)
except Exception: # Catching anything zip throws, pylint:disable=W0703
info("Problem with the downloaded zip file, deleting it.")
deleteFile(depends_exe, must_exist=False)
deleteFile(nuitka_depends_zip, must_exist=True)
sys.exit(
"Error, need '%s' as extracted from '%s'." % (depends_exe, depends_url)
)
assert os.path.isfile(depends_exe)
return depends_exe
def compileTree():
source_dir = OutputDirectories.getSourceDirectoryPath()
general.info("Completed Python level compilation and optimization.")
if not Options.shallOnlyExecCCompilerCall():
general.info("Generating source code for C backend compiler.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before generating C code: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
# Now build the target language code for the whole tree.
makeSourceDirectory()
bytecode_accessor = ConstantAccessor(
data_filename="__bytecode.const", top_level_name="bytecode_data"
)
# This should take all bytecode values, even ones needed for frozen or
# not produce anything.
loader_code = LoaderCodes.getMetapathLoaderBodyCode(bytecode_accessor)
writeSourceCode(
filename=os.path.join(source_dir, "__loader.c"), source_code=loader_code
)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
general.sysexit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.is_debug:
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
general.info("Running data composer tool for optimal constant value handling.")
runDataComposer(source_dir)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(
filename=os.path.join(target_dir, filename), source_code=source_code
)
general.info("Running C compilation via Scons.")
# Run the Scons to build things.
result, options = runSconsBackend(quiet=not Options.isShowScons())
return result, options
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule(),
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCCompilerCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing=None,
module_name=package_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate package %r." % package_name)
Recursion.checkPluginPath(
plugin_filename=package_directory, module_package=package_package
)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing=None,
module_name=module_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate module %r." % module_name)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_package
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module