def _detectBinaryPathDLLsMacOS(original_dir, binary_filename, keep_unresolved):
result = set()
process = subprocess.Popen(
args=["otool", "-L", binary_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _stderr = process.communicate()
system_paths = (b"/usr/lib/", b"/System/Library/Frameworks/")
for line in stdout.split(b"\n")[1:]:
if not line:
continue
filename = line.split(b" (")[0].strip()
stop = False
for w in system_paths:
if filename.startswith(w):
stop = True
break
if not stop:
if python_version >= 0x300:
filename = filename.decode("utf-8")
# print("adding", filename)
result.add(filename)
resolved_result = _resolveBinaryPathDLLsMacOS(original_dir,
binary_filename, result,
keep_unresolved)
return resolved_result
def _detectBinaryRPathsMacOS(original_dir, binary_filename):
result = set()
process = subprocess.Popen(
args=["otool", "-l", binary_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _stderr = process.communicate()
lines = stdout.split(b"\n")
for i, o in enumerate(lines):
if o.endswith(b"cmd LC_RPATH"):
line = lines[i + 2]
if python_version >= 0x300:
line = line.decode("utf-8")
line = line.split("path ")[1]
line = line.split(" (offset")[0]
if line.startswith("@loader_path"):
line = os.path.join(original_dir, line[13:])
elif line.startswith("@executable_path"):
continue
result.add(line)
return result
def getCompilerArch(mingw_mode, msvc_mode, the_cc_name, compiler_path):
if mingw_mode:
if compiler_path not in _compiler_arch:
_compiler_arch[compiler_path] = _getBinaryArch(
binary=compiler_path, mingw_mode=mingw_mode)
elif msvc_mode:
cmdline = [compiler_path]
if "-cl" in the_cc_name:
cmdline.append("--version")
proc = subprocess.Popen(
cmdline,
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
)
# The cl.exe without further args will give error output indicating
# arch, while clang outputs in stdout.
stdout, stderr = proc.communicate()
_rv = proc.wait()
if b"x86" in stderr or b"i686" in stdout:
_compiler_arch[compiler_path] = "pei-i386"
elif b"x64" in stderr or b"x86_64" in stdout:
_compiler_arch[compiler_path] = "pei-x86-64"
else:
assert False, (stdout, stderr)
else:
assert False, compiler_path
return _compiler_arch[compiler_path]
def setupCacheHashSalt(test_code_path):
assert os.path.exists(test_code_path)
if os.path.exists(os.path.join(test_code_path, ".git")):
git_cmd = ["git", "ls-tree", "-r", "HEAD", test_code_path]
process = subprocess.Popen(
args=git_cmd,
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_git, stderr_git = process.communicate()
assert process.returncode == 0, stderr_git
salt_value = hashlib.md5(stdout_git)
else:
salt_value = hashlib.md5()
for filename in getFileList(test_code_path):
if filename.endswith(".py"):
salt_value.update(getFileContents(filename, mode="rb"))
os.environ["NUITKA_HASH_SALT"] = salt_value.hexdigest()
def _getBinaryArch(binary, mingw_mode):
if "linux" in sys.platform or mingw_mode:
assert os.path.exists(binary), binary
command = ["objdump", "-f", binary]
try:
proc = subprocess.Popen(
command,
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
)
except OSError:
return None
data, _err = proc.communicate()
rv = proc.wait()
if rv != 0:
return None
if str is not bytes:
data = decodeData(data)
for line in data.splitlines():
if " file format " in line:
return line.split(" file format ")[-1]
else:
# TODO: Missing for macOS, FreeBSD, other Linux
return None
def detectDLLsWithDependencyWalker(binary_filename, scan_dirs):
dwp_filename = binary_filename + ".dwp"
output_filename = binary_filename + ".depends"
# User query should only happen once if at all.
with withFileLock(
"Finding out dependency walker path and creating DWP file for %s" %
binary_filename):
depends_exe = getDependsExePath()
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
with open(dwp_filename, "w") as dwp_file:
dwp_file.write(
"""\
%(scan_dirs)s
SxS
""" % {
"scan_dirs":
"\n".join("UserDir %s" % getExternalUsePath(dirname)
for dirname in scan_dirs)
})
# Starting the process while locked, so file handles are not duplicated.
depends_exe_process = subprocess.Popen(
(
depends_exe,
"-c",
"-ot%s" % output_filename,
"-d:%s" % dwp_filename,
"-f1",
"-pa1",
"-ps1",
binary_filename,
),
stdin=getNullInput(),
cwd=getExternalUsePath(os.getcwd()),
)
# TODO: Exit code should be checked.
depends_exe_process.wait()
if not os.path.exists(output_filename):
inclusion_logger.sysexit(
"Error, depends.exe failed to produce expected output.")
# Opening the result under lock, so it is not getting locked by new processes.
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
result = _parseDependsExeOutput(output_filename)
deleteFile(output_filename, must_exist=True)
deleteFile(dwp_filename, must_exist=True)
return result
def _executePylint(filenames, pylint_options, extra_options):
# This is kind of a singleton module, pylint: disable=global-statement
global our_exit_code
command = ([os.environ["PYTHON"], "-m", "pylint"] + pylint_options +
extra_options + filenames)
process = subprocess.Popen(
args=command,
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
)
stdout, stderr = process.communicate()
exit_code = process.returncode
if exit_code == -11:
sys.exit("Error, segfault from pylint.")
stdout = _cleanupPylintOutput(stdout)
stderr = _cleanupPylintOutput(stderr)
if stderr:
our_exit_code = 1
for line in stderr:
my_print(line)
if stdout:
# If we filtered everything away, remove the leading file name reports.
while stdout and stdout[-1].startswith("******"):
del stdout[-1]
for line in stdout:
my_print(line)
if stdout:
our_exit_code = 1
sys.stdout.flush()
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None and not Utils.isPosixWindows():
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
"$ORIGIN", os.path.dirname(sys.executable))
ld_library_path = OrderedSet()
if _detected_python_rpath:
ld_library_path.add(_detected_python_rpath)
ld_library_path.update(getPackageSpecificDLLDirectories(package_name))
if original_dir is not None:
ld_library_path.add(original_dir)
# ld_library_path.update(getSubDirectories(original_dir, ignore_dirs=("__pycache__",)))
with withEnvironmentPathAdded("LD_LIBRARY_PATH", *ld_library_path):
process = subprocess.Popen(
args=["ldd", dll_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
stderr = b"\n".join(
line for line in stderr.splitlines() if not line.startswith(
b"ldd: warning: you do not have execution permission for"))
inclusion_logger.debug("ldd output for %s is:\n%s" %
(dll_filename, stdout))
if stderr:
inclusion_logger.debug("ldd error for %s is:\n%s" %
(dll_filename, stderr))
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found or supplied by ldd itself.
if filename in ("not found", "ldd"):
continue
# Do not include kernel / glibc specific libraries. This list has been
# assembled by looking what are the most common .so files provided by
# glibc packages from ArchLinux, Debian Stretch and CentOS.
#
# Online sources:
# - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html
# - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html
# - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html
# - https://packages.debian.org/stretch/amd64/libc6/filelist
#
# Note: This list may still be incomplete. Some additional libraries
# might be provided by glibc - it may vary between the package versions
# and between Linux distros. It might or might not be a problem in the
# future, but it should be enough for now.
if os.path.basename(filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
result.add(filename)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsPosix(
dll_filename=sub_dll_filename,
package_name=package_name,
original_dir=original_dir,
))
return sub_result
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 0x300:
command = command.encode("utf8")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args=[sys.executable, "-s", "-S", "-v", tmp_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dict(os.environ, PYTHONIOENCODING="utf_8"),
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf-8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300:
if module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "shlib", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "shlib", filename))
for module_name, _prio, kind, filename in sorted(detections):
if kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "shlib":
_detectedShlibFile(filename=filename, module_name=module_name)
else:
assert False, kind
return result
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir, binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
if not locateDLL("fuse"):
postprocessing_logger.sysexit(
"""\
Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed
for onefile creation to work on Linux."""
)
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
with open(apprun_filename, "w") as output_file:
output_file.write(
"""\
#!/bin/bash
exec -a $ARGV0 $APPDIR/%s $@"""
% os.path.basename(binary_filename)
)
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
assert icon_paths
extension = os.path.splitext(icon_paths[0])[1].lower()
shutil.copyfile(icon_paths[0], getResultBasepath() + extension)
with open(getResultBasepath() + ".desktop", "w") as output_file:
output_file.write(
"""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;"""
% {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
}
)
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while."
)
stdout_filename = binary_filename + ".appimage.stdout.txt"
stderr_filename = binary_filename + ".appimage.stderr.txt"
stdout_file = open(stdout_filename, "wb")
stderr_file = open(stderr_filename, "wb")
# Starting the process while locked, so file handles are not duplicated.
appimagetool_process = subprocess.Popen(
(
_getAppImageToolPath(
for_operation=True, assume_yes_for_downloads=assumeYesForDownloads()
),
dist_dir,
"--comp",
"xz",
"-n",
onefile_output_filename,
),
shell=False,
stdin=getNullInput(),
stdout=stdout_file,
stderr=stderr_file,
)
result = appimagetool_process.wait()
stdout_file.close()
stderr_file.close()
if not os.path.exists(onefile_output_filename):
postprocessing_logger.sysexit(
"Error, expected output file %r not created by AppImage, check its outputs %r and %r."
% (onefile_output_filename, stdout_filename, stderr_filename)
)
if result != 0:
# Useless now.
os.unlink(onefile_output_filename)
if b"Text file busy" in getFileContents(stderr_filename, mode="rb"):
postprocessing_logger.sysexit(
"Error, error exit from AppImage because target file is locked."
)
postprocessing_logger.sysexit(
"Error, error exit from AppImage, check its outputs %r and %r."
% (stdout_filename, stderr_filename)
)
os.unlink(stdout_filename)
os.unlink(stderr_filename)
postprocessing_logger.info("Completed onefile creation.")
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir,
binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
if not locateDLL("fuse"):
postprocessing_logger.sysexit("""\
Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed
for onefile creation to work on Linux.""")
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
putTextFileContents(
apprun_filename,
contents="""\
#!/bin/bash
exec -a $ARGV0 $APPDIR/%s \"$@\"""" % os.path.basename(binary_filename),
)
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
assert icon_paths
extension = os.path.splitext(icon_paths[0])[1].lower()
copyFile(icon_paths[0], getResultBasepath() + extension)
putTextFileContents(
getResultBasepath() + ".desktop",
contents="""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;""" % {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
},
)
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
stdout_filename = binary_filename + ".appimage.stdout.txt"
stderr_filename = binary_filename + ".appimage.stderr.txt"
stdout_file = openTextFile(stdout_filename, "wb")
stderr_file = openTextFile(stderr_filename, "wb")
command = (
_getAppImageToolPath(for_operation=True,
assume_yes_for_downloads=assumeYesForDownloads()),
dist_dir,
"--comp",
getAppImageCompression(),
"-n",
onefile_output_filename,
)
stderr_file.write(b"Executed %r\n" % " ".join(command))
# Starting the process while locked, so file handles are not duplicated, we
# need fine grained control over process here, therefore we cannot use the
# Execution.executeProcess() function without making it too complex and not
# all Python versions allow using with, pylint: disable=consider-using-with
# pylint: disable
appimagetool_process = subprocess.Popen(
command,
shell=False,
stdin=getNullInput(),
stdout=stdout_file,
stderr=stderr_file,
)
result = appimagetool_process.wait()
stdout_file.close()
stderr_file.close()
if result != 0:
# Useless result if there were errors, so now remove it.
deleteFile(onefile_output_filename, must_exist=False)
stderr = getFileContents(stderr_filename, mode="rb")
if b"Text file busy" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because target file is locked."
)
if b"modprobe fuse" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because fuse kernel module was not loaded."
)
postprocessing_logger.sysexit(
"Error, error exit from AppImage, check its outputs '%s' and '%s'."
% (stdout_filename, stderr_filename))
if not os.path.exists(onefile_output_filename):
postprocessing_logger.sysexit(
"Error, expected output file %r not created by AppImage, check its outputs '%s' and '%s'."
% (onefile_output_filename, stdout_filename, stderr_filename))
deleteFile(stdout_filename, must_exist=True)
deleteFile(stderr_filename, must_exist=True)
postprocessing_logger.info("Completed onefile creation.")
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir,
binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
if not locateDLL("fuse"):
postprocessing_logger.sysexit(
"Error, the fuse library (libfuse.so.x) must be installed for onefile to work on Linux."
)
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
with open(apprun_filename, "w") as output_file:
output_file.write("""\
#!/bin/sh
exec $APPDIR/%s $@""" % os.path.basename(binary_filename))
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
assert icon_paths
extension = os.path.splitext(icon_paths[0])[1].lower()
shutil.copyfile(icon_paths[0], getResultBasepath() + extension)
with open(getResultBasepath() + ".desktop", "w") as output_file:
output_file.write(
"""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;""" % {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
})
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
# Starting the process while locked, so file handles are not duplicated.
appimagetool_process = subprocess.Popen(
(
getAppImageToolPath(),
dist_dir,
"--comp",
"xz",
"-n",
onefile_output_filename,
),
shell=False,
stdin=getNullInput(),
stderr=getNullOutput(),
stdout=getNullOutput(),
)
# TODO: Exit code should be checked.
result = appimagetool_process.wait()
if not os.path.exists(onefile_output_filename):
postprocessing_logger.sysexit(
"Error, expected output file %s not created by AppImage." %
onefile_output_filename)
postprocessing_logger.info("Completed onefile creation.")
assert result == 0, result
def _callDebchange(*args):
args = ["debchange"] + list(args)
os.environ["EDITOR"] = ""
check_call(args, stdin=getNullInput())
