def makeLogoImages(): basePathLogo = "doc/Logo/Nuitka-Logo-%s" for logo in ("Vertical", "Symbol", "Horizontal"): cmd = "convert -background none %s.svg %s.png" % (basePathLogo, basePathLogo) check_call((cmd % (logo, logo)).split()) optimize_pngs( [basePathLogo % item for item in ("Vertical", "Symbol", "Horizontal")]) if os.path.exists("../Nuitka-website"): cmd = "convert -resize %s doc/Logo/Nuitka-Logo-Symbol.svg %s" for icon, size in { "../Nuitka-website/files/favicon.ico": "32x32", "../Nuitka-website/files/favicon.png": "32x32", "../Nuitka-website/doc/_static/favicon.ico": "32x32", "../Nuitka-website/doc/_static/favicon.png": "32x32", "../Nuitka-website/doc/_static/apple-touch-icon-ipad.png": "72x72", "../Nuitka-website/doc/_static/apple-touch-icon-ipad3.png": "144x144", "../Nuitka-website/doc/_static/apple-touch-icon-iphone.png": "57x57", "../Nuitka-website/doc/_static/apple-touch-icon-iphone4.png": "114x114", "../Nuitka-website/doc/_static/apple-touch-icon-180x180.png": "180x180", }: check_call((cmd % (icon, size)).split())
def _callDebchange(*args): args = ["debchange"] + list(args) os.environ["EDITOR"] = "" with open(os.devnull, "r") as devnull: check_call(args, stdin=devnull)
def withVirtualenv(env_name, base_dir=None, python=None, delete=True, style=None): """ Create a virtualenv and change into it. Activating for actual use will be your task. """ print("Creating virtualenv for quick test:") if python is None: python = sys.executable if base_dir is not None: env_dir = os.path.join(base_dir, env_name) else: env_dir = env_name removeDirectory(env_dir, ignore_errors=False) with withDirectoryChange(base_dir, allow_none=True): command = [python, "-m", "virtualenv", env_name] if style is not None: my_print("Executing: %s" % " ".join(command)) check_call(command) yield Virtualenv(env_dir) if delete: removeDirectory(env_dir, ignore_errors=False)
def copyToGlobalCoverageData(source, target): coverage_dir = os.environ.get("COVERAGE_DIR", None) if coverage_dir is None: return check_call(("scp", source, os.path.join(coverage_dir, target)))
def main(): parser = OptionParser() parser.add_option( "--checked-module", action="store", dest="checked_module", help="""\ Module with main() function to be checked for reference count stability.""", ) parser.add_option( "--explain", action="store_true", dest="explain", default=False, help="""\ Try to explain the differences by comparing object counts.""", ) options, positional_args = parser.parse_args() if positional_args and options.checked_module is None: options.checked_module = positional_args.pop() if positional_args and options.checked_module: parser.print_help() sys.exit("\nError, no positional argument allowed.") # First with pure Python. checked_module = importFileAsModule(options.checked_module) print("Using", checked_module.main) checkReferenceCount(checked_module.main, explain=options.explain) temp_dir = getTempDir() command = [ sys.executable, "-m", "nuitka", "--module", options.checked_module, "--output-dir", temp_dir, ] if hasattr(sys, "gettotalrefcount"): command.append("--python-debug") # print(command) check_call(command) module_name = os.path.basename(options.checked_module).split(".")[0] sys.path.insert(0, temp_dir) checked_module = __import__(module_name) print("Using", checked_module.main) checkReferenceCount(checked_module.main)
def updateFileIndex(diff_entry, new_object_hash): check_call([ "git", "update-index", "--cacheinfo", "%s,%s,%s" % (diff_entry["dst_mode"], new_object_hash, diff_entry["src_path"]), ])
def _cleanupRstFmt(filename): updated_contents = contents = getFileContents(filename, mode="rb") for keyword in extra_rst_keywords: updated_contents = updated_contents.replace(b".. %s::" % keyword, b".. raw:: %s" % keyword) if updated_contents != contents: with open(filename, "wb") as out_file: out_file.write(updated_contents) rstfmt_call = _getPythonBinaryCall("rstfmt") check_call(rstfmt_call + [ filename, ], # stdout=devnull, ) cleanupWindowsNewlines(filename) contents = getFileContents(filename, mode="rb") # Enforce choice between "bash" and "sh" for code directive. Use bash as # more people will know it. updated_contents = contents.replace(b".. code:: sh\n", b".. code:: bash\n") for keyword in extra_rst_keywords: updated_contents = updated_contents.replace(b".. raw:: %s" % keyword, b".. %s::" % keyword) lines = [] inside = False needs_empty = False for line in updated_contents.splitlines(): if line.startswith(b"-"): if inside and needs_empty: lines.append(b"") inside = True needs_empty = True lines.append(line) elif inside and line == b"": needs_empty = False lines.append(line) elif inside and line.startswith(b" "): needs_empty = True lines.append(line) else: inside = False lines.append(line) updated_contents = b"\n".join(lines) + b"\n" if updated_contents != contents: with open(filename, "wb") as out_file: out_file.write(updated_contents)
def runPy2dsc(filename, new_name): check_call(["py2dsc", new_name]) # Fixup for py2dsc not taking our custom suffix into account, so we need # to rename it ourselves. before_deb_name = filename[:-7].lower().replace("-", "_") after_deb_name = before_deb_name.replace("rc", "~rc") os.rename( "deb_dist/%s.orig.tar.gz" % before_deb_name, "deb_dist/%s+ds.orig.tar.gz" % after_deb_name, ) check_call(["rm -f deb_dist/*_source*"], shell=True) # Remove the now useless input, py2dsc has copied it, and we don't # publish it. os.unlink(new_name) # Assert that the unpacked directory is there and find it. Otherwise fail badly. entry = None for fullname, entry in listDir("deb_dist"): if ( os.path.isdir(fullname) and entry.startswith("nuitka") and not entry.endswith(".orig") ): break if entry is None: assert False # Import the "debian" directory from above. It's not in the original tar and # overrides fully what py2dsc did. check_call(["rm -r deb_dist/%s/debian/*" % entry], shell=True) check_call( [ "rsync", "-a", "--exclude", "pbuilder-hookdir", "../debian/", "deb_dist/%s/debian/" % entry, ] ) check_call(["rm deb_dist/*.dsc deb_dist/*.debian.tar.xz"], shell=True) return entry
def _cleanupImportSortOrder(filename): _cleanupImportRelative(filename) isort_call = _getPythonBinaryCall("isort") contents = getFileContents(filename) start_index = None if "\n# isort:start" in contents: parts = contents.splitlines() start_index = parts.index("# isort:start") contents = "\n".join(parts[start_index + 1 :]) with open(filename, "w") as out_file: out_file.write(contents) with open(os.devnull, "w") as devnull: check_call( isort_call + [ "-q", # quiet, but stdout is still garbage "--overwrite-in-place", # avoid using another temp file, this is already on one. "-ot", # Order imports by type in addition to alphabetically "-m3", # "vert-hanging" "-tc", # Trailing commas "-p", # make sure nuitka is first party package in import sorting. "nuitka", "-o", "SCons", filename, ], stdout=devnull, ) if start_index is not None: contents = getFileContents(filename) contents = "\n".join(parts[: start_index + 1]) + "\n" + contents with open(filename, "w") as out_file: out_file.write(contents)
def _cleanupImportSortOrder(filename): _cleanupImportRelative(filename) isort_call = _getPythonBinaryCall("isort") contents = getFileContents(filename) start_index = None if "\n# isort:start" in contents: parts = contents.splitlines() start_index = parts.index("# isort:start") contents = "\n".join(parts[start_index + 1:]) with open(filename, "w") as out_file: out_file.write(contents) with open(os.devnull, "w") as devnull: check_call( isort_call + [ "-q", # quiet, but stdout is still garbage "-ot", # Order imports by type in addition to alphabetically "-m3", # "vert-hanging" "-up", # Prefer braces () over \ for line continuation. "-tc", # Trailing commas "-p", # make sure nuitka is first party package in import sorting. "nuitka", "-ns", # Do not ignore those: "__init__.py", filename, ], stdout=devnull, ) if start_index is not None: contents = getFileContents(filename) contents = "\n".join(parts[:start_index + 1]) + "\n" + contents with open(filename, "w") as out_file: out_file.write(contents)
def _cleanupRstFmt(filename): rstfmt_call = _getPythonBinaryCall("rstfmt") check_call( rstfmt_call + [ filename, ], # stdout=devnull, ) cleanupWindowsNewlines(filename) with open(filename, "rb") as f: contents = f.read() updated_contents = contents.replace(b":\n\n.. code::\n", b"::\n") if updated_contents != contents: with open(filename, "wb") as out_file: out_file.write(updated_contents)
def makeManpage(python, suffix): cmd = [ "help2man", "-n", "the Python compiler", "--no-discard-stderr", "--no-info", "--include", "doc/nuitka-man-include.txt", "%s ./bin/nuitka" % python, ] with openTextFile("doc/nuitka%s.1" % suffix, "wb") as output: check_call(cmd, stdout=output) cmd[-1] += "-run" with openTextFile("doc/nuitka%s-run.1" % suffix, "wb") as output: check_call(cmd, stdout=output) for manpage in ("doc/nuitka%s.1" % suffix, "doc/nuitka%s-run.1" % suffix): manpage_contents = getFileContents(manpage).splitlines() new_contents = [] mark = False for count, line in enumerate(manpage_contents): if mark: line = ".SS " + line + ".BR\n" mark = False elif line == ".IP\n" and manpage_contents[count + 1].endswith(":\n"): mark = True continue if line == r"\fB\-\-g\fR++\-only" + "\n": line = r"\fB\-\-g\++\-only\fR" + "\n" new_contents.append(line) putTextFileContents(manpage, contents=new_contents)
def _compressFile(self, filename, use_cache): upx_options = ["-q", "--no-progress"] if use_cache: if self.upx_binary_hash is None: self.upx_binary_hash = getFileContentsHash(self.upx_binary, as_string=False) upx_hash = Hash() upx_hash.updateFromBytes(self.upx_binary_hash) upx_hash.updateFromValues(*upx_options) upx_hash.updateFromFile(filename) # TODO: Repeating pattern upx_cache_dir = os.path.join(getCacheDir(), "upx") makePath(upx_cache_dir) upx_cache_filename = os.path.join(upx_cache_dir, upx_hash.asHexDigest() + ".bin") if os.path.exists(upx_cache_filename): copyFile(upx_cache_filename, filename) return if use_cache: self.info("Uncached file, compressing '%s' may take a while." % os.path.basename(filename)) else: self.info("Compressing '%s'." % filename) check_call( [self.upx_binary] + upx_options + [filename], stdout=getNullOutput(), shell=False, ) if use_cache: copyFile(filename, upx_cache_filename)
def cleanupTarfileForDebian(filename, new_name): """Remove files that shouldn't be in Debian. The inline copies should definitely not be there. Also remove the PDF files. """ copyFile(filename, new_name) check_call(["gunzip", new_name]) check_call( [ "tar", "--wildcards", "--delete", "--file", new_name[:-3], "Nuitka*/*.pdf", "Nuitka*/build/inline_copy", ], ) check_call(["gzip", "-9", "-n", new_name[:-3]])
def _build(self, build_lib): # High complexity, pylint: disable=too-many-branches,too-many-locals # Nuitka wants the main package by filename, probably we should stop # needing that. old_dir = os.getcwd() os.chdir(build_lib) # Search in the build directory preferably. setMainScriptDirectory(os.path.abspath(old_dir)) for is_package, module_name in self._find_to_build(): module_name, main_filename, finding = locateModule( module_name=module_name, parent_package=None, level=0, ) package = module_name.getPackageName() # Check expectations, e.g. do not compile built-in modules. assert finding == "absolute", finding if package is not None: output_dir = os.path.join(build_lib, package.asPath()) else: output_dir = build_lib command = [ sys.executable, "-m", "nuitka", "--module", "--enable-plugin=pylint-warnings", "--output-dir=%s" % output_dir, "--nofollow-import-to=*.tests", "--remove-output", ] if is_package: command.append("--include-package=%s" % module_name) else: command.append("--include-module=%s" % module_name) # Process any extra options from setuptools if "nuitka" in self.distribution.command_options: for option, value in self.distribution.command_options[ "nuitka"].items(): option = "--" + option.lstrip("-") if (type(value) is tuple and len(value) == 2 and value[0] == "setup.py"): value = value[1] if value is None: command.append(option) elif isinstance(value, bool): option = "--" + ("no" if not value else "") + option.lstrip("-") command.append(option) elif isinstance(value, Iterable) and not isinstance( value, (unicode, bytes, str)): for val in value: command.append("%s=%s" % (option, val)) else: command.append("%s=%s" % (option, value)) command.append(main_filename) # Adding traces for clarity wheel_logger.info( "Building: '%s' with command %r" % (module_name.asString(), command), style="blue", ) check_call(command, cwd=build_lib) wheel_logger.info("Finished compilation of '%s'." % module_name.asString(), style="green") for root, _, filenames in os.walk(build_lib): for filename in filenames: fullpath = os.path.join(root, filename) if fullpath.lower().endswith( (".py", ".pyw", ".pyc", ".pyo")): os.unlink(fullpath) self.build_lib = build_lib os.chdir(old_dir)
def _build(self, build_lib): # High complexity, pylint: disable=too-many-branches,too-many-locals # Nuitka wants the main package by filename, probably we should stop # needing that. from nuitka.__past__ import ( # pylint: disable=I0021,redefined-builtin Iterable, unicode, ) from nuitka.importing.Importing import ( findModule, setMainScriptDirectory, ) from nuitka.utils.ModuleNames import ModuleName old_dir = os.getcwd() os.chdir(build_lib) # Search in the build directory preferably. setMainScriptDirectory(".") to_builds = self._find_to_build() for to_build in to_builds: package, main_filename, finding = findModule( importing=None, module_name=ModuleName(to_build.module_name), parent_package=None, level=0, warn=False, ) # Check expectations, e.g. do not compile built-in modules. assert finding == "absolute", finding if package is not None: output_dir = os.path.join(build_lib, package) else: output_dir = build_lib command = [ sys.executable, "-m", "nuitka", "--module", "--plugin-enable=pylint-warnings", "--output-dir=%s" % output_dir, "--nofollow-import-to=*.tests", "--show-modules", "--remove-output", ] if type(to_build) is PyPackage: command += ("--include-package=%s" % package_name.replace("/", ".") for package_name in to_build.related_packages) else: # type(to_build) is PyModule command += ("--include-module=%s" % module_name for module_name in to_build.related_modules) # Process any extra options from setuptools if "nuitka" in self.distribution.command_options: for option, value in self.distribution.command_options[ "nuitka"].items(): option = "--" + option.lstrip("-") if value is None: command.append(option) elif isinstance(value, bool): option = "--" + ("no" if not value else "") + option.lstrip("-") command.append(option) elif isinstance(value, Iterable) and not isinstance( value, (unicode, bytes, str)): for val in value: command.append("%s=%s" % (option, val)) else: command.append("%s=%s" % (option, value)) command.append(main_filename) # added for clarity my_print("Building: %s" % to_build, style="yellow") check_call(command, cwd=build_lib) for root, _, filenames in os.walk(build_lib): for filename in filenames: fullpath = os.path.join(root, filename) if fullpath.lower().endswith( (".py", ".pyw", ".pyc", ".pyo")): os.unlink(fullpath) # If the Python module has more than one parent package (e.g. # 'a.b.mod'), the compiled module will be in 'a.b/mod.so'. Move it # to 'a/b/mod.so', to make imports work. if package and "." in package: compiled_package_path = os.path.join(build_lib, package) assert os.path.isdir( compiled_package_path), compiled_package_path parts = package.split(".") fixed_package_path = os.path.join(build_lib, *parts) copyTree(compiled_package_path, fixed_package_path) removeDirectory(compiled_package_path, ignore_errors=False) os.chdir(old_dir) self.build_lib = build_lib
def main(): # Complex stuff, pylint: disable=too-many-statements # Make sure error messages are in English. os.environ["LANG"] = "C" parser = OptionParser() parser.add_option( "--no-pbuilder-update", action="store_false", dest="update_pbuilder", default=True, help="""\ Update the pbuilder chroot before building. Default %default.""", ) options, positional_args = parser.parse_args() assert len(positional_args) == 1, positional_args codename = positional_args[0] nuitka_version = getNuitkaVersion() branch_name = checkBranchName() category = getBranchCategory(branch_name) if category == "stable": if nuitka_version.count(".") == 1: assert checkChangeLog("New upstream release.") else: assert checkChangeLog("New upstream hotfix release.") else: assert checkChangeLog("New upstream pre-release.") shutil.rmtree("dist", ignore_errors=True) shutil.rmtree("build", ignore_errors=True) createReleaseDocumentation() assert os.system("python setup.py sdist --formats=gztar") == 0 os.chdir("dist") # Clean the stage for the debian package. The name "deb_dist" is what "py2dsc" # uses for its output later on. shutil.rmtree("deb_dist", ignore_errors=True) # Provide a re-packed tar.gz for the Debian package as input. # Create it as a "+ds" file, removing: # - the benchmarks (too many sources, not useful to end users, potential license # issues) # - the inline copy of scons (not wanted for Debian) for filename in os.listdir("."): if filename.endswith(".tar.gz"): new_name = filename[:-7] + "+ds.tar.gz" cleanupTarfileForDebian(filename, new_name) entry = runPy2dsc(filename, new_name) break else: assert False os.chdir("deb_dist") os.chdir(entry) # Build the debian package, but disable the running of tests, will be done later # in the pbuilder test steps. assert os.system("debuild --set-envvar=DEB_BUILD_OPTIONS=nocheck") == 0 os.chdir("../../..") assert os.path.exists("dist/deb_dist") # Check with pylint in pedantic mode and don't proceed if there were any # warnings given. Nuitka is lintian clean and shall remain that way. For # hotfix releases, i.e. "stable" builds, we skip this test though. if category == "stable": my_print("Skipped lintian checks for stable releases.", style="blue") else: assert os.system( "lintian --pedantic --allow-root dist/deb_dist/*.changes") == 0 # Move the created debian package files out. os.system("cp dist/deb_dist/*.deb dist/") # Build inside the pbuilder chroot, and output to dedicated directory. shutil.rmtree("package", ignore_errors=True) os.makedirs("package") # Now update the pbuilder. if options.update_pbuilder: command = """\ sudo /usr/sbin/pbuilder --update --basetgz /var/cache/pbuilder/%s.tgz""" % ( codename) assert os.system(command) == 0, codename (dsc_filename, ) = resolveShellPatternToFilenames("dist/deb_dist/*.dsc") # Execute the package build in the pbuilder with tests. command = ( "sudo", "/usr/sbin/pbuilder", "--build", "--basetgz", "/var/cache/pbuilder/%s.tgz" % codename, "--hookdir", "debian/pbuilder-hookdir", "--debemail", "Kay Hayen <*****@*****.**>", "--buildresult", "package", dsc_filename, ) check_call(command, shell=False) # Cleanup the build directory, not needed anymore. shutil.rmtree("build", ignore_errors=True) # Now build the repository. my_print("Building repository ...", style="blue") os.chdir("package") os.makedirs("repo") os.chdir("repo") os.makedirs("conf") putTextFileContents( "conf/distributions", contents="""\ Origin: Nuitka Label: Nuitka Codename: %(codename)s Architectures: i386 amd64 armel armhf powerpc Components: main Description: Apt repository for project Nuitka %(codename)s SignWith: D96ADCA1377F1CEB6B5103F11BFC33752912B99C """ % {"codename": codename}, ) command = ["reprepro", "includedeb", codename] command.extend(resolveShellPatternToFilenames("../*.deb")) check_call(command, shell=False) my_print("Uploading ...", style="blue") # Create repo folder unless already done. This is needed for the first # build only. assert (os.system("ssh [email protected] mkdir -p /var/www/deb/%s/%s/" % (category, codename)) == 0) # Update the repository on the web site. assert (os.system( "rsync -avz --delete dists pool --chown www-data [email protected]:/var/www/deb/%s/%s/" % (category, codename)) == 0) my_print("Finished.", style="blue")
def _callDebchange(*args): args = ["debchange"] + list(args) os.environ["EDITOR"] = "" check_call(args, stdin=getNullInput())
def _build(self, build_lib): # High complexity, pylint: disable=too-many-branches,too-many-locals # Nuitka wants the main package by filename, probably we should stop # needing that. old_dir = os.getcwd() os.chdir(build_lib) if self.distribution.package_dir and "" in self.distribution.package_dir: main_package_dir = self.distribution.package_dir.get("") else: main_package_dir = os.path.abspath(old_dir) # Search in the build directory preferably. setMainScriptDirectory(main_package_dir) for is_package, module_name in self._find_to_build(): module_name, main_filename, finding = locateModule( module_name=module_name, parent_package=None, level=0, ) package = module_name.getPackageName() # Check expectations, e.g. do not compile built-in modules. assert finding == "absolute", finding if package is not None: output_dir = os.path.join(build_lib, package.asPath()) else: output_dir = build_lib command = [ sys.executable, "-m", "nuitka", "--module", "--enable-plugin=pylint-warnings", "--output-dir=%s" % output_dir, "--nofollow-import-to=*.tests", "--remove-output", ] if is_package: command.append("--include-package=%s" % module_name) else: command.append("--include-module=%s" % module_name) toml_filename = os.environ.get("NUITKA_TOML_FILE") if toml_filename: # Import toml parser like "build" module does. try: from tomli import loads as toml_loads except ImportError: from toml import loads as toml_loads # Cannot use FileOperations.getFileContents() here, because of non-Nuitka process # pylint: disable=unspecified-encoding with open(toml_filename) as toml_file: toml_options = toml_loads(toml_file.read()) for option, value in toml_options.get("nuitka", {}).items(): command.extend(self._parseOptionsEntry(option, value)) # Process any extra options from setuptools if "nuitka" in self.distribution.command_options: for option, value in self.distribution.command_options[ "nuitka"].items(): command.extend(self._parseOptionsEntry(option, value)) command.append(main_filename) # Adding traces for clarity wheel_logger.info( "Building: '%s' with command %r" % (module_name.asString(), command), style="blue", ) check_call(command, cwd=build_lib) wheel_logger.info("Finished compilation of '%s'." % module_name.asString(), style="green") for root, _, filenames in os.walk(build_lib): for filename in filenames: fullpath = os.path.join(root, filename) if fullpath.lower().endswith( (".py", ".pyw", ".pyc", ".pyo")): os.unlink(fullpath) self.build_lib = build_lib os.chdir(old_dir)
def main(): goHome() parser = OptionParser() parser.add_option( "--upload", action="store_true", dest="upload", default=False, help="""\ Upload to https://nuitka.net/apidoc requires access rights and is done by the official servers automatically only. Without this, create the local html folder only. Default is %default.""", ) options, _positional_args = parser.parse_args() shutil.rmtree("html", ignore_errors=True) doxygen_path = getExecutablePath("doxygen") # Extra ball on Windows, check default installation PATH too. if not doxygen_path and getOS() == "Windows": with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"): doxygen_path = getExecutablePath("doxygen") if not doxygen_path: sys.exit( "Error, need to install Doxygen and add it to PATH for this to work." ) try: import doxypypy # pylint: disable=I0021,unused-import,unused-variable except ImportError: sys.exit("Error, needs to install doxypypy into this Python.") with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file: doxy_config = getFileContents("doc/Doxyfile.template") with withTemporaryFile( suffix=".bat" if getOS() == "Windows" else ".sh", delete=False) as doxy_batch_file: if getOS() == "Windows": doxy_batch_file.write("%s -m doxypypy.doxypypy -a -c %%1" % sys.executable) else: doxy_batch_file.write( "#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1" % sys.executable) doxy_batch_filename = doxy_batch_file.name st = os.stat(doxy_batch_filename) os.chmod(doxy_batch_filename, st.st_mode | stat.S_IEXEC) doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename) doxy_file.write(doxy_config) doxy_filename = doxy_file.name print("Running doxygen:") try: check_call([doxygen_path, doxy_filename]) finally: os.unlink(doxy_filename) os.unlink(doxy_batch_filename) # Update the repository on the web site. if options.upload: assert (os.system( "rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/" ) == 0) print("Finished.")
def optimize_pngs(pngList): for png in pngList: check_call(["optipng", "-o2", "%s.png" % png])
def createRstPDF(document, args): check_call(["rst2pdf"] + args + [document])
def main(): # Complex stuff, not broken down yet # pylint: disable=too-many-branches,too-many-locals,too-many-statements parser = OptionParser() parser.add_option("--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", "")) parser.add_option( "--cpython", action="store", dest="cpython", default=os.environ.get("PYTHON", sys.executable), ) parser.add_option("--code-diff", action="store", dest="diff_filename", default="") parser.add_option("--copy-source-to", action="store", dest="target_dir", default="") options, positional_args = parser.parse_args() if len(positional_args) != 1: sys.exit( "Error, need to give test case file name as positional argument.") test_case = positional_args[0] if os.path.exists(test_case): test_case = os.path.abspath(test_case) if options.cpython == "no": options.cpython = "" nuitka = options.nuitka if os.path.exists(nuitka): nuitka = os.path.abspath(nuitka) elif nuitka: sys.exit("Error, nuitka binary '%s' not found." % nuitka) python_version = setup(silent=True, go_main=False) assert os.path.exists(test_case), (test_case, os.getcwd()) my_print("PYTHON='%s'" % python_version) my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"]) with open(test_case, "rb") as f: my_print("TEST_CASE_HASH='%s'" % hashlib.md5(f.read()).hexdigest()) needs_2to3 = (python_version.startswith("3") and not test_case.endswith("32.py") and not test_case.endswith("33.py")) if options.target_dir: shutil.copyfile( test_case, os.path.join(options.target_dir, os.path.basename(test_case))) # First produce two variants. temp_dir = getTempDir() test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case)) test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case)) with open(test_case) as f: case_1_source, case_2_source = generateConstructCases(f.read()) with open(test_case_1, "w") as case_1_file: case_1_file.write(case_1_source) with open(test_case_2, "w") as case_2_file: case_2_file.write(case_2_source) if needs_2to3: test_case_1, _needs_delete = convertUsing2to3(test_case_1) test_case_2, _needs_delete = convertUsing2to3(test_case_2) os.environ["PYTHONHASHSEED"] = "0" if nuitka: nuitka_id = check_output("cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True) nuitka_id = nuitka_id.strip() if sys.version_info > (3, ): nuitka_id = nuitka_id.decode() my_print("NUITKA_COMMIT='%s'" % nuitka_id) os.chdir(getTempDir()) if nuitka: nuitka_call = [ os.environ["PYTHON"], nuitka, "--python-flag=-S", os.path.basename(test_case), ] nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()) # We want to compile under the same filename to minimize differences, and # then copy the resulting files afterwards. shutil.copyfile(test_case_1, os.path.basename(test_case)) check_call(nuitka_call) if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")): exe_suffix = ".exe" else: exe_suffix = ".bin" os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_1).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_1).replace(".py", exe_suffix), ) shutil.copyfile(test_case_2, os.path.basename(test_case)) check_call(nuitka_call) os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_2).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_2).replace(".py", exe_suffix), ) if options.diff_filename: suffixes = [".c", ".cpp"] for suffix in suffixes: cpp_1 = os.path.join(test_case_1.replace(".py", ".build"), "module.__main__" + suffix) if os.path.exists(cpp_1): break else: assert False for suffix in suffixes: cpp_2 = os.path.join(test_case_2.replace(".py", ".build"), "module.__main__" + suffix) if os.path.exists(cpp_2): break else: assert False import difflib with open(options.diff_filename, "w") as f: with open(cpp_1) as cpp1: with open(cpp_2) as cpp2: f.write(difflib.HtmlDiff().make_table( cpp1.readlines(), cpp2.readlines(), "Construct", "Baseline", True, )) nuitka_1 = runValgrind( "Nuitka construct", "callgrind", (test_case_1.replace(".py", exe_suffix), ), include_startup=True, ) nuitka_2 = runValgrind( "Nuitka baseline", "callgrind", (test_case_2.replace(".py", exe_suffix), ), include_startup=True, ) nuitka_diff = nuitka_1 - nuitka_2 my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call), file=sys.stderr) my_print("NUITKA_RAW=%s" % nuitka_1) my_print("NUITKA_BASE=%s" % nuitka_2) my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff) if options.cpython: cpython_1 = runValgrind( "CPython construct", "callgrind", (os.environ["PYTHON"], "-S", test_case_1), include_startup=True, ) cpython_2 = runValgrind( "CPython baseline", "callgrind", (os.environ["PYTHON"], "-S", test_case_2), include_startup=True, ) cpython_diff = cpython_1 - cpython_2 my_print("CPYTHON_RAW=%d" % cpython_1) my_print("CPYTHON_BASE=%d" % cpython_2) my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff) if options.cpython and options.nuitka: if nuitka_diff == 0: nuitka_gain = float("inf") else: nuitka_gain = float(100 * cpython_diff) / nuitka_diff my_print("NUITKA_GAIN=%.3f" % nuitka_gain) my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1)) my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))