def makeLogoImages():
basePathLogo = "doc/Logo/Nuitka-Logo-%s"
for logo in ("Vertical", "Symbol", "Horizontal"):
cmd = "convert -background none %s.svg %s.png" % (basePathLogo,
basePathLogo)
check_call((cmd % (logo, logo)).split())
optimize_pngs(
[basePathLogo % item for item in ("Vertical", "Symbol", "Horizontal")])
if os.path.exists("../Nuitka-website"):
cmd = "convert -resize %s doc/Logo/Nuitka-Logo-Symbol.svg %s"
for icon, size in {
"../Nuitka-website/files/favicon.ico":
"32x32",
"../Nuitka-website/files/favicon.png":
"32x32",
"../Nuitka-website/doc/_static/favicon.ico":
"32x32",
"../Nuitka-website/doc/_static/favicon.png":
"32x32",
"../Nuitka-website/doc/_static/apple-touch-icon-ipad.png":
"72x72",
"../Nuitka-website/doc/_static/apple-touch-icon-ipad3.png":
"144x144",
"../Nuitka-website/doc/_static/apple-touch-icon-iphone.png":
"57x57",
"../Nuitka-website/doc/_static/apple-touch-icon-iphone4.png":
"114x114",
"../Nuitka-website/doc/_static/apple-touch-icon-180x180.png":
"180x180",
}:
check_call((cmd % (icon, size)).split())
def _callDebchange(*args):
args = ["debchange"] + list(args)
os.environ["EDITOR"] = ""
with open(os.devnull, "r") as devnull:
check_call(args, stdin=devnull)
def withVirtualenv(env_name, base_dir=None, python=None, delete=True, style=None):
""" Create a virtualenv and change into it.
Activating for actual use will be your task.
"""
print("Creating virtualenv for quick test:")
if python is None:
python = sys.executable
if base_dir is not None:
env_dir = os.path.join(base_dir, env_name)
else:
env_dir = env_name
removeDirectory(env_dir, ignore_errors=False)
with withDirectoryChange(base_dir, allow_none=True):
command = [python, "-m", "virtualenv", env_name]
if style is not None:
my_print("Executing: %s" % " ".join(command))
check_call(command)
yield Virtualenv(env_dir)
if delete:
removeDirectory(env_dir, ignore_errors=False)
def copyToGlobalCoverageData(source, target):
coverage_dir = os.environ.get("COVERAGE_DIR", None)
if coverage_dir is None:
return
check_call(("scp", source, os.path.join(coverage_dir, target)))
def main():
parser = OptionParser()
parser.add_option(
"--checked-module",
action="store",
dest="checked_module",
help="""\
Module with main() function to be checked for reference count stability.""",
)
parser.add_option(
"--explain",
action="store_true",
dest="explain",
default=False,
help="""\
Try to explain the differences by comparing object counts.""",
)
options, positional_args = parser.parse_args()
if positional_args and options.checked_module is None:
options.checked_module = positional_args.pop()
if positional_args and options.checked_module:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
# First with pure Python.
checked_module = importFileAsModule(options.checked_module)
print("Using", checked_module.main)
checkReferenceCount(checked_module.main, explain=options.explain)
temp_dir = getTempDir()
command = [
sys.executable,
"-m",
"nuitka",
"--module",
options.checked_module,
"--output-dir",
temp_dir,
]
if hasattr(sys, "gettotalrefcount"):
command.append("--python-debug")
# print(command)
check_call(command)
module_name = os.path.basename(options.checked_module).split(".")[0]
sys.path.insert(0, temp_dir)
checked_module = __import__(module_name)
print("Using", checked_module.main)
checkReferenceCount(checked_module.main)
def updateFileIndex(diff_entry, new_object_hash):
check_call([
"git",
"update-index",
"--cacheinfo",
"%s,%s,%s" %
(diff_entry["dst_mode"], new_object_hash, diff_entry["src_path"]),
])
def _cleanupRstFmt(filename):
updated_contents = contents = getFileContents(filename, mode="rb")
for keyword in extra_rst_keywords:
updated_contents = updated_contents.replace(b".. %s::" % keyword,
b".. raw:: %s" % keyword)
if updated_contents != contents:
with open(filename, "wb") as out_file:
out_file.write(updated_contents)
rstfmt_call = _getPythonBinaryCall("rstfmt")
check_call(rstfmt_call + [
filename,
],
# stdout=devnull,
)
cleanupWindowsNewlines(filename)
contents = getFileContents(filename, mode="rb")
# Enforce choice between "bash" and "sh" for code directive. Use bash as
# more people will know it.
updated_contents = contents.replace(b".. code:: sh\n", b".. code:: bash\n")
for keyword in extra_rst_keywords:
updated_contents = updated_contents.replace(b".. raw:: %s" % keyword,
b".. %s::" % keyword)
lines = []
inside = False
needs_empty = False
for line in updated_contents.splitlines():
if line.startswith(b"-"):
if inside and needs_empty:
lines.append(b"")
inside = True
needs_empty = True
lines.append(line)
elif inside and line == b"":
needs_empty = False
lines.append(line)
elif inside and line.startswith(b" "):
needs_empty = True
lines.append(line)
else:
inside = False
lines.append(line)
updated_contents = b"\n".join(lines) + b"\n"
if updated_contents != contents:
with open(filename, "wb") as out_file:
out_file.write(updated_contents)
def runPy2dsc(filename, new_name):
check_call(["py2dsc", new_name])
# Fixup for py2dsc not taking our custom suffix into account, so we need
# to rename it ourselves.
before_deb_name = filename[:-7].lower().replace("-", "_")
after_deb_name = before_deb_name.replace("rc", "~rc")
os.rename(
"deb_dist/%s.orig.tar.gz" % before_deb_name,
"deb_dist/%s+ds.orig.tar.gz" % after_deb_name,
)
check_call(["rm -f deb_dist/*_source*"], shell=True)
# Remove the now useless input, py2dsc has copied it, and we don't
# publish it.
os.unlink(new_name)
# Assert that the unpacked directory is there and find it. Otherwise fail badly.
entry = None
for fullname, entry in listDir("deb_dist"):
if (
os.path.isdir(fullname)
and entry.startswith("nuitka")
and not entry.endswith(".orig")
):
break
if entry is None:
assert False
# Import the "debian" directory from above. It's not in the original tar and
# overrides fully what py2dsc did.
check_call(["rm -r deb_dist/%s/debian/*" % entry], shell=True)
check_call(
[
"rsync",
"-a",
"--exclude",
"pbuilder-hookdir",
"../debian/",
"deb_dist/%s/debian/" % entry,
]
)
check_call(["rm deb_dist/*.dsc deb_dist/*.debian.tar.xz"], shell=True)
return entry
def _cleanupImportSortOrder(filename):
_cleanupImportRelative(filename)
isort_call = _getPythonBinaryCall("isort")
contents = getFileContents(filename)
start_index = None
if "\n# isort:start" in contents:
parts = contents.splitlines()
start_index = parts.index("# isort:start")
contents = "\n".join(parts[start_index + 1 :])
with open(filename, "w") as out_file:
out_file.write(contents)
with open(os.devnull, "w") as devnull:
check_call(
isort_call
+ [
"-q", # quiet, but stdout is still garbage
"--overwrite-in-place", # avoid using another temp file, this is already on one.
"-ot", # Order imports by type in addition to alphabetically
"-m3", # "vert-hanging"
"-tc", # Trailing commas
"-p", # make sure nuitka is first party package in import sorting.
"nuitka",
"-o",
"SCons",
filename,
],
stdout=devnull,
)
if start_index is not None:
contents = getFileContents(filename)
contents = "\n".join(parts[: start_index + 1]) + "\n" + contents
with open(filename, "w") as out_file:
out_file.write(contents)
def _cleanupImportSortOrder(filename):
_cleanupImportRelative(filename)
isort_call = _getPythonBinaryCall("isort")
contents = getFileContents(filename)
start_index = None
if "\n# isort:start" in contents:
parts = contents.splitlines()
start_index = parts.index("# isort:start")
contents = "\n".join(parts[start_index + 1:])
with open(filename, "w") as out_file:
out_file.write(contents)
with open(os.devnull, "w") as devnull:
check_call(
isort_call + [
"-q", # quiet, but stdout is still garbage
"-ot", # Order imports by type in addition to alphabetically
"-m3", # "vert-hanging"
"-up", # Prefer braces () over \ for line continuation.
"-tc", # Trailing commas
"-p", # make sure nuitka is first party package in import sorting.
"nuitka",
"-ns", # Do not ignore those:
"__init__.py",
filename,
],
stdout=devnull,
)
if start_index is not None:
contents = getFileContents(filename)
contents = "\n".join(parts[:start_index + 1]) + "\n" + contents
with open(filename, "w") as out_file:
out_file.write(contents)
def _cleanupRstFmt(filename):
rstfmt_call = _getPythonBinaryCall("rstfmt")
check_call(
rstfmt_call
+ [
filename,
],
# stdout=devnull,
)
cleanupWindowsNewlines(filename)
with open(filename, "rb") as f:
contents = f.read()
updated_contents = contents.replace(b":\n\n.. code::\n", b"::\n")
if updated_contents != contents:
with open(filename, "wb") as out_file:
out_file.write(updated_contents)
def makeManpage(python, suffix):
cmd = [
"help2man",
"-n",
"the Python compiler",
"--no-discard-stderr",
"--no-info",
"--include",
"doc/nuitka-man-include.txt",
"%s ./bin/nuitka" % python,
]
with openTextFile("doc/nuitka%s.1" % suffix, "wb") as output:
check_call(cmd, stdout=output)
cmd[-1] += "-run"
with openTextFile("doc/nuitka%s-run.1" % suffix, "wb") as output:
check_call(cmd, stdout=output)
for manpage in ("doc/nuitka%s.1" % suffix,
"doc/nuitka%s-run.1" % suffix):
manpage_contents = getFileContents(manpage).splitlines()
new_contents = []
mark = False
for count, line in enumerate(manpage_contents):
if mark:
line = ".SS " + line + ".BR\n"
mark = False
elif line == ".IP\n" and manpage_contents[count +
1].endswith(":\n"):
mark = True
continue
if line == r"\fB\-\-g\fR++\-only" + "\n":
line = r"\fB\-\-g\++\-only\fR" + "\n"
new_contents.append(line)
putTextFileContents(manpage, contents=new_contents)
def _compressFile(self, filename, use_cache):
upx_options = ["-q", "--no-progress"]
if use_cache:
if self.upx_binary_hash is None:
self.upx_binary_hash = getFileContentsHash(self.upx_binary,
as_string=False)
upx_hash = Hash()
upx_hash.updateFromBytes(self.upx_binary_hash)
upx_hash.updateFromValues(*upx_options)
upx_hash.updateFromFile(filename)
# TODO: Repeating pattern
upx_cache_dir = os.path.join(getCacheDir(), "upx")
makePath(upx_cache_dir)
upx_cache_filename = os.path.join(upx_cache_dir,
upx_hash.asHexDigest() + ".bin")
if os.path.exists(upx_cache_filename):
copyFile(upx_cache_filename, filename)
return
if use_cache:
self.info("Uncached file, compressing '%s' may take a while." %
os.path.basename(filename))
else:
self.info("Compressing '%s'." % filename)
check_call(
[self.upx_binary] + upx_options + [filename],
stdout=getNullOutput(),
shell=False,
)
if use_cache:
copyFile(filename, upx_cache_filename)
def cleanupTarfileForDebian(filename, new_name):
"""Remove files that shouldn't be in Debian.
The inline copies should definitely not be there. Also remove the
PDF files.
"""
copyFile(filename, new_name)
check_call(["gunzip", new_name])
check_call(
[
"tar",
"--wildcards",
"--delete",
"--file",
new_name[:-3],
"Nuitka*/*.pdf",
"Nuitka*/build/inline_copy",
],
)
check_call(["gzip", "-9", "-n", new_name[:-3]])
def _build(self, build_lib):
# High complexity, pylint: disable=too-many-branches,too-many-locals
# Nuitka wants the main package by filename, probably we should stop
# needing that.
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferably.
setMainScriptDirectory(os.path.abspath(old_dir))
for is_package, module_name in self._find_to_build():
module_name, main_filename, finding = locateModule(
module_name=module_name,
parent_package=None,
level=0,
)
package = module_name.getPackageName()
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
if package is not None:
output_dir = os.path.join(build_lib, package.asPath())
else:
output_dir = build_lib
command = [
sys.executable,
"-m",
"nuitka",
"--module",
"--enable-plugin=pylint-warnings",
"--output-dir=%s" % output_dir,
"--nofollow-import-to=*.tests",
"--remove-output",
]
if is_package:
command.append("--include-package=%s" % module_name)
else:
command.append("--include-module=%s" % module_name)
# Process any extra options from setuptools
if "nuitka" in self.distribution.command_options:
for option, value in self.distribution.command_options[
"nuitka"].items():
option = "--" + option.lstrip("-")
if (type(value) is tuple and len(value) == 2
and value[0] == "setup.py"):
value = value[1]
if value is None:
command.append(option)
elif isinstance(value, bool):
option = "--" + ("no" if not value else
"") + option.lstrip("-")
command.append(option)
elif isinstance(value, Iterable) and not isinstance(
value, (unicode, bytes, str)):
for val in value:
command.append("%s=%s" % (option, val))
else:
command.append("%s=%s" % (option, value))
command.append(main_filename)
# Adding traces for clarity
wheel_logger.info(
"Building: '%s' with command %r" %
(module_name.asString(), command),
style="blue",
)
check_call(command, cwd=build_lib)
wheel_logger.info("Finished compilation of '%s'." %
module_name.asString(),
style="green")
for root, _, filenames in os.walk(build_lib):
for filename in filenames:
fullpath = os.path.join(root, filename)
if fullpath.lower().endswith(
(".py", ".pyw", ".pyc", ".pyo")):
os.unlink(fullpath)
self.build_lib = build_lib
os.chdir(old_dir)
def _build(self, build_lib):
# High complexity, pylint: disable=too-many-branches,too-many-locals
# Nuitka wants the main package by filename, probably we should stop
# needing that.
from nuitka.__past__ import ( # pylint: disable=I0021,redefined-builtin
Iterable, unicode,
)
from nuitka.importing.Importing import (
findModule,
setMainScriptDirectory,
)
from nuitka.utils.ModuleNames import ModuleName
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferably.
setMainScriptDirectory(".")
to_builds = self._find_to_build()
for to_build in to_builds:
package, main_filename, finding = findModule(
importing=None,
module_name=ModuleName(to_build.module_name),
parent_package=None,
level=0,
warn=False,
)
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
if package is not None:
output_dir = os.path.join(build_lib, package)
else:
output_dir = build_lib
command = [
sys.executable,
"-m",
"nuitka",
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % output_dir,
"--nofollow-import-to=*.tests",
"--show-modules",
"--remove-output",
]
if type(to_build) is PyPackage:
command += ("--include-package=%s" %
package_name.replace("/", ".")
for package_name in to_build.related_packages)
else: # type(to_build) is PyModule
command += ("--include-module=%s" % module_name
for module_name in to_build.related_modules)
# Process any extra options from setuptools
if "nuitka" in self.distribution.command_options:
for option, value in self.distribution.command_options[
"nuitka"].items():
option = "--" + option.lstrip("-")
if value is None:
command.append(option)
elif isinstance(value, bool):
option = "--" + ("no" if not value else
"") + option.lstrip("-")
command.append(option)
elif isinstance(value, Iterable) and not isinstance(
value, (unicode, bytes, str)):
for val in value:
command.append("%s=%s" % (option, val))
else:
command.append("%s=%s" % (option, value))
command.append(main_filename)
# added for clarity
my_print("Building: %s" % to_build, style="yellow")
check_call(command, cwd=build_lib)
for root, _, filenames in os.walk(build_lib):
for filename in filenames:
fullpath = os.path.join(root, filename)
if fullpath.lower().endswith(
(".py", ".pyw", ".pyc", ".pyo")):
os.unlink(fullpath)
# If the Python module has more than one parent package (e.g.
# 'a.b.mod'), the compiled module will be in 'a.b/mod.so'. Move it
# to 'a/b/mod.so', to make imports work.
if package and "." in package:
compiled_package_path = os.path.join(build_lib, package)
assert os.path.isdir(
compiled_package_path), compiled_package_path
parts = package.split(".")
fixed_package_path = os.path.join(build_lib, *parts)
copyTree(compiled_package_path, fixed_package_path)
removeDirectory(compiled_package_path, ignore_errors=False)
os.chdir(old_dir)
self.build_lib = build_lib
def main():
# Complex stuff, pylint: disable=too-many-statements
# Make sure error messages are in English.
os.environ["LANG"] = "C"
parser = OptionParser()
parser.add_option(
"--no-pbuilder-update",
action="store_false",
dest="update_pbuilder",
default=True,
help="""\
Update the pbuilder chroot before building. Default %default.""",
)
options, positional_args = parser.parse_args()
assert len(positional_args) == 1, positional_args
codename = positional_args[0]
nuitka_version = getNuitkaVersion()
branch_name = checkBranchName()
category = getBranchCategory(branch_name)
if category == "stable":
if nuitka_version.count(".") == 1:
assert checkChangeLog("New upstream release.")
else:
assert checkChangeLog("New upstream hotfix release.")
else:
assert checkChangeLog("New upstream pre-release.")
shutil.rmtree("dist", ignore_errors=True)
shutil.rmtree("build", ignore_errors=True)
createReleaseDocumentation()
assert os.system("python setup.py sdist --formats=gztar") == 0
os.chdir("dist")
# Clean the stage for the debian package. The name "deb_dist" is what "py2dsc"
# uses for its output later on.
shutil.rmtree("deb_dist", ignore_errors=True)
# Provide a re-packed tar.gz for the Debian package as input.
# Create it as a "+ds" file, removing:
# - the benchmarks (too many sources, not useful to end users, potential license
# issues)
# - the inline copy of scons (not wanted for Debian)
for filename in os.listdir("."):
if filename.endswith(".tar.gz"):
new_name = filename[:-7] + "+ds.tar.gz"
cleanupTarfileForDebian(filename, new_name)
entry = runPy2dsc(filename, new_name)
break
else:
assert False
os.chdir("deb_dist")
os.chdir(entry)
# Build the debian package, but disable the running of tests, will be done later
# in the pbuilder test steps.
assert os.system("debuild --set-envvar=DEB_BUILD_OPTIONS=nocheck") == 0
os.chdir("../../..")
assert os.path.exists("dist/deb_dist")
# Check with pylint in pedantic mode and don't proceed if there were any
# warnings given. Nuitka is lintian clean and shall remain that way. For
# hotfix releases, i.e. "stable" builds, we skip this test though.
if category == "stable":
my_print("Skipped lintian checks for stable releases.", style="blue")
else:
assert os.system(
"lintian --pedantic --allow-root dist/deb_dist/*.changes") == 0
# Move the created debian package files out.
os.system("cp dist/deb_dist/*.deb dist/")
# Build inside the pbuilder chroot, and output to dedicated directory.
shutil.rmtree("package", ignore_errors=True)
os.makedirs("package")
# Now update the pbuilder.
if options.update_pbuilder:
command = """\
sudo /usr/sbin/pbuilder --update --basetgz /var/cache/pbuilder/%s.tgz""" % (
codename)
assert os.system(command) == 0, codename
(dsc_filename, ) = resolveShellPatternToFilenames("dist/deb_dist/*.dsc")
# Execute the package build in the pbuilder with tests.
command = (
"sudo",
"/usr/sbin/pbuilder",
"--build",
"--basetgz",
"/var/cache/pbuilder/%s.tgz" % codename,
"--hookdir",
"debian/pbuilder-hookdir",
"--debemail",
"Kay Hayen <*****@*****.**>",
"--buildresult",
"package",
dsc_filename,
)
check_call(command, shell=False)
# Cleanup the build directory, not needed anymore.
shutil.rmtree("build", ignore_errors=True)
# Now build the repository.
my_print("Building repository ...", style="blue")
os.chdir("package")
os.makedirs("repo")
os.chdir("repo")
os.makedirs("conf")
putTextFileContents(
"conf/distributions",
contents="""\
Origin: Nuitka
Label: Nuitka
Codename: %(codename)s
Architectures: i386 amd64 armel armhf powerpc
Components: main
Description: Apt repository for project Nuitka %(codename)s
SignWith: D96ADCA1377F1CEB6B5103F11BFC33752912B99C
""" % {"codename": codename},
)
command = ["reprepro", "includedeb", codename]
command.extend(resolveShellPatternToFilenames("../*.deb"))
check_call(command, shell=False)
my_print("Uploading ...", style="blue")
# Create repo folder unless already done. This is needed for the first
# build only.
assert (os.system("ssh [email protected] mkdir -p /var/www/deb/%s/%s/" %
(category, codename)) == 0)
# Update the repository on the web site.
assert (os.system(
"rsync -avz --delete dists pool --chown www-data [email protected]:/var/www/deb/%s/%s/"
% (category, codename)) == 0)
my_print("Finished.", style="blue")
def _callDebchange(*args):
args = ["debchange"] + list(args)
os.environ["EDITOR"] = ""
check_call(args, stdin=getNullInput())
def _build(self, build_lib):
# High complexity, pylint: disable=too-many-branches,too-many-locals
# Nuitka wants the main package by filename, probably we should stop
# needing that.
old_dir = os.getcwd()
os.chdir(build_lib)
if self.distribution.package_dir and "" in self.distribution.package_dir:
main_package_dir = self.distribution.package_dir.get("")
else:
main_package_dir = os.path.abspath(old_dir)
# Search in the build directory preferably.
setMainScriptDirectory(main_package_dir)
for is_package, module_name in self._find_to_build():
module_name, main_filename, finding = locateModule(
module_name=module_name,
parent_package=None,
level=0,
)
package = module_name.getPackageName()
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
if package is not None:
output_dir = os.path.join(build_lib, package.asPath())
else:
output_dir = build_lib
command = [
sys.executable,
"-m",
"nuitka",
"--module",
"--enable-plugin=pylint-warnings",
"--output-dir=%s" % output_dir,
"--nofollow-import-to=*.tests",
"--remove-output",
]
if is_package:
command.append("--include-package=%s" % module_name)
else:
command.append("--include-module=%s" % module_name)
toml_filename = os.environ.get("NUITKA_TOML_FILE")
if toml_filename:
# Import toml parser like "build" module does.
try:
from tomli import loads as toml_loads
except ImportError:
from toml import loads as toml_loads
# Cannot use FileOperations.getFileContents() here, because of non-Nuitka process
# pylint: disable=unspecified-encoding
with open(toml_filename) as toml_file:
toml_options = toml_loads(toml_file.read())
for option, value in toml_options.get("nuitka", {}).items():
command.extend(self._parseOptionsEntry(option, value))
# Process any extra options from setuptools
if "nuitka" in self.distribution.command_options:
for option, value in self.distribution.command_options[
"nuitka"].items():
command.extend(self._parseOptionsEntry(option, value))
command.append(main_filename)
# Adding traces for clarity
wheel_logger.info(
"Building: '%s' with command %r" %
(module_name.asString(), command),
style="blue",
)
check_call(command, cwd=build_lib)
wheel_logger.info("Finished compilation of '%s'." %
module_name.asString(),
style="green")
for root, _, filenames in os.walk(build_lib):
for filename in filenames:
fullpath = os.path.join(root, filename)
if fullpath.lower().endswith(
(".py", ".pyw", ".pyc", ".pyo")):
os.unlink(fullpath)
self.build_lib = build_lib
os.chdir(old_dir)
def main():
goHome()
parser = OptionParser()
parser.add_option(
"--upload",
action="store_true",
dest="upload",
default=False,
help="""\
Upload to https://nuitka.net/apidoc requires access rights and is done by the
official servers automatically only. Without this, create the local html folder
only.
Default is %default.""",
)
options, _positional_args = parser.parse_args()
shutil.rmtree("html", ignore_errors=True)
doxygen_path = getExecutablePath("doxygen")
# Extra ball on Windows, check default installation PATH too.
if not doxygen_path and getOS() == "Windows":
with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"):
doxygen_path = getExecutablePath("doxygen")
if not doxygen_path:
sys.exit(
"Error, need to install Doxygen and add it to PATH for this to work."
)
try:
import doxypypy # pylint: disable=I0021,unused-import,unused-variable
except ImportError:
sys.exit("Error, needs to install doxypypy into this Python.")
with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file:
doxy_config = getFileContents("doc/Doxyfile.template")
with withTemporaryFile(
suffix=".bat" if getOS() == "Windows" else ".sh",
delete=False) as doxy_batch_file:
if getOS() == "Windows":
doxy_batch_file.write("%s -m doxypypy.doxypypy -a -c %%1" %
sys.executable)
else:
doxy_batch_file.write(
"#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1" %
sys.executable)
doxy_batch_filename = doxy_batch_file.name
st = os.stat(doxy_batch_filename)
os.chmod(doxy_batch_filename, st.st_mode | stat.S_IEXEC)
doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename)
doxy_file.write(doxy_config)
doxy_filename = doxy_file.name
print("Running doxygen:")
try:
check_call([doxygen_path, doxy_filename])
finally:
os.unlink(doxy_filename)
os.unlink(doxy_batch_filename)
# Update the repository on the web site.
if options.upload:
assert (os.system(
"rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/"
) == 0)
print("Finished.")
def optimize_pngs(pngList):
for png in pngList:
check_call(["optipng", "-o2", "%s.png" % png])
def createRstPDF(document, args):
check_call(["rst2pdf"] + args + [document])
def main():
# Complex stuff, not broken down yet
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
parser = OptionParser()
parser.add_option("--nuitka",
action="store",
dest="nuitka",
default=os.environ.get("NUITKA", ""))
parser.add_option(
"--cpython",
action="store",
dest="cpython",
default=os.environ.get("PYTHON", sys.executable),
)
parser.add_option("--code-diff",
action="store",
dest="diff_filename",
default="")
parser.add_option("--copy-source-to",
action="store",
dest="target_dir",
default="")
options, positional_args = parser.parse_args()
if len(positional_args) != 1:
sys.exit(
"Error, need to give test case file name as positional argument.")
test_case = positional_args[0]
if os.path.exists(test_case):
test_case = os.path.abspath(test_case)
if options.cpython == "no":
options.cpython = ""
nuitka = options.nuitka
if os.path.exists(nuitka):
nuitka = os.path.abspath(nuitka)
elif nuitka:
sys.exit("Error, nuitka binary '%s' not found." % nuitka)
python_version = setup(silent=True, go_main=False)
assert os.path.exists(test_case), (test_case, os.getcwd())
my_print("PYTHON='%s'" % python_version)
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
with open(test_case, "rb") as f:
my_print("TEST_CASE_HASH='%s'" % hashlib.md5(f.read()).hexdigest())
needs_2to3 = (python_version.startswith("3")
and not test_case.endswith("32.py")
and not test_case.endswith("33.py"))
if options.target_dir:
shutil.copyfile(
test_case,
os.path.join(options.target_dir, os.path.basename(test_case)))
# First produce two variants.
temp_dir = getTempDir()
test_case_1 = os.path.join(temp_dir,
"Variant1_" + os.path.basename(test_case))
test_case_2 = os.path.join(temp_dir,
"Variant2_" + os.path.basename(test_case))
with open(test_case) as f:
case_1_source, case_2_source = generateConstructCases(f.read())
with open(test_case_1, "w") as case_1_file:
case_1_file.write(case_1_source)
with open(test_case_2, "w") as case_2_file:
case_2_file.write(case_2_source)
if needs_2to3:
test_case_1, _needs_delete = convertUsing2to3(test_case_1)
test_case_2, _needs_delete = convertUsing2to3(test_case_2)
os.environ["PYTHONHASHSEED"] = "0"
if nuitka:
nuitka_id = check_output("cd %s; git rev-parse HEAD" %
os.path.dirname(nuitka),
shell=True)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3, ):
nuitka_id = nuitka_id.decode()
my_print("NUITKA_COMMIT='%s'" % nuitka_id)
os.chdir(getTempDir())
if nuitka:
nuitka_call = [
os.environ["PYTHON"],
nuitka,
"--python-flag=-S",
os.path.basename(test_case),
]
nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())
# We want to compile under the same filename to minimize differences, and
# then copy the resulting files afterwards.
shutil.copyfile(test_case_1, os.path.basename(test_case))
check_call(nuitka_call)
if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
exe_suffix = ".exe"
else:
exe_suffix = ".bin"
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_1).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_1).replace(".py", exe_suffix),
)
shutil.copyfile(test_case_2, os.path.basename(test_case))
check_call(nuitka_call)
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_2).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_2).replace(".py", exe_suffix),
)
if options.diff_filename:
suffixes = [".c", ".cpp"]
for suffix in suffixes:
cpp_1 = os.path.join(test_case_1.replace(".py", ".build"),
"module.__main__" + suffix)
if os.path.exists(cpp_1):
break
else:
assert False
for suffix in suffixes:
cpp_2 = os.path.join(test_case_2.replace(".py", ".build"),
"module.__main__" + suffix)
if os.path.exists(cpp_2):
break
else:
assert False
import difflib
with open(options.diff_filename, "w") as f:
with open(cpp_1) as cpp1:
with open(cpp_2) as cpp2:
f.write(difflib.HtmlDiff().make_table(
cpp1.readlines(),
cpp2.readlines(),
"Construct",
"Baseline",
True,
))
nuitka_1 = runValgrind(
"Nuitka construct",
"callgrind",
(test_case_1.replace(".py", exe_suffix), ),
include_startup=True,
)
nuitka_2 = runValgrind(
"Nuitka baseline",
"callgrind",
(test_case_2.replace(".py", exe_suffix), ),
include_startup=True,
)
nuitka_diff = nuitka_1 - nuitka_2
my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call),
file=sys.stderr)
my_print("NUITKA_RAW=%s" % nuitka_1)
my_print("NUITKA_BASE=%s" % nuitka_2)
my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)
if options.cpython:
cpython_1 = runValgrind(
"CPython construct",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_1),
include_startup=True,
)
cpython_2 = runValgrind(
"CPython baseline",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_2),
include_startup=True,
)
cpython_diff = cpython_1 - cpython_2
my_print("CPYTHON_RAW=%d" % cpython_1)
my_print("CPYTHON_BASE=%d" % cpython_2)
my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)
if options.cpython and options.nuitka:
if nuitka_diff == 0:
nuitka_gain = float("inf")
else:
nuitka_gain = float(100 * cpython_diff) / nuitka_diff
my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
