def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
# Many cases to deal with, pylint: disable=too-many-branches
if isWin32Windows():
sys.exit("Error, valgrind is not available on Windows.")
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ("--tool=callgrind",
"--callgrind-out-file=%s" % log_filename)
elif tool == "massif":
command += ("--tool=massif", "--massif-out-file=%s" % log_filename)
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += (
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
)
command.extend(args)
_stdout_valgrind, stderr_valgrind, exit_valgrind = executeProcess(
command)
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
copyFile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def _getCPythonResults(cpython_cmd, send_kill):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(args=cpython_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if send_kill:
# Doing it per loop iteration hopefully, pylint: disable=cell-var-from-loop
executeAfterTimePassed(
1.0,
lambda: killProcess("Uncompiled Python program", process.pid))
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def autoformat(filename, abort=False):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
my_print("Consider", filename, end=": ")
old_code = open(filename, "r").read()
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
new_name = filename + ".new"
with open(new_name, "w") as source_code:
source_code.write(red.dumps())
if os.name == "nt":
cleanupWindowsNewlines(new_name)
# There is no way to safely replace a file on Windows, but lets try on Linux
# at least.
old_stat = os.stat(filename)
try:
os.rename(new_name, filename)
except OSError:
shutil.copyfile(new_name, filename)
os.unlink(new_name)
os.chmod(filename, old_stat.st_mode)
my_print("updated.")
changed = 1
else:
my_print("OK.")
changed = 0
return changed
def subprocess_spawn(args):
sh, _cmd, args, env = args
_stdout, stderr, exit_code = executeProcess(
command=[sh, "-c", " ".join(args)], env=env)
ignore_next = False
for line in stderr.splitlines():
if ignore_next:
ignore_next = False
continue
if isIgnoredError(line):
ignore_next = True
continue
if str is not bytes:
line = decodeData(line)
if exit_code != 0 and "terminated with signal 11" in line:
exit_code = -11
my_print(line, style="yellow", file=sys.stderr)
return exit_code
def show(stats):
p = stats.top_profile()
if not p:
my_print("no stats")
return
p.sort(key=lambda x: x[1], reverse=True)
top = p[0][1]
max_len = max([_namelen(e[0]) for e in p])
my_print(" vmprof output:")
my_print(" %: name:" + " " * (max_len - 3) + "location:")
for k, v in p:
v = "%.1f%%" % (float(v) / top * 100)
if v == "0.0%":
v = "<0.1%"
if k.startswith("py:"):
_, func_name, lineno, filename = k.split(":", 3)
lineno = int(lineno)
my_print(
" %s %s %s:%d"
% (v.ljust(7), func_name.ljust(max_len + 1), filename, lineno)
)
else:
my_print(" %s %s" % (v.ljust(7), k.ljust(max_len + 1)))
def main():
module_name = ModuleName(sys.argv[1])
setMainScriptDirectory(os.getcwd())
my_print(" ".join(
locateModule(module_name=module_name, parent_package=None, level=0)))
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Default is %default.""",
)
parser.add_option(
"--from-commit",
action="store_true",
dest="from_commit",
default=False,
help=
"""From commit hook, do not descend into directories. Default is %default.""",
)
parser.add_option(
"--abort-on-parsing-error",
action="store_true",
dest="abort",
default=False,
help="""Stop if an error occurs, or continue. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.from_commit:
assert not positional_args
for desc in getStagedFileChangeDesc():
autoformat(desc["src_path"], git_stage=desc, abort=options.abort)
else:
if not positional_args:
positional_args = [
"bin", "nuitka", "setup.py", "tests/*/run_all.py"
]
my_print("Working on:", positional_args)
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
goHome()
filenames = list(
scanTargets(
positional_args,
(".py", ".scons", ".rst", ".txt", ".j2", ".md", ".c", ".h"),
))
if not filenames:
sys.exit("No files found.")
for filename in filenames:
autoformat(filename, git_stage=False, abort=options.abort)
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka
)
)
return exit_code_stdout, exit_code_stderr, exit_code_return
def _cleanupPyLintComments(filename, abort):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
old_code = getFileContents(filename)
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Show what it is doing. Default is %default.""",
)
parser.add_option(
"--write",
"-w",
action="store_true",
dest="write",
default=False,
help="""Write changes to the files. Default is %default.""",
)
options, positional_args = parser.parse_args()
if not positional_args:
positional_args = [
"bin",
"nuitka",
"rpm",
"misc",
"tests/*/run_all.py",
"*.rst",
]
goHome()
my_print("Working on:", positional_args)
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
filenames = list(
scanTargets(
positional_args,
suffixes=(".py", ".scons", ".rst", ".txt", ".j2", ".md", ".c",
".h"),
))
if not filenames:
sys.exit("No files found.")
result = runCodespell(filenames=filenames,
verbose=options.verbose,
write=options.write)
if result:
my_print("OK.")
else:
sys.exit(
"\nError, please correct the spelling problems found or extend 'misc/codespell-ignore.txt' if applicable."
)
def _cleanupPyLintComments(filename, abort):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
old_code = getFileContents(filename)
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def setup(suite="", needs_io_encoding=False, silent=False, go_main=True):
if go_main:
goMainDir()
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
# Allow test code to use this to make caching specific.
os.environ["NUITKA_TEST_SUITE"] = suite
# Allow providing 33, 27, and expand that to python2.7
if (len(os.environ["PYTHON"]) == 2 and os.environ["PYTHON"].isdigit()
and os.name != "nt"):
os.environ["PYTHON"] = "python%s.%s" % (
os.environ["PYTHON"][0],
os.environ["PYTHON"][1],
)
if needs_io_encoding and "PYTHONIOENCODING" not in os.environ:
os.environ["PYTHONIOENCODING"] = "utf-8"
version_output = check_output(
(
os.environ["PYTHON"],
"-c",
"""\
import sys, os;\
print(".".join(str(s) for s in list(sys.version_info)[:3]));\
print(("x86_64" if "AMD64" in sys.version else "x86") if os.name == "nt" else os.uname()[4]);\
print(sys.executable);\
""",
),
stderr=subprocess.STDOUT,
)
global _python_version, _python_arch, _python_executable # singleton, pylint: disable=global-statement
_python_version = version_output.split(b"\n")[0].strip()
_python_arch = version_output.split(b"\n")[1].strip()
_python_executable = version_output.split(b"\n")[2].strip()
if sys.version.startswith("3"):
_python_arch = _python_arch.decode("utf-8")
_python_version = _python_version.decode("utf-8")
_python_executable = _python_executable.decode("utf-8")
if not silent:
my_print("Using concrete python", _python_version, "on", _python_arch)
assert type(_python_version) is str, repr(_python_version)
assert type(_python_arch) is str, repr(_python_arch)
assert type(_python_executable) is str, repr(_python_executable)
if "COVERAGE_FILE" not in os.environ:
os.environ["COVERAGE_FILE"] = os.path.join(os.path.dirname(__file__),
"..", "..", "..",
".coverage")
return _python_version
def compareOutput(
kind, out_cpython, out_nuitka, ignore_warnings, ignore_infos, syntax_errors
):
fromdate = ""
todate = ""
diff = difflib.unified_diff(
makeDiffable(out_cpython, ignore_warnings, ignore_infos, syntax_errors),
makeDiffable(out_nuitka, ignore_warnings, ignore_infos, syntax_errors),
"{program} ({detail})".format(program=os.environ["PYTHON"], detail=kind),
"{program} ({detail})".format(program="nuitka", detail=kind),
fromdate,
todate,
n=3,
)
result = list(diff)
if result:
for line in result:
my_print(line, end="\n" if not line.startswith("---") else "")
return 1
else:
return 0
def _getCPythonResults(cpython_cmd):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(
args=cpython_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython
):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def _findModule(module_name):
# Not a good module name. TODO: Push this to ModuleName() creation maybe.
assert module_name != ""
if _debug_module_finding:
my_print("_findModule: Enter to search '%s'." % (module_name,))
assert module_name.getBasename(), module_name
key = module_name
if key in module_search_cache:
result = module_search_cache[key]
if _debug_module_finding:
my_print("_findModule: Cached result (see previous call).")
if result is ImportError:
raise ImportError
return result
module_search_cache[key] = _findModuleInPath(module_name)
return module_search_cache[key]
def _cleanupPyLintComments(filename, abort):
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, )
old_code = getFileContents(filename)
# Baron does assertions too, and all kinds of strange errors, pylint: disable=broad-except
try:
red = RedBaron(old_code)
except Exception:
if abort:
raise
return
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def compileLibraryPath(search_mode, path, stage_dir, decide, action):
my_print("Checking standard library path:", path)
for root, dirnames, filenames in os.walk(path):
dirnames_to_remove = [
dirname for dirname in dirnames if '-' in dirname
]
for dirname in dirnames_to_remove:
dirnames.remove(dirname)
dirnames.sort()
filenames = [
filename for filename in filenames if decide(root, filename)
]
for filename in sorted(filenames):
if not search_mode.consider(root, filename):
continue
full_path = os.path.join(root, filename)
my_print(full_path, ':', end=' ')
sys.stdout.flush()
action(stage_dir, path, full_path)
def myDetectVersion(env, cc):
"""Return the version of the GNU compiler, or None if it is not a GNU compiler."""
cc = env.subst(cc)
if not cc:
return None
if "++" in os.path.basename(cc):
return None
version = None
clvar = tuple(SCons.Util.CLVar(cc))
if clvar in v_cache:
return v_cache[clvar]
clvar0 = os.path.basename(clvar[0])
if isGccName(clvar0) or "clang" in clvar0:
command = list(clvar) + ["-dumpversion"]
else:
command = list(clvar) + ["--version"]
# pipe = SCons.Action._subproc(env, SCons.Util.CLVar(cc) + ['-dumpversion'],
pipe = SCons.Action._subproc( # pylint: disable=protected-access
env,
command,
stdin="devnull",
stderr="devnull",
stdout=subprocess.PIPE)
line = pipe.stdout.readline()
# Non-GNU compiler's output (like AIX xlc's) may exceed the stdout buffer:
# So continue with reading to let the child process actually terminate.
while pipe.stdout.readline():
pass
ret = pipe.wait()
if ret != 0:
return None
if str is not bytes and type(line) is bytes:
line = decodeData(line)
line = line.strip()
match = re.findall(r"[0-9]+(?:\.[0-9]+)+", line)
if match:
version = match[0]
else:
# gcc 8 or higher
version = line.strip()
version = tuple(int(part) for part in version.split("."))
if show_scons_mode:
my_print("Scons: CC %r version check gives %r from %r" %
(cc, version, line))
v_cache[clvar] = version
return version
def compareOutput(
kind, out_cpython, out_nuitka, ignore_warnings, ignore_infos, syntax_errors
):
fromdate = ""
todate = ""
diff = difflib.unified_diff(
makeDiffable(out_cpython, ignore_warnings, ignore_infos, syntax_errors),
makeDiffable(out_nuitka, ignore_warnings, ignore_infos, syntax_errors),
"{program} ({detail})".format(program=os.environ["PYTHON"], detail=kind),
"{program} ({detail})".format(program="nuitka", detail=kind),
fromdate,
todate,
n=3,
)
result = list(diff)
if result:
for line in result:
my_print(line, end="\n" if not line.startswith("---") else "")
return 1
else:
return 0
def main():
parser = OptionParser()
_options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["nuitka/plugins/standard/*.yml"]
my_print("Working on:", positional_args)
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
goHome()
filenames = list(scanTargets(
positional_args,
suffixes=(".yaml", ),
))
if not filenames:
sys.exit("No files found.")
for filename in filenames:
checkYamllint(filename)
def wrapExpressionWithSideEffects(side_effects, old_node, new_node):
assert new_node.isExpression()
from .SideEffectNodes import ExpressionSideEffects
if side_effects:
try:
side_effects = sum(
(
side_effect.extractSideEffects()
for side_effect in side_effects
if side_effect.mayHaveSideEffects()
),
(),
)
except AttributeError:
my_print("Problem with side effects:", side_effects)
raise
if side_effects:
new_node = ExpressionSideEffects(
expression=new_node,
side_effects=side_effects,
source_ref=old_node.getSourceReference(),
)
return new_node
def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ("--tool=callgrind",
"--callgrind-out-file=%s" % log_filename)
elif tool == "massif":
command += ("--tool=massif", "--massif-out-file=%s" % log_filename)
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += (
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
)
command.extend(args)
process = subprocess.Popen(args=command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_stdout_valgrind, stderr_valgrind = process.communicate()
exit_valgrind = process.returncode
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
shutil.copyfile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def displayFileContents(name, path):
test_logger.info("Contents of %s %r:" % (name, path))
if os.path.exists(path):
for line in getFileContentByLine(path):
my_print(line)
else:
test_logger.info("Does not exist.")
def displayOutput(stdout, stderr):
if type(stdout) is not str:
stdout = stdout.decode("utf-8" if os.name != "nt" else "cp850")
stderr = stderr.decode("utf-8" if os.name != "nt" else "cp850")
my_print(stdout, end=" ")
if stderr:
my_print(stderr)
def getCPythonResults(cpython_cmd, cpython_cached):
cached = False
if cpython_cached:
# TODO: Hashing stuff and creating cache filename is duplicate code
# and should be shared.
hash_input = " -- ".join(cpython_cmd)
if str is not bytes:
hash_input = hash_input.encode("utf8")
command_hash = hashlib.md5(hash_input)
cache_filename = os.path.join(
getTestingCPythonOutputsCacheDir(),
command_hash.hexdigest()
)
if os.path.exists(cache_filename):
with open(cache_filename, "rb") as cache_file:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = \
pickle.load(cache_file)
cached = True
if not cached:
start_time = time.time()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(
args = cpython_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
if checkNoPermissionError(stdout_cpython) and \
checkNoPermissionError(stderr_cpython):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
start_time = time.time()
cpython_time = time.time() - start_time
if cpython_cached:
with open(cache_filename, "wb") as cache_file:
pickle.dump(
(cpython_time, stdout_cpython, stderr_cpython, exit_cpython),
cache_file
)
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ["--tool=callgrind", "--callgrind-out-file=%s" % log_filename]
elif tool == "massif":
command += ["--tool=massif", "--massif-out-file=%s" % log_filename]
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += [
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
]
command.extend(args)
process = subprocess.Popen(
args=command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
_stdout_valgrind, stderr_valgrind = process.communicate()
exit_valgrind = process.returncode
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
shutil.copyfile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def checkDebugPython():
if not hasattr(sys, "gettotalrefcount"):
my_print("Warning, using non-debug Python makes this test ineffective.")
sys.gettotalrefcount = lambda: 0
elif sys.version_info >= (3, 7, 0) and sys.version_info < (3, 7, 1):
my_print(
"Warning, bug of CPython 3.7.0/1 breaks reference counting and makes this test ineffective."
)
sys.gettotalrefcount = lambda: 0
def main():
parser = OptionParser()
parser.add_option(
"--checked-module",
action="store",
dest="checked_module",
help="""\
Module with main() function to be checked for reference count stability.""",
)
parser.add_option(
"--explain",
action="store_true",
dest="explain",
default=False,
help="""\
Try to explain the differences by comparing object counts.""",
)
options, positional_args = parser.parse_args()
if positional_args and options.checked_module is None:
options.checked_module = positional_args.pop()
if positional_args and options.checked_module:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
# First with pure Python.
checked_module = importFileAsModule(options.checked_module)
my_print("Using %s" % checked_module.main, style="blue")
checkReferenceCount(checked_module.main, explain=options.explain)
temp_dir = getTempDir()
command = [
sys.executable,
"-m",
"nuitka",
"--module",
options.checked_module,
"--output-dir=%s" % temp_dir,
]
if hasattr(sys, "gettotalrefcount"):
command.append("--python-debug")
check_call(command)
module_name = os.path.basename(options.checked_module).split(".")[0]
sys.path.insert(0, temp_dir)
checked_module = __import__(module_name)
my_print("Using %s" % checked_module.main, style="blue")
checkReferenceCount(checked_module.main)
def setup(needs_io_encoding=False, silent=False):
# Go its own directory, to have it easy with path knowledge.
os.chdir(os.path.dirname(os.path.abspath(
sys.modules["__main__"].__file__)))
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
# Allow providing 33, 27, and expand that to python2.7
if len(os.environ["PYTHON"]) == 2 and \
os.environ["PYTHON"].isdigit() and \
os.name != "nt":
os.environ["PYTHON"] = "python%s.%s" % (os.environ["PYTHON"][0],
os.environ["PYTHON"][1])
if needs_io_encoding and "PYTHONIOENCODING" not in os.environ:
os.environ["PYTHONIOENCODING"] = "utf-8"
version_output = check_output((
os.environ["PYTHON"],
"-c",
"""\
import sys, os;\
print(".".join(str(s) for s in list(sys.version_info)[:3]));\
print(("x86_64" if "AMD64" in sys.version else "x86") if os.name == "nt" else os.uname()[4]);\
print(sys.executable);\
""",
),
stderr=subprocess.STDOUT)
global _python_version, _python_arch, _python_executable # singleton, pylint: disable=global-statement
_python_version = version_output.split(b"\n")[0].strip()
_python_arch = version_output.split(b"\n")[1].strip()
_python_executable = version_output.split(b"\n")[2].strip()
if sys.version.startswith('3'):
_python_arch = _python_arch.decode("utf-8")
_python_version = _python_version.decode("utf-8")
_python_executable = _python_executable.decode("utf-8")
if not silent:
my_print("Using concrete python", _python_version, "on", _python_arch)
assert type(_python_version) is str, repr(_python_version)
assert type(_python_arch) is str, repr(_python_arch)
assert type(_python_executable) is str, repr(_python_executable)
if "COVERAGE_FILE" not in os.environ:
os.environ["COVERAGE_FILE"] = os.path.join(os.path.dirname(__file__),
"..", "..", "..",
".coverage")
return _python_version
def main():
parser = OptionParser()
parser.add_option(
"--mode",
action="store",
dest="mode",
default=None,
help="""\
The mode of update, prerelease, hotfix, release, auto (default auto determines from branch).""",
)
options, positional_args = parser.parse_args()
if positional_args:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
# Go its own directory, to have it easy with path knowledge.
goHome()
with openTextFile("nuitka/Version.py", "r") as f:
option_lines = f.readlines()
(version_line,) = [line for line in option_lines if line.startswith("Nuitka V")]
old_version = version_line[8:].rstrip()
mode = options.mode
branch_name = getBranchName()
if mode is None:
if branch_name.startswith("hotfix/"):
mode = "hotfix"
elif branch_name == "main" or branch_name.startswith("release/"):
mode = "release"
elif branch_name == "develop":
mode = "prerelease"
else:
sys.exit("Error, cannot detect mode from branch name '%s'." % branch_name)
new_version = getBumpedVersion(mode, old_version)
my_print("Bumped %s '%s' -> '%s'." % (mode, old_version, new_version))
with openTextFile("nuitka/Version.py", "w") as options_file:
for line in option_lines:
if line.startswith("Nuitka V"):
line = "Nuitka V" + new_version + "\n"
options_file.write(line)
# Debian is currently in not freeze, change to "experimental" once that changes.
updateDebianChangelog(old_version, new_version, "unstable")
def compareWithCPython(dirname, filename, extra_flags, search_mode,
needs_2to3):
""" Call the comparison tool. For a given directory filename.
The search mode decides if the test case aborts on error or gets extra
flags that are exceptions.
"""
if dirname is None:
path = filename
else:
path = os.path.join(dirname, filename)
# Apply 2to3 conversion if necessary.
if needs_2to3:
path, converted = convertUsing2to3(path)
else:
converted = False
command = [
sys.executable,
os.path.join("..", "..", "bin", "compare_with_cpython"),
path,
"silent",
]
if extra_flags is not None:
command += extra_flags
command += search_mode.getExtraFlags(dirname, filename)
# Cleanup before and after test stage directory.
_removeCPythonTestSuiteDir()
try:
result = subprocess.call(command)
except KeyboardInterrupt:
result = 2
# Cleanup before and after test stage directory.
_removeCPythonTestSuiteDir()
if result != 0 and result != 2 and search_mode.abortOnFinding(
dirname, filename):
my_print("Error exit!", result)
sys.exit(result)
if converted:
os.unlink(path)
if result == 2:
sys.stderr.write("Interrupted, with CTRL-C\n")
sys.exit(2)
def main():
branch_name = checkBranchName()
assert branch_name == "main"
createReleaseDocumentation()
assert os.system("python setup.py sdist --formats=gztar") == 0
os.chdir("dist")
# Clean the stage for the debian package. The name "deb_dist" is what "py2dsc"
# uses for its output later on.
shutil.rmtree("deb_dist", ignore_errors=True)
# Provide a re-packed tar.gz for the Debian package as input.
# Create it as a "+ds" file, removing:
# - the benchmarks (too many sources, not useful to end users, potential license
# issues)
# - the inline copy of scons (not wanted for Debian)
# Then run "py2dsc" on it.
for filename in os.listdir("."):
if filename.endswith(".tar.gz"):
new_name = filename[:-7] + "+ds.tar.gz"
cleanupTarfileForDebian(filename, new_name)
entry = runPy2dsc(filename, new_name)
break
else:
assert False
os.chdir("deb_dist")
os.chdir(entry)
# Build the debian package, but disable the running of tests, will be done later
# in the pbuilder test steps.
assert os.system("debuild --set-envvar=DEB_BUILD_OPTIONS=nocheck") == 0
os.chdir("../../..")
assert os.path.exists("dist/deb_dist")
# Cleanup the build directory, not needed anymore.
shutil.rmtree("build", ignore_errors=True)
my_print("Uploading...", style="blue")
os.chdir("dist/deb_dist")
assert os.system("dput mentors *.changes") == 0
my_print("Finished.", style="blue")
def main():
msi_filename = createMSIPackage()
assert (subprocess.call(
(
"scp",
msi_filename,
"[email protected]:/var/www/releases/" +
os.path.basename(msi_filename),
),
shell=True, # scan scp in PATH.
) == 0)
my_print("OK, uploaded '%s'." % msi_filename, style="blue")
def main():
if os.name != "nt":
sys.exit("Error, this is only for use on Windows where SxS exists.")
setup(needs_io_encoding=True)
search_mode = createSearchMode()
tmp_dir = tempfile.gettempdir()
compileLibraryTest(
search_mode=search_mode,
stage_dir=os.path.join(tmp_dir, "find_sxs_modules"),
decide=decide,
action=action,
)
my_print("FINISHED, all extension modules checked.")
def action(stage_dir, root, path):
# We need only the actual path, pylint: disable=unused-argument
sxs = getSxsFromDLL(path)
if sxs:
my_print(path, sxs)
def autoformat(filename, git_stage, abort):
# This does a lot of distinctions, pylint:disable=too-many-branches
filename = os.path.normpath(filename)
my_print("Consider", filename, end=": ")
is_python = _isPythonFile(filename)
is_c = filename.endswith((".c", ".h"))
# Some parts of Nuitka must not be re-formatted with black or clang-format
# as they have different intentions.
if _shouldNotFormatCode(filename):
is_python = is_c = False
# Work on a temporary copy
tmp_filename = filename + ".tmp"
if git_stage:
old_code = getFileHashContent(git_stage["dst_hash"])
else:
old_code = getFileContents(filename)
with open(tmp_filename, "wb") as output_file:
output_file.write(old_code)
try:
_cleanupWindowsNewlines(tmp_filename)
if is_python:
_cleanupPyLintComments(tmp_filename, abort)
_cleanupImportSortOrder(tmp_filename)
if is_python:
black_call = _getPythonBinaryCall("black")
subprocess.call(black_call + ["-q", tmp_filename])
elif is_c:
_cleanupClangFormat(filename)
else:
_cleanupTrailingWhitespace(tmp_filename)
_cleanupWindowsNewlines(tmp_filename)
changed = False
if old_code != getFileContents(tmp_filename):
my_print("Updated.")
if git_stage:
new_hash_value = putFileHashContent(tmp_filename)
updateFileIndex(git_stage, new_hash_value)
updateWorkingFile(filename, git_stage["dst_hash"], new_hash_value)
else:
renameFile(tmp_filename, filename)
changed = True
else:
my_print("OK.")
return changed
finally:
if os.path.exists(tmp_filename):
os.unlink(tmp_filename)
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Default is %default.""",
)
parser.add_option(
"--from-commit",
action="store_true",
dest="from_commit",
default=False,
help="""From commit hook, do not descend into directories. Default is %default.""",
)
parser.add_option(
"--abort-on-parsing-error",
action="store_true",
dest="abort",
default=False,
help="""Stop if an error occurs, or continue. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.from_commit:
assert not positional_args
for desc in getStagedFileChangeDesc():
autoformat(desc["src_path"], git_stage=desc, abort=options.abort)
else:
if not positional_args:
positional_args = [
"bin",
"nuitka",
# "tests/*/run_all.py"
]
my_print("Working on:", positional_args)
positional_args = sum(
(
resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args
),
[],
)
goHome()
filenames = list(
scanTargets(positional_args, (".py", ".scons", ".rst", ".txt"))
)
if not filenames:
sys.exit("No files found.")
for filename in filenames:
autoformat(filename, git_stage=False, abort=options.abort)
