def compileTree():
source_dir = OutputDirectories.getSourceDirectoryPath()
general.info("Completed Python level compilation and optimization.")
if not Options.shallOnlyExecCCompilerCall():
general.info("Generating source code for C backend compiler.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before generating C code: {memory}:".
format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()))
# Now build the target language code for the whole tree.
makeSourceDirectory()
bytecode_accessor = ConstantAccessor(data_filename="__bytecode.const",
top_level_name="bytecode_data")
# This should take all bytecode values, even ones needed for frozen or
# not produce anything.
loader_code = LoaderCodes.getMetapathLoaderBodyCode(bytecode_accessor)
writeSourceCode(filename=os.path.join(source_dir, "__loader.c"),
source_code=loader_code)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
general.sysexit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()))
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.is_debug:
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
general.info(
"Running data composer tool for optimal constant value handling.")
# TODO: On Windows, we could run this in parallel to Scons, on Linux we need it
# for linking.
runDataComposer(source_dir)
general.info("Running C level backend compilation via Scons.")
# Run the Scons to build things.
result, options = runSconsBackend(quiet=not Options.isShowScons())
return result, options
def compileTree(main_module):
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
if Options.isShowProgress() or Options.isShowMemory():
info(
"Total memory usage before generating C code: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
# Now build the target language code for the whole tree.
makeSourceDirectory(main_module=main_module)
frozen_code = generateBytecodeFrozenCode()
if frozen_code is not None:
writeSourceCode(
filename=os.path.join(source_dir, "__frozen.c"), source_code=frozen_code
)
if not isWin32Windows():
writeBinaryData(
filename=os.path.join(source_dir, "__constants.bin"),
binary_data=ConstantCodes.stream_data.getBytes(),
)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
sys.exit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.isDebug():
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
# Run the Scons to build things.
result, options = runScons(main_module=main_module, quiet=not Options.isShowScons())
return result, options
def _traceProgress(current_module):
output = """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that.""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
)
if Options.isShowMemory():
output += "Memory usage {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage())
info(output)
def _traceProgress(current_module):
output = """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that.""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
)
if Options.isShowMemory():
output += "Memory usage {memory}:".format(
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
printLine(output)
def compileTree(main_module):
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
# Now build the target language code for the whole tree.
makeSourceDirectory(
main_module = main_module
)
frozen_code = generateBytecodeFrozenCode()
if frozen_code is not None:
writeSourceCode(
filename = Utils.joinpath(
source_dir,
"__frozen.c"
),
source_code = frozen_code
)
writeBinaryData(
filename = Utils.joinpath(source_dir, "__constants.bin"),
binary_data = ConstantCodes.stream_data.getBytes()
)
else:
source_dir = getSourceDirectoryPath(main_module)
if not Utils.isFile(Utils.joinpath(source_dir, "__helpers.h")):
sys.exit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
Tracing.printLine(
"Total memory usage before running scons: {memory}:".format(
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.shallNotDoExecCppCall():
return True, {}
# Run the Scons to build things.
result, options = runScons(
main_module = main_module,
quiet = not Options.isShowScons()
)
return result, options
def compileTree(main_module):
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
# Now build the target language code for the whole tree.
makeSourceDirectory(
main_module = main_module
)
frozen_code = generateBytecodeFrozenCode()
if frozen_code is not None:
writeSourceCode(
filename = Utils.joinpath(
source_dir,
"__frozen.cpp"
),
source_code = frozen_code
)
writeBinaryData(
filename = Utils.joinpath(source_dir, "__constants.bin"),
binary_data = ConstantCodes.stream_data.getBytes()
)
else:
source_dir = getSourceDirectoryPath(main_module)
if not Utils.isFile(Utils.joinpath(source_dir, "__helpers.hpp")):
sys.exit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
Tracing.printLine(
"Total memory usage before running scons: {memory}:".format(
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.shallNotDoExecCppCall():
return True, {}
# Run the Scons to build things.
result, options = runScons(
main_module = main_module,
quiet = not Options.isShowScons()
)
return result, options
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if VariableRegistry.considerCompletion():
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection=function.constraint_collection,
new_collection=None)
function.constraint_collection = None
if VariableRegistry.considerCompletion():
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)