def makeSourceDirectory(main_module):
""" Get the full list of modules imported, create code for all of them.
"""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches,too-many-locals,too-many-statements
assert main_module.isCompiledPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneUserModules():
if module.getFullName() == any_case_module:
break
else:
general.warning("Not recursing to unused '%s'." % any_case_module)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=ModuleRegistry.getDoneModules())
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
try:
prepared_modules[
c_filename] = CodeGeneration.prepareModuleCode(
global_context=global_context,
module=module,
module_name=module.getFullName(),
)
except Exception:
general.warning("Problem creating code for module %r." %
module)
raise
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[c_filename][1].getConstantCode(0)
# Second pass, generate the actual module code into the files.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
template_values, module_context = prepared_modules[c_filename]
source_code = CodeGeneration.generateModuleCode(
module_context=module_context, template_values=template_values)
writeSourceCode(filename=c_filename, source_code=source_code)
if Options.isShowInclusion():
general.info("Included compiled module '%s'." %
module.getFullName())
elif module.isPythonShlibModule():
target_filename = os.path.join(
OutputDirectories.getStandaloneDirectoryPath(),
*module.getFullName().split("."))
target_filename += Utils.getSharedLibrarySuffix()
target_dir = os.path.dirname(target_filename)
if not os.path.isdir(target_dir):
makePath(target_dir)
shutil.copyfile(module.getFilename(), target_filename)
standalone_entry_points.append((
module.getFilename(),
target_filename,
module.getFullName().getPackageName(),
))
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
general.info("Included uncompiled module '%s'." %
module.getFullName())
else:
assert False, module
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=ConstantCodes.getConstantsDefinitionCode(
context=global_context),
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
ModuleRegistry.getDoneUserModules())
writeSourceCode(filename=os.path.join(source_dir, "__helpers.h"),
source_code=helper_decl_code)
writeSourceCode(filename=os.path.join(source_dir, "__helpers.c"),
source_code=helper_impl_code)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(filename=os.path.join(target_dir, filename),
source_code=source_code)
def compileTree():
source_dir = OutputDirectories.getSourceDirectoryPath()
general.info("Completed Python level compilation and optimization.")
if not Options.shallOnlyExecCCompilerCall():
general.info("Generating source code for C backend compiler.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before generating C code: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
# Now build the target language code for the whole tree.
makeSourceDirectory()
bytecode_accessor = ConstantAccessor(
data_filename="__bytecode.const", top_level_name="bytecode_data"
)
# This should take all bytecode values, even ones needed for frozen or
# not produce anything.
loader_code = LoaderCodes.getMetapathLoaderBodyCode(bytecode_accessor)
writeSourceCode(
filename=os.path.join(source_dir, "__loader.c"), source_code=loader_code
)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
general.sysexit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.is_debug:
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
general.info("Running data composer tool for optimal constant value handling.")
runDataComposer(source_dir)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(
filename=os.path.join(target_dir, filename), source_code=source_code
)
general.info("Running C compilation via Scons.")
# Run the Scons to build things.
result, options = runSconsBackend(quiet=not Options.isShowScons())
return result, options
def makeSourceDirectory():
"""Get the full list of modules imported, create code for all of them."""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneModules():
if module.getFullName() == any_case_module:
break
else:
general.warning("Not recursing to unused '%s'." % any_case_module)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=ModuleRegistry.getDoneModules())
# Generate code for modules.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
source_code = CodeGeneration.generateModuleCode(
module=module,
data_filename=os.path.basename(c_filename +
"onst"), # Really .const
)
writeSourceCode(filename=c_filename, source_code=source_code)
if Options.isShowInclusion():
inclusion_logger.info("Included compiled module '%s'." %
module.getFullName())
elif module.isPythonShlibModule():
addShlibEntryPoint(module)
if Options.isShowInclusion():
inclusion_logger.info("Included extension module '%s'." %
module.getFullName())
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
inclusion_logger.info("Included uncompiled module '%s'." %
module.getFullName())
else:
assert False, module
(
helper_decl_code,
helper_impl_code,
constants_header_code,
constants_body_code,
) = CodeGeneration.generateHelpersCode()
writeSourceCode(filename=os.path.join(source_dir, "__helpers.h"),
source_code=helper_decl_code)
writeSourceCode(filename=os.path.join(source_dir, "__helpers.c"),
source_code=helper_impl_code)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.h"),
source_code=constants_header_code,
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=constants_body_code,
)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(filename=os.path.join(target_dir, filename),
source_code=source_code)
