def add_dependency_and_wait(module_name: str):
if Path(CompilationManager.path_from_mod_name(module_name)).exists():
if module_name not in ParserState.module().dependencies:
ParserState.module().dependencies.append(module_name)
CompilationManager.request_module(module_name)
CompilationManager.wait_for_module_compiled(module_name)
return True
return False
def check_cache(path: str) -> bool:
if CompilationManager.config.raw_opts.disable_cache:
return False
module = Cache.get_cached_module(path)
if module is None:
return False
dependency_invalid = False
request_recompilation = list()
with CompilationManager.compiled_lock:
for dependency in module.dependencies:
# If dependency has been compiled
if CompilationManager.check_module_already_compiled(dependency):
# But not cached, then it's invalid and we need to recompile
if not dependency in CompilationManager.cached_modules:
dependency_invalid = True
# If it has not been compiled then we request it
else:
request_recompilation.append(dependency)
for dependency in request_recompilation:
CompilationManager.request_module(dependency)
for dependency in request_recompilation:
CompilationManager.wait_for_module_compiled(dependency)
if dependency_invalid:
return False
# Check cache again
if len(request_recompilation) > 0:
return check_cache(path)
CompilationManager.modules[path] = module
for struct in module.structs:
context.global_context.get_identified_type(struct.name)
context.global_context.identified_types[struct.name] = struct
for key in module.builtin_type_methods.keys():
if not key in ParserState.builtin_types:
ParserState.builtin_types[key] = dict()
for fun in module.builtin_type_methods[key]:
ParserState.builtin_types[key][fun] = next(
(func for func in module.functions if func.name == func), None)
with CompilationManager.compiled_lock:
CompilationManager.cached_modules.append(module.name)
return True
def save_module(self, module: RIALModule, path: str):
from rial.compilation_manager import CompilationManager
with run_with_profiling(CompilationManager.filename_from_path(path),
ExecutionStep.WRITE_CACHE):
self._check_dirs_exist(path)
with open(path, "wb") as file:
pickle.dump(module, file)
def load_module(self, path: str) -> Optional[RIALModule]:
from rial.compilation_manager import CompilationManager
with run_with_profiling(CompilationManager.filename_from_path(path),
ExecutionStep.READ_CACHE):
try:
with open(path, "rb") as file:
return pickle.load(file)
except Exception:
return None
def imported(self, nodes):
mutability = nodes[0]
if mutability != VariableMutabilityModifier.CONST:
raise PermissionError(
"Cannot import modules as anything but const variables")
var_name = nodes[1].value
if var_name in self.module.dependencies:
raise NameError(var_name)
mod_name = ':'.join([node.value for node in nodes[3:]])
if mod_name.startswith("core") or mod_name.startswith(
"std") or mod_name.startswith("startup"):
mod_name = f"rial:{mod_name}"
CompilationManager.request_module(mod_name)
self.module.dependencies[var_name] = mod_name
raise Discard()
def get_definition(self, identifier: str):
variable = self.get_exact_definition(identifier)
# Search with module specifier
if variable is None:
if ':' in identifier:
parts = identifier.split(':')
module_name = ':'.join(parts[0:-1])
if CompilationManager.check_module_already_compiled(module_name):
return None
if ParserState.add_dependency_and_wait(module_name):
return self.get_definition(identifier)
return None
return variable
def find_global(name: str) -> Optional[RIALVariable]:
# Search with just its name
glob: RIALVariable = ParserState.module().get_rial_variable(name)
# Search with module specifier
if glob is None:
glob = ParserState.module().get_rial_variable(
mangle_global_name(ParserState.module().name, name))
# Go through usings to find it
if glob is None:
globs_found: List[Tuple] = list()
for using in ParserState.module().dependencies:
path = CompilationManager.path_from_mod_name(using)
if path not in CompilationManager.modules:
continue
module = CompilationManager.modules[path]
gl = module.get_rial_variable(name)
if gl is None:
gl = module.get_rial_variable(
mangle_global_name(using, name))
if gl is not None:
globs_found.append((using, gl))
if len(globs_found) == 0:
return None
if len(globs_found) > 1:
raise KeyError(name)
glob = globs_found[0][1]
if glob.access_modifier == RIALAccessModifier.PRIVATE:
raise PermissionError(name)
if glob.access_modifier == RIALAccessModifier.INTERNAL and glob.backing_value.parent.split(':')[0] != \
ParserState.module().name.split(':')[
0]:
raise PermissionError(name)
return glob
def main(options):
start = timer()
# Monkey patch functions in
monkey_patch()
init()
if options.profile_mem:
import tracemalloc
tracemalloc.start()
start_snapshot = tracemalloc.take_snapshot()
set_profiling(options.profile)
self_dir = Path(rreplace(__file__.replace("main.py", ""), "/rial", "",
1)).joinpath("std")
project_path = str(os.path.abspath(options.workdir))
project_name = project_path.split('/')[-1]
source_path = Path(os.path.join(project_path, "src"))
cache_path = Path(os.path.join(project_path, "cache"))
output_path = Path(os.path.join(project_path, "output"))
bin_path = Path(os.path.join(project_path, "bin"))
cache_path.mkdir(parents=False, exist_ok=True)
if output_path.exists():
shutil.rmtree(output_path)
output_path.mkdir(parents=False, exist_ok=False)
if bin_path.exists():
shutil.rmtree(bin_path)
bin_path.mkdir(parents=False, exist_ok=False)
if not source_path.exists():
raise FileNotFoundError(str(source_path))
config = Configuration(project_name, source_path, cache_path, output_path,
bin_path, self_dir, options)
CompilationManager.init(config)
CompilationManager.compiler()
CompilationManager.fini()
end = timer()
if options.profile:
print("----- PROFILING -----")
total = 0
for event in execution_events:
print(event)
total += event.time_taken_seconds
print(f"TOTAL : {(end - start).__round__(3)}s")
print("")
if options.profile_mem:
import tracemalloc
end_snapshot = tracemalloc.take_snapshot()
display_top(end_snapshot)
def find_function(
full_function_name: str,
rial_arg_types: List[str] = None) -> Optional[RIALFunction]:
# Check by canonical name if we got args to check
if rial_arg_types is not None:
func = ParserState.module().get_function(full_function_name)
# If couldn't find it, iterate through usings and try to find function
if func is None:
functions_found: List[Tuple[str, RIALFunction]] = list()
for use in ParserState.module().dependencies:
path = CompilationManager.path_from_mod_name(use)
if path not in CompilationManager.modules:
continue
module = CompilationManager.modules[path]
function = module.get_function(full_function_name)
if function is None:
continue
functions_found.append((
use,
function,
))
# Check each function if the arguments match the passed arguments
if len(functions_found) > 1:
for tup in functions_found:
function = tup[1]
matches = True
for i, arg in enumerate(function.definition.rial_args):
if arg[0] != rial_arg_types[i]:
matches = False
break
if matches:
func = function
break
# Check for number of functions found
elif len(functions_found) == 1:
func = functions_found[0][1]
if func is not None:
# Function is in current module and only a declaration, safe to assume that it's a redeclared function
# from another module or originally declared in this module anyways
if func.module.name != ParserState.module(
).name and not func.is_declaration:
# Function cannot be accessed if:
# - Function is not public and
# - Function is internal but not in same TLM (top level module) or
# - Function is private but not in same module
func_def: FunctionDefinition = func.definition
if func_def.access_modifier != RIALAccessModifier.PUBLIC and \
((func_def.access_modifier == RIALAccessModifier.INTERNAL and
func.module.name.split(':')[0] != ParserState.module().name.split(':')[0]) or
(func_def.access_modifier == RIALAccessModifier.PRIVATE and
func.module.name != ParserState.module().name)):
log_fail(
f"Cannot access method {full_function_name} in module {func.module.name}!"
)
return None
# Try to find function in current module
func: RIALFunction = ParserState.module().get_global_safe(
full_function_name)
# Try to find function in current module with module specifier
if func is None:
func = ParserState.module().get_global_safe(
f"{ParserState.module().name}:{full_function_name}")
# If func isn't in current module
if func is None:
# If couldn't find it, iterate through usings and try to find function
if func is None:
functions_found: List[Tuple[str, RIALFunction]] = list()
for use in ParserState.module().dependencies:
path = CompilationManager.path_from_mod_name(use)
if path not in CompilationManager.modules:
continue
module = CompilationManager.modules[path]
function = module.get_global_safe(full_function_name)
if function is None:
function = module.get_global_safe(
f"{use}:{full_function_name}")
if function is None:
continue
functions_found.append((
use,
function,
))
if len(functions_found) > 1:
log_fail(
f"Function {full_function_name} has been declared multiple times!"
)
log_fail(
f"Specify the specific function to use by adding the namespace to the function call"
)
log_fail(
f"E.g. {functions_found[0][0]}:{full_function_name}()")
return None
# Check for number of functions found
if len(functions_found) == 1:
func = functions_found[0][1]
if func is not None:
# Function is in current module and only a declaration, safe to assume that it's a redeclared function
# from another module or originally declared in this module anyways
if func.module.name != ParserState.module(
).name and not func.is_declaration:
# Function cannot be accessed if:
# - Function is not public and
# - Function is internal but not in same TLM (top level module) or
# - Function is private but not in same module
func_def: FunctionDefinition = func.definition
if func_def.access_modifier != RIALAccessModifier.PUBLIC and \
((func_def.access_modifier == RIALAccessModifier.INTERNAL and
func.module.name.split(':')[0] != ParserState.module().name.split(':')[0]) or
(func_def.access_modifier == RIALAccessModifier.PRIVATE and
func.module.name != ParserState.module().name)):
log_fail(
f"Cannot access method {full_function_name} in module {func.module.name}!"
)
return None
return func
def compiler():
global exceptions
exceptions = False
if not CompilationManager.config.raw_opts.disable_cache:
Cache.load_cache()
if CompilationManager.config.raw_opts.file is not None:
path = Path(CompilationManager.config.raw_opts.file)
else:
path = CompilationManager.config.rial_path.joinpath(
"builtin").joinpath("start.rial")
if not path.exists():
raise FileNotFoundError(str(path))
# Clear global data that can sometimes remain (especially during test execution).
context.global_context.scope._useset.clear()
context.global_context.identified_types.clear()
# Collect all always imported paths
builtin_path = str(
CompilationManager.config.rial_path.joinpath("builtin").joinpath(
"always_imported"))
for file in [
join(builtin_path, f) for f in listdir(builtin_path)
if isfile(join(builtin_path, f))
]:
CompilationManager.request_file(file)
module_name = CompilationManager.mod_name_from_path(
CompilationManager.filename_from_path(str(file)))
CompilationManager.always_imported.append(module_name)
# Request main file
CompilationManager.request_file(str(path))
if CompilationManager.config.raw_opts.profile_gil:
import gil_load
gil_load.init()
gil_load.start()
futures = list()
with concurrent.futures.ThreadPoolExecutor() as executor:
while True:
for future in futures:
if future.done():
futures.remove(future)
try:
# This timeout is so small, it shouldn't matter.
# However it serves a crucial role in regards to Python's GIL.
# Setting this on a higher timeout obviously means that there may be some excessive time spent waiting
# for nothing to appear.
# BUT, setting it to non-blocking means that the GIL never gives any of the compilation threads priority
# as well as holding up the internal threading.Lock(s).
# So setting this timeout so small means not much time is spent waiting for nothing, but it gives other
# things the opportunity to get control of the GIL and execute.
path = CompilationManager.files_to_compile.get(timeout=0.01)
future = executor.submit(compile_file, path)
futures.append(future)
except Empty:
pass
if len(futures) == 0 and CompilationManager.files_to_compile.empty(
):
break
executor.shutdown(True)
CompilationManager.files_to_compile.join()
if CompilationManager.config.raw_opts.profile_gil:
import gil_load
gil_load.stop()
print(f"Main Thread ID: {threading.current_thread().ident}")
print(gil_load.format(gil_load.get()))
if exceptions:
return
if not CompilationManager.config.raw_opts.disable_cache:
Cache.save_cache()
execute_stage(Stage.POST_IR_GEN)
modules: Dict[str, ModuleRef] = dict()
for key, mod in CompilationManager.modules.items():
if not CompilationManager.config.raw_opts.disable_cache:
cache_path = str(
CompilationManager.get_cache_path_str(key)).replace(
".rial", ".cache")
if not Path(cache_path).exists():
CompilationManager.codegen.save_module(mod, cache_path)
with run_with_profiling(CompilationManager.filename_from_path(key),
ExecutionStep.COMPILE_MOD):
try:
modules[key] = CompilationManager.codegen.compile_ir(mod)
except Exception as e:
log_fail(f"Exception when compiling module {mod.name}")
log_fail(e)
log_fail(traceback.format_exc())
return
object_files: List[str] = list()
CompilationManager.codegen.generate_final_modules(list(modules.values()))
for path in list(modules.keys()):
mod = modules[path]
if CompilationManager.config.raw_opts.print_ir:
ir_file = str(
CompilationManager.get_output_path_str(path)).replace(
".rial", ".ll")
if check_needs_output(mod.name, ir_file):
CompilationManager.codegen.save_ir(ir_file, mod)
if CompilationManager.config.raw_opts.print_asm:
asm_file = str(
CompilationManager.get_cache_path_str(path)).replace(
".rial", ".asm")
if check_needs_output(mod.name, asm_file):
CompilationManager.codegen.save_assembly(asm_file, mod)
if not CompilationManager.config.raw_opts.use_object_files:
llvm_bitcode_file = str(
CompilationManager.get_output_path_str(path)).replace(
".rial", ".o")
if check_needs_output(mod.name, llvm_bitcode_file):
CompilationManager.codegen.save_llvm_bitcode(
llvm_bitcode_file, mod)
object_files.append(llvm_bitcode_file)
else:
object_file = str(
CompilationManager.get_output_path_str(path)).replace(
".rial", ".o")
if check_needs_output(mod.name, object_file):
CompilationManager.codegen.save_object(object_file, mod)
object_files.append(object_file)
with run_with_profiling(CompilationManager.config.project_name,
ExecutionStep.LINK_EXE):
exe_path = str(
CompilationManager.config.bin_path.joinpath(
f"{CompilationManager.config.project_name}{Platform.get_exe_file_extension()}"
))
Linker.link_files(
object_files, exe_path,
CompilationManager.config.raw_opts.print_link_command,
CompilationManager.config.raw_opts.strip)
def compile_file(path: str):
global exceptions
try:
last_modified = os.path.getmtime(path)
filename = CompilationManager.filename_from_path(path)
if check_cache(path):
CompilationManager.finish_file(path)
CompilationManager.files_to_compile.task_done()
return
module_name = CompilationManager.mod_name_from_path(filename)
module = CompilationManager.codegen.get_module(
module_name, filename, str(CompilationManager.config.source_path))
ParserState.set_module(module)
ParserState.set_llvmgen(LLVMGen())
if not ParserState.module().name.startswith(
"rial:builtin:always_imported"):
ParserState.module().dependencies.extend(
CompilationManager.always_imported)
# Remove the current module in case it's in the always imported list
if module_name in ParserState.module().dependencies:
ParserState.module().dependencies.remove(module_name)
with run_with_profiling(filename, ExecutionStep.READ_FILE):
with open(path, "r") as file:
contents = file.read()
desugar_transformer = DesugarTransformer()
primitive_transformer = PrimitiveASTTransformer()
parser = Lark_StandAlone(postlex=Postlexer())
# Parse the file
with run_with_profiling(filename, ExecutionStep.PARSE_FILE):
try:
ast = parser.parse(contents)
except Exception as e:
log_fail(f"Exception when parsing {filename}")
log_fail(e)
exceptions = True
CompilationManager.finish_file(path)
CompilationManager.files_to_compile.task_done()
return
if CompilationManager.config.raw_opts.print_tokens:
print(ast.pretty())
# Generate primitive IR (things we don't need other modules for)
with run_with_profiling(filename, ExecutionStep.GEN_IR):
ast = desugar_transformer.transform(ast)
ast = primitive_transformer.transform(ast)
struct_declaration_transformer = StructDeclarationTransformer()
function_declaration_transformer = FunctionDeclarationTransformer()
global_declaration_transformer = GlobalDeclarationTransformer()
transformer = ASTVisitor()
with run_with_profiling(filename, ExecutionStep.GEN_IR):
ast = struct_declaration_transformer.visit(ast)
if ast is not None:
ast = function_declaration_transformer.visit(ast)
if ast is not None:
ast = global_declaration_transformer.visit(ast)
# Declarations are all already collected so we can move on.
CompilationManager.modules[str(path)] = ParserState.module()
CompilationManager.finish_file(path)
if ast is not None:
transformer.visit(ast)
cache_path = str(CompilationManager.get_cache_path_str(path)).replace(
".rial", ".cache")
if Path(cache_path).exists(
) and CompilationManager.config.raw_opts.disable_cache:
os.remove(cache_path)
elif not CompilationManager.config.raw_opts.disable_cache:
Cache.cache_module(ParserState.module(), path, cache_path,
last_modified)
CompilationManager.files_to_compile.task_done()
except Exception as e:
log_fail("Internal Compiler Error: ")
log_fail(traceback.format_exc())
exceptions = True
CompilationManager.finish_file(path)
CompilationManager.files_to_compile.task_done()