def inject_rtl_params_to_loop(self, file_dict: dict, omp_rtl_params: list):
if not omp_rtl_params:
return
c_file_path = file_dict['file_full_path']
e.assert_file_exist(c_file_path)
with open(c_file_path, 'r') as input_file:
c_code = input_file.read()
e.assert_file_is_empty(c_code)
for loop_id in range(1, self.files_loop_dict[file_dict['file_id_by_rel_path']][0] + 1):
start_loop_marker = f'{Fragmentator.get_start_label()}{loop_id}'
end_loop_marker = f'{Fragmentator.get_end_label()}{loop_id}'
start_marker_pattern = rf'{start_loop_marker}[ \t]*\n'
loop_pattern = rf'{start_marker_pattern}.*{end_loop_marker}[ \t]*\n'
loop_before_changes = re.search(loop_pattern, c_code, re.DOTALL).group()
loop_prefix_pattern = rf'{start_marker_pattern}.*?(?=[\t ]*for[ \t]*\()'
loop_prefix = re.search(loop_prefix_pattern, loop_before_changes, re.DOTALL).group()
loop_body_and_suffix = re.sub(re.escape(loop_prefix), '', loop_before_changes)
params_code = f'{start_loop_marker}\n'
for param in omp_rtl_params:
omp_function_name = re.search(r'.+(?=\(.*\))', param).group()
if omp_function_name in loop_prefix:
loop_prefix = re.sub(rf'{omp_function_name}[^;]*;[^\n]*\n', '', loop_prefix, re.DOTALL)
params_code += f'{param}\n'
loop_prefix = re.sub(rf'{start_marker_pattern}', params_code, loop_prefix)
new_loop = f'{loop_prefix}{loop_body_and_suffix}'
c_code = c_code.replace(loop_before_changes, new_loop)
try:
with open(c_file_path, 'w') as output_file:
output_file.write(c_code)
except OSError as err:
raise e.FileError(str(err))
def __init__(self, file_path: str, code_with_markers: bool = False):
assert_file_exist(file_path)
self.__file_path = file_path
self.__file_content = ''
self.__loops_list = []
self.__fragments = []
self.__occurrences_index_list = []
self.code_with_markers = code_with_markers
def __init__(self, file_path: str, code_with_markers: bool = False):
e.assert_file_exist(file_path)
self.__input_file_path = file_path
c_file_name = str(os.path.basename(file_path).split('.')[0])
self.__time_result_file = c_file_name + TimerConfig.LOOPS_RUNTIME_RESULTS_SUFFIX
self.__time_result_file = self.__time_result_file.replace(
';', '') # the file name cannot contains semicolon
self.__number_of_loops = 0
self.__fragmentation = Fragmentator(file_path, code_with_markers)
def inject_c_code_to_loop(c_file_path: str, loop_id: str, c_code_to_inject: str):
e.assert_file_exist(c_file_path)
with open(c_file_path, 'r') as input_file:
c_code = input_file.read()
e.assert_file_is_empty(c_code)
loop_id_with_inject_code = loop_id + '\n' + c_code_to_inject
c_code = re.sub(loop_id + '[ ]*\n', loop_id_with_inject_code, c_code)
try:
with open(c_file_path, 'w') as output_file:
output_file.write(c_code)
except OSError as err:
raise e.FileError(str(err))
def add_to_loop_details_about_comp_and_combination(file_path: str, start_label: str, combination_id: str,
comp_name: str):
e.assert_file_exist(file_path)
e.assert_file_is_empty(file_path)
with open(file_path, 'r') as file:
file_text = file.read()
to_replace = ''
to_replace += f'{start_label}\n{ComparConfig.COMBINATION_ID_C_COMMENT}{combination_id}\n'
to_replace += f'{ComparConfig.COMPILER_NAME_C_COMMENT}{comp_name}\n'
file_text = re.sub(f'{start_label}[ ]*\\n', to_replace, file_text)
try:
with open(file_path, 'w') as file:
file.write(file_text)
except OSError as err:
raise e.FileError(str(err))
def inject_directive_params_to_loop(self, file_dict: dict, omp_directive_params: list):
if not omp_directive_params:
return
c_file_path = file_dict['file_full_path']
e.assert_file_exist(c_file_path)
format_c_code([c_file_path, ]) # mainly to arrange directives in a single line
with open(c_file_path, 'r') as input_file:
c_code = input_file.read()
e.assert_file_is_empty(c_code)
for loop_id in range(1, self.files_loop_dict[file_dict['file_id_by_rel_path']][0] + 1):
start_loop_marker = f'{Fragmentator.get_start_label()}{loop_id}'
end_loop_marker = f'{Fragmentator.get_end_label()}{loop_id}'
start_marker_pattern = rf'{start_loop_marker}[ \t]*\n'
loop_pattern = rf'{start_marker_pattern}.*{end_loop_marker}[ \t]*\n'
loop_before_changes = re.search(loop_pattern, c_code, re.DOTALL).group()
loop_prefix_pattern = rf'{start_marker_pattern}.*?(?=[\t ]*for[ \t]*\()'
loop_prefix = re.search(loop_prefix_pattern, loop_before_changes, re.DOTALL).group()
loop_body_and_suffix = re.sub(re.escape(loop_prefix), '', loop_before_changes)
pragma_pattern = r'#pragma omp[^\n]+\n'
pragma = re.search(rf'{pragma_pattern}', loop_prefix)
if pragma:
pragma = pragma.group().replace('\n', '')
new_directives = ''
for directive in omp_directive_params:
pragma_type, directive = directive.split('_', 1)
pragma_name = re.search(r'[^(]+', directive).group()
if not re.search(rf' {pragma_type} ?', pragma):
if pragma_type == CombinatorConfig.PARALLEL_DIRECTIVE_PREFIX:
pragma = re.sub(r'pragma omp', f'pragma omp {pragma_type}', pragma)
if pragma_type == CombinatorConfig.FOR_DIRECTIVE_PREFIX:
if re.search(rf' {CombinatorConfig.PARALLEL_DIRECTIVE_PREFIX} ?', pragma):
new_text = f'pragma omp {CombinatorConfig.PARALLEL_DIRECTIVE_PREFIX} {pragma_type} '
pragma = re.sub(rf'pragma omp {CombinatorConfig.PARALLEL_DIRECTIVE_PREFIX} ?',
new_text,
pragma)
else:
pragma = re.sub(r'pragma omp', f'pragma omp {pragma_type}', pragma)
if pragma_name in pragma:
pragma = re.sub(rf'{"pragma_name"}(?:\([^)]+\))? ?', '', pragma)
new_directives += f' {directive}'
new_pragma = f'{pragma} {new_directives}\n'
new_prefix = re.sub(rf'{pragma_pattern}', new_pragma, loop_prefix)
c_code = c_code.replace(loop_before_changes, f'{new_prefix}{loop_body_and_suffix}')
try:
with open(c_file_path, 'w') as output_file:
output_file.write(c_code)
except OSError as err:
raise e.FileError(str(err))
def set_input_file_path(self, file_path: str):
e.assert_file_exist(file_path)
self.__input_file_path = file_path
def set_file_path(self, new_file_path: str):
assert_file_exist(new_file_path)
self.__file_path = new_file_path
self.__reset_data()
def __init__(self,
input_dir: str,
output_dir: str,
project_name: str,
main_file_rel_path: str,
binary_compiler_type: str = "",
binary_compiler_version: str = None,
binary_compiler_flags: list = None,
save_combinations_folders: bool = False,
is_make_file: bool = False,
makefile_commands: list = None,
makefile_exe_folder_rel_path: str = "",
makefile_output_exe_file_name: str = "",
ignored_rel_paths: list = None,
include_dirs_list: list = None,
main_file_parameters: list = None,
slurm_parameters: list = None,
extra_files: list = None,
time_limit: str = None,
slurm_partition=ComparConfig.DEFAULT_SLURM_PARTITION,
test_file_path: str = '',
mode=ComparConfig.MODES[ComparConfig.DEFAULT_MODE],
code_with_markers: bool = False,
clear_db: bool = False,
multiple_combinations: int = 1,
log_level: int = logger.DEFAULT_LOG_LEVEL):
self.db = Database(project_name, mode)
working_directory = os.path.join(output_dir,
self.db.get_project_name())
if mode == ComparMode.CONTINUE:
e.assert_original_files_folder_exists(working_directory)
else:
if os.path.exists(working_directory):
shutil.rmtree(working_directory)
os.makedirs(working_directory, exist_ok=True)
logger.initialize(log_level, working_directory)
logger.info('Starting ComPar execution')
e.assert_rel_path_starts_without_sep(makefile_exe_folder_rel_path)
e.assert_rel_path_starts_without_sep(main_file_rel_path)
if slurm_parameters and len(slurm_parameters) == 1:
slurm_parameters = str(slurm_parameters[0]).split(' ')
e.assert_folder_exist(input_dir)
e.assert_user_json_structure()
if not is_make_file:
e.assert_only_files(input_dir)
if not include_dirs_list:
include_dirs_list = []
if not makefile_commands:
makefile_commands = []
if not binary_compiler_flags:
binary_compiler_flags = []
if not main_file_parameters:
main_file_parameters = []
if not slurm_parameters:
slurm_parameters = ComparConfig.DEFAULT_SLURM_PARAMETERS
if not ignored_rel_paths:
ignored_rel_paths = []
if not test_file_path:
test_file_path = CombinationValidator.UNIT_TEST_DEFAULT_PATH
if not extra_files:
extra_files = []
self.binary_compiler = None
self.__timer = None
self.serial_run_time = {}
self.files_loop_dict = {}
self.main_file_rel_path = main_file_rel_path
self.save_combinations_folders = save_combinations_folders
self.binary_compiler_version = binary_compiler_version
self.ignored_rel_paths = ignored_rel_paths
self.include_dirs_list = include_dirs_list
self.time_limit = time_limit
self.slurm_partition = slurm_partition
self.parallel_jobs_pool_executor = JobExecutor(Compar.NUM_OF_THREADS)
self.mode = mode
self.code_with_markers = code_with_markers
self.clear_db = clear_db
self.multiple_combinations = multiple_combinations
# Unit test
self.test_file_path = test_file_path
e.assert_file_exist(self.test_file_path)
e.assert_test_file_name(os.path.basename(self.test_file_path))
e.assert_test_file_function_name(self.test_file_path)
# Initiate Compar environment
e.assert_forbidden_characters(working_directory)
self.working_directory = working_directory
self.backup_files_dir = os.path.join(working_directory,
ComparConfig.BACKUP_FOLDER_NAME)
self.original_files_dir = os.path.join(
working_directory, ComparConfig.ORIGINAL_FILES_FOLDER_NAME)
if self.mode == ComparMode.CONTINUE:
e.assert_folder_exist(self.original_files_dir)
self.__delete_combination_folder(
os.path.join(working_directory,
self.COMPAR_COMBINATION_FOLDER_NAME))
self.__delete_combination_folder(
os.path.join(working_directory,
self.FINAL_RESULTS_FOLDER_NAME))
self.combinations_dir = os.path.join(
working_directory, ComparConfig.COMBINATIONS_FOLDER_NAME)
self.__create_directories_structure(input_dir)
# Compilers variables
self.relative_c_file_list = self.make_relative_c_file_list(
self.original_files_dir)
if self.code_with_markers:
file_paths = [
file['file_full_path'] for file in
self.make_absolute_file_list(self.original_files_dir)
]
self.remove_optimal_combinations_details(file_paths)
self.binary_compiler_type = binary_compiler_type
self.parallelizers = dict()
for name, ctor in parallelizers.items():
self.parallelizers[name] = ctor(
"",
include_dirs_list=self.include_dirs_list,
extra_files=extra_files)
# Compiler flags
self.user_binary_compiler_flags = binary_compiler_flags
# Makefile
self.is_make_file = is_make_file
self.makefile_commands = makefile_commands
self.makefile_exe_folder_rel_path = makefile_exe_folder_rel_path
self.makefile_output_exe_file_name = makefile_output_exe_file_name
# Main file
self.main_file_parameters = main_file_parameters
# SLURM
self.slurm_parameters = slurm_parameters
# Initialization
if not is_make_file:
self.__initialize_binary_compiler()
self.db.create_collections()
