def calculate_multiple_combinations_average(self):
for combination_json in self.db.combinations_iterator():
total_results = dict()
combination_id = Combination.json_to_obj(
combination_json).combination_id
logger.info(f'Calculating average of {combination_id} combination')
for i in range(self.multiple_combinations):
current_id = f'{combination_id}_{i}'
current_results = self.db.get_combination_results(current_id)
if 'error' in current_results.keys():
current_results[
'error'] = f"from {current_id}: {current_results['error']}"
if not total_results:
total_results = current_results
else:
if 'error' in current_results.keys():
total_results[
'error'] = f"{total_results.get('error', '')}\n{current_results['error']}"
elif 'error' not in total_results.keys():
total_results['total_run_time'] += current_results[
'total_run_time']
for j, file in enumerate(
current_results['run_time_results']):
if 'dead_code_file' in file.keys():
total_results['run_time_results'][j] = file
elif 'dead_code_file' not in total_results[
'run_time_results'][j].keys():
for k, loop in enumerate(file['loops']):
if 'dead_code' in loop.keys():
total_results['run_time_results'][j][
'loops'][k] = loop
elif 'dead_code' not in total_results[
'run_time_results'][j]['loops'][
k].keys():
total_results['run_time_results'][j][
'loops'][k]['run_time'] += loop[
'run_time']
self.db.delete_combination(current_id)
if 'error' not in total_results.keys():
try:
total_results[
'total_run_time'] /= self.multiple_combinations
except KeyError as ex:
total_results['error'] = str(ex)
for file in total_results['run_time_results']:
if 'dead_code_file' not in file.keys():
for loop in file['loops']:
if 'dead_code' not in loop.keys():
loop['run_time'] /= self.multiple_combinations
key = (file['file_id_by_rel_path'],
loop['loop_label'])
try:
loop['speedup'] = self.serial_run_time[
key] / loop['run_time']
except ZeroDivisionError:
loop['speedup'] = float('inf')
except Exception as ex: # if serial loop is dead_code and parallel is not, KeyError
total_results['error'] = str(ex)
total_results['_id'] = combination_id
self.db.insert_new_combination_results(total_results)
def run_parallel_combinations(self):
logger.info('Start to work on parallel combinations')
self.parallel_jobs_pool_executor.create_jobs_pool()
# if equal to one - we don't need to concatenate the number of repetitions to combination id nor calculate avg
is_multiple_combinations = self.multiple_combinations > 1
for combination_json in self.db.combinations_iterator():
original_combination_obj = Combination.json_to_obj(combination_json)
logger.info(LogPhrases.NEW_COMBINATION.format(original_combination_obj.combination_id))
for i in range(self.multiple_combinations):
if is_multiple_combinations:
combination_obj = copy.deepcopy(original_combination_obj)
combination_obj.combination_id = f'{combination_obj.combination_id}_{i}'
logger.info(f'#{i} repetition of {original_combination_obj.combination_id} combination')
else:
combination_obj = original_combination_obj
combination_folder_path = self.create_combination_folder(str(combination_obj.get_combination_id()))
try:
self.parallel_compilation_of_one_combination(combination_obj, combination_folder_path)
self.compile_combination_to_binary(combination_folder_path)
except Exception as ex:
logger.info_error(f'Exception at {Compar.__name__}: {ex}')
logger.debug_error(f'{traceback.format_exc()}')
self.save_combination_as_failure(combination_obj.get_combination_id(), str(ex),
combination_folder_path)
continue
job = Job(combination_folder_path, combination_obj, self.main_file_parameters)
self.parallel_jobs_pool_executor.run_job_in_thread(self.run_and_save_job, job)
self.parallel_jobs_pool_executor.wait_and_finish_pool()
if is_multiple_combinations:
self.calculate_multiple_combinations_average()
logger.info('Finish to work on all the parallel combinations')
def generate_summary_file(self, optimal_data: list, dir_path: str):
file_path = os.path.join(dir_path, ComparConfig.SUMMARY_FILE_NAME)
with open(file_path, 'w', newline='') as file:
writer = csv.writer(file)
writer.writerow([
"File", "Loop", "Combination", "Compiler", "Compilation Flags",
"OMP RTL Parameters", "OMP Directive Parameters", "Runtime",
"Speedup"
])
for curr_file in optimal_data:
if 'dead_code_file' in curr_file.keys():
writer.writerow([
curr_file['file_id_by_rel_path'], ""
'dead code file', "", "", "", "", ""
])
continue
for loop in curr_file['optimal_loops']:
if 'dead_code' in loop.keys():
writer.writerow([
curr_file['file_id_by_rel_path'],
loop['loop_label'], 'dead code loop', "", "", "",
"", ""
])
else:
combination_obj = Combination.json_to_obj(
self.db.get_combination_from_static_db(
loop['_id']))
writer.writerow([
curr_file['file_id_by_rel_path'],
loop['loop_label'], loop['_id'],
combination_obj.get_compiler(),
combination_obj.get_parameters(
).get_compilation_params(),
combination_obj.get_parameters(
).get_omp_rtl_params(),
combination_obj.get_parameters(
).get_omp_directives_params(), loop['run_time'],
loop['speedup']
])
def generate_optimal_code(self):
logger.info('Start to combine the Compar combination')
optimal_loops_data = []
# copy final results into this folder
compar_combination_folder_path = self.create_combination_folder(
self.COMPAR_COMBINATION_FOLDER_NAME,
base_dir=self.working_directory)
final_files_list = self.make_absolute_file_list(
compar_combination_folder_path)
for file_id_by_rel_path, loops in self.files_loop_dict.items():
current_file = {
"file_id_by_rel_path": file_id_by_rel_path,
'optimal_loops': []
}
for loop_id in range(1, loops[0] + 1):
start_label = Fragmentator.get_start_label() + str(loop_id)
end_label = Fragmentator.get_end_label() + str(loop_id)
try:
current_optimal_id, current_loop = self.db.find_optimal_loop_combination(
file_id_by_rel_path, str(loop_id))
# update the optimal loops list
current_loop['_id'] = current_optimal_id
current_file["optimal_loops"].append(current_loop)
except e.DeadCodeFile:
current_file["dead_code_file"] = True
break
except e.DeadCodeLoop:
current_file["optimal_loops"].append({
'_id':
Database.SERIAL_COMBINATION_ID,
'loop_label':
str(loop_id),
'dead_code':
True
})
current_optimal_id = Database.SERIAL_COMBINATION_ID
# if the optimal combination is the serial => do nothing
if current_optimal_id != Database.SERIAL_COMBINATION_ID:
current_optimal_combination = Combination.json_to_obj(
self.db.get_combination_from_static_db(
current_optimal_id))
current_combination_folder_path = self.create_combination_folder(
ComparConfig.OPTIMAL_CURRENT_COMBINATION_FOLDER_NAME,
base_dir=self.working_directory)
files_list = self.make_absolute_file_list(
current_combination_folder_path)
current_comp_name = current_optimal_combination.compiler_name
# get direct file path to inject params
src_file_path = list(
filter(
lambda x: x['file_id_by_rel_path'] ==
file_id_by_rel_path, files_list))
src_file_path = src_file_path[0]['file_full_path']
# parallelize and inject
self.parallel_compilation_of_one_combination(
current_optimal_combination,
current_combination_folder_path)
# replace loop in c file using final_files_list
target_file_path = list(
filter(
lambda x: x['file_id_by_rel_path'] ==
file_id_by_rel_path, final_files_list))
target_file_path = target_file_path[0]['file_full_path']
Compar.replace_loops_in_files(src_file_path,
target_file_path,
start_label, end_label)
Compar.add_to_loop_details_about_comp_and_combination(
target_file_path, start_label, current_optimal_id,
current_comp_name)
sleep(1) # prevent IO error
shutil.rmtree(current_combination_folder_path)
optimal_loops_data.append(current_file)
# remove timers code
Timer.remove_timer_code(
self.make_absolute_file_list(compar_combination_folder_path))
# inject new code
Timer.inject_timer_to_compar_mixed_file(
os.path.join(compar_combination_folder_path,
self.main_file_rel_path),
compar_combination_folder_path)
self.generate_summary_file(optimal_loops_data,
compar_combination_folder_path)
try:
logger.info('Compiling Compar combination')
self.compile_combination_to_binary(compar_combination_folder_path,
inject=False)
job = Job(
compar_combination_folder_path,
Combination(Database.COMPAR_COMBINATION_ID,
ComparConfig.MIXED_COMPILER_NAME, None), [])
logger.info('Running Compar combination')
self.execute_job(job, self.serial_run_time)
except Exception as ex:
msg = f'Exception in Compar: {ex}\ngenerate_optimal_code: cannot compile compar combination'
self.save_combination_as_failure(Database.COMPAR_COMBINATION_ID,
msg,
compar_combination_folder_path)
logger.info(
LogPhrases.NEW_COMBINATION.format(
Database.FINAL_RESULTS_COMBINATION_ID))
# Check for best total runtime
best_runtime_combination_id = self.db.get_total_runtime_best_combination(
)
best_combination_obj = None
if best_runtime_combination_id != Database.COMPAR_COMBINATION_ID:
logger.info(
f'Combination #{best_runtime_combination_id} is more optimal than Compar combination'
)
best_combination_obj = Combination.json_to_obj(
self.db.get_combination_from_static_db(
best_runtime_combination_id))
final_results_folder_path = self.create_combination_folder(
self.FINAL_RESULTS_FOLDER_NAME, self.working_directory)
try:
if best_runtime_combination_id != Database.SERIAL_COMBINATION_ID:
self.parallel_compilation_of_one_combination(
best_combination_obj, final_results_folder_path)
self.compile_combination_to_binary(final_results_folder_path)
summary_file_path = os.path.join(
compar_combination_folder_path,
ComparConfig.SUMMARY_FILE_NAME)
summary_file_new_path = os.path.join(
final_results_folder_path, ComparConfig.SUMMARY_FILE_NAME)
shutil.move(summary_file_path, summary_file_new_path)
except Exception as ex:
raise Exception(
f"Total runtime calculation - The optimal file could not be compiled, combination"
f" {best_runtime_combination_id}.\n{ex}")
else:
logger.info(f'Compar combination is the optimal combination')
final_folder_path = os.path.join(self.working_directory,
self.FINAL_RESULTS_FOLDER_NAME)
if os.path.exists(final_folder_path):
shutil.rmtree(final_folder_path)
shutil.copytree(compar_combination_folder_path, final_folder_path)
# remove compar code from all the files in final result folder
final_folder_path = os.path.join(self.working_directory,
self.FINAL_RESULTS_FOLDER_NAME)
Timer.remove_timer_code(
self.make_absolute_file_list(final_folder_path))
final_combination_results = self.db.get_combination_results(
best_runtime_combination_id)
if final_combination_results:
final_combination_results[
'_id'] = Database.FINAL_RESULTS_COMBINATION_ID
final_combination_results[
'from_combination'] = best_runtime_combination_id
self.db.insert_new_combination_results(final_combination_results)
with open(
os.path.join(final_folder_path,
Timer.TOTAL_RUNTIME_FILENAME), 'w') as f:
f.write(str(final_combination_results['total_run_time']))
self.update_summary_file(
final_folder_path, best_runtime_combination_id,
final_combination_results['total_run_time'],
best_combination_obj)
# format all optimal files
self.format_c_files([
file_dict['file_full_path']
for file_dict in self.make_absolute_file_list(final_folder_path)
])
self.db.remove_unused_data(Database.COMPAR_COMBINATION_ID)
self.db.remove_unused_data(Database.FINAL_RESULTS_COMBINATION_ID)
final_result_speedup, final_result_runtime = self.db.get_final_result_speedup_and_runtime(
)
logger.info(
LogPhrases.FINAL_RESULTS_SUMMARY.format(final_result_speedup,
final_result_runtime))
if self.clear_db:
self.clear_related_collections()
self.db.close_connection()
