def calculate_multiple_combinations_average(self): for combination_json in self.db.combinations_iterator(): total_results = dict() combination_id = Combination.json_to_obj( combination_json).combination_id logger.info(f'Calculating average of {combination_id} combination') for i in range(self.multiple_combinations): current_id = f'{combination_id}_{i}' current_results = self.db.get_combination_results(current_id) if 'error' in current_results.keys(): current_results[ 'error'] = f"from {current_id}: {current_results['error']}" if not total_results: total_results = current_results else: if 'error' in current_results.keys(): total_results[ 'error'] = f"{total_results.get('error', '')}\n{current_results['error']}" elif 'error' not in total_results.keys(): total_results['total_run_time'] += current_results[ 'total_run_time'] for j, file in enumerate( current_results['run_time_results']): if 'dead_code_file' in file.keys(): total_results['run_time_results'][j] = file elif 'dead_code_file' not in total_results[ 'run_time_results'][j].keys(): for k, loop in enumerate(file['loops']): if 'dead_code' in loop.keys(): total_results['run_time_results'][j][ 'loops'][k] = loop elif 'dead_code' not in total_results[ 'run_time_results'][j]['loops'][ k].keys(): total_results['run_time_results'][j][ 'loops'][k]['run_time'] += loop[ 'run_time'] self.db.delete_combination(current_id) if 'error' not in total_results.keys(): try: total_results[ 'total_run_time'] /= self.multiple_combinations except KeyError as ex: total_results['error'] = str(ex) for file in total_results['run_time_results']: if 'dead_code_file' not in file.keys(): for loop in file['loops']: if 'dead_code' not in loop.keys(): loop['run_time'] /= self.multiple_combinations key = (file['file_id_by_rel_path'], loop['loop_label']) try: loop['speedup'] = self.serial_run_time[ key] / loop['run_time'] except ZeroDivisionError: loop['speedup'] = float('inf') except Exception as ex: # if serial loop is dead_code and parallel is not, KeyError total_results['error'] = str(ex) total_results['_id'] = combination_id self.db.insert_new_combination_results(total_results)
def run_parallel_combinations(self): logger.info('Start to work on parallel combinations') self.parallel_jobs_pool_executor.create_jobs_pool() # if equal to one - we don't need to concatenate the number of repetitions to combination id nor calculate avg is_multiple_combinations = self.multiple_combinations > 1 for combination_json in self.db.combinations_iterator(): original_combination_obj = Combination.json_to_obj(combination_json) logger.info(LogPhrases.NEW_COMBINATION.format(original_combination_obj.combination_id)) for i in range(self.multiple_combinations): if is_multiple_combinations: combination_obj = copy.deepcopy(original_combination_obj) combination_obj.combination_id = f'{combination_obj.combination_id}_{i}' logger.info(f'#{i} repetition of {original_combination_obj.combination_id} combination') else: combination_obj = original_combination_obj combination_folder_path = self.create_combination_folder(str(combination_obj.get_combination_id())) try: self.parallel_compilation_of_one_combination(combination_obj, combination_folder_path) self.compile_combination_to_binary(combination_folder_path) except Exception as ex: logger.info_error(f'Exception at {Compar.__name__}: {ex}') logger.debug_error(f'{traceback.format_exc()}') self.save_combination_as_failure(combination_obj.get_combination_id(), str(ex), combination_folder_path) continue job = Job(combination_folder_path, combination_obj, self.main_file_parameters) self.parallel_jobs_pool_executor.run_job_in_thread(self.run_and_save_job, job) self.parallel_jobs_pool_executor.wait_and_finish_pool() if is_multiple_combinations: self.calculate_multiple_combinations_average() logger.info('Finish to work on all the parallel combinations')
def generate_summary_file(self, optimal_data: list, dir_path: str): file_path = os.path.join(dir_path, ComparConfig.SUMMARY_FILE_NAME) with open(file_path, 'w', newline='') as file: writer = csv.writer(file) writer.writerow([ "File", "Loop", "Combination", "Compiler", "Compilation Flags", "OMP RTL Parameters", "OMP Directive Parameters", "Runtime", "Speedup" ]) for curr_file in optimal_data: if 'dead_code_file' in curr_file.keys(): writer.writerow([ curr_file['file_id_by_rel_path'], "" 'dead code file', "", "", "", "", "" ]) continue for loop in curr_file['optimal_loops']: if 'dead_code' in loop.keys(): writer.writerow([ curr_file['file_id_by_rel_path'], loop['loop_label'], 'dead code loop', "", "", "", "", "" ]) else: combination_obj = Combination.json_to_obj( self.db.get_combination_from_static_db( loop['_id'])) writer.writerow([ curr_file['file_id_by_rel_path'], loop['loop_label'], loop['_id'], combination_obj.get_compiler(), combination_obj.get_parameters( ).get_compilation_params(), combination_obj.get_parameters( ).get_omp_rtl_params(), combination_obj.get_parameters( ).get_omp_directives_params(), loop['run_time'], loop['speedup'] ])
def generate_optimal_code(self): logger.info('Start to combine the Compar combination') optimal_loops_data = [] # copy final results into this folder compar_combination_folder_path = self.create_combination_folder( self.COMPAR_COMBINATION_FOLDER_NAME, base_dir=self.working_directory) final_files_list = self.make_absolute_file_list( compar_combination_folder_path) for file_id_by_rel_path, loops in self.files_loop_dict.items(): current_file = { "file_id_by_rel_path": file_id_by_rel_path, 'optimal_loops': [] } for loop_id in range(1, loops[0] + 1): start_label = Fragmentator.get_start_label() + str(loop_id) end_label = Fragmentator.get_end_label() + str(loop_id) try: current_optimal_id, current_loop = self.db.find_optimal_loop_combination( file_id_by_rel_path, str(loop_id)) # update the optimal loops list current_loop['_id'] = current_optimal_id current_file["optimal_loops"].append(current_loop) except e.DeadCodeFile: current_file["dead_code_file"] = True break except e.DeadCodeLoop: current_file["optimal_loops"].append({ '_id': Database.SERIAL_COMBINATION_ID, 'loop_label': str(loop_id), 'dead_code': True }) current_optimal_id = Database.SERIAL_COMBINATION_ID # if the optimal combination is the serial => do nothing if current_optimal_id != Database.SERIAL_COMBINATION_ID: current_optimal_combination = Combination.json_to_obj( self.db.get_combination_from_static_db( current_optimal_id)) current_combination_folder_path = self.create_combination_folder( ComparConfig.OPTIMAL_CURRENT_COMBINATION_FOLDER_NAME, base_dir=self.working_directory) files_list = self.make_absolute_file_list( current_combination_folder_path) current_comp_name = current_optimal_combination.compiler_name # get direct file path to inject params src_file_path = list( filter( lambda x: x['file_id_by_rel_path'] == file_id_by_rel_path, files_list)) src_file_path = src_file_path[0]['file_full_path'] # parallelize and inject self.parallel_compilation_of_one_combination( current_optimal_combination, current_combination_folder_path) # replace loop in c file using final_files_list target_file_path = list( filter( lambda x: x['file_id_by_rel_path'] == file_id_by_rel_path, final_files_list)) target_file_path = target_file_path[0]['file_full_path'] Compar.replace_loops_in_files(src_file_path, target_file_path, start_label, end_label) Compar.add_to_loop_details_about_comp_and_combination( target_file_path, start_label, current_optimal_id, current_comp_name) sleep(1) # prevent IO error shutil.rmtree(current_combination_folder_path) optimal_loops_data.append(current_file) # remove timers code Timer.remove_timer_code( self.make_absolute_file_list(compar_combination_folder_path)) # inject new code Timer.inject_timer_to_compar_mixed_file( os.path.join(compar_combination_folder_path, self.main_file_rel_path), compar_combination_folder_path) self.generate_summary_file(optimal_loops_data, compar_combination_folder_path) try: logger.info('Compiling Compar combination') self.compile_combination_to_binary(compar_combination_folder_path, inject=False) job = Job( compar_combination_folder_path, Combination(Database.COMPAR_COMBINATION_ID, ComparConfig.MIXED_COMPILER_NAME, None), []) logger.info('Running Compar combination') self.execute_job(job, self.serial_run_time) except Exception as ex: msg = f'Exception in Compar: {ex}\ngenerate_optimal_code: cannot compile compar combination' self.save_combination_as_failure(Database.COMPAR_COMBINATION_ID, msg, compar_combination_folder_path) logger.info( LogPhrases.NEW_COMBINATION.format( Database.FINAL_RESULTS_COMBINATION_ID)) # Check for best total runtime best_runtime_combination_id = self.db.get_total_runtime_best_combination( ) best_combination_obj = None if best_runtime_combination_id != Database.COMPAR_COMBINATION_ID: logger.info( f'Combination #{best_runtime_combination_id} is more optimal than Compar combination' ) best_combination_obj = Combination.json_to_obj( self.db.get_combination_from_static_db( best_runtime_combination_id)) final_results_folder_path = self.create_combination_folder( self.FINAL_RESULTS_FOLDER_NAME, self.working_directory) try: if best_runtime_combination_id != Database.SERIAL_COMBINATION_ID: self.parallel_compilation_of_one_combination( best_combination_obj, final_results_folder_path) self.compile_combination_to_binary(final_results_folder_path) summary_file_path = os.path.join( compar_combination_folder_path, ComparConfig.SUMMARY_FILE_NAME) summary_file_new_path = os.path.join( final_results_folder_path, ComparConfig.SUMMARY_FILE_NAME) shutil.move(summary_file_path, summary_file_new_path) except Exception as ex: raise Exception( f"Total runtime calculation - The optimal file could not be compiled, combination" f" {best_runtime_combination_id}.\n{ex}") else: logger.info(f'Compar combination is the optimal combination') final_folder_path = os.path.join(self.working_directory, self.FINAL_RESULTS_FOLDER_NAME) if os.path.exists(final_folder_path): shutil.rmtree(final_folder_path) shutil.copytree(compar_combination_folder_path, final_folder_path) # remove compar code from all the files in final result folder final_folder_path = os.path.join(self.working_directory, self.FINAL_RESULTS_FOLDER_NAME) Timer.remove_timer_code( self.make_absolute_file_list(final_folder_path)) final_combination_results = self.db.get_combination_results( best_runtime_combination_id) if final_combination_results: final_combination_results[ '_id'] = Database.FINAL_RESULTS_COMBINATION_ID final_combination_results[ 'from_combination'] = best_runtime_combination_id self.db.insert_new_combination_results(final_combination_results) with open( os.path.join(final_folder_path, Timer.TOTAL_RUNTIME_FILENAME), 'w') as f: f.write(str(final_combination_results['total_run_time'])) self.update_summary_file( final_folder_path, best_runtime_combination_id, final_combination_results['total_run_time'], best_combination_obj) # format all optimal files self.format_c_files([ file_dict['file_full_path'] for file_dict in self.make_absolute_file_list(final_folder_path) ]) self.db.remove_unused_data(Database.COMPAR_COMBINATION_ID) self.db.remove_unused_data(Database.FINAL_RESULTS_COMBINATION_ID) final_result_speedup, final_result_runtime = self.db.get_final_result_speedup_and_runtime( ) logger.info( LogPhrases.FINAL_RESULTS_SUMMARY.format(final_result_speedup, final_result_runtime)) if self.clear_db: self.clear_related_collections() self.db.close_connection()