def collect_training_data(kernel_folders, kernel_folder_pattern): """ Collect training data from log files resulting of autotuning """ # =============================================================================== # For each folder: n_kernels = len(kernel_folders) for i, kernel_folder in enumerate(kernel_folders): print("\nProcess folder {} ({}/{:,})".format(kernel_folder, i + 1, n_kernels)) # Find (m, n, k) # Each folder contains data for just one (m, n, k) but potentially mutliple algorithms match = kernel_folder_pattern.search(kernel_folder).groups() m = int(match[0]) n = int(match[1]) k = int(match[2]) # =============================================================================== # Collect info from log files data = read_log_file(kernel_folder, m, n, k) # =============================================================================== # Write parameters to CSV for name_algo, kernel_algo in kernel_algorithm.items(): # if applicable to this mnk if name_algo in data["algorithm"].values: # Does collected csv file exist already? raw_parameters_file_name = os.path.join( kernel_folder, "raw_training_data_" + to_string(m, n, k) + "_" + name_algo + ".csv", ) if os.path.exists(raw_parameters_file_name): print( "\tFound csv file:", raw_parameters_file_name, ", skipping ..." ) else: # Get the data corresponding to this algorithm data_algo = data[data["algorithm"] == name_algo] # Write raw parameters pars_to_get = kernel_algo.launch_parameters + ["perf (Gflop/s)"] data_algo[pars_to_get].to_csv(raw_parameters_file_name, index=False) print("\tWrote", raw_parameters_file_name)
def get_predive_model_performances(y_true, y_pred, x_mnk, max_performances_ref, max_performances_algo): predictive_model_perf_scaled = dict() for mnk_string in x_mnk["mnk"].unique(): idx_mnk = np.where(x_mnk == mnk_string)[0].tolist() assert len(idx_mnk) > 0, "idx_mnk is empty" m, n, k = to_tuple(mnk_string) perf_chosen_idx = np.argmax(y_pred[idx_mnk]) perf_effective = y_true.iloc[idx_mnk].iloc[ perf_chosen_idx].values.item() predictive_model_perf_scaled[( m, n, k)] = perf_effective # 'scaled' between 0 and 1 predictive_model_perf = dict( zip( predictive_model_perf_scaled.keys(), [ perf_scaled * max_performances_ref[to_string(mnk)] for mnk, perf_scaled in predictive_model_perf_scaled.items() ], )) # Re-scale performances by algorithm for a fair comparison predictive_model_perf_scaled = dict( zip( predictive_model_perf.keys(), [ perf / max_performances_algo[mnk] for mnk, perf in predictive_model_perf.items() ], )) return predictive_model_perf, predictive_model_perf_scaled
def plot_choice_goodness( m, n, k, baseline_performances, max_performances, y_true, y_pred, train, pp, scaled=True, ): # Sort in ascending performances data_mnk = pd.DataFrame() if scaled: data_mnk["perf_true"] = (100 * y_true.flatten()).tolist() data_mnk["perf_pred"] = (100 * y_pred).tolist() else: data_mnk["perf_true"] = y_true.flatten().tolist() data_mnk["perf_pred"] = y_pred.tolist() data_mnk.sort_values(by="perf_true", inplace=True) # Plot plt.figure() marker_size = 1 par_set_ids = range(len(data_mnk.index.values)) plt.plot( par_set_ids, data_mnk["perf_true"], "b.", markersize=marker_size, label="measured performances", ) plt.xlabel("Parameter set id") plt.ylabel("Performance scaled [%]") type = "train" if train else "test" plt.title("Performance profile of parameter sets for " + str((m, n, k)) + "-triplet (" + type + ")") # Annotate x = [0, len(y_true)] y = np.array([1, 1]) perf_num = "{:2.2f}" # autotuning perf_autotuned_algo = data_mnk["perf_true"].max() plt.plot( x, perf_autotuned_algo * y, "k-", label="max (for this algo): " + perf_num.format(perf_autotuned_algo), ) # chosen idx_perf_chosen = data_mnk["perf_pred"].idxmax() perf_chosen = data_mnk["perf_true"][idx_perf_chosen] plt.plot(x, perf_chosen * y, "r-", label="chosen: " + perf_num.format(perf_chosen)) # baseline if scaled: # baseline = per algo, scale it to 0-1 perf_baseline = (100 * baseline_performances[to_string(m, n, k)] / max_performances["{}x{}x{}".format(m, n, k)]) else: perf_baseline = baseline_performances[to_string(m, n, k)] plt.plot(x, perf_baseline * y, "g-", label="baseline: " + perf_num.format(perf_baseline)) plt.legend(loc="lower right") pp.savefig()
def main(tunedir, arch): """ This script is part of the workflow for predictive modelling of optimal libcusmm parameters. For more details, see predict.md. After downloading raw data from the dedicated repository, use this script to - Compute derived training data and write it to a CSV file - Record maximum and baseline performances of (m,n,k)-triplets in JSON files """ # =============================================================================== # Read GPU properties and autotuning properties with open("kernels/gpu_properties.json") as f: gpu_properties = json.load(f)["sm_" + str(arch)] with open("kernels/autotuning_properties.json") as f: autotuning_properties = json.load(f) # =============================================================================== # Loop over algorithms max_performances_per_mnk = dict() baseline_performances_per_algo_per_mnk = { "tiny": dict(), "small": dict(), "medium": dict(), "largeDB1": dict(), "largeDB2": dict(), } for name_algo, kernel_algo in kernel_algorithm.items(): raw_training_data_filename = os.path.join( tunedir, "raw_training_data_{}.csv".format(name_algo)) print("\nReading from {}".format(raw_training_data_filename)) # Read CSV and loop over chunks chunk_size = 10000 # Number of rows of CSV file to process at a time chunk_count = 0 for data_chunk in pd.read_csv(raw_training_data_filename, chunksize=chunk_size): # Print progress chunk_count += 1 print("Read chunk {:5>}".format(chunk_count)) # Get max_performance_per_mnk max_performances = get_max_performances_per_mnk(data_chunk) max_performances_per_mnk.update( dict( zip(to_string(*max_performances.keys()), max_performances.values()))) # Get baseline_per_mnk baseline_performances_algo = get_baseline_performances_per_mnk( data_chunk, name_algo, gpu_properties, autotuning_properties) baseline_performances_per_algo_per_mnk[name_algo].update( dict( zip( to_string(*baseline_performances_algo.keys()), baseline_performances_algo.values(), ))) # Compute derived parameters data_chunk["algorithm"] = [name_algo] * len( data_chunk.index) # add 'algorithm' column manually parameter_sets = PredictiveParameters(data_chunk, gpu_properties, autotuning_properties, max_performances) pars_to_get = derived_parameters["common"] + derived_parameters[ name_algo] new_data = parameter_sets.get_features(pars_to_get) # Write derived parameters derived_training_data_filename = os.path.join( tunedir, "training_data_{}_{}.csv".format(name_algo, chunk_count - 1)) new_data[pars_to_get].to_csv(derived_training_data_filename, index=False) print("\tWrote", derived_training_data_filename) # =============================================================================== print("\nRead all raw and computed all derived data") # Print header lines & merge instructions print("\n$ # Merge instructions:") print("$ cd {}".format(tunedir)) for name_algo, kernel_algo in kernel_algorithm.items(): # Print header line derived_training_data_filename_base = "training_data_{}_{}.csv" derived_training_data_filename_chunk = derived_training_data_filename_base.format( name_algo, 0) with open(derived_training_data_filename_chunk, "r") as f: header_line = f.readline() derived_training_data_filename = "training_data_{}.csv".format( name_algo) with open(derived_training_data_filename, "w") as f: f.write(header_line) print("$ # Wrote header line to {}".format( derived_training_data_filename)) # Print merge instructions print("$ # Wrote header line to {}".format( derived_training_data_filename)) print("$ # Append training data chunks to {} by running:".format( derived_training_data_filename)) derived_training_data_filename_wildcard = derived_training_data_filename_base.format( name_algo, "*") print("$ tail -n +2 -q {to_merge} >> {training_data_file}".format( to_merge=derived_training_data_filename_wildcard, training_data_file=derived_training_data_filename, )) # Print max performances max_performances_per_mnk_file = os.path.join(tunedir, "max_performances.json") with open(max_performances_per_mnk_file, "w") as f: json.dump(max_performances_per_mnk, f) print("\nWrote maximum performances to:\n", max_performances_per_mnk_file) # Print baseline baseline_performances_per_algo_per_mnk_file = os.path.join( tunedir, "baseline_performances_by_algo.json") with open(baseline_performances_per_algo_per_mnk_file, "w") as f: json.dump(baseline_performances_per_algo_per_mnk, f) print( "\nWrote baseline performances to:\n", baseline_performances_per_algo_per_mnk_file, )
def print_merging_commands(kernel_folders, kernel_folder_pattern, tunedir): """ Print commands to execute in order to merge CSV files """ for algorithm in kernel_algorithm.keys(): for data_type in ("raw_", ""): data_type_name = ("raw" if data_type == "raw_" else "for predictive modelling") print( "\n$ # Merge instructions for algorithm", algorithm, "(", data_type_name, ")", ) training_data_file = "{data_type}training_data_{algorithm}.csv".format( data_type=data_type, algorithm=algorithm) if os.path.exists(training_data_file): print("$ # Found {}, append new training data to this file:". format(training_data_file)) else: # Find an (m, n, k) for this algorithm to get its header line for i, kernel_folder in enumerate(kernel_folders): # Find (m, n, k) match = kernel_folder_pattern.search( kernel_folder).groups() m = int(match[0]) n = int(match[1]) k = int(match[2]) file_name = os.path.join( kernel_folder, "{data_type}training_data_{mnk}_{algorithm}.csv". format( data_type=data_type, mnk=to_string(m, n, k), algorithm=algorithm, ), ) if os.path.exists(file_name): print("$ head -1 {base_file} > {training_data_file}". format( base_file=file_name, training_data_file=training_data_file, )) break else: print( "None: did not find any existing files for algorithm", algorithm, "and data", data_type_name, ) continue print( "$ tail -n +2 -q {tunedir}tune_*/{data_type}training_data_*_{algorithm}.csv >> {training_data_file}" .format( tunedir=tunedir, data_type=data_type, algorithm=algorithm, training_data_file=training_data_file, ))
def collect_training_data( kernel_folders, kernel_folder_pattern, gpu_properties, autotuning_properties, max_performances_per_mnk, baseline_performances_per_algo_per_mnk, ): """ Collect training data from log files resulting of autotuning """ n_kernels = len(kernel_folders) # For each folder: for i, kernel_folder in enumerate(kernel_folders): print("\nProcess folder {} ({}/{:,})".format(kernel_folder, i + 1, n_kernels)) # Find (m, n, k) match = kernel_folder_pattern.search(kernel_folder).groups() m = int(match[0]) n = int(match[1]) k = int(match[2]) # =============================================================================== # Collect info from log files data = read_log_file(kernel_folder, m, n, k) # Collect max performances per (m, n, k) max_performances = get_max_performances_per_mnk(data) max_performances_per_mnk.update( dict( zip(to_string(*max_performances.keys()), max_performances.values()))) # =============================================================================== # Write parameters to CSV for name_algo, kernel_algo in kernel_algorithm.items(): # if applicable to this mnk if name_algo in data["algorithm"].values: # Get the data corresponding to this algorithm data_algo = data[data["algorithm"] == name_algo] # Collect baseline performances per algo, per (m, n, k) baseline_performances_algo = get_baseline_performances_per_mnk( data_algo, name_algo, gpu_properties, autotuning_properties) baseline_performances_per_algo_per_mnk[name_algo].update( dict( zip( to_string(*baseline_performances_algo.keys()), baseline_performances_algo.values(), ))) # Does collected csv file exist already? raw_parameters_file_name = os.path.join( kernel_folder, "raw_training_data_" + to_string(m, n, k) + "_" + name_algo + ".csv", ) derived_parameters_file_name = os.path.join( kernel_folder, "training_data_" + to_string(m, n, k) + "_" + name_algo + ".csv", ) if os.path.exists(raw_parameters_file_name): print("\tFound csv file:", raw_parameters_file_name, ", skipping ...") else: # Write raw parameters pars_to_get = kernel_algo.launch_parameters + [ "perf (Gflop/s)" ] data_algo[pars_to_get].to_csv(raw_parameters_file_name, index=False) print("\tWrote", raw_parameters_file_name) if os.path.exists(derived_parameters_file_name): print( "\tFound csv file:", derived_parameters_file_name, ", skipping ...", ) else: # Compute derived parameters parameter_sets = PredictiveParameters( data_algo, gpu_properties, autotuning_properties, max_performances, ) pars_to_get = (derived_parameters["common"] + derived_parameters[name_algo]) new_df = parameter_sets.get_features(pars_to_get) data_algo.merge(new_df) # Write derived parameters data_algo[pars_to_get].to_csv(derived_parameters_file_name, index=False) print("\tWrote", derived_parameters_file_name)