def encode_srcs(data_folder, df: pd.DataFrame) -> np.array: from keras.preprocessing.sequence import pad_sequences # Get the 'unknown' vocab index. with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab: unk_index = vocab.unknown_token_index # Get list of source file names data_folder = os.path.join(data_folder, "kernels_seq") input_files = df["benchmark"].values # list of strings of benchmark names dataset = df["dataset"].values # list of strings of dataset descriptions num_files = len(input_files) num_unks = 0 seq_lengths = list() app.Log(1, "Preparing to read %d input files from folder %s", num_files, data_folder) seqs = list() for i in range(num_files): file = input_files[i] dat = dataset[i] if file[:3] == "npb": # concatenate data set size file += "_" + str(dat) file = os.path.join(data_folder, file + "_seq.csv") if os.path.exists(file): # load sequence with open(file, "r") as f: seq = f.read().splitlines() assert len(seq) > 0, "Found empty file: " + file num_unks += seq.count(str(unk_index)) seq_lengths.append(len(seq)) seqs.append([int(s) for s in seq]) else: assert True, "input file not found: " + file max_len = max(seq_lengths) app.Log( 1, "Sequence lengths: min=%d, avg=%.2f, max=%d", min(seq_lengths), np.mean(seq_lengths), max_len, ) app.Log(1, "Number of 'UNK': %d", num_unks) app.Log( 1, "Percentage of 'UNK': %.3f %% among all stmts", (num_unks * 100) / sum(seq_lengths), ) app.Log(1, "'UNK' index: %d", unk_index) encoded = np.array(pad_sequences(seqs, maxlen=max_len, value=unk_index)) return np.vstack([np.expand_dims(x, axis=0) for x in encoded]), max_len
def EncodeAndPadSources(self, df: pd.DataFrame, maxlen: typing.Optional[int] = None ) -> typing.Tuple[np.array, int]: """Encode and pad source sequences.""" # TODO(cec): This is hardcoded to OpenClDeviceMappingsDataset, and is # mighty slow. with DEEPTUNE_INST2VEC_DATA_ARCHIVE as datafolder: with inst2vec_vocabulary.VocabularyZipFile( self.vocabulary_file) as vocab: return EncodeAndPadSourcesWithInst2Vec(df, vocab, datafolder, maxlen)
def encode_srcs(data_folder, df: pd.DataFrame): """ encode and pad source code for learning """ from keras.preprocessing.sequence import pad_sequences # Get the 'unknown' vocab index. with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab: unk_index = vocab.unknown_token_index # Get list of source file names data_folder = os.path.join(data_folder, "kernels_seq") input_files = df["kernel"].values # list of strings of kernel names num_files = len(input_files) num_unks = 0 seq_lengths = list() print("\n--- Preparing to read", num_files, "input files from folder", data_folder) seqs = list() for file in input_files: file = os.path.join(data_folder, file + "_seq.csv") assert os.path.exists(file), "input file not found: " + file with open(file, "r") as f: seq = f.read().splitlines() assert len(seq) > 0, "Found empty file: " + file num_unks += seq.count(str(unk_index)) seq_lengths.append(len(seq)) seqs.append([int(s) for s in seq]) print("\tShortest sequence : {:>5}".format(min(seq_lengths))) maxlen = max(seq_lengths) print("\tLongest sequence : {:>5}".format(maxlen)) print("\tMean sequence length : {:>5} (rounded down)".format( math.floor(np.mean(seq_lengths)))) print("\tNumber of 'UNK' : {:>5}".format(num_unks)) print("\tPercentage of 'UNK' : {:>8.4} (% among all stmts)".format( (num_unks * 100) / sum(seq_lengths))) print("\t'UNK' index : {:>5}".format(unk_index)) encoded = np.array(pad_sequences(seqs, maxlen=maxlen, value=unk_index)) return np.vstack([np.expand_dims(x, axis=0) for x in encoded]), maxlen
def EncodeGraphs( self, data: typing.Iterable[ typing.Tuple[typing.Dict[str, typing.Any], llvm_util.LlvmControlFlowGraph] ], ) -> typing.Iterable[ typing.Tuple[typing.Dict[str, typing.Any], llvm_util.LlvmControlFlowGraph] ]: """Encode inst2vec attributes on graphs. Args: data: An iterator of <row,cfg> tuples. Returns: An iterator <row,cfg> tuples. """ with inst2vec_vocabulary.VocabularyZipFile(self.vocabulary_file) as vocab: # Create embedding lookup op. embedding_lookup_input_ph = tf.compat.v1.placeholder(dtype=tf.int32) normalized_embedding_matrix = tf.nn.l2_normalize( self.embedding_matrix, axis=1 ) embedding_lookup_op = tf.nn.embedding_lookup( normalized_embedding_matrix, embedding_lookup_input_ph ) with tf.compat.v1.Session() as session: for i, (row, graph) in enumerate(data): app.Log(1, "Encoding graph %d %s", i, row["program:benchmark_name"]) yield row, EncodeGraph( graph, vocab, session, embedding_lookup_op, embedding_lookup_input_ph, )
def main(argv): if len(argv) > 1: raise app.UsageError("Unrecognized command line flags.") #################################################################################################################### # Setup # Get flag values embeddings = task_utils.ReadEmbeddingFileFromFlags() input_data = FLAGS.input_data out = FLAGS.out if not os.path.exists(out): os.makedirs(out) device = FLAGS.device assert device in [ "all", "Cypress", "Tahiti", "Fermi", "Kepler", ], "Choose device among: all, Cypress, Tahiti, Fermi, Kepler" dense_layer_size = FLAGS.dense_layer print_summary = FLAGS.print_summary num_epochs = FLAGS.num_epochs batch_size = FLAGS.batch_size # Unpack data archive if necessary. if not os.path.exists(os.path.join(input_data, "kernels_ir")): dataset = bazelutil.DataArchive( "phd/deeplearning/ncc/published_results/task_threadcoarsening.zip") dataset.ExtractAll(pathlib.Path(input_data)) with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab: task_utils.CreateSeqDirFromIr(os.path.join(input_data, "kernels_ir"), vocab) #################################################################################################################### # Reference values # Values copied from papers and github magni_pl_sp_vals = [1.21, 1.01, 0.86, 0.94] magni_sp_mean = 1.005 deeptune_pl_sp_vals = [1.10, 1.05, 1.10, 0.99] deeptune_sp_mean = 1.06 deeptuneTL_pl_sp_vals = [1.17, 1.23, 1.14, 0.93] deeptuneTL_sp_mean = 1.1175 #################################################################################################################### # Train model # Evaluate NCC_threadcoarsening print("\nEvaluating NCC_threadcoarsening ...") ncc_threadcoarsening = evaluate( NCC_threadcoarsening(), device, input_data, out, embeddings, dense_layer_size, print_summary, num_epochs, batch_size, ) #################################################################################################################### # Print results print( "\n", ncc_threadcoarsening.groupby("Platform")["Platform", "Speedup", "Oracle"].mean(), ) d = np.array([ncc_threadcoarsening[["Speedup", "Oracle"]].mean()]).T print( "\n", pd.DataFrame(d, columns=["DeepTuneInst2Vec"], index=["Speedup", "Oracle"]), ) # Model comparison: speedups print("\nModel comparison: speedups") d = list() d.append(np.append(magni_pl_sp_vals, magni_sp_mean)) d.append(np.append(deeptune_pl_sp_vals, deeptune_sp_mean)) d.append(np.append(deeptuneTL_pl_sp_vals, deeptuneTL_sp_mean)) d.append( np.append( ncc_threadcoarsening.groupby(["Platform" ])["Speedup"].mean().values, ncc_threadcoarsening["Speedup"].mean(), )) if FLAGS.device == "all": d = np.array(d).T.reshape(5, 4) devs = [ "AMD Radeon HD 5900", "AMD Tahiti 7970", "NVIDIA GTX 480", "NVIDIA Tesla K20c", "Average", ] else: d = np.array(d).T.reshape(1, 4) devs = [_FLAG_TO_DEVICE_NAME[FLAGS.device]] print( "\n", pd.DataFrame( d, columns=[ "Magni et al.", "DeepTune", "DeepTuneTL", "DeepTuneInst2Vec" ], index=devs, ), )
def main(argv): if len(argv) > 1: raise app.UsageError("Unrecognized command line flags.") #################################################################################################################### # Setup # Get flag values embeddings = task_utils.ReadEmbeddingFileFromFlags() folder_results = FLAGS.out assert ( len(folder_results) > 0 ), "Please specify a path to the results folder using --folder_results" folder_data = FLAGS.input_data dense_layer_size = FLAGS.dense_layer print_summary = FLAGS.print_summary num_epochs = FLAGS.num_epochs batch_size = FLAGS.batch_size train_samples = FLAGS.train_samples pathlib.Path(folder_data).mkdir(parents=True, exist_ok=True) # Acquire data if not os.path.exists(folder_data + "_train"): # Download data task_utils.download_and_unzip( "https://polybox.ethz.ch/index.php/s/JOBjrfmAjOeWCyl/download", "classifyapp_training_data", folder_data, ) with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab: task_utils.CreateSeqDirFromIr(folder_data + "_train", vocab) assert os.path.exists(folder_data + "_val"), ("Folder not found: " + folder_data + "_val") task_utils.CreateSeqDirFromIr(folder_data + "_val", vocab) assert os.path.exists(folder_data + "_test"), ("Folder not found: " + folder_data + "_test") task_utils.CreateSeqDirFromIr(folder_data + "_test", vocab) # Create directories if they do not exist if not os.path.exists(folder_results): os.makedirs(folder_results) if not os.path.exists(os.path.join(folder_results, "models")): os.makedirs(os.path.join(folder_results, "models")) if not os.path.exists(os.path.join(folder_results, "predictions")): os.makedirs(os.path.join(folder_results, "predictions")) #################################################################################################################### # Train model # Evaluate Classifyapp print("\nEvaluating ClassifyappInst2Vec ...") classifyapp_accuracy = evaluate( NCC_classifyapp(), embeddings, folder_data, train_samples, folder_results, dense_layer_size, print_summary, num_epochs, batch_size, ) #################################################################################################################### # Print results print( "\nTest accuracy:", sum(classifyapp_accuracy) * 100 / len(classifyapp_accuracy), "%", )
def evaluate( model, embeddings, folder_data, samples_per_class, folder_results, dense_layer_size, print_summary, num_epochs, batch_size, ): # Set seed for reproducibility seed = 204 #################################################################################################################### # Get data vsamples_per_class = FLAGS.vsamples # Data acquisition num_classes = 104 y_train = np.empty(0) # training X_train = list() folder_data_train = folder_data + "_train" y_val = np.empty(0) # validation X_val = list() folder_data_val = folder_data + "_val" y_test = np.empty(0) # testing X_test = list() folder_data_test = folder_data + "_test" print("Getting file names for", num_classes, "classes from folders:") print(folder_data_train) print(folder_data_val) print(folder_data_test) for i in range(1, num_classes + 1): # loop over classes # training: Read data file names folder = os.path.join(folder_data_train, str(i)) assert os.path.exists(folder), "Folder: " + folder + " does not exist" print("\ttraining : Read file names from folder ", folder) listing = os.listdir(folder + "/") seq_files = [ os.path.join(folder, f) for f in listing if f[-4:] == ".rec" ] # training: Randomly pick programs assert len(seq_files) >= samples_per_class, ( "Cannot sample " + str(samples_per_class) + " from " + str(len(seq_files)) + " files found in " + folder) X_train += resample(seq_files, replace=False, n_samples=samples_per_class, random_state=seed) y_train = np.concatenate( [y_train, np.array([int(i)] * samples_per_class, dtype=np.int32)]) # validation: Read data file names folder = os.path.join(folder_data_val, str(i)) assert os.path.exists(folder), "Folder: " + folder + " does not exist" print("\tvalidation: Read file names from folder ", folder) listing = os.listdir(folder + "/") seq_files = [ os.path.join(folder, f) for f in listing if f[-4:] == ".rec" ] # validation: Randomly pick programs if vsamples_per_class > 0: assert len(seq_files) >= vsamples_per_class, ( "Cannot sample " + str(vsamples_per_class) + " from " + str(len(seq_files)) + " files found in " + folder) X_val += resample( seq_files, replace=False, n_samples=vsamples_per_class, random_state=seed, ) y_val = np.concatenate([ y_val, np.array([int(i)] * vsamples_per_class, dtype=np.int32) ]) else: assert len(seq_files) > 0, "No .rec files found in" + folder X_val += seq_files y_val = np.concatenate( [y_val, np.array([int(i)] * len(seq_files), dtype=np.int32)]) # test: Read data file names folder = os.path.join(folder_data_test, str(i)) assert os.path.exists(folder), "Folder: " + folder + " does not exist" print("\ttest : Read file names from folder ", folder) listing = os.listdir(folder + "/") seq_files = [ os.path.join(folder, f) for f in listing if f[-4:] == ".rec" ] assert len(seq_files) > 0, "No .rec files found in" + folder X_test += seq_files y_test = np.concatenate( [y_test, np.array([int(i)] * len(seq_files), dtype=np.int32)]) # Get the 'unknown' vocab index. with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab: unk_index = vocab.unknown_token_index # Encode source codes and get max. sequence length X_seq_train, maxlen_train = encode_srcs(X_train, "training", unk_index) X_seq_val, maxlen_val = encode_srcs(X_val, "validation", unk_index) X_seq_test, maxlen_test = encode_srcs(X_test, "testing", unk_index) maxlen = max(maxlen_train, maxlen_test, maxlen_val) print("Max. sequence length overall:", maxlen) print("Padding sequences") X_seq_train = pad_src(X_seq_train, maxlen, unk_index) X_seq_val = pad_src(X_seq_val, maxlen, unk_index) X_seq_test = pad_src(X_seq_test, maxlen, unk_index) # Get one-hot vectors for classification print("YTRAIN\n", y_train) y_1hot_train = get_onehot(y_train, num_classes) y_1hot_val = get_onehot(y_val, num_classes) #################################################################################################################### # Setup paths # Set up names paths model_name = model.__name__ model_path = os.path.join(folder_results, "classifyapp/models/{}.model".format(model_name)) predictions_path = os.path.join( folder_results, "classifyapp/predictions/{}.result".format(model_name)) # If predictions have already been made with these embeddings, load them if fs.exists(predictions_path): print("\tFound predictions in", predictions_path, ", skipping...") with open(predictions_path, "rb") as infile: p = pickle.load(infile) else: # could not find predictions already computed with these embeddings # Embeddings from third_party.py.tensorflow import tf # for embeddings lookup embedding_matrix_normalized = tf.nn.l2_normalize(embeddings, axis=1) vocabulary_size, embedding_dimension = embedding_matrix_normalized.shape print("XSEQ:\n", X_seq_train) print("EMB:\n", embedding_matrix_normalized) gen_test = EmbeddingPredictionSequence(batch_size, X_seq_test, embedding_matrix_normalized) # If models have already been made with these embeddings, load them if fs.exists(model_path): print("\n\tFound trained model in", model_path, ", skipping...") model.restore(model_path) else: # could not find models already computed with these embeddings gen_train = EmbeddingSequence(batch_size, X_seq_train, y_1hot_train, embedding_matrix_normalized) gen_val = EmbeddingSequence(batch_size, X_seq_val, y_1hot_val, embedding_matrix_normalized) ############################################################################################################ # Train # Create a new model and train it print("\n--- Initializing model...") model.init( seed=seed, maxlen=maxlen, embedding_dim=int(embedding_dimension), num_classes=num_classes, dense_layer_size=dense_layer_size, ) if print_summary: model.model.summary() print("\n--- Training model...") model.train_gen( train_generator=gen_train, validation_generator=gen_val, verbose=True, epochs=num_epochs, ) # Save the model fs.mkdir(fs.dirname(model_path)) model.save(model_path) print("\tsaved model to", model_path) ################################################################################################################ # Test # Test model print("\n--- Testing model...") p = model.predict_gen(generator=gen_test)[0] # cache the prediction fs.mkdir(fs.dirname(predictions_path)) with open(predictions_path, "wb") as outfile: pickle.dump(p, outfile) print("\tsaved predictions to", predictions_path) #################################################################################################################### # Return accuracy accuracy = p == y_test # prediction accuracy return accuracy
def vocab() -> vocabulary.VocabularyZipFile: """Test fixture which yields a vocabulary zip file instance as a ctx mngr.""" with vocabulary.VocabularyZipFile(VOCABULARY_PATH) as v: yield v
def main(argv): if len(argv) > 1: raise app.UsageError("Unrecognized command line flags.") # Don't truncate output when printing pandas tables. pd.set_option("display.max_columns", None) pd.set_option("display.max_rows", None) # Setup # Get flag values embeddings = task_utils.ReadEmbeddingFileFromFlags() out = FLAGS.out if not os.path.exists(out): os.makedirs(out) device = FLAGS.device assert device in [ "all", "amd", "nvidia", ], "Choose device among: all, amd, nvidia" dense_layer_size = FLAGS.dense_layer print_summary = FLAGS.print_summary num_epochs = FLAGS.num_epochs batch_size = FLAGS.batch_size input_data = FLAGS.input_data # Unpack data archive if necessary. if not os.path.exists(os.path.join(input_data, "kernels_ir")): dataset = bazelutil.DataArchive( "phd/deeplearning/ncc/published_results/task_devmap.zip") dataset.ExtractAll(pathlib.Path(input_data)) with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab: task_utils.CreateSeqDirFromIr(os.path.join(input_data, "kernels_ir"), vocab) # Reference values copied from: # https://github.com/ChrisCummins/paper-end2end-dl/blob/master/code/Case%20Study%20A.ipynb static_pred_vals = [58.823529, 56.911765] static_pred_mean = 57.867647 static_sp_vals = [1.0, 1.0] static_sp_mean = 1.0 grewe_pred_vals = [73.382353, 72.941176] grewe_pred_mean = 73.161765 grewe_sp_vals = [2.905822, 1.264801] grewe_sp_mean = 2.085312 deeptune_pred_vals = [83.676471, 80.294118] deeptune_pred_mean = 81.985294 deeptune_sp_vals = [3.335612, 1.412222] deeptune_sp_mean = 2.373917 # Train model app.Log(1, "Evaluating ncc model") ncc_devmap = evaluate( NCC_devmap(), device, input_data, out, embeddings, dense_layer_size, print_summary, num_epochs, batch_size, ) # Print results print("--- Prediction results") print( ncc_devmap.groupby(["Platform", "Benchmark Suite"])["Platform", "Correct?", "Speedup"].mean()) print("--- Prediction results (summarized)") print( ncc_devmap.groupby(["Platform"])["Platform", "Correct?", "Speedup"].mean()) # Model comparison: prediction accuracy print("--- Model comparison: prediction accuracy") d = list() d.append(np.append(static_pred_vals, static_pred_mean)) d.append(np.append(grewe_pred_vals, grewe_pred_mean)) d.append(np.append(deeptune_pred_vals, deeptune_pred_mean)) d.append( np.append( ncc_devmap.groupby(["Platform"])["Correct?"].mean().values * 100, ncc_devmap["Correct?"].mean() * 100, )) d = np.array(d).T.reshape(3, 4) print( pd.DataFrame( d, columns=[ "Static mapping", "Grewe et al.", "DeepTune", "DeepTuneInst2Vec", ], index=["AMD Tahiti 7970", "NVIDIA GTX 970", "Average"], )) # Model comparison: speedups print("--- Model comparison: speedups") d = list() d.append(np.append(static_sp_vals, static_sp_mean)) d.append(np.append(grewe_sp_vals, grewe_sp_mean)) d.append(np.append(deeptune_sp_vals, deeptune_sp_mean)) d.append( np.append( ncc_devmap.groupby(["Platform"])["Speedup"].mean().values, ncc_devmap["Speedup"].mean(), )) d = np.array(d).T.reshape(3, 4) print( pd.DataFrame( d, columns=[ "Static mapping", "Grewe et al.", "DeepTune", "DeepTuneInst2Vec", ], index=["AMD Tahiti 7970", "NVIDIA GTX 970", "Average"], )) app.Log(1, "done")