def encode_srcs(data_folder, df: pd.DataFrame) -> np.array:
from keras.preprocessing.sequence import pad_sequences
# Get the 'unknown' vocab index.
with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab:
unk_index = vocab.unknown_token_index
# Get list of source file names
data_folder = os.path.join(data_folder, "kernels_seq")
input_files = df["benchmark"].values # list of strings of benchmark names
dataset = df["dataset"].values # list of strings of dataset descriptions
num_files = len(input_files)
num_unks = 0
seq_lengths = list()
app.Log(1, "Preparing to read %d input files from folder %s", num_files,
data_folder)
seqs = list()
for i in range(num_files):
file = input_files[i]
dat = dataset[i]
if file[:3] == "npb":
# concatenate data set size
file += "_" + str(dat)
file = os.path.join(data_folder, file + "_seq.csv")
if os.path.exists(file):
# load sequence
with open(file, "r") as f:
seq = f.read().splitlines()
assert len(seq) > 0, "Found empty file: " + file
num_unks += seq.count(str(unk_index))
seq_lengths.append(len(seq))
seqs.append([int(s) for s in seq])
else:
assert True, "input file not found: " + file
max_len = max(seq_lengths)
app.Log(
1,
"Sequence lengths: min=%d, avg=%.2f, max=%d",
min(seq_lengths),
np.mean(seq_lengths),
max_len,
)
app.Log(1, "Number of 'UNK': %d", num_unks)
app.Log(
1,
"Percentage of 'UNK': %.3f %% among all stmts",
(num_unks * 100) / sum(seq_lengths),
)
app.Log(1, "'UNK' index: %d", unk_index)
encoded = np.array(pad_sequences(seqs, maxlen=max_len, value=unk_index))
return np.vstack([np.expand_dims(x, axis=0) for x in encoded]), max_len
def EncodeAndPadSources(self,
df: pd.DataFrame,
maxlen: typing.Optional[int] = None
) -> typing.Tuple[np.array, int]:
"""Encode and pad source sequences."""
# TODO(cec): This is hardcoded to OpenClDeviceMappingsDataset, and is
# mighty slow.
with DEEPTUNE_INST2VEC_DATA_ARCHIVE as datafolder:
with inst2vec_vocabulary.VocabularyZipFile(
self.vocabulary_file) as vocab:
return EncodeAndPadSourcesWithInst2Vec(df, vocab, datafolder,
maxlen)
def encode_srcs(data_folder, df: pd.DataFrame):
"""
encode and pad source code for learning
"""
from keras.preprocessing.sequence import pad_sequences
# Get the 'unknown' vocab index.
with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab:
unk_index = vocab.unknown_token_index
# Get list of source file names
data_folder = os.path.join(data_folder, "kernels_seq")
input_files = df["kernel"].values # list of strings of kernel names
num_files = len(input_files)
num_unks = 0
seq_lengths = list()
print("\n--- Preparing to read", num_files, "input files from folder",
data_folder)
seqs = list()
for file in input_files:
file = os.path.join(data_folder, file + "_seq.csv")
assert os.path.exists(file), "input file not found: " + file
with open(file, "r") as f:
seq = f.read().splitlines()
assert len(seq) > 0, "Found empty file: " + file
num_unks += seq.count(str(unk_index))
seq_lengths.append(len(seq))
seqs.append([int(s) for s in seq])
print("\tShortest sequence : {:>5}".format(min(seq_lengths)))
maxlen = max(seq_lengths)
print("\tLongest sequence : {:>5}".format(maxlen))
print("\tMean sequence length : {:>5} (rounded down)".format(
math.floor(np.mean(seq_lengths))))
print("\tNumber of 'UNK' : {:>5}".format(num_unks))
print("\tPercentage of 'UNK' : {:>8.4} (% among all stmts)".format(
(num_unks * 100) / sum(seq_lengths)))
print("\t'UNK' index : {:>5}".format(unk_index))
encoded = np.array(pad_sequences(seqs, maxlen=maxlen, value=unk_index))
return np.vstack([np.expand_dims(x, axis=0) for x in encoded]), maxlen
def EncodeGraphs(
self,
data: typing.Iterable[
typing.Tuple[typing.Dict[str, typing.Any], llvm_util.LlvmControlFlowGraph]
],
) -> typing.Iterable[
typing.Tuple[typing.Dict[str, typing.Any], llvm_util.LlvmControlFlowGraph]
]:
"""Encode inst2vec attributes on graphs.
Args:
data: An iterator of <row,cfg> tuples.
Returns:
An iterator <row,cfg> tuples.
"""
with inst2vec_vocabulary.VocabularyZipFile(self.vocabulary_file) as vocab:
# Create embedding lookup op.
embedding_lookup_input_ph = tf.compat.v1.placeholder(dtype=tf.int32)
normalized_embedding_matrix = tf.nn.l2_normalize(
self.embedding_matrix, axis=1
)
embedding_lookup_op = tf.nn.embedding_lookup(
normalized_embedding_matrix, embedding_lookup_input_ph
)
with tf.compat.v1.Session() as session:
for i, (row, graph) in enumerate(data):
app.Log(1, "Encoding graph %d %s", i, row["program:benchmark_name"])
yield row, EncodeGraph(
graph,
vocab,
session,
embedding_lookup_op,
embedding_lookup_input_ph,
)
def main(argv):
if len(argv) > 1:
raise app.UsageError("Unrecognized command line flags.")
####################################################################################################################
# Setup
# Get flag values
embeddings = task_utils.ReadEmbeddingFileFromFlags()
input_data = FLAGS.input_data
out = FLAGS.out
if not os.path.exists(out):
os.makedirs(out)
device = FLAGS.device
assert device in [
"all",
"Cypress",
"Tahiti",
"Fermi",
"Kepler",
], "Choose device among: all, Cypress, Tahiti, Fermi, Kepler"
dense_layer_size = FLAGS.dense_layer
print_summary = FLAGS.print_summary
num_epochs = FLAGS.num_epochs
batch_size = FLAGS.batch_size
# Unpack data archive if necessary.
if not os.path.exists(os.path.join(input_data, "kernels_ir")):
dataset = bazelutil.DataArchive(
"phd/deeplearning/ncc/published_results/task_threadcoarsening.zip")
dataset.ExtractAll(pathlib.Path(input_data))
with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab:
task_utils.CreateSeqDirFromIr(os.path.join(input_data, "kernels_ir"),
vocab)
####################################################################################################################
# Reference values
# Values copied from papers and github
magni_pl_sp_vals = [1.21, 1.01, 0.86, 0.94]
magni_sp_mean = 1.005
deeptune_pl_sp_vals = [1.10, 1.05, 1.10, 0.99]
deeptune_sp_mean = 1.06
deeptuneTL_pl_sp_vals = [1.17, 1.23, 1.14, 0.93]
deeptuneTL_sp_mean = 1.1175
####################################################################################################################
# Train model
# Evaluate NCC_threadcoarsening
print("\nEvaluating NCC_threadcoarsening ...")
ncc_threadcoarsening = evaluate(
NCC_threadcoarsening(),
device,
input_data,
out,
embeddings,
dense_layer_size,
print_summary,
num_epochs,
batch_size,
)
####################################################################################################################
# Print results
print(
"\n",
ncc_threadcoarsening.groupby("Platform")["Platform", "Speedup",
"Oracle"].mean(),
)
d = np.array([ncc_threadcoarsening[["Speedup", "Oracle"]].mean()]).T
print(
"\n",
pd.DataFrame(d,
columns=["DeepTuneInst2Vec"],
index=["Speedup", "Oracle"]),
)
# Model comparison: speedups
print("\nModel comparison: speedups")
d = list()
d.append(np.append(magni_pl_sp_vals, magni_sp_mean))
d.append(np.append(deeptune_pl_sp_vals, deeptune_sp_mean))
d.append(np.append(deeptuneTL_pl_sp_vals, deeptuneTL_sp_mean))
d.append(
np.append(
ncc_threadcoarsening.groupby(["Platform"
])["Speedup"].mean().values,
ncc_threadcoarsening["Speedup"].mean(),
))
if FLAGS.device == "all":
d = np.array(d).T.reshape(5, 4)
devs = [
"AMD Radeon HD 5900",
"AMD Tahiti 7970",
"NVIDIA GTX 480",
"NVIDIA Tesla K20c",
"Average",
]
else:
d = np.array(d).T.reshape(1, 4)
devs = [_FLAG_TO_DEVICE_NAME[FLAGS.device]]
print(
"\n",
pd.DataFrame(
d,
columns=[
"Magni et al.", "DeepTune", "DeepTuneTL", "DeepTuneInst2Vec"
],
index=devs,
),
)
def main(argv):
if len(argv) > 1:
raise app.UsageError("Unrecognized command line flags.")
####################################################################################################################
# Setup
# Get flag values
embeddings = task_utils.ReadEmbeddingFileFromFlags()
folder_results = FLAGS.out
assert (
len(folder_results) > 0
), "Please specify a path to the results folder using --folder_results"
folder_data = FLAGS.input_data
dense_layer_size = FLAGS.dense_layer
print_summary = FLAGS.print_summary
num_epochs = FLAGS.num_epochs
batch_size = FLAGS.batch_size
train_samples = FLAGS.train_samples
pathlib.Path(folder_data).mkdir(parents=True, exist_ok=True)
# Acquire data
if not os.path.exists(folder_data + "_train"):
# Download data
task_utils.download_and_unzip(
"https://polybox.ethz.ch/index.php/s/JOBjrfmAjOeWCyl/download",
"classifyapp_training_data",
folder_data,
)
with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab:
task_utils.CreateSeqDirFromIr(folder_data + "_train", vocab)
assert os.path.exists(folder_data + "_val"), ("Folder not found: " +
folder_data + "_val")
task_utils.CreateSeqDirFromIr(folder_data + "_val", vocab)
assert os.path.exists(folder_data + "_test"), ("Folder not found: " +
folder_data + "_test")
task_utils.CreateSeqDirFromIr(folder_data + "_test", vocab)
# Create directories if they do not exist
if not os.path.exists(folder_results):
os.makedirs(folder_results)
if not os.path.exists(os.path.join(folder_results, "models")):
os.makedirs(os.path.join(folder_results, "models"))
if not os.path.exists(os.path.join(folder_results, "predictions")):
os.makedirs(os.path.join(folder_results, "predictions"))
####################################################################################################################
# Train model
# Evaluate Classifyapp
print("\nEvaluating ClassifyappInst2Vec ...")
classifyapp_accuracy = evaluate(
NCC_classifyapp(),
embeddings,
folder_data,
train_samples,
folder_results,
dense_layer_size,
print_summary,
num_epochs,
batch_size,
)
####################################################################################################################
# Print results
print(
"\nTest accuracy:",
sum(classifyapp_accuracy) * 100 / len(classifyapp_accuracy),
"%",
)
def evaluate(
model,
embeddings,
folder_data,
samples_per_class,
folder_results,
dense_layer_size,
print_summary,
num_epochs,
batch_size,
):
# Set seed for reproducibility
seed = 204
####################################################################################################################
# Get data
vsamples_per_class = FLAGS.vsamples
# Data acquisition
num_classes = 104
y_train = np.empty(0) # training
X_train = list()
folder_data_train = folder_data + "_train"
y_val = np.empty(0) # validation
X_val = list()
folder_data_val = folder_data + "_val"
y_test = np.empty(0) # testing
X_test = list()
folder_data_test = folder_data + "_test"
print("Getting file names for", num_classes, "classes from folders:")
print(folder_data_train)
print(folder_data_val)
print(folder_data_test)
for i in range(1, num_classes + 1): # loop over classes
# training: Read data file names
folder = os.path.join(folder_data_train, str(i))
assert os.path.exists(folder), "Folder: " + folder + " does not exist"
print("\ttraining : Read file names from folder ", folder)
listing = os.listdir(folder + "/")
seq_files = [
os.path.join(folder, f) for f in listing if f[-4:] == ".rec"
]
# training: Randomly pick programs
assert len(seq_files) >= samples_per_class, (
"Cannot sample " + str(samples_per_class) + " from " +
str(len(seq_files)) + " files found in " + folder)
X_train += resample(seq_files,
replace=False,
n_samples=samples_per_class,
random_state=seed)
y_train = np.concatenate(
[y_train,
np.array([int(i)] * samples_per_class, dtype=np.int32)])
# validation: Read data file names
folder = os.path.join(folder_data_val, str(i))
assert os.path.exists(folder), "Folder: " + folder + " does not exist"
print("\tvalidation: Read file names from folder ", folder)
listing = os.listdir(folder + "/")
seq_files = [
os.path.join(folder, f) for f in listing if f[-4:] == ".rec"
]
# validation: Randomly pick programs
if vsamples_per_class > 0:
assert len(seq_files) >= vsamples_per_class, (
"Cannot sample " + str(vsamples_per_class) + " from " +
str(len(seq_files)) + " files found in " + folder)
X_val += resample(
seq_files,
replace=False,
n_samples=vsamples_per_class,
random_state=seed,
)
y_val = np.concatenate([
y_val,
np.array([int(i)] * vsamples_per_class, dtype=np.int32)
])
else:
assert len(seq_files) > 0, "No .rec files found in" + folder
X_val += seq_files
y_val = np.concatenate(
[y_val,
np.array([int(i)] * len(seq_files), dtype=np.int32)])
# test: Read data file names
folder = os.path.join(folder_data_test, str(i))
assert os.path.exists(folder), "Folder: " + folder + " does not exist"
print("\ttest : Read file names from folder ", folder)
listing = os.listdir(folder + "/")
seq_files = [
os.path.join(folder, f) for f in listing if f[-4:] == ".rec"
]
assert len(seq_files) > 0, "No .rec files found in" + folder
X_test += seq_files
y_test = np.concatenate(
[y_test,
np.array([int(i)] * len(seq_files), dtype=np.int32)])
# Get the 'unknown' vocab index.
with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab:
unk_index = vocab.unknown_token_index
# Encode source codes and get max. sequence length
X_seq_train, maxlen_train = encode_srcs(X_train, "training", unk_index)
X_seq_val, maxlen_val = encode_srcs(X_val, "validation", unk_index)
X_seq_test, maxlen_test = encode_srcs(X_test, "testing", unk_index)
maxlen = max(maxlen_train, maxlen_test, maxlen_val)
print("Max. sequence length overall:", maxlen)
print("Padding sequences")
X_seq_train = pad_src(X_seq_train, maxlen, unk_index)
X_seq_val = pad_src(X_seq_val, maxlen, unk_index)
X_seq_test = pad_src(X_seq_test, maxlen, unk_index)
# Get one-hot vectors for classification
print("YTRAIN\n", y_train)
y_1hot_train = get_onehot(y_train, num_classes)
y_1hot_val = get_onehot(y_val, num_classes)
####################################################################################################################
# Setup paths
# Set up names paths
model_name = model.__name__
model_path = os.path.join(folder_results,
"classifyapp/models/{}.model".format(model_name))
predictions_path = os.path.join(
folder_results, "classifyapp/predictions/{}.result".format(model_name))
# If predictions have already been made with these embeddings, load them
if fs.exists(predictions_path):
print("\tFound predictions in", predictions_path, ", skipping...")
with open(predictions_path, "rb") as infile:
p = pickle.load(infile)
else: # could not find predictions already computed with these embeddings
# Embeddings
from third_party.py.tensorflow import tf # for embeddings lookup
embedding_matrix_normalized = tf.nn.l2_normalize(embeddings, axis=1)
vocabulary_size, embedding_dimension = embedding_matrix_normalized.shape
print("XSEQ:\n", X_seq_train)
print("EMB:\n", embedding_matrix_normalized)
gen_test = EmbeddingPredictionSequence(batch_size, X_seq_test,
embedding_matrix_normalized)
# If models have already been made with these embeddings, load them
if fs.exists(model_path):
print("\n\tFound trained model in", model_path, ", skipping...")
model.restore(model_path)
else: # could not find models already computed with these embeddings
gen_train = EmbeddingSequence(batch_size, X_seq_train,
y_1hot_train,
embedding_matrix_normalized)
gen_val = EmbeddingSequence(batch_size, X_seq_val, y_1hot_val,
embedding_matrix_normalized)
############################################################################################################
# Train
# Create a new model and train it
print("\n--- Initializing model...")
model.init(
seed=seed,
maxlen=maxlen,
embedding_dim=int(embedding_dimension),
num_classes=num_classes,
dense_layer_size=dense_layer_size,
)
if print_summary:
model.model.summary()
print("\n--- Training model...")
model.train_gen(
train_generator=gen_train,
validation_generator=gen_val,
verbose=True,
epochs=num_epochs,
)
# Save the model
fs.mkdir(fs.dirname(model_path))
model.save(model_path)
print("\tsaved model to", model_path)
################################################################################################################
# Test
# Test model
print("\n--- Testing model...")
p = model.predict_gen(generator=gen_test)[0]
# cache the prediction
fs.mkdir(fs.dirname(predictions_path))
with open(predictions_path, "wb") as outfile:
pickle.dump(p, outfile)
print("\tsaved predictions to", predictions_path)
####################################################################################################################
# Return accuracy
accuracy = p == y_test # prediction accuracy
return accuracy
def vocab() -> vocabulary.VocabularyZipFile:
"""Test fixture which yields a vocabulary zip file instance as a ctx mngr."""
with vocabulary.VocabularyZipFile(VOCABULARY_PATH) as v:
yield v
def main(argv):
if len(argv) > 1:
raise app.UsageError("Unrecognized command line flags.")
# Don't truncate output when printing pandas tables.
pd.set_option("display.max_columns", None)
pd.set_option("display.max_rows", None)
# Setup
# Get flag values
embeddings = task_utils.ReadEmbeddingFileFromFlags()
out = FLAGS.out
if not os.path.exists(out):
os.makedirs(out)
device = FLAGS.device
assert device in [
"all",
"amd",
"nvidia",
], "Choose device among: all, amd, nvidia"
dense_layer_size = FLAGS.dense_layer
print_summary = FLAGS.print_summary
num_epochs = FLAGS.num_epochs
batch_size = FLAGS.batch_size
input_data = FLAGS.input_data
# Unpack data archive if necessary.
if not os.path.exists(os.path.join(input_data, "kernels_ir")):
dataset = bazelutil.DataArchive(
"phd/deeplearning/ncc/published_results/task_devmap.zip")
dataset.ExtractAll(pathlib.Path(input_data))
with vocabulary.VocabularyZipFile(FLAGS.vocabulary_zip_path) as vocab:
task_utils.CreateSeqDirFromIr(os.path.join(input_data, "kernels_ir"),
vocab)
# Reference values copied from:
# https://github.com/ChrisCummins/paper-end2end-dl/blob/master/code/Case%20Study%20A.ipynb
static_pred_vals = [58.823529, 56.911765]
static_pred_mean = 57.867647
static_sp_vals = [1.0, 1.0]
static_sp_mean = 1.0
grewe_pred_vals = [73.382353, 72.941176]
grewe_pred_mean = 73.161765
grewe_sp_vals = [2.905822, 1.264801]
grewe_sp_mean = 2.085312
deeptune_pred_vals = [83.676471, 80.294118]
deeptune_pred_mean = 81.985294
deeptune_sp_vals = [3.335612, 1.412222]
deeptune_sp_mean = 2.373917
# Train model
app.Log(1, "Evaluating ncc model")
ncc_devmap = evaluate(
NCC_devmap(),
device,
input_data,
out,
embeddings,
dense_layer_size,
print_summary,
num_epochs,
batch_size,
)
# Print results
print("--- Prediction results")
print(
ncc_devmap.groupby(["Platform",
"Benchmark Suite"])["Platform", "Correct?",
"Speedup"].mean())
print("--- Prediction results (summarized)")
print(
ncc_devmap.groupby(["Platform"])["Platform", "Correct?",
"Speedup"].mean())
# Model comparison: prediction accuracy
print("--- Model comparison: prediction accuracy")
d = list()
d.append(np.append(static_pred_vals, static_pred_mean))
d.append(np.append(grewe_pred_vals, grewe_pred_mean))
d.append(np.append(deeptune_pred_vals, deeptune_pred_mean))
d.append(
np.append(
ncc_devmap.groupby(["Platform"])["Correct?"].mean().values * 100,
ncc_devmap["Correct?"].mean() * 100,
))
d = np.array(d).T.reshape(3, 4)
print(
pd.DataFrame(
d,
columns=[
"Static mapping",
"Grewe et al.",
"DeepTune",
"DeepTuneInst2Vec",
],
index=["AMD Tahiti 7970", "NVIDIA GTX 970", "Average"],
))
# Model comparison: speedups
print("--- Model comparison: speedups")
d = list()
d.append(np.append(static_sp_vals, static_sp_mean))
d.append(np.append(grewe_sp_vals, grewe_sp_mean))
d.append(np.append(deeptune_sp_vals, deeptune_sp_mean))
d.append(
np.append(
ncc_devmap.groupby(["Platform"])["Speedup"].mean().values,
ncc_devmap["Speedup"].mean(),
))
d = np.array(d).T.reshape(3, 4)
print(
pd.DataFrame(
d,
columns=[
"Static mapping",
"Grewe et al.",
"DeepTune",
"DeepTuneInst2Vec",
],
index=["AMD Tahiti 7970", "NVIDIA GTX 970", "Average"],
))
app.Log(1, "done")
