def evaluate(self):
""" Evaluate the model on itself
"""
## We should be evaluating on dev dataset as well, so commenting x_test
#self.model_score = self.model.evaluate(self.x_test, self.y_test_oh, batch_size=2048)
self.model_score = self.model.evaluate(self.x_dev, self.y_dev_oh, batch_size=2048)
print("%s score = %f\n" %(self.modelName, self.model_score[1]))
##Saving atucal vs predicted predictions
##np.argmax returns the index where it see's 1 in the row
#y_pred = np.argmax(self.model.predict(self.x_test, batch_size=2048), axis=1)
y_pred = np.argmax(self.model.predict(self.x_dev, batch_size=2048), axis=1)
## vstack will stack them in 2 rows, so we use Trasnpose to get them in column stack
#output_predict = np.vstack((np.argmax(self.y_test_oh, axis=1), y_pred)).T
output_predict = np.vstack((np.argmax(self.y_dev_oh, axis=1), y_pred)).T
self.outputFile = self.resultDir + '/' + self.configName + '/' + self.modelName + '_' + str(len(self.hiddenLayer)) + 'HL_' + str(self.history.epoch[-1]+1) + 'epochs_' + '{0:.2f}'.format(self.model_score[1]*100).replace('.', 'p') + '_acc_' + "outputPredict.csv"
np.savetxt(self.outputFile, output_predict, fmt="%5.0f", delimiter=",")
##Error Analysis of the prediction
errorAnalysis(self.outputFile)
return self.model_score
def evaluateTest(self):
""" Evaluate the model on itself
"""
## We should be evaluating on dev dataset as well, so commenting x_test
self.model_score = self.model.evaluate(self.x_test,
self.y_test_oh,
batch_size=2048)
print("%s score = %f\n" % (self.modelName, self.model_score[1]))
##Saving atucal vs predicted predictions
##np.argmax returns the index where it see's 1 in the row
y_pred = np.argmax(self.model.predict(self.x_test, batch_size=2048),
axis=1)
## vstack will stack them in 2 rows, so we use Trasnpose to get them in column stack
output_predict = np.vstack((np.argmax(self.y_test_oh,
axis=1), y_pred)).T
outputFile = self.resultDir + "/outputPredict.csv"
np.savetxt(outputFile, output_predict, fmt="%5.0f", delimiter=",")
##Error Analysis of the prediction
errorAnalysis(outputFile)
return self.model_score
def evaluate(self):
""" Evaluate the model on itself
"""
## We should be evaluating on dev dataset as well, so commenting x_test
#self.model_score = self.model.evaluate(self.x_test, self.y_test_oh, batch_size=2048)
self.model_score = self.model.evaluate(self.x_dev, self.y_dev_oh, batch_size=2048)
print("%s score = %f\n" %(self.modelName, self.model_score[1]))
##Saving atucal vs predicted predictions
##np.argmax returns the index where it see's 1 in the row
#y_pred = np.argmax(self.model.predict(self.x_test, batch_size=2048), axis=1)
y_pred = np.argmax(self.model.predict(self.x_dev, batch_size=2048), axis=1)
## vstack will stack them in 2 rows, so we use Trasnpose to get them in column stack
#output_predict = np.vstack((np.argmax(self.y_test_oh, axis=1), y_pred)).T
output_predict = np.vstack((np.argmax(self.y_dev_oh, axis=1), y_pred)).T
outputFile = self.resultDir + '/' + self.modelName +'_8HLw_1000_700_500_500_300_300_256_256' + str(self.history.epoch[-1]+1) + 'epochs_' + 'Dropout_' + str(self.drop1).replace('.', 'p') + '_' + str(self.drop2).replace('.', 'p') + '_' + str(self.drop3).replace('.', 'p') + '_' + str(self.drop4).replace('.', 'p') +'_' + str(self.drop5).replace('.', 'p') + '_' + str(self.drop6).replace('.', 'p') + '_' + str(self.drop7).replace('.', 'p') + '_' + str(self.drop8).replace('.', 'p') + "_outputPredict.csv"
np.savetxt(outputFile, output_predict, fmt="%5.0f", delimiter=",")
##Error Analysis of the prediction
errorAnalysis(outputFile)
return self.model_score
## Convert from one-hot to numerical prediction
y_pred = np.argmax(model.predict(x_dev[:, :inputTraces],
batch_size=256),
axis=1)
## vstack the actual and predicted output and take transpose
output_predict = np.vstack((np.argmax(y_dev_oh, axis=1), y_pred)).T
## Save it to csv file for future analysis
## Split the modelName so that it has config and run number in the name
outputFile = modelDir + "/" + devDataConfig + "/" + "dataOf_" + devDataConfig + "_modelOf_" + "_".join(
modelName.split("_")[1:2]) + "_dev.csv"
np.savetxt(outputFile, output_predict, fmt="%5.0f", delimiter=",")
##Error Analysis
errorAnalysis(outputFile)
df = pd.read_csv(outputFile, header=None)
error_df = df[df[0] != df[1]].astype('category')
error_df[2] = error_df[0].astype(str).str.cat(error_df[1].astype(str),
sep="-")
totalCount = df[0].count()
errorCount = error_df[2].count()
accuracy = (
(df[0].count() - error_df[2].count()) / df[0].count()) * 100
## to get the accuracy of individual keys, we need to count the number of rows in error_df for the same key
## dubtract it from total data elements and divide it by the total number of data elements for each.
