def model_evaluate(loaded_model, folderName='./rtt_series/valid_data'):
# evaluate loaded model on test data
SAMPLE_LEN = 100
file_csv = os.listdir(folderName)
N_SAMPLE = len(file_csv)
X = np.array([])
Y = np.array([])
for f in file_csv:
fileName = folderName + '/' + f
test = csvio.csv2list(fileName, "rtt", sep=';', decimal='.')
cp = csvio.csv2list(fileName, "cp", sep=';', decimal='.')
# Pre-Treatment
min_test = min(test)
for i in range(len(test)):
test[i] = test[i] - min_test
X = np.concatenate([X, test])
Y = np.concatenate([Y, cp])
X = X.reshape(N_SAMPLE, SAMPLE_LEN, 1)
Y = Y.reshape(N_SAMPLE, SAMPLE_LEN, 1)
loaded_model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
score = loaded_model.evaluate(X, Y, verbose=1)
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1] * 100))
print("%s: %.2f" % (loaded_model.metrics_names[0], score[0] * 100))
def single_test(loaded_model, fileName='rtt_series/valid_data/5021.csv'):
# Test for an input
#test = np.array([189.21973,189.105955,189.0866,189.19152,189.18938,189.21477,189.363065,189.060735,189.018825,189.497595,189.460615,194.76307,196.72467,189.280095,189.014585,189.02478,189.03338,189.087215,88.99757,189.041955])
#test = csvio.csv2list("rtt_series/real_trace_labelled/11119.csv", "rtt")
test = csvio.csv2list(fileName, "rtt", sep=';', decimal='.')
l = max(test)
plt.plot(test)
testLen = len(test)
min_test = min(test)
# Pre-Treatment
test = test.reshape(1, testLen, 1)
# Prediction
temp = loaded_model.predict(test)
temp = temp.reshape(testLen)
#print("temp =")
#print(temp)
res = np.zeros(testLen)
for i in range(testLen):
if temp[i] >= 0.5:
res[i] = 1
cp = csvio.csv2list(fileName, "cp", sep=';', decimal='.')
print("cp =")
np.set_printoptions(threshold=np.nan)
print(cp)
print(sum(cp))
print("res =")
np.set_printoptions(threshold=np.nan)
print(res)
print(sum(res))
'''
for i in range(len(cp)):
if (cp[i]==1):
print('position cp: ')
print(i)
for i in range(len(res)):
if(res[i]==1):
print('position res')
print(i)
'''
print("evaluation = ")
a = eval.evaluation(cp, res)
for i in range(len(res)):
if (res[i] > 0):
res[i] = l
plt.plot(res, 'r')
plt.show()
print(a)
return a
def model_evaluate2precision_recall(
loaded_model,
folderName='./rtt_series/valid_data',
resultName='./results/resultNeuroNet_w.csv'):
# generate precision and recall for every test file
file_csv = os.listdir(folderName)
precision = []
recall = []
for f in file_csv:
fileName = folderName + '/' + f
test = csvio.csv2list(fileName, "rtt", sep=';', decimal='.')
cp = csvio.csv2list(fileName, "cp", sep=';', decimal='.')
# Pre-Treatment
min_test = min(test)
for i in range(len(test)):
test[i] = test[i] - min_test
testLen = len(test)
test = test.reshape(1, testLen, 1)
temp = loaded_model.predict(test)
temp = temp.reshape(testLen)
res = np.zeros(testLen)
for i in range(testLen):
if temp[i] >= 0.5:
res[i] = 1
''' Binary to index
temp = []
for i in range(len(res)):
if(res[i] == 1):
temp.append(i)
res = temp
temp = []
for i in range(len(cp)):
if(cp[i] - 1 < 0.001):
temp.append(i)
cp = temp
'''
cp = cp.astype(int)
res = res.astype(int)
if sum(res) > sum(cp):
print('res=')
print(res)
print('cp=')
print(cp)
print(f)
#temp = eval.evaluation(cp, res)
#precision.append(temp["precision"])
#recall.append(temp["recall"])
#csvio.list2csv(resultName, [file_csv, precision, recall], ['fileName', 'precision', 'recall'])
print("Prediction & Recall saved.")
def comparison_recall():
recallC = csv.csv2list('./results/resultCUSUM_w.csv', 'recall')
recallB = csv.csv2list('./results/resultBayesian_w.csv', 'recall')
cdf = [float(k + 1) / len(recallC) for k in range(len(recallC))]
recallC.sort()
recallB.sort()
plt.plot(recallC, cdf, "r", label="recall of cusum method")
plt.plot(recallB, cdf, "b", label="recall of bayesian method")
plt.legend()
plt.xlabel("recall")
plt.ylabel("cdf")
plt.title("comparison of cdf (recall)")
plt.show()
return
def fetch_data(folder):
file_csv = os.listdir(folder)
x = []
y = []
for f in file_csv:
f = folder + '/' + f
# Attention for the column name, separator and decimal
temp = csvio.csv2list(f, 'trace', sep=',', decimal='.')
x.append(temp)
y.append(csvio.csv2list(f, 'cpt', sep=',', decimal='.'))
inputs=[[x[i][j]-min(x[i]) for j in range(len(x[i]))] for i in range(len(x))]
outputs=[1 if sum(y[i])>0 else 0 for i in range(len(y))]
inputs=np.array(inputs)
outputs=np.array(outputs)
inputs=inputs.reshape(N_SAMPLE,SAMPLE_LEN,1)
outputs=outputs.reshape(N_SAMPLE,1)
return (inputs,outputs)
def cdf_recall_baysian():
recall = csv.csv2list('./results/resultBayesian_w.csv', 'recall')
cdf = [float(k + 1) / len(recall) for k in range(len(recall))]
recall.sort()
plt.plot(recall, cdf)
plt.xlabel("recall")
plt.ylabel("cdf")
plt.title("cdf of recall for Bayesian method")
plt.show()
return
def cdf_precision_baysian():
precision = csv.csv2list('./results/resultBayesian_w.csv', 'precision')
cdf = [float(k + 1) / len(precision) for k in range(len(precision))]
precision.sort()
plt.plot(precision, cdf)
plt.xlabel("precision")
plt.ylabel("cdf")
plt.title("cdf of precision for Bayesian method")
plt.show()
return
def comparison_precision():
precisionC = csv.csv2list('./results/resultCUSUM_w.csv', 'precision')
precisionB = csv.csv2list('./results/resultBayesian_w.csv', 'precision')
precisionN = csv.csv2list(
'./results/resultNeuroNet_w.csv',
'precision',
)
cdf = [float(k + 1) / len(precisionC) for k in range(len(precisionC))]
precisionC.sort()
precisionB.sort()
precisionN.sort()
plt.plot(precisionC, cdf, "r", label="precision of cusum method")
plt.plot(precisionB, cdf, "b", label="precision of bayesian method")
plt.plot(precisionN, cdf, "g", label="precision of NeuroNet")
plt.legend()
plt.xlabel("precision")
plt.ylabel("cdf")
plt.title("comparison of cdf (precision)")
plt.show()
return
def evaluationDataSet(folder):
# Use CUSUM and Bayesian to detect change point
file_csv = os.listdir(folder)
precisionB = []
recallB = []
precisionC = []
recallC = []
for f in file_csv:
f = folder + '/' + f
reality = csv.csv2list(f, 'rtt', sep=';', decimal='.')
detectionB = baycpd.baysiancpt(reality)
detectionC = cusum.cusum_var(reality)
fact = csv.csv2list(f, 'cp', sep=';', decimal='.')
# Change the binary array to index array
detectionB = binary2index(detectionB)
detectionC = binary2index(detectionC)
fact = binary2index(fact)
#print(fact)
temp = eval.evaluation_window_adp(fact, detectionB, 2)
precisionB.append(temp["precision"])
recallB.append(temp["recall"])
temp = eval.evaluation_window_adp(fact, detectionC, 2)
precisionC.append(temp["precision"])
recallC.append(temp["recall"])
csv.list2csv('./results/resultBayesian_w.csv',
[file_csv, precisionB, recallB],
['fileName', 'precision', 'recall'])
csv.list2csv('./results/resultCUSUM_w.csv',
[file_csv, precisionC, recallC],
['fileName', 'precision', 'recall'])
def Fn_score(n):
recallBayesian = csv.csv2list('./results/resultBayesian_w.csv', 'recall')
precisionBayesian = csv.csv2list('./results/resultBayesian_w.csv',
'precision')
Fn_bayesian = []
for i in range(len(recallBayesian)):
if (recallBayesian[i] != 0 or precisionBayesian[i] != 0):
Fn_bayesian.append(
(1 + n * n) * (precisionBayesian[i] * recallBayesian[i]) /
(n * n * (precisionBayesian[i] + recallBayesian[i])))
else:
Fn_bayesian.append(0)
recallCusum = csv.csv2list('./results/resultCUSUM_w.csv', 'recall')
precisionCusum = csv.csv2list('./results/resultCUSUM_w.csv', 'precision')
Fn_cusum = []
for i in range(len(recallCusum)):
if (recallCusum[i] != 0 or precisionCusum[i] != 0):
Fn_cusum.append(
(1 + n * n) * (precisionCusum[i] * recallCusum[i]) /
(n * n * precisionCusum[i] + recallCusum[i]))
else:
Fn_cusum.append(0)
plt.plot(Fn_bayesian, "x", label="Bayesian method")
plt.plot(Fn_cusum, "x", label="Cusum method")
plt.legend()
plt.xlabel("Index of datasets")
plt.ylabel("Fn_measure")
plt.title("F Score for n=" + str(n) +
" applied to the results of Bayesian and Cusum methods")
plt.show()
return
def recall_precision():
precisionC = csv.csv2list('./results/resultCUSUM_w.csv', 'precision')
precisionB = csv.csv2list('./results/resultBayesian_w.csv', 'precision')
precisionN = csv.csv2list('./results/resultNeuroNet_w.csv', 'precision')
recallC = csv.csv2list('./results/resultCUSUM_w.csv', 'recall')
recallB = csv.csv2list('./results/resultBayesian_w.csv', 'recall')
recallN = csv.csv2list('./results/resultNeuroNet_w.csv', 'recall')
plt.plot(precisionC, recallC, "rx", label="cusum method")
plt.plot(precisionB, recallB, "bx", label="bayesian method")
plt.plot(precisionN, recallN, "gx", label="NeuroNet method")
plt.legend()
plt.xlabel("precision")
plt.ylabel("recall")
plt.title("scatterplot (precision,recall)")
plt.show()
return
'''
Run only once
to create the fichier of series split with number_x.csv and number_y.csv
to use this fonction 1. you have to create a folder name "dataset_split"
2. you have to take all the fichier.csv to the folder "dataset_original". For example 11017.csv.
'''
length = 500
path_original = './tools/dataset_original/'
path_generate = "./dataset_split/"
flag = 0
for (path_original, dirs, files) in os.walk(path_original):
for filename in files:
print(filename)
data_rtt_test = csv_t.csv2list(path_original + str(filename), "rtt")
data_cp_test = csv_t.csv2list(path_original + str(filename), "cp")
#write data x and y in nx.csv and ny.csv
for i in range(int(len(data_rtt_test) / length)):
val_cp = []
val_rtt = []
header = ["rtt", "cp"]
filename_x = str(flag + i) + "xy" + ".csv"
with open(path_generate + filename_x, 'wt') as f:
writer = csv.writer(f)
begin = i * length
end = (i + 1) * length
for val in data_rtt_test[begin:end]:
val_rtt.append(val)
for val in data_cp_test[begin:end]:
import tools.csvio as csvio
import ANNTool as md
# global config
SAMPLE_LEN = 100
N_EPOCH = 200
MTX = ['acc']
folderName = './rtt_series/artificial_dataset'
# Read data from the folder
file_csv = os.listdir(folderName)
x = []
y = []
for f in file_csv:
f = folderName + '/' + f
temp = csvio.csv2list(f, 'trace', sep=',', decimal='.')
x.append(temp)
y.append(csvio.csv2list(f, 'cpt', sep=',', decimal='.'))
# add some zeros at the end
data_x = []
data_y = []
for i in range(len(x)):
data_x.extend(x[i].tolist())
data_y.extend(y[i].tolist())
N_SAMPLE = len(data_x) / SAMPLE_LEN
reste = len(data_x) % SAMPLE_LEN
if (reste != 0):
N_SAMPLE += 1