def start_testing(trained_model_file):
parser = CoreNLPParser(url='http://localhost:9080')
emotions = ['happiness', 'sadness', 'anger', 'disgust', 'surprise', 'fear']
dirname = os.path.dirname(os.path.realpath(__file__)) + "/"
glove_model = read_glove_vectors(dirname + "Pickle/gloveModel")
hidden_size = 256
num_layers = 2
bidirectional = False
batchnorm = False
dropout_hidden = 0.3
dropout_output = 0.9
model = LSTM(300, hidden_size, num_layers, bidirectional, batchnorm,
dropout_hidden, dropout_output).to(device)
with torch.no_grad():
model.load_state_dict(torch.load(trained_model_file))
print(model)
model.eval()
while True:
test_sentence = input("Give a test sentence: ")
sentence = list(parser.tokenize(test_sentence))
input1, sent_length = get_input_vector(glove_model, sentence)
class_pred = model(input1, sent_length)
print("Sentence: " + test_sentence)
_, pred = class_pred.max(dim=1)
print("Prediction:\t" + emotions[pred[0]])
print("Output Values:")
percentages = torch.nn.functional.softmax(class_pred, dim=1) * 100
for i in range(len(emotions)):
print(emotions[i] + " %" +
str(percentages.data.tolist()[0][i]))
def test(self, data_path, event_name,
window): #API for testing on the other datasets
x, y0 = Data_Loader.load(data_path, window)
INPUT_SIZE = x.shape[1]
true_pred = pd.DataFrame()
Event_list = [
'prefix_hijack', 'route_leak', 'breakout', 'edge', 'defcon'
]
print(INPUT_SIZE)
import pickle
for Event_name in Event_list:
scaler = pickle.load(
open('../params/' + Event_name + '_scaler.pkl', 'rb'))
x0 = scaler.transform(x.values)
test_x, test_y = Data_Loader.to_timestep(x0, y0.values, window)
test_x = torch.tensor(test_x, dtype=torch.float32)
test_y = torch.tensor(np.array(test_y))
Path = '../params/best_lstm_params_' + Event_name + '.pkl'
model = LSTM()
model.load_state_dict(torch.load(Path))
test_output = model(test_x)
output = test_output.detach().numpy()
output_event_conf = output[:, 1]
# print(len(output_event_conf))
# print(output_event_conf[output_event_conf>0.8])
condition_1 = np.array(output_event_conf > 0.9, dtype=int)
condition_2 = torch.max(test_output, 1)[1].cpu().data.numpy()
pred_y = condition_1 & condition_2
from sklearn.metrics import classification_report
target_l = ['normal']
target_l.append(event_name)
print(
classification_report(y_true=test_y,
y_pred=pred_y,
target_names=target_l))
true_pred['pred_' + Event_name] = pred_y
true_pred['true'] = test_y
true_pred.to_csv('../result_doc/pred_true_' + event_name + '.csv')
def test_route_leak(self):
'''
description: directly use the after-trained model (from weak dataset) to pred the well-known dataset
:return: csv file
'''
global SA_LSTM_flag
loader = Data_Loader(Event_name='route_leak', Event_num=2)
self.INPUT_SIZE = loader.INPUT_SIZE
x, y0 = loader.loadroute_leak()
INPUT_SIZE = x.shape[1]
true_pred = pd.DataFrame()
Event_list = [
'prefix_hijack', 'route_leak', 'breakout', 'edge', 'defcon'
]
print(INPUT_SIZE)
import pickle
for Event_name in Event_list:
scaler = pickle.load(
open('../params/' + Event_name + '_scaler.pkl', 'rb'))
x0 = scaler.transform(x.values)
test_x, test_y = loader.to_timestep(x=x0,
y=y0.values,
event_len=1440)
test_x = torch.tensor(test_x, dtype=torch.float32)
test_y = torch.tensor(np.array(test_y))
Path = '../params/best_lstm_params_' + Event_name + '.pkl'
if SA_LSTM_flag:
model = SA_LSTM(WINDOW_SIZE=self.WINDOW_SIZE,
INPUT_SIZE=self.INPUT_SIZE,
Hidden_SIZE=128,
LSTM_layer_NUM=1)
model.load_state_dict(torch.load(Path))
test_output, attn = model(test_x)
else:
model = LSTM(WINDOW_SIZE=self.WINDOW_SIZE,
INPUT_SIZE=self.INPUT_SIZE,
Hidden_SIZE=128,
LSTM_layer_NUM=1)
model.load_state_dict(torch.load(Path))
test_output = model(test_x)
pred_y = torch.max(test_output, 1)[1].cpu().data.numpy()
from sklearn.metrics import classification_report
print(
classification_report(y_true=test_y,
y_pred=pred_y,
target_names=['Normal', 'Abnomarl ']))
true_pred['pred_' + Event_name] = pred_y
true_pred['true'] = test_y
true_pred.to_csv('../result_doc/pred_true_route_leak_train.csv')
dataLoader = DataLoader(dataSet,
batch_size=64,
shuffle=True,
num_workers=0,
pin_memory=True,
collate_fn=None)
num_epochs = 100
learning_rate = 0.001
input_size = 5
hidden_size = 128
num_layers = 3
device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu") # device
model = LSTM(input_size, hidden_size, num_layers).to(device)
loss_function = torch.nn.MSELoss() # mean-squared error for regression
optimizer = optim.Adam(model.parameters(), lr=learning_rate) # adam optimizer
scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer,
step_size=30,
gamma=0.9)
# losses = []
# minLoss = 1
# model.train()
# for epoch in tqdm.tqdm(range(num_epochs), desc='Epoch'):
# for chart, target in dataLoader:
# output = model.forward(Variable(chart).to(device))
#
# optimizer.zero_grad()
# loss = loss_function(output, target.to(device))
# loss.backward()
def train(self,
Event_num,
read_from_file=True,
include_MANRS_data=True,
WINDOW_SIZE=30,
HIDDEN_SIZE=128,
save_epoch=10): #the implement of training model
'''
:param Event_num: the index type of the anomaly
:param read_from_file: already have the file to read
:param include_MANRS_data: join the legitimate data
:param WINDOW_SIZE: the sliding window size
:param HIDDEN_SIZE: the hidden size of LSTM model
:return: save model under the path ../params/best_lstm_params_' + Event_name + '2.pkl
'''
self.WINDOW_SIZE = WINDOW_SIZE
self.Hidden_SIZE = HIDDEN_SIZE
global SA_LSTM_flag
Event_name = self.Event_list[Event_num - 1]
Path = '../params/best_lstm_params_' + Event_name + '.pkl'
target_list = ['normal', Event_name]
loader = Data_Loader(Event_name=Event_name,
Event_num=Event_num,
TIME_STEP=1,
WINDOW_SIZE=self.WINDOW_SIZE)
train_x, train_y, test_x, test_y, eval_x, eval_y = loader.loadDataSet(
read_from_file=read_from_file,
include_MANRS_data=include_MANRS_data)
self.INPUT_SIZE = loader.INPUT_SIZE
datasets = Data.TensorDataset(train_x, train_y)
train_loader = Data.DataLoader(dataset=datasets,
batch_size=self.BATCH_SIZE,
shuffle=True,
num_workers=2)
eval_x = eval_x.cuda()
eval_y = eval_y.numpy()
test_x = test_x.cuda()
test_y = test_y.numpy()
if SA_LSTM_flag:
lstm = SA_LSTM(WINDOW_SIZE=self.WINDOW_SIZE,
INPUT_SIZE=self.INPUT_SIZE,
Hidden_SIZE=self.Hidden_SIZE,
LSTM_layer_NUM=self.LSTM_layer_NUM)
else:
lstm = LSTM(WINDOW_SIZE=self.WINDOW_SIZE,
INPUT_SIZE=self.INPUT_SIZE,
Hidden_SIZE=self.Hidden_SIZE,
LSTM_layer_NUM=self.LSTM_layer_NUM)
lstm = lstm.cuda()
optimizer = torch.optim.Adam(lstm.parameters(), lr=self.LR)
loss_func = nn.CrossEntropyLoss()
h_state = None
from sklearn.metrics import f1_score
best_f1_score = 0.0
best_epoch = 0
#train_length = len(train_loader)
for epoch in range(self.EPOCH):
for step, (x, y) in tqdm(enumerate(train_loader)):
x = x.cuda()
y = y.cuda()
if SA_LSTM_flag:
output, attn_weights = lstm(x)
else:
output = lstm(x)
loss = loss_func(output, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % 10000 == 0:
if SA_LSTM_flag:
eval_output, attn_weights = lstm(test_x)
else:
eval_output = lstm(test_x)
pred_y = torch.max(eval_output, 1)[1].cpu().data.numpy()
#print(pred_y)
#print(eval_y)
accuracy = float(np.sum(pred_y == test_y)) / float(
test_y.size)
print('Epoch: ', epoch,
'| train loss: %.4f' % loss.cpu().data.numpy(),
'| test accuracy: %.2f' % accuracy)
from sklearn.metrics import classification_report
temp_str = classification_report(y_true=test_y,
y_pred=pred_y,
target_names=target_list)
temp_f1 = f1_score(y_pred=pred_y,
y_true=test_y,
average='macro')
print('temp_f1', temp_f1)
# temp_sum=temp_f1+temp_route_f1
#if (best_f1_score < temp_f1):
if (epoch == save_epoch):
print(temp_str + '\n' + str(temp_f1))
with open(
'../result_doc/test_best_f1' + Event_name +
'.txt', 'a') as f:
message = 'epoch:' + str(
epoch) + ' f1_score:' + str(temp_f1) + '\n'
f.write(message)
best_f1_score = temp_f1
best_epoch = epoch
torch.save(lstm.state_dict(), Path)
lstm.load_state_dict(torch.load(Path))
if SA_LSTM_flag:
test_output, attn_weights = lstm(test_x)
else:
test_output = lstm(test_x)
pred_y = torch.max(test_output, 1)[1].cpu().data.numpy()
from sklearn.metrics import classification_report
test_report = classification_report(y_true=test_y,
y_pred=pred_y,
target_names=target_list)
test_parameter_path = '../result_doc/test_parameter' + '_' + Event_name + '.txt'
with open(test_parameter_path, 'a') as f:
message = "TimeStep:" + str(
self.TIME_STEP
) + '\tWINDOW_SIZE:' + str(
self.WINDOW_SIZE
) + "\tLSTM_NUM: " + str(self.LSTM_NUM) + '\tLayer num: ' + str(
self.LSTM_layer_NUM
) + '\tLR:' + str(self.LR) + '\tBatch_size: ' + str(
self.BATCH_SIZE) + '\tHidden_size: ' + str(
self.Hidden_SIZE
) + '\tNormalizer：MinMaxScaler' + '\t epoch:' + str(
best_epoch) + '\tf1_score:' + str(
best_f1_score) + '\n' + 'include_MANRS_data:' + str(
include_MANRS_data
) + '\t time_bins:60s' + '\n' + test_report + '\n\n'
print(message)
f.write(message)
self.models.append(lstm)
#attn_weights_df=pd.DataFrame(best_attn_weights)
#print(attn_weights_df)
#attn_weights_df.to_csv('../result_doc/atten_weights' + '_' + Event_name + '.csv')
torch.save(lstm, '../params/lstm' + Event_name + '.pkl')
def train():
number_of_epochs = 1000
n_splits = 5
early_stopping_patience = 15
outputs, filelines = zip(*filtereddata)
outputs = torch.LongTensor(outputs).to(device)
padded_vectors, targets, sent_lengths = ready_data(outputs, filelines)
targets_on_cpu = targets.cpu()
skf = StratifiedKFold(n_splits=n_splits, shuffle=False)
i = 0
train_classification_reports, test_classification_reports = [], []
all_train_conf_matrices, all_test_conf_matrices = [], []
for train_index, test_index in skf.split(np.zeros(len(targets)), targets_on_cpu):
i += 1
x_train, x_test = padded_vectors[:, train_index,
:], padded_vectors[:, test_index, :]
y_train, y_test = targets[train_index], targets[test_index]
sent_len_train, sent_len_test = sent_lengths[train_index], sent_lengths[test_index]
x_test, y_test, sent_len_test = get_batch(
0, len(y_test), x_test, y_test, sent_len_test)
model = LSTM(300, hidden_size, num_layers, bidirectional,
batchnorm, dropout_hidden, dropout_output).to(device)
# https://discuss.pytorch.org/t/vgg-output-layer-no-softmax/9273/5
loss_function = nn.CrossEntropyLoss().to(device)
#optimizer = optim.Adam(model.parameters(), lr=0.01)
optimizer = optim.Adam(model.parameters(), lr=learning_rate, betas=(
0.9, 0.999), eps=1e-08, weight_decay=1e-5, amsgrad=False)
early_stopping = EarlyStopping(
patience=early_stopping_patience, verbose=True)
N = len(y_train)
trainlosses, testlosses = [None], [None]
fold_train_classification_reports, fold_test_classification_reports = [], []
fold_train_conf_matrices, fold_test_conf_matrices = [], []
for epoch in range(1, number_of_epochs + 1):
###################
# train the model #
###################
model.train() # prep model for training
shuffleindices = torch.randperm(len(y_train))
x_train.copy_(x_train[:, shuffleindices, :])
y_train.copy_(y_train[shuffleindices])
sent_len_train.copy_(sent_len_train[shuffleindices])
for batch in range(math.ceil(N / batch_size)):
# clear the gradients of all optimized variables
optimizer.zero_grad()
# get_data gets the data from the dataset (sequence batch, size batch_size)
x_batch, targets_batch, sent_lengths_batch = get_batch(
batch, batch_size, x_train, y_train, sent_len_train)
# forward pass: compute predicted outputs by passing inputs to the model
class_pred = model(x_batch, sent_lengths_batch)
# calculate the loss
loss = loss_function(class_pred, targets_batch)
# backward pass: compute gradient of the loss with respect to model parameters
loss.backward()
# perform a single optimization step (parameter update)
optimizer.step()
"""===================================================================================================================="""
directory = dirname + "results/es_n" + str(n_splits) + "+b" + str(batch_size) + "+e" + str(number_of_epochs) + "+lr" + str(learning_rate) + "+hidden" + str(hidden_size) + "+ly" + str(num_layers) \
+ ("+bd" if bidirectional else "") + ("+bn" if batchnorm else "") + \
"+dp_h" + str(dropout_hidden) + "+dp_o" + \
str(dropout_output) + "/"
fold = i
saveFile = epoch == number_of_epochs
"""===================================================================================================================="""
######################
# validate the model #
######################
model.eval() # prep model for evaluation
"""===================================================================================================================="""
######################
# FIND TEST LOSS #
######################
# forward pass: compute predicted outputs by passing inputs to the model
class_pred = model(x_test, sent_len_test)
# calculate the loss
loss = loss_function(class_pred, y_test)
_, pred = class_pred.cpu().detach().max(dim=1)
testlosses.append(loss.item()) # record validation loss
minposs = testlosses.index(min(testlosses[1:])) - 1
y_test2 = y_test.cpu()
classification_report = metrics.classification_report(
y_true=y_test2, y_pred=pred, target_names=emotions, output_dict=True, zero_division=0)
test_conf_matrix = metrics.confusion_matrix(
y_true=y_test2, y_pred=pred)
fold_test_classification_reports.append(
fix_report(classification_report))
fold_test_conf_matrices.append(test_conf_matrix)
del classification_report
del test_conf_matrix
del loss
del y_test2
tobeprintedtest = ["", "", "", "", "", "", "", "", "", "", "", " TEST DATA",
report_to_string(fold_test_classification_reports[minposs])]
tobeprintedtest = '\n'.join(tobeprintedtest)
"""===================================================================================================================="""
"""===================================================================================================================="""
######################
# FIND TRAINING LOSS #
######################
x_train2, y_train2, sent_len_train2 = get_batch(
0, len(y_train), x_train, y_train, sent_len_train)
# forward pass: compute predicted outputs by passing inputs to the model
class_pred = model(x_train2, sent_len_train2)
# calculate the loss
loss = loss_function(class_pred, y_train2)
_, pred = class_pred.cpu().detach().max(dim=1)
trainlosses.append(loss.item()) # record training loss
y_train2 = y_train2.cpu()
classification_report = metrics.classification_report(
y_true=y_train2, y_pred=pred, target_names=emotions, output_dict=True, zero_division=0)
train_conf_matrix = metrics.confusion_matrix(
y_true=y_train2, y_pred=pred)
fold_train_classification_reports.append(
fix_report(classification_report))
fold_train_conf_matrices.append(train_conf_matrix)
del classification_report
del train_conf_matrix
del loss
del x_train2
del y_train2
del sent_len_train2
tobeprintedtrain = ["LEARNING RATE = " + str(learning_rate), "BATCH SIZE = " + str(batch_size), "HIDDEN_SIZE = " + str(hidden_size),
str(num_layers) + " LAYERS", "BIDIRECTIONAL " + ("YES" if bidirectional else "NO"), "BATCHNORM " + (
"YES" if batchnorm else "NO"), "DROPOUT HIDDEN = " + str(dropout_hidden),
"DROPOUT OUTPUT = " + str(dropout_output),
"", "FOLD " + str(fold) + "/" + str(n_splits), "EPOCH " + str(
epoch) + "/" + str(number_of_epochs), " TRAIN DATA",
report_to_string(fold_train_classification_reports[minposs])]
tobeprintedtrain = '\n'.join(tobeprintedtrain)
"""===================================================================================================================="""
"""===================================================================================================================="""
"""===================================================================================================================="""
# early_stopping needs the validation loss to check if it has decresed,
# and if it has, it will make a checkpoint of the current model
early_stopping(testlosses[-1], model)
"""===================================================================================================================="""
"""===================================================================================================================="""
if saveFile or early_stopping.early_stop:
image1 = plottraintest(trainlosses, testlosses)
image2 = showResults(tobeprintedtrain, tobeprintedtest,
fold_train_conf_matrices[minposs], fold_test_conf_matrices[minposs], emotions)
showSaveImage(image1, image2, directory, fold,
saveFile or early_stopping.early_stop, False)
train_classification_reports.append(
fold_train_classification_reports[minposs])
all_train_conf_matrices.append(
fold_train_conf_matrices[minposs])
fold_train_conf_matrices = []
test_classification_reports.append(
fold_test_classification_reports[minposs])
fold_test_classification_reports = []
all_test_conf_matrices.append(fold_test_conf_matrices[minposs])
"""===================================================================================================================="""
if n_splits == i and (epoch == number_of_epochs or early_stopping.early_stop):
train_conf_matrices_average = np.round(
np.mean(all_train_conf_matrices, axis=0), 1)
test_conf_matrices_average = np.round(
np.mean(all_test_conf_matrices, axis=0), 1)
train_classification_average = report_average(
train_classification_reports)
test_classification_average = report_average(
test_classification_reports)
tobeprintedtrain = ["LEARNING RATE = " + str(learning_rate), "BATCH SIZE = " + str(batch_size), "HIDDEN_SIZE = " + str(hidden_size),
str(num_layers) + " LAYERS", "BIDIRECTIONAL " + ("YES" if bidirectional else "NO"), "BATCHNORM " + (
"YES" if batchnorm else "NO"), "DROPOUT HIDDEN = " + str(dropout_hidden),
"DROPOUT OUTPUT = " + str(dropout_output),
"", str(fold) + " FOLD AVERAGE", "EPOCH " +
str(epoch), " TRAIN DATA",
train_classification_average]
tobeprintedtrain = '\n'.join(tobeprintedtrain)
tobeprintedtest = ["", "", "", "", "", "", "", "", "", "", "", " TEST DATA",
test_classification_average]
tobeprintedtest = '\n'.join(tobeprintedtest)
averageImage2 = showResults(
tobeprintedtrain, tobeprintedtest, train_conf_matrices_average, test_conf_matrices_average, emotions)
showSaveImage(None, averageImage2, directory, str(fold) + " fold average",
n_splits == i and (epoch == number_of_epochs or early_stopping.early_stop), True)
if early_stopping.early_stop:
print("Early stopping")
break
"""===================================================================================================================="""
print('epoch: {} memory use: {}MB'.format(
epoch, torch.cuda.memory_allocated()/2.**20))
torch.cuda.empty_cache()
# load the last checkpoint with the best model
model.load_state_dict(torch.load('checkpoint.pt'))
torch.save(model.state_dict(), dirname + "models/es_n" + str(i) + "+b" + str(batch_size) + "+e" + str(number_of_epochs) + "+lr" + str(learning_rate) + "+hidden" + str(hidden_size) + "+ly" + str(num_layers)
+ ("+bd" if bidirectional else "") + ("+bn" if batchnorm else "") + "+dp_h" + str(dropout_hidden) + "+dp_o" + str(dropout_output) + ".pth")
del x_train
del x_test
del y_train
del y_test
del pred
del sent_len_train
del sent_len_test
del trainlosses
del testlosses
del loss_function
del optimizer
del early_stopping
del model
gc.collect()
"""
##### Parameters ######################
from LSTM_model import get_params
params = get_params()
num_epochs = params["num_epochs"]
learning_rate = params["learning_rate"]
batch_size = params["batch_size"]
input_size = params["input_size"]
hidden_size = params["hidden_size"]
num_layers = params["num_layers"]
num_classes = params["num_classes"]
#####Init the Model #######################
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(device)
lstm = LSTM(num_classes, input_size, hidden_size, num_layers, device).to(device)
##### Set Criterion Optimzer and scheduler ####################
criterion = torch.nn.MSELoss().to(device) # mean-squared error for regression
best_val_loss, best_model = \
train_eval_loop(lstm,
train,
valid,
criterion,
lr=learning_rate,
epoch_n = num_epochs,
batch_size=batch_size,
device=device,
l2_reg_alpha=1e-5,