#plt.xlabel("it")
#plt.ylabel("loss")
#plt.plot(losses_per_it,'b')
#plt.savefig(os.path.join(RESULTS,model.name+"_ep"+str(e)+"_losses_per_epoch"))
#plt.clf()
print("Epoch:{}/{} it:{} Loss:{}".format(e, EPOCHS, it, loss))
loss = round(sum(losses_per_it) / len(losses_per_it), 4)
losses_per_it = []
it = 0
if loss < min_loss:
print("loss decreased {} --> {}, saving model".format(
min_loss, loss))
min_loss = loss
model.save()
losses_per_epoch.append(loss.astype(np.float16))
plt.suptitle(NAME)
plt.xlabel("epoch")
plt.ylabel("loss")
plt.plot(losses_per_epoch, 'b')
plt.savefig(os.path.join(RESULTS, model.name + "_losses"))
plt.clf()
#
# Saving the state to resume training
# State file has one line for each value:
# -> current_epoch
# -> min_loss
# -> losses per epoch
class Detector(DetectorBase):
def __init__(self, key, seq_length=10):
super(Detector, self).__init__(key, seq_length)
self.key = str(key)
self.packet_length = 1500
self.mini_batch = 30
self.epochs = 50
self.train_buffer = []
self.exec_buffer = []
self.set_buffer = []
self.max_round = inf
self.train_round = 0
self.model = Autoencoder(self.packet_length, seq_length, self.epochs)
self.clf = OneClassSVM(kernel='rbf', gamma=0.1, nu=0.05)
self.model_path = os.path.join('model_{}'.format(seq_length), self.key)
self.stats_path = os.path.join('stats_{}'.format(seq_length),
self.key + '.pkl')
self.eval_path = os.path.join('evaluation_{}'.format(seq_length),
self.key + '.csv')
self.loss_path = os.path.join('evaluation_{}'.format(seq_length),
self.key + '_loss.csv')
if self.model.exist(self.model_path):
print('Using existing model: {}'.format(self.key))
self.model.load(self.model_path)
if os.path.exists(self.stats_path):
print('Using existing stats')
self.clf = joblib.load(self.stats_path)
def update_buffer(self, seq, mode, info=None):
seq = deepcopy(seq)
if mode == 'T' and self.train_round <= self.max_round:
self.train_buffer.append(seq)
if len(self.train_buffer) == self.mini_batch:
random.shuffle(self.train_buffer)
X = np.array(self.train_buffer)
self.train(X)
self.train_buffer = []
self.train_round += 1
elif mode == 'E':
self.exec_buffer.append(seq)
if len(self.exec_buffer) == 1:
X = np.array(self.exec_buffer)
self.execute(X, info)
self.exec_buffer = []
else:
X = np.array(seq)
X = X.reshape((1, X.shape[0], X.shape[1]))
self.eval(X)
def train(self, X):
if self.train_round < self.max_round:
history = self.model.fit(X)
with open(self.loss_path, 'a') as f_loss:
writer_loss = csv.writer(f_loss)
if self.train_round == 0:
writer_loss.writerow([history.history['loss'][0]])
writer_loss.writerow([history.history['loss'][-1]])
print('Detector {} saved'.format(self.key))
def eval(self, X):
Y = self.model.predict(X)
mse = mean_squared_error(X[0], Y[0])
print('Calculating mse of {}: {}'.format(self.key, mse))
self.set_buffer.append(mse)
def set_threshold(self):
self.clf = OneClassSVM(kernel='rbf', gamma=0.1, nu=0.05)
self.clf.fit(np.array(self.set_buffer).reshape(-1, 1))
joblib.dump(self.clf, self.stats_path)
def execute(self, X, info=None):
start = time.time()
Y = self.model.predict(X)
dur = time.time() - start
with open(self.eval_path, 'a') as f:
writer = csv.writer(f)
for x, y in zip(X, Y):
mse = mean_squared_error(x, y)
print('Execute on {}: {}'.format(self.key, mse))
label = self.clf.predict(np.array(mse).reshape(-1, 1))
result = 'Normal' if label == 1 else 'Malicious'
if info:
writer.writerow([str(mse), result, str(info)])
else:
writer.writerow([str(mse), result])
def wrap_up(self, mode):
if mode == 'T':
self.model.save(self.model_path)
elif mode == 'S':
self.set_threshold()
# шумы
trainNoise = np.random.normal(loc=0.5, scale=0.5, size=trainX.shape)
testNoise = np.random.normal(loc=0.5, scale=0.5, size=testX.shape)
trainXNoisy = np.clip(trainX + trainNoise, 0, 1)
testXNoisy = np.clip(testX + testNoise, 0, 1)
print("[INFO] building autoencoder...")
opt = Adam(lr=1e-4)
autoencoder = Autoencoder().build(IMAGE_HEIGHT, IMAGE_WIDTH, 3)
autoencoder.compile(loss="mse", optimizer=opt)
H = autoencoder.fit(trainXNoisy,
trainX,
validation_data=(testXNoisy, testX),
epochs=EPOCHS,
batch_size=BS)
N = np.arange(0, EPOCHS)
plt.style.use("ggplot")
plt.figure()
plt.plot(N, H.history["loss"], label="train_loss")
plt.plot(N, H.history["val_loss"], label="val_loss")
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch")
plt.ylabel("Loss/Accuracy")
plt.legend(loc="lower left")
plt.savefig(PLOT_PATH)
autoencoder.save(MODEL)