#plt.xlabel("it") #plt.ylabel("loss") #plt.plot(losses_per_it,'b') #plt.savefig(os.path.join(RESULTS,model.name+"_ep"+str(e)+"_losses_per_epoch")) #plt.clf() print("Epoch:{}/{} it:{} Loss:{}".format(e, EPOCHS, it, loss)) loss = round(sum(losses_per_it) / len(losses_per_it), 4) losses_per_it = [] it = 0 if loss < min_loss: print("loss decreased {} --> {}, saving model".format( min_loss, loss)) min_loss = loss model.save() losses_per_epoch.append(loss.astype(np.float16)) plt.suptitle(NAME) plt.xlabel("epoch") plt.ylabel("loss") plt.plot(losses_per_epoch, 'b') plt.savefig(os.path.join(RESULTS, model.name + "_losses")) plt.clf() # # Saving the state to resume training # State file has one line for each value: # -> current_epoch # -> min_loss # -> losses per epoch
class Detector(DetectorBase): def __init__(self, key, seq_length=10): super(Detector, self).__init__(key, seq_length) self.key = str(key) self.packet_length = 1500 self.mini_batch = 30 self.epochs = 50 self.train_buffer = [] self.exec_buffer = [] self.set_buffer = [] self.max_round = inf self.train_round = 0 self.model = Autoencoder(self.packet_length, seq_length, self.epochs) self.clf = OneClassSVM(kernel='rbf', gamma=0.1, nu=0.05) self.model_path = os.path.join('model_{}'.format(seq_length), self.key) self.stats_path = os.path.join('stats_{}'.format(seq_length), self.key + '.pkl') self.eval_path = os.path.join('evaluation_{}'.format(seq_length), self.key + '.csv') self.loss_path = os.path.join('evaluation_{}'.format(seq_length), self.key + '_loss.csv') if self.model.exist(self.model_path): print('Using existing model: {}'.format(self.key)) self.model.load(self.model_path) if os.path.exists(self.stats_path): print('Using existing stats') self.clf = joblib.load(self.stats_path) def update_buffer(self, seq, mode, info=None): seq = deepcopy(seq) if mode == 'T' and self.train_round <= self.max_round: self.train_buffer.append(seq) if len(self.train_buffer) == self.mini_batch: random.shuffle(self.train_buffer) X = np.array(self.train_buffer) self.train(X) self.train_buffer = [] self.train_round += 1 elif mode == 'E': self.exec_buffer.append(seq) if len(self.exec_buffer) == 1: X = np.array(self.exec_buffer) self.execute(X, info) self.exec_buffer = [] else: X = np.array(seq) X = X.reshape((1, X.shape[0], X.shape[1])) self.eval(X) def train(self, X): if self.train_round < self.max_round: history = self.model.fit(X) with open(self.loss_path, 'a') as f_loss: writer_loss = csv.writer(f_loss) if self.train_round == 0: writer_loss.writerow([history.history['loss'][0]]) writer_loss.writerow([history.history['loss'][-1]]) print('Detector {} saved'.format(self.key)) def eval(self, X): Y = self.model.predict(X) mse = mean_squared_error(X[0], Y[0]) print('Calculating mse of {}: {}'.format(self.key, mse)) self.set_buffer.append(mse) def set_threshold(self): self.clf = OneClassSVM(kernel='rbf', gamma=0.1, nu=0.05) self.clf.fit(np.array(self.set_buffer).reshape(-1, 1)) joblib.dump(self.clf, self.stats_path) def execute(self, X, info=None): start = time.time() Y = self.model.predict(X) dur = time.time() - start with open(self.eval_path, 'a') as f: writer = csv.writer(f) for x, y in zip(X, Y): mse = mean_squared_error(x, y) print('Execute on {}: {}'.format(self.key, mse)) label = self.clf.predict(np.array(mse).reshape(-1, 1)) result = 'Normal' if label == 1 else 'Malicious' if info: writer.writerow([str(mse), result, str(info)]) else: writer.writerow([str(mse), result]) def wrap_up(self, mode): if mode == 'T': self.model.save(self.model_path) elif mode == 'S': self.set_threshold()
# шумы trainNoise = np.random.normal(loc=0.5, scale=0.5, size=trainX.shape) testNoise = np.random.normal(loc=0.5, scale=0.5, size=testX.shape) trainXNoisy = np.clip(trainX + trainNoise, 0, 1) testXNoisy = np.clip(testX + testNoise, 0, 1) print("[INFO] building autoencoder...") opt = Adam(lr=1e-4) autoencoder = Autoencoder().build(IMAGE_HEIGHT, IMAGE_WIDTH, 3) autoencoder.compile(loss="mse", optimizer=opt) H = autoencoder.fit(trainXNoisy, trainX, validation_data=(testXNoisy, testX), epochs=EPOCHS, batch_size=BS) N = np.arange(0, EPOCHS) plt.style.use("ggplot") plt.figure() plt.plot(N, H.history["loss"], label="train_loss") plt.plot(N, H.history["val_loss"], label="val_loss") plt.title("Training Loss and Accuracy") plt.xlabel("Epoch") plt.ylabel("Loss/Accuracy") plt.legend(loc="lower left") plt.savefig(PLOT_PATH) autoencoder.save(MODEL)