class NNQ: def __init__(self, input_space, action_space): #HyperParameters self.GAMMA = 0.95 self.LEARNING_RATE = 0.002 self.MEMORY_SIZE = 1000000 self.BATCH_SIZE = 30 self.EXPLORATION_MAX = 1.0 self.EXPLORATION_MIN = 0.01 self.EXPLORATION_DECAY = 0.997 self.exploration_rate = self.EXPLORATION_MAX self.reward = 0 self.actions = action_space #Experience Replay self.memory = deque(maxlen=self.MEMORY_SIZE) #Create the NN model self.model = Sequential() self.model.add( Dense(64, input_shape=(input_space, ), activation="relu")) self.model.add(Dense(64, activation="relu")) self.model.add(Dense(self.actions, activation="softmax")) self.model.compile(loss="mse", optimizer=Adam(lr=self.LEARNING_RATE)) def act(self, state): #Exploration vs Exploitation if np.random.rand() < self.exploration_rate: return random.randrange(self.actions) q_values = self.model.predict(state) return np.argmax(q_values[0]) def remember(self, state, action, reward, next_state, done): #in every action put in the memory self.memory.append((state, action, reward, next_state, done)) def experience_replay(self): #When the memory is filled up take a batch and train the network if len(self.memory) < self.MEMORY_SIZE: return batch = random.sample(self.memory, self.BATCH_SIZE) for state, action, reward, next_state, terminal in batch: q_update = reward if not terminal: q_update = ( reward + self.GAMMA * np.amax(self.model.predict(next_state)[0])) q_values = self.model.predict(state) q_values[0][action] = q_update self.model.fit(state, q_values, verbose=0) if self.exploration_rate > self.EXPLORATION_MIN: self.exploration_rate *= self.EXPLORATION_DECAY
class Model: __batch_size = 100 def __init__(self): self.model = Sequential([ Dense(40, input_shape=(4, ), activation="relu"), Dense(40, activation="relu"), Dense(40, activation="relu"), Dense(4, activation="tanh") ]) # TODO: xavier initialization? self.model.compile(loss=keras.losses.mean_squared_error, optimizer=keras.optimizers.Adam(lr=0.001)) self.memory = deque(maxlen=10000) def experience_replay(self): if len(self.memory) < self.__batch_size: return batch = random.sample(self.memory, self.__batch_size) states = np.vstack([state for state, _, _, _, _ in batch]) next_states = np.vstack( [next_state for _, _, _, next_state, _ in batch]) predicted_states = self.model.predict(states) predicted_next_states = self.model.predict(next_states) max_nex_state_values = np.max(predicted_next_states, 1) for index, (_, action, reward, _, terminal) in enumerate(batch): q_update = reward if not terminal: discount_factor = 0.95 q_update += discount_factor * max_nex_state_values[index] learning_rate = 0.95 predicted_states[index][action] = ( (1 - learning_rate) * predicted_states[index][action] + learning_rate * q_update) self.model.fit(states, predicted_states, verbose=0)
n_chars = len(alphabet) n_in_seq_length = n_numbers * ceil(log10(largest + 1)) + n_numbers - 1 n_out_seq_length = ceil(log10(n_numbers * (largest + 1))) n_batch = 100 n_epoch = 500 model = Sequential([ LSTM(100, input_shape=(n_in_seq_length, n_chars)), RepeatVector(n_out_seq_length), LSTM(50, return_sequences=True), TimeDistributed(Dense(n_chars, activation='softmax')) ]) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) for i in range(n_epoch): x, y = generate_data(n_samples, largest, alphabet) model.fit(x, y, epochs=1, batch_size=n_batch) model.save('training/keras_classifier.h5') # evaluate on some new patterns x, y = generate_data(n_samples, largest, alphabet) result = model.predict(x, batch_size=n_batch, verbose=0) # calculate error expected = [invert(x, alphabet) for x in y] predicted = [invert(x, alphabet) for x in result] # show some examples for i in range(20): print('Expected=%s, Predicted=%s' % (expected[i], predicted[i]))
#Plot model loss plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title("Model Loss") plt.ylabel('Loss') plt.xlabel('Epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() #Evaluate model on test data accuracy = model.evaluate(x_test, y_test) print(f"Test accuracy: {accuracy}") #Generate predictions for 10 samples print("Predictions for 10 samples") predictions = model.predict(x_test[:10]) y_new = model.predict_classes(x_test[:10]) y_pred = model.predict(x_test) print("Shape of predictions") print(y_pred.shape) #Generate confusion matrix # rounded_predictions = model.predict_classes(x_test, batch_size=128, verbose=0 Y_pred = np.argmax(y_pred, 1) Y_test = np.argmax(y_test, 1) matrix = skl.confusion_matrix(Y_test, Y_pred) sns.heatmap(matrix.T, square=True,