def __init__(self, in_channels, action_size):
super(DQNModel, self).__init__(
) # the line you give simply calls the __init__ method of ClassNames parent class.
self.dict = {} # contain all the parameters
self.conv1 = ConvLayer(in_channels, 32, 84, 8, 4, "relu",
learning_rate, BATCH_SIZE, self.dict, "conv1")
self.conv2 = ConvLayer(32, 64, self.conv1.output_size, 4, 2, "relu",
learning_rate, BATCH_SIZE, self.dict, "conv2")
self.conv3 = ConvLayer(64, 64, self.conv2.output_size, 3, 1, "relu",
learning_rate, BATCH_SIZE, self.dict, "conv3")
self.fc1 = FCLayer(64 * self.conv3.output_size**2, 512, "relu",
learning_rate, BATCH_SIZE, self.dict, "fc1")
self.out = FCLayer(512, action_size, "identity", learning_rate,
BATCH_SIZE, self.dict, "out")
self.action_size = action_size
self.in_channels = in_channels
def __init__(self, input_size, output_size, layers=None):
self.input_layer = InputLayer(input_size)
self.current_layer = self.input_layer
if layers is not None:
for l in layers:
self.current_layer.add_next_layer(
FCLayer(self.current_layer, l, self.learning_rate))
self.current_layer = self.current_layer.get_next_layer()
self.current_layer.add_next_layer(
OutputLayer(self.current_layer, output_size, self.learning_rate))
self.current_layer = self.current_layer.get_next_layer()
self.output_layer = self.current_layer
# Normalisation des données
x_train = x_train.reshape(x_train.shape[0], 1, 28 * 28)
x_train = x_train.astype('float32')
x_train /= 255
y_train = np_utils.to_categorical(y_train)
x_test = x_test.reshape(x_test.shape[0], 1, 28 * 28)
x_test = x_test.astype('float32')
x_test /= 255
y_test = np_utils.to_categorical(y_test)
# Réseau de neurone
net = Network()
net.add(FCLayer(28 * 28,
100)) # input_shape=(1, 28*28) ; output_shape=(1, 100)
net.add(ActivationLayer(tanh, tanh_prime))
net.add(FCLayer(100, 50)) # input_shape=(1, 100) ; output_shape=(1, 50)
net.add(ActivationLayer(tanh, tanh_prime))
net.add(FCLayer(50, 10)) # input_shape=(1, 50) ; output_shape=(1, 10)
net.add(ActivationLayer(tanh, tanh_prime))
net.use(mse, mse_prime)
net.fit(x_train[0:1000], y_train[0:1000], epochs=35, learning_rate=0.1)
# Test sur 3 exemples
out = net.predict(x_test[0:3])
net.print_error()
print("\n")
class DQNModel():
def __init__(self, in_channels, action_size):
super(DQNModel, self).__init__(
) # the line you give simply calls the __init__ method of ClassNames parent class.
self.dict = {} # contain all the parameters
self.conv1 = ConvLayer(in_channels, 32, 84, 8, 4, "relu",
learning_rate, BATCH_SIZE, self.dict, "conv1")
self.conv2 = ConvLayer(32, 64, self.conv1.output_size, 4, 2, "relu",
learning_rate, BATCH_SIZE, self.dict, "conv2")
self.conv3 = ConvLayer(64, 64, self.conv2.output_size, 3, 1, "relu",
learning_rate, BATCH_SIZE, self.dict, "conv3")
self.fc1 = FCLayer(64 * self.conv3.output_size**2, 512, "relu",
learning_rate, BATCH_SIZE, self.dict, "fc1")
self.out = FCLayer(512, action_size, "identity", learning_rate,
BATCH_SIZE, self.dict, "out")
self.action_size = action_size
self.in_channels = in_channels
def forward(self, state):
self.conv1.forward(state)
self.conv2.forward(self.conv1.output_tensor)
self.conv3.forward(self.conv2.output_tensor)
self.fc1.forward(
self.conv3.output_tensor.reshape(-1,
64 * self.conv3.output_size**2))
self.out.forward(self.fc1.output_tensor_act)
return self.out.output_tensor_act
def edit_tensor(self, tensor):
'''
reshape the delta tensor, transforming (B,H) to (B,C,H,W)
'''
return tensor.reshape(BATCH_SIZE, -1, self.conv3.output_size,
self.conv3.output_size)
def backward(self, target, local, loss_name, actions):
self.out.backward(
self.out.input_tensor,
self.loss_gradient(self.out.output_tensor_act, target, local,
loss_name, actions))
self.fc1.backward(self.fc1.input_tensor, self.out.delta_tensor)
self.conv3.backward(self.conv3.input_tensor,
self.edit_tensor(self.fc1.delta_tensor), "relu")
self.conv2.backward(self.conv2.input_tensor, self.conv3.delta_tensor,
"relu")
self.conv1.backward(self.conv1.input_tensor, self.conv2.delta_tensor,
"relu")
def step(self):
self.conv1.update()
self.conv2.update()
self.conv3.update()
self.fc1.update()
self.out.update()
def loss_gradient(self, tensor, target, local, loss_name, actions):
'''
calculate the loss that depends on the defintion of the loss
'''
g = torch.zeros_like(tensor, device=device)
if loss_name == "huber":
n = torch.abs(local - target)
loss_g = torch.where(n < 1, local - target, n**0)
loss_g = torch.where(local - target < -1, -n**0, loss_g)
for b in range(BATCH_SIZE):
action = actions[b, 0]
g[b, action.int()] = loss_g[b, 0]
return g
raise
def soft_update(self, local_network, TAU):
'''
a method for updating in soft way
'''
self.conv1.filters.weights = TAU * local_network.conv1.filters.weights + (
1 - TAU) * self.conv1.filters.weights
self.conv1.filters.bias = TAU * local_network.conv1.filters.bias + (
1 - TAU) * self.conv1.filters.bias
self.conv2.filters.weights = TAU * local_network.conv2.filters.weights + (
1 - TAU) * self.conv2.filters.weights
self.conv2.filters.bias = TAU * local_network.conv2.filters.bias + (
1 - TAU) * self.conv2.filters.bias
self.conv3.filters.weights = TAU * local_network.conv3.filters.weights + (
1 - TAU) * self.conv3.filters.weights
self.conv3.filters.bias = TAU * local_network.conv3.filters.bias + (
1 - TAU) * self.conv3.filters.bias
self.fc1.filter.weights = TAU * local_network.fc1.filter.weights + (
1 - TAU) * self.fc1.filter.weights
self.fc1.filter.bias = TAU * local_network.fc1.filter.bias + (
1 - TAU) * self.fc1.filter.bias
self.out.filter.weights = TAU * local_network.out.filter.weights + (
1 - TAU) * self.out.filter.weights
self.out.filter.bias = TAU * local_network.out.filter.bias + (
1 - TAU) * self.out.filter.bias
def get_dict(self):
'''
return the dictionary of all the learned parameters of the model
'''
return self.dict
def load_model(self, dict):
'''
a method to upload the dict of previous model
'''
self.dict.update(dict)
class QNetwork():
def __init__(self, state_size, action_size, seed):
super(QNetwork, self).__init__() # the line you give simply calls the __init__ method of ClassNames parent class.
self.seed = seed.random(seed)
self.fc1 = FCLayer(state_size, 256,ReluActivator(),learning_rate)
self.fc2 = FCLayer(256,128,ReluActivator(),learning_rate)
self.fc3 = FCLayer(128,64,ReluActivator(),learning_rate)
self.out = FCLayer(64, action_size,IdentityActivator() ,learning_rate)
self.data = [self.fc1.parameters,self.fc2.parameters,self.fc3.parameters,self.out.parameters]
def forward(self, state):
self.fc1.forward(state.reshape(-1,1))
self.fc2.forward(self.fc1.output_array_act)
self.fc3.forward(self.fc2.output_array_act)
self.out.forward(self.fc3.output_array_act)
return self.out.output_array_act
def backward(self,target, loss_name ,action) :
self.out.backward(self.out.output_array, self.loss_gradient(self.out.output_array_act, target, loss_name, action))
self.fc3.backward(self.fc3.output_array, self.out.delta_array)
self.fc2.backward(self.fc2.output_array, self.fc3.delta_array)
self.fc1.backward(self.fc1.output_array, self.fc2.delta_array)
def step (self):
'''
update the network
'''
self.fc1.update()
self.fc2.update()
self.fc3.update()
self.out.update()
def loss_gradient(self, local ,target, loss_name, action):
'''
the loss_gradient depends on the defintion of the loss
'''
ABS =abs (target-local[action])
grad = np.zeros(local.shape)
if loss_name == "MSE":
grad[action][0] += 2* (local[action][0] - target)
return grad
raise
def soft_update (self, local_network,TAU):
'''
a method for updating in soft way
'''
self.fc1.filter.weights = TAU* local_network.fc1.filter.weights + (1-TAU) *self.fc1.filter.weights
self.fc1.filter.bias = TAU* local_network.fc1.filter.bias + (1-TAU) *self.fc1.filter.bias
self.fc2.filter.weights = TAU* local_network.fc2.filter.weights + (1-TAU) *self.fc2.filter.weights
self.fc2.filter.bias = TAU* local_network.fc2.filter.bias +(1-TAU) * self.fc2.filter.bias
self.fc3.filter.weights = TAU* local_network.fc3.filter.weights +(1-TAU) * self.fc3.filter.weights
self.fc3.filter.bias = TAU* local_network.fc3.filter.bias + (1-TAU) *self.fc3.filter.bias
self.out.filter.weights = TAU* local_network.out.filter.weights + (1-TAU) *self.out.filter.weights
self.out.filter.bias = TAU* local_network.out.filter.bias + (1-TAU) *self.out.filter.bias
def load_model(self, path):
'''
a method to upload the dict of previous model
'''
self.data = pickle.load(open(path,'rb'))
def main():
print("Starting Network...")
print("-------------------------------------------------------")
print("Reading Data sets...")
# MNIST Data sets
#train_img, test_img, train_lbl, test_lbl = load(file_name="mnist")
# CIFAR-10 Data sets
train_img, test_img, train_lbl, test_lbl = load(file_name="cifar")
Y = train_lbl[:].astype(int)
X = train_img[:] / 255.
Y_test = test_lbl[:].astype(int)
X_test = test_img[:] / 255.
#preprocess data
#X = preprocess_data(X)
#X_test = preprocess_data(X_test)
#model
model = Model()
model.add(
ConvNet(filter_size=(5, 5),
filter_no=6,
zero_padding=0,
stride=(1, 1),
activation="relu"))
model.add(Pool(pool_size=(2, 2), stride=(2, 2), pool_type="max"))
model.add(
ConvNet(filter_size=(5, 5),
filter_no=6,
zero_padding=0,
stride=(1, 1),
activation="relu"))
model.add(Pool(pool_size=(2, 2), stride=(2, 2), pool_type="max"))
model.add(Flatten())
model.add(
FCLayer(activation="relu",
n_neurons=32,
l_rate=0.001,
is_drop_out=True,
drop_out=0.7))
model.add(FCLayer(activation="softmax", n_neurons=10, l_rate=0.001))
print("-------------------------------------------------------")
print("CNN Layers:")
print("-------------------------------------------------------")
model.print_layers()
print("-------------------------------------------------------")
print("Begin Training...")
model.train(X, Y, n_epochs=150, print_loss=True, batch_size=32)
print("End Training.")
print("-------------------------------------------------------")
print("Begin Testing...")
train_accuracy = model.test(X, Y)
test_accuracy = model.test(X_test, Y_test)
print("End Testing.")
print("-------------------------------------------------------")
print('Training Accuracy: {0:0.2f} %'.format(train_accuracy))
print('Test Accuracy: {0:0.2f} %'.format(test_accuracy))
model.show_graph()
def relu(z):
return np.maximum(0, z)
def relu_prime(z):
z[z < 0] = 0
z[z > 0] = 1
return z
def loss(y_true, y_predict):
return 0.5 * (y_predict - y_true)**2
def loss_prime(y_true, y_predict):
return y_predict - y_true
x_train = np.array([[[0, 0]], [[0, 1]], [[1, 0]], [[1, 1]]])
y_train = np.array([[[0]], [[1]], [[1]], [[0]]])
net = Network()
net.add(FCLayer((1, 2), (1, 3)))
net.add(ActivationLayer((1, 3), (1, 3), relu, relu_prime))
net.add(FCLayer((1, 3), (1, 1)))
net.add(ActivationLayer((1, 1), (1, 1), relu, relu_prime))
net.setup_loss(loss, loss_prime)
net.fit(x_train, y_train, epochs=1000, learning_rate=0.01)
out = net.predict(np.array([[0, 1]]))
print(out)