class Dense(Layer):
def __init__(self,
input_shape,
size,
activation=None,
init=None,
lr=0.001):
self.input_shape = input_shape
self.size = size
self.init = init
self.activation = Linear() if activation == None else activation
self.lr = lr
bias = np.zeros(shape=size)
weights = init_matrix(size=(self.input_shape, self.size),
init=self.init)
self.bias = tf.Variable(bias, dtype=tf.float32)
self.weights = tf.Variable(weights, dtype=tf.float32)
###################################################################
def get_weights(self):
assert (False)
def num_params(self):
assert (False)
###################################################################
def forward(self, X):
Z = tf.matmul(X, self.weights) + self.bias
A = self.activation.forward(Z)
return A, None
def backward(self, AI, AO, DO, cache):
DO = DO * self.activation.gradient(AO)
DI = tf.matmul(DO, tf.transpose(self.weights))
DW = tf.matmul(tf.transpose(AI), DO)
DB = tf.reduce_sum(DO, axis=0)
return DI, [(DW, self.weights), (DB, self.bias)]
class Dense(Layer):
def __init__(self,
input_shape,
size,
activation=None,
init=None,
lr=0.001):
self.input_shape = input_shape
self.size = size
self.init = init
self.activation = Linear() if activation == None else activation
self.lr = lr
self.bias = np.zeros(shape=size)
self.weights = init_matrix(size=(self.input_shape, self.size),
init=self.init)
###################################################################
def get_weights(self):
assert (False)
def num_params(self):
assert (False)
###################################################################
def forward(self, X):
Z = X @ self.weights + self.bias
A = self.activation.forward(Z)
return A, None
def backward(self, AI, AO, DO, cache):
DO = DO * self.activation.gradient(AO)
DI = DO @ self.weights.T
DW = AI.T @ DO
DB = np.sum(DO, axis=0)
self.weights -= self.lr * DW
self.bias -= self.lr * DB
return DI