def derivative(self):
"""
Returns
-------
float32
The derivative of Softmax function.
"""
return np.ones(self.last_forward.shape, dtype=get_dtype())
def derivative(self):
"""The backward also return identity matrix.
Returns
-------
float32
The derivative of linear function.
"""
return np.ones(self.last_forward.shape, dtype=get_dtype())
def derivative(self, input=None):
"""Backward propagation.
Returns
-------
float32
The derivative of Softmax function.
"""
last_forward = input if input else self.last_forward
return np.ones(last_forward.shape, dtype=get_dtype())
def derivative(self, input=None):
"""Backward propagation.
The backward also return identity matrix.
Returns
-------
float32
The derivative of linear function.
"""
last_forward = input if input else self.last_forward
return np.ones(last_forward.shape, dtype=get_dtype())
def derivative(self):
"""The point-wise derivative for ReLU is :math:`\\frac{dy}{dx} = 1`, if
:math:`x>0`, or :math:`\\frac{dy}{dx} = 0`, if :math:`x<=0`.
Returns
-------
float32
The derivative of ReLU function.
"""
res = np.zeros(self.last_forward.shape, dtype=get_dtype())
res[self.last_forward > 0] = 1.
return res
def test_model():
X = np.random.random((10, 20)).astype(get_dtype())
Y = one_hot(np.random.randint(0, 3, 10)).astype(get_dtype())
n_classes = np.unique(Y).size
valid_X = X[:2]
valid_Y = Y[:2]
model = Model()
model.add(
npdl.layers.Dense(n_out=500,
n_in=64,
activation=npdl.activations.ReLU()))
model.add(
npdl.layers.Dense(n_out=n_classes,
activation=npdl.activations.Softmax()))
model.compile(loss=npdl.objectives.SCCE(),
optimizer=npdl.optimizers.SGD(lr=0.005))
with pytest.raises(NotImplementedError):
model.evaluate(X, Y)
model.fit(X, Y, max_iter=0, validation_data=(valid_X, valid_Y))
def fit(self,
X,
Y,
max_iter=100,
batch_size=64,
shuffle=True,
validation_split=0.,
validation_data=None,
file=sys.stdout):
# prepare data
train_X = X.astype(get_dtype()) if np.issubdtype(np.float64,
X.dtype) else X
train_Y = Y.astype(get_dtype()) if np.issubdtype(np.float64,
Y.dtype) else Y
if 1. > validation_split > 0.:
split = int(train_Y.shape[0] * validation_split)
valid_X, valid_Y = train_X[-split:], train_Y[-split:]
train_X, train_Y = train_X[:-split], train_Y[:-split]
elif validation_data is not None:
valid_X, valid_Y = validation_data
else:
valid_X, valid_Y = None, None
iter_idx = 0
while iter_idx < max_iter:
iter_idx += 1
# shuffle
if shuffle:
seed = get_rng().randint(111, 1111111)
np.random.seed(seed)
np.random.shuffle(train_X)
np.random.seed(seed)
np.random.shuffle(train_Y)
# train
train_losses, train_predicts, train_targets = [], [], []
for b in range(train_Y.shape[0] // batch_size):
batch_begin = b * batch_size
batch_end = batch_begin + batch_size
x_batch = train_X[batch_begin:batch_end]
y_batch = train_Y[batch_begin:batch_end]
# forward propagation
y_pred = self.predict(x_batch)
# backward propagation
next_grad = self.loss.backward(y_pred, y_batch)
for layer in self.layers[::-1]:
next_grad = layer.backward(next_grad)
# get parameter and gradients
params = []
grads = []
for layer in self.layers:
params += layer.params
grads += layer.grads
# update parameters
self.optimizer.update(params, grads)
# got loss and predict
train_losses.append(self.loss.forward(y_pred, y_batch))
train_predicts.extend(y_pred)
train_targets.extend(y_batch)
# output train status
runout = "iter %d, train-[loss %.4f, acc %.4f]; " % (
iter_idx, float(np.mean(train_losses)),
float(self.accuracy(train_predicts, train_targets)))
# runout = "iter %d, train-[loss %.4f, ]; " % (
# iter_idx, float(np.mean(train_losses)))
if valid_X is not None and valid_Y is not None:
# valid
valid_losses, valid_predicts, valid_targets = [], [], []
for b in range(valid_X.shape[0] // batch_size):
batch_begin = b * batch_size
batch_end = batch_begin + batch_size
x_batch = valid_X[batch_begin:batch_end]
y_batch = valid_Y[batch_begin:batch_end]
# forward propagation
y_pred = self.predict(x_batch)
# got loss and predict
valid_losses.append(self.loss.forward(y_pred, y_batch))
valid_predicts.extend(y_pred)
valid_targets.extend(y_batch)
# output valid status
runout += "valid-[loss %.4f, acc %.4f]; " % (
float(np.mean(valid_losses)),
float(self.accuracy(valid_predicts, valid_targets)))
print(runout, file=file)
def _cast_dtype(res):
return np.array(res, dtype=get_dtype())