def learn(in_data: ndarray, out_data: ndarray, test_in_data: ndarray,
test_out_data: ndarray, model_func: Callable[[Tensor, Tensor, bool], Tensor],
loss_func: Callable[[Tensor, Tensor, bool], Tensor], optimizer: Optimizer,
score_func: Callable[[Tensor, ndarray], float]=None,
batch_size: int=100, epoch_number: int=100):
input_data = in_data.astype(np.float32)
output_data = out_data.astype(np.float32)
test_input_data = test_in_data.astype(np.float32)
test_output_data = test_out_data.astype(np.float32)
train_loss_values = []
test_loss_values = []
test_score_values = []
start = time()
for i in range(0, epoch_number):
bit = BatchIterator(input_data, output_data, batch_size)
iter_loss = 0
for b_in, b_out in bit:
x = Tensor(b_in)
y = Tensor(b_out)
model = model_func(x, y, True)
loss = loss_func(y, model, True)
iter_loss += loss.data[0] / input_data.shape[0]
optimizer.step(loss)
if score_func is not None:
test_loss, err_ratio = score_test(test_input_data, test_output_data, model_func, loss_func, score_func)
else:
err_ratio = 'N/A'
test_loss = 'N/A'
train_loss_values.append(iter_loss)
test_loss_values.append(test_loss)
test_score_values.append(err_ratio)
print("Iteration {0} train-loss: {1}, test-loss: {2}, score: {3}%".format(i, iter_loss, test_loss, err_ratio))
end = time()
print("Execution time: {0}s".format(end - start))
return train_loss_values, test_loss_values, test_score_values
def step(self, loss: Tensor):
loss.calc_gradients()
self.t = self.t + 1
for i in range(0, len(self.parameters)):
self.m[i] = running_avg(self.m[i], self.beta, self.parameters[i].grad)
m_hat = running_avg_bias_correction(self.m[i], self.beta, self.t) if self.bias_correction else self.m[i]
self.parameters[i].data -= self.learning_rate * m_hat
def score_test(input_data, out_data, model_func, loss_func, score_func):
err_ratio = 0
y = Tensor(out_data)
actual = model_func(Tensor(input_data))
loss = loss_func(y, actual, False)
err_ratio += score_func(actual.data, out_data)
return loss.data[0] / input_data.shape[0], err_ratio
def __init__(self, dim: Tuple[int, int], optimizer: Optimizer = None):
self.dim = dim
self.w = Tensor(np.random.normal(0, 0.01, dim).astype(np.float32), diff=True)
self.b = Tensor(np.zeros((1, dim[1]), dtype=np.float32), diff=True)
self.parameters = [self.w, self.b]
if optimizer is not None:
self.optimizer = optimizer
self.optimizer.add_parameters(self.parameters)
def compute(self, tensor_in: Tensor, train_mode=False):
if train_mode:
m = Tensor(mean(tensor_in.data), diff=False)
v = Tensor(var(tensor_in.data, m.data), diff=False)
self.ravg_mean = running_avg(self.ravg_mean, 0.9, m.data)
self.ravg_var = running_avg(self.ravg_var, 0.9, v.data)
else:
m = Tensor(self.ravg_mean)
v = Tensor(self.ravg_var)
x = (tensor_in - m) / op.sqrt(v + self.epsilon)
y = self.gamma * x + self.beta
return y
def __init__(self, dim: Tuple[int, ...], optimizer: Optimizer):
self.dim = dim
self.gamma = Tensor(np.expand_dims(np.ones(dim, dtype=np.float32),
axis=0),
diff=True)
self.beta = Tensor(np.expand_dims(np.zeros(dim, dtype=np.float32),
axis=0),
diff=True)
self.ravg_mean = np.zeros(dim, dtype=np.float32)
self.ravg_var = np.zeros(dim, dtype=np.float32)
self.epsilon = 1e-7
optimizer.add_parameters([self.beta])
def step(self, loss: Tensor):
loss.calc_gradients()
self.t = self.t + 1
for i in range(0, len(self.parameters)):
self.v[i] = running_avg_squared(self.v[i], self.beta,
self.parameters[i].grad)
v_hat = running_avg_bias_correction(
self.v[i], self.beta,
self.t) if self.bias_correction else self.v[i]
self.parameters[i].data -= grad_square_delta(
self.learning_rate, self.parameters[i].grad, v_hat,
self.epsilon)
def dropout(tensor_in: Tensor, keep_prob: float=0.5, train_mode=False):
if train_mode:
mask = Tensor(binomial(keep_prob, tensor_in.shape()), diff=False)
return tensor_in * mask
else:
return tensor_in
def step(self, loss: Tensor):
loss.calc_gradients()
for p in self.parameters:
p.data -= self.learning_rate * p.grad