def test_constant_tensor_interchange(self):
const_A = Constant(self.A.value)
const_B = Constant(self.B.value)
self.assertTrue(
np.array_equal(self.C.value,
self.operation(const_A, self.B).value))
self.assertTrue(
np.array_equal(self.C.value,
self.operation(self.A, const_B).value))
def setUp(self):
self.A = Tensor([[1, 2], [3, 4]], name='A')
self.B = Tensor([[1, 2], [3, 4]], name='B')
self.C = Tensor([[5, 6], [7, 8]], name='C')
self.D = Constant([[2, 1], [3, 2]], name='D')
# Define an involved computation graph with the constants and variables above
e = TensorAddition(self.A, self.B)
f = TensorElemMultiply(Constant(2 * np.ones(e.shape)), e)
g = TensorNegLog(f)
h = TensorAddition(self.A, g)
i = TensorMultiply(h, self.C)
j = TensorSubtraction(i, self.D)
l = TensorSum(j)
self.bp = BackwardsPass(l)
def setUp(self):
self.data = np.array([
[[1, 2, 3], [4, 5, 6]],
[[7, 8, 9], [10, 11, 12]],
])
self.const = Constant(self.data)
def l2_loss(self):
weights_mag = None
for var in self.vars:
weights = TensorSum(TensorSquared(self.vars[var]))
if weights_mag:
weights_mag = TensorAddition(weights_mag, weights)
else:
weights_mag = weights
return TensorElemMultiply(Constant([self.l2]), weights_mag)
def log_loss(y, y_hat):
""" The log loss of a single prediction and target outcome. """
pos_prob_log = TensorNegLog(y_hat)
pos_prob_rect = TensorElemMultiply(pos_prob_log, y)
neg_prob = TensorSubtraction(
Constant(np.ones(y_hat.shape)),
y_hat,
)
neg_prob_log = TensorNegLog(neg_prob)
neg_outcomes = TensorSubtraction(
Constant(np.ones(y.shape)),
y,
)
neg_prob_rect = TensorElemMultiply(neg_prob_log, neg_outcomes)
log_probs = TensorAddition(neg_prob_rect, pos_prob_rect)
return TensorSum(log_probs)
def optimize(self, nn, x, y, batch_size, epochs, early_stopping):
self._print_optimization_message(nn)
x, y = self._shuffle(x, y)
x_train, x_test, y_train, y_test = self._train_val_split(x, y)
n_batches = int(x_train.shape[0] / batch_size)
for epoch in range(epochs):
x_train, y_train = self._shuffle(x_train, y_train)
for batch in range(n_batches):
batch_grad = {}
start_splice = batch * batch_size
end_splice = (batch + 1) * batch_size
x_batch = x_train[start_splice:end_splice, ...]
y_batch = y_train[start_splice:end_splice, ...]
# forward pass
y_hat = nn.forward_pass(Tensor(x_batch))
loss = self.loss(Constant(y_batch), y_hat)
# backwards pass
grad = BackwardsPass(loss).execute()
for var in grad:
if var not in batch_grad:
batch_grad[var] = grad[var]
else:
batch_grad[var] += grad[var]
for var in grad:
grad[var] = grad[var] / batch_size
# update w/ optimization alg
self._update(nn, grad)
# control outputs
self._handle_prints(epoch, batch, n_batches)
# eval performance
train_loss = self._eval_perf(x_train, y_train, nn)
validation_loss = self._eval_perf(x_test, y_test, nn)
# early stopping
if early_stopping and epoch > 0:
if validation_loss > lst_epch_val_loss:
self._handle_prints(epoch, batch, n_batches, train_loss,
validation_loss)
break
lst_epch_val_loss = validation_loss
self._handle_prints(epoch, batch, n_batches, train_loss,
validation_loss)
def _eval_perf(self, x, y, model):
n_evals = x.shape[0]
y_hat = model.forward_pass(Tensor(x))
loss = self.loss(Constant(y), y_hat).value[0]
return loss / n_evals