def predict(self, X, maxBatch=None):
'''X should be 2d numpy array'''
if maxBatch is None:
maxBatch = len(X)
if X.shape[1] == self.inputSize_ - 1:
X = np.append(np.ones((X.shape[0], 1)), X, 1)
yPred = Dual(np.zeros((len(X), self.outputSize_)), 0, self.weight_.n_)
lower = 0
for bx in batch(X, maxBatch=maxBatch):
raw = Dual(bx, n=self.nParameters_).matmul(self.weight_)
upper = lower + len(bx)
yPred[lower:upper] = self.outputActivation_(raw)
lower += maxBatch
return yPred
def predictIndependent_(self, X, elide):
self.valueOut_ = Dual(np.zeros((len(X), self.valueOutSize_)), 0,
self.nParameters_)
if X.shape[1] < self.valueInSize_:
X = np.append(
X, np.zeros((X.shape[0], self.valueInSize_ - X.shape[1])), 1)
self.valueIn_ = Dual(X, 0, self.nParameters_)
for n in range(self.valueOutSize_):
if n in elide:
continue
m = n + 1
self.valueOut_[:, n:m] = self.valueIn_.matmul(self.weight_[:, n:m])
if n < self.hiddenSize_:
self.valueIn_[:, self.inputSize_ + n:self.inputSize_ +
m] = self.hiddenActivation_(self.valueOut_[:,
n:m])
return self.outputActivation_(self.valueOut_[:, self.hiddenSize_:])
def predictRecurrent_(self, X, elide):
pred = np.zeros((len(X), self.outputSize_))
if not self.valueIn_:
self.valueIn_ = Dual(np.zeros(self.valueInSize_), 0,
self.nParameters_)
self.valueIn_[0] = 1
self.valueOut_ = Dual(np.zeros(self.valueOutSize_), 0,
self.nParameters_)
for s in range(len(X)):
self.valueIn_[:self.inputSize_] = X[s]
for n in range(self.valueOutSize_):
if n in elide:
continue
m = n + 1
self.valueOut_[n:m] = self.valueIn_.matmul(self.weight_[:,
n:m])
self.valueIn_[self.inputSize_ + n:self.inputSize_ +
m] = self.hiddenActivation_(self.valueOut_[n:m])
pred[s] = self.outputActivation_(self.valueOut_[self.hiddenSize_:])
return pred
def predict(self, X, maxBatch=None, elide=[]):
'''Perform forward pass.
idempotent when recurrent=False.
use elide list to leave out nodes (zero out, no update)'''
if maxBatch is None:
maxBatch = len(X)
if X.shape[1] == self.inputSize_ - 1:
X = np.append(np.ones((X.shape[0], 1)), X, 1)
if not isinstance(elide, list):
elide = [elide]
yPred = Dual(np.zeros((len(X), self.outputSize_)), 0, self.weight_.n_)
lower = 0
by = None
for bx in batch(X):
if self.recurrent_:
by = self.predictRecurrent_(bx, elide)
else:
by = self.predictIndependent_(bx, elide)
upper = lower + len(bx)
yPred[lower:upper] = by
lower += maxBatch
return yPred
def zeroTest():
tv = (Dual([-1, 10], 1), 0, 0)
testCase(Regularize.Zero(), *tv)
testCase(tr.Regularizer(tr.zero), *tv)
def klDivergenceTest():
tv = ( np.ones( 2, dtype=np.float32 ), Dual( [ 0.5, 1 ], 1 ), [ np.log( 2 ), 0 ], [ -2, -1 ] )
testCase( Error.KlDivergence(), *tv )
testCase( tl.Loss( tl.klDivergence ), *tv )
def JsDivergenceTest():
tv = ( np.ones( 2, dtype=np.float32 ), Dual( [ 0.5, 1 ], 1 ), [ ( np.log( 1 / 0.75 ) + 0.5 * np.log( 0.5 / 0.75 ) ) / 2, 0 ], [ -0.20273255, 0 ] )
testCase( Error.JsDivergence(), *tv )
testCase( tl.Loss( tl.jsDivergence ), *tv )
def hingeTest():
tv = ( np.array( [ 1, -1 ], dtype=np.float32 ), Dual( [ 2, 2 ], 1 ), [ 0, 3 ], [ 0, 1 ] )
testCase( Error.Hinge(), *tv )
testCase( tl.Loss( tl.hinge ), *tv )
def crossEntropyTest():
tv = ( np.ones( 2, dtype=np.float32 ), Dual( [ 0.5, 1 ], 1 ), [ np.log( 2 ), 0 ], [ -2, -1 ] )
testCase( Error.CrossEntropy(), *tv )
testCase( tl.Loss( tl.crossEntropy ), *tv )
def reluTest():
tv = (Dual([-1, 10], 1), [0, 10], [0, 1])
testCase(Activation.Relu(), *tv)
testCase(ta.Activation(ta.relu), *tv)
def huberTest():
tv = ( np.ones( 4, dtype=np.float32 ), Dual( [ 0.9, 1.1, 0, 2 ], 1 ), [ 0.005, 0.005, 0.5, 0.5 ], [ -0.1, 0.1, -1, 1 ] )
testCase( Error.Huber(), *tv )
testCase( tl.Loss( tl.huber ), *tv )
def zero(shape, total, first=0):
real = np.zeros(shape)
inft = makeInft(real.shape, total, first)
return Dual(real, inft, total)
def elasticTest():
tv = (Dual([-1, 10], 1), [0.1, 5.5], [-0.15, 1.05])
testCase(tr.Regularizer(tr.elasticNet, 0.1, 0.1), *tv)
def tanhTest():
tv = (Dual([-1, 10], 1), np.tanh([-1, 10]), 1 - np.tanh([-1, 10])**2)
testCase(Activation.Tanh(), *tv)
testCase(ta.Activation(ta.tanh), *tv)
def softmaxTest():
tv = (Dual([1, 2], 1), np.exp([1, 2]) / np.sum(np.exp([1, 2])), [0, 0])
testCase(Activation.Softmax(), *tv)
testCase(ta.Activation(ta.softmax, tf.constant(1.0)), *tv)
def logisticTest():
tv = (Dual(np.log([0.1, 10]), 1), [1 / 11,
1 / 1.1], [10 / 121, 0.1 / 1.21])
testCase(Activation.Logistic(), *tv)
testCase(ta.Activation(ta.logistic), *tv)
def eluTest():
tv = (Dual([-1, 10], 1), [np.exp(-1) - 1, 10], [np.exp(-1), 1])
testCase(Activation.Elu(), *tv)
testCase(ta.Activation(ta.elu), *tv)
def leakyReluTest():
tv = (Dual([-1, 10], 1), [-0.01, 10], [0.01, 1])
testCase(Activation.LeakyRelu(), *tv)
testCase(ta.Activation(ta.leaky, tf.constant(0.01)), *tv)
def ridgeTest():
tv = (Dual([-1, 10], 1), [0.05, 5], [-0.1, 1])
testCase(Regularize.Ridge(0.1), *tv)
testCase(tr.Regularizer(tr.ridge, 0.1), *tv)
def identityTest():
tv = (Dual([-1, 10], 1), [-1, 10], [1, 1])
testCase(Activation.Identity(), *tv)
testCase(ta.Activation(ta.identity), *tv)
def lassoTest():
tv = (Dual([-1, 10], 1), [0.05, 0.5], [-0.05, 0.05])
testCase(tr.Regularizer(tr.lasso, 0.1), *tv)
def softplusTest():
tv = (Dual([-1, 10],
1), np.log(1.0 + np.exp([-1, 10])), 1 / (1 + np.exp([1, -10])))
testCase(Activation.Softplus(), *tv)
testCase(ta.Activation(ta.softplus), *tv)
def normal(shape, total, first=0, mean=0, std=1):
'''randomize initial weights'''
real = np.random.standard_normal(shape) * std + mean
inft = makeInft(real.shape, total, first)
return Dual(real, inft, total)
def mseTest():
tv = ( np.ones( 2, dtype=np.float32 ), Dual( [ 0, 2 ], 1 ), [ 1, 1 ], [ -2, 2 ] )
testCase( Error.Mse(), *tv )
testCase( tl.Loss( tl.mse ), *tv )