def __init__(self,
inputSize,
outputSize,
outputAct=Activation.Identity(),
iweight=Initialize.zero,
update=Update.Momentum(),
error=Error.Mse(),
regularization=Regularize.Zero()):
# structure settings
self.inputSize_ = inputSize + 1
self.outputSize_ = outputSize
self.nParameters_ = self.inputSize_ * self.outputSize_
self.outputActivationObj_ = outputAct
self.outputActivation_ = self.outputActivationObj_.f
# training settings
self.iweight_ = iweight
self.errorObj_ = error
self.errorFunc_ = error.f
self.updateObj_ = update
self.update_ = update.step
self.regObj_ = regularization
self.regLoss_ = self.regObj_.f
# put in clean state
self.initialize_()
def test():
# base data
X = np.random.randn( 1000, 1 ) * 10 + 50
Y = X * 2 - 10
# add noise
X += np.random.randn( 1000, 1 ) * 2
Y += np.random.randn( 1000, 1 ) * 2
# split
trainX = X[ :900 ]
trainY = Y[ :900 ]
testX = X[ 900: ]
testY = Y[ 900: ]
# for prediction line
plotX = np.array( [ min( X ), max( X ) ] )
iters = 2000
name = [ "RMSProp", "Momentum", "Nesterov", "SGD", "Rprop", "Adam" ]
model = [ LLS( 1, 1, update=Update.RmsProp() ),
LLS( 1, 1, update=Update.Momentum( 1e-7 ) ),
LLS( 1, 1, update=Update.NesterovMomentum( 1e-7 ) ),
LLS( 1, 1, update=Update.Sgd( 1e-7 ) ),
LLS( 1, 1, update=Update.Rprop() ),
LLS( 1, 1, update=Update.Adam() ) ]
error = np.zeros( ( len( model ), iters ) )
for i in range( iters ):
for m in range( len( model ) ):
error[ m, i ] = model[ m ].partial_fit( trainX, trainY )
print( i + 1, "complete" )
# plot results
plt.figure()
plt.title( 'Data Space' )
plt.scatter( trainX, trainY, label='train' )
plt.scatter( testX, testY, label='test' )
plt.plot( plotX, model[ 4 ].predict( plotX ).x_, label='prediction' )
plt.legend()
plt.figure()
plt.title( 'Error Curves' )
for m in range( len( model ) ):
plt.semilogy( error[ m ], label=name[ m ] )
plt.legend()
plt.show()
def __init__(self,
inputSize,
outputSize,
hiddenSize=100,
recurrent=False,
hiddenAct=Activation.Relu(),
outputAct=Activation.Identity(),
iweight=Initialize.he,
update=Update.Momentum(),
error=Error.Mse(),
regularization=Regularize.Zero()):
# structure settings
self.inputSize_ = inputSize + 1
self.outputSize_ = outputSize
self.hiddenSize_ = hiddenSize
self.valueInSize_ = self.inputSize_ + hiddenSize
self.valueOutSize_ = hiddenSize + outputSize
self.nParameters_ = self.valueInSize_ * self.valueOutSize_
self.hiddenActivationObj_ = hiddenAct
self.hiddenActivation_ = self.hiddenActivationObj_.f
self.outputActivationObj_ = outputAct
self.outputActivation_ = self.outputActivationObj_.f
self.recurrent_ = recurrent
# training settings
self.iweight_ = iweight
self.errorObj_ = error
self.errorFunc_ = error.f
self.updateObj_ = update
self.update_ = update.step
self.regObj_ = regularization
self.regLoss_ = self.regObj_.f
# put in clean state
self.initialize_()
