def UpdateAlgorithm(alg, params, gparams, pdecay=None, lr=None, l2reg=None):
## currently the code mainly work for Adam, AdamW and SGDM. Other algorithms are not fully tested.
other_params = []
#alg = modelSpecs['algorithm']
if alg == 'SGD':
updates = GD(params, gparams, lr)
elif alg == 'SGDM':
updates, other_params = SGDM(params, gparams, np.float32(0.90), lr)
elif alg == 'SGDM2':
updates, other_params = SGDM2(params, gparams, np.float32(0.90), lr)
elif alg == 'SGNA':
updates, other_params = Nesterov(params, gparams, np.float32(0.90), lr)
elif alg == 'Adam':
updates, other_params = Adam(params, gparams, lr=lr)
elif alg == 'AMSGrad':
updates, other_params = AMSGrad(params, gparams, lr=lr)
elif alg == 'AdamW':
assert (lr is not None)
assert (l2reg is not None)
updates, other_params = AdamW(params,
gparams,
pdecay=pdecay,
l2reg=l2reg,
lr=lr)
elif alg == 'AdamWAMS':
assert (lr is not None)
assert (l2reg is not None)
updates, other_params = AdamWAMS(params,
gparams,
pdecay=pdecay,
l2reg=l2reg,
lr=lr)
else:
raise NotImplementedError
return updates, other_params
def UpdateAlgorithm(alg, params, gparams, pdecay, lr=None, l2reg=None):
other_params = []
if alg == 'SGD':
updates = GD(params, gparams, lr)
elif alg == 'SGDM':
updates, other_params = SGDM(params, gparams, np.float32(0.90), lr)
elif alg == 'SGDM2':
updates, other_params = SGDM2(params, gparams, np.float32(0.90), lr)
elif alg == 'SGNA':
updates, other_params = Nesterov(params, gparams, np.float32(0.90), lr)
elif alg == 'Adam':
updates, other_params = Adam(params, gparams, lr=lr)
elif alg == 'AMSGrad':
updates, other_params = AMSGrad(params, gparams, lr=lr)
elif alg == 'AdamW':
assert (l2reg is not None)
updates, other_params = AdamW(params,
gparams,
pdecay=pdecay,
l2reg=l2reg,
lr=lr)
elif alg == 'AdamWAMS':
assert (l2reg is not None)
updates, other_params = AdamWAMS(params,
gparams,
pdecay=pdecay,
l2reg=l2reg,
lr=lr)
else:
raise NotImplementedError
return updates, other_params
def testNN4LogReg(learning_rate=0.01,
L1_reg=0.00,
L2_reg=0.0001,
n_epochs=2000,
n_hiddens=[200, 200],
trainData=None,
testData=None):
## generate some random train and test data
batchSize = 200000
nFeatures = 30
trainX = numpy.random.uniform(0, 1,
(batchSize, nFeatures)).astype(numpy.float32)
trainXsum = numpy.sum(trainX**2, axis=1, keepdims=True)
trainY = numpy.zeros((batchSize, 1), dtype=numpy.int32)
numpy.putmask(trainY, trainXsum > 5, 1)
numpy.putmask(trainY, trainXsum > 10, 2)
numpy.putmask(trainY, trainXsum > 15, 3)
testBatchSize = 50
testX = numpy.random.uniform(0, 1, (testBatchSize, nFeatures)).astype(
numpy.float32)
testXsum = numpy.sum(testX**2, axis=1, keepdims=True)
testY = numpy.zeros((testBatchSize, 1), dtype=numpy.int32)
numpy.putmask(testY, testXsum > 5, 1)
numpy.putmask(testY, testXsum > 10, 2)
numpy.putmask(testY, testXsum > 15, 3)
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
x = T.matrix('x') # the data is presented as rasterized images
y = T.imatrix('y') # the labels
rng = numpy.random.RandomState()
regressor = NN4LogReg(rng,
input=x,
n_in=trainX.shape[1],
n_hiddens=n_hiddens,
n_out=4,
activation=T.nnet.relu)
loss = regressor.loss(y)
error = regressor.errors(y)
cost = loss + L1_reg * regressor.paramL1 + L2_reg * regressor.paramL2
gparams = [T.grad(cost, param) for param in regressor.params]
param_shapes = [param.shape.eval() for param in regressor.params]
updates, others = Adam(regressor.params, gparams)
train = theano.function(
inputs=[x, y],
outputs=[loss, error, regressor.paramL1, regressor.paramL2],
updates=updates)
test = theano.function(inputs=[x, y], outputs=error)
calculate = theano.function(inputs=[x], outputs=regressor.output)
step = 200
numEpochs = 30
for j in range(0, numEpochs):
results = []
for i in range(0, trainX.shape[0], step):
los, err, l1, l2 = train(trainX[i:i + step, :],
trainY[i:i + step, :])
results.append(los)
if i % 5000 == 0:
print 'i=', i, ' loss=', los, ' error=', err, ' L1norm=', l1, ' L2norm=', l2
print 'j=', j, ' avgLos, avgErr=', numpy.mean(results, axis=0)
out = calculate(testX)
print numpy.around(
numpy.concatenate((out, testY.reshape(testBatchSize, 1)), axis=1), 2)
print 'err=', test(testX, testY)
def testNN4Normal(learning_rate=0.01,
L1_reg=0.00,
L2_reg=0.0001,
n_epochs=2000,
n_hiddens=[100, 200],
trainData=None,
testData=None):
## generate some random train and test data
batchSize = 200000
nFeatures = 50
trainX = numpy.random.uniform(0, 1,
(batchSize, nFeatures)).astype(numpy.float32)
u1 = numpy.sum(trainX[:, :30], axis=1, keepdims=True)
u2 = numpy.sum(trainX[:, 21:], axis=1, keepdims=True)
trainY = (numpy.random.normal(0, 2., (batchSize, 2)) + numpy.concatenate(
(u1, u2), axis=1)).astype(numpy.float32)
testBatchSize = 500
testX = numpy.random.uniform(0, 1, (testBatchSize, nFeatures)).astype(
numpy.float32)
testu1 = numpy.sum(testX[:, :30], axis=1, keepdims=True)
testu2 = numpy.sum(testX[:, 21:], axis=1, keepdims=True)
testY = (numpy.random.normal(0, 2., (testBatchSize, 2)) +
numpy.concatenate((testu1, testu2), axis=1)).astype(numpy.float32)
testCorr = numpy.sum(testX[:, 21:30], axis=1, keepdims=True) / numpy.sum(
testX, axis=1, keepdims=True)
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# allocate symbolic variables for the data
x = T.matrix('x') # the input feature
y = T.matrix('y') # the response
rng = numpy.random.RandomState()
regressor = NN4Normal(rng,
input=x,
n_in=trainX.shape[1],
n_variables=2,
n_out=5,
n_hiddens=n_hiddens,
sigma_sqr_min=0.01)
loss = regressor.loss(y)
cost = loss + L1_reg * regressor.paramL1 + L2_reg * regressor.paramL2
error = regressor.errors(y)
gparams = [T.grad(cost, param) for param in regressor.params]
param_shapes = [param.shape.eval() for param in regressor.params]
updates, others = Adam(regressor.params, gparams)
train = theano.function(
inputs=[x, y],
outputs=[loss, error, regressor.paramL1, regressor.paramL2],
updates=updates)
test = theano.function(inputs=[x, y], outputs=error)
calculate = theano.function(inputs=[x], outputs=regressor.output)
step = 200
numEpochs = 13
for j in range(0, numEpochs):
results = []
for i in range(0, trainX.shape[0], step):
los, err, l1, l2 = train(trainX[i:i + step, :],
trainY[i:i + step, :])
results.append(los)
if i % 5000 == 0:
print 'i=', i, ' loss=', los, ' error=', err, ' L1norm=', l1, ' L2norm=', l2
print 'j=', j, ' avgLos, avgErr=', numpy.mean(results, axis=0)
out = calculate(testX)
print numpy.concatenate((out, testCorr, testY),
axis=1).astype(numpy.float16)
print 'err=', test(testX, testY)
corr = numpy.concatenate((out[:, 4:5], testCorr), axis=1)
print numpy.corrcoef(numpy.transpose(corr))
import scipy
print scipy.stats.mstats.spearmanr(corr[:, 0], corr[:, 1])
def testNN4PhiPsi(learning_rate=0.01, L1_reg=0.00, L2_reg=0.01, n_epochs=2000, n_hiddens=[100, 200, 200], trainData=None, testData=None):
## generate some random train and test data
batchSize = 200000
nFeatures = 50
trainX = numpy.random.uniform(0, 12, (batchSize, nFeatures)).astype(numpy.float32)
u1 = numpy.sum( trainX[:,:30], axis=1, keepdims=True)
u2 = numpy.sum( trainX[:,21:], axis=1, keepdims=True)
numpy.putmask(u1, u1>180, u1-360)
numpy.putmask(u2, u2>180, u2-360)
u1 = u1/180.0 * numpy.pi
u2 = u2/180.0 * numpy.pi
trainY = (numpy.random.normal(0, 0.4, (batchSize, 2)) + numpy.concatenate( (u1, u2), axis=1) ).astype(numpy.float32)
numpy.putmask(trainY, trainY > numpy.pi, trainY - 2*numpy.pi)
numpy.putmask(trainY, trainY < -numpy.pi, trainY + 2*numpy.pi)
testBatchSize = 500
testX = numpy.random.uniform(0, 12, (testBatchSize, nFeatures)).astype(numpy.float32)
testu1 = numpy.sum(testX[:,:30], axis=1, keepdims=True)
testu2 = numpy.sum(testX[:,21:], axis=1, keepdims=True)
testCorr = numpy.sum(testX[:,21:30], axis=1, keepdims=True)/numpy.sum(testX, axis=1, keepdims=True)
numpy.putmask(testu1, testu1>180, testu1-360)
numpy.putmask(testu2, testu2>180, testu2-360)
testu1 = testu1/180.0 * numpy.pi
testu2 = testu2/180.0 * numpy.pi
testY = (numpy.random.normal(0, 0.4, (testBatchSize, 2)) + numpy.concatenate( (testu1, testu2), axis=1) ).astype(numpy.float32)
numpy.putmask(testY, testY > numpy.pi, testY - 2*numpy.pi)
numpy.putmask(testY, testY < -numpy.pi, testY + 2*numpy.pi)
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# allocate symbolic variables for the data
x = T.matrix('x') # the input feature
y = T.matrix('y') # the response
rng = numpy.random.RandomState()
regressor = NN4PhiPsi(rng, input=x, n_in=trainX.shape[1], n_hiddens=n_hiddens)
loss = regressor.loss(y)
cost = loss + L1_reg * regressor.paramL1 + L2_reg * regressor.paramL2
error = regressor.errors(y)
gparams = [T.grad(cost, param) for param in regressor.params]
param_shapes = [ param.shape.eval() for param in regressor.params ]
updates, others = Adam(regressor.params, gparams)
train = theano.function( inputs=[x,y], outputs=[loss, error, regressor.paramL1, regressor.paramL2], updates=updates)
calculate = theano.function( inputs=[x], outputs=regressor.output )
#test = theano.function( inputs=[x,y], outputs=regressor.NLL(y))
test = theano.function( inputs=[x,y], outputs=regressor.errors(y))
step = 500
numEpochs = 12
for j in range(0, numEpochs):
for i in range(0,trainX.shape[0], step):
los, err, l1, l2 = train(trainX[i:i+step, :], trainY[i:i+step, :])
if i%10000 == 0:
print 'j=', j, ' i=', i, ' loss=', los, ' error=', err, ' L1norm=', l1, ' L2norm=', l2
out = calculate(testX)
print numpy.concatenate( (out, testCorr, testY), axis=1)
print 'err=', test(testX, testY)
corr = numpy.concatenate( (out[:,4:5], testCorr), axis=1)
print numpy.corrcoef( numpy.transpose(corr) )
import scipy
print scipy.stats.mstats.spearmanr(corr[:,0], corr[:,1])