def run(simulationIndex, X, Y):
"""Run the model"""
print(("Training with:", simulationIndex))
dropoutRate = 0.25
mainGraph = ga.Graph()
ffeed = mainGraph.addOperation(ga.Variable(X), doGradient=False, feederOperation=True)
feedDrop = mainGraph.addOperation(ga.DropoutOperation(
ffeed, dropoutRate), doGradient=False, finalOperation=False)
l1 = ga.addDenseLayer(mainGraph, 100,
inputOperation=feedDrop,
activation=ga.ReLUActivation,
dropoutRate=dropoutRate,
batchNormalisation=True)
l2 = ga.addDenseLayer(mainGraph, 10,
inputOperation=l1,
activation=ga.SoftmaxActivation,
dropoutRate=0,
batchNormalisation=False)
fcost = mainGraph.addOperation(
ga.CrossEntropyCostSoftmax(l2, Y),
doGradient=False,
finalOperation=True)
def fprime(p, data, labels):
mainGraph.feederOperation.assignData(data)
mainGraph.resetAll()
mainGraph.finalOperation.assignLabels(labels)
mainGraph.attachParameters(p)
c = mainGraph.feedForward()
mainGraph.feedBackward()
g = mainGraph.unrollGradients()
return c, g
param0 = mainGraph.unrollGradientParameters()
adamGrad = ga.adaptiveSGD(trainingData=X,
trainingLabels=Y,
param0=param0,
epochs=1e2,
miniBatchSize=20,
initialLearningRate=1e-2,
beta1=0.9,
beta2=0.999,
epsilon=1e-8,
testFrequency=1e2,
function=fprime)
pickleFilename = "minimizerParamsDense_" + str(simulationIndex) + ".pkl"
# with open(pickleFilename, "rb") as fp:
# adamParams = pickle.load(fp)
# adamGrad.restoreState(adamParams)
# params = adamParams["params"]
params = adamGrad.minimize(printTrainigCost=True, printUpdateRate=False,
dumpParameters=pickleFilename)
mainGraph.attachParameters(params)
return mainGraph
def run():
"""Run the model"""
N, D, H1, H2 = 10, 3, 4, 2
trainData = np.arange(0, N * D).reshape(N, D).astype(np.float)
trainLabels = np.arange(0, N * H2).reshape(N, H2).astype(np.float)
mainGraph = ga.Graph()
ffeed = mainGraph.addOperation(ga.Variable(trainData),
doGradient=False,
feederOperation=True)
feedDrop = mainGraph.addOperation(ga.DropoutOperation(ffeed, 0.0),
doGradient=False,
finalOperation=False)
l1 = ga.addDenseLayer(mainGraph,
H1,
inputOperation=feedDrop,
activation=ga.ReLUActivation,
dropoutRate=0.0,
batchNormalisation=True)
l2 = ga.addDenseLayer(mainGraph,
H2,
inputOperation=l1,
activation=ga.SoftmaxActivation,
dropoutRate=0.0,
batchNormalisation=False)
fcost = mainGraph.addOperation(ga.CrossEntropyCostSoftmax(l2, trainLabels),
doGradient=False,
finalOperation=True)
def f(x):
mainGraph.attachParameters(x)
return mainGraph.getValue()
def fprime(p, data, labels):
mainGraph.feederOperation.assignData(data)
mainGraph.resetAll()
mainGraph.finalOperation.assignLabels(labels)
mainGraph.attachParameters(p)
c = mainGraph.feedForward()
mainGraph.feedBackward()
g = mainGraph.unrollGradients()
return c, g
params = mainGraph.unrollGradientParameters()
numGrad = scipy.optimize.approx_fprime(params, f, 1e-8)
analCostGraph, analGradientGraph = fprime(params, trainData, trainLabels)
return numGrad, analGradientGraph, analCostGraph, mainGraph
def run(simulationIndex, X, Y):
"""Run the model"""
# index = int(sys.argv[1])
print("Training with:", simulationIndex)
dropValueL = 0.1
dropValueS = 0.05
# ------ Build a LeNet archicture CNN
mainGraph = ga.Graph()
feed = mainGraph.addOperation(ga.Variable(X),
doGradient=False,
feederOperation=True)
feedDrop = mainGraph.addOperation(ga.DropoutOperation(feed, dropValueS),
doGradient=False,
finalOperation=False)
cnn1 = ga.addConv2dLayer(mainGraph,
inputOperation=feedDrop,
nFilters=20,
filterHeigth=5,
filterWidth=5,
padding="SAME",
convStride=1,
activation=ga.ReLUActivation,
batchNormalisation=False,
pooling=ga.MaxPoolOperation,
poolHeight=2,
poolWidth=2,
poolStride=2)
cnn2 = ga.addConv2dLayer(mainGraph,
inputOperation=cnn1,
nFilters=50,
filterHeigth=5,
filterWidth=5,
padding="SAME",
convStride=1,
activation=ga.ReLUActivation,
batchNormalisation=True,
pooling=ga.MaxPoolOperation,
poolHeight=2,
poolWidth=2,
poolStride=2)
flattenOp = mainGraph.addOperation(ga.FlattenFeaturesOperation(cnn2))
flattenDrop = mainGraph.addOperation(ga.DropoutOperation(
flattenOp, dropValueL),
doGradient=False,
finalOperation=False)
l1 = ga.addDenseLayer(mainGraph,
500,
inputOperation=flattenDrop,
activation=ga.ReLUActivation,
dropoutRate=dropValueL,
batchNormalisation=True)
l2 = ga.addDenseLayer(mainGraph,
10,
inputOperation=l1,
activation=ga.SoftmaxActivation,
dropoutRate=0.0,
batchNormalisation=False)
fcost = mainGraph.addOperation(ga.CrossEntropyCostSoftmax(l2, Y),
doGradient=False,
finalOperation=True)
def fprime(p, data, labels):
mainGraph.feederOperation.assignData(data)
mainGraph.resetAll()
mainGraph.finalOperation.assignLabels(labels)
mainGraph.attachParameters(p)
c = mainGraph.feedForward()
mainGraph.feedBackward()
g = mainGraph.unrollGradients()
return c, g
param0 = mainGraph.unrollGradientParameters()
adamGrad = ga.adaptiveSGD(trainingData=X,
trainingLabels=Y,
param0=param0,
epochs=10,
miniBatchSize=10,
initialLearningRate=1e-2,
beta1=0.9,
beta2=0.999,
epsilon=1e-8,
testFrequency=1e1,
function=fprime)
pickleFilename = "minimizerParamsCNN_" + str(simulationIndex) + ".pkl"
# with open(pickleFilename, "rb") as fp:
# adamParams = pickle.load(fp)
# adamGrad.restoreState(adamParams)
# params = adamParams["params"]
params = adamGrad.minimize(printTrainigCost=True,
printUpdateRate=False,
dumpParameters=pickleFilename)
mainGraph.attachParameters(params)
return mainGraph
Xcheck2 = np.arange(1, 7).reshape((2, 3)).astype(np.float)
Ycheck2 = np.array([[1, 0], [0, 1]]).astype(np.float)
mainGraph = ga.Graph()
ffeed = mainGraph.addOperation(ga.Variable(Xcheck2),
doGradient=False,
feederOperation=True)
feedDrop = mainGraph.addOperation(ga.DropoutOperation(ffeed, 0.0),
doGradient=False,
finalOperation=False)
l1 = ga.addDenseLayer(mainGraph,
4,
inputOperation=feedDrop,
activation=ga.ReLUActivation,
dropoutRate=0.0,
w=None,
b=None)
l2 = ga.addDenseLayer(mainGraph,
2,
inputOperation=l1,
activation=ga.SoftmaxActivation,
dropoutRate=0.0,
w=None,
b=None)
fcost = mainGraph.addOperation(ga.CrossEntropyCostSoftmax(l2, Ycheck2),
doGradient=False,
finalOperation=True)
print(mainGraph)
def run():
"""Run the model"""
trainData = np.random.random((5, 1, 10, 10))
trainLabels = np.random.random((5, 10))
# ------ conv2D operation testing
mainGraph = ga.Graph()
feed = mainGraph.addOperation(ga.Variable(trainData),
doGradient=False,
feederOperation=True)
cnn1 = ga.addConv2dLayer(mainGraph,
inputOperation=feed,
nFilters=3,
filterHeigth=5,
filterWidth=5,
padding="SAME",
convStride=1,
activation=ga.ReLUActivation,
batchNormalisation=True,
pooling=ga.MaxPoolOperation,
poolHeight=2,
poolWidth=2,
poolStride=2)
flattenOp = mainGraph.addOperation(ga.FlattenFeaturesOperation(cnn1))
flattenDrop = mainGraph.addOperation(ga.DropoutOperation(flattenOp, 0.0),
doGradient=False,
finalOperation=False)
l1 = ga.addDenseLayer(mainGraph,
20,
inputOperation=flattenDrop,
activation=ga.ReLUActivation,
dropoutRate=0.0,
batchNormalisation=False)
l2 = ga.addDenseLayer(mainGraph,
10,
inputOperation=l1,
activation=ga.SoftmaxActivation,
dropoutRate=0.0,
batchNormalisation=False)
fcost = mainGraph.addOperation(ga.CrossEntropyCostSoftmax(l2, trainLabels),
doGradient=False,
finalOperation=True)
def fprime(p, data, labels):
mainGraph.feederOperation.assignData(data)
mainGraph.resetAll()
mainGraph.finalOperation.assignLabels(labels)
mainGraph.attachParameters(p)
c = mainGraph.feedForward()
mainGraph.feedBackward()
g = mainGraph.unrollGradients()
return c, g
def f(p):
data = trainData
labels = trainLabels
mainGraph.feederOperation.assignData(data)
mainGraph.resetAll()
mainGraph.finalOperation.assignLabels(labels)
mainGraph.attachParameters(p)
c = mainGraph.feedForward()
return c
params = mainGraph.unrollGradientParameters()
numGrad = scipy.optimize.approx_fprime(params, f, 1e-8)
analCostGraph, analGradientGraph = fprime(params, trainData, trainLabels)
return numGrad, analGradientGraph, analCostGraph, mainGraph