def trainModel(modelFilename, allTrainDataImage, patchSize, margin, mu=None):
# Split into train and validation.
trainDataImage, validationDataImage = utils.splitData(allTrainDataImage, 0.9)
trainData = utils.makeClassificationPatches(trainDataImage, patchSize, margin)
validationData = utils.makeClassificationPatches(validationDataImage, patchSize, margin)
trainDataImage = None
validationDataImage = None
allTrainDataImage = None
print("Training data size {0}, Validation data size {1}".format(len(trainData), len(validationData)))
# Create shared
train_x, train_y = make_shared(trainData, patchSize, mu)
rng = np.random.RandomState(1234)
lasagne.random.set_rng(rng)
# allocate symbolic variables for the data
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are 0-1 labels.
input_dimension = patchSize**2
output_dimension = 2
#classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,),
# (500, nnlayer.ReluLayer),
# (output_dimension, nnlayer.LogisticRegressionLayer)])
classifier = nnlayer.ClassificationNet(input=x, topology=[(input_dimension,),
(nnlayer.LasangeNet.Reshape, (-1, 1, patchSize, patchSize)),
(nnlayer.LasangeNet.Conv, 32, 3, {'pad':'same'}),
(nnlayer.LasangeNet.Pool,),
(nnlayer.LasangeNet.Conv, 64, 3, {'pad':'same'}),
(nnlayer.LasangeNet.Pool,),
#(nnlayer.LasangeNet.DropoutLayer, 0.2),
#(nnlayer.LasangeNet.BatchNorm, nnlayer.LasangeNet.ReluLayer, 500),
#(nnlayer.LasangeNet.ReluLayer, 500),
#(nnlayer.LasangeNet.DropoutLayer, ),
#(nnlayer.LasangeNet.ReluLayer, 500),
#(nnlayer.LasangeNet.DropoutLayer, ),
(nnlayer.LasangeNet.SoftmaxLayer, output_dimension)])
cost = classifier.cost(y) #+ 0.0003*classifier.L2
costParams = []
costParams.extend(classifier.params)
costFunction = (costParams, cost, classifier.accuracy(y))
tt = MLPBatchTrainer()
# Create shared
validation_x, validation_y = make_shared(validationData, patchSize, mu)
v_start = T.iscalar()
v_end = T.iscalar()
bv_func = theano.function(inputs = [v_start, v_end],
outputs = [classifier.validation_cost(y), classifier.accuracy(y)],
givens = {x:validation_x[v_start:v_end], y:validation_y[v_start:v_end]})
def batch_validation():
maxIdx = len(validationData)
nv_batches = maxIdx // 128
tc = [0, 0]
for i in range(nv_batches):
d_start = min(i*128, maxIdx)
d_end = min((i + 1)*128, maxIdx)
bc = bv_func(d_start, d_end)
factor = (d_end - d_start) / 128.0
tc[0] += bc[0]*factor
tc[1] += bc[1]*factor
tc[0] /= nv_batches
tc[1] /= nv_batches
return tc
variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData)))
epochFunction, stateMananger = tt.getEpochTrainer(costFunction,
variableAndData,
batch_size=128,
rms = True,
momentum=0.9,
randomize=True,
updateFunction=MLPBatchTrainer.wrapUpdate(lasagne.updates.rmsprop))
# Train with adaptive learning rate.
stats = tt.trainALR2(epochFunction,
#valid_func,
batch_validation,
#initial_learning_rate=0.001,
initial_learning_rate=0.01,
max_runs=100,
state_manager = stateMananger)
validation_scores = [item["validation_outputs"][1] for item in stats]
train_scorees = [item["training_outputs"][1] for item in stats]
plt.plot(validation_scores, 'g')
plt.plot(train_scorees, 'r')
plt.show()
#e_func = theano.function(inputs = [],
# outputs = classifier.accuracy(y),
# #outputs = [classifier.cost(y)],
# givens = {x:validation_x, y:validation_y})
##print("avg error: {0}".format(np.mean(e_func())))
#print("validation accuracy: {0}".format(e_func()))
mgr = PersistenceManager()
mgr.set_filename(modelFilename)
mgr.set_model(x, y, classifier)
mgr.save_model()
# Use convnet
#conv_net = nnlayer.ConvNet(rng, x, (28, 28), [(20, 5, 5), (50, 5, 5)], rectified=True)
conv_net = nnlayer.ConvNet(rng, x, (28, 28), [(20, 5, 5)], rectified=True)
# the cost we minimize during training is the negative log likelihood of
# the model in symbolic format
classifier = nnlayer.MLPReg(rng=rng, input=conv_net.output, topology=[(conv_net.output_size,),
#(256, nnlayer.TanhLayer),
(10, nnlayer.LogisticRegressionLayer)])
cost = classifier.cost(y) + 0.0001*classifier.L2_sqr + 0.0001*conv_net.L2_sqr
params = conv_net.params + classifier.params
costFunction = (params, cost)
# Create trainer
tt = MLPBatchTrainer()
valid_func = theano.function(inputs = [],
outputs = [classifier.cost(y)],
givens = {x:validation_set_x, y:validation_set_y})
variableAndData = (VariableAndData(x, train_set_x), VariableAndData(y, train_set_y, size=train_set_x.get_value(borrow=True).shape[0]))
epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True, momentum=0.9)
# Train with adaptive learning rate.
stats = tt.trainALR(epochFunction,
valid_func,
initial_learning_rate=0.001,
epochs=3,
convergence_criteria=0.0001,
max_runs=100,
state_manager = stateMananger)
def buildData(filename, outName, resultSize):
loader = CSVLoader.Loader()
rawData = loader.LoadAsItems(filename)
rawData = rawData[:100]
rawData = sorted(rawData, key=lambda x:int(x['Count']), reverse=True)
labeledData = [item for item in rawData if not (item['Label'].upper() == 'UNKNOWN')]
trainData, testData = splitData(labeledData, split=0.99)
trainData, validationData = splitData(trainData, split=0.99)
print("Train {0}, Test {1}, Validation {2}".format(len(trainData), len(testData), len(validationData)))
#modality = fm.BagOfItemsMap(lambda x: x[1], fm.splitUpper)
#modality.build(modality.getUniqueValues(trainData))
#code = fm.BagOfItemsMap(lambda x: x[2], lambda x: [p[0:min(len(p), 3)] for p in fm.splitUpper(x)])
#code.build(code.getUniqueValues(trainData))
#body = fm.BagOfItemsMap(lambda x: x[3], fm.splitUpper)
#body.build(fm.getCommonTerms(body, trainData, minCount = None, size = 200))
#description = fm.BagOfItemsMap(lambda x: x[4], fm.splitUpper)
#description.build(fm.getCommonTerms(description, trainData, minCount = 10, size = None))
#label = fm.LabelMap(lambda x: x[labelIndex], fm.splitUpper)
#label.build(label.getUniqueValues(trainData))
#itemMapper = fm.ItemMapper([modality, code, body, description], label)
desc = [
{"key":'Modality', "type":"dict"},
{"key":'Code', "type":"dict", "valueLength":3},
{"key":'Body Part', "type":"dict", "size":200},
{"key":'Description', "type":"dict", "minCount":10},
{"key":'Label', "type":"label"}
]
builder = fm.ItemMapperBuilder(desc)
builder.build(trainData)
itemMapper = builder.pipe
print("Beginning mapping of {0} samples".format(len(trainData)))
mappedTrainX, mappedTrainY = itemMapper.map(trainData)
mappedValidationX, mappedValidationY = itemMapper.map(validationData)
print("Map completed")
mu = np.mean(mappedTrainX, axis=0)
sdev = np.std(mappedTrainX, axis=0) + 1e-5
mappedTrainX = (mappedTrainX - mu) / sdev
mappedValidationX = (mappedValidationX - mu) / sdev
# Create Theano shared data
train_x = theano.shared(mappedTrainX, borrow=True)
train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32')
validation_x = theano.shared(mappedValidationX, borrow=True)
validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32')
rng = np.random.RandomState(1234)
# allocate symbolic variables for the data
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
# the cost we minimize during training is the negative log likelihood of
# the model in symbolic format
input_dimension = itemMapper.dimension
output_dimension = itemMapper.range
classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,),
(100, nnlayer.ReluLayer),
(output_dimension, nnlayer.LogisticRegressionLayer)])
cost = classifier.cost(y) + 0.0001*classifier.L2_sqr
costParams = []
costParams.extend(classifier.params)
costFunction = (costParams, cost)
cum_dim = 0
for p in classifier.params:
cum_dim += p.get_value(borrow=True).size
print("Model dimension: {0}".format(cum_dim))
# Create validation function.
valid_func = theano.function(inputs = [],
outputs = [classifier.cost(y)],
givens = {x:validation_x, y:validation_y})
# Create trainer
tt = MLPBatchTrainer()
variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData)))
epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True)
# Train with adaptive learning rate.
stats = tt.trainALR(epochFunction,
valid_func,
initial_learning_rate=0.01,
epochs=2,
convergence_criteria=0.0001,
max_runs=10,
state_manager = stateMananger)
validation_scores = [item["validation_score"] for item in stats]
train_scorees = [item["training_costs"][-1] for item in stats]
#train_scorees = stats[0]["training_costs"]
plt.plot(validation_scores, 'g')
plt.plot(train_scorees, 'r')
plt.show()
input("Enter to continue:>")
mappedTestX, mappedTestY = itemMapper.map(testData)
#Normalize
mappedTestX = (mappedTestX - mu)/sdev
# Create Theano shared data
test_x = theano.shared(mappedTestX, borrow=True)
test_y = T.cast(theano.shared(mappedTestY, borrow=True), 'int32')
# Setup test function
batch_size=1
index = T.lscalar() # index to a [mini]batch
test_model = theano.function(inputs=[index],
outputs=(classifier.errors(y), classifier.y_pred),
givens={
x: test_x[index * batch_size: (index + 1) * batch_size],
y: test_y[index * batch_size: (index + 1) * batch_size]})
n_test_batch = int(test_x.get_value(borrow=True).shape[0] / batch_size)
errorVector = [test_model(i) for i in range(n_test_batch)]
#print("Avg. error {0}".format(np.average(errorVector)))
errCount = 0
for i in range(len(errorVector)):
if errorVector[i][0] > 0.0:
errCount += 1
print("Error: {0}, Label:{1}, Predicted:{2}".format(testData[i], testData[i]['Label'], itemMapper.labelMapper.inverseMap(int(errorVector[i][1]))))
print("Avg: {0}".format(errCount / len(errorVector)))
input("Enter to continue")
def train(self, trainArgs):
# Get train data
experimentDir = getCreateExperimentDir(self.id)
trainDataFilename = "{0}/traindata.pkl".format(experimentDir)
trainData, validationData = load(trainDataFilename)
#Build the mapper based on the training data.
itemMapperFilename = "{0}/mapper.pkl".format(experimentDir)
itemMapper = load(itemMapperFilename)[0]
print("Beginning mapping of {0} samples".format(len(trainData)))
mappedTrainX, mappedTrainY = itemMapper.map(trainData)
mappedValidationX, mappedValidationY = itemMapper.map(validationData)
#Normalize
#mu = np.mean(mappedTrainX, axis=0)
#sdev = np.std(mappedTrainX, axis=0) + 1e-5
#mappedTrainX = (mappedTrainX - mu) / sdev
#mappedValidationX = (mappedValidationX - mu) / sdev
# Create Theano shared data
train_x = theano.shared(mappedTrainX, borrow=True)
train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32')
validation_x = theano.shared(mappedValidationX, borrow=True)
validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32')
rng = np.random.RandomState(1234)
# allocate symbolic variables for the data
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
# the cost we minimize during training is the negative log likelihood of
# the model in symbolic format
input_dimension = itemMapper.dimension
output_dimension = itemMapper.range
# Check the train args for layers.
layerArgs = trainArgs["hiddenLayers"]
# Start with input
selectedLayers = [(input_dimension,)]
for l in layerArgs:
selectedLayers.append((l, nnlayer.ReluLayer))
# Add regression layer
selectedLayers.append((output_dimension, nnlayer.LogisticRegressionLayer))
classifier = nnlayer.MLPReg(rng=rng, input=x, topology=selectedLayers)
# classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,),
# (100, nnlayer.ReluLayer),
# (output_dimension, nnlayer.LogisticRegressionLayer)])
settings = trainArgs["settings"]
regularization = float(settings.get("regularization", "0.0001"))
cost = classifier.cost(y) + regularization*classifier.L2_sqr
costParams = []
costParams.extend(classifier.params)
costFunction = (costParams, cost)
cum_dim = 0
for p in classifier.params:
cum_dim += p.get_value(borrow=True).size
print("Model dimension: {0}".format(cum_dim))
# Create validation function.
valid_func = theano.function(inputs = [],
outputs = [classifier.cost(y)],
givens = {x:validation_x, y:validation_y})
# Create trainer
tt = MLPBatchTrainer()
variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData)))
epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True)
# Train with adaptive learning rate.
initial_lr = float(settings.get("learningRate", "0.01"))
eprun = int(settings.get("epochsPerRun", "1"))
runs = int(settings.get("runs", "10"))
stats = tt.trainALR(epochFunction,
valid_func,
initial_learning_rate=initial_lr,
epochs=eprun,
convergence_criteria=0.0001,
max_runs=runs,
state_manager = stateMananger)
experimentDir = getCreateExperimentDir(self.id)
modelFilename = "{0}/model.pkl".format(experimentDir)
# Save all relevant model data, include normalization if this is used.
save(modelFilename, x, y, classifier)
# Save the training plots.
validation_scores = [item["validation_score"] for item in stats]
train_scorees = [item["training_costs"][-1] for item in stats]
plt.plot(train_scorees, 'r', label="Training (regularized)")
plt.plot(validation_scores, 'g', label="Validation")
plt.legend(loc="upper right")
plt.ylabel("Score")
plt.xlabel("Run #")
figureFilename = "{0}/fig.png".format(experimentDir)
plt.savefig(figureFilename)
plt.close()
def runExperiment(experimentId, runArgs):
type = runArgs["type"]
if type == "create_datamapping":
return saveDatamapping(experimentId, runArgs["data"])
elif type == "train":
return train(experimentId, runArgs["data"])
# Load stored experiment data.
dataMapping = loadDatamapping(experimentId)
data, headers = getDataFile(experimentId)
#Split data into train/validation/test
trainData, testData = featuremapping.splitData(data, split=0.99)
trainData, validationData = featuremapping.splitData(trainData, split=0.99)
#Build the mapper based on the training data.
itemMapper = buildMapper(trainData, headers, dataMapping)
print("Beginning mapping of {0} samples".format(len(trainData)))
mappedTrainX, mappedTrainY = itemMapper.map(trainData)
mappedValidationX, mappedValidationY = itemMapper.map(validationData)
#Normalize
#mu = np.mean(mappedTrainX, axis=0)
#sdev = np.std(mappedTrainX, axis=0) + 1e-5
#mappedTrainX = (mappedTrainX - mu) / sdev
#mappedValidationX = (mappedValidationX - mu) / sdev
# Create Theano shared data
train_x = theano.shared(mappedTrainX, borrow=True)
train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32')
validation_x = theano.shared(mappedValidationX, borrow=True)
validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32')
rng = np.random.RandomState(1234)
# allocate symbolic variables for the data
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
# the cost we minimize during training is the negative log likelihood of
# the model in symbolic format
input_dimension = itemMapper.dimension
output_dimension = itemMapper.range
classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,),
(100, nnlayer.ReluLayer),
(output_dimension, nnlayer.LogisticRegressionLayer)])
cost = classifier.cost(y) + 0.0001*classifier.L2_sqr
costParams = []
costParams.extend(classifier.params)
costFunction = (costParams, cost)
cum_dim = 0
for p in classifier.params:
cum_dim += p.get_value(borrow=True).size
print("Model dimension: {0}".format(cum_dim))
# Create validation function.
valid_func = theano.function(inputs = [],
outputs = [classifier.cost(y)],
givens = {x:validation_x, y:validation_y})
# Create trainer
tt = MLPBatchTrainer()
variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData)))
epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True)
# Train with adaptive learning rate.
stats = tt.trainALR(epochFunction,
valid_func,
initial_learning_rate=0.01,
epochs=2,
convergence_criteria=0.0001,
max_runs=10,
state_manager = stateMananger)
validation_scores = [item["validation_score"] for item in stats]
train_scorees = [item["training_costs"][-1] for item in stats]
plt.plot(validation_scores, 'g')
plt.plot(train_scorees, 'r')
plt.show()
mappedTestX, mappedTestY = itemMapper.map(testData)
#Normalize
#mappedTestX = (mappedTestX - mu)/sdev
# Create Theano shared data
test_x = theano.shared(mappedTestX, borrow=True)
test_y = T.cast(theano.shared(mappedTestY, borrow=True), 'int32')
# Setup test function
batch_size=1
index = T.lscalar() # index to a [mini]batch
test_model = theano.function(inputs=[index],
outputs=(classifier.errors(y), classifier.y_pred),
givens={
x: test_x[index * batch_size: (index + 1) * batch_size],
y: test_y[index * batch_size: (index + 1) * batch_size]})
n_test_batch = int(test_x.get_value(borrow=True).shape[0] / batch_size)
errorVector = [test_model(i) for i in range(n_test_batch)]
#print("Avg. error {0}".format(np.average(errorVector)))
errCount = 0
for i in range(len(errorVector)):
if errorVector[i][0] > 0.0:
errCount += 1
print("Error: {0}, Predicted:{1}".format(testData[i], itemMapper.labelMapper.inverseMap(int(errorVector[i][1]))))
print("Avg: {0}".format(errCount / len(errorVector)))
def train(experimentId, trainingArgs):
# Load stored experiment data.
dataMapping = loadDatamapping(experimentId)
data, headers = getDataFile(experimentId)
#Split data into train/validation/test
trainData, testData = featuremapping.splitData(data, split=0.99)
trainData, validationData = featuremapping.splitData(trainData, split=0.99)
#Build the mapper based on the training data.
itemMapper = buildMapper(trainData, headers, dataMapping)
print("Beginning mapping of {0} samples".format(len(trainData)))
mappedTrainX, mappedTrainY = itemMapper.map(trainData)
mappedValidationX, mappedValidationY = itemMapper.map(validationData)
#Normalize
#mu = np.mean(mappedTrainX, axis=0)
#sdev = np.std(mappedTrainX, axis=0) + 1e-5
#mappedTrainX = (mappedTrainX - mu) / sdev
#mappedValidationX = (mappedValidationX - mu) / sdev
# Create Theano shared data
train_x = theano.shared(mappedTrainX, borrow=True)
train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32')
validation_x = theano.shared(mappedValidationX, borrow=True)
validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32')
rng = np.random.RandomState(1234)
# allocate symbolic variables for the data
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
# the cost we minimize during training is the negative log likelihood of
# the model in symbolic format
input_dimension = itemMapper.dimension
output_dimension = itemMapper.range
classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,),
(100, nnlayer.ReluLayer),
(output_dimension, nnlayer.LogisticRegressionLayer)])
cost = classifier.cost(y) + 0.0001*classifier.L2_sqr
costParams = []
costParams.extend(classifier.params)
costFunction = (costParams, cost)
cum_dim = 0
for p in classifier.params:
cum_dim += p.get_value(borrow=True).size
print("Model dimension: {0}".format(cum_dim))
# Create validation function.
valid_func = theano.function(inputs = [],
outputs = [classifier.cost(y)],
givens = {x:validation_x, y:validation_y})
# Create trainer
tt = MLPBatchTrainer()
variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData)))
epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True)
# Train with adaptive learning rate.
stats = tt.trainALR(epochFunction,
valid_func,
initial_learning_rate=0.01,
epochs=2,
convergence_criteria=0.0001,
max_runs=10,
state_manager = stateMananger)
experimentDir = getCreateExperimentDir(experimentId)
modelFilename = "{0}/model.pkl".format(experimentDir)
# Save all relevant model data, include normalization if this is used.
save(modelFilename, x, y, classifier, itemMapper)
def pretrainConv(trainData, nFilters, shape):
# Randomly sample patches from the trainData
# Each patch should be of size shape.
nSamples = 50000
data_x = np.zeros((nSamples, shape[0]*shape[1]), dtype = 'float32')
rng = np.random.RandomState(4321)
maxIndex = len(trainData) - 1
# Assuming that all samples have the same shape.
halfY = np.ceil(shape[1]/2)
minY = halfY
maxY = trainData[0].height - halfY
halfX = np.ceil(shape[0]/2)
minX = halfX
maxX = trainData[0].width - halfX
imageIndexes = rng.uniform(0, maxIndex, size =(nSamples))
xPositions = rng.uniform(minX, maxX, size=(nSamples))
yPositions = rng.uniform(minY, maxY, size=(nSamples))
for idx in range(nSamples):
i = int(imageIndexes[idx])
x = int(xPositions[idx])
y = int(yPositions[idx])
mammoSample = trainData[i]
xStart = x - halfX
yStart = y - halfY
xEnd = xStart + shape[0]
yEnd = yStart + shape[1]
data_x[idx] = mammoSample.pixelData[yStart:yEnd, xStart:xEnd].ravel()
#plt.gray()
#plt.imshow(data_x[idx].reshape((shape[0], shape[1])))
#plt.show()
# Prepare testdata.
nTrain = nSamples * 0.95
xTrain = data_x[:nTrain]
xValidation = data_x[nTrain:]
train_x = theano.shared(xTrain, borrow=True)
validation_x = theano.shared(xValidation, borrow=True)
# allocate symbolic variables for the data
x = T.matrix('x') # the data is presented as rasterized images
y = T.matrix('y') # the labels are (x,y) coordinates.
layer = nnlayer.ReluLayer(rng, x, shape[0]*shape[1], nFilters)
tt = MLPBatchTrainer()
#pre-training
dae = nnlayer.DAE(rng, layer, 0.2)
pretrainFunction, preStateManager = tt.getEpochTrainer((dae.params, dae.cost), [VariableAndData(x, train_x)], batch_size = 64, rms = True)
preTrainValid = theano.function(inputs = [],
outputs = [dae.cost],
givens = {x:validation_x})
preStats = tt.trainALR(pretrainFunction,
preTrainValid,
initial_learning_rate = 0.03,
epochs = 5,
#max_runs = 25,
max_runs = 1,
state_manager = preStateManager)
pre_scores = [item["validation_score"] for item in preStats]
plt.plot(pre_scores)
plt.show()
W = layer.W.get_value()
b = layer.b.get_value()
plotutils.plot_columns(W, (shape[0], shape[1]))
with open(r'.\SavedModels\conv_filters.pkl', 'wb') as f:
pickle.dump((W, b), f)
def trainModel(modelFilename, trainData, validationData, size):
#pretrainConv(trainData, 10, (5, 5))
with open(r'.\SavedModels\conv_filters.pkl', 'rb') as f:
W_conv, b_conv = pickle.load(f)
# Prepare testdata.
train_x, train_y = make_shared(trainData, size)
validation_x, validation_y = make_shared(validationData, size)
rng = np.random.RandomState(1234)
# allocate symbolic variables for the data
x = T.matrix('x') # the data is presented as rasterized images
y = T.matrix('y') # the labels are (x,y) coordinates.
batch_size = 512
tst = nnlayer.ConvNet(rng, x, (50, 50), [(10, 5, 5)], rectified=True)
#plotutils.plot_columns(W_conv, (5, 5))
W_t = np.zeros((10, 1, 5, 5), dtype='float32')
for i in range(10):
filt = W_conv[:, i].reshape((5, 5))
filt = np.fliplr(filt)
filt = np.flipud(filt)
W_t[i, 0] = filt
#plotutils.plot_tensor_image(W_t)
tst.layers[0].set_params(W_t, b_conv)
Wbefore = tst.layers[0].W.get_value()
#plotutils.plot_tensor_image(Wbefore)
print("Size {0}".format(tst.output_size))
input_dimension = size**2#tst.output_size
output_dimension = 2
classifier = nnlayer.MLPReg(rng=rng,
#input=tst.output,
input=x,
topology=[(input_dimension,),
(100, nnlayer.ReluLayer),
(output_dimension, nnlayer.LinearRegressionLayer)])
#classifier = nnlayer.MLPReg(rng=rng, input=tst.output, topology=[input_dimension, output_dimension], rectified=True, dropout_rate=0.5)
cost = classifier.cost(y) #+ 0.003*classifier.L2_sqr #+ 0.003*tst.L2_sqr
costParams = []
#costParams.extend(tst.params)
costParams.extend(classifier.params)
costFunction = (costParams, cost)
tt = MLPBatchTrainer()
#pre-training
#dae = nnlayer.DAE(rng, classifier.hiddenLayers[0], 0.2)
#pretrainFunction, preStateManager = tt.getEpochTrainer((dae.params, dae.cost), [VariableAndData(x, train_x)], batch_size = 64, rms = True)
#preTrainValid = theano.function(inputs = [],
# outputs = [dae.cost],
# givens = {x:validation_x})
#preStats = tt.trainALR(pretrainFunction,
# preTrainValid,
# initial_learning_rate = 0.001,
# epochs = 5,
# max_runs = 5,
# state_manager = preStateManager)
#pre_scores = [item["validation_score"] for item in preStats]
#plt.plot(pre_scores)
#plt.show()
#plotutils.plot_columns(classifier.hiddenLayers[0].W.get_value(), (50, 50))
# Supervised training
valid_func = theano.function(inputs = [],
outputs = [classifier.cost(y)],
givens = {x:validation_x, y:validation_y})
variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y))
epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True)
# Train with adaptive learning rate.
stats = tt.trainALR(epochFunction,
valid_func,
initial_learning_rate=0.0003,
epochs=1,
convergence_criteria=0.0001,
max_runs=15,
state_manager = stateMananger)
validation_scores = [item["validation_score"] for item in stats]
#train_scorees = [item["training_costs"][-1] for item in stats]
train_scorees = stats[0]["training_costs"]
plt.plot(validation_scores, 'g')
plt.plot(train_scorees, 'r')
plt.show()
Wafter = tst.layers[0].W.get_value()
#plotutils.plot_tensor_image(Wafter)
# Save model to disk
persistence = PersistenceManager()
persistence.set_filename(modelFilename)
persistence.set_model(x, y, classifier)
persistence.save_model()
