def trainModel(modelFilename, allTrainDataImage, patchSize, margin, mu=None): # Split into train and validation. trainDataImage, validationDataImage = utils.splitData(allTrainDataImage, 0.9) trainData = utils.makeClassificationPatches(trainDataImage, patchSize, margin) validationData = utils.makeClassificationPatches(validationDataImage, patchSize, margin) trainDataImage = None validationDataImage = None allTrainDataImage = None print("Training data size {0}, Validation data size {1}".format(len(trainData), len(validationData))) # Create shared train_x, train_y = make_shared(trainData, patchSize, mu) rng = np.random.RandomState(1234) lasagne.random.set_rng(rng) # allocate symbolic variables for the data x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are 0-1 labels. input_dimension = patchSize**2 output_dimension = 2 #classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,), # (500, nnlayer.ReluLayer), # (output_dimension, nnlayer.LogisticRegressionLayer)]) classifier = nnlayer.ClassificationNet(input=x, topology=[(input_dimension,), (nnlayer.LasangeNet.Reshape, (-1, 1, patchSize, patchSize)), (nnlayer.LasangeNet.Conv, 32, 3, {'pad':'same'}), (nnlayer.LasangeNet.Pool,), (nnlayer.LasangeNet.Conv, 64, 3, {'pad':'same'}), (nnlayer.LasangeNet.Pool,), #(nnlayer.LasangeNet.DropoutLayer, 0.2), #(nnlayer.LasangeNet.BatchNorm, nnlayer.LasangeNet.ReluLayer, 500), #(nnlayer.LasangeNet.ReluLayer, 500), #(nnlayer.LasangeNet.DropoutLayer, ), #(nnlayer.LasangeNet.ReluLayer, 500), #(nnlayer.LasangeNet.DropoutLayer, ), (nnlayer.LasangeNet.SoftmaxLayer, output_dimension)]) cost = classifier.cost(y) #+ 0.0003*classifier.L2 costParams = [] costParams.extend(classifier.params) costFunction = (costParams, cost, classifier.accuracy(y)) tt = MLPBatchTrainer() # Create shared validation_x, validation_y = make_shared(validationData, patchSize, mu) v_start = T.iscalar() v_end = T.iscalar() bv_func = theano.function(inputs = [v_start, v_end], outputs = [classifier.validation_cost(y), classifier.accuracy(y)], givens = {x:validation_x[v_start:v_end], y:validation_y[v_start:v_end]}) def batch_validation(): maxIdx = len(validationData) nv_batches = maxIdx // 128 tc = [0, 0] for i in range(nv_batches): d_start = min(i*128, maxIdx) d_end = min((i + 1)*128, maxIdx) bc = bv_func(d_start, d_end) factor = (d_end - d_start) / 128.0 tc[0] += bc[0]*factor tc[1] += bc[1]*factor tc[0] /= nv_batches tc[1] /= nv_batches return tc variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData))) epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=128, rms = True, momentum=0.9, randomize=True, updateFunction=MLPBatchTrainer.wrapUpdate(lasagne.updates.rmsprop)) # Train with adaptive learning rate. stats = tt.trainALR2(epochFunction, #valid_func, batch_validation, #initial_learning_rate=0.001, initial_learning_rate=0.01, max_runs=100, state_manager = stateMananger) validation_scores = [item["validation_outputs"][1] for item in stats] train_scorees = [item["training_outputs"][1] for item in stats] plt.plot(validation_scores, 'g') plt.plot(train_scorees, 'r') plt.show() #e_func = theano.function(inputs = [], # outputs = classifier.accuracy(y), # #outputs = [classifier.cost(y)], # givens = {x:validation_x, y:validation_y}) ##print("avg error: {0}".format(np.mean(e_func()))) #print("validation accuracy: {0}".format(e_func())) mgr = PersistenceManager() mgr.set_filename(modelFilename) mgr.set_model(x, y, classifier) mgr.save_model()
# Use convnet #conv_net = nnlayer.ConvNet(rng, x, (28, 28), [(20, 5, 5), (50, 5, 5)], rectified=True) conv_net = nnlayer.ConvNet(rng, x, (28, 28), [(20, 5, 5)], rectified=True) # the cost we minimize during training is the negative log likelihood of # the model in symbolic format classifier = nnlayer.MLPReg(rng=rng, input=conv_net.output, topology=[(conv_net.output_size,), #(256, nnlayer.TanhLayer), (10, nnlayer.LogisticRegressionLayer)]) cost = classifier.cost(y) + 0.0001*classifier.L2_sqr + 0.0001*conv_net.L2_sqr params = conv_net.params + classifier.params costFunction = (params, cost) # Create trainer tt = MLPBatchTrainer() valid_func = theano.function(inputs = [], outputs = [classifier.cost(y)], givens = {x:validation_set_x, y:validation_set_y}) variableAndData = (VariableAndData(x, train_set_x), VariableAndData(y, train_set_y, size=train_set_x.get_value(borrow=True).shape[0])) epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True, momentum=0.9) # Train with adaptive learning rate. stats = tt.trainALR(epochFunction, valid_func, initial_learning_rate=0.001, epochs=3, convergence_criteria=0.0001, max_runs=100, state_manager = stateMananger)
def buildData(filename, outName, resultSize): loader = CSVLoader.Loader() rawData = loader.LoadAsItems(filename) rawData = rawData[:100] rawData = sorted(rawData, key=lambda x:int(x['Count']), reverse=True) labeledData = [item for item in rawData if not (item['Label'].upper() == 'UNKNOWN')] trainData, testData = splitData(labeledData, split=0.99) trainData, validationData = splitData(trainData, split=0.99) print("Train {0}, Test {1}, Validation {2}".format(len(trainData), len(testData), len(validationData))) #modality = fm.BagOfItemsMap(lambda x: x[1], fm.splitUpper) #modality.build(modality.getUniqueValues(trainData)) #code = fm.BagOfItemsMap(lambda x: x[2], lambda x: [p[0:min(len(p), 3)] for p in fm.splitUpper(x)]) #code.build(code.getUniqueValues(trainData)) #body = fm.BagOfItemsMap(lambda x: x[3], fm.splitUpper) #body.build(fm.getCommonTerms(body, trainData, minCount = None, size = 200)) #description = fm.BagOfItemsMap(lambda x: x[4], fm.splitUpper) #description.build(fm.getCommonTerms(description, trainData, minCount = 10, size = None)) #label = fm.LabelMap(lambda x: x[labelIndex], fm.splitUpper) #label.build(label.getUniqueValues(trainData)) #itemMapper = fm.ItemMapper([modality, code, body, description], label) desc = [ {"key":'Modality', "type":"dict"}, {"key":'Code', "type":"dict", "valueLength":3}, {"key":'Body Part', "type":"dict", "size":200}, {"key":'Description', "type":"dict", "minCount":10}, {"key":'Label', "type":"label"} ] builder = fm.ItemMapperBuilder(desc) builder.build(trainData) itemMapper = builder.pipe print("Beginning mapping of {0} samples".format(len(trainData))) mappedTrainX, mappedTrainY = itemMapper.map(trainData) mappedValidationX, mappedValidationY = itemMapper.map(validationData) print("Map completed") mu = np.mean(mappedTrainX, axis=0) sdev = np.std(mappedTrainX, axis=0) + 1e-5 mappedTrainX = (mappedTrainX - mu) / sdev mappedValidationX = (mappedValidationX - mu) / sdev # Create Theano shared data train_x = theano.shared(mappedTrainX, borrow=True) train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32') validation_x = theano.shared(mappedValidationX, borrow=True) validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32') rng = np.random.RandomState(1234) # allocate symbolic variables for the data x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels # the cost we minimize during training is the negative log likelihood of # the model in symbolic format input_dimension = itemMapper.dimension output_dimension = itemMapper.range classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,), (100, nnlayer.ReluLayer), (output_dimension, nnlayer.LogisticRegressionLayer)]) cost = classifier.cost(y) + 0.0001*classifier.L2_sqr costParams = [] costParams.extend(classifier.params) costFunction = (costParams, cost) cum_dim = 0 for p in classifier.params: cum_dim += p.get_value(borrow=True).size print("Model dimension: {0}".format(cum_dim)) # Create validation function. valid_func = theano.function(inputs = [], outputs = [classifier.cost(y)], givens = {x:validation_x, y:validation_y}) # Create trainer tt = MLPBatchTrainer() variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData))) epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True) # Train with adaptive learning rate. stats = tt.trainALR(epochFunction, valid_func, initial_learning_rate=0.01, epochs=2, convergence_criteria=0.0001, max_runs=10, state_manager = stateMananger) validation_scores = [item["validation_score"] for item in stats] train_scorees = [item["training_costs"][-1] for item in stats] #train_scorees = stats[0]["training_costs"] plt.plot(validation_scores, 'g') plt.plot(train_scorees, 'r') plt.show() input("Enter to continue:>") mappedTestX, mappedTestY = itemMapper.map(testData) #Normalize mappedTestX = (mappedTestX - mu)/sdev # Create Theano shared data test_x = theano.shared(mappedTestX, borrow=True) test_y = T.cast(theano.shared(mappedTestY, borrow=True), 'int32') # Setup test function batch_size=1 index = T.lscalar() # index to a [mini]batch test_model = theano.function(inputs=[index], outputs=(classifier.errors(y), classifier.y_pred), givens={ x: test_x[index * batch_size: (index + 1) * batch_size], y: test_y[index * batch_size: (index + 1) * batch_size]}) n_test_batch = int(test_x.get_value(borrow=True).shape[0] / batch_size) errorVector = [test_model(i) for i in range(n_test_batch)] #print("Avg. error {0}".format(np.average(errorVector))) errCount = 0 for i in range(len(errorVector)): if errorVector[i][0] > 0.0: errCount += 1 print("Error: {0}, Label:{1}, Predicted:{2}".format(testData[i], testData[i]['Label'], itemMapper.labelMapper.inverseMap(int(errorVector[i][1])))) print("Avg: {0}".format(errCount / len(errorVector))) input("Enter to continue")
def train(self, trainArgs): # Get train data experimentDir = getCreateExperimentDir(self.id) trainDataFilename = "{0}/traindata.pkl".format(experimentDir) trainData, validationData = load(trainDataFilename) #Build the mapper based on the training data. itemMapperFilename = "{0}/mapper.pkl".format(experimentDir) itemMapper = load(itemMapperFilename)[0] print("Beginning mapping of {0} samples".format(len(trainData))) mappedTrainX, mappedTrainY = itemMapper.map(trainData) mappedValidationX, mappedValidationY = itemMapper.map(validationData) #Normalize #mu = np.mean(mappedTrainX, axis=0) #sdev = np.std(mappedTrainX, axis=0) + 1e-5 #mappedTrainX = (mappedTrainX - mu) / sdev #mappedValidationX = (mappedValidationX - mu) / sdev # Create Theano shared data train_x = theano.shared(mappedTrainX, borrow=True) train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32') validation_x = theano.shared(mappedValidationX, borrow=True) validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32') rng = np.random.RandomState(1234) # allocate symbolic variables for the data x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels # the cost we minimize during training is the negative log likelihood of # the model in symbolic format input_dimension = itemMapper.dimension output_dimension = itemMapper.range # Check the train args for layers. layerArgs = trainArgs["hiddenLayers"] # Start with input selectedLayers = [(input_dimension,)] for l in layerArgs: selectedLayers.append((l, nnlayer.ReluLayer)) # Add regression layer selectedLayers.append((output_dimension, nnlayer.LogisticRegressionLayer)) classifier = nnlayer.MLPReg(rng=rng, input=x, topology=selectedLayers) # classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,), # (100, nnlayer.ReluLayer), # (output_dimension, nnlayer.LogisticRegressionLayer)]) settings = trainArgs["settings"] regularization = float(settings.get("regularization", "0.0001")) cost = classifier.cost(y) + regularization*classifier.L2_sqr costParams = [] costParams.extend(classifier.params) costFunction = (costParams, cost) cum_dim = 0 for p in classifier.params: cum_dim += p.get_value(borrow=True).size print("Model dimension: {0}".format(cum_dim)) # Create validation function. valid_func = theano.function(inputs = [], outputs = [classifier.cost(y)], givens = {x:validation_x, y:validation_y}) # Create trainer tt = MLPBatchTrainer() variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData))) epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True) # Train with adaptive learning rate. initial_lr = float(settings.get("learningRate", "0.01")) eprun = int(settings.get("epochsPerRun", "1")) runs = int(settings.get("runs", "10")) stats = tt.trainALR(epochFunction, valid_func, initial_learning_rate=initial_lr, epochs=eprun, convergence_criteria=0.0001, max_runs=runs, state_manager = stateMananger) experimentDir = getCreateExperimentDir(self.id) modelFilename = "{0}/model.pkl".format(experimentDir) # Save all relevant model data, include normalization if this is used. save(modelFilename, x, y, classifier) # Save the training plots. validation_scores = [item["validation_score"] for item in stats] train_scorees = [item["training_costs"][-1] for item in stats] plt.plot(train_scorees, 'r', label="Training (regularized)") plt.plot(validation_scores, 'g', label="Validation") plt.legend(loc="upper right") plt.ylabel("Score") plt.xlabel("Run #") figureFilename = "{0}/fig.png".format(experimentDir) plt.savefig(figureFilename) plt.close()
def runExperiment(experimentId, runArgs): type = runArgs["type"] if type == "create_datamapping": return saveDatamapping(experimentId, runArgs["data"]) elif type == "train": return train(experimentId, runArgs["data"]) # Load stored experiment data. dataMapping = loadDatamapping(experimentId) data, headers = getDataFile(experimentId) #Split data into train/validation/test trainData, testData = featuremapping.splitData(data, split=0.99) trainData, validationData = featuremapping.splitData(trainData, split=0.99) #Build the mapper based on the training data. itemMapper = buildMapper(trainData, headers, dataMapping) print("Beginning mapping of {0} samples".format(len(trainData))) mappedTrainX, mappedTrainY = itemMapper.map(trainData) mappedValidationX, mappedValidationY = itemMapper.map(validationData) #Normalize #mu = np.mean(mappedTrainX, axis=0) #sdev = np.std(mappedTrainX, axis=0) + 1e-5 #mappedTrainX = (mappedTrainX - mu) / sdev #mappedValidationX = (mappedValidationX - mu) / sdev # Create Theano shared data train_x = theano.shared(mappedTrainX, borrow=True) train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32') validation_x = theano.shared(mappedValidationX, borrow=True) validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32') rng = np.random.RandomState(1234) # allocate symbolic variables for the data x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels # the cost we minimize during training is the negative log likelihood of # the model in symbolic format input_dimension = itemMapper.dimension output_dimension = itemMapper.range classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,), (100, nnlayer.ReluLayer), (output_dimension, nnlayer.LogisticRegressionLayer)]) cost = classifier.cost(y) + 0.0001*classifier.L2_sqr costParams = [] costParams.extend(classifier.params) costFunction = (costParams, cost) cum_dim = 0 for p in classifier.params: cum_dim += p.get_value(borrow=True).size print("Model dimension: {0}".format(cum_dim)) # Create validation function. valid_func = theano.function(inputs = [], outputs = [classifier.cost(y)], givens = {x:validation_x, y:validation_y}) # Create trainer tt = MLPBatchTrainer() variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData))) epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True) # Train with adaptive learning rate. stats = tt.trainALR(epochFunction, valid_func, initial_learning_rate=0.01, epochs=2, convergence_criteria=0.0001, max_runs=10, state_manager = stateMananger) validation_scores = [item["validation_score"] for item in stats] train_scorees = [item["training_costs"][-1] for item in stats] plt.plot(validation_scores, 'g') plt.plot(train_scorees, 'r') plt.show() mappedTestX, mappedTestY = itemMapper.map(testData) #Normalize #mappedTestX = (mappedTestX - mu)/sdev # Create Theano shared data test_x = theano.shared(mappedTestX, borrow=True) test_y = T.cast(theano.shared(mappedTestY, borrow=True), 'int32') # Setup test function batch_size=1 index = T.lscalar() # index to a [mini]batch test_model = theano.function(inputs=[index], outputs=(classifier.errors(y), classifier.y_pred), givens={ x: test_x[index * batch_size: (index + 1) * batch_size], y: test_y[index * batch_size: (index + 1) * batch_size]}) n_test_batch = int(test_x.get_value(borrow=True).shape[0] / batch_size) errorVector = [test_model(i) for i in range(n_test_batch)] #print("Avg. error {0}".format(np.average(errorVector))) errCount = 0 for i in range(len(errorVector)): if errorVector[i][0] > 0.0: errCount += 1 print("Error: {0}, Predicted:{1}".format(testData[i], itemMapper.labelMapper.inverseMap(int(errorVector[i][1])))) print("Avg: {0}".format(errCount / len(errorVector)))
def train(experimentId, trainingArgs): # Load stored experiment data. dataMapping = loadDatamapping(experimentId) data, headers = getDataFile(experimentId) #Split data into train/validation/test trainData, testData = featuremapping.splitData(data, split=0.99) trainData, validationData = featuremapping.splitData(trainData, split=0.99) #Build the mapper based on the training data. itemMapper = buildMapper(trainData, headers, dataMapping) print("Beginning mapping of {0} samples".format(len(trainData))) mappedTrainX, mappedTrainY = itemMapper.map(trainData) mappedValidationX, mappedValidationY = itemMapper.map(validationData) #Normalize #mu = np.mean(mappedTrainX, axis=0) #sdev = np.std(mappedTrainX, axis=0) + 1e-5 #mappedTrainX = (mappedTrainX - mu) / sdev #mappedValidationX = (mappedValidationX - mu) / sdev # Create Theano shared data train_x = theano.shared(mappedTrainX, borrow=True) train_y = T.cast(theano.shared(mappedTrainY, borrow=True), 'int32') validation_x = theano.shared(mappedValidationX, borrow=True) validation_y = T.cast(theano.shared(mappedValidationY, borrow=True), 'int32') rng = np.random.RandomState(1234) # allocate symbolic variables for the data x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels # the cost we minimize during training is the negative log likelihood of # the model in symbolic format input_dimension = itemMapper.dimension output_dimension = itemMapper.range classifier = nnlayer.MLPReg(rng=rng, input=x, topology=[(input_dimension,), (100, nnlayer.ReluLayer), (output_dimension, nnlayer.LogisticRegressionLayer)]) cost = classifier.cost(y) + 0.0001*classifier.L2_sqr costParams = [] costParams.extend(classifier.params) costFunction = (costParams, cost) cum_dim = 0 for p in classifier.params: cum_dim += p.get_value(borrow=True).size print("Model dimension: {0}".format(cum_dim)) # Create validation function. valid_func = theano.function(inputs = [], outputs = [classifier.cost(y)], givens = {x:validation_x, y:validation_y}) # Create trainer tt = MLPBatchTrainer() variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y, size=len(trainData))) epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True) # Train with adaptive learning rate. stats = tt.trainALR(epochFunction, valid_func, initial_learning_rate=0.01, epochs=2, convergence_criteria=0.0001, max_runs=10, state_manager = stateMananger) experimentDir = getCreateExperimentDir(experimentId) modelFilename = "{0}/model.pkl".format(experimentDir) # Save all relevant model data, include normalization if this is used. save(modelFilename, x, y, classifier, itemMapper)
def pretrainConv(trainData, nFilters, shape): # Randomly sample patches from the trainData # Each patch should be of size shape. nSamples = 50000 data_x = np.zeros((nSamples, shape[0]*shape[1]), dtype = 'float32') rng = np.random.RandomState(4321) maxIndex = len(trainData) - 1 # Assuming that all samples have the same shape. halfY = np.ceil(shape[1]/2) minY = halfY maxY = trainData[0].height - halfY halfX = np.ceil(shape[0]/2) minX = halfX maxX = trainData[0].width - halfX imageIndexes = rng.uniform(0, maxIndex, size =(nSamples)) xPositions = rng.uniform(minX, maxX, size=(nSamples)) yPositions = rng.uniform(minY, maxY, size=(nSamples)) for idx in range(nSamples): i = int(imageIndexes[idx]) x = int(xPositions[idx]) y = int(yPositions[idx]) mammoSample = trainData[i] xStart = x - halfX yStart = y - halfY xEnd = xStart + shape[0] yEnd = yStart + shape[1] data_x[idx] = mammoSample.pixelData[yStart:yEnd, xStart:xEnd].ravel() #plt.gray() #plt.imshow(data_x[idx].reshape((shape[0], shape[1]))) #plt.show() # Prepare testdata. nTrain = nSamples * 0.95 xTrain = data_x[:nTrain] xValidation = data_x[nTrain:] train_x = theano.shared(xTrain, borrow=True) validation_x = theano.shared(xValidation, borrow=True) # allocate symbolic variables for the data x = T.matrix('x') # the data is presented as rasterized images y = T.matrix('y') # the labels are (x,y) coordinates. layer = nnlayer.ReluLayer(rng, x, shape[0]*shape[1], nFilters) tt = MLPBatchTrainer() #pre-training dae = nnlayer.DAE(rng, layer, 0.2) pretrainFunction, preStateManager = tt.getEpochTrainer((dae.params, dae.cost), [VariableAndData(x, train_x)], batch_size = 64, rms = True) preTrainValid = theano.function(inputs = [], outputs = [dae.cost], givens = {x:validation_x}) preStats = tt.trainALR(pretrainFunction, preTrainValid, initial_learning_rate = 0.03, epochs = 5, #max_runs = 25, max_runs = 1, state_manager = preStateManager) pre_scores = [item["validation_score"] for item in preStats] plt.plot(pre_scores) plt.show() W = layer.W.get_value() b = layer.b.get_value() plotutils.plot_columns(W, (shape[0], shape[1])) with open(r'.\SavedModels\conv_filters.pkl', 'wb') as f: pickle.dump((W, b), f)
def trainModel(modelFilename, trainData, validationData, size): #pretrainConv(trainData, 10, (5, 5)) with open(r'.\SavedModels\conv_filters.pkl', 'rb') as f: W_conv, b_conv = pickle.load(f) # Prepare testdata. train_x, train_y = make_shared(trainData, size) validation_x, validation_y = make_shared(validationData, size) rng = np.random.RandomState(1234) # allocate symbolic variables for the data x = T.matrix('x') # the data is presented as rasterized images y = T.matrix('y') # the labels are (x,y) coordinates. batch_size = 512 tst = nnlayer.ConvNet(rng, x, (50, 50), [(10, 5, 5)], rectified=True) #plotutils.plot_columns(W_conv, (5, 5)) W_t = np.zeros((10, 1, 5, 5), dtype='float32') for i in range(10): filt = W_conv[:, i].reshape((5, 5)) filt = np.fliplr(filt) filt = np.flipud(filt) W_t[i, 0] = filt #plotutils.plot_tensor_image(W_t) tst.layers[0].set_params(W_t, b_conv) Wbefore = tst.layers[0].W.get_value() #plotutils.plot_tensor_image(Wbefore) print("Size {0}".format(tst.output_size)) input_dimension = size**2#tst.output_size output_dimension = 2 classifier = nnlayer.MLPReg(rng=rng, #input=tst.output, input=x, topology=[(input_dimension,), (100, nnlayer.ReluLayer), (output_dimension, nnlayer.LinearRegressionLayer)]) #classifier = nnlayer.MLPReg(rng=rng, input=tst.output, topology=[input_dimension, output_dimension], rectified=True, dropout_rate=0.5) cost = classifier.cost(y) #+ 0.003*classifier.L2_sqr #+ 0.003*tst.L2_sqr costParams = [] #costParams.extend(tst.params) costParams.extend(classifier.params) costFunction = (costParams, cost) tt = MLPBatchTrainer() #pre-training #dae = nnlayer.DAE(rng, classifier.hiddenLayers[0], 0.2) #pretrainFunction, preStateManager = tt.getEpochTrainer((dae.params, dae.cost), [VariableAndData(x, train_x)], batch_size = 64, rms = True) #preTrainValid = theano.function(inputs = [], # outputs = [dae.cost], # givens = {x:validation_x}) #preStats = tt.trainALR(pretrainFunction, # preTrainValid, # initial_learning_rate = 0.001, # epochs = 5, # max_runs = 5, # state_manager = preStateManager) #pre_scores = [item["validation_score"] for item in preStats] #plt.plot(pre_scores) #plt.show() #plotutils.plot_columns(classifier.hiddenLayers[0].W.get_value(), (50, 50)) # Supervised training valid_func = theano.function(inputs = [], outputs = [classifier.cost(y)], givens = {x:validation_x, y:validation_y}) variableAndData = (VariableAndData(x, train_x), VariableAndData(y, train_y)) epochFunction, stateMananger = tt.getEpochTrainer(costFunction, variableAndData, batch_size=64, rms = True) # Train with adaptive learning rate. stats = tt.trainALR(epochFunction, valid_func, initial_learning_rate=0.0003, epochs=1, convergence_criteria=0.0001, max_runs=15, state_manager = stateMananger) validation_scores = [item["validation_score"] for item in stats] #train_scorees = [item["training_costs"][-1] for item in stats] train_scorees = stats[0]["training_costs"] plt.plot(validation_scores, 'g') plt.plot(train_scorees, 'r') plt.show() Wafter = tst.layers[0].W.get_value() #plotutils.plot_tensor_image(Wafter) # Save model to disk persistence = PersistenceManager() persistence.set_filename(modelFilename) persistence.set_model(x, y, classifier) persistence.save_model()