def run(LEARNING_RATE=0.04, UPDATE_MOMENTUM=0.9,UPDATE_RHO=None, NUM_OF_EPOCH=50, OUTPUT_SIZE = 20 , input_width=300, input_height=140, dataset='ISH.pkl.gz', TRAIN_VALIDATION_SPLIT=0.2, #activation=lasagne.nonlinearities.tanh, #rectify NUM_UNITS_HIDDEN_LAYER=[5, 10, 20, 40], BATCH_SIZE=40, toShuffleInput = False , withZeroMeaning = False): global counter FILE_PREFIX = os.path.split(dataset)[1][:-6] #os.path.split(__file__)[1][:-3] FOLDER_PREFIX = "results/"+FILE_PREFIX+"/run_"+str(counter)+"/" if not os.path.exists(FOLDER_PREFIX): os.makedirs(FOLDER_PREFIX) PARAMS_FILE_NAME = FOLDER_PREFIX + "parameters.txt" HIDDEN_LAYER_OUTPUT_FILE_NAME = FOLDER_PREFIX +"hiddenLayerOutput.pickle" FIG_FILE_NAME = FOLDER_PREFIX + "fig" PICKLES_NET_FILE_NAME = FOLDER_PREFIX + "picklesNN.pickle" SVM_FILE_NAME = FOLDER_PREFIX + "svmData.txt" # VALIDATION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_validation_"+str(counter)+".txt" # PREDICTION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_prediction.txt" counter +=1 outputFile = open(PARAMS_FILE_NAME, "w") def load2d(dataset='ISH.pkl.gz', toShuffleInput = False , withZeroMeaning = False): print 'loading data...' datasets = load_data(dataset, toShuffleInput, withZeroMeaning) train_set_x, train_set_y = datasets[0] # valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] train_set_x = train_set_x.reshape(-1, 1, input_width, input_height) # valid_set_x = valid_set_x.reshape(-1, 1, input_width, input_height) test_set_x = test_set_x.reshape(-1, 1, input_width, input_height) print(train_set_x.shape[0], 'train samples') return train_set_x, train_set_y, test_set_x, test_set_y def createNNwithMomentom(input_height, input_width): net2 = NeuralNet(layers=[ ('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('conv4', layers.Conv2DLayer), ('pool4', layers.MaxPool2DLayer), ('hidden5', layers.DenseLayer), ('hidden6', layers.DenseLayer), ('hidden7', layers.DenseLayer), ('output', layers.DenseLayer)], input_shape=(None, 1, input_width, input_height), conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2), conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2), conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2), conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2), hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100, output_num_units=20, output_nonlinearity=None, update_learning_rate=LEARNING_RATE, update_momentum=UPDATE_MOMENTUM, update=nesterov_momentum, train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT), batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE), regression=True, max_epochs=NUM_OF_EPOCH, verbose=1, hiddenLayer_to_output=-2) # on_training_finished=last_hidden_layer, return net2 def createNNwithDecay(input_height, input_width): net2 = NeuralNet(layers=[ ('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('conv4', layers.Conv2DLayer), ('pool4', layers.MaxPool2DLayer), ('hidden5', layers.DenseLayer), ('hidden6', layers.DenseLayer), ('hidden7', layers.DenseLayer), ('output', layers.DenseLayer)], input_shape=(None, 1, input_width, input_height), conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2), conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2), conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2), conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2), hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100, output_num_units=20, output_nonlinearity=None, update_learning_rate=LEARNING_RATE, update_rho=UPDATE_RHO, update=rmsprop, train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT), batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE), regression=True, max_epochs=NUM_OF_EPOCH, verbose=1, hiddenLayer_to_output=-2) # on_training_finished=last_hidden_layer, return net2 def last_hidden_layer(s, h): print s.output_last_hidden_layer_(X) # input_layer = s.get_all_layers()[0] # last_h_layer = s.get_all_layers()[-2] # f = theano.function(s.X_inputs, last_h_layer.get_output(last_h_layer),allow_input_downcast=True) # myFunc = theano.function( # inputs=s.input_X, # outputs=s.h_predict, # allow_input_downcast=True, # ) # print s.output_last_hidden_layer_(X,-2) def writeOutputFile(outputFile,train_history,layer_info): # save the network's parameters outputFile.write("Validation set Prediction rate is: "+str((1-train_history[-1]['valid_accuracy'])*100) + "%\n") outputFile.write("Run time[minutes] is: "+str(run_time) + "\n\n") outputFile.write("Learning rate: " + str(LEARNING_RATE) + "\n") outputFile.write("Momentum: " + str(UPDATE_MOMENTUM) + "\n") if (UPDATE_RHO == None) else outputFile.write("Decay Factor: " + str(UPDATE_RHO) + "\n") outputFile.write("Batch size: " + str(BATCH_SIZE) + "\n") outputFile.write("Num epochs: " + str(NUM_OF_EPOCH) + "\n") outputFile.write("Num units hidden layers: " + str(NUM_UNITS_HIDDEN_LAYER) + "\n") # outputFile.write("activation func: " + str(activation) + "\n") outputFile.write("Train/validation split: " + str(TRAIN_VALIDATION_SPLIT) + "\n") outputFile.write("toShuffleInput: " + str(toShuffleInput) + "\n") outputFile.write("withZeroMeaning: " + str(withZeroMeaning) + "\n\n") outputFile.write("history: " + str(train_history) + "\n\n") outputFile.write("layer_info:\n" + str(layer_info) + "\n") start_time = time.clock() net2 = createNNwithMomentom(input_height, input_width) if UPDATE_RHO == None else createNNwithDecay(input_height, input_width) X, y, test_x, test_y = load2d() # load 2-d data net2.fit(X, y) run_time = (time.clock() - start_time) / 60. print "outputing last hidden layer" train_last_hiddenLayer = net2.output_hiddenLayer(X) test_last_hiddenLayer = net2.output_hiddenLayer(test_x) # ohlFile = open(HIDDEN_LAYER_OUTPUT_FILE_NAME+".txt", "w") # for line in train_last_hiddenLayer: # ohlFile.write(str(line) + "\n") with open(HIDDEN_LAYER_OUTPUT_FILE_NAME,'wb') as f: ob = (train_last_hiddenLayer,y,test_last_hiddenLayer,test_y) pickle.dump(ob, f, -1) f.close() writeOutputFile(outputFile,net2.train_history_,PrintLayerInfo._get_layer_info_plain(net2)) errorRates = runSvm(HIDDEN_LAYER_OUTPUT_FILE_NAME) errorRate = np.average(errorRates) outputFile.write("\nSVM Total Prediction rate is: "+str(100-errorRate) + "\n\n") outputFile.write("SVM Error rate is:\n"+str(errorRates) + "\n") outputFile.close() # write svm data # writeDataToFile(HIDDEN_LAYER_OUTPUT_FILE_NAME,SVM_FILE_NAME) ############################################## train_loss = np.array([i["train_loss"] for i in net2.train_history_]) valid_loss = np.array([i["valid_loss"] for i in net2.train_history_]) pyplot.plot(train_loss, linewidth=3, label="train") pyplot.plot(valid_loss, linewidth=3, label="valid") pyplot.grid() pyplot.legend() pyplot.xlabel("epoch") pyplot.ylabel("loss") pyplot.ylim(1e-3, 1) pyplot.yscale("log") pyplot.savefig(FIG_FILE_NAME) ################################################# # def plot_sample(x, y, axis): # img = x.reshape(96, 96) # axis.imshow(img, cmap='gray') # axis.scatter(y[0::2] * 48 + 48, y[1::2] * 48 + 48, marker='x', s=10) # # X, _ = load(test=True) # y_pred = net1.predict(X) # # fig = pyplot.figure(figsize=(6, 6)) # fig.subplots_adjust( # left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05) # # for i in range(16): # ax = fig.add_subplot(4, 4, i + 1, xticks=[], yticks=[]) # plot_sample(X[i], y_pred[i], ax) # # pyplot.show() ########## pickle the network ########## print "pickling" with open(PICKLES_NET_FILE_NAME,'wb') as f: pickle.dump(net2, f, -1) f.close()
def run(loadedData=None,FOLDER_NAME="defualt",LEARNING_RATE=0.04, UPDATE_MOMENTUM=0.9, UPDATE_RHO=None, NUM_OF_EPOCH=15, input_width=300, input_height=140, dataset='withOutDataSet', TRAIN_VALIDATION_SPLIT=0.2, MULTI_POSITIVES=20, dropout_percent=0.1, USE_NUM_CAT=20,end_index=16351, #activation=lasagne.nonlinearities.tanh, #rectify NUM_UNITS_HIDDEN_LAYER=[5, 10, 20, 40], BATCH_SIZE=40, toShuffleInput = False , withZeroMeaning = False): global counter # FILE_PREFIX = os.path.split(dataset)[1][:-6] #os.path.split(__file__)[1][:-3] FOLDER_PREFIX = "results/"+FOLDER_NAME+"/run_"+str(counter)+"/" if not os.path.exists(FOLDER_PREFIX): os.makedirs(FOLDER_PREFIX) PARAMS_FILE_NAME = FOLDER_PREFIX + "parameters.txt" HIDDEN_LAYER_OUTPUT_FILE_NAME = FOLDER_PREFIX +"hiddenLayerOutput.pkl.gz" FIG_FILE_NAME = FOLDER_PREFIX + "fig" PICKLES_NET_FILE_NAME = FOLDER_PREFIX + "picklesNN.pkl.gz" SVM_FILE_NAME = FOLDER_PREFIX + "svmData.txt" # VALIDATION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_validation_"+str(counter)+".txt" # PREDICTION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_prediction.txt" counter +=1 outputFile = open(PARAMS_FILE_NAME, "w") def createNNwithMomentom(input_height, input_width): if USE_NUM_CAT==20: outputLayerSize=20 else: outputLayerSize=15 net2 = NeuralNet(layers=[ ('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('conv4', layers.Conv2DLayer), ('pool4', layers.MaxPool2DLayer), ('hidden5', layers.DenseLayer), ('hidden6', layers.DenseLayer), ('hidden7', layers.DenseLayer), ('output', layers.DenseLayer)], input_shape=(None, 1, input_width, input_height), conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2), conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2), conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2), conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2), hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100, output_num_units=outputLayerSize, output_nonlinearity=None, update_learning_rate=LEARNING_RATE, update_momentum=UPDATE_MOMENTUM, update=nesterov_momentum, train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT), batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE), regression=True, max_epochs=NUM_OF_EPOCH, verbose=1, hiddenLayer_to_output=-2) # on_training_finished=last_hidden_layer, return net2 def createNNwithDecay(input_height, input_width): if USE_NUM_CAT==20: outputLayerSize=20 else: outputLayerSize=15 net2 = NeuralNet(layers=[ ('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('conv4', layers.Conv2DLayer), ('pool4', layers.MaxPool2DLayer), ('hidden5', layers.DenseLayer), ('hidden6', layers.DenseLayer), ('hidden7', layers.DenseLayer), ('output', layers.DenseLayer)], input_shape=(None, 1, input_width, input_height), conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2), conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2), conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2), conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2), hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100, output_num_units=outputLayerSize, output_nonlinearity=None, update_learning_rate=LEARNING_RATE, update_rho=UPDATE_RHO, update=rmsprop, train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT), batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE), regression=True, max_epochs=NUM_OF_EPOCH, verbose=1, hiddenLayer_to_output=-2) # on_training_finished=last_hidden_layer, return net2 def last_hidden_layer(s, h): print s.output_last_hidden_layer_(X) # input_layer = s.get_all_layers()[0] # last_h_layer = s.get_all_layers()[-2] # f = theano.function(s.X_inputs, last_h_layer.get_output(last_h_layer),allow_input_downcast=True) # myFunc = theano.function( # inputs=s.input_X, # outputs=s.h_predict, # allow_input_downcast=True, # ) # print s.output_last_hidden_layer_(X,-2) def outputLastLayer_CNN(net2, X, y, test_x, test_y): print "outputing last hidden layer" # train_last_hiddenLayer = net2.output_hiddenLayer(X) quarter_x = np.floor(X.shape[0] / 4) train_last_hiddenLayer1 = net2.output_hiddenLayer(X[:quarter_x]) print "after first quarter train output" train_last_hiddenLayer2 = net2.output_hiddenLayer(X[quarter_x:2 * quarter_x]) print "after seconed quarter train output" train_last_hiddenLayer3 = net2.output_hiddenLayer(X[2 * quarter_x:3 * quarter_x]) print "after third quarter train output" train_last_hiddenLayer4 = net2.output_hiddenLayer(X[3 * quarter_x:]) print "after all train output" test_last_hiddenLayer = net2.output_hiddenLayer(test_x) print "after test output" # lastLayerOutputs = (train_last_hiddenLayer,y,test_last_hiddenLayer,test_y) lastLayerOutputs = np.concatenate((train_last_hiddenLayer1, train_last_hiddenLayer2, train_last_hiddenLayer3, train_last_hiddenLayer4), axis=0), y, test_last_hiddenLayer, test_y return lastLayerOutputs def writeOutputFile(outputFile,train_history,layer_info): # save the network's parameters outputFile.write("Validation set Prediction rate is: "+str((1-train_history[-1]['valid_accuracy'])*100) + "%\n") outputFile.write("Run time[minutes] is: "+str(run_time) + "\n\n") outputFile.write("Training NN on: " + ("20 Top Categorys\n" if USE_NUM_CAT==20 else "Article Categorys\n")) outputFile.write("Learning rate: " + str(LEARNING_RATE) + "\n") outputFile.write(("Momentum: " + str(UPDATE_MOMENTUM)+ "\n") if (UPDATE_RHO == None) else ("Decay Factor: " + str(UPDATE_RHO)+ "\n") ) outputFile.write("Batch size: " + str(BATCH_SIZE) + "\n") outputFile.write("Num epochs: " + str(NUM_OF_EPOCH) + "\n") outputFile.write("Num units hidden layers: " + str(NUM_UNITS_HIDDEN_LAYER) + "\n\n") # outputFile.write("activation func: " + str(activation) + "\n") outputFile.write("Multipuly Positives by: " + str(MULTI_POSITIVES) + "\n") outputFile.write("New Positives Dropout rate: " + str(dropout_percent) + "\n") outputFile.write("Train/validation split: " + str(TRAIN_VALIDATION_SPLIT) + "\n") outputFile.write("toShuffleInput: " + str(toShuffleInput) + "\n") outputFile.write("withZeroMeaning: " + str(withZeroMeaning) + "\n\n") outputFile.write("history: " + str(train_history) + "\n\n") outputFile.write("layer_info:\n" + str(layer_info) + "\n") outputFile.flush() start_time = time.clock() print "Start time: " , time.ctime() net2 = createNNwithMomentom(input_height, input_width) if UPDATE_RHO == None else createNNwithDecay(input_height, input_width) if loadedData is None: X, y, test_x, test_y = load2d(USE_NUM_CAT,outputFile,input_width,input_height,end_index,MULTI_POSITIVES,dropout_percent) # load 2-d data else: X, y, test_x, test_y = loadedData net2.fit(X, y) run_time = (time.clock() - start_time) / 60. writeOutputFile(outputFile,net2.train_history_,PrintLayerInfo._get_layer_info_plain(net2)) lastLayerOutputs = outputLastLayer_CNN(net2, X, y, test_x, test_y) print "running Category Classifier" errorRates, aucScores = runSvm(lastLayerOutputs,15) #HIDDEN_LAYER_OUTPUT_FILE_NAME,15) # errorRates, aucScores = runCrossSvm(lastLayerOutputs,15) # errorRates, aucScores = runNNclassifier(lastLayerOutputs,15) errorRate = np.average(errorRates) aucScore = np.average(aucScores) outputFile.write("\nClassifiers Total Prediction rate is: "+str(100-errorRate) + "\n\n") outputFile.write("Classifiers Error rates are:\n"+str(errorRates) + "\n") outputFile.write("\nClassifiers Total AUC Score is: "+str(aucScore) + "\n\n") outputFile.write("Classifiers AUC Scores are:\n"+str(aucScores) + "\n") outputFile.close() print "saving last layer outputs" # with open(HIDDEN_LAYER_OUTPUT_FILE_NAME,'wb') as f: # pickle.dump(lastLayerOutputs, f, -1) # f.close() f = gzip.open(HIDDEN_LAYER_OUTPUT_FILE_NAME,'wb') cPickle.dump(lastLayerOutputs, f, protocol=2) f.close() # write svm data # writeDataToFile(HIDDEN_LAYER_OUTPUT_FILE_NAME,SVM_FILE_NAME) ############################################## train_loss = np.array([i["train_loss"] for i in net2.train_history_]) valid_loss = np.array([i["valid_loss"] for i in net2.train_history_]) pyplot.plot(train_loss, linewidth=3, label="train") pyplot.plot(valid_loss, linewidth=3, label="valid") pyplot.grid() pyplot.legend() pyplot.xlabel("epoch") pyplot.ylabel("loss") pyplot.ylim(1e-3, 1) pyplot.yscale("log") pyplot.savefig(FIG_FILE_NAME) ################################################# # def plot_sample(x, y, axis): # img = x.reshape(96, 96) # axis.imshow(img, cmap='gray') # axis.scatter(y[0::2] * 48 + 48, y[1::2] * 48 + 48, marker='x', s=10) # # X, _ = load(test=True) # y_pred = net1.predict(X) # # fig = pyplot.figure(figsize=(6, 6)) # fig.subplots_adjust( # left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05) # # for i in range(16): # ax = fig.add_subplot(4, 4, i + 1, xticks=[], yticks=[]) # plot_sample(X[i], y_pred[i], ax) # # pyplot.show() ########## pickle the network ########## print "pickling" # with open(PICKLES_NET_FILE_NAME,'wb') as f: # pickle.dump(net2, f, -1) # f.close() f = gzip.open(PICKLES_NET_FILE_NAME,'wb') cPickle.dump(net2, f, protocol=2) f.close()