def run(LEARNING_RATE=0.04, UPDATE_MOMENTUM=0.9,UPDATE_RHO=None, NUM_OF_EPOCH=50, OUTPUT_SIZE = 20 , input_width=300, input_height=140,
dataset='ISH.pkl.gz', TRAIN_VALIDATION_SPLIT=0.2, #activation=lasagne.nonlinearities.tanh, #rectify
NUM_UNITS_HIDDEN_LAYER=[5, 10, 20, 40], BATCH_SIZE=40, toShuffleInput = False , withZeroMeaning = False):
global counter
FILE_PREFIX = os.path.split(dataset)[1][:-6] #os.path.split(__file__)[1][:-3]
FOLDER_PREFIX = "results/"+FILE_PREFIX+"/run_"+str(counter)+"/"
if not os.path.exists(FOLDER_PREFIX):
os.makedirs(FOLDER_PREFIX)
PARAMS_FILE_NAME = FOLDER_PREFIX + "parameters.txt"
HIDDEN_LAYER_OUTPUT_FILE_NAME = FOLDER_PREFIX +"hiddenLayerOutput.pickle"
FIG_FILE_NAME = FOLDER_PREFIX + "fig"
PICKLES_NET_FILE_NAME = FOLDER_PREFIX + "picklesNN.pickle"
SVM_FILE_NAME = FOLDER_PREFIX + "svmData.txt"
# VALIDATION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_validation_"+str(counter)+".txt"
# PREDICTION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_prediction.txt"
counter +=1
outputFile = open(PARAMS_FILE_NAME, "w")
def load2d(dataset='ISH.pkl.gz', toShuffleInput = False , withZeroMeaning = False):
print 'loading data...'
datasets = load_data(dataset, toShuffleInput, withZeroMeaning)
train_set_x, train_set_y = datasets[0]
# valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
train_set_x = train_set_x.reshape(-1, 1, input_width, input_height)
# valid_set_x = valid_set_x.reshape(-1, 1, input_width, input_height)
test_set_x = test_set_x.reshape(-1, 1, input_width, input_height)
print(train_set_x.shape[0], 'train samples')
return train_set_x, train_set_y, test_set_x, test_set_y
def createNNwithMomentom(input_height, input_width):
net2 = NeuralNet(layers=[
('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('conv2', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('conv3', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('conv4', layers.Conv2DLayer),
('pool4', layers.MaxPool2DLayer),
('hidden5', layers.DenseLayer),
('hidden6', layers.DenseLayer),
('hidden7', layers.DenseLayer),
('output', layers.DenseLayer)],
input_shape=(None, 1, input_width, input_height),
conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2),
conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2),
conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2),
conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2),
hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100,
output_num_units=20, output_nonlinearity=None,
update_learning_rate=LEARNING_RATE,
update_momentum=UPDATE_MOMENTUM,
update=nesterov_momentum,
train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT),
batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE),
regression=True,
max_epochs=NUM_OF_EPOCH,
verbose=1,
hiddenLayer_to_output=-2)
# on_training_finished=last_hidden_layer,
return net2
def createNNwithDecay(input_height, input_width):
net2 = NeuralNet(layers=[
('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('conv2', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('conv3', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('conv4', layers.Conv2DLayer),
('pool4', layers.MaxPool2DLayer),
('hidden5', layers.DenseLayer),
('hidden6', layers.DenseLayer),
('hidden7', layers.DenseLayer),
('output', layers.DenseLayer)],
input_shape=(None, 1, input_width, input_height),
conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2),
conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2),
conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2),
conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2),
hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100,
output_num_units=20, output_nonlinearity=None,
update_learning_rate=LEARNING_RATE,
update_rho=UPDATE_RHO,
update=rmsprop,
train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT),
batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE),
regression=True,
max_epochs=NUM_OF_EPOCH,
verbose=1,
hiddenLayer_to_output=-2)
# on_training_finished=last_hidden_layer,
return net2
def last_hidden_layer(s, h):
print s.output_last_hidden_layer_(X)
# input_layer = s.get_all_layers()[0]
# last_h_layer = s.get_all_layers()[-2]
# f = theano.function(s.X_inputs, last_h_layer.get_output(last_h_layer),allow_input_downcast=True)
# myFunc = theano.function(
# inputs=s.input_X,
# outputs=s.h_predict,
# allow_input_downcast=True,
# )
# print s.output_last_hidden_layer_(X,-2)
def writeOutputFile(outputFile,train_history,layer_info):
# save the network's parameters
outputFile.write("Validation set Prediction rate is: "+str((1-train_history[-1]['valid_accuracy'])*100) + "%\n")
outputFile.write("Run time[minutes] is: "+str(run_time) + "\n\n")
outputFile.write("Learning rate: " + str(LEARNING_RATE) + "\n")
outputFile.write("Momentum: " + str(UPDATE_MOMENTUM) + "\n") if (UPDATE_RHO == None) else outputFile.write("Decay Factor: " + str(UPDATE_RHO) + "\n")
outputFile.write("Batch size: " + str(BATCH_SIZE) + "\n")
outputFile.write("Num epochs: " + str(NUM_OF_EPOCH) + "\n")
outputFile.write("Num units hidden layers: " + str(NUM_UNITS_HIDDEN_LAYER) + "\n")
# outputFile.write("activation func: " + str(activation) + "\n")
outputFile.write("Train/validation split: " + str(TRAIN_VALIDATION_SPLIT) + "\n")
outputFile.write("toShuffleInput: " + str(toShuffleInput) + "\n")
outputFile.write("withZeroMeaning: " + str(withZeroMeaning) + "\n\n")
outputFile.write("history: " + str(train_history) + "\n\n")
outputFile.write("layer_info:\n" + str(layer_info) + "\n")
start_time = time.clock()
net2 = createNNwithMomentom(input_height, input_width) if UPDATE_RHO == None else createNNwithDecay(input_height, input_width)
X, y, test_x, test_y = load2d() # load 2-d data
net2.fit(X, y)
run_time = (time.clock() - start_time) / 60.
print "outputing last hidden layer"
train_last_hiddenLayer = net2.output_hiddenLayer(X)
test_last_hiddenLayer = net2.output_hiddenLayer(test_x)
# ohlFile = open(HIDDEN_LAYER_OUTPUT_FILE_NAME+".txt", "w")
# for line in train_last_hiddenLayer:
# ohlFile.write(str(line) + "\n")
with open(HIDDEN_LAYER_OUTPUT_FILE_NAME,'wb') as f:
ob = (train_last_hiddenLayer,y,test_last_hiddenLayer,test_y)
pickle.dump(ob, f, -1)
f.close()
writeOutputFile(outputFile,net2.train_history_,PrintLayerInfo._get_layer_info_plain(net2))
errorRates = runSvm(HIDDEN_LAYER_OUTPUT_FILE_NAME)
errorRate = np.average(errorRates)
outputFile.write("\nSVM Total Prediction rate is: "+str(100-errorRate) + "\n\n")
outputFile.write("SVM Error rate is:\n"+str(errorRates) + "\n")
outputFile.close()
# write svm data
# writeDataToFile(HIDDEN_LAYER_OUTPUT_FILE_NAME,SVM_FILE_NAME)
##############################################
train_loss = np.array([i["train_loss"] for i in net2.train_history_])
valid_loss = np.array([i["valid_loss"] for i in net2.train_history_])
pyplot.plot(train_loss, linewidth=3, label="train")
pyplot.plot(valid_loss, linewidth=3, label="valid")
pyplot.grid()
pyplot.legend()
pyplot.xlabel("epoch")
pyplot.ylabel("loss")
pyplot.ylim(1e-3, 1)
pyplot.yscale("log")
pyplot.savefig(FIG_FILE_NAME)
#################################################
# def plot_sample(x, y, axis):
# img = x.reshape(96, 96)
# axis.imshow(img, cmap='gray')
# axis.scatter(y[0::2] * 48 + 48, y[1::2] * 48 + 48, marker='x', s=10)
#
# X, _ = load(test=True)
# y_pred = net1.predict(X)
#
# fig = pyplot.figure(figsize=(6, 6))
# fig.subplots_adjust(
# left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05)
#
# for i in range(16):
# ax = fig.add_subplot(4, 4, i + 1, xticks=[], yticks=[])
# plot_sample(X[i], y_pred[i], ax)
#
# pyplot.show()
########## pickle the network ##########
print "pickling"
with open(PICKLES_NET_FILE_NAME,'wb') as f:
pickle.dump(net2, f, -1)
f.close()
def run(loadedData=None,FOLDER_NAME="defualt",LEARNING_RATE=0.04, UPDATE_MOMENTUM=0.9, UPDATE_RHO=None, NUM_OF_EPOCH=15, input_width=300, input_height=140,
dataset='withOutDataSet', TRAIN_VALIDATION_SPLIT=0.2, MULTI_POSITIVES=20, dropout_percent=0.1, USE_NUM_CAT=20,end_index=16351, #activation=lasagne.nonlinearities.tanh, #rectify
NUM_UNITS_HIDDEN_LAYER=[5, 10, 20, 40], BATCH_SIZE=40, toShuffleInput = False , withZeroMeaning = False):
global counter
# FILE_PREFIX = os.path.split(dataset)[1][:-6] #os.path.split(__file__)[1][:-3]
FOLDER_PREFIX = "results/"+FOLDER_NAME+"/run_"+str(counter)+"/"
if not os.path.exists(FOLDER_PREFIX):
os.makedirs(FOLDER_PREFIX)
PARAMS_FILE_NAME = FOLDER_PREFIX + "parameters.txt"
HIDDEN_LAYER_OUTPUT_FILE_NAME = FOLDER_PREFIX +"hiddenLayerOutput.pkl.gz"
FIG_FILE_NAME = FOLDER_PREFIX + "fig"
PICKLES_NET_FILE_NAME = FOLDER_PREFIX + "picklesNN.pkl.gz"
SVM_FILE_NAME = FOLDER_PREFIX + "svmData.txt"
# VALIDATION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_validation_"+str(counter)+".txt"
# PREDICTION_FILE_NAME = "results/"+os.path.split(__file__)[1][:-3]+"_prediction.txt"
counter +=1
outputFile = open(PARAMS_FILE_NAME, "w")
def createNNwithMomentom(input_height, input_width):
if USE_NUM_CAT==20:
outputLayerSize=20
else:
outputLayerSize=15
net2 = NeuralNet(layers=[
('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('conv2', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('conv3', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('conv4', layers.Conv2DLayer),
('pool4', layers.MaxPool2DLayer),
('hidden5', layers.DenseLayer),
('hidden6', layers.DenseLayer),
('hidden7', layers.DenseLayer),
('output', layers.DenseLayer)],
input_shape=(None, 1, input_width, input_height),
conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2),
conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2),
conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2),
conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2),
hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100,
output_num_units=outputLayerSize, output_nonlinearity=None,
update_learning_rate=LEARNING_RATE,
update_momentum=UPDATE_MOMENTUM,
update=nesterov_momentum,
train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT),
batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE),
regression=True,
max_epochs=NUM_OF_EPOCH,
verbose=1,
hiddenLayer_to_output=-2)
# on_training_finished=last_hidden_layer,
return net2
def createNNwithDecay(input_height, input_width):
if USE_NUM_CAT==20:
outputLayerSize=20
else:
outputLayerSize=15
net2 = NeuralNet(layers=[
('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('conv2', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('conv3', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('conv4', layers.Conv2DLayer),
('pool4', layers.MaxPool2DLayer),
('hidden5', layers.DenseLayer),
('hidden6', layers.DenseLayer),
('hidden7', layers.DenseLayer),
('output', layers.DenseLayer)],
input_shape=(None, 1, input_width, input_height),
conv1_num_filters=NUM_UNITS_HIDDEN_LAYER[0], conv1_filter_size=(5, 5), pool1_pool_size=(2, 2),
conv2_num_filters=NUM_UNITS_HIDDEN_LAYER[1], conv2_filter_size=(9, 9), pool2_pool_size=(2, 2),
conv3_num_filters=NUM_UNITS_HIDDEN_LAYER[2], conv3_filter_size=(11, 11), pool3_pool_size=(4, 2),
conv4_num_filters=NUM_UNITS_HIDDEN_LAYER[3], conv4_filter_size=(8, 5), pool4_pool_size=(2, 2),
hidden5_num_units=500, hidden6_num_units=200, hidden7_num_units=100,
output_num_units=outputLayerSize, output_nonlinearity=None,
update_learning_rate=LEARNING_RATE,
update_rho=UPDATE_RHO,
update=rmsprop,
train_split=TrainSplit(eval_size=TRAIN_VALIDATION_SPLIT),
batch_iterator_train=BatchIterator(batch_size=BATCH_SIZE),
regression=True,
max_epochs=NUM_OF_EPOCH,
verbose=1,
hiddenLayer_to_output=-2)
# on_training_finished=last_hidden_layer,
return net2
def last_hidden_layer(s, h):
print s.output_last_hidden_layer_(X)
# input_layer = s.get_all_layers()[0]
# last_h_layer = s.get_all_layers()[-2]
# f = theano.function(s.X_inputs, last_h_layer.get_output(last_h_layer),allow_input_downcast=True)
# myFunc = theano.function(
# inputs=s.input_X,
# outputs=s.h_predict,
# allow_input_downcast=True,
# )
# print s.output_last_hidden_layer_(X,-2)
def outputLastLayer_CNN(net2, X, y, test_x, test_y):
print "outputing last hidden layer" # train_last_hiddenLayer = net2.output_hiddenLayer(X)
quarter_x = np.floor(X.shape[0] / 4)
train_last_hiddenLayer1 = net2.output_hiddenLayer(X[:quarter_x])
print "after first quarter train output"
train_last_hiddenLayer2 = net2.output_hiddenLayer(X[quarter_x:2 * quarter_x])
print "after seconed quarter train output"
train_last_hiddenLayer3 = net2.output_hiddenLayer(X[2 * quarter_x:3 * quarter_x])
print "after third quarter train output"
train_last_hiddenLayer4 = net2.output_hiddenLayer(X[3 * quarter_x:])
print "after all train output"
test_last_hiddenLayer = net2.output_hiddenLayer(test_x)
print "after test output" # lastLayerOutputs = (train_last_hiddenLayer,y,test_last_hiddenLayer,test_y)
lastLayerOutputs = np.concatenate((train_last_hiddenLayer1, train_last_hiddenLayer2, train_last_hiddenLayer3, train_last_hiddenLayer4), axis=0), y, test_last_hiddenLayer, test_y
return lastLayerOutputs
def writeOutputFile(outputFile,train_history,layer_info):
# save the network's parameters
outputFile.write("Validation set Prediction rate is: "+str((1-train_history[-1]['valid_accuracy'])*100) + "%\n")
outputFile.write("Run time[minutes] is: "+str(run_time) + "\n\n")
outputFile.write("Training NN on: " + ("20 Top Categorys\n" if USE_NUM_CAT==20 else "Article Categorys\n"))
outputFile.write("Learning rate: " + str(LEARNING_RATE) + "\n")
outputFile.write(("Momentum: " + str(UPDATE_MOMENTUM)+ "\n") if (UPDATE_RHO == None) else ("Decay Factor: " + str(UPDATE_RHO)+ "\n") )
outputFile.write("Batch size: " + str(BATCH_SIZE) + "\n")
outputFile.write("Num epochs: " + str(NUM_OF_EPOCH) + "\n")
outputFile.write("Num units hidden layers: " + str(NUM_UNITS_HIDDEN_LAYER) + "\n\n")
# outputFile.write("activation func: " + str(activation) + "\n")
outputFile.write("Multipuly Positives by: " + str(MULTI_POSITIVES) + "\n")
outputFile.write("New Positives Dropout rate: " + str(dropout_percent) + "\n")
outputFile.write("Train/validation split: " + str(TRAIN_VALIDATION_SPLIT) + "\n")
outputFile.write("toShuffleInput: " + str(toShuffleInput) + "\n")
outputFile.write("withZeroMeaning: " + str(withZeroMeaning) + "\n\n")
outputFile.write("history: " + str(train_history) + "\n\n")
outputFile.write("layer_info:\n" + str(layer_info) + "\n")
outputFile.flush()
start_time = time.clock()
print "Start time: " , time.ctime()
net2 = createNNwithMomentom(input_height, input_width) if UPDATE_RHO == None else createNNwithDecay(input_height, input_width)
if loadedData is None:
X, y, test_x, test_y = load2d(USE_NUM_CAT,outputFile,input_width,input_height,end_index,MULTI_POSITIVES,dropout_percent) # load 2-d data
else:
X, y, test_x, test_y = loadedData
net2.fit(X, y)
run_time = (time.clock() - start_time) / 60.
writeOutputFile(outputFile,net2.train_history_,PrintLayerInfo._get_layer_info_plain(net2))
lastLayerOutputs = outputLastLayer_CNN(net2, X, y, test_x, test_y)
print "running Category Classifier"
errorRates, aucScores = runSvm(lastLayerOutputs,15) #HIDDEN_LAYER_OUTPUT_FILE_NAME,15)
# errorRates, aucScores = runCrossSvm(lastLayerOutputs,15)
# errorRates, aucScores = runNNclassifier(lastLayerOutputs,15)
errorRate = np.average(errorRates)
aucScore = np.average(aucScores)
outputFile.write("\nClassifiers Total Prediction rate is: "+str(100-errorRate) + "\n\n")
outputFile.write("Classifiers Error rates are:\n"+str(errorRates) + "\n")
outputFile.write("\nClassifiers Total AUC Score is: "+str(aucScore) + "\n\n")
outputFile.write("Classifiers AUC Scores are:\n"+str(aucScores) + "\n")
outputFile.close()
print "saving last layer outputs"
# with open(HIDDEN_LAYER_OUTPUT_FILE_NAME,'wb') as f:
# pickle.dump(lastLayerOutputs, f, -1)
# f.close()
f = gzip.open(HIDDEN_LAYER_OUTPUT_FILE_NAME,'wb')
cPickle.dump(lastLayerOutputs, f, protocol=2)
f.close()
# write svm data
# writeDataToFile(HIDDEN_LAYER_OUTPUT_FILE_NAME,SVM_FILE_NAME)
##############################################
train_loss = np.array([i["train_loss"] for i in net2.train_history_])
valid_loss = np.array([i["valid_loss"] for i in net2.train_history_])
pyplot.plot(train_loss, linewidth=3, label="train")
pyplot.plot(valid_loss, linewidth=3, label="valid")
pyplot.grid()
pyplot.legend()
pyplot.xlabel("epoch")
pyplot.ylabel("loss")
pyplot.ylim(1e-3, 1)
pyplot.yscale("log")
pyplot.savefig(FIG_FILE_NAME)
#################################################
# def plot_sample(x, y, axis):
# img = x.reshape(96, 96)
# axis.imshow(img, cmap='gray')
# axis.scatter(y[0::2] * 48 + 48, y[1::2] * 48 + 48, marker='x', s=10)
#
# X, _ = load(test=True)
# y_pred = net1.predict(X)
#
# fig = pyplot.figure(figsize=(6, 6))
# fig.subplots_adjust(
# left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05)
#
# for i in range(16):
# ax = fig.add_subplot(4, 4, i + 1, xticks=[], yticks=[])
# plot_sample(X[i], y_pred[i], ax)
#
# pyplot.show()
########## pickle the network ##########
print "pickling"
# with open(PICKLES_NET_FILE_NAME,'wb') as f:
# pickle.dump(net2, f, -1)
# f.close()
f = gzip.open(PICKLES_NET_FILE_NAME,'wb')
cPickle.dump(net2, f, protocol=2)
f.close()
