def main(): ################ # LOAD DATASET # ################ dataset = './data/ubiquitous_aug.hkl' kfd = './data/ubiquitous_kfold.hkl' print('Loading dataset {}...'.format(dataset)) X, y = hkl.load(open(dataset, 'r')) X = X.reshape(-1, 4, 1, 400).astype(floatX) y = y.astype('int32') print('X shape: {}, y shape: {}'.format(X.shape, y.shape)) kf = hkl.load(open(kfd, 'r')) kfold = [(train, test) for train, test in kf] (train, test) = kfold[0] print('train_set size: {}, test_set size: {}'.format(len(train), len(test))) # shuffle +/- labels in minibatch print('shuffling train_set and test_set') shuffle(train) shuffle(test) X_train = X[train] X_test = X[test] y_train = y[train] y_test = y[test] print('data prepared!') layers = [ (InputLayer, {'shape': (None, 4, 1, 400)}), (Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 4)}), (Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 3)}), (Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 3)}), (MaxPool2DLayer, {'pool_size': (1, 2)}), (Conv2DLayer, {'num_filters': 32, 'filter_size': (1, 2)}), (Conv2DLayer, {'num_filters': 32, 'filter_size': (1, 2)}), (Conv2DLayer, {'num_filters': 32, 'filter_size': (1, 2)}), (MaxPool2DLayer, {'pool_size': (1, 2)}), (DenseLayer, {'num_units': 64}), (DropoutLayer, {}), (DenseLayer, {'num_units': 64}), (DenseLayer, {'num_units': 2, 'nonlinearity': softmax})] net = NeuralNet( layers=layers, max_epochs=100, update=adam, update_learning_rate=1e-4, train_split=TrainSplit(eval_size=0.1), on_epoch_finished=[ AdjustVariable(1e-4, target=0, half_life=20)], verbose=2) net.fit(X_train, y_train) plot_loss(net)
def test(): l = 300 dataset = './data/ubiquitous_test.hkl' print 'Loading dataset {}'.format(dataset) X_test, y_test = hkl.load(dataset) X_test = X_test.reshape(-1, 4, 1, l).astype(floatX) y_test = np.array(y_test, dtype='int32') print 'X_test shape: {}, y_test shape: {}'.format(X_test.shape, y_test.shape) test_pos_ids = hkl.load('./data/ubiquitous_test_pos_ids.hkl') net = pkl.load(open('./models/net.pkl', 'r')) y_prob = net.predict_proba(X_test) y_prob = y_prob[:, 1] f = open('./results/y_score.txt', 'w') f.write('id:\t predicted probability(ies)') y_prob_new = [] temp = [] for i, id, lastid in zip( range(len(test_pos_ids)), test_pos_ids, [None] + test_pos_ids[:-1]): if id != lastid: y_prob_new.append(temp) temp = [] f.write('\n%s:\t' % id) temp.append(y_prob[i]) f.write('%.4f\t' % y_prob[i]) y_prob_new.append(temp) y_prob_new = [max(item) for item in y_prob_new[1:]] n_pos = len(y_prob_new) n_neg = y_test.shape[0] - len(test_pos_ids) y_prob_new.extend(list(y_prob[-n_neg:])) y_test = [1] * n_pos + [0] * n_neg y_pred = np.array(np.array(y_prob_new) > 0.5, dtype='int32') print 'ROC AUC score is {}'.format(metrics.roc_auc_score(y_test, y_prob_new)) print 'Precision score is {}'.format(metrics.precision_score(y_test, y_pred)) print 'Accuracy score is {}'.format(metrics.accuracy_score(y_test, y_pred)) plot_loss(net)
answ_wl=ANSW_SENT_L, rnn_size=rnn_size, rnn_type='LSTM', pool_size=4, answ_n=4, dence_l=dence, dropout=0.5, batch_size=BATCH_SIZE, emb_size=EMB_SIZE, grad_clip=40, init_std=0.1, num_hops=3, rnn_style=False, nonlin=LN.softmax, init_W=None, rng=None, art_pool=4, lr=0.01, mom=0, updates=LU.adagrad, valid_indices=0.2, permute_answ=True, permute_cont=True) #----------------------------------------------------------------------train NN _helpScripts.print_msg('train NN') net.fit({'l_in_q': train_quest[:quest_n_train], 'l_in_a': train_answ[:quest_n_train], 'l_in_q_pe': float32(q_pe_train), 'l_in_a_pe': float32(a_pe_train), 'l_in_cont': int32(context_train), 'l_in_cont_pe': float32(cont_pe_train)}, train_targets[:quest_n_train], epochs=N_EPOCHS) plot_loss(net) #-----------------------------------------------------------------------save NN net.save_params_to(saveDir + 'params.picle') # net.save_weights_to(saveDir + 'weights.picle') print strftime("%Y-%m-%d %H:%M:%S") print "completed in ", datetime.datetime.now() - startTime #--------------------------------------------------------------predict for test q_pe_test = positional_encoding(QUEST_SENT_L, 1, EMB_SIZE, quest_n_test) a_pe_test = positional_encoding(ANSW_SENT_L, 4, EMB_SIZE, quest_n_test) cont_pe_test = positional_encoding(MAX_SENT_LENGTH, ARTICLES_QUEST*MAX_SENT_ART, EMB_SIZE, quest_n_test) pred_answ_test = net.predict({'l_in_q': test_quest[:quest_n_test], 'l_in_a': train_answ[:quest_n_test], 'l_in_q_pe': float32(q_pe_test), 'l_in_a_pe': float32(a_pe_test), 'l_in_cont': int32(context_test), 'l_in_cont_pe': float32(cont_pe_test)}) #------------------------------------------------------------------------submit
def main(model='MLP'): if (model=='MLP'): x_train,y_train = load_TrainData() ###########training process ##################### net = build_mlp() print "Start training:" net.fit(x_train,y_train) ################## calculate the precision################### print "======================== accuracy==========================" x_test = np.array(pd.read_csv('x_test.csv').iloc[:,1:]/255) #y_test = np.array(pd.read_csv('y_test.csv').iloc[:,1:]) y_label = np.array(pd.read_csv('y_testLabel.csv').iloc[:,1:]) #print x_test y_pred = net.predict(x_test) pd.DataFrame(y_pred).to_csv('y_pred_MLP.csv') y = [] for i in range(y_pred.shape[0]): y.append(y_pred[i].tolist().index(max(y_pred[i]))+1) pd.DataFrame(y).to_csv('y_predlabel_MLP.csv') print classification_report(y_label, y) ##############save the MLP weights to file########################## net.save_params_to('MLP_weights_file') elif (model=='CNN'): x_train,y_train = load_TrainData2D() #print x_train.shape print y_train ###########training process ##################### net = buildCNN() print "Start training:" net.fit(x_train,y_train) plot_loss(net) plot_conv_weights(net.layers_[1], figsize=(4, 4)) plt.show() ################## calculate the precision################### print "======================== accuracy==========================" x_test ,y_test = load_TestData2D(); y_label = np.array(pd.read_csv('y_testLabel.csv').iloc[:,1:]) #print x_test y_pred = net.predict(x_test) pd.DataFrame(y_pred).to_csv('y_pred_CNN.csv') y = [] for i in range(y_pred.shape[0]): y.append(y_pred[i].tolist().index(max(y_pred[i]))+1) pd.DataFrame(y).to_csv('y_predlabel_CNN.csv') print classification_report(y_label, y) #print classification_report(y_pred, y_label) ##############save the MLP weights to file########################## net.save_params_to('CNN_weights_file') else: print 'ERROR: Please select a model #MLP or #CNN !'
def test_plot_loss(self, net_fitted): from nolearn.lasagne.visualize import plot_loss plot_loss(net_fitted) plt.clf() plt.cla()
#----------------------------------------------------------------------train NN _helpScripts.print_msg('train NN') net.fit( { 'l_in_q': train_quest[:quest_n_train], 'l_in_a': train_answ[:quest_n_train], 'l_in_q_pe': float32(q_pe_train), 'l_in_a_pe': float32(a_pe_train), 'l_in_cont': int32(context_train), 'l_in_cont_pe': float32(cont_pe_train) }, train_targets[:quest_n_train], epochs=N_EPOCHS) plot_loss(net) #-----------------------------------------------------------------------save NN net.save_params_to(saveDir + 'params.picle') # net.save_weights_to(saveDir + 'weights.picle') print strftime("%Y-%m-%d %H:%M:%S") print "completed in ", datetime.datetime.now() - startTime #--------------------------------------------------------------predict for test q_pe_test = positional_encoding(QUEST_SENT_L, 1, EMB_SIZE, quest_n_test) a_pe_test = positional_encoding(ANSW_SENT_L, 4, EMB_SIZE, quest_n_test) cont_pe_test = positional_encoding(MAX_SENT_LENGTH, ARTICLES_QUEST * MAX_SENT_ART, EMB_SIZE, quest_n_test) pred_answ_test = net.predict({ 'l_in_q': test_quest[:quest_n_test], 'l_in_a': train_answ[:quest_n_test],
def train(self, X, y, param_file=None, out_param_file=None, filter_file=None, load_params=False, pretrain=False): if pretrain: params = createNoLearnParams(param_file) print "Parameters", params[1].shape conv0_W = np.concatenate((params[0], params[0]), axis=1) conv0_W = np.concatenate((conv0_W, params[0]), axis=1) conv0_W = conv0_W[:, :7, :, :] conv0_b = np.concatenate((params[1], params[1]), axis=0) conv0_b = np.concatenate((conv0_b, params[1]), axis=0) conv0_b = conv0_b[:96] conv1_W = np.concatenate((params[2], params[2]), axis=1) conv1_W = np.concatenate((conv1_W, params[2]), axis=1) conv1_W = conv1_W[:, :96, :, :] conv1_b = np.concatenate((params[3], params[3]), axis=0) conv1_b = np.concatenate((conv1_b, params[3]), axis=0) conv1_b = conv1_b[:256] conv2_W = np.concatenate((params[4], params[4]), axis=1) conv2_W = np.concatenate((conv2_W, params[4]), axis=1) conv2_W = conv2_W[:, :256, :, :] conv2_b = np.concatenate((params[5], params[5]), axis=0) conv2_b = np.concatenate((conv2_b, params[5]), axis=0) conv2_b = conv2_b[:512] conv3_W = np.concatenate((params[6], params[6]), axis=1) conv3_W = np.concatenate((conv3_W, params[6]), axis=1) conv3_W = conv3_W[:, :512, :, :] conv3_b = np.concatenate((params[7], params[7]), axis=0) conv3_b = np.concatenate((conv3_b, params[7]), axis=0) conv3_b = conv3_b[:512] conv4_W = np.concatenate((params[8], params[8]), axis=1) conv4_W = np.concatenate((conv4_W, params[8]), axis=1) conv4_W = conv4_W[:, :512, :, :] conv4_b = np.concatenate((params[9], params[9]), axis=0) conv4_b = np.concatenate((conv4_b, params[9]), axis=0) conv4_b = conv4_b[:512] dense0_W = np.concatenate((params[10], params[10]), axis=1) dense0_W = np.concatenate((dense0_W, params[10]), axis=1) dense0_W = dense0_W[:2560, :4096] dense0_b = np.concatenate((params[11], params[11]), axis=0) dense0_b = np.concatenate((dense0_b, params[11]), axis=0) dense0_b = dense0_b[:4096] dense1_W = np.concatenate((params[12], params[12]), axis=1) dense1_W = np.concatenate((dense1_W, params[12]), axis=1) dense1_W = dense1_W[:4096, :4096] dense1_b = np.concatenate((params[13], params[13]), axis=0) dense1_b = np.concatenate((dense1_b, params[13]), axis=0) dense1_b = dense1_b[:4096] #http://arxiv.org/pdf/1405.3531v4.pdf self.net = NeuralNet( layers=self.layers, conv0_W=np.array(conv0_W), conv0_b=np.array(conv0_b), conv1_W=np.array(conv1_W), conv1_b=np.array(conv1_b), conv2_W=np.array(conv2_W), conv2_b=np.array(conv2_b), conv3_W=np.array(conv3_W), conv3_b=np.array(conv3_b), conv4_W=np.array(conv4_W), conv4_b=np.array(conv4_b), dense0_W=np.array(dense0_W), dense0_b=np.array(dense0_b), dense1_W=np.array(dense1_W), dense1_b=np.array(dense1_b), update_learning_rate=0.015, update=L.updates.nesterov_momentum, update_momentum=0.9, #update=L.updates.sgd, regression=True, verbose=1, eval_size=0.15, objective_loss_function=L.objectives.binary_crossentropy, max_epochs=200) if load_params: print "Loading parameters from ", param_file self.net.load_params_from(param_file) print "TRAINING!" print "input shape: ", X.shape print "output shape: ", y.shape print "Example X", X[0] print "Example Y", y[0] #print self.net.get_params() self.net.fit(X, y) print(self.net.score(X, y)) print "Saving network parameters to ", out_param_file, "..." file = open(out_param_file, 'w+') file.close() self.net.save_weights_to(out_param_file) print "Saving filters to ", filter_file self.write_filters_to_file(filter_file) plt = visualize.plot_loss(self.net) plt.show() plt.savefig(DIR_PROJ + 'loss.png') plt.clf() plt.cla() print "Sample predictions" for i in range(10): pred = self.net.predict(np.array([X[i]])) print "---------------------------------" print i print "prediction", pred print y[i]
update = adam, # For 'adam', a small learning rate is best update_learning_rate = 0.0002, objective_l2 = 0.0025, # L2 regularization train_split = TrainSplit(eval_size = 0.25), verbose = 1 ) net0.fit(X_train, y_train) # visualization from nolearn.lasagne.visualize import draw_to_notebook, plot_loss from nolearn.lasagne.visualize import plot_conv_weights, plot_conv_activity from nolearn.lasagne.visualize import plot_occlusion, plot_saliency draw_to_notebook(net0) plot_loss(net0) #plot helps determine if we are overfitting: #If the train loss is much lower than the validation loss, #we should probably do something to regularize the net. # visualize layer weights plot_conv_weights(net0.layers_[1], figsize = (4,4)) #If the weights just look like noise, we might have to do something #(e.g. use more filters so that each can specialize better). # visualize layers' activities x = X_train[0:1] # an image in the bc01 format (so use X[0:1] instead of just X[0]). plot_conv_activity(net0.layers_[1], x) plot_occlusion(net0, X_train[:5], y_train[:5]) plot_saliency(net0, X_train[:5])
#optimization parameters: update=nesterov_momentum, update_learning_rate=0.05, update_momentum=0.9, regression=True, max_epochs=50, verbose=1, batch_iterator_train=BatchIterator(batch_size=25), on_epoch_finished=[ EarlyStopping(patience=20), ], train_split=TrainSplit(eval_size=0.5)) net2.fit(train_x, train_y) plot_loss(net2) plt.savefig("/home/sandeep/Desktop/Major/Results/plotloss.png") plot_conv_weights(net2.layers_[1], figsize=(4, 4)) plt.savefig("/home/sandeep/Desktop/Major/Results/convweights.png") #layer_info = PrintLayerInfo() #layer_info(net2) import cPickle as pickle with open('/home/sandeep/Desktop/Major/Results/net2.pickle', 'wb') as f: pickle.dump(net2, f, -1) y_pred2 = net2.predict(test_x) print "The accuracy of this network is: %0.2f" % (abs(y_pred2 - test_y)).mean() do_test("/home/sandeep/Desktop/Major/train",
def cae_eval(self): """Draw evaluation lines <TODO> """ from nolearn.lasagne.visualize import plot_loss plot_loss(self.cae)
def test_visualize_functions_with_cnn(mnist): # this test simply tests that no exception is raised when using # the plotting functions from nolearn.lasagne import NeuralNet from nolearn.lasagne.visualize import plot_conv_activity from nolearn.lasagne.visualize import plot_conv_weights from nolearn.lasagne.visualize import plot_loss from nolearn.lasagne.visualize import plot_occlusion X, y = mnist X_train, y_train = X[:100].reshape(-1, 1, 28, 28), y[:100] X_train = X_train.reshape(-1, 1, 28, 28) num_epochs = 3 nn = NeuralNet( layers=[ ('input', InputLayer), ('conv1', Conv2DLayer), ('conv2', Conv2DLayer), ('pool2', MaxPool2DLayer), ('conv3', Conv2DLayer), ('conv4', Conv2DLayer), ('pool4', MaxPool2DLayer), ('hidden1', DenseLayer), ('output', DenseLayer), ], input_shape=(None, 1, 28, 28), output_num_units=10, output_nonlinearity=softmax, more_params=dict( conv1_filter_size=(5, 5), conv1_num_filters=16, conv2_filter_size=(3, 3), conv2_num_filters=16, pool2_ds=(3, 3), conv3_filter_size=(3, 3), conv3_num_filters=16, conv4_filter_size=(3, 3), conv4_num_filters=16, pool4_ds=(2, 2), hidden1_num_units=16, ), update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=num_epochs, ) nn.fit(X_train, y_train) plot_loss(nn) plot_conv_weights(nn.layers_['conv1']) plot_conv_weights(nn.layers_['conv2'], figsize=(1, 2)) plot_conv_activity(nn.layers_['conv3'], X_train[:1]) plot_conv_activity(nn.layers_['conv4'], X_train[10:11], figsize=(3, 4)) plot_occlusion(nn, X_train[:1], y_train[:1]) plot_occlusion(nn, X_train[2:4], y_train[2:4], square_length=3, figsize=(5, 5)) # clear figures from memory plt.clf() plt.cla()
acc_net = accuracy_score(Y_test,preds_net) precision_net = precision_score(Y_test,preds_net) recall_net = recall_score(Y_test,preds_net) print("----------------------------------------------------------------") print("NET: Matthews Correlation Coefficient: \t" + str(mcc_net)) print("NET: Accuracy: \t\t\t\t" + str(acc_net)) print("NET: Precision: \t\t\t\t" + str(precision_net)) print("NET: Recall: \t\t\t\t" + str(recall_net)) netFile = logPath + "/net" net.save_params_to(netFile) plt = plot_loss(net) plt.savefig(logPath + "/valid_train_loss.png") numbers_net = [mcc_net,acc_net,precision_net,recall_net] numbers_file_net = open(logPath + "/net_numbers.csv",'wb') wr_net = csv.writer(numbers_file_net) wr_net.writerow(numbers_net) numbers_file_net.close() f = open(logPath+'/net_info.txt','w') f.write("Net layers:\n") f.write(str(net.layers)) f.write('\n') f.write("Seed: ") f.write(str(args.seed)) f.close() fold+=1
def load_cnn(path, cnn_type=None): # load cnn with open(path, 'rb') as f: cnn = pickle.load(f) # make sure we have the correct test batch iterator # cnn.batch_iterator_test = MyTestBatchIterator(cnn.batch_iterator_train.batch_size) if cnn_type == None: cnn_type = os.path.basename(os.path.dirname(path)) input_names = [] input_values = [] if cnn_type.startswith('RGBA'): # this is a rgba network if cnn_type.find('large') != -1: # large border X_test, y_test = mlp.Patch.load_rgba_test_only('cylinder1_rgba', border_prefix = 'larger_border') else: # small border X_test, y_test = mlp.Patch.load_rgba_test_only('cylinder1_rgba') test_inputs = X_test input_names.append('RGBA') elif cnn_type.startswith('RGB'): # this is a RGB net X_test, y_test = mlp.Patch.load_rgb_test_only('cylinder2_rgb') # test_inputs = X_test[:,:-1,:,:] test_inputs = X_test input_names.append('RGB') else: # load patches X_train, y_train, X_test, y_test = mlp.Patch.load('cylinder2') test_inputs = collections.OrderedDict() for l in cnn.layers: layer_name, layer_type = l if layer_type == lasagne.layers.input.InputLayer: input_name = layer_name.split('_')[0] if input_name == 'binary': input_name = 'merged_array' if input_name == 'border': input_name = 'border_overlap' if path.find('larger_border_overlap') != -1: input_name = 'larger_border_overlap' input_names.append(layer_name) input_values.append(input_name) test_inputs[layer_name] = X_test[input_name] print 'Using test set:', input_values # calc F1 test_prediction = cnn.predict(test_inputs) test_prediction_prob = cnn.predict_proba(test_inputs) print print 'Precision/Recall:' print classification_report(y_test, test_prediction) # calc test accuracy test_acc = cnn.score(test_inputs, y_test) acc_score = accuracy_score(y_test, test_prediction) print 'Test Accuracy:', test_acc print 'Accuracy Score:', acc_score # ROC/AUC fpr, tpr, _ = roc_curve(y_test, test_prediction_prob[:,1]) roc_auc = auc(fpr, tpr) # attach patch selection cnn.input_names = input_names cnn.input_values = input_values cnn.uuid = cnn_type#os.path.basename(os.path.dirname(path)) # plot loss output_folder = '/home/d/netstats/'+cnn.uuid+'/' if not os.path.exists(output_folder): os.makedirs(output_folder) font = {'family' : 'normal', # 'weight' : 'bold', 'size' : 26} plt.rc('font', **font) plt.figure(figsize=(22,22)) loss_plot = plot_loss(cnn) loss_plot.savefig(output_folder+'/loss.pdf') data = {} data['CNN'] = (fpr, tpr, roc_auc) mlp.Legacy.plot_roc(data, output_folder+'/cnn_roc.pdf') return cnn
max_epochs=20, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.975, regression=True, verbose=1 ) # ae.initialize() # PrintLayerInfo()(ae) ae.fit(X, X_out) from nolearn.lasagne.visualize import plot_loss plot_loss(ae) # ae.save_params_to('mnist/conv_ae.np') pickle.dump(ae, open('mnist/conv_ae.pkl','w')) # ae = pickle.load(open('mnist/conv_ae.pkl','r')) # ae.layers X_pred = ae.predict(X).reshape(-1, 28, 28) X_pred = np.rint(256. * X_pred).astype(int) X_pred = np.clip(X_pred, a_min = 0, a_max = 255) X_pred = X_pred.astype('uint8') print X_pred.shape , X.shape ### show random inputs / outputs side by side def get_picture_array(X, rescale=4):