verbose=1, ) ae.fit(X_train, X_out) print ### expect training / val error of about 0.087 with these parameters ### if your GPU not fast enough, reduce the number of filters in the conv/deconv step # <codecell> import pickle import sys sys.setrecursionlimit(10000) pickle.dump(ae, open('mnist/conv_ae.pkl', 'w')) #ae = pickle.load(open('mnist/conv_ae.pkl','r')) ae.save_weights_to('mnist/conv_ae.np') # <codecell> X_train_pred = ae.predict(X_train).reshape(-1, 28, 28) * sigma + mu X_pred = np.rint(X_train_pred).astype(int) X_pred = np.clip(X_pred, a_min=0, a_max=255) X_pred = X_pred.astype('uint8') print X_pred.shape, X.shape # <codecell> ### show random inputs / outputs side by side def get_picture_array(X, index):
dropout_p=0.5, output_num_units=num_classes, output_nonlinearity=lasagne.nonlinearities.softmax, output_W = GlorotUniform(gain = 1.0), # ----------------------- ConvNet Params ------------------------------------------- update = nesterov_momentum, update_learning_rate = learning_rate, update_momentum = momentum, max_epochs = num_epochs, verbose = 1, ) tic = time.time() for i in range(12): convNet.fit(dataset['X_train'], dataset['Y_train']) fl = './model1/saved_model_data' + str(i+1) + '.npz' convNet.save_weights_to(fl) print 'Model saved to file :- ', fl toc = time.time() fl = './model1/saved_model_data' + str(6) + '.npz' convNet.load_weights_from(fl) y_pred = convNet.predict(dataset['X_test']) print classification_report(Y_test, y_pred) print accuracy_score(Y_test, y_pred) print 'Time taken to train the data :- ', toc-tic, 'seconds'
output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, eval_size=0.2, verbose=1, max_epochs=200) print "fitting nn model.." net0.fit(X_train, y_train) print "predicting probabilities using nn model..." proba = net0.predict_proba(X_test) ll.append(calc_ll_from_proba(proba, y_test)) print metrics.confusion_matrix( y_test.astype(int), np.argmax(proba, axis=1).astype(int)) print "logloss: ", ll[ncv] print "saving nn model..." net0.save_weights_to('weights/nn_%d.pkl' % ncv) net0 = None ncv += 1 ll = np.array(ll) print "logloss: ", ll # make_submission(net0, X_test, ids, encoder)
# output output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=2, # optimization method params update=nesterov_momentum, update_learning_rate=0.007, update_momentum=0.9, max_epochs=16, verbose=1, ) """Loading data and training Lasagne network using nolearn""" trainImg = np.load('trainImg_stage1.npy') trainVal2 = np.load('trainVal_stage1.npy') trainImg2 = trainImg.astype(np.float32).swapaxes(1, 3) trainVal2 = trainVal2.astype(np.uint8) print "Training Classifier: 70/30 split" nn.fit(trainImg2, trainVal2) print "Saving Classifier" pickle.dump(nn, open("nn_stage1.pkl", "wb")) nn.save_weights_to('weights_stage1')
class graspNet(): def __init__(self, param_file=None): net_divider = 1.0 self.layers = [ (L.layers.InputLayer, { 'shape': (None, 7, X_H, X_W), 'name': 'input' }), (L.layers.Conv2DLayer, { 'num_filters': 96, 'stride': 1, 'pad': 3, 'filter_size': (7, 7), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv0' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.MaxPool2DLayer, { 'pool_size': 3, 'ignore_border': False }), (L.layers.Conv2DLayer, { 'num_filters': 256, 'stride': 1, 'filter_size': (5, 5), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv1' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.MaxPool2DLayer, { 'pool_size': 2, 'ignore_border': False }), (L.layers.Conv2DLayer, { 'num_filters': 512, 'stride': 1, 'pad': 1, 'filter_size': (3, 3), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv2' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.Conv2DLayer, { 'num_filters': 512, 'stride': 1, 'pad': 1, 'filter_size': (3, 3), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv3' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.Conv2DLayer, { 'num_filters': 512, 'stride': 1, 'pad': 1, 'filter_size': (3, 3), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv4' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.MaxPool2DLayer, { 'pool_size': 3, 'ignore_border': False }), (L.layers.DenseLayer, { 'num_units': 4096, 'nonlinearity': L.nonlinearities.sigmoid, 'name': 'dense0' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.DenseLayer, { 'num_units': 4096, 'nonlinearity': L.nonlinearities.sigmoid, 'name': 'dense1' }), (L.layers.DenseLayer, { 'num_units': 1, 'nonlinearity': L.nonlinearities.sigmoid }), ] self.net = NeuralNet( layers=self.layers, update_learning_rate=0.015, update=L.updates.nesterov_momentum, update_momentum=0.9, #update=L.updates.sgd, regression=True, verbose=1, eval_size=0.15, objective_loss_function=L.objectives.binary_crossentropy, max_epochs=200) if param_file != None: print "Loading parameters from ", param_file self.net.load_params_from(param_file) def write_filters_to_file(self, fname): params = self.net.get_all_params() layer_counter = 0 for p in params: print p layer_counter += 1 filter_counter = 0 weights = p.get_value() if len(weights.shape) > 2: for f in weights: kernel = np.asarray(f, dtype=np.float32) kernel = kernel * 255 + 128 viz = np.zeros(shape=kernel[0].shape) viz = cv2.resize(kernel[0], None, fx=20, fy=20, interpolation=cv2.INTER_CUBIC) cv2.normalize(viz, viz) viz = viz * 50 #print viz #cv2.imshow('ha', viz) #cv2.waitKey(-1) viz = viz / 50 viz = viz * 12000 viz = viz #print viz cv2.imwrite( fname + "_" + str(layer_counter) + "_" + str(filter_counter) + ".png", viz) filter_counter += 1 # evaluates the input array over the neural net def eval(self, x): y = np.zeros((x.shape[0], )) for i in range(x.shape[0]): pred = self.net.predict(np.array([x[i]])) y[i] = pred print i, pred return y # train the network in input (x,y) # param_file: input parameter file (perhaps from previous trainings) # out_param_file: output path to write resulting params to # filter_file: output path to write images of filters for visualization # load_params: boolean flag, when set: 1 loads input parameters from <param_file> # pretrain: boolean flag, when set: 1 loads parameters from pretrained network def train(self, X, y, param_file=None, out_param_file=None, filter_file=None, load_params=False, pretrain=False): if pretrain: params = createNoLearnParams(param_file) print "Parameters", params[1].shape conv0_W = np.concatenate((params[0], params[0]), axis=1) conv0_W = np.concatenate((conv0_W, params[0]), axis=1) conv0_W = conv0_W[:, :7, :, :] conv0_b = np.concatenate((params[1], params[1]), axis=0) conv0_b = np.concatenate((conv0_b, params[1]), axis=0) conv0_b = conv0_b[:96] conv1_W = np.concatenate((params[2], params[2]), axis=1) conv1_W = np.concatenate((conv1_W, params[2]), axis=1) conv1_W = conv1_W[:, :96, :, :] conv1_b = np.concatenate((params[3], params[3]), axis=0) conv1_b = np.concatenate((conv1_b, params[3]), axis=0) conv1_b = conv1_b[:256] conv2_W = np.concatenate((params[4], params[4]), axis=1) conv2_W = np.concatenate((conv2_W, params[4]), axis=1) conv2_W = conv2_W[:, :256, :, :] conv2_b = np.concatenate((params[5], params[5]), axis=0) conv2_b = np.concatenate((conv2_b, params[5]), axis=0) conv2_b = conv2_b[:512] conv3_W = np.concatenate((params[6], params[6]), axis=1) conv3_W = np.concatenate((conv3_W, params[6]), axis=1) conv3_W = conv3_W[:, :512, :, :] conv3_b = np.concatenate((params[7], params[7]), axis=0) conv3_b = np.concatenate((conv3_b, params[7]), axis=0) conv3_b = conv3_b[:512] conv4_W = np.concatenate((params[8], params[8]), axis=1) conv4_W = np.concatenate((conv4_W, params[8]), axis=1) conv4_W = conv4_W[:, :512, :, :] conv4_b = np.concatenate((params[9], params[9]), axis=0) conv4_b = np.concatenate((conv4_b, params[9]), axis=0) conv4_b = conv4_b[:512] dense0_W = np.concatenate((params[10], params[10]), axis=1) dense0_W = np.concatenate((dense0_W, params[10]), axis=1) dense0_W = dense0_W[:2560, :4096] dense0_b = np.concatenate((params[11], params[11]), axis=0) dense0_b = np.concatenate((dense0_b, params[11]), axis=0) dense0_b = dense0_b[:4096] dense1_W = np.concatenate((params[12], params[12]), axis=1) dense1_W = np.concatenate((dense1_W, params[12]), axis=1) dense1_W = dense1_W[:4096, :4096] dense1_b = np.concatenate((params[13], params[13]), axis=0) dense1_b = np.concatenate((dense1_b, params[13]), axis=0) dense1_b = dense1_b[:4096] #http://arxiv.org/pdf/1405.3531v4.pdf self.net = NeuralNet( layers=self.layers, conv0_W=np.array(conv0_W), conv0_b=np.array(conv0_b), conv1_W=np.array(conv1_W), conv1_b=np.array(conv1_b), conv2_W=np.array(conv2_W), conv2_b=np.array(conv2_b), conv3_W=np.array(conv3_W), conv3_b=np.array(conv3_b), conv4_W=np.array(conv4_W), conv4_b=np.array(conv4_b), dense0_W=np.array(dense0_W), dense0_b=np.array(dense0_b), dense1_W=np.array(dense1_W), dense1_b=np.array(dense1_b), update_learning_rate=0.015, update=L.updates.nesterov_momentum, update_momentum=0.9, #update=L.updates.sgd, regression=True, verbose=1, eval_size=0.15, objective_loss_function=L.objectives.binary_crossentropy, max_epochs=200) if load_params: print "Loading parameters from ", param_file self.net.load_params_from(param_file) print "TRAINING!" print "input shape: ", X.shape print "output shape: ", y.shape print "Example X", X[0] print "Example Y", y[0] #print self.net.get_params() self.net.fit(X, y) print(self.net.score(X, y)) print "Saving network parameters to ", out_param_file, "..." file = open(out_param_file, 'w+') file.close() self.net.save_weights_to(out_param_file) print "Saving filters to ", filter_file self.write_filters_to_file(filter_file) plt = visualize.plot_loss(self.net) plt.show() plt.savefig(DIR_PROJ + 'loss.png') plt.clf() plt.cla() print "Sample predictions" for i in range(10): pred = self.net.predict(np.array([X[i]])) print "---------------------------------" print i print "prediction", pred print y[i]
verbose=1, ) ae.fit(X_train, X_out) print ### expect training / val error of about 0.087 with these parameters ### if your GPU not fast enough, reduce the number of filters in the conv/deconv step # <codecell> import pickle import sys sys.setrecursionlimit(10000) pickle.dump(ae, open('mnist/conv_ae.pkl','w')) #ae = pickle.load(open('mnist/conv_ae.pkl','r')) ae.save_weights_to('mnist/conv_ae.np') # <codecell> X_train_pred = ae.predict(X_train).reshape(-1, 28, 28) * sigma + mu X_pred = np.rint(X_train_pred).astype(int) X_pred = np.clip(X_pred, a_min = 0, a_max = 255) X_pred = X_pred.astype('uint8') print X_pred.shape , X.shape # <codecell> ### show random inputs / outputs side by side def get_picture_array(X, index): array = X[index].reshape(28,28)
verbose=1, ) ae.fit(X_train, X_out) print ### expect training / val error of about 0.087 with these parameters ### if your GPU not fast enough, reduce the number of filters in the conv/deconv step # <codecell> import pickle import sys sys.setrecursionlimit(10000) pickle.dump(ae, open('ish/conv_ae.pkl','w')) #ae = pickle.load(open('mnist/conv_ae.pkl','r')) ae.save_weights_to('ish/conv_ae.np') # <codecell> X_train_pred = ae.predict(X_train).reshape(-1, IMAGE_WIDTH, IMAGE_HEIGHT) * sigma + mu X_pred = np.rint(X_train_pred).astype(int) X_pred = np.clip(X_pred, a_min = 0, a_max = 255) X_pred = X_pred.astype('uint8') print X_pred.shape , X.shape # <codecell> ### show random inputs / outputs side by side def get_picture_array(X, index): array = X[index].reshape(IMAGE_WIDTH,IMAGE_HEIGHT)
def createCSAE(input_height, input_width, X_train, X_out): X_train = np.random.binomial(1, 1-dropout_percent, size=X_train.shape) * X_train cnn = NeuralNet(layers=[ ('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), ('conv11', layers.Conv2DLayer), # ('conv12', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2', layers.Conv2DLayer), ('conv21', layers.Conv2DLayer), # ('conv22', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3', layers.Conv2DLayer), # ('conv31', layers.Conv2DLayer), ('conv32', layers.Conv2DLayer), ('unpool1', Unpool2DLayer), ('conv4', layers.Conv2DLayer), ('conv41', layers.Conv2DLayer), # ('conv42', layers.Conv2DLayer), ('unpool2', Unpool2DLayer), ('conv5', layers.Conv2DLayer), ('conv51', layers.Conv2DLayer), # ('conv52', layers.Conv2DLayer), ('conv6', layers.Conv2DLayer), ('output_layer', ReshapeLayer), ], input_shape=(None, 1, input_width, input_height), # Layer current size - 1x300x140 conv1_num_filters=layers_size[0], conv1_filter_size=filter_1, conv1_nonlinearity=activation, # conv1_border_mode="same", conv1_pad="same", conv11_num_filters=layers_size[0], conv11_filter_size=filter_1, conv11_nonlinearity=activation, # conv11_border_mode="same", conv11_pad="same", # conv12_num_filters=layers_size[0], conv12_filter_size=filter_1, conv12_nonlinearity=activation, # # conv12_border_mode="same", # conv12_pad="same", pool1_pool_size=(2, 2), conv2_num_filters=layers_size[1], conv2_filter_size=filter_2, conv2_nonlinearity=activation, # conv2_border_mode="same", conv2_pad="same", conv21_num_filters=layers_size[1], conv21_filter_size=filter_2, conv21_nonlinearity=activation, # conv21_border_mode="same", conv21_pad="same", # conv22_num_filters=layers_size[1], conv22_filter_size=filter_2, conv22_nonlinearity=activation, # # conv22_border_mode="same", # conv22_pad="same", pool2_pool_size=(2, 2), conv3_num_filters=layers_size[2], conv3_filter_size=filter_3, conv3_nonlinearity=activation, # conv3_border_mode="same", conv3_pad="same", # conv31_num_filters=layers_size[2], conv31_filter_size=filter_3, conv31_nonlinearity=activation, # # conv31_border_mode="same", # conv31_pad="same", conv32_num_filters=1, conv32_filter_size=filter_3, conv32_nonlinearity=activation, # conv32_border_mode="same", conv32_pad="same", unpool1_ds=(2, 2), conv4_num_filters=layers_size[3], conv4_filter_size=filter_4, conv4_nonlinearity=activation, # conv4_border_mode="same", conv4_pad="same", conv41_num_filters=layers_size[3], conv41_filter_size=filter_4, conv41_nonlinearity=activation, # conv41_border_mode="same", conv41_pad="same", # conv42_num_filters=layers_size[3], conv42_filter_size=filter_4, conv42_nonlinearity=activation, # # conv42_border_mode="same", # conv42_pad="same", unpool2_ds=(2, 2), conv5_num_filters=layers_size[4], conv5_filter_size=filter_5, conv5_nonlinearity=activation, # conv5_border_mode="same", conv5_pad="same", conv51_num_filters=layers_size[4], conv51_filter_size=filter_5, conv51_nonlinearity=activation, # conv51_border_mode="same", conv51_pad="same", # conv52_num_filters=layers_size[4], conv52_filter_size=filter_5, conv52_nonlinearity=activation, # # conv52_border_mode="same", # conv52_pad="same", conv6_num_filters=1, conv6_filter_size=filter_6, conv6_nonlinearity=last_layer_activation, # conv6_border_mode="same", conv6_pad="same", output_layer_shape=(([0], -1)), update_learning_rate=learning_rate, update_momentum=update_momentum, update=nesterov_momentum, train_split=TrainSplit(eval_size=train_valid_split), batch_iterator_train=FlipBatchIterator(batch_size=batch_size) if flip_batch else BatchIterator(batch_size=batch_size), regression=True, max_epochs=epochs, verbose=1, hiddenLayer_to_output=-9) cnn.fit(X_train, X_out) try: pickle.dump(cnn, open(folder_path + 'conv_ae.pkl', 'w')) # cnn = pickle.load(open(folder_path + 'conv_ae.pkl','r')) cnn.save_weights_to(folder_path + 'conv_ae.np') except: print ("Could not pickle cnn") X_pred = cnn.predict(X_train).reshape(-1, input_height, input_width) # * sigma + mu # # X_pred = np.rint(X_pred).astype(int) # # X_pred = np.clip(X_pred, a_min=0, a_max=255) # # X_pred = X_pred.astype('uint8') # # try: # trian_last_hiddenLayer = cnn.output_hiddenLayer(X_train) # # test_last_hiddenLayer = cnn.output_hiddenLayer(test_x) # pickle.dump(trian_last_hiddenLayer, open(folder_path + 'encode.pkl', 'w')) # except: # print "Could not save encoded images" print ("Saving some images....") for i in range(10): index = np.random.randint(train_x.shape[0]) print (index) def get_picture_array(X, index): array = np.rint(X[index] * 256).astype(np.int).reshape(input_height, input_width) array = np.clip(array, a_min=0, a_max=255) return array.repeat(4, axis=0).repeat(4, axis=1).astype(np.uint8()) original_image = Image.fromarray(get_picture_array(X_out, index)) # original_image.save(folder_path + 'original' + str(index) + '.png', format="PNG") # # array = np.rint(trian_last_hiddenLayer[index] * 256).astype(np.int).reshape(input_height/2, input_width/2) # array = np.clip(array, a_min=0, a_max=255) # encode_image = Image.fromarray(array.repeat(4, axis=0).repeat(4, axis=1).astype(np.uint8())) # encode_image.save(folder_path + 'encode' + str(index) + '.png', format="PNG") new_size = (original_image.size[0] * 3, original_image.size[1]) new_im = Image.new('L', new_size) new_im.paste(original_image, (0, 0)) pred_image = Image.fromarray(get_picture_array(X_pred, index)) # pred_image.save(folder_path + 'pred' + str(index) + '.png', format="PNG") new_im.paste(pred_image, (original_image.size[0], 0)) noise_image = Image.fromarray(get_picture_array(X_train, index)) new_im.paste(noise_image, (original_image.size[0]*2, 0)) new_im.save(folder_path+'origin_prediction_noise-'+str(index)+'.png', format="PNG") # diff = ImageChops.difference(original_image, pred_image) # diff = diff.convert('L') # diff.save(folder_path + 'diff' + str(index) + '.png', format="PNG") # plt.imshow(new_im) # new_size = (original_image.size[0] * 2, original_image.size[1]) # new_im = Image.new('L', new_size) # new_im.paste(original_image, (0, 0)) # pred_image = Image.fromarray(get_picture_array(X_train, index)) # # pred_image.save(folder_path + 'noisyInput' + str(index) + '.png', format="PNG") # new_im.paste(pred_image, (original_image.size[0], 0)) # new_im.save(folder_path+'origin_VS_noise-'+str(index)+'.png', format="PNG") # plt.imshow(new_im) return cnn
hidden1_W=GlorotUniform('relu'), # hidden2_num_units=256, hidden2_nonlinearity=lasagne.nonlinearities.rectify, dropout_p=0.5, output_num_units=num_classes, output_nonlinearity=lasagne.nonlinearities.softmax, output_W=GlorotUniform(gain=1.0), # ----------------------- ConvNet Params ------------------------------------------- update=nesterov_momentum, update_learning_rate=learning_rate, update_momentum=momentum, max_epochs=num_epochs, verbose=1, ) tic = time.time() for i in range(12): convNet.fit(dataset['X_train'], dataset['Y_train']) fl = './model1/saved_model_data' + str(i + 1) + '.npz' convNet.save_weights_to(fl) print 'Model saved to file :- ', fl toc = time.time() fl = './model1/saved_model_data' + str(6) + '.npz' convNet.load_weights_from(fl) y_pred = convNet.predict(dataset['X_test']) print classification_report(Y_test, y_pred) print accuracy_score(Y_test, y_pred) print 'Time taken to train the data :- ', toc - tic, 'seconds'
update_momentum=theano.shared(np.float32(0.9)), # Decay the learning rate on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.99), ], regression=True, y_tensor_type=T.imatrix, objective_loss_function=binary_crossentropy, #batch_iterator_train = BatchIterator(batch_size = 256), max_epochs=20, eval_size=0.1, #train_split =0.0, verbose=2, ) seednumber = 1235 np.random.seed(seednumber) net.fit(train, labels) preds = net.predict_proba(test)[:, 0] submission = pd.read_csv('sample_submission.csv') submission["PredictedProb"] = preds submission.to_csv('submission.csv', index=False) print("Saving the Model") #https://github.com/dnouri/nolearn/issues/58 net.save_weights_to('./model/nnet_qu_150_40.pkl')
verbose=1, ) ae.fit(X_train, X_out) print ### expect training / val error of about 0.087 with these parameters ### if your GPU not fast enough, reduce the number of filters in the conv/deconv step # <codecell> import pickle import sys sys.setrecursionlimit(10000) pickle.dump(ae, open('conv_ae_100.pkl', 'w')) #ae = pickle.load(open('mnist/conv_ae.pkl','r')) ae.save_weights_to('conv_ae_100.np') # <codecell> X_train_pred = ae.predict(X_train).reshape(-1, 28, 28) * sigma + mu X_pred = np.rint(X_train_pred).astype(int) X_pred = np.clip(X_pred, a_min=0, a_max=255) X_pred = X_pred.astype('uint8') print X_pred.shape, X.shape # <codecell> ### show random inputs / outputs side by side def get_picture_array(X, index):