verbose=1,
)
ae.fit(X_train, X_out)
print
### expect training / val error of about 0.087 with these parameters
### if your GPU not fast enough, reduce the number of filters in the conv/deconv step
# <codecell>
import pickle
import sys
sys.setrecursionlimit(10000)
pickle.dump(ae, open('mnist/conv_ae.pkl', 'w'))
#ae = pickle.load(open('mnist/conv_ae.pkl','r'))
ae.save_weights_to('mnist/conv_ae.np')
# <codecell>
X_train_pred = ae.predict(X_train).reshape(-1, 28, 28) * sigma + mu
X_pred = np.rint(X_train_pred).astype(int)
X_pred = np.clip(X_pred, a_min=0, a_max=255)
X_pred = X_pred.astype('uint8')
print X_pred.shape, X.shape
# <codecell>
### show random inputs / outputs side by side
def get_picture_array(X, index):
dropout_p=0.5, output_num_units=num_classes, output_nonlinearity=lasagne.nonlinearities.softmax, output_W = GlorotUniform(gain = 1.0), # ----------------------- ConvNet Params ------------------------------------------- update = nesterov_momentum, update_learning_rate = learning_rate, update_momentum = momentum, max_epochs = num_epochs, verbose = 1, ) tic = time.time() for i in range(12): convNet.fit(dataset['X_train'], dataset['Y_train']) fl = './model1/saved_model_data' + str(i+1) + '.npz' convNet.save_weights_to(fl) print 'Model saved to file :- ', fl toc = time.time() fl = './model1/saved_model_data' + str(6) + '.npz' convNet.load_weights_from(fl) y_pred = convNet.predict(dataset['X_test']) print classification_report(Y_test, y_pred) print accuracy_score(Y_test, y_pred) print 'Time taken to train the data :- ', toc-tic, 'seconds'
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
eval_size=0.2,
verbose=1,
max_epochs=200)
print "fitting nn model.."
net0.fit(X_train, y_train)
print "predicting probabilities using nn model..."
proba = net0.predict_proba(X_test)
ll.append(calc_ll_from_proba(proba, y_test))
print metrics.confusion_matrix(
y_test.astype(int), np.argmax(proba, axis=1).astype(int))
print "logloss: ", ll[ncv]
print "saving nn model..."
net0.save_weights_to('weights/nn_%d.pkl' % ncv)
net0 = None
ncv += 1
ll = np.array(ll)
print "logloss: ", ll
# make_submission(net0, X_test, ids, encoder)
# output
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=2,
# optimization method params
update=nesterov_momentum,
update_learning_rate=0.007,
update_momentum=0.9,
max_epochs=16,
verbose=1,
)
"""Loading data and training Lasagne network using nolearn"""
trainImg = np.load('trainImg_stage1.npy')
trainVal2 = np.load('trainVal_stage1.npy')
trainImg2 = trainImg.astype(np.float32).swapaxes(1, 3)
trainVal2 = trainVal2.astype(np.uint8)
print "Training Classifier: 70/30 split"
nn.fit(trainImg2, trainVal2)
print "Saving Classifier"
pickle.dump(nn, open("nn_stage1.pkl", "wb"))
nn.save_weights_to('weights_stage1')
class graspNet():
def __init__(self, param_file=None):
net_divider = 1.0
self.layers = [
(L.layers.InputLayer, {
'shape': (None, 7, X_H, X_W),
'name': 'input'
}),
(L.layers.Conv2DLayer, {
'num_filters': 96,
'stride': 1,
'pad': 3,
'filter_size': (7, 7),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv0'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.MaxPool2DLayer, {
'pool_size': 3,
'ignore_border': False
}),
(L.layers.Conv2DLayer, {
'num_filters': 256,
'stride': 1,
'filter_size': (5, 5),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv1'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.MaxPool2DLayer, {
'pool_size': 2,
'ignore_border': False
}),
(L.layers.Conv2DLayer, {
'num_filters': 512,
'stride': 1,
'pad': 1,
'filter_size': (3, 3),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv2'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.Conv2DLayer, {
'num_filters': 512,
'stride': 1,
'pad': 1,
'filter_size': (3, 3),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv3'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.Conv2DLayer, {
'num_filters': 512,
'stride': 1,
'pad': 1,
'filter_size': (3, 3),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv4'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.MaxPool2DLayer, {
'pool_size': 3,
'ignore_border': False
}),
(L.layers.DenseLayer, {
'num_units': 4096,
'nonlinearity': L.nonlinearities.sigmoid,
'name': 'dense0'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.DenseLayer, {
'num_units': 4096,
'nonlinearity': L.nonlinearities.sigmoid,
'name': 'dense1'
}),
(L.layers.DenseLayer, {
'num_units': 1,
'nonlinearity': L.nonlinearities.sigmoid
}),
]
self.net = NeuralNet(
layers=self.layers,
update_learning_rate=0.015,
update=L.updates.nesterov_momentum,
update_momentum=0.9,
#update=L.updates.sgd,
regression=True,
verbose=1,
eval_size=0.15,
objective_loss_function=L.objectives.binary_crossentropy,
max_epochs=200)
if param_file != None:
print "Loading parameters from ", param_file
self.net.load_params_from(param_file)
def write_filters_to_file(self, fname):
params = self.net.get_all_params()
layer_counter = 0
for p in params:
print p
layer_counter += 1
filter_counter = 0
weights = p.get_value()
if len(weights.shape) > 2:
for f in weights:
kernel = np.asarray(f, dtype=np.float32)
kernel = kernel * 255 + 128
viz = np.zeros(shape=kernel[0].shape)
viz = cv2.resize(kernel[0],
None,
fx=20,
fy=20,
interpolation=cv2.INTER_CUBIC)
cv2.normalize(viz, viz)
viz = viz * 50
#print viz
#cv2.imshow('ha', viz)
#cv2.waitKey(-1)
viz = viz / 50
viz = viz * 12000
viz = viz
#print viz
cv2.imwrite(
fname + "_" + str(layer_counter) + "_" +
str(filter_counter) + ".png", viz)
filter_counter += 1
# evaluates the input array over the neural net
def eval(self, x):
y = np.zeros((x.shape[0], ))
for i in range(x.shape[0]):
pred = self.net.predict(np.array([x[i]]))
y[i] = pred
print i, pred
return y
# train the network in input (x,y)
# param_file: input parameter file (perhaps from previous trainings)
# out_param_file: output path to write resulting params to
# filter_file: output path to write images of filters for visualization
# load_params: boolean flag, when set: 1 loads input parameters from <param_file>
# pretrain: boolean flag, when set: 1 loads parameters from pretrained network
def train(self,
X,
y,
param_file=None,
out_param_file=None,
filter_file=None,
load_params=False,
pretrain=False):
if pretrain:
params = createNoLearnParams(param_file)
print "Parameters", params[1].shape
conv0_W = np.concatenate((params[0], params[0]), axis=1)
conv0_W = np.concatenate((conv0_W, params[0]), axis=1)
conv0_W = conv0_W[:, :7, :, :]
conv0_b = np.concatenate((params[1], params[1]), axis=0)
conv0_b = np.concatenate((conv0_b, params[1]), axis=0)
conv0_b = conv0_b[:96]
conv1_W = np.concatenate((params[2], params[2]), axis=1)
conv1_W = np.concatenate((conv1_W, params[2]), axis=1)
conv1_W = conv1_W[:, :96, :, :]
conv1_b = np.concatenate((params[3], params[3]), axis=0)
conv1_b = np.concatenate((conv1_b, params[3]), axis=0)
conv1_b = conv1_b[:256]
conv2_W = np.concatenate((params[4], params[4]), axis=1)
conv2_W = np.concatenate((conv2_W, params[4]), axis=1)
conv2_W = conv2_W[:, :256, :, :]
conv2_b = np.concatenate((params[5], params[5]), axis=0)
conv2_b = np.concatenate((conv2_b, params[5]), axis=0)
conv2_b = conv2_b[:512]
conv3_W = np.concatenate((params[6], params[6]), axis=1)
conv3_W = np.concatenate((conv3_W, params[6]), axis=1)
conv3_W = conv3_W[:, :512, :, :]
conv3_b = np.concatenate((params[7], params[7]), axis=0)
conv3_b = np.concatenate((conv3_b, params[7]), axis=0)
conv3_b = conv3_b[:512]
conv4_W = np.concatenate((params[8], params[8]), axis=1)
conv4_W = np.concatenate((conv4_W, params[8]), axis=1)
conv4_W = conv4_W[:, :512, :, :]
conv4_b = np.concatenate((params[9], params[9]), axis=0)
conv4_b = np.concatenate((conv4_b, params[9]), axis=0)
conv4_b = conv4_b[:512]
dense0_W = np.concatenate((params[10], params[10]), axis=1)
dense0_W = np.concatenate((dense0_W, params[10]), axis=1)
dense0_W = dense0_W[:2560, :4096]
dense0_b = np.concatenate((params[11], params[11]), axis=0)
dense0_b = np.concatenate((dense0_b, params[11]), axis=0)
dense0_b = dense0_b[:4096]
dense1_W = np.concatenate((params[12], params[12]), axis=1)
dense1_W = np.concatenate((dense1_W, params[12]), axis=1)
dense1_W = dense1_W[:4096, :4096]
dense1_b = np.concatenate((params[13], params[13]), axis=0)
dense1_b = np.concatenate((dense1_b, params[13]), axis=0)
dense1_b = dense1_b[:4096]
#http://arxiv.org/pdf/1405.3531v4.pdf
self.net = NeuralNet(
layers=self.layers,
conv0_W=np.array(conv0_W),
conv0_b=np.array(conv0_b),
conv1_W=np.array(conv1_W),
conv1_b=np.array(conv1_b),
conv2_W=np.array(conv2_W),
conv2_b=np.array(conv2_b),
conv3_W=np.array(conv3_W),
conv3_b=np.array(conv3_b),
conv4_W=np.array(conv4_W),
conv4_b=np.array(conv4_b),
dense0_W=np.array(dense0_W),
dense0_b=np.array(dense0_b),
dense1_W=np.array(dense1_W),
dense1_b=np.array(dense1_b),
update_learning_rate=0.015,
update=L.updates.nesterov_momentum,
update_momentum=0.9,
#update=L.updates.sgd,
regression=True,
verbose=1,
eval_size=0.15,
objective_loss_function=L.objectives.binary_crossentropy,
max_epochs=200)
if load_params:
print "Loading parameters from ", param_file
self.net.load_params_from(param_file)
print "TRAINING!"
print "input shape: ", X.shape
print "output shape: ", y.shape
print "Example X", X[0]
print "Example Y", y[0]
#print self.net.get_params()
self.net.fit(X, y)
print(self.net.score(X, y))
print "Saving network parameters to ", out_param_file, "..."
file = open(out_param_file, 'w+')
file.close()
self.net.save_weights_to(out_param_file)
print "Saving filters to ", filter_file
self.write_filters_to_file(filter_file)
plt = visualize.plot_loss(self.net)
plt.show()
plt.savefig(DIR_PROJ + 'loss.png')
plt.clf()
plt.cla()
print "Sample predictions"
for i in range(10):
pred = self.net.predict(np.array([X[i]]))
print "---------------------------------"
print i
print "prediction", pred
print y[i]
verbose=1,
)
ae.fit(X_train, X_out)
print
### expect training / val error of about 0.087 with these parameters
### if your GPU not fast enough, reduce the number of filters in the conv/deconv step
# <codecell>
import pickle
import sys
sys.setrecursionlimit(10000)
pickle.dump(ae, open('mnist/conv_ae.pkl','w'))
#ae = pickle.load(open('mnist/conv_ae.pkl','r'))
ae.save_weights_to('mnist/conv_ae.np')
# <codecell>
X_train_pred = ae.predict(X_train).reshape(-1, 28, 28) * sigma + mu
X_pred = np.rint(X_train_pred).astype(int)
X_pred = np.clip(X_pred, a_min = 0, a_max = 255)
X_pred = X_pred.astype('uint8')
print X_pred.shape , X.shape
# <codecell>
### show random inputs / outputs side by side
def get_picture_array(X, index):
array = X[index].reshape(28,28)
verbose=1,
)
ae.fit(X_train, X_out)
print
### expect training / val error of about 0.087 with these parameters
### if your GPU not fast enough, reduce the number of filters in the conv/deconv step
# <codecell>
import pickle
import sys
sys.setrecursionlimit(10000)
pickle.dump(ae, open('ish/conv_ae.pkl','w'))
#ae = pickle.load(open('mnist/conv_ae.pkl','r'))
ae.save_weights_to('ish/conv_ae.np')
# <codecell>
X_train_pred = ae.predict(X_train).reshape(-1, IMAGE_WIDTH, IMAGE_HEIGHT) * sigma + mu
X_pred = np.rint(X_train_pred).astype(int)
X_pred = np.clip(X_pred, a_min = 0, a_max = 255)
X_pred = X_pred.astype('uint8')
print X_pred.shape , X.shape
# <codecell>
### show random inputs / outputs side by side
def get_picture_array(X, index):
array = X[index].reshape(IMAGE_WIDTH,IMAGE_HEIGHT)
def createCSAE(input_height, input_width, X_train, X_out):
X_train = np.random.binomial(1, 1-dropout_percent, size=X_train.shape) * X_train
cnn = NeuralNet(layers=[
('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('conv11', layers.Conv2DLayer),
# ('conv12', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('conv2', layers.Conv2DLayer),
('conv21', layers.Conv2DLayer),
# ('conv22', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('conv3', layers.Conv2DLayer),
# ('conv31', layers.Conv2DLayer),
('conv32', layers.Conv2DLayer),
('unpool1', Unpool2DLayer),
('conv4', layers.Conv2DLayer),
('conv41', layers.Conv2DLayer),
# ('conv42', layers.Conv2DLayer),
('unpool2', Unpool2DLayer),
('conv5', layers.Conv2DLayer),
('conv51', layers.Conv2DLayer),
# ('conv52', layers.Conv2DLayer),
('conv6', layers.Conv2DLayer),
('output_layer', ReshapeLayer),
],
input_shape=(None, 1, input_width, input_height),
# Layer current size - 1x300x140
conv1_num_filters=layers_size[0], conv1_filter_size=filter_1, conv1_nonlinearity=activation,
# conv1_border_mode="same",
conv1_pad="same",
conv11_num_filters=layers_size[0], conv11_filter_size=filter_1, conv11_nonlinearity=activation,
# conv11_border_mode="same",
conv11_pad="same",
# conv12_num_filters=layers_size[0], conv12_filter_size=filter_1, conv12_nonlinearity=activation,
# # conv12_border_mode="same",
# conv12_pad="same",
pool1_pool_size=(2, 2),
conv2_num_filters=layers_size[1], conv2_filter_size=filter_2, conv2_nonlinearity=activation,
# conv2_border_mode="same",
conv2_pad="same",
conv21_num_filters=layers_size[1], conv21_filter_size=filter_2, conv21_nonlinearity=activation,
# conv21_border_mode="same",
conv21_pad="same",
# conv22_num_filters=layers_size[1], conv22_filter_size=filter_2, conv22_nonlinearity=activation,
# # conv22_border_mode="same",
# conv22_pad="same",
pool2_pool_size=(2, 2),
conv3_num_filters=layers_size[2], conv3_filter_size=filter_3, conv3_nonlinearity=activation,
# conv3_border_mode="same",
conv3_pad="same",
# conv31_num_filters=layers_size[2], conv31_filter_size=filter_3, conv31_nonlinearity=activation,
# # conv31_border_mode="same",
# conv31_pad="same",
conv32_num_filters=1, conv32_filter_size=filter_3, conv32_nonlinearity=activation,
# conv32_border_mode="same",
conv32_pad="same",
unpool1_ds=(2, 2),
conv4_num_filters=layers_size[3], conv4_filter_size=filter_4, conv4_nonlinearity=activation,
# conv4_border_mode="same",
conv4_pad="same",
conv41_num_filters=layers_size[3], conv41_filter_size=filter_4, conv41_nonlinearity=activation,
# conv41_border_mode="same",
conv41_pad="same",
# conv42_num_filters=layers_size[3], conv42_filter_size=filter_4, conv42_nonlinearity=activation,
# # conv42_border_mode="same",
# conv42_pad="same",
unpool2_ds=(2, 2),
conv5_num_filters=layers_size[4], conv5_filter_size=filter_5, conv5_nonlinearity=activation,
# conv5_border_mode="same",
conv5_pad="same",
conv51_num_filters=layers_size[4], conv51_filter_size=filter_5, conv51_nonlinearity=activation,
# conv51_border_mode="same",
conv51_pad="same",
# conv52_num_filters=layers_size[4], conv52_filter_size=filter_5, conv52_nonlinearity=activation,
# # conv52_border_mode="same",
# conv52_pad="same",
conv6_num_filters=1, conv6_filter_size=filter_6, conv6_nonlinearity=last_layer_activation,
# conv6_border_mode="same",
conv6_pad="same",
output_layer_shape=(([0], -1)),
update_learning_rate=learning_rate,
update_momentum=update_momentum,
update=nesterov_momentum,
train_split=TrainSplit(eval_size=train_valid_split),
batch_iterator_train=FlipBatchIterator(batch_size=batch_size) if flip_batch else BatchIterator(batch_size=batch_size),
regression=True,
max_epochs=epochs,
verbose=1,
hiddenLayer_to_output=-9)
cnn.fit(X_train, X_out)
try:
pickle.dump(cnn, open(folder_path + 'conv_ae.pkl', 'w'))
# cnn = pickle.load(open(folder_path + 'conv_ae.pkl','r'))
cnn.save_weights_to(folder_path + 'conv_ae.np')
except:
print ("Could not pickle cnn")
X_pred = cnn.predict(X_train).reshape(-1, input_height, input_width) # * sigma + mu
# # X_pred = np.rint(X_pred).astype(int)
# # X_pred = np.clip(X_pred, a_min=0, a_max=255)
# # X_pred = X_pred.astype('uint8')
#
# try:
# trian_last_hiddenLayer = cnn.output_hiddenLayer(X_train)
# # test_last_hiddenLayer = cnn.output_hiddenLayer(test_x)
# pickle.dump(trian_last_hiddenLayer, open(folder_path + 'encode.pkl', 'w'))
# except:
# print "Could not save encoded images"
print ("Saving some images....")
for i in range(10):
index = np.random.randint(train_x.shape[0])
print (index)
def get_picture_array(X, index):
array = np.rint(X[index] * 256).astype(np.int).reshape(input_height, input_width)
array = np.clip(array, a_min=0, a_max=255)
return array.repeat(4, axis=0).repeat(4, axis=1).astype(np.uint8())
original_image = Image.fromarray(get_picture_array(X_out, index))
# original_image.save(folder_path + 'original' + str(index) + '.png', format="PNG")
#
# array = np.rint(trian_last_hiddenLayer[index] * 256).astype(np.int).reshape(input_height/2, input_width/2)
# array = np.clip(array, a_min=0, a_max=255)
# encode_image = Image.fromarray(array.repeat(4, axis=0).repeat(4, axis=1).astype(np.uint8()))
# encode_image.save(folder_path + 'encode' + str(index) + '.png', format="PNG")
new_size = (original_image.size[0] * 3, original_image.size[1])
new_im = Image.new('L', new_size)
new_im.paste(original_image, (0, 0))
pred_image = Image.fromarray(get_picture_array(X_pred, index))
# pred_image.save(folder_path + 'pred' + str(index) + '.png', format="PNG")
new_im.paste(pred_image, (original_image.size[0], 0))
noise_image = Image.fromarray(get_picture_array(X_train, index))
new_im.paste(noise_image, (original_image.size[0]*2, 0))
new_im.save(folder_path+'origin_prediction_noise-'+str(index)+'.png', format="PNG")
# diff = ImageChops.difference(original_image, pred_image)
# diff = diff.convert('L')
# diff.save(folder_path + 'diff' + str(index) + '.png', format="PNG")
# plt.imshow(new_im)
# new_size = (original_image.size[0] * 2, original_image.size[1])
# new_im = Image.new('L', new_size)
# new_im.paste(original_image, (0, 0))
# pred_image = Image.fromarray(get_picture_array(X_train, index))
# # pred_image.save(folder_path + 'noisyInput' + str(index) + '.png', format="PNG")
# new_im.paste(pred_image, (original_image.size[0], 0))
# new_im.save(folder_path+'origin_VS_noise-'+str(index)+'.png', format="PNG")
# plt.imshow(new_im)
return cnn
hidden1_W=GlorotUniform('relu'),
# hidden2_num_units=256, hidden2_nonlinearity=lasagne.nonlinearities.rectify,
dropout_p=0.5,
output_num_units=num_classes,
output_nonlinearity=lasagne.nonlinearities.softmax,
output_W=GlorotUniform(gain=1.0),
# ----------------------- ConvNet Params -------------------------------------------
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=momentum,
max_epochs=num_epochs,
verbose=1,
)
tic = time.time()
for i in range(12):
convNet.fit(dataset['X_train'], dataset['Y_train'])
fl = './model1/saved_model_data' + str(i + 1) + '.npz'
convNet.save_weights_to(fl)
print 'Model saved to file :- ', fl
toc = time.time()
fl = './model1/saved_model_data' + str(6) + '.npz'
convNet.load_weights_from(fl)
y_pred = convNet.predict(dataset['X_test'])
print classification_report(Y_test, y_pred)
print accuracy_score(Y_test, y_pred)
print 'Time taken to train the data :- ', toc - tic, 'seconds'
update_momentum=theano.shared(np.float32(0.9)),
# Decay the learning rate
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.99),
],
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
#batch_iterator_train = BatchIterator(batch_size = 256),
max_epochs=20,
eval_size=0.1,
#train_split =0.0,
verbose=2,
)
seednumber = 1235
np.random.seed(seednumber)
net.fit(train, labels)
preds = net.predict_proba(test)[:, 0]
submission = pd.read_csv('sample_submission.csv')
submission["PredictedProb"] = preds
submission.to_csv('submission.csv', index=False)
print("Saving the Model")
#https://github.com/dnouri/nolearn/issues/58
net.save_weights_to('./model/nnet_qu_150_40.pkl')
verbose=1,
)
ae.fit(X_train, X_out)
print
### expect training / val error of about 0.087 with these parameters
### if your GPU not fast enough, reduce the number of filters in the conv/deconv step
# <codecell>
import pickle
import sys
sys.setrecursionlimit(10000)
pickle.dump(ae, open('conv_ae_100.pkl', 'w'))
#ae = pickle.load(open('mnist/conv_ae.pkl','r'))
ae.save_weights_to('conv_ae_100.np')
# <codecell>
X_train_pred = ae.predict(X_train).reshape(-1, 28, 28) * sigma + mu
X_pred = np.rint(X_train_pred).astype(int)
X_pred = np.clip(X_pred, a_min=0, a_max=255)
X_pred = X_pred.astype('uint8')
print X_pred.shape, X.shape
# <codecell>
### show random inputs / outputs side by side
def get_picture_array(X, index):
