train_labels = np.asfarray(train_data[:, :1]) test_labels = np.asfarray(test_data[:, :1]) print('train and test images converted to float (0, 1)') train_labels_one_hot = (label == train_labels).astype(np.float32) test_labels_one_hot = (label == test_labels).astype(np.float32) print("train and test one-hot vectors") # train_labels = np.vectorize(lambda y: pycl_array.to_device(cls.queue, # train_labels_one_hot[y]))(np.arange(len_train_data)) # test_labels = np.vectorize(lambda y: pycl_array.to_device(cls.queue, # test_labels_one_hot[y]))(np.arange(len_test_data)) # print("train and test labels created") # train_dataset = np.vectorize(lambda i: (train_imgs[i], train_labels[i]))(np.arange(len_train_data)) # test_dataset = np.vectorize(lambda i: (test_imgs[i], test_labels[i]))(np.arange(len_test_data)) # print("train and test dataset created") print("Building NN") nn = NeuralNet(Layer(image_pixels, activation_type=TANH), Layer(512, activation_type=SOFTMAX), Output(no_of_different_labels)) batch = 100 nn.fit(train_imgs, train_labels_one_hot, batch_size=batch, num_epochs=len_train_data // batch, len_dataset=len_train_data, print_every=10)
# --------learning-------- nn = NeuralNet(n_output=3, n_features=X_train.shape[1], n_hidden=50, l2=0.1, l1=0.0, epochs=500, eta=0.001, alpha=0.001, decrease_const=0.00001, shuffle=True, minibatches=50, random_state=1) nn.fit(X_train, y_train, print_progress=True) # --------end-learning-------- print('\n') # --------examples-type-1-------- ##-1- print(nn.detection(nn, 20)) # return 1 ##-2- print(nn.detection(nn, 30)) # return 2 ##-3-
class BlendedModel(BaseEnsemble): def __init__(self, models=[], blending='average',nbFeatures=4): self.models = models self.blending = blending self.logR = LogisticRegression(C=10)#,multi_class='multinomial',solver='lbfgs', max_iter=10000) self.logRT= LogisticRegression(C=10)#,multi_class='multinomial',solver='lbfgs', max_iter=10000) self.nn=NeuralNet(nbFeatures) self.XGB=ModifiedXGBClassifier() if self.blending not in ['average', 'most_confident']: raise Exception('Wrong blending method') ##fit the stochastic gradient boosting trees classifier def fit(self, X, y): for model in self.models: print 'Training model :' print model.get_params() model.fit(X, y) return self ##predict the outputs according to the average of the classifier (or according an entropy based voting scheme that does not work well) def predict_proba(self, X): preds = np.array( [model.predict_proba(X) for model in self.models] ) if self.blending == 'average': return np.mean(preds , axis=0 ) elif self.blending == 'most_confident': def dirac_weights(entropies): w = (entropies == np.min(entropies)).astype(float) return w/np.sum(w) def shannon_entropy(l): l = [min(max(1e-5,p),1-1e-5) for p in l] l = np.array(l)/sum(l) return sum([-p*math.log(p) for p in l]) shannon_entropy_array = lambda l : np.apply_along_axis(shannon_entropy, 1, l) entropies = np.array([shannon_entropy_array(pred) for pred in preds]) weights = np.apply_along_axis(dirac_weights, 0, entropies) return np.sum(np.multiply(weights.T, preds.T).T, axis = 0) ##fit the logistic regression stacking def fitLog(self,X,y,mod=0): if mod==0: ##without features engineering preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))]) self.logR.fit(features, y) elif mod==1: ##with features engineering preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))]) self.logRT.fit(features, y) return self ##predict the outputs of the logistic regression stack def predict_Logproba(self, X,mod=0): if mod==0: preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))]) preds=self.logR.predict_proba(features) return preds elif mod==1: preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))]) preds=self.logRT.predict_proba(features) return preds ##I also try to use a gradient boosting as a stacking classifier, but it does not work well def fitXGB(self,X,y): preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))]) #features= np.append(features, X, axis=1) self.XGB.fit(features,y) return self def predict_XGBproba(self,X): preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))]) #features= np.append(features, X, axis=1) return self.XGB.predict_proba(features) ##neural network stack classifier def fitNN(self,X,y,lambda1=0.00000001,lambda2=0.00005,new=0,teX=[],teY=[],lr=0.001): preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))]) features= np.append(features, X, axis=1) if len(teX)>0: preds = np.array( [model.predict_proba(teX) for model in self.models] ) featuresteX=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(teX))]) featuresteX= np.append(featuresteX, teX, axis=1) else: featuresteX=[] self.nn.fit(features,y,lambda1,lambda2,new,featuresteX,teY,lr) return self ##predict nn stack classifier def predict_NNproba(self,X): preds = np.array( [model.predict_proba(X) for model in self.models] ) features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))]) features= np.append(features, X, axis=1) return self.nn.predict_proba(features)