train_labels = np.asfarray(train_data[:, :1])
test_labels = np.asfarray(test_data[:, :1])
print('train and test images converted to float (0, 1)')
train_labels_one_hot = (label == train_labels).astype(np.float32)
test_labels_one_hot = (label == test_labels).astype(np.float32)
print("train and test one-hot vectors")
# train_labels = np.vectorize(lambda y: pycl_array.to_device(cls.queue,
# train_labels_one_hot[y]))(np.arange(len_train_data))
# test_labels = np.vectorize(lambda y: pycl_array.to_device(cls.queue,
# test_labels_one_hot[y]))(np.arange(len_test_data))
# print("train and test labels created")
# train_dataset = np.vectorize(lambda i: (train_imgs[i], train_labels[i]))(np.arange(len_train_data))
# test_dataset = np.vectorize(lambda i: (test_imgs[i], test_labels[i]))(np.arange(len_test_data))
# print("train and test dataset created")
print("Building NN")
nn = NeuralNet(Layer(image_pixels, activation_type=TANH),
Layer(512, activation_type=SOFTMAX),
Output(no_of_different_labels))
batch = 100
nn.fit(train_imgs,
train_labels_one_hot,
batch_size=batch,
num_epochs=len_train_data // batch,
len_dataset=len_train_data,
print_every=10)
# --------learning--------
nn = NeuralNet(n_output=3,
n_features=X_train.shape[1],
n_hidden=50,
l2=0.1,
l1=0.0,
epochs=500,
eta=0.001,
alpha=0.001,
decrease_const=0.00001,
shuffle=True,
minibatches=50,
random_state=1)
nn.fit(X_train, y_train, print_progress=True)
# --------end-learning--------
print('\n')
# --------examples-type-1--------
##-1-
print(nn.detection(nn, 20))
# return 1
##-2-
print(nn.detection(nn, 30))
# return 2
##-3-
class BlendedModel(BaseEnsemble):
def __init__(self, models=[], blending='average',nbFeatures=4):
self.models = models
self.blending = blending
self.logR = LogisticRegression(C=10)#,multi_class='multinomial',solver='lbfgs', max_iter=10000)
self.logRT= LogisticRegression(C=10)#,multi_class='multinomial',solver='lbfgs', max_iter=10000)
self.nn=NeuralNet(nbFeatures)
self.XGB=ModifiedXGBClassifier()
if self.blending not in ['average', 'most_confident']:
raise Exception('Wrong blending method')
##fit the stochastic gradient boosting trees classifier
def fit(self, X, y):
for model in self.models:
print 'Training model :'
print model.get_params()
model.fit(X, y)
return self
##predict the outputs according to the average of the classifier (or according an entropy based voting scheme that does not work well)
def predict_proba(self, X):
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
if self.blending == 'average':
return np.mean(preds , axis=0 )
elif self.blending == 'most_confident':
def dirac_weights(entropies):
w = (entropies == np.min(entropies)).astype(float)
return w/np.sum(w)
def shannon_entropy(l):
l = [min(max(1e-5,p),1-1e-5) for p in l]
l = np.array(l)/sum(l)
return sum([-p*math.log(p) for p in l])
shannon_entropy_array = lambda l : np.apply_along_axis(shannon_entropy, 1, l)
entropies = np.array([shannon_entropy_array(pred) for pred in preds])
weights = np.apply_along_axis(dirac_weights, 0, entropies)
return np.sum(np.multiply(weights.T, preds.T).T, axis = 0)
##fit the logistic regression stacking
def fitLog(self,X,y,mod=0):
if mod==0: ##without features engineering
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))])
self.logR.fit(features, y)
elif mod==1: ##with features engineering
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))])
self.logRT.fit(features, y)
return self
##predict the outputs of the logistic regression stack
def predict_Logproba(self, X,mod=0):
if mod==0:
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))])
preds=self.logR.predict_proba(features)
return preds
elif mod==1:
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))])
preds=self.logRT.predict_proba(features)
return preds
##I also try to use a gradient boosting as a stacking classifier, but it does not work well
def fitXGB(self,X,y):
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))])
#features= np.append(features, X, axis=1)
self.XGB.fit(features,y)
return self
def predict_XGBproba(self,X):
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([[math.log(preds[j][i][k]/(1-preds[j][i][k])) for k in range(4)] for j in range(len(self.models))]).flatten() for i in range(len(X))])
#features= np.append(features, X, axis=1)
return self.XGB.predict_proba(features)
##neural network stack classifier
def fitNN(self,X,y,lambda1=0.00000001,lambda2=0.00005,new=0,teX=[],teY=[],lr=0.001):
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))])
features= np.append(features, X, axis=1)
if len(teX)>0:
preds = np.array(
[model.predict_proba(teX) for model in self.models]
)
featuresteX=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(teX))])
featuresteX= np.append(featuresteX, teX, axis=1)
else:
featuresteX=[]
self.nn.fit(features,y,lambda1,lambda2,new,featuresteX,teY,lr)
return self
##predict nn stack classifier
def predict_NNproba(self,X):
preds = np.array(
[model.predict_proba(X) for model in self.models]
)
features=np.array([np.array([preds[j][i] for j in range(len(self.models))]).flatten() for i in range(len(X))])
features= np.append(features, X, axis=1)
return self.nn.predict_proba(features)
