# label[0]: score; label[1]: #
print data_train.target_names[label[1]], label[0]
print
#####################################
# decision_function and predict_proba
print clf_nb
pred_prob = clf_nb.predict_proba(X_new)
print pred_prob
print
print clf_lsvc
pred_decision = clf_lsvc.decision_function(X_new)
print pred_decision
print
print clf_svc
# SVC should have the decision_function method, but got error:
# error - ValueError: setting an array element with a sequence
# pred_decision = clf_svc.decision_function(X_new)
pred_prob = clf_svc.predict_proba(X_new)
print pred_prob
print
print clf_sgd
pred_decision = clf_sgd.decision_function(X_new)
# pred_prob is only supported for binary classification!
# pred_prob = clf_sgd.predict_proba(X_new)
print pred_decision
print
X_den_train, X_den_test = X_den[train_index], X_den[test_index]
# feed models
clf_mNB.fit(X_train, y_train)
clf_ridge.fit(X_train, y_train)
clf_SGD.fit(X_train, y_train)
clf_lSVC.fit(X_train, y_train)
clf_SVC.fit(X_train, y_train)
# get prediction for this fold run
prob_mNB = clf_mNB.predict_proba(X_test)
prob_ridge = clf_ridge.decision_function(X_test)
prob_SGD = clf_SGD.decision_function(X_test)
prob_lSVC = clf_lSVC.decision_function(X_test)
prob_SVC = clf_SVC.predict_proba(X_test)
# add prob functions into the z 2d-array
z_temp = (prob_mNB + prob_ridge + prob_SGD + prob_lSVC + prob_SVC)
z = np.append(z, z_temp, axis=0)
# remove the first sub-1d-array of z, due to the creation with 0s
z = np.delete(z, 0, 0)
# the result of z is a 2d array with shape of (n_samples, n_categories)
# the elements are the sum of probabilities of classifiers on each (sample,category) pair
print z
print 'z shape: ', z.shape
# Possible preprocessing on z
# X_den_train, X_den_test = X_den[train_index], X_den[test_index]
# feed models
clf_mNB.fit(X_train_train, y_train_train)
# clf_kNN.fit(X_train_train, y_train_train)
clf_ridge.fit(X_train_train, y_train_train)
clf_lSVC.fit(X_train_train, y_train_train)
clf_SVC.fit(X_train_train, y_train_train)
# get prediction for this fold run
prob_mNB = clf_mNB.predict_proba(X_train_test)
# prob_kNN = clf_kNN.predict_proba(X_train_test)
prob_ridge = clf_ridge.decision_function(X_train_test)
prob_lSVC = clf_lSVC.decision_function(X_train_test)
prob_SVC = clf_SVC.predict_proba(X_train_test)
# update z array for each model
# z_temp = prob_lSVC
# z_temp = (prob_ridge + prob_lSVC)
z_temp = (prob_mNB + prob_ridge + prob_lSVC + prob_SVC)
z = np.append(z, z_temp, axis=0)
# remove the first sub-1d-array of z, due to the creation with 0s
z = np.delete(z, 0, 0)
# the result of z is a 2d array with shape of (n_samples, n_categories)
# the elements are the sum of probabilities of classifiers on each (sample,category) pair
# Possible preprocessing on z
# z = normalize(z, norm="l2")
# z = scale(z)
