def linear_multiclassify(X, Ya, X_test, Ya_test, num_classes,
create_classifier):
Yp = np.zeros((Ya_test.shape[0], num_classes))
for i in range(1, num_classes):
Yb = np.zeros((Ya.shape[0]))
Yb[Ya>=i] = 1
classifier = create_classifier(X, Yb)
logging.debug(classifier.classify(X, Yb))
Yb_test = np.zeros((Ya_test.shape[0]))
Yb_test[Ya_test>=i] = 1
util.report_accuracy(classifier.classify(X_test, Yb_test))
Yp_i = classifier.predict(X_test)
Yp[:, i] = Yp_i
Yp[:, 0] = 1 #it's gotta be positive by definition.
for i in range(num_classes-1):
Yp[:, i] *= Yp[:, i+1]
Yp[:, num_classes-1] = - Yp[:, num_classes-1] # pretend n+1 col wudve been negative
Y_guess = np.argmin(Yp, axis=1)
c_matrix = util.confusion_matrix(Ya_test, Y_guess, num_classes)
logging.info('Overall test acc: %f%%', util.get_accuracy(c_matrix))
logging.debug('%s', c_matrix)
return c_matrix
def test_one_vs_one(X_test, Y_test, num_classes, classifier_list):
logging.info("Testing one vs one multiclassifier ")
Yp = np.zeros((Y_test.shape[0], num_classes, num_classes))
c = 0
for i in range(num_classes - 1):
for j in range(i+1, num_classes):
logging.debug("=========== Testing %s vs %s", i, j)
Xa_test, Yb_test = one_vs_one_partition(X_test, Y_test, i, j)
classifier = classifier_list[c]
c += 1
# Record test prediction for later comparison
Yp[:, i, j] = classifier.predict(Xa_test)
Yp[:, j, i] = -Yp[:, i, j]
logging.info("============== DONE ALL ===============")
logging.debug("")
Yp = np.sum(Yp, axis=2)
Y_guess = np.argmax(Yp, axis=1)
c_matrix = util.confusion_matrix(Y_test, Y_guess, num_classes)
logging.info('Overall test acc: %f%%', util.get_accuracy(c_matrix))
logging.debug('%s', c_matrix)
logging.debug(".......")
return c_matrix
def classify_class(X, Y, X_val, Y_val, class_validation_helper):
logging.info("Running classifier")
best_classifier, best_acc_v, acc_list_v = \
class_validation_helper(X, Y, X_val, Y_val, info=None)
logging.debug("")
c_matrix = best_classifier.classify(X_val, Y_val)
logging.info('Validation acc: %f%%', util.get_accuracy(c_matrix))
logging.debug('%s', c_matrix)
logging.debug(".......")
return c_matrix, acc_list_v
def train_nn(X, Y, X_val, Y_val, num_classes, hidden_layer_sizes,
learning_rate, reg_constant, num_iterations, should_plot=False):
logging.info("Running NN training on %d classes -----", num_classes)
Y_spread = spread(Y, num_classes)
Y_val_spread = spread(Y_val, num_classes)
from ml_lib.n_net import NNet
nn = NNet(X, Y_spread, hidden_layer_sizes, learning_rate, reg_constant)
ITERS_PER_RUN = 25
J_e = [] # output error cost
J_r = [] # regularization cost
sat_list = [] # %age saturated
tr_acc_list = [] # training accuracy
v_acc_s = [] # validation accuracy
It = []
for train_loops in range(num_iterations // ITERS_PER_RUN):
nn.train(ITERS_PER_RUN)
(it, stat_J_e, stat_J_r, stat_perc_saturated, tr_acc) = nn.get_stats()
It.append(it)
J_e.append(stat_J_e)
J_r.append(stat_J_r)
sat_list.append(stat_perc_saturated)
tr_acc_list.append(tr_acc)
val_acc = util.get_accuracy(
nn.classify(X_val, Y_val_spread, should_report=False))
v_acc_s.append(val_acc)
logging.debug("Iteration %d. Cost=%0.2f \t Tr=%0.2f Val=%0.2f Sat=%0.2f",
it, stat_J_e + stat_J_r, tr_acc, val_acc, stat_perc_saturated)
if should_plot:
util.plot_all(It,
#[J_e, J_r],
#[J_e, sat_list, tr_acc_list, VAcc_s],
[tr_acc_list, v_acc_s],
"iteration", "",
#["Cost", "Saturation", "Training", "Validation"],
["Training", "Validation"],
nn.prefix())
return nn
def one_vs_one_multiclassify(X, Y, X_val, Y_val, num_classes,
c_helper):
logging.info("Running one vs one multiclassifier " + \
"on %d classes -----", num_classes)
Yp = np.zeros((Y_val.shape[0], num_classes, num_classes))
acc_list = []
classifier_list = []
for i in range(num_classes - 1):
for j in range(i+1, num_classes):
logging.debug("=========== Classifying %s vs %s", i, j)
Xa, Yb = one_vs_one_partition(X, Y, i, j)
Xa_val, Yb_val = one_vs_one_partition(X_val, Y_val, i, j)
classifier, acc_v, acc_list_v = c_helper(Xa, Yb,
Xa_val, Yb_val,
(i, j))
acc_list.append(acc_list_v)
classifier_list.append(classifier)
logging.debug("")
logging.info("=========== DONE classifying %s vs %s ============",
i, j)
logging.debug("")
cm_b = classifier.classify(Xa_val, Yb_val)
logging.debug("%s", cm_b)
# Record test prediction for later comparison
Yp[:, i, j] = classifier.predict(X_val)
Yp[:, j, i] = -Yp[:, i, j]
logging.info("============== DONE ALL ===============")
logging.debug("")
Yp = np.sum(Yp, axis=2)
Y_guess = np.argmax(Yp, axis=1)
c_matrix = util.confusion_matrix(Y_val, Y_guess, num_classes)
logging.info('Best validation acc: %f%%', util.get_accuracy(c_matrix))
logging.debug('%s', c_matrix)
logging.debug(".......")
return c_matrix, acc_list, classifier_list
def one_vs_all_multiclassify(X, Y, X_val, Y_val, num_classes,
c_helper):
logging.info("Running one vs all multiclassifier " + \
"validation on %d classes -----", num_classes)
Yp = np.zeros((Y_val.shape[0], num_classes))
acc_list = []
classifier_list = []
for i in range(num_classes):
Yb = np.zeros((Y.shape[0]))
Yb[Y==i] = 1
Yb_val = np.zeros((Y_val.shape[0]))
Yb_val[Y_val==i] = 1
classifier, acc_v, acc_list_v = c_helper(X, Yb, X_val, Yb_val, (i,))
acc_list.append(acc_list_v)
classifier_list.append(classifier)
logging.debug("")
logging.info("============== DONE class %d ===============", i)
logging.debug("")
cm_b = classifier.classify(X_val, Yb_val)
logging.debug("%s", cm_b)
# Record prediction for later one-vs-all comparison
Yp_i = classifier.predict(X_val)
Yp[:, i] = Yp_i
logging.info("============== DONE ALL ===============")
logging.debug("")
Yguess = np.argmax(Yp, axis=1)
c_matrix = util.confusion_matrix(Y_val, Yguess, num_classes)
logging.info('Best validation acc: %f%%', util.get_accuracy(c_matrix))
logging.debug('%s', c_matrix)
logging.debug(".......")
return c_matrix, acc_list, classifier_list
def classifier_and_accuracy(var1, var2):
classifier = create_classifier_validated(X, Y, var1, var2)
cm_v = classifier.classify(X_val, Y_val)
acc = util.get_accuracy(cm_v)
return (classifier, acc, acc)
def classifier_helper(X, Y, X_val, Y_val, create_classifier_validated):
classifier = create_classifier_validated(X, Y)
cm_v = classifier.classify(X_val, Y_val)
acc = util.get_accuracy(cm_v)
return (classifier, acc, acc)
def test_nn(nn, X_test, Y_test, num_classes):
logger.debug("Testing NN")
Y_test_spread = spread(Y_test, num_classes)
return util.get_accuracy(
nn.classify(X_test, Y_test_spread))
def main():
args = cmd_line.parse_args()
util.prefix_init(args)
util.pre_dataset = "wdbc"
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
log.configure_logger(logger, util.pre_dataset)
logger.info("--- WDBC dataset ---")
data = np.genfromtxt(args.file, delimiter=",",
converters={1: lambda x: 1.0 if x=='M' else 0.0})
Y = data[:, 1].astype(int)
X = data[:, features_to_use()]
X, Y, X_valid, Y_valid, X_test, Y_test = \
data_util.split_into_train_test_sets(X, Y, args.validation_portion,
args.test_portion)
logger.debug("%s %s", X.shape, X_test.shape)
if args.normalize:
logger.info("Normalizing...")
util.pre_norm = "n"
X, X_valid, X_test = data_util.normalize_all(X, X_valid, X_test)
if args.draw_classes_histogram:
draw_classes_histogram(X, Y)
if args.draw_classes_data:
util.draw_classes_data(X, Y, 5, 6)
if args.bayes:
logger.info("Bayes classifier...")
util.pre_alg = "bayes"
from ml_lib.gaussian_plugin_classifier import GaussianPlugInClassifier
# Gaussian plug-in classifier
gpi_classifier = GaussianPlugInClassifier(X, Y, 2)
# util.report_accuracy(gpi_classifier.classify(X, Y, 0.5)[0])
util.report_accuracy(
gpi_classifier.classify(X_test, Y_test, [0.5, 0.5])[0])
util.draw_ROC_curve(X_test, Y_test, gpi_classifier)
# util.draw_classes_pdf(X, Y, gpi_classifier, [0.5, 0.5], 3)
if args.naive:
logger.info("Naive Bayes classifier...")
util.pre_alg = "naive"
from ml_lib.gaussian_naive_classifier import GaussianNaiveClassifier
# Gaussian naive classifier
gn_classifier = GaussianNaiveClassifier(X, Y, 2)
# util.report_accuracy(gn_classifier.classify(X, Y, 0.5)[0])
util.report_accuracy(
gn_classifier.classify(X_test, Y_test, [0.5, 0.5])[0])
util.draw_ROC_curve(X_test, Y_test, gn_classifier)
if args.sklearn_perceptron:
logger.info("Scikit-learn Perceptron...")
util.pre_alg = "scikitperceptron"
from sklearn.linear_model import Perceptron
perceptron = Perceptron(tol=None, max_iter=300000)
perceptron.fit(X, Y)
logger.info("Mean accuracy: %s%%", 100 * perceptron.score(X, Y))
if args.perceptron:
logger.info("Perceptron...")
util.pre_alg = "perceptron"
from ml_lib.perceptron import Perceptron
helper.classify_one_vs_one([],
X, Y, X_test, Y_test, 2,
lambda X, Y: Perceptron(X, Y, args.stochastic, 1, 30000, 0))
if args.logistic:
logger.info("Logistic Regression...")
util.pre_alg = "logistic"
from ml_lib.logistic import Logistic
helper.classify_one_vs_one([],
X, Y, X_test, Y_test, 2,
lambda X, Y: Logistic(X, Y, step_size=0.001, max_steps=15000,
reg_constant=1))
if args.knn:
logger.info("k-Nearest Neighbor...")
util.pre_alg = "knn"
from ml_lib.knn import KNN
k_range = 10
p_range = 6 # / 2.0
a_matrix = np.zeros((k_range, p_range))
for k in range(k_range):
logger.info("%s-NN", k+1)
for p in range(p_range):
knn_classifier = KNN(X, Y, 1+k, dist_p=(p+1)/2.0)
a_matrix[k, p] = util.get_accuracy(
knn_classifier.classify(X_test, Y_test))
logger.info("%s", a_matrix)
if args.svm:
logger.info("Support Vector Machine...")
util.pre_alg = "svm"
from ml_lib.svm import SVM, RBFKernel
single_svm_test = False
if single_svm_test:
cm = SVM(X, Y, lam=None).classify(X_test, Y_test)
util.report_accuracy(cm)
single_svm_rbf_test = False
if single_svm_rbf_test:
svm = SVM(X, Y, lam=100, kernel=RBFKernel(0.3))
cm = svm.classify(X_test, Y_test)
util.report_accuracy(cm)
linear_svm_validation = False
if linear_svm_validation:
#lam_val = [math.pow(1.2, p) for p in range(-10,20)]
lam_val = [p/2 for p in range(1,200)]
acc = np.zeros(len(lam_val))
for i, lam in enumerate(lam_val):
svm_classifier = SVM(X, Y, lam)#), kernel=RBFKernel(1))
#util.report_accuracy(svm_classifier.classify(X, Y))
cm = svm_classifier.classify(X_valid, Y_valid)
util.report_accuracy(cm)
acc[i] = util.get_accuracy(cm)
logger.info("\nAccuracies found for lambda:")
for i, lam in enumerate(lam_val):
logger.info("%f: \t%f", lam, acc[i])
util.plot_accuracy(acc, lam_val)
rbf_svm_validation = False
if rbf_svm_validation:
for reps in range(2):
pre_svm_cv_x = "b" if reps == 0 else "l"
if pre_svm_cv_x == "b":
lam_val = [math.pow(1.5, p+1)*10 for p in range(7)]
b_val = [(p+1)/20 for p in range(27)]
elif pre_svm_cv_x == "l":
lam_val = [math.pow(1.2, p+1)*10 for p in range(27)]
b_val = [(p+1)/10 for p in range(7)]
logger.debug(lam_val)
logger.debug(b_val)
# Use a single instance so K matrix can be shared better
single_svm = SVM(X)
lmbd_classifier = lambda X, Y, b, lam, svm=single_svm: \
svm.initialize(Y, lam, RBFKernel(b))
cm, acc_2d_list = helper.classify([b_val, lam_val], X, Y,
X_valid, Y_valid,
lmbd_classifier)
acc_matrix = np.array(acc_2d_list)
logger.info("%s", acc_matrix)
suff = "val_%s"%(pre_svm_cv_x)
np.savetxt(util.prefix() + suff + ".csv",
acc_matrix, delimiter=",", fmt='%.3f')
if pre_svm_cv_x == 'b':
util.plot_accuracies(acc_matrix.T, b_val, "RBF width b",
lam_val, "Lambda (C)", suff)
elif pre_svm_cv_x == 'l':
util.plot_accuracies(acc_matrix, lam_val, "Lambda (C)",
b_val, "RBF width b", suff)
def train(self, num_iterations):
''' Train the neural network for the given number of iterations
'''
L = len(self.layer_sizes)
X = self.X
Y = self.Y
m = X.shape[0] # number of data points
Ts = self.thetas
for iter in range(num_iterations):
self.iteration += 1
A_s = []
# Forward propogation
A = X.T
for i in range(L - 1):
A_ = np.append(np.ones((1, A.shape[1])), A, axis=0)
A_s.append(A_)
Z = np.dot(Ts[i], A_)
A = self.sigmoid(Z)
if iter == num_iterations - 1:
# Cost
error_cost = -np.sum(Y.T * np.log(A) + (1 - Y.T) *
(np.log(1 - A))) / m
reg_cost = 0
for i in range(L - 1): # regularization
reg_cost += np.sum(Ts[i] * Ts[i]) * self.lam / (2 * m)
#cost = error_cost + reg_cost
# monitor activation saturation
num_saturated = 0
num_activations = 0
for A_ in A_s:
#sum_activations += np.sum(np.absolute(A))
num_saturated += np.sum(
np.logical_or(A_ > 0.999, A_ < 0.001))
num_activations += A_.shape[1] * A_.shape[0]
perc_saturated = 100 * num_saturated / num_activations
tr_accuracy = util.get_accuracy(
self.classify(X, Y, should_report=False))
# Back propagation
# Precompute delta D for last (output) layer
D = A - Y.T # initialize to error cost of output
for i in reversed(range(L - 1)):
# Compute cost gradient for theta = D*A matrix (averaged over
# all training examples)
Tcost_grad = np.dot(D, A_s[i].T)
# Regularization
T_reg = np.copy(Ts[i])
T_reg[:, 0] = 0
Tcost_grad += self.lam * T_reg
Tcost_grad /= m
# Precompute delta D for next loop iteration (earlier layer)
D = np.dot(Ts[i].T, D) * A_s[i] * (1 - A_s[i])
D = D[1:, :] # Because bias value (=1) is not to be modified
# Update theta for the current layer
Ts[i] -= Tcost_grad * self.learning_rate
# TODO: gradient checking
self.stat_J_e = error_cost
self.stat_J_r = reg_cost
self.stat_perc_saturated = perc_saturated
self.stat_acc = tr_accuracy