t, p = stats.ttest_ind(NaiveBayes, Logistic, equal_var=False) print('t-value is: %f, p-vaule is: %f' % (t, p)) if p < alpha: print("Reject h0,NaiveBayes and Logistic are different!") else: print("Fail to reject h0") if __name__ == '__main__': trainsize = 90000 testsize = 10000 numruns = 10 classalgs = { #'Linear Regression': algs.LinearRegressionClass(), 'Naive Bayes': algs.NaiveBayes({'usecolumnones': False}), 'Logistic Regression': algs.LogitReg(), 'Neural Network': algs.NeuralNet({'epochs': 100}), #'KernelLogitReg': algs.KernelLogitReg({'kernel': 'linear'}) } numalgs = len(classalgs) parameters = ( # best parameters for now { 'regwgt': 0.1, 'nh': 12, 'eta': 0.0001 }, # {'regwgt': 0.1, 'nh': 2,'eta':0.01}, # {'regwgt': 0.01, 'nh': 4,'eta':0.001},
learnername + ': ' + str(STDeviation(std[learnername], errors[learnername]))) learner = min(errors, key=errors.get) best_algorithm = learner return best_algorithm if __name__ == '__main__': trainsize = 5000 testsize = 5000 numruns = 3 classalgs = { #'Random': algs.Classifier(), 'Naive Bayes': algs.NaiveBayes({'usecolumnones': False}), 'Naive Bayes Ones': algs.NaiveBayes({'usecolumnones': True}), 'Linear Regression': algs.LinearRegressionClass(), 'Logistic Regression Reg': algs.LogitReg({ 'regularizer': 'l2', 'lamb': 0.001, 'stepsize': 0.001 }), 'Logistic Regression': algs.LogitReg({ 'lamb': 0.001, 'stepsize': 0.001 }),
trainset, testset = loadsusy() """The choice of the number of folds should be user-input""" fold=10 trainlabel=np.reshape(trainset[1],(-1,1)) trset = np.hstack((trainset[0],trainlabel)) numinputs = trset.shape[1]-1 np.random.shuffle(trset) parts = [trset[i::fold] for i in xrange(fold)] obj=[] print('Running on train={0} and test={1} samples').format(trainset[0].shape[0], testset[0].shape[0]) parm_pass={'Neural Network':{'ni': trset.shape[1]-1, 'nh': 0, 'no': 1}, 'Logistic Regression':{'regwt':0,'type':"L2"}} classalgs = {'Linear Regression': algs.LinearRegressionClass(), 'Naive Bayes Ones': algs.NaiveBayes(), 'Logistic Regression': algs.LogitReg(parm_pass['Logistic Regression']), 'Neural Network': algs.NeuralNet(parm_pass['Neural Network']) } classalgs1 = collections.OrderedDict(sorted(classalgs.items())) best_parm=[] for learnername , learner in classalgs1.iteritems(): print 'Running learner = ' + learnername # # Train model parm_accuracy={}