def _test_size(self): ''' Compare several competing methods changing the ratio of the positive class in the dataset. We use binary class dataset for the easy of interpretation. ''' for set_size in numpy.arange(100, 1000, 100): X_train_full, y_train_full, X_test, y_test = nc_rna_reader.toNumpy() X_train, y_train= self.get_sub_set_with_size([X_train_full, y_train_full], set_size) assert(len(y_train) == set_size) train_set = (X_train, y_train) test_set_original = (X_test, y_test) ms = MS2(LogisticRegression) ms.fit(train_set) r = 0.05 X_test_new, y_test_new = SetGen.with_pos_ratio(test_set_original, r, pos_label=1) test_set = [X_test_new, y_test_new] dist_true = DE.arrayToDist(y_test_new) dist_est = ms.predict(X_test_new) err = rms(dist_est, dist_true) print dist_est print "size: %d, err: %f" % (set_size, err)
def test_class_ratio(self): ''' Compare several competing methods changing the ratio of the positive class in the dataset. We use binary class dataset for the easy of interpretation. ''' X_train_full, y_train_full, X_test, y_test = nc_rna_reader.toNumpy() set_size = 400 # an arbitrary number X_train, y_train= self.get_sub_set_with_size( [X_train_full, y_train_full], set_size) assert(len(y_train) == set_size) train_set = (X_train, y_train) test_set_original = (X_test, y_test) cc = CC2(LogisticRegression) ac = AC2(LogisticRegression) ms = MS2(LogisticRegression) en = EN2(LogisticRegression) ests = [cc, ac, ms, en] print "We compare the performance as changing the positive class ratio." print "The training set size is %d" % set_size print "Training classifiers" map(lambda e: e.fit(train_set), ests) print "ratio\tcc\tac\tms\ten" for r in numpy.arange(0.05, 1.0, 0.05): X_test_new, y_test_new = SetGen.with_pos_ratio(test_set_original, r, pos_label=1) test_set = [X_test_new, y_test_new] errs = map(lambda e: self.run_for_estimator(e, test_set), ests) print ("%.2f" + "\t%.4f" * 4) % (r, errs[0], errs[1], errs[2], errs[3])
def _test_debug(self): X_train_full, y_train_full, X_test, y_test = nc_rna_reader.toNumpy() set_size = 300 # an arbitrary number X_train, y_train = self.get_sub_set_with_size([X_train_full, y_train_full], set_size) assert len(y_train) == set_size train_set = (X_train, y_train) test_set_original = (X_test, y_test) X_test_new, y_test_new = SetGen.with_pos_ratio(test_set_original, 0.05, pos_label=1) test_set = [X_test_new, y_test_new] cc = CC(LogisticRegression) en = EN(LogisticRegression, debug=True) err = self.run_for_estimator(en, train_set, test_set, debug=True) print "err", err
def _test_avg(self): dataset = nc_rna_reader.toNumpy() train_set_size = 200 X_train_full, y_train_full, X_test_full, y_test_full = dataset X_train, y_train = self.get_sub_set_with_size([X_train_full, y_train_full], train_set_size) X_test, y_test = self.get_sub_set_with_size([X_test_full, y_test_full], 10000) train_set = (X_train, y_train) test_set_original = (X_test, y_test) clf_class = LinearSVC for split_r in numpy.arange(0.1, 1.0, 0.1): ra = RA(clf_class, ac_method="ac", subsample_count=200, split_r=split_r) ra.fit(train_set) err = self.compute_avg_error(ra, test_set_original) print "%f\t%f" % (split_r, err)
def _test_rna_change_training_size(self): ''' Do compare using ncRNA dataset. The dataset is from Andrew V Uzilov, Joshua M Keegan, and David H Mathews. Detection of non-coding RNAs on the basis of predicted secondary structure formation free energy change. BMC Bioinformatics, 7(173), 2006. ''' X_train_full, y_train_full, X_test, y_test = nc_rna_reader.toNumpy() test_set_original = [X_test, y_test] pos_ratio = 0.8 # arbtrary ratio X_test_new, y_test_new = SetGen.with_pos_ratio(test_set_original, pos_ratio, pos_label=1) test_set = [X_test_new, y_test_new] print "We compare performance as chaning the training set size." print "Positive class ratio is %f" % pos_ratio print "ratio\tcc\tac\tms\ten" #for set_size in [600, 300, 400, 500, 700, 800, 900, 1000]: for set_size in [800, 900, 1000, 1500, 2000, 2500, 3000]: #for set_size in numpy.arange(1500, 5000, 500): #for set_size in [500, 600, 700, 800, 900, 1000]: cc = CC2(LogisticRegression) ac = AC2(LogisticRegression) ms = MS2(LogisticRegression) en = EN2(LogisticRegression) ests = [cc, ac, ms, en] X_train_sub, y_train_sub = self.get_sub_set_with_size( [X_train_full, y_train_full], set_size) train_set = [X_train_sub, y_train_sub] map(lambda e: e.fit(train_set), ests) errs = map(lambda e: self.run_for_estimator(e, test_set), ests) print ("%d" + "\t%.4f" * 4) % (set_size, errs[0], errs[1], errs[2], errs[3])
def run_training_size(self, pos_ratio): """ Do compare using ncRNA dataset. The dataset is from Andrew V Uzilov, Joshua M Keegan, and David H Mathews. Detection of non-coding RNAs on the basis of predicted secondary structure formation free energy change. BMC Bioinformatics, 7(173), 2006. """ X_train_full, y_train_full, X_test, y_test = nc_rna_reader.toNumpy() test_set_original = [X_test, y_test] X_test_new, y_test_new = SetGen.with_pos_ratio(test_set_original, pos_ratio, pos_label=1) test_set = [X_test_new, y_test_new] print "RNA dataset" print "We compare performance as chaning the training set size." print "Positive class ratio is %f" % pos_ratio print "size\tcc\tac\tms\tra\trc\trb\trd" for set_size in numpy.arange(100, 1100, 100).tolist() + [2000, 3000, 4000, 5000, 10000, 20000]: cc = CC2(LogisticRegression) ac = AC2(LogisticRegression) ms = MS2(LogisticRegression) ra = RA(LogisticRegression, ac_method="ac") rc = RA(LogisticRegression, ac_method="cac") rb = RA(LogisticRegression, ac_method="bac") rd = RA(LogisticRegression, ac_method="dac") ests = [cc, ac, ms, ra, rc, rb, rd] X_train_sub, y_train_sub = self.get_sub_set_with_size([X_train_full, y_train_full], set_size) train_set = [X_train_sub, y_train_sub] map(lambda e: e.fit(train_set), ests) errs = map(lambda e: self.run_for_estimator(e, test_set), ests) print ("%d" + "\t%.4f" * 7) % (set_size, errs[0], errs[1], errs[2], errs[3], errs[4], errs[5], errs[6])
def test_ratio(self): dataset = nc_rna_reader.toNumpy() for set_size in numpy.arange(100, 1100, 100): self.run_ratio(dataset, set_size) print