############################################################################ # load classifiers, set thresholds, then sigmoid clfs = [] for clfnametup in csortme: clffile = clfnametup[1] assert clffile.endswith('_LDAresult.hdf5') clf = QuadraticDiscriminantAnalysis(store_covariances=False) clf.classes_ = np.array([0, 1], dtype=np.int64) h5file = tables.open_file(clffile, mode='r') clf.means_ = np.array(h5file.root.means[:]) clf.priors_ = np.array(h5file.root.priors[:]) clf.rotations_ = np.array(h5file.root.rotations[:]) clf.scalings_ = np.array(h5file.root.scalings[:]) h5file.close() assert clf.priors_.shape == (2, ), str(clf.priors_) assert np.sum(np.fabs(clf.priors_ - np.array([0.5, 0.5]))) < 1e-9, str( clf.priors_) clf.priors_ = np.array([1. - fraction_manip, fraction_manip]) clfs.append(clf) #clfthreshfile = clffile[:-len('result.hdf5')]+'highestROC.pkl' #highestroc = cPickle.load(open(clfthreshfile,'rb')) #distsfromdiag = (highestroc[3] - highestroc[2]) / np.sqrt(2.) #maxfromdiag_idx = np.argmax(distsfromdiag)
afttime = time.time() print("AFTER: tr_X.shape " + str(tr_X.shape) + ", dtype " + str(tr_X.dtype)) print("fitting took " + str(afttime - beftime) + " sec") fraction_manip = 0.50 # delete training set, since it is no longer needed del tr_X del tr_Y gc.collect() clf.means_ = np.array(clf.means_) clf.priors_ = np.array([1. - fraction_manip, fraction_manip]) # clf.rotations_ = np.array(clf.rotations_) clf.scalings_ = np.array(clf.scalings_) print("clf.priors_ == " + str(clf.priors_)) # re-load validation set tr_X, tr_Y = load_hdf5_train_set(validfilename + '.hdf5', fraction_manip=fraction_manip) describe("VALIDATION: BEFORE MEAN SUBTR: tr_X", tr_X) #tr_X -= tr_mean_vec #tr_X /= tr_scale_vec #describe("VALIDATION: AFTER MEAN SUBTR: tr_X", tr_X) h5file = tables.open_file(scoresfname + '_LDAresult.hdf5', mode='w')