def main_with_validation(fileName, intercept=False, evalFunc = Util.brierScore): trainX, trainY, testX, testY = Util.readData(fileName, intercept=intercept) clf = GLS() regs = np.logspace(-3, 3, 7) xis = np.linspace(-1., 1.5, 26) bestScore, bestXi, bestReg = 1e10, None, None for xi in xis: clf.setXi(xi) print("Current xi = ", xi) score, reg = Util.crossValidate(clf, trainX, trainY, \ evalFunc, 5, "Regular", regs) if score < bestScore: bestScore, bestXi, bestReg = score, xi, reg print("bestScore, bestXi, bestReg = ", bestScore, bestXi, bestReg) clf.setXi(bestXi) clf.setRegular(bestReg) clf.fit(trainX, trainY) testScore = evalFunc(clf.predict(testX), testY) with open("log/GLS_final_log.txt", 'a') as f: log = ','.join([dt.now().strftime("%Y/%m/%d %H:%M"), str(fileName), \ str(bestReg), str(bestXi), str(intercept), \ evalFunc.__name__, str(bestScore), str(testScore)]) f.write(log + '\n')
def main_with_validation(fileName, preproc=False, evalFunc = Util.brierScore): validateX, validateY, \ trainX, trainY, \ testX, testY = Util.readData(fileName, True, preproc) clf = GLS() xis = np.linspace(-1., 1.5, 26) bestScore, bestXi = Util.crossValidate(clf, validateX, validateY, \ evalFunc, 5, "Xi", xis) print("bestScore, bestXi = ", bestScore, bestXi) with open("log/GLS_validation_log.txt", 'a') as f: log = ','.join([dt.now().strftime("%Y/%m/%d %H:%M"), str(fileName), \ str(bestXi), str(preproc), evalFunc.__name__]) f.write(log + '\n')
def worker(chosenNum): print("Current chosen number: ", chosenNum) # global validateX, validateY, trainX, trainY, regs, xis, lock, clfs clf = GLS() hda = HighDimAlg(clf, "convolve") hda.setWidth(28) hda.setKernels([(2,2), (4,4), (7,7)]) target = validateY.copy() target[target == chosenNum] = 999 target[target != 999] = 0 target[target == 999] = 1 bestScore, bestXi, bestReg = 1e10, None, None for xi in xis: hda.setXi(xi) score, reg = Util.crossValidate(hda, validateX, target, \ Util.brierScore, 2, "Regular", regs) if score < bestScore: bestScore, bestXi, bestReg = score, xi, reg print("bestScore, bestXi, bestReg = ", bestScore, bestXi, bestReg) hda.setXi(bestXi) hda.setRegular(bestReg) target = trainY.copy() target[target == chosenNum] = 999 target[target != 999] = 0 target[target == 999] = 1 hda.fit(trainX, target) res = {} res['num'] = chosenNum res['clf'] = hda res['validateScore'] = bestScore lock.acquire() try: clfs.append(res) finally: lock.release()
scores = np.zeros(10 * 10).reshape(10, 10) for i in range(10): trainX, trainY, testX, testY = Util.readData(source=os.path.join(path, f)) est = MAPGEV.MAPGEV(iterations=20) est.fit(trainX, trainY) beta, xi = est.beta.reshape(-1, 1), est.xi clf = GLS.GLS() # clf.manualInitial = True if xi < -1: clf.setXi(-1) elif xi > 50: clf.setXi(50) else: clf.setXi(xi) print("xi = %f clf.xi = %f" % (xi, clf.xi)) bestScore, bestReg = Util.crossValidate(clf, trainX, trainY, Util.brierScore, 5, "Regular", regs) # bestScore, bestReg = 1., 0. # for reg in regs: # clf.setRegular(reg) # clf.setBeta(beta) # clf.fit(trainX, trainY) # score = Util.brierScore(clf.predict(testX), testY) # if score < bestScore: # bestScore = score # bestReg = reg print("bestScore = %f bestReg = %f" % (bestScore, bestReg)) clf.setRegular(bestReg) clf.setBeta(beta) clf.manualInitial = True clf.fit(trainX, trainY) scores[i] = Util.evaluate(clf.predict(testX), testY)