def glm_alpha_array_lambda_null(): # first test: compare coefficients and deviance d = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/prostate.csv")) mL = glm(family='binomial',alpha=[0.1,0.5,0.9],solver='COORDINATE_DESCENT') mL.train(training_frame=d,x=[2,3,4,5,6,7,8],y=1) r = glm.getGLMRegularizationPath(mL) regKeys = ["alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train"] best_submodel_index = mL._model_json["output"]["best_submodel_index"] m2 = glm.makeGLMModel(model=mL,coefs=r['coefficients'][best_submodel_index]) dev1 = r['explained_deviance_train'][best_submodel_index] p2 = m2.model_performance(d) dev2 = 1-p2.residual_deviance()/p2.null_deviance() print(dev1," =?= ",dev2) assert abs(dev1 - dev2) < 1e-6 for l in range(0,len(r['lambdas'])): m = glm(family='binomial',alpha=[r['alphas'][l]],Lambda=[r['lambdas'][l]],solver='COORDINATE_DESCENT') m.train(training_frame=d,x=[2,3,4,5,6,7,8],y=1) mr = glm.getGLMRegularizationPath(m) cs = r['coefficients'][l] cs_norm = r['coefficients_std'][l] pyunit_utils.assertEqualCoeffDicts(cs, m.coef()) pyunit_utils.assertEqualCoeffDicts(cs_norm, m.coef_norm()) p = m.model_performance(d) devm = 1-p.residual_deviance()/p.null_deviance() devn = r['explained_deviance_train'][l] assert abs(devm - devn) < 1e-4 pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr) if (l == best_submodel_index): # check training metrics, should equal for best submodel index pyunit_utils.assertEqualModelMetrics(m._model_json["output"]["training_metrics"], mL._model_json["output"]["training_metrics"]) else: # for other submodel, should have worse residual_deviance() than best submodel assert p.residual_deviance() >= p2.residual_deviance(), "Best submodel does not have lowerest " \ "residual_deviance()!"
def glm_alpha_array_lambda_null(): # first test: compare coefficients and deviance keySets = ["MSE", "null_deviance", "logloss", "RMSE", "r2"] d = h2o.import_file( path=pyunit_utils.locate("smalldata/covtype/covtype.20k.data")) mL = glm(family='multinomial', alpha=[0.1, 0.5, 0.9], Lambda=[0.1, 0.5, 0.9], cold_start=True) d[54] = d[54].asfactor() mL.train(training_frame=d, x=list(range(0, 54)), y=54) r = glm.getGLMRegularizationPath(mL) regKeys = [ "alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train" ] best_submodel_index = mL._model_json["output"]["best_submodel_index"] coefClassSet = [ 'coefs_class_0', 'coefs_class_1', 'coefs_class_2', 'coefs_class_3', 'coefs_class_4', 'coefs_class_5', 'coefs_class_6', 'coefs_class_7' ] coefClassSetNorm = [ 'std_coefs_class_0', 'std_coefs_class_1', 'std_coefs_class_2', 'std_coefs_class_3', 'std_coefs_class_4', 'std_coefs_class_5', 'std_coefs_class_6', 'std_coefs_class_7' ] groupedClass = d.group_by("C55") groupedClass.count() classFrame = groupedClass.get_frame() classProb = classFrame[1] / d.nrow coeffIndex = [52, 105, 158, 211, 264, 317, 370] startVal = [0] * 371 for ind in range(classProb.nrow): startVal[coeffIndex[ind]] = math.log(classProb[ind, 0]) for l in range(0, len(r['lambdas'])): m = glm(family='multinomial', alpha=[r['alphas'][l]], Lambda=[r['lambdas'][l]], startval=startVal) m.train(training_frame=d, x=list(range(0, 54)), y=54) mr = glm.getGLMRegularizationPath(m) cs = r['coefficients'][l] cs_norm = r['coefficients_std'][l] pyunit_utils.assertCoefEqual(cs, m.coef(), coefClassSet) pyunit_utils.assertCoefEqual(cs_norm, m.coef_norm(), coefClassSetNorm) devm = 1 - m.residual_deviance() / m.null_deviance() devn = r['explained_deviance_train'][l] assert abs(devm - devn) < 1e-4 pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr) if (l == best_submodel_index ): # check training metrics, should equal for best submodel index pyunit_utils.assertEqualModelMetrics( m._model_json["output"]["training_metrics"], mL._model_json["output"]["training_metrics"], tol=1e-2, keySet=keySets) else: # for other submodel, should have worse residual_deviance() than best submodel assert m.logloss() >= mL.logloss(), "Best submodel does not have lowerest " \ "logloss()!"
def glm_alpha_lambda_arrays(): # compare coefficients and deviance when only training dataset is available train = h2o.import_file(path=pyunit_utils.locate( "smalldata/glm_test/binomial_20_cols_10KRows.csv")) for ind in range(10): train[ind] = train[ind].asfactor() train["C21"] = train["C21"].asfactor() frames = train.split_frame(ratios=[0.8], seed=12345) d = frames[0] d_test = frames[1] regKeys = [ "alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train" ] # compare results when validation dataset is present mLVal = glm(family='binomial', alpha=[0.1, 0.5], lambda_search=True, solver='COORDINATE_DESCENT', nlambdas=3) # train with validations set mLVal.train(training_frame=d, x=list(range(20)), y=20, validation_frame=d_test) rVal = glm.getGLMRegularizationPath(mLVal) best_submodel_indexVal = mLVal._model_json["output"]["best_submodel_index"] m2Val = glm.makeGLMModel( model=mLVal, coefs=rVal['coefficients'][best_submodel_indexVal]) dev1Val = rVal['explained_deviance_valid'][best_submodel_indexVal] p2Val = m2Val.model_performance(d_test) dev2Val = 1 - p2Val.residual_deviance() / p2Val.null_deviance() assert abs(dev1Val - dev2Val) < 1e-6 for l in range(0, len(rVal['lambdas'])): m = glm(family='binomial', alpha=[rVal['alphas'][l]], Lambda=rVal['lambdas'][l], solver='COORDINATE_DESCENT') m.train(training_frame=d, x=list(range(20)), y=20, validation_frame=d_test) mr = glm.getGLMRegularizationPath(m) p = m.model_performance(d_test) cs = rVal['coefficients'][l] cs_norm = rVal['coefficients_std'][l] print("Comparing submodel index {0}".format(l)) pyunit_utils.assertEqualCoeffDicts(cs, m.coef(), tol=1e-1) pyunit_utils.assertEqualCoeffDicts(cs_norm, m.coef_norm(), tol=1e-1) pyunit_utils.assertEqualRegPaths(regKeys, rVal, l, mr, tol=1e-3) dVal = 1 - p.residual_deviance() / p.null_deviance() if l == best_submodel_indexVal: # check training metrics, should equal for best submodel index pyunit_utils.assertEqualModelMetrics( m._model_json["output"]["validation_metrics"], mLVal._model_json["output"]["validation_metrics"], tol=1e-2) else: # for other submodel, should have worse residual_deviance() than best submodel assert dVal <= dev2Val, "Best submodel does not have highest explained deviance_valid for submodel: !".format( l)
def glm_alpha_array_lambda_null(): # first test: compare coefficients and deviance d = h2o.import_file( path=pyunit_utils.locate("smalldata/covtype/covtype.20k.data")) mL = glm(family='multinomial', alpha=[0.1, 0.5, 0.9], lambda_search=True, solver='COORDINATE_DESCENT', cold_start=True, nlambdas=5) d[54] = d[54].asfactor() mL.train(training_frame=d, x=list(range(0, 54)), y=54) r = glm.getGLMRegularizationPath(mL) regKeys = [ "alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train" ] best_submodel_index = mL._model_json["output"]["best_submodel_index"] coefClassSet = [ 'coefs_class_0', 'coefs_class_1', 'coefs_class_2', 'coefs_class_3', 'coefs_class_4', 'coefs_class_5', 'coefs_class_6', 'coefs_class_7' ] coefClassSetNorm = [ 'std_coefs_class_0', 'std_coefs_class_1', 'std_coefs_class_2', 'std_coefs_class_3', 'std_coefs_class_4', 'std_coefs_class_5', 'std_coefs_class_6', 'std_coefs_class_7' ] for l in range(0, len(r['lambdas'])): print("compare models for index {0}, alpha {1}, lambda{2}".format( l, r['alphas'][l], r['lambdas'][l])) m = glm(family='multinomial', alpha=[r['alphas'][l]], Lambda=[r['lambdas'][l]], solver='COORDINATE_DESCENT') m.train(training_frame=d, x=list(range(0, 54)), y=54) mr = glm.getGLMRegularizationPath(m) cs = r['coefficients'][l] cs_norm = r['coefficients_std'][l] pyunit_utils.assertCoefEqual(cs, m.coef(), coefClassSet) pyunit_utils.assertCoefEqual(cs_norm, m.coef_norm(), coefClassSetNorm) devm = 1 - m.residual_deviance() / m.null_deviance() devn = r['explained_deviance_train'][l] assert abs(devm - devn) < 1e-6 pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr) if (l == best_submodel_index ): # check training metrics, should equal for best submodel index pyunit_utils.assertEqualModelMetrics( m._model_json["output"]["training_metrics"], mL._model_json["output"]["training_metrics"], keySet=["MSE", "null_deviance", "logloss", "RMSE", "r2"], tol=5e-1) else: # for other submodel, should have worse residual_deviance() than best submodel assert devm <= r['explained_deviance_train'][best_submodel_index], "Best submodel does not best " \ "explained_deviance_train!"
def glm_alpha_lambda_arrays(): # read in the dataset and construct training set (and validation set) d = h2o.import_file( path=pyunit_utils.locate("smalldata/logreg/prostate.csv")) mL = glm(family='binomial', Lambda=[0.9, 0.5, 0.1], alpha=[0.1, 0.5, 0.9], solver='COORDINATE_DESCENT') mL.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1) r = glm.getGLMRegularizationPath(mL) regKeys = [ "alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train" ] best_submodel_index = mL._model_json["output"]["best_submodel_index"] m2 = glm.makeGLMModel(model=mL, coefs=r['coefficients'][best_submodel_index]) dev1 = r['explained_deviance_train'][best_submodel_index] p2 = m2.model_performance(d) dev2 = 1 - p2.residual_deviance() / p2.null_deviance() assert abs(dev1 - dev2) < 1e-6 for l in range(0, len(r['lambdas'])): m = glm(family='binomial', alpha=[r['alphas'][l]], Lambda=[r['lambdas'][l]], solver='COORDINATE_DESCENT') m.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1) mr = glm.getGLMRegularizationPath(m) cs = r['coefficients'][l] cs_norm = r['coefficients_std'][l] diff = 0 diff2 = 0 for n in cs.keys(): diff = max(diff, abs((cs[n] - m.coef()[n]))) diff2 = max(diff2, abs((cs_norm[n] - m.coef_norm()[n]))) assert diff < 1e-2 assert diff2 < 1e-2 p = m.model_performance(d) devm = 1 - p.residual_deviance() / p.null_deviance() devn = r['explained_deviance_train'][l] assert abs(devm - devn) < 1e-4 pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr, tol=1e-5) if (l == best_submodel_index ): # check training metrics, should equal for best submodel index pyunit_utils.assertEqualModelMetrics( m._model_json["output"]["training_metrics"], mL._model_json["output"]["training_metrics"], tol=1e-5) else: # for other submodel, should have worse residual_deviance() than best submodel assert p.residual_deviance() >= p2.residual_deviance(), "Best submodel does not have lowerest " \ "residual_deviance()!"
def glm_alpha_array_lambda_null(): # first test: compare coefficients and deviance d = h2o.import_file( path=pyunit_utils.locate("smalldata/covtype/covtype.20k.data")) mL = glm(family='multinomial', alpha=[0.1, 0.5, 0.9]) d[54] = d[54].asfactor() mL.train(training_frame=d, x=list(range(0, 54)), y=54) r = glm.getGLMRegularizationPath(mL) regKeys = [ "alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train" ] best_submodel_index = mL._model_json["output"]["best_submodel_index"] coefClassSet = [ 'coefs_class_0', 'coefs_class_1', 'coefs_class_2', 'coefs_class_3', 'coefs_class_4', 'coefs_class_5', 'coefs_class_6', 'coefs_class_7' ] coefClassSetNorm = [ 'std_coefs_class_0', 'std_coefs_class_1', 'std_coefs_class_2', 'std_coefs_class_3', 'std_coefs_class_4', 'std_coefs_class_5', 'std_coefs_class_6', 'std_coefs_class_7' ] for l in range(0, len(r['lambdas'])): m = glm(family='multinomial', alpha=[r['alphas'][l]], Lambda=[r['lambdas'][l]]) m.train(training_frame=d, x=list(range(0, 54)), y=54) mr = glm.getGLMRegularizationPath(m) cs = r['coefficients'][l] cs_norm = r['coefficients_std'][l] pyunit_utils.assertCoefEqual(cs, m.coef(), coefClassSet, tol=1e-5) pyunit_utils.assertCoefEqual(cs_norm, m.coef_norm(), coefClassSetNorm, tol=1e-5) devm = 1 - m.residual_deviance() / m.null_deviance() devn = r['explained_deviance_train'][l] assert abs(devm - devn) < 1e-4 pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr) if (l == best_submodel_index ): # check training metrics, should equal for best submodel index pyunit_utils.assertEqualModelMetrics( m._model_json["output"]["training_metrics"], mL._model_json["output"]["training_metrics"], tol=1e-2) else: # for other submodel, should have worse residual_deviance() than best submodel assert m.logloss() >= mL.logloss(), "Best submodel does not have lowerest " \ "logloss()!"
def glm_alpha_lambda_arrays(): # read in the dataset and construct training set (and validation set) d = h2o.import_file( path=pyunit_utils.locate("smalldata/logreg/prostate.csv")) mL = glm(family='binomial', Lambda=[0.9, 0.5, 0.1], alpha=[0.1, 0.5, 0.9], solver='COORDINATE_DESCENT', cold_start=False) mL.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1) r = glm.getGLMRegularizationPath(mL) regKeys = [ "alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train" ] best_submodel_index = mL._model_json["output"]["best_submodel_index"] m2 = glm.makeGLMModel(model=mL, coefs=r['coefficients'][best_submodel_index]) dev1 = r['explained_deviance_train'][best_submodel_index] p2 = m2.model_performance(d) dev2 = 1 - p2.residual_deviance() / p2.null_deviance() print(dev1, " =?= ", dev2) assert abs(dev1 - dev2) < 1e-6 responseMean = d[1].mean() initIntercept = math.log(responseMean / (1.0 - responseMean)) startValInit = [0, 0, 0, 0, 0, 0, 0, initIntercept] startVal = [0, 0, 0, 0, 0, 0, 0, initIntercept] orderedCoeffNames = [ "AGE", "RACE", "DPROS", "DCAPS", "PSA", "VOL", "GLEASON", "Intercept" ] for l in range(0, len(r['lambdas'])): m = glm(family='binomial', alpha=[r['alphas'][l]], Lambda=[r['lambdas'][l]], solver='COORDINATE_DESCENT', startval=startVal) m.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1) mr = glm.getGLMRegularizationPath(m) cs = r['coefficients'][l] cs_norm = r['coefficients_std'][l] pyunit_utils.assertEqualCoeffDicts(cs, m.coef(), tol=1e-3) pyunit_utils.assertEqualCoeffDicts(cs_norm, m.coef_norm(), 1e-3) if (l + 1) < len( r['lambdas']) and r['alphas'][l] != r['alphas'][l + 1]: startVal = startValInit else: startVal = pyunit_utils.extractNextCoeff( cs_norm, orderedCoeffNames, startVal) # prepare startval for next round p = m.model_performance(d) devm = 1 - p.residual_deviance() / p.null_deviance() devn = r['explained_deviance_train'][l] assert abs(devm - devn) < 1e-4 pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr, tol=1e-4) if (l == best_submodel_index ): # check training metrics, should equal for best submodel index pyunit_utils.assertEqualModelMetrics( m._model_json["output"]["training_metrics"], mL._model_json["output"]["training_metrics"], tol=1e-4) else: # for other submodel, should have worse residual_deviance() than best submodel assert p.residual_deviance() >= p2.residual_deviance(), "Best submodel does not have lowerest " \ "residual_deviance()!"