def glm_alpha_array_lambda_null():
# first test: compare coefficients and deviance
d = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/prostate.csv"))
mL = glm(family='binomial',alpha=[0.1,0.5,0.9],solver='COORDINATE_DESCENT')
mL.train(training_frame=d,x=[2,3,4,5,6,7,8],y=1)
r = glm.getGLMRegularizationPath(mL)
regKeys = ["alphas", "lambdas", "explained_deviance_valid", "explained_deviance_train"]
best_submodel_index = mL._model_json["output"]["best_submodel_index"]
m2 = glm.makeGLMModel(model=mL,coefs=r['coefficients'][best_submodel_index])
dev1 = r['explained_deviance_train'][best_submodel_index]
p2 = m2.model_performance(d)
dev2 = 1-p2.residual_deviance()/p2.null_deviance()
print(dev1," =?= ",dev2)
assert abs(dev1 - dev2) < 1e-6
for l in range(0,len(r['lambdas'])):
m = glm(family='binomial',alpha=[r['alphas'][l]],Lambda=[r['lambdas'][l]],solver='COORDINATE_DESCENT')
m.train(training_frame=d,x=[2,3,4,5,6,7,8],y=1)
mr = glm.getGLMRegularizationPath(m)
cs = r['coefficients'][l]
cs_norm = r['coefficients_std'][l]
pyunit_utils.assertEqualCoeffDicts(cs, m.coef())
pyunit_utils.assertEqualCoeffDicts(cs_norm, m.coef_norm())
p = m.model_performance(d)
devm = 1-p.residual_deviance()/p.null_deviance()
devn = r['explained_deviance_train'][l]
assert abs(devm - devn) < 1e-4
pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr)
if (l == best_submodel_index): # check training metrics, should equal for best submodel index
pyunit_utils.assertEqualModelMetrics(m._model_json["output"]["training_metrics"],
mL._model_json["output"]["training_metrics"])
else: # for other submodel, should have worse residual_deviance() than best submodel
assert p.residual_deviance() >= p2.residual_deviance(), "Best submodel does not have lowerest " \
"residual_deviance()!"
def glm_alpha_array_lambda_null():
# first test: compare coefficients and deviance
keySets = ["MSE", "null_deviance", "logloss", "RMSE", "r2"]
d = h2o.import_file(
path=pyunit_utils.locate("smalldata/covtype/covtype.20k.data"))
mL = glm(family='multinomial',
alpha=[0.1, 0.5, 0.9],
Lambda=[0.1, 0.5, 0.9],
cold_start=True)
d[54] = d[54].asfactor()
mL.train(training_frame=d, x=list(range(0, 54)), y=54)
r = glm.getGLMRegularizationPath(mL)
regKeys = [
"alphas", "lambdas", "explained_deviance_valid",
"explained_deviance_train"
]
best_submodel_index = mL._model_json["output"]["best_submodel_index"]
coefClassSet = [
'coefs_class_0', 'coefs_class_1', 'coefs_class_2', 'coefs_class_3',
'coefs_class_4', 'coefs_class_5', 'coefs_class_6', 'coefs_class_7'
]
coefClassSetNorm = [
'std_coefs_class_0', 'std_coefs_class_1', 'std_coefs_class_2',
'std_coefs_class_3', 'std_coefs_class_4', 'std_coefs_class_5',
'std_coefs_class_6', 'std_coefs_class_7'
]
groupedClass = d.group_by("C55")
groupedClass.count()
classFrame = groupedClass.get_frame()
classProb = classFrame[1] / d.nrow
coeffIndex = [52, 105, 158, 211, 264, 317, 370]
startVal = [0] * 371
for ind in range(classProb.nrow):
startVal[coeffIndex[ind]] = math.log(classProb[ind, 0])
for l in range(0, len(r['lambdas'])):
m = glm(family='multinomial',
alpha=[r['alphas'][l]],
Lambda=[r['lambdas'][l]],
startval=startVal)
m.train(training_frame=d, x=list(range(0, 54)), y=54)
mr = glm.getGLMRegularizationPath(m)
cs = r['coefficients'][l]
cs_norm = r['coefficients_std'][l]
pyunit_utils.assertCoefEqual(cs, m.coef(), coefClassSet)
pyunit_utils.assertCoefEqual(cs_norm, m.coef_norm(), coefClassSetNorm)
devm = 1 - m.residual_deviance() / m.null_deviance()
devn = r['explained_deviance_train'][l]
assert abs(devm - devn) < 1e-4
pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr)
if (l == best_submodel_index
): # check training metrics, should equal for best submodel index
pyunit_utils.assertEqualModelMetrics(
m._model_json["output"]["training_metrics"],
mL._model_json["output"]["training_metrics"],
tol=1e-2,
keySet=keySets)
else: # for other submodel, should have worse residual_deviance() than best submodel
assert m.logloss() >= mL.logloss(), "Best submodel does not have lowerest " \
"logloss()!"
def glm_alpha_lambda_arrays():
# compare coefficients and deviance when only training dataset is available
train = h2o.import_file(path=pyunit_utils.locate(
"smalldata/glm_test/binomial_20_cols_10KRows.csv"))
for ind in range(10):
train[ind] = train[ind].asfactor()
train["C21"] = train["C21"].asfactor()
frames = train.split_frame(ratios=[0.8], seed=12345)
d = frames[0]
d_test = frames[1]
regKeys = [
"alphas", "lambdas", "explained_deviance_valid",
"explained_deviance_train"
]
# compare results when validation dataset is present
mLVal = glm(family='binomial',
alpha=[0.1, 0.5],
lambda_search=True,
solver='COORDINATE_DESCENT',
nlambdas=3) # train with validations set
mLVal.train(training_frame=d,
x=list(range(20)),
y=20,
validation_frame=d_test)
rVal = glm.getGLMRegularizationPath(mLVal)
best_submodel_indexVal = mLVal._model_json["output"]["best_submodel_index"]
m2Val = glm.makeGLMModel(
model=mLVal, coefs=rVal['coefficients'][best_submodel_indexVal])
dev1Val = rVal['explained_deviance_valid'][best_submodel_indexVal]
p2Val = m2Val.model_performance(d_test)
dev2Val = 1 - p2Val.residual_deviance() / p2Val.null_deviance()
assert abs(dev1Val - dev2Val) < 1e-6
for l in range(0, len(rVal['lambdas'])):
m = glm(family='binomial',
alpha=[rVal['alphas'][l]],
Lambda=rVal['lambdas'][l],
solver='COORDINATE_DESCENT')
m.train(training_frame=d,
x=list(range(20)),
y=20,
validation_frame=d_test)
mr = glm.getGLMRegularizationPath(m)
p = m.model_performance(d_test)
cs = rVal['coefficients'][l]
cs_norm = rVal['coefficients_std'][l]
print("Comparing submodel index {0}".format(l))
pyunit_utils.assertEqualCoeffDicts(cs, m.coef(), tol=1e-1)
pyunit_utils.assertEqualCoeffDicts(cs_norm, m.coef_norm(), tol=1e-1)
pyunit_utils.assertEqualRegPaths(regKeys, rVal, l, mr, tol=1e-3)
dVal = 1 - p.residual_deviance() / p.null_deviance()
if l == best_submodel_indexVal: # check training metrics, should equal for best submodel index
pyunit_utils.assertEqualModelMetrics(
m._model_json["output"]["validation_metrics"],
mLVal._model_json["output"]["validation_metrics"],
tol=1e-2)
else: # for other submodel, should have worse residual_deviance() than best submodel
assert dVal <= dev2Val, "Best submodel does not have highest explained deviance_valid for submodel: !".format(
l)
def glm_alpha_array_lambda_null():
# first test: compare coefficients and deviance
d = h2o.import_file(
path=pyunit_utils.locate("smalldata/covtype/covtype.20k.data"))
mL = glm(family='multinomial',
alpha=[0.1, 0.5, 0.9],
lambda_search=True,
solver='COORDINATE_DESCENT',
cold_start=True,
nlambdas=5)
d[54] = d[54].asfactor()
mL.train(training_frame=d, x=list(range(0, 54)), y=54)
r = glm.getGLMRegularizationPath(mL)
regKeys = [
"alphas", "lambdas", "explained_deviance_valid",
"explained_deviance_train"
]
best_submodel_index = mL._model_json["output"]["best_submodel_index"]
coefClassSet = [
'coefs_class_0', 'coefs_class_1', 'coefs_class_2', 'coefs_class_3',
'coefs_class_4', 'coefs_class_5', 'coefs_class_6', 'coefs_class_7'
]
coefClassSetNorm = [
'std_coefs_class_0', 'std_coefs_class_1', 'std_coefs_class_2',
'std_coefs_class_3', 'std_coefs_class_4', 'std_coefs_class_5',
'std_coefs_class_6', 'std_coefs_class_7'
]
for l in range(0, len(r['lambdas'])):
print("compare models for index {0}, alpha {1}, lambda{2}".format(
l, r['alphas'][l], r['lambdas'][l]))
m = glm(family='multinomial',
alpha=[r['alphas'][l]],
Lambda=[r['lambdas'][l]],
solver='COORDINATE_DESCENT')
m.train(training_frame=d, x=list(range(0, 54)), y=54)
mr = glm.getGLMRegularizationPath(m)
cs = r['coefficients'][l]
cs_norm = r['coefficients_std'][l]
pyunit_utils.assertCoefEqual(cs, m.coef(), coefClassSet)
pyunit_utils.assertCoefEqual(cs_norm, m.coef_norm(), coefClassSetNorm)
devm = 1 - m.residual_deviance() / m.null_deviance()
devn = r['explained_deviance_train'][l]
assert abs(devm - devn) < 1e-6
pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr)
if (l == best_submodel_index
): # check training metrics, should equal for best submodel index
pyunit_utils.assertEqualModelMetrics(
m._model_json["output"]["training_metrics"],
mL._model_json["output"]["training_metrics"],
keySet=["MSE", "null_deviance", "logloss", "RMSE", "r2"],
tol=5e-1)
else: # for other submodel, should have worse residual_deviance() than best submodel
assert devm <= r['explained_deviance_train'][best_submodel_index], "Best submodel does not best " \
"explained_deviance_train!"
def glm_alpha_lambda_arrays():
# read in the dataset and construct training set (and validation set)
d = h2o.import_file(
path=pyunit_utils.locate("smalldata/logreg/prostate.csv"))
mL = glm(family='binomial',
Lambda=[0.9, 0.5, 0.1],
alpha=[0.1, 0.5, 0.9],
solver='COORDINATE_DESCENT')
mL.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1)
r = glm.getGLMRegularizationPath(mL)
regKeys = [
"alphas", "lambdas", "explained_deviance_valid",
"explained_deviance_train"
]
best_submodel_index = mL._model_json["output"]["best_submodel_index"]
m2 = glm.makeGLMModel(model=mL,
coefs=r['coefficients'][best_submodel_index])
dev1 = r['explained_deviance_train'][best_submodel_index]
p2 = m2.model_performance(d)
dev2 = 1 - p2.residual_deviance() / p2.null_deviance()
assert abs(dev1 - dev2) < 1e-6
for l in range(0, len(r['lambdas'])):
m = glm(family='binomial',
alpha=[r['alphas'][l]],
Lambda=[r['lambdas'][l]],
solver='COORDINATE_DESCENT')
m.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1)
mr = glm.getGLMRegularizationPath(m)
cs = r['coefficients'][l]
cs_norm = r['coefficients_std'][l]
diff = 0
diff2 = 0
for n in cs.keys():
diff = max(diff, abs((cs[n] - m.coef()[n])))
diff2 = max(diff2, abs((cs_norm[n] - m.coef_norm()[n])))
assert diff < 1e-2
assert diff2 < 1e-2
p = m.model_performance(d)
devm = 1 - p.residual_deviance() / p.null_deviance()
devn = r['explained_deviance_train'][l]
assert abs(devm - devn) < 1e-4
pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr, tol=1e-5)
if (l == best_submodel_index
): # check training metrics, should equal for best submodel index
pyunit_utils.assertEqualModelMetrics(
m._model_json["output"]["training_metrics"],
mL._model_json["output"]["training_metrics"],
tol=1e-5)
else: # for other submodel, should have worse residual_deviance() than best submodel
assert p.residual_deviance() >= p2.residual_deviance(), "Best submodel does not have lowerest " \
"residual_deviance()!"
def glm_alpha_array_lambda_null():
# first test: compare coefficients and deviance
d = h2o.import_file(
path=pyunit_utils.locate("smalldata/covtype/covtype.20k.data"))
mL = glm(family='multinomial', alpha=[0.1, 0.5, 0.9])
d[54] = d[54].asfactor()
mL.train(training_frame=d, x=list(range(0, 54)), y=54)
r = glm.getGLMRegularizationPath(mL)
regKeys = [
"alphas", "lambdas", "explained_deviance_valid",
"explained_deviance_train"
]
best_submodel_index = mL._model_json["output"]["best_submodel_index"]
coefClassSet = [
'coefs_class_0', 'coefs_class_1', 'coefs_class_2', 'coefs_class_3',
'coefs_class_4', 'coefs_class_5', 'coefs_class_6', 'coefs_class_7'
]
coefClassSetNorm = [
'std_coefs_class_0', 'std_coefs_class_1', 'std_coefs_class_2',
'std_coefs_class_3', 'std_coefs_class_4', 'std_coefs_class_5',
'std_coefs_class_6', 'std_coefs_class_7'
]
for l in range(0, len(r['lambdas'])):
m = glm(family='multinomial',
alpha=[r['alphas'][l]],
Lambda=[r['lambdas'][l]])
m.train(training_frame=d, x=list(range(0, 54)), y=54)
mr = glm.getGLMRegularizationPath(m)
cs = r['coefficients'][l]
cs_norm = r['coefficients_std'][l]
pyunit_utils.assertCoefEqual(cs, m.coef(), coefClassSet, tol=1e-5)
pyunit_utils.assertCoefEqual(cs_norm,
m.coef_norm(),
coefClassSetNorm,
tol=1e-5)
devm = 1 - m.residual_deviance() / m.null_deviance()
devn = r['explained_deviance_train'][l]
assert abs(devm - devn) < 1e-4
pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr)
if (l == best_submodel_index
): # check training metrics, should equal for best submodel index
pyunit_utils.assertEqualModelMetrics(
m._model_json["output"]["training_metrics"],
mL._model_json["output"]["training_metrics"],
tol=1e-2)
else: # for other submodel, should have worse residual_deviance() than best submodel
assert m.logloss() >= mL.logloss(), "Best submodel does not have lowerest " \
"logloss()!"
def glm_alpha_lambda_arrays():
# read in the dataset and construct training set (and validation set)
d = h2o.import_file(
path=pyunit_utils.locate("smalldata/logreg/prostate.csv"))
mL = glm(family='binomial',
Lambda=[0.9, 0.5, 0.1],
alpha=[0.1, 0.5, 0.9],
solver='COORDINATE_DESCENT',
cold_start=False)
mL.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1)
r = glm.getGLMRegularizationPath(mL)
regKeys = [
"alphas", "lambdas", "explained_deviance_valid",
"explained_deviance_train"
]
best_submodel_index = mL._model_json["output"]["best_submodel_index"]
m2 = glm.makeGLMModel(model=mL,
coefs=r['coefficients'][best_submodel_index])
dev1 = r['explained_deviance_train'][best_submodel_index]
p2 = m2.model_performance(d)
dev2 = 1 - p2.residual_deviance() / p2.null_deviance()
print(dev1, " =?= ", dev2)
assert abs(dev1 - dev2) < 1e-6
responseMean = d[1].mean()
initIntercept = math.log(responseMean / (1.0 - responseMean))
startValInit = [0, 0, 0, 0, 0, 0, 0, initIntercept]
startVal = [0, 0, 0, 0, 0, 0, 0, initIntercept]
orderedCoeffNames = [
"AGE", "RACE", "DPROS", "DCAPS", "PSA", "VOL", "GLEASON", "Intercept"
]
for l in range(0, len(r['lambdas'])):
m = glm(family='binomial',
alpha=[r['alphas'][l]],
Lambda=[r['lambdas'][l]],
solver='COORDINATE_DESCENT',
startval=startVal)
m.train(training_frame=d, x=[2, 3, 4, 5, 6, 7, 8], y=1)
mr = glm.getGLMRegularizationPath(m)
cs = r['coefficients'][l]
cs_norm = r['coefficients_std'][l]
pyunit_utils.assertEqualCoeffDicts(cs, m.coef(), tol=1e-3)
pyunit_utils.assertEqualCoeffDicts(cs_norm, m.coef_norm(), 1e-3)
if (l + 1) < len(
r['lambdas']) and r['alphas'][l] != r['alphas'][l + 1]:
startVal = startValInit
else:
startVal = pyunit_utils.extractNextCoeff(
cs_norm, orderedCoeffNames,
startVal) # prepare startval for next round
p = m.model_performance(d)
devm = 1 - p.residual_deviance() / p.null_deviance()
devn = r['explained_deviance_train'][l]
assert abs(devm - devn) < 1e-4
pyunit_utils.assertEqualRegPaths(regKeys, r, l, mr, tol=1e-4)
if (l == best_submodel_index
): # check training metrics, should equal for best submodel index
pyunit_utils.assertEqualModelMetrics(
m._model_json["output"]["training_metrics"],
mL._model_json["output"]["training_metrics"],
tol=1e-4)
else: # for other submodel, should have worse residual_deviance() than best submodel
assert p.residual_deviance() >= p2.residual_deviance(), "Best submodel does not have lowerest " \
"residual_deviance()!"