def iterateRidge(iterNums, stepSizes, regParam, train, valid):
from pyspark.mllib.regression import RidgeRegressionWithSGD
import math
for numIter in iterNums:
for step in stepSizes:
alg = RidgeRegressionWithSGD()
model = alg.train(train,
intercept=True,
regParam=regParam,
iterations=numIter,
step=step)
rescaledPredicts = train.map(lambda x:
(model.predict(x.features), x.label))
validPredicts = valid.map(lambda x:
(model.predict(x.features), x.label))
meanSquared = math.sqrt(
rescaledPredicts.map(lambda p: pow(p[0] - p[1], 2)).mean())
meanSquaredValid = math.sqrt(
validPredicts.map(lambda p: pow(p[0] - p[1], 2)).mean())
print("%d, %5.3f -> %.4f, %.4f" %
(numIter, step, meanSquared, meanSquaredValid))
