def test_linear_regression_sk2(self): diabetes = datasets.load_diabetes() diabetes_X = diabetes.data[:, np.newaxis, 2] diabetes_X_train = diabetes_X[:-20] diabetes_X_test = diabetes_X[-20:] diabetes_y_train = diabetes.target[:-20] diabetes_y_test = diabetes.target[-20:] regr = LinearRegression(sqlCtx, transferUsingDF=True) regr.fit(diabetes_X_train, diabetes_y_train) score = regr.score(diabetes_X_test, diabetes_y_test) self.failUnless(score > 0.4) # TODO: Improve r2-score (may be I am using it incorrectly)
def testLinearRegressionSK2(self): diabetes = datasets.load_diabetes() diabetes_X = diabetes.data[:, np.newaxis, 2] diabetes_X_train = diabetes_X[:-20] diabetes_X_test = diabetes_X[-20:] diabetes_y_train = diabetes.target[:-20] diabetes_y_test = diabetes.target[-20:] regr = LinearRegression(sqlCtx, transferUsingDF=True) regr.fit(diabetes_X_train, diabetes_y_train) score = regr.score(diabetes_X_test, diabetes_y_test) self.failUnless(score > 0.4) # TODO: Improve r2-score (may be I am using it incorrectly)
def test_linear_regression_cg(self): diabetes = datasets.load_diabetes() diabetes_X = diabetes.data[:, np.newaxis, 2] diabetes_X_train = diabetes_X[:-20] diabetes_X_test = diabetes_X[-20:] diabetes_y_train = diabetes.target[:-20] diabetes_y_test = diabetes.target[-20:] regr = LinearRegression(sparkSession, solver='newton-cg', transferUsingDF=True) regr.fit(diabetes_X_train, diabetes_y_train) mllearn_predicted = regr.predict(diabetes_X_test) sklearn_regr = linear_model.LinearRegression() sklearn_regr.fit(diabetes_X_train, diabetes_y_train) self.failUnless(r2_score(sklearn_regr.predict(diabetes_X_test), mllearn_predicted) > 0.95) # We are comparable to a similar algorithm in scikit learn