def test_run_nested_kfold_cross_validation_with_list_input(self): print "test nested k-fold cross validation with list input..." train_test_model_class = SklearnRandomForestTrainTestModel model_param_search_range = \ {'norm_type':['none'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0] } output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, self.features, [[0, 3, 2], [8, 6, 5], [4, 1, 7]]) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.26666666666666666, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.15272340058922063, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.22222222222222221, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 1.452887116343635, places=4) expected_top_model_param = {'norm_type':'none', 'n_estimators':10, 'max_depth':None, 'random_state':0 } expected_top_ratio = 0.6666666666666666 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_randomforest(self): print "test nested k-fold cross validation on random forest..." train_test_model_class = SklearnRandomForestTrainTestModel model_param_search_range = \ {'norm_type':['normalize'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0]} output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, self.features, 3) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.40167715620274708, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.11009919053282299, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.14085904245475275, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 1.3681348274719265, places=4) expected_top_model_param = {'norm_type':'normalize', 'n_estimators':10, 'max_depth':None, 'random_state':0 } expected_top_ratio = 0.6666666666666666 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_libsvmnusvr(self): print "test nested k-fold cross validation on libsvmnusvr..." train_test_model_class = LibsvmNusvrTrainTestModel model_param_search_range = \ {'norm_type':['normalize', 'clip_0to1', 'clip_minus1to1'], 'kernel':['rbf'], 'nu': [0.5], 'C': [1, 2], 'gamma': [0.0] } output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, self.features, 3) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.30962614123961751, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], -0.1535643705229309, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.14085904245475275, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 1.5853397658781734, places=4) expected_top_model_param = {'norm_type':'clip_0to1', 'kernel':'rbf', 'nu':0.5, 'C':1, 'gamma':0.0, } expected_top_ratio = 1.0 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_with_list_input(self): print "test nested k-fold cross validation with list input..." train_test_model_class = RandomForestTrainTestModel model_param_search_range = \ {'norm_type':['normalize'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0] } feature_df_file = config.ROOT + \ "/python/test/resource/sample_feature_extraction_results.json" feature_df = pd.DataFrame.from_dict(eval(open(feature_df_file, "r").read())) feature_df = feature_df[:200] output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, feature_df, [range(0,50), range(130, 200), range(50, 130)] ) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.92549459243170684) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.93070443071372855) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.76385104263763215) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 0.43223946862572299) expected_top_model_param = {'norm_type':'normalize', 'n_estimators':90, 'max_depth':3, 'random_state':0 } expected_top_ratio = 0.6666666666666666 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_libsvmnusvr(self): print "test nested k-fold cross validation on libsvmnusvr..." train_test_model_class = LibsvmnusvrTrainTestModel model_param_search_range = \ {'norm_type':['normalize', 'clip_0to1', 'clip_minus1to1'], 'kernel':['rbf'], 'nu': [0.5, 1.0], 'C': [1, 2], 'gamma': [0.0] } feature_df_file = config.ROOT + \ "/python/test/resource/sample_feature_extraction_results.json" feature_df = pd.DataFrame.from_dict(eval(open(feature_df_file, "r").read())) output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, feature_df, 6) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.93704238362264514) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.94734024567912978) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.77785381654919195) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 0.34039563991411448) expected_top_model_param = {'norm_type':'clip_0to1', 'kernel':'rbf', 'nu':1.0, 'C':1, 'gamma':0.0, } expected_top_ratio = 0.5 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_randomforest(self): print "test nested k-fold cross validation on random forest..." train_test_model_class = RandomForestTrainTestModel model_param_search_range = \ {'norm_type':['normalize'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0] } feature_df_file = config.ROOT + \ "/python/test/resource/sample_feature_extraction_results.json" feature_df = pd.DataFrame.from_dict(eval(open(feature_df_file, "r").read())) output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, feature_df, 6) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.92805802153293737) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.94066838465382363) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.76196220071567478) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 0.37660623901376861) expected_top_model_param = {'norm_type':'normalize', 'n_estimators':90, 'max_depth':None, 'random_state':0 } expected_top_ratio = 0.5 # self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_with_list_input(self): print "test nested k-fold cross validation with list input..." train_test_model_class = SklearnRandomForestTrainTestModel model_param_search_range = \ {'norm_type':['none'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0] } output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, self.features, [[0, 3, 2], [8, 6, 5], [4, 1, 7]]) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.26666666666666666, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.15272340058922063, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.22222222222222221, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 1.452887116343635, places=4) expected_top_model_param = {'norm_type':'none', 'n_estimators':10, 'max_depth':None, 'random_state':0 } expected_top_ratio = 0.6666666666666666 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_libsvmnusvr(self): print "test nested k-fold cross validation on libsvmnusvr..." train_test_model_class = LibsvmNusvrTrainTestModel model_param_search_range = \ {'norm_type':['normalize', 'clip_0to1', 'clip_minus1to1'], 'kernel':['rbf'], 'nu': [0.5], 'C': [1, 2], 'gamma': [0.0] } output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, self.features, 3) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.30962614123961751, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], -0.1535643705229309, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.14085904245475275, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 1.5853397658781734, places=4) expected_top_model_param = {'norm_type':'clip_0to1', 'kernel':'rbf', 'nu':0.5, 'C':1, 'gamma':0.0, } expected_top_ratio = 1.0 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_randomforest(self): print "test nested k-fold cross validation on random forest..." train_test_model_class = SklearnRandomForestTrainTestModel model_param_search_range = \ {'norm_type':['normalize'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0]} output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, self.features, 3) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.40167715620274708, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.11009919053282299, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.14085904245475275, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 1.3681348274719265, places=4) expected_top_model_param = {'norm_type':'normalize', 'n_estimators':10, 'max_depth':None, 'random_state':0 } expected_top_ratio = 0.6666666666666666 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_with_list_input(self): print "test nested k-fold cross validation with list input..." train_test_model_class = RandomForestTrainTestModel model_param_search_range = \ {'norm_type':['normalize'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0] } feature_df_file = config.ROOT + \ "/python/test/resource/sample_feature_extraction_results.json" feature_df = pd.DataFrame.from_dict( eval(open(feature_df_file, "r").read())) feature_df = feature_df[:200] output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, feature_df, [range(0, 50), range(130, 200), range(50, 130)]) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.92549459243170684, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.93070443071372855, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.76385104263763215, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 0.43223946862572299, places=4) expected_top_model_param = { 'norm_type': 'normalize', 'n_estimators': 90, 'max_depth': 3, 'random_state': 0 } expected_top_ratio = 0.6666666666666666 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_libsvmnusvr(self): print "test nested k-fold cross validation on libsvmnusvr..." train_test_model_class = LibsvmnusvrTrainTestModel model_param_search_range = \ {'norm_type':['normalize', 'clip_0to1', 'clip_minus1to1'], 'kernel':['rbf'], 'nu': [0.5, 1.0], 'C': [1, 2], 'gamma': [0.0] } feature_df_file = config.ROOT + \ "/python/test/resource/sample_feature_extraction_results.json" feature_df = pd.DataFrame.from_dict( eval(open(feature_df_file, "r").read())) output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, feature_df, 6) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.93704238362264514, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.94734024567912978, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.77785381654919195, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 0.34039563991411448, places=4) expected_top_model_param = { 'norm_type': 'clip_0to1', 'kernel': 'rbf', 'nu': 1.0, 'C': 1, 'gamma': 0.0, } expected_top_ratio = 0.5 self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)
def test_run_nested_kfold_cross_validation_randomforest(self): print "test nested k-fold cross validation on random forest..." train_test_model_class = RandomForestTrainTestModel model_param_search_range = \ {'norm_type':['normalize'], 'n_estimators':[10, 90], 'max_depth':[None, 3], 'random_state': [0] } feature_df_file = config.ROOT + \ "/python/test/resource/sample_feature_extraction_results.json" feature_df = pd.DataFrame.from_dict( eval(open(feature_df_file, "r").read())) output = ModelCrossValidation.run_nested_kfold_cross_validation( train_test_model_class, model_param_search_range, feature_df, 6) self.assertAlmostEquals(output['aggr_stats']['SRCC'], 0.92805802153293737, places=4) self.assertAlmostEquals(output['aggr_stats']['PCC'], 0.94066838465382363, places=4) self.assertAlmostEquals(output['aggr_stats']['KENDALL'], 0.76196220071567478, places=4) self.assertAlmostEquals(output['aggr_stats']['RMSE'], 0.37660623901376861, places=4) expected_top_model_param = { 'norm_type': 'normalize', 'n_estimators': 90, 'max_depth': None, 'random_state': 0 } expected_top_ratio = 0.5 # self.assertEquals(output['top_model_param'], expected_top_model_param) self.assertEquals(output['top_ratio'], expected_top_ratio)