def test_run_experiment_lr_predict_with_dictionary(): """Test rsmpredict using the dictionary object, rather than a file.""" source = 'lr-predict-dict' # set up a temporary directory since # we will be using getcwd temp_dir = tempfile.TemporaryDirectory(prefix=getcwd()) old_file_dict = { 'feature_file': 'data/files/test.csv', 'experiment_dir': 'data/experiments/lr-predict-dict/existing_experiment' } new_file_dict = copy_data_files(temp_dir.name, old_file_dict, rsmtool_test_dir) config_dict = { "id_column": "ID", "input_features_file": new_file_dict['feature_file'], "experiment_dir": new_file_dict['experiment_dir'], "experiment_id": "lr" } check_run_prediction(source, given_test_dir=rsmtool_test_dir, config_obj_or_dict=config_dict)
def test_run_experiment_lr_compare_with_dictionary(): ''' test rsmcompare using the dictionary object, rather than a file ''' source = 'lr-self-compare-dict' experiment_id = 'lr_self_compare_dict' # set up a temporary directory since # we will be using getcwd temp_dir = tempfile.TemporaryDirectory(prefix=getcwd()) old_file_dict = { 'experiment_dir': 'data/experiments/lr-self-compare-dict/lr-subgroups' } new_file_dict = copy_data_files(temp_dir.name, old_file_dict, rsmtool_test_dir) config_dict = { "comparison_id": "lr_self_compare_dict", "experiment_dir_old": new_file_dict['experiment_dir'], "experiment_id_old": "lr_subgroups", "description_old": "Using all features with a LinearRegression model.", "use_scaled_predictions_old": True, "experiment_dir_new": new_file_dict['experiment_dir'], "experiment_id_new": "lr_subgroups", "description_new": "Using all features with a LinearRegression model.", "use_scaled_predictions_new": True, "subgroups": ["QUESTION"] } check_run_comparison(source, experiment_id, config_obj_or_dict=config_dict)
def test_run_experiment_lr_eval_with_dictionary(): ''' test rsmeval using the dictionary object, rather than a file ''' source = 'lr-eval-dict' experiment_id = 'lr_eval_dict' # set up a temporary directory since # we will be using getcwd temp_dir = tempfile.TemporaryDirectory(prefix=getcwd()) old_file_dict = { 'pred': 'data/files/predictions_scaled_with_subgroups.csv' } new_file_dict = copy_data_files(temp_dir.name, old_file_dict, rsmtool_test_dir) config_dict = { "predictions_file": new_file_dict['pred'], "system_score_column": "score", "description": "An evaluation of LinearRegression predictions.", "human_score_column": "h1", "id_column": "id", "experiment_id": "lr_eval_dict", "subgroups": "QUESTION", "scale_with": "asis", "trim_min": 1, "trim_max": 6 } check_run_evaluation(source, experiment_id, config_obj_or_dict=config_dict)
def test_run_experiment_lr_with_object_no_configdir(): """Test rsmtool using a Configuration object and no specified configdir.""" source = 'lr-object-no-path' experiment_id = 'lr_object_no_path' # set up a temporary directory since # we will be using getcwd old_file_dict = {'train': 'data/files/train.csv', 'test': 'data/files/test.csv', 'features': 'data/experiments/lr-object-no-path/features.csv'} temp_dir = tempfile.TemporaryDirectory(prefix=getcwd()) new_file_dict = copy_data_files(temp_dir.name, old_file_dict, rsmtool_test_dir) config_dict = {"train_file": new_file_dict['train'], "id_column": "ID", "use_scaled_predictions": True, "test_label_column": "score", "train_label_column": "score", "test_file": new_file_dict['test'], "trim_max": 6, "features": new_file_dict['features'], "trim_min": 1, "model": "LinearRegression", "experiment_id": "lr_object_no_path", "description": "Using all features with an LinearRegression model."} config_obj = Configuration(config_dict) check_run_experiment(source, experiment_id, config_obj_or_dict=config_obj)
def test_run_experiment_lr_with_dictionary(): # Passing a dictionary as input. source = 'lr-dictionary' experiment_id = 'lr_dictionary' old_file_dict = {'train': 'data/files/train.csv', 'test': 'data/files/test.csv', 'features': 'data/experiments/lr-dictionary/features.csv'} temp_dir = tempfile.TemporaryDirectory(prefix=getcwd()) new_file_dict = copy_data_files(temp_dir.name, old_file_dict, rsmtool_test_dir) config_dict = {"train_file": new_file_dict['train'], "id_column": "ID", "use_scaled_predictions": True, "test_label_column": "score", "train_label_column": "score", "test_file": new_file_dict['test'], "trim_max": 6, "features": new_file_dict['features'], "trim_min": 1, "model": "LinearRegression", "experiment_id": "lr_dictionary", "description": "Using all features with an LinearRegression model."} check_run_experiment(source, experiment_id, config_obj_or_dict=config_dict)
def test_copy_data_files_directory(self): file_dict = {'exp_dir': 'data/experiments/lr-self-compare/lr-subgroups'} expected_dict = {'exp_dir': join('temp_test_copy_dirs', 'lr-subgroups')} self.dirs_to_remove.append('temp_test_copy_dirs') output_dict = copy_data_files('temp_test_copy_dirs', file_dict, rsmtool_test_dir) for file_type in expected_dict: eq_(output_dict[file_type], expected_dict[file_type]) ok_(Path(output_dict[file_type]).exists()) ok_(Path(output_dict[file_type]).is_dir())
def test_copy_data_files(self): file_dict = {'train': 'data/files/train.csv', 'features': 'data/experiments/lr/features.csv'} expected_dict = {'train': join('temp_test_copy_data_file', 'train.csv'), 'features': join('temp_test_copy_data_file', 'features.csv')} self.dirs_to_remove.append('temp_test_copy_data_file') output_dict = copy_data_files('temp_test_copy_data_file', file_dict, rsmtool_test_dir) for file_type in expected_dict: eq_(output_dict[file_type], expected_dict[file_type]) ok_(Path(output_dict[file_type]).exists()) ok_(Path(output_dict[file_type]).is_file())
def test_run_experiment_lr_summary_dictionary(): ''' Test rsmsummarize using the dictionary object, rather than a file; ''' source = 'lr-self-summary-dict' # set up a temporary directory since # we will be using getcwd temp_dir = tempfile.TemporaryDirectory(prefix=getcwd()) old_file_dict = {'experiment_dir': 'data/experiments/lr-self-summary-dict/lr-subgroups'} new_file_dict = copy_data_files(temp_dir.name, old_file_dict, rsmtool_test_dir) config_dict = {"summary_id": "model_comparison", "experiment_dirs": [new_file_dict['experiment_dir'], new_file_dict['experiment_dir'], new_file_dict['experiment_dir']], "description": "Comparison of rsmtool experiment with itself."} check_run_summary(source, config_obj_or_dict=config_dict)