def test_run_experiment_lr_predict_with_dictionary():
"""Test rsmpredict using the dictionary object, rather than a file."""
source = 'lr-predict-dict'
# set up a temporary directory since
# we will be using getcwd
temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
old_file_dict = {
'feature_file': 'data/files/test.csv',
'experiment_dir':
'data/experiments/lr-predict-dict/existing_experiment'
}
new_file_dict = copy_data_files(temp_dir.name, old_file_dict,
rsmtool_test_dir)
config_dict = {
"id_column": "ID",
"input_features_file": new_file_dict['feature_file'],
"experiment_dir": new_file_dict['experiment_dir'],
"experiment_id": "lr"
}
check_run_prediction(source,
given_test_dir=rsmtool_test_dir,
config_obj_or_dict=config_dict)
def test_run_experiment_lr_compare_with_dictionary():
'''
test rsmcompare using the dictionary object, rather than a file
'''
source = 'lr-self-compare-dict'
experiment_id = 'lr_self_compare_dict'
# set up a temporary directory since
# we will be using getcwd
temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
old_file_dict = {
'experiment_dir': 'data/experiments/lr-self-compare-dict/lr-subgroups'
}
new_file_dict = copy_data_files(temp_dir.name, old_file_dict,
rsmtool_test_dir)
config_dict = {
"comparison_id": "lr_self_compare_dict",
"experiment_dir_old": new_file_dict['experiment_dir'],
"experiment_id_old": "lr_subgroups",
"description_old": "Using all features with a LinearRegression model.",
"use_scaled_predictions_old": True,
"experiment_dir_new": new_file_dict['experiment_dir'],
"experiment_id_new": "lr_subgroups",
"description_new": "Using all features with a LinearRegression model.",
"use_scaled_predictions_new": True,
"subgroups": ["QUESTION"]
}
check_run_comparison(source, experiment_id, config_obj_or_dict=config_dict)
def test_run_experiment_lr_eval_with_dictionary():
'''
test rsmeval using the dictionary object, rather than a file
'''
source = 'lr-eval-dict'
experiment_id = 'lr_eval_dict'
# set up a temporary directory since
# we will be using getcwd
temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
old_file_dict = {
'pred': 'data/files/predictions_scaled_with_subgroups.csv'
}
new_file_dict = copy_data_files(temp_dir.name, old_file_dict,
rsmtool_test_dir)
config_dict = {
"predictions_file": new_file_dict['pred'],
"system_score_column": "score",
"description": "An evaluation of LinearRegression predictions.",
"human_score_column": "h1",
"id_column": "id",
"experiment_id": "lr_eval_dict",
"subgroups": "QUESTION",
"scale_with": "asis",
"trim_min": 1,
"trim_max": 6
}
check_run_evaluation(source, experiment_id, config_obj_or_dict=config_dict)
def test_run_experiment_lr_with_object_no_configdir():
"""Test rsmtool using a Configuration object and no specified configdir."""
source = 'lr-object-no-path'
experiment_id = 'lr_object_no_path'
# set up a temporary directory since
# we will be using getcwd
old_file_dict = {'train': 'data/files/train.csv',
'test': 'data/files/test.csv',
'features': 'data/experiments/lr-object-no-path/features.csv'}
temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
new_file_dict = copy_data_files(temp_dir.name,
old_file_dict,
rsmtool_test_dir)
config_dict = {"train_file": new_file_dict['train'],
"id_column": "ID",
"use_scaled_predictions": True,
"test_label_column": "score",
"train_label_column": "score",
"test_file": new_file_dict['test'],
"trim_max": 6,
"features": new_file_dict['features'],
"trim_min": 1,
"model": "LinearRegression",
"experiment_id": "lr_object_no_path",
"description": "Using all features with an LinearRegression model."}
config_obj = Configuration(config_dict)
check_run_experiment(source,
experiment_id,
config_obj_or_dict=config_obj)
def test_run_experiment_lr_with_dictionary():
# Passing a dictionary as input.
source = 'lr-dictionary'
experiment_id = 'lr_dictionary'
old_file_dict = {'train': 'data/files/train.csv',
'test': 'data/files/test.csv',
'features': 'data/experiments/lr-dictionary/features.csv'}
temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
new_file_dict = copy_data_files(temp_dir.name,
old_file_dict,
rsmtool_test_dir)
config_dict = {"train_file": new_file_dict['train'],
"id_column": "ID",
"use_scaled_predictions": True,
"test_label_column": "score",
"train_label_column": "score",
"test_file": new_file_dict['test'],
"trim_max": 6,
"features": new_file_dict['features'],
"trim_min": 1,
"model": "LinearRegression",
"experiment_id": "lr_dictionary",
"description": "Using all features with an LinearRegression model."}
check_run_experiment(source,
experiment_id,
config_obj_or_dict=config_dict)
def test_copy_data_files_directory(self):
file_dict = {'exp_dir': 'data/experiments/lr-self-compare/lr-subgroups'}
expected_dict = {'exp_dir': join('temp_test_copy_dirs', 'lr-subgroups')}
self.dirs_to_remove.append('temp_test_copy_dirs')
output_dict = copy_data_files('temp_test_copy_dirs',
file_dict,
rsmtool_test_dir)
for file_type in expected_dict:
eq_(output_dict[file_type], expected_dict[file_type])
ok_(Path(output_dict[file_type]).exists())
ok_(Path(output_dict[file_type]).is_dir())
def test_copy_data_files(self):
file_dict = {'train': 'data/files/train.csv',
'features': 'data/experiments/lr/features.csv'}
expected_dict = {'train': join('temp_test_copy_data_file', 'train.csv'),
'features': join('temp_test_copy_data_file', 'features.csv')}
self.dirs_to_remove.append('temp_test_copy_data_file')
output_dict = copy_data_files('temp_test_copy_data_file',
file_dict,
rsmtool_test_dir)
for file_type in expected_dict:
eq_(output_dict[file_type], expected_dict[file_type])
ok_(Path(output_dict[file_type]).exists())
ok_(Path(output_dict[file_type]).is_file())
def test_run_experiment_lr_summary_dictionary():
'''
Test rsmsummarize using the dictionary object, rather than a file;
'''
source = 'lr-self-summary-dict'
# set up a temporary directory since
# we will be using getcwd
temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
old_file_dict = {'experiment_dir': 'data/experiments/lr-self-summary-dict/lr-subgroups'}
new_file_dict = copy_data_files(temp_dir.name,
old_file_dict,
rsmtool_test_dir)
config_dict = {"summary_id": "model_comparison",
"experiment_dirs": [new_file_dict['experiment_dir'],
new_file_dict['experiment_dir'],
new_file_dict['experiment_dir']],
"description": "Comparison of rsmtool experiment with itself."}
check_run_summary(source, config_obj_or_dict=config_dict)
