def test_run_experiment_lr_predict_no_numeric_feature_values(): # rsmpredict experiment with missing post-processing file source = 'lr-predict-no-numeric-feature-values' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_no_input_feature_file(): # rsmpredict experiment with missing feature file source = 'lr-predict-no-input-feature-file' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_lr_predict_missing_model_file(): # rsmpredict experiment with missing model file source = 'lr-predict-missing-model-file' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_lr_predict_with_repeated_ids(): # rsmpredict experiment with non-unique ids source = 'lr-predict-with-repeated-ids' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_no_experiment_dir(): # rsmpredict experiment with missing experiment dir source = 'lr-predict-no-experiment-dir' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_no_experiment_id(): # rsmpredict experiment ehere the experiment_dir # does not contain the experiment with the stated id source = 'lr-predict-no-experiment-id' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_lr_predict_missing_feature(): ''' rsmpredict experiment with missing features ''' source = 'lr-predict-missing-feature' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_lr_predict_missing_columns(): # rsmpredict experiment with missing columns # from the config file source = 'lr-predict-missing-columns' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_expected_scores_builtin_model(): # rsmpredict experiment for expected scores but with # a built-in model which is not supporte source = 'lr-predict-expected-scores-builtin-model' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_expected_scores_wrong_skll_model(): # rsmpredict experiment for expected scores but with # a non-probabilistic SKLL learner source = 'predict-expected-scores-wrong-skll-model' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_expected_scores_non_probablistic_svc(): # rsmpredict experiment for expected scores but with # a non-probabilistic SKLL learner source = 'predict-expected-scores-non-probabilistic-svc' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_no_output_dir(): # rsmpredict experiment where experiment_dir # does not containt output directory source = 'lr-predict-no-output-dir' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_lr_predict_missing_feature(): # rsmpredict experiment with missing features source = 'lr-predict-missing-feature' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_lr_rsmtool_and_rsmpredict(): # this test is to make sure that both rsmtool # and rsmpredict generate the same files source = 'lr-rsmtool-rsmpredict' experiment_id = 'lr_rsmtool_rsmpredict' rsmtool_config_file = join(rsmtool_test_dir, 'data', 'experiments', source, '{}.json'.format(experiment_id)) do_run_experiment(source, experiment_id, rsmtool_config_file) rsmpredict_config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, rsmpredict_config_file) output_dir = join('test_outputs', source, 'output') expected_output_dir = join(rsmtool_test_dir, 'data', 'experiments', source, 'output') csv_files = glob(join(output_dir, '*.csv')) html_report = join('test_outputs', source, 'report', '{}_report.html'.format(experiment_id)) # Check the results for rsmtool for csv_file in csv_files: csv_filename = basename(csv_file) expected_csv_file = join(expected_output_dir, csv_filename) if exists(expected_csv_file): yield check_file_output, csv_file, expected_csv_file yield check_scaled_coefficients, source, experiment_id yield check_generated_output, csv_files, experiment_id, 'rsmtool' yield check_report, html_report # check that the rsmpredict generated the same results for csv_pair in [('predictions.csv', '{}_pred_processed.csv'.format(experiment_id)), ('preprocessed_features.csv', '{}_test_preprocessed_features.csv'.format(experiment_id))]: output_file = join(output_dir, csv_pair[0]) expected_output_file = join(expected_output_dir, csv_pair[1]) yield check_file_output, output_file, expected_output_file
def test_run_experiment_lr_rsmtool_and_rsmpredict(): ''' this test is to make sure that both rsmtool and rsmpredict generate the same files ''' source = 'lr-rsmtool-rsmpredict' experiment_id = 'lr_rsmtool_rsmpredict' rsmtool_config_file = join(rsmtool_test_dir, 'data', 'experiments', source, '{}.json'.format(experiment_id)) do_run_experiment(source, experiment_id, rsmtool_config_file) rsmpredict_config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, rsmpredict_config_file) output_dir = join('test_outputs', source, 'output') expected_output_dir = join(rsmtool_test_dir, 'data', 'experiments', source, 'output') csv_files = glob(join(output_dir, '*.csv')) html_report = join('test_outputs', source, 'report', '{}_report.html'.format(experiment_id)) # Check the results for rsmtool for csv_file in csv_files: csv_filename = basename(csv_file) expected_csv_file = join(expected_output_dir, csv_filename) if exists(expected_csv_file): yield check_file_output, csv_file, expected_csv_file yield check_scaled_coefficients, source, experiment_id yield check_generated_output, csv_files, experiment_id, 'rsmtool' yield check_report, html_report # check that the rsmpredict generated the same results for csv_pair in [ ('predictions.csv', '{}_pred_processed.csv'.format(experiment_id)), ('preprocessed_features.csv', '{}_test_preprocessed_features.csv'.format(experiment_id)) ]: output_file = join(output_dir, csv_pair[0]) expected_output_file = join(expected_output_dir, csv_pair[1]) yield check_file_output, output_file, expected_output_file
def test_run_experiment_predict_expected_scores_wrong_skll_model(): """Run rsmpredict experiment for expected scores with an unsupported SKLL learner.""" source = 'predict-expected-scores-wrong-skll-model' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_expected_scores_builtin_model(): """Run rsmpredict experiment for expected scores with unsupported built-in model.""" source = 'lr-predict-expected-scores-builtin-model' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_no_experiment_id(): """Run rsmpredict experiment with no experiment ID.""" source = 'lr-predict-no-experiment-id' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_predict_no_output_dir(): """Run rsmpredict experiment with a missing "output" directory.""" source = 'lr-predict-no-output-dir' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)
def test_run_experiment_lr_predict_missing_postprocessing_file(): """Run rsmpredict experiment with missing post-processing file.""" source = 'lr-predict-missing-postprocessing-file' config_file = join(rsmtool_test_dir, 'data', 'experiments', source, 'rsmpredict.json') do_run_prediction(source, config_file)