def test_run_experiment_lr_eval_with_cfg():
# basic evaluation experiment using rsmeval
source = 'lr-eval-cfg'
experiment_id = 'lr_eval_cfg'
config_file = join(rsmtool_test_dir,
'data',
'experiments',
source,
'{}.cfg'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
output_dir = join('test_outputs', source, 'output')
expected_output_dir = join(rsmtool_test_dir, 'data', 'experiments', source, 'output')
html_report = join('test_outputs', source, 'report', '{}_report.html'.format(experiment_id))
csv_files = glob(join(output_dir, '*.csv'))
for csv_file in csv_files:
csv_filename = basename(csv_file)
expected_csv_file = join(expected_output_dir, csv_filename)
if exists(expected_csv_file):
yield check_file_output, csv_file, expected_csv_file
yield check_report, html_report
def test_run_experiment_lr_eval_with_repeated_ids():
# rsmeval experiment with non-unique ids
source = 'lr-eval-with-repeated-ids'
experiment_id = 'lr_eval_with_repeated_ids'
config_file = join(rsmtool_test_dir, 'data', 'experiments', source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_lr_eval_wrong_path():
# basic rsmeval experiment with wrong path to the
# predictions file
source = 'lr-eval-with-wrong-path'
experiment_id = 'lr_eval_with_h2'
config_file = join(rsmtool_test_dir, 'data', 'experiments', source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_eval_lr_with_missing_candidate_column():
# rsmeval experiment with `candidate_column`
# set to a column that does not exist in the given
# predictions file
source = 'lr-eval-with-missing-candidate-column'
experiment_id = 'lr_eval_with_missing_candidate_column'
config_file = join(rsmtool_test_dir, 'data', 'experiments', source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_lr_eval_same_system_human_score():
# rsmeval experiment with the same value supplied
# for both human score ans system score
source = 'lr-eval-same-system-human-score'
experiment_id = 'lr_eval_same_system_human_score'
config_file = join(rsmtool_test_dir, 'data', 'experiments', source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_lr_eval_all_non_numeric_machine_scores():
# rsmeval experiment with all the machine scores`
# being non-numeric and all getting filtered out
# which should raise an exception
source = 'lr-eval-with-all-non-numeric-machine-scores'
experiment_id = 'lr_eval_all_non_numeric_machine_scores'
config_file = join(rsmtool_test_dir, 'data', 'experiments', source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_lr_eval_with_repeated_ids():
# rsmeval experiment with non-unique ids
source = 'lr-eval-with-repeated-ids'
experiment_id = 'lr_eval_with_repeated_ids'
config_file = join(rsmtool_test_dir,
'data',
'experiments',
source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_lr_eval_wrong_path():
# basic rsmeval experiment with wrong path to the
# predictions file
source = 'lr-eval-with-wrong-path'
experiment_id = 'lr_eval_with_h2'
config_file = join(rsmtool_test_dir,
'data',
'experiments',
source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_eval_lr_with_missing_candidate_column():
# rsmeval experiment with `candidate_column`
# set to a column that does not exist in the given
# predictions file
source = 'lr-eval-with-missing-candidate-column'
experiment_id = 'lr_eval_with_missing_candidate_column'
config_file = join(rsmtool_test_dir,
'data',
'experiments',
source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_lr_eval_same_system_human_score():
# rsmeval experiment with the same value supplied
# for both human score ans system score
source = 'lr-eval-same-system-human-score'
experiment_id = 'lr_eval_same_system_human_score'
config_file = join(rsmtool_test_dir,
'data',
'experiments',
source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
def test_run_experiment_lr_eval_all_non_numeric_scores():
# rsmeval experiment with all values for the human
# score being non-numeric and all getting filtered out
# which should raise an exception
source = 'lr-eval-with-all-non-numeric-scores'
experiment_id = 'lr_eval_all_non_numeric_scores'
config_file = join(rsmtool_test_dir,
'data',
'experiments',
source,
'{}.json'.format(experiment_id))
do_run_evaluation(source, experiment_id, config_file)
