def test_config_with_inverted_custom_metric(): """Test config with a lower-is-better custom metric""" # run the first experiment that uses a lower-is-better custom metric # for grid saerch defined as simply 1 minus the macro-averaged F1 score input_dir = join(_my_dir, "other") train_file = join(input_dir, "examples_train.jsonlines") test_file = join(input_dir, "examples_test.jsonlines") config_path1 = fill_in_config_paths_for_single_file( join(_my_dir, "configs", "test_custom_" "metrics_kwargs1" ".template.cfg"), train_file, test_file) run_configuration(config_path1, local=True, quiet=True) # laod the results with open( join(_my_dir, 'output', ('test_custom_metrics_kwargs1_train_' 'examples_train.jsonlines_' 'LogisticRegression.results.json'))) as f: result_dict1 = json.load(f) grid_score1 = result_dict1['grid_score'] grid_results_dict1 = result_dict1['grid_search_cv_results'] # now run the second experiment that is identical except that # that it uses the regular macro-averaged F1 score for grid search config_path2 = fill_in_config_paths_for_single_file( join(_my_dir, "configs", "test_custom_" "metrics_kwargs2" ".template.cfg"), train_file, test_file) run_configuration(config_path2, local=True, quiet=True) # laod the results with open( join(_my_dir, 'output', ('test_custom_metrics_kwargs2_train_' 'examples_train.jsonlines_' 'LogisticRegression.results.json'))) as f: result_dict2 = json.load(f) grid_score2 = result_dict2['grid_score'] grid_results_dict2 = result_dict2['grid_search_cv_results'] # for both experiments the ranking of the C hyperparameter should be # should be the identical since when we defined one_minus_precision # we set the `greater_is_better` keyword argument to `False` assert_array_equal(grid_results_dict1['rank_test_score'], grid_results_dict2['rank_test_score']) # furthermore, the final grid score and the mean scores for each # C hyperparameter value should follow the same 1-X relationship # except that our custom metric should be negated due to the # keyword argument that we set when we defined it assert_almost_equal(1 - grid_score2, -1 * grid_score1, places=6) assert_array_almost_equal( 1 - np.array(grid_results_dict2['mean_test_score']), -1 * np.array(grid_results_dict1['mean_test_score']), decimal=6)
def test_custom_metric_config_experiment_with_kappa_filename(): """Test config with metric defined in a file named kappa""" # Run experiment input_dir = join(_my_dir, "other") train_file = join(input_dir, "examples_train.jsonlines") test_file = join(input_dir, "examples_test.jsonlines") config_path = fill_in_config_paths_for_single_file( join(_my_dir, "configs", "test_custom_metrics_" "kappa.template.cfg"), train_file, test_file) run_configuration(config_path, local=True, quiet=True) # Check results for objective functions and output metrics # objective function f075_macro with open( join(_my_dir, 'output', ('test_custom_metrics_kappa_train_' 'examples_train.jsonlines_test_' 'examples_test.jsonlines_' 'LogisticRegression.results.json'))) as f: result_dict = json.load(f)[0] test_objective_value = result_dict['score'] test_output_metrics_dict = result_dict['additional_scores'] test_accuracy_value = test_output_metrics_dict["balanced_accuracy"] test_dummy_metric_value = test_output_metrics_dict["dummy_metric"] # check that the values are as expected assert_almost_equal(test_objective_value, 0.9699, places=4) assert_almost_equal(test_accuracy_value, 0.9792, places=4) eq_(test_dummy_metric_value, 1.0)
def test_folds_file_logging_num_folds(): """ Test when using `folds_file`, log shows number of folds and appropriate warning. """ # Run experiment suffix = '.jsonlines' train_path = join(_my_dir, 'train', 'f0{}'.format(suffix)) config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs", "test_folds_file" ".template.cfg"), train_path, None) run_configuration(config_path, quiet=True) # Check experiment log output with open(join(_my_dir, 'output', 'test_folds_file_logging.log')) as f: cv_file_pattern = re.compile('Specifying "folds_file" overrides both explicit and default "num_cv_folds".') matches = re.findall(cv_file_pattern, f.read()) assert_equal(len(matches), 1) # Check job log output with open(join(_my_dir, 'output', 'test_folds_file_logging_train_f0.' 'jsonlines_LogisticRegression.log')) as f: cv_folds_pattern = re.compile("(Task: cross_validate\n)(.+)(Cross-validating \([0-9]+ folds\))") matches = re.findall(cv_folds_pattern, f.read()) assert_equal(len(matches), 1)
def test_config_with_custom_prob_metric(): """Test config with custom probabilistic metric""" # run the first experiment that uses a custom probabilistic metric # for grid search but with a learner that does not produce probabilities input_dir = join(_my_dir, "other") train_file = join(input_dir, "examples_train.jsonlines") test_file = join(input_dir, "examples_test.jsonlines") config_path = fill_in_config_paths_for_single_file( join(_my_dir, "configs", "test_custom_" "metrics_kwargs3" ".template.cfg"), train_file, test_file) # this should fail as expected assert_raises_regex(AttributeError, r"has no attribute 'predict_proba'", run_configuration, config_path, local=True, quiet=True) # now run the second experiment that is identical except that # the learner now produces probabilities config_path = fill_in_config_paths_for_single_file( join(_my_dir, "configs", "test_custom_" "metrics_kwargs4" ".template.cfg"), train_file, test_file) # this should succeed and produce results run_configuration(config_path, local=True, quiet=True) # laod the results and verify them with open( join(_my_dir, 'output', ('test_custom_metrics_kwargs4_train_' 'examples_train.jsonlines_' 'SVC.results.json'))) as f: result_dict = json.load(f) grid_score = result_dict['grid_score'] ok_(grid_score > 0.95)
def test_custom_metric_config_with_invalid_custom_metric(): """Test config with a valid and an invalid custom metric""" # Run experiment input_dir = join(_my_dir, "other") train_file = join(input_dir, "examples_train.jsonlines") test_file = join(input_dir, "examples_test.jsonlines") config_path = fill_in_config_paths_for_single_file( join(_my_dir, "configs", "test_custom_metrics_bad" ".template.cfg"), train_file, test_file) # since this configuration file consists of an invalid # metric, this should raise an error assert_raises_regex(ValueError, r"invalid metrics specified:.*missing_metric", run_configuration, config_path, local=True, quiet=True)
def test_cross_validate_task(): """ Test that 10-fold cross_validate experiments work and fold ids get saved. """ # Run experiment suffix = '.jsonlines' train_path = join(_my_dir, 'train', 'f0{}'.format(suffix)) config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs", "test_save_cv_folds" ".template.cfg"), train_path, None) run_configuration(config_path, quiet=True) # Check final average results with open(join(_my_dir, 'output', 'test_save_cv_folds_train_f0.' + 'jsonlines_LogisticRegression.results.json')) as f: result_dict = json.load(f)[10] assert_almost_equal(result_dict['accuracy'], 0.517) # Check that the fold ids were saved correctly expected_skll_ids = {} examples = _load_featureset(train_path, '', suffix, quiet=True) kfold = StratifiedKFold(n_splits=10) for fold_num, (_, test_indices) in enumerate(kfold.split(examples.features, examples.labels)): for index in test_indices: expected_skll_ids[examples.ids[index]] = fold_num skll_fold_ids = {} with open(join(_my_dir, 'output', 'test_save_cv_folds_skll_fold_ids.csv')) as f: reader = csv.DictReader(f) for row in reader: skll_fold_ids[row['id']] = row['cv_test_fold'] # convert the dictionary to strings (sorted by key) for quick comparison skll_fold_ids_str = ''.join('{}{}'.format(key, val) for key, val in sorted(skll_fold_ids.items())) expected_skll_ids_str = ''.join('{}{}'.format(key, val) for key, val in sorted(expected_skll_ids.items())) assert_equal(skll_fold_ids_str, expected_skll_ids_str)
def test_cross_validate_task_save_cv_models(): """ Test that 10-fold cross_validate experiments work and that CV models are correctly saved. """ suffix = '.jsonlines' train_path = join(_my_dir, 'train', 'f0{}'.format(suffix)) config_path = \ fill_in_config_paths_for_single_file(join(_my_dir, "configs", "test_save_cv_models.template.cfg"), train_path, None) run_configuration(config_path, quiet=True) output_dir = join(_my_dir, 'output') cv_model_prefix = \ "test_save_cv_models_train_f0.jsonlines_LogisticRegression_fold" for i in range(1, 11): assert exists(join(output_dir, "{}{}.model".format(cv_model_prefix, i))) is True
def test_folds_file_logging_grid_search(): """ Test that, when `folds_file` is used but `use_folds_file` for grid search is specified, that we get an appropriate message in the log. """ # Run experiment suffix = '.jsonlines' train_path = join(_my_dir, 'train', 'f0{}'.format(suffix)) config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs", "test_folds_file_grid" ".template.cfg"), train_path, None) run_configuration(config_path, quiet=True) # Check experiment log output with open(join(_my_dir, 'output', 'test_folds_file_logging.log')) as f: cv_file_pattern = re.compile('Specifying "folds_file" overrides both explicit and default "num_cv_folds".\n(.+)The specified "folds_file" will not be used for inner grid search.') matches = re.findall(cv_file_pattern, f.read()) assert_equal(len(matches), 1)
def test_folds_file_with_fewer_ids_than_featureset(): """ Test when using `folds_file`, log shows warning for extra IDs in featureset. """ # Run experiment with a special featureset that has extra IDs suffix = '.jsonlines' train_path = join(_my_dir, 'train', 'f5{}'.format(suffix)) config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs", "test_folds_file" ".template.cfg"), train_path, None) run_configuration(config_path, quiet=True) # Check job log output with open(join(_my_dir, 'output', 'test_folds_file_logging_train_f5.' 'jsonlines_LogisticRegression.log')) as f: cv_file_pattern = re.compile('Feature set contains IDs that are not in folds dictionary. Skipping those IDs.') matches = re.findall(cv_file_pattern, f.read()) assert_equal(len(matches), 1)