def test_config_parsing_relative_input_paths():
train_dir = '../train'
train_file = join(train_dir, 'f0.jsonlines')
test_file = join(train_dir, 'f1.jsonlines')
output_dir = '../output'
custom_learner_path_input = join('other', 'majority_class_learner.py')
# make a simple config file that has relative paths
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_file': train_file,
'test_file': test_file,
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_micro'}
config_template_path = join(_my_dir, 'configs',
'test_relative_paths.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'relative_paths')
(experiment_name, task, sampler, fixed_sampler_parameters,
feature_hasher, hasher_features, id_col, label_col, train_set_name,
test_set_name, suffix, featuresets, do_shuffle, model_path,
do_grid_search, grid_objective, probability, results_path,
pos_label_str, feature_scaling, min_feature_count,
grid_search_jobs, grid_search_folds, cv_folds, save_cv_folds, do_stratified_folds,
fixed_parameter_list, param_grid_list, featureset_names, learners,
prediction_dir, log_path, train_path, test_path, ids_to_floats,
class_map, custom_learner_path) = _parse_config_file(config_path)
def test_config_parsing_bad_cv_folds():
"""
Test to ensure config file parsing raises an error with an invalid cv_folds
"""
train_dir = join(_my_dir, 'train')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad value for cv_folds
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'num_cv_folds': 'random',
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_macro'}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'bad_cv_folds')
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_featurenames():
# Test to ensure config file parsing raises an error with badly specified
# featureset names
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir,
'results': output_dir
}
for fname, sub_prefix in zip(["['set_a']", "['1', 2]", "set_a", "1"], [
'wrong_num_names', 'wrong_type_names', 'wrong_num_and_type1',
'wrong_num_and_type2'
]):
if fname is not None:
values_to_fill_dict['featureset_names'] = fname
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_hashing():
"""
Test to ensure config file parsing raises an error when feature_hasher is specified but not hasher_features
"""
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'feature_hasher': 'True'}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'bad_hashing')
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_cv_folds():
"""
Test to ensure config file parsing raises an error with an invalid cv_folds
"""
train_dir = join(_my_dir, 'train')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad value for cv_folds
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'num_cv_folds': 'random',
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_macro'
}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, 'bad_cv_folds')
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_objective_and_objectives():
"""
Test to ensure config file parsing raises an error with
a grid objectives and objective both given non default values
"""
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
values_to_fill_dict = {
'train_directory': train_dir,
'log': output_dir,
'results': output_dir,
'objectives': "['accuracy']",
'objective': "accuracy"
}
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'bad_objective_and_objectives')
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_objectives():
"""
Test to ensure config file parsing raises an error with a grid objectives given as a string
"""
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objectives': "accuracy"}
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'bad_objectives')
yield check_config_parsing_type_error, config_path
def test_default_learning_curve_options():
train_dir = join(_my_dir, 'train')
output_dir = join(_my_dir, 'output')
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'learning_curve',
'train_directory': train_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression', 'MultinomialNB']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_macro'
}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'default_learning_curve')
(experiment_name, task, sampler, fixed_sampler_parameters, feature_hasher,
hasher_features, id_col, label_col, train_set_name, test_set_name, suffix,
featuresets, do_shuffle, model_path, do_grid_search, grid_objective,
probability, results_path, pos_label_str, feature_scaling,
min_feature_count, grid_search_jobs, grid_search_folds, cv_folds,
save_cv_folds, do_stratified_folds, fixed_parameter_list, param_grid_list,
featureset_names, learners, prediction_dir, log_path, train_path,
test_path, ids_to_floats, class_map, custom_learner_path,
learning_curve_cv_folds_list,
learning_curve_train_sizes) = _parse_config_file(config_path)
eq_(learning_curve_cv_folds_list, [10, 10])
ok_(np.all(learning_curve_train_sizes == np.linspace(0.1, 1.0, 5)))
def test_config_parsing_bad_objectives():
"""
Test to ensure config file parsing raises an error with a grid objectives given as a string
"""
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objectives': "accuracy"
}
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, 'bad_objectives')
yield check_config_parsing_type_error, config_path
def test_config_parsing_relative_input_path():
train_dir = join('..', 'train')
output_dir = join(_my_dir, 'output')
# make a simple config file that has an invalid option
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_macro'
}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'mislocated_input_file')
(experiment_name, task, sampler, fixed_sampler_parameters, feature_hasher,
hasher_features, id_col, label_col, train_set_name, test_set_name, suffix,
featuresets, do_shuffle, model_path, do_grid_search, grid_objective,
probability, results_path, pos_label_str, feature_scaling,
min_feature_count, grid_search_jobs, grid_search_folds, cv_folds,
save_cv_folds, do_stratified_folds, fixed_parameter_list, param_grid_list,
featureset_names, learners, prediction_dir, log_path, train_path,
test_path, ids_to_floats, class_map,
custom_learner_path) = _parse_config_file(config_path)
eq_(normpath(train_path), (join(_my_dir, 'train')))
def test_config_parsing_bad_task():
# Test to ensure config file parsing raises an error with invalid or
# missing task
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir}
for task_value, sub_prefix in zip([None, '', 'procrastinate'],
['no_task', 'missing_task', 'bad_task']):
if task_value is not None:
values_to_fill_dict['task'] = task_value
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_hashing():
"""
Test to ensure config file parsing raises an error when feature_hasher is specified but not hasher_features
"""
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'feature_hasher': 'True'
}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, 'bad_hashing')
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_featuresets():
# Test to ensure config file parsing raises an error with badly specified
# featuresets
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir
}
for featuresets, sub_prefix in zip(
[None, '[]', "{'f1', 'f2', 'f3'}", "[['f1', 'f2'], 'f3', 'f4']"],
['no_feats', 'empty_feats', 'non_list_feats1', 'non_list_feats2']):
if featuresets is not None:
values_to_fill_dict['featuresets'] = featuresets
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_task():
# Test to ensure config file parsing raises an error with invalid or
# missing task
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir
}
for task_value, sub_prefix in zip([None, '', 'procrastinate'],
['no_task', 'missing_task', 'bad_task']):
if task_value is not None:
values_to_fill_dict['task'] = task_value
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_learner():
# Test to ensure config file parsing raises an error with missing, bad and
# duplicate learners
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'log': output_dir,
'results': output_dir
}
for learners_list, sub_prefix in zip([
None, '[]', 'LogisticRegression', "['LogisticRegression', "
"'LogisticRegression']"
], [
'no_learner', 'empty_learner', 'not_list_learner',
'duplicate_learner'
]):
if learners_list is not None:
values_to_fill_dict['learners'] = learners_list
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_featurenames():
# Test to ensure config file parsing raises an error with badly specified
# featureset names
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir,
'results': output_dir}
for fname, sub_prefix in zip(["['set_a']", "['1', 2]", "set_a", "1"],
['wrong_num_names', 'wrong_type_names',
'wrong_num_and_type1',
'wrong_num_and_type2']):
if fname is not None:
values_to_fill_dict['featureset_names'] = fname
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_scaling():
# Test to ensure config file parsing raises an error with invalid scaling
# type
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir,
'results': output_dir}
for scaling_type, sub_prefix in zip(["foo", "True", "False"],
['bad_scaling1', 'bad_scaling2',
'bad_scaling3']):
values_to_fill_dict['feature_scaling'] = scaling_type
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_learner():
# Test to ensure config file parsing raises an error with missing, bad and
# duplicate learners
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'log': output_dir,
'results': output_dir}
for learners_list, sub_prefix in zip([None, '[]', 'LogisticRegression',
"['LogisticRegression', "
"'LogisticRegression']"],
['no_learner', 'empty_learner',
'not_list_learner',
'duplicate_learner']):
if learners_list is not None:
values_to_fill_dict['learners'] = learners_list
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_option_in_wrong_section():
train_dir = join(_my_dir, 'train')
output_dir = join(_my_dir, 'output')
# make a simple config file that has an option in the wrong section
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'probability': 'true',
'objective': 'f1_score_macro'
}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'option_in_wrong_section')
yield check_config_parsing_key_error, config_path
def test_config_parsing_bad_featuresets():
# Test to ensure config file parsing raises an error with badly specified
# featuresets
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir}
for featuresets, sub_prefix in zip([None, '[]', "{'f1', 'f2', 'f3'}",
"[['f1', 'f2'], 'f3', 'f4']"],
['no_feats', 'empty_feats',
'non_list_feats1', 'non_list_feats2']):
if featuresets is not None:
values_to_fill_dict['featuresets'] = featuresets
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_mislocated_input_path():
train_dir = 'train'
output_dir = join(_my_dir, 'output')
# make a simple config file that has an invalid option
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_macro'
}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'mislocated_input_file')
yield check_config_parsing_file_not_found_error, config_path
def test_config_parsing_bad_scaling():
# Test to ensure config file parsing raises an error with invalid scaling
# type
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir,
'results': output_dir
}
for scaling_type, sub_prefix in zip(
["foo", "True", "False"],
['bad_scaling1', 'bad_scaling2', 'bad_scaling3']):
values_to_fill_dict['feature_scaling'] = scaling_type
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, sub_prefix)
yield check_config_parsing_value_error, config_path
def test_config_parsing_relative_input_paths():
train_dir = '../train'
train_file = join(train_dir, 'f0.jsonlines')
test_file = join(train_dir, 'f1.jsonlines')
output_dir = '../output'
custom_learner_path_input = join('other', 'majority_class_learner.py')
# make a simple config file that has relative paths
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_file': train_file,
'test_file': test_file,
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_micro'
}
config_template_path = join(_my_dir, 'configs',
'test_relative_paths.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, 'relative_paths')
(experiment_name, task, sampler, fixed_sampler_parameters, feature_hasher,
hasher_features, id_col, label_col, train_set_name, test_set_name, suffix,
featuresets, do_shuffle, model_path, do_grid_search, grid_objective,
probability, results_path, pos_label_str, feature_scaling,
min_feature_count, grid_search_jobs, grid_search_folds, cv_folds,
save_cv_folds, do_stratified_folds, fixed_parameter_list, param_grid_list,
featureset_names, learners, prediction_dir, log_path, train_path,
test_path, ids_to_floats, class_map,
custom_learner_path) = _parse_config_file(config_path)
def test_setting_number_of_cv_folds():
train_dir = join(_my_dir, 'train')
output_dir = join(_my_dir, 'output')
# make a simple config file that does not set cv_folds
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'num_cv_folds': "5",
'objective': 'f1_score_macro'}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'default_cv_folds')
(experiment_name, task, sampler, fixed_sampler_parameters,
feature_hasher, hasher_features, id_col, label_col, train_set_name,
test_set_name, suffix, featuresets, do_shuffle, model_path,
do_grid_search, grid_objective, probability, results_path,
pos_label_str, feature_scaling, min_feature_count,
grid_search_jobs, grid_search_folds, cv_folds, save_cv_folds, do_stratified_folds,
fixed_parameter_list, param_grid_list, featureset_names, learners,
prediction_dir, log_path, train_path, test_path, ids_to_floats,
class_map, custom_learner_path) = _parse_config_file(config_path)
eq_(cv_folds, 5)
def test_config_parsing_bad_test():
# Test to ensure config file parsing raises an error with invalid test path
# specifications
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir,
'results': output_dir
}
for sub_prefix in [
'both_test_path_and_file', 'nonexistent_test_path',
'nonexistent_test_file'
]:
if sub_prefix == 'both_test_path_and_file':
test_fh = tempfile.NamedTemporaryFile(suffix='jsonlines',
prefix=join(
_my_dir, 'other',
('test_config_'
'parsing_')))
values_to_fill_dict['test_file'] = test_fh.name
values_to_fill_dict['test_directory'] = test_dir
elif sub_prefix == 'nonexistent_test_path':
values_to_fill_dict['test_directory'] = join(test_dir, 'foo')
elif sub_prefix == 'nonexistent_test_file':
values_to_fill_dict['test_file'] = 'foo.jsonlines'
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, sub_prefix)
yield check_config_parsing_value_error, config_path
if sub_prefix == 'both_test_path_and_file':
test_fh.close()
def test_config_parsing_bad_train():
# Test to ensure config file parsing raises an error with invalid train
# path specifications
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'test_directory': test_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir,
'results': output_dir}
for sub_prefix in ['no_train_path_or_file',
'both_train_path_and_file',
'nonexistent_train_path',
'nonexistent_test_file']:
if sub_prefix == 'both_train_path_and_file':
train_fh = tempfile.NamedTemporaryFile(suffix='jsonlines',
prefix=join(_my_dir,
'other',
('test_config_'
'parsing_')))
values_to_fill_dict['train_file'] = train_fh.name
values_to_fill_dict['train_directory'] = train_dir
elif sub_prefix == 'nonexistent_train_path':
values_to_fill_dict['train_directory'] = join(train_dir, 'foo')
elif sub_prefix == 'nonexistent_test_file':
values_to_fill_dict['train_file'] = 'foo.jsonlines'
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
sub_prefix)
yield check_config_parsing_value_error, config_path
if sub_prefix == 'both_train_path_and_file':
train_fh.close()
def test_setting_fixed_parameters():
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that does not set cv_folds
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LinearSVC']",
'log': output_dir,
'results': output_dir,
'fixed_parameters': "[{'C': [1e-6, 0.001, 1, 10, 100, 1e5]}]",
'objective': 'f1_score_macro'
}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'fixed_parameters')
(experiment_name, task, sampler, fixed_sampler_parameters, feature_hasher,
hasher_features, id_col, label_col, train_set_name, test_set_name, suffix,
featuresets, do_shuffle, model_path, do_grid_search, grid_objective,
probability, results_path, pos_label_str, feature_scaling,
min_feature_count, grid_search_jobs, grid_search_folds, cv_folds,
save_cv_folds, do_stratified_folds, fixed_parameter_list, param_grid_list,
featureset_names, learners, prediction_dir, log_path, train_path,
test_path, ids_to_floats, class_map, custom_learner_path,
learning_curve_cv_folds_list,
learning_curve_train_sizes) = _parse_config_file(config_path)
eq_(fixed_parameter_list[0]['C'][0], 1e-6)
eq_(fixed_parameter_list[0]['C'][1], 1e-3)
eq_(fixed_parameter_list[0]['C'][2], 1)
eq_(fixed_parameter_list[0]['C'][3], 10)
eq_(fixed_parameter_list[0]['C'][4], 100)
eq_(fixed_parameter_list[0]['C'][5], 1e5)
def test_setting_fixed_parameters():
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that does not set cv_folds
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'evaluate',
'train_directory': train_dir,
'test_directory': test_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LinearSVC']",
'log': output_dir,
'results': output_dir,
'fixed_parameters': "[{'C': [1e-6, 0.001, 1, 10, 100, 1e5]}]",
'objective': 'f1_score_macro'}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'fixed_parameters')
(experiment_name, task, sampler, fixed_sampler_parameters,
feature_hasher, hasher_features, id_col, label_col, train_set_name,
test_set_name, suffix, featuresets, do_shuffle, model_path,
do_grid_search, grid_objective, probability, results_path,
pos_label_str, feature_scaling, min_feature_count,
grid_search_jobs, grid_search_folds, cv_folds, save_cv_folds, do_stratified_folds,
fixed_parameter_list, param_grid_list, featureset_names, learners,
prediction_dir, log_path, train_path, test_path, ids_to_floats,
class_map, custom_learner_path) = _parse_config_file(config_path)
eq_(fixed_parameter_list[0]['C'][0], 1e-6)
eq_(fixed_parameter_list[0]['C'][1], 1e-3)
eq_(fixed_parameter_list[0]['C'][2], 1)
eq_(fixed_parameter_list[0]['C'][3], 10)
eq_(fixed_parameter_list[0]['C'][4], 100)
eq_(fixed_parameter_list[0]['C'][5], 1e5)
def test_config_parsing_mislocated_input_path():
train_dir = 'train'
output_dir = join(_my_dir, 'output')
# make a simple config file that has an invalid option
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'objective': 'f1_score_macro'}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'mislocated_input_file')
yield check_config_parsing_file_not_found_error, config_path
def test_config_parsing_option_in_wrong_section():
train_dir = join(_my_dir, 'train')
output_dir = join(_my_dir, 'output')
# make a simple config file that has an option in the wrong section
values_to_fill_dict = {'experiment_name': 'config_parsing',
'task': 'cross_validate',
'train_directory': train_dir,
'featuresets': "[['f1', 'f2', 'f3']]",
'learners': "['LogisticRegression']",
'log': output_dir,
'results': output_dir,
'probability': 'true',
'objective': 'f1_score_macro'}
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'option_in_wrong_section')
yield check_config_parsing_key_error, config_path
def test_config_parsing_bad_objective_and_objectives():
"""
Test to ensure config file parsing raises an error with
a grid objectives and objective both given non default values
"""
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
values_to_fill_dict = {'train_directory': train_dir,
'log': output_dir,
'results': output_dir,
'objectives': "['accuracy']",
'objective': "accuracy"}
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'bad_objective_and_objectives')
yield check_config_parsing_value_error, config_path
def test_config_parsing_bad_task_paths():
# Test to ensure config file parsing raises an error with various
# incorrectly set path
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {
'experiment_name': 'config_parsing',
'train_directory': train_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir
}
for sub_prefix in [
'predict_no_test', 'evaluate_no_test', 'xv_with_test_path',
'train_with_test_path', 'xv_with_test_file',
'train_with_test_file', 'train_with_results',
'predict_with_results', 'train_no_model', 'train_with_predictions',
'xv_with_model'
]:
if sub_prefix == 'predict_no_test':
values_to_fill_dict['task'] = 'predict'
values_to_fill_dict['predictions'] = output_dir
elif sub_prefix == 'evaluate_no_test':
values_to_fill_dict['task'] = 'evaluate'
values_to_fill_dict['results'] = output_dir
elif sub_prefix == 'xv_with_test_path':
values_to_fill_dict['task'] = 'cross_validate'
values_to_fill_dict['results'] = output_dir
values_to_fill_dict['test_directory'] = test_dir
elif sub_prefix == 'train_with_test_path':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
values_to_fill_dict['test_directory'] = test_dir
elif sub_prefix == 'xv_with_test_file':
values_to_fill_dict['task'] = 'cross_validate'
values_to_fill_dict['results'] = output_dir
test_fh1 = tempfile.NamedTemporaryFile(suffix='jsonlines',
prefix=join(
_my_dir, 'other',
('test_config_'
'parsing_')))
values_to_fill_dict['test_file'] = test_fh1.name
elif sub_prefix == 'train_with_test_file':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
test_fh2 = tempfile.NamedTemporaryFile(suffix='jsonlines',
prefix=join(
_my_dir, 'other',
('test_config_'
'parsing_')))
values_to_fill_dict['test_file'] = test_fh2.name
elif sub_prefix == 'train_with_results':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
values_to_fill_dict['results'] = output_dir
elif sub_prefix == 'predict_with_results':
values_to_fill_dict['task'] = 'predict'
values_to_fill_dict['test_directory'] = test_dir
values_to_fill_dict['predictions'] = output_dir
values_to_fill_dict['results'] = output_dir
elif sub_prefix == 'train_no_model':
values_to_fill_dict['task'] = 'train'
elif sub_prefix == 'train_with_predictions':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
values_to_fill_dict['predictions'] = output_dir
elif sub_prefix == 'xv_with_model':
values_to_fill_dict['task'] = 'cross_validate'
values_to_fill_dict['results'] = output_dir
values_to_fill_dict['models'] = output_dir
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, sub_prefix)
yield check_config_parsing_value_error, config_path
if sub_prefix == 'xv_with_test_file':
test_fh1.close()
elif sub_prefix == 'train_with_test_file':
test_fh2.close()
def test_config_parsing_bad_task_paths():
# Test to ensure config file parsing raises an error with various
# incorrectly set path
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
# make a simple config file that has a bad task
# but everything else is correct
values_to_fill_dict = {'experiment_name': 'config_parsing',
'train_directory': train_dir,
'learners': "['LogisticRegression']",
'featuresets': ("[['f1', 'f2', 'f3'], ['f4', 'f5', "
"'f6']]"),
'log': output_dir}
for sub_prefix in ['predict_no_test', 'evaluate_no_test',
'xv_with_test_path', 'train_with_test_path',
'xv_with_test_file', 'train_with_test_file',
'train_with_results', 'predict_with_results',
'train_no_model', 'train_with_predictions',
'xv_with_model']:
if sub_prefix == 'predict_no_test':
values_to_fill_dict['task'] = 'predict'
values_to_fill_dict['predictions'] = output_dir
elif sub_prefix == 'evaluate_no_test':
values_to_fill_dict['task'] = 'evaluate'
values_to_fill_dict['results'] = output_dir
elif sub_prefix == 'xv_with_test_path':
values_to_fill_dict['task'] = 'cross_validate'
values_to_fill_dict['results'] = output_dir
values_to_fill_dict['test_directory'] = test_dir
elif sub_prefix == 'train_with_test_path':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
values_to_fill_dict['test_directory'] = test_dir
elif sub_prefix == 'xv_with_test_file':
values_to_fill_dict['task'] = 'cross_validate'
values_to_fill_dict['results'] = output_dir
test_fh1 = tempfile.NamedTemporaryFile(suffix='jsonlines',
prefix=join(_my_dir,
'other',
('test_config_'
'parsing_')))
values_to_fill_dict['test_file'] = test_fh1.name
elif sub_prefix == 'train_with_test_file':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
test_fh2 = tempfile.NamedTemporaryFile(suffix='jsonlines',
prefix=join(_my_dir,
'other',
('test_config_'
'parsing_')))
values_to_fill_dict['test_file'] = test_fh2.name
elif sub_prefix == 'train_with_results':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
values_to_fill_dict['results'] = output_dir
elif sub_prefix == 'predict_with_results':
values_to_fill_dict['task'] = 'predict'
values_to_fill_dict['test_directory'] = test_dir
values_to_fill_dict['predictions'] = output_dir
values_to_fill_dict['results'] = output_dir
elif sub_prefix == 'train_no_model':
values_to_fill_dict['task'] = 'train'
elif sub_prefix == 'train_with_predictions':
values_to_fill_dict['task'] = 'train'
values_to_fill_dict['models'] = output_dir
values_to_fill_dict['predictions'] = output_dir
elif sub_prefix == 'xv_with_model':
values_to_fill_dict['task'] = 'cross_validate'
values_to_fill_dict['results'] = output_dir
values_to_fill_dict['models'] = output_dir
config_template_path = join(_my_dir, 'configs',
'test_config_parsing.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
sub_prefix)
yield check_config_parsing_value_error, config_path
if sub_prefix == 'xv_with_test_file':
test_fh1.close()
elif sub_prefix == 'train_with_test_file':
test_fh2.close()
def check_xval_fancy_results_file(do_grid_search,
use_folds_file,
use_folds_file_for_grid_search,
use_additional_metrics):
train_path = join(_my_dir, 'train', 'f0.jsonlines')
output_dir = join(_my_dir, 'output')
# make a simple config file for cross-validation
values_to_fill_dict = {'experiment_name': 'test_fancy_xval',
'train_file': train_path,
'task': 'cross_validate',
'grid_search': 'true',
'objectives': "['f1_score_micro']",
'featureset_names': '["f0"]',
'num_cv_folds': '6',
'grid_search_folds': '4',
'learners': "['LogisticRegression']",
'log': output_dir,
'predictions': output_dir,
'results': output_dir}
folds_file_path = join(_my_dir, 'train', 'folds_file_test.csv')
if use_folds_file:
values_to_fill_dict['folds_file'] = folds_file_path
values_to_fill_dict['grid_search'] = str(do_grid_search)
values_to_fill_dict['use_folds_file_for_grid_search'] = str(use_folds_file_for_grid_search)
if use_additional_metrics:
if PY2:
values_to_fill_dict['metrics'] = str([b"accuracy", b"unweighted_kappa"])
else:
values_to_fill_dict['metrics'] = str(["accuracy", "unweighted_kappa"])
config_template_path = join(_my_dir,
'configs',
'test_fancy.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'xval')
# run the experiment
run_configuration(config_path, quiet=True)
# now make sure that the results file was produced
results_file_path = join(_my_dir, 'output', 'test_fancy_xval_f0_LogisticRegression.results')
ok_(exists(results_file_path))
# read in all the lines and look at the lines up to where we print the "Total Time"
with open(results_file_path, 'r') as resultsf:
results_lines = resultsf.readlines()
end_idx = [results_lines.index(l) for l in results_lines if l.startswith('Total Time:')][0]
results_lines = results_lines[:end_idx+1]
# read in the "keys" and "values" separated by colons into a dictionary
results_dict = dict([rl.strip().split(': ') for rl in results_lines])
# check that the fields we expect in the results file are there
# and the ones that we do not expect aren't
if do_grid_search:
eq_(results_dict['Grid Search'], 'True')
eq_(results_dict['Grid Objective Function'], 'f1_score_micro')
else:
eq_(results_dict['Grid Search'], 'False')
ok_('Grid Search Folds' not in results_dict)
ok_('Grid Objective Function' not in results_dict)
if use_folds_file:
eq_(results_dict['Number of Folds'], '5 via folds file')
ok_('Stratified Folds' not in results_dict)
eq_(results_dict['Specified Folds File'], folds_file_path)
if do_grid_search:
if use_folds_file_for_grid_search:
eq_(results_dict['Grid Search Folds'], '5 via folds file')
eq_(results_dict['Using Folds File for Grid Search'], 'True')
else:
eq_(results_dict['Grid Search Folds'], '4')
eq_(results_dict['Using Folds File for Grid Search'], 'False')
else:
eq_(results_dict['Number of Folds'], '6')
eq_(results_dict['Stratified Folds'], 'True')
ok_('Using Folds File for Grid Search' not in results_dict)
ok_('Specified Folds File' not in results_dict)
if do_grid_search:
eq_(results_dict['Grid Search Folds'], '4')
if use_additional_metrics:
expected_metrics = [b"accuracy", b"unweighted_kappa"] if PY2 else ["accuracy", "unweighted_kappa"]
eq_(sorted(literal_eval(results_dict['Additional Evaluation Metrics'])),
sorted(expected_metrics))
def check_grid_search_cv_results(task, do_grid_search):
learners = ['LogisticRegression', 'SVC']
expected_path = join(_my_dir, 'other', 'cv_results')
time_field = lambda x: x.endswith('_time')
train_path = join(_my_dir, 'train', 'f0.jsonlines')
output_dir = join(_my_dir, 'output')
exp_name = ('test_grid_search_cv_results_{}_{}'
.format(task, "gs" if do_grid_search else "nogs"))
# make a simple config file for cross-validation
values_to_fill_dict = {'experiment_name': exp_name,
'train_file': train_path,
'task': task,
'grid_search': json.dumps(do_grid_search),
'objectives': "['f1_score_micro']",
'featureset_names': "['f0']",
'learners': '{}'.format(json.dumps(learners)),
'log': output_dir,
'results': output_dir}
if task == 'train':
values_to_fill_dict['models'] = output_dir
elif task == 'cross_validate':
values_to_fill_dict['predictions'] = output_dir
elif task in ['evaluate', 'predict']:
values_to_fill_dict['predictions'] = output_dir
values_to_fill_dict['test_file'] = \
values_to_fill_dict['train_file']
# In the case where grid search is on and the task is
# learning curve, grid search will automatically be turned
# off, so simply turn it off here as well since it should
# result in the same situation
elif task == 'learning_curve':
values_to_fill_dict['metrics'] = values_to_fill_dict.pop('objectives')
if do_grid_search:
do_grid_search = False
config_template_path = join(_my_dir,
'configs',
'test_cv_results.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
task)
# run the experiment
if task in ['train', 'predict']:
if do_grid_search:
run_configuration(config_path, quiet=True)
else:
assert_raises(ValueError, run_configuration, config_path, quiet=True)
# Short-circuit the test since a ValueError is
# expected and is fatal
return
else:
run_configuration(config_path, quiet=True)
# now make sure that the results json file was produced
for learner in learners:
results_file_name = ('{}_f0_{}.results.json'
.format(exp_name, learner))
actual_results_file_path = join(_my_dir, 'output',
results_file_name)
expected_results_file_path = join(expected_path,
results_file_name)
ok_(exists(actual_results_file_path))
with open(expected_results_file_path) as expected, \
open(actual_results_file_path) as actual:
expected_lines = [json.loads(line) for line in expected][0]
actual_lines = [json.loads(line) for line in actual][0]
assert len(expected_lines) == len(actual_lines)
if task == 'cross_validate':
# All but the last line will have grid search-related
# results
for (expected_gs_cv_results,
actual_gs_cv_results) in zip(expected_lines[:-1],
actual_lines[:-1]):
assert len(expected_gs_cv_results) == len(actual_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if do_grid_search:
assert (set(expected_gs_cv_results)
.intersection(actual_gs_cv_results) ==
set(expected_gs_cv_results))
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
else:
if field == 'grid_score':
assert actual_gs_cv_results[field] == 0.0
else:
assert actual_gs_cv_results[field] is None
# The last line should be for the "average" and should
# not contain any grid search results
assert actual_lines[-1]['fold'] == 'average'
for field in ['grid_score', 'grid_search_cv_results']:
assert field not in actual_lines[-1]
elif task == 'evaluate':
for (expected_gs_cv_results,
actual_gs_cv_results) in zip(expected_lines,
actual_lines):
assert len(expected_gs_cv_results) == len(actual_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if do_grid_search:
assert (set(expected_gs_cv_results)
.intersection(actual_gs_cv_results) ==
set(expected_gs_cv_results))
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
else:
if field == 'grid_score':
assert actual_gs_cv_results[field] == 0.0
else:
assert actual_gs_cv_results[field] is None
elif task in ['train', 'predict']:
expected_gs_cv_results = expected_lines
actual_gs_cv_results = actual_lines
assert set(expected_gs_cv_results).intersection(actual_gs_cv_results) == \
set(expected_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
else:
for expected_line, actual_line in zip(expected_lines,
actual_lines):
expected_fields = set(list(expected_line))
actual_fields = set(list(actual_line))
assert expected_fields.intersection(actual_fields) == \
expected_fields
assert all(field not in actual_fields
for field in ['grid_score',
'grid_search_cv_results'])
def check_xval_fancy_results_file(do_grid_search, use_folds_file,
use_folds_file_for_grid_search,
use_additional_metrics):
train_path = join(_my_dir, 'train', 'f0.jsonlines')
output_dir = join(_my_dir, 'output')
# make a simple config file for cross-validation
values_to_fill_dict = {
'experiment_name': 'test_fancy_xval',
'train_file': train_path,
'task': 'cross_validate',
'featureset_names': '["f0"]',
'num_cv_folds': '6',
'grid_search_folds': '4',
'learners': "['LogisticRegression']",
'log': output_dir,
'predictions': output_dir,
'results': output_dir
}
folds_file_path = join(_my_dir, 'train', 'folds_file_test.csv')
if use_folds_file:
values_to_fill_dict['folds_file'] = folds_file_path
values_to_fill_dict['grid_search'] = str(do_grid_search)
values_to_fill_dict['use_folds_file_for_grid_search'] = str(
use_folds_file_for_grid_search)
if use_additional_metrics:
if PY2:
values_to_fill_dict['metrics'] = str(
[b"accuracy", b"unweighted_kappa"])
else:
values_to_fill_dict['metrics'] = str(
["accuracy", "unweighted_kappa"])
config_template_path = join(_my_dir, 'configs', 'test_fancy.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict, 'xval')
# run the experiment
run_configuration(config_path, quiet=True)
# now make sure that the results file was produced
results_file_path = join(_my_dir, 'output',
'test_fancy_xval_f0_LogisticRegression.results')
ok_(exists(results_file_path))
# read in all the lines and look at the lines up to where we print the "Total Time"
with open(results_file_path, 'r') as resultsf:
results_lines = resultsf.readlines()
end_idx = [
results_lines.index(l) for l in results_lines
if l.startswith('Total Time:')
][0]
results_lines = results_lines[:end_idx + 1]
# read in the "keys" and "values" separated by colons into a dictionary
results_dict = dict([rl.strip().split(': ') for rl in results_lines])
# check that the fields we expect in the results file are there
# and the ones that we do not expect aren't
if do_grid_search:
eq_(results_dict['Grid Search'], 'True')
eq_(results_dict['Grid Objective Function'], 'f1_score_micro')
else:
eq_(results_dict['Grid Search'], 'False')
ok_('Grid Search Folds' not in results_dict)
ok_('Grid Objective Function' not in results_dict)
if use_folds_file:
eq_(results_dict['Number of Folds'], '5 via folds file')
ok_('Stratified Folds' not in results_dict)
eq_(results_dict['Specified Folds File'], folds_file_path)
if do_grid_search:
if use_folds_file_for_grid_search:
eq_(results_dict['Grid Search Folds'], '5 via folds file')
eq_(results_dict['Using Folds File for Grid Search'], 'True')
else:
eq_(results_dict['Grid Search Folds'], '4')
eq_(results_dict['Using Folds File for Grid Search'], 'False')
else:
eq_(results_dict['Number of Folds'], '6')
eq_(results_dict['Stratified Folds'], 'True')
ok_('Using Folds File for Grid Search' not in results_dict)
ok_('Specified Folds File' not in results_dict)
if do_grid_search:
eq_(results_dict['Grid Search Folds'], '4')
if use_additional_metrics:
expected_metrics = [b"accuracy", b"unweighted_kappa"
] if PY2 else ["accuracy", "unweighted_kappa"]
eq_(
sorted(literal_eval(
results_dict['Additional Evaluation Metrics'])),
sorted(expected_metrics))