def test_ablation_cv_feature_hasher_all_combos_sampler():
"""
Test to validate whether ablation works with cross-validate
and feature_hasher
"""
make_ablation_data()
config_template_path = join(_my_dir, 'configs', ('test_ablation_feature_'
'hasher_sampler.template'
'.cfg'))
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True, ablation=None)
# read in the summary file and make sure it has
# 10 ablated featuresets * (10 folds + 1 average line) * 2 learners = 220
# lines
with open(join(_my_dir, 'output',
'ablation_cv_feature_hasher_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
num_rows = check_ablation_rows(reader)
eq_(num_rows, 220)
# make sure there are 10 ablated featuresets * 2 learners = 20 results
# files
num_result_files = len(glob.glob(join(_my_dir, 'output',
('ablation_cv_feature_hasher_'
'*results'))))
eq_(num_result_files, 20)
def check_specified_cv_folds(numeric_ids):
make_cv_folds_data(numeric_ids)
# test_cv_folds1.cfg has prespecified folds and should have ~50% accuracy
# test_cv_folds2.cfg doesn't have prespecified folds and >95% accuracy
for experiment_name, test_func, grid_size in [('test_cv_folds1',
lambda x: x < 0.6,
3),
('test_cv_folds2',
lambda x: x > 0.95,
10)]:
config_template_file = '{}.template.cfg'.format(experiment_name)
config_template_path = os.path.join(_my_dir, 'configs',
config_template_file)
config_path = os.path.join(_my_dir,
fill_in_config_paths(config_template_path))
# Modify config file to change ids_to_floats depending on numeric_ids
# setting
with open(config_path, 'r+') as config_template_file:
lines = config_template_file.readlines()
config_template_file.seek(0)
config_template_file.truncate()
for line in lines:
if line.startswith('ids_to_floats='):
if numeric_ids:
line = 'ids_to_floats=true\n'
else:
line = 'ids_to_floats=false\n'
config_template_file.write(line)
run_configuration(config_path, quiet=True)
result_filename = ('{}_test_cv_folds_LogisticRegression.' +
'results').format(experiment_name)
with open(os.path.join(_my_dir, 'output', result_filename)) as f:
# check held out scores
outstr = f.read()
score = float(SCORE_OUTPUT_RE.search(outstr).groups()[-1])
assert test_func(score)
grid_score_matches = GRID_RE.findall(outstr)
assert len(grid_score_matches) == grid_size
for match_str in grid_score_matches:
assert test_func(float(match_str))
# try the same tests for just training (and specifying the folds for the
# grid search)
dirpath = os.path.join(_my_dir, 'train')
suffix = '.jsonlines'
featureset = ['test_cv_folds']
examples = _load_featureset(dirpath, featureset, suffix, quiet=True)
clf = Learner('LogisticRegression', probability=True)
cv_folds = _load_cv_folds(os.path.join(_my_dir, 'train',
'test_cv_folds.csv'))
grid_search_score = clf.train(examples, grid_search_folds=cv_folds,
grid_objective='accuracy', grid_jobs=1)
assert grid_search_score < 0.6
grid_search_score = clf.train(examples, grid_search_folds=5,
grid_objective='accuracy', grid_jobs=1)
assert grid_search_score > 0.95
def test_train_file_test_file_ablation():
"""
Test that specifying ablation with train and test file is ignored
"""
# Create data files
make_single_file_featureset_data()
# Run experiment
config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs",
"test_single_file"
".template.cfg"),
join(_my_dir, 'train',
'train_single_file'
'.jsonlines'),
join(_my_dir, 'test',
'test_single_file.'
'jsonlines'))
run_configuration(config_path, quiet=True, ablation=None)
# check that we see the message that ablation was ignored in the experiment log
# Check experiment log output
with open(join(_my_dir,
'output',
'train_test_single_file.log')) as f:
cv_file_pattern = re.compile('Not enough featuresets for ablation. Ignoring.')
matches = re.findall(cv_file_pattern, f.read())
eq_(len(matches), 1)
def test_predict_on_subset_with_existing_model():
"""
Test generating predictions on subset with existing model
"""
# Create data files
make_single_file_featureset_data()
# train and save a model on the training file
train_fs = NDJReader.for_path(join(_my_dir, 'train', 'train_single_file.jsonlines')).read()
learner = Learner('RandomForestClassifier')
learner.train(train_fs, grid_search=True, grid_objective="accuracy")
model_filename = join(_my_dir, 'output', ('train_test_single_file_train_train_'
'single_file.jsonlines_test_test_single'
'_file_subset.jsonlines_RandomForestClassifier'
'.model'))
learner.save(model_filename)
# Run experiment
config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs",
"test_single_file_saved_subset"
".template.cfg"),
join(_my_dir, 'train', 'train_single_file.jsonlines'),
join(_my_dir, 'test',
'test_single_file_subset.'
'jsonlines'))
run_configuration(config_path, quiet=True, overwrite=False)
# Check results
with open(join(_my_dir, 'output', ('train_test_single_file_train_train_'
'single_file.jsonlines_test_test_single'
'_file_subset.jsonlines_RandomForestClassifier'
'.results.json'))) as f:
result_dict = json.load(f)[0]
assert_almost_equal(result_dict['score'], 0.7333333)
def test_ablation_cv_feature_hasher_all_combos_sampler():
"""
Test ablation all-combos + cross-validation + feature hashing + samplers
"""
config_template_path = join(
_my_dir, 'configs',
'test_ablation_feature_hasher_sampler_all_combos.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True, ablation=None)
# read in the summary file and make sure it has
# 10 ablated featuresets * (10 folds + 1 average line) * 2 learners = 220
# lines
with open(
join(_my_dir, 'output',
'ablation_cv_feature_hasher_all_combos_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
num_rows = check_ablation_rows(reader)
eq_(num_rows, 220)
# make sure there are 10 ablated featuresets * 2 learners = 20 results
# files
num_result_files = len(
glob(
join(_my_dir, 'output',
'ablation_cv_feature_hasher_sampler_all_combos*.results')))
eq_(num_result_files, 20)
def test_learning_curve_output():
"""
Test learning curve output for experiment with metrics option
"""
# Test to validate learning curve output
make_learning_curve_data()
config_template_path = join(_my_dir, 'configs', 'test_learning_curve.template.cfg')
config_path = fill_in_config_paths(config_template_path)
# run the learning curve experiment
run_configuration(config_path, quiet=True)
outprefix = 'test_learning_curve'
# make sure that the TSV file is created with the right columns
output_tsv_path = join(_my_dir, 'output', '{}_summary.tsv'.format(outprefix))
ok_(exists(output_tsv_path))
with open(output_tsv_path, 'r') as tsvf:
r = csv.reader(tsvf, dialect=csv.excel_tab)
header = next(r)
# make sure we have the expected number of columns
eq_(len(header), 11)
num_rows = len(list(r))
# we should have 2 featuresets x 3 learners x 2 objectives x 5 (default)
# training sizes = 60 rows
eq_(num_rows, 60)
# make sure that the two PNG files (one per featureset) are created
# if the requirements are satisfied
if _HAVE_SEABORN:
for featureset_name in ["test_learning_curve1", "test_learning_curve2"]:
ok_(exists(join(_my_dir,
'output',
'{}_{}.png'.format(outprefix, featureset_name))))
def test_train_file_test_file():
"""
Test that train_file and test_file experiments work
"""
# Create data files
make_single_file_featureset_data()
# Run experiment
config_path = fill_in_config_paths_for_single_file(
join(_my_dir, "configs", "test_single_file"
".template.cfg"),
join(_my_dir, 'train', 'train_single_file'
'.jsonlines'),
join(_my_dir, 'test', 'test_single_file.'
'jsonlines'))
run_configuration(config_path, quiet=True)
# Check results
with open(
join(_my_dir, 'output', ('train_test_single_file_train_train_'
'single_file.jsonlines_test_test_single'
'_file.jsonlines_RandomForestClassifier'
'.results.json'))) as f:
result_dict = json.load(f)[0]
assert_almost_equal(result_dict['score'], 0.925)
def test_predict():
'''
This tests whether predict task runs.
'''
make_regression_data()
config_template_path = os.path.join(_my_dir, 'configs',
'test_predict.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(os.path.join(_my_dir, config_path), quiet=True)
with open(os.path.join(_my_dir, 'test',
'test_regression1.jsonlines')) as test_file:
inputs = [x for x in test_file]
assert len(inputs) == 1000
with open(
os.path.join(
_my_dir, 'output',
'test_predict_test_regression1_RescaledRidge.predictions')
) as outfile:
reader = csv.DictReader(outfile, dialect=csv.excel_tab)
predictions = [x['prediction'] for x in reader]
assert len(predictions) == len(inputs)
def test_predict_on_subset_with_existing_model():
"""
Test generating predictions on subset with existing model
"""
# Create data files
make_single_file_featureset_data()
# train and save a model on the training file
train_fs = NDJReader.for_path(join(_my_dir, 'train', 'train_single_file.jsonlines')).read()
learner = Learner('RandomForestClassifier')
learner.train(train_fs, grid_search=True, grid_objective="accuracy")
model_filename = join(_my_dir, 'output', ('train_test_single_file_train_train_'
'single_file.jsonlines_test_test_single'
'_file_subset.jsonlines_RandomForestClassifier'
'.model'))
learner.save(model_filename)
# Run experiment
config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs",
"test_single_file_saved_subset"
".template.cfg"),
join(_my_dir, 'train', 'train_single_file.jsonlines'),
join(_my_dir, 'test',
'test_single_file_subset.'
'jsonlines'))
run_configuration(config_path, quiet=True, overwrite=False)
# Check results
with open(join(_my_dir, 'output', ('train_test_single_file_train_train_'
'single_file.jsonlines_test_test_single'
'_file_subset.jsonlines_RandomForestClassifier'
'.results.json'))) as f:
result_dict = json.load(f)[0]
assert_almost_equal(result_dict['accuracy'], 0.7333333)
def test_folds_file_logging_num_folds():
"""
Test when using `folds_file`, log shows number of folds and appropriate warning.
"""
# Run experiment
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(join(_my_dir,
"configs",
"test_folds_file"
".template.cfg"),
train_path,
None)
run_configuration(config_path, quiet=True)
# Check experiment log output
with open(join(_my_dir,
'output',
'test_folds_file_logging.log')) as f:
cv_file_pattern = re.compile('Specifying "folds_file" overrides both explicit and default "num_cv_folds".')
matches = re.findall(cv_file_pattern, f.read())
assert_equal(len(matches), 1)
# Check job log output
with open(join(_my_dir,
'output',
'test_folds_file_logging_train_f0.'
'jsonlines_LogisticRegression.log')) as f:
cv_folds_pattern = re.compile("(Task: cross_validate\n)(.+)(Cross-validating \([0-9]+ folds\))")
matches = re.findall(cv_folds_pattern, f.read())
assert_equal(len(matches), 1)
def test_regression1():
'''
This is a bit of a contrived test, but it should fail
if anything drastic happens to the regression code.
'''
y = make_regression_data()
config_template_path = os.path.join(_my_dir, 'configs', 'test_regression1.template.cfg')
config_path = fill_in_config_paths(config_template_path)
config_template_path = "test_regression1.cfg"
run_configuration(os.path.join(_my_dir, config_path), quiet=True)
with open(os.path.join(_my_dir, 'output', 'test_regression1_test_regression1_RescaledRidge.results')) as f:
# check held out scores
outstr = f.read()
score = float(SCORE_OUTPUT_RE.search(outstr).groups()[-1])
assert score > 0.7
with open(os.path.join(_my_dir, 'output', 'test_regression1_test_regression1_RescaledRidge.predictions'), 'r') as f:
reader = csv.reader(f, dialect='excel-tab')
next(reader)
pred = [float(row[1]) for row in reader]
assert np.min(pred) >= np.min(y)
assert np.max(pred) <= np.max(y)
assert abs(np.mean(pred) - np.mean(y)) < 0.1
assert abs(np.std(pred) - np.std(y)) < 0.1
def test_ablation_cv_feature_hasher_all_combos():
"""
Test ablation all-combos + cross-validation + feature hashing
"""
config_template_path = join(_my_dir,
'configs',
'test_ablation_feature_hasher_all_combos.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True, ablation=None)
# read in the summary file and make sure it has
# 10 ablated featuresets
# * (10 folds + 1 average line)
# * 2 learners
# = 220 lines in total
with open(join(_my_dir,
'output',
'ablation_cv_feature_hasher_all_combos_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
num_rows = check_ablation_rows(reader)
eq_(num_rows, 220)
# make sure there are 10 ablated featuresets * 2 learners = 20 results
# files
num_result_files = len(glob(join(_my_dir,
'output',
'ablation_cv_feature_hasher_all_combos*.results')))
eq_(num_result_files, 20)
def test_folds_file_logging_num_folds():
"""
Test that, when `folds_file` is used, the log prints the number of folds,
instead of the entire cv_folds data. And that the folds file warning shows up
in the log file.
"""
# Run experiment
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(
join(_my_dir, "configs", "test_folds_file"
".template.cfg"), train_path, None)
run_configuration(config_path, quiet=True)
# Check experiment log output
with open(join(_my_dir, 'output', 'test_folds_file_logging.log')) as f:
cv_file_pattern = re.compile(
'Specifying "folds_file" overrides both explicit and default "num_cv_folds".'
)
matches = re.findall(cv_file_pattern, f.read())
assert_equal(len(matches), 1)
# Check job log output
with open(
join(
_my_dir, 'output', 'test_folds_file_logging_train_f0.'
'jsonlines_LogisticRegression.log')) as f:
cv_folds_pattern = re.compile(
"(Task: cross_validate\n)(.+)(Cross-validating \([0-9]+ folds\))")
matches = re.findall(cv_folds_pattern, f.read())
assert_equal(len(matches), 1)
def test_train_file_test_file():
"""
Test that train_file and test_file experiments work
"""
# Create data files
make_single_file_featureset_data()
# Run experiment
config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs",
"test_single_file"
".template.cfg"),
join(_my_dir, 'train',
'train_single_file'
'.jsonlines'),
join(_my_dir, 'test',
'test_single_file.'
'jsonlines'))
run_configuration(config_path, quiet=True)
# Check results
with open(join(_my_dir, 'output', ('train_test_single_file_train_train_'
'single_file.jsonlines_test_test_single'
'_file.jsonlines_RandomForestClassifier'
'.results.json'))) as f:
result_dict = json.load(f)[0]
assert_almost_equal(result_dict['score'], 0.925)
def test_summary():
'''
Test to validate summary file scores
'''
make_summary_data()
config_template_path = os.path.join(_my_dir, 'configs',
'test_summary.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
with open(
os.path.join(
_my_dir, 'output',
'test_summary_test_summary_LogisticRegression.results')) as f:
outstr = f.read()
logistic_result_score = float(
SCORE_OUTPUT_RE.search(outstr).groups()[0])
with open(
os.path.join(
_my_dir, 'output',
'test_summary_test_summary_MultinomialNB.results')) as f:
outstr = f.read()
naivebayes_result_score = float(
SCORE_OUTPUT_RE.search(outstr).groups()[0])
with open(
os.path.join(_my_dir, 'output',
'test_summary_test_summary_SVC.results')) as f:
outstr = f.read()
svm_result_score = float(SCORE_OUTPUT_RE.search(outstr).groups()[0])
with open(os.path.join(_my_dir, 'output', 'test_summary_summary.tsv'),
'r') as f:
reader = csv.DictReader(f, dialect='excel-tab')
for row in reader:
# the learner results dictionaries should have 18 rows,
# and all of these except results_table
# should be printed (though some columns will be blank).
eq_(len(row), 18)
assert row['model_params']
assert row['grid_score']
assert row['score']
if row['learner_name'] == 'LogisticRegression':
logistic_summary_score = float(row['score'])
elif row['learner_name'] == 'MultinomialNB':
naivebayes_summary_score = float(row['score'])
elif row['learner_name'] == 'SVC':
svm_summary_score = float(row['score'])
for result_score, summary_score, learner_name in [
(logistic_result_score, logistic_summary_score, 'LogisticRegression'),
(naivebayes_result_score, naivebayes_summary_score, 'MultinomialNB'),
(svm_result_score, svm_summary_score, 'SVC')
]:
yield check_summary_score, result_score, summary_score, learner_name
def test_scaling():
'''
Test to validate whether feature scaling works
'''
make_scaling_data()
# run the experiment without scaling
config_template_path = os.path.join(_my_dir, 'configs', 'test_scaling_without.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
# now run the version with scaling
config_template_path = os.path.join(_my_dir, 'configs', 'test_scaling_with.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
# make sure that the result with and without scaling aren't the same
with open(os.path.join(_my_dir, 'output', 'without_scaling_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
row = list(reader)[0]
without_scaling_score = row['score']
without_scaling_scaling_value = row['feature_scaling']
with open(os.path.join(_my_dir, 'output', 'with_scaling_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
row = list(reader)[0]
with_scaling_score = row['score']
with_scaling_scaling_value = row['feature_scaling']
assert_not_equal(without_scaling_score, with_scaling_score)
eq_(without_scaling_scaling_value, 'none')
eq_(with_scaling_scaling_value, 'both')
def test_train_file_test_file_ablation():
"""
Test that specifying ablation with train and test file is ignored
"""
# Create data files
make_single_file_featureset_data()
# Run experiment
config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs",
"test_single_file"
".template.cfg"),
join(_my_dir, 'train',
'train_single_file'
'.jsonlines'),
join(_my_dir, 'test',
'test_single_file.'
'jsonlines'))
run_configuration(config_path, quiet=True, ablation=None)
# check that we see the message that ablation was ignored in the experiment log
# Check experiment log output
with open(join(_my_dir,
'output',
'train_test_single_file.log')) as f:
cv_file_pattern = re.compile('Not enough featuresets for ablation. Ignoring.')
matches = re.findall(cv_file_pattern, f.read())
eq_(len(matches), 1)
def test_ablation_cv_feature_hasher_sampler():
"""
Test to validate whether ablation works with cross-validate
and feature_hasher
"""
make_ablation_data()
config_template_path = join(_my_dir, 'configs', ('test_ablation_feature_'
'hasher_sampler.template'
'.cfg'))
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True, ablation=1)
# read in the summary file and make sure it has
# 7 ablated featuresets * (10 folds + 1 average line) * 2 learners = 154
# lines
with open(join(_my_dir, 'output',
'ablation_cv_feature_hasher_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
num_rows = check_ablation_rows(reader)
eq_(num_rows, 154)
# make sure there are 6 ablated featuresets * 2 learners = 12 results files
num_result_files = len(
glob.glob(
join(_my_dir, 'output', ('ablation_cv_feature_hasher_'
'*.results'))))
eq_(num_result_files, 14)
def test_ablation_cv_feature_hasher():
"""
Test if ablation works with cross-validate and feature_hasher
"""
make_ablation_data()
config_template_path = join(_my_dir, 'configs',
'test_ablation_feature_hasher.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True, ablation=1)
# read in the summary file and make sure it has
# 7 ablated featuresets * (10 folds + 1 average line) * 2 learners = 154
# lines
with open(join(_my_dir, 'output',
'ablation_cv_feature_hasher_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
num_rows = check_ablation_rows(reader)
eq_(num_rows, 154)
# make sure there are 6 ablated featuresets * 2 learners = 12 results files
num_result_files = len(glob.glob(join(_my_dir, 'output',
('ablation_cv_feature_hasher_'
'*.results'))))
eq_(num_result_files, 14)
def test_custom_learner_model_loading():
num_labels = 10
class_weights = [(0.5 / (num_labels - 1))
for x in range(num_labels - 1)] + [0.5]
train_fs, test_fs = make_classification_data(num_examples=600,
train_test_ratio=0.8,
num_labels=num_labels,
num_features=5,
non_negative=True,
class_weights=class_weights)
# Write training feature set to a file
train_path = join(_my_dir, 'train',
'test_model_custom_learner.jsonlines')
writer = NDJWriter(train_path, train_fs)
writer.write()
# Write test feature set to a file
test_path = join(_my_dir, 'test',
'test_model_custom_learner.jsonlines')
writer = NDJWriter(test_path, test_fs)
writer.write()
# run the configuration that trains the custom model and saves it
cfgfile = 'test_model_save_custom_learner.template.cfg'
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
# save the predictions from disk into memory
# and delete the predictions file
outprefix = 'test_model_custom_learner'
pred_file = join(_my_dir, 'output',
'{}_{}_CustomLogisticRegressionWrapper'
'.predictions'.format(outprefix,
outprefix))
preds1 = read_predictions(pred_file)
os.unlink(pred_file)
# run the configuration that loads the saved model
# and generates the predictions again
cfgfile = 'test_model_load_custom_learner.template.cfg'
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, overwrite=False, quiet=True)
# load the newly generated predictions
preds2 = read_predictions(pred_file)
# make sure that they are the same as before
assert_array_equal(preds1, preds2)
def train_rst_parsing_model(working_path, model_path, parameter_settings):
'''
parameter_settings is a dict of scikit-learn hyperparameter settings
'''
C_value = parameter_settings['C']
working_subdir = os.path.join(working_path, 'C{}'.format(C_value))
assert not os.path.exists(working_subdir)
os.makedirs(working_subdir)
if not os.path.exists(model_path):
os.makedirs(model_path)
learner_name = 'LogisticRegression'
fixed_parameters = [{'random_state': 123456789, 'penalty': 'l1',
'C': C_value}]
# Make the SKLL config file.
cfg_dict = {"General": {"task": "train",
"experiment_name": "rst_parsing"},
"Input": {"train_location": working_path,
"ids_to_floats": "False",
"featuresets": json.dumps([["rst_parsing"]]),
"featureset_names": json.dumps(["all_feats"]),
"suffix": '.jsonlines',
"fixed_parameters": json.dumps(fixed_parameters),
"learners": json.dumps([learner_name])},
"Tuning": {"feature_scaling": "none",
"grid_search": "False",
"min_feature_count": "1"},
"Output": {"probability": "True",
"models": model_path,
"log": working_subdir}
}
# write config file
cfg_path = os.path.join(working_subdir, 'rst_parsing.cfg')
cfg = ConfigParser()
for section_name, section_dict in list(cfg_dict.items()):
cfg.add_section(section_name)
for key, val in section_dict.items():
cfg.set(section_name, key, val)
assert not os.path.exists(cfg_path)
with open(cfg_path, 'w') as config_file:
cfg.write(config_file)
# run SKLL
run_configuration(cfg_path)
# make the model smaller/faster
minimize_model(model_path,
'rst_parsing_all_feats_LogisticRegression.model')
def test_class_map():
make_class_map_data()
config_template_path = os.path.join(_my_dir, 'configs', 'test_class_map.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
with open(os.path.join(_my_dir, 'output', 'test_class_map_test_class_map_LogisticRegression.results')) as f:
outstr = f.read()
logistic_result_score = float(SCORE_OUTPUT_RE.search(outstr).groups()[0])
assert_almost_equal(logistic_result_score, 0.5)
def test_cross_validate_task():
"""
Test that 10-fold cross_validate experiments work.
Test that fold ids get correctly saved.
"""
# Run experiment
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(
join(_my_dir, "configs", "test_save_cv_folds"
".template.cfg"), train_path, None)
run_configuration(config_path, quiet=True)
# Check final average results
with open(
join(
_my_dir, 'output', 'test_save_cv_folds_train_f0.' +
'jsonlines_LogisticRegression.results.json')) as f:
result_dict = json.load(f)[10]
assert_almost_equal(result_dict['score'], 0.517)
# Check that the fold ids were saved correctly
expected_skll_ids = {}
examples = _load_featureset(train_path, '', suffix, quiet=True)
kfold = StratifiedKFold(n_splits=10)
for fold_num, (_, test_indices) in enumerate(
kfold.split(examples.features, examples.labels)):
for index in test_indices:
expected_skll_ids[examples.ids[index]] = fold_num
skll_fold_ids = {}
with open(join(_my_dir, 'output',
'test_save_cv_folds_skll_fold_ids.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
skll_fold_ids[row['id']] = row['cv_test_fold']
# convert the dictionary to strings (sorted by key) for quick comparison
skll_fold_ids_str = ''.join('{}{}'.format(key, val)
for key, val in sorted(skll_fold_ids.items()))
expected_skll_ids_str = ''.join(
'{}{}'.format(key, val)
for key, val in sorted(expected_skll_ids.items()))
assert_equal(skll_fold_ids_str, expected_skll_ids_str)
def run_configuration_and_check_outputs(config_path):
"""
Run the configuration, and then check the JSON results
against expected JSON files
"""
# run this experiment, get the `results_json_path`
results_json_path = run_configuration(config_path, local=True, quiet=True)[0]
# if the results path exists, check the output
if exists(results_json_path):
results_json_exp_path = join(_my_dir, 'other', 'expected', basename(results_json_path))
results_obj = json.load(open(results_json_path, 'r'))[0]
results_exp_obj = json.load(open(results_json_exp_path, 'r'))[0]
# we check a subset of the values, just to make sure
# that nothing weird is going on with our output
for key in ["train_set_size", "test_set_size",
"learner_name", "cv_folds", "feature_scaling",
"grid_score", "grid_objective",
"accuracy", "score", "pearson"]:
# we obviously want to skip any keys that we aren't expecting
if key in results_exp_obj:
actual = results_obj[key]
expected = results_exp_obj[key]
# if this is a float, then we check with less precision (4 decimals);
# otherwise, we check to make sure things are matching exactly
if isinstance(expected, float):
assert_almost_equal(actual, expected, places=4)
else:
eq_(actual, expected)
def test_sparse_predict():
'''
Test to validate whether predict works with sparse data
'''
make_sparse_data()
config_template_path = os.path.join(_my_dir, 'configs', 'test_sparse.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
with open(os.path.join(_my_dir, 'output', 'test_sparse_test_sparse_LogisticRegression.results')) as f:
outstr = f.read()
logistic_result_score = float(SCORE_OUTPUT_RE.search(outstr).groups()[0])
assert_almost_equal(logistic_result_score, 0.5)
def test_summary():
'''
Test to validate summary file scores
'''
make_summary_data()
config_template_path = os.path.join(_my_dir, 'configs', 'test_summary.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
with open(os.path.join(_my_dir, 'output', 'test_summary_test_summary_LogisticRegression.results')) as f:
outstr = f.read()
logistic_result_score = float(SCORE_OUTPUT_RE.search(outstr).groups()[0])
with open(os.path.join(_my_dir, 'output', 'test_summary_test_summary_MultinomialNB.results')) as f:
outstr = f.read()
naivebayes_result_score = float(SCORE_OUTPUT_RE.search(outstr).groups()[0])
with open(os.path.join(_my_dir, 'output', 'test_summary_test_summary_SVC.results')) as f:
outstr = f.read()
svm_result_score = float(SCORE_OUTPUT_RE.search(outstr).groups()[0])
with open(os.path.join(_my_dir, 'output', 'test_summary_summary.tsv'), 'r') as f:
reader = csv.DictReader(f, dialect='excel-tab')
for row in reader:
# the learner results dictionaries should have 18 rows,
# and all of these except results_table
# should be printed (though some columns will be blank).
eq_(len(row), 18)
assert row['model_params']
assert row['grid_score']
assert row['score']
if row['learner_name'] == 'LogisticRegression':
logistic_summary_score = float(row['score'])
elif row['learner_name'] == 'MultinomialNB':
naivebayes_summary_score = float(row['score'])
elif row['learner_name'] == 'SVC':
svm_summary_score = float(row['score'])
for result_score, summary_score, learner_name in [(logistic_result_score, logistic_summary_score, 'LogisticRegression'), (naivebayes_result_score, naivebayes_summary_score, 'MultinomialNB'), (svm_result_score, svm_summary_score, 'SVC')]:
yield check_summary_score, result_score, summary_score, learner_name
def test_cross_validate_task():
"""
Test that 10-fold cross_validate experiments work.
Test that fold ids get correctly saved.
"""
# Run experiment
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(join(_my_dir, "configs",
"test_save_cv_folds"
".template.cfg"),
train_path,
None)
run_configuration(config_path, quiet=True)
# Check final average results
with open(join(_my_dir, 'output', 'test_save_cv_folds_train_f0.' +
'jsonlines_LogisticRegression.results.json')) as f:
result_dict = json.load(f)[10]
assert_almost_equal(result_dict['score'], 0.517)
# Check that the fold ids were saved correctly
expected_skll_ids = {}
examples = _load_featureset(train_path, '', suffix, quiet=True)
kfold = StratifiedKFold(examples.labels, n_folds=10)
for fold_num, (_, test_indices) in enumerate(kfold):
for index in test_indices:
expected_skll_ids[examples.ids[index]] = fold_num
skll_fold_ids = {}
with open(join(_my_dir, 'output', 'test_save_cv_folds_skll_fold_ids.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
skll_fold_ids[row['id']] = row['cv_test_fold']
# convert the dictionary to strings (sorted by key) for quick comparison
skll_fold_ids_str = ''.join('{}{}'.format(key, val) for key, val in sorted(skll_fold_ids.items()))
expected_skll_ids_str = ''.join('{}{}'.format(key, val) for key, val in sorted(expected_skll_ids.items()))
assert_equal(skll_fold_ids_str, expected_skll_ids_str)
def test_learning_curve_plots():
"""
Test learning curve plots for experiment with metrics option
"""
# Test to validate learning curve output
make_learning_curve_data()
config_template_path = join(_my_dir, 'configs', 'test_learning_curve.template.cfg')
config_path = fill_in_config_paths(config_template_path)
# run the learning curve experiment
run_configuration(config_path, quiet=True)
outprefix = 'test_learning_curve'
# make sure that the two PNG files (one per featureset) are created
for featureset_name in ["test_learning_curve1", "test_learning_curve2"]:
ok_(exists(join(_my_dir,
'output',
'{}_{}.png'.format(outprefix, featureset_name))))
def test_predict():
'''
This tests whether predict task runs.
'''
make_regression_data()
config_template_path = os.path.join(_my_dir, 'configs', 'test_predict.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(os.path.join(_my_dir, config_path), quiet=True)
with open(os.path.join(_my_dir, 'test', 'test_regression1.jsonlines')) as test_file:
inputs = [x for x in test_file]
assert len(inputs) == 1000
with open(os.path.join(_my_dir, 'output', 'test_predict_test_regression1_RescaledRidge.predictions')) as outfile:
reader = csv.DictReader(outfile, dialect=csv.excel_tab)
predictions = [x['prediction'] for x in reader]
assert len(predictions) == len(inputs)
def test_cross_validate_task_save_cv_models():
"""
Test that 10-fold cross_validate experiments work and that CV models
are correctly saved.
"""
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = \
fill_in_config_paths_for_single_file(join(_my_dir, "configs",
"test_save_cv_models.template.cfg"),
train_path,
None)
run_configuration(config_path, quiet=True)
output_dir = join(_my_dir, 'output')
cv_model_prefix = \
"test_save_cv_models_train_f0.jsonlines_LogisticRegression_fold"
for i in range(1, 11):
assert exists(join(output_dir,
"{}{}.model".format(cv_model_prefix, i))) is True
def test_logistic_custom_learner():
num_labels = 10
class_weights = [(0.5 / (num_labels - 1))
for x in range(num_labels - 1)] + [0.5]
train_fs, test_fs = make_classification_data(num_examples=600,
train_test_ratio=0.8,
num_labels=num_labels,
num_features=5,
non_negative=True,
class_weights=class_weights)
# Write training feature set to a file
train_path = join(_my_dir, 'train',
'test_logistic_custom_learner.jsonlines')
writer = NDJWriter(train_path, train_fs)
writer.write()
# Write test feature set to a file
test_path = join(_my_dir, 'test',
'test_logistic_custom_learner.jsonlines')
writer = NDJWriter(test_path, test_fs)
writer.write()
cfgfile = 'test_logistic_custom_learner.template.cfg'
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
outprefix = 'test_logistic_custom_learner'
preds = read_predictions(join(_my_dir, 'output',
('{}_{}_CustomLogisticRegressionWrapper'
'.predictions'.format(outprefix,
outprefix))))
expected = read_predictions(join(_my_dir, 'output',
('{}_{}_LogisticRegression.predictions'
.format(outprefix, outprefix))))
assert_array_equal(preds, expected)
def test_logistic_custom_learner():
num_labels = 10
class_weights = [(0.5 / (num_labels - 1))
for x in range(num_labels - 1)] + [0.5]
train_fs, test_fs = make_classification_data(num_examples=600,
train_test_ratio=0.8,
num_labels=num_labels,
num_features=5,
non_negative=True,
class_weights=class_weights)
# Write training feature set to a file
train_path = join(_my_dir, 'train',
'test_logistic_custom_learner.jsonlines')
writer = NDJWriter(train_path, train_fs)
writer.write()
# Write test feature set to a file
test_path = join(_my_dir, 'test',
'test_logistic_custom_learner.jsonlines')
writer = NDJWriter(test_path, test_fs)
writer.write()
cfgfile = 'test_logistic_custom_learner.template.cfg'
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
outprefix = 'test_logistic_custom_learner'
preds = read_predictions(join(_my_dir, 'output',
('{}_{}_CustomLogisticRegressionWrapper'
'_predictions.tsv'.format(outprefix,
outprefix))))
expected = read_predictions(join(_my_dir, 'output',
('{}_{}_LogisticRegression_predictions.tsv'
.format(outprefix, outprefix))))
assert_array_equal(preds, expected)
def test_folds_file_logging_grid_search():
"""
Test that, when `folds_file` is used but `use_folds_file` for grid search
is specified, that we get an appropriate message in the log.
"""
# Run experiment
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(
join(_my_dir, "configs", "test_folds_file_grid"
".template.cfg"), train_path, None)
run_configuration(config_path, quiet=True)
# Check experiment log output
with open(join(_my_dir, 'output', 'test_folds_file_logging.log')) as f:
cv_file_pattern = re.compile(
'Specifying "folds_file" overrides both explicit and default "num_cv_folds".\n(.+)The specified "folds_file" will not be used for inner grid search.'
)
matches = re.findall(cv_file_pattern, f.read())
assert_equal(len(matches), 1)
def test_majority_class_custom_learner():
num_labels = 10
# This will make data where the last class happens about 50% of the time.
class_weights = [(0.5 / (num_labels - 1))
for x in range(num_labels - 1)] + [0.5]
train_fs, test_fs = make_classification_data(num_examples=600,
train_test_ratio=0.8,
num_labels=num_labels,
num_features=5,
non_negative=True,
class_weights=class_weights)
# Write training feature set to a file
train_path = join(_my_dir, 'train',
'test_majority_class_custom_learner.jsonlines')
writer = NDJWriter(train_path, train_fs)
writer.write()
# Write test feature set to a file
test_path = join(_my_dir, 'test',
'test_majority_class_custom_learner.jsonlines')
writer = NDJWriter(test_path, test_fs)
writer.write()
cfgfile = 'test_majority_class_custom_learner.template.cfg'
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
outprefix = 'test_majority_class_custom_learner'
preds = read_predictions(
join(_my_dir, 'output',
('{}_{}_MajorityClassLearner_predictions.tsv'.format(
outprefix, outprefix))))
expected = np.array([float(num_labels - 1) for x in preds])
assert_array_equal(preds, expected)
def test_majority_class_custom_learner():
num_labels = 10
# This will make data where the last class happens about 50% of the time.
class_weights = [(0.5 / (num_labels - 1))
for x in range(num_labels - 1)] + [0.5]
train_fs, test_fs = make_classification_data(num_examples=600,
train_test_ratio=0.8,
num_labels=num_labels,
num_features=5,
non_negative=True,
class_weights=class_weights)
# Write training feature set to a file
train_path = join(_my_dir, 'train',
'test_majority_class_custom_learner.jsonlines')
writer = NDJWriter(train_path, train_fs)
writer.write()
# Write test feature set to a file
test_path = join(_my_dir, 'test',
'test_majority_class_custom_learner.jsonlines')
writer = NDJWriter(test_path, test_fs)
writer.write()
cfgfile = 'test_majority_class_custom_learner.template.cfg'
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
outprefix = 'test_majority_class_custom_learner'
preds = read_predictions(join(_my_dir, 'output',
('{}_{}_MajorityClassLearner_predictions.tsv'
.format(outprefix, outprefix))))
expected = np.array([float(num_labels - 1) for x in preds])
assert_array_equal(preds, expected)
def test_class_map_feature_hasher():
"""
Test class maps with feature hashing
"""
make_class_map_data()
config_template_path = join(_my_dir, 'configs',
'test_class_map_feature_hasher.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
with open(join(_my_dir, 'output', ('test_class_map_test_class_map_'
'LogisticRegression.results.'
'json'))) as f:
# outstr = f.read()
outd = json.loads(f.read())
# logistic_result_score = float(
# SCORE_OUTPUT_RE.search(outstr).groups()[0])
logistic_result_score = outd[0]['score']
assert_almost_equal(logistic_result_score, 0.5)
def test_cv_folds_file_logging():
"""
Test that, when `cv_folds_file` is used, the log prints the number of folds,
instead of the entire cv_folds data.
"""
# Run experiment
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(
join(_my_dir, "configs", "test_cv_folds_file"
".template.cfg"), train_path, None)
run_configuration(config_path, quiet=True)
# Check log output
with open(
join(
_my_dir, 'output', 'test_cv_folds_file_logging_train_f0.' +
'jsonlines_LogisticRegression.log')) as f:
cv_folds_pattern = re.compile(
"Task: cross_validate\nCross-validating \([0-9]+ folds\)")
matches = re.findall(cv_folds_pattern, f.read())
assert_equal(len(matches), 1)
def test_ablation_cv():
"""
Test ablation + cross-validation
"""
config_template_path = join(_my_dir, 'configs',
'test_ablation.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True, ablation=1)
# read in the summary file and make sure it has
# 7 ablated featuresets * (10 folds + 1 average line) * 2 learners = 154
# lines
with open(join(_my_dir, 'output', 'ablation_cv_plain_summary.tsv')) as f:
reader = csv.DictReader(f, dialect=csv.excel_tab)
num_rows = check_ablation_rows(reader)
eq_(num_rows, 154)
# make sure there are 7 ablated featuresets * 2 learners = 12 results files
num_result_files = len(
glob(join(_my_dir, 'output', 'ablation_cv_plain*.results')))
eq_(num_result_files, 14)
def test_class_map():
"""
Test class maps
"""
make_class_map_data()
config_template_path = join(_my_dir, 'configs',
'test_class_map.template.cfg')
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
with open(
join(_my_dir, 'output', ('test_class_map_test_class_map_Logistic'
'Regression.results.json'))) as f:
outd = json.loads(f.read())
# outstr = f.read()
# logistic_result_score = float(
# SCORE_OUTPUT_RE.search(outstr).groups()[0])
logistic_result_score = outd[0]['accuracy']
assert_almost_equal(logistic_result_score, 0.5)
def test_int_labels():
"""
Testing that SKLL can take integer input.
This is just to test that SKLL can take int labels in the input
(rather than floats or strings). For v1.0.0, it could not because the
json package doesn't know how to serialize numpy.int64 objects.
"""
config_template_path = join(_my_dir, 'configs',
'test_int_labels_cv.template.cfg')
config_path = join(_my_dir, 'configs', 'test_int_labels_cv.cfg')
output_dir = join(_my_dir, 'output')
config = _setup_config_parser(config_template_path, validate=False)
config.set("Input", "train_file",
join(_my_dir, 'other', 'test_int_labels_cv.jsonlines'))
config.set("Output", "results", output_dir)
config.set("Output", "log", output_dir)
config.set("Output", "predictions", output_dir)
with open(config_path, 'w') as new_config_file:
config.write(new_config_file)
run_configuration(config_path, quiet=True)
def test_int_labels():
"""
Testing that SKLL can take integer input.
This is just to test that SKLL can take int labels in the input
(rather than floats or strings). For v1.0.0, it could not because the
json package doesn't know how to serialize numpy.int64 objects.
"""
config_template_path = join(_my_dir, 'configs',
'test_int_labels_cv.template.cfg')
config_path = join(_my_dir, 'configs', 'test_int_labels_cv.cfg')
output_dir = join(_my_dir, 'output')
config = _setup_config_parser(config_template_path, validate=False)
config.set("Input", "train_file",
join(_my_dir, 'other', 'test_int_labels_cv.jsonlines'))
config.set("Output", "results", output_dir)
config.set("Output", "log", output_dir)
config.set("Output", "predictions", output_dir)
with open(config_path, 'w') as new_config_file:
config.write(new_config_file)
run_configuration(config_path, quiet=True)
def test_folds_file_with_fewer_ids_than_featureset():
"""
Test when using `folds_file`, log shows warning for extra IDs in featureset.
"""
# Run experiment with a special featureset that has extra IDs
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f5{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(join(_my_dir,
"configs",
"test_folds_file"
".template.cfg"),
train_path,
None)
run_configuration(config_path, quiet=True)
# Check job log output
with open(join(_my_dir,
'output',
'test_folds_file_logging_train_f5.'
'jsonlines_LogisticRegression.log')) as f:
cv_file_pattern = re.compile('Feature set contains IDs that are not in folds dictionary. Skipping those IDs.')
matches = re.findall(cv_file_pattern, f.read())
assert_equal(len(matches), 1)
def test_folds_file_logging_grid_search():
"""
Test that, when `folds_file` is used but `use_folds_file` for grid search
is specified, that we get an appropriate message in the log.
"""
# Run experiment
suffix = '.jsonlines'
train_path = join(_my_dir, 'train', 'f0{}'.format(suffix))
config_path = fill_in_config_paths_for_single_file(join(_my_dir,
"configs",
"test_folds_file_grid"
".template.cfg"),
train_path,
None)
run_configuration(config_path, quiet=True)
# Check experiment log output
with open(join(_my_dir,
'output',
'test_folds_file_logging.log')) as f:
cv_file_pattern = re.compile('Specifying "folds_file" overrides both explicit and default "num_cv_folds".\n(.+)The specified "folds_file" will not be used for inner grid search.')
matches = re.findall(cv_file_pattern, f.read())
assert_equal(len(matches), 1)
def main(argv=None):
'''
Handles command line arguments and gets things started.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv[1:]`` is used instead.
:type argv: list of str
'''
# Get command line arguments
parser = argparse.ArgumentParser(
description="Runs the scikit-learn experiments in a given config file.\
If Grid Map is installed, jobs will automatically be \
created and run on a DRMAA-compatible cluster.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve')
parser.add_argument('config_file',
help='Configuration file describing the sklearn task\
to run.',
nargs='+')
parser.add_argument('-a',
'--ablation',
help='Runs an ablation study where repeated \
experiments are conducted where the specified \
number of features in each featureset in the \
configuration file are held out.',
type=int,
default=0,
metavar='NUM_FEATURES')
parser.add_argument('-A',
'--ablation_all',
help='Runs an ablation study where repeated \
experiments are conducted with all combinations \
of features in each featureset in the \
configuration file. Overrides --ablation \
setting.',
action='store_true')
parser.add_argument('-k',
'--keep_models',
help='If trained models already exists, re-use them\
instead of overwriting them.',
action='store_true')
parser.add_argument('-l',
'--local',
help='Do not use the Grid Engine for running jobs and\
just run everything sequentially on the local \
machine. This is for debugging.',
action='store_true')
parser.add_argument('-m',
'--machines',
help="comma-separated list of machines to add to\
gridmap's whitelist (if not specified, all\
available machines are used). Note that full \
names must be specified, e.g., \
\"nlp.research.ets.org\"",
default=None)
parser.add_argument('-q',
'--queue',
help="Use this queue for gridmap.",
default='all.q')
parser.add_argument(
'-r',
'--resume',
help='If result files already exist for an experiment, \
do not overwrite them. This is very useful when \
doing a large ablation experiment and part of it \
crashes.',
action='store_true')
parser.add_argument('-v',
'--verbose',
help='Print more status information. For every ' +
'additional time this flag is specified, ' +
'output gets more verbose.',
default=0,
action='count')
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
# Logging levels are really integer multiples of 10, so convert verbose flag
log_level = max(logging.WARNING - (args.verbose * 10), logging.DEBUG)
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'),
level=log_level)
machines = None
if args.machines:
machines = args.machines.split(',')
# Create partial function to map onto list of config files
ablation = args.ablation
if args.ablation_all:
ablation = None
# Run each config file sequentially
for config_file in args.config_file:
run_configuration(config_file,
local=args.local,
overwrite=not args.keep_models,
queue=args.queue,
hosts=machines,
ablation=ablation,
resume=args.resume)
def check_summary_score(use_feature_hashing=False):
# Test to validate summary file scores
make_summary_data()
cfgfile = ('test_summary_feature_hasher.template.cfg' if
use_feature_hashing else 'test_summary.template.cfg')
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
outprefix = ('test_summary_feature_hasher_test_summary' if
use_feature_hashing else 'test_summary_test_summary')
summprefix = ('test_summary_feature_hasher' if use_feature_hashing else
'test_summary')
with open(join(_my_dir, 'output', ('{}_LogisticRegression.results.'
'json'.format(outprefix)))) as f:
outd = json.loads(f.read())
logistic_result_score = outd[0]['score']
with open(join(_my_dir, 'output',
'{}_SVC.results.json'.format(outprefix))) as f:
outd = json.loads(f.read())
svm_result_score = outd[0]['score']
# note that Naive Bayes doesn't work with feature hashing
if not use_feature_hashing:
with open(join(_my_dir, 'output', ('{}_MultinomialNB.results.'
'json'.format(outprefix)))) as f:
outd = json.loads(f.read())
naivebayes_result_score = outd[0]['score']
with open(join(_my_dir, 'output', '{}_summary.tsv'.format(summprefix)),
'r') as f:
reader = csv.DictReader(f, dialect='excel-tab')
for row in reader:
# the learner results dictionaries should have 27 rows,
# and all of these except results_table
# should be printed (though some columns will be blank).
eq_(len(row), 27)
assert row['model_params']
assert row['grid_score']
assert row['score']
if row['learner_name'] == 'LogisticRegression':
logistic_summary_score = float(row['score'])
elif row['learner_name'] == 'MultinomialNB':
naivebayes_summary_score = float(row['score'])
elif row['learner_name'] == 'SVC':
svm_summary_score = float(row['score'])
test_tuples = [(logistic_result_score,
logistic_summary_score,
'LogisticRegression'),
(svm_result_score,
svm_summary_score,
'SVC')]
if not use_feature_hashing:
test_tuples.append((naivebayes_result_score,
naivebayes_summary_score,
'MultinomialNB'))
for result_score, summary_score, learner_name in test_tuples:
assert_almost_equal(result_score, summary_score,
err_msg=('mismatched scores for {} '
'(result:{}, summary:'
'{})').format(learner_name, result_score,
summary_score))
def test_fancy_output():
"""
Test the descriptive statistics output in the results file for a regressor
"""
train_fs, test_fs, _ = make_regression_data(num_examples=2000,
num_features=3)
# train a regression model using the train feature set
learner = Learner('LinearRegression')
learner.train(train_fs, grid_search=True, grid_objective='pearson')
# evaluate the trained model using the test feature set
resultdict = learner.evaluate(test_fs)
actual_stats_from_api = dict(resultdict[2]['descriptive']['actual'])
pred_stats_from_api = dict(resultdict[2]['descriptive']['predicted'])
# write out the training and test feature set
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
train_writer = NDJWriter(join(train_dir, 'fancy_train.jsonlines'),
train_fs)
train_writer.write()
test_writer = NDJWriter(join(test_dir, 'fancy_test.jsonlines'), test_fs)
test_writer.write()
# now get the config file template, fill it in and run it
# so that we can get a results file
config_template_path = join(_my_dir, 'configs',
'test_regression_fancy_output.template.cfg')
config_path = fill_in_config_paths_for_fancy_output(config_template_path)
run_configuration(config_path, quiet=True)
# read in the results file and get the descriptive statistics
actual_stats_from_file = {}
pred_stats_from_file = {}
with open(
join(output_dir, ('regression_fancy_output_train_fancy_train.'
'jsonlines_test_fancy_test.jsonlines'
'_LinearRegression.results')), 'r') as resultf:
result_output = resultf.read().strip().split('\n')
for desc_stat_line in result_output[26:30]:
desc_stat_line = desc_stat_line.strip()
if not desc_stat_line:
continue
else:
m = re.search(
r'([A-Za-z]+)\s+=\s+(-?[0-9]+.?[0-9]*)\s+'
r'\((actual)\),\s+(-?[0-9]+.?[0-9]*)\s+'
r'\((predicted)\)', desc_stat_line)
stat_type, actual_value, _, pred_value, _ = m.groups()
actual_stats_from_file[stat_type.lower()] = float(actual_value)
pred_stats_from_file[stat_type.lower()] = float(pred_value)
for stat_type in actual_stats_from_api:
assert_almost_equal(actual_stats_from_file[stat_type],
actual_stats_from_api[stat_type],
places=4)
assert_almost_equal(pred_stats_from_file[stat_type],
pred_stats_from_api[stat_type],
places=4)
def test_fancy_output():
"""
Test the descriptive statistics output in the results file for a regressor
"""
train_fs, test_fs, _ = make_regression_data(num_examples=2000,
num_features=3)
# train a regression model using the train feature set
learner = Learner('LinearRegression')
learner.train(train_fs, grid_objective='pearson')
# evaluate the trained model using the test feature set
resultdict = learner.evaluate(test_fs)
actual_stats_from_api = dict(resultdict[2]['descriptive']['actual'])
pred_stats_from_api = dict(resultdict[2]['descriptive']['predicted'])
# write out the training and test feature set
train_dir = join(_my_dir, 'train')
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
train_writer = NDJWriter(join(train_dir, 'fancy_train.jsonlines'),
train_fs)
train_writer.write()
test_writer = NDJWriter(join(test_dir, 'fancy_test.jsonlines'), test_fs)
test_writer.write()
# now get the config file template, fill it in and run it
# so that we can get a results file
config_template_path = join(_my_dir, 'configs',
'test_regression_fancy_output.template.cfg')
config_path = fill_in_config_paths_for_fancy_output(config_template_path)
run_configuration(config_path, quiet=True)
# read in the results file and get the descriptive statistics
actual_stats_from_file = {}
pred_stats_from_file = {}
with open(join(output_dir, ('regression_fancy_output_train_fancy_train.'
'jsonlines_test_fancy_test.jsonlines'
'_LinearRegression.results')),
'r') as resultf:
result_output = resultf.read().strip().split('\n')
for desc_stat_line in result_output[27:31]:
desc_stat_line = desc_stat_line.strip()
if not desc_stat_line:
continue
else:
m = re.search(r'([A-Za-z]+)\s+=\s+(-?[0-9]+.?[0-9]*)\s+'
r'\((actual)\),\s+(-?[0-9]+.?[0-9]*)\s+'
r'\((predicted)\)', desc_stat_line)
stat_type, actual_value, _, pred_value, _ = m.groups()
actual_stats_from_file[stat_type.lower()] = float(actual_value)
pred_stats_from_file[stat_type.lower()] = float(pred_value)
for stat_type in actual_stats_from_api:
assert_almost_equal(actual_stats_from_file[stat_type],
actual_stats_from_api[stat_type],
places=4)
assert_almost_equal(pred_stats_from_file[stat_type],
pred_stats_from_api[stat_type],
places=4)
def main(argv=None):
"""
Handles command line arguments and gets things started.
Parameters
----------
argv : list of str
List of arguments, as if specified on the command-line.
If None, ``sys.argv[1:]`` is used instead.
"""
# Get command line arguments
parser = ArgumentParser(description='Runs the scikit-learn '
'experiments in a given config file. '
'If Grid Map is installed, jobs will '
'automatically be created and run on '
'a DRMAA-compatible cluster.',
formatter_class=ArgumentDefaultsHelpFormatter,
conflict_handler='resolve')
parser.add_argument('config_file',
help='Configuration file describing the task to run.',
nargs='+')
parser.add_argument('-a',
'--ablation',
help='Runs an ablation study where repeated '
'experiments are conducted where the specified '
'number of features in each featureset in the '
'configuration file are held out.',
type=int,
default=0,
metavar='NUM_FEATURES')
parser.add_argument('-A',
'--ablation_all',
help='Runs an ablation study where repeated '
'experiments are conducted with all combinations '
'of features in each featureset in the '
'configuration file. Overrides --ablation '
'setting.',
action='store_true')
parser.add_argument('-k',
'--keep_models',
help='If trained models already exists, re-use them '
'instead of overwriting them.',
action='store_true')
parser.add_argument('-l',
'--local',
help='Do not use the Grid Engine for running jobs and '
'just run everything sequentially on the local '
'machine. ',
action='store_true')
parser.add_argument('-m',
'--machines',
help='comma-separated list of machines to add to '
'the gridmap whitelist (if not specified, all '
'available machines are used). Note that full '
'names must be specified, e.g., '
'"nlp.research.ets.org"',
default=None)
parser.add_argument('-q',
'--queue',
help="Use this queue for gridmap.",
default='all.q')
parser.add_argument('-r',
'--resume',
help='If result files already exist for an experiment '
'do not overwrite them. This is very useful when '
'doing a large ablation experiment and part of it '
'crashes.',
action='store_true')
parser.add_argument(
'-v',
'--verbose',
help='Include debug information in the logging output.',
default=False,
action='store_true')
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
# Default logging level is INFO unless we are being verbose
log_level = logging.DEBUG if args.verbose else logging.INFO
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'),
level=log_level)
machines = None
if args.machines:
machines = args.machines.split(',')
ablation = args.ablation
if args.ablation_all:
ablation = None
# Run each config file sequentially
for config_file in args.config_file:
run_configuration(config_file,
local=args.local,
overwrite=not args.keep_models,
queue=args.queue,
hosts=machines,
ablation=ablation,
resume=args.resume,
log_level=log_level)
def check_summary_score(use_feature_hashing=False):
# Test to validate summary file scores
make_summary_data()
cfgfile = ('test_summary_feature_hasher.template.cfg' if
use_feature_hashing else 'test_summary.template.cfg')
config_template_path = join(_my_dir, 'configs', cfgfile)
config_path = fill_in_config_paths(config_template_path)
run_configuration(config_path, quiet=True)
outprefix = ('test_summary_feature_hasher_test_summary' if
use_feature_hashing else 'test_summary_test_summary')
summprefix = ('test_summary_feature_hasher' if use_feature_hashing else
'test_summary')
with open(join(_my_dir, 'output', ('{}_LogisticRegression.results.'
'json'.format(outprefix)))) as f:
outd = json.loads(f.read())
logistic_result_score = outd[0]['score']
with open(join(_my_dir, 'output',
'{}_SVC.results.json'.format(outprefix))) as f:
outd = json.loads(f.read())
svm_result_score = outd[0]['score']
# note that Naive Bayes doesn't work with feature hashing
if not use_feature_hashing:
with open(join(_my_dir, 'output', ('{}_MultinomialNB.results.'
'json'.format(outprefix)))) as f:
outd = json.loads(f.read())
naivebayes_result_score = outd[0]['score']
with open(join(_my_dir, 'output', '{}_summary.tsv'.format(summprefix)),
'r') as f:
reader = csv.DictReader(f, dialect='excel-tab')
for row in reader:
# the learner results dictionaries should have 29 rows,
# and all of these except results_table
# should be printed (though some columns will be blank).
eq_(len(row), 29)
assert row['model_params']
assert row['grid_score']
assert row['score']
if row['learner_name'] == 'LogisticRegression':
logistic_summary_score = float(row['score'])
elif row['learner_name'] == 'MultinomialNB':
naivebayes_summary_score = float(row['score'])
elif row['learner_name'] == 'SVC':
svm_summary_score = float(row['score'])
test_tuples = [(logistic_result_score,
logistic_summary_score,
'LogisticRegression'),
(svm_result_score,
svm_summary_score,
'SVC')]
if not use_feature_hashing:
test_tuples.append((naivebayes_result_score,
naivebayes_summary_score,
'MultinomialNB'))
for result_score, summary_score, learner_name in test_tuples:
assert_almost_equal(result_score, summary_score,
err_msg=('mismatched scores for {} '
'(result:{}, summary:'
'{})').format(learner_name, result_score,
summary_score))
# We itereate over each model with an expected
# accuracy score. T est proves that the report
# written out at least as a correct format for
# this line. See _print_fancy_output
for report_name, val in (("LogisticRegression", .5),
("MultinomialNB", .5),
("SVC", .7)):
filename = "test_summary_test_summary_{}.results".format(report_name)
results_path = join(_my_dir, 'output', filename)
with open(results_path) as results_file:
report = results_file.read()
expected_string = "Accuracy = {:.1f}".format(val)
eq_(expected_string in report, # approximate
True,
msg="{} is not in {}".format(expected_string,
report))
def check_xval_fancy_results_file(do_grid_search,
use_folds_file,
use_folds_file_for_grid_search,
use_additional_metrics):
train_path = join(_my_dir, 'train', 'f0.jsonlines')
output_dir = join(_my_dir, 'output')
# make a simple config file for cross-validation
values_to_fill_dict = {'experiment_name': 'test_fancy_xval',
'train_file': train_path,
'task': 'cross_validate',
'grid_search': 'true',
'objectives': "['f1_score_micro']",
'featureset_names': '["f0"]',
'num_cv_folds': '6',
'grid_search_folds': '4',
'learners': "['LogisticRegression']",
'log': output_dir,
'predictions': output_dir,
'results': output_dir}
folds_file_path = join(_my_dir, 'train', 'folds_file_test.csv')
if use_folds_file:
values_to_fill_dict['folds_file'] = folds_file_path
values_to_fill_dict['grid_search'] = str(do_grid_search)
values_to_fill_dict['use_folds_file_for_grid_search'] = str(use_folds_file_for_grid_search)
if use_additional_metrics:
if PY2:
values_to_fill_dict['metrics'] = str([b"accuracy", b"unweighted_kappa"])
else:
values_to_fill_dict['metrics'] = str(["accuracy", "unweighted_kappa"])
config_template_path = join(_my_dir,
'configs',
'test_fancy.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'xval')
# run the experiment
run_configuration(config_path, quiet=True)
# now make sure that the results file was produced
results_file_path = join(_my_dir, 'output', 'test_fancy_xval_f0_LogisticRegression.results')
ok_(exists(results_file_path))
# read in all the lines and look at the lines up to where we print the "Total Time"
with open(results_file_path, 'r') as resultsf:
results_lines = resultsf.readlines()
end_idx = [results_lines.index(l) for l in results_lines if l.startswith('Total Time:')][0]
results_lines = results_lines[:end_idx+1]
# read in the "keys" and "values" separated by colons into a dictionary
results_dict = dict([rl.strip().split(': ') for rl in results_lines])
# check that the fields we expect in the results file are there
# and the ones that we do not expect aren't
if do_grid_search:
eq_(results_dict['Grid Search'], 'True')
eq_(results_dict['Grid Objective Function'], 'f1_score_micro')
else:
eq_(results_dict['Grid Search'], 'False')
ok_('Grid Search Folds' not in results_dict)
ok_('Grid Objective Function' not in results_dict)
if use_folds_file:
eq_(results_dict['Number of Folds'], '5 via folds file')
ok_('Stratified Folds' not in results_dict)
eq_(results_dict['Specified Folds File'], folds_file_path)
if do_grid_search:
if use_folds_file_for_grid_search:
eq_(results_dict['Grid Search Folds'], '5 via folds file')
eq_(results_dict['Using Folds File for Grid Search'], 'True')
else:
eq_(results_dict['Grid Search Folds'], '4')
eq_(results_dict['Using Folds File for Grid Search'], 'False')
else:
eq_(results_dict['Number of Folds'], '6')
eq_(results_dict['Stratified Folds'], 'True')
ok_('Using Folds File for Grid Search' not in results_dict)
ok_('Specified Folds File' not in results_dict)
if do_grid_search:
eq_(results_dict['Grid Search Folds'], '4')
if use_additional_metrics:
expected_metrics = [b"accuracy", b"unweighted_kappa"] if PY2 else ["accuracy", "unweighted_kappa"]
eq_(sorted(literal_eval(results_dict['Additional Evaluation Metrics'])),
sorted(expected_metrics))
def main(argv=None):
'''
Handles command line arguments and gets things started.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv[1:]`` is used instead.
:type argv: list of str
'''
# Get command line arguments
parser = argparse.ArgumentParser(
description="Runs the scikit-learn experiments in a given config file.\
If Grid Map is installed, jobs will automatically be \
created and run on a DRMAA-compatible cluster.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve')
parser.add_argument('config_file',
help='Configuration file describing the sklearn task\
to run.',
nargs='+')
parser.add_argument('-a', '--ablation',
help='Runs an ablation study where repeated \
experiments are conducted where the specified \
number of features in each featureset in the \
configuration file are held out.',
type=int, default=0,
metavar='NUM_FEATURES')
parser.add_argument('-A', '--ablation_all',
help='Runs an ablation study where repeated \
experiments are conducted with all combinations \
of features in each featureset in the \
configuration file. Overrides --ablation \
setting.',
action='store_true')
parser.add_argument('-k', '--keep_models',
help='If trained models already exists, re-use them\
instead of overwriting them.',
action='store_true')
parser.add_argument('-l', '--local',
help='Do not use the Grid Engine for running jobs and\
just run everything sequentially on the local \
machine. This is for debugging.',
action='store_true')
parser.add_argument('-m', '--machines',
help="comma-separated list of machines to add to\
gridmap's whitelist (if not specified, all\
available machines are used). Note that full \
names must be specified, e.g., \
\"nlp.research.ets.org\"",
default=None)
parser.add_argument('-q', '--queue',
help="Use this queue for gridmap.",
default='all.q')
parser.add_argument('-r', '--resume',
help='If result files already exist for an experiment, \
do not overwrite them. This is very useful when \
doing a large ablation experiment and part of it \
crashes.',
action='store_true')
parser.add_argument('-v', '--verbose',
help='Print more status information. For every ' +
'additional time this flag is specified, ' +
'output gets more verbose.',
default=0, action='count')
parser.add_argument('--version', action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
# Logging levels are really integer multiples of 10, so convert verbose flag
log_level = max(logging.WARNING - (args.verbose * 10), logging.DEBUG)
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'), level=log_level)
machines = None
if args.machines:
machines = args.machines.split(',')
# Create partial function to map onto list of config files
ablation = args.ablation
if args.ablation_all:
ablation = None
# Run each config file sequentially
for config_file in args.config_file:
run_configuration(config_file, local=args.local, overwrite=not
args.keep_models, queue=args.queue, hosts=machines,
ablation=ablation, resume=args.resume)
def check_grid_search_cv_results(task, do_grid_search):
learners = ['LogisticRegression', 'SVC']
expected_path = join(_my_dir, 'other', 'cv_results')
time_field = lambda x: x.endswith('_time')
train_path = join(_my_dir, 'train', 'f0.jsonlines')
output_dir = join(_my_dir, 'output')
exp_name = ('test_grid_search_cv_results_{}_{}'
.format(task, "gs" if do_grid_search else "nogs"))
# make a simple config file for cross-validation
values_to_fill_dict = {'experiment_name': exp_name,
'train_file': train_path,
'task': task,
'grid_search': json.dumps(do_grid_search),
'objectives': "['f1_score_micro']",
'featureset_names': "['f0']",
'learners': '{}'.format(json.dumps(learners)),
'log': output_dir,
'results': output_dir}
if task == 'train':
values_to_fill_dict['models'] = output_dir
elif task == 'cross_validate':
values_to_fill_dict['predictions'] = output_dir
elif task in ['evaluate', 'predict']:
values_to_fill_dict['predictions'] = output_dir
values_to_fill_dict['test_file'] = \
values_to_fill_dict['train_file']
# In the case where grid search is on and the task is
# learning curve, grid search will automatically be turned
# off, so simply turn it off here as well since it should
# result in the same situation
elif task == 'learning_curve':
values_to_fill_dict['metrics'] = values_to_fill_dict.pop('objectives')
if do_grid_search:
do_grid_search = False
config_template_path = join(_my_dir,
'configs',
'test_cv_results.template.cfg')
config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
task)
# run the experiment
if task in ['train', 'predict']:
if do_grid_search:
run_configuration(config_path, quiet=True)
else:
assert_raises(ValueError, run_configuration, config_path, quiet=True)
# Short-circuit the test since a ValueError is
# expected and is fatal
return
else:
run_configuration(config_path, quiet=True)
# now make sure that the results json file was produced
for learner in learners:
results_file_name = ('{}_f0_{}.results.json'
.format(exp_name, learner))
actual_results_file_path = join(_my_dir, 'output',
results_file_name)
expected_results_file_path = join(expected_path,
results_file_name)
ok_(exists(actual_results_file_path))
with open(expected_results_file_path) as expected, \
open(actual_results_file_path) as actual:
expected_lines = [json.loads(line) for line in expected][0]
actual_lines = [json.loads(line) for line in actual][0]
assert len(expected_lines) == len(actual_lines)
if task == 'cross_validate':
# All but the last line will have grid search-related
# results
for (expected_gs_cv_results,
actual_gs_cv_results) in zip(expected_lines[:-1],
actual_lines[:-1]):
assert len(expected_gs_cv_results) == len(actual_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if do_grid_search:
assert (set(expected_gs_cv_results)
.intersection(actual_gs_cv_results) ==
set(expected_gs_cv_results))
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
else:
if field == 'grid_score':
assert actual_gs_cv_results[field] == 0.0
else:
assert actual_gs_cv_results[field] is None
# The last line should be for the "average" and should
# not contain any grid search results
assert actual_lines[-1]['fold'] == 'average'
for field in ['grid_score', 'grid_search_cv_results']:
assert field not in actual_lines[-1]
elif task == 'evaluate':
for (expected_gs_cv_results,
actual_gs_cv_results) in zip(expected_lines,
actual_lines):
assert len(expected_gs_cv_results) == len(actual_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if do_grid_search:
assert (set(expected_gs_cv_results)
.intersection(actual_gs_cv_results) ==
set(expected_gs_cv_results))
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
else:
if field == 'grid_score':
assert actual_gs_cv_results[field] == 0.0
else:
assert actual_gs_cv_results[field] is None
elif task in ['train', 'predict']:
expected_gs_cv_results = expected_lines
actual_gs_cv_results = actual_lines
assert set(expected_gs_cv_results).intersection(actual_gs_cv_results) == \
set(expected_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
else:
for expected_line, actual_line in zip(expected_lines,
actual_lines):
expected_fields = set(list(expected_line))
actual_fields = set(list(actual_line))
assert expected_fields.intersection(actual_fields) == \
expected_fields
assert all(field not in actual_fields
for field in ['grid_score',
'grid_search_cv_results'])
