def test_custom_learner_model_loading(): num_labels = 10 class_weights = [(0.5 / (num_labels - 1)) for x in range(num_labels - 1)] + [0.5] train_fs, test_fs = make_classification_data(num_examples=600, train_test_ratio=0.8, num_labels=num_labels, num_features=5, non_negative=True, class_weights=class_weights) # Write training feature set to a file train_path = join(_my_dir, 'train', 'test_model_custom_learner.jsonlines') writer = NDJWriter(train_path, train_fs) writer.write() # Write test feature set to a file test_path = join(_my_dir, 'test', 'test_model_custom_learner.jsonlines') writer = NDJWriter(test_path, test_fs) writer.write() # run the configuration that trains the custom model and saves it cfgfile = 'test_model_save_custom_learner.template.cfg' config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) # save the predictions from disk into memory # and delete the predictions file outprefix = 'test_model_custom_learner' pred_file = join(_my_dir, 'output', '{}_{}_CustomLogisticRegressionWrapper' '.predictions'.format(outprefix, outprefix)) preds1 = read_predictions(pred_file) os.unlink(pred_file) # run the configuration that loads the saved model # and generates the predictions again cfgfile = 'test_model_load_custom_learner.template.cfg' config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, overwrite=False, quiet=True) # load the newly generated predictions preds2 = read_predictions(pred_file) # make sure that they are the same as before assert_array_equal(preds1, preds2)
def test_ablation_cv_feature_hasher_all_combos_sampler(): """ Test ablation all-combos + cross-validation + feature hashing + samplers """ config_template_path = join( _my_dir, 'configs', 'test_ablation_feature_hasher_sampler_all_combos.template.cfg') config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True, ablation=None) # read in the summary file and make sure it has # 10 ablated featuresets * (10 folds + 1 average line) * 2 learners = 220 # lines with open( join(_my_dir, 'output', 'ablation_cv_feature_hasher_all_combos_summary.tsv')) as f: reader = csv.DictReader(f, dialect=csv.excel_tab) num_rows = check_ablation_rows(reader) eq_(num_rows, 220) # make sure there are 10 ablated featuresets * 2 learners = 20 results # files num_result_files = len( glob( join(_my_dir, 'output', 'ablation_cv_feature_hasher_sampler_all_combos*.results'))) eq_(num_result_files, 20)
def test_ablation_cv_feature_hasher_sampler(): """ Test to validate whether ablation works with cross-validate and feature_hasher """ make_ablation_data() config_template_path = join(_my_dir, 'configs', ('test_ablation_feature_' 'hasher_sampler.template' '.cfg')) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True, ablation=1) # read in the summary file and make sure it has # 7 ablated featuresets * (10 folds + 1 average line) * 2 learners = 154 # lines with open(join(_my_dir, 'output', 'ablation_cv_feature_hasher_summary.tsv')) as f: reader = csv.DictReader(f, dialect=csv.excel_tab) num_rows = check_ablation_rows(reader) eq_(num_rows, 154) # make sure there are 6 ablated featuresets * 2 learners = 12 results files num_result_files = len( glob.glob( join(_my_dir, 'output', ('ablation_cv_feature_hasher_' '*.results')))) eq_(num_result_files, 14)
def test_learning_curve_output(): """ Test learning curve output for experiment with metrics option """ # Test to validate learning curve output make_learning_curve_data() config_template_path = join(_my_dir, 'configs', 'test_learning_curve.template.cfg') config_path = fill_in_config_paths(config_template_path) # run the learning curve experiment run_configuration(config_path, quiet=True) outprefix = 'test_learning_curve' # make sure that the TSV file is created with the right columns output_tsv_path = join(_my_dir, 'output', '{}_summary.tsv'.format(outprefix)) ok_(exists(output_tsv_path)) with open(output_tsv_path, 'r') as tsvf: r = csv.reader(tsvf, dialect=csv.excel_tab) header = next(r) # make sure we have the expected number of columns eq_(len(header), 11) num_rows = len(list(r)) # we should have 2 featuresets x 3 learners x 2 objectives x 5 (default) # training sizes = 60 rows eq_(num_rows, 60) # make sure that the two PNG files (one per featureset) are created # if the requirements are satisfied if _HAVE_SEABORN: for featureset_name in ["test_learning_curve1", "test_learning_curve2"]: ok_(exists(join(_my_dir, 'output', '{}_{}.png'.format(outprefix, featureset_name))))
def test_ablation_cv_feature_hasher(): """ Test if ablation works with cross-validate and feature_hasher """ make_ablation_data() config_template_path = join(_my_dir, 'configs', 'test_ablation_feature_hasher.template.cfg') config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True, ablation=1) # read in the summary file and make sure it has # 7 ablated featuresets * (10 folds + 1 average line) * 2 learners = 154 # lines with open(join(_my_dir, 'output', 'ablation_cv_feature_hasher_summary.tsv')) as f: reader = csv.DictReader(f, dialect=csv.excel_tab) num_rows = check_ablation_rows(reader) eq_(num_rows, 154) # make sure there are 6 ablated featuresets * 2 learners = 12 results files num_result_files = len(glob.glob(join(_my_dir, 'output', ('ablation_cv_feature_hasher_' '*.results')))) eq_(num_result_files, 14)
def test_ablation_cv_feature_hasher_all_combos(): """ Test ablation all-combos + cross-validation + feature hashing """ config_template_path = join(_my_dir, 'configs', 'test_ablation_feature_hasher_all_combos.template.cfg') config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True, ablation=None) # read in the summary file and make sure it has # 10 ablated featuresets # * (10 folds + 1 average line) # * 2 learners # = 220 lines in total with open(join(_my_dir, 'output', 'ablation_cv_feature_hasher_all_combos_summary.tsv')) as f: reader = csv.DictReader(f, dialect=csv.excel_tab) num_rows = check_ablation_rows(reader) eq_(num_rows, 220) # make sure there are 10 ablated featuresets * 2 learners = 20 results # files num_result_files = len(glob(join(_my_dir, 'output', 'ablation_cv_feature_hasher_all_combos*.results'))) eq_(num_result_files, 20)
def test_ablation_cv_feature_hasher_all_combos_sampler(): """ Test to validate whether ablation works with cross-validate and feature_hasher """ make_ablation_data() config_template_path = join(_my_dir, 'configs', ('test_ablation_feature_' 'hasher_sampler.template' '.cfg')) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True, ablation=None) # read in the summary file and make sure it has # 10 ablated featuresets * (10 folds + 1 average line) * 2 learners = 220 # lines with open(join(_my_dir, 'output', 'ablation_cv_feature_hasher_summary.tsv')) as f: reader = csv.DictReader(f, dialect=csv.excel_tab) num_rows = check_ablation_rows(reader) eq_(num_rows, 220) # make sure there are 10 ablated featuresets * 2 learners = 20 results # files num_result_files = len(glob.glob(join(_my_dir, 'output', ('ablation_cv_feature_hasher_' '*results')))) eq_(num_result_files, 20)
def test_learning_curve_plots(): """ Test learning curve plots for experiment with metrics option """ # Test to validate learning curve output make_learning_curve_data() config_template_path = join(_my_dir, 'configs', 'test_learning_curve.template.cfg') config_path = fill_in_config_paths(config_template_path) # run the learning curve experiment run_configuration(config_path, quiet=True) outprefix = 'test_learning_curve' # make sure that the two PNG files (one per featureset) are created for featureset_name in ["test_learning_curve1", "test_learning_curve2"]: ok_(exists(join(_my_dir, 'output', '{}_{}.png'.format(outprefix, featureset_name))))
def test_logistic_custom_learner(): num_labels = 10 class_weights = [(0.5 / (num_labels - 1)) for x in range(num_labels - 1)] + [0.5] train_fs, test_fs = make_classification_data(num_examples=600, train_test_ratio=0.8, num_labels=num_labels, num_features=5, non_negative=True, class_weights=class_weights) # Write training feature set to a file train_path = join(_my_dir, 'train', 'test_logistic_custom_learner.jsonlines') writer = NDJWriter(train_path, train_fs) writer.write() # Write test feature set to a file test_path = join(_my_dir, 'test', 'test_logistic_custom_learner.jsonlines') writer = NDJWriter(test_path, test_fs) writer.write() cfgfile = 'test_logistic_custom_learner.template.cfg' config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) outprefix = 'test_logistic_custom_learner' preds = read_predictions(join(_my_dir, 'output', ('{}_{}_CustomLogisticRegressionWrapper' '.predictions'.format(outprefix, outprefix)))) expected = read_predictions(join(_my_dir, 'output', ('{}_{}_LogisticRegression.predictions' .format(outprefix, outprefix)))) assert_array_equal(preds, expected)
def test_logistic_custom_learner(): num_labels = 10 class_weights = [(0.5 / (num_labels - 1)) for x in range(num_labels - 1)] + [0.5] train_fs, test_fs = make_classification_data(num_examples=600, train_test_ratio=0.8, num_labels=num_labels, num_features=5, non_negative=True, class_weights=class_weights) # Write training feature set to a file train_path = join(_my_dir, 'train', 'test_logistic_custom_learner.jsonlines') writer = NDJWriter(train_path, train_fs) writer.write() # Write test feature set to a file test_path = join(_my_dir, 'test', 'test_logistic_custom_learner.jsonlines') writer = NDJWriter(test_path, test_fs) writer.write() cfgfile = 'test_logistic_custom_learner.template.cfg' config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) outprefix = 'test_logistic_custom_learner' preds = read_predictions(join(_my_dir, 'output', ('{}_{}_CustomLogisticRegressionWrapper' '_predictions.tsv'.format(outprefix, outprefix)))) expected = read_predictions(join(_my_dir, 'output', ('{}_{}_LogisticRegression_predictions.tsv' .format(outprefix, outprefix)))) assert_array_equal(preds, expected)
def test_majority_class_custom_learner(): num_labels = 10 # This will make data where the last class happens about 50% of the time. class_weights = [(0.5 / (num_labels - 1)) for x in range(num_labels - 1)] + [0.5] train_fs, test_fs = make_classification_data(num_examples=600, train_test_ratio=0.8, num_labels=num_labels, num_features=5, non_negative=True, class_weights=class_weights) # Write training feature set to a file train_path = join(_my_dir, 'train', 'test_majority_class_custom_learner.jsonlines') writer = NDJWriter(train_path, train_fs) writer.write() # Write test feature set to a file test_path = join(_my_dir, 'test', 'test_majority_class_custom_learner.jsonlines') writer = NDJWriter(test_path, test_fs) writer.write() cfgfile = 'test_majority_class_custom_learner.template.cfg' config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) outprefix = 'test_majority_class_custom_learner' preds = read_predictions( join(_my_dir, 'output', ('{}_{}_MajorityClassLearner_predictions.tsv'.format( outprefix, outprefix)))) expected = np.array([float(num_labels - 1) for x in preds]) assert_array_equal(preds, expected)
def test_majority_class_custom_learner(): num_labels = 10 # This will make data where the last class happens about 50% of the time. class_weights = [(0.5 / (num_labels - 1)) for x in range(num_labels - 1)] + [0.5] train_fs, test_fs = make_classification_data(num_examples=600, train_test_ratio=0.8, num_labels=num_labels, num_features=5, non_negative=True, class_weights=class_weights) # Write training feature set to a file train_path = join(_my_dir, 'train', 'test_majority_class_custom_learner.jsonlines') writer = NDJWriter(train_path, train_fs) writer.write() # Write test feature set to a file test_path = join(_my_dir, 'test', 'test_majority_class_custom_learner.jsonlines') writer = NDJWriter(test_path, test_fs) writer.write() cfgfile = 'test_majority_class_custom_learner.template.cfg' config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) outprefix = 'test_majority_class_custom_learner' preds = read_predictions(join(_my_dir, 'output', ('{}_{}_MajorityClassLearner_predictions.tsv' .format(outprefix, outprefix)))) expected = np.array([float(num_labels - 1) for x in preds]) assert_array_equal(preds, expected)
def test_ablation_cv(): """ Test ablation + cross-validation """ config_template_path = join(_my_dir, 'configs', 'test_ablation.template.cfg') config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True, ablation=1) # read in the summary file and make sure it has # 7 ablated featuresets * (10 folds + 1 average line) * 2 learners = 154 # lines with open(join(_my_dir, 'output', 'ablation_cv_plain_summary.tsv')) as f: reader = csv.DictReader(f, dialect=csv.excel_tab) num_rows = check_ablation_rows(reader) eq_(num_rows, 154) # make sure there are 7 ablated featuresets * 2 learners = 12 results files num_result_files = len( glob(join(_my_dir, 'output', 'ablation_cv_plain*.results'))) eq_(num_result_files, 14)
def test_class_map_feature_hasher(): """ Test class maps with feature hashing """ make_class_map_data() config_template_path = join(_my_dir, 'configs', 'test_class_map_feature_hasher.template.cfg') config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) with open(join(_my_dir, 'output', ('test_class_map_test_class_map_' 'LogisticRegression.results.' 'json'))) as f: # outstr = f.read() outd = json.loads(f.read()) # logistic_result_score = float( # SCORE_OUTPUT_RE.search(outstr).groups()[0]) logistic_result_score = outd[0]['score'] assert_almost_equal(logistic_result_score, 0.5)
def test_class_map(): """ Test class maps """ make_class_map_data() config_template_path = join(_my_dir, 'configs', 'test_class_map.template.cfg') config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) with open( join(_my_dir, 'output', ('test_class_map_test_class_map_Logistic' 'Regression.results.json'))) as f: outd = json.loads(f.read()) # outstr = f.read() # logistic_result_score = float( # SCORE_OUTPUT_RE.search(outstr).groups()[0]) logistic_result_score = outd[0]['accuracy'] assert_almost_equal(logistic_result_score, 0.5)
def check_summary_score(use_feature_hashing=False): # Test to validate summary file scores make_summary_data() cfgfile = ('test_summary_feature_hasher.template.cfg' if use_feature_hashing else 'test_summary.template.cfg') config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) outprefix = ('test_summary_feature_hasher_test_summary' if use_feature_hashing else 'test_summary_test_summary') summprefix = ('test_summary_feature_hasher' if use_feature_hashing else 'test_summary') with open(join(_my_dir, 'output', ('{}_LogisticRegression.results.' 'json'.format(outprefix)))) as f: outd = json.loads(f.read()) logistic_result_score = outd[0]['score'] with open(join(_my_dir, 'output', '{}_SVC.results.json'.format(outprefix))) as f: outd = json.loads(f.read()) svm_result_score = outd[0]['score'] # note that Naive Bayes doesn't work with feature hashing if not use_feature_hashing: with open(join(_my_dir, 'output', ('{}_MultinomialNB.results.' 'json'.format(outprefix)))) as f: outd = json.loads(f.read()) naivebayes_result_score = outd[0]['score'] with open(join(_my_dir, 'output', '{}_summary.tsv'.format(summprefix)), 'r') as f: reader = csv.DictReader(f, dialect='excel-tab') for row in reader: # the learner results dictionaries should have 27 rows, # and all of these except results_table # should be printed (though some columns will be blank). eq_(len(row), 27) assert row['model_params'] assert row['grid_score'] assert row['score'] if row['learner_name'] == 'LogisticRegression': logistic_summary_score = float(row['score']) elif row['learner_name'] == 'MultinomialNB': naivebayes_summary_score = float(row['score']) elif row['learner_name'] == 'SVC': svm_summary_score = float(row['score']) test_tuples = [(logistic_result_score, logistic_summary_score, 'LogisticRegression'), (svm_result_score, svm_summary_score, 'SVC')] if not use_feature_hashing: test_tuples.append((naivebayes_result_score, naivebayes_summary_score, 'MultinomialNB')) for result_score, summary_score, learner_name in test_tuples: assert_almost_equal(result_score, summary_score, err_msg=('mismatched scores for {} ' '(result:{}, summary:' '{})').format(learner_name, result_score, summary_score))
def check_summary_score(use_feature_hashing=False): # Test to validate summary file scores make_summary_data() cfgfile = ('test_summary_feature_hasher.template.cfg' if use_feature_hashing else 'test_summary.template.cfg') config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) outprefix = ('test_summary_feature_hasher_test_summary' if use_feature_hashing else 'test_summary_test_summary') summprefix = ('test_summary_feature_hasher' if use_feature_hashing else 'test_summary') with open(join(_my_dir, 'output', ('{}_LogisticRegression.results.' 'json'.format(outprefix)))) as f: outd = json.loads(f.read()) logistic_result_score = outd[0]['score'] with open(join(_my_dir, 'output', '{}_SVC.results.json'.format(outprefix))) as f: outd = json.loads(f.read()) svm_result_score = outd[0]['score'] # note that Naive Bayes doesn't work with feature hashing if not use_feature_hashing: with open(join(_my_dir, 'output', ('{}_MultinomialNB.results.' 'json'.format(outprefix)))) as f: outd = json.loads(f.read()) naivebayes_result_score = outd[0]['score'] with open(join(_my_dir, 'output', '{}_summary.tsv'.format(summprefix)), 'r') as f: reader = csv.DictReader(f, dialect='excel-tab') for row in reader: # the learner results dictionaries should have 29 rows, # and all of these except results_table # should be printed (though some columns will be blank). eq_(len(row), 29) assert row['model_params'] assert row['grid_score'] assert row['score'] if row['learner_name'] == 'LogisticRegression': logistic_summary_score = float(row['score']) elif row['learner_name'] == 'MultinomialNB': naivebayes_summary_score = float(row['score']) elif row['learner_name'] == 'SVC': svm_summary_score = float(row['score']) test_tuples = [(logistic_result_score, logistic_summary_score, 'LogisticRegression'), (svm_result_score, svm_summary_score, 'SVC')] if not use_feature_hashing: test_tuples.append((naivebayes_result_score, naivebayes_summary_score, 'MultinomialNB')) for result_score, summary_score, learner_name in test_tuples: assert_almost_equal(result_score, summary_score, err_msg=('mismatched scores for {} ' '(result:{}, summary:' '{})').format(learner_name, result_score, summary_score)) # We itereate over each model with an expected # accuracy score. T est proves that the report # written out at least as a correct format for # this line. See _print_fancy_output for report_name, val in (("LogisticRegression", .5), ("MultinomialNB", .5), ("SVC", .7)): filename = "test_summary_test_summary_{}.results".format(report_name) results_path = join(_my_dir, 'output', filename) with open(results_path) as results_file: report = results_file.read() expected_string = "Accuracy = {:.1f}".format(val) eq_(expected_string in report, # approximate True, msg="{} is not in {}".format(expected_string, report))
def check_summary_score(use_feature_hashing=False): # Test to validate summary file scores make_summary_data() cfgfile = ('test_summary_feature_hasher.template.cfg' if use_feature_hashing else 'test_summary.template.cfg') config_template_path = join(_my_dir, 'configs', cfgfile) config_path = fill_in_config_paths(config_template_path) run_configuration(config_path, quiet=True) outprefix = ('test_summary_feature_hasher_test_summary' if use_feature_hashing else 'test_summary_test_summary') summprefix = ('test_summary_feature_hasher' if use_feature_hashing else 'test_summary') with open( join(_my_dir, 'output', ('{}_LogisticRegression.results.' 'json'.format(outprefix)))) as f: outd = json.loads(f.read()) logistic_result_score = outd[0]['score'] with open(join(_my_dir, 'output', '{}_SVC.results.json'.format(outprefix))) as f: outd = json.loads(f.read()) svm_result_score = outd[0]['score'] # note that Naive Bayes doesn't work with feature hashing if not use_feature_hashing: with open( join(_my_dir, 'output', ('{}_MultinomialNB.results.' 'json'.format(outprefix)))) as f: outd = json.loads(f.read()) naivebayes_result_score = outd[0]['score'] with open(join(_my_dir, 'output', '{}_summary.tsv'.format(summprefix)), 'r') as f: reader = csv.DictReader(f, dialect='excel-tab') for row in reader: # the learner results dictionaries should have 29 rows, # and all of these except results_table # should be printed (though some columns will be blank). eq_(len(row), 29) assert row['model_params'] assert row['grid_score'] assert row['score'] if row['learner_name'] == 'LogisticRegression': logistic_summary_score = float(row['score']) elif row['learner_name'] == 'MultinomialNB': naivebayes_summary_score = float(row['score']) elif row['learner_name'] == 'SVC': svm_summary_score = float(row['score']) test_tuples = [(logistic_result_score, logistic_summary_score, 'LogisticRegression'), (svm_result_score, svm_summary_score, 'SVC')] if not use_feature_hashing: test_tuples.append((naivebayes_result_score, naivebayes_summary_score, 'MultinomialNB')) for result_score, summary_score, learner_name in test_tuples: assert_almost_equal(result_score, summary_score, err_msg=('mismatched scores for {} ' '(result:{}, summary:' '{})').format(learner_name, result_score, summary_score)) # We iterate over each model with an expected # accuracy score. Test proves that the report # written out at least has a correct format for # this line. See _print_fancy_output for report_name, val in (("LogisticRegression", .5), ("MultinomialNB", .5), ("SVC", .6333)): filename = "test_summary_test_summary_{}.results".format(report_name) results_path = join(_my_dir, 'output', filename) with open(results_path) as results_file: report = results_file.read() expected_string = "Accuracy = {:.1f}".format(val) eq_( expected_string in report, # approximate True, msg="{} is not in {}".format(expected_string, report))