def _generate_loglikelihood_reports(loglikelihoods, y, target_names=None): report = metrics.multilabel_loglikelihood_report(y, loglikelihoods) report += '\n\n' report += metrics.multilabel_loglikelihood_summary_report( y, loglikelihoods, target_names=target_names) return report
def _evaluate_model(dataset, iterator, args, print_results=False): loglikelihood_method = args.loglikelihood_method # Collect stats train_loglikelihoods = [] train_predictions = [] train_labels = [] test_loglikelihoods = [] test_predictions = [] test_labels = [] for rnd, (train_indexes, test_indexes) in enumerate(iterator): assert len(set(train_indexes).intersection(set(test_indexes))) == 0 transformers = data.transformers_from_args(args) train, test = dataset.split_train_test(train_indexes, test_indexes, transformers) assert train.n_samples == len(train_indexes) assert test.n_samples == len(test_indexes) train_labels.append(train.y) test_labels.append(test.y) classifier = get_classifier(args) if print_results: print('evaluation round %d' % (rnd + 1)) print(' train split: %s' % train_indexes) print(' test split: %s' % test_indexes) print(' training classifier on training samples ...') start = timeit.default_timer() classifier.fit(train.X, train.y) stop = timeit.default_timer() if args.output_dir is not None: name = 'rnd%d_model.pkl' % (rnd+1) with open(os.path.join(args.output_dir, name), 'wb') as f: pickle.dump(classifier, f) if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on train dataset ...' % loglikelihood_method) start = timeit.default_timer() train_ll = classifier.loglikelihoods(train.X, method=loglikelihood_method) train_loglikelihoods.append(train_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on test dataset ...' % loglikelihood_method) start = timeit.default_timer() test_ll = classifier.loglikelihoods(test.X, method=loglikelihood_method) test_loglikelihoods.append(test_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) decision_maker = decision.decision_maker_from_args(args) train_pred, test_pred = None, None if decision_maker is not None: if hasattr(decision_maker, 'fit') and callable(decision_maker.fit): if print_results: print(' training decision maker %s on train loglikelihoods ...' % args.decision_maker) start = timeit.default_timer() decision_maker.fit(train_ll, train.y) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on train dataset ...') start = timeit.default_timer() train_pred = decision_maker.predict(train_ll) train_predictions.append(train_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on test dataset ...') start = timeit.default_timer() test_pred = decision_maker.predict(test_ll) test_predictions.append(test_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print('') # Save round results if args.output_dir is not None: save_results(args.output_dir, train.y, train_pred, train_ll, prefix='rnd%d_train' % (rnd+1)) save_results(args.output_dir, test.y, test_pred, test_ll, prefix='rnd%d_test' % (rnd+1)) # Combine and save combined results train_y_combined = np.vstack(train_labels) train_ll_combined = np.vstack(train_loglikelihoods) train_pred_combined = np.vstack(train_predictions) if len(train_predictions) > 0 else None test_ll_combined = np.vstack(test_loglikelihoods) test_y_combined = np.vstack(test_labels) test_pred_combined = np.vstack(test_predictions) if len(test_predictions) > 0 else None if args.output_dir is not None: save_results(args.output_dir, train_y_combined, train_pred_combined, train_ll_combined, 'combined_train') save_results(args.output_dir, test_y_combined, test_pred_combined, test_ll_combined, 'combined_test') if print_results: # Print report label_names = dataset.unique_labels print('*** train dataset summary ***') print('') print(metrics.multilabel_loglikelihood_summary_report(train_y_combined, train_ll_combined, target_names=label_names)) print('') if train_pred_combined is not None: print(metrics.multilabel_classification_report(train_y_combined, train_pred_combined, target_names=label_names)) print('total accuracy: %.3f' % sk_metrics.accuracy_score(train_y_combined, train_pred_combined)) print('') print('') print('*** test dataset summary ***') print('') print(metrics.multilabel_loglikelihood_summary_report(test_y_combined, test_ll_combined, target_names=label_names)) print('') if test_pred_combined is not None: print(metrics.multilabel_classification_report(test_y_combined, test_pred_combined, target_names=label_names)) print('total accuracy: %.3f' % sk_metrics.accuracy_score(test_y_combined, test_pred_combined)) print('') return train_loglikelihoods, train_labels, test_loglikelihoods, test_labels
def _generate_loglikelihood_reports(loglikelihoods, y, target_names=None): report = metrics.multilabel_loglikelihood_report(y, loglikelihoods) report += '\n\n' report += metrics.multilabel_loglikelihood_summary_report(y, loglikelihoods, target_names=target_names) return report
def _evaluate_model(dataset, iterator, args, print_results=False): loglikelihood_method = args.loglikelihood_method # Collect stats train_loglikelihoods = [] train_predictions = [] train_labels = [] test_loglikelihoods = [] test_predictions = [] test_labels = [] for rnd, (train_indexes, test_indexes) in enumerate(iterator): assert len(set(train_indexes).intersection(set(test_indexes))) == 0 transformers = data.transformers_from_args(args) train, test = dataset.split_train_test(train_indexes, test_indexes, transformers) assert train.n_samples == len(train_indexes) assert test.n_samples == len(test_indexes) train_labels.append(train.y) test_labels.append(test.y) classifier = get_classifier(args) if print_results: print('evaluation round %d' % (rnd + 1)) print(' train split: %s' % train_indexes) print(' test split: %s' % test_indexes) print(' training classifier on training samples ...') start = timeit.default_timer() classifier.fit(train.X, train.y) stop = timeit.default_timer() if args.output_dir is not None: name = 'rnd%d_model.pkl' % (rnd + 1) with open(os.path.join(args.output_dir, name), 'wb') as f: pickle.dump(classifier, f) if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on train dataset ...' % loglikelihood_method) start = timeit.default_timer() train_ll = classifier.loglikelihoods(train.X, method=loglikelihood_method) train_loglikelihoods.append(train_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on test dataset ...' % loglikelihood_method) start = timeit.default_timer() test_ll = classifier.loglikelihoods(test.X, method=loglikelihood_method) test_loglikelihoods.append(test_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) decision_maker = decision.decision_maker_from_args(args) train_pred, test_pred = None, None if decision_maker is not None: if hasattr(decision_maker, 'fit') and callable(decision_maker.fit): if print_results: print( ' training decision maker %s on train loglikelihoods ...' % args.decision_maker) start = timeit.default_timer() decision_maker.fit(train_ll, train.y) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on train dataset ...') start = timeit.default_timer() train_pred = decision_maker.predict(train_ll) train_predictions.append(train_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on test dataset ...') start = timeit.default_timer() test_pred = decision_maker.predict(test_ll) test_predictions.append(test_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print('') # Save round results if args.output_dir is not None: save_results(args.output_dir, train.y, train_pred, train_ll, prefix='rnd%d_train' % (rnd + 1)) save_results(args.output_dir, test.y, test_pred, test_ll, prefix='rnd%d_test' % (rnd + 1)) # Combine and save combined results train_y_combined = np.vstack(train_labels) train_ll_combined = np.vstack(train_loglikelihoods) train_pred_combined = np.vstack( train_predictions) if len(train_predictions) > 0 else None test_ll_combined = np.vstack(test_loglikelihoods) test_y_combined = np.vstack(test_labels) test_pred_combined = np.vstack( test_predictions) if len(test_predictions) > 0 else None if args.output_dir is not None: save_results(args.output_dir, train_y_combined, train_pred_combined, train_ll_combined, 'combined_train') save_results(args.output_dir, test_y_combined, test_pred_combined, test_ll_combined, 'combined_test') if print_results: # Print report label_names = dataset.unique_labels print('*** train dataset summary ***') print('') print( metrics.multilabel_loglikelihood_summary_report( train_y_combined, train_ll_combined, target_names=label_names)) print('') if train_pred_combined is not None: print( metrics.multilabel_classification_report( train_y_combined, train_pred_combined, target_names=label_names)) print( 'total accuracy: %.3f' % sk_metrics.accuracy_score( train_y_combined, train_pred_combined)) print('') print('') print('*** test dataset summary ***') print('') print( metrics.multilabel_loglikelihood_summary_report( test_y_combined, test_ll_combined, target_names=label_names)) print('') if test_pred_combined is not None: print( metrics.multilabel_classification_report( test_y_combined, test_pred_combined, target_names=label_names)) print( 'total accuracy: %.3f' % sk_metrics.accuracy_score(test_y_combined, test_pred_combined)) print('') return train_loglikelihoods, train_labels, test_loglikelihoods, test_labels