def _generate_classification_reports(y_true, y_pred, target_names=None):
# Calculate additional stats
total_accuracy = accuracy_score(y_true, y_pred)
cov_error = coverage_error(y_true, y_pred)
lrap = label_ranking_average_precision_score(y_true, y_pred)
report = metrics.multilabel_prediction_report(y_true, y_pred)
report += '\n\n'
report += metrics.multilabel_classification_report(y_true, y_pred, target_names=target_names)
report += '\n\n'
report += 'coverage error: %.3f' % cov_error
report += '\n'
report += 'LRAP: %.3f' % lrap
report += '\n'
report += 'total accuracy: %.3f' % total_accuracy
return report
def _generate_classification_reports(y_true, y_pred, target_names=None):
# Calculate additional stats
total_accuracy = accuracy_score(y_true, y_pred)
cov_error = coverage_error(y_true, y_pred)
lrap = label_ranking_average_precision_score(y_true, y_pred)
report = metrics.multilabel_prediction_report(y_true, y_pred)
report += '\n\n'
report += metrics.multilabel_classification_report(
y_true, y_pred, target_names=target_names)
report += '\n\n'
report += 'coverage error: %.3f' % cov_error
report += '\n'
report += 'LRAP: %.3f' % lrap
report += '\n'
report += 'total accuracy: %.3f' % total_accuracy
return report
def _evaluate_model(dataset, iterator, args, print_results=False):
loglikelihood_method = args.loglikelihood_method
# Collect stats
train_loglikelihoods = []
train_predictions = []
train_labels = []
test_loglikelihoods = []
test_predictions = []
test_labels = []
for rnd, (train_indexes, test_indexes) in enumerate(iterator):
assert len(set(train_indexes).intersection(set(test_indexes))) == 0
transformers = data.transformers_from_args(args)
train, test = dataset.split_train_test(train_indexes, test_indexes, transformers)
assert train.n_samples == len(train_indexes)
assert test.n_samples == len(test_indexes)
train_labels.append(train.y)
test_labels.append(test.y)
classifier = get_classifier(args)
if print_results:
print('evaluation round %d' % (rnd + 1))
print(' train split: %s' % train_indexes)
print(' test split: %s' % test_indexes)
print(' training classifier on training samples ...')
start = timeit.default_timer()
classifier.fit(train.X, train.y)
stop = timeit.default_timer()
if args.output_dir is not None:
name = 'rnd%d_model.pkl' % (rnd+1)
with open(os.path.join(args.output_dir, name), 'wb') as f:
pickle.dump(classifier, f)
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' computing %s loglikelihoods on train dataset ...' % loglikelihood_method)
start = timeit.default_timer()
train_ll = classifier.loglikelihoods(train.X, method=loglikelihood_method)
train_loglikelihoods.append(train_ll)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' computing %s loglikelihoods on test dataset ...' % loglikelihood_method)
start = timeit.default_timer()
test_ll = classifier.loglikelihoods(test.X, method=loglikelihood_method)
test_loglikelihoods.append(test_ll)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
decision_maker = decision.decision_maker_from_args(args)
train_pred, test_pred = None, None
if decision_maker is not None:
if hasattr(decision_maker, 'fit') and callable(decision_maker.fit):
if print_results:
print(' training decision maker %s on train loglikelihoods ...' % args.decision_maker)
start = timeit.default_timer()
decision_maker.fit(train_ll, train.y)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' predicting labels on train dataset ...')
start = timeit.default_timer()
train_pred = decision_maker.predict(train_ll)
train_predictions.append(train_pred)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' predicting labels on test dataset ...')
start = timeit.default_timer()
test_pred = decision_maker.predict(test_ll)
test_predictions.append(test_pred)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print('')
# Save round results
if args.output_dir is not None:
save_results(args.output_dir, train.y, train_pred, train_ll, prefix='rnd%d_train' % (rnd+1))
save_results(args.output_dir, test.y, test_pred, test_ll, prefix='rnd%d_test' % (rnd+1))
# Combine and save combined results
train_y_combined = np.vstack(train_labels)
train_ll_combined = np.vstack(train_loglikelihoods)
train_pred_combined = np.vstack(train_predictions) if len(train_predictions) > 0 else None
test_ll_combined = np.vstack(test_loglikelihoods)
test_y_combined = np.vstack(test_labels)
test_pred_combined = np.vstack(test_predictions) if len(test_predictions) > 0 else None
if args.output_dir is not None:
save_results(args.output_dir, train_y_combined, train_pred_combined, train_ll_combined, 'combined_train')
save_results(args.output_dir, test_y_combined, test_pred_combined, test_ll_combined, 'combined_test')
if print_results:
# Print report
label_names = dataset.unique_labels
print('*** train dataset summary ***')
print('')
print(metrics.multilabel_loglikelihood_summary_report(train_y_combined, train_ll_combined, target_names=label_names))
print('')
if train_pred_combined is not None:
print(metrics.multilabel_classification_report(train_y_combined, train_pred_combined, target_names=label_names))
print('total accuracy: %.3f' % sk_metrics.accuracy_score(train_y_combined, train_pred_combined))
print('')
print('')
print('*** test dataset summary ***')
print('')
print(metrics.multilabel_loglikelihood_summary_report(test_y_combined, test_ll_combined, target_names=label_names))
print('')
if test_pred_combined is not None:
print(metrics.multilabel_classification_report(test_y_combined, test_pred_combined, target_names=label_names))
print('total accuracy: %.3f' % sk_metrics.accuracy_score(test_y_combined, test_pred_combined))
print('')
return train_loglikelihoods, train_labels, test_loglikelihoods, test_labels
def _evaluate_model(dataset, iterator, args, print_results=False):
loglikelihood_method = args.loglikelihood_method
# Collect stats
train_loglikelihoods = []
train_predictions = []
train_labels = []
test_loglikelihoods = []
test_predictions = []
test_labels = []
for rnd, (train_indexes, test_indexes) in enumerate(iterator):
assert len(set(train_indexes).intersection(set(test_indexes))) == 0
transformers = data.transformers_from_args(args)
train, test = dataset.split_train_test(train_indexes, test_indexes,
transformers)
assert train.n_samples == len(train_indexes)
assert test.n_samples == len(test_indexes)
train_labels.append(train.y)
test_labels.append(test.y)
classifier = get_classifier(args)
if print_results:
print('evaluation round %d' % (rnd + 1))
print(' train split: %s' % train_indexes)
print(' test split: %s' % test_indexes)
print(' training classifier on training samples ...')
start = timeit.default_timer()
classifier.fit(train.X, train.y)
stop = timeit.default_timer()
if args.output_dir is not None:
name = 'rnd%d_model.pkl' % (rnd + 1)
with open(os.path.join(args.output_dir, name), 'wb') as f:
pickle.dump(classifier, f)
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' computing %s loglikelihoods on train dataset ...' %
loglikelihood_method)
start = timeit.default_timer()
train_ll = classifier.loglikelihoods(train.X,
method=loglikelihood_method)
train_loglikelihoods.append(train_ll)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' computing %s loglikelihoods on test dataset ...' %
loglikelihood_method)
start = timeit.default_timer()
test_ll = classifier.loglikelihoods(test.X,
method=loglikelihood_method)
test_loglikelihoods.append(test_ll)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
decision_maker = decision.decision_maker_from_args(args)
train_pred, test_pred = None, None
if decision_maker is not None:
if hasattr(decision_maker, 'fit') and callable(decision_maker.fit):
if print_results:
print(
' training decision maker %s on train loglikelihoods ...'
% args.decision_maker)
start = timeit.default_timer()
decision_maker.fit(train_ll, train.y)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' predicting labels on train dataset ...')
start = timeit.default_timer()
train_pred = decision_maker.predict(train_ll)
train_predictions.append(train_pred)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print(' predicting labels on test dataset ...')
start = timeit.default_timer()
test_pred = decision_maker.predict(test_ll)
test_predictions.append(test_pred)
stop = timeit.default_timer()
if print_results:
print(' done, took %fs' % (stop - start))
if print_results:
print('')
# Save round results
if args.output_dir is not None:
save_results(args.output_dir,
train.y,
train_pred,
train_ll,
prefix='rnd%d_train' % (rnd + 1))
save_results(args.output_dir,
test.y,
test_pred,
test_ll,
prefix='rnd%d_test' % (rnd + 1))
# Combine and save combined results
train_y_combined = np.vstack(train_labels)
train_ll_combined = np.vstack(train_loglikelihoods)
train_pred_combined = np.vstack(
train_predictions) if len(train_predictions) > 0 else None
test_ll_combined = np.vstack(test_loglikelihoods)
test_y_combined = np.vstack(test_labels)
test_pred_combined = np.vstack(
test_predictions) if len(test_predictions) > 0 else None
if args.output_dir is not None:
save_results(args.output_dir, train_y_combined, train_pred_combined,
train_ll_combined, 'combined_train')
save_results(args.output_dir, test_y_combined, test_pred_combined,
test_ll_combined, 'combined_test')
if print_results:
# Print report
label_names = dataset.unique_labels
print('*** train dataset summary ***')
print('')
print(
metrics.multilabel_loglikelihood_summary_report(
train_y_combined, train_ll_combined, target_names=label_names))
print('')
if train_pred_combined is not None:
print(
metrics.multilabel_classification_report(
train_y_combined,
train_pred_combined,
target_names=label_names))
print(
'total accuracy: %.3f' % sk_metrics.accuracy_score(
train_y_combined, train_pred_combined))
print('')
print('')
print('*** test dataset summary ***')
print('')
print(
metrics.multilabel_loglikelihood_summary_report(
test_y_combined, test_ll_combined, target_names=label_names))
print('')
if test_pred_combined is not None:
print(
metrics.multilabel_classification_report(
test_y_combined,
test_pred_combined,
target_names=label_names))
print(
'total accuracy: %.3f' %
sk_metrics.accuracy_score(test_y_combined, test_pred_combined))
print('')
return train_loglikelihoods, train_labels, test_loglikelihoods, test_labels
