def _evaluate_combinations(values, train_splits, test_splits, args): scores = [] keys = values.keys() combinations = list(itertools.product(*values.values())) for combination in combinations: for idx, value in enumerate(combination): setattr(args, "decision_maker_" + keys[idx], value) print("evaluating combination %s ..." % str(combination)) start = timeit.default_timer() y_true = [] y_pred = [] for rnd, (train, test) in enumerate(zip(train_splits, test_splits)): decision_maker = decision.decision_maker_from_args(args) y_pred.append(evaluate_decision_maker(decision_maker, train, test, args)) y_true.append(test[1]) y_true = np.vstack(y_true) y_pred = np.vstack(y_pred) assert y_true.shape == y_pred.shape score = _compute_score(y_true, y_pred, args) scores.append(score) print("score: %f" % score) print("done, took %fs" % (timeit.default_timer() - start)) print("") best_idx = _select_best_score(scores, args) print("best combination with score %f: %s" % (scores[best_idx], str(combinations[best_idx]))) # Save results if args.output is not None: with open(args.output, "wb") as f: writer = csv.writer(f, delimiter=";") writer.writerow(["", "idx", "score"] + keys) for idx, (score, combination) in enumerate(zip(scores, combinations)): selected = "*" if best_idx == idx else "" writer.writerow([selected, "%d" % idx, "%f" % score] + list(combination))
def _evaluate_combinations(values, train_splits, test_splits, args): scores = [] keys = values.keys() combinations = list(itertools.product(*values.values())) for combination in combinations: for idx, value in enumerate(combination): setattr(args, 'decision_maker_' + keys[idx], value) print('evaluating combination %s ...' % str(combination)) start = timeit.default_timer() y_true = [] y_pred = [] for rnd, (train, test) in enumerate(zip(train_splits, test_splits)): decision_maker = decision.decision_maker_from_args(args) y_pred.append( evaluate_decision_maker(decision_maker, train, test, args)) y_true.append(test[1]) y_true = np.vstack(y_true) y_pred = np.vstack(y_pred) assert y_true.shape == y_pred.shape score = _compute_score(y_true, y_pred, args) scores.append(score) print('score: %f' % score) print('done, took %fs' % (timeit.default_timer() - start)) print('') best_idx = _select_best_score(scores, args) print('best combination with score %f: %s' % (scores[best_idx], str(combinations[best_idx]))) # Save results if args.output is not None: with open(args.output, 'wb') as f: writer = csv.writer(f, delimiter=';') writer.writerow(['', 'idx', 'score'] + keys) for idx, (score, combination) in enumerate(zip(scores, combinations)): selected = '*' if best_idx == idx else '' writer.writerow([selected, '%d' % idx, '%f' % score] + list(combination))
def evaluate_end_to_end(dataset, iterator, args): # Things to evaluate feature_sets = [('normalized_root_rot_norm',), ('normalized_root_pos', 'normalized_root_vel', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm'), ('normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm',), ('normalized_root_pos', 'normalized_root_vel', 'normalized_com_pos', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm'), ('normalized_root_vel', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm'), ('normalized_root_pos', 'normalized_root_vel', 'normalized_com_pos', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm', 'normalized_marker_vel_norm')] values = {'features': range(len(feature_sets)), 'hyperparams': [('left-to-right-full', 5), ('left-to-right-1', 6), ('left-to-right-2', 5), ('full', 8)], 'init': [('uniform', 'k-means', 'diag')], 'model': [('hmm', None), ('fhmm-seq', 2)], 'decision': ['all']} decision_makers = ['log-regression', 'svm', 'decision-tree', 'random-forest', 'zero', 'max'] datasets = [] print('selecting features ...') start = timeit.default_timer() for feature_set in feature_sets: features = _explode_features(feature_set) curr_dataset = dataset.dataset_from_feature_names(features) datasets.append(curr_dataset) dataset = None # ensure that dataset is not usable hereinafter assert len(datasets) == len(feature_sets) print('done, took %fs' % (timeit.default_timer() - start)) print('') # Save state output_dir = args.output_dir # Stats combinations = [] total_accuracies = [] precisions_mean = [] precisions_std = [] precisions_min = [] precisions_max = [] recalls_mean = [] recalls_std = [] recalls_min = [] recalls_max = [] fscores_mean = [] fscores_std = [] fscores_max = [] fscores_min = [] pos_ll_means = [] pos_ll_stds = [] neg_ll_means = [] neg_ll_stds = [] keys = values.keys() iterable_combinations = list(itertools.product(*values.values())) curr_step = 0 for idx, combination in enumerate(iterable_combinations): print('(%.3d/%.3d) evaluating combination %s + decision makers ...' % (idx + 1, len(iterable_combinations), str(combination))) start = timeit.default_timer() curr_dataset = datasets[combination[keys.index('features')]] topology, n_states = combination[keys.index('hyperparams')] transition_init, emission_init, covar_type = combination[keys.index('init')] model, n_chains = combination[keys.index('model')] if output_dir is not None: curr_path = os.path.join(output_dir, '%.3d' % curr_step) os.mkdir(curr_path) args.output_dir = curr_path # Configure HMMs args.topology = topology args.n_states = n_states args.transition_init = transition_init args.emission_init = emission_init args.covar_type = covar_type args.model = model args.n_chains = n_chains args.decision_maker = None train_ll, train_y, test_ll, test_y = _evaluate_model(curr_dataset, iterator, args, print_results=False) assert len(train_ll) == len(train_y) assert len(train_ll) == len(test_ll) assert len(train_ll) == len(test_y) train_ll_combined = np.vstack(train_ll) test_ll_combined = np.vstack(test_ll) test_y_combined = np.vstack(test_y) assert train_ll_combined.shape == train_ll_combined.shape assert test_ll_combined.shape == test_y_combined.shape assert train_ll_combined.shape[0] > test_ll_combined.shape[0] # just a sanity check so that the both are not confused n_samples, n_labels = test_ll_combined.shape curr_pos_ll_means = [] curr_pos_ll_stds = [] curr_neg_ll_means = [] curr_neg_ll_stds = [] for label_idx in xrange(n_labels): label_y = test_y_combined[:, label_idx] pos_indexes = np.where(label_y == 1)[0] neg_indexes = np.where(label_y == 0)[0] pos_ll = test_ll_combined[pos_indexes, label_idx] neg_ll = test_ll_combined[neg_indexes, label_idx] assert np.size(pos_ll) + np.size(neg_ll) == n_samples curr_pos_ll_means.append(np.mean(pos_ll)) curr_pos_ll_stds.append(np.std(pos_ll)) curr_neg_ll_means.append(np.mean(neg_ll)) curr_neg_ll_stds.append(np.std(neg_ll)) for name in decision_makers: args.decision_maker = name modified_combination = list(combination) modified_combination[keys.index('decision')] = name if name == 'svm': args.decision_maker_C = 1e-2 args.decision_maker_penalty = 'l1' elif name == 'log-regression': args.decision_maker_C = 1e-3 args.decision_maker_penalty = 'l1' elif name == 'decision-tree': args.decision_maker_criterion = 'entropy' args.decision_maker_max_depth = 15 elif name == 'random-forest': args.decision_maker_criterion = 'entropy' args.decision_maker_n_estimators = 40 args.decision_maker_max_depth = 15 curr_preds = [] for curr_train_ll, curr_train_y, curr_test_ll in zip(train_ll, train_y, test_ll): print('training decision maker') assert curr_train_ll.shape == curr_train_y.shape assert curr_train_ll.shape[0] > curr_test_ll.shape[0] decision_maker = decision.decision_maker_from_args(args) assert decision_maker is not None # Fit and predict using the decision maker if hasattr(decision_maker, 'fit') and callable(decision_maker.fit): decision_maker.fit(curr_train_ll, curr_train_y) curr_preds.append(decision_maker.predict(curr_test_ll)) print('') test_pred_combined = np.vstack(curr_preds) assert test_y_combined.shape == test_pred_combined.shape # Track everything combinations.append(modified_combination) total_accuracies.append(sk_metrics.accuracy_score(test_y_combined, test_pred_combined)) precision, recall, fscore, _ = sk_metrics.precision_recall_fscore_support(test_y_combined, test_pred_combined) precisions_mean.append(np.mean(precision)) precisions_std.append(np.std(precision)) precisions_min.append(np.min(precision)) precisions_max.append(np.max(precision)) recalls_mean.append(np.mean(recall)) recalls_std.append(np.std(recall)) recalls_min.append(np.min(recall)) recalls_max.append(np.max(recall)) fscores_mean.append(np.mean(fscore)) fscores_std.append(np.std(fscore)) fscores_min.append(np.min(fscore)) fscores_max.append(np.max(fscore)) pos_ll_means.append(np.array(np.median(curr_pos_ll_means))) pos_ll_stds.append(np.array(np.median(curr_pos_ll_stds))) neg_ll_means.append(np.array(np.median(curr_neg_ll_means))) neg_ll_stds.append(np.array(np.median(curr_neg_ll_stds))) curr_step += 1 print('done, took %fs' % (timeit.default_timer() - start)) print('') assert len(combinations) == len(fscores_mean) assert len(combinations) == len(fscores_std) assert len(combinations) == len(fscores_min) assert len(combinations) == len(fscores_max) assert len(combinations) == len(precisions_mean) assert len(combinations) == len(precisions_std) assert len(combinations) == len(precisions_min) assert len(combinations) == len(precisions_max) assert len(combinations) == len(recalls_mean) assert len(combinations) == len(recalls_std) assert len(combinations) == len(recalls_min) assert len(combinations) == len(recalls_max) assert len(combinations) == len(pos_ll_means) assert len(combinations) == len(pos_ll_stds) assert len(combinations) == len(neg_ll_means) assert len(combinations) == len(neg_ll_stds) assert len(combinations) == len(total_accuracies) # Save results if output_dir is not None: filename = 'results.csv' with open(os.path.join(output_dir, filename), 'wb') as f: writer = csv.writer(f, delimiter=';') writer.writerow(['idx', 'combination', 'f1-score-mean', 'f1-score-std', 'f1-score-min', 'f1-score-max', 'precision-mean', 'precision-std', 'precision-min', 'precision-max', 'recall-mean', 'recall-std', 'recall-min', 'recall-max', 'total-accuracy', 'pos-ll-mean', 'pos-ll-std', 'neg-ll-mean', 'neg-ll-std']) for idx, d in enumerate(zip(combinations, fscores_mean, fscores_std, fscores_min, fscores_max, precisions_mean, precisions_std, precisions_min, precisions_max, recalls_mean, recalls_std, recalls_min, recalls_max, total_accuracies, pos_ll_means, pos_ll_stds, neg_ll_means, neg_ll_stds)): combination = ', '.join([str(x) for x in d[0]]) new_data = ['%d' % idx] + list((combination, ) + d[1:]) writer.writerow(new_data) print len(combinations)
def _evaluate_model(dataset, iterator, args, print_results=False): loglikelihood_method = args.loglikelihood_method # Collect stats train_loglikelihoods = [] train_predictions = [] train_labels = [] test_loglikelihoods = [] test_predictions = [] test_labels = [] for rnd, (train_indexes, test_indexes) in enumerate(iterator): assert len(set(train_indexes).intersection(set(test_indexes))) == 0 transformers = data.transformers_from_args(args) train, test = dataset.split_train_test(train_indexes, test_indexes, transformers) assert train.n_samples == len(train_indexes) assert test.n_samples == len(test_indexes) train_labels.append(train.y) test_labels.append(test.y) classifier = get_classifier(args) if print_results: print('evaluation round %d' % (rnd + 1)) print(' train split: %s' % train_indexes) print(' test split: %s' % test_indexes) print(' training classifier on training samples ...') start = timeit.default_timer() classifier.fit(train.X, train.y) stop = timeit.default_timer() if args.output_dir is not None: name = 'rnd%d_model.pkl' % (rnd+1) with open(os.path.join(args.output_dir, name), 'wb') as f: pickle.dump(classifier, f) if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on train dataset ...' % loglikelihood_method) start = timeit.default_timer() train_ll = classifier.loglikelihoods(train.X, method=loglikelihood_method) train_loglikelihoods.append(train_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on test dataset ...' % loglikelihood_method) start = timeit.default_timer() test_ll = classifier.loglikelihoods(test.X, method=loglikelihood_method) test_loglikelihoods.append(test_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) decision_maker = decision.decision_maker_from_args(args) train_pred, test_pred = None, None if decision_maker is not None: if hasattr(decision_maker, 'fit') and callable(decision_maker.fit): if print_results: print(' training decision maker %s on train loglikelihoods ...' % args.decision_maker) start = timeit.default_timer() decision_maker.fit(train_ll, train.y) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on train dataset ...') start = timeit.default_timer() train_pred = decision_maker.predict(train_ll) train_predictions.append(train_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on test dataset ...') start = timeit.default_timer() test_pred = decision_maker.predict(test_ll) test_predictions.append(test_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print('') # Save round results if args.output_dir is not None: save_results(args.output_dir, train.y, train_pred, train_ll, prefix='rnd%d_train' % (rnd+1)) save_results(args.output_dir, test.y, test_pred, test_ll, prefix='rnd%d_test' % (rnd+1)) # Combine and save combined results train_y_combined = np.vstack(train_labels) train_ll_combined = np.vstack(train_loglikelihoods) train_pred_combined = np.vstack(train_predictions) if len(train_predictions) > 0 else None test_ll_combined = np.vstack(test_loglikelihoods) test_y_combined = np.vstack(test_labels) test_pred_combined = np.vstack(test_predictions) if len(test_predictions) > 0 else None if args.output_dir is not None: save_results(args.output_dir, train_y_combined, train_pred_combined, train_ll_combined, 'combined_train') save_results(args.output_dir, test_y_combined, test_pred_combined, test_ll_combined, 'combined_test') if print_results: # Print report label_names = dataset.unique_labels print('*** train dataset summary ***') print('') print(metrics.multilabel_loglikelihood_summary_report(train_y_combined, train_ll_combined, target_names=label_names)) print('') if train_pred_combined is not None: print(metrics.multilabel_classification_report(train_y_combined, train_pred_combined, target_names=label_names)) print('total accuracy: %.3f' % sk_metrics.accuracy_score(train_y_combined, train_pred_combined)) print('') print('') print('*** test dataset summary ***') print('') print(metrics.multilabel_loglikelihood_summary_report(test_y_combined, test_ll_combined, target_names=label_names)) print('') if test_pred_combined is not None: print(metrics.multilabel_classification_report(test_y_combined, test_pred_combined, target_names=label_names)) print('total accuracy: %.3f' % sk_metrics.accuracy_score(test_y_combined, test_pred_combined)) print('') return train_loglikelihoods, train_labels, test_loglikelihoods, test_labels
def _evaluate_model(dataset, iterator, args, print_results=False): loglikelihood_method = args.loglikelihood_method # Collect stats train_loglikelihoods = [] train_predictions = [] train_labels = [] test_loglikelihoods = [] test_predictions = [] test_labels = [] for rnd, (train_indexes, test_indexes) in enumerate(iterator): assert len(set(train_indexes).intersection(set(test_indexes))) == 0 transformers = data.transformers_from_args(args) train, test = dataset.split_train_test(train_indexes, test_indexes, transformers) assert train.n_samples == len(train_indexes) assert test.n_samples == len(test_indexes) train_labels.append(train.y) test_labels.append(test.y) classifier = get_classifier(args) if print_results: print('evaluation round %d' % (rnd + 1)) print(' train split: %s' % train_indexes) print(' test split: %s' % test_indexes) print(' training classifier on training samples ...') start = timeit.default_timer() classifier.fit(train.X, train.y) stop = timeit.default_timer() if args.output_dir is not None: name = 'rnd%d_model.pkl' % (rnd + 1) with open(os.path.join(args.output_dir, name), 'wb') as f: pickle.dump(classifier, f) if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on train dataset ...' % loglikelihood_method) start = timeit.default_timer() train_ll = classifier.loglikelihoods(train.X, method=loglikelihood_method) train_loglikelihoods.append(train_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' computing %s loglikelihoods on test dataset ...' % loglikelihood_method) start = timeit.default_timer() test_ll = classifier.loglikelihoods(test.X, method=loglikelihood_method) test_loglikelihoods.append(test_ll) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) decision_maker = decision.decision_maker_from_args(args) train_pred, test_pred = None, None if decision_maker is not None: if hasattr(decision_maker, 'fit') and callable(decision_maker.fit): if print_results: print( ' training decision maker %s on train loglikelihoods ...' % args.decision_maker) start = timeit.default_timer() decision_maker.fit(train_ll, train.y) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on train dataset ...') start = timeit.default_timer() train_pred = decision_maker.predict(train_ll) train_predictions.append(train_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print(' predicting labels on test dataset ...') start = timeit.default_timer() test_pred = decision_maker.predict(test_ll) test_predictions.append(test_pred) stop = timeit.default_timer() if print_results: print(' done, took %fs' % (stop - start)) if print_results: print('') # Save round results if args.output_dir is not None: save_results(args.output_dir, train.y, train_pred, train_ll, prefix='rnd%d_train' % (rnd + 1)) save_results(args.output_dir, test.y, test_pred, test_ll, prefix='rnd%d_test' % (rnd + 1)) # Combine and save combined results train_y_combined = np.vstack(train_labels) train_ll_combined = np.vstack(train_loglikelihoods) train_pred_combined = np.vstack( train_predictions) if len(train_predictions) > 0 else None test_ll_combined = np.vstack(test_loglikelihoods) test_y_combined = np.vstack(test_labels) test_pred_combined = np.vstack( test_predictions) if len(test_predictions) > 0 else None if args.output_dir is not None: save_results(args.output_dir, train_y_combined, train_pred_combined, train_ll_combined, 'combined_train') save_results(args.output_dir, test_y_combined, test_pred_combined, test_ll_combined, 'combined_test') if print_results: # Print report label_names = dataset.unique_labels print('*** train dataset summary ***') print('') print( metrics.multilabel_loglikelihood_summary_report( train_y_combined, train_ll_combined, target_names=label_names)) print('') if train_pred_combined is not None: print( metrics.multilabel_classification_report( train_y_combined, train_pred_combined, target_names=label_names)) print( 'total accuracy: %.3f' % sk_metrics.accuracy_score( train_y_combined, train_pred_combined)) print('') print('') print('*** test dataset summary ***') print('') print( metrics.multilabel_loglikelihood_summary_report( test_y_combined, test_ll_combined, target_names=label_names)) print('') if test_pred_combined is not None: print( metrics.multilabel_classification_report( test_y_combined, test_pred_combined, target_names=label_names)) print( 'total accuracy: %.3f' % sk_metrics.accuracy_score(test_y_combined, test_pred_combined)) print('') return train_loglikelihoods, train_labels, test_loglikelihoods, test_labels
def evaluate_end_to_end(dataset, iterator, args): # Things to evaluate feature_sets = [ ('normalized_root_rot_norm', ), ('normalized_root_pos', 'normalized_root_vel', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm'), ( 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm', ), ('normalized_root_pos', 'normalized_root_vel', 'normalized_com_pos', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm'), ('normalized_root_vel', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm'), ('normalized_root_pos', 'normalized_root_vel', 'normalized_com_pos', 'normalized_extremity_pos', 'normalized_root_rot', 'normalized_root_rot_norm', 'normalized_marker_vel_norm') ] values = { 'features': range(len(feature_sets)), 'hyperparams': [('left-to-right-full', 5), ('left-to-right-1', 6), ('left-to-right-2', 5), ('full', 8)], 'init': [('uniform', 'k-means', 'diag')], 'model': [('hmm', None), ('fhmm-seq', 2)], 'decision': ['all'] } decision_makers = [ 'log-regression', 'svm', 'decision-tree', 'random-forest', 'zero', 'max' ] datasets = [] print('selecting features ...') start = timeit.default_timer() for feature_set in feature_sets: features = _explode_features(feature_set) curr_dataset = dataset.dataset_from_feature_names(features) datasets.append(curr_dataset) dataset = None # ensure that dataset is not usable hereinafter assert len(datasets) == len(feature_sets) print('done, took %fs' % (timeit.default_timer() - start)) print('') # Save state output_dir = args.output_dir # Stats combinations = [] total_accuracies = [] precisions_mean = [] precisions_std = [] precisions_min = [] precisions_max = [] recalls_mean = [] recalls_std = [] recalls_min = [] recalls_max = [] fscores_mean = [] fscores_std = [] fscores_max = [] fscores_min = [] pos_ll_means = [] pos_ll_stds = [] neg_ll_means = [] neg_ll_stds = [] keys = values.keys() iterable_combinations = list(itertools.product(*values.values())) curr_step = 0 for idx, combination in enumerate(iterable_combinations): print('(%.3d/%.3d) evaluating combination %s + decision makers ...' % (idx + 1, len(iterable_combinations), str(combination))) start = timeit.default_timer() curr_dataset = datasets[combination[keys.index('features')]] topology, n_states = combination[keys.index('hyperparams')] transition_init, emission_init, covar_type = combination[keys.index( 'init')] model, n_chains = combination[keys.index('model')] if output_dir is not None: curr_path = os.path.join(output_dir, '%.3d' % curr_step) os.mkdir(curr_path) args.output_dir = curr_path # Configure HMMs args.topology = topology args.n_states = n_states args.transition_init = transition_init args.emission_init = emission_init args.covar_type = covar_type args.model = model args.n_chains = n_chains args.decision_maker = None train_ll, train_y, test_ll, test_y = _evaluate_model( curr_dataset, iterator, args, print_results=False) assert len(train_ll) == len(train_y) assert len(train_ll) == len(test_ll) assert len(train_ll) == len(test_y) train_ll_combined = np.vstack(train_ll) test_ll_combined = np.vstack(test_ll) test_y_combined = np.vstack(test_y) assert train_ll_combined.shape == train_ll_combined.shape assert test_ll_combined.shape == test_y_combined.shape assert train_ll_combined.shape[0] > test_ll_combined.shape[ 0] # just a sanity check so that the both are not confused n_samples, n_labels = test_ll_combined.shape curr_pos_ll_means = [] curr_pos_ll_stds = [] curr_neg_ll_means = [] curr_neg_ll_stds = [] for label_idx in xrange(n_labels): label_y = test_y_combined[:, label_idx] pos_indexes = np.where(label_y == 1)[0] neg_indexes = np.where(label_y == 0)[0] pos_ll = test_ll_combined[pos_indexes, label_idx] neg_ll = test_ll_combined[neg_indexes, label_idx] assert np.size(pos_ll) + np.size(neg_ll) == n_samples curr_pos_ll_means.append(np.mean(pos_ll)) curr_pos_ll_stds.append(np.std(pos_ll)) curr_neg_ll_means.append(np.mean(neg_ll)) curr_neg_ll_stds.append(np.std(neg_ll)) for name in decision_makers: args.decision_maker = name modified_combination = list(combination) modified_combination[keys.index('decision')] = name if name == 'svm': args.decision_maker_C = 1e-2 args.decision_maker_penalty = 'l1' elif name == 'log-regression': args.decision_maker_C = 1e-3 args.decision_maker_penalty = 'l1' elif name == 'decision-tree': args.decision_maker_criterion = 'entropy' args.decision_maker_max_depth = 15 elif name == 'random-forest': args.decision_maker_criterion = 'entropy' args.decision_maker_n_estimators = 40 args.decision_maker_max_depth = 15 curr_preds = [] for curr_train_ll, curr_train_y, curr_test_ll in zip( train_ll, train_y, test_ll): print('training decision maker') assert curr_train_ll.shape == curr_train_y.shape assert curr_train_ll.shape[0] > curr_test_ll.shape[0] decision_maker = decision.decision_maker_from_args(args) assert decision_maker is not None # Fit and predict using the decision maker if hasattr(decision_maker, 'fit') and callable( decision_maker.fit): decision_maker.fit(curr_train_ll, curr_train_y) curr_preds.append(decision_maker.predict(curr_test_ll)) print('') test_pred_combined = np.vstack(curr_preds) assert test_y_combined.shape == test_pred_combined.shape # Track everything combinations.append(modified_combination) total_accuracies.append( sk_metrics.accuracy_score(test_y_combined, test_pred_combined)) precision, recall, fscore, _ = sk_metrics.precision_recall_fscore_support( test_y_combined, test_pred_combined) precisions_mean.append(np.mean(precision)) precisions_std.append(np.std(precision)) precisions_min.append(np.min(precision)) precisions_max.append(np.max(precision)) recalls_mean.append(np.mean(recall)) recalls_std.append(np.std(recall)) recalls_min.append(np.min(recall)) recalls_max.append(np.max(recall)) fscores_mean.append(np.mean(fscore)) fscores_std.append(np.std(fscore)) fscores_min.append(np.min(fscore)) fscores_max.append(np.max(fscore)) pos_ll_means.append(np.array(np.median(curr_pos_ll_means))) pos_ll_stds.append(np.array(np.median(curr_pos_ll_stds))) neg_ll_means.append(np.array(np.median(curr_neg_ll_means))) neg_ll_stds.append(np.array(np.median(curr_neg_ll_stds))) curr_step += 1 print('done, took %fs' % (timeit.default_timer() - start)) print('') assert len(combinations) == len(fscores_mean) assert len(combinations) == len(fscores_std) assert len(combinations) == len(fscores_min) assert len(combinations) == len(fscores_max) assert len(combinations) == len(precisions_mean) assert len(combinations) == len(precisions_std) assert len(combinations) == len(precisions_min) assert len(combinations) == len(precisions_max) assert len(combinations) == len(recalls_mean) assert len(combinations) == len(recalls_std) assert len(combinations) == len(recalls_min) assert len(combinations) == len(recalls_max) assert len(combinations) == len(pos_ll_means) assert len(combinations) == len(pos_ll_stds) assert len(combinations) == len(neg_ll_means) assert len(combinations) == len(neg_ll_stds) assert len(combinations) == len(total_accuracies) # Save results if output_dir is not None: filename = 'results.csv' with open(os.path.join(output_dir, filename), 'wb') as f: writer = csv.writer(f, delimiter=';') writer.writerow([ 'idx', 'combination', 'f1-score-mean', 'f1-score-std', 'f1-score-min', 'f1-score-max', 'precision-mean', 'precision-std', 'precision-min', 'precision-max', 'recall-mean', 'recall-std', 'recall-min', 'recall-max', 'total-accuracy', 'pos-ll-mean', 'pos-ll-std', 'neg-ll-mean', 'neg-ll-std' ]) for idx, d in enumerate( zip(combinations, fscores_mean, fscores_std, fscores_min, fscores_max, precisions_mean, precisions_std, precisions_min, precisions_max, recalls_mean, recalls_std, recalls_min, recalls_max, total_accuracies, pos_ll_means, pos_ll_stds, neg_ll_means, neg_ll_stds)): combination = ', '.join([str(x) for x in d[0]]) new_data = ['%d' % idx] + list((combination, ) + d[1:]) writer.writerow(new_data) print len(combinations)