def raise_if_mean_differs_from(accuracy_balanced, class_sizes, reference_level=None, eps_chance_acc=None, method_descr=''): """ Check if the performance is close to chance. Generic method that works for multi-class too!""" if eps_chance_acc is None: total_num_classes = len(class_sizes) eps_chance_acc = max(0.02, 0.1 / total_num_classes) if reference_level is None: reference_level = chance_accuracy(class_sizes) elif not 0.0 < reference_level <= 1.0: raise ValueError('invalid reference_level: must be in (0, 1]') # chance calculation expects "average", not median mean_bal_acc = np.mean(accuracy_balanced, axis=0) for ma in mean_bal_acc: print('for {},\n reference level accuracy expected: {} ' '-- Estimated via CV: {}'.format(method_descr, reference_level, ma)) abs_diff = abs(ma - reference_level) if abs_diff > eps_chance_acc: raise ValueError('they substantially differ by {:.4f} that is ' 'more than {:.4f}!'.format( abs_diff, eps_chance_acc))
def raise_if_mean_differs_from_chance(accuracy_balanced, class_sizes, eps_chance_acc=None): "Check if the performance is close to chance. Generic method that works for multi-class too!" if eps_chance_acc is None: total_num_classes = len(class_sizes) eps_chance_acc = max(0.02, 0.1 / total_num_classes) chance_acc = chance_accuracy(class_sizes) # chance calculation expects "average", not median mean_bal_acc = np.mean(accuracy_balanced, axis=0) for ma in mean_bal_acc: print('Chance accuracy expected: {} -- Estimated via CV: {}'.format(chance_acc, ma)) if abs(ma - chance_acc) > eps_chance_acc: raise ValueError('they substantially differ by more than {:.4f}!'.format(eps_chance_acc))
def _summarize_expt(self): """Summarize the experiment for user info""" print('\nCURRENT EXPERIMENT:\n{line}'.format(line='-' * 50)) print('Training percentage : {:.2}'.format(self.train_perc)) print('Number of CV repetitions : {}'.format(self.num_rep_cv)) print('Number of processors : {}'.format(self.num_procs)) print('Dim reduction method : {}'.format(self.dim_red_method)) print('Dim reduction size : {}'.format(self.reduced_dim)) print('Predictive model chosen : {}'.format(self.pred_model)) print('Grid search level : {}\n'.format(self.grid_search_level)) if len(self.covariates) > 0: print('Covarites selected : {}'.format(', '.join( self.covariates))) print('Deconfoudning method : {}\n'.format(self.deconfounder)) if self._workflow_type == 'classify': self._target_sizes = list(self.datasets.target_sizes.values()) self._chance_accuracy = chance_accuracy(self._target_sizes, 'balanced') print('Estimated chance accuracy : {:.3f}\n' ''.format(self._chance_accuracy))
def metric_distribution(metric, labels, output_path, class_sizes, num_classes=2, metric_label='balanced accuracy'): """ Distribution plots of various metrics such as balanced accuracy! metric is expected to be ndarray of size [num_repetitions, num_datasets] """ num_repetitions = metric.shape[0] num_datasets = metric.shape[1] if len(labels) < num_datasets: raise ValueError( "Insufficient number of labels for {} features!".format( num_datasets)) method_ticks = 1.0 + np.arange(num_datasets) fig, ax = plt.subplots(figsize=cfg.COMMON_FIG_SIZE) line_coll = ax.violinplot(metric, widths=cfg.violin_width, bw_method=cfg.violin_bandwidth, showmedians=True, showextrema=False, positions=method_ticks) cmap = cm.get_cmap(cfg.CMAP_DATASETS, num_datasets) for cc, ln in enumerate(line_coll['bodies']): ln.set_facecolor(cmap(cc)) ln.set_label(labels[cc]) ax.tick_params(axis='both', which='major', labelsize=15) ax.grid(axis='y', which='major', linewidth=cfg.LINE_WIDTH, zorder=0) lower_lim = np.round(np.min([np.float64(0.9 / num_classes), metric.min()]), cfg.PRECISION_METRICS) upper_lim = np.round(np.min([1.01, metric.max()]), cfg.PRECISION_METRICS) step_tick = 0.05 ax.set_ylim(lower_lim, upper_lim) ax.set_xlim(np.min(method_ticks) - 1, np.max(method_ticks) + 1) ax.set_xticks(method_ticks) # ax.set_xticklabels(labels, rotation=45) # 'vertical' ytick_loc = np.arange(lower_lim, upper_lim, step_tick) # add a tick for chance accuracy and/or % of majority class # given the classifier trained on stratified set, we must use the balanced version chance_acc = chance_accuracy(class_sizes, 'balanced') # rounding to ensure improved labels chance_acc = np.round(chance_acc, cfg.PRECISION_METRICS) ytick_loc = np.round(np.append(ytick_loc, chance_acc), cfg.PRECISION_METRICS) ax.set_yticks(ytick_loc) ax.set_yticklabels(ytick_loc) plt.text(0.05, chance_acc, 'chance accuracy') plt.ylabel(metric_label, fontsize=cfg.FONT_SIZE) plt.tick_params(axis='both', which='major', labelsize=cfg.FONT_SIZE) # numbered labels numbered_labels = [ '{} {}'.format(int(ix), lbl) for ix, lbl in zip(method_ticks, labels) ] # putting legends outside the plot below. fig.subplots_adjust(bottom=0.2) leg = ax.legend(numbered_labels, ncol=2, loc=9, bbox_to_anchor=(0.5, -0.1)) # setting colors manually as plot has been through arbitray jumps for ix, lh in enumerate(leg.legendHandles): lh.set_color(cmap(ix)) leg.set_frame_on(False) # making leg background transparent # fig.savefig(output_path + '.png', transparent=True, dpi=300, # bbox_extra_artists=(leg,), bbox_inches='tight') fig.savefig(output_path + '.pdf', bbox_extra_artists=(leg, ), bbox_inches='tight') plt.close() return