def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" savedir = opj(BASEPATH, DATASETNAME, 'i9_InterRaterStats') _maybe_mkdir(savedir) # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(opj(savedir, '..')) # plot kappa matrix and MDS plot for clg in ['main', 'super']: where = opj(savedir, clg) _maybe_mkdir(where) # compare various evalsets in terms of inter-rater concordance plot_interrater_boxplots(dbcon=dbcon, where=where, clsgroup=clg) for evalset in ir.MAIN_EVALSET_NAMES: plot_interrater_pairs(dbcon=dbcon, where=where, evalset=evalset, clsgroup=clg)
def plot_krippendorph_summary(savepath, clsgroup): """""" # connect to database dbcon = _connect_to_anchor_db(opj(savepath, '..')) # get krippendorph summary table krippendorph_summary = read_sql_query( f""" SELECT * FROM "Krippendorph_byAnchorSubsets" WHERE "class_grouping" = "{clsgroup}" ;""", dbcon) # now plot savedir = opj(savepath, '..', 'i10_Krippendorph', f'plots_{clsgroup}') _maybe_mkdir(savedir) _ = [ plot_krippendorph_figure( savedir=savedir, krippendorph_summary=krippendorph_summary, unbiased_is_truth=unbiased_is_truth, evalset=evalset, whoistruth=whoistruth, who=who, whichanchors=whichanchors, ) for evalset in ir.MAIN_EVALSET_NAMES for unbiased_is_truth in [True, False] for whoistruth in ir.CONSENSUS_WHOS for who in ir.CONSENSUS_WHOS for whichanchors in ['v2.1_consensus', 'v2.2_excluded'] ]
def run_constrained_clustering_by_fov_experiment( savepath, gc, whoistruth, evalset='U-control'): """Get medoids and vis constraining on a couple of fovs.""" # connect to sqlite database -- anchors dbcon = { y: _connect_to_anchor_db(opj(savepath, '..'), constrained=y) for y in [True, False] } savedir = opj(savepath, f"{evalset}_{whoistruth}_AreTruth") _maybe_mkdir(savedir) # get fovs where there's a difference anchor_counts = _get_fov_anchor_counts( dbcon=dbcon, evalset=evalset, whoistruth=whoistruth) fovs_to_vis = list(anchor_counts.loc[anchor_counts.loc[ :, f'diff_mean_n_matches_{whoistruth}'] > 0, :].index) for fovname in fovs_to_vis: print(f"visualizing {fovname}") # plot effect of clustering constraint plot_effect_of_iouthresh_and_constraint( gc=gc, dbcon=dbcon, fovname=fovname, whoistruth=whoistruth, evalset=evalset, savename=opj(savedir, f'constraintEffect_{fovname}.png'), )
def plot_krippendorph_figure(savedir: str, krippendorph_summary: DataFrame, unbiased_is_truth: bool, evalset: str, whoistruth: str, who: str, whichanchors: str): """""" path0 = opj(savedir, f'{whoistruth}AreTruth') path1 = opj(path0, f'{evalset}') for path in [path0, path1]: _maybe_mkdir(path) ubstr = ir._ubstr(unbiased_is_truth) print(f'Plotting Krippendorph for {evalset}: {ubstr}{whoistruth}AreTruth: ' f'{who}: {whichanchors} anchors') ksummary = krippendorph_summary.loc[ krippendorph_summary.loc[:, 'unbiased_is_truth'] == unbiased_is_truth, :] ksummary = ksummary.loc[ksummary.loc[:, 'evalset'] == evalset, :] ksummary = ksummary.loc[ksummary.loc[:, 'whoistruth'] == whoistruth, :] ksummary = ksummary.loc[ksummary.loc[:, 'who'] == who, :] ksummary = ksummary.loc[ksummary.loc[:, 'whichanchors'] == whichanchors, :] cutoff = ir.MIN_DETECTIONS_PER_ANCHOR if any([ (evalset == 'U-control') and (who == whoistruth), (evalset != 'U-control') and (who == whoistruth) and (not unbiased_is_truth), # noqa ]) else None if ksummary.shape[0] < 1: return nrows = 1 nperrow = 3 fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) for axno, axis in enumerate(ax.ravel()): if axno == 0: _kripp_subplot(axis=axis, what='n_anchors', is_agreement=False, cutoff=cutoff, krippendorph_summary=ksummary) elif axno == 1: _kripp_subplot(axis=axis, what='detection_and_classification', cutoff=cutoff, is_agreement=True, krippendorph_summary=ksummary) elif axno == 2: _kripp_subplot(axis=axis, what='classification', is_agreement=True, cutoff=cutoff, krippendorph_summary=ksummary) plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) plt.savefig( opj( path1, f'krippendorph_{evalset}_{who}_{ubstr}{whoistruth}_AreTruth' f'_{whichanchors}.svg')) plt.close()
def parse_anchors_dataset(dbcon, gc, savedir: str, whoistruth: str, evalset: str, min_side: int = 100): """""" where = opj(savedir, f'{whoistruth}AreTruth_{evalset}') _maybe_mkdir(where) _maybe_mkdir(opj(where, 'rgb')) _maybe_mkdir(opj(where, 'mask')) _maybe_mkdir(opj(where, 'vis')) _maybe_mkdir(opj(where, 'contours')) # GT codes dict for parsing into label mask GTCODE_PATH = '/home/mtageld/Desktop/cTME/ctme/configs/nucleus_GTcodes.csv' GTCodes_dict = read_csv(GTCODE_PATH) GTCodes_dict.index = GTCodes_dict.loc[:, 'group'] GTCodes_dict = GTCodes_dict.to_dict(orient='index') GTCodes_df = DataFrame.from_dict(GTCodes_dict, orient='index') # get the metadata for fovs fovmetas = _get_and_maybe_update_fovmeta(dbcon) for _, fmeta in fovmetas.iterrows(): contours, fovbounds = _get_and_fix_contours(dbcon, whoistruth=whoistruth, evalset=evalset, fmeta=fmeta) # handle edge cases if contours.shape[0] < 1: continue if ((fovbounds['XMAX'] - fovbounds['XMIN']) < min_side) or ( (fovbounds['YMAX'] - fovbounds['YMIN']) < min_side): continue # parse rgb, mask, visualization, and contours roi_out = _get_roi_from_contours(gc=gc, contours=contours, GTCodes_df=GTCodes_df, fovbounds=fovbounds, fmeta=fmeta) # save output roinamestr = \ f"{fmeta['FOVID']}_{fmeta['slide_name']}" \ f"_left-{fovbounds['XMIN']}_top-{fovbounds['YMIN']}" \ f"_bottom-{fovbounds['YMAX']}_right-{fovbounds['XMAX']}" print('Saving', roinamestr) for imtype in ['mask', 'rgb', 'vis']: savename = opj(where, imtype, roinamestr + '.png') imwrite(im=roi_out[imtype], uri=savename) contours.to_csv(opj(where, 'contours', roinamestr + '.csv'))
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" SAVEDIR = opj(BASEPATH, DATASETNAME, 'i5_ParticipantAccuracy') _maybe_mkdir(SAVEDIR) # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(opj(SAVEDIR, '..')) # Go through various evaluation sets & participant groups for clsgroup in ['main', 'super']: savedir = opj(SAVEDIR, clsgroup) _maybe_mkdir(savedir) for whoistruth in ['Ps']: for unbiased_is_truth in [False]: ubstr = "UNBIASED_" if unbiased_is_truth else "" print(f'{clsgroup.upper()}: {ubstr}{whoistruth}_AreTruth') for evalset in ir.MAIN_EVALSET_NAMES: # accuracy stats for a single avalset plot_participant_accuracy_stats( dbcon=dbcon, savedir=savedir, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, evalset=evalset, clsgroup=clsgroup, ) # compare accuracy stats for various evalsets plot_participant_accuracy_stats_v2( dbcon=dbcon, savedir=savedir, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, clsgroup=clsgroup, ) # superimpose AUROC for various evalsets if whoistruth == 'Ps': plot_participant_accuracy_stats_v3( dbcon=dbcon, savedir=savedir, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, clsgroup=clsgroup, )
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = '/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/' SAVEPATH = opj(BASEPATH, DATASETNAME, 'i1_anchors') kpath = opj(BASEPATH, DATASETNAME, 'i10_Krippendorph') _maybe_mkdir(kpath) # get krippendorph summary for clsgroup in ['main', 'super']: plot_krippendorph_summary(savepath=SAVEPATH, clsgroup=clsgroup)
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" SAVEDIR = opj(BASEPATH, DATASETNAME, 'i8_IntraRaterStats') _maybe_mkdir(SAVEDIR) # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(opj(SAVEDIR, '..')) # compare same participant on various evalsets for clg in ['main', 'super']: plot_intrarater_stats( dbcon=dbcon, savedir=opj(SAVEDIR, clg), clsgroup=clg)
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = '/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/' SAVEPATH = opj(BASEPATH, DATASETNAME, 'i2_ConstraintEffect') _maybe_mkdir(SAVEPATH) # to get FOV RGBs and visualize cluster medoids etc gc = CandygramAPI.connect_to_candygram() # check effect of constrained clustering on a couple of random FOVs for whoistruth in ['Ps']: # Interrater.CONSENSUS_WHOS: for evalset in ['U-control', 'E']: run_constrained_clustering_by_fov_experiment( savepath=SAVEPATH, gc=gc, whoistruth=whoistruth, evalset=evalset)
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" savedir = opj(BASEPATH, DATASETNAME, 'i6_SegmentationAccuracy') _maybe_mkdir(savedir) # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(opj(savedir, '..')) # Go through various evaluation sets & participant groups unbiased_is_truth = False for whoistruth in ir.CONSENSUS_WHOS: ubstr = "UNBIASED_" if unbiased_is_truth else "" print(f'{ubstr}{whoistruth}_AreTruth') # plot proportion of anchors that were agreed upon (by Ps) as # correctly segmented by the algorithm. # NOTE: Since the anchors here are paired, and the legend shows the # no of FOVs per anchor, this by definition uses the unbiased control # as a reference. plot_proportion_segmented(dbcon=dbcon, savedir=savedir, whoistruth=whoistruth) # compare accuracy stats for evalsets (coupled) plot_segmentation_accuracy_stats_v1( dbcon=dbcon, savedir=savedir, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth) # compare accuracy stats for evalsets (independent) plot_segmentation_accuracy_stats_v2( dbcon=dbcon, savedir=savedir, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth)
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" SAVEDIR = opj(BASEPATH, DATASETNAME, 'i11_NPsAccuracySimulations') _maybe_mkdir(SAVEDIR) # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(opj(SAVEDIR, '..')) # Go through various evaluation sets & participant groups for evalset in ['E']: for clsgroup in ['super']: savedir = opj(SAVEDIR, clsgroup) _maybe_mkdir(savedir) plot_simulation_stats( dbcon=dbcon, savedir=savedir, evalset=evalset, clsgroup=clsgroup)
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" SAVEPATH = opj(BASEPATH, DATASETNAME, 'i4_DetectionAndClassificationTally') _maybe_mkdir(SAVEPATH) # Go through various evaluation sets & participant groups for whoistruth in Interrater.CONSENSUS_WHOS: for unbiased_is_truth in [True, False]: for who in Interrater.CONSENSUS_WHOS: if (whoistruth == 'NPs') and (who == 'Ps'): continue for evalset in Interrater.EVALSET_NAMES: print(f'{whoistruth}IsTruth: {who}: {evalset}') # now run main experiment get_and_plot_detection_and_classification_tally( savedir=SAVEPATH, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, who=who, evalset=evalset)
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" SAVEDIR = opj(BASEPATH, DATASETNAME, 'i7_ParicipantConfusions') _maybe_mkdir(SAVEDIR) # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(opj(SAVEDIR, '..')) # Go through various evaluation sets & participant groups for clsgroup in ['main', 'super']: savedir = opj(SAVEDIR, clsgroup) _maybe_mkdir(savedir) for whoistruth in ['Ps']: # ir.CONSENSUS_WHOS: for unbiased_is_truth in [False]: # [True, False] for who in ir.CONSENSUS_WHOS: if (whoistruth == 'NPs') and (who == 'Ps'): continue for evalset in ['E', 'U-control']: # ir.MAIN_EVALSET_NAMES ubstr = "UNBIASED_" if unbiased_is_truth else "" print( f'{clsgroup.upper()}: ' f'{ubstr}{whoistruth}_AreTruth: {who}: {evalset}') # compare accuracy stats for various evalsets plot_participant_confusions( dbcon=dbcon, savedir=savedir, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, who=who, evalset=evalset, clsgroup=clsgroup, )
def plot_interrater_boxplots(dbcon, where: str, clsgroup: str): """""" _maybe_mkdir(opj(where, 'plots')) _maybe_mkdir(opj(where, 'csv')) # Note that for a fair comparison and to avoid confusion, we only include # the U-control and E set. This is because (almost) all participants # annotated both these setc, so we can later do a paired t-test (or # Wilcoxon), but only about half the participants did the B-control. # Besides, we're already "properly" compared the B-control when we did # the intra-rater stats evalsets = ['U-control', 'E'] # read intrarater stats stats = _get_interrater(dbcon, evalsets=evalsets, clsgroup=clsgroup, reorder=True) # organize canvas and plot metrics = ['detection_and_classification', 'classification'] nperrow = len(metrics) nrows = 1 fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) axno = -1 for axis in ax.ravel(): axno += 1 _evalset_comparison_subplot(axis=axis, stats=stats, metric=metrics[axno], evalsets=evalsets) savename = f'interrater_boxplots' plt.savefig(opj(where, 'plots', savename + '.svg')) plt.close() # raw numbers stats.to_csv(opj(where, 'csv', savename + '.csv'))
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" SAVEPATH = opj(BASEPATH, DATASETNAME, 'i3_AnchorSummary') _maybe_mkdir(SAVEPATH) VisConfigs.CATEG_COLORS['other_nucleus'] = [180] * 3 # Go through various evaluation sets & participant groups for whoistruth in ['Ps']: # Interrater.CONSENSUS_WHOS: for unbiased_is_truth in [False]: # [True, False] for who in Interrater.CONSENSUS_WHOS: for evalset in ['E', 'U-control']: if (whoistruth == 'NPs') and (who == 'Ps'): continue for clsgroup in ['main', 'super']: print(f'{clsgroup.upper()}: ' f'{whoistruth}IsTruth: {who}: {evalset}') savedir = opj(SAVEPATH, clsgroup) _maybe_mkdir(savedir) # now run main experiment get_and_plot_all_summary_counts( savedir=savedir, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, who=who, evalset=evalset, clsgroup=clsgroup)
def main(): DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' # where to save stuff BASEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" savedir = opj(BASEPATH, DATASETNAME, 'i1_anchors', 'DATASET') # savedir = opj(BASEPATH, DATASETNAME, 'i1_anchors', 'TMP') _maybe_mkdir(savedir) # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(opj(savedir, '..', '..')) # to get FOV RGBs and visualize cluster medoids etc gc = CandygramAPI.connect_to_candygram() # Create datasets using different inferred truths for whoistruth in ir.CONSENSUS_WHOS: for evalset in ['E', 'U-control']: parse_anchors_dataset(dbcon=dbcon, gc=gc, savedir=savedir, whoistruth=whoistruth, evalset=evalset)
def plot_segmentation_accuracy_stats_v2(dbcon, savedir: str, unbiased_is_truth: bool, whoistruth: str): """""" truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}' \ f'{whoistruth}_AreTruth' where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) # get df df = _get_segmentation_accuracies_v2(dbcon=dbcon, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth) # organize canvas and plot nperrow = 2 nrows = 1 fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) for axno, metric in enumerate(['DICE', 'IOU']): axis = ax.ravel()[axno] # Draw a nested violinplot and split the violins for easier comparison axis = sns.violinplot(data=df, x='evalset', y=metric, hue='iscorrect', ax=axis, split=True, inner="quart", linewidth=1, palette={ 0: 'gold', 1: 'orangered' }) axis.set_ylim(0., 1.) axis.set_title(metric, fontsize=14, fontweight='bold') axis.set_ylabel(metric, fontsize=11) plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'{truthstr}_evalset_violinplot_comparison' plt.savefig(opj(where, 'plots', savename + '.svg')) plt.close() # save raw numbers df.to_csv(opj(where, 'csv', savename + '.csv'))
def plot_proportion_segmented(savedir: str, dbcon, whoistruth: str, cls: str = 'all'): """""" # NOTE: In order to be able to plot this nicely, the total number # of final anchors must be the same, so unbiased Ps must be the # reference group unbiased_is_truth = True ubstr = 'UNBIASED_' if unbiased_is_truth else '' truthstr = f'{ubstr}{whoistruth}_AreTruth' where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) # get the no of anchors and proportion segmented df = _get_proportion_segmented(dbcon=dbcon, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, cls=cls) # scatter, with size indicating number plt.figure(figsize=(5 * 1, 5.5 * 1)) axis = sns.scatterplot( data=df, x=f'avg_is_segmentation_B-control', y=f'avg_is_segmentation_E', size='n_anchors', sizes=(10, 200), alpha=0.7, color='dimgray', ) minn = -0.02 maxn = 1.02 axis.plot([0., maxn], [0., maxn], color='gray', linestyle='--') axis.set_xlim(minn, maxn) axis.set_ylim(minn, maxn) axis.set_xlabel('B-control', fontsize=11) axis.set_ylabel('E', fontsize=11) plt.title('N. segmentations / N. anchors', fontsize=14, fontweight='bold') plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'{truthstr}_evalset_proportionSegmented_comparison' plt.savefig(opj(where, 'plots', savename + '.svg')) plt.close() # save raw numbers df.to_csv(opj(where, 'csv', savename + '.csv'))
def plot_segmentation_accuracy_stats_v1(dbcon, savedir: str, unbiased_is_truth: bool, whoistruth: str): """""" truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}' \ f'{whoistruth}_AreTruth' where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) for metric in ['DICE', 'IOU']: df = _get_segmentation_accuracies_v1( dbcon=dbcon, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, metric=metric) # organize canvas and plot nperrow = 1 nrows = 1 plt.figure(figsize=(5 * nperrow, 5.5 * nrows)) scprops = {'marker': 'o', 'alpha': 0.5, 's': 4**2, 'color': 'dimgray'} # scatter plot with marginals and KDE density g = (sns.jointplot(x=f'{metric}_B-control', y=f'{metric}_E', data=df, kind="scatter", xlim=(0.5, 1), ylim=(0.5, 1), **scprops).plot_joint(sns.kdeplot, zorder=0, n_levels=10, cmap='YlOrRd', alpha=1.)) g.ax_joint.plot([0.5, 1.], [0.5, 1.], color='gray', linestyle='--') g.ax_joint.set_xlabel('B-control', fontsize=11) g.ax_joint.set_ylabel('E', fontsize=11) g.fig.suptitle(metric, fontsize=14, fontweight='bold') plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'{truthstr}_evalset_{metric}_comparison' plt.savefig(opj(where, 'plots', savename + '.svg')) plt.close() # save raw numbers df.to_csv(opj(where, 'csv', savename + '.csv'))
def plot_participant_confusions(dbcon, savedir: str, unbiased_is_truth: bool, whoistruth: str, who: str, evalset: str, clsgroup: str, discard_unmatched: bool = True): """""" truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}' \ f'{whoistruth}_AreTruth' where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) # get confusion matrices by participant and seed params = { 'dbcon': dbcon, 'unbiased_is_truth': unbiased_is_truth, 'whoistruth': whoistruth, 'who': who, 'evalset': evalset, 'clsgroup': clsgroup, } confusions = { 'per_participant': _get_confmat_per_participant(discard_unmatched=discard_unmatched, **params) } confusions['per_anchor'], n_who = _get_confmat_per_anchor(**params) # save csv savename = f'{who}_{truthstr}_{evalset}_confusions' for cs, confmat in confusions.items(): confmat.to_csv(opj(where, 'csv', savename + '.csv')) # Now plot both conusion matrices _plot_confusion( confmat_by_anchor=confusions['per_anchor'], confmat_by_participant=confusions['per_participant'], n_participants=n_who, discard_unmatched=discard_unmatched, savename=opj(where, 'plots', savename + '.svg'), )
def get_and_plot_detection_and_classification_tally( savedir: str, unbiased_is_truth: bool, whoistruth: str, who: str, evalset: str): """Get a tally of detection and classification. For example, a tally dataframe for tumor nuclei, having a value of 43 at row 3, column 5 means that there are 43 tumor nuclei (i.e. their REAL label is 'tumor') that were detected by 5 people, but only 3 of these people called it 'tumor'. """ truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}{whoistruth}_AreTruth' # noqa where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) # connect to sqlite database -- anchors dbcon_anchors = _connect_to_anchor_db(opj(savedir, '..')) # get combined tally of detection and classification tallydfs = _get_detection_and_classification_tally( dbcon_anchors=dbcon_anchors, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, evalset=evalset, who=who) # save csvs prepend = f'{Interrater.TRUTHMETHOD}_{evalset}_{who}_{truthstr}' for cls, tallydf in tallydfs.items(): tallydf.to_csv(opj( where, 'csv', f'{prepend}_{cls}_detection_and_classification_tally.csv'), ) # now plot vis_detection_and_classification_tally( tallydfs=tallydfs, savename=opj( where, 'plots', f'{prepend}_detection_and_classification_tally.svg'), )
def plot_participant_accuracy_stats(dbcon, savedir: str, unbiased_is_truth: bool, whoistruth: str, evalset: str, clsgroup: str): """""" truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}' \ f'{whoistruth}_AreTruth' # noqa where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) classes, accuracy = _get_accuracy_stats( dbcon=dbcon, whoistruth=whoistruth, clsgroup=clsgroup, unbiased_is_truth=unbiased_is_truth, evalset=evalset) if whoistruth == 'Ps': tpr, fpr, roc_auc = get_roc_and_auroc_for_who( dbcon=dbcon, evalset=evalset, who='NPs', whoistruth=whoistruth, unbiased_is_truth=unbiased_is_truth, clsgroup=clsgroup) # to save raw values for calculating p-values later overalldf = [] # organize canvas and plot nperrow = 4 if len(classes) <= 4 else 3 nrows = int(np.ceil((len(classes)) / nperrow)) fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) scprops = {'alpha': 0.75, 's': 9**2, 'edgecolor': 'k'} axno = -1 for axis in ax.ravel(): axno += 1 if axno == len(classes): break cls = classes[axno] isdetection = cls == 'detection' for who in ['NPs', 'JPs', 'SPs']: pstyle = ir.PARTICIPANT_STYLES[who] scprops.update({k: pstyle[k] for k in ['c', 'marker']}) keep = accuracy.loc[:, 'participant'].apply( lambda x: x in ir.who[who]) dfslice = accuracy.loc[keep, :] dfslice = dfslice.loc[dfslice.loc[:, 'class'] == cls, :] overalldf.append(dfslice) # add PR / ROC curve for inferred truth (from NPs) # versus the "actual" inferred truth (from SPs) if (whoistruth == 'Ps') and (who == 'NPs'): lprops = {'color': scprops['c'], 'alpha': 0.7, 'linewidth': 2} if isdetection: # get precision-recalll curve prc = get_precision_recall_for_who( dbcon=dbcon, evalset=evalset, who='NPs', whoistruth=whoistruth, unbiased_is_truth=unbiased_is_truth) # plot axis.plot(prc['recall'], prc['precision'], linestyle='-', label=f'{who} "Truth" (AP=%0.2f)' % prc['AP'], **lprops) axis.axhline(prc['random'], xmin=0., xmax=1., c='gray', linestyle='--', label='Random guess') elif cls == 'classification': axis.plot( fpr['micro'], tpr['micro'], linestyle='-', # noqa label=f'{who} "Truth" - MicroAvg (AUC=%0.2f)' % roc_auc['micro'], # noqa **lprops) axis.plot(fpr['macro'], tpr['macro'], linestyle='--', label=f'{who} "Truth" - MacroAvg (AUC=%0.2f)' % roc_auc['macro'], **lprops) else: axis.plot( fpr[cls], tpr[cls], linestyle='-', # noqa label=f'{who} "Truth" (AUC=%0.2f)' % roc_auc[cls], # noqa **lprops) # scatter the various participants if isdetection: axis.scatter(dfslice.loc[:, 'recall'], dfslice.loc[:, 'precision'], label=f'{who}', **scprops) else: axis.scatter(1 - dfslice.loc[:, 'specificity'], dfslice.loc[:, 'sensitivity'], label=f'{who}', **scprops) if isdetection: xlab, ylab = ('Recall (Sensitivity)', 'Precision (PPV)') else: axis.plot([0., 0.5, 1.0], [0., 0.5, 1.0], c='gray', linestyle='--', label='Random guess') xlab, ylab = ('1 - Specificity (FPR)', 'Sensitivity (TPR)') axis.set_xlim(-0.02, 1.02) axis.set_ylim(-0.02, 1.02) axis.set_aspect('equal') axis.set_title(cls.capitalize(), fontsize=14, fontweight='bold') axis.set_xlabel(xlab, fontsize=11) axis.set_ylabel(ylab, fontsize=11) axis.legend(fontsize=8) # save plot plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'{truthstr}_{evalset}_accuracy_stats' plt.savefig(opj(where, 'plots', savename + '.svg')) plt.close() # save raw numbers overalldf = concat(overalldf, axis=0, ignore_index=True) overalldf.to_csv(opj(where, 'csv', savename + '.csv'))
def get_and_plot_all_summary_counts(savedir: str, unbiased_is_truth: bool, whoistruth: str, who: str, evalset: str, clsgroup: str): """""" assert clsgroup in ['raw', 'main', 'super'] truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}{whoistruth}_AreTruth' # noqa where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) clmap, class_list = _get_clmap(clsgroup) class_list.remove('AMBIGUOUS') clmap['undetected'] = 'undetected' clmap['DidNotAnnotateFOV'] = 'DidNotAnnotateFOV' # connect to sqlite database -- anchors dbcon_anchors = _connect_to_anchor_db(opj(savedir, '..', '..')) # restrict to relevant FOV subset and anchors out = get_fovs_annotated_by_almost_everyone( dbcon_anchors=dbcon_anchors, unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, evalset=evalset, who=who) # group classes as needed out['anchors'] = remap_classes_in_anchorsdf(anchors=out['anchors'], clsgroup=clsgroup) # Get tally of nuclei was detected by AT LEAST 6 observers, etc cumulative_counts_table = get_summary_counts_table( anchors=out['anchors'], maxn=out['maxn'], unbiased_is_truth=unbiased_is_truth, whoistruth=whoistruth, who=who, class_list=class_list) detection_composition, Inferred_label_breakdown = \ _get_summary_percent_table( cumulative_counts_table, who=who, class_list=class_list) # save for reference prepend = f'{Interrater.TRUTHMETHOD}_{evalset}_{who}_{truthstr}' cumulative_counts_table.to_csv( opj(where, 'csv', f'{prepend}_counts_table.csv')) detection_composition.to_csv( opj(where, 'csv', f'{prepend}_detection_composition.csv')) Inferred_label_breakdown.to_csv( opj(where, 'csv', f'{prepend}_inferred_label_breakdown.csv')) # now plot _plot_counts_summaries( cumulative_counts_table=cumulative_counts_table, detection_composition=detection_composition, Inferred_label_breakdown=Inferred_label_breakdown, who=who, class_list=class_list, savename=opj(where, 'plots', f'{prepend}_count_summaries.svg'), )
def plot_participant_accuracy_stats_v3(dbcon, savedir: str, unbiased_is_truth: bool, whoistruth: str, clsgroup: str): """""" truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}' \ f'{whoistruth}_AreTruth' # noqa where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'csv')) _maybe_mkdir(opj(where, 'plots')) tpr = {} fpr = {} roc_auc = {} for evalset in ir.MAIN_EVALSET_NAMES: tpr[evalset], fpr[evalset], roc_auc[evalset] = \ get_roc_and_auroc_for_who( dbcon=dbcon, evalset=evalset, who='NPs', clsgroup=clsgroup, whoistruth=whoistruth, unbiased_is_truth=unbiased_is_truth) # organize canvas and plot classes = ['detection', 'micro', 'macro'] + CLS[clsgroup] nperrow = 3 nrows = int(np.ceil((len(classes)) / nperrow)) fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) axno = -1 for axis in ax.ravel(): axno += 1 if axno == len(classes): break cls = classes[axno] for evalset in ir.MAIN_EVALSET_NAMES: who = 'NPs' lprops = { 'color': ir.PARTICIPANT_STYLES[who]['c'], 'alpha': 1., 'linewidth': 2 } lprops.update(ir.EVALSET_STYLES[evalset]) if cls == 'detection': # get precision-recall curve prc = get_precision_recall_for_who( dbcon=dbcon, evalset=evalset, who='NPs', whoistruth=whoistruth, unbiased_is_truth=unbiased_is_truth) # plot axis.plot(prc['recall'], prc['precision'], label=f'{evalset} (AP=%0.2f)' % prc['AP'], **lprops) axis.axhline(prc['random'], xmin=0., xmax=1., c='gray', linestyle=lprops['linestyle'], label=f'Random guess ({evalset})') else: axis.plot(fpr[evalset][cls], tpr[evalset][cls], label=f'{evalset} (AUC=%0.2f)' % roc_auc[evalset][cls], **lprops) if cls == 'detection': xlab, ylab = ('Recall (Sensitivity)', 'Precision (PPV)') else: axis.plot([0., 0.5, 1.0], [0., 0.5, 1.0], c='gray', linestyle='--', label='Random guess') xlab, ylab = ('1 - Specificity (FPR)', 'Sensitivity (TPR)') axis.set_xlim(-0.02, 1.02) axis.set_ylim(-0.02, 1.02) axis.set_aspect('equal') axis.set_title(cls.capitalize(), fontsize=14, fontweight='bold') axis.set_xlabel(xlab, fontsize=11) axis.set_ylabel(ylab, fontsize=11) axis.legend(fontsize=8) # save plot plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'{truthstr}_superimposed_auroc_curves' plt.savefig(opj(where, 'plots', savename + '.svg')) plt.close()
def plot_participant_accuracy_stats_v2(dbcon, savedir: str, unbiased_is_truth: bool, whoistruth: str, clsgroup: str): """""" truthstr = f'{"UNBIASED_" if unbiased_is_truth else ""}' \ f'{whoistruth}_AreTruth' where = opj(savedir, truthstr) _maybe_mkdir(where) _maybe_mkdir(opj(where, 'plots')) classes, accuracy = _get_accuracy_stats( dbcon=dbcon, whoistruth=whoistruth, clsgroup=clsgroup, unbiased_is_truth=unbiased_is_truth) # to save raw values for calculating p-values later overalldf = [] # reorder evalsets tmp = [] for evalset in ir.MAIN_EVALSET_NAMES: tmp.append(accuracy.loc[accuracy.loc[:, 'evalset'] == evalset, :]) accuracy = concat(tmp, axis=0) # organize canvas and plot nperrow = 4 if len(classes) <= 4 else 3 nrows = int(np.ceil((len(classes)) / nperrow)) fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) scprops = {'alpha': 0.7, 's': 7**2, 'edgecolor': 'k'} axno = -1 for axis in ax.ravel(): axno += 1 if axno == len(classes): break cls = classes[axno] metric = 'F1' if cls == 'detection' else 'MCC' dfslice = accuracy.loc[accuracy.loc[:, 'class'] == cls, :].copy() dfslice.index = dfslice.loc[:, 'participant'] dfslice.loc[:, 'who'] = 'NPs' for who in ['JPs', 'SPs']: for p in dfslice.index: if p in ir.who[who]: dfslice.loc[p, 'who'] = who dfslice.loc[:, 'swho'] = dfslice.loc[:, 'who'].copy() dfslice.loc[dfslice.loc[:, 'swho'] == 'SPs', 'swho'] = 'Ps' dfslice.loc[dfslice.loc[:, 'swho'] == 'JPs', 'swho'] = 'Ps' dfslice = dfslice.loc[:, ['class', 'evalset', metric, 'who', 'swho']] overalldf.append(dfslice) # main boxplots bppr = {'alpha': 0.5} sns.boxplot( ax=axis, data=dfslice, x='evalset', y=metric, hue='swho', palette=[ir.PARTICIPANT_STYLES[who]['c'] for who in ['Ps', 'NPs']], boxprops=bppr, whiskerprops=bppr, capprops=bppr, medianprops=bppr, showfliers=False, # notch=True, bootstrap=5000, ) # scatter each participant group for who in ['NPs', 'JPs', 'SPs']: pstyle = ir.PARTICIPANT_STYLES[who] scprops.update({k: pstyle[k] for k in ['c', 'marker']}) plotme = dfslice.loc[dfslice.loc[:, 'who'] == who, :].copy() offset = -0.2 if who in ['JPs', 'SPs'] else 0.2 plotme.loc[:, 'x'] = plotme.loc[:, 'evalset'].apply( lambda x: ir.MAIN_EVALSET_NAMES.index(x) + offset) plotme = np.array(plotme.loc[:, ['x', metric]]) # add jitter plotme[:, 0] += 0.05 * np.random.randn(plotme.shape[0]) # now scatter axis.scatter(plotme[:, 0], plotme[:, 1], label=f'{who}', **scprops) axis.set_ylim(0., 1.) # axis.set_ylim(0.5, 1.) axis.set_title(cls.capitalize(), fontsize=14, fontweight='bold') axis.set_ylabel(metric.capitalize(), fontsize=11) axis.legend() plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'{truthstr}_evalset_accuracy_comparison' plt.savefig(opj(where, 'plots', savename + '.svg')) plt.close() # save raw numbers overalldf = concat(overalldf, axis=0, ignore_index=True) overalldf.to_csv(opj(where, 'csv', savename + '.csv'))
def plot_intrarater_stats(dbcon, savedir: str, clsgroup: str): """""" _maybe_mkdir(savedir) _maybe_mkdir(opj(savedir, 'csv')) _maybe_mkdir(opj(savedir, 'plots')) # read intrarater stats evalsets = ['B-control', 'E'] stats = read_sql_query(f""" SELECT "participant", "second_evalset" AS "evalset" , "detection_and_classification", "classification" , "n_anchors_second_evalset" AS "n_anchors" , "n_clicks_second_evalset" AS "n_clicks" FROM "intra-rater_{clsgroup}ClassGroup" WHERE "participant" IN ({ir._get_sqlite_usrstr_for_who('All')}) AND "first_evalset" = "U-control" AND "second_evalset" IN ({ir._get_sqlitestr_for_list(evalsets)}) ;""", dbcon) stats.loc[:, 'psegmented'] = stats.loc[ :, 'n_clicks'] / stats.loc[:, 'n_anchors'] # to save raw values for calculating p-values later overalldf = [] # reorder evalsets tmp = [] for evalset in ir.EVALSET_NAMES: tmp.append(stats.loc[stats.loc[:, 'evalset'] == evalset, :]) stats = concat(tmp, axis=0) # organize canvas and plot nperrow = 3 nrows = 1 fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) scprops = {'alpha': 0.7, 's': 7 ** 2, 'edgecolor': 'k'} axno = -1 metrics = ['psegmented', 'detection_and_classification', 'classification'] mdict = { 'psegmented': 'N. segmentations / N. anchors', 'detection_and_classification': "Kappa vs U-control (det. & classif.)", 'classification': "Kappa vs U-control (classif.)", } for axis in ax.ravel(): axno += 1 metric = metrics[axno] dfslice = stats.copy() dfslice.index = dfslice.loc[:, 'participant'] dfslice.loc[:, 'who'] = 'NPs' for who in ['JPs', 'SPs']: for p in dfslice.index: if p in ir.who[who]: dfslice.loc[p, 'who'] = who dfslice.loc[:, 'swho'] = dfslice.loc[:, 'who'].copy() dfslice.loc[dfslice.loc[:, 'swho'] == 'SPs', 'swho'] = 'Ps' dfslice.loc[dfslice.loc[:, 'swho'] == 'JPs', 'swho'] = 'Ps' dfslice = dfslice.loc[:, ['evalset', metric, 'who', 'swho']] overalldf.append(dfslice) # annotate agreement ranges if axno > 0: _annotate_krippendorph_ranges( axis=axis, minx=0, maxx=2, shades=False) # main boxplots bppr = {'alpha': 0.5} sns.boxplot( ax=axis, data=dfslice, x='evalset', y=metric, hue='swho', palette=[ir.PARTICIPANT_STYLES[who]['c'] for who in ['Ps', 'NPs']], boxprops=bppr, whiskerprops=bppr, capprops=bppr, medianprops=bppr, showfliers=False, notch=False, bootstrap=5000) # scatter each participant group for who in ['NPs', 'JPs', 'SPs']: pstyle = ir.PARTICIPANT_STYLES[who] scprops.update({k: pstyle[k] for k in ['c', 'marker']}) plotme = dfslice.loc[dfslice.loc[:, 'who'] == who, :].copy() offset = -0.2 if who in ['JPs', 'SPs'] else 0.2 plotme.loc[:, 'x'] = plotme.loc[:, 'evalset'].apply( lambda x: evalsets.index(x) + offset) plotme = np.array(plotme.loc[:, ['x', metric]]) # add jitter plotme[:, 0] += 0.05 * np.random.randn(plotme.shape[0]) # now scatter axis.scatter( plotme[:, 0], plotme[:, 1], label=f'{who}', **scprops) axis.set_ylim(0., 1.) axis.set_title(mdict[metric], fontsize=14, fontweight='bold') axis.set_ylabel(metric.capitalize(), fontsize=11) axis.legend() plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'intra-rater_comparison' plt.savefig(opj(savedir, 'plots', savename + '.svg')) plt.close() # save raw numbers overalldf = concat(overalldf, axis=0) overalldf.to_csv(opj(savedir, 'csv', savename + '.csv'))
def plot_simulation_stats( dbcon, savedir: str, evalset: str, clsgroup: str): """""" _maybe_mkdir(opj(savedir, 'csv')) _maybe_mkdir(opj(savedir, 'plots')) # control for total no of unique pathologists nnps = 18 if evalset == 'E' else 19 # get simulation stats for evalset stats = read_sql_query(f""" SELECT * FROM "NPs_AccuracySimulations_{clsgroup}ClassGroup" WHERE "evalset" = "{evalset}" AND "n_unique_NPs" = {nnps} ;""", dbcon) # organize canvas and plot _, tmp_classes = _get_clmap(clsgroup) tmp_classes.remove('AMBIGUOUS') classes = ['detection', 'classification', 'micro', 'macro'] + tmp_classes nperrow = 4 nrows = int(np.ceil((len(classes)) / nperrow)) fig, ax = plt.subplots(nrows, nperrow, figsize=(5 * nperrow, 5.5 * nrows)) axno = -1 for axis in ax.ravel(): axno += 1 cls = classes[axno] isdetection = cls == 'detection' default_color = ir.PARTICIPANT_STYLES['NPs']['c'] if isdetection: metric = 'detection-F1' lab = 'Detection F1 score' # ymin = 0. ymin = 0.5 color = default_color elif cls == 'classification': metric = 'classification-all-MCC' lab = 'Classification MCC' ymin = 0 color = default_color else: if cls == 'micro': clstr = 'Micro-Average' color = default_color elif cls == 'macro': clstr = 'Macro-Average' color = default_color else: clstr = cls.capitalize() color = [j / 255. for j in VisConfigs.CATEG_COLORS[cls]] metric = f'auroc-{cls}' lab = f'AUROC - {clstr}' ymin = 0.5 # main boxplot bppr = {'alpha': 0.6, 'color': color} sns.boxplot( ax=axis, data=stats, x='NPs_per_fov', y=metric, boxprops=bppr, whiskerprops=bppr, capprops=bppr, medianprops=bppr, showfliers=False, color=color, notch=True, bootstrap=5000, # notch=False, ) axis.set_ylim(ymin, 1.) axis.set_title(lab, fontsize=14, fontweight='bold') axis.set_ylabel(lab, fontsize=11) axis.set_xlabel('No. of NPs per FOV', fontsize=11) plt.tight_layout(pad=0.3, w_pad=0.5, h_pad=0.3) savename = f'NPs_AccuracySimulation_PsAreTruth_{evalset}' plt.savefig(opj(savedir, 'plots', savename + '.svg')) plt.close()
def main(): # Where are the masks, contours, etc DATASETNAME = 'CURATED_v1_2020-03-29_EVAL' DATASETPATH = "/home/mtageld/Desktop/cTME/data/tcga-nucleus/" DATASETPATH = opj(DATASETPATH, DATASETNAME) # where to save stuff SAVEPATH = "/home/mtageld/Desktop/cTME/results/tcga-nucleus/interrater/" SAVEPATH = opj(SAVEPATH, DATASETNAME) _maybe_mkdir(SAVEPATH) _maybe_mkdir(opj(SAVEPATH, 'i1_anchors')) # get + save everyone's alias alias = ir.PARTICIPANT_ALIASES aliasdf = DataFrame.from_dict(alias, orient='index') aliasdf.to_csv(opj(SAVEPATH, 'i1_anchors', 'participant_aliases.csv')) # connect to sqlite database -- annotations db_path = opj(DATASETPATH, DATASETNAME + ".sqlite") sql_engine = create_engine('sqlite:///' + db_path, echo=False) dbcon_annots = sql_engine.connect() # to get FOV RGBs and visualize cluster medoids etc gc = CandygramAPI.connect_to_candygram() MPP = 0.2 MAG = None # get information per evaluation set, user, and fov fovinfos = get_fovinfos_for_interrater(dbcon=dbcon_annots) with open(opj(SAVEPATH, 'i1_anchors', "fovinfos.json"), 'w') as f: json.dump(fovinfos, f, indent=4) # ------------------------------------------------------------------------- for constrained in [True, False]: # connect to sqlite database -- anchors dbcon = _connect_to_anchor_db(SAVEPATH, constrained=constrained) # Get nucleus anchors, using pathologists (SP/JP) as truth # but also get the false anchors gana_kwargs = { 'fovinfos': fovinfos, 'get_medoids_kwargs': { 'dbcon': dbcon_annots, # annotations 'who': 'All', 'add_relative_bounds': True, 'gc': gc, 'MPP': MPP, 'MAG': MAG, 'constrained': constrained, }, 'dbcon': dbcon, # anchors # 'min_ious': np.arange(0.125, 0.76, 0.125), 'min_ious': [0.25, 0.5, 0.75], 'fovs_to_use': None, 'constrained': constrained, } get_all_nucleus_anchors_gtruth(**gana_kwargs) # Add Expectation-Maximization inferred labels add_all_EM_inferred_labels(dbcon=dbcon) # Add unbiased labels to all the eval sets add_unbiased_labels_to_db(dbcon=dbcon) # create convenience virtual tables create_convenience_table_views(dbcon=dbcon)