コード例 #1
0
ファイル: experiment_printres.py プロジェクト: Erotemic/ibeis
    def print_confusion_stats():
        """
        CommandLine:
            python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
        """
        # Prints nextbest ranks
        print('-------------')
        print('ScoreDiffMatStats: %s' % testnameid)
        print('column_lbls = %r' % (column_lbls,))

        #cfgx2_gt_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
        #cfgx2_gf_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')

        gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
        gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)

        tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
        fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
        tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
        fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)

        tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(tp_rawscores, 'tp_rawscores', cfgx2_lbl)
        fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(fp_rawscores, 'fp_rawscores', cfgx2_lbl)
        tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(tn_rawscores, 'tn_rawscores', cfgx2_lbl)
        fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(fn_rawscores, 'fn_rawscores', cfgx2_lbl)

        #print(tp_rawscores_str)
        #print(fp_rawscores_str)
        #print(tn_rawscores_str)
        #print(fn_rawscores_str)

        print(tp_rawscore_statstr)
        print(fp_rawscore_statstr)
        print(tn_rawscore_statstr)
        print(fn_rawscore_statstr)
コード例 #2
0
ファイル: __init__.py プロジェクト: simplesoftMX/ibeis
    def build_commandline(e=e, **kwargs):
        # Equivalent command line version of this func
        import ibeis.dev
        valid_e_flags = ut.flatten(
            [[tup[1]] if len(tup) == 2 else [tup[1]] + tup[2]
             for tup in ibeis.dev.REGISTERED_DOCTEST_EXPERIMENTS])
        if e in valid_e_flags:
            epref = '-e'
        else:
            # hack to use tf
            epref = '--tf'

        if dbdir is not None:
            db_flag = '--dbdir'
            db_value = dbdir
        else:
            db_flag = '--db'
            db_value = db
        command_parts = [
            'ibeis',
            epref,
            e,
            db_flag,
            db_value,
            '-a',
            ' '.join(a).replace('(', '\(').replace(')', '\)'),
            '-t',
            ' '.join(t),
        ]
        if qaid_override is not None:
            command_parts.extend(
                ['--qaid=' + ','.join(map(str, qaid_override))])
        if daid_override is not None:
            command_parts.extend(
                ['--daid-override=' + ','.join(map(str, daid_override))])
        if 'disttype' in kwargs:
            command_parts.extend(
                ['--disttype=', ','.join(map(str, kwargs['disttype']))])

        # hack parse out important args that were on command line
        if 'f' in kwargs:
            command_parts.extend(['-f', ' '.join(kwargs['f'])])
        if 'test_cfgx_slice' in kwargs:
            # very hacky, much more than checking for f
            slice_ = kwargs['test_cfgx_slice']
            slice_attrs = [
                getattr(slice_, attr, '')
                for attr in ['start', 'stop', 'step']
            ]
            slice_attrs = ut.replace_nones(slice_attrs, '')
            slicestr = ':'.join(map(str, slice_attrs))
            command_parts.extend(['--test_cfgx_slice', slicestr])

        command_parts.extend(['--show'])

        command_line_str = ' '.join(command_parts)
        # Warning, not always equivalent
        print('Equivalent Command Line:')
        print(command_line_str)
        return command_line_str
コード例 #3
0
 def _make_lnbnn_scores(infr, edges):
     edge_to_data = infr._get_cm_edge_data(edges)
     edges = list(edge_to_data.keys())
     edge_scores = list(ut.take_column(edge_to_data.values(), 'score'))
     edge_scores = ut.replace_nones(edge_scores, np.nan)
     edge_scores = np.array(edge_scores)
     # take the inf-norm
     normscores = edge_scores / vt.safe_max(edge_scores, nans=False)
     return normscores
コード例 #4
0
    def print_confusion_stats():
        """
        CommandLine:
            python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
        """
        # Prints nextbest ranks
        print('-------------')
        print('ScoreDiffMatStats: %s' % testnameid)
        print('column_lbls = %r' % (column_lbls, ))

        #cfgx2_gt_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
        #cfgx2_gf_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')

        gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
        gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)

        tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
        fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
        tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
        fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)

        tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(
            tp_rawscores, 'tp_rawscores', cfgx2_lbl)
        fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(
            fp_rawscores, 'fp_rawscores', cfgx2_lbl)
        tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(
            tn_rawscores, 'tn_rawscores', cfgx2_lbl)
        fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(
            fn_rawscores, 'fn_rawscores', cfgx2_lbl)

        #print(tp_rawscores_str)
        #print(fp_rawscores_str)
        #print(tn_rawscores_str)
        #print(fn_rawscores_str)

        print(tp_rawscore_statstr)
        print(fp_rawscore_statstr)
        print(tn_rawscore_statstr)
        print(fn_rawscore_statstr)
コード例 #5
0
    def apply_match_scores(infr):
        """

        Applies precomputed matching scores to edges that already exist in the
        graph. Typically you should run infr.apply_match_edges() before running
        this.

        CommandLine:
            python -m wbia.algo.graph.core apply_match_scores --show

        Example:
            >>> # xdoctest: +REQUIRES(--slow)
            >>> # ENABLE_DOCTEST
            >>> from wbia.algo.graph.core import *  # NOQA
            >>> infr = testdata_infr('PZ_MTEST')
            >>> infr.exec_matching()
            >>> infr.apply_match_edges()
            >>> infr.apply_match_scores()
            >>> infr.get_edge_attrs('score')
        """
        if infr.cm_list is None:
            infr.print('apply_match_scores - no scores to apply!')
            return
        infr.print('apply_match_scores', 1)
        edges = list(infr.graph.edges())
        edge_to_data = infr._get_cm_edge_data(edges)

        # Remove existing attrs
        ut.nx_delete_edge_attr(infr.graph, 'score')
        ut.nx_delete_edge_attr(infr.graph, 'rank')
        ut.nx_delete_edge_attr(infr.graph, 'normscore')

        edges = list(edge_to_data.keys())
        edge_scores = list(ut.take_column(edge_to_data.values(), 'score'))
        edge_scores = ut.replace_nones(edge_scores, np.nan)
        edge_scores = np.array(edge_scores)
        edge_ranks = np.array(ut.take_column(edge_to_data.values(), 'rank'))
        # take the inf-norm
        normscores = edge_scores / vt.safe_max(edge_scores, nans=False)

        # Add new attrs
        infr.set_edge_attrs('score', ut.dzip(edges, edge_scores))
        infr.set_edge_attrs('rank', ut.dzip(edges, edge_ranks))

        # Hack away zero probabilites
        # probs = np.vstack([p_nomatch, p_match, p_notcomp]).T + 1e-9
        # probs = vt.normalize(probs, axis=1, ord=1, out=probs)
        # entropy = -(np.log2(probs) * probs).sum(axis=1)
        infr.set_edge_attrs('normscore', dict(zip(edges, normscores)))
コード例 #6
0
def get_annotedge_viewdist(ibs, edges):
    edges = np.array(edges)
    unique_annots = ibs.annots(np.unique(edges)).view()
    annots1 = unique_annots.view(edges.T[0])
    annots2 = unique_annots.view(edges.T[1])
    view_ints1 = annots1.viewpoint_int
    view_ints2 = annots2.viewpoint_int

    DIST = ibs.const.VIEW.DIST
    view_dists = [
        DIST[tup] if tup in DIST else DIST[tup[::-1]]
        for tup in zip(view_ints1, view_ints2)
    ]
    view_dists = np.array(ut.replace_nones(view_dists, np.nan))
    return view_dists
コード例 #7
0
ファイル: __init__.py プロジェクト: heroinlin/ibeis
    def build_commandline(e=e, **kwargs):
        # Equivalent command line version of this func
        import ibeis.dev
        valid_e_flags = ut.flatten([[tup[1]] if len(tup) == 2 else [tup[1]] + tup[2] for tup in ibeis.dev.REGISTERED_DOCTEST_EXPERIMENTS])
        if e in valid_e_flags:
            epref = '-e'
        else:
            # hack to use tf
            epref = '--tf'
        command_parts = ['ibeis',
                         epref, e,
                         '--db', db,
                         '-a', ' '.join(a).replace('(', '\(').replace(')', '\)'),
                         '-t', ' '.join(t),
                        ]
        if qaid_override is not None:
            command_parts.extend(['--qaid=' + ','.join(map(str, qaid_override))])
        if daid_override is not None:
            command_parts.extend(['--daid-override=' + ','.join(map(str, daid_override))])
        if 'disttypes' in kwargs:
            command_parts.extend(['--disttypes=', ','.join(map(str, kwargs['disttypes']))])

        # hack parse out important args that were on command line
        if 'f' in kwargs:
            command_parts.extend(['-f', ' '.join(kwargs['f'])])
        if 'test_cfgx_slice' in kwargs:
            # very hacky, much more than checking for f
            slice_ = kwargs['test_cfgx_slice']
            slice_attrs = [getattr(slice_, attr, '')
                           for attr in ['start', 'stop', 'step']]
            slice_attrs = ut.replace_nones(slice_attrs, '')
            slicestr = ':'.join(map(str, slice_attrs))
            command_parts.extend(['--test_cfgx_slice', slicestr])

        command_parts.extend(['--show'])

        command_line_str = ' '.join(command_parts)
        # Warning, not always equivalent
        print('Equivalent Command Line:')
        print(command_line_str)
        return command_line_str
コード例 #8
0
def split_analysis(ibs):
    """
    CommandLine:
        python -m ibeis.other.dbinfo split_analysis --show
        python -m ibeis split_analysis --show
        python -m ibeis split_analysis --show --good

    Ignore:
        # mount
        sshfs -o idmap=user lev:/ ~/lev

        # unmount
        fusermount -u ~/lev

    Example:
        >>> # DISABLE_DOCTEST GGR
        >>> from ibeis.other.dbinfo import *  # NOQA
        >>> import ibeis
        >>> dbdir = '/media/danger/GGR/GGR-IBEIS'
        >>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS')
        >>> ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False)
        >>> import guitool_ibeis as gt
        >>> gt.ensure_qtapp()
        >>> win = split_analysis(ibs)
        >>> ut.quit_if_noshow()
        >>> import plottool_ibeis as pt
        >>> gt.qtapp_loop(qwin=win)
        >>> #ut.show_if_requested()
    """
    #nid_list = ibs.get_valid_nids(filter_empty=True)
    import datetime
    day1 = datetime.date(2016, 1, 30)
    day2 = datetime.date(2016, 1, 31)

    filter_kw = {
        'multiple': None,
        #'view': ['right'],
        #'minqual': 'good',
        'is_known': True,
        'min_pername': 1,
    }
    aids1 = ibs.filter_annots_general(filter_kw=ut.dict_union(
        filter_kw, {
            'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
            'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 1.0)),
        })
    )
    aids2 = ibs.filter_annots_general(filter_kw=ut.dict_union(
        filter_kw, {
            'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 0.0)),
            'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
        })
    )
    all_aids = aids1 + aids2
    all_annots = ibs.annots(all_aids)
    print('%d annots on day 1' % (len(aids1)) )
    print('%d annots on day 2' % (len(aids2)) )
    print('%d annots overall' % (len(all_annots)) )
    print('%d names overall' % (len(ut.unique(all_annots.nids))) )

    nid_list, annots_list = all_annots.group(all_annots.nids)

    REVIEWED_EDGES = True
    if REVIEWED_EDGES:
        aids_list = [annots.aids for annots in annots_list]
        #aid_pairs = [annots.get_am_aidpairs() for annots in annots_list]  # Slower
        aid_pairs = ibs.get_unflat_am_aidpairs(aids_list)  # Faster
    else:
        # ALL EDGES
        aid_pairs = [annots.get_aidpairs() for annots in annots_list]

    speeds_list = ibs.unflat_map(ibs.get_annotpair_speeds, aid_pairs)
    import vtool_ibeis as vt
    max_speeds = np.array([vt.safe_max(s, nans=False) for s in speeds_list])

    nan_idx = np.where(np.isnan(max_speeds))[0]
    inf_idx = np.where(np.isinf(max_speeds))[0]
    bad_idx = sorted(ut.unique(ut.flatten([inf_idx, nan_idx])))
    ok_idx = ut.index_complement(bad_idx, len(max_speeds))

    print('#nan_idx = %r' % (len(nan_idx),))
    print('#inf_idx = %r' % (len(inf_idx),))
    print('#ok_idx = %r' % (len(ok_idx),))

    ok_speeds = max_speeds[ok_idx]
    ok_nids = ut.take(nid_list, ok_idx)
    ok_annots = ut.take(annots_list, ok_idx)
    sortx = np.argsort(ok_speeds)[::-1]

    sorted_speeds = np.array(ut.take(ok_speeds, sortx))
    sorted_annots = np.array(ut.take(ok_annots, sortx))
    sorted_nids = np.array(ut.take(ok_nids, sortx))  # NOQA

    sorted_speeds = np.clip(sorted_speeds, 0, 100)

    #idx = vt.find_elbow_point(sorted_speeds)
    #EXCESSIVE_SPEED = sorted_speeds[idx]
    # http://www.infoplease.com/ipa/A0004737.html
    # http://www.speedofanimals.com/animals/zebra
    #ZEBRA_SPEED_MAX  = 64  # km/h
    #ZEBRA_SPEED_RUN  = 50  # km/h
    ZEBRA_SPEED_SLOW_RUN  = 20  # km/h
    #ZEBRA_SPEED_FAST_WALK = 10  # km/h
    #ZEBRA_SPEED_WALK = 7  # km/h

    MAX_SPEED = ZEBRA_SPEED_SLOW_RUN
    #MAX_SPEED = ZEBRA_SPEED_WALK
    #MAX_SPEED = EXCESSIVE_SPEED

    flags = sorted_speeds > MAX_SPEED
    flagged_ok_annots = ut.compress(sorted_annots, flags)
    inf_annots = ut.take(annots_list, inf_idx)
    flagged_annots = inf_annots + flagged_ok_annots

    print('MAX_SPEED = %r km/h' % (MAX_SPEED,))
    print('%d annots with infinite speed' % (len(inf_annots),))
    print('%d annots with large speed' % (len(flagged_ok_annots),))
    print('Marking all pairs of annots above the threshold as non-matching')

    from ibeis.algo.graph import graph_iden
    import networkx as nx
    progkw = dict(freq=1, bs=True, est_window=len(flagged_annots))

    bad_edges_list = []
    good_edges_list = []
    for annots in ut.ProgIter(flagged_annots, lbl='flag speeding names', **progkw):
        edge_to_speeds = annots.get_speeds()
        bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]
        good_edges = [edge for edge, speed in edge_to_speeds.items() if speed <= MAX_SPEED]
        bad_edges_list.append(bad_edges)
        good_edges_list.append(good_edges)
    all_bad_edges = ut.flatten(bad_edges_list)
    good_edges_list = ut.flatten(good_edges_list)
    print('num_bad_edges = %r' % (len(ut.flatten(bad_edges_list)),))
    print('num_bad_edges = %r' % (len(ut.flatten(good_edges_list)),))

    if 1:
        from ibeis.viz import viz_graph2
        import guitool_ibeis as gt
        gt.ensure_qtapp()

        if ut.get_argflag('--good'):
            print('Looking at GOOD (no speed problems) edges')
            aid_pairs = good_edges_list
        else:
            print('Looking at BAD (speed problems) edges')
            aid_pairs = all_bad_edges
        aids = sorted(list(set(ut.flatten(aid_pairs))))
        infr = graph_iden.AnnotInference(ibs, aids, verbose=False)
        infr.initialize_graph()

        # Use random scores to randomize sort order
        rng = np.random.RandomState(0)
        scores = (-rng.rand(len(aid_pairs)) * 10).tolist()
        infr.graph.add_edges_from(aid_pairs)

        if True:
            edge_sample_size = 250
            pop_nids = ut.unique(ibs.get_annot_nids(ut.unique(ut.flatten(aid_pairs))))
            sorted_pairs = ut.sortedby(aid_pairs, scores)[::-1][0:edge_sample_size]
            sorted_nids = ibs.get_annot_nids(ut.take_column(sorted_pairs, 0))
            sample_size = len(ut.unique(sorted_nids))
            am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(*zip(*sorted_pairs))
            flags = ut.not_list(ut.flag_None_items(am_rowids))
            #am_rowids = ut.compress(am_rowids, flags)
            positive_tags = ['SplitCase', 'Photobomb']
            flags_list = [ut.replace_nones(ibs.get_annotmatch_prop(tag, am_rowids), 0)
                          for tag in positive_tags]
            print('edge_case_hist: ' + ut.repr3(
                ['%s %s' % (txt, sum(flags_)) for flags_, txt in zip(flags_list, positive_tags)]))
            is_positive = ut.or_lists(*flags_list)
            num_positive = sum(ut.lmap(any, ut.group_items(is_positive, sorted_nids).values()))
            pop = len(pop_nids)
            print('A positive is any edge flagged as a %s' % (ut.conj_phrase(positive_tags, 'or'),))
            print('--- Sampling wrt edges ---')
            print('edge_sample_size  = %r' % (edge_sample_size,))
            print('edge_population_size = %r' % (len(aid_pairs),))
            print('num_positive_edges = %r' % (sum(is_positive)))
            print('--- Sampling wrt names ---')
            print('name_population_size = %r' % (pop,))
            vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level=.95)

        nx.set_edge_attributes(infr.graph, name='score', values=dict(zip(aid_pairs, scores)))

        win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False,
                                          init_mode=None)
        win.populate_edge_model()
        win.show()
        return win
        # Make review interface for only bad edges

    infr_list = []
    iter_ = list(zip(flagged_annots, bad_edges_list))
    for annots, bad_edges in ut.ProgIter(iter_, lbl='creating inference', **progkw):
        aids = annots.aids
        nids = [1] * len(aids)
        infr = graph_iden.AnnotInference(ibs, aids, nids, verbose=False)
        infr.initialize_graph()
        infr.reset_feedback()
        infr_list.append(infr)

    # Check which ones are user defined as incorrect
    #num_positive = 0
    #for infr in infr_list:
    #    flag = np.any(infr.get_feedback_probs()[0] == 0)
    #    num_positive += flag
    #print('num_positive = %r' % (num_positive,))
    #pop = len(infr_list)
    #print('pop = %r' % (pop,))

    iter_ = list(zip(infr_list, bad_edges_list))
    for infr, bad_edges in ut.ProgIter(iter_, lbl='adding speed edges', **progkw):
        flipped_edges = []
        for aid1, aid2 in bad_edges:
            if infr.graph.has_edge(aid1, aid2):
                flipped_edges.append((aid1, aid2))
            infr.add_feedback((aid1, aid2), NEGTV)
        nx.set_edge_attributes(infr.graph, name='_speed_split', values='orig')
        nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'new' for edge in bad_edges})
        nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'flip' for edge in flipped_edges})

    #for infr in ut.ProgIter(infr_list, lbl='flagging speeding edges', **progkw):
    #    annots = ibs.annots(infr.aids)
    #    edge_to_speeds = annots.get_speeds()
    #    bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]

    def inference_stats(infr_list_):
        relabel_stats = []
        for infr in infr_list_:
            num_ccs, num_inconsistent = infr.relabel_using_reviews()
            state_hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, 'decision').values())
            if POSTV not in state_hist:
                state_hist[POSTV] = 0
            hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, '_speed_split').values())

            subgraphs = infr.positive_connected_compoments()
            subgraph_sizes = [len(g) for g in subgraphs]

            info = ut.odict([
                ('num_nonmatch_edges', state_hist[NEGTV]),
                ('num_match_edges', state_hist[POSTV]),
                ('frac_nonmatch_edges',  state_hist[NEGTV] / (state_hist[POSTV] + state_hist[NEGTV])),
                ('num_inconsistent', num_inconsistent),
                ('num_ccs', num_ccs),
                ('edges_flipped', hist.get('flip', 0)),
                ('edges_unchanged', hist.get('orig', 0)),
                ('bad_unreviewed_edges', hist.get('new', 0)),
                ('orig_size', len(infr.graph)),
                ('new_sizes', subgraph_sizes),
            ])
            relabel_stats.append(info)
        return relabel_stats

    relabel_stats = inference_stats(infr_list)

    print('\nAll Split Info:')
    lines = []
    for key in relabel_stats[0].keys():
        data = ut.take_column(relabel_stats, key)
        if key == 'new_sizes':
            data = ut.flatten(data)
        lines.append('stats(%s) = %s' % (key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
    print('\n'.join(ut.align_lines(lines, '=')))

    num_incon_list = np.array(ut.take_column(relabel_stats, 'num_inconsistent'))
    can_split_flags = num_incon_list == 0
    print('Can trivially split %d / %d' % (sum(can_split_flags), len(can_split_flags)))

    splittable_infrs = ut.compress(infr_list, can_split_flags)

    relabel_stats = inference_stats(splittable_infrs)

    print('\nTrival Split Info:')
    lines = []
    for key in relabel_stats[0].keys():
        if key in ['num_inconsistent']:
            continue
        data = ut.take_column(relabel_stats, key)
        if key == 'new_sizes':
            data = ut.flatten(data)
        lines.append('stats(%s) = %s' % (
            key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
    print('\n'.join(ut.align_lines(lines, '=')))

    num_match_edges = np.array(ut.take_column(relabel_stats, 'num_match_edges'))
    num_nonmatch_edges = np.array(ut.take_column(relabel_stats, 'num_nonmatch_edges'))
    flags1 = np.logical_and(num_match_edges > num_nonmatch_edges, num_nonmatch_edges < 3)
    reasonable_infr = ut.compress(splittable_infrs, flags1)

    new_sizes_list = ut.take_column(relabel_stats, 'new_sizes')
    flags2 = [len(sizes) == 2 and sum(sizes) > 4 and (min(sizes) / max(sizes)) > .3
              for sizes in new_sizes_list]
    reasonable_infr = ut.compress(splittable_infrs, flags2)
    print('#reasonable_infr = %r' % (len(reasonable_infr),))

    for infr in ut.InteractiveIter(reasonable_infr):
        annots = ibs.annots(infr.aids)
        edge_to_speeds = annots.get_speeds()
        print('max_speed = %r' % (max(edge_to_speeds.values())),)
        infr.initialize_visual_node_attrs()
        infr.show_graph(use_image=True, only_reviewed=True)

    rest = ~np.logical_or(flags1, flags2)
    nonreasonable_infr = ut.compress(splittable_infrs, rest)
    rng = np.random.RandomState(0)
    random_idx = ut.random_indexes(len(nonreasonable_infr) - 1, 15, rng=rng)
    random_infr = ut.take(nonreasonable_infr, random_idx)
    for infr in ut.InteractiveIter(random_infr):
        annots = ibs.annots(infr.aids)
        edge_to_speeds = annots.get_speeds()
        print('max_speed = %r' % (max(edge_to_speeds.values())),)
        infr.initialize_visual_node_attrs()
        infr.show_graph(use_image=True, only_reviewed=True)

    #import scipy.stats as st
    #conf_interval = .95
    #st.norm.cdf(conf_interval)
    # view-source:http://www.surveysystem.com/sscalc.htm
    #zval = 1.96  # 95 percent confidence
    #zValC = 3.8416  #
    #zValC = 6.6564

    #import statsmodels.stats.api as sms
    #es = sms.proportion_effectsize(0.5, 0.75)
    #sms.NormalIndPower().solve_power(es, power=0.9, alpha=0.05, ratio=1)

    pop = 279
    num_positive = 3
    sample_size = 15
    conf_level = .95
    #conf_level = .99
    vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level)
    print('---')
    vt.calc_error_bars_from_sample(sample_size + 38, num_positive, pop, conf_level)
    print('---')
    vt.calc_error_bars_from_sample(sample_size + 38 / 3, num_positive, pop, conf_level)
    print('---')

    vt.calc_error_bars_from_sample(15 + 38, num_positive=3, pop=675, conf_level=.95)
    vt.calc_error_bars_from_sample(15, num_positive=3, pop=675, conf_level=.95)

    pop = 279
    #err_frac = .05  # 5%
    err_frac = .10  # 10%
    conf_level = .95
    vt.calc_sample_from_error_bars(err_frac, pop, conf_level)

    pop = 675
    vt.calc_sample_from_error_bars(err_frac, pop, conf_level)
    vt.calc_sample_from_error_bars(.05, pop, conf_level=.95, prior=.1)
    vt.calc_sample_from_error_bars(.05, pop, conf_level=.68, prior=.2)
    vt.calc_sample_from_error_bars(.10, pop, conf_level=.68)

    vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.95)
    vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.68)
コード例 #9
0
    def _update_rows(model, rebuild_structure=True):
        """
        Uses the current ider and col_sort_index to create
        row_indices
        """
        if VERBOSE:
            print('[APIItemModel] +-----------')
            print('[APIItemModel] _update_rows')
        # this is not slow
        #with ut.Timer('update_rows'):
        #printDBG('UPDATE ROWS!')
        #print('UPDATE ROWS!')
        #print('num_rows=%r' % len(model.col_level_list))
        #print('UPDATE model(%s) rows' % model.name)
        #print('[api_model] UPDATE ROWS: %r' % (model.name,))
        #print(ut.get_caller_name(range(4, 12)))
        if len(model.col_level_list) == 0:
            return
        #old_root = model.root_node  # NOQA
        if rebuild_structure:
            #with ut.Timer('[%s] _update_rows: %r' %
            #                 ('cyth' if _atn.CYTHONIZED else 'pyth',
            #                  model.name,), newline=False):
                model.root_node = _atn.build_internal_structure(model)
        #print('-----')
        #def lazy_update_rows():
        #    with ut.Timer('lazy updater: %r' % (model.name,)):
        #        printDBG('[model] calling lazy updater: %r' % (model.name,))
        # REMOVING LAZY FUNCTION BECAUSE IT MIGHT HAVE CAUSED PROBLEMS
        #with ut.Timer('[%s] _update_rows2: %r' %
        #                 ('cyth' if _atn.CYTHONIZED else 'pyth',
        #                  model.name,), newline=False):
        if VERBOSE:
            print('[APIItemModel] lazy_update_rows')
        model.level_index_list = []
        sort_index = 0 if model.col_sort_index is None else model.col_sort_index
        #print('[item_model] sort_index=%r' % (sort_index,))
        children = model.root_node.get_children()  # THIS IS THE LINE THAT TAKES FOREVER
        id_list = [child.get_id() for child in children]
        #print('ids_ generated')
        nodes = []
        if len(id_list) != 0:
            if VERBOSE:
                print('[APIItemModel] lazy_update_rows len(id_list) = %r' % (len(id_list)))
            # start sort
            if model.col_sort_index is not None:
                type_ = model.col_type_list[sort_index]
                getter = model.col_getter_list[sort_index]
                values = getter(id_list)
                if type_ == 'PIXMAP':
                    # TODO: find a better sorting metric for pixmaps
                    values = ut.get_list_column(values, 0)
                #print('values got')
            else:
                type_ = int
                values = id_list
            reverse = model.col_sort_reverse

            #ut.embed()
            # <NUMPY MULTIARRAY SORT>
            #with ut.embed_on_exception_context:
            if type_ is float:
                values = np.array(ut.replace_nones(values, np.nan))
                #values = np.array(values)
                values[np.isnan(values)] = -np.inf  # Force nan to be the smallest number
            import vtool as vt
            sortx = vt.argsort_records([values, id_list], reverse=reverse)
            #sorting_records = np.rec.fromarrays([values, id_list])
            #sort_stride = (-reverse * 2) + 1
            #sortx = sorting_records.argsort()[::sort_stride]
            # </NUMPY MULTIARRAY SORT>
            nodes = ut.take(children, sortx)

            #sorted_pairs = sorted(zip(values, id_list, children), reverse=reverse)
            #nodes = [child for (value, id_, child) in sorted_pairs]
            level = model.col_level_list[sort_index]
            #print("row_indices sorted")
            if level == 0:
                model.root_node.set_children(nodes)
            # end sort
        if ut.USE_ASSERT:
            assert nodes is not None, 'no indices'
        model.level_index_list = nodes
        #if VERBOSE:
        #    print('[APIItemModel] lazy_update_rows emmiting _rows_updated')

        # EMIT THE NUMERR OF ROWS AND THE NAME OF FOR THE VIEW TO DISPLAY
        model._rows_updated.emit(model.name, len(model.level_index_list))

        # lazy method didn't work. Eagerly evaluate
        #lazy_update_rows()
        # HACK TO MAKE SURE TREE NODES DONT DELETE THEMSELVES
        #if VERBOSE:
        #    print('[APIItemModel] build_scope_hack_list')
        # SCOPE HACK SEEMS TO HAVE NOT HALPED
        #model.scope_hack_list = []
        #_atn.build_scope_hack_list(model.root_node, model.scope_hack_list)
        #model.lazy_updater = lazy_update_rows
        #print("Rows updated")
        if VERBOSE:
            print('[APIItemModel] finished _update_rows')
            print('[APIItemModel] L__________')
コード例 #10
0
ファイル: harness.py プロジェクト: whaozl/ibeis
def get_query_result_info(qreq_):
    """
    Helper function.

    Runs queries of a specific configuration returns the best rank of each query

    Args:
        qaids (list) : query annotation ids
        daids (list) : database annotation ids

    Returns:
        qx2_bestranks

    CommandLine:
        python -m ibeis.expt.harness --test-get_query_result_info
        python -m ibeis.expt.harness --test-get_query_result_info:0
        python -m ibeis.expt.harness --test-get_query_result_info:1
        python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1
        python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 --cmd

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.expt.harness import *  # NOQA
        >>> import ibeis
        >>> qreq_ = ibeis.main_helpers.testdata_qreq_(a=['default:qindex=0:3,dindex=0:5'])
        >>> #ibs = ibeis.opendb('PZ_MTEST')
        >>> #qaids = ibs.get_valid_aids()[0:3]
        >>> #daids = ibs.get_valid_aids()[0:5]
        >>> #qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
        >>> cfgres_info = get_query_result_info(qreq_)
        >>> print(ut.dict_str(cfgres_info))

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.expt.harness import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb('PZ_MTEST')
        >>> #cfgdict = dict(codename='vsone')
        >>> # ibs.cfg.query_cfg.codename = 'vsone'
        >>> qaids = ibs.get_valid_aids()[0:3]
        >>> daids = ibs.get_valid_aids()[0:5]
        >>> qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
        >>> cfgres_info = get_query_result_info(qreq_)
        >>> print(ut.dict_str(cfgres_info))

    Ignore:

        ibeis -e rank_cdf --db humpbacks -a default:has_any=hasnotch,mingt=2 -t default:proot=BC_DTW --show --nocache-big

        ibeis -e rank_cdf --db humpbacks -a default:is_known=True,mingt=2 -t default:pipeline_root=BC_DTW
        --show --debug-depc
        ibeis -e rank_cdf --db humpbacks -a default:is_known=True -t default:pipeline_root=BC_DTW --qaid=1,9,15,16,18 --daid-override=1,9,15,16,18,21,22 --show --debug-depc
        --clear-all-depcache
    """
    try:
        ibs = qreq_.ibs
    except AttributeError:
        ibs = qreq_.depc.controller
    import vtool as vt
    cm_list = qreq_.execute()
    #qreq_.ibs.query_chips(qreq_=qreq_, use_bigcache=False)
    qx2_cm = cm_list
    qaids = qreq_.qaids
    #qaids2 = [cm.qaid for cm in cm_list]
    qnids = ibs.get_annot_name_rowids(qaids)

    import utool
    with utool.embed_on_exception_context:
        unique_dnids = np.unique(ibs.get_annot_name_rowids(qreq_.daids))

        unique_qnids, groupxs = vt.group_indices(qnids)
        cm_group_list = ut.apply_grouping(cm_list, groupxs)
        qnid2_aggnamescores = {}

    qnx2_nameres_info = []

    #import utool
    #utool.embed()

    # Ranked list aggregation-ish
    nameres_info_list = []
    for qnid, cm_group in zip(unique_qnids, cm_group_list):
        nid2_name_score_group = [
            dict([(nid, cm.name_score_list[nidx])
                  for nid, nidx in cm.nid2_nidx.items()]) for cm in cm_group
        ]
        aligned_name_scores = np.array([
            ut.dict_take(nid2_name_score, unique_dnids.tolist(), -np.inf)
            for nid2_name_score in nid2_name_score_group
        ]).T
        name_score_list = np.nanmax(aligned_name_scores, axis=1)
        qnid2_aggnamescores[qnid] = name_score_list
        # sort
        sortx = name_score_list.argsort()[::-1]
        sorted_namescores = name_score_list[sortx]
        sorted_dnids = unique_dnids[sortx]

        ## infer agg name results
        is_positive = sorted_dnids == qnid
        is_negative = np.logical_and(~is_positive, sorted_dnids > 0)
        gt_name_rank = None if not np.any(is_positive) else np.where(
            is_positive)[0][0]
        gf_name_rank = None if not np.any(is_negative) else np.nonzero(
            is_negative)[0][0]
        gt_nid = sorted_dnids[gt_name_rank]
        gf_nid = sorted_dnids[gf_name_rank]
        gt_name_score = sorted_namescores[gt_name_rank]
        gf_name_score = sorted_namescores[gf_name_rank]
        qnx2_nameres_info = {}
        qnx2_nameres_info['qnid'] = qnid
        qnx2_nameres_info['gt_nid'] = gt_nid
        qnx2_nameres_info['gf_nid'] = gf_nid
        qnx2_nameres_info['gt_name_rank'] = gt_name_rank
        qnx2_nameres_info['gf_name_rank'] = gf_name_rank
        qnx2_nameres_info['gt_name_score'] = gt_name_score
        qnx2_nameres_info['gf_name_score'] = gf_name_score

        nameres_info_list.append(qnx2_nameres_info)
        nameres_info = ut.dict_stack(nameres_info_list, 'qnx2_')

    qaids = qreq_.qaids
    daids = qreq_.daids
    qx2_gtaids = ibs.get_annot_groundtruth(qaids, daid_list=daids)
    # Get the groundtruth ranks and accuracy measures
    qx2_qresinfo = [get_qres_name_result_info(ibs, cm, qreq_) for cm in qx2_cm]

    cfgres_info = ut.dict_stack(qx2_qresinfo, 'qx2_')
    #for key in qx2_qresinfo[0].keys():
    #    'qx2_' + key
    #    ut.get_list_column(qx2_qresinfo, key)

    if False:
        qx2_avepercision = np.array([
            cm.get_average_percision(ibs=ibs, gt_aids=gt_aids)
            for (cm, gt_aids) in zip(qx2_cm, qx2_gtaids)
        ])
        cfgres_info['qx2_avepercision'] = qx2_avepercision
    # Compute mAP score  # TODO: use mAP score
    # (Actually map score doesn't make much sense if using name scoring
    #mAP = qx2_avepercision[~np.isnan(qx2_avepercision)].mean()  # NOQA
    cfgres_info['qx2_bestranks'] = ut.replace_nones(
        cfgres_info['qx2_bestranks'], -1)
    cfgres_info.update(nameres_info)
    return cfgres_info
コード例 #11
0
ファイル: harness.py プロジェクト: Erotemic/ibeis
def get_query_result_info(qreq_):
    """
    Helper function.

    Runs queries of a specific configuration returns the best rank of each query

    Args:
        qaids (list) : query annotation ids
        daids (list) : database annotation ids

    Returns:
        qx2_bestranks

    CommandLine:
        python -m ibeis.expt.harness --test-get_query_result_info
        python -m ibeis.expt.harness --test-get_query_result_info:0
        python -m ibeis.expt.harness --test-get_query_result_info:1
        python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1
        python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 --cmd

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.expt.harness import *  # NOQA
        >>> import ibeis
        >>> qreq_ = ibeis.main_helpers.testdata_qreq_(a=['default:qindex=0:3,dindex=0:5'])
        >>> #ibs = ibeis.opendb('PZ_MTEST')
        >>> #qaids = ibs.get_valid_aids()[0:3]
        >>> #daids = ibs.get_valid_aids()[0:5]
        >>> #qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
        >>> cfgres_info = get_query_result_info(qreq_)
        >>> print(ut.dict_str(cfgres_info))

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.expt.harness import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb('PZ_MTEST')
        >>> #cfgdict = dict(codename='vsone')
        >>> # ibs.cfg.query_cfg.codename = 'vsone'
        >>> qaids = ibs.get_valid_aids()[0:3]
        >>> daids = ibs.get_valid_aids()[0:5]
        >>> qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
        >>> cfgres_info = get_query_result_info(qreq_)
        >>> print(ut.dict_str(cfgres_info))

    Ignore:

        ibeis -e rank_cdf --db humpbacks -a default:has_any=hasnotch,mingt=2 -t default:proot=BC_DTW --show --nocache-big

        ibeis -e rank_cdf --db humpbacks -a default:is_known=True,mingt=2 -t default:pipeline_root=BC_DTW
        --show --debug-depc
        ibeis -e rank_cdf --db humpbacks -a default:is_known=True -t default:pipeline_root=BC_DTW --qaid=1,9,15,16,18 --daid-override=1,9,15,16,18,21,22 --show --debug-depc
        --clear-all-depcache
    """
    try:
        ibs = qreq_.ibs
    except AttributeError:
        ibs = qreq_.depc.controller
    import vtool as vt
    cm_list = qreq_.execute()
    #qreq_.ibs.query_chips(qreq_=qreq_, use_bigcache=False)
    qx2_cm = cm_list
    qaids = qreq_.qaids
    #qaids2 = [cm.qaid for cm in cm_list]
    qnids = ibs.get_annot_name_rowids(qaids)

    import utool
    with utool.embed_on_exception_context:
        unique_dnids = np.unique(ibs.get_annot_name_rowids(qreq_.daids))

        unique_qnids, groupxs = vt.group_indices(qnids)
        cm_group_list = ut.apply_grouping(cm_list, groupxs)
        qnid2_aggnamescores = {}

    qnx2_nameres_info = []

    #import utool
    #utool.embed()

    # Ranked list aggregation-ish
    nameres_info_list = []
    for qnid, cm_group in zip(unique_qnids, cm_group_list):
        nid2_name_score_group = [
            dict([(nid, cm.name_score_list[nidx]) for nid, nidx in cm.nid2_nidx.items()])
            for cm in cm_group
        ]
        aligned_name_scores = np.array([
            ut.dict_take(nid2_name_score, unique_dnids.tolist(), -np.inf)
            for nid2_name_score in nid2_name_score_group
        ]).T
        name_score_list = np.nanmax(aligned_name_scores, axis=1)
        qnid2_aggnamescores[qnid] = name_score_list
        # sort
        sortx = name_score_list.argsort()[::-1]
        sorted_namescores = name_score_list[sortx]
        sorted_dnids = unique_dnids[sortx]

        ## infer agg name results
        is_positive = sorted_dnids == qnid
        is_negative = np.logical_and(~is_positive, sorted_dnids > 0)
        gt_name_rank = None if not np.any(is_positive) else np.where(is_positive)[0][0]
        gf_name_rank = None if not np.any(is_negative) else np.nonzero(is_negative)[0][0]
        gt_nid = sorted_dnids[gt_name_rank]
        gf_nid = sorted_dnids[gf_name_rank]
        gt_name_score = sorted_namescores[gt_name_rank]
        gf_name_score = sorted_namescores[gf_name_rank]
        qnx2_nameres_info = {}
        qnx2_nameres_info['qnid'] = qnid
        qnx2_nameres_info['gt_nid'] = gt_nid
        qnx2_nameres_info['gf_nid'] = gf_nid
        qnx2_nameres_info['gt_name_rank'] = gt_name_rank
        qnx2_nameres_info['gf_name_rank'] = gf_name_rank
        qnx2_nameres_info['gt_name_score'] = gt_name_score
        qnx2_nameres_info['gf_name_score'] = gf_name_score

        nameres_info_list.append(qnx2_nameres_info)
        nameres_info = ut.dict_stack(nameres_info_list, 'qnx2_')

    qaids = qreq_.qaids
    daids = qreq_.daids
    qx2_gtaids = ibs.get_annot_groundtruth(qaids, daid_list=daids)
    # Get the groundtruth ranks and accuracy measures
    qx2_qresinfo = [get_qres_name_result_info(ibs, cm, qreq_) for cm in qx2_cm]

    cfgres_info = ut.dict_stack(qx2_qresinfo, 'qx2_')
    #for key in qx2_qresinfo[0].keys():
    #    'qx2_' + key
    #    ut.get_list_column(qx2_qresinfo, key)

    if False:
        qx2_avepercision = np.array(
            [cm.get_average_percision(ibs=ibs, gt_aids=gt_aids) for
             (cm, gt_aids) in zip(qx2_cm, qx2_gtaids)])
        cfgres_info['qx2_avepercision'] = qx2_avepercision
    # Compute mAP score  # TODO: use mAP score
    # (Actually map score doesn't make much sense if using name scoring
    #mAP = qx2_avepercision[~np.isnan(qx2_avepercision)].mean()  # NOQA
    cfgres_info['qx2_bestranks'] = ut.replace_nones(cfgres_info['qx2_bestranks'] , -1)
    cfgres_info.update(nameres_info)
    return cfgres_info
コード例 #12
0
    def _update_rows(model, rebuild_structure=True):
        """
        Uses the current ider and col_sort_index to create
        row_indices
        """
        # with ut.Timer('[gt] update_rows (%s)' % (model.name,)):
        if True:
            # flag = model.blockSignals(True)
            if VERBOSE_MODEL:
                logger.info('[APIItemModel] +-----------')
                logger.info('[APIItemModel] _update_rows')
            # this is not slow
            # logger.info('UPDATE ROWS!')
            if len(model.col_level_list) == 0:
                return
            # old_root = model.root_node  # NOQA
            if rebuild_structure:
                # logger.info('Rebuilging api_item_model internal structure')
                model.beginResetModel(
                )  # I think this is preventing a segfault
                model.root_node = _atn.build_internal_structure(model)
                model.endResetModel()
            if VERBOSE_MODEL:
                logger.info('[APIItemModel] lazy_update_rows')
            model.level_index_list = []
            sort_index = 0 if model.col_sort_index is None else model.col_sort_index
            children = (model.root_node.get_children()
                        )  # THIS IS THE LINE THAT TAKES FOREVER
            id_list = [child.get_id() for child in children]
            # logger.info('ids_ generated')
            nodes = []
            if len(id_list) != 0:
                if VERBOSE_MODEL:
                    logger.info(
                        '[APIItemModel] lazy_update_rows len(id_list) = %r' %
                        (len(id_list)))
                # start sort
                if model.col_sort_index is not None:
                    type_ = model.col_type_list[sort_index]
                    getter = model.col_getter_list[sort_index]
                    values = getter(id_list)
                    if type_ == 'PIXMAP':
                        # TODO: find a better sorting metric for pixmaps
                        values = ut.get_list_column(values, 0)
                else:
                    type_ = int
                    values = id_list
                reverse = model.col_sort_reverse

                # <NUMPY MULTIARRAY SORT>
                if True:
                    if values is None:
                        logger.info('SORTING VALUES IS NONE. VERY WEIRD')
                    if type_ is float:
                        values = np.array(ut.replace_nones(values, np.nan))
                        # Force nan to be the smallest number
                        values[np.isnan(values)] = -np.inf
                    elif type_ is str:
                        values = ut.replace_nones(values, '')
                    import vtool as vt

                    sortx = vt.argsort_records([values, id_list],
                                               reverse=reverse)
                    # </NUMPY MULTIARRAY SORT>
                    nodes = ut.take(children, sortx)
                    level = model.col_level_list[sort_index]
                    if level == 0:
                        model.root_node.set_children(nodes)
                    # end sort
            if ut.USE_ASSERT:
                assert nodes is not None, 'no indices'
            model.level_index_list = nodes

            # Book keeping for lazy loading rows
            model.num_rows_total = len(model.level_index_list)
            # model.num_cols_total = len(model.col_name_list)
            model.num_cols_loaded = 0

            if model.lazy_rows:
                model.num_rows_loaded = 0
            else:
                model.num_rows_loaded = model.num_rows_total
            # emit the numerr of rows and the name of for the view to display
            # model.blockSignals(flag)
            model._rows_updated.emit(model.name, model.num_rows_total)
            if VERBOSE_MODEL:
                logger.info('[APIItemModel] finished _update_rows')
                logger.info('[APIItemModel] L__________')
コード例 #13
0
ファイル: experiment_printres.py プロジェクト: Erotemic/ibeis
def print_results(ibs, testres):
    """
    Prints results from an experiment harness run.
    Rows store different qaids (query annotation ids)
    Cols store different configurations (algorithm parameters)

    Args:
        ibs (IBEISController):  ibeis controller object
        testres (test_result.TestResult):

    CommandLine:
        python dev.py -e print --db PZ_MTEST -a default:dpername=1,qpername=[1,2]  -t default:fg_on=False

        python dev.py -e print -t best --db seals2 --allgt --vz
        python dev.py -e print --db PZ_MTEST --allgt -t custom --print-confusion-stats
        python dev.py -e print --db PZ_MTEST --allgt --noqcache --index 0:10:2 -t custom:rrvsone_on=True --print-confusion-stats
        python dev.py -e print --db PZ_MTEST --allgt --noqcache --qaid4 -t custom:rrvsone_on=True --print-confusion-stats
        python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
        python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
        python -m ibeis --tf print_results --db PZ_MTEST -a default -t default:lnbnn_on=True default:lnbnn_on=False,bar_l2_on=True default:lnbnn_on=False,normonly_on=True

    CommandLine:
        python -m ibeis.expt.experiment_printres --test-print_results
        utprof.py -m ibeis.expt.experiment_printres --test-print_results

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.expt.experiment_printres import *  # NOQA
        >>> from ibeis.init import main_helpers
        >>> ibs, testres = main_helpers.testdata_expts(
        >>>     'PZ_MTEST', a='default:dpername=1,qpername=[1,2]', t='default:fg_on=False')
        >>> result = print_results(ibs, testres)
        >>> print(result)
    """

    (cfg_list, cfgx2_cfgresinfo, testnameid, cfgx2_lbl, cfgx2_qreq_) = ut.dict_take(
        testres.__dict__, ['cfg_list', 'cfgx2_cfgresinfo', 'testnameid', 'cfgx2_lbl', 'cfgx2_qreq_'])

    # cfgx2_cfgresinfo is a list of dicts of lists
    # Parse result info out of the lists
    cfgx2_nextbestranks  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_next_bestranks')
    cfgx2_gt_rawscores   = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
    cfgx2_gf_rawscores   = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')
    #cfgx2_aveprecs       = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_avepercision')

    cfgx2_scorediffs     = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_scorediff')
    #cfgx2_gt_raw_score   = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')

    column_lbls = [ut.remove_chars(ut.remove_vowels(lbl), [' ', ','])
                   for lbl in cfgx2_lbl]

    scorediffs_mat     = np.array(ut.replace_nones(cfgx2_scorediffs, np.nan))

    print(' --- PRINT RESULTS ---')
    print(' use --rank-lt-list=1,5 to specify X_LIST')
    if True:
        # Num of ranks less than to score
        X_LIST = testres.get_X_LIST()
        #X_LIST = [1, 5]

        #nConfig = len(cfg_list)
        #nQuery = len(testres.qaids)
        cfgx2_nQuery = list(map(len, testres.cfgx2_qaids))
        #cfgx2_qx2_ranks = testres.get_infoprop_list('qx2_bestranks')
        #--------------------

        # A positive scorediff indicates the groundtruth was better than the
        # groundfalse scores
        istrue_list  = [scorediff > 0 for scorediff in scorediffs_mat]
        isfalse_list = [~istrue for istrue in istrue_list]

        #------------
        # Build Colscore
        nLessX_dict = testres.get_nLessX_dict()

        #------------
        best_rankscore_summary = []
        #to_intersect_list = []
        # print each configs scores less than X=thresh
        for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
            max_nLessX = cfgx2_nLessX.max()
            bestX_cfgx_list = np.where(cfgx2_nLessX == max_nLessX)[0]
            best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestX_cfgx_list)
            # FIXME
            best_rankscore += rankscore_str(X, max_nLessX, cfgx2_nQuery[bestX_cfgx_list[0]])
            best_rankscore_summary += [best_rankscore]
            #to_intersect_list.append(ut.take(cfgx2_lbl, max_nLessX))

        #intersected = to_intersect_list[0] if len(to_intersect_list) > 0 else []
        #for ix in range(1, len(to_intersect_list)):
        #    intersected = np.intersect1d(intersected, to_intersect_list[ix])

    #if False:
    #    #gt_raw_score_mat = np.vstack(cfgx2_gt_raw_score).T

    #    #rank_mat = testres.get_rank_mat()

    #    #------------
    #    # Build row lbls
    #    if False:
    #        qx2_lbl = np.array([
    #            'qx=%d) q%s ' % (qx, ibsfuncs.aidstr(testres.qaids[qx], ibs=ibs, notes=True))
    #            for qx in range(nQuery)])

    #    #------------
    #    # Build Colscore and hard cases
    #    if False:
    #        qx2_min_rank = []
    #        qx2_argmin_rank = []
    #        new_hard_qaids = []
    #        new_hardtup_list = []

    #        for qx in range(nQuery):
    #            ranks = rank_mat[qx]
    #            valid_ranks = ranks[ranks >= 0]
    #            min_rank = ranks.min() if len(valid_ranks) > 0 else -3
    #            bestCFG_X = np.where(ranks == min_rank)[0]
    #            qx2_min_rank.append(min_rank)
    #            # Find the best rank over all configurations
    #            qx2_argmin_rank.append(bestCFG_X)

    #@ut.memoize
    #def get_new_hard_qx_list(testres):
    #    """ Mark any query as hard if it didnt get everything correct """
    #    rank_mat = testres.get_rank_mat()
    #    is_new_hard_list = rank_mat.max(axis=1) > 0
    #    new_hard_qx_list = np.where(is_new_hard_list)[0]
    #    return new_hard_qx_list

    #        new_hard_qx_list = testres.get_new_hard_qx_list()

    #        for qx in new_hard_qx_list:
    #            # New list is in aid format instead of cx format
    #            # because you should be copying and pasting it
    #            notes = ' ranks = ' + str(rank_mat[qx])
    #            qaid = testres.qaids[qx]
    #            name = ibs.get_annot_names(qaid)
    #            new_hardtup_list += [(qaid, name + " - " + notes)]
    #            new_hard_qaids += [qaid]

    @ut.argv_flag_dec
    def intersect_hack():
        failed = testres.rank_mat > 0
        colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
        #failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
        #failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
        failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
        failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
        gt_aids1 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
        gt_aids2 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)

        qaids_expt = failed_col1_only_aids
        gt_avl_aids1 = ut.flatten(gt_aids1)
        gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))

        ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
        ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
        #jsontext = ut.to_json({
        #    'qaids': list(qaids_expt),
        #    'dinclude_aids1': list(gt_aids_expt1),
        #    'dinclude_aids2': list(gt_aids_expt2),
        #})
        #annotation_configs.varysize_pzm
        #from ibeis.expt import annotation_configs

        acfg = testres.acfg_list[0]
        import copy
        acfg1 = copy.deepcopy(acfg)
        acfg2 = copy.deepcopy(acfg)
        acfg1['qcfg']['min_pername'] = None
        acfg2['qcfg']['min_pername'] = None
        acfg1['dcfg']['min_pername'] = None
        acfg2['dcfg']['min_gt_per_name'] = None

        acfg1['qcfg']['default_aids'] = qaids_expt
        acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
        acfg2['qcfg']['default_aids'] = qaids_expt
        acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2

        from ibeis.init import filter_annots
        from ibeis.expt import experiment_helpers

        annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
        annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)

        acfg_name_list = dict(  # NOQA
            acfg_list=[acfg1, acfg2],
            expanded_aids_list=[annots1, annots2],
        )
        test_cfg_name_list = ['candidacy_k']
        cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs)

        t1, t2 = testres_list  # NOQA
    #ut.embed()
    #intersect_hack()

    #@ut.argv_flag_dec
    #def print_rowlbl():
    #    print('=====================')
    #    print('[harn] Row/Query Labels: %s' % testnameid)
    #    print('=====================')
    #    print('[harn] queries:\n%s' % '\n'.join(qx2_lbl))
    #print_rowlbl()
    #------------

    @ut.argv_flag_dec
    def print_collbl():
        print('=====================')
        print('[harn] Col/Config Labels: %s' % testnameid)
        print('=====================')
        enum_cfgx2_lbl = ['%2d) %s' % (count, cfglbl)
                            for count, cfglbl in enumerate(cfgx2_lbl)]
        print('[harn] cfglbl:\n%s' % '\n'.join(enum_cfgx2_lbl))
    print_collbl()

    #------------

    @ut.argv_flag_dec
    def print_cfgstr():
        print('=====================')
        print('[harn] Config Strings: %s' % testnameid)
        print('=====================')
        cfgstr_list = [query_cfg.get_cfgstr() for query_cfg in cfg_list]
        enum_cfgstr_list = ['%2d) %s' % (count, cfgstr)
                            for count, cfgstr in enumerate(cfgstr_list)]
        print('\n[harn] cfgstr:\n%s' % '\n'.join(enum_cfgstr_list))
    print_cfgstr()

    #------------

    #@ut.argv_flag_dec
    #def print_rowscore():
    #    print('=======================')
    #    print('[harn] Scores per Query: %s' % testnameid)
    #    print('=======================')
    #    for qx in range(nQuery):
    #        bestCFG_X = qx2_argmin_rank[qx]
    #        min_rank = qx2_min_rank[qx]
    #        minimizing_cfg_str = ut.indentjoin(cfgx2_lbl[bestCFG_X], '\n  * ')
    #        #minimizing_cfg_str = str(bestCFG_X)

    #        print('-------')
    #        print(qx2_lbl[qx])
    #        print(' best_rank = %d ' % min_rank)
    #        if len(cfgx2_lbl) != 1:
    #            print(' minimizing_cfg_x\'s = %s ' % minimizing_cfg_str)
    #print_rowscore()

    #------------

    #@ut.argv_flag_dec
    #def print_row_ave_precision():
    #    print('=======================')
    #    print('[harn] Scores per Query: %s' % testnameid)
    #    print('=======================')
    #    for qx in range(nQuery):
    #        aveprecs = ', '.join(['%.2f' % (aveprecs[qx],) for aveprecs in cfgx2_aveprecs])
    #        print('-------')
    #        print(qx2_lbl[qx])
    #        print(' aveprecs = %s ' % aveprecs)
    #print_row_ave_precision()

    ##------------

    #@ut.argv_flag_dec
    #def print_hardcase():
    #    print('--- hard new_hardtup_list (w.r.t these configs): %s' % testnameid)
    #    print('\n'.join(map(repr, new_hardtup_list)))
    #    print('There are %d hard cases ' % len(new_hardtup_list))
    #    aid_list = [aid_notes[0] for aid_notes in new_hardtup_list]
    #    name_list = ibs.get_annot_names(aid_list)
    #    name_set = set(name_list)
    #    print(sorted(aid_list))
    #    print('Names: %r' % (name_set,))
    #print_hardcase()
    #default=not ut.get_argflag('--allhard'))

    #------------

    #@ut.argv_flag_dec
    #def echo_hardcase():
    #    print('--- hardcase commandline: %s' % testnameid)
    #    # Show index for current query where hardids reside
    #    #print('--index ' + (' '.join(map(str, new_hard_qx_list))))
    #    #print('--take new_hard_qx_list')
    #    #hardaids_str = ' '.join(map(str, ['    ', '--qaid'] + new_hard_qaids))
    #    hardaids_str = ' '.join(map(str, ['    ', '--set-aids-as-hard'] + new_hard_qaids))
    #    print(hardaids_str)
    ##echo_hardcase(default=not ut.get_argflag('--allhard'))
    #echo_hardcase()

    #@ut.argv_flag_dec
    #def print_bestcfg():
    #    print('==========================')
    #    print('[harn] Best Configurations: %s' % testnameid)
    #    print('==========================')
    #    # print each configs scores less than X=thresh
    #    for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
    #        max_LessX = cfgx2_nLessX.max()
    #        bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
    #        best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
    #        best_rankscore += rankscore_str(X, max_LessX, nQuery)
    #        cfglbl_list = cfgx2_lbl[bestCFG_X]

    #        best_rankcfg = format_cfgstr_list(cfglbl_list)
    #        #indent('\n'.join(cfgstr_list), '    ')
    #        print(best_rankscore)
    #        print(best_rankcfg)
    #    print('[cfg*]  %d cfg(s) are the best of %d total cfgs' % (len(intersected), nConfig))
    #    print(format_cfgstr_list(intersected))
    #print_bestcfg()

    #------------

    #@ut.argv_flag_dec
    #def print_gtscore():
    #    # Prints best ranks
    #    print('gtscore_mat: %s' % testnameid)
    #    print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
    #    header = (' labled rank matrix: rows=queries, cols=cfgs:')
    #    print('\n'.join(cfgx2_lbl))
    #    column_list = gt_raw_score_mat.T
    #    print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
    #                            column_lbls=column_lbls, header=header,
    #                            transpose=False,
    #                            use_lbl_width=len(cfgx2_lbl) < 5))
    #print_gtscore()

    #------------

    #@ut.argv_flag_dec
    #def print_best_rankmat():
    #    # Prints best ranks
    #    print('-------------')
    #    print('RankMat: %s' % testnameid)
    #    print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
    #    header = (' labled rank matrix: rows=queries, cols=cfgs:')
    #    print('\n'.join(cfgx2_lbl))
    #    column_list = rank_mat.T
    #    print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
    #                            column_lbls=column_lbls, header=header,
    #                            transpose=False,
    #                            use_lbl_width=len(cfgx2_lbl) < 5))
    #print_best_rankmat()

    #@ut.argv_flag_dec
    #def print_diffmat():
    #    # score differences over configs
    #    print('-------------')
    #    print('Diffmat: %s' % testnameid)
    #    diff_matstr = get_diffmat_str(rank_mat, testres.qaids, nConfig)
    #    print(diff_matstr)
    #print_diffmat()

    #@ut.argv_flag_dec
    #def print_rankhist_time():
    #    print('A rank histogram is a dictionary. '
    #          'The keys denote the range of the ranks that the values fall in')
    #    # TODO: rectify this code with other hist code

    #    config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
    #    config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()

    #    _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
    #    for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
    #        #full_cfgstr = testres.cfgx2_qreq_[cfgx].get_full_cfgstr()
    #        #ut.print_dict(ut.dict_hist(ranks), 'rank histogram', sorted_=True)
    #        # find the qxs that belong to each bin
    #        aid_list1 = testres.qaids
    #        aid_list2 = qx2_gt_aid
    #        ibs.assert_valid_aids(aid_list1)
    #        ibs.assert_valid_aids(aid_list2)
    #        timedelta_list = ibs.get_annot_pair_timdelta(aid_list1, aid_list2)
    #        #timedelta_str_list = [ut.get_posix_timedelta_str2(delta)
    #        #                      for delta in timedelta_list]

    #        bin_edges = testres.get_rank_histogram_bin_edges()
    #        timedelta_groups = ut.dict_take(ut.group_items(timedelta_list, config_binxs), np.arange(len(bin_edges)), [])

    #        timedelta_stats = [ut.get_stats(deltas, use_nan=True, datacast=ut.get_posix_timedelta_str2) for deltas in timedelta_groups]
    #        print('Time statistics for each rank range:')
    #        print(ut.dict_str(dict(zip(bin_edges, timedelta_stats)), sorted_=True))
    #print_rankhist_time()

    #@ut.argv_flag_dec
    #def print_rankhist():
    #    print('A rank histogram is a dictionary. '
    #          'The keys denote the range of the ranks that the values fall in')
    #    # TODO: rectify this code with other hist code

    #    config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
    #    config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()

    #    _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
    #    for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
    #        print('Frequency of rank ranges:')
    #        ut.print_dict(agg_hist_dict, 'agg rank histogram', sorted_=True)
    #print_rankhist()

    #------------
    # Print summary
    #print(' --- SUMMARY ---')

    #------------

    #@ut.argv_flag_dec
    #def print_colmap():
    #    print('==================')
    #    print('[harn] mAP per Config: %s (sorted by mAP)' % testnameid)
    #    print('==================')
    #    cfgx2_mAP = np.array([aveprec_list.mean() for aveprec_list in cfgx2_aveprecs])
    #    sortx = cfgx2_mAP.argsort()
    #    for cfgx in sortx:
    #        print('[mAP] cfgx=%r) mAP=%.3f -- %s' % (cfgx, cfgx2_mAP[cfgx], cfgx2_lbl[cfgx]))
    #    #print('L___ Scores per Config ___')
    #print_colmap()
    #------------

    @ut.argv_flag_dec_true
    def print_colscore():
        print('==================')
        print('[harn] Scores per Config: %s' % testnameid)
        print('==================')
        #for cfgx in range(nConfig):
        #    print('[score] %s' % (cfgx2_lbl[cfgx]))
        #    for X in X_LIST:
        #        nLessX_ = nLessX_dict[int(X)][cfgx]
        #        print('        ' + rankscore_str(X, nLessX_, nQuery))
        print('\n[harn] ... sorted scores')
        for X in X_LIST:
            print('\n[harn] Sorted #ranks < %r scores' % (X))
            sortx = np.array(nLessX_dict[int(X)]).argsort()
            #frac_list = (nLessX_dict[int(X)] / cfgx2_nQuery)[:, None]
            #print('cfgx2_nQuery = %r' % (cfgx2_nQuery,))
            #print('frac_list = %r' % (frac_list,))
            #print('Pairwise Difference: ' + str(ut.safe_pdist(frac_list, metric=ut.absdiff)))
            for cfgx in sortx:
                nLessX_ = nLessX_dict[int(X)][cfgx]
                rankstr = rankscore_str(X, nLessX_, cfgx2_nQuery[cfgx], withlbl=False)
                print('[score] %s --- %s' % (rankstr, cfgx2_lbl[cfgx]))
    print_colscore()

    #------------

    ut.argv_flag_dec(print_latexsum)(ibs, testres)

    @ut.argv_flag_dec
    def print_next_rankmat():
        # Prints nextbest ranks
        print('-------------')
        print('NextRankMat: %s' % testnameid)
        header = (' top false rank matrix: rows=queries, cols=cfgs:')
        print('\n'.join(cfgx2_lbl))
        column_list = cfgx2_nextbestranks
        print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
                                column_lbls=column_lbls, header=header,
                                transpose=False,
                                use_lbl_width=len(cfgx2_lbl) < 5))
    print_next_rankmat()

    #------------

    @ut.argv_flag_dec
    def print_scorediff_mat():
        # Prints nextbest ranks
        print('-------------')
        print('ScoreDiffMat: %s' % testnameid)
        header = (' score difference between top true and top false: rows=queries, cols=cfgs:')
        print('\n'.join(cfgx2_lbl))
        column_list = cfgx2_scorediffs
        column_type = [float] * len(column_list)
        print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
                                column_lbls=column_lbls,
                                column_type=column_type,
                                header=header,
                                transpose=False,
                                use_lbl_width=len(cfgx2_lbl) < 5))
    print_scorediff_mat(alias_flags=['--sdm'])

    #------------
    def jagged_stats_info(arr_, lbl, col_lbls):
        arr = ut.recursive_replace(arr_, np.inf, np.nan)
        # Treat infinite as nan
        stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True)
        sel_stat_dict, sel_indices = ut.find_interesting_stats(stat_dict, col_lbls)
        sel_col_lbls = ut.take(col_lbls, sel_indices)
        statstr_kw   = dict(precision=3, newlines=True, lbl=lbl, align=True)
        stat_str     = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw)
        sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw)
        sel_stat_str = 'sel_col_lbls = %s' % (ut.list_str(sel_col_lbls),) + '\n' + sel_stat_str
        return stat_str, sel_stat_str

    @ut.argv_flag_dec
    def print_confusion_stats():
        """
        CommandLine:
            python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
        """
        # Prints nextbest ranks
        print('-------------')
        print('ScoreDiffMatStats: %s' % testnameid)
        print('column_lbls = %r' % (column_lbls,))

        #cfgx2_gt_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
        #cfgx2_gf_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')

        gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
        gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)

        tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
        fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
        tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
        fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)

        tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(tp_rawscores, 'tp_rawscores', cfgx2_lbl)
        fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(fp_rawscores, 'fp_rawscores', cfgx2_lbl)
        tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(tn_rawscores, 'tn_rawscores', cfgx2_lbl)
        fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(fn_rawscores, 'fn_rawscores', cfgx2_lbl)

        #print(tp_rawscores_str)
        #print(fp_rawscores_str)
        #print(tn_rawscores_str)
        #print(fn_rawscores_str)

        print(tp_rawscore_statstr)
        print(fp_rawscore_statstr)
        print(tn_rawscore_statstr)
        print(fn_rawscore_statstr)

    print_confusion_stats(alias_flags=['--cs'])

    ut.argv_flag_dec_true(testres.print_percent_identification_success)()

    sumstrs = []
    sumstrs.append('')
    sumstrs.append('||===========================')
    sumstrs.append('|| [cfg*] SUMMARY: %s' % testnameid)
    sumstrs.append('||---------------------------')
    sumstrs.append(ut.joins('\n|| ', best_rankscore_summary))
    sumstrs.append('||===========================')
    summary_str = '\n' + '\n'.join(sumstrs) + '\n'
    #print(summary_str)
    ut.colorprint(summary_str, 'blue')

    print('To enable all printouts add --print-all to the commandline')
コード例 #14
0
def print_results(ibs, testres):
    """
    Prints results from an experiment harness run.
    Rows store different qaids (query annotation ids)
    Cols store different configurations (algorithm parameters)

    Args:
        ibs (IBEISController):  ibeis controller object
        testres (test_result.TestResult):

    CommandLine:
        python dev.py -e print --db PZ_MTEST -a default:dpername=1,qpername=[1,2]  -t default:fg_on=False

        python dev.py -e print -t best --db seals2 --allgt --vz
        python dev.py -e print --db PZ_MTEST --allgt -t custom --print-confusion-stats
        python dev.py -e print --db PZ_MTEST --allgt --noqcache --index 0:10:2 -t custom:rrvsone_on=True --print-confusion-stats
        python dev.py -e print --db PZ_MTEST --allgt --noqcache --qaid4 -t custom:rrvsone_on=True --print-confusion-stats
        python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
        python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
        python -m ibeis --tf print_results --db PZ_MTEST -a default -t default:lnbnn_on=True default:lnbnn_on=False,bar_l2_on=True default:lnbnn_on=False,normonly_on=True

    CommandLine:
        python -m ibeis.expt.experiment_printres --test-print_results
        utprof.py -m ibeis.expt.experiment_printres --test-print_results

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.expt.experiment_printres import *  # NOQA
        >>> from ibeis.init import main_helpers
        >>> ibs, testres = main_helpers.testdata_expts(
        >>>     'PZ_MTEST', a='default:dpername=1,qpername=[1,2]', t='default:fg_on=False')
        >>> result = print_results(ibs, testres)
        >>> print(result)
    """

    (cfg_list, cfgx2_cfgresinfo, testnameid, cfgx2_lbl,
     cfgx2_qreq_) = ut.dict_take(testres.__dict__, [
         'cfg_list', 'cfgx2_cfgresinfo', 'testnameid', 'cfgx2_lbl',
         'cfgx2_qreq_'
     ])

    # cfgx2_cfgresinfo is a list of dicts of lists
    # Parse result info out of the lists
    cfgx2_nextbestranks = ut.get_list_column(cfgx2_cfgresinfo,
                                             'qx2_next_bestranks')
    cfgx2_gt_rawscores = ut.get_list_column(cfgx2_cfgresinfo,
                                            'qx2_gt_raw_score')
    cfgx2_gf_rawscores = ut.get_list_column(cfgx2_cfgresinfo,
                                            'qx2_gf_raw_score')
    #cfgx2_aveprecs       = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_avepercision')

    cfgx2_scorediffs = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_scorediff')
    #cfgx2_gt_raw_score   = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')

    column_lbls = [
        ut.remove_chars(ut.remove_vowels(lbl), [' ', ',']) for lbl in cfgx2_lbl
    ]

    scorediffs_mat = np.array(ut.replace_nones(cfgx2_scorediffs, np.nan))

    print(' --- PRINT RESULTS ---')
    print(' use --rank-lt-list=1,5 to specify X_LIST')
    if True:
        # Num of ranks less than to score
        X_LIST = testres.get_X_LIST()
        #X_LIST = [1, 5]

        #nConfig = len(cfg_list)
        #nQuery = len(testres.qaids)
        cfgx2_nQuery = list(map(len, testres.cfgx2_qaids))
        #cfgx2_qx2_ranks = testres.get_infoprop_list('qx2_bestranks')
        #--------------------

        # A positive scorediff indicates the groundtruth was better than the
        # groundfalse scores
        istrue_list = [scorediff > 0 for scorediff in scorediffs_mat]
        isfalse_list = [~istrue for istrue in istrue_list]

        #------------
        # Build Colscore
        nLessX_dict = testres.get_nLessX_dict()

        #------------
        best_rankscore_summary = []
        #to_intersect_list = []
        # print each configs scores less than X=thresh
        for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
            max_nLessX = cfgx2_nLessX.max()
            bestX_cfgx_list = np.where(cfgx2_nLessX == max_nLessX)[0]
            best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestX_cfgx_list)
            # FIXME
            best_rankscore += rankscore_str(X, max_nLessX,
                                            cfgx2_nQuery[bestX_cfgx_list[0]])
            best_rankscore_summary += [best_rankscore]
            #to_intersect_list.append(ut.take(cfgx2_lbl, max_nLessX))

        #intersected = to_intersect_list[0] if len(to_intersect_list) > 0 else []
        #for ix in range(1, len(to_intersect_list)):
        #    intersected = np.intersect1d(intersected, to_intersect_list[ix])

    #if False:
    #    #gt_raw_score_mat = np.vstack(cfgx2_gt_raw_score).T

    #    #rank_mat = testres.get_rank_mat()

    #    #------------
    #    # Build row lbls
    #    if False:
    #        qx2_lbl = np.array([
    #            'qx=%d) q%s ' % (qx, ibsfuncs.aidstr(testres.qaids[qx], ibs=ibs, notes=True))
    #            for qx in range(nQuery)])

    #    #------------
    #    # Build Colscore and hard cases
    #    if False:
    #        qx2_min_rank = []
    #        qx2_argmin_rank = []
    #        new_hard_qaids = []
    #        new_hardtup_list = []

    #        for qx in range(nQuery):
    #            ranks = rank_mat[qx]
    #            valid_ranks = ranks[ranks >= 0]
    #            min_rank = ranks.min() if len(valid_ranks) > 0 else -3
    #            bestCFG_X = np.where(ranks == min_rank)[0]
    #            qx2_min_rank.append(min_rank)
    #            # Find the best rank over all configurations
    #            qx2_argmin_rank.append(bestCFG_X)

    #@ut.memoize
    #def get_new_hard_qx_list(testres):
    #    """ Mark any query as hard if it didnt get everything correct """
    #    rank_mat = testres.get_rank_mat()
    #    is_new_hard_list = rank_mat.max(axis=1) > 0
    #    new_hard_qx_list = np.where(is_new_hard_list)[0]
    #    return new_hard_qx_list

    #        new_hard_qx_list = testres.get_new_hard_qx_list()

    #        for qx in new_hard_qx_list:
    #            # New list is in aid format instead of cx format
    #            # because you should be copying and pasting it
    #            notes = ' ranks = ' + str(rank_mat[qx])
    #            qaid = testres.qaids[qx]
    #            name = ibs.get_annot_names(qaid)
    #            new_hardtup_list += [(qaid, name + " - " + notes)]
    #            new_hard_qaids += [qaid]

    @ut.argv_flag_dec
    def intersect_hack():
        failed = testres.rank_mat > 0
        colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
        #failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
        #failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
        failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
        failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
        gt_aids1 = ibs.get_annot_groundtruth(
            failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
        gt_aids2 = ibs.get_annot_groundtruth(
            failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)

        qaids_expt = failed_col1_only_aids
        gt_avl_aids1 = ut.flatten(gt_aids1)
        gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))

        ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
        ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
        #jsontext = ut.to_json({
        #    'qaids': list(qaids_expt),
        #    'dinclude_aids1': list(gt_aids_expt1),
        #    'dinclude_aids2': list(gt_aids_expt2),
        #})
        #annotation_configs.varysize_pzm
        #from ibeis.expt import annotation_configs

        acfg = testres.acfg_list[0]
        import copy
        acfg1 = copy.deepcopy(acfg)
        acfg2 = copy.deepcopy(acfg)
        acfg1['qcfg']['min_pername'] = None
        acfg2['qcfg']['min_pername'] = None
        acfg1['dcfg']['min_pername'] = None
        acfg2['dcfg']['min_gt_per_name'] = None

        acfg1['qcfg']['default_aids'] = qaids_expt
        acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
        acfg2['qcfg']['default_aids'] = qaids_expt
        acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2

        from ibeis.init import filter_annots
        from ibeis.expt import experiment_helpers

        annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
        annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)

        acfg_name_list = dict(  # NOQA
            acfg_list=[acfg1, acfg2],
            expanded_aids_list=[annots1, annots2],
        )
        test_cfg_name_list = ['candidacy_k']
        cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(
            test_cfg_name_list, ibs=ibs)

        t1, t2 = testres_list  # NOQA

    #ut.embed()
    #intersect_hack()

    #@ut.argv_flag_dec
    #def print_rowlbl():
    #    print('=====================')
    #    print('[harn] Row/Query Labels: %s' % testnameid)
    #    print('=====================')
    #    print('[harn] queries:\n%s' % '\n'.join(qx2_lbl))
    #print_rowlbl()
    #------------

    @ut.argv_flag_dec
    def print_collbl():
        print('=====================')
        print('[harn] Col/Config Labels: %s' % testnameid)
        print('=====================')
        enum_cfgx2_lbl = [
            '%2d) %s' % (count, cfglbl)
            for count, cfglbl in enumerate(cfgx2_lbl)
        ]
        print('[harn] cfglbl:\n%s' % '\n'.join(enum_cfgx2_lbl))

    print_collbl()

    #------------

    @ut.argv_flag_dec
    def print_cfgstr():
        print('=====================')
        print('[harn] Config Strings: %s' % testnameid)
        print('=====================')
        cfgstr_list = [query_cfg.get_cfgstr() for query_cfg in cfg_list]
        enum_cfgstr_list = [
            '%2d) %s' % (count, cfgstr)
            for count, cfgstr in enumerate(cfgstr_list)
        ]
        print('\n[harn] cfgstr:\n%s' % '\n'.join(enum_cfgstr_list))

    print_cfgstr()

    #------------

    #@ut.argv_flag_dec
    #def print_rowscore():
    #    print('=======================')
    #    print('[harn] Scores per Query: %s' % testnameid)
    #    print('=======================')
    #    for qx in range(nQuery):
    #        bestCFG_X = qx2_argmin_rank[qx]
    #        min_rank = qx2_min_rank[qx]
    #        minimizing_cfg_str = ut.indentjoin(cfgx2_lbl[bestCFG_X], '\n  * ')
    #        #minimizing_cfg_str = str(bestCFG_X)

    #        print('-------')
    #        print(qx2_lbl[qx])
    #        print(' best_rank = %d ' % min_rank)
    #        if len(cfgx2_lbl) != 1:
    #            print(' minimizing_cfg_x\'s = %s ' % minimizing_cfg_str)
    #print_rowscore()

    #------------

    #@ut.argv_flag_dec
    #def print_row_ave_precision():
    #    print('=======================')
    #    print('[harn] Scores per Query: %s' % testnameid)
    #    print('=======================')
    #    for qx in range(nQuery):
    #        aveprecs = ', '.join(['%.2f' % (aveprecs[qx],) for aveprecs in cfgx2_aveprecs])
    #        print('-------')
    #        print(qx2_lbl[qx])
    #        print(' aveprecs = %s ' % aveprecs)
    #print_row_ave_precision()

    ##------------

    #@ut.argv_flag_dec
    #def print_hardcase():
    #    print('--- hard new_hardtup_list (w.r.t these configs): %s' % testnameid)
    #    print('\n'.join(map(repr, new_hardtup_list)))
    #    print('There are %d hard cases ' % len(new_hardtup_list))
    #    aid_list = [aid_notes[0] for aid_notes in new_hardtup_list]
    #    name_list = ibs.get_annot_names(aid_list)
    #    name_set = set(name_list)
    #    print(sorted(aid_list))
    #    print('Names: %r' % (name_set,))
    #print_hardcase()
    #default=not ut.get_argflag('--allhard'))

    #------------

    #@ut.argv_flag_dec
    #def echo_hardcase():
    #    print('--- hardcase commandline: %s' % testnameid)
    #    # Show index for current query where hardids reside
    #    #print('--index ' + (' '.join(map(str, new_hard_qx_list))))
    #    #print('--take new_hard_qx_list')
    #    #hardaids_str = ' '.join(map(str, ['    ', '--qaid'] + new_hard_qaids))
    #    hardaids_str = ' '.join(map(str, ['    ', '--set-aids-as-hard'] + new_hard_qaids))
    #    print(hardaids_str)
    ##echo_hardcase(default=not ut.get_argflag('--allhard'))
    #echo_hardcase()

    #@ut.argv_flag_dec
    #def print_bestcfg():
    #    print('==========================')
    #    print('[harn] Best Configurations: %s' % testnameid)
    #    print('==========================')
    #    # print each configs scores less than X=thresh
    #    for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
    #        max_LessX = cfgx2_nLessX.max()
    #        bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
    #        best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
    #        best_rankscore += rankscore_str(X, max_LessX, nQuery)
    #        cfglbl_list = cfgx2_lbl[bestCFG_X]

    #        best_rankcfg = format_cfgstr_list(cfglbl_list)
    #        #indent('\n'.join(cfgstr_list), '    ')
    #        print(best_rankscore)
    #        print(best_rankcfg)
    #    print('[cfg*]  %d cfg(s) are the best of %d total cfgs' % (len(intersected), nConfig))
    #    print(format_cfgstr_list(intersected))
    #print_bestcfg()

    #------------

    #@ut.argv_flag_dec
    #def print_gtscore():
    #    # Prints best ranks
    #    print('gtscore_mat: %s' % testnameid)
    #    print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
    #    header = (' labled rank matrix: rows=queries, cols=cfgs:')
    #    print('\n'.join(cfgx2_lbl))
    #    column_list = gt_raw_score_mat.T
    #    print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
    #                            column_lbls=column_lbls, header=header,
    #                            transpose=False,
    #                            use_lbl_width=len(cfgx2_lbl) < 5))
    #print_gtscore()

    #------------

    #@ut.argv_flag_dec
    #def print_best_rankmat():
    #    # Prints best ranks
    #    print('-------------')
    #    print('RankMat: %s' % testnameid)
    #    print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
    #    header = (' labled rank matrix: rows=queries, cols=cfgs:')
    #    print('\n'.join(cfgx2_lbl))
    #    column_list = rank_mat.T
    #    print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
    #                            column_lbls=column_lbls, header=header,
    #                            transpose=False,
    #                            use_lbl_width=len(cfgx2_lbl) < 5))
    #print_best_rankmat()

    #@ut.argv_flag_dec
    #def print_diffmat():
    #    # score differences over configs
    #    print('-------------')
    #    print('Diffmat: %s' % testnameid)
    #    diff_matstr = get_diffmat_str(rank_mat, testres.qaids, nConfig)
    #    print(diff_matstr)
    #print_diffmat()

    #@ut.argv_flag_dec
    #def print_rankhist_time():
    #    print('A rank histogram is a dictionary. '
    #          'The keys denote the range of the ranks that the values fall in')
    #    # TODO: rectify this code with other hist code

    #    config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
    #    config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()

    #    _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
    #    for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
    #        #full_cfgstr = testres.cfgx2_qreq_[cfgx].get_full_cfgstr()
    #        #ut.print_dict(ut.dict_hist(ranks), 'rank histogram', sorted_=True)
    #        # find the qxs that belong to each bin
    #        aid_list1 = testres.qaids
    #        aid_list2 = qx2_gt_aid
    #        ibs.assert_valid_aids(aid_list1)
    #        ibs.assert_valid_aids(aid_list2)
    #        timedelta_list = ibs.get_annot_pair_timdelta(aid_list1, aid_list2)
    #        #timedelta_str_list = [ut.get_posix_timedelta_str2(delta)
    #        #                      for delta in timedelta_list]

    #        bin_edges = testres.get_rank_histogram_bin_edges()
    #        timedelta_groups = ut.dict_take(ut.group_items(timedelta_list, config_binxs), np.arange(len(bin_edges)), [])

    #        timedelta_stats = [ut.get_stats(deltas, use_nan=True, datacast=ut.get_posix_timedelta_str2) for deltas in timedelta_groups]
    #        print('Time statistics for each rank range:')
    #        print(ut.dict_str(dict(zip(bin_edges, timedelta_stats)), sorted_=True))
    #print_rankhist_time()

    #@ut.argv_flag_dec
    #def print_rankhist():
    #    print('A rank histogram is a dictionary. '
    #          'The keys denote the range of the ranks that the values fall in')
    #    # TODO: rectify this code with other hist code

    #    config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
    #    config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()

    #    _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
    #    for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
    #        print('Frequency of rank ranges:')
    #        ut.print_dict(agg_hist_dict, 'agg rank histogram', sorted_=True)
    #print_rankhist()

    #------------
    # Print summary
    #print(' --- SUMMARY ---')

    #------------

    #@ut.argv_flag_dec
    #def print_colmap():
    #    print('==================')
    #    print('[harn] mAP per Config: %s (sorted by mAP)' % testnameid)
    #    print('==================')
    #    cfgx2_mAP = np.array([aveprec_list.mean() for aveprec_list in cfgx2_aveprecs])
    #    sortx = cfgx2_mAP.argsort()
    #    for cfgx in sortx:
    #        print('[mAP] cfgx=%r) mAP=%.3f -- %s' % (cfgx, cfgx2_mAP[cfgx], cfgx2_lbl[cfgx]))
    #    #print('L___ Scores per Config ___')
    #print_colmap()
    #------------

    @ut.argv_flag_dec_true
    def print_colscore():
        print('==================')
        print('[harn] Scores per Config: %s' % testnameid)
        print('==================')
        #for cfgx in range(nConfig):
        #    print('[score] %s' % (cfgx2_lbl[cfgx]))
        #    for X in X_LIST:
        #        nLessX_ = nLessX_dict[int(X)][cfgx]
        #        print('        ' + rankscore_str(X, nLessX_, nQuery))
        print('\n[harn] ... sorted scores')
        for X in X_LIST:
            print('\n[harn] Sorted #ranks < %r scores' % (X))
            sortx = np.array(nLessX_dict[int(X)]).argsort()
            #frac_list = (nLessX_dict[int(X)] / cfgx2_nQuery)[:, None]
            #print('cfgx2_nQuery = %r' % (cfgx2_nQuery,))
            #print('frac_list = %r' % (frac_list,))
            #print('Pairwise Difference: ' + str(ut.safe_pdist(frac_list, metric=ut.absdiff)))
            for cfgx in sortx:
                nLessX_ = nLessX_dict[int(X)][cfgx]
                rankstr = rankscore_str(X,
                                        nLessX_,
                                        cfgx2_nQuery[cfgx],
                                        withlbl=False)
                print('[score] %s --- %s' % (rankstr, cfgx2_lbl[cfgx]))

    print_colscore()

    #------------

    ut.argv_flag_dec(print_latexsum)(ibs, testres)

    @ut.argv_flag_dec
    def print_next_rankmat():
        # Prints nextbest ranks
        print('-------------')
        print('NextRankMat: %s' % testnameid)
        header = (' top false rank matrix: rows=queries, cols=cfgs:')
        print('\n'.join(cfgx2_lbl))
        column_list = cfgx2_nextbestranks
        print(
            ut.make_csv_table(column_list,
                              row_lbls=testres.qaids,
                              column_lbls=column_lbls,
                              header=header,
                              transpose=False,
                              use_lbl_width=len(cfgx2_lbl) < 5))

    print_next_rankmat()

    #------------

    @ut.argv_flag_dec
    def print_scorediff_mat():
        # Prints nextbest ranks
        print('-------------')
        print('ScoreDiffMat: %s' % testnameid)
        header = (
            ' score difference between top true and top false: rows=queries, cols=cfgs:'
        )
        print('\n'.join(cfgx2_lbl))
        column_list = cfgx2_scorediffs
        column_type = [float] * len(column_list)
        print(
            ut.make_csv_table(column_list,
                              row_lbls=testres.qaids,
                              column_lbls=column_lbls,
                              column_type=column_type,
                              header=header,
                              transpose=False,
                              use_lbl_width=len(cfgx2_lbl) < 5))

    print_scorediff_mat(alias_flags=['--sdm'])

    #------------
    def jagged_stats_info(arr_, lbl, col_lbls):
        arr = ut.recursive_replace(arr_, np.inf, np.nan)
        # Treat infinite as nan
        stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True)
        sel_stat_dict, sel_indices = ut.find_interesting_stats(
            stat_dict, col_lbls)
        sel_col_lbls = ut.take(col_lbls, sel_indices)
        statstr_kw = dict(precision=3, newlines=True, lbl=lbl, align=True)
        stat_str = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw)
        sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw)
        sel_stat_str = 'sel_col_lbls = %s' % (
            ut.list_str(sel_col_lbls), ) + '\n' + sel_stat_str
        return stat_str, sel_stat_str

    @ut.argv_flag_dec
    def print_confusion_stats():
        """
        CommandLine:
            python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
        """
        # Prints nextbest ranks
        print('-------------')
        print('ScoreDiffMatStats: %s' % testnameid)
        print('column_lbls = %r' % (column_lbls, ))

        #cfgx2_gt_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
        #cfgx2_gf_rawscores  = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')

        gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
        gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)

        tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
        fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
        tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
        fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)

        tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(
            tp_rawscores, 'tp_rawscores', cfgx2_lbl)
        fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(
            fp_rawscores, 'fp_rawscores', cfgx2_lbl)
        tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(
            tn_rawscores, 'tn_rawscores', cfgx2_lbl)
        fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(
            fn_rawscores, 'fn_rawscores', cfgx2_lbl)

        #print(tp_rawscores_str)
        #print(fp_rawscores_str)
        #print(tn_rawscores_str)
        #print(fn_rawscores_str)

        print(tp_rawscore_statstr)
        print(fp_rawscore_statstr)
        print(tn_rawscore_statstr)
        print(fn_rawscore_statstr)

    print_confusion_stats(alias_flags=['--cs'])

    ut.argv_flag_dec_true(testres.print_percent_identification_success)()

    sumstrs = []
    sumstrs.append('')
    sumstrs.append('||===========================')
    sumstrs.append('|| [cfg*] SUMMARY: %s' % testnameid)
    sumstrs.append('||---------------------------')
    sumstrs.append(ut.joins('\n|| ', best_rankscore_summary))
    sumstrs.append('||===========================')
    summary_str = '\n' + '\n'.join(sumstrs) + '\n'
    #print(summary_str)
    ut.colorprint(summary_str, 'blue')

    print('To enable all printouts add --print-all to the commandline')