Example #1
0
def delete_features(ibs, feat_rowid_list, config2_=None):
    """ deletes images from the database that belong to fids"""
    from ibeis.algo.preproc import preproc_feat
    if ut.VERBOSE:
        print('[ibs] deleting %d features' % len(feat_rowid_list))
    # remove non-sql external dependeinces of these rowids
    preproc_feat.on_delete(ibs, feat_rowid_list)
    # remove dependants of these rowids
    featweight_rowid_list = ut.filter_Nones(ibs.get_feat_featweight_rowids(feat_rowid_list, config2_=config2_, ensure=False))
    ibs.delete_featweight(featweight_rowid_list)
    # remove these rowids
    ibs.dbcache.delete_rowids(const.FEATURE_TABLE, feat_rowid_list)
    num_deleted = len(ut.filter_Nones(feat_rowid_list))
    return num_deleted
def delete_annot_featweight(ibs, aid_list, config2_=None):
    """ annot.featweight.delete(aid_list)

    Args:
        aid_list

    TemplateInfo:
        Tdeleter_rl_depenant
        root = annot
        leaf = featweight

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.control._autogen_featweight_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_ibs()
        >>> aid_list = ibs._get_all_aids()[:1]
        >>> # Ensure there are  some leafs to delete
        >>> featweight_rowid_list = ibs.get_annot_featweight_rowids(aid_list, config2_=config2_, ensure=True)
        >>> num_deleted1 = ibs.delete_annot_featweight(aid_list, config2_=config2_)
        >>> num_deleted2 = ibs.delete_annot_featweight(aid_list, config2_=config2_)
        >>> # The first delete should remove everything
        >>> ut.assert_eq(num_deleted1, len(featweight_rowid_list))
        >>> # The second delete should have removed nothing
        >>> ut.assert_eq(num_deleted2, 0)
    """
    if ut.VERBOSE:
        print('[ibs] deleting %d annots leaf nodes' % len(aid_list))
    # Delete any dependants
    _featweight_rowid_list = ibs.get_annot_featweight_rowids(
        aid_list, config2_=config2_, ensure=False)
    featweight_rowid_list = ut.filter_Nones(_featweight_rowid_list)
    num_deleted = ibs.delete_featweight(featweight_rowid_list)
    return num_deleted
def delete_annotmatch(ibs, annotmatch_rowid_list):
    r""" annotmatch.delete(annotmatch_rowid_list)

    delete annotmatch rows

    Args:
        annotmatch_rowid_list

    Returns:
        int: num_deleted

    TemplateInfo:
        Tdeleter_native_tbl
        tbl = annotmatch

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.control.manual_annotmatch_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_annotmatch()
        >>> annotmatch_rowid_list = ibs._get_all_annotmatch_rowids()[:2]
        >>> num_deleted = ibs.delete_annotmatch(annotmatch_rowid_list)
        >>> print('num_deleted = %r' % (num_deleted,))
    """
    #from ibeis.algo.preproc import preproc_annotmatch
    # NO EXTERN IMPORT
    if ut.VERBOSE:
        print('[ibs] deleting %d annotmatch rows' % len(annotmatch_rowid_list))
    # Prepare: Delete externally stored data (if any)
    #preproc_annotmatch.on_delete(ibs, annotmatch_rowid_list, config2_=config2_)
    # NO EXTERN DELETE
    # Finalize: Delete self
    ibs.db.delete_rowids(const.ANNOTMATCH_TABLE, annotmatch_rowid_list)
    num_deleted = len(ut.filter_Nones(annotmatch_rowid_list))
    return num_deleted
def delete_annotmatch(ibs, annotmatch_rowid_list, config2_=None):
    r""" annotmatch.delete(annotmatch_rowid_list)

    delete annotmatch rows

    Args:
        annotmatch_rowid_list

    Returns:
        int: num_deleted

    TemplateInfo:
        Tdeleter_native_tbl
        tbl = annotmatch

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.control._autogen_annotmatch_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_annotmatch()
        >>> annotmatch_rowid_list = ibs._get_all_annotmatch_rowids()[:2]
        >>> num_deleted = ibs.delete_annotmatch(annotmatch_rowid_list)
        >>> print('num_deleted = %r' % (num_deleted,))
    """
    #from ibeis.algo.preproc import preproc_annotmatch
    # NO EXTERN IMPORT
    if ut.VERBOSE:
        print('[ibs] deleting %d annotmatch rows' % len(annotmatch_rowid_list))
    # Prepare: Delete externally stored data (if any)
    #preproc_annotmatch.on_delete(ibs, annotmatch_rowid_list, config2_=config2_)
    # NO EXTERN DELETE
    # Finalize: Delete self
    ibs.db.delete_rowids(const.ANNOTMATCH_TABLE, annotmatch_rowid_list)
    num_deleted = len(ut.filter_Nones(annotmatch_rowid_list))
    return num_deleted
Example #5
0
def delete_annot_feats(ibs, aid_list, config2_=None):
    """ annot.feat.delete(aid_list)

    Args:
        aid_list

    TemplateInfo:
        Tdeleter_rl_depenant
        root = annot
        leaf = feat

    CommandLine:
        python -m ibeis.control.manual_feat_funcs --test-delete_annot_feats
        python -m ibeis.control.manual_feat_funcs --test-delete_annot_feats --verb-control

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.control.manual_feat_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_ibs()
        >>> aid_list = ibs._get_all_aids()[:1]
        >>> fids_list = ibs.get_annot_feat_rowids(aid_list, config2_=config2_, ensure=True)
        >>> num_deleted1 = ibs.delete_annot_feats(aid_list, config2_=config2_)
        >>> ut.assert_eq(num_deleted1, len(fids_list))
        >>> num_deleted2 = ibs.delete_annot_feats(aid_list, config2_=config2_)
        >>> ut.assert_eq(num_deleted2, 0)
    """
    if ut.VERBOSE:
        print('[ibs] deleting %d annots leaf nodes' % len(aid_list))
    # Delete any dependants
    _feat_rowid_list = ibs.get_annot_feat_rowids(
        aid_list, config2_=config2_, ensure=False)
    feat_rowid_list = ut.filter_Nones(_feat_rowid_list)
    num_deleted = ibs.delete_feat(feat_rowid_list)
    return num_deleted
Example #6
0
def delete_chip_feats(ibs, chip_rowid_list, config2_=None):
    """ chip.feat.delete(chip_rowid_list)

    Args:
        chip_rowid_list

    TemplateInfo:
        Tdeleter_rl_depenant
        parent = chip
        leaf = feat

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.control.manual_feat_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_ibs()
        >>> chip_rowid_list = ibs._get_all_chip_rowids()[::3]
        >>> ibs.delete_chip_feats(chip_rowid_list, config2_=config2_)
    """
    if ut.VERBOSE:
        print('[ibs] deleting %d chips leaf nodes' % len(chip_rowid_list))
    # Delete any dependants
    _feat_rowid_list = ibs.get_chip_feat_rowid(
        chip_rowid_list, config2_=config2_, ensure=False)
    feat_rowid_list = ut.filter_Nones(_feat_rowid_list)
    num_deleted = ibs.delete_feat(feat_rowid_list)
    return num_deleted
def delete_gar(ibs, gar_rowid_list, config2_=None):
    """ gar.delete(gar_rowid_list)

    delete gar rows

    Args:
        gar_rowid_list

    Returns:
        int: num_deleted

    TemplateInfo:
        Tdeleter_native_tbl
        tbl = gar

    Example:
        >>> # DISABLE_DOCTEST
        >>> from wbia.control.manual_garelate_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_ibs()
        >>> gar_rowid_list = ibs._get_all_gar_rowids()[:2]
        >>> num_deleted = ibs.delete_gar(gar_rowid_list)
        >>> print('num_deleted = %r' % (num_deleted,))
    """
    # from wbia.algo.preproc import preproc_gar
    # NO EXTERN IMPORT
    if ut.VERBOSE:
        logger.info('[ibs] deleting %d gar rows' % len(gar_rowid_list))
    # Prepare: Delete externally stored data (if any)
    # preproc_gar.on_delete(ibs, gar_rowid_list, config2_=config2_)
    # NO EXTERN DELETE
    # Finalize: Delete self
    ibs.db.delete_rowids(const.GA_RELATION_TABLE, gar_rowid_list)
    num_deleted = len(ut.filter_Nones(gar_rowid_list))
    return num_deleted
Example #8
0
    def _get_cm_edge_data(infr, edges, cm_list=None):
        symmetric = True

        if cm_list is None:
            cm_list = infr.cm_list
        # Find scores for the edges that exist in the graph
        edge_to_data = ut.ddict(dict)
        aid_to_cm = {cm.qaid: cm for cm in cm_list}
        for u, v in edges:
            if symmetric:
                u, v = e_(u, v)
            cm1 = aid_to_cm.get(u, None)
            cm2 = aid_to_cm.get(v, None)
            scores = []
            ranks = []
            for cm in ut.filter_Nones([cm1, cm2]):
                for aid in [u, v]:
                    idx = cm.daid2_idx.get(aid, None)
                    if idx is None:
                        continue
                    score = cm.annot_score_list[idx]
                    rank = cm.get_annot_ranks([aid])[0]
                    scores.append(score)
                    ranks.append(rank)
            if len(scores) == 0:
                score = None
                rank = None
            else:
                # Choose whichever one gave the best score
                idx = vt.safe_argmax(scores, nans=False)
                score = scores[idx]
                rank = ranks[idx]
            edge_to_data[(u, v)]['score'] = score
            edge_to_data[(u, v)]['rank'] = rank
        return edge_to_data
def delete_feat_featweight(ibs, feat_rowid_list, config2_=None):
    """ feat.featweight.delete(feat_rowid_list)

    Args:
        feat_rowid_list

    Returns:
         int: num_deleted

    TemplateInfo:
        Tdeleter_rl_depenant
        parent = feat
        leaf = featweight

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.control._autogen_featweight_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_ibs()
        >>> feat_rowid_list = ibs._get_all_feat_rowids()[:2]
        >>> ibs.delete_feat_featweight(feat_rowid_list, config2_=config2_)
    """
    if ut.VERBOSE:
        print('[ibs] deleting %d feats leaf nodes' % len(feat_rowid_list))
    # Delete any dependants
    _featweight_rowid_list = ibs.get_feat_featweight_rowids(
        feat_rowid_list, config2_=config2_, ensure=False)
    featweight_rowid_list = ut.filter_Nones(_featweight_rowid_list)
    num_deleted = ibs.delete_featweight(featweight_rowid_list)
    return num_deleted
Example #10
0
def ingest_rawdata(ibs, ingestable, localize=False):
    """
    Ingests rawdata into an ibeis database.

    if ingest_type == 'named_folders':
        Converts folder structure where folders = name, to ibsdb
    if ingest_type == 'named_images':
        Converts imgname structure where imgnames = name_id.ext, to ibsdb
    """
    img_dir         = ingestable.img_dir
    ingest_type     = ingestable.ingest_type
    fmtkey          = ingestable.fmtkey
    adjust_percent  = ingestable.adjust_percent
    postingest_func = ingestable.postingest_func
    print('[ingest] ingesting rawdata: img_dir=%r, injest_type=%r' % (img_dir, ingest_type))
    # Get images in the image directory
    gpath_list  = list_ingestable_images(img_dir, recursive=True)
    # Parse structure for image names
    if ingest_type == 'named_folders':
        name_list = ibsfuncs.get_name_text_from_parent_folder(gpath_list, img_dir, fmtkey)
        pass
    if ingest_type == 'named_images':
        name_list = ibsfuncs.get_name_text_from_gnames(gpath_list, img_dir, fmtkey)
    if ingest_type == 'unknown':
        name_list = [constants.INDIVIDUAL_KEY for _ in xrange(len(gpath_list))]

    # Add Images
    gid_list_ = ibs.add_images(gpath_list)
    # <DEBUG>
    #print('added: ' + utool.indentjoin(map(str, zip(gid_list_, gpath_list))))
    unique_gids = list(set(gid_list_))
    print("[ingest] Length gid list: %d" % len(gid_list_))
    print("[ingest] Length unique gid list: %d" % len(unique_gids))
    assert len(gid_list_) == len(gpath_list)
    for gid in gid_list_:
        if gid is None:
            print('[ingest] big fat warning')
    # </DEBUG>
    gid_list = utool.filter_Nones(gid_list_)
    unique_gids, unique_names, unique_notes = ibsfuncs.resolve_name_conflicts(
        gid_list, name_list)
    # Add ANNOTATIONs with names and notes
    aid_list = ibsfuncs.use_images_as_annotations(ibs, unique_gids,
                                                  name_list=unique_names,
                                                  notes_list=unique_notes,
                                                  adjust_percent=adjust_percent)
    if localize:
        ibsfuncs.localize_images(ibs)
    if postingest_func is not None:
        postingest_func(ibs)
    # Print to show success
    #ibs.print_image_table()
    #ibs.print_tables()
    #ibs.print_annotation_table()
    #ibs.print_alr_table()
    #ibs.print_lblannot_table()
    #ibs.print_image_table()
    return aid_list
Example #11
0
def config_graph_subattrs(cfg, depc):
    # TODO: if this hack is fully completed need a way of getting the
    # full config belonging to both chip + feat
    # cfg = request.config.feat_cfg
    import networkx as netx
    tablename = ut.invert_dict(depc.configclass_dict)[cfg.__class__]
    #tablename = cfg.get_config_name()
    ancestors = netx.dag.ancestors(depc.graph, tablename)
    subconfigs_ = ut.dict_take(depc.configclass_dict, ancestors, None)
    subconfigs = ut.filter_Nones(subconfigs_)  # NOQA
Example #12
0
 def delete_rows(table, rowid_list):
     #from ibeis.algo.preproc import preproc_feat
     if table.on_delete is not None:
         table.on_delete()
     if ut.VERBOSE:
         print('deleting %d rows' % len(rowid_list))
     # Finalize: Delete table
     table.db.delete_rowids(table.tablename, rowid_list)
     num_deleted = len(ut.filter_Nones(rowid_list))
     return num_deleted
Example #13
0
def config_graph_subattrs(cfg, depc):
    # TODO: if this hack is fully completed need a way of getting the
    # full config belonging to both chip + feat
    # cfg = request.config.feat_cfg
    import networkx as netx
    tablename = ut.invert_dict(depc.configclass_dict)[cfg.__class__]
    #tablename = cfg.get_config_name()
    ancestors = netx.dag.ancestors(depc.graph, tablename)
    subconfigs_ = ut.dict_take(depc.configclass_dict, ancestors, None)
    subconfigs = ut.filter_Nones(subconfigs_)  # NOQA
def get_review_rowids_between(ibs, aids1, aids2=None, method=None):
    """
    Find staging rowids between sets of aids

    Doctest:
        >>> from wbia.control.manual_review_funcs import *
        >>> import wbia
        >>> ibs = wbia.opendb('PZ_MTEST')
        >>> aids1 = aids2 = [1, 2, 3, 4, 5, 6]
        >>> rowids_between = ibs.get_review_rowids_between
        >>> ids1 = sorted(rowids_between(aids1, aids2, method=1))
        >>> ids2 = sorted(rowids_between(aids1, aids2, method=2))
        >>> assert len(ub.find_duplicates(ids1)) == 0
        >>> assert len(ub.find_duplicates(ids2)) == 0
        >>> assert ids1 == ids2
    """
    if aids2 is None:
        aids2 = aids1
    if method is None:
        if len(aids1) * len(aids2) > 5000:
            method = 1
        else:
            method = 2

    if method == 1:
        # Strategy 1: get all existing rows and see what intersects
        # This is better when the enumerated set of rows would be larger than
        # the database size
        rowids11 = set(ut.flatten(ibs.get_review_rowids_from_aid1(aids1)))
        rowids12 = set(ut.flatten(ibs.get_review_rowids_from_aid2(aids1)))
        if aids1 is aids2:
            rowids = list(reduce(set.intersection, [rowids11, rowids12]))
        else:
            rowids21 = set(ut.flatten(ibs.get_review_rowids_from_aid1(aids2)))
            rowids22 = set(ut.flatten(ibs.get_review_rowids_from_aid2(aids2)))
            rowids = list(
                reduce(set.intersection, [rowids11, rowids12, rowids21, rowids22])
            )
    elif method == 2:
        # Strategy 2: enumerate what rows could exist and see what does exist
        # This is better when the enumerated set of rows would be smaller than
        # the database size
        edges = list(ut.product_nonsame(aids1, aids2))
        if len(edges) == 0:
            rowids = []
        else:
            rowids = ibs.get_review_rowids_from_edges(edges, directed=True)
            if rowids is None:
                rowids = []
            rowids = ut.filter_Nones(rowids)
            rowids = ut.flatten(rowids)
    else:
        raise ValueError('no method=%r' % (method,))
    return rowids
Example #15
0
def delete_annot_chips(ibs, aid_list, qreq_=None):
    """ Clears annotation data but does not remove the annotation """
    _cid_list = ibs.get_annot_chip_rowids(aid_list, ensure=False, qreq_=qreq_)
    cid_list = ut.filter_Nones(_cid_list)
    ibs.delete_chips(cid_list)
    # HACK FIX: if annot chips are None then the image thumbnail
    # will not be invalidated
    if len(_cid_list) != len(cid_list):
        aid_list_ = [aid for aid, _cid in zip(aid_list, _cid_list) if _cid is None]
        gid_list_ = ibs.get_annot_gids(aid_list_)
        ibs.delete_image_thumbs(gid_list_)
Example #16
0
def get_annotmatch_rowids_in_cliques(ibs, aids_list):
    # Equivalent call:
    #ibs.get_annotmatch_rowids_between_groups(ibs, aids_list, aids_list)
    import itertools
    ams_list = [
        ibs.get_annotmatch_rowid_from_undirected_superkey(*zip(
            *itertools.combinations(aids, 2)))
        for aids in ut.ProgIter(aids_list, lbl='loading clique am rowids')
    ]
    ams_list = [[] if ams is None else ut.filter_Nones(ams)
                for ams in ams_list]
    return ams_list
Example #17
0
def make_netx_graph_from_aids(ibs, aids_list, full=False):
    #aids_list, nid_list = ibs.group_annots_by_name(aid_list)
    unique_aids = list(ut.flatten(aids_list))

    aids1, aids2 = get_name_rowid_edges_from_aids2(ibs, aids_list)

    if not full:
        annotmatch_rowids = ibs.get_annotmatch_rowid_from_superkey(aids1, aids2)
        annotmatch_rowids = ut.filter_Nones(annotmatch_rowids)
        aids1 = ibs.get_annotmatch_aid1(annotmatch_rowids)
        aids2 = ibs.get_annotmatch_aid2(annotmatch_rowids)

    return make_netx_graph_from_aidpairs(ibs, aids1, aids2, unique_aids=unique_aids)
Example #18
0
def delete_chips(ibs, cid_list, verbose=ut.VERBOSE, qreq_=None):
    """ deletes images from the database that belong to gids"""
    from ibeis.model.preproc import preproc_chip
    if verbose:
        print('[ibs] deleting %d annotation-chips' % len(cid_list))
    # Delete chip-images from disk
    #preproc_chip.delete_chips(ibs, cid_list, verbose=verbose)
    preproc_chip.on_delete(ibs, cid_list, verbose=verbose)
    # Delete chip features from sql
    _fid_list = ibs.get_chip_fids(cid_list, ensure=False, qreq_=qreq_)
    fid_list = ut.filter_Nones(_fid_list)
    ibs.delete_features(fid_list)
    # Delete chips from sql
    ibs.dbcache.delete_rowids(const.CHIP_TABLE, cid_list)
Example #19
0
def delete_annot_chips(ibs, aid_list, config2_=None):
    r"""
    Clears annotation data (does not remove the annotation)

    RESTful:
        Method: DELETE
        URL:    /api/annot_chip/
    """
    _cid_list = ibs.get_annot_chip_rowids(aid_list, ensure=False, config2_=config2_)
    cid_list = ut.filter_Nones(_cid_list)
    ibs.delete_chips(cid_list)
    # HACK FIX: if annot chips are None then the image thumbnail
    # will not be invalidated
    if len(_cid_list) != len(cid_list):
        aid_list_ = [aid for aid, _cid in zip(aid_list, _cid_list) if _cid is None]
        gid_list_ = ibs.get_annot_gids(aid_list_)
        ibs.delete_image_thumbs(gid_list_)
Example #20
0
    def _new_repo_url(repo, host=None, user=None, reponame=None, fmt=None):
        import utool as ut

        if reponame is None:
            reponame = repo.reponame
        if host is None:
            host = "github.com"
        if fmt is None:
            fmt = "ssh"
        if host is not None:
            assert user is not None, "github needs a user"
        url_fmts = {"https": ("https://", "/"), "ssh": ("git@", ":")}
        prefix, sep = url_fmts[fmt]
        parts = [prefix, host, sep, user, "/", reponame, ".git"]
        parts = ut.filter_Nones(parts)
        url = "".join(parts)
        return url
Example #21
0
def remove_existing_fpaths(fpath_list, verbose=VERBOSE, quiet=QUIET,
                           strict=False, print_caller=PRINT_CALLER, lbl='files'):
    """ checks existance before removing. then tries to remove exisint paths """
    import utool as ut
    if print_caller:
        print(util_dbg.get_caller_name(range(1, 4)) + ' called remove_existing_fpaths')
    fpath_list_ = ut.filter_Nones(fpath_list)
    exists_list = list(map(exists, fpath_list_))
    if verbose:
        nTotal = len(fpath_list)
        nValid = len(fpath_list_)
        nExist = sum(exists_list)
        print('[util_path.remove_existing_fpaths] requesting delete of %d %s' % (nTotal, lbl))
        if nValid != nTotal:
            print('[util_path.remove_existing_fpaths] trying to delete %d/%d non None %s ' % (nValid, nTotal, lbl))
        print('[util_path.remove_existing_fpaths] %d/%d exist and need to be deleted' % (nExist, nValid))
    existing_fpath_list = ut.filter_items(fpath_list_, exists_list)
    return remove_fpaths(existing_fpath_list, verbose=verbose, quiet=quiet,
                            strict=strict, print_caller=False, lbl=lbl)
Example #22
0
 def _new_repo_url(repo, host=None, user=None, reponame=None, fmt=None):
     import utool as ut
     if reponame is None:
         reponame = repo.reponame
     if host is None:
         host = 'github.com'
     if fmt is None:
         fmt = 'ssh'
     if host is not None:
         assert user is not None, 'github needs a user'
     url_fmts = {
         'https': ('https://', '/'),
         'ssh':   ('git@', ':'),
     }
     prefix, sep = url_fmts[fmt]
     parts = [prefix, host, sep, user, '/', reponame, '.git']
     parts = ut.filter_Nones(parts)
     url = ''.join(parts)
     return url
Example #23
0
 def _new_remote_url(repo, host=None, user=None, reponame=None, fmt=None):
     import utool as ut
     if reponame is None:
         reponame = repo.reponame
     if host is None:
         host = 'github.com'
     if fmt is None:
         fmt = 'ssh'
     if host == 'github.com':
         assert user is not None, 'github needs a user'
     url_fmts = {
         'https': ('https://', '/'),
         'ssh':   ('git@', ':'),
     }
     prefix, sep = url_fmts[fmt]
     user_ = '' if user is None else user + '/'
     parts = [prefix, host, sep, user_, reponame, '.git']
     parts = ut.filter_Nones(parts)
     url = ''.join(parts)
     return url
Example #24
0
def get_default_cell_template_list(ibs):
    cells = notebook_cells

    noexample = ut.get_argflag('--noexample')

    cell_template_list = [
        cells.initialize,

        None if ibs.get_dbname() != 'humpbacks' else cells.fluke_select,

        cells.pipe_config_info,
        cells.annot_config_info,

        None if noexample else cells.timestamp_distribution,
        None if noexample else cells.example_annotations,
        None if noexample else cells.example_names,

        cells.per_annotation_accuracy,
        cells.per_name_accuracy,
        cells.timedelta_distribution,
        cells.config_overlap,
        #cells.dbsize_expt,

        None if ibs.get_dbname() == 'humpbacks' else cells.feat_score_sep,

        cells.all_annot_scoresep,
        cells.success_annot_scoresep,
        cells.easy_success_cases,
        cells.hard_success_cases,
        cells.failure_type1_cases,
        cells.failure_type2_cases,
        #cells.investigate_specific_case,
        #cells.view_intereseting_tags,

        # TODO:
        # show query chips

    ]
    cell_template_list = ut.filter_Nones(cell_template_list)
    return cell_template_list
Example #25
0
def delete_chips(ibs, cid_list, verbose=ut.VERBOSE, config2_=None):
    r"""
    deletes images from the database that belong to gids

    RESTful:
        Method: DELETE
        URL:    /api/chip/
    """
    from ibeis.algo.preproc import preproc_chip
    if verbose:
        print('[ibs] deleting %d annotation-chips' % len(cid_list))
    # Delete sql-external (on-disk) information
    preproc_chip.on_delete(ibs, cid_list)
    # Delete sql-dependencies
    fid_list = ut.filter_Nones(ibs.get_chip_feat_rowid(cid_list, config2_=config2_, ensure=False))
    aid_list = ibs.get_chip_aids(cid_list)
    gid_list = ibs.get_annot_gids(aid_list)
    ibs.delete_image_thumbs(gid_list)
    ibs.delete_annot_chip_thumbs(aid_list)
    ibs.delete_features(fid_list, config2_=config2_)
    # Delete chips from sql
    ibs.dbcache.delete_rowids(const.CHIP_TABLE, cid_list)
Example #26
0
def delete_gar(ibs, gar_rowid_list, config2_=None):
    """ gar.delete(gar_rowid_list)

    delete gar rows

    Args:
        gar_rowid_list

    Returns:
        int: num_deleted

    TemplateInfo:
        Tdeleter_native_tbl
        tbl = gar

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.control.manual_garelate_funcs import *  # NOQA
        >>> ibs, config2_ = testdata_ibs()
        >>> gar_rowid_list = ibs._get_all_gar_rowids()[:2]
        >>> num_deleted = ibs.delete_gar(gar_rowid_list)
        >>> print('num_deleted = %r' % (num_deleted,))
    """
    #from ibeis.algo.preproc import preproc_gar
    # NO EXTERN IMPORT
    if ut.VERBOSE:
        print('[ibs] deleting %d gar rows' %
              len(gar_rowid_list))
    # Prepare: Delete externally stored data (if any)
    #preproc_gar.on_delete(ibs, gar_rowid_list, config2_=config2_)
    # NO EXTERN DELETE
    # Finalize: Delete self
    ibs.db.delete_rowids(
        const.GA_RELATION_TABLE, gar_rowid_list)
    num_deleted = len(
        ut.filter_Nones(gar_rowid_list))
    return num_deleted
Example #27
0
def build_alias_map(regex_map, tag_vocab):
    """
    Constructs explicit mapping. Order of items in regex map matters.
    Items at top are given preference.

    Example:
        >>> tags_list = [['t1', 't2'], [], ['t3'], ['t4', 't5']]
        >>> tag_vocab = ut.flat_unique(*tags_list)
        >>> regex_map = [('t[3-4]', 'A9'), ('t0', 'a0')]
        >>> unmapped = list(set(tag_vocab) - set(alias_map.keys()))
    """
    import utool as ut
    import re
    alias_map = ut.odict([])
    for pats, new_tag in reversed(regex_map):
        pats = ut.ensure_iterable(pats)
        for pat in pats:
            flags = [re.match(pat, t) for t in tag_vocab]
            for old_tag in ut.compress(tag_vocab, flags):
                alias_map[old_tag] = new_tag
    identity_map = ut.take_column(regex_map, 1)
    for tag in ut.filter_Nones(identity_map):
        alias_map[tag] = tag
    return alias_map
Example #28
0
def ingest_rawdata(ibs, ingestable, localize=False):
    """
    Ingests rawdata into an ibeis database.

    if ingest_type == 'named_folders':
        Converts folder structure where folders = name, to ibsdb
    if ingest_type == 'named_images':
        Converts imgname structure where imgnames = name_id.ext, to ibsdb

    CommandLine:
        python ibeis/dbio/ingest_database.py --db seals_drop2

    Args:
        ibs (IBEISController):  ibeis controller object
        ingestable (?):
        localize (bool): (default = False)

    Returns:
        list: aid_list -  list of annotation rowids

    CommandLine:
        python -m ibeis.dbio.ingest_database --exec-ingest_rawdata
        python -m ibeis.dbio.ingest_database --exec-ingest_rawdata --db snow-leopards --imgdir /raid/raw_rsync/snow-leopards

    Example:
        >>> # SCRIPT
        >>> # General ingest script
        >>> from ibeis.dbio.ingest_database import *  # NOQA
        >>> import ibeis
        >>> dbname = ut.get_argval('--db', str, None)  # 'snow-leopards')
        >>> force_delete = ut.get_argflag(('--force_delete', '--force-delete'))
        >>> img_dir = ut.get_argval('--imgdir', type_=str, default=None)
        >>> assert img_dir is not None, 'specify img dir'
        >>> assert dbname is not None, 'specify dbname'
        >>> ingestable = Ingestable(dbname,
        >>>                   img_dir=img_dir,
        >>>                   ingest_type='unknown',
        >>>                   fmtkey=None,
        >>>                   species=None,
        >>>                   images_as_annots=False,
        >>>                   adjust_percent=0.00)
        >>> from ibeis.control import IBEISControl
        >>> dbdir = ibeis.sysres.db_to_dbdir(dbname, allow_newdir=True, use_sync=False)
        >>> ut.ensuredir(dbdir, verbose=True)
        >>> ibs = IBEISControl.request_IBEISController(dbdir)
        >>> localize = False
        >>> aid_list = ingest_rawdata(ibs, ingestable, localize)
        >>> result = ('aid_list = %s' % (str(aid_list),))
        >>> print(result)
    """
    print('[ingest_rawdata] Ingestable' + str(ingestable))

    if ingestable.zipfile is not None:
        zipfile_fpath = ut.truepath(join(ibeis.sysres.get_workdir(), ingestable.zipfile))
        ingestable.img_dir = ut.unarchive_file(zipfile_fpath)

    img_dir         = realpath(ingestable.img_dir)
    ingest_type     = ingestable.ingest_type
    fmtkey          = ingestable.fmtkey
    adjust_percent  = ingestable.adjust_percent
    species_text    = ingestable.species
    postingest_func = ingestable.postingest_func
    print('[ingest] ingesting rawdata: img_dir=%r, injest_type=%r' % (img_dir, ingest_type))
    # Get images in the image directory

    unzipped_file_base_dir = join(ibs.get_dbdir(), 'unzipped_files')

    def extract_zipfile_images(ibs, ingestable):
        import utool as ut  # NOQA
        zipfile_list = ut.glob(ingestable.img_dir, '*.zip', recursive=True)
        if len(zipfile_list) > 0:
            print('Found zipfile_list = %r' % (zipfile_list,))
            ut.ensuredir(unzipped_file_base_dir)
            for zipfile in zipfile_list:
                unziped_file_relpath = dirname(relpath(relpath(realpath(zipfile), realpath(ingestable.img_dir))))
                unzipped_file_dir = join(unzipped_file_base_dir, unziped_file_relpath)
                ut.ensuredir(unzipped_file_dir)
                ut.unzip_file(zipfile, output_dir=unzipped_file_dir, overwrite=False)
            gpath_list = ut.list_images(unzipped_file_dir, fullpath=True, recursive=True)
        else:
            gpath_list = []
        return gpath_list

    def list_images(img_dir):
        """ lists images that are not in an internal cache """
        import utool as ut  # NOQA
        ignore_list = ['_hsdb', '.hs_internals', '_ibeis_cache', '_ibsdb']
        gpath_list = ut.list_images(img_dir,
                                    fullpath=True,
                                    recursive=True,
                                    ignore_list=ignore_list)
        return gpath_list

    # FIXME ensure python3 works with this
    gpath_list1 = ut.ensure_unicode_strlist(list_images(img_dir))
    gpath_list2 = ut.ensure_unicode_strlist(extract_zipfile_images(ibs, ingestable))
    gpath_list = gpath_list1 + gpath_list2

    #__STR__ = const.__STR__
    #map(__STR__, gpath_list)
    #ut.embed()
    # Parse structure for image names
    if ingest_type == 'named_folders':
        name_list1 = get_name_texts_from_parent_folder(gpath_list1, img_dir, fmtkey)
        name_list2 = get_name_texts_from_parent_folder(gpath_list2, unzipped_file_base_dir, fmtkey)
        name_list = name_list1 + name_list2
        pass
    elif ingest_type == 'named_images':
        name_list = get_name_texts_from_gnames(gpath_list, img_dir, fmtkey)
    elif ingest_type == 'unknown':
        name_list = [const.UNKNOWN for _ in range(len(gpath_list))]
    else:
        raise NotImplementedError('unknwon ingest_type=%r' % (ingest_type,))

    # Find names likely to be the same?
    RECTIFY_NAMES_HUERISTIC = True
    if RECTIFY_NAMES_HUERISTIC:
        names = sorted(list(set(name_list)))
        splitchars = [' ', '/']

        def multisplit(str_, splitchars):
            import utool as ut
            n = [str_]
            for char in splitchars:
                n = ut.flatten([_.split(char) for _ in n])
            return n

        groupids = [multisplit(n1, splitchars)[0] for n1 in names]
        grouped_names = ut.group_items(names, groupids)
        fixed_names = {newkey: key for key, val in grouped_names.items() if len(val) > 1 for newkey in val}
        name_list = [fixed_names.get(name, name) for name in name_list]

    # Add Images
    gpath_list = [gpath.replace('\\', '/') for gpath in gpath_list]
    gid_list_ = ibs.add_images(gpath_list)
    # <DEBUG>
    #print('added: ' + ut.indentjoin(map(str, zip(gid_list_, gpath_list))))
    unique_gids = list(set(gid_list_))
    print("[ingest] Length gid list: %d" % len(gid_list_))
    print("[ingest] Length unique gid list: %d" % len(unique_gids))
    assert len(gid_list_) == len(gpath_list)
    for gid in gid_list_:
        if gid is None:
            print('[ingest] big fat warning')
    # </DEBUG>
    gid_list = ut.filter_Nones(gid_list_)
    unique_gids, unique_names, unique_notes = resolve_name_conflicts(
        gid_list, name_list)
    # Add ANNOTATIONs with names and notes
    if ingestable.images_as_annots:
        aid_list = ibs.use_images_as_annotations(unique_gids,
                                                 adjust_percent=adjust_percent)
        ibs.set_annot_names(aid_list, unique_names)
        ibs.set_annot_notes(aid_list, unique_notes)
        if species_text is not None:
            ibs.set_annot_species(aid_list, [species_text] * len(aid_list))
    if localize:
        ibs.localize_images()
    if postingest_func is not None:
        postingest_func(ibs)
Example #29
0
def get_annotmatch_rowids_between(ibs, aids1, aids2, method=None):
    """

    Example:
        >>> # ENABLE_DOCTEST
        >>> from wbia.annotmatch_funcs import *  # NOQA
        >>> import wbia
        >>> ibs = wbia.opendb('PZ_MTEST')
        >>> aids1 = aids2 = [1, 2, 3, 4, 5, 6]
        >>> rowids_between = ibs.get_annotmatch_rowids_between
        >>> ams1 = sorted(rowids_between(aids1, aids2, method=1))
        >>> ams2 = sorted(rowids_between(aids1, aids2, method=2))
        >>> assert len(ub.find_duplicates(ams1)) == 0
        >>> assert len(ub.find_duplicates(ams2)) == 0
        >>> assert sorted(ams2) == sorted(ams1)
    """
    if method is None:
        if len(aids1) * len(aids2) > 5000:
            method = 1
        else:
            method = 2
    if method == 1:
        # Strategy 1: get all existing rows and see what intersects
        # This is better when the enumerated set of rows would be larger than
        # the database size
        unflat_rowids1L = ibs.get_annotmatch_rowids_from_aid1(aids1)
        unflat_rowids1R = ibs.get_annotmatch_rowids_from_aid2(aids1)
        unflat_rowids2L = ibs.get_annotmatch_rowids_from_aid1(aids2)
        unflat_rowids2R = ibs.get_annotmatch_rowids_from_aid2(aids2)

        am_rowids1L = {
            r
            for r in ut.iflatten(unflat_rowids1L) if r is not None
        }
        am_rowids1R = {
            r
            for r in ut.iflatten(unflat_rowids1R) if r is not None
        }
        am_rowids2L = {
            r
            for r in ut.iflatten(unflat_rowids2L) if r is not None
        }
        am_rowids2R = {
            r
            for r in ut.iflatten(unflat_rowids2R) if r is not None
        }

        ams12 = am_rowids1L.intersection(am_rowids2R)
        ams21 = am_rowids2L.intersection(am_rowids1R)
        ams = sorted(ams12.union(ams21))
        # ams = sorted(am_rowids1.intersection(am_rowids2))
        # rowids2 = ibs.get_annotmatch_rowids_from_aid2(aid_list)
        # unflat_rowids1 = ibs.get_annotmatch_rowids_from_aid(aids1)
        # unflat_rowids2 = ibs.get_annotmatch_rowids_from_aid(aids2)
        # am_rowids1 = {r for r in ut.iflatten(unflat_rowids1) if r is not None}
        # am_rowids2 = {r for r in ut.iflatten(unflat_rowids2) if r is not None}
        # ams = sorted(am_rowids1.intersection(am_rowids2))
        # ams = ut.isect(am_rowids1, am_rowids2)
    elif method == 2:
        # Strategy 2: enumerate what rows could exist and see what does exist
        # This is better when the enumerated set of rows would be smaller than
        # the database size
        edges = list(ut.product_nonsame(aids1, aids2))
        if len(edges) == 0:
            ams = []
        else:
            aids1_, aids2_ = ut.listT(edges)
            # ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1_, aids2_)
            ams = ibs.get_annotmatch_rowid_from_superkey(aids1_, aids2_)
            if ams is None:
                ams = []
            ams = ut.filter_Nones(ams)
    return ams
Example #30
0
def make_score_tabular(row_lbls,
                       col_lbls,
                       values,
                       title=None,
                       out_of=None,
                       bold_best=False,
                       flip=False,
                       bigger_is_better=True,
                       multicol_lbls=None,
                       FORCE_INT=False,
                       precision=None,
                       SHORTEN_ROW_LBLS=False,
                       col_align='l',
                       col_sep='|',
                       multicol_sep='|',
                       centerline=True,
                       astable=False,
                       table_position='',
                       AUTOFIX_LATEX=True,
                       **kwargs):
    r"""
    makes a LaTeX tabular for displaying scores or errors

    Args:
        row_lbls (list of str):
        col_lbls (list of str):
        values (ndarray):
        title (str):  (default = None)
        out_of (None): (default = None)
        bold_best (bool): (default = True)
        flip (bool): (default = False)
        table_position (str) : eg '[h]'

    Returns:
        str: tabular_str

    CommandLine:
        python -m utool.util_latex --test-make_score_tabular:0 --show
        python -m utool.util_latex --test-make_score_tabular:1 --show
        python -m utool.util_latex --test-make_score_tabular:2 --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_latex import *  # NOQA
        >>> import utool as ut
        >>> row_lbls = ['config1', 'config2']
        >>> col_lbls = ['score \leq 1', 'metric2']
        >>> values = np.array([[1.2, 2], [3.2, 4]])
        >>> title = 'title'
        >>> out_of = 10
        >>> bold_best = True
        >>> flip = False
        >>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
        >>> result = tabular_str
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> render_latex_text(tabular_str)

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_latex import *  # NOQA
        >>> import utool as ut
        >>> row_lbls = ['config1']
        >>> col_lbls = ['score \leq 1', 'metric2']
        >>> values = np.array([[1.2, 2]])
        >>> title = 'title'
        >>> out_of = 10
        >>> bold_best = True
        >>> flip = False
        >>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
        >>> result = tabular_str
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> render_latex_text(tabular_str)

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_latex import *  # NOQA
        >>> import utool as ut
        >>> row_lbls = ['config1', 'config2']
        >>> col_lbls = ['score \leq 1', 'metric2', 'foobar']
        >>> multicol_lbls = [('spam', 1), ('eggs', 2)]
        >>> values = np.array([[1.2, 2, -3], [3.2, 4, -2]])
        >>> title = 'title'
        >>> out_of = 10
        >>> bold_best = True
        >>> flip = False
        >>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip, multicol_lbls=multicol_lbls)
        >>> result = tabular_str
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> render_latex_text(tabular_str)
    """
    import utool as ut
    if flip:
        bigger_is_better = not bigger_is_better
        flip_repltups = [('<=', '>'), ('>', '<='), ('\\leq', '\\gt'),
                         ('\\geq', '\\lt'), ('score', 'error')]
        col_lbls = [replace_all(lbl, flip_repltups) for lbl in col_lbls]
        if title is not None:
            title = replace_all(title, flip_repltups)
        if out_of is not None:
            values = out_of - values

    # Abbreviate based on common substrings
    common_rowlbl = None
    if SHORTEN_ROW_LBLS:
        if isinstance(row_lbls, list):
            row_lbl_list = row_lbls
        else:
            row_lbl_list = row_lbls.flatten().tolist()
        # Split the rob labels into the alg components
        #algcomp_list = [lbl.split(')_') for lbl in row_lbl_list]
        longest = long_substr(row_lbl_list)
        common_strs = []
        while len(longest) > 10:
            common_strs += [longest]
            row_lbl_list = [
                row.replace(longest, '...') for row in row_lbl_list
            ]
            longest = long_substr(row_lbl_list)
        common_rowlbl = ('...'.join(common_strs)).replace(')_', ')_\n')
        row_lbls = row_lbl_list
        if len(row_lbl_list) == 1:
            common_rowlbl = row_lbl_list[0]
            row_lbls = ['0']

    # Stack values into a tabular body
    # TODO: need ability to specify datatypes
    def ensurelist(row_values):
        try:
            return row_values.tolist()
        except AttributeError:
            return row_values

    if False:
        # Numpy formatting
        def padvec(shape=(1, 1)):
            pad = np.array([[' ' for c in range(shape[1])]
                            for r in range(shape[0])])
            return pad

        col_lbls = ensure_rowvec(col_lbls)
        row_lbls = ensure_colvec(row_lbls)
        _0 = np.vstack([padvec(), row_lbls])
        _1 = np.vstack([col_lbls, values])
        body = np.hstack([_0, _1])
        body = [[str_ for str_ in row] for row in body]
    else:
        assert len(row_lbls) == len(values)
        body = [[' '] + col_lbls]
        body += [[row_lbl] + ensurelist(row_values)
                 for row_lbl, row_values in zip(row_lbls, values)]
    #import utool as ut
    # Fix things in each body cell
    DO_PERCENT = True
    try:
        for r in range(len(body)):
            for c in range(len(body[0])):
                # In data land
                if r > 0 and c > 0:
                    if precision is not None:
                        # Hack
                        if ut.is_float(body[r][c]):
                            fmtstr = '%.' + str(precision) + 'f'
                            body[r][c] = fmtstr % (float(body[r][c]), )
                    # Force integer
                    if FORCE_INT:
                        body[r][c] = str(int(float(body[r][c])))
                body[r][c] = str(body[r][c])
                # Remove bad formatting;
                if AUTOFIX_LATEX:
                    body[r][c] = escape_latex(body[r][c])
    except Exception as ex:
        import utool as ut
        print('len(row_lbls) = %r' % (len(row_lbls), ))
        print('len(col_lbls) = %r' % (len(col_lbls), ))
        print('len(values) = %r' % (values, ))
        print('ut.depth_profile(values) = %r' % (ut.depth_profile(values), ))
        util_dbg.printex(ex, keys=['r', 'c'])
        raise

    # Bold the best values
    if bold_best:
        best_col_scores = values.max(0) if bigger_is_better else values.min(0)
        rows_to_bold = [
            np.where(values[:, colx] == best_col_scores[colx])[0]
            for colx in range(len(values.T))
        ]
        for colx, rowx_list in enumerate(rows_to_bold):
            for rowx in rowx_list:
                body[rowx + 1][colx +
                               1] = '\\txtbf{' + body[rowx + 1][colx + 1] + '}'

    # More fixing after the bold is in place
    for r in range(len(body)):
        for c in range(len(body[0])):
            # In data land
            if r > 0 and c > 0:
                if out_of is not None:
                    body[r][c] = body[r][c] + '/' + str(out_of)
                    if DO_PERCENT:
                        percent = ' = %.1f%%' % float(
                            100 * values[r - 1, c - 1] / out_of)
                        body[r][c] += escape_latex(percent)

    # Align columns for pretty printing
    body = np.array(body)
    ALIGN_BODY = True
    if ALIGN_BODY:
        new_body_cols = []
        for col in body.T:
            colstrs = list(map(str, ensurelist(col)))
            collens = list(map(len, colstrs))
            maxlen = max(collens)
            newcols = [str_ + (' ' * (maxlen - len(str_))) for str_ in colstrs]
            new_body_cols += [newcols]
        body = np.array(new_body_cols).T

    # Build Body (and row layout)
    HLINE_SEP = True
    rowvalsep = ''
    colvalsep = ' & '
    endl = '\\\\\n'
    hline = r'\hline'
    #extra_rowsep_pos_list = [1]  # rows to insert an extra hline after
    extra_rowsep_pos_list = []  # rows to insert an extra hline after
    if HLINE_SEP:
        rowvalsep = hline + '\n'
    # rowstr list holds blocks of rows
    rowstr_list = [colvalsep.join(row) + endl for row in body]
    #rowstr_list = [row[0] + rowlbl_sep + colvalsep.join(row[1:]) + endl for row in body]
    #rowstr_list = [(
    #    ('' if len(row) == 0 else row[0])
    #    if len(row) <= 1 else
    #    row[0] + rowlblcol_sep + colvalsep.join(row[1:]) + endl)
    #    for row in body]
    rowsep_list = [rowvalsep for row in rowstr_list[0:-1]
                   ]  # should be len 1 less than rowstr_list
    # Insert multicolumn names
    if multicol_lbls is not None:
        # TODO: label of the row labels
        multicol_sep
        multicols = [
            latex_multicolumn(multicol, size, 'c' + multicol_sep)
            for multicol, size in multicol_lbls
        ]
        multicol_str = latex_multirow(
            '', 2) + colvalsep + colvalsep.join(multicols) + endl
        ncols = sum([tup[1] for tup in multicol_lbls])
        mcol_sep = '\\cline{2-%d}\n' % (ncols + 1, )
        rowstr_list = [multicol_str] + rowstr_list
        rowsep_list = [mcol_sep] + rowsep_list
        #extra_rowsep_pos_list += [1]

    # Insert title
    if title is not None and not astable:
        tex_title = latex_multicolumn(title, len(body[0])) + endl
        rowstr_list = [tex_title] + rowstr_list
        rowsep_list = [rowvalsep] + rowsep_list
        #extra_rowsep_pos_list += [2]

    # Apply an extra hline (for label)
    #extra_rowsep_pos_list = []
    for pos in sorted(extra_rowsep_pos_list)[::-1]:
        rowstr_list.insert(pos, '')
        rowsep_list.insert(pos, rowvalsep)
    #tabular_body = rowvalsep.join(rowstr_list)
    from six.moves import zip_longest
    tabular_body = ''.join([
        row if sep is None else row + sep
        for row, sep in zip_longest(rowstr_list, rowsep_list)
    ])

    # Build Column Layout
    col_align_list = [col_align] * len(body[0])
    #extra_collayoutsep_pos_list = [1]
    extra_collayoutsep_pos_list = []
    for pos in sorted(extra_collayoutsep_pos_list)[::-1]:
        col_align_list.insert(pos, '')
    #col_layaout_sep_list = rowlblcol_sep  # TODO

    rowlblcol_sep = '|'
    # Build build internal seprations between column alignments
    # Defaults to just the normal col_sep
    col_align_sep_list = [col_sep] * (len(col_align_list) - 1)
    # Adjust for the separations between row labels and the actual row data
    if len(col_align_sep_list) > 0:
        col_align_sep_list[0] = rowlblcol_sep
    # Continue multicolumn sepratation
    if multicol_lbls is not None:
        multicol_offsets = ut.cumsum(ut.get_list_column(multicol_lbls, 1))
        for offset in multicol_offsets:
            if offset < len(col_align_sep_list):
                col_align_sep_list[offset] = multicol_sep

    from six.moves import zip_longest
    _tmp = [
        ut.filter_Nones(tup)
        for tup in zip_longest(col_align_list, col_align_sep_list)
    ]
    col_layout = ''.join(ut.flatten(_tmp))

    #if len(col_align_list) > 1:
    #    col_layout = col_align_list[0] + rowlblcol_sep + col_sep.join(col_align_list[1:])
    #else:
    #    col_layout = col_sep.join(col_align_list)

    tabular_head = (r'\begin{tabular}{|%s|}' % col_layout) + '\n'
    tabular_tail = r'\end{tabular}'

    if centerline:
        tabular_head = r'\centerline{' + '\n' + tabular_head
        tabular_tail = tabular_tail + '}'

    if astable:
        #tabular_head = r'\begin{centering}' + '\n' + tabular_head
        tabular_head = r'\centering' + '\n' + tabular_head
        tabular_head = r'\begin{table}' + table_position + '\n' + tabular_head

        lblstr = latex_sanitize_command_name(kwargs.get('label', title))
        caption = title
        if AUTOFIX_LATEX:
            caption = escape_latex(caption)
        caption = '\n% ---\n' + caption + '\n% ---\n'
        #tabular_head = r'\end{centering}' + '\n' + tabular_head
        tabular_tail = tabular_tail + '\n\caption[%s]{%s}\n\label{tbl:%s}\n\end{table}' % (
            lblstr, caption, lblstr)

    tabular_str = rowvalsep.join([tabular_head, tabular_body, tabular_tail])
    topsep = '\\hline\n' if True else '\\toprule\n'
    botsep = '\\hline\n' if True else '\\bottomrule\n'
    tabular_str = tabular_head + topsep + tabular_body + botsep + tabular_tail

    if common_rowlbl is not None:
        #tabular_str += escape_latex('\n\nThe following parameters were held fixed:\n' + common_rowlbl)
        pass
    return tabular_str
Example #31
0
 def stagger_group(list_):
     return ut.filter_Nones(ut.iflatten(zip_longest(*list_)))
Example #32
0
def get_dbinfo(ibs, verbose=True,
               with_imgsize=False,
               with_bytes=False,
               with_contrib=False,
               with_agesex=False,
               with_header=True,
               short=False,
               tag='dbinfo',
               aid_list=None):
    """

    Returns dictionary of digestable database information
    Infostr is a string summary of all the stats. Prints infostr in addition to
    returning locals

    Args:
        ibs (IBEISController):
        verbose (bool):
        with_imgsize (bool):
        with_bytes (bool):

    Returns:
        dict:

    CommandLine:
        python -m ibeis.other.dbinfo --exec-get_dbinfo:0
        python -m ibeis.other.dbinfo --test-get_dbinfo:1
        python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3
        python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1
        python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL
        python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints
        python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1

        python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a ctrl
        python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA
        python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0

        python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA
        python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0

    Example1:
        >>> # SCRIPT
        >>> from ibeis.other.dbinfo import *  # NOQA
        >>> import ibeis
        >>> defaultdb = 'testdb1'
        >>> ibs, aid_list = ibeis.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1')
        >>> kwargs = ut.get_kwdefaults(get_dbinfo)
        >>> kwargs['verbose'] = False
        >>> kwargs['aid_list'] = aid_list
        >>> kwargs = ut.parse_dict_from_argv(kwargs)
        >>> output = get_dbinfo(ibs, **kwargs)
        >>> result = (output['info_str'])
        >>> print(result)
        >>> #ibs = ibeis.opendb(defaultdb='testdb1')
        >>> # <HACK FOR FILTERING>
        >>> #from ibeis.expt import cfghelpers
        >>> #from ibeis.expt import annotation_configs
        >>> #from ibeis.init import filter_annots
        >>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
        >>> #                                   annotation_configs.TEST_NAMES)
        >>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
        >>> #                               ut.get_list_column(named_defaults_dict, 'qcfg')))
        >>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0]
        >>> #aid_list = ibs.get_valid_aids()
        >>> # </HACK FOR FILTERING>

    Example1:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.other.dbinfo import *  # NOQA
        >>> import ibeis
        >>> verbose = True
        >>> short = True
        >>> #ibs = ibeis.opendb(db='GZ_ALL')
        >>> #ibs = ibeis.opendb(db='PZ_Master0')
        >>> ibs = ibeis.opendb('testdb1')
        >>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS'
        >>> ibs.delete_contributors(ibs.get_valid_contrib_rowids())
        >>> ibs.delete_empty_nids()
        >>> #ibs = ibeis.opendb(db='PZ_MTEST')
        >>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True)
        >>> result = (output['info_str'])
        >>> print(result)
        +============================
        DB Info:  testdb1
        DB Notes: None
        DB NumContrib: 0
        ----------
        # Names                      = 7
        # Names (unassociated)       = 0
        # Names (singleton)          = 5
        # Names (multiton)           = 2
        ----------
        # Annots                     = 13
        # Annots (unknown)           = 4
        # Annots (singleton)         = 5
        # Annots (multiton)          = 4
        ----------
        # Img                        = 13
        L============================
    """
    # TODO Database size in bytes
    # TODO: occurrence, contributors, etc...

    # Basic variables
    request_annot_subset = False
    _input_aid_list = aid_list  # NOQA
    if aid_list is None:
        valid_aids = ibs.get_valid_aids()
        valid_nids = ibs.get_valid_nids()
        valid_gids = ibs.get_valid_gids()
    else:
        if isinstance(aid_list, str):
            # Hack to get experiment stats on aids
            acfg_name_list = [aid_list]
            print('Specified custom aids via acfgname %s' % (acfg_name_list,))
            from ibeis.expt import experiment_helpers
            acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
                ibs, acfg_name_list)
            aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list)))))
            #aid_list =
        if verbose:
            print('Specified %d custom aids' % (len(aid_list,)))
        request_annot_subset = True
        valid_aids = aid_list
        valid_nids = list(
            set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False)) -
            {const.UNKNOWN_NAME_ROWID}
        )
        valid_gids = list(set(ibs.get_annot_gids(aid_list)))
    #associated_nids = ibs.get_valid_nids(filter_empty=True)  # nids with at least one annotation
    FILTER_HACK = True
    if FILTER_HACK:
        # HUGE HACK - get only images and names with filtered aids
        valid_aids_ = ibs.filter_aids_custom(valid_aids)
        valid_nids_ = ibs.filter_nids_custom(valid_nids)
        valid_gids_ = ibs.filter_gids_custom(valid_gids)
        if verbose:
            print('Filtered %d names' % (len(valid_nids) - len(valid_nids_)))
            print('Filtered %d images' % (len(valid_gids) - len(valid_gids_)))
            print('Filtered %d annots' % (len(valid_aids) - len(valid_aids_)))
        valid_gids = valid_gids_
        valid_nids = valid_nids_
        valid_aids = valid_aids_
        #associated_nids = ut.compress(associated_nids, map(any,
        #ibs.unflat_map(ibs.get_annot_custom_filterflags,
        #               ibs.get_name_aids(associated_nids))))

    # Image info
    if verbose:
        print('Checking Image Info')
    gx2_aids = ibs.get_image_aids(valid_gids)
    if FILTER_HACK:
        gx2_aids = [ibs.filter_aids_custom(aids) for aids in gx2_aids]  # HACK FOR FILTER
    if request_annot_subset:
        # remove annots not in this subset
        valid_aids_set = set(valid_aids)
        gx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in gx2_aids]

    gx2_nAnnots = np.array(list(map(len, gx2_aids)))
    image_without_annots = len(np.where(gx2_nAnnots == 0)[0])
    gx2_nAnnots_stats  = ut.get_stats_str(gx2_nAnnots, newlines=True, use_median=True)
    image_reviewed_list = ibs.get_image_reviewed(valid_gids)

    # Name stats
    if verbose:
        print('Checking Name Info')
    nx2_aids = ibs.get_name_aids(valid_nids)
    if FILTER_HACK:
        nx2_aids =  [ibs.filter_aids_custom(aids) for aids in nx2_aids]    # HACK FOR FILTER
    if request_annot_subset:
        # remove annots not in this subset
        valid_aids_set = set(valid_aids)
        nx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in nx2_aids]
    associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids)))

    ibs.check_name_mapping_consistency(nx2_aids)

    # Occurrence Info
    def compute_annot_occurrence_ids(ibs, aid_list):
        from ibeis.algo.preproc import preproc_occurrence
        gid_list = ibs.get_annot_gids(aid_list)
        gid2_aids = ut.group_items(aid_list, gid_list)
        flat_imgsetids, flat_gids = preproc_occurrence.ibeis_compute_occurrences(ibs, gid_list, seconds_thresh=4 * 60 * 60, verbose=False)
        occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
        occurid2_aids = {oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items()}
        return occurid2_aids

    import utool
    with utool.embed_on_exception_context:
        occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids)
        occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values())
        occur_unique_nids = [ut.unique(nids) for nids in occur_nids]
        nid2_occurxs = ut.ddict(list)
        for occurx, nids in enumerate(occur_unique_nids):
            for nid in nids:
                nid2_occurxs[nid].append(occurx)

    nid2_occurx_single = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1}
    nid2_occurx_resight = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1}
    singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys())

    singlesight_annot_stats = ut.get_stats(list(map(len, singlesight_encounters)), use_median=True, use_sum=True)
    resight_name_stats = ut.get_stats(list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True)

    try:
        aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0)
        undirected_tags = ibs.get_aidpair_tags(aid_pairs.T[0], aid_pairs.T[1], directed=False)
        tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags))
        tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags)
        pair_tag_info = ut.map_dict_vals(len, tag_dict)

        num_reviewed_pairs = sum(ibs.get_annot_pair_is_reviewed(aid_pairs.T[0], aid_pairs.T[1]))
        pair_tag_info['num_reviewed'] = num_reviewed_pairs
    except Exception:
        pair_tag_info = {}

    #print(ut.dict_str(pair_tag_info))

    # Annot Stats
    # TODO: number of images where chips cover entire image
    # TODO: total image coverage of annotation
    # TODO: total annotation overlap
    """
    ax2_unknown = ibs.is_aid_unknown(valid_aids)
    ax2_nid = ibs.get_annot_name_rowids(valid_aids)
    assert all([nid < 0 if unknown else nid > 0 for nid, unknown in
                zip(ax2_nid, ax2_unknown)]), 'bad annot nid'
    """
    #
    if verbose:
        print('Checking Annot Species')
    unknown_aids = ut.compress(valid_aids, ibs.is_aid_unknown(valid_aids))
    species_list = ibs.get_annot_species_texts(valid_aids)
    species2_aids = ut.group_items(valid_aids, species_list)
    species2_nAids = {key: len(val) for key, val in species2_aids.items()}

    if verbose:
        print('Checking Multiton/Singleton Species')
    nx2_nAnnots = np.array(list(map(len, nx2_aids)))
    # Seperate singleton / multitons
    multiton_nxs  = np.where(nx2_nAnnots > 1)[0]
    singleton_nxs = np.where(nx2_nAnnots == 1)[0]
    unassociated_nxs = np.where(nx2_nAnnots == 0)[0]
    assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names'
    valid_nxs      = np.hstack([multiton_nxs, singleton_nxs])
    num_names_with_gt = len(multiton_nxs)

    # Annot Info
    if verbose:
        print('Checking Annot Info')
    multiton_aids_list = ut.take(nx2_aids, multiton_nxs)
    assert len(set(multiton_nxs)) == len(multiton_nxs)
    if len(multiton_aids_list) == 0:
        multiton_aids = np.array([], dtype=np.int)
    else:
        multiton_aids = np.hstack(multiton_aids_list)
        assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot'
    singleton_aids = ut.take(nx2_aids, singleton_nxs)
    multiton_nid2_nannots = list(map(len, multiton_aids_list))

    # Image size stats
    if with_imgsize:
        if verbose:
            print('Checking ImageSize Info')
        gpath_list = ibs.get_image_paths(valid_gids)
        def wh_print_stats(wh_list):
            if len(wh_list) == 0:
                return '{empty}'
            wh_list = np.asarray(wh_list)
            stat_dict = OrderedDict(
                [( 'max', wh_list.max(0)),
                 ( 'min', wh_list.min(0)),
                 ('mean', wh_list.mean(0)),
                 ( 'std', wh_list.std(0))])
            def arr2str(var):
                return ('[' + (
                    ', '.join(list(map(lambda x: '%.1f' % x, var)))
                ) + ']')
            ret = (',\n    '.join([
                '%s:%s' % (key, arr2str(val))
                for key, val in stat_dict.items()
            ]))
            return '{\n    ' + ret + '\n}'

        print('reading image sizes')
        # Image size stats
        img_size_list  = ibs.get_image_sizes(valid_gids)
        img_size_stats  = wh_print_stats(img_size_list)

        # Chip size stats
        annotation_bbox_list = ibs.get_annot_bboxes(valid_aids)
        annotation_bbox_arr = np.array(annotation_bbox_list)
        if len(annotation_bbox_arr) == 0:
            annotation_size_list = []
        else:
            annotation_size_list = annotation_bbox_arr[:, 2:4]
        chip_size_stats = wh_print_stats(annotation_size_list)
        imgsize_stat_lines = [
            (' # Img in dir                 = %d' % len(gpath_list)),
            (' Image Size Stats  = %s' % (img_size_stats,)),
            (' * Chip Size Stats = %s' % (chip_size_stats,)),
        ]
    else:
        imgsize_stat_lines = []

    if verbose:
        print('Building Stats String')

    multiton_stats = ut.get_stats_str(multiton_nid2_nannots, newlines=True, use_median=True)

    # Time stats
    unixtime_list = ibs.get_image_unixtime(valid_gids)
    unixtime_list = ut.list_replace(unixtime_list, -1, float('nan'))
    #valid_unixtime_list = [time for time in unixtime_list if time != -1]
    #unixtime_statstr = ibs.get_image_time_statstr(valid_gids)
    if ut.get_argflag('--hackshow-unixtime'):
        show_time_distributions(ibs, unixtime_list)
        ut.show_if_requested()
    unixtime_statstr = ut.get_timestats_str(unixtime_list, newlines=True, full=True)

    # GPS stats
    gps_list_ = ibs.get_image_gps(valid_gids)
    gpsvalid_list = [gps != (-1, -1) for gps in gps_list_]
    gps_list  = ut.compress(gps_list_, gpsvalid_list)

    def get_annot_age_stats(aid_list):
        annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list)
        annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list)
        age_dict = ut.ddict((lambda : 0))
        for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max):
            if (min_age is None or min_age < 12) and max_age < 12:
                age_dict['Infant'] += 1
            elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36:
                age_dict['Juvenile'] += 1
            elif 36 <= min_age and (36 <= max_age or max_age is None):
                age_dict['Adult'] += 1
            else:
                print('Found UNKNOWN Age: %r, %r' % (min_age, max_age, ))
                age_dict['UNKNOWN'] += 1
        return age_dict

    def get_annot_sex_stats(aid_list):
        annot_sextext_list = ibs.get_annot_sex_texts(aid_list)
        sextext2_aids = ut.group_items(aid_list, annot_sextext_list)
        sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys())
        assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str(set(annot_sextext_list) - set(sex_keys))
        sextext2_nAnnots = ut.odict([(key, len(sextext2_aids.get(key, []))) for key in sex_keys])
        # Filter 0's
        sextext2_nAnnots = {key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0}
        return sextext2_nAnnots

    if verbose:
        print('Checking Other Annot Stats')

    qualtext2_nAnnots = ibs.get_annot_qual_stats(valid_aids)
    yawtext2_nAnnots = ibs.get_annot_yaw_stats(valid_aids)
    agetext2_nAnnots = get_annot_age_stats(valid_aids)
    sextext2_nAnnots = get_annot_sex_stats(valid_aids)

    if verbose:
        print('Checking Contrib Stats')

    # Contributor Statistics
    # hack remove colon for image alignment
    def fix_tag_list(tag_list):
        return [None if tag is None else tag.replace(':', ';') for tag in tag_list]
    image_contrib_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids))
    annot_contrib_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids))
    contrib_tag_to_gids = ut.group_items(valid_gids, image_contrib_tags)
    contrib_tag_to_aids = ut.group_items(valid_aids, annot_contrib_tags)

    contrib_tag_to_qualstats = {key: ibs.get_annot_qual_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)}
    contrib_tag_to_viewstats = {key: ibs.get_annot_yaw_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)}

    contrib_tag_to_nImages = {key: len(val) for key, val in six.iteritems(contrib_tag_to_gids)}
    contrib_tag_to_nAnnots = {key: len(val) for key, val in six.iteritems(contrib_tag_to_aids)}

    if verbose:
        print('Summarizing')

    # Summarize stats
    num_names = len(valid_nids)
    num_names_unassociated = len(valid_nids) - len(associated_nids)
    num_names_singleton = len(singleton_nxs)
    num_names_multiton =  len(multiton_nxs)

    num_singleton_annots = len(singleton_aids)
    num_multiton_annots = len(multiton_aids)
    num_unknown_annots = len(unknown_aids)
    num_annots = len(valid_aids)

    if with_bytes:
        if verbose:
            print('Checking Disk Space')
        ibsdir_space   = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir()))
        dbdir_space    = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir()))
        imgdir_space   = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir()))
        cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir()))

    if True:
        if verbose:
            print('Check asserts')
        try:
            bad_aids = np.intersect1d(multiton_aids, unknown_aids)
            _num_names_total_check = num_names_singleton + num_names_unassociated + num_names_multiton
            _num_annots_total_check = num_unknown_annots + num_singleton_annots + num_multiton_annots
            assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids'
            assert _num_names_total_check == num_names, 'inconsistent num names'
            #if not request_annot_subset:
            # dont check this if you have an annot subset
            assert _num_annots_total_check == num_annots, 'inconsistent num annots'
        except Exception as ex:
            ut.printex(ex, keys=[
                '_num_names_total_check',
                'num_names',
                '_num_annots_total_check',
                'num_annots',
                'num_names_singleton',
                'num_names_multiton',
                'num_unknown_annots',
                'num_multiton_annots',
                'num_singleton_annots',
            ])
            raise

    # Get contributor statistics
    contrib_rowids = ibs.get_valid_contrib_rowids()
    num_contributors = len(contrib_rowids)

    # print
    num_tabs = 5

    def align2(str_):
        return ut.align(str_, ':', ' :')

    def align_dict2(dict_):
        str_ = ut.dict_str(dict_)
        return align2(str_)

    header_block_lines = (
        [('+============================'), ] + (
            [
                ('+ singleton := single sighting'),
                ('+ multiton  := multiple sightings'),
                ('--' * num_tabs),
            ] if not short and with_header else []
        )
    )

    source_block_lines = [
        ('DB Info:  ' + ibs.get_dbname()),
        ('DB Notes: ' + ibs.get_dbnotes()),
        ('DB NumContrib: %d' % num_contributors),
    ]

    bytes_block_lines = [
        ('--' * num_tabs),
        ('DB Bytes: '),
        ('     +- dbdir nBytes:         ' + dbdir_space),
        ('     |  +- _ibsdb nBytes:     ' + ibsdir_space),
        ('     |  |  +-imgdir nBytes:   ' + imgdir_space),
        ('     |  |  +-cachedir nBytes: ' + cachedir_space),
    ] if with_bytes else []

    name_block_lines = [
        ('--' * num_tabs),
        ('# Names                      = %d' % num_names),
        ('# Names (unassociated)       = %d' % num_names_unassociated),
        ('# Names (singleton)          = %d' % num_names_singleton),
        ('# Names (multiton)           = %d' % num_names_multiton),
    ]

    subset_str = '        ' if not request_annot_subset else '(SUBSET)'

    annot_block_lines = [
        ('--' * num_tabs),
        ('# Annots %s            = %d' % (subset_str, num_annots,)),
        ('# Annots (unknown)           = %d' % num_unknown_annots),
        ('# Annots (singleton)         = %d' % num_singleton_annots),
        ('# Annots (multiton)          = %d' % num_multiton_annots),
    ]

    annot_per_basic_block_lines = [
        ('--' * num_tabs),
        ('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)),
        ('# Annots per Image           = %s' % (align2(gx2_nAnnots_stats),)),
        ('# Annots per Species         = %s' % (align_dict2(species2_nAids),)),
    ] if not short else []

    occurrence_block_lines = [
        ('--' * num_tabs),
        ('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)),
        ('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)),
        ('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)),
    ] if not short else []

    annot_per_qualview_block_lines = [
        None if short else '# Annots per Viewpoint = %s' % align_dict2(yawtext2_nAnnots),
        None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots),
    ]

    annot_per_agesex_block_lines = [
        '# Annots per Age = %s' % align_dict2(agetext2_nAnnots),
        '# Annots per Sex = %s' % align_dict2(sextext2_nAnnots),
    ] if not short  and with_agesex else []

    contrib_block_lines = [
        '# Images per contributor       = ' + align_dict2(contrib_tag_to_nImages),
        '# Annots per contributor       = ' + align_dict2(contrib_tag_to_nAnnots),
        '# Quality per contributor      = ' + ut.dict_str(contrib_tag_to_qualstats, sorted_=True),
        '# Viewpoint per contributor    = ' + ut.dict_str(contrib_tag_to_viewstats, sorted_=True),
    ] if with_contrib else []

    img_block_lines = [
        ('--' * num_tabs),
        ('# Img                        = %d' % len(valid_gids)),
        None if short else ('# Img reviewed               = %d' % sum(image_reviewed_list)),
        None if short else ('# Img with gps               = %d' % len(gps_list)),
        #('# Img with timestamp         = %d' % len(valid_unixtime_list)),
        None if short else ('Img Time Stats               = %s' % (align2(unixtime_statstr),)),
    ]

    info_str_lines = (
        header_block_lines +
        bytes_block_lines +
        source_block_lines +
        name_block_lines +
        annot_block_lines +
        annot_per_basic_block_lines +
        occurrence_block_lines +
        annot_per_qualview_block_lines +
        annot_per_agesex_block_lines +
        img_block_lines +
        contrib_block_lines +
        imgsize_stat_lines +
        [('L============================'), ]
    )
    info_str = '\n'.join(ut.filter_Nones(info_str_lines))
    info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag))
    if verbose:
        print(info_str2)
    locals_ = locals()
    return locals_
Example #33
0
def make_netx_graph_from_aid_groups(ibs,
                                    aids_list,
                                    only_reviewed_matches=True,
                                    invis_edges=None,
                                    ensure_edges=None,
                                    temp_nids=None,
                                    allow_directed=False):
    r"""
    Args:
        ibs (ibeis.IBEISController): image analysis api
        aids_list (list):

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.viz.viz_graph import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(defaultdb='testdb1')
        >>> aids_list = [[1, 2, 3, 4], [5, 6, 7]]
        >>> invis_edges = [(1, 5)]
        >>> only_reviewed_matches = True
        >>> graph = make_netx_graph_from_aid_groups(ibs, aids_list,
        >>>                                         only_reviewed_matches,
        >>>                                         invis_edges)
        >>> list(nx.connected_components(graph.to_undirected()))
    """
    #aids_list, nid_list = ibs.group_annots_by_name(aid_list)
    unique_aids = list(ut.flatten(aids_list))

    # grouped version
    unflat_edges = (list(itertools.product(aids, aids)) for aids in aids_list)
    aid_pairs = [tup for tup in ut.iflatten(unflat_edges) if tup[0] != tup[1]]
    aids1 = ut.get_list_column(aid_pairs, 0)
    aids2 = ut.get_list_column(aid_pairs, 1)

    if only_reviewed_matches:
        annotmatch_rowids = ibs.get_annotmatch_rowid_from_superkey(
            aids1, aids2)
        annotmatch_rowids = ut.filter_Nones(annotmatch_rowids)
        aids1 = ibs.get_annotmatch_aid1(annotmatch_rowids)
        aids2 = ibs.get_annotmatch_aid2(annotmatch_rowids)

    graph = make_netx_graph_from_aidpairs(ibs,
                                          aids1,
                                          aids2,
                                          unique_aids=unique_aids)

    if ensure_edges is not None:
        if ensure_edges == 'all':
            ensure_edges = list(ut.upper_diag_self_prodx(list(graph.nodes())))
        ensure_edges_ = []
        for edge in ensure_edges:
            edge = tuple(edge)
            redge = tuple(edge[::-1])  # HACK
            if graph.has_edge(*edge):
                ensure_edges_.append(edge)
                pass
                #nx.set_edge_attributes(graph, 'weight', {edge: .001})
            elif (not allow_directed) and graph.has_edge(*redge):
                ensure_edges_.append(redge)
                #nx.set_edge_attributes(graph, 'weight', {redge: .001})
                pass
            else:
                ensure_edges_.append(edge)
                #graph.add_edge(*edge, weight=.001)
                graph.add_edge(*edge)

    if temp_nids is None:
        unique_nids = ibs.get_annot_nids(list(graph.nodes()))
    else:
        # HACK
        unique_nids = [1] * len(list(graph.nodes()))
        #unique_nids = temp_nids

    nx.set_node_attributes(graph, 'nid', dict(zip(graph.nodes(), unique_nids)))

    import plottool as pt
    ensure_names_are_connected(graph, aids_list)

    # Color edges by nid
    color_by_nids(graph, unique_nids=unique_nids)
    if invis_edges:
        for edge in invis_edges:
            if graph.has_edge(*edge):
                nx.set_edge_attributes(graph, 'style', {edge: 'invis'})
                nx.set_edge_attributes(graph, 'invisible', {edge: True})
            else:
                graph.add_edge(*edge, style='invis', invisible=True)

    # Hack color images orange
    if ensure_edges:
        nx.set_edge_attributes(
            graph, 'color', {tuple(edge): pt.ORANGE
                             for edge in ensure_edges_})

    return graph
Example #34
0
def get_default_cell_template_list(ibs):
    """
    Defines the order of ipython notebook cells
    """

    cells = notebook_cells

    noexample = ut.get_argflag('--noexample')
    asreport = ut.get_argflag('--asreport')
    withtags = ut.get_argflag('--withtags')

    cell_template_list = []

    info_cells = [
        cells.pipe_config_info,
        cells.annot_config_info,
        cells.timestamp_distribution,
    ]

    dev_analysis = [
        cells.config_overlap,
        #cells.dbsize_expt,
        None if ibs.get_dbname() == 'humpbacks' else cells.feat_score_sep,
        cells.all_annot_scoresep,
        cells.success_annot_scoresep,
    ]

    cell_template_list += [
        cells.introduction if asreport else None,
        cells.initialize,
        None if ibs.get_dbname() != 'humpbacks' else cells.fluke_select,
    ]

    if not asreport:
        cell_template_list += info_cells

    if not noexample:
        cell_template_list += [
            cells.example_annotations,
            cells.example_names,
        ]

    cell_template_list += [
        cells.per_annotation_accuracy,
        cells.per_name_accuracy,
        cells.easy_success_cases,
        cells.hard_success_cases,
        cells.failure_type1_cases,
        cells.failure_type2_cases,
        cells.timedelta_distribution,
    ]

    if withtags:
        cell_template_list += [
            cells.investigate_specific_case,
            cells.view_intereseting_tags,
        ]

    if asreport:
        # Append our debug stuff at the bottom
        cell_template_list += [cells.IGNOREAFTER]
        cell_template_list += info_cells

    cell_template_list += dev_analysis

    cell_template_list += [
        cells.config_disagree_cases,
    ]

    cell_template_list = ut.filter_Nones(cell_template_list)

    return cell_template_list
Example #35
0
def get_default_cell_template_list(ibs):
    """
    Defines the order of ipython notebook cells
    """

    cells = notebook_cells

    noexample = not ut.get_argflag('--examples')
    asreport = ut.get_argflag('--asreport')
    withtags = ut.get_argflag('--withtags')

    cell_template_list = []

    info_cells = [
        cells.pipe_config_info,
        cells.annot_config_info,
        # cells.per_encounter_stats,
        cells.timestamp_distribution,
    ]

    dev_analysis = [
        cells.config_overlap,
        #cells.dbsize_expt,
        # None if ibs.get_dbname() == 'humpbacks' else cells.feat_score_sep,
        cells.all_annot_scoresep,
        cells.success_annot_scoresep,
    ]

    cell_template_list += [
        cells.introduction if asreport else None,
        cells.nb_init,
        cells.db_init,
        None if ibs.get_dbname() != 'humpbacks' else cells.fluke_select,
    ]

    if not asreport:
        cell_template_list += info_cells

    if not noexample:
        cell_template_list += [
            cells.example_annotations,
            cells.example_names,
        ]

    cell_template_list += [
        cells.per_annotation_accuracy,
        cells.per_name_accuracy,
        cells.easy_success_cases,
        cells.hard_success_cases,
        cells.failure_type1_cases,
        cells.failure_type2_cases,
        cells.total_failure_cases,
        cells.timedelta_distribution,
    ]

    if withtags:
        cell_template_list += [
            cells.investigate_specific_case,
            cells.view_intereseting_tags,
        ]

    if asreport:
        # Append our debug stuff at the bottom
        cell_template_list += [cells.IGNOREAFTER]
        cell_template_list += info_cells

    cell_template_list += dev_analysis

    cell_template_list += [
        cells.config_disagree_cases,
    ]

    cell_template_list = ut.filter_Nones(cell_template_list)

    cell_template_list = ut.lmap(ut.normalize_cells, cell_template_list)

    if not asreport:
        # Remove all of the extra fluff
        cell_template_list = [(header.split('\n')[0], code, None)
                              for (header, code, footer) in cell_template_list]

    return cell_template_list
Example #36
0
def convert_ggr2018_to_wbia(ggr_path,
                            dbdir=None,
                            purge=True,
                            dry_run=False,
                            apply_updates=True,
                            **kwargs):
    r"""Convert the raw GGR2 (2018) data to an wbia database.

    Args
        ggr_path (str): Directory to folder *containing* raw GGR 2018 data
        dbdir (str): Output directory

    CommandLine:
        python -m wbia convert_ggr2018_to_wbia

    Example:
        >>> # SCRIPT
        >>> from wbia.dbio.ingest_ggr import *  # NOQA
        >>> default_ggr_path = join('/', 'data', 'wbia', 'GGR2', 'GGR2018data')
        >>> default_dbdir = join('/', 'data', 'wbia', 'GGR2-IBEIS')
        >>> dbdir = ut.get_argval('--dbdir', type_=str, default=default_dbdir)
        >>> ggr_path = ut.get_argval('--ggr', type_=str, default=default_ggr_path)
        >>> result = convert_ggr2018_to_wbia(ggr_path, dbdir=dbdir, purge=False, dry_run=True, apply_updates=False)
        >>> print(result)
    """
    ALLOWED_NUMBERS = list(range(1, 250))
    ALLOWED_LETTERS = ['A', 'B', 'C', 'D', 'E', 'F']

    ################################################################################

    if apply_updates:
        _fix_ggr2018_directory_structure(ggr_path)

    ################################################################################

    blacklist_filepath_set = set([
        join(ggr_path, 'Cameras info.numbers'),
        join(ggr_path, 'Cameras info.xlsx'),
        join(ggr_path, 'GGR_photos_MRC_29.1.18.ods'),
        join(ggr_path, 'Cameras info-2.numbers'),
    ])

    # Check root files
    direct = Directory(ggr_path)
    for filepath in direct.files(recursive=False):
        try:
            assert filepath in blacklist_filepath_set
            ut.delete(filepath)
        except AssertionError:
            logger.info('Unresolved root file found in %r' % (filepath, ))
            continue

    ################################################################################

    if purge:
        ut.delete(dbdir)
    ibs = wbia.opendb(dbdir=dbdir)

    ################################################################################

    # Check folder structure
    assert exists(ggr_path)
    direct = Directory(ggr_path, recursive=0)
    direct1_list = direct.directories()
    direct1_list.sort(key=lambda x: int(x.base()), reverse=False)
    for direct1 in direct1_list:
        if not dry_run:
            logger.info('Processing directory: %r' % (direct1, ))
        base1 = direct1.base()

        try:
            int(base1)
        except ValueError:
            logger.info('Error found in %r' % (direct1, ))
            continue

        try:
            assert len(direct1.files(recursive=False)) == 0
        except AssertionError:
            logger.info('Files found in %r' % (direct1, ))
            continue

        seen_letter_list = []
        direct1_ = Directory(direct1.absolute_directory_path, recursive=0)
        direct2_list = direct1_.directories()
        direct2_list.sort(key=lambda x: x.base(), reverse=False)
        for direct2 in direct2_list:
            base2 = direct2.base()

            try:
                assert base2.startswith(base1)
            except AssertionError:
                logger.info('Folder name heredity conflict %r with %r' %
                            (direct2, direct1))
                continue

            try:
                assert len(base2) >= 2
                assert ' ' not in base2
                number = base2[:-1]
                letter = base2[-1]
                number = int(number)
                letter = letter.upper()
                assert number in ALLOWED_NUMBERS
                assert letter in ALLOWED_LETTERS
                seen_letter_list.append(letter)
            except ValueError:
                logger.info('Error found in %r' % (direct2, ))
                continue
            except AssertionError:
                logger.info('Folder name format error found in %r' %
                            (direct2, ))
                continue

            direct2_ = Directory(direct2.absolute_directory_path,
                                 recursive=True,
                                 images=True)
            try:
                assert len(direct2_.directories()) == 0
            except AssertionError:
                logger.info('Folders exist in file only level %r' %
                            (direct2, ))
                continue

            filepath_list = sorted(direct2_.files())

            if not dry_run:
                try:
                    gid_list = ibs.add_images(filepath_list)
                    gid_list = ut.filter_Nones(gid_list)
                    gid_list = sorted(list(set(gid_list)))

                    imageset_text = 'GGR2,%d,%s' % (number, letter)
                    note_list = [
                        '%s,%05d' % (imageset_text, index + 1)
                        for index, gid in enumerate(gid_list)
                    ]
                    ibs.set_image_notes(gid_list, note_list)
                    ibs.set_image_imagesettext(gid_list,
                                               [imageset_text] * len(gid_list))
                except Exception as ex:  # NOQA
                    ut.embed()

        seen_letter_set = set(seen_letter_list)
        try:
            assert len(seen_letter_set) == len(seen_letter_list)
        except AssertionError:
            logger.info('Duplicate letters in %r with letters %r' %
                        (direct1, seen_letter_list))
            continue

        try:
            assert 'A' in seen_letter_set
        except AssertionError:
            logger.info('WARNING: A camera not found in %r' % (direct1, ))
            continue

    return ibs
Example #37
0
def get_layer_info_str(output_layer, batch_size=128):
    r"""
    Args:
        output_layer (lasange.layers.Layer):

    CommandLine:
        python -m ibeis_cnn.net_strs --test-get_layer_info_str:0
        python -m ibeis_cnn.net_strs --test-get_layer_info_str:1

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.net_strs import *  # NOQA
        >>> from ibeis_cnn import models
        >>> model = models.DummyModel(data_shape=(24, 24, 3), autoinit=True)
        >>> output_layer = model.output_layer
        >>> result = get_layer_info_str(output_layer)
        >>> result = '\n'.join([x.rstrip() for x in result.split('\n')])
        >>> print(result)
        Network Structure:
         index  Layer    Outputs    Bytes OutShape         Params
         0      Input      1,728   55,296 (8, 3, 24, 24)   []
         1      Conv2D     7,744  249,600 (8, 16, 22, 22)  [W(16,3,3,3, {t,r}), b(16, {t})]
         2      Conv2D     7,056  229,952 (8, 16, 21, 21)  [W(16,16,2,2, {t,r}), b(16, {t})]
         3      Dense          8  226,080 (8, 8)           [W(7056,8, {t,r}), b(8, {t})]
         4      Dense          5      340 (8, 5)           [W(8,5, {t,r}), b(5, {t})]
        ...this model has 57,989 learnable parameters
        ...this model will use 761,268 bytes = 743.43 KB

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.net_strs import *  # NOQA
        >>> from ibeis_cnn import models
        >>> model = models.mnist.MNISTModel(batch_size=128, output_dims=10,
        >>>                                 data_shape=(24, 24, 3))
        >>> model.init_arch()
        >>> output_layer = model.output_layer
        >>> result = get_layer_info_str(output_layer)
        >>> result = '\n'.join([x.rstrip() for x in result.split('\n')])
        >>> print(result)
    """
    import ibeis_cnn.__LASAGNE__ as lasagne
    info_lines = []
    _print = info_lines.append
    with warnings.catch_warnings():
        warnings.filterwarnings('ignore', '.*topo.*')
        nn_layers = lasagne.layers.get_all_layers(output_layer)
        _print('Network Structure:')

        columns_ = ut.ddict(list)

        for index, layer in enumerate(nn_layers):

            layer_info = get_layer_info(layer)

            columns_['index'].append(index)
            columns_['name'].append(layer_info['name'])
            #columns_['type'].append(getattr(layer, 'type', None))
            #columns_['layer'].append(layer_info['classname'])
            columns_['layer'].append(layer_info['classalias'])
            columns_['num_outputs'].append('{:,}'.format(int(layer_info['num_outputs'])))
            columns_['output_shape'].append(str(layer_info['output_shape'] ))
            columns_['params'].append(layer_info['param_str'])
            columns_['param_type'].append(layer_info['param_type_str'])
            columns_['mem'].append(layer_info['total_memory'])
            columns_['bytes'].append('{:,}'.format(int(layer_info['total_bytes'])))
            #ut.embed()

        header_nice = {
            'index'        : 'index',
            'name'         : 'Name',
            'layer'        : 'Layer',
            'type'         : 'Type',
            'num_outputs'  : 'Outputs',
            'output_shape' : 'OutShape',
            'params'       : 'Params',
            'param_type'   : 'ParamType',
            'mem'          : 'Mem',
            'bytes'        : 'Bytes',
        }

        header_align = {
            'index'       : '<',
            'params'      : '<',
            'bytes'       : '>',
            'num_outputs' : '>',
        }

        def get_col_maxval(key):
            header_len = len(header_nice[key])
            val_len = max(list(map(len, map(str, columns_[key]))))
            return max(val_len, header_len)

        header_order = ['index']
        if len(ut.filter_Nones(columns_['name'])) > 0:
            header_order += ['name']
        header_order += ['layer', 'num_outputs']
        #header_order += ['mem']
        header_order += ['bytes']
        header_order += ['output_shape', 'params' ]
        #'param_type']

        max_len = {key: str(get_col_maxval(key) + 1) for key, col in six.iteritems(columns_)}

        fmtstr = ' ' + ' '.join(
            [
                '{:' + align + len_ + '}'
                for align, len_ in zip(ut.dict_take(header_align, header_order, '<'),
                                       ut.dict_take(max_len, header_order))
            ]
        )
        _print(fmtstr.format(*ut.dict_take(header_nice, header_order)))

        row_list = zip(*ut.dict_take(columns_, header_order))
        for row in row_list:
            try:
                row = ['' if _ is None else _ for _ in row]
                str_ = fmtstr.format(*row)
                _print(str_)
            except TypeError:
                print('Error printing %r with args %r' % (fmtstr, row, ))

        total_bytes = count_bytes(output_layer)
        num_params = lasagne.layers.count_params(output_layer)

        _print('...this model has {:,} learnable parameters'.format(num_params))
        _print('...this model will use ~{:,} bytes = {} per input'.format(
            total_bytes, ut.byte_str2(total_bytes)))
        _print('...this model will use ~{:,} bytes = {} per batch with a batch size of {}'.format(
            total_bytes * batch_size, ut.byte_str2(total_bytes * batch_size), batch_size))
    info_str = '\n'.join(info_lines)
    return info_str
Example #38
0
    def make_inference(infr):
        cm_list = infr.cm_list
        unique_nids, prob_names = infr.make_prob_names()
        cluster_tuples = infr.make_clusters()

        # Make pair list for output
        if infr.user_feedback is not None:
            keys = list(
                zip(infr.user_feedback['aid1'], infr.user_feedback['aid2']))
            feedback_lookup = ut.make_index_lookup(keys)
            user_feedback = infr.user_feedback
            p_bg = 0
            user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
            part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
            part2 = p_bg * user_feedback['p_notcomp']
            p_same_list = part1 + part2
        else:
            feedback_lookup = {}
        infr.user_feedback
        needs_review_list = []
        num_top = 4
        for cm, row in zip(cm_list, prob_names):
            # Find top scoring names for this chip match in the posterior distribution
            idxs = row.argsort()[::-1]
            top_idxs = idxs[:num_top]
            nids = ut.take(unique_nids, top_idxs)
            # Find the matched annotations in the pairwise prior distributions
            nidxs = ut.dict_take(cm.nid2_nidx, nids, None)
            name_groupxs = ut.take(cm.name_groupxs, ut.filter_Nones(nidxs))
            daids_list = ut.take(cm.daid_list, name_groupxs)
            for daids in daids_list:
                ut.take(cm.score_list, ut.take(cm.daid2_idx, daids))
                scores_all = cm.annot_score_list / cm.annot_score_list.sum()
                idxs = ut.take(cm.daid2_idx, daids)
                scores = scores_all.take(idxs)
                raw_scores = cm.score_list.take(idxs)
                scorex = scores.argmax()
                raw_score = raw_scores[scorex]
                daid = daids[scorex]
                import scipy.special
                # SUPER HACK: these are not probabilities
                # TODO: set a and b based on dbsize and param configuration
                # python -m plottool.draw_func2 --exec-plot_func --show --range=0,3 --func="lambda x: scipy.special.expit(2 * x - 2)"
                #a = 2.0
                a = 1.5
                b = 2
                p_same = scipy.special.expit(b * raw_score - a)
                #confidence = scores[scorex]
                #p_diff = 1 - p_same
                #decision = 'same' if confidence > thresh else 'diff'
                #confidence = p_same if confidence > thresh else p_diff
                #tup = (cm.qaid, daid, decision, confidence, raw_score)
                confidence = (2 * np.abs(0.5 - p_same))**2
                #if infr.user_feedback is not None:
                #    import utool
                #    utool.embed(
                key = (cm.qaid, daid)
                fb_idx = feedback_lookup.get(key)
                if fb_idx is not None:
                    confidence = p_same_list[fb_idx]
                tup = (cm.qaid, daid, p_same, confidence, raw_score)
                needs_review_list.append(tup)

        # Sort resulting list by confidence
        sortx = ut.argsort(ut.take_column(needs_review_list, 3))
        needs_review_list = ut.take(needs_review_list, sortx)

        infr.needs_review_list = needs_review_list
        infr.cluster_tuples = cluster_tuples
Example #39
0
def show_name_matches(ibs, qaid, name_daid_list, name_fm_list, name_fs_list,
                      name_H1_list, name_featflag_list, qreq_=None, **kwargs):
    """
    Called from chip_match.py

    Args:
        ibs (IBEISController):  ibeis controller object
        qaid (int):  query annotation id
        name_daid_list (list):
        name_fm_list (list):
        name_fs_list (list):
        name_H1_list (list):
        name_featflag_list (list):
        qreq_ (QueryRequest):  query request object with hyper-parameters(default = None)

    Kwargs:
        draw_fmatches, name_rank, fnum, pnum, colorbar_, nonvote_mode,
        fastmode, show_matches, fs, fm_norm, lbl1, lbl2, rect, draw_border,
        cmap, H1, H2, scale_factor1, scale_factor2, draw_pts, draw_ell,
        draw_lines, show_nMatches, all_kpts, in_image, show_query, draw_lbl,
        name_annot_scores, score, rawscore, aid2_raw_rank, show_name,
        show_nid, show_aid, show_annot_score, show_truth, name_score,
        show_name_score, show_name_rank, show_timedelta

    CommandLine:
        python -m ibeis.viz.viz_matches --exec-show_name_matches
        python -m ibeis.viz.viz_matches --test-show_name_matches --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.viz.viz_matches import *  # NOQA
        >>> from ibeis.algo.hots import chip_match
        >>> from ibeis.algo.hots import name_scoring
        >>> import vtool as vt
        >>> from ibeis.algo.hots import _pipeline_helpers as plh  # NOQA
        >>> import numpy as np
        >>> func = chip_match.ChipMatch.show_single_namematch
        >>> sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True,
        >>>                                     strip_docstr=True)
        >>> setup = ut.regex_replace('viz_matches.show_name_matches', '#', sourcecode)
        >>> homog = False
        >>> print(ut.indent(setup, '>>> '))
        >>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[1])
        >>> cm = cm_list[0]
        >>> cm.score_nsum(qreq_)
        >>> dnid = ibs.get_annot_nids(cm.qaid)
        >>> # +--- COPIED SECTION
        >>> locals_ = locals()
        >>> var_list = ut.exec_func_src(
        >>>     func, locals_=locals_,
        >>>     sentinal='name_annot_scores = cm.annot_score_list.take(sorted_groupxs')
        >>> exec(ut.execstr_dict(var_list))
        >>> # L___ COPIED SECTION
        >>> kwargs = {}
        >>> show_name_matches(ibs, qaid, name_daid_list, name_fm_list,
        >>>                   name_fs_list, name_h1_list, name_featflag_list,
        >>>                   qreq_=qreq_, **kwargs)
        >>> ut.quit_if_noshow()
        >>> ut.show_if_requested()
    """
    #print("SHOW NAME MATCHES")
    #print(ut.repr2(kwargs, nl=True))
    #from ibeis import constants as const
    from ibeis import tag_funcs
    draw_fmatches = kwargs.pop('draw_fmatches', True)
    rchip1, kpts1 = get_query_annot_pair_info(ibs, qaid, qreq_, draw_fmatches)
    rchip2_list, kpts2_list = get_data_annot_pair_info(ibs, name_daid_list,
                                                       qreq_, draw_fmatches)
    fm_list = name_fm_list
    fs_list = name_fs_list
    featflag_list = name_featflag_list
    offset_list, sf_list, bbox_list = show_multichip_match(rchip1, rchip2_list,
                                                           kpts1, kpts2_list,
                                                           fm_list, fs_list,
                                                           featflag_list,
                                                           **kwargs)
    aid_list = [qaid] + name_daid_list
    annotate_matches3(ibs, aid_list, bbox_list, offset_list, name_fm_list,
                      name_fs_list, qreq_=None, **kwargs)
    ax = pt.gca()
    title = vh.get_query_text(ibs, None, name_daid_list, False, qaid=qaid,
                              **kwargs)

    pt.set_title(title, ax)

    # Case tags
    annotmatch_rowid_list = ibs.get_annotmatch_rowid_from_superkey(
        [qaid] * len(name_daid_list), name_daid_list)
    annotmatch_rowid_list = ut.filter_Nones(annotmatch_rowid_list)
    tags_list = ibs.get_annotmatch_case_tags(annotmatch_rowid_list)
    if not ut.get_argflag('--show'):  # False:
        tags_list = tag_funcs.consolodate_annotmatch_tags(tags_list)
    tag_list = ut.unique_ordered(ut.flatten(tags_list))

    name_rank = kwargs.get('name_rank', None)
    truth = get_multitruth(ibs, aid_list)

    xlabel = {1: 'Correct ID', 0: 'Incorrect ID', 2: 'Unknown ID'}[truth]

    if False:
        if name_rank is None:
            xlabel = {1: 'Genuine', 0: 'Imposter', 2: 'Unknown'}[truth]
            #xlabel = {1: 'True', 0: 'False', 2: 'Unknown'}[truth]
        else:
            if name_rank == 0:
                xlabel = {
                    1: 'True Positive', 0: 'False Positive', 2: 'Unknown'}[truth]
            else:
                xlabel = {
                    1: 'False Negative', 0: 'True Negative', 2: 'Unknown'}[truth]

    if len(tag_list) > 0:
        xlabel += '\n' + ', '.join(tag_list)

    pt.set_xlabel(xlabel)
    return ax
Example #40
0
    def make_inference(infr):
        cm_list = infr.cm_list
        unique_nids, prob_names = infr.make_prob_names()
        cluster_tuples = infr.make_clusters()

        # Make pair list for output
        if infr.user_feedback is not None:
            keys = list(zip(infr.user_feedback['aid1'], infr.user_feedback['aid2']))
            feedback_lookup = ut.make_index_lookup(keys)
            user_feedback = infr.user_feedback
            p_bg = 0
            user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
            part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
            part2 = p_bg * user_feedback['p_notcomp']
            p_same_list = part1 + part2
        else:
            feedback_lookup = {}
        infr.user_feedback
        needs_review_list = []
        num_top = 4
        for cm, row in zip(cm_list, prob_names):
            # Find top scoring names for this chip match in the posterior distribution
            idxs = row.argsort()[::-1]
            top_idxs = idxs[:num_top]
            nids = ut.take(unique_nids, top_idxs)
            # Find the matched annotations in the pairwise prior distributions
            nidxs = ut.dict_take(cm.nid2_nidx, nids, None)
            name_groupxs = ut.take(cm.name_groupxs, ut.filter_Nones(nidxs))
            daids_list = ut.take(cm.daid_list, name_groupxs)
            for daids in daids_list:
                ut.take(cm.score_list, ut.take(cm.daid2_idx, daids))
                scores_all = cm.annot_score_list / cm.annot_score_list.sum()
                idxs = ut.take(cm.daid2_idx, daids)
                scores = scores_all.take(idxs)
                raw_scores = cm.score_list.take(idxs)
                scorex = scores.argmax()
                raw_score = raw_scores[scorex]
                daid = daids[scorex]
                import scipy.special
                # SUPER HACK: these are not probabilities
                # TODO: set a and b based on dbsize and param configuration
                # python -m plottool.draw_func2 --exec-plot_func --show --range=0,3 --func="lambda x: scipy.special.expit(2 * x - 2)"
                #a = 2.0
                a = 1.5
                b = 2
                p_same = scipy.special.expit(b * raw_score - a)
                #confidence = scores[scorex]
                #p_diff = 1 - p_same
                #decision = 'same' if confidence > thresh else 'diff'
                #confidence = p_same if confidence > thresh else p_diff
                #tup = (cm.qaid, daid, decision, confidence, raw_score)
                confidence = (2 * np.abs(0.5 - p_same)) ** 2
                #if infr.user_feedback is not None:
                #    import utool
                #    utool.embed(
                key = (cm.qaid, daid)
                fb_idx = feedback_lookup.get(key)
                if fb_idx is not None:
                    confidence = p_same_list[fb_idx]
                tup = (cm.qaid, daid, p_same, confidence, raw_score)
                needs_review_list.append(tup)

        # Sort resulting list by confidence
        sortx = ut.argsort(ut.take_column(needs_review_list, 3))
        needs_review_list = ut.take(needs_review_list, sortx)

        infr.needs_review_list = needs_review_list
        infr.cluster_tuples = cluster_tuples
Example #41
0
def get_dbinfo(
    ibs,
    verbose=True,
    with_imgsize=False,
    with_bytes=False,
    with_contrib=False,
    with_agesex=False,
    with_header=True,
    short=False,
    tag='dbinfo',
    aid_list=None,
    aids=None,
):
    """

    Returns dictionary of digestable database information
    Infostr is a string summary of all the stats. Prints infostr in addition to
    returning locals

    Args:
        ibs (IBEISController):
        verbose (bool):
        with_imgsize (bool):
        with_bytes (bool):

    Returns:
        dict:

    SeeAlso:
        python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --use-hist=True --old=False --per_name_vpedge=False
        python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --all

    CommandLine:
        python -m wbia.other.dbinfo --exec-get_dbinfo:0
        python -m wbia.other.dbinfo --test-get_dbinfo:1
        python -m wbia.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3
        python -m wbia.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1
        python -m wbia.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL
        python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints
        python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1

        python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db LF_Bajo_bonito -a default
        python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db DETECT_SEATURTLES -a default --readonly

        python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a ctrl
        python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA
        python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0

        python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA
        python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0

    Example1:
        >>> # SCRIPT
        >>> from wbia.other.dbinfo import *  # NOQA
        >>> import wbia
        >>> defaultdb = 'testdb1'
        >>> ibs, aid_list = wbia.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1')
        >>> kwargs = ut.get_kwdefaults(get_dbinfo)
        >>> kwargs['verbose'] = False
        >>> kwargs['aid_list'] = aid_list
        >>> kwargs = ut.parse_dict_from_argv(kwargs)
        >>> output = get_dbinfo(ibs, **kwargs)
        >>> result = (output['info_str'])
        >>> print(result)
        >>> #ibs = wbia.opendb(defaultdb='testdb1')
        >>> # <HACK FOR FILTERING>
        >>> #from wbia.expt import cfghelpers
        >>> #from wbia.expt import annotation_configs
        >>> #from wbia.init import filter_annots
        >>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
        >>> #                                   annotation_configs.TEST_NAMES)
        >>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
        >>> #                               ut.get_list_column(named_defaults_dict, 'qcfg')))
        >>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0]
        >>> #aid_list = ibs.get_valid_aids()
        >>> # </HACK FOR FILTERING>

    Example1:
        >>> # ENABLE_DOCTEST
        >>> from wbia.other.dbinfo import *  # NOQA
        >>> import wbia
        >>> verbose = True
        >>> short = True
        >>> #ibs = wbia.opendb(db='GZ_ALL')
        >>> #ibs = wbia.opendb(db='PZ_Master0')
        >>> ibs = wbia.opendb('testdb1')
        >>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS'
        >>> ibs.delete_contributors(ibs.get_valid_contributor_rowids())
        >>> ibs.delete_empty_nids()
        >>> #ibs = wbia.opendb(db='PZ_MTEST')
        >>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True)
        >>> result = (output['info_str'])
        >>> print(result)
        +============================
        DB Info:  testdb1
        DB Notes: None
        DB NumContrib: 0
        ----------
        # Names                      = 7
        # Names (unassociated)       = 0
        # Names (singleton)          = 5
        # Names (multiton)           = 2
        ----------
        # Annots                     = 13
        # Annots (unknown)           = 4
        # Annots (singleton)         = 5
        # Annots (multiton)          = 4
        ----------
        # Img                        = 13
        L============================
    """
    # TODO Database size in bytes
    # TODO: occurrence, contributors, etc...
    if aids is not None:
        aid_list = aids

    # Basic variables
    request_annot_subset = False
    _input_aid_list = aid_list  # NOQA
    if aid_list is None:
        valid_aids = ibs.get_valid_aids()
        valid_nids = ibs.get_valid_nids()
        valid_gids = ibs.get_valid_gids()
    else:
        if isinstance(aid_list, str):
            # Hack to get experiment stats on aids
            acfg_name_list = [aid_list]
            logger.info('Specified custom aids via acfgname %s' % (acfg_name_list,))
            from wbia.expt import experiment_helpers

            acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
                ibs, acfg_name_list
            )
            aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list)))))
            # aid_list =
        if verbose:
            logger.info('Specified %d custom aids' % (len(aid_list,)))
        request_annot_subset = True
        valid_aids = aid_list
        valid_nids = list(
            set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False))
            - {const.UNKNOWN_NAME_ROWID}
        )
        valid_gids = list(set(ibs.get_annot_gids(aid_list)))
    # associated_nids = ibs.get_valid_nids(filter_empty=True)  # nids with at least one annotation
    valid_images = ibs.images(valid_gids)
    valid_annots = ibs.annots(valid_aids)

    # Image info
    if verbose:
        logger.info('Checking Image Info')
    gx2_aids = valid_images.aids
    if request_annot_subset:
        # remove annots not in this subset
        valid_aids_set = set(valid_aids)
        gx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in gx2_aids]

    gx2_nAnnots = np.array(list(map(len, gx2_aids)))
    image_without_annots = len(np.where(gx2_nAnnots == 0)[0])
    gx2_nAnnots_stats = ut.repr4(
        ut.get_stats(gx2_nAnnots, use_median=True), nl=0, precision=2, si=True
    )
    image_reviewed_list = ibs.get_image_reviewed(valid_gids)

    # Name stats
    if verbose:
        logger.info('Checking Name Info')
    nx2_aids = ibs.get_name_aids(valid_nids)
    if request_annot_subset:
        # remove annots not in this subset
        valid_aids_set = set(valid_aids)
        nx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in nx2_aids]
    associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids)))

    ibs.check_name_mapping_consistency(nx2_aids)

    if False:
        # Occurrence Info
        def compute_annot_occurrence_ids(ibs, aid_list):
            from wbia.algo.preproc import preproc_occurrence

            gid_list = ibs.get_annot_gids(aid_list)
            gid2_aids = ut.group_items(aid_list, gid_list)
            config = {'seconds_thresh': 4 * 60 * 60}
            flat_imgsetids, flat_gids = preproc_occurrence.wbia_compute_occurrences(
                ibs, gid_list, config=config, verbose=False
            )
            occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
            occurid2_aids = {
                oid: ut.flatten(ut.take(gid2_aids, gids))
                for oid, gids in occurid2_gids.items()
            }
            return occurid2_aids

        import utool

        with utool.embed_on_exception_context:
            occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids)
            occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values())
            occur_unique_nids = [ut.unique(nids) for nids in occur_nids]
            nid2_occurxs = ut.ddict(list)
            for occurx, nids in enumerate(occur_unique_nids):
                for nid in nids:
                    nid2_occurxs[nid].append(occurx)

        nid2_occurx_single = {
            nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1
        }
        nid2_occurx_resight = {
            nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1
        }
        singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys())

        singlesight_annot_stats = ut.get_stats(
            list(map(len, singlesight_encounters)), use_median=True, use_sum=True
        )
        resight_name_stats = ut.get_stats(
            list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True
        )

    # Encounter Info
    def break_annots_into_encounters(aids):
        from wbia.algo.preproc import occurrence_blackbox
        import datetime

        thresh_sec = datetime.timedelta(minutes=30).seconds
        posixtimes = np.array(ibs.get_annot_image_unixtimes_asfloat(aids))
        # latlons = ibs.get_annot_image_gps(aids)
        labels = occurrence_blackbox.cluster_timespace2(
            posixtimes, None, thresh_sec=thresh_sec
        )
        return labels
        # ave_enc_time = [np.mean(times) for lbl, times in ut.group_items(posixtimes, labels).items()]
        # ut.square_pdist(ave_enc_time)

    try:
        am_rowids = ibs.get_annotmatch_rowids_between_groups([valid_aids], [valid_aids])[
            0
        ]
        aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0, am_rowids=am_rowids)
        undirected_tags = ibs.get_aidpair_tags(
            aid_pairs.T[0], aid_pairs.T[1], directed=False
        )
        tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags))
        tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags)
        pair_tag_info = ut.map_dict_vals(len, tag_dict)
    except Exception:
        pair_tag_info = {}

    # logger.info(ut.repr2(pair_tag_info))

    # Annot Stats
    # TODO: number of images where chips cover entire image
    # TODO: total image coverage of annotation
    # TODO: total annotation overlap
    """
    ax2_unknown = ibs.is_aid_unknown(valid_aids)
    ax2_nid = ibs.get_annot_name_rowids(valid_aids)
    assert all([nid < 0 if unknown else nid > 0 for nid, unknown in
                zip(ax2_nid, ax2_unknown)]), 'bad annot nid'
    """
    #
    if verbose:
        logger.info('Checking Annot Species')
    unknown_annots = valid_annots.compress(ibs.is_aid_unknown(valid_annots))
    species_list = valid_annots.species_texts
    species2_annots = valid_annots.group_items(valid_annots.species_texts)
    species2_nAids = {key: len(val) for key, val in species2_annots.items()}

    if verbose:
        logger.info('Checking Multiton/Singleton Species')
    nx2_nAnnots = np.array(list(map(len, nx2_aids)))
    # Seperate singleton / multitons
    multiton_nxs = np.where(nx2_nAnnots > 1)[0]
    singleton_nxs = np.where(nx2_nAnnots == 1)[0]
    unassociated_nxs = np.where(nx2_nAnnots == 0)[0]
    assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names'
    valid_nxs = np.hstack([multiton_nxs, singleton_nxs])
    num_names_with_gt = len(multiton_nxs)

    # Annot Info
    if verbose:
        logger.info('Checking Annot Info')
    multiton_aids_list = ut.take(nx2_aids, multiton_nxs)
    assert len(set(multiton_nxs)) == len(multiton_nxs)
    if len(multiton_aids_list) == 0:
        multiton_aids = np.array([], dtype=np.int)
    else:
        multiton_aids = np.hstack(multiton_aids_list)
        assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot'
    singleton_aids = ut.take(nx2_aids, singleton_nxs)
    multiton_nid2_nannots = list(map(len, multiton_aids_list))

    # Image size stats
    if with_imgsize:
        if verbose:
            logger.info('Checking ImageSize Info')
        gpath_list = ibs.get_image_paths(valid_gids)

        def wh_print_stats(wh_list):
            if len(wh_list) == 0:
                return '{empty}'
            wh_list = np.asarray(wh_list)
            stat_dict = collections.OrderedDict(
                [
                    ('max', wh_list.max(0)),
                    ('min', wh_list.min(0)),
                    ('mean', wh_list.mean(0)),
                    ('std', wh_list.std(0)),
                ]
            )

            def arr2str(var):
                return '[' + (', '.join(list(map(lambda x: '%.1f' % x, var)))) + ']'

            ret = ',\n    '.join(
                ['%s:%s' % (key, arr2str(val)) for key, val in stat_dict.items()]
            )
            return '{\n    ' + ret + '\n}'

        logger.info('reading image sizes')
        # Image size stats
        img_size_list = ibs.get_image_sizes(valid_gids)
        img_size_stats = wh_print_stats(img_size_list)

        # Chip size stats
        annotation_bbox_list = ibs.get_annot_bboxes(valid_aids)
        annotation_bbox_arr = np.array(annotation_bbox_list)
        if len(annotation_bbox_arr) == 0:
            annotation_size_list = []
        else:
            annotation_size_list = annotation_bbox_arr[:, 2:4]
        chip_size_stats = wh_print_stats(annotation_size_list)
        imgsize_stat_lines = [
            (' # Img in dir                 = %d' % len(gpath_list)),
            (' Image Size Stats  = %s' % (img_size_stats,)),
            (' * Chip Size Stats = %s' % (chip_size_stats,)),
        ]
    else:
        imgsize_stat_lines = []

    if verbose:
        logger.info('Building Stats String')

    multiton_stats = ut.repr3(
        ut.get_stats(multiton_nid2_nannots, use_median=True), nl=0, precision=2, si=True
    )

    # Time stats
    unixtime_list = valid_images.unixtime2
    # valid_unixtime_list = [time for time in unixtime_list if time != -1]
    # unixtime_statstr = ibs.get_image_time_statstr(valid_gids)
    if ut.get_argflag('--hackshow-unixtime'):
        show_time_distributions(ibs, unixtime_list)
        ut.show_if_requested()
    unixtime_statstr = ut.repr3(ut.get_timestats_dict(unixtime_list, full=True), si=True)

    # GPS stats
    gps_list_ = ibs.get_image_gps(valid_gids)
    gpsvalid_list = [gps != (-1, -1) for gps in gps_list_]
    gps_list = ut.compress(gps_list_, gpsvalid_list)

    def get_annot_age_stats(aid_list):
        annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list)
        annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list)
        age_dict = ut.ddict((lambda: 0))
        for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max):
            if max_age is None:
                max_age = min_age
            if min_age is None:
                min_age = max_age
            if max_age is None and min_age is None:
                logger.info('Found UNKNOWN Age: %r, %r' % (min_age, max_age,))
                age_dict['UNKNOWN'] += 1
            elif (min_age is None or min_age < 12) and max_age < 12:
                age_dict['Infant'] += 1
            elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36:
                age_dict['Juvenile'] += 1
            elif 36 <= min_age and (max_age is None or 36 <= max_age):
                age_dict['Adult'] += 1
        return age_dict

    def get_annot_sex_stats(aid_list):
        annot_sextext_list = ibs.get_annot_sex_texts(aid_list)
        sextext2_aids = ut.group_items(aid_list, annot_sextext_list)
        sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys())
        assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str(
            set(annot_sextext_list) - set(sex_keys)
        )
        sextext2_nAnnots = ut.odict(
            [(key, len(sextext2_aids.get(key, []))) for key in sex_keys]
        )
        # Filter 0's
        sextext2_nAnnots = {
            key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0
        }
        return sextext2_nAnnots

    def get_annot_qual_stats(ibs, aid_list):
        annots = ibs.annots(aid_list)
        qualtext2_nAnnots = ut.order_dict_by(
            ut.map_vals(len, annots.group_items(annots.quality_texts)),
            list(ibs.const.QUALITY_TEXT_TO_INT.keys()),
        )
        return qualtext2_nAnnots

    def get_annot_viewpoint_stats(ibs, aid_list):
        annots = ibs.annots(aid_list)
        viewcode2_nAnnots = ut.order_dict_by(
            ut.map_vals(len, annots.group_items(annots.viewpoint_code)),
            list(ibs.const.VIEW.CODE_TO_INT.keys()) + [None],
        )
        return viewcode2_nAnnots

    if verbose:
        logger.info('Checking Other Annot Stats')

    qualtext2_nAnnots = get_annot_qual_stats(ibs, valid_aids)
    viewcode2_nAnnots = get_annot_viewpoint_stats(ibs, valid_aids)
    agetext2_nAnnots = get_annot_age_stats(valid_aids)
    sextext2_nAnnots = get_annot_sex_stats(valid_aids)

    if verbose:
        logger.info('Checking Contrib Stats')

    # Contributor Statistics
    # hack remove colon for image alignment
    def fix_tag_list(tag_list):
        return [None if tag is None else tag.replace(':', ';') for tag in tag_list]

    image_contributor_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids))
    annot_contributor_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids))
    contributor_tag_to_gids = ut.group_items(valid_gids, image_contributor_tags)
    contributor_tag_to_aids = ut.group_items(valid_aids, annot_contributor_tags)

    contributor_tag_to_qualstats = {
        key: get_annot_qual_stats(ibs, aids)
        for key, aids in six.iteritems(contributor_tag_to_aids)
    }
    contributor_tag_to_viewstats = {
        key: get_annot_viewpoint_stats(ibs, aids)
        for key, aids in six.iteritems(contributor_tag_to_aids)
    }

    contributor_tag_to_nImages = {
        key: len(val) for key, val in six.iteritems(contributor_tag_to_gids)
    }
    contributor_tag_to_nAnnots = {
        key: len(val) for key, val in six.iteritems(contributor_tag_to_aids)
    }

    if verbose:
        logger.info('Summarizing')

    # Summarize stats
    num_names = len(valid_nids)
    num_names_unassociated = len(valid_nids) - len(associated_nids)
    num_names_singleton = len(singleton_nxs)
    num_names_multiton = len(multiton_nxs)

    num_singleton_annots = len(singleton_aids)
    num_multiton_annots = len(multiton_aids)
    num_unknown_annots = len(unknown_annots)
    num_annots = len(valid_aids)

    if with_bytes:
        if verbose:
            logger.info('Checking Disk Space')
        ibsdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir()))
        dbdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir()))
        imgdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir()))
        cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir()))

    if True:
        if verbose:
            logger.info('Check asserts')
        try:
            bad_aids = np.intersect1d(multiton_aids, unknown_annots)
            _num_names_total_check = (
                num_names_singleton + num_names_unassociated + num_names_multiton
            )
            _num_annots_total_check = (
                num_unknown_annots + num_singleton_annots + num_multiton_annots
            )
            assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids'
            assert _num_names_total_check == num_names, 'inconsistent num names'
            # if not request_annot_subset:
            # dont check this if you have an annot subset
            assert _num_annots_total_check == num_annots, 'inconsistent num annots'
        except Exception as ex:
            ut.printex(
                ex,
                keys=[
                    '_num_names_total_check',
                    'num_names',
                    '_num_annots_total_check',
                    'num_annots',
                    'num_names_singleton',
                    'num_names_multiton',
                    'num_unknown_annots',
                    'num_multiton_annots',
                    'num_singleton_annots',
                ],
            )
            raise

    # Get contributor statistics
    contributor_rowids = ibs.get_valid_contributor_rowids()
    num_contributors = len(contributor_rowids)

    # print
    num_tabs = 5

    def align2(str_):
        return ut.align(str_, ':', ' :')

    def align_dict2(dict_):
        str_ = ut.repr2(dict_, si=True)
        return align2(str_)

    header_block_lines = [('+============================')] + (
        [
            ('+ singleton := single sighting'),
            ('+ multiton  := multiple sightings'),
            ('--' * num_tabs),
        ]
        if not short and with_header
        else []
    )

    source_block_lines = [
        ('DB Info:  ' + ibs.get_dbname()),
        ('DB Notes: ' + ibs.get_dbnotes()),
        ('DB NumContrib: %d' % num_contributors),
    ]

    bytes_block_lines = (
        [
            ('--' * num_tabs),
            ('DB Bytes: '),
            ('     +- dbdir nBytes:         ' + dbdir_space),
            ('     |  +- _ibsdb nBytes:     ' + ibsdir_space),
            ('     |  |  +-imgdir nBytes:   ' + imgdir_space),
            ('     |  |  +-cachedir nBytes: ' + cachedir_space),
        ]
        if with_bytes
        else []
    )

    name_block_lines = [
        ('--' * num_tabs),
        ('# Names                      = %d' % num_names),
        ('# Names (unassociated)       = %d' % num_names_unassociated),
        ('# Names (singleton)          = %d' % num_names_singleton),
        ('# Names (multiton)           = %d' % num_names_multiton),
    ]

    subset_str = '        ' if not request_annot_subset else '(SUBSET)'

    annot_block_lines = [
        ('--' * num_tabs),
        ('# Annots %s            = %d' % (subset_str, num_annots,)),
        ('# Annots (unknown)           = %d' % num_unknown_annots),
        ('# Annots (singleton)         = %d' % num_singleton_annots),
        ('# Annots (multiton)          = %d' % num_multiton_annots),
    ]

    annot_per_basic_block_lines = (
        [
            ('--' * num_tabs),
            ('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)),
            ('# Annots per Image           = %s' % (align2(gx2_nAnnots_stats),)),
            ('# Annots per Species         = %s' % (align_dict2(species2_nAids),)),
        ]
        if not short
        else []
    )

    occurrence_block_lines = (
        [
            ('--' * num_tabs),
            # ('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)),
            # ('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)),
            ('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)),
        ]
        if not short
        else []
    )

    annot_per_qualview_block_lines = [
        None if short else '# Annots per Viewpoint = %s' % align_dict2(viewcode2_nAnnots),
        None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots),
    ]

    annot_per_agesex_block_lines = (
        [
            '# Annots per Age = %s' % align_dict2(agetext2_nAnnots),
            '# Annots per Sex = %s' % align_dict2(sextext2_nAnnots),
        ]
        if not short and with_agesex
        else []
    )

    contributor_block_lines = (
        [
            '# Images per contributor       = ' + align_dict2(contributor_tag_to_nImages),
            '# Annots per contributor       = ' + align_dict2(contributor_tag_to_nAnnots),
            '# Quality per contributor      = '
            + ut.repr2(contributor_tag_to_qualstats, sorted_=True),
            '# Viewpoint per contributor    = '
            + ut.repr2(contributor_tag_to_viewstats, sorted_=True),
        ]
        if with_contrib
        else []
    )

    img_block_lines = [
        ('--' * num_tabs),
        ('# Img                        = %d' % len(valid_gids)),
        None
        if short
        else ('# Img reviewed               = %d' % sum(image_reviewed_list)),
        None if short else ('# Img with gps               = %d' % len(gps_list)),
        # ('# Img with timestamp         = %d' % len(valid_unixtime_list)),
        None
        if short
        else ('Img Time Stats               = %s' % (align2(unixtime_statstr),)),
    ]

    info_str_lines = (
        header_block_lines
        + bytes_block_lines
        + source_block_lines
        + name_block_lines
        + annot_block_lines
        + annot_per_basic_block_lines
        + occurrence_block_lines
        + annot_per_qualview_block_lines
        + annot_per_agesex_block_lines
        + img_block_lines
        + contributor_block_lines
        + imgsize_stat_lines
        + [('L============================')]
    )
    info_str = '\n'.join(ut.filter_Nones(info_str_lines))
    info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag))
    if verbose:
        logger.info(info_str2)
    locals_ = locals()
    return locals_
Example #42
0
 def stagger_group(list_):
     return ut.filter_Nones(ut.iflatten(zip_longest(*list_)))
Example #43
0
def make_netx_graph_from_aid_groups(ibs, aids_list, only_reviewed_matches=True,
                                    invis_edges=None, ensure_edges=None,
                                    temp_nids=None, allow_directed=False):
    r"""
    Args:
        ibs (ibeis.IBEISController): image analysis api
        aids_list (list):

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.viz.viz_graph import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(defaultdb='testdb1')
        >>> aids_list = [[1, 2, 3, 4], [5, 6, 7]]
        >>> invis_edges = [(1, 5)]
        >>> only_reviewed_matches = True
        >>> graph = make_netx_graph_from_aid_groups(ibs, aids_list,
        >>>                                         only_reviewed_matches,
        >>>                                         invis_edges)
        >>> list(nx.connected_components(graph.to_undirected()))
    """
    #aids_list, nid_list = ibs.group_annots_by_name(aid_list)
    unique_aids = list(ut.flatten(aids_list))

    # grouped version
    unflat_edges = (list(itertools.product(aids, aids)) for aids in aids_list)
    aid_pairs = [tup for tup in ut.iflatten(unflat_edges) if tup[0] != tup[1]]
    aids1 = ut.get_list_column(aid_pairs, 0)
    aids2 = ut.get_list_column(aid_pairs, 1)

    if only_reviewed_matches:
        annotmatch_rowids = ibs.get_annotmatch_rowid_from_superkey(aids1, aids2)
        annotmatch_rowids = ut.filter_Nones(annotmatch_rowids)
        aids1 = ibs.get_annotmatch_aid1(annotmatch_rowids)
        aids2 = ibs.get_annotmatch_aid2(annotmatch_rowids)

    graph = make_netx_graph_from_aidpairs(ibs, aids1, aids2, unique_aids=unique_aids)

    if ensure_edges is not None:
        if ensure_edges == 'all':
            ensure_edges = list(ut.upper_diag_self_prodx(list(graph.nodes())))
        ensure_edges_ = []
        for edge in ensure_edges:
            edge = tuple(edge)
            redge = tuple(edge[::-1])  # HACK
            if graph.has_edge(*edge):
                ensure_edges_.append(edge)
                pass
                #nx.set_edge_attributes(graph, 'weight', {edge: .001})
            elif (not allow_directed) and graph.has_edge(*redge):
                ensure_edges_.append(redge)
                #nx.set_edge_attributes(graph, 'weight', {redge: .001})
                pass
            else:
                ensure_edges_.append(edge)
                #graph.add_edge(*edge, weight=.001)
                graph.add_edge(*edge)

    if temp_nids is None:
        unique_nids = ibs.get_annot_nids(list(graph.nodes()))
    else:
        # HACK
        unique_nids = [1] * len(list(graph.nodes()))
        #unique_nids = temp_nids

    nx.set_node_attributes(graph, 'nid', dict(zip(graph.nodes(), unique_nids)))

    import plottool as pt
    ensure_names_are_connected(graph, aids_list)

    # Color edges by nid
    color_by_nids(graph, unique_nids=unique_nids)
    if invis_edges:
        for edge in invis_edges:
            if graph.has_edge(*edge):
                nx.set_edge_attributes(graph, 'style', {edge: 'invis'})
                nx.set_edge_attributes(graph, 'invisible', {edge: True})
            else:
                graph.add_edge(*edge, style='invis', invisible=True)

    # Hack color images orange
    if ensure_edges:
        nx.set_edge_attributes(graph, 'color',
                               {tuple(edge): pt.ORANGE for edge in ensure_edges_})

    return graph
Example #44
0
def show_name_matches(ibs,
                      qaid,
                      name_daid_list,
                      name_fm_list,
                      name_fs_list,
                      name_H1_list,
                      name_featflag_list,
                      qreq_=None,
                      **kwargs):
    """
    Called from chip_match.py

    Args:
        ibs (IBEISController):  ibeis controller object
        qaid (int):  query annotation id
        name_daid_list (list):
        name_fm_list (list):
        name_fs_list (list):
        name_H1_list (list):
        name_featflag_list (list):
        qreq_ (QueryRequest):  query request object with hyper-parameters(default = None)

    Kwargs:
        draw_fmatches, name_rank, fnum, pnum, colorbar_, nonvote_mode,
        fastmode, show_matches, fs, fm_norm, lbl1, lbl2, rect, draw_border,
        cmap, H1, H2, scale_factor1, scale_factor2, draw_pts, draw_ell,
        draw_lines, show_nMatches, all_kpts, in_image, show_query, draw_lbl,
        name_annot_scores, score, rawscore, aid2_raw_rank, show_name,
        show_nid, show_aid, show_annot_score, show_truth, name_score,
        show_name_score, show_name_rank, show_timedelta

    CommandLine:
        python -m ibeis.viz.viz_matches --exec-show_name_matches
        python -m ibeis.viz.viz_matches --test-show_name_matches --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.viz.viz_matches import *  # NOQA
        >>> from ibeis.algo.hots import chip_match
        >>> from ibeis.algo.hots import name_scoring
        >>> import vtool as vt
        >>> from ibeis.algo.hots import _pipeline_helpers as plh  # NOQA
        >>> import numpy as np
        >>> func = chip_match.ChipMatch.show_single_namematch
        >>> sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True,
        >>>                                     strip_docstr=True)
        >>> setup = ut.regex_replace('viz_matches.show_name_matches', '#', sourcecode)
        >>> homog = False
        >>> print(ut.indent(setup, '>>> '))
        >>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[1])
        >>> cm = cm_list[0]
        >>> cm.score_nsum(qreq_)
        >>> dnid = ibs.get_annot_nids(cm.qaid)
        >>> # +--- COPIED SECTION
        >>> locals_ = locals()
        >>> var_list = ut.exec_func_src(
        >>>     func, locals_=locals_,
        >>>     sentinal='name_annot_scores = cm.annot_score_list.take(sorted_groupxs')
        >>> exec(ut.execstr_dict(var_list))
        >>> # L___ COPIED SECTION
        >>> kwargs = {}
        >>> show_name_matches(ibs, qaid, name_daid_list, name_fm_list,
        >>>                   name_fs_list, name_h1_list, name_featflag_list,
        >>>                   qreq_=qreq_, **kwargs)
        >>> ut.quit_if_noshow()
        >>> ut.show_if_requested()
    """
    #print("SHOW NAME MATCHES")
    #print(ut.repr2(kwargs, nl=True))
    #from ibeis import constants as const
    from ibeis import tag_funcs
    draw_fmatches = kwargs.pop('draw_fmatches', True)
    rchip1, kpts1 = get_query_annot_pair_info(ibs, qaid, qreq_, draw_fmatches)
    rchip2_list, kpts2_list = get_data_annot_pair_info(ibs, name_daid_list,
                                                       qreq_, draw_fmatches)
    fm_list = name_fm_list
    fs_list = name_fs_list
    featflag_list = name_featflag_list
    offset_list, sf_list, bbox_list = show_multichip_match(
        rchip1, rchip2_list, kpts1, kpts2_list, fm_list, fs_list,
        featflag_list, **kwargs)
    aid_list = [qaid] + name_daid_list
    annotate_matches3(ibs,
                      aid_list,
                      bbox_list,
                      offset_list,
                      name_fm_list,
                      name_fs_list,
                      qreq_=None,
                      **kwargs)
    ax = pt.gca()
    title = vh.get_query_text(ibs,
                              None,
                              name_daid_list,
                              False,
                              qaid=qaid,
                              **kwargs)

    pt.set_title(title, ax)

    # Case tags
    annotmatch_rowid_list = ibs.get_annotmatch_rowid_from_superkey(
        [qaid] * len(name_daid_list), name_daid_list)
    annotmatch_rowid_list = ut.filter_Nones(annotmatch_rowid_list)
    tags_list = ibs.get_annotmatch_case_tags(annotmatch_rowid_list)
    if not ut.get_argflag('--show'):  # False:
        tags_list = tag_funcs.consolodate_annotmatch_tags(tags_list)
    tag_list = ut.unique_ordered(ut.flatten(tags_list))

    name_rank = kwargs.get('name_rank', None)
    truth = get_multitruth(ibs, aid_list)

    xlabel = {1: 'Correct ID', 0: 'Incorrect ID', 2: 'Unknown ID'}[truth]

    if False:
        if name_rank is None:
            xlabel = {1: 'Genuine', 0: 'Imposter', 2: 'Unknown'}[truth]
            #xlabel = {1: 'True', 0: 'False', 2: 'Unknown'}[truth]
        else:
            if name_rank == 0:
                xlabel = {
                    1: 'True Positive',
                    0: 'False Positive',
                    2: 'Unknown'
                }[truth]
            else:
                xlabel = {
                    1: 'False Negative',
                    0: 'True Negative',
                    2: 'Unknown'
                }[truth]

    if len(tag_list) > 0:
        xlabel += '\n' + ', '.join(tag_list)

    pt.set_xlabel(xlabel)
    return ax
Example #45
0
def make_score_tabular(
        row_lbls, col_lbls, values, title=None, out_of=None, bold_best=False,
        flip=False, bigger_is_better=True, multicol_lbls=None, FORCE_INT=False,
        precision=None, SHORTEN_ROW_LBLS=False, col_align='l', col_sep='|',
        multicol_sep='|', centerline=True, astable=False, table_position='',
        AUTOFIX_LATEX=True, **kwargs):
    r"""
    makes a LaTeX tabular for displaying scores or errors

    Args:
        row_lbls (list of str):
        col_lbls (list of str):
        values (ndarray):
        title (str):  (default = None)
        out_of (None): (default = None)
        bold_best (bool): (default = True)
        flip (bool): (default = False)
        table_position (str) : eg '[h]'

    Returns:
        str: tabular_str

    CommandLine:
        python -m utool.util_latex --test-make_score_tabular:0 --show
        python -m utool.util_latex --test-make_score_tabular:1 --show
        python -m utool.util_latex --test-make_score_tabular:2 --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_latex import *  # NOQA
        >>> import utool as ut
        >>> row_lbls = ['config1', 'config2']
        >>> col_lbls = ['score \leq 1', 'metric2']
        >>> values = np.array([[1.2, 2], [3.2, 4]])
        >>> title = 'title'
        >>> out_of = 10
        >>> bold_best = True
        >>> flip = False
        >>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
        >>> result = tabular_str
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> render_latex_text(tabular_str)

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_latex import *  # NOQA
        >>> import utool as ut
        >>> row_lbls = ['config1']
        >>> col_lbls = ['score \leq 1', 'metric2']
        >>> values = np.array([[1.2, 2]])
        >>> title = 'title'
        >>> out_of = 10
        >>> bold_best = True
        >>> flip = False
        >>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
        >>> result = tabular_str
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> render_latex_text(tabular_str)

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_latex import *  # NOQA
        >>> import utool as ut
        >>> row_lbls = ['config1', 'config2']
        >>> col_lbls = ['score \leq 1', 'metric2', 'foobar']
        >>> multicol_lbls = [('spam', 1), ('eggs', 2)]
        >>> values = np.array([[1.2, 2, -3], [3.2, 4, -2]])
        >>> title = 'title'
        >>> out_of = 10
        >>> bold_best = True
        >>> flip = False
        >>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip, multicol_lbls=multicol_lbls)
        >>> result = tabular_str
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> render_latex_text(tabular_str)
    """
    import utool as ut
    if flip:
        bigger_is_better = not bigger_is_better
        flip_repltups = [
            ('<=', '>'),
            ('>', '<='),
            ('\\leq', '\\gt'),
            ('\\geq', '\\lt'),
            ('score', 'error')
        ]
        col_lbls = [replace_all(lbl, flip_repltups) for lbl in col_lbls]
        if title is not None:
            title = replace_all(title, flip_repltups)
        if out_of is not None:
            values = out_of - values

    # Abbreviate based on common substrings
    common_rowlbl = None
    if SHORTEN_ROW_LBLS:
        if isinstance(row_lbls, list):
            row_lbl_list = row_lbls
        else:
            row_lbl_list = row_lbls.flatten().tolist()
        # Split the rob labels into the alg components
        #algcomp_list = [lbl.split(')_') for lbl in row_lbl_list]
        longest = long_substr(row_lbl_list)
        common_strs = []
        while len(longest) > 10:
            common_strs += [longest]
            row_lbl_list = [row.replace(longest, '...') for row in row_lbl_list]
            longest = long_substr(row_lbl_list)
        common_rowlbl = ('...'.join(common_strs)).replace(')_', ')_\n')
        row_lbls = row_lbl_list
        if len(row_lbl_list) == 1:
            common_rowlbl = row_lbl_list[0]
            row_lbls = ['0']

    # Stack values into a tabular body
    # TODO: need ability to specify datatypes
    def ensurelist(row_values):
        try:
            return row_values.tolist()
        except AttributeError:
            return row_values

    if False:
        # Numpy formatting
        def padvec(shape=(1, 1)):
            pad = np.array([[' ' for c in range(shape[1])] for r in range(shape[0])])
            return pad
        col_lbls = ensure_rowvec(col_lbls)
        row_lbls = ensure_colvec(row_lbls)
        _0 = np.vstack([padvec(), row_lbls])
        _1 = np.vstack([col_lbls, values])
        body = np.hstack([_0, _1])
        body = [[str_ for str_ in row] for row in body]
    else:
        assert len(row_lbls) == len(values)
        body = [[' '] + col_lbls]
        body += [[row_lbl] + ensurelist(row_values) for row_lbl, row_values in zip(row_lbls, values)]
    #import utool as ut
    # Fix things in each body cell
    DO_PERCENT = True
    try:
        for r in range(len(body)):
            for c in range(len(body[0])):
                # In data land
                if r > 0 and c > 0:
                    if precision is not None:
                        # Hack
                        if ut.is_float(body[r][c]):
                            fmtstr = '%.' + str(precision) + 'f'
                            body[r][c] = fmtstr % (float(body[r][c]),)
                    # Force integer
                    if FORCE_INT:
                        body[r][c] = str(int(float(body[r][c])))
                body[r][c] = str(body[r][c])
                # Remove bad formatting;
                if AUTOFIX_LATEX:
                    body[r][c] = escape_latex(body[r][c])
    except Exception as ex:
        import utool as ut
        print('len(row_lbls) = %r' % (len(row_lbls),))
        print('len(col_lbls) = %r' % (len(col_lbls),))
        print('len(values) = %r' % (values,))
        print('ut.depth_profile(values) = %r' % (ut.depth_profile(values),))
        util_dbg.printex(ex, keys=['r', 'c'])
        raise

    # Bold the best values
    if bold_best:
        best_col_scores = values.max(0) if bigger_is_better else values.min(0)
        rows_to_bold = [np.where(values[:, colx] == best_col_scores[colx])[0]
                        for colx in range(len(values.T))]
        for colx, rowx_list in enumerate(rows_to_bold):
            for rowx in rowx_list:
                body[rowx + 1][colx + 1] = '\\txtbf{' + body[rowx + 1][colx + 1] + '}'

    # More fixing after the bold is in place
    for r in range(len(body)):
        for c in range(len(body[0])):
            # In data land
            if r > 0 and c > 0:
                if out_of is not None:
                    body[r][c] = body[r][c] + '/' + str(out_of)
                    if DO_PERCENT:
                        percent = ' = %.1f%%' % float(100 * values[r - 1, c - 1] / out_of)
                        body[r][c] += escape_latex(percent)

    # Align columns for pretty printing
    body = np.array(body)
    ALIGN_BODY = True
    if ALIGN_BODY:
        new_body_cols = []
        for col in body.T:
            colstrs = list(map(str, ensurelist(col)))
            collens = list(map(len, colstrs))
            maxlen = max(collens)
            newcols = [str_ + (' ' * (maxlen - len(str_))) for str_ in colstrs]
            new_body_cols += [newcols]
        body = np.array(new_body_cols).T

    # Build Body (and row layout)
    HLINE_SEP = True
    rowvalsep = ''
    colvalsep = ' & '
    endl = '\\\\\n'
    hline = r'\hline'
    #extra_rowsep_pos_list = [1]  # rows to insert an extra hline after
    extra_rowsep_pos_list = []  # rows to insert an extra hline after
    if HLINE_SEP:
        rowvalsep = hline + '\n'
    # rowstr list holds blocks of rows
    rowstr_list = [colvalsep.join(row) + endl for row in body]
    #rowstr_list = [row[0] + rowlbl_sep + colvalsep.join(row[1:]) + endl for row in body]
    #rowstr_list = [(
    #    ('' if len(row) == 0 else row[0])
    #    if len(row) <= 1 else
    #    row[0] + rowlblcol_sep + colvalsep.join(row[1:]) + endl)
    #    for row in body]
    rowsep_list = [rowvalsep for row in rowstr_list[0:-1]]  # should be len 1 less than rowstr_list
    # Insert multicolumn names
    if multicol_lbls is not None:
        # TODO: label of the row labels
        multicol_sep
        multicols = [latex_multicolumn(multicol, size, 'c' + multicol_sep) for multicol, size in multicol_lbls]
        multicol_str = latex_multirow('', 2) + colvalsep + colvalsep.join(multicols) + endl
        ncols = sum([tup[1] for tup in multicol_lbls])
        mcol_sep = '\\cline{2-%d}\n' % (ncols + 1,)
        rowstr_list = [multicol_str] + rowstr_list
        rowsep_list = [mcol_sep] + rowsep_list
        #extra_rowsep_pos_list += [1]

    # Insert title
    if title is not None and not astable:
        tex_title = latex_multicolumn(title, len(body[0])) + endl
        rowstr_list = [tex_title] + rowstr_list
        rowsep_list = [rowvalsep] + rowsep_list
        #extra_rowsep_pos_list += [2]

    # Apply an extra hline (for label)
    #extra_rowsep_pos_list = []
    for pos in sorted(extra_rowsep_pos_list)[::-1]:
        rowstr_list.insert(pos, '')
        rowsep_list.insert(pos, rowvalsep)
    #tabular_body = rowvalsep.join(rowstr_list)
    from six.moves import zip_longest
    tabular_body = ''.join([row if sep is None else row + sep for row, sep in zip_longest(rowstr_list, rowsep_list)])

    # Build Column Layout
    col_align_list = [col_align] * len(body[0])
    #extra_collayoutsep_pos_list = [1]
    extra_collayoutsep_pos_list = []
    for pos in  sorted(extra_collayoutsep_pos_list)[::-1]:
        col_align_list.insert(pos, '')
    #col_layaout_sep_list = rowlblcol_sep  # TODO

    rowlblcol_sep = '|'
    # Build build internal seprations between column alignments
    # Defaults to just the normal col_sep
    col_align_sep_list = [col_sep] * (len(col_align_list) - 1)
    # Adjust for the separations between row labels and the actual row data
    if len(col_align_sep_list) > 0:
        col_align_sep_list[0] = rowlblcol_sep
    # Continue multicolumn sepratation
    if multicol_lbls is not None:
        multicol_offsets = ut.cumsum(ut.get_list_column(multicol_lbls, 1))
        for offset in multicol_offsets:
            if offset < len(col_align_sep_list):
                col_align_sep_list[offset] = multicol_sep

    from six.moves import zip_longest
    _tmp = [ut.filter_Nones(tup) for tup in zip_longest(col_align_list, col_align_sep_list)]
    col_layout = ''.join(ut.flatten(_tmp))

    #if len(col_align_list) > 1:
    #    col_layout = col_align_list[0] + rowlblcol_sep + col_sep.join(col_align_list[1:])
    #else:
    #    col_layout = col_sep.join(col_align_list)

    tabular_head = (r'\begin{tabular}{|%s|}' % col_layout) + '\n'
    tabular_tail = r'\end{tabular}'

    if centerline:
        tabular_head = r'\centerline{' + '\n' + tabular_head
        tabular_tail = tabular_tail + '}'

    if astable:
        #tabular_head = r'\begin{centering}' + '\n' + tabular_head
        tabular_head = r'\centering' + '\n' + tabular_head
        tabular_head = r'\begin{table}' + table_position + '\n' + tabular_head

        lblstr = latex_sanitize_command_name(kwargs.get('label', title))
        caption = title
        if AUTOFIX_LATEX:
            caption = escape_latex(caption)
        caption = '\n% ---\n' + caption + '\n% ---\n'
        #tabular_head = r'\end{centering}' + '\n' + tabular_head
        tabular_tail = tabular_tail + '\n\caption[%s]{%s}\n\label{tbl:%s}\n\end{table}' % (lblstr, caption, lblstr)

    tabular_str = rowvalsep.join([tabular_head, tabular_body, tabular_tail])
    topsep = '\\hline\n' if True else '\\toprule\n'
    botsep = '\\hline\n' if True else '\\bottomrule\n'
    tabular_str = tabular_head + topsep + tabular_body + botsep + tabular_tail

    if common_rowlbl is not None:
        #tabular_str += escape_latex('\n\nThe following parameters were held fixed:\n' + common_rowlbl)
        pass
    return tabular_str