Beispiel #1
0
def _init_ibeis(dbdir=None, verbose=None, use_cache=True, web=None, **kwargs):
    """
    Private function that calls code to create an ibeis controller
    """
    import utool as ut
    from ibeis import params
    from ibeis.control import IBEISControl
    if verbose is None:
        verbose = ut.VERBOSE
    if verbose and NOT_QUIET:
        print('[main] _init_ibeis()')
    # Use command line dbdir unless user specifies it
    if dbdir is None:
        ibs = None
        ut.printWARN('[main!] WARNING args.dbdir is None')
    else:
        kwargs = kwargs.copy()
        request_dbversion = kwargs.pop('request_dbversion', None)
        asproxy = kwargs.pop('asproxy', None)
        ibs = IBEISControl.request_IBEISController(
            dbdir=dbdir, use_cache=use_cache,
            request_dbversion=request_dbversion,
            asproxy=asproxy)
        if web is None:
            web = ut.get_argflag(('--webapp', '--webapi', '--web', '--browser'),
                                 help_='automatically launch the web app / web api')
            #web = params.args.webapp
        if web:
            from ibeis.web import app
            port = params.args.webport
            app.start_from_ibeis(ibs, port=port, **kwargs)
    return ibs
Beispiel #2
0
def _init_ibeis(dbdir=None, verbose=None, use_cache=True, web=None, **kwargs):
    """
    Private function that calls code to create an ibeis controller
    """
    import utool as ut
    from ibeis import params
    from ibeis.control import IBEISControl
    if verbose is None:
        verbose = ut.VERBOSE
    if verbose and NOT_QUIET:
        print('[main] _init_ibeis()')
    # Use command line dbdir unless user specifies it
    if dbdir is None:
        ibs = None
        ut.printWARN('[main!] WARNING args.dbdir is None')
    else:
        kwargs = kwargs.copy()
        request_dbversion = kwargs.pop('request_dbversion', None)
        asproxy = kwargs.pop('asproxy', None)
        ibs = IBEISControl.request_IBEISController(
            dbdir=dbdir,
            use_cache=use_cache,
            request_dbversion=request_dbversion,
            asproxy=asproxy)
        if web is None:
            web = ut.get_argflag(
                ('--webapp', '--webapi', '--web', '--browser'),
                help_='automatically launch the web app / web api')
            #web = params.args.webapp
        if web:
            from ibeis.web import app
            port = params.args.webport
            app.start_from_ibeis(ibs, port=port, **kwargs)
    return ibs
Beispiel #3
0
def merge_species_databases(species_prefix):
    """ Build a merged database """
    from ibeis.control import IBEISControl
    from ibeis.dev import sysres
    print('[ibsfuncs] Merging species with prefix: %r' % species_prefix)
    ut.util_parallel.ensure_pool(warn=False)
    with ut.Indenter('    '):
        # Build / get target database
        all_db = '__ALL_' + species_prefix + '_'
        all_dbdir = sysres.db_to_dbdir(all_db, allow_newdir=True)
        ibs_target = IBEISControl.IBEISController(all_dbdir)
        # Build list of databases to merge
        species_dbdir_list = get_species_dbs(species_prefix)
        ibs_source_list = []
        for dbdir in species_dbdir_list:
            ibs_source = IBEISControl.IBEISController(dbdir)
            ibs_source_list.append(ibs_source)
    print('[ibsfuncs] Destination database: %r' % all_db)
    print('[ibsfuncs] Source databases:' +
          ut.indentjoin(species_dbdir_list, '\n *   '))
    #Merge the databases into ibs_target
    merge_databases(ibs_target, ibs_source_list)
    return ibs_target
Beispiel #4
0
def ingest_standard_database(dbname, force_delete=False):
    """
    ingest_standard_database

    Args:
        dbname (str): database name
        force_delete (bool):

    Example:
        >>> from ibeis.dbio.ingest_database import *  # NOQA
        >>> dbname = 'testdb1'
        >>> force_delete = False
        >>> result = ingest_standard_database(dbname, force_delete)
        >>> print(result)
    """
    from ibeis.control import IBEISControl
    print('[ingest] Ingest Standard Database: dbname=%r' % (dbname,))
    ingestable = get_standard_ingestable(dbname)
    dbdir = ibeis.sysres.db_to_dbdir(ingestable.dbname, allow_newdir=True, use_sync=False)
    ut.ensuredir(dbdir, verbose=True)
    if force_delete:
        ibsfuncs.delete_ibeis_database(dbdir)
    ibs = IBEISControl.request_IBEISController(dbdir)
    ingest_rawdata(ibs, ingestable)
Beispiel #5
0
def convert_hsdb_to_ibeis(hsdb_dir, **kwargs):
    r"""
    Args:
        hsdb_dir (str):

    CommandLine:
        python -m ibeis.dbio.ingest_hsdb --test-convert_hsdb_to_ibeis:0 --db ~/work/Frogs

    Example:
        >>> # SCRIPT
        >>> from ibeis.dbio.ingest_hsdb import *  # NOQA
        >>> hsdb_dir = ut.get_argval('--dbdir', type_=str, default=None)
        >>> result = convert_hsdb_to_ibeis(hsdb_dir)
        >>> print(result)
    """
    from ibeis.control import IBEISControl
    import utool as ut
    assert is_hsdb(
        hsdb_dir
    ), 'not a hotspotter database. cannot even force convert: hsdb_dir=%r' % (
        hsdb_dir, )
    assert not is_succesful_convert(
        hsdb_dir), 'hsdb_dir=%r is already converted' % (hsdb_dir, )
    #print('FORCE DELETE: %r' % (hsdb_dir,))
    #ibsfuncs.delete_ibeis_database(hsdb_dir)
    print('[ingest] Ingesting hsdb: %r' % hsdb_dir)
    imgdir = join(hsdb_dir, 'images')

    ibs = IBEISControl.IBEISController(dbdir=hsdb_dir, **kwargs)

    # READ NAME TABLE
    names_name_list = ['____']
    name_nid_list = [0]

    internal_dir = get_hsinternal(hsdb_dir)

    with open(join(internal_dir, 'name_table.csv'), 'rb') as nametbl_file:
        name_reader = csv.reader(nametbl_file)
        for ix, row in enumerate(name_reader):
            #if ix >= 3:
            if len(row) == 0 or row[0].strip().startswith('#'):
                continue
            else:
                nid = int(row[0])
                name = row[1].strip()
                names_name_list.append(name)
                name_nid_list.append(nid)

    # ADD NAMES TABLE
    nid_list = ibs.add_names(names_name_list)
    #print(names_name_list)
    #print(nid_list)

    assert len(nid_list) == len(names_name_list), 'bad name adder'

    image_gid_list = []
    image_gname_list = []
    image_reviewed_list = []
    with open(join(internal_dir, 'image_table.csv'), 'rb') as imgtb_file:
        image_reader = csv.reader(imgtb_file)
        for ix, row in enumerate(image_reader):
            #if ix >= 3:
            if len(row) == 0 or row[0].strip().startswith('#'):
                continue
            else:
                gid = int(row[0])
                gname_ = row[1].strip()
                reviewed = bool(
                    row[2]
                )  # aif in hotspotter is equivilant to reviewed in IBEIS
                image_gid_list.append(gid)
                image_gname_list.append(gname_)
                image_reviewed_list.append(reviewed)

    image_gpath_list = [join(imgdir, gname) for gname in image_gname_list]

    ut.debug_duplicate_items(image_gpath_list)
    #print(image_gpath_list)
    flags = list(map(exists, image_gpath_list))
    for image_gpath, flag in zip(image_gpath_list, flags):
        if not flag:
            print('Image does not exist: %s' % image_gpath)

    assert all(flags), 'some images dont exist'

    # Add Images Table
    gid_list = ibs.add_images(image_gpath_list)  # any failed gids will be None
    assert len(gid_list) == len(image_gpath_list), 'bad image adder'
    # Build mappings to new indexes
    names_nid_to_nid = {
        names_nid: nid
        for (names_nid, nid) in zip(name_nid_list, nid_list)
    }
    names_nid_to_nid[1] = names_nid_to_nid[0]  # hsdb unknknown is 0 or 1
    images_gid_to_gid = {
        images_gid: gid
        for (images_gid, gid) in zip(image_gid_list, gid_list)
    }

    #get annotations from chip_table
    chip_bbox_list = []
    chip_theta_list = []
    chip_nid_list = []
    chip_gid_list = []
    chip_note_list = []
    with open(join(internal_dir, 'chip_table.csv'), 'rb') as chiptbl_file:
        chip_reader = csv.reader(chiptbl_file)
        for ix, row in enumerate(chip_reader):
            if len(row) == 0 or row[0].strip().startswith('#'):
                continue
            else:
                images_gid = int(row[1])
                names_nid = int(row[2])
                bbox_text = row[3]
                theta = float(row[4])
                notes = '<COMMA>'.join([item.strip() for item in row[5:]])
                try:
                    nid = names_nid_to_nid[names_nid]
                except KeyError:
                    print(names_nid_to_nid)
                    print('Error no names_nid: %r' % names_nid)
                    raise
                gid = images_gid_to_gid[images_gid]
                bbox_text = bbox_text.replace('[', '').replace(']', '').strip()
                bbox_text = re.sub('  *', ' ', bbox_text)
                bbox_strlist = bbox_text.split(' ')
                bbox = tuple(map(int, bbox_strlist))
                if gid is None:
                    print(
                        'Not adding the ix=%r-th Chip. Its image is corrupted image.'
                        % (ix, ))
                    continue
                #bbox = [int(item) for item in bbox_strlist]
                chip_nid_list.append(nid)
                chip_gid_list.append(gid)
                chip_bbox_list.append(bbox)
                chip_theta_list.append(theta)
                chip_note_list.append(notes)

    # Add Chips Table
    ibs.add_annots(chip_gid_list,
                   chip_bbox_list,
                   chip_theta_list,
                   nid_list=chip_nid_list,
                   notes_list=chip_note_list)

    # Set all injested RIDS as exemplars
    aid_list = ibs.get_valid_aids()
    flag_list = [True] * len(aid_list)
    ibs.set_annot_exemplar_flags(aid_list, flag_list)
    assert (all(
        ibs.get_annot_exemplar_flags(aid_list))), 'exemplars not set correctly'

    # Write file flagging successful conversion
    with open(join(ibs.get_ibsdir(), SUCCESS_FLAG_FNAME), 'w') as file_:
        file_.write('Successfully converted hsdb_dir=%r' % (hsdb_dir, ))
    print('finished ingest')
    return ibs
Beispiel #6
0
    print('''
          =====================
          FINISHED PROCESS
          =====================
          ''')

    def _sloppy_data(string):
        string = string.replace('0212', '2012')
        string = string.replace('1212', '2012')
        string = string.replace('"', '')
        return string

    dbdir = join(prefix, 'converted')
    ibsfuncs.delete_ibeis_database(dbdir)
    ibs = IBEISControl.IBEISController(dbdir=dbdir)
    image_gpath_list = [item[0] for item in processed]
    notes_list = [','.join( [basename(item[0])] + item[2:5] ) for item in processed]
    times_list = [ utool.exiftime_to_unixtime(_sloppy_data(item[1]), timestamp_format=2)
                    for item in processed]
    assert all(map(exists, image_gpath_list)), 'some images dont exist'

    gid_list = ibs.add_images(image_gpath_list)
    ibs.localize_images()
    ibs.set_image_notes(gid_list, notes_list)
    ibs.set_image_unixtime(gid_list, times_list)
    bbox_list = [ (0, 0, w, h) for (w, h) in ibs.get_image_sizes(gid_list) ]
    aid_list = ibs.add_annots(gid_list, bbox_list)
    name_list = [ basename(image_path).split(".")[0] for image_path in image_gpath_list ]
    ibs.set_annot_names(aid_list, name_list)
    ibs.db.commit()
Beispiel #7
0
def convert_hsdb_to_ibeis(hsdir, dbdir=None, **kwargs):
    r"""
    Args
        hsdir (str): Directory to folder *containing* _hsdb
        dbdir (str): Output directory (defaults to same as  hsdb)

    CommandLine:
        python -m ibeis convert_hsdb_to_ibeis --dbdir ~/work/Frogs
        python -m ibeis convert_hsdb_to_ibeis --hsdir "/raid/raw/RotanTurtles/Roatan HotSpotter Nov_21_2016"

    Ignore:
        from ibeis.dbio.ingest_hsdb import *  # NOQA
        hsdir = "/raid/raw/RotanTurtles/Roatan HotSpotter Nov_21_2016"
        dbdir = "~/work/RotanTurtles"

    Example:
        >>> # SCRIPT
        >>> from ibeis.dbio.ingest_hsdb import *  # NOQA
        >>> dbdir = ut.get_argval('--dbdir', type_=str, default=None)
        >>> hsdir = ut.get_argval('--hsdir', type_=str, default=dbdir)
        >>> result = convert_hsdb_to_ibeis(hsdir)
        >>> print(result)
    """
    from ibeis.control import IBEISControl
    import utool as ut

    if dbdir is None:
        dbdir = hsdir
    print('[ingest] Ingesting hsdb: %r -> %r' % (hsdir, dbdir))

    assert is_hsdb(
        hsdir
    ), 'not a hotspotter database. cannot even force convert: hsdir=%r' % (
        hsdir, )
    assert not is_succesful_convert(dbdir), 'hsdir=%r is already converted' % (
        hsdir, )
    #print('FORCE DELETE: %r' % (hsdir,))
    #ibsfuncs.delete_ibeis_database(hsdir)
    imgdir = join(hsdir, 'images')

    internal_dir = get_hsinternal(hsdir)
    nametbl_fpath = join(internal_dir, 'name_table.csv')
    imgtbl_fpath = join(internal_dir, 'image_table.csv')
    chiptbl_fpath = join(internal_dir, 'chip_table.csv')

    # READ NAME TABLE
    name_text_list = ['____']
    name_hs_nid_list = [0]
    with open(nametbl_fpath, 'r') as nametbl_file:
        name_reader = csv.reader(nametbl_file)
        for ix, row in enumerate(name_reader):
            #if ix >= 3:
            if len(row) == 0 or row[0].strip().startswith('#'):
                continue
            else:
                hs_nid = int(row[0])
                name = row[1].strip()
                name_text_list.append(name)
                name_hs_nid_list.append(hs_nid)

    # READ IMAGE TABLE
    iamge_hs_gid_list = []
    image_gname_list = []
    image_reviewed_list = []
    with open(imgtbl_fpath, 'r') as imgtb_file:
        image_reader = csv.reader(imgtb_file)
        for ix, row in enumerate(image_reader):
            if len(row) == 0 or row[0].strip().startswith('#'):
                continue
            else:
                hs_gid = int(row[0])
                gname_ = row[1].strip()
                # aif in hotspotter is equivilant to reviewed in IBEIS
                reviewed = bool(row[2])
                iamge_hs_gid_list.append(hs_gid)
                image_gname_list.append(gname_)
                image_reviewed_list.append(reviewed)

    image_gpath_list = [join(imgdir, gname) for gname in image_gname_list]

    ut.debug_duplicate_items(image_gpath_list)
    #print(image_gpath_list)
    image_exist_flags = list(map(exists, image_gpath_list))
    missing_images = []
    for image_gpath, flag in zip(image_gpath_list, image_exist_flags):
        if not flag:
            missing_images.append(image_gpath)
            print('Image does not exist: %s' % image_gpath)

    if not all(image_exist_flags):
        print('Only %d / %d image exist' %
              (sum(image_exist_flags), len(image_exist_flags)))

    SEARCH_FOR_IMAGES = False
    if SEARCH_FOR_IMAGES:
        # Hack to try and find the missing images
        from os.path import basename
        subfiles = ut.glob(hsdir,
                           '*',
                           recursive=True,
                           fullpath=True,
                           with_files=True)
        basename_to_existing = ut.group_items(subfiles,
                                              ut.lmap(basename, subfiles))

        can_copy_list = []
        for gpath in missing_images:
            gname = basename(gpath)
            if gname not in basename_to_existing:
                print('gname = %r' % (gname, ))
                pass
            else:
                existing = basename_to_existing[gname]
                can_choose = True
                if len(existing) > 1:
                    if not ut.allsame(ut.lmap(ut.get_file_uuid, existing)):
                        can_choose = False
                if can_choose:
                    found = existing[0]
                    can_copy_list.append((found, gpath))
                else:
                    print(existing)

        src, dst = ut.listT(can_copy_list)
        ut.copy_list(src, dst)

    # READ CHIP TABLE
    chip_bbox_list = []
    chip_theta_list = []
    chip_hs_nid_list = []
    chip_hs_gid_list = []
    chip_note_list = []
    with open(chiptbl_fpath, 'r') as chiptbl_file:
        chip_reader = csv.reader(chiptbl_file)
        for ix, row in enumerate(chip_reader):
            if len(row) == 0 or row[0].strip().startswith('#'):
                continue
            else:
                hs_gid = int(row[1])
                hs_nid = int(row[2])
                bbox_text = row[3]
                theta = float(row[4])
                notes = '<COMMA>'.join([item.strip() for item in row[5:]])

                bbox_text = bbox_text.replace('[', '').replace(']', '').strip()
                bbox_text = re.sub('  *', ' ', bbox_text)
                bbox_strlist = bbox_text.split(' ')
                bbox = tuple(map(int, bbox_strlist))
                #bbox = [int(item) for item in bbox_strlist]
                chip_hs_nid_list.append(hs_nid)
                chip_hs_gid_list.append(hs_gid)
                chip_bbox_list.append(bbox)
                chip_theta_list.append(theta)
                chip_note_list.append(notes)

    names = ut.ColumnLists({
        'hs_nid': name_hs_nid_list,
        'text': name_text_list,
    })

    images = ut.ColumnLists({
        'hs_gid': iamge_hs_gid_list,
        'gpath': image_gpath_list,
        'reviewed': image_reviewed_list,
        'exists': image_exist_flags,
    })

    chips = ut.ColumnLists({
        'hs_gid': chip_hs_gid_list,
        'hs_nid': chip_hs_nid_list,
        'bbox': chip_bbox_list,
        'theta': chip_theta_list,
        'note': chip_note_list,
    })

    IGNORE_MISSING_IMAGES = True
    if IGNORE_MISSING_IMAGES:
        # Ignore missing information
        print('pre')
        print('chips = %r' % (chips, ))
        print('images = %r' % (images, ))
        print('names = %r' % (names, ))
        missing_gxs = ut.where(ut.not_list(images['exists']))
        missing_gids = ut.take(images['hs_gid'], missing_gxs)
        gid_to_cxs = ut.dzip(*chips.group_indicies('hs_gid'))
        missing_cxs = ut.flatten(ut.take(gid_to_cxs, missing_gids))
        # Remove missing images and dependant chips
        images = images.remove(missing_gxs)
        chips = chips.remove(missing_cxs)
        valid_nids = set(chips['hs_nid'] + [0])
        isvalid = [nid in valid_nids for nid in names['hs_nid']]
        names = names.compress(isvalid)
        print('post')
        print('chips = %r' % (chips, ))
        print('images = %r' % (images, ))
        print('names = %r' % (names, ))

    assert all(images['exists']), 'some images dont exist'

    # if gid is None:
    #     print('Not adding the ix=%r-th Chip. Its image is corrupted image.' % (ix,))
    #     # continue
    # # Build mappings to new indexes
    # names_nid_to_nid  = {names_nid: nid for (names_nid, nid) in zip(hs_nid_list, nid_list)}
    # names_nid_to_nid[1] = names_nid_to_nid[0]  # hsdb unknknown is 0 or 1
    # images_gid_to_gid = {images_gid: gid for (images_gid, gid) in zip(hs_gid_list, gid_list)}

    ibs = IBEISControl.request_IBEISController(dbdir=dbdir,
                                               check_hsdb=False,
                                               **kwargs)
    assert len(ibs.get_valid_gids()) == 0, 'target database is not empty'

    # Add names, images, and annotations
    names['ibs_nid'] = ibs.add_names(names['text'])
    images['ibs_gid'] = ibs.add_images(
        images['gpath'])  # any failed gids will be None

    if True:
        # Remove corrupted images
        print('pre')
        print('chips = %r' % (chips, ))
        print('images = %r' % (images, ))
        print('names = %r' % (names, ))
        missing_gxs = ut.where(ut.flag_None_items(images['ibs_gid']))
        missing_gids = ut.take(images['hs_gid'], missing_gxs)
        gid_to_cxs = ut.dzip(*chips.group_indicies('hs_gid'))
        missing_cxs = ut.flatten(ut.take(gid_to_cxs, missing_gids))
        # Remove missing images and dependant chips
        chips = chips.remove(missing_cxs)
        images = images.remove(missing_gxs)
        print('post')
        print('chips = %r' % (chips, ))
        print('images = %r' % (images, ))
        print('names = %r' % (names, ))

    # Index chips using new ibs rowids
    ibs_gid_lookup = ut.dzip(images['hs_gid'], images['ibs_gid'])
    ibs_nid_lookup = ut.dzip(names['hs_nid'], names['ibs_nid'])
    try:
        chips['ibs_gid'] = ut.take(ibs_gid_lookup, chips['hs_gid'])
    except KeyError:
        chips['ibs_gid'] = [
            ibs_gid_lookup.get(index, None) for index in chips['hs_gid']
        ]
    try:
        chips['ibs_nid'] = ut.take(ibs_nid_lookup, chips['hs_nid'])
    except KeyError:
        chips['ibs_nid'] = [
            ibs_nid_lookup.get(index, None) for index in chips['hs_nid']
        ]

    ibs.add_annots(chips['ibs_gid'],
                   bbox_list=chips['bbox'],
                   theta_list=chips['theta'],
                   nid_list=chips['ibs_nid'],
                   notes_list=chips['note'])

    # aid_list = ibs.get_valid_aids()
    # flag_list = [True] * len(aid_list)
    # ibs.set_annot_exemplar_flags(aid_list, flag_list)
    # assert(all(ibs.get_annot_exemplar_flags(aid_list))), 'exemplars not set correctly'

    # Write file flagging successful conversion
    with open(join(ibs.get_ibsdir(), SUCCESS_FLAG_FNAME), 'w') as file_:
        file_.write('Successfully converted hsdir=%r' % (hsdir, ))
    print('finished ingest')
    return ibs
Beispiel #8
0
dbname = ut.get_argval('--db', type_=str, default=None)
imageset = ut.get_argval('--imageset', type_=str, default=None)
refset = ut.get_argval('--refset', type_=str, default=None)
output_file = ut.get_argval('--o', type_=str, default="id_" + imageset + "_" + refset + ".txt")
no_cache = ut.get_argflag('--no_cache')
use_edge = ut.get_argflag('--edge')

assert dbname is not None, 'specify dbname'
assert imageset is not None, 'specify imageset'
assert refset is not None, 'specify reference set'

dbdir = ibeis.sysres.db_to_dbdir(dbname, allow_newdir=False)
ut.ensuredir(dbdir, verbose=True)

ibs = IBEISControl.request_IBEISController(dbdir)

qgid_list = ut.flatten(get_imageset_gids(ibs, add_imagesets(ibs, [imageset])))
dgid_list = ut.flatten(get_imageset_gids(ibs, add_imagesets(ibs, [refset])))

qaids = ut.flatten(ibs.get_image_aids(qgid_list))
daids = ut.flatten(ibs.get_image_aids(dgid_list))

print(no_cache)

cfgdict = None
if (use_edge):
    #
    # This is how you specify IBEIS to use the edge matching routine for humpback flukes. Only works I think if species
    # is set to humpback_whale
    #