def merge_species_databases(species_prefix): """ Build a merged database """ from ibeis.control import IBEISControl from ibeis.dev import sysres print('[ibsfuncs] Merging species with prefix: %r' % species_prefix) ut.util_parallel.ensure_pool(warn=False) with ut.Indenter(' '): # Build / get target database all_db = '__ALL_' + species_prefix + '_' all_dbdir = sysres.db_to_dbdir(all_db, allow_newdir=True) ibs_target = IBEISControl.IBEISController(all_dbdir) # Build list of databases to merge species_dbdir_list = get_species_dbs(species_prefix) ibs_source_list = [] for dbdir in species_dbdir_list: ibs_source = IBEISControl.IBEISController(dbdir) ibs_source_list.append(ibs_source) print('[ibsfuncs] Destination database: %r' % all_db) print('[ibsfuncs] Source databases:' + ut.indentjoin(species_dbdir_list, '\n * ')) #Merge the databases into ibs_target merge_databases(ibs_target, ibs_source_list) return ibs_target
def convert_hsdb_to_ibeis(hsdb_dir, **kwargs): r""" Args: hsdb_dir (str): CommandLine: python -m ibeis.dbio.ingest_hsdb --test-convert_hsdb_to_ibeis:0 --db ~/work/Frogs Example: >>> # SCRIPT >>> from ibeis.dbio.ingest_hsdb import * # NOQA >>> hsdb_dir = ut.get_argval('--dbdir', type_=str, default=None) >>> result = convert_hsdb_to_ibeis(hsdb_dir) >>> print(result) """ from ibeis.control import IBEISControl import utool as ut assert is_hsdb( hsdb_dir ), 'not a hotspotter database. cannot even force convert: hsdb_dir=%r' % ( hsdb_dir, ) assert not is_succesful_convert( hsdb_dir), 'hsdb_dir=%r is already converted' % (hsdb_dir, ) #print('FORCE DELETE: %r' % (hsdb_dir,)) #ibsfuncs.delete_ibeis_database(hsdb_dir) print('[ingest] Ingesting hsdb: %r' % hsdb_dir) imgdir = join(hsdb_dir, 'images') ibs = IBEISControl.IBEISController(dbdir=hsdb_dir, **kwargs) # READ NAME TABLE names_name_list = ['____'] name_nid_list = [0] internal_dir = get_hsinternal(hsdb_dir) with open(join(internal_dir, 'name_table.csv'), 'rb') as nametbl_file: name_reader = csv.reader(nametbl_file) for ix, row in enumerate(name_reader): #if ix >= 3: if len(row) == 0 or row[0].strip().startswith('#'): continue else: nid = int(row[0]) name = row[1].strip() names_name_list.append(name) name_nid_list.append(nid) # ADD NAMES TABLE nid_list = ibs.add_names(names_name_list) #print(names_name_list) #print(nid_list) assert len(nid_list) == len(names_name_list), 'bad name adder' image_gid_list = [] image_gname_list = [] image_reviewed_list = [] with open(join(internal_dir, 'image_table.csv'), 'rb') as imgtb_file: image_reader = csv.reader(imgtb_file) for ix, row in enumerate(image_reader): #if ix >= 3: if len(row) == 0 or row[0].strip().startswith('#'): continue else: gid = int(row[0]) gname_ = row[1].strip() reviewed = bool( row[2] ) # aif in hotspotter is equivilant to reviewed in IBEIS image_gid_list.append(gid) image_gname_list.append(gname_) image_reviewed_list.append(reviewed) image_gpath_list = [join(imgdir, gname) for gname in image_gname_list] ut.debug_duplicate_items(image_gpath_list) #print(image_gpath_list) flags = list(map(exists, image_gpath_list)) for image_gpath, flag in zip(image_gpath_list, flags): if not flag: print('Image does not exist: %s' % image_gpath) assert all(flags), 'some images dont exist' # Add Images Table gid_list = ibs.add_images(image_gpath_list) # any failed gids will be None assert len(gid_list) == len(image_gpath_list), 'bad image adder' # Build mappings to new indexes names_nid_to_nid = { names_nid: nid for (names_nid, nid) in zip(name_nid_list, nid_list) } names_nid_to_nid[1] = names_nid_to_nid[0] # hsdb unknknown is 0 or 1 images_gid_to_gid = { images_gid: gid for (images_gid, gid) in zip(image_gid_list, gid_list) } #get annotations from chip_table chip_bbox_list = [] chip_theta_list = [] chip_nid_list = [] chip_gid_list = [] chip_note_list = [] with open(join(internal_dir, 'chip_table.csv'), 'rb') as chiptbl_file: chip_reader = csv.reader(chiptbl_file) for ix, row in enumerate(chip_reader): if len(row) == 0 or row[0].strip().startswith('#'): continue else: images_gid = int(row[1]) names_nid = int(row[2]) bbox_text = row[3] theta = float(row[4]) notes = '<COMMA>'.join([item.strip() for item in row[5:]]) try: nid = names_nid_to_nid[names_nid] except KeyError: print(names_nid_to_nid) print('Error no names_nid: %r' % names_nid) raise gid = images_gid_to_gid[images_gid] bbox_text = bbox_text.replace('[', '').replace(']', '').strip() bbox_text = re.sub(' *', ' ', bbox_text) bbox_strlist = bbox_text.split(' ') bbox = tuple(map(int, bbox_strlist)) if gid is None: print( 'Not adding the ix=%r-th Chip. Its image is corrupted image.' % (ix, )) continue #bbox = [int(item) for item in bbox_strlist] chip_nid_list.append(nid) chip_gid_list.append(gid) chip_bbox_list.append(bbox) chip_theta_list.append(theta) chip_note_list.append(notes) # Add Chips Table ibs.add_annots(chip_gid_list, chip_bbox_list, chip_theta_list, nid_list=chip_nid_list, notes_list=chip_note_list) # Set all injested RIDS as exemplars aid_list = ibs.get_valid_aids() flag_list = [True] * len(aid_list) ibs.set_annot_exemplar_flags(aid_list, flag_list) assert (all( ibs.get_annot_exemplar_flags(aid_list))), 'exemplars not set correctly' # Write file flagging successful conversion with open(join(ibs.get_ibsdir(), SUCCESS_FLAG_FNAME), 'w') as file_: file_.write('Successfully converted hsdb_dir=%r' % (hsdb_dir, )) print('finished ingest') return ibs
print(''' ===================== FINISHED PROCESS ===================== ''') def _sloppy_data(string): string = string.replace('0212', '2012') string = string.replace('1212', '2012') string = string.replace('"', '') return string dbdir = join(prefix, 'converted') ibsfuncs.delete_ibeis_database(dbdir) ibs = IBEISControl.IBEISController(dbdir=dbdir) image_gpath_list = [item[0] for item in processed] notes_list = [','.join( [basename(item[0])] + item[2:5] ) for item in processed] times_list = [ utool.exiftime_to_unixtime(_sloppy_data(item[1]), timestamp_format=2) for item in processed] assert all(map(exists, image_gpath_list)), 'some images dont exist' gid_list = ibs.add_images(image_gpath_list) ibs.localize_images() ibs.set_image_notes(gid_list, notes_list) ibs.set_image_unixtime(gid_list, times_list) bbox_list = [ (0, 0, w, h) for (w, h) in ibs.get_image_sizes(gid_list) ] aid_list = ibs.add_annots(gid_list, bbox_list) name_list = [ basename(image_path).split(".")[0] for image_path in image_gpath_list ] ibs.set_annot_names(aid_list, name_list) ibs.db.commit()