def merge_species_databases(species_prefix):
""" Build a merged database """
from ibeis.control import IBEISControl
from ibeis.dev import sysres
print('[ibsfuncs] Merging species with prefix: %r' % species_prefix)
ut.util_parallel.ensure_pool(warn=False)
with ut.Indenter(' '):
# Build / get target database
all_db = '__ALL_' + species_prefix + '_'
all_dbdir = sysres.db_to_dbdir(all_db, allow_newdir=True)
ibs_target = IBEISControl.IBEISController(all_dbdir)
# Build list of databases to merge
species_dbdir_list = get_species_dbs(species_prefix)
ibs_source_list = []
for dbdir in species_dbdir_list:
ibs_source = IBEISControl.IBEISController(dbdir)
ibs_source_list.append(ibs_source)
print('[ibsfuncs] Destination database: %r' % all_db)
print('[ibsfuncs] Source databases:' +
ut.indentjoin(species_dbdir_list, '\n * '))
#Merge the databases into ibs_target
merge_databases(ibs_target, ibs_source_list)
return ibs_target
def convert_hsdb_to_ibeis(hsdb_dir, **kwargs):
r"""
Args:
hsdb_dir (str):
CommandLine:
python -m ibeis.dbio.ingest_hsdb --test-convert_hsdb_to_ibeis:0 --db ~/work/Frogs
Example:
>>> # SCRIPT
>>> from ibeis.dbio.ingest_hsdb import * # NOQA
>>> hsdb_dir = ut.get_argval('--dbdir', type_=str, default=None)
>>> result = convert_hsdb_to_ibeis(hsdb_dir)
>>> print(result)
"""
from ibeis.control import IBEISControl
import utool as ut
assert is_hsdb(
hsdb_dir
), 'not a hotspotter database. cannot even force convert: hsdb_dir=%r' % (
hsdb_dir, )
assert not is_succesful_convert(
hsdb_dir), 'hsdb_dir=%r is already converted' % (hsdb_dir, )
#print('FORCE DELETE: %r' % (hsdb_dir,))
#ibsfuncs.delete_ibeis_database(hsdb_dir)
print('[ingest] Ingesting hsdb: %r' % hsdb_dir)
imgdir = join(hsdb_dir, 'images')
ibs = IBEISControl.IBEISController(dbdir=hsdb_dir, **kwargs)
# READ NAME TABLE
names_name_list = ['____']
name_nid_list = [0]
internal_dir = get_hsinternal(hsdb_dir)
with open(join(internal_dir, 'name_table.csv'), 'rb') as nametbl_file:
name_reader = csv.reader(nametbl_file)
for ix, row in enumerate(name_reader):
#if ix >= 3:
if len(row) == 0 or row[0].strip().startswith('#'):
continue
else:
nid = int(row[0])
name = row[1].strip()
names_name_list.append(name)
name_nid_list.append(nid)
# ADD NAMES TABLE
nid_list = ibs.add_names(names_name_list)
#print(names_name_list)
#print(nid_list)
assert len(nid_list) == len(names_name_list), 'bad name adder'
image_gid_list = []
image_gname_list = []
image_reviewed_list = []
with open(join(internal_dir, 'image_table.csv'), 'rb') as imgtb_file:
image_reader = csv.reader(imgtb_file)
for ix, row in enumerate(image_reader):
#if ix >= 3:
if len(row) == 0 or row[0].strip().startswith('#'):
continue
else:
gid = int(row[0])
gname_ = row[1].strip()
reviewed = bool(
row[2]
) # aif in hotspotter is equivilant to reviewed in IBEIS
image_gid_list.append(gid)
image_gname_list.append(gname_)
image_reviewed_list.append(reviewed)
image_gpath_list = [join(imgdir, gname) for gname in image_gname_list]
ut.debug_duplicate_items(image_gpath_list)
#print(image_gpath_list)
flags = list(map(exists, image_gpath_list))
for image_gpath, flag in zip(image_gpath_list, flags):
if not flag:
print('Image does not exist: %s' % image_gpath)
assert all(flags), 'some images dont exist'
# Add Images Table
gid_list = ibs.add_images(image_gpath_list) # any failed gids will be None
assert len(gid_list) == len(image_gpath_list), 'bad image adder'
# Build mappings to new indexes
names_nid_to_nid = {
names_nid: nid
for (names_nid, nid) in zip(name_nid_list, nid_list)
}
names_nid_to_nid[1] = names_nid_to_nid[0] # hsdb unknknown is 0 or 1
images_gid_to_gid = {
images_gid: gid
for (images_gid, gid) in zip(image_gid_list, gid_list)
}
#get annotations from chip_table
chip_bbox_list = []
chip_theta_list = []
chip_nid_list = []
chip_gid_list = []
chip_note_list = []
with open(join(internal_dir, 'chip_table.csv'), 'rb') as chiptbl_file:
chip_reader = csv.reader(chiptbl_file)
for ix, row in enumerate(chip_reader):
if len(row) == 0 or row[0].strip().startswith('#'):
continue
else:
images_gid = int(row[1])
names_nid = int(row[2])
bbox_text = row[3]
theta = float(row[4])
notes = '<COMMA>'.join([item.strip() for item in row[5:]])
try:
nid = names_nid_to_nid[names_nid]
except KeyError:
print(names_nid_to_nid)
print('Error no names_nid: %r' % names_nid)
raise
gid = images_gid_to_gid[images_gid]
bbox_text = bbox_text.replace('[', '').replace(']', '').strip()
bbox_text = re.sub(' *', ' ', bbox_text)
bbox_strlist = bbox_text.split(' ')
bbox = tuple(map(int, bbox_strlist))
if gid is None:
print(
'Not adding the ix=%r-th Chip. Its image is corrupted image.'
% (ix, ))
continue
#bbox = [int(item) for item in bbox_strlist]
chip_nid_list.append(nid)
chip_gid_list.append(gid)
chip_bbox_list.append(bbox)
chip_theta_list.append(theta)
chip_note_list.append(notes)
# Add Chips Table
ibs.add_annots(chip_gid_list,
chip_bbox_list,
chip_theta_list,
nid_list=chip_nid_list,
notes_list=chip_note_list)
# Set all injested RIDS as exemplars
aid_list = ibs.get_valid_aids()
flag_list = [True] * len(aid_list)
ibs.set_annot_exemplar_flags(aid_list, flag_list)
assert (all(
ibs.get_annot_exemplar_flags(aid_list))), 'exemplars not set correctly'
# Write file flagging successful conversion
with open(join(ibs.get_ibsdir(), SUCCESS_FLAG_FNAME), 'w') as file_:
file_.write('Successfully converted hsdb_dir=%r' % (hsdb_dir, ))
print('finished ingest')
return ibs
print('''
=====================
FINISHED PROCESS
=====================
''')
def _sloppy_data(string):
string = string.replace('0212', '2012')
string = string.replace('1212', '2012')
string = string.replace('"', '')
return string
dbdir = join(prefix, 'converted')
ibsfuncs.delete_ibeis_database(dbdir)
ibs = IBEISControl.IBEISController(dbdir=dbdir)
image_gpath_list = [item[0] for item in processed]
notes_list = [','.join( [basename(item[0])] + item[2:5] ) for item in processed]
times_list = [ utool.exiftime_to_unixtime(_sloppy_data(item[1]), timestamp_format=2)
for item in processed]
assert all(map(exists, image_gpath_list)), 'some images dont exist'
gid_list = ibs.add_images(image_gpath_list)
ibs.localize_images()
ibs.set_image_notes(gid_list, notes_list)
ibs.set_image_unixtime(gid_list, times_list)
bbox_list = [ (0, 0, w, h) for (w, h) in ibs.get_image_sizes(gid_list) ]
aid_list = ibs.add_annots(gid_list, bbox_list)
name_list = [ basename(image_path).split(".")[0] for image_path in image_gpath_list ]
ibs.set_annot_names(aid_list, name_list)
ibs.db.commit()
