def test(gpath_list, canonical_weight_filepath=None, **kwargs):
from wbia.detecttools.directory import Directory
# Get correct weight if specified with shorthand
archive_url = None
ensemble_index = None
if canonical_weight_filepath is not None and ':' in canonical_weight_filepath:
assert canonical_weight_filepath.count(':') == 1
canonical_weight_filepath, ensemble_index = canonical_weight_filepath.split(
':')
ensemble_index = int(ensemble_index)
if canonical_weight_filepath in ARCHIVE_URL_DICT:
archive_url = ARCHIVE_URL_DICT[canonical_weight_filepath]
archive_path = ut.grab_file_url(archive_url,
appname='wbia',
check_hash=True)
else:
raise RuntimeError('canonical_weight_filepath %r not recognized' %
(canonical_weight_filepath, ))
assert os.path.exists(archive_path)
archive_path = ut.truepath(archive_path)
ensemble_path = archive_path.strip('.zip')
if not os.path.exists(ensemble_path):
ut.unarchive_file(archive_path, output_dir=ensemble_path)
assert os.path.exists(ensemble_path)
direct = Directory(ensemble_path,
include_file_extensions=['weights'],
recursive=True)
weights_path_list = direct.files()
weights_path_list = sorted(weights_path_list)
assert len(weights_path_list) > 0
if ensemble_index is not None:
assert 0 <= ensemble_index and ensemble_index < len(weights_path_list)
weights_path_list = [weights_path_list[ensemble_index]]
assert len(weights_path_list) > 0
logger.info('Using weights in the ensemble: %s ' %
(ut.repr3(weights_path_list), ))
result_list = test_ensemble(gpath_list, weights_path_list, **kwargs)
for result in result_list:
x0 = max(result['x0'], 0.0)
y0 = max(result['y0'], 0.0)
x1 = max(result['x1'], 0.0)
y1 = max(result['y1'], 0.0)
yield (
x0,
y0,
x1,
y1,
)
def test(
gpath_list,
classifier_weight_filepath=None,
return_dict=False,
multiclass=False,
**kwargs,
):
from wbia.detecttools.directory import Directory
# Get correct weight if specified with shorthand
archive_url = None
ensemble_index = None
if classifier_weight_filepath is not None and ':' in classifier_weight_filepath:
assert classifier_weight_filepath.count(':') == 1
classifier_weight_filepath, ensemble_index = classifier_weight_filepath.split(
':')
ensemble_index = int(ensemble_index)
if classifier_weight_filepath in ARCHIVE_URL_DICT:
archive_url = ARCHIVE_URL_DICT[classifier_weight_filepath]
archive_path = ut.grab_file_url(archive_url,
appname='wbia',
check_hash=True)
else:
logger.info('classifier_weight_filepath %r not recognized' %
(classifier_weight_filepath, ))
raise RuntimeError
assert os.path.exists(archive_path)
archive_path = ut.truepath(archive_path)
ensemble_path = archive_path.strip('.zip')
if not os.path.exists(ensemble_path):
ut.unarchive_file(archive_path, output_dir=ensemble_path)
assert os.path.exists(ensemble_path)
direct = Directory(ensemble_path,
include_file_extensions=['weights'],
recursive=True)
weights_path_list = direct.files()
weights_path_list = sorted(weights_path_list)
assert len(weights_path_list) > 0
kwargs.pop('classifier_algo', None)
logger.info('Using weights in the ensemble, index %r: %s ' %
(ensemble_index, ut.repr3(weights_path_list)))
result_list = test_ensemble(
gpath_list,
weights_path_list,
classifier_weight_filepath,
ensemble_index,
multiclass=multiclass,
**kwargs,
)
for result in result_list:
best_key = None
best_score = -1.0
for key, score in result.items():
if score > best_score:
best_key = key
best_score = score
assert best_score >= 0.0 and best_key is not None
if return_dict:
yield best_score, best_key, result
else:
yield best_score, best_key
def ingest_rawdata(ibs, ingestable, localize=False):
"""
Ingests rawdata into an ibeis database.
if ingest_type == 'named_folders':
Converts folder structure where folders = name, to ibsdb
if ingest_type == 'named_images':
Converts imgname structure where imgnames = name_id.ext, to ibsdb
CommandLine:
python ibeis/dbio/ingest_database.py --db seals_drop2
Args:
ibs (IBEISController): ibeis controller object
ingestable (?):
localize (bool): (default = False)
Returns:
list: aid_list - list of annotation rowids
CommandLine:
python -m ibeis.dbio.ingest_database --exec-ingest_rawdata
python -m ibeis.dbio.ingest_database --exec-ingest_rawdata --db snow-leopards --imgdir /raid/raw_rsync/snow-leopards
Example:
>>> # SCRIPT
>>> # General ingest script
>>> from ibeis.dbio.ingest_database import * # NOQA
>>> import ibeis
>>> dbname = ut.get_argval('--db', str, None) # 'snow-leopards')
>>> force_delete = ut.get_argflag(('--force_delete', '--force-delete'))
>>> img_dir = ut.get_argval('--imgdir', type_=str, default=None)
>>> assert img_dir is not None, 'specify img dir'
>>> assert dbname is not None, 'specify dbname'
>>> ingestable = Ingestable(dbname,
>>> img_dir=img_dir,
>>> ingest_type='unknown',
>>> fmtkey=None,
>>> species=None,
>>> images_as_annots=False,
>>> adjust_percent=0.00)
>>> from ibeis.control import IBEISControl
>>> dbdir = ibeis.sysres.db_to_dbdir(dbname, allow_newdir=True, use_sync=False)
>>> ut.ensuredir(dbdir, verbose=True)
>>> ibs = IBEISControl.request_IBEISController(dbdir)
>>> localize = False
>>> aid_list = ingest_rawdata(ibs, ingestable, localize)
>>> result = ('aid_list = %s' % (str(aid_list),))
>>> print(result)
"""
print('[ingest_rawdata] Ingestable' + str(ingestable))
if ingestable.zipfile is not None:
zipfile_fpath = ut.truepath(join(ibeis.sysres.get_workdir(), ingestable.zipfile))
ingestable.img_dir = ut.unarchive_file(zipfile_fpath)
img_dir = realpath(ingestable.img_dir)
ingest_type = ingestable.ingest_type
fmtkey = ingestable.fmtkey
adjust_percent = ingestable.adjust_percent
species_text = ingestable.species
postingest_func = ingestable.postingest_func
print('[ingest] ingesting rawdata: img_dir=%r, injest_type=%r' % (img_dir, ingest_type))
# Get images in the image directory
unzipped_file_base_dir = join(ibs.get_dbdir(), 'unzipped_files')
def extract_zipfile_images(ibs, ingestable):
import utool as ut # NOQA
zipfile_list = ut.glob(ingestable.img_dir, '*.zip', recursive=True)
if len(zipfile_list) > 0:
print('Found zipfile_list = %r' % (zipfile_list,))
ut.ensuredir(unzipped_file_base_dir)
for zipfile in zipfile_list:
unziped_file_relpath = dirname(relpath(relpath(realpath(zipfile), realpath(ingestable.img_dir))))
unzipped_file_dir = join(unzipped_file_base_dir, unziped_file_relpath)
ut.ensuredir(unzipped_file_dir)
ut.unzip_file(zipfile, output_dir=unzipped_file_dir, overwrite=False)
gpath_list = ut.list_images(unzipped_file_dir, fullpath=True, recursive=True)
else:
gpath_list = []
return gpath_list
def list_images(img_dir):
""" lists images that are not in an internal cache """
import utool as ut # NOQA
ignore_list = ['_hsdb', '.hs_internals', '_ibeis_cache', '_ibsdb']
gpath_list = ut.list_images(img_dir,
fullpath=True,
recursive=True,
ignore_list=ignore_list)
return gpath_list
# FIXME ensure python3 works with this
gpath_list1 = ut.ensure_unicode_strlist(list_images(img_dir))
gpath_list2 = ut.ensure_unicode_strlist(extract_zipfile_images(ibs, ingestable))
gpath_list = gpath_list1 + gpath_list2
#__STR__ = const.__STR__
#map(__STR__, gpath_list)
#ut.embed()
# Parse structure for image names
if ingest_type == 'named_folders':
name_list1 = get_name_texts_from_parent_folder(gpath_list1, img_dir, fmtkey)
name_list2 = get_name_texts_from_parent_folder(gpath_list2, unzipped_file_base_dir, fmtkey)
name_list = name_list1 + name_list2
pass
elif ingest_type == 'named_images':
name_list = get_name_texts_from_gnames(gpath_list, img_dir, fmtkey)
elif ingest_type == 'unknown':
name_list = [const.UNKNOWN for _ in range(len(gpath_list))]
else:
raise NotImplementedError('unknwon ingest_type=%r' % (ingest_type,))
# Find names likely to be the same?
RECTIFY_NAMES_HUERISTIC = True
if RECTIFY_NAMES_HUERISTIC:
names = sorted(list(set(name_list)))
splitchars = [' ', '/']
def multisplit(str_, splitchars):
import utool as ut
n = [str_]
for char in splitchars:
n = ut.flatten([_.split(char) for _ in n])
return n
groupids = [multisplit(n1, splitchars)[0] for n1 in names]
grouped_names = ut.group_items(names, groupids)
fixed_names = {newkey: key for key, val in grouped_names.items() if len(val) > 1 for newkey in val}
name_list = [fixed_names.get(name, name) for name in name_list]
# Add Images
gpath_list = [gpath.replace('\\', '/') for gpath in gpath_list]
gid_list_ = ibs.add_images(gpath_list)
# <DEBUG>
#print('added: ' + ut.indentjoin(map(str, zip(gid_list_, gpath_list))))
unique_gids = list(set(gid_list_))
print("[ingest] Length gid list: %d" % len(gid_list_))
print("[ingest] Length unique gid list: %d" % len(unique_gids))
assert len(gid_list_) == len(gpath_list)
for gid in gid_list_:
if gid is None:
print('[ingest] big fat warning')
# </DEBUG>
gid_list = ut.filter_Nones(gid_list_)
unique_gids, unique_names, unique_notes = resolve_name_conflicts(
gid_list, name_list)
# Add ANNOTATIONs with names and notes
if ingestable.images_as_annots:
aid_list = ibs.use_images_as_annotations(unique_gids,
adjust_percent=adjust_percent)
ibs.set_annot_names(aid_list, unique_names)
ibs.set_annot_notes(aid_list, unique_notes)
if species_text is not None:
ibs.set_annot_species(aid_list, [species_text] * len(aid_list))
if localize:
ibs.localize_images()
if postingest_func is not None:
postingest_func(ibs)