def train_identification_pz():
r"""
CommandLine:
python -m ibeis_cnn.train --test-train_identification_pz
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis_cnn.train import * # NOQA
>>> train_identification_pz()
"""
print('get_identification_decision_training_data')
import ibeis
ibs = ibeis.opendb('NNP_Master3')
base_size = 128
#max_examples = 1001
#max_examples = None
max_examples = 400
data_fpath, labels_fpath, training_dpath = ibsplugin.get_identify_training_fpaths(
ibs, base_size=base_size, max_examples=max_examples)
model = models.SiameseModel()
config = dict(
equal_batch_sizes=True,
patience=50,
batch_size=32,
learning_rate=.03,
show_confusion=False,
run_test=None,
)
nets_dir = ibs.get_neuralnet_dir()
ut.ensuredir(nets_dir)
weights_fpath = join(training_dpath, 'ibeis_cnn_weights.pickle')
train_harness.train(model, data_fpath, labels_fpath, weights_fpath,
training_dpath, **config)
def load_qcx2_res(ibs, qrid_list, nocache=False):
'Prefrosm / loads all queries'
qreq = mc3.quickly_ensure_qreq(ibs, qrids=qrid_list)
# Build query big cache rowid
query_rowid = qreq.get_rowid()
hs_rowid = ibs.get_db_name()
qcxs_rowid = utool.hashstr_arr(qrid_list, lbl='_qcxs')
qres_rowid = hs_rowid + query_rowid + qcxs_rowid
cache_dir = join(ibs.dirs.cache_dir, 'query_results_bigcache')
print('[rr2] load_qcx2_res(): %r' % qres_rowid)
io_kwargs = dict(dpath=cache_dir, fname='query_results', rowid=qres_rowid, ext='.cPkl')
# Return cache if available
if not params.args.nocache_query and (not nocache):
qrid2_qres = io.smart_load(**io_kwargs)
if qrid2_qres is not None:
print('[rr2] * cache hit')
return qrid2_qres
print('[rr2] * cache miss')
else:
print('[rr2] * cache off')
# Individually load / compute queries
if isinstance(qrid_list, list):
qcx_set = set(qrid_list)
else:
qcx_set = set(qrid_list.tolist())
qcx_max = max(qrid_list) + 1
qrid2_qres = [ibs.query(qrid) if qrid in qcx_set else None for qrid in xrange(qcx_max)]
# Save to the cache
print('[rr2] Saving query_results to bigcache: %r' % qres_rowid)
utool.ensuredir(cache_dir)
io.smart_save(qrid2_qres, **io_kwargs)
return qrid2_qres
def get_smart_patrol_dir(ibs, ensure=True):
"""
Args:
ensure (bool):
Returns:
str smart_patrol_dpath
CommandLine:
python -m wbia.control.IBEISControl --test-get_smart_patrol_dir
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.control.IBEISControl import * # NOQA
>>> import wbia
>>> # build test data
>>> ibs = wbia.opendb('testdb1')
>>> ensure = True
>>> # execute function
>>> smart_patrol_dpath = ibs.get_smart_patrol_dir(ensure)
>>> # verify results
>>> ut.assertpath(smart_patrol_dpath, verbose=True)
"""
smart_patrol_dpath = join(ibs.dbdir, const.PATH_NAMES.smartpatrol)
if ensure:
ut.ensuredir(smart_patrol_dpath)
return smart_patrol_dpath
def get_global_species_scorenorm_cachedir(ibs, species_text, ensure=True):
"""
Args:
species_text (str):
ensure (bool):
Returns:
str: species_cachedir
CommandLine:
python -m ibeis.control.IBEISControl --test-get_global_species_scorenorm_cachedir
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.control.IBEISControl import * # NOQA
>>> import ibeis # NOQA
>>> ibs = ibeis.opendb('testdb1')
>>> species_text = ibeis.const.TEST_SPECIES.ZEB_GREVY
>>> ensure = True
>>> species_cachedir = ibs.get_global_species_scorenorm_cachedir(species_text, ensure)
>>> resourcedir = ibs.get_ibeis_resource_dir()
>>> result = ut.relpath_unix(species_cachedir, resourcedir)
>>> print(result)
scorenorm/zebra_grevys
"""
scorenorm_cachedir = join(ibs.get_ibeis_resource_dir(),
const.PATH_NAMES.scorenormdir)
species_cachedir = join(scorenorm_cachedir, species_text)
if ensure:
ut.ensurepath(scorenorm_cachedir)
ut.ensuredir(species_cachedir)
return species_cachedir
def get_smart_patrol_dir(ibs, ensure=True):
"""
Args:
ensure (bool):
Returns:
str smart_patrol_dpath
CommandLine:
python -m ibeis.control.IBEISControl --test-get_smart_patrol_dir
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.control.IBEISControl import * # NOQA
>>> import ibeis
>>> # build test data
>>> ibs = ibeis.opendb('testdb1')
>>> ensure = True
>>> # execute function
>>> smart_patrol_dpath = ibs.get_smart_patrol_dir(ensure)
>>> # verify results
>>> ut.assertpath(smart_patrol_dpath, verbose=True)
"""
smart_patrol_dpath = join(ibs.dbdir, const.PATH_NAMES.smartpatrol)
if ensure:
ut.ensuredir(smart_patrol_dpath)
return smart_patrol_dpath
def set_logdir(log_dir):
from os.path import realpath, expanduser
log_dir = realpath(expanduser(log_dir))
ut.ensuredir(log_dir, verbose=True)
ut.stop_logging()
_ibeis_cache_write(LOGDIR_CACHEID, log_dir)
ut.start_logging(appname=__APPNAME__)
def create_shortcut(r, what, where=''):
# TODO Move to windows helpers
print('\n\n+---- Creating Shortcut ----')
print('What = %s\n Where=%s' % (what, where))
run_in = ''
what_args = ''
if isinstance(what, tuple):
tup = what
what = tup[0]
what_args = tup[1]
run_in = tup[2]
if run_in == ' ':
run_in = ''
if what_args == ' ':
what_args = ''
if where == '':
target = what + '.lnk'
else:
import utool as ut
ut.ensuredir(where)
base_what = os.path.basename(what)
if len(base_what) > 0:
if base_what[-1] in ['"', "'"]:
base_what = base_what[0:-1]
target = where + '/' + base_what + '.lnk'
helpers_vbs = r.f.create_shortcut_vbs
cmd = 'cscript "%s" "%s" "%s" "%s" "%s"' % (helpers_vbs, target, what,
what_args, run_in)
print(cmd)
call(cmd)
def get_global_distinctiveness_modeldir(ensure=True):
# DEPRICATE
resource_dir = get_wbia_resource_dir()
global_distinctdir = join(resource_dir, const.PATH_NAMES.distinctdir)
if ensure:
ut.ensuredir(global_distinctdir)
return global_distinctdir
def vizualize_vocabulary(ibs, invindex):
"""
cleaned up version of dump_word_patches. Makes idf scatter plots and dumps
the patches that contributed to each word.
CommandLine:
python -m ibeis.algo.hots.smk.smk_plots --test-vizualize_vocabulary
python -m ibeis.algo.hots.smk.smk_plots --test-vizualize_vocabulary --vf
Example:
>>> from ibeis.algo.hots.smk.smk_plots import * # NOQA
>>> from ibeis.algo.hots.smk import smk_debug
>>> from ibeis.algo.hots.smk import smk_repr
>>> #tup = smk_debug.testdata_raw_internals0(db='GZ_ALL', nWords=64000)
>>> #tup = smk_debug.testdata_raw_internals0(db='GZ_ALL', nWords=8000)
>>> tup = smk_debug.testdata_raw_internals0(db='PZ_Master0', nWords=64000)
>>> #tup = smk_debug.testdata_raw_internals0(db='PZ_Mothers', nWords=8000)
>>> ibs, annots_df, daids, qaids, invindex, qreq_ = tup
>>> smk_repr.compute_data_internals_(invindex, qreq_.qparams, delete_rawvecs=False)
>>> vizualize_vocabulary(ibs, invindex)
"""
invindex.idx2_wxs = np.array(invindex.idx2_wxs)
print('[smk_plots] Vizualizing vocabulary')
# DUMPING PART --- dumps patches to disk
figdir = ibs.get_fig_dir()
ut.ensuredir(figdir)
if ut.get_argflag('--vf'):
ut.view_directory(figdir)
# Compute Word Statistics
metrics = compute_word_metrics(invindex)
wx2_nMembers, wx2_pdist_stats, wx2_wdist_stats = metrics
#(wx2_pdist, wx2_wdist, wx2_nMembers, wx2_pdist_stats, wx2_wdist_stats) = metrics
#wx2_prad = {wx: pdist_stats['max'] for wx, pdist_stats in six.iteritems(wx2_pdist_stats) if 'max' in pdist_stats}
#wx2_wrad = {wx: wdist_stats['max'] for wx, wdist_stats in six.iteritems(wx2_wdist_stats) if 'max' in wdist_stats}
wx2_prad = {wx: stats['max'] for wx, stats in wx2_pdist_stats.items() if 'max' in stats}
wx2_wrad = {wx: stats['max'] for wx, stats in wx2_wdist_stats.items() if 'max' in stats}
#wx2_prad = get_metric(metrics, 'wx2_pdist_stats', 'max')
#wx2_wrad = get_metric(metrics, 'wx2_wdist_stats', 'max')
wx_sample1 = select_by_metric(wx2_nMembers)
wx_sample2 = select_by_metric(wx2_prad)
wx_sample3 = select_by_metric(wx2_wrad)
wx_sample = wx_sample1 + wx_sample2 + wx_sample3
overlap123 = len(wx_sample) - len(set(wx_sample))
print('overlap123 = %r' % overlap123)
wx_sample = set(wx_sample)
print('len(wx_sample) = %r' % len(wx_sample))
#make_scatterplots(ibs, figdir, invindex, metrics)
vocabdir = join(figdir, 'vocab_patches2')
wx2_dpath = get_word_dpaths(vocabdir, wx_sample, metrics)
make_wordfigures(ibs, metrics, invindex, figdir, wx_sample, wx2_dpath)
def get_global_species_scorenorm_cachedir(ibs, species_text, ensure=True):
"""
Args:
species_text (str):
ensure (bool):
Returns:
str: species_cachedir
CommandLine:
python -m ibeis.control.IBEISControl --test-get_global_species_scorenorm_cachedir
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.control.IBEISControl import * # NOQA
>>> import ibeis # NOQA
>>> ibs = ibeis.opendb('testdb1')
>>> species_text = ibeis.const.TEST_SPECIES.ZEB_GREVY
>>> ensure = True
>>> species_cachedir = ibs.get_global_species_scorenorm_cachedir(species_text, ensure)
>>> resourcedir = ibs.get_ibeis_resource_dir()
>>> result = ut.relpath_unix(species_cachedir, resourcedir)
>>> print(result)
scorenorm/zebra_grevys
"""
scorenorm_cachedir = join(ibs.get_ibeis_resource_dir(),
const.PATH_NAMES.scorenormdir)
species_cachedir = join(scorenorm_cachedir, species_text)
if ensure:
ut.ensurepath(scorenorm_cachedir)
ut.ensuredir(species_cachedir)
return species_cachedir
def get_numpy_dataset(data_fpath, labels_fpath, training_dpath):
"""
"""
import numpy as np
# hack for caching num_labels
data = np.load(data_fpath)
data_shape = data.shape[1:]
labels = np.load(labels_fpath)
num_labels = len(labels)
alias_key = 'temp'
ut.ensuredir(training_dpath)
dataset = DataSet.new_training_set(
alias_key=alias_key,
data_fpath=data_fpath,
labels_fpath=labels_fpath,
metadata_fpath=None,
training_dpath=training_dpath,
data_shape=data_shape,
data_per_label=1,
output_dims=1,
num_labels=num_labels,
)
return dataset
def _setup_links(self, cfg_prefix, config=None):
"""
Called only when setting up an experiment to make a measurement.
Creates symlinks such that all data is written to a directory that
depends on a computer name, cfg_prefix and an arbitrary configuration
dict.
Then force the link in the basic directory to point to abs_dpath.
"""
# Setup directory
from os.path import expanduser
assert self.dname is not None
computer_id = ut.get_argval('--comp', default=ut.get_computer_name())
conf_dpath = ut.ensuredir((expanduser(self.base_dpath), 'configured'))
comp_dpath = ut.ensuredir((join(conf_dpath, computer_id)))
link_dpath = ut.ensuredir((self.base_dpath, 'link'))
# if True:
# # move to new system
# old_dpath = join(conf_dpath, self.dbname + '_' + computer_id)
# if exists(old_dpath):
# ut.move(old_dpath, join(comp_dpath, self.dbname))
try:
cfgstr = ut.repr3(config.getstate_todict_recursive())
except AttributeError:
cfgstr = ut.repr3(config)
hashid = ut.hash_data(cfgstr)[0:6]
suffix = '_'.join([cfg_prefix, hashid])
dbcode = self.dbname + '_' + suffix
abs_dpath = ut.ensuredir(join(comp_dpath, dbcode))
self.dname = dbcode
self.dpath = abs_dpath
self.abs_dpath = abs_dpath
# Place a basic link in the base link directory
links = []
links.append(expanduser(join(link_dpath, self.dbname)))
# # Make a configured but computer agnostic link
# links.append(expanduser(join(conf_dpath, self.dbname)))
for link in links:
try:
# Overwrite any existing link so the most recently used is
# the default
self.link = ut.symlink(abs_dpath, link, overwrite=True)
except Exception:
if exists(abs_dpath):
newpath = ut.non_existing_path(abs_dpath, suffix='_old')
ut.move(link, newpath)
self.link = ut.symlink(abs_dpath, link)
ut.writeto(join(abs_dpath, 'info.txt'), cfgstr)
def visualize_bounding_boxes(
ibs,
config,
version,
gid_list=None,
randomize=False,
num_images=10,
t_width=500,
output_path=None,
):
if gid_list is None:
gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **config)
else:
num_images = None
if randomize:
random.shuffle(gid_list)
if num_images not in [-1, None]:
num_images = min(num_images, len(gid_list))
gid_list = gid_list[:num_images]
uuid_list = ibs.get_image_uuids(gid_list)
assert version is not None
version = version.lower()
if version == 'prediction':
logger.info('\tGather Predictions')
val_dict = localizer_parse_pred(ibs, test_gid_list=gid_list, **config)
elif version == 'ground_truth':
logger.info('\tGather Ground-Truth')
val_dict = general_parse_gt(ibs, test_gid_list=gid_list, **config)
if output_path is None:
output_path = abspath(expanduser(join('~', 'Desktop', 'bboxes')))
ut.ensuredir(output_path)
filepath_dict = {}
for gid, image_uuid in zip(gid_list, uuid_list):
image = ibs.get_images(gid)
image = _resize(image, t_width=t_width)
h, w, c = image.shape
val_list = val_dict[image_uuid]
for val in val_list:
xbr = int(np.around(val['xbr'] * w))
ybr = int(np.around(val['ybr'] * h))
xtl = int(np.around(val['xtl'] * w))
ytl = int(np.around(val['ytl'] * h))
cv2.rectangle(image, (xtl, ytl), (xbr, ybr), (0, 140, 255), 4)
write_filename = 'bboxes_%d_%s.png' % (gid, version)
write_filepath = join(output_path, write_filename)
logger.info(write_filepath)
cv2.imwrite(write_filepath, image)
filepath_dict[gid] = write_filepath
return filepath_dict
def get_annot_probchip_fpath_list(ibs, aid_list, config2_=None, species=None):
""" Build probability chip file paths based on the current IBEIS configuration
Args:
ibs (IBEISController):
aid_list (list):
suffix (None):
Returns:
probchip_fpath_list
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.preproc.preproc_probchip import * # NOQA
>>> from os.path import basename
>>> ibs, aid_list = preproc_chip.testdata_ibeis()
>>> config2_ = ibs.new_query_params(dict(fg_on=False))
>>> probchip_fpath_list = get_annot_probchip_fpath_list(ibs, aid_list, config2_=config2_)
>>> result = ut.relpath_unix(probchip_fpath_list[1], ibs.get_dbdir())
>>> print(result)
_ibsdb/_ibeis_cache/prob_chips/probchip_avuuid=5a1a53ba-fd44-b113-7f8c-fcf248d7047f_CHIP(sz450)_FEATWEIGHT(OFF).png
_ibsdb/_ibeis_cache/prob_chips/probchip_avuuid=5a1a53ba-fd44-b113-7f8c-fcf248d7047f_CHIP(sz450)_FEATWEIGHT(ON,uselabel,rf).png
probchip_aid=5_bbox=(0,0,1072,804)_theta=0.0tau_gid=5_CHIP(sz450)_FEATWEIGHT(ON,uselabel,rf)_CHIP().png
"""
ibs.probchipdir = ibs.get_probchip_dir()
cachedir = ibs.get_probchip_dir()
ut.ensuredir(cachedir)
probchip_fname_fmt = get_probchip_fname_fmt(ibs, config2_=config2_, species=species)
annot_visual_uuid_list = ibs.get_annot_visual_uuids(aid_list)
probchip_fpath_list = [ut.unixjoin(cachedir, probchip_fname_fmt.format(avuuid=avuuid))
for avuuid in annot_visual_uuid_list]
return probchip_fpath_list
def get_flann_fpath(dpts,
cache_dir='default',
cfgstr='',
flann_params={},
use_params_hash=True,
use_data_hash=True,
appname='vtool',
verbose=True):
""" returns filepath for flann index """
if cache_dir == 'default':
if verbose:
print('[flann] using default cache dir')
cache_dir = utool.get_app_resource_dir(appname)
utool.ensuredir(cache_dir)
flann_cfgstr = get_flann_cfgstr(dpts,
flann_params,
cfgstr,
use_params_hash=use_params_hash,
use_data_hash=use_data_hash)
if verbose:
print('...flann_cache cfgstr = %r: ' % flann_cfgstr)
# Append any user labels
flann_fname = 'flann_index' + flann_cfgstr + '.flann'
flann_fpath = normpath(join(cache_dir, flann_fname))
return flann_fpath
def TEST_SQL_NAMES():
# -------- INIT DATABASE ------------
#
# Create new temp database
sqldb_fname = 'temp_test_sql_names.sqlite3'
sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
utool.ensuredir(sqldb_dpath)
print('Remove Old Temp Database')
utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
print('New Temp Database')
db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
sqldb_fname=sqldb_fname)
#
# Define the schema
__define_schema(db)
#
# -------- RUN INSERTS --------------
print('[TEST] --- INSERT NAMES --- ')
test_names = [
'fred',
'sue',
'Robert\');DROP TABLE Students;--',
'joe',
'rob',
]
__insert_names(db, test_names)
__insert_names(db, test_names[2:4])
#
# -------- RUN SELECT NAMES --------------
print('[TEST] --- SELECT NAMES ---')
name_text_results = db.executeone('SELECT name_text FROM names', [])
print(' * name_text_results=%r' % name_text_results)
#assert name_text_results == test_names, 'unexpected results from select names'
#
# -------- RUN SELECT NIDS --------------
print('[TEST] --- SELECT NIDS ---')
query_names = test_names[::2] + ['missingno']
nid_list = db.executemany(
operation='''
SELECT name_rowid
FROM names
WHERE name_text=?
''',
params_iter=((name,) for name in query_names))
# Get the parameter indexes that failed
failx_list = [count for count, nid in enumerate(nid_list) if nid is None]
assert failx_list == [3]
failed_names = [query_names[failx] for failx in failx_list] # NOQA
utool.printvar2('failed_names')
# We selected a name not in the table.
# Its return index is an empty list
print('[TEST] nid_list=%r' % nid_list)
print('[TEST] query_names=%r' % query_names)
print('[TEST] test_names=%r' % test_names)
# SQL INTEGERS START AT 1 APPARENTLY
#expected_names = [test_names[nid - 1] for nid in nid_list]
#assert expected_names == query_names, 'unexpected results from select names'
return locals()
def new_database(back, new_dbdir=None):
""" File -> New Database"""
if new_dbdir is None:
new_dbname = back.user_input(
msg='What do you want to name the new database?',
title='New Database')
if new_dbname is None or len(new_dbname) == 0:
print('Abort new database. new_dbname=%r' % new_dbname)
return
reply = back.user_option(
msg='Where should I put the new database?',
title='Import Images',
options=['Choose Directory', 'My Work Dir'],
use_cache=False)
if reply == 'Choose Directory':
print('[back] new_database(): SELECT A DIRECTORY')
putdir = guitool.select_directory('Select new database directory')
elif reply == 'My Work Dir':
putdir = back.get_work_directory()
else:
print('Abort new database')
return
new_dbdir = join(putdir, new_dbname)
if not exists(putdir):
raise ValueError('Directory %r does not exist.' % putdir)
if exists(new_dbdir):
raise ValueError('New DB %r already exists.' % new_dbdir)
utool.ensuredir(new_dbdir)
print('[back] new_database(new_dbdir=%r)' % new_dbdir)
back.open_database(dbdir=new_dbdir)
def try_download_baseline_ibeis_normalizer(ibs, qreq_):
"""
tries to download a baseline normalizer for some species.
creates an empty normalizer if it cannot
"""
baseline_url_dict = {
# TODO: Populate
}
species_text = '_'.join(qreq_.get_unique_species()) # HACK
query_cfgstr = qreq_.qparams.query_cfgstr
cachedir = qreq_.ibs.get_global_species_scorenorm_cachedir(species_text)
key = species_text + query_cfgstr
baseline_url = baseline_url_dict.get(key, None)
if baseline_url is not None:
try:
cachedir = qreq_.ibs.get_global_species_scorenorm_cachedir(species_text)
baseline_cachedir = join(cachedir, 'baseline')
ut.ensuredir(baseline_cachedir)
encoder = vt.ScoreNormalizer(cfgstr=query_cfgstr, prefix=species_text)
encoder.load(baseline_cachedir)
except Exception:
encoder = None
else:
encoder = None
if encoder is None:
if False and ut.is_developer(['hyrule']):
# train new encoder. only do this on hyrule
print('Baseline does not exist and cannot be downlaoded. Training baseline')
encoder = train_baseline_ibeis_normalizer(qreq_.ibs)
else:
# return empty score encoder
encoder = vt.ScoreNormalizer(cfgstr=query_cfgstr, prefix=species_text)
print('returning empty encoder')
#raise NotImplementedError('return the nodata noramlizer with 1/2 default')
return encoder
def set_logdir(log_dir):
from os.path import realpath, expanduser
log_dir = realpath(expanduser(log_dir))
ut.ensuredir(log_dir, verbose=True)
ut.stop_logging()
_ibeis_cache_write(LOGDIR_CACHEID, log_dir)
ut.start_logging(appname=__APPNAME__)
def TEST_SQL_NAMES():
# -------- INIT DATABASE ------------
#
# Create new temp database
sqldb_fname = 'temp_test_sql_names.sqlite3'
sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
utool.ensuredir(sqldb_dpath)
print('Remove Old Temp Database')
utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
print('New Temp Database')
db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
sqldb_fname=sqldb_fname)
#
# Define the schema
__define_schema(db)
#
# -------- RUN INSERTS --------------
print('[TEST] --- INSERT NAMES --- ')
test_names = [
'fred',
'sue',
'Robert\');DROP TABLE names;--',
'joe',
'rob',
]
__insert_names(db, test_names)
__insert_names(db, test_names[2:3])
#
# -------- RUN SELECT NAMES --------------
print('[TEST] --- SELECT NAMES ---')
name_text_results = db.executeone('SELECT name_text FROM names', [])
print(' * name_text_results=%r' % name_text_results)
#assert name_text_results == test_names, 'unexpected results from select names'
return locals()
def copy_ibeisdb(source_dbdir, dest_dbdir):
# TODO: rectify with rsync, script, and merge script.
from os.path import normpath
import ibeis
exclude_dirs_ = (ibeis.const.EXCLUDE_COPY_REL_DIRS +
['_hsdb', '.hs_internals'])
exclude_dirs = [ut.ensure_unixslash(normpath(rel))
for rel in exclude_dirs_]
rel_tocopy = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs,
recursive=True, with_files=True, with_dirs=False,
fullpath=False)
rel_tocopy_dirs = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs,
recursive=True, with_files=False, with_dirs=True,
fullpath=False)
src_list = [join(source_dbdir, relpath) for relpath in rel_tocopy]
dst_list = [join(dest_dbdir, relpath) for relpath in rel_tocopy]
# ensure directories exist
rel_tocopy_dirs = [dest_dbdir] + [join(dest_dbdir, dpath_)
for dpath_ in rel_tocopy_dirs]
for dpath in rel_tocopy_dirs:
ut.ensuredir(dpath)
# copy files
ut.copy(src_list, dst_list)
def TEST_SQL_NAMES():
# -------- INIT DATABASE ------------
#
# Create new temp database
sqldb_fname = 'temp_test_sql_names.sqlite3'
sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
utool.ensuredir(sqldb_dpath)
print('Remove Old Temp Database')
utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
print('New Temp Database')
db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
sqldb_fname=sqldb_fname)
#
# Define the schema
__define_schema(db)
#
# -------- RUN INSERTS --------------
print('[TEST] --- INSERT NAMES --- ')
test_names = [
'fred',
'sue',
'Robert\');DROP TABLE Students;--',
'joe',
'rob',
]
__insert_names(db, test_names)
__insert_names(db, test_names[2:4])
#
# -------- RUN SELECT NAMES --------------
print('[TEST] --- SELECT NAMES ---')
name_text_results = db.executeone('SELECT name_text FROM names', [])
print(' * name_text_results=%r' % name_text_results)
#assert name_text_results == test_names, 'unexpected results from select names'
#
# -------- RUN SELECT NIDS --------------
print('[TEST] --- SELECT NIDS ---')
query_names = test_names[::2] + ['missingno']
nid_list = db.executemany(operation='''
SELECT name_rowid
FROM names
WHERE name_text=?
''',
params_iter=((name, ) for name in query_names))
# Get the parameter indexes that failed
failx_list = [count for count, nid in enumerate(nid_list) if nid is None]
assert failx_list == [3]
failed_names = [query_names[failx] for failx in failx_list] # NOQA
utool.printvar2('failed_names')
# We selected a name not in the table.
# Its return index is an empty list
print('[TEST] nid_list=%r' % nid_list)
print('[TEST] query_names=%r' % query_names)
print('[TEST] test_names=%r' % test_names)
# SQL INTEGERS START AT 1 APPARENTLY
#expected_names = [test_names[nid - 1] for nid in nid_list]
#assert expected_names == query_names, 'unexpected results from select names'
return locals()
def train(dark, voc_path, weight_path, **kwargs):
"""
Train a new forest with the given positive chips and negative chips.
Args:
train_pos_chip_path_list (list of str): list of positive training chips
train_neg_chip_path_list (list of str): list of negative training chips
trees_path (str): string path of where the newly trained trees are to be saved
Kwargs:
chips_norm_width (int, optional): Chip normalization width for resizing;
the chip is resized to have a width of chips_norm_width and
whatever resulting height in order to best match the original
aspect ratio; defaults to 128
If both chips_norm_width and chips_norm_height are specified,
the original aspect ratio of the chip is not respected
chips_norm_height (int, optional): Chip normalization height for resizing;
the chip is resized to have a height of chips_norm_height and
whatever resulting width in order to best match the original
aspect ratio; defaults to None
If both chips_norm_width and chips_norm_height are specified,
the original aspect ratio of the chip is not respected
verbose (bool, optional): verbose flag; defaults to object's verbose or
selectively enabled for this function
Returns:
None
"""
# Default values
params = odict(
[
("weight_filepath", None), # This value always gets overwritten
("verbose", dark.verbose),
("quiet", dark.quiet),
]
)
# params.update(kwargs)
ut.update_existing(params, kwargs)
# Make the tree path absolute
weight_path = abspath(weight_path)
ut.ensuredir(weight_path)
# Setup training files and folder structures
results = dark._train_setup(voc_path, weight_path)
manifest_filename, num_images, config_filepath, class_filepath = results
# Run training algorithm
params_list = [dark.net, manifest_filename, weight_path, num_images] + list(params.values())
DARKNET_CLIB.train(*params_list)
weight_filepath = params["weight_filepath"]
if not params["quiet"]:
print("\n\n[pydarknet py] *************************************")
print("[pydarknet py] Training Completed")
print("[pydarknet py] Weight file saved to: %s" % (weight_filepath,))
return weight_filepath, config_filepath, class_filepath
def labeler_train(ibs,
species_list=None,
species_mapping=None,
viewpoint_mapping=None,
ensembles=3,
**kwargs):
from ibeis_cnn.ingest_ibeis import get_cnn_labeler_training_images_pytorch
from ibeis.algo.detect import densenet
species = '-'.join(species_list)
args = (species, )
data_path = join(ibs.get_cachedir(), 'extracted-labeler-%s' % args)
extracted_path = get_cnn_labeler_training_images_pytorch(
ibs,
category_list=species_list,
category_mapping=species_mapping,
viewpoint_mapping=viewpoint_mapping,
dest_path=data_path,
**kwargs)
weights_path_list = []
for ensemble_num in range(ensembles):
args = (
species,
ensemble_num,
)
output_path = join(ibs.get_cachedir(), 'training',
'labeler-%s-ensemble-%d' % args)
if exists(output_path):
ut.delete(output_path)
weights_path = densenet.train(extracted_path,
output_path,
blur=False,
flip=False)
weights_path_list.append(weights_path)
args = (species, )
output_name = 'labeler.%s' % args
ensemble_path = join(ibs.get_cachedir(), 'training', output_name)
ut.ensuredir(ensemble_path)
archive_path = '%s.zip' % (ensemble_path)
ensemble_weights_path_list = []
for index, weights_path in enumerate(sorted(weights_path_list)):
assert exists(weights_path)
ensemble_weights_path = join(ensemble_path,
'labeler.%d.weights' % (index, ))
ut.copy(weights_path, ensemble_weights_path)
ensemble_weights_path_list.append(ensemble_weights_path)
ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list
ut.archive_files(archive_path,
ensemble_weights_path_list,
overwrite=True,
common_prefix=True)
return archive_path
def db_to_dbdir(db, allow_newdir=False, extra_workdirs=[], use_sync=False):
""" Implicitly gets dbdir. Searches for db inside of workdir """
work_dir = get_workdir()
dbalias_dict = get_dbalias_dict()
workdir_list = []
for extra_dir in extra_workdirs:
if exists(extra_dir):
workdir_list.append(extra_dir)
if use_sync:
sync_dir = join(work_dir, '../sync')
if exists(sync_dir):
workdir_list.append(sync_dir)
workdir_list.append(work_dir) # TODO: Allow multiple workdirs
# Check all of your work directories for the database
for _dir in workdir_list:
dbdir = realpath(join(_dir, db))
# Use db aliases
if not exists(dbdir) and db.upper() in dbalias_dict:
dbdir = join(_dir, dbalias_dict[db.upper()])
if exists(dbdir):
break
# Create the database if newdbs are allowed in the workdir
#print('allow_newdir=%r' % allow_newdir)
if allow_newdir:
utool.ensuredir(dbdir, verbose=True)
# Complain if the implicit dbdir does not exist
if not exists(dbdir):
print('!!!')
print('[sysres] WARNING: db=%r not found in work_dir=%r' %
(db, work_dir))
fname_list = os.listdir(work_dir)
lower_list = [fname.lower() for fname in fname_list]
index = util_list.listfind(lower_list, db.lower())
if index is not None:
print('[sysres] WARNING: db capitalization seems to be off')
if not utool.STRICT:
print('[sysres] attempting to fix it')
db = fname_list[index]
dbdir = join(work_dir, db)
print('[sysres] dbdir=%r' % dbdir)
print('[sysres] db=%r' % db)
if not exists(dbdir):
msg = '[sysres!] ERROR: Database does not exist'
print('<!!!>')
print(msg)
print('[sysres!] Here is a list of valid dbs: ' +
utool.indentjoin(fname_list, '\n * '))
print('[sysres!] dbdir=%r' % dbdir)
print('[sysres!] db=%r' % db)
print('[sysres!] work_dir=%r' % work_dir)
print('</!!!>')
raise AssertionError(msg)
print('!!!')
return dbdir
def TEST_SQL_NUMPY():
sqldb_fname = 'temp_test_sql_numpy.sqlite3'
sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
utool.ensuredir(sqldb_dpath)
utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
db = sqldbc.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
sqldb_fname=sqldb_fname)
db.add_table('temp', [
('temp_id', 'INTEGER PRIMARY KEY'),
('temp_hash', 'NUMPY'),
])
tt = utool.tic()
feats_list = grab_numpy_testdata(shape=(3e3, 128), dtype=np.uint8)
print(' * numpy.new time=%r sec' % utool.toc(tt))
print('[TEST] insert numpy arrays')
tt = utool.tic()
feats_iter = ((feats, ) for feats in feats_list)
db.executemany(operation='''
INSERT
INTO temp
(
temp_hash
)
VALUES (?)
''',
params_iter=feats_iter)
print(' * execute insert time=%r sec' % utool.toc(tt))
print('[TEST] save sql database')
tt = utool.tic()
#db.cur.commit()
db.connection.commit()
print(' * commit time=%r sec' % utool.toc(tt))
print('[TEST] read from sql database')
tt = utool.tic()
db.cur.execute('SELECT temp_hash FROM temp', [])
print(' * execute select time=%r sec' % utool.toc(tt))
tt = utool.tic()
result_list = _results_gen(db.cur)
print(' * iter results time=%r sec' % utool.toc(tt))
print(' * memory(result_list) = %s' %
utool.byte_str2(utool.get_object_size(result_list)))
del result_list
#print('[TEST] result_list=%r' % result_list)
print('[TEST] dump sql database')
tt = utool.tic()
db.dump('temp.dump.txt')
print(' * dump time=%r sec' % utool.toc(tt))
#with open('temp.dump.txt') as file_:
# print(file_.read())
return locals()
def ingest_standard_database(db, force_delete=False):
print('[ingest] Ingest Standard Database: db=%r' % (db,))
ingestable = get_standard_ingestable(db)
dbdir = ibeis.sysres.db_to_dbdir(ingestable.db, allow_newdir=True, use_sync=False)
utool.ensuredir(dbdir, verbose=True)
if force_delete:
ibsfuncs.delete_ibeis_database(dbdir)
ibs = IBEISControl.IBEISController(dbdir)
ingest_rawdata(ibs, ingestable)
def _make_empty_controller():
print('make_empty_controller')
sqldb_fname = 'temp_test_sql_control.sqlite3'
sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
utool.ensuredir(sqldb_dpath)
utool.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
sqldb_fname=sqldb_fname)
return db
def get_local_species_scorenorm_cachedir(ibs, species_text, ensure=True):
""""""
scorenorm_cachedir = join(ibs.get_cachedir(),
const.PATH_NAMES.scorenormdir)
species_cachedir = join(scorenorm_cachedir, species_text)
if ensure:
ut.ensuredir(scorenorm_cachedir)
ut.ensuredir(species_cachedir)
return species_cachedir
def canonical_classifier_train(ibs,
species,
ensembles=3,
extracted_path=None,
**kwargs):
from wbia.other.detectexport import (
get_cnn_classifier_canonical_training_images_pytorch, )
from wbia.algo.detect import densenet
args = (species, )
data_path = join(ibs.get_cachedir(),
'extracted-classifier-canonical-%s' % args)
if extracted_path is None:
extracted_path = get_cnn_classifier_canonical_training_images_pytorch(
ibs,
species,
dest_path=data_path,
)
weights_path_list = []
for ensemble_num in range(ensembles):
args = (
species,
ensemble_num,
)
output_path = join(ibs.get_cachedir(), 'training',
'classifier-canonical-%s-ensemble-%d' % args)
if exists(output_path):
ut.delete(output_path)
weights_path = densenet.train(extracted_path,
output_path,
blur=False,
flip=False)
weights_path_list.append(weights_path)
args = (species, )
output_name = 'classifier.canonical.%s' % args
ensemble_path = join(ibs.get_cachedir(), 'training', output_name)
ut.ensuredir(ensemble_path)
archive_path = '%s.zip' % (ensemble_path)
ensemble_weights_path_list = []
for index, weights_path in enumerate(sorted(weights_path_list)):
assert exists(weights_path)
ensemble_weights_path = join(
ensemble_path, 'classifier.canonical.%d.weights' % (index, ))
ut.copy(weights_path, ensemble_weights_path)
ensemble_weights_path_list.append(ensemble_weights_path)
ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list
ut.archive_files(archive_path,
ensemble_weights_path_list,
overwrite=True,
common_prefix=True)
return archive_path
def localizer_train(ibs, species_list=None, **kwargs):
from pydarknet import Darknet_YOLO_Detector
data_path = ibs.export_to_xml(species_list=species_list, **kwargs)
output_path = join(ibs.get_cachedir(), 'training', 'localizer')
ut.ensuredir(output_path)
dark = Darknet_YOLO_Detector()
model_path = dark.train(data_path, output_path)
del dark
return model_path
def _make_empty_controller():
print('make_empty_controller')
sqldb_fname = 'temp_test_sql_control.sqlite3'
sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
utool.ensuredir(sqldb_dpath)
utool.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
db = SQLDatabaseControl.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
sqldb_fname=sqldb_fname)
return db
def dump_tables_to_csv(db):
""" Convenience: Dumps all csv database files to disk """
dump_dir = join(db.dir_, 'CSV_DUMP')
utool.ensuredir(dump_dir)
for tablename in six.iterkeys(db.table_columns):
table_fname = tablename + '.csv'
table_csv = db.get_table_csv(tablename)
with open(join(dump_dir, table_fname), 'w') as file_:
file_.write(table_csv)
def TEST_SQL_NUMPY():
sqldb_fname = 'temp_test_sql_numpy.sqlite3'
sqldb_dpath = utool.util_cplat.get_app_resource_dir('ibeis', 'testfiles')
utool.ensuredir(sqldb_dpath)
utool.util_path.remove_file(join(sqldb_dpath, sqldb_fname), dryrun=False)
db = sqldbc.SQLDatabaseController(sqldb_dpath=sqldb_dpath,
sqldb_fname=sqldb_fname)
db.schema('temp', [
('temp_id', 'INTEGER PRIMARY KEY'),
('temp_hash', 'NUMPY'),
])
tt = utool.tic()
feats_list = grab_numpy_testdata(shape=(3e3, 128), dtype=np.uint8)
print(' * numpy.new time=%r sec' % utool.toc(tt))
print('[TEST] insert numpy arrays')
tt = utool.tic()
feats_iter = ((feats, ) for feats in feats_list)
db.executemany(operation='''
INSERT
INTO temp
(
temp_hash
)
VALUES (?)
''', params_iter=feats_iter)
print(' * execute insert time=%r sec' % utool.toc(tt))
print('[TEST] save sql database')
tt = utool.tic()
#db.cur.commit()
db.connection.commit()
print(' * commit time=%r sec' % utool.toc(tt))
print('[TEST] read from sql database')
tt = utool.tic()
db.cur.execute('SELECT temp_hash FROM temp', [])
print(' * execute select time=%r sec' % utool.toc(tt))
tt = utool.tic()
result_list = _results_gen(db.cur)
print(' * iter results time=%r sec' % utool.toc(tt))
print(' * memory(result_list) = %s' % utool.byte_str2(utool.get_object_size(result_list)))
del result_list
#print('[TEST] result_list=%r' % result_list)
print('[TEST] dump sql database')
tt = utool.tic()
db.dump('temp.dump.txt')
print(' * dump time=%r sec' % utool.toc(tt))
#with open('temp.dump.txt') as file_:
# print(file_.read())
return locals()
def test_akmeans(full_test=False,
plot_test=False,
num_pca_dims=2,
data_dim=2,
nump=1000):
import numpy as np
from vtool import clustering
nump = nump
dims = data_dim # 128
dtype = np.uint8
print('Make %d random %d-dimensional %s points.' % (nump, dims, dtype))
# Seed for a determenistic test
np.random.seed(42)
data = np.array(np.random.randint(0, 255, (nump, dims)), dtype=dtype)
num_clusters = 10
max_iters = 2
ave_unchanged_thresh = 0
ave_unchanged_iterwin = 10
flann_params = {}
cache_dir = utool.get_app_resource_dir('vtool', 'test_cache')
utool.ensuredir(cache_dir)
# Test precomputing
dx2_label, centers = clustering.precompute_akmeans(data,
num_clusters,
max_iters=max_iters,
cache_dir=cache_dir)
# internal names
datax2_clusterx, centroids = dx2_label, centers
if plot_test:
clustering.plot_clusters(data,
datax2_clusterx,
centroids,
num_pca_dims=num_pca_dims)
assert centers.shape == (num_clusters, dims), 'sanity check'
assert dx2_label.shape == (nump, ), 'sanity check'
# Test regular computing
if full_test:
dx2_label, centers = clustering.akmeans(data,
num_clusters,
max_iters=max_iters)
assert centers.shape == (num_clusters, dims), 'sanity check'
assert dx2_label.shape == (nump, ), 'sanity check'
if False:
# other test (development)
import pyflann
flann_lib_inst = pyflann.flann
flann_class_inst = pyflann.FLANN()
flann_class_inst.build_index(data)
return locals()
def export_ibeis_to_hotspotter(ibs):
print('export to hsdb')
ibs.inject_func(get_hsdb_image_gpaths)
hsdb_dir = ibs.get_dbdir()
internal_hsdb = join(hsdb_dir, '_hsdb')
utool.ensuredir(internal_hsdb)
# Build Image Table
gid_list = ibs.get_valid_gids()
gpath_list = ibs.get_hsdb_image_gpaths(gid_list)
reviewed_list = ibs.get_image_reviewed(gid_list)
# aif in hotspotter is equivilant to reviewed in IBEIS
image_table_csv = utool.make_csv_table(
[gid_list, gpath_list, reviewed_list],
['gid', 'gname', 'aif'],
'# image table')
# Build Name Table
nid_list = ibs.get_valid_nids()
name_list = ibs.get_name_text(nid_list)
name_table_csv = utool.make_csv_table(
[nid_list, name_list],
['nid', 'name'],
'# name table')
# Build Chip Table
aid_list = ibs.get_valid_aids()
annotationgid_list = ibs.get_annot_gids(aid_list)
annotationnid_list = ibs.get_annot_nids(aid_list)
bbox_list = map(list, ibs.get_annot_bboxes(aid_list))
theta_list = ibs.get_annot_thetas(aid_list)
notes_list = ibs.get_annot_notes(aid_list)
chip_column_list = [aid_list, annotationgid_list, annotationnid_list, bbox_list, theta_list, notes_list]
chip_column_lbls = ['cid', 'gid', 'nid', '[tlx tly w h]', 'theta', 'notes']
chip_column_types = [int, int, int, list, float, str]
chip_table_csv = utool.make_csv_table(
chip_column_list,
chip_column_lbls,
'# chip table', chip_column_types)
if utool.VERBOSE:
if len(aid_list) < 87:
print(chip_table_csv)
if len(nid_list) < 87:
print(name_table_csv)
if len(gid_list) < 87:
print(image_table_csv)
# Write Tables
with open(join(internal_hsdb, 'HSDB_image_table.csv'), 'wb') as imgtbl_file:
imgtbl_file.write(image_table_csv)
with open(join(internal_hsdb, 'HSDB_name_table.csv'), 'wb') as nametbl_file:
nametbl_file.write(name_table_csv)
with open(join(internal_hsdb, 'HSDB_chip_table.csv'), 'wb') as chiptbl_file:
chiptbl_file.write(chip_table_csv)
def imwrite_theano_symbolic_graph(thean_expr):
import theano
graph_dpath = '.'
graph_fname = 'symbolic_graph.png'
graph_fpath = ut.unixjoin(graph_dpath, graph_fname)
ut.ensuredir(graph_dpath)
theano.printing.pydotprint(
thean_expr, outfile=graph_fpath, var_with_name_simple=True)
ut.startfile(graph_fpath)
return graph_fpath
def get_word_dpaths(vocabdir, wx_sample, metrics):
"""
Gets word folder names and ensure they exist
"""
ut.ensuredir(vocabdir)
wx2_dpath = {wx: join(vocabdir, get_word_dname(wx, metrics)) for wx in wx_sample}
iter_ = ut.progiter(six.itervalues(wx2_dpath), lbl='Ensuring word_dpath: ', freq=200)
for dpath in iter_:
ut.ensuredir(dpath)
return wx2_dpath
def get_local_species_scorenorm_cachedir(ibs, species_text, ensure=True):
"""
"""
scorenorm_cachedir = join(ibs.get_cachedir(),
const.PATH_NAMES.scorenormdir)
species_cachedir = join(scorenorm_cachedir, species_text)
if ensure:
ut.ensuredir(scorenorm_cachedir)
ut.ensuredir(species_cachedir)
return species_cachedir
def __init__(self):
import utool as ut
self.modname = None
code_dpath = ut.truepath(ut.get_argval('--code-dir', default='~/code'))
self.code_dpath = ut.unexpanduser(code_dpath)
self.repo_fname = (ut.get_argval(('--repo', '--repo-name'), type_=str))
self.repo_dpath = join(code_dpath, self.repo_fname)
self.modname = ut.get_argval('--modname', default=self.repo_fname)
self.regenfmt = 'python -m utool SetupRepo.{cmd} --modname={modname} --repo={repo_fname} --codedir={code_dpath}'
ut.ensuredir(self.repo_dpath, verbose=True)
def detector_train(ibs):
results = ibs.localizer_train()
localizer_weight_path, localizer_config_path, localizer_class_path = results
classifier_model_path = ibs.classifier_binary_train()
labeler_model_path = ibs.labeler_train()
output_path = join(ibs.get_cachedir(), 'training', 'detector')
ut.ensuredir(output_path)
ut.copy(localizer_weight_path, join(output_path, 'localizer.weights'))
ut.copy(localizer_config_path, join(output_path, 'localizer.config'))
ut.copy(localizer_class_path, join(output_path, 'localizer.classes'))
ut.copy(classifier_model_path, join(output_path, 'classifier.npy'))
ut.copy(labeler_model_path, join(output_path, 'labeler.npy'))
def build_sphinx_apidoc_cmdstr():
print('')
print('if this fails try: sudo pip install sphinx')
print('')
apidoc = 'sphinx-apidoc'
if ut.WIN32:
winprefix = 'C:/Python27/Scripts/'
sphinx_apidoc_exe = winprefix + apidoc + '.exe'
else:
sphinx_apidoc_exe = apidoc
apidoc_argfmt_list = [
sphinx_apidoc_exe,
'--force',
'--full',
'--maxdepth="{maxdepth}"',
'--doc-author="{author}"',
'--doc-version="{doc_version}"',
'--doc-release="{doc_release}"',
'--output-dir="_doc"',
#'--separate', # Put documentation for each module on its own page
'--private', # Include "_private" modules
'{pkgdir}',
]
outputdir = '_doc'
author = ut.parse_author()
packages = ut.find_packages(maxdepth=1)
assert len(
packages) != 0, 'directory must contain at least one package'
if len(packages) > 1:
assert len(packages) == 1,\
('FIXME I dont know what to do with more than one root package: %r'
% (packages,))
pkgdir = packages[0]
version = ut.parse_package_for_version(pkgdir)
modpath = dirname(ut.truepath(pkgdir))
apidoc_fmtdict = {
'author': author,
'maxdepth': '8',
'pkgdir': pkgdir,
'doc_version': version,
'doc_release': version,
'outputdir': outputdir,
}
ut.assert_exists('setup.py')
ut.ensuredir('_doc')
apidoc_fmtstr = ' '.join(apidoc_argfmt_list)
apidoc_cmdstr = apidoc_fmtstr.format(**apidoc_fmtdict)
print('[util_setup] autogenerate sphinx docs for %r' % (pkgdir, ))
if ut.VERBOSE:
print(ut.dict_str(apidoc_fmtdict))
return apidoc_cmdstr, modpath, outputdir
def export_ibeis_to_hotspotter(ibs):
# Dumps the files
hsdb_dir = ibs.get_dbdir()
internal_hsdb = join(hsdb_dir, "_hsdb")
ut.ensuredir(internal_hsdb)
image_table_csv, name_table_csv, chip_table_csv = get_hots_table_strings(ibs)
# Write Tables
with open(join(internal_hsdb, "HSDB_image_table.csv"), "wb") as imgtbl_file:
imgtbl_file.write(image_table_csv)
with open(join(internal_hsdb, "HSDB_name_table.csv"), "wb") as nametbl_file:
nametbl_file.write(name_table_csv)
with open(join(internal_hsdb, "HSDB_chip_table.csv"), "wb") as chiptbl_file:
chiptbl_file.write(chip_table_csv)
def build_sphinx_apidoc_cmdstr():
print('')
print('if this fails try: sudo pip install sphinx')
print('')
apidoc = 'sphinx-apidoc'
if ut.WIN32:
winprefix = 'C:/Python27/Scripts/'
sphinx_apidoc_exe = winprefix + apidoc + '.exe'
else:
sphinx_apidoc_exe = apidoc
apidoc_argfmt_list = [
sphinx_apidoc_exe,
'--force',
'--full',
'--maxdepth="{maxdepth}"',
'--doc-author="{author}"',
'--doc-version="{doc_version}"',
'--doc-release="{doc_release}"',
'--output-dir="_doc"',
#'--separate', # Put documentation for each module on its own page
'--private', # Include "_private" modules
'{pkgdir}',
]
outputdir = '_doc'
author = ut.parse_author()
packages = ut.find_packages(maxdepth=1)
assert len(packages) != 0, 'directory must contain at least one package'
if len(packages) > 1:
assert len(packages) == 1,\
('FIXME I dont know what to do with more than one root package: %r'
% (packages,))
pkgdir = packages[0]
version = ut.parse_package_for_version(pkgdir)
modpath = dirname(ut.truepath(pkgdir))
apidoc_fmtdict = {
'author': author,
'maxdepth': '8',
'pkgdir': pkgdir,
'doc_version': version,
'doc_release': version,
'outputdir': outputdir,
}
ut.assert_exists('setup.py')
ut.ensuredir('_doc')
apidoc_fmtstr = ' '.join(apidoc_argfmt_list)
apidoc_cmdstr = apidoc_fmtstr.format(**apidoc_fmtdict)
print('[util_setup] autogenerate sphinx docs for %r' % (pkgdir,))
if ut.VERBOSE:
print(ut.dict_str(apidoc_fmtdict))
return apidoc_cmdstr, modpath, outputdir
def test_labeler(output_path):
print('[labeler] Loading the labeler training data')
data_list, label_list = load_images()
print('[mnist] Loading the data into a JPCNN_Data')
data = JPCNN_Data()
data.set_data_list(data_list)
data.set_label_list(label_list)
print('[labeler] Create the JPCNN_Model used for testing')
model = Labeler_Model('model.npy')
print('[mnist] Create the JPCNN_network and start testing')
net = JPCNN_Network(model, data)
test_results = net.test(output_path, best_weights=True)
prediction_list = test_results['label_list']
confidence_list = test_results['confidence_list']
best_errors = np.inf
conf_list = [ _ / 100.0 for _ in range(0, 101) ]
# conf_list = [ 0.81 ] # FOR MODEL.5.NPY
for conf in conf_list:
failure_path = join(output_path, 'failures')
ut.ensuredir(failure_path)
error_list = [0, 0, 0, 0]
zipped = zip(data_list, label_list, prediction_list, confidence_list)
for index, (data, label, prediction, confidence) in enumerate(zipped):
if prediction == 'negative' and confidence < conf:
prediction = 'positive'
confidence == 1.0 - confidence
if label == prediction and label == 'positive':
error_list[0] += 1
elif label == prediction and label == 'negative':
error_list[1] += 1
elif label != prediction:
if label == 'positive':
error_list[2] += 1
elif label == 'negative':
error_list[3] += 1
args = (confidence, index, label, prediction)
failure_filename = 'failure_%0.05f_%06d_%s_%s.png' % args
failure_filepath = join(failure_path, failure_filename)
cv2.imwrite(failure_filepath, data)
errors = error_list[2] + error_list[3]
total = sum(error_list)
if errors < best_errors:
best_errors = errors
print(error_list)
args = (conf, errors / total, errors, total, )
print('Error rate %0.2f: %0.03f [ %d / %d ]' % args)
def dummy_manual_chipmask(depc, parent_rowids, config=None):
import vtool as vt
from plottool import interact_impaint
mask_dpath = join(depc.cache_dpath, 'ManualChipMask')
ut.ensuredir(mask_dpath)
if config is None:
config = {}
print('Requesting user defined chip mask')
for rowid in parent_rowids:
img = vt.imread(gpath_list[rowid])
mask = interact_impaint.impaint_mask2(img)
mask_fpath = join(mask_dpath, 'mask%d.png' % (rowid, ))
vt.imwrite(mask_fpath, mask)
w, h = vt.get_size(mask)
yield (w, h), mask_fpath
def extract_zipfile_images(ibs, ingestable):
import utool as ut # NOQA
zipfile_list = ut.glob(ingestable.img_dir, '*.zip', recursive=True)
if len(zipfile_list) > 0:
print('Found zipfile_list = %r' % (zipfile_list,))
ut.ensuredir(unzipped_file_base_dir)
for zipfile in zipfile_list:
unziped_file_relpath = dirname(relpath(relpath(realpath(zipfile), realpath(ingestable.img_dir))))
unzipped_file_dir = join(unzipped_file_base_dir, unziped_file_relpath)
ut.ensuredir(unzipped_file_dir)
ut.unzip_file(zipfile, output_dir=unzipped_file_dir, overwrite=False)
gpath_list = ut.list_images(unzipped_file_dir, fullpath=True, recursive=True)
else:
gpath_list = []
return gpath_list
def get_word_dpaths(vocabdir, wx_sample, metrics):
"""
Gets word folder names and ensure they exist
"""
ut.ensuredir(vocabdir)
wx2_dpath = {
wx: join(vocabdir, get_word_dname(wx, metrics))
for wx in wx_sample
}
iter_ = ut.progiter(six.itervalues(wx2_dpath),
lbl='Ensuring word_dpath: ',
freq=200)
for dpath in iter_:
ut.ensuredir(dpath)
return wx2_dpath
def dummy_manual_chipmask(depc, parent_rowids, config=None):
import vtool as vt
from plottool import interact_impaint
mask_dpath = ut.unixjoin(depc.cache_dpath, 'ManualChipMask')
ut.ensuredir(mask_dpath)
if config is None:
config = {}
print('Requesting user defined chip mask')
for rowid in parent_rowids:
img = vt.imread(gpath_list[rowid])
mask = interact_impaint.impaint_mask2(img)
mask_fpath = ut.unixjoin(mask_dpath, 'mask%d.png' % (rowid,))
vt.imwrite(mask_fpath, mask)
w, h = vt.get_size(mask)
yield (w, h), mask_fpath
def __init__(qreq_, ibs=None, qaids=None, daids=None, config=None):
super(SMKRequest, qreq_).__init__()
if config is None:
config = {}
qreq_.ibs = ibs
qreq_.qaids = qaids
qreq_.daids = daids
qreq_.config = config
# qreq_.vocab = None
# qreq_.dinva = None
qreq_.qinva = None
qreq_.dinva = None
qreq_.smk = SMK()
# Hack to work with existing hs code
qreq_.stack_config = SMKRequestConfig(**config)
# Flat config
qreq_.qparams = dtool.base.StackedConfig(
[dict(qreq_.stack_config.parse_items())])
# # TODO: add vocab, inva, features
qreq_.cachedir = ut.ensuredir((ibs.cachedir, 'smk'))
def test_akmeans(full_test=False, plot_test=False, num_pca_dims=2, data_dim=2,
nump=1000):
import numpy as np
from vtool import clustering
nump = nump
dims = data_dim # 128
dtype = np.uint8
print('Make %d random %d-dimensional %s points.' % (nump, dims, dtype))
# Seed for a determenistic test
np.random.seed(42)
data = np.array(np.random.randint(0, 255, (nump, dims)), dtype=dtype)
num_clusters = 10
max_iters = 2
ave_unchanged_thresh = 0
ave_unchanged_iterwin = 10
flann_params = {}
cache_dir = utool.get_app_resource_dir('vtool', 'test_cache')
utool.ensuredir(cache_dir)
# Test precomputing
dx2_label, centers = clustering.precompute_akmeans(data, num_clusters,
max_iters=max_iters,
cache_dir=cache_dir)
# internal names
datax2_clusterx, centroids = dx2_label, centers
if plot_test:
clustering.plot_clusters(data, datax2_clusterx, centroids, num_pca_dims=num_pca_dims)
assert centers.shape == (num_clusters, dims), 'sanity check'
assert dx2_label.shape == (nump,), 'sanity check'
# Test regular computing
if full_test:
dx2_label, centers = clustering.akmeans(data, num_clusters, max_iters=max_iters)
assert centers.shape == (num_clusters, dims), 'sanity check'
assert dx2_label.shape == (nump,), 'sanity check'
if False:
# other test (development)
import pyflann
flann_lib_inst = pyflann.flann
flann_class_inst = pyflann.FLANN()
flann_class_inst.build_index(data)
return locals()
def image_upload(cleanup=True, **kwargs):
r"""
Returns the gid for an uploaded image.
Args:
image (image binary): the POST variable containing the binary
(multi-form) image data
**kwargs: Arbitrary keyword arguments; the kwargs are passed down to
the add_images function
Returns:
gid (rowids): gid corresponding to the image submitted.
lexigraphical order.
RESTful:
Method: POST
URL: /api/upload/image/
"""
ibs = current_app.ibs
logger.info('request.files = %s' % (request.files, ))
filestore = request.files.get('image', None)
if filestore is None:
raise controller_inject.WebMissingInput(
'Missing required image parameter', 'image')
# raise IOError('Image not given')
uploads_path = ibs.get_uploadsdir()
ut.ensuredir(uploads_path)
current_time = time.strftime('%Y_%m_%d_%H_%M_%S')
modifier = 1
upload_filename = 'upload_%s.png' % (current_time)
while exists(upload_filename):
upload_filename = 'upload_%s_%04d.png' % (current_time, modifier)
modifier += 1
upload_filepath = join(uploads_path, upload_filename)
filestore.save(upload_filepath)
gid_list = ibs.add_images([upload_filepath], **kwargs)
gid = gid_list[0]
if cleanup and exists(upload_filepath):
ut.delete(upload_filepath)
return gid
def canonical_localizer_train(ibs, species, ensembles=3, **kwargs):
from ibeis_cnn.ingest_ibeis import get_cnn_localizer_canonical_training_images_pytorch
from ibeis.algo.detect import canonical
args = (species, )
data_path = join(ibs.get_cachedir(),
'extracted-localizer-canonical-%s' % args)
extracted_path = get_cnn_localizer_canonical_training_images_pytorch(
ibs,
species,
dest_path=data_path,
)
weights_path_list = []
for ensemble_num in range(ensembles):
args = (
species,
ensemble_num,
)
output_path = join(ibs.get_cachedir(), 'training',
'localizer-canonical-%s-ensemble-%d' % args)
weights_path = canonical.train(extracted_path, output_path)
weights_path_list.append(weights_path)
args = (species, )
output_name = 'localizer.canonical.%s' % args
ensemble_path = join(ibs.get_cachedir(), 'training', output_name)
ut.ensuredir(ensemble_path)
archive_path = '%s.zip' % (ensemble_path)
ensemble_weights_path_list = []
for index, weights_path in enumerate(sorted(weights_path_list)):
assert exists(weights_path)
ensemble_weights_path = join(
ensemble_path, 'localizer.canonical.%d.weights' % (index, ))
ut.copy(weights_path, ensemble_weights_path)
ensemble_weights_path_list.append(ensemble_weights_path)
ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list
ut.archive_files(archive_path,
ensemble_weights_path_list,
overwrite=True,
common_prefix=True)
return archive_path
def export_wbia_to_hotspotter(ibs):
# Dumps the files
hsdb_dir = ibs.get_dbdir()
internal_hsdb = join(hsdb_dir, '_hsdb')
ut.ensuredir(internal_hsdb)
image_table_csv, name_table_csv, chip_table_csv = get_hots_table_strings(
ibs)
# Write Tables
with open(join(internal_hsdb, 'HSDB_image_table.csv'),
'wb') as imgtbl_file:
imgtbl_file.write(image_table_csv)
with open(join(internal_hsdb, 'HSDB_name_table.csv'),
'wb') as nametbl_file:
nametbl_file.write(name_table_csv)
with open(join(internal_hsdb, 'HSDB_chip_table.csv'),
'wb') as chiptbl_file:
chiptbl_file.write(chip_table_csv)
def image_upload(cleanup=True, **kwargs):
r"""
Returns the gid for an uploaded image.
Args:
image (image binary): the POST variable containing the binary
(multi-form) image data
**kwargs: Arbitrary keyword arguments; the kwargs are passed down to
the add_images function
Returns:
gid (rowids): gid corresponding to the image submitted.
lexigraphical order.
RESTful:
Method: POST
URL: /api/image/
"""
ibs = current_app.ibs
print('request.files = %s' % (request.files,))
filestore = request.files.get('image', None)
if filestore is None:
raise IOError('Image not given')
uploads_path = ibs.get_uploadsdir()
ut.ensuredir(uploads_path)
current_time = time.strftime('%Y_%m_%d_%H_%M_%S')
modifier = 1
upload_filename = 'upload_%s.png' % (current_time)
while exists(upload_filename):
upload_filename = 'upload_%s_%04d.png' % (current_time, modifier)
modifier += 1
upload_filepath = join(uploads_path, upload_filename)
filestore.save(upload_filepath)
gid_list = ibs.add_images([upload_filepath], **kwargs)
gid = gid_list[0]
if cleanup:
ut.remove_dirs(upload_filepath)
return gid
def get_flann_fpath(dpts, cache_dir='default', cfgstr='', flann_params={},
use_params_hash=True, use_data_hash=True, appname='vtool',
verbose=True):
""" returns filepath for flann index """
if cache_dir == 'default':
if verbose:
print('[flann] using default cache dir')
cache_dir = utool.get_app_resource_dir(appname)
utool.ensuredir(cache_dir)
flann_cfgstr = get_flann_cfgstr(dpts, flann_params, cfgstr,
use_params_hash=use_params_hash,
use_data_hash=use_data_hash)
if verbose:
print('...flann_cache cfgstr = %r: ' % flann_cfgstr)
# Append any user labels
flann_fname = 'flann_index' + flann_cfgstr + '.flann'
flann_fpath = normpath(join(cache_dir, flann_fname))
return flann_fpath
def testdata_expts(defaultdb='testdb1',
default_acfgstr_name_list=['default:qindex=0:10:4,dindex=0:20'],
default_test_cfg_name_list=['default'],
a=None,
t=None,
qaid_override=None,
daid_override=None,
initial_aids=None,
):
"""
Use this if you want data from an experiment.
Command line interface to quickly get testdata for test_results.
Command line flags can be used to specify db, aidcfg, pipecfg, qaid
override, daid override (and maybe initial aids).
"""
print('[main_helpers] testdata_expts')
import ibeis
from ibeis.expt import experiment_harness
from ibeis.expt import test_result
if a is not None:
default_acfgstr_name_list = a
if t is not None:
default_test_cfg_name_list = t
if isinstance(default_acfgstr_name_list, six.string_types):
default_acfgstr_name_list = [default_acfgstr_name_list]
if isinstance(default_test_cfg_name_list, six.string_types):
default_test_cfg_name_list = [default_test_cfg_name_list]
#from ibeis.expt import experiment_helpers
ibs = ibeis.opendb(defaultdb=defaultdb)
acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list,
default=default_acfgstr_name_list)
test_cfg_name_list = ut.get_argval(('-t', '-p'), type_=list, default=default_test_cfg_name_list)
daid_override = ut.get_argval(('--daid-override', '--daids-override'), type_=list, default=daid_override)
qaid_override = ut.get_argval(('--qaid', '--qaids-override', '--qaid-override'), type_=list, default=qaid_override)
# Hack a cache here
use_bigtest_cache3 = not ut.get_argflag(('--nocache', '--nocache-hs'))
use_bigtest_cache3 &= ut.is_developer()
use_bigtest_cache3 &= False
if use_bigtest_cache3:
from os.path import dirname, join
cache_dir = ut.ensuredir(join(dirname(ut.get_module_dir(ibeis)), 'BIG_TESTLIST_CACHE3'))
load_testres = ut.cached_func('testreslist', cache_dir=cache_dir)(experiment_harness.run_test_configurations2)
else:
load_testres = experiment_harness.run_test_configurations2
testres_list = load_testres(
ibs, acfg_name_list, test_cfg_name_list, qaid_override=qaid_override,
daid_override=daid_override, initial_aids=initial_aids)
testres = test_result.combine_testres_list(ibs, testres_list)
print(testres)
return ibs, testres
def copy_ibeisdb(source_dbdir, dest_dbdir):
# TODO; rectify with rsycn script
from os.path import normpath
import ibeis
exclude_dirs = [ut.ensure_unixslash(normpath(rel)) for rel in ibeis.const.EXCLUDE_COPY_REL_DIRS + ['_hsdb', '.hs_internals']]
rel_tocopy = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs, recursive=True, with_files=True, with_dirs=False, fullpath=False)
rel_tocopy_dirs = ut.glob(source_dbdir, '*', exclude_dirs=exclude_dirs, recursive=True, with_files=False, with_dirs=True, fullpath=False)
src_list = [join(source_dbdir, relpath) for relpath in rel_tocopy]
dst_list = [join(dest_dbdir, relpath) for relpath in rel_tocopy]
# ensure directories exist
rel_tocopy_dirs = [dest_dbdir] + [join(dest_dbdir, dpath_) for dpath_ in rel_tocopy_dirs]
for dpath in rel_tocopy_dirs:
ut.ensuredir(dpath)
# copy files
ut.copy(src_list, dst_list)
