def execute_and_save(qreq_miss):
# Iterate over vsone queries in chunks.
total_chunks = ut.get_num_chunks(len(qreq_miss.qaids), qreq_miss.chunksize)
qaid_chunk_iter = ut.ichunks(qreq_miss.qaids, qreq_miss.chunksize)
_prog = ut.ProgPartial(
length=total_chunks,
freq=1,
label='[mc5] query chunk: ',
prog_hook=qreq_miss.prog_hook,
bs=False,
)
qaid_chunk_iter = iter(_prog(qaid_chunk_iter))
qaid_to_cm = {}
for qaids in qaid_chunk_iter:
sub_qreq = qreq_miss.shallowcopy(qaids=qaids)
cm_batch = sub_qreq.execute_pipeline()
assert len(cm_batch) == len(qaids), 'bad alignment'
assert all([qaid == cm.qaid for qaid, cm in zip(qaids, cm_batch)])
# TODO: we already computed the fpaths
# should be able to pass them in
fpath_list = sub_qreq.get_chipmatch_fpaths(qaids)
_prog = ut.ProgPartial(
length=len(cm_batch),
adjust=True,
freq=1,
label='saving chip matches',
bs=True,
)
for cm, fpath in _prog(zip(cm_batch, fpath_list)):
cm.save_to_fpath(fpath, verbose=False)
qaid_to_cm.update({cm.qaid: cm for cm in cm_batch})
return qaid_to_cm
def compute_gammas(inva, alpha, thresh):
"""
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.smk.inverted_index import * # NOQA
>>> qreq_, inva = testdata_inva()
>>> inva.wx_to_weight = inva.compute_word_weights('uniform')
>>> alpha = 3.0
>>> thresh = 0.0
>>> gamma_list = inva.compute_gammas(alpha, thresh)
"""
# TODO: sep
wx_to_weight = inva.wx_to_weight
_prog = ut.ProgPartial(length=len(inva.wx_lists),
bs=True,
lbl='gamma',
adjust=True)
_iter = zip(inva.wx_lists, inva.agg_rvecs, inva.agg_flags)
gamma_list = []
for wx_list, phiX_list, flagsX_list in _prog(_iter):
if inva.int_rvec:
phiX_list = smk_funcs.uncast_residual_integer(phiX_list)
weight_list = np.array(ut.take(wx_to_weight, wx_list))
gammaX = smk_funcs.gamma_agg(phiX_list, flagsX_list, weight_list,
alpha, thresh)
gamma_list.append(gammaX)
return gamma_list
def get_patches(inva, wx, ibs, verbose=True):
"""
Loads the patches assigned to a particular word in this stack
>>> inva.wx_to_aids = inva.compute_inverted_list()
>>> verbose=True
"""
config = inva.config
aid_list = inva.wx_to_aids[wx]
X_list = [inva.get_annot(aid) for aid in aid_list]
fxs_groups = [X.fxs(wx) for X in X_list]
all_kpts_list = ibs.depc.d.get_feat_kpts(aid_list, config=config)
sub_kpts_list = vt.ziptake(all_kpts_list, fxs_groups, axis=0)
total_patches = sum(ut.lmap(len, fxs_groups))
chip_list = ibs.depc_annot.d.get_chips_img(aid_list, config=config)
# convert to approprate colorspace
#if colorspace is not None:
# chip_list = vt.convert_image_list_colorspace(chip_list, colorspace)
# ut.print_object_size(chip_list, 'chip_list')
patch_size = 64
shape = (total_patches, patch_size, patch_size, 3)
_prog = ut.ProgPartial(enabled=verbose, lbl='warping patches', bs=True)
_patchiter = ut.iflatten([
vt.get_warped_patches(chip, kpts, patch_size=patch_size)[0]
#vt.get_warped_patches(chip, kpts, patch_size=patch_size, use_cpp=True)[0]
for chip, kpts in _prog(zip(chip_list, sub_kpts_list),
length=len(aid_list))
])
word_patches = vt.fromiter_nd(_patchiter, shape, dtype=np.uint8)
return word_patches
def predict_matches(smk, qreq_, verbose=True):
"""
>>> from wbia.algo.smk.smk_pipeline import * # NOQA
>>> ibs, smk, qreq_ = testdata_smk()
>>> verbose = True
"""
logger.info('Predicting matches')
# assert qreq_.qinva.vocab is qreq_.dinva.vocab
# X_list = qreq_.qinva.inverted_annots(qreq_.qaids)
# Y_list = qreq_.dinva.inverted_annots(qreq_.daids)
# verbose = 2
_prog = ut.ProgPartial(lbl='smk query',
bs=verbose <= 1,
enabled=verbose)
daids = np.array(qreq_.daids)
cm_list = [
smk.match_single(qaid, daids, qreq_, verbose=verbose > 1)
for qaid in _prog(qreq_.qaids)
]
return cm_list
def match_single(smk, qaid, daids, qreq_, verbose=True):
"""
CommandLine:
python -m wbia.algo.smk.smk_pipeline SMK.match_single --profile
python -m wbia.algo.smk.smk_pipeline SMK.match_single --show
python -m wbia SMK.match_single -a ctrl:qmingt=2 --profile --db PZ_Master1
python -m wbia SMK.match_single -a ctrl --profile --db GZ_ALL
Example:
>>> # FUTURE_ENABLE
>>> from wbia.algo.smk.smk_pipeline import * # NOQA
>>> import wbia
>>> qreq_ = wbia.testdata_qreq_(defaultdb='PZ_MTEST')
>>> ibs = qreq_.ibs
>>> daids = qreq_.daids
>>> #ibs, daids = wbia.testdata_aids(defaultdb='PZ_MTEST', default_set='dcfg')
>>> qreq_ = SMKRequest(ibs, daids[0:1], daids, {'agg': True,
>>> 'num_words': 1000,
>>> 'sv_on': True})
>>> qreq_.ensure_data()
>>> qaid = qreq_.qaids[0]
>>> daids = qreq_.daids
>>> daid = daids[1]
>>> verbose = True
>>> cm = qreq_.smk.match_single(qaid, daids, qreq_)
>>> ut.quit_if_noshow()
>>> ut.qtensure()
>>> cm.ishow_analysis(qreq_)
>>> ut.show_if_requested()
"""
from wbia.algo.hots import chip_match
from wbia.algo.hots import pipeline
alpha = qreq_.qparams['smk_alpha']
thresh = qreq_.qparams['smk_thresh']
agg = qreq_.qparams['agg']
# nAnnotPerName = qreq_.qparams.nAnnotPerNameSVER
sv_on = qreq_.qparams.sv_on
if sv_on:
nNameShortList = qreq_.qparams.nNameShortlistSVER
shortsize = nNameShortList
else:
shortsize = None
X = qreq_.qinva.get_annot(qaid)
# Determine which database annotations need to be checked
# with ut.Timer('searching qaid=%r' % (qaid,), verbose=verbose):
hit_inva_wxs = ut.take(qreq_.dinva.wx_to_aids, X.wx_list)
hit_daids = np.array(list(set(ut.iflatten(hit_inva_wxs))))
# Mark impossible daids
# with ut.Timer('checking impossible daids=%r' % (qaid,), verbose=verbose):
valid_flags = check_can_match(qaid, hit_daids, qreq_)
valid_daids = hit_daids.compress(valid_flags)
shortlist = ut.Shortlist(shortsize)
# gammaX = smk.gamma(X, wx_to_weight, agg, alpha, thresh)
_prog = ut.ProgPartial(lbl='smk scoring qaid=%r' % (qaid, ),
enabled=verbose,
bs=True,
adjust=True)
wx_to_weight = qreq_.dinva.wx_to_weight
debug = False
if debug:
qnid = qreq_.get_qreq_annot_nids([qaid])[0]
daids = np.array(qreq_.daids)
dnids = qreq_.get_qreq_annot_nids(daids)
correct_aids = daids[np.where(dnids == qnid)[0]]
daid = correct_aids[0]
if agg:
for daid in _prog(valid_daids):
Y = qreq_.dinva.get_annot(daid)
item = match_kernel_agg(X, Y, wx_to_weight, alpha, thresh)
shortlist.insert(item)
else:
for daid in _prog(valid_daids):
Y = qreq_.dinva.get_annot(daid)
item = match_kernel_sep(X, Y, wx_to_weight, alpha, thresh)
shortlist.insert(item)
# Build chipmatches for the shortlist results
# with ut.Timer('build cms', verbose=verbose):
cm = chip_match.ChipMatch(qaid=qaid, fsv_col_lbls=['smk'])
cm.daid_list = []
cm.fm_list = []
cm.fsv_list = []
_prog = ut.ProgPartial(lbl='smk build cm qaid=%r' % (qaid, ),
enabled=verbose,
bs=True,
adjust=True)
for item in _prog(shortlist):
(score, score_list, Y, X_idx, Y_idx) = item
X_fxs = ut.take(X.fxs_list, X_idx)
Y_fxs = ut.take(Y.fxs_list, Y_idx)
# Only build matches for those that sver will use
if agg:
X_maws = ut.take(X.maws_list, X_idx)
Y_maws = ut.take(Y.maws_list, Y_idx)
fm, fs = smk_funcs.build_matches_agg(X_fxs, Y_fxs, X_maws,
Y_maws, score_list)
else:
fm, fs = smk_funcs.build_matches_sep(X_fxs, Y_fxs, score_list)
if len(fm) > 0:
# assert not np.any(np.isnan(fs))
daid = Y.aid
fsv = fs[:, None]
cm.daid_list.append(daid)
cm.fm_list.append(fm)
cm.fsv_list.append(fsv)
cm._update_daid_index()
cm.arraycast_self()
cm.score_name_maxcsum(qreq_)
# if False:
# cm.assert_self(qreq_=qreq_, verbose=True)
if sv_on:
cm = pipeline.sver_single_chipmatch(qreq_, cm, verbose=verbose)
cm.score_name_maxcsum(qreq_)
return cm
def run_asmk_script():
with ut.embed_on_exception_context: # NOQA
"""
>>> from wbia.algo.smk.script_smk import *
"""
# NOQA
# ==============================================
# PREPROCESSING CONFIGURATION
# ==============================================
config = {
# 'data_year': 2013,
'data_year': None,
'dtype': 'float32',
# 'root_sift': True,
'root_sift': False,
# 'centering': True,
'centering': False,
'num_words': 2**16,
# 'num_words': 1E6
# 'num_words': 8000,
'kmeans_impl': 'sklearn.mini',
'extern_words': False,
'extern_assign': False,
'assign_algo': 'kdtree',
'checks': 1024,
'int_rvec': True,
'only_xy': False,
}
# Define which params are relevant for which operations
relevance = {}
relevance['feats'] = ['dtype', 'root_sift', 'centering', 'data_year']
relevance['words'] = relevance['feats'] + [
'num_words',
'extern_words',
'kmeans_impl',
]
relevance['assign'] = relevance['words'] + [
'checks',
'extern_assign',
'assign_algo',
]
# relevance['ydata'] = relevance['assign'] + ['int_rvec']
# relevance['xdata'] = relevance['assign'] + ['only_xy', 'int_rvec']
nAssign = 1
class SMKCacher(ut.Cacher):
def __init__(self, fname, ext='.cPkl'):
relevant_params = relevance[fname]
relevant_cfg = ut.dict_subset(config, relevant_params)
cfgstr = ut.get_cfg_lbl(relevant_cfg)
dbdir = ut.truepath('/raid/work/Oxford/')
super(SMKCacher, self).__init__(fname,
cfgstr,
cache_dir=dbdir,
ext=ext)
# ==============================================
# LOAD DATASET, EXTRACT AND POSTPROCESS FEATURES
# ==============================================
if config['data_year'] == 2007:
data = load_oxford_2007()
elif config['data_year'] == 2013:
data = load_oxford_2013()
elif config['data_year'] is None:
data = load_oxford_wbia()
offset_list = data['offset_list']
all_kpts = data['all_kpts']
raw_vecs = data['all_vecs']
query_uri_order = data['query_uri_order']
data_uri_order = data['data_uri_order']
# del data
# ================
# PRE-PROCESS
# ================
import vtool as vt
# Alias names to avoid errors in interactive sessions
proc_vecs = raw_vecs
del raw_vecs
feats_cacher = SMKCacher('feats', ext='.npy')
all_vecs = feats_cacher.tryload()
if all_vecs is None:
if config['dtype'] == 'float32':
logger.info('Converting vecs to float32')
proc_vecs = proc_vecs.astype(np.float32)
else:
proc_vecs = proc_vecs
raise NotImplementedError('other dtype')
if config['root_sift']:
with ut.Timer('Apply root sift'):
np.sqrt(proc_vecs, out=proc_vecs)
vt.normalize(proc_vecs, ord=2, axis=1, out=proc_vecs)
if config['centering']:
with ut.Timer('Apply centering'):
mean_vec = np.mean(proc_vecs, axis=0)
# Center and then re-normalize
np.subtract(proc_vecs, mean_vec[None, :], out=proc_vecs)
vt.normalize(proc_vecs, ord=2, axis=1, out=proc_vecs)
if config['dtype'] == 'int8':
smk_funcs
all_vecs = proc_vecs
feats_cacher.save(all_vecs)
del proc_vecs
# =====================================
# BUILD VISUAL VOCABULARY
# =====================================
if config['extern_words']:
words = data['words']
assert config['num_words'] is None or len(
words) == config['num_words']
else:
word_cacher = SMKCacher('words')
words = word_cacher.tryload()
if words is None:
with ut.embed_on_exception_context:
if config['kmeans_impl'] == 'sklearn.mini':
import sklearn.cluster
rng = np.random.RandomState(13421421)
# init_size = int(config['num_words'] * 8)
init_size = int(config['num_words'] * 4)
# converged after 26043 iterations
clusterer = sklearn.cluster.MiniBatchKMeans(
config['num_words'],
init_size=init_size,
batch_size=1000,
compute_labels=False,
max_iter=20,
random_state=rng,
n_init=1,
verbose=1,
)
clusterer.fit(all_vecs)
words = clusterer.cluster_centers_
elif config['kmeans_impl'] == 'yael':
from yael import ynumpy
centroids, qerr, dis, assign, nassign = ynumpy.kmeans(
all_vecs,
config['num_words'],
init='kmeans++',
verbose=True,
output='all',
)
words = centroids
word_cacher.save(words)
# =====================================
# ASSIGN EACH VECTOR TO ITS NEAREST WORD
# =====================================
if config['extern_assign']:
assert config[
'extern_words'], 'need extern cluster to extern assign'
idx_to_wxs = vt.atleast_nd(data['idx_to_wx'], 2)
idx_to_maws = np.ones(idx_to_wxs.shape, dtype=np.float32)
idx_to_wxs = np.ma.array(idx_to_wxs)
idx_to_maws = np.ma.array(idx_to_maws)
else:
from wbia.algo.smk import vocab_indexer
vocab = vocab_indexer.VisualVocab(words)
dassign_cacher = SMKCacher('assign')
assign_tup = dassign_cacher.tryload()
if assign_tup is None:
vocab.flann_params['algorithm'] = config['assign_algo']
vocab.build()
# Takes 12 minutes to assign jegous vecs to 2**16 vocab
with ut.Timer('assign vocab neighbors'):
_idx_to_wx, _idx_to_wdist = vocab.nn_index(
all_vecs, nAssign, checks=config['checks'])
if nAssign > 1:
idx_to_wxs, idx_to_maws = smk_funcs.weight_multi_assigns(
_idx_to_wx,
_idx_to_wdist,
massign_alpha=1.2,
massign_sigma=80.0,
massign_equal_weights=True,
)
else:
idx_to_wxs = np.ma.masked_array(_idx_to_wx,
fill_value=-1)
idx_to_maws = np.ma.ones(idx_to_wxs.shape,
fill_value=-1,
dtype=np.float32)
idx_to_maws.mask = idx_to_wxs.mask
assign_tup = (idx_to_wxs, idx_to_maws)
dassign_cacher.save(assign_tup)
idx_to_wxs, idx_to_maws = assign_tup
# Breakup vectors, keypoints, and word assignments by annotation
wx_lists = [
idx_to_wxs[left:right] for left, right in ut.itertwo(offset_list)
]
maw_lists = [
idx_to_maws[left:right] for left, right in ut.itertwo(offset_list)
]
vecs_list = [
all_vecs[left:right] for left, right in ut.itertwo(offset_list)
]
kpts_list = [
all_kpts[left:right] for left, right in ut.itertwo(offset_list)
]
# =======================
# FIND QUERY SUBREGIONS
# =======================
ibs, query_annots, data_annots, qx_to_dx = load_ordered_annots(
data_uri_order, query_uri_order)
daids = data_annots.aids
qaids = query_annots.aids
query_super_kpts = ut.take(kpts_list, qx_to_dx)
query_super_vecs = ut.take(vecs_list, qx_to_dx)
query_super_wxs = ut.take(wx_lists, qx_to_dx)
query_super_maws = ut.take(maw_lists, qx_to_dx)
# Mark which keypoints are within the bbox of the query
query_flags_list = []
only_xy = config['only_xy']
for kpts_, bbox in zip(query_super_kpts, query_annots.bboxes):
flags = kpts_inside_bbox(kpts_, bbox, only_xy=only_xy)
query_flags_list.append(flags)
logger.info('Queries are crops of existing database images.')
logger.info('Looking at average percents')
percent_list = [
flags_.sum() / flags_.shape[0] for flags_ in query_flags_list
]
percent_stats = ut.get_stats(percent_list)
logger.info('percent_stats = %s' % (ut.repr4(percent_stats), ))
import vtool as vt
query_kpts = vt.zipcompress(query_super_kpts, query_flags_list, axis=0)
query_vecs = vt.zipcompress(query_super_vecs, query_flags_list, axis=0)
query_wxs = vt.zipcompress(query_super_wxs, query_flags_list, axis=0)
query_maws = vt.zipcompress(query_super_maws, query_flags_list, axis=0)
# =======================
# CONSTRUCT QUERY / DATABASE REPR
# =======================
# int_rvec = not config['dtype'].startswith('float')
int_rvec = config['int_rvec']
X_list = []
_prog = ut.ProgPartial(length=len(qaids),
label='new X',
bs=True,
adjust=True)
for aid, fx_to_wxs, fx_to_maws in _prog(
zip(qaids, query_wxs, query_maws)):
X = new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec)
X_list.append(X)
# ydata_cacher = SMKCacher('ydata')
# Y_list = ydata_cacher.tryload()
# if Y_list is None:
Y_list = []
_prog = ut.ProgPartial(length=len(daids),
label='new Y',
bs=True,
adjust=True)
for aid, fx_to_wxs, fx_to_maws in _prog(zip(daids, wx_lists,
maw_lists)):
Y = new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec)
Y_list.append(Y)
# ydata_cacher.save(Y_list)
# ======================
# Add in some groundtruth
logger.info('Add in some groundtruth')
for Y, nid in zip(Y_list, ibs.get_annot_nids(daids)):
Y.nid = nid
for X, nid in zip(X_list, ibs.get_annot_nids(qaids)):
X.nid = nid
for Y, qual in zip(Y_list, ibs.get_annot_quality_texts(daids)):
Y.qual = qual
# ======================
# Add in other properties
for Y, vecs, kpts in zip(Y_list, vecs_list, kpts_list):
Y.vecs = vecs
Y.kpts = kpts
imgdir = ut.truepath('/raid/work/Oxford/oxbuild_images')
for Y, imgid in zip(Y_list, data_uri_order):
gpath = ut.unixjoin(imgdir, imgid + '.jpg')
Y.gpath = gpath
for X, vecs, kpts in zip(X_list, query_vecs, query_kpts):
X.kpts = kpts
X.vecs = vecs
# ======================
logger.info('Building inverted list')
daids = [Y.aid for Y in Y_list]
# wx_list = sorted(ut.list_union(*[Y.wx_list for Y in Y_list]))
wx_list = sorted(set.union(*[Y.wx_set for Y in Y_list]))
assert daids == data_annots.aids
assert len(wx_list) <= config['num_words']
wx_to_aids = smk_funcs.invert_lists(daids, [Y.wx_list for Y in Y_list],
all_wxs=wx_list)
# Compute IDF weights
logger.info('Compute IDF weights')
ndocs_total = len(daids)
# Use only the unique number of words
ndocs_per_word = np.array([len(set(wx_to_aids[wx])) for wx in wx_list])
logger.info('ndocs_perword stats: ' +
ut.repr4(ut.get_stats(ndocs_per_word)))
idf_per_word = smk_funcs.inv_doc_freq(ndocs_total, ndocs_per_word)
wx_to_weight = dict(zip(wx_list, idf_per_word))
logger.info('idf stats: ' +
ut.repr4(ut.get_stats(wx_to_weight.values())))
# Filter junk
Y_list_ = [Y for Y in Y_list if Y.qual != 'junk']
# =======================
# CHOOSE QUERY KERNEL
# =======================
params = {
'asmk': dict(alpha=3.0, thresh=0.0),
'bow': dict(),
'bow2': dict(),
}
# method = 'bow'
method = 'bow2'
method = 'asmk'
smk = SMK(wx_to_weight, method=method, **params[method])
# Specific info for the type of query
if method == 'asmk':
# Make residual vectors
if True:
# The stacked way is 50x faster
# TODO: extend for multi-assignment and record fxs
flat_query_vecs = np.vstack(query_vecs)
flat_query_wxs = np.vstack(query_wxs)
flat_query_offsets = np.array(
[0] + ut.cumsum(ut.lmap(len, query_wxs)))
flat_wxs_assign = flat_query_wxs
flat_offsets = flat_query_offsets
flat_vecs = flat_query_vecs
tup = smk_funcs.compute_stacked_agg_rvecs(
words, flat_wxs_assign, flat_vecs, flat_offsets)
all_agg_vecs, all_error_flags, agg_offset_list = tup
if int_rvec:
all_agg_vecs = smk_funcs.cast_residual_integer(
all_agg_vecs)
agg_rvecs_list = [
all_agg_vecs[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
agg_flags_list = [
all_error_flags[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
for X, agg_rvecs, agg_flags in zip(X_list, agg_rvecs_list,
agg_flags_list):
X.agg_rvecs = agg_rvecs
X.agg_flags = agg_flags[:, None]
flat_wxs_assign = idx_to_wxs
flat_offsets = offset_list
flat_vecs = all_vecs
tup = smk_funcs.compute_stacked_agg_rvecs(
words, flat_wxs_assign, flat_vecs, flat_offsets)
all_agg_vecs, all_error_flags, agg_offset_list = tup
if int_rvec:
all_agg_vecs = smk_funcs.cast_residual_integer(
all_agg_vecs)
agg_rvecs_list = [
all_agg_vecs[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
agg_flags_list = [
all_error_flags[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
for Y, agg_rvecs, agg_flags in zip(Y_list, agg_rvecs_list,
agg_flags_list):
Y.agg_rvecs = agg_rvecs
Y.agg_flags = agg_flags[:, None]
else:
# This non-stacked way is about 500x slower
_prog = ut.ProgPartial(label='agg Y rvecs',
bs=True,
adjust=True)
for Y in _prog(Y_list_):
make_agg_vecs(Y, words, Y.vecs)
_prog = ut.ProgPartial(label='agg X rvecs',
bs=True,
adjust=True)
for X in _prog(X_list):
make_agg_vecs(X, words, X.vecs)
elif method == 'bow2':
# Hack for orig tf-idf bow vector
nwords = len(words)
for X in ut.ProgIter(X_list, label='make bow vector'):
ensure_tf(X)
bow_vector(X, wx_to_weight, nwords)
for Y in ut.ProgIter(Y_list_, label='make bow vector'):
ensure_tf(Y)
bow_vector(Y, wx_to_weight, nwords)
if method != 'bow2':
for X in ut.ProgIter(X_list, 'compute X gamma'):
X.gamma = smk.gamma(X)
for Y in ut.ProgIter(Y_list_, 'compute Y gamma'):
Y.gamma = smk.gamma(Y)
# Execute matches (could go faster by enumerating candidates)
scores_list = []
for X in ut.ProgIter(X_list, label='query %s' % (smk, )):
scores = [smk.kernel(X, Y) for Y in Y_list_]
scores = np.array(scores)
scores = np.nan_to_num(scores)
scores_list.append(scores)
import sklearn.metrics
avep_list = []
_iter = list(zip(scores_list, X_list))
_iter = ut.ProgIter(_iter, label='evaluate %s' % (smk, ))
for scores, X in _iter:
truth = [X.nid == Y.nid for Y in Y_list_]
avep = sklearn.metrics.average_precision_score(truth, scores)
avep_list.append(avep)
avep_list = np.array(avep_list)
mAP = np.mean(avep_list)
logger.info('mAP = %r' % (mAP, ))
def run_expt(
ibs,
acfg_name_list,
test_cfg_name_list,
use_cache=None,
qaid_override=None,
daid_override=None,
initial_aids=None,
):
r"""
Loops over annot configs.
Try and use this function as a starting point to clean up this module.
The code is getting too untenable.
CommandLine:
python -m wbia.expt.harness run_expt --acfginfo
python -m wbia.expt.harness run_expt --pcfginfo
python -m wbia.expt.harness run_expt
Ignore:
test_cfg_name_list = [p]
Example:
>>> # SLOW_DOCTEST
>>> from wbia.expt.harness import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='PZ_MTEST')
>>> default_acfgstrs = ['ctrl:qsize=20,dpername=1,dsize=10',
>>> 'ctrl:qsize=20,dpername=10,dsize=20']
>>> acfg_name_list = default_acfgstrs
>>> test_cfg_name_list = ['default:proot=smk', 'default']
>>> #test_cfg_name_list = ['custom', 'custom:fg_on=False']
>>> use_cache = False
>>> testres_list = run_expt(ibs, acfg_name_list, test_cfg_name_list, use_cache)
"""
logger.info('[harn] run_expt')
# Generate list of database annotation configurations
if len(acfg_name_list) == 0:
raise ValueError('must give acfg name list')
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
ibs,
acfg_name_list,
qaid_override=qaid_override,
daid_override=daid_override,
initial_aids=initial_aids,
use_cache=use_cache,
)
# Generate list of query pipeline param configs
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(
test_cfg_name_list, ibs=ibs)
cfgx2_lbl = experiment_helpers.get_varied_pipecfg_lbls(cfgdict_list)
# NOTE: Can specify --pcfginfo or --acfginfo
if ut.NOT_QUIET:
ut.colorprint(
textwrap.dedent("""
[harn]================
[harn] harness.test_configurations2()""").strip(),
'white',
)
msg = '[harn] Running %s using %s and %s' % (
ut.quantstr('test',
len(acfg_list) * len(cfgdict_list)),
ut.quantstr('pipeline config', len(cfgdict_list)),
ut.quantstr('annot config', len(acfg_list)),
)
ut.colorprint(msg, 'white')
testres_list = []
nAcfg = len(acfg_list)
testnameid = ibs.get_dbname() + ' ' + str(test_cfg_name_list) + str(
acfg_name_list)
lbl = '[harn] TEST_CFG ' + str(test_cfg_name_list) + str(acfg_name_list)
expanded_aids_iter = ut.ProgIter(
expanded_aids_list,
lbl='annot config',
freq=1,
autoadjust=False,
enabled=ut.NOT_QUIET,
)
for acfgx, (qaids, daids) in enumerate(expanded_aids_iter):
assert len(qaids) != 0, '[harness] No query annots specified'
assert len(daids) != 0, '[harness] No database annotas specified'
acfg = acfg_list[acfgx]
if ut.NOT_QUIET:
ut.colorprint('\n---Annot config testnameid=%r' % (testnameid, ),
'brightcyan')
subindexer_partial = ut.ProgPartial(parent_index=acfgx,
parent_length=nAcfg,
enabled=ut.NOT_QUIET)
testres_ = make_single_testres(
ibs,
qaids,
daids,
pipecfg_list,
cfgx2_lbl,
cfgdict_list,
lbl,
testnameid,
use_cache=use_cache,
subindexer_partial=subindexer_partial,
)
if DRY_RUN:
continue
testres_.acfg = acfg
testres_.test_cfg_name_list = test_cfg_name_list
testres_list.append(testres_)
if DRY_RUN:
logger.info('DRYRUN: Cannot continue past run_expt')
sys.exit(0)
testres = test_result.combine_testres_list(ibs, testres_list)
# testres.print_results()
logger.info('Returning Test Result')
return testres
