def testdata_pipecfg(p=None, t=None):
r"""
Returns:
dict: pcfgdict
CommandLine:
python -m ibeis.init.main_helpers --exec-testdata_pipecfg
python -m ibeis.init.main_helpers --exec-testdata_pipecfg -t default:AI=False
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> pcfgdict = testdata_pipecfg()
>>> result = ('pcfgdict = %s' % (ut.dict_str(pcfgdict),))
>>> print(result)
"""
print('[main_helpers] testdata_pipecfg')
if t is not None and p is None:
p = t
if p is None:
p = ['default']
from ibeis.expt import experiment_helpers
test_cfg_name_list = ut.get_argval(('-t', '-p'), type_=list, default=p)
pcfgdict_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list)[0]
assert len(pcfgdict_list) == 1, 'can only specify one pipeline config here'
pcfgdict = pcfgdict_list[0]
return pcfgdict
def testdata_pipecfg(p=None, t=None):
r"""
Returns:
dict: pcfgdict
CommandLine:
python -m ibeis.init.main_helpers --exec-testdata_pipecfg
python -m ibeis.init.main_helpers --exec-testdata_pipecfg -t default:AI=False
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> pcfgdict = testdata_pipecfg()
>>> result = ('pcfgdict = %s' % (ut.dict_str(pcfgdict),))
>>> print(result)
"""
print('[main_helpers] testdata_pipecfg')
if t is not None and p is None:
p = t
if p is None:
p = ['default']
from ibeis.expt import experiment_helpers
test_cfg_name_list = ut.get_argval(('-t', '-p'), type_=list, default=p)
pcfgdict_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list)[0]
assert len(pcfgdict_list) == 1, 'can only specify one pipeline config here'
pcfgdict = pcfgdict_list[0]
return pcfgdict
def intersect_hack():
failed = testres.rank_mat > 0
colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
#failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
#failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
gt_aids1 = ibs.get_annot_groundtruth(
failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
gt_aids2 = ibs.get_annot_groundtruth(
failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)
qaids_expt = failed_col1_only_aids
gt_avl_aids1 = ut.flatten(gt_aids1)
gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
#jsontext = ut.to_json({
# 'qaids': list(qaids_expt),
# 'dinclude_aids1': list(gt_aids_expt1),
# 'dinclude_aids2': list(gt_aids_expt2),
#})
#annotation_configs.varysize_pzm
#from ibeis.expt import annotation_configs
acfg = testres.acfg_list[0]
import copy
acfg1 = copy.deepcopy(acfg)
acfg2 = copy.deepcopy(acfg)
acfg1['qcfg']['min_pername'] = None
acfg2['qcfg']['min_pername'] = None
acfg1['dcfg']['min_pername'] = None
acfg2['dcfg']['min_gt_per_name'] = None
acfg1['qcfg']['default_aids'] = qaids_expt
acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
acfg2['qcfg']['default_aids'] = qaids_expt
acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2
from ibeis.init import filter_annots
from ibeis.expt import experiment_helpers
annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)
acfg_name_list = dict( # NOQA
acfg_list=[acfg1, acfg2],
expanded_aids_list=[annots1, annots2],
)
test_cfg_name_list = ['candidacy_k']
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(
test_cfg_name_list, ibs=ibs)
t1, t2 = testres_list # NOQA
def intersect_hack():
failed = testres.rank_mat > 0
colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
#failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
#failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
gt_aids1 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
gt_aids2 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)
qaids_expt = failed_col1_only_aids
gt_avl_aids1 = ut.flatten(gt_aids1)
gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
#jsontext = ut.to_json({
# 'qaids': list(qaids_expt),
# 'dinclude_aids1': list(gt_aids_expt1),
# 'dinclude_aids2': list(gt_aids_expt2),
#})
#annotation_configs.varysize_pzm
#from ibeis.expt import annotation_configs
acfg = testres.acfg_list[0]
import copy
acfg1 = copy.deepcopy(acfg)
acfg2 = copy.deepcopy(acfg)
acfg1['qcfg']['min_pername'] = None
acfg2['qcfg']['min_pername'] = None
acfg1['dcfg']['min_pername'] = None
acfg2['dcfg']['min_gt_per_name'] = None
acfg1['qcfg']['default_aids'] = qaids_expt
acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
acfg2['qcfg']['default_aids'] = qaids_expt
acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2
from ibeis.init import filter_annots
from ibeis.expt import experiment_helpers
annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)
acfg_name_list = dict( # NOQA
acfg_list=[acfg1, acfg2],
expanded_aids_list=[annots1, annots2],
)
test_cfg_name_list = ['candidacy_k']
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs)
t1, t2 = testres_list # NOQA
def testdata_pipecfg(p=None, t=None, ibs=None, verbose=None):
r"""
Returns:
dict: pcfgdict
CommandLine:
python -m ibeis testdata_pipecfg
python -m ibeis testdata_pipecfg -t default:AI=False
Ignore:
from jedi.evaluate import docstrings
script = jedi.Script(ut.readfrom(main_helpers.__file__))
mod = script._get_module()
func = mod.names_dict['testdata_pipecfg'][0].parent
docstrings.find_return_types(script._evaluator, func)
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> pcfgdict = testdata_pipecfg()
>>> result = ('pcfgdict = %s' % (ut.repr2(pcfgdict),))
>>> print(result)
"""
if verbose is None or verbose >= 1:
print('[main_helpers] testdata_pipecfg')
if t is not None and p is None:
p = t
print('WARNING DO NOT USE t. Use p instead')
if p is None:
p = ['default']
from ibeis.expt import experiment_helpers
test_cfg_name_list, _spec = ut.get_argval(('-t', '-p'),
type_=list,
default=p,
return_was_specified=True)
if not _spec and isinstance(p, dict):
# allow explict default spec
return p
pcfgdict_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list,
ibs=ibs)[0]
assert len(pcfgdict_list) == 1, 'can only specify one pipeline config here'
pcfgdict = pcfgdict_list[0]
return pcfgdict
def run_expt(ibs,
acfg_name_list,
test_cfg_name_list,
use_cache=None,
qaid_override=None,
daid_override=None,
initial_aids=None):
r"""
Loops over annot configs.
Try and use this function as a starting point to clean up this module.
The code is getting too untenable.
CommandLine:
python -m ibeis.expt.harness run_expt --acfginfo
python -m ibeis.expt.harness run_expt --pcfginfo
python -m ibeis.expt.harness run_expt
Ignore:
test_cfg_name_list = [p]
Example:
>>> # SLOW_DOCTEST
>>> from ibeis.expt.harness import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> default_acfgstrs = ['ctrl:qsize=20,dpername=1,dsize=10',
>>> 'ctrl:qsize=20,dpername=10,dsize=20']
>>> acfg_name_list = default_acfgstrs
>>> test_cfg_name_list = ['default:proot=smk', 'default']
>>> #test_cfg_name_list = ['custom', 'custom:fg_on=False']
>>> use_cache = False
>>> testres_list = run_expt(ibs, acfg_name_list, test_cfg_name_list, use_cache)
"""
print('[harn] run_expt')
# Generate list of database annotation configurations
if len(acfg_name_list) == 0:
raise ValueError('must give acfg name list')
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
ibs,
acfg_name_list,
qaid_override=qaid_override,
daid_override=daid_override,
initial_aids=initial_aids,
use_cache=use_cache)
# Generate list of query pipeline param configs
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(
test_cfg_name_list, ibs=ibs)
cfgx2_lbl = experiment_helpers.get_varied_pipecfg_lbls(cfgdict_list)
# NOTE: Can specify --pcfginfo or --acfginfo
if ut.NOT_QUIET:
ut.colorprint(
textwrap.dedent("""
[harn]================
[harn] harness.test_configurations2()""").strip(), 'white')
msg = '[harn] Running %s using %s and %s' % (
ut.quantstr('test',
len(acfg_list) * len(cfgdict_list)),
ut.quantstr('pipeline config', len(cfgdict_list)),
ut.quantstr('annot config', len(acfg_list)),
)
ut.colorprint(msg, 'white')
testres_list = []
nAcfg = len(acfg_list)
testnameid = (ibs.get_dbname() + ' ' + str(test_cfg_name_list) +
str(acfg_name_list))
lbl = '[harn] TEST_CFG ' + str(test_cfg_name_list) + str(acfg_name_list)
expanded_aids_iter = ut.ProgIter(expanded_aids_list,
lbl='annot config',
freq=1,
autoadjust=False,
enabled=ut.NOT_QUIET)
for acfgx, (qaids, daids) in enumerate(expanded_aids_iter):
assert len(qaids) != 0, ('[harness] No query annots specified')
assert len(daids) != 0, ('[harness] No database annotas specified')
acfg = acfg_list[acfgx]
if ut.NOT_QUIET:
ut.colorprint('\n---Annot config testnameid=%r' % (testnameid, ),
'turquoise')
subindexer_partial = ut.ProgPartial(parent_index=acfgx,
parent_length=nAcfg,
enabled=ut.NOT_QUIET)
testres_ = make_single_testres(ibs,
qaids,
daids,
pipecfg_list,
cfgx2_lbl,
cfgdict_list,
lbl,
testnameid,
use_cache=use_cache,
subindexer_partial=subindexer_partial)
if DRY_RUN:
continue
testres_.acfg = acfg
testres_.test_cfg_name_list = test_cfg_name_list
testres_list.append(testres_)
if DRY_RUN:
print('DRYRUN: Cannot continue past run_expt')
sys.exit(0)
testres = test_result.combine_testres_list(ibs, testres_list)
# testres.print_results()
print('Returning Test Result')
return testres
def run_test_configurations2(ibs, acfg_name_list, test_cfg_name_list,
use_cache=None, qaid_override=None,
daid_override=None, initial_aids=None):
"""
Loops over annot configs.
Try and use this function as a starting point to clean up this module.
The code is getting too untenable.
CommandLine:
python -m ibeis.expt.harness --exec-run_test_configurations2
Example:
>>> # SLOW_DOCTEST
>>> from ibeis.expt.harness import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> default_acfgstrs = ['controlled:qsize=20,dpername=1,dsize=10', 'controlled:qsize=20,dpername=10,dsize=100']
>>> acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list, default=default_acfgstrs)
>>> test_cfg_name_list = ut.get_argval(('-t', '-p')), type_=list, default=['custom', 'custom:fg_on=False'])
>>> use_cache = False
>>> testres_list = run_test_configurations2(ibs, acfg_name_list, test_cfg_name_list, use_cache)
"""
print('[harn] run_test_configurations2')
# Generate list of database annotation configurations
if len(acfg_name_list) == 0:
raise ValueError('must give acfg name list')
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
ibs, acfg_name_list, qaid_override=qaid_override,
daid_override=daid_override, initial_aids=initial_aids,
use_cache=use_cache)
# Generate list of query pipeline param configs
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(
test_cfg_name_list, ibs=ibs)
cfgx2_lbl = experiment_helpers.get_varied_pipecfg_lbls(cfgdict_list)
# NOTE: Can specify --pcfginfo or --acfginfo
if ut.NOT_QUIET:
ut.colorprint(textwrap.dedent("""
[harn]================
[harn] harness.test_configurations2()""").strip(), 'white')
msg = '[harn] Running %s using %s and %s' % (
ut.quantstr('test', len(acfg_list) * len(pipecfg_list)),
ut.quantstr('pipeline config', len(pipecfg_list)),
ut.quantstr('annot config', len(acfg_list)),
)
ut.colorprint(msg, 'white')
testres_list = []
nAcfg = len(acfg_list)
testnameid = ibs.get_dbname() + ' ' + str(test_cfg_name_list) + str(acfg_name_list)
lbl = '[harn] TEST_CFG ' + str(test_cfg_name_list) + str(acfg_name_list)
expanded_aids_iter = ut.ProgressIter(expanded_aids_list,
lbl='annot config',
freq=1, autoadjust=False,
enabled=ut.NOT_QUIET)
for acfgx, (qaids, daids) in enumerate(expanded_aids_iter):
assert len(qaids) != 0, (
'[harness] No query annotations specified')
assert len(daids) != 0, (
'[harness] No database annotations specified')
acfg = acfg_list[acfgx]
if ut.NOT_QUIET:
ut.colorprint('\n---Annot config testnameid=%r' % (
testnameid,), 'turquoise')
subindexer_partial = partial(ut.ProgressIter, parent_index=acfgx,
parent_nTotal=nAcfg, enabled=ut.NOT_QUIET)
testres = make_single_testres(ibs, qaids, daids, pipecfg_list,
cfgx2_lbl, cfgdict_list, lbl, testnameid,
use_cache=use_cache,
subindexer_partial=subindexer_partial)
if DRY_RUN:
continue
testres.acfg = acfg
testres.test_cfg_name_list = test_cfg_name_list
testres_list.append(testres)
if DRY_RUN:
print('DRYRUN: Cannot continue past run_test_configurations2')
sys.exit(0)
return testres_list
def precfg(ibs, acfg_name_list, test_cfg_name_list):
r"""
Helper to precompute information
Args:
ibs (IBEISController): ibeis controller object
qaids (list): query annotation ids
daids (list): database annotation ids
test_cfg_name_list (list):
CommandLine:
python -m ibeis.expt.precomputer --exec-precfg -t custom --expt-preload
python -m ibeis.expt.precomputer --exec-precfg -t candidacy -a default:qaids=allgt --preload
python -m ibeis.expt.precomputer --exec-precfg -t candidacy_invariance -a default:qaids=allgt --preload
python -m ibeis.expt.precomputer --exec-precfg --delete-nn-cache
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.expt.precomputer import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> default_acfgstrs = ['default:qaids=allgt']
>>> acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list, default=default_acfgstrs)
>>> test_cfg_name_list = ut.get_argval('-t', type_=list, default=['custom'])
>>> result = precfg(ibs, acfg_name_list, test_cfg_name_list)
>>> print(result)
"""
# Generate list of database annotation configurations
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(ibs, acfg_name_list)
# Generate list of query pipeline param configs
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs)
#cfgx2_lbl = experiment_helpers.get_varied_cfg_lbls(cfgdict_list)
expanded_aids_iter = ut.ProgressIter(expanded_aids_list, lbl='annot config', freq=1, autoadjust=False)
nAcfg = len(acfg_list)
for acfgx, (qaids, daids) in enumerate(expanded_aids_iter):
if len(qaids) == 0:
print('[harness] WARNING No query annotations specified')
continue
if len(daids) == 0:
print('[harness] WARNING No database annotations specified')
continue
ut.colorprint('\n---Annot config', 'turquoise')
nCfg = len(pipecfg_list) # number of configurations (cols)
dbname = ibs.get_dbname()
cfgiter = ut.ProgressIter(pipecfg_list, lbl='query config', freq=1, autoadjust=False, parent_index=acfgx, parent_nTotal=nAcfg)
flag = False
if ut.get_argflag('--delete-nn-cache'):
ibs.delete_neighbor_cache()
flag = True
for cfgx, query_cfg in enumerate(cfgiter):
print('')
ut.colorprint(query_cfg.get_cfgstr(), 'turquoise')
verbose = True
with ut.Indenter('[%s cfg %d/%d]' % (dbname, (acfgx * nCfg) + cfgx * + 1, nCfg * nAcfg)):
qreq_ = ibs.new_query_request(qaids, daids, verbose=True, query_cfg=query_cfg)
if ut.get_argflag('--preload'):
qreq_.lazy_preload(verbose=verbose)
flag = True
if ut.get_argflag('--preload-chip'):
qreq_.ensure_chips(verbose=verbose, extra_tries=1)
flag = True
if ut.get_argflag('--preload-feat'):
qreq_.ensure_features(verbose=verbose)
flag = True
if ut.get_argflag('--preload-featweight'):
qreq_.ensure_featweights(verbose=verbose)
flag = True
if ut.get_argflag('--preindex'):
flag = True
if qreq_.qparams.pipeline_root in ['vsone', 'vsmany']:
qreq_.load_indexer(verbose=verbose)
assert flag is True, 'no flag specified'
assert flag is True, 'no flag specified'