def get_pipecfg_list(test_cfg_name_list, ibs=None):
r"""
Builds a list of varied query configurations. Only custom configs depend on
an ibs object. The order of the output is not gaurenteed to aggree with
input order.
Args:
test_cfg_name_list (list): list of strs
ibs (IBEISController): ibeis controller object (optional)
Returns:
tuple: (cfg_list, cfgx2_lbl) -
cfg_list (list): list of config objects
cfgx2_lbl (list): denotes which parameters are being varied.
If there is just one config then nothing is varied
CommandLine:
python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:0
python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:1 --db humpbacks
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> #test_cfg_name_list = ['best', 'custom', 'custom:sv_on=False']
>>> #test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> # execute function
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> # verify results
>>> assert pipecfg_list[0].sv_cfg.sv_on is True
>>> assert pipecfg_list[1].sv_cfg.sv_on is False
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls))
>>> print(result)
pipecfg_lbls = [
'default:',
'default:sv_on=False',
]
Example1:
>>> # DISABLE_DOCTEST
>>> import ibeis_flukematch.plugin
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='humpbacks')
>>> test_cfg_name_list = ['default:pipeline_root=BC_DTW,decision=average', 'default:K=[1,4]']
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls))
>>> print(result)
>>> print_pipe_configs(pcfgdict_list, pipecfg_list)
"""
if ut.VERBOSE:
print('[expt_help.get_pipecfg_list] building pipecfg_list using: %s' %
test_cfg_name_list)
if isinstance(test_cfg_name_list, six.string_types):
test_cfg_name_list = [test_cfg_name_list]
_standard_cfg_names = []
_pcfgdict_list = []
# HACK: Parse out custom configs first
for test_cfg_name in test_cfg_name_list:
if test_cfg_name.startswith('custom:') or test_cfg_name == 'custom':
print('[expthelpers] Parsing nonstandard custom config')
if test_cfg_name.startswith('custom:'):
# parse out modifications to custom
cfgstr_list = ':'.join(test_cfg_name.split(':')[1:]).split(',')
augcfgdict = ut.parse_cfgstr_list(cfgstr_list, smartcast=True)
else:
augcfgdict = {}
# Take the configuration from the ibeis object
pipe_cfg = ibs.cfg.query_cfg.deepcopy()
# Update with augmented params
pipe_cfg.update_query_cfg(**augcfgdict)
# Parse out a standard cfgdict
cfgdict = dict(pipe_cfg.parse_items())
cfgdict['_cfgname'] = 'custom'
cfgdict['_cfgstr'] = test_cfg_name
_pcfgdict_list.append(cfgdict)
else:
_standard_cfg_names.append(test_cfg_name)
# Handle stanndard configs next
if len(_standard_cfg_names) > 0:
# Get parsing information
#cfg_default_dict = dict(Config.QueryConfig().parse_items())
#valid_keys = list(cfg_default_dict.keys())
cfgstr_list = _standard_cfg_names
named_defaults_dict = ut.dict_subset(
experiment_configs.__dict__, experiment_configs.TEST_NAMES)
alias_keys = experiment_configs.ALIAS_KEYS
# Parse standard pipeline cfgstrings
metadata = {'ibs': ibs}
dict_comb_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list,
named_defaults_dict,
cfgtype=None,
alias_keys=alias_keys,
# Hack out valid keys for humpbacks
#valid_keys=valid_keys,
strict=False,
metadata=metadata
)
# Get varied params (there may be duplicates)
_pcfgdict_list.extend(ut.flatten(dict_comb_list))
# TODO: respsect different algorithm parameters
# like flukes
# Expand cfgdicts into PipelineConfig config objects
if ibs is None:
configclass_list = [Config.QueryConfig] * len(_pcfgdict_list)
else:
root_to_config = ibs.depc.configclass_dict
configclass_list = [
root_to_config.get(_cfgdict.get('pipeline_root', 'vsmany'), Config.QueryConfig)
for _cfgdict in _pcfgdict_list]
_pipecfg_list = [cls(**_cfgdict) for cls, _cfgdict in zip(configclass_list, _pcfgdict_list)]
# Enforce rule that removes duplicate configs
# by using feasiblity from ibeis.algo.Config
# TODO: Move this unique finding code to its own function
# and then move it up one function level so even the custom
# configs can be uniquified
_flag_list = ut.flag_unique_items(_pipecfg_list)
cfgdict_list = ut.compress(_pcfgdict_list, _flag_list)
pipecfg_list = ut.compress(_pipecfg_list, _flag_list)
if ut.NOT_QUIET:
print('[harn.help] return %d / %d unique pipeline configs from: %r' %
(len(cfgdict_list), len(_pcfgdict_list), test_cfg_name_list))
if ut.get_argflag(('--pcfginfo', '--pinfo', '--pipecfginfo')):
import sys
ut.colorprint('Requested PcfgInfo for tests... ', 'red')
print_pipe_configs(cfgdict_list, pipecfg_list)
ut.colorprint('Finished Reporting PcfgInfo. Exiting', 'red')
sys.exit(1)
return (cfgdict_list, pipecfg_list)
def testdata_aids(defaultdb=None, a=None, adefault='default', ibs=None,
return_acfg=False, verbose=None, default_aids=None):
r"""
Grabs default testdata for functions, but is command line overrideable
CommandLine:
python -m ibeis --tf testdata_aids --verbtd --db PZ_ViewPoints
python -m ibeis --tf testdata_aids --verbtd --db NNP_Master3 -a is_known=True,view_pername='#primary>0&#primary1>=1'
python -m ibeis --tf testdata_aids --verbtd --db PZ_Master1 -a default:is_known=True,view_pername='#primary>0&#primary1>=1'
python -m ibeis --tf testdata_aids --verbtd --db PZ_Master1 -a default:species=primary,minqual=ok --verbtd
python -m ibeis.other.dbinfo --test-latex_dbstats --dblist
CommandLine:
python -m ibeis.init.main_helpers --exec-testdata_aids --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> from ibeis.expt import annotation_configs
>>> import ibeis
>>> #ibs = ibeis.opendb(defaultdb='PZ_ViewPoints')
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> a = None
>>> adefault = 'default:is_known=True'
>>> aids, aidcfg = testdata_aids(ibs=ibs, a=a, adefault=adefault, return_acfg=True)
>>> print('\n RESULT:')
>>> annotation_configs.print_acfg(aidcfg, aids, ibs, per_name_vpedge=None)
"""
from ibeis.init import filter_annots
from ibeis.expt import annotation_configs
from ibeis.expt import cfghelpers
import ibeis
print('[main_helpers] testdata_aids')
if a is None:
a = adefault
a, _specified_a = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=str, default=a, return_was_specified=True)
return_ibs = False
if ibs is None:
return_ibs = True
if defaultdb is None:
defaultdb = 'testdb1'
ibs = ibeis.opendb(defaultdb=defaultdb)
named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
annotation_configs.TEST_NAMES)
named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
ut.get_list_column(named_defaults_dict, 'qcfg')))
# Allow command line override
aids, _specified_aids = ut.get_argval(('--aid', '--aids'), type_=list,
default=default_aids,
return_was_specified=True)
aidcfg = None
have_aids = aids is not None
need_expand = (not have_aids) or (_specified_a and not _specified_aids)
#(not aid) or (sa and (not said))
if need_expand:
#base_cfg = annotation_configs.single_default
aidcfg_combo_list = cfghelpers.parse_cfgstr_list2(
[a], named_qcfg_defaults, 'acfg', annotation_configs.ALIAS_KEYS,
expand_nested=False, is_nestedcfgtype=False)
aidcfg_combo = aidcfg_combo_list[0]
if len(aidcfg_combo_list) != 1:
raise AssertionError('Error: combinations not handled for single cfg setting')
if len(aidcfg_combo) != 1:
raise AssertionError('Error: combinations not handled for single cfg setting')
aidcfg = aidcfg_combo[0]
aids = filter_annots.expand_single_acfg(ibs, aidcfg, verbose=verbose)
if return_ibs:
return ibs, aids
if return_acfg:
return aids, aidcfg
else:
return aids
def parse_acfg_combo_list(acfg_name_list):
r"""
Args:
acfg_name_list (list):
Returns:
list: acfg_combo_list
CommandLine:
python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list
python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list:1
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> from ibeis.expt import annotation_configs
>>> acfg_name_list = testdata_acfg_names(['default', 'uncontrolled'])
>>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list)
>>> acfg_list = ut.flatten(acfg_combo_list)
>>> printkw = dict()
>>> annotation_configs.print_acfg_list(acfg_list, **printkw)
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> from ibeis.expt import annotation_configs
>>> # double colon :: means expand consistently and force const size
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'ctrl::unctrl'])
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::unctrl'])
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::varysize', 'ctrl::unctrl'])
>>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list)
>>> acfg_list = ut.flatten(acfg_combo_list)
>>> printkw = dict()
>>> annotation_configs.print_acfg_list(acfg_list, **printkw)
"""
from ibeis.expt import annotation_configs
named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
annotation_configs.TEST_NAMES)
named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
ut.get_list_column(named_defaults_dict, 'qcfg')))
named_dcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
ut.get_list_column(named_defaults_dict, 'dcfg')))
alias_keys = annotation_configs.ALIAS_KEYS
# need to have the cfgstr_lists be the same for query and database so they
# can be combined properly for now
# Apply this flag to any case joined with ::
special_join_dict = {'force_const_size': True}
# Parse Query Annot Config
nested_qcfg_combo_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list=acfg_name_list,
named_defaults_dict=named_qcfg_defaults,
cfgtype='qcfg',
alias_keys=alias_keys,
expand_nested=False,
special_join_dict=special_join_dict,
is_nestedcfgtype=True)
#print('acfg_name_list = %r' % (acfg_name_list,))
#print(len(acfg_name_list))
#print(ut.depth_profile(nested_qcfg_combo_list))
# Parse Data Annot Config
nested_dcfg_combo_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list=acfg_name_list,
named_defaults_dict=named_dcfg_defaults,
cfgtype='dcfg',
alias_keys=alias_keys,
expand_nested=False,
special_join_dict=special_join_dict,
is_nestedcfgtype=True)
#print(ut.depth_profile(nested_dcfg_combo_list))
acfg_combo_list = []
#print('--')
for nested_qcfg_combo, nested_dcfg_combo in zip(nested_qcfg_combo_list, nested_dcfg_combo_list):
#print('\n\n++++')
#print(len(nested_dcfg_combo))
#print(len(nested_qcfg_combo))
acfg_combo = []
# Only the inner nested combos are combinatorial
for qcfg_combo, dcfg_combo in zip(nested_qcfg_combo, nested_dcfg_combo):
#print('---++++')
#print('---- ' + str(len(qcfg_combo)))
#print('---- ' + str(len(dcfg_combo)))
_combo = [
dict([('qcfg', qcfg), ('dcfg', dcfg)])
for qcfg, dcfg in list(itertools.product(qcfg_combo, dcfg_combo))
]
#print('---- len(_combo) = %r' % (len(_combo),))
acfg_combo.extend(_combo)
acfg_combo_list.append(acfg_combo)
#print('LLL--')
#print(ut.depth_profile(acfg_combo_list))
return acfg_combo_list
def get_pipecfg_list(test_cfg_name_list, ibs=None):
r"""
Builds a list of varied query configurations. Only custom configs depend on
an ibs object. The order of the output is not gaurenteed to aggree with
input order.
Args:
test_cfg_name_list (list): list of strs
ibs (IBEISController): ibeis controller object (optional)
Returns:
tuple: (cfg_list, cfgx2_lbl) -
cfg_list (list): list of config objects
cfgx2_lbl (list): denotes which parameters are being varied.
If there is just one config then nothing is varied
CommandLine:
python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:0
python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:1 --db humpbacks
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> #test_cfg_name_list = ['best', 'custom', 'custom:sv_on=False']
>>> #test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> # execute function
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> # verify results
>>> assert pipecfg_list[0].sv_cfg.sv_on is True
>>> assert pipecfg_list[1].sv_cfg.sv_on is False
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls))
>>> print(result)
pipecfg_lbls = [
'default:',
'default:sv_on=False',
]
Example1:
>>> # DISABLE_DOCTEST
>>> import ibeis_flukematch.plugin
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='humpbacks')
>>> test_cfg_name_list = ['default:pipeline_root=BC_DTW,decision=average,crop_dim_size=[960,500]', 'default:K=[1,4]']
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls))
>>> print(result)
>>> print_pipe_configs(pcfgdict_list, pipecfg_list)
"""
if ut.VERBOSE:
print('[expt_help.get_pipecfg_list] building pipecfg_list using: %s' %
test_cfg_name_list)
if isinstance(test_cfg_name_list, six.string_types):
test_cfg_name_list = [test_cfg_name_list]
_standard_cfg_names = []
_pcfgdict_list = []
# HACK: Parse out custom configs first
for test_cfg_name in test_cfg_name_list:
if test_cfg_name.startswith('custom:') or test_cfg_name == 'custom':
print('[expthelpers] Parsing nonstandard custom config')
if test_cfg_name.startswith('custom:'):
# parse out modifications to custom
cfgstr_list = ':'.join(test_cfg_name.split(':')[1:]).split(',')
augcfgdict = ut.parse_cfgstr_list(cfgstr_list, smartcast=True)
else:
augcfgdict = {}
# Take the configuration from the ibeis object
pipe_cfg = ibs.cfg.query_cfg.deepcopy()
# Update with augmented params
pipe_cfg.update_query_cfg(**augcfgdict)
# Parse out a standard cfgdict
cfgdict = dict(pipe_cfg.parse_items())
cfgdict['_cfgname'] = 'custom'
cfgdict['_cfgstr'] = test_cfg_name
_pcfgdict_list.append(cfgdict)
else:
_standard_cfg_names.append(test_cfg_name)
# Handle stanndard configs next
if len(_standard_cfg_names) > 0:
# Get parsing information
#cfg_default_dict = dict(Config.QueryConfig().parse_items())
#valid_keys = list(cfg_default_dict.keys())
cfgstr_list = _standard_cfg_names
named_defaults_dict = ut.dict_subset(experiment_configs.__dict__,
experiment_configs.TEST_NAMES)
alias_keys = experiment_configs.ALIAS_KEYS
# Parse standard pipeline cfgstrings
metadata = {'ibs': ibs}
dict_comb_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list,
named_defaults_dict,
cfgtype=None,
alias_keys=alias_keys,
# Hack out valid keys for humpbacks
#valid_keys=valid_keys,
strict=False,
metadata=metadata)
# Get varied params (there may be duplicates)
_pcfgdict_list.extend(ut.flatten(dict_comb_list))
# Expand cfgdicts into PipelineConfig config objects
# TODO: respsect different algorithm parameters like flukes
if ibs is None:
configclass_list = [Config.QueryConfig] * len(_pcfgdict_list)
else:
root_to_config = ibs.depc_annot.configclass_dict
configclass_list = [
root_to_config.get(_cfgdict.get('pipeline_root', 'vsmany'),
Config.QueryConfig)
for _cfgdict in _pcfgdict_list
]
_pipecfg_list = [
cls(**_cfgdict)
for cls, _cfgdict in zip(configclass_list, _pcfgdict_list)
]
# Enforce rule that removes duplicate configs
# by using feasiblity from ibeis.algo.Config
# TODO: Move this unique finding code to its own function
# and then move it up one function level so even the custom
# configs can be uniquified
_flag_list = ut.flag_unique_items(_pipecfg_list)
cfgdict_list = ut.compress(_pcfgdict_list, _flag_list)
pipecfg_list = ut.compress(_pipecfg_list, _flag_list)
if ut.NOT_QUIET:
#for cfg in _pipecfg_list:
# print(cfg.get_cfgstr())
# print(cfg)
print('[harn.help] return %d / %d unique pipeline configs from: %r' %
(len(cfgdict_list), len(_pcfgdict_list), test_cfg_name_list))
if ut.get_argflag(('--pcfginfo', '--pinfo', '--pipecfginfo')):
import sys
ut.colorprint('Requested PcfgInfo for tests... ', 'red')
print_pipe_configs(cfgdict_list, pipecfg_list)
ut.colorprint('Finished Reporting PcfgInfo. Exiting', 'red')
sys.exit(0)
return (cfgdict_list, pipecfg_list)
def parse_acfg_combo_list(acfg_name_list):
r"""
Parses the name list into a list of config dicts
Args:
acfg_name_list (list): a list of annotation config strings
Returns:
list: acfg_combo_list
CommandLine:
python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list
python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list:1
Example:
>>> # ENABLE_DOCTET
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> from ibeis.expt import annotation_configs
>>> acfg_name_list = testdata_acfg_names(['default', 'uncontrolled'])
>>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list)
>>> acfg_list = ut.flatten(acfg_combo_list)
>>> printkw = dict()
>>> annotation_configs.print_acfg_list(acfg_list, **printkw)
>>> result = list(acfg_list[0].keys())
>>> print(result)
[u'qcfg', u'dcfg']
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> from ibeis.expt import annotation_configs
>>> # double colon :: means expand consistently and force const size
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'ctrl::unctrl'])
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::unctrl'])
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::varysize', 'ctrl::unctrl'])
>>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list)
>>> acfg_list = ut.flatten(acfg_combo_list)
>>> printkw = dict()
>>> annotation_configs.print_acfg_list(acfg_list, **printkw)
"""
from ibeis.expt import annotation_configs
named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
annotation_configs.TEST_NAMES)
named_qcfg_defaults = dict(
zip(annotation_configs.TEST_NAMES,
ut.get_list_column(named_defaults_dict, 'qcfg')))
named_dcfg_defaults = dict(
zip(annotation_configs.TEST_NAMES,
ut.get_list_column(named_defaults_dict, 'dcfg')))
alias_keys = annotation_configs.ALIAS_KEYS
# need to have the cfgstr_lists be the same for query and database so they
# can be combined properly for now
# Apply this flag to any case joined with ::
special_join_dict = {'force_const_size': True}
# Parse Query Annot Config
nested_qcfg_combo_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list=acfg_name_list,
named_defaults_dict=named_qcfg_defaults,
cfgtype='qcfg',
alias_keys=alias_keys,
expand_nested=False,
special_join_dict=special_join_dict,
is_nestedcfgtype=True)
#print('acfg_name_list = %r' % (acfg_name_list,))
#print(len(acfg_name_list))
#print(ut.depth_profile(nested_qcfg_combo_list))
# Parse Data Annot Config
nested_dcfg_combo_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list=acfg_name_list,
named_defaults_dict=named_dcfg_defaults,
cfgtype='dcfg',
alias_keys=alias_keys,
expand_nested=False,
special_join_dict=special_join_dict,
is_nestedcfgtype=True)
#print(ut.depth_profile(nested_dcfg_combo_list))
acfg_combo_list = []
#print('--')
for nested_qcfg_combo, nested_dcfg_combo in zip(nested_qcfg_combo_list,
nested_dcfg_combo_list):
#print('\n\n++++')
#print(len(nested_dcfg_combo))
#print(len(nested_qcfg_combo))
acfg_combo = []
# Only the inner nested combos are combinatorial
for qcfg_combo, dcfg_combo in zip(nested_qcfg_combo,
nested_dcfg_combo):
#print('---++++')
#print('---- ' + str(len(qcfg_combo)))
#print('---- ' + str(len(dcfg_combo)))
_combo = [
dict([('qcfg', qcfg), ('dcfg', dcfg)]) for qcfg, dcfg in list(
itertools.product(qcfg_combo, dcfg_combo))
]
#print('---- len(_combo) = %r' % (len(_combo),))
acfg_combo.extend(_combo)
acfg_combo_list.append(acfg_combo)
#print('LLL--')
#print(ut.depth_profile(acfg_combo_list))
return acfg_combo_list
def testdata_aids(defaultdb=None,
a=None,
adefault='default',
ibs=None,
return_acfg=False,
verbose=None,
default_aids=None,
default_set='qcfg'):
r"""
Grabs default testdata for functions, but is command line overrideable
CommandLine:
python -m ibeis testdata_aids --verbtd --db PZ_ViewPoints
python -m ibeis testdata_aids --verbtd --db NNP_Master3 -a is_known=True,view_pername='#primary>0&#primary1>=1'
python -m ibeis testdata_aids --verbtd --db PZ_Master1 -a default:is_known=True,view_pername='#primary>0&#primary1>=1'
python -m ibeis testdata_aids --verbtd --db PZ_Master1 -a default:species=primary,minqual=ok --verbtd
python -m ibeis.other.dbinfo --test-latex_dbstats --dblist
python -m ibeis testdata_aids --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> from ibeis.expt import annotation_configs
>>> import ibeis
>>> #ibs = ibeis.opendb(defaultdb='PZ_ViewPoints')
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> a = None
>>> adefault = 'default:is_known=True'
>>> aids, aidcfg = testdata_aids(ibs=ibs, a=a, adefault=adefault, return_acfg=True)
>>> print('\n RESULT:')
>>> annotation_configs.print_acfg(aidcfg, aids, ibs, per_name_vpedge=None)
"""
import ibeis
from ibeis.init import filter_annots
from ibeis.expt import annotation_configs
from ibeis.expt import cfghelpers
if verbose is None or verbose >= 1:
print('[main_helpers] testdata_aids')
if a is None:
a = adefault
a, _specified_a = ut.get_argval(('--aidcfg', '--acfg', '-a'),
type_=str,
default=a,
return_was_specified=True)
return_ibs = False
if ibs is None:
return_ibs = True
if defaultdb is None:
defaultdb = 'testdb1'
ibs = ibeis.opendb(defaultdb=defaultdb)
named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
annotation_configs.TEST_NAMES)
named_acfg_defaults = dict(
zip(annotation_configs.TEST_NAMES,
ut.get_list_column(named_defaults_dict, default_set)))
# Allow command line override
aids, _specified_aids = ut.get_argval(('--aid', '--aids'),
type_=list,
default=default_aids,
return_was_specified=True)
aidcfg = None
have_aids = aids is not None
need_expand = (not have_aids) or (_specified_a and not _specified_aids)
#(not aid) or (sa and (not said))
if need_expand:
#base_cfg = annotation_configs.single_default
aidcfg_combo_list = cfghelpers.parse_cfgstr_list2(
[a],
named_acfg_defaults,
'acfg',
annotation_configs.ALIAS_KEYS,
expand_nested=False,
is_nestedcfgtype=False)
aidcfg_combo = aidcfg_combo_list[0]
if len(aidcfg_combo_list) != 1:
raise AssertionError(
'Error: combinations not handled for single cfg setting')
if len(aidcfg_combo) != 1:
raise AssertionError(
'Error: combinations not handled for single cfg setting')
aidcfg = aidcfg_combo[0]
aids = filter_annots.expand_single_acfg(ibs, aidcfg, verbose=verbose)
if return_ibs:
return ibs, aids
if return_acfg:
return aids, aidcfg
else:
return aids
def get_individual_result_sample(testres, filt_cfg=None, **kwargs):
"""
The selected rows are the query annotation you are interested in viewing
The selected cols are the parameter configuration you are interested in viewing
Args:
testres (TestResult): test result object
filt_cfg (dict): config dict
Kwargs:
all, hard, hard2, easy, interesting, hist
Returns:
tuple: (sel_rows, sel_cols, flat_case_labels)
CommandLine:
python -m ibeis --tf -get_individual_result_sample --db PZ_Master1 -a ctrl
python -m ibeis --tf -get_individual_result_sample --db PZ_Master1 -a ctrl --filt :fail=True,min_gtrank=5,gtrank_lt=20
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.expt.experiment_drawing import * # NOQA
>>> from ibeis.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> filt_cfg = {'fail': True, 'success': True, 'min_gtrank': 5, 'max_gtrank': 40}
>>> sel_rows, sel_cols, flat_case_labels = get_individual_result_sample(testres, filt_cfg)
>>> result = ('(sel_rows, sel_cols, flat_case_labels) = %s' % (str((sel_rows, sel_cols, flat_case_labels)),))
>>> print(result)
"""
#from ibeis.expt import cfghelpers
#sample_cfgstr_list = ut.get_argval('--filt', type_=list, default=None)
#from ibeis.expt import cfghelpers
#if sample_cfgstr_list is None:
print('filt_cfg = %r' % (filt_cfg,))
if filt_cfg is None or isinstance(filt_cfg, list):
# Hack to check if specified on command line
#if not show_in_notebook:
# from ibeis.init import main_helpers
# filt_cfg = main_helpers.testdata_filtcfg(default=filt_cfg)
#else:
from ibeis.expt import cfghelpers
if filt_cfg is None:
filt_cfg = ['']
filt_cfg = ut.flatten(cfghelpers.parse_cfgstr_list2(filt_cfg, strict=False))[0]
cfg_list = testres.cfg_list
#qaids = testres.qaids
qaids = testres.get_common_qaids()
view_all = kwargs.get('all', ut.get_argflag(('--view-all', '--va')))
view_hard = kwargs.get('hard', ut.get_argflag(('--view-hard', '--vh')))
view_hard2 = kwargs.get('hard2', ut.get_argflag(('--view-hard2', '--vh2')))
view_easy = kwargs.get('easy', ut.get_argflag(('--view-easy', '--vz')))
view_interesting = kwargs.get('interesting', ut.get_argflag(('--view-interesting', '--vn')))
hist_sample = kwargs.get('hist', ut.get_argflag(('--hs', '--hist-sample')))
view_differ_cases = kwargs.get('differcases', ut.get_argflag(('--diff-cases', '--dc')))
view_cases = kwargs.get('cases', ut.get_argflag(('--view-cases', '--vc')))
if ut.get_argval('--qaid', type_=str, default=None) is not None:
# hack
view_all = True
#sel_cols = params.args.sel_cols # FIXME
#sel_rows = params.args.sel_rows # FIXME
#sel_cols = [] if sel_cols is None else sel_cols
#sel_rows = [] if sel_rows is None else sel_rows
sel_rows = []
sel_cols = []
flat_case_labels = None
if ut.NOT_QUIET:
print('remember to inspect with --show --sel-rows (-r) and --sel-cols (-c) ')
print('other options:')
print(' --vf - view figure dir')
print(' --va - view all (--filt :)')
print(' --vh - view hard (--filt :fail=True)')
print(' --ve - view easy (--filt :success=True)')
print(' --vn - view iNteresting')
print(' --hs - hist sample')
print(' --filt - result filtering config (new way to do this func)')
print(' --gv, --guiview - gui result inspection')
if len(sel_rows) > 0 and len(sel_cols) == 0:
sel_cols = list(range(len(cfg_list)))
if len(sel_cols) > 0 and len(sel_rows) == 0:
sel_rows = list(range(len(qaids)))
if view_all:
sel_rows = list(range(len(qaids)))
sel_cols = list(range(len(cfg_list)))
if view_hard:
new_hard_qx_list = testres.get_new_hard_qx_list()
sel_rows.extend(np.array(new_hard_qx_list).tolist())
sel_cols.extend(list(range(len(cfg_list))))
# sample-cases
def convert_case_pos_to_cfgx(case_pos_list, case_labels_list):
# Convert to all cfgx format
qx_list = ut.unique_ordered(np.array(case_pos_list).T[0])
ut.dict_take(ut.group_items(case_pos_list, case_pos_list.T[0]), qx_list)
if case_labels_list is not None:
grouped_labels = ut.dict_take(
ut.group_items(case_labels_list, case_pos_list.T[0]),
qx_list)
flat_case_labels = list(map(ut.unique_ordered, map(ut.flatten, grouped_labels)))
else:
flat_case_labels = None
new_rows = np.array(qx_list).tolist()
new_cols = list(range(len(cfg_list)))
return new_rows, new_cols, flat_case_labels
if view_differ_cases:
# Cases that passed on config but failed another
case_pos_list, case_labels_list = testres.case_type_sample(
1, with_success=True, min_success_diff=1)
new_rows, new_cols, flat_case_labels = convert_case_pos_to_cfgx(
case_pos_list, case_labels_list)
sel_rows.extend(new_rows)
sel_cols.extend(new_cols)
if view_cases:
case_pos_list, case_labels_list = testres.case_type_sample(1, with_success=False)
new_rows, new_cols, flat_case_labels = convert_case_pos_to_cfgx(
case_pos_list, case_labels_list)
sel_rows.extend(new_rows)
sel_cols.extend(new_cols)
if view_hard2:
# TODO handle returning case_pos_list
#samplekw = ut.argparse_dict(dict(per_group=5))
samplekw = ut.argparse_dict(dict(per_group=None))
case_pos_list = testres.get_case_positions(mode='failure', samplekw=samplekw)
failure_qx_list = ut.unique_ordered(case_pos_list.T[0])
sel_rows.extend(np.array(failure_qx_list).tolist())
sel_cols.extend(list(range(len(cfg_list))))
if view_easy:
new_hard_qx_list = testres.get_new_hard_qx_list()
new_easy_qx_list = np.setdiff1d(np.arange(len(qaids)), new_hard_qx_list).tolist()
sel_rows.extend(new_easy_qx_list)
sel_cols.extend(list(range(len(cfg_list))))
if view_interesting:
interesting_qx_list = testres.get_interesting_ranks()
sel_rows.extend(interesting_qx_list)
# TODO: grab the best scoring and most interesting configs
if len(sel_cols) == 0:
sel_cols.extend(list(range(len(cfg_list))))
if hist_sample:
# Careful if there is more than one config
config_rand_bin_qxs = testres.get_rank_histogram_qx_sample(size=10)
sel_rows = np.hstack(ut.flatten(config_rand_bin_qxs))
# TODO: grab the best scoring and most interesting configs
if len(sel_cols) == 0:
sel_cols.extend(list(range(len(cfg_list))))
if filt_cfg is not None:
# NEW WAY OF SAMPLING
verbose = kwargs.get('verbose', None)
case_pos_list = testres.case_sample2(filt_cfg, verbose=verbose)
new_rows, new_cols, flat_case_labels = convert_case_pos_to_cfgx(case_pos_list, None)
sel_rows.extend(new_rows)
sel_cols.extend(new_cols)
pass
sel_rows = ut.unique_ordered(sel_rows)
sel_cols = ut.unique_ordered(sel_cols)
sel_cols = list(sel_cols)
sel_rows = list(sel_rows)
sel_rowxs = ut.get_argval('-r', type_=list, default=None)
sel_colxs = ut.get_argval('-c', type_=list, default=None)
if sel_rowxs is not None:
sel_rows = ut.take(sel_rows, sel_rowxs)
print('sel_rows = %r' % (sel_rows,))
if sel_colxs is not None:
sel_cols = ut.take(sel_cols, sel_colxs)
if ut.NOT_QUIET:
print('Returning Case Selection')
print('len(sel_rows) = %r/%r' % (len(sel_rows), len(qaids)))
print('len(sel_cols) = %r/%r' % (len(sel_cols), len(cfg_list)))
return sel_rows, sel_cols, flat_case_labels
