def get_pipecfg_list(test_cfg_name_list, ibs=None): r""" Builds a list of varied query configurations. Only custom configs depend on an ibs object. The order of the output is not gaurenteed to aggree with input order. Args: test_cfg_name_list (list): list of strs ibs (IBEISController): ibeis controller object (optional) Returns: tuple: (cfg_list, cfgx2_lbl) - cfg_list (list): list of config objects cfgx2_lbl (list): denotes which parameters are being varied. If there is just one config then nothing is varied CommandLine: python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:0 python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:1 --db humpbacks Example: >>> # ENABLE_DOCTEST >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> #test_cfg_name_list = ['best', 'custom', 'custom:sv_on=False'] >>> #test_cfg_name_list = ['default', 'default:sv_on=False', 'best'] >>> test_cfg_name_list = ['default', 'default:sv_on=False', 'best'] >>> # execute function >>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs) >>> # verify results >>> assert pipecfg_list[0].sv_cfg.sv_on is True >>> assert pipecfg_list[1].sv_cfg.sv_on is False >>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list) >>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls)) >>> print(result) pipecfg_lbls = [ 'default:', 'default:sv_on=False', ] Example1: >>> # DISABLE_DOCTEST >>> import ibeis_flukematch.plugin >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='humpbacks') >>> test_cfg_name_list = ['default:pipeline_root=BC_DTW,decision=average', 'default:K=[1,4]'] >>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs) >>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list) >>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls)) >>> print(result) >>> print_pipe_configs(pcfgdict_list, pipecfg_list) """ if ut.VERBOSE: print('[expt_help.get_pipecfg_list] building pipecfg_list using: %s' % test_cfg_name_list) if isinstance(test_cfg_name_list, six.string_types): test_cfg_name_list = [test_cfg_name_list] _standard_cfg_names = [] _pcfgdict_list = [] # HACK: Parse out custom configs first for test_cfg_name in test_cfg_name_list: if test_cfg_name.startswith('custom:') or test_cfg_name == 'custom': print('[expthelpers] Parsing nonstandard custom config') if test_cfg_name.startswith('custom:'): # parse out modifications to custom cfgstr_list = ':'.join(test_cfg_name.split(':')[1:]).split(',') augcfgdict = ut.parse_cfgstr_list(cfgstr_list, smartcast=True) else: augcfgdict = {} # Take the configuration from the ibeis object pipe_cfg = ibs.cfg.query_cfg.deepcopy() # Update with augmented params pipe_cfg.update_query_cfg(**augcfgdict) # Parse out a standard cfgdict cfgdict = dict(pipe_cfg.parse_items()) cfgdict['_cfgname'] = 'custom' cfgdict['_cfgstr'] = test_cfg_name _pcfgdict_list.append(cfgdict) else: _standard_cfg_names.append(test_cfg_name) # Handle stanndard configs next if len(_standard_cfg_names) > 0: # Get parsing information #cfg_default_dict = dict(Config.QueryConfig().parse_items()) #valid_keys = list(cfg_default_dict.keys()) cfgstr_list = _standard_cfg_names named_defaults_dict = ut.dict_subset( experiment_configs.__dict__, experiment_configs.TEST_NAMES) alias_keys = experiment_configs.ALIAS_KEYS # Parse standard pipeline cfgstrings metadata = {'ibs': ibs} dict_comb_list = cfghelpers.parse_cfgstr_list2( cfgstr_list, named_defaults_dict, cfgtype=None, alias_keys=alias_keys, # Hack out valid keys for humpbacks #valid_keys=valid_keys, strict=False, metadata=metadata ) # Get varied params (there may be duplicates) _pcfgdict_list.extend(ut.flatten(dict_comb_list)) # TODO: respsect different algorithm parameters # like flukes # Expand cfgdicts into PipelineConfig config objects if ibs is None: configclass_list = [Config.QueryConfig] * len(_pcfgdict_list) else: root_to_config = ibs.depc.configclass_dict configclass_list = [ root_to_config.get(_cfgdict.get('pipeline_root', 'vsmany'), Config.QueryConfig) for _cfgdict in _pcfgdict_list] _pipecfg_list = [cls(**_cfgdict) for cls, _cfgdict in zip(configclass_list, _pcfgdict_list)] # Enforce rule that removes duplicate configs # by using feasiblity from ibeis.algo.Config # TODO: Move this unique finding code to its own function # and then move it up one function level so even the custom # configs can be uniquified _flag_list = ut.flag_unique_items(_pipecfg_list) cfgdict_list = ut.compress(_pcfgdict_list, _flag_list) pipecfg_list = ut.compress(_pipecfg_list, _flag_list) if ut.NOT_QUIET: print('[harn.help] return %d / %d unique pipeline configs from: %r' % (len(cfgdict_list), len(_pcfgdict_list), test_cfg_name_list)) if ut.get_argflag(('--pcfginfo', '--pinfo', '--pipecfginfo')): import sys ut.colorprint('Requested PcfgInfo for tests... ', 'red') print_pipe_configs(cfgdict_list, pipecfg_list) ut.colorprint('Finished Reporting PcfgInfo. Exiting', 'red') sys.exit(1) return (cfgdict_list, pipecfg_list)
def testdata_aids(defaultdb=None, a=None, adefault='default', ibs=None, return_acfg=False, verbose=None, default_aids=None): r""" Grabs default testdata for functions, but is command line overrideable CommandLine: python -m ibeis --tf testdata_aids --verbtd --db PZ_ViewPoints python -m ibeis --tf testdata_aids --verbtd --db NNP_Master3 -a is_known=True,view_pername='#primary>0&#primary1>=1' python -m ibeis --tf testdata_aids --verbtd --db PZ_Master1 -a default:is_known=True,view_pername='#primary>0&#primary1>=1' python -m ibeis --tf testdata_aids --verbtd --db PZ_Master1 -a default:species=primary,minqual=ok --verbtd python -m ibeis.other.dbinfo --test-latex_dbstats --dblist CommandLine: python -m ibeis.init.main_helpers --exec-testdata_aids --show Example: >>> # ENABLE_DOCTEST >>> from ibeis.init.main_helpers import * # NOQA >>> from ibeis.expt import annotation_configs >>> import ibeis >>> #ibs = ibeis.opendb(defaultdb='PZ_ViewPoints') >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> a = None >>> adefault = 'default:is_known=True' >>> aids, aidcfg = testdata_aids(ibs=ibs, a=a, adefault=adefault, return_acfg=True) >>> print('\n RESULT:') >>> annotation_configs.print_acfg(aidcfg, aids, ibs, per_name_vpedge=None) """ from ibeis.init import filter_annots from ibeis.expt import annotation_configs from ibeis.expt import cfghelpers import ibeis print('[main_helpers] testdata_aids') if a is None: a = adefault a, _specified_a = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=str, default=a, return_was_specified=True) return_ibs = False if ibs is None: return_ibs = True if defaultdb is None: defaultdb = 'testdb1' ibs = ibeis.opendb(defaultdb=defaultdb) named_defaults_dict = ut.dict_take(annotation_configs.__dict__, annotation_configs.TEST_NAMES) named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES, ut.get_list_column(named_defaults_dict, 'qcfg'))) # Allow command line override aids, _specified_aids = ut.get_argval(('--aid', '--aids'), type_=list, default=default_aids, return_was_specified=True) aidcfg = None have_aids = aids is not None need_expand = (not have_aids) or (_specified_a and not _specified_aids) #(not aid) or (sa and (not said)) if need_expand: #base_cfg = annotation_configs.single_default aidcfg_combo_list = cfghelpers.parse_cfgstr_list2( [a], named_qcfg_defaults, 'acfg', annotation_configs.ALIAS_KEYS, expand_nested=False, is_nestedcfgtype=False) aidcfg_combo = aidcfg_combo_list[0] if len(aidcfg_combo_list) != 1: raise AssertionError('Error: combinations not handled for single cfg setting') if len(aidcfg_combo) != 1: raise AssertionError('Error: combinations not handled for single cfg setting') aidcfg = aidcfg_combo[0] aids = filter_annots.expand_single_acfg(ibs, aidcfg, verbose=verbose) if return_ibs: return ibs, aids if return_acfg: return aids, aidcfg else: return aids
def parse_acfg_combo_list(acfg_name_list): r""" Args: acfg_name_list (list): Returns: list: acfg_combo_list CommandLine: python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list:1 Example: >>> # DISABLE_DOCTEST >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> from ibeis.expt import annotation_configs >>> acfg_name_list = testdata_acfg_names(['default', 'uncontrolled']) >>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list) >>> acfg_list = ut.flatten(acfg_combo_list) >>> printkw = dict() >>> annotation_configs.print_acfg_list(acfg_list, **printkw) Example: >>> # DISABLE_DOCTEST >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> from ibeis.expt import annotation_configs >>> # double colon :: means expand consistently and force const size >>> acfg_name_list = testdata_acfg_names(['unctrl', 'ctrl::unctrl']) >>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::unctrl']) >>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::varysize', 'ctrl::unctrl']) >>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list) >>> acfg_list = ut.flatten(acfg_combo_list) >>> printkw = dict() >>> annotation_configs.print_acfg_list(acfg_list, **printkw) """ from ibeis.expt import annotation_configs named_defaults_dict = ut.dict_take(annotation_configs.__dict__, annotation_configs.TEST_NAMES) named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES, ut.get_list_column(named_defaults_dict, 'qcfg'))) named_dcfg_defaults = dict(zip(annotation_configs.TEST_NAMES, ut.get_list_column(named_defaults_dict, 'dcfg'))) alias_keys = annotation_configs.ALIAS_KEYS # need to have the cfgstr_lists be the same for query and database so they # can be combined properly for now # Apply this flag to any case joined with :: special_join_dict = {'force_const_size': True} # Parse Query Annot Config nested_qcfg_combo_list = cfghelpers.parse_cfgstr_list2( cfgstr_list=acfg_name_list, named_defaults_dict=named_qcfg_defaults, cfgtype='qcfg', alias_keys=alias_keys, expand_nested=False, special_join_dict=special_join_dict, is_nestedcfgtype=True) #print('acfg_name_list = %r' % (acfg_name_list,)) #print(len(acfg_name_list)) #print(ut.depth_profile(nested_qcfg_combo_list)) # Parse Data Annot Config nested_dcfg_combo_list = cfghelpers.parse_cfgstr_list2( cfgstr_list=acfg_name_list, named_defaults_dict=named_dcfg_defaults, cfgtype='dcfg', alias_keys=alias_keys, expand_nested=False, special_join_dict=special_join_dict, is_nestedcfgtype=True) #print(ut.depth_profile(nested_dcfg_combo_list)) acfg_combo_list = [] #print('--') for nested_qcfg_combo, nested_dcfg_combo in zip(nested_qcfg_combo_list, nested_dcfg_combo_list): #print('\n\n++++') #print(len(nested_dcfg_combo)) #print(len(nested_qcfg_combo)) acfg_combo = [] # Only the inner nested combos are combinatorial for qcfg_combo, dcfg_combo in zip(nested_qcfg_combo, nested_dcfg_combo): #print('---++++') #print('---- ' + str(len(qcfg_combo))) #print('---- ' + str(len(dcfg_combo))) _combo = [ dict([('qcfg', qcfg), ('dcfg', dcfg)]) for qcfg, dcfg in list(itertools.product(qcfg_combo, dcfg_combo)) ] #print('---- len(_combo) = %r' % (len(_combo),)) acfg_combo.extend(_combo) acfg_combo_list.append(acfg_combo) #print('LLL--') #print(ut.depth_profile(acfg_combo_list)) return acfg_combo_list
def get_pipecfg_list(test_cfg_name_list, ibs=None): r""" Builds a list of varied query configurations. Only custom configs depend on an ibs object. The order of the output is not gaurenteed to aggree with input order. Args: test_cfg_name_list (list): list of strs ibs (IBEISController): ibeis controller object (optional) Returns: tuple: (cfg_list, cfgx2_lbl) - cfg_list (list): list of config objects cfgx2_lbl (list): denotes which parameters are being varied. If there is just one config then nothing is varied CommandLine: python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:0 python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:1 --db humpbacks Example: >>> # ENABLE_DOCTEST >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> #test_cfg_name_list = ['best', 'custom', 'custom:sv_on=False'] >>> #test_cfg_name_list = ['default', 'default:sv_on=False', 'best'] >>> test_cfg_name_list = ['default', 'default:sv_on=False', 'best'] >>> # execute function >>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs) >>> # verify results >>> assert pipecfg_list[0].sv_cfg.sv_on is True >>> assert pipecfg_list[1].sv_cfg.sv_on is False >>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list) >>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls)) >>> print(result) pipecfg_lbls = [ 'default:', 'default:sv_on=False', ] Example1: >>> # DISABLE_DOCTEST >>> import ibeis_flukematch.plugin >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='humpbacks') >>> test_cfg_name_list = ['default:pipeline_root=BC_DTW,decision=average,crop_dim_size=[960,500]', 'default:K=[1,4]'] >>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs) >>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list) >>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls)) >>> print(result) >>> print_pipe_configs(pcfgdict_list, pipecfg_list) """ if ut.VERBOSE: print('[expt_help.get_pipecfg_list] building pipecfg_list using: %s' % test_cfg_name_list) if isinstance(test_cfg_name_list, six.string_types): test_cfg_name_list = [test_cfg_name_list] _standard_cfg_names = [] _pcfgdict_list = [] # HACK: Parse out custom configs first for test_cfg_name in test_cfg_name_list: if test_cfg_name.startswith('custom:') or test_cfg_name == 'custom': print('[expthelpers] Parsing nonstandard custom config') if test_cfg_name.startswith('custom:'): # parse out modifications to custom cfgstr_list = ':'.join(test_cfg_name.split(':')[1:]).split(',') augcfgdict = ut.parse_cfgstr_list(cfgstr_list, smartcast=True) else: augcfgdict = {} # Take the configuration from the ibeis object pipe_cfg = ibs.cfg.query_cfg.deepcopy() # Update with augmented params pipe_cfg.update_query_cfg(**augcfgdict) # Parse out a standard cfgdict cfgdict = dict(pipe_cfg.parse_items()) cfgdict['_cfgname'] = 'custom' cfgdict['_cfgstr'] = test_cfg_name _pcfgdict_list.append(cfgdict) else: _standard_cfg_names.append(test_cfg_name) # Handle stanndard configs next if len(_standard_cfg_names) > 0: # Get parsing information #cfg_default_dict = dict(Config.QueryConfig().parse_items()) #valid_keys = list(cfg_default_dict.keys()) cfgstr_list = _standard_cfg_names named_defaults_dict = ut.dict_subset(experiment_configs.__dict__, experiment_configs.TEST_NAMES) alias_keys = experiment_configs.ALIAS_KEYS # Parse standard pipeline cfgstrings metadata = {'ibs': ibs} dict_comb_list = cfghelpers.parse_cfgstr_list2( cfgstr_list, named_defaults_dict, cfgtype=None, alias_keys=alias_keys, # Hack out valid keys for humpbacks #valid_keys=valid_keys, strict=False, metadata=metadata) # Get varied params (there may be duplicates) _pcfgdict_list.extend(ut.flatten(dict_comb_list)) # Expand cfgdicts into PipelineConfig config objects # TODO: respsect different algorithm parameters like flukes if ibs is None: configclass_list = [Config.QueryConfig] * len(_pcfgdict_list) else: root_to_config = ibs.depc_annot.configclass_dict configclass_list = [ root_to_config.get(_cfgdict.get('pipeline_root', 'vsmany'), Config.QueryConfig) for _cfgdict in _pcfgdict_list ] _pipecfg_list = [ cls(**_cfgdict) for cls, _cfgdict in zip(configclass_list, _pcfgdict_list) ] # Enforce rule that removes duplicate configs # by using feasiblity from ibeis.algo.Config # TODO: Move this unique finding code to its own function # and then move it up one function level so even the custom # configs can be uniquified _flag_list = ut.flag_unique_items(_pipecfg_list) cfgdict_list = ut.compress(_pcfgdict_list, _flag_list) pipecfg_list = ut.compress(_pipecfg_list, _flag_list) if ut.NOT_QUIET: #for cfg in _pipecfg_list: # print(cfg.get_cfgstr()) # print(cfg) print('[harn.help] return %d / %d unique pipeline configs from: %r' % (len(cfgdict_list), len(_pcfgdict_list), test_cfg_name_list)) if ut.get_argflag(('--pcfginfo', '--pinfo', '--pipecfginfo')): import sys ut.colorprint('Requested PcfgInfo for tests... ', 'red') print_pipe_configs(cfgdict_list, pipecfg_list) ut.colorprint('Finished Reporting PcfgInfo. Exiting', 'red') sys.exit(0) return (cfgdict_list, pipecfg_list)
def parse_acfg_combo_list(acfg_name_list): r""" Parses the name list into a list of config dicts Args: acfg_name_list (list): a list of annotation config strings Returns: list: acfg_combo_list CommandLine: python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list python -m ibeis.expt.experiment_helpers --exec-parse_acfg_combo_list:1 Example: >>> # ENABLE_DOCTET >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> from ibeis.expt import annotation_configs >>> acfg_name_list = testdata_acfg_names(['default', 'uncontrolled']) >>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list) >>> acfg_list = ut.flatten(acfg_combo_list) >>> printkw = dict() >>> annotation_configs.print_acfg_list(acfg_list, **printkw) >>> result = list(acfg_list[0].keys()) >>> print(result) [u'qcfg', u'dcfg'] Example: >>> # DISABLE_DOCTEST >>> from ibeis.expt.experiment_helpers import * # NOQA >>> import ibeis >>> from ibeis.expt import annotation_configs >>> # double colon :: means expand consistently and force const size >>> acfg_name_list = testdata_acfg_names(['unctrl', 'ctrl::unctrl']) >>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::unctrl']) >>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::varysize', 'ctrl::unctrl']) >>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list) >>> acfg_list = ut.flatten(acfg_combo_list) >>> printkw = dict() >>> annotation_configs.print_acfg_list(acfg_list, **printkw) """ from ibeis.expt import annotation_configs named_defaults_dict = ut.dict_take(annotation_configs.__dict__, annotation_configs.TEST_NAMES) named_qcfg_defaults = dict( zip(annotation_configs.TEST_NAMES, ut.get_list_column(named_defaults_dict, 'qcfg'))) named_dcfg_defaults = dict( zip(annotation_configs.TEST_NAMES, ut.get_list_column(named_defaults_dict, 'dcfg'))) alias_keys = annotation_configs.ALIAS_KEYS # need to have the cfgstr_lists be the same for query and database so they # can be combined properly for now # Apply this flag to any case joined with :: special_join_dict = {'force_const_size': True} # Parse Query Annot Config nested_qcfg_combo_list = cfghelpers.parse_cfgstr_list2( cfgstr_list=acfg_name_list, named_defaults_dict=named_qcfg_defaults, cfgtype='qcfg', alias_keys=alias_keys, expand_nested=False, special_join_dict=special_join_dict, is_nestedcfgtype=True) #print('acfg_name_list = %r' % (acfg_name_list,)) #print(len(acfg_name_list)) #print(ut.depth_profile(nested_qcfg_combo_list)) # Parse Data Annot Config nested_dcfg_combo_list = cfghelpers.parse_cfgstr_list2( cfgstr_list=acfg_name_list, named_defaults_dict=named_dcfg_defaults, cfgtype='dcfg', alias_keys=alias_keys, expand_nested=False, special_join_dict=special_join_dict, is_nestedcfgtype=True) #print(ut.depth_profile(nested_dcfg_combo_list)) acfg_combo_list = [] #print('--') for nested_qcfg_combo, nested_dcfg_combo in zip(nested_qcfg_combo_list, nested_dcfg_combo_list): #print('\n\n++++') #print(len(nested_dcfg_combo)) #print(len(nested_qcfg_combo)) acfg_combo = [] # Only the inner nested combos are combinatorial for qcfg_combo, dcfg_combo in zip(nested_qcfg_combo, nested_dcfg_combo): #print('---++++') #print('---- ' + str(len(qcfg_combo))) #print('---- ' + str(len(dcfg_combo))) _combo = [ dict([('qcfg', qcfg), ('dcfg', dcfg)]) for qcfg, dcfg in list( itertools.product(qcfg_combo, dcfg_combo)) ] #print('---- len(_combo) = %r' % (len(_combo),)) acfg_combo.extend(_combo) acfg_combo_list.append(acfg_combo) #print('LLL--') #print(ut.depth_profile(acfg_combo_list)) return acfg_combo_list
def testdata_aids(defaultdb=None, a=None, adefault='default', ibs=None, return_acfg=False, verbose=None, default_aids=None, default_set='qcfg'): r""" Grabs default testdata for functions, but is command line overrideable CommandLine: python -m ibeis testdata_aids --verbtd --db PZ_ViewPoints python -m ibeis testdata_aids --verbtd --db NNP_Master3 -a is_known=True,view_pername='#primary>0&#primary1>=1' python -m ibeis testdata_aids --verbtd --db PZ_Master1 -a default:is_known=True,view_pername='#primary>0&#primary1>=1' python -m ibeis testdata_aids --verbtd --db PZ_Master1 -a default:species=primary,minqual=ok --verbtd python -m ibeis.other.dbinfo --test-latex_dbstats --dblist python -m ibeis testdata_aids --show Example: >>> # ENABLE_DOCTEST >>> from ibeis.init.main_helpers import * # NOQA >>> from ibeis.expt import annotation_configs >>> import ibeis >>> #ibs = ibeis.opendb(defaultdb='PZ_ViewPoints') >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> a = None >>> adefault = 'default:is_known=True' >>> aids, aidcfg = testdata_aids(ibs=ibs, a=a, adefault=adefault, return_acfg=True) >>> print('\n RESULT:') >>> annotation_configs.print_acfg(aidcfg, aids, ibs, per_name_vpedge=None) """ import ibeis from ibeis.init import filter_annots from ibeis.expt import annotation_configs from ibeis.expt import cfghelpers if verbose is None or verbose >= 1: print('[main_helpers] testdata_aids') if a is None: a = adefault a, _specified_a = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=str, default=a, return_was_specified=True) return_ibs = False if ibs is None: return_ibs = True if defaultdb is None: defaultdb = 'testdb1' ibs = ibeis.opendb(defaultdb=defaultdb) named_defaults_dict = ut.dict_take(annotation_configs.__dict__, annotation_configs.TEST_NAMES) named_acfg_defaults = dict( zip(annotation_configs.TEST_NAMES, ut.get_list_column(named_defaults_dict, default_set))) # Allow command line override aids, _specified_aids = ut.get_argval(('--aid', '--aids'), type_=list, default=default_aids, return_was_specified=True) aidcfg = None have_aids = aids is not None need_expand = (not have_aids) or (_specified_a and not _specified_aids) #(not aid) or (sa and (not said)) if need_expand: #base_cfg = annotation_configs.single_default aidcfg_combo_list = cfghelpers.parse_cfgstr_list2( [a], named_acfg_defaults, 'acfg', annotation_configs.ALIAS_KEYS, expand_nested=False, is_nestedcfgtype=False) aidcfg_combo = aidcfg_combo_list[0] if len(aidcfg_combo_list) != 1: raise AssertionError( 'Error: combinations not handled for single cfg setting') if len(aidcfg_combo) != 1: raise AssertionError( 'Error: combinations not handled for single cfg setting') aidcfg = aidcfg_combo[0] aids = filter_annots.expand_single_acfg(ibs, aidcfg, verbose=verbose) if return_ibs: return ibs, aids if return_acfg: return aids, aidcfg else: return aids
def get_individual_result_sample(testres, filt_cfg=None, **kwargs): """ The selected rows are the query annotation you are interested in viewing The selected cols are the parameter configuration you are interested in viewing Args: testres (TestResult): test result object filt_cfg (dict): config dict Kwargs: all, hard, hard2, easy, interesting, hist Returns: tuple: (sel_rows, sel_cols, flat_case_labels) CommandLine: python -m ibeis --tf -get_individual_result_sample --db PZ_Master1 -a ctrl python -m ibeis --tf -get_individual_result_sample --db PZ_Master1 -a ctrl --filt :fail=True,min_gtrank=5,gtrank_lt=20 Example: >>> # DISABLE_DOCTEST >>> from ibeis.expt.experiment_drawing import * # NOQA >>> from ibeis.init import main_helpers >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST') >>> filt_cfg = {'fail': True, 'success': True, 'min_gtrank': 5, 'max_gtrank': 40} >>> sel_rows, sel_cols, flat_case_labels = get_individual_result_sample(testres, filt_cfg) >>> result = ('(sel_rows, sel_cols, flat_case_labels) = %s' % (str((sel_rows, sel_cols, flat_case_labels)),)) >>> print(result) """ #from ibeis.expt import cfghelpers #sample_cfgstr_list = ut.get_argval('--filt', type_=list, default=None) #from ibeis.expt import cfghelpers #if sample_cfgstr_list is None: print('filt_cfg = %r' % (filt_cfg,)) if filt_cfg is None or isinstance(filt_cfg, list): # Hack to check if specified on command line #if not show_in_notebook: # from ibeis.init import main_helpers # filt_cfg = main_helpers.testdata_filtcfg(default=filt_cfg) #else: from ibeis.expt import cfghelpers if filt_cfg is None: filt_cfg = [''] filt_cfg = ut.flatten(cfghelpers.parse_cfgstr_list2(filt_cfg, strict=False))[0] cfg_list = testres.cfg_list #qaids = testres.qaids qaids = testres.get_common_qaids() view_all = kwargs.get('all', ut.get_argflag(('--view-all', '--va'))) view_hard = kwargs.get('hard', ut.get_argflag(('--view-hard', '--vh'))) view_hard2 = kwargs.get('hard2', ut.get_argflag(('--view-hard2', '--vh2'))) view_easy = kwargs.get('easy', ut.get_argflag(('--view-easy', '--vz'))) view_interesting = kwargs.get('interesting', ut.get_argflag(('--view-interesting', '--vn'))) hist_sample = kwargs.get('hist', ut.get_argflag(('--hs', '--hist-sample'))) view_differ_cases = kwargs.get('differcases', ut.get_argflag(('--diff-cases', '--dc'))) view_cases = kwargs.get('cases', ut.get_argflag(('--view-cases', '--vc'))) if ut.get_argval('--qaid', type_=str, default=None) is not None: # hack view_all = True #sel_cols = params.args.sel_cols # FIXME #sel_rows = params.args.sel_rows # FIXME #sel_cols = [] if sel_cols is None else sel_cols #sel_rows = [] if sel_rows is None else sel_rows sel_rows = [] sel_cols = [] flat_case_labels = None if ut.NOT_QUIET: print('remember to inspect with --show --sel-rows (-r) and --sel-cols (-c) ') print('other options:') print(' --vf - view figure dir') print(' --va - view all (--filt :)') print(' --vh - view hard (--filt :fail=True)') print(' --ve - view easy (--filt :success=True)') print(' --vn - view iNteresting') print(' --hs - hist sample') print(' --filt - result filtering config (new way to do this func)') print(' --gv, --guiview - gui result inspection') if len(sel_rows) > 0 and len(sel_cols) == 0: sel_cols = list(range(len(cfg_list))) if len(sel_cols) > 0 and len(sel_rows) == 0: sel_rows = list(range(len(qaids))) if view_all: sel_rows = list(range(len(qaids))) sel_cols = list(range(len(cfg_list))) if view_hard: new_hard_qx_list = testres.get_new_hard_qx_list() sel_rows.extend(np.array(new_hard_qx_list).tolist()) sel_cols.extend(list(range(len(cfg_list)))) # sample-cases def convert_case_pos_to_cfgx(case_pos_list, case_labels_list): # Convert to all cfgx format qx_list = ut.unique_ordered(np.array(case_pos_list).T[0]) ut.dict_take(ut.group_items(case_pos_list, case_pos_list.T[0]), qx_list) if case_labels_list is not None: grouped_labels = ut.dict_take( ut.group_items(case_labels_list, case_pos_list.T[0]), qx_list) flat_case_labels = list(map(ut.unique_ordered, map(ut.flatten, grouped_labels))) else: flat_case_labels = None new_rows = np.array(qx_list).tolist() new_cols = list(range(len(cfg_list))) return new_rows, new_cols, flat_case_labels if view_differ_cases: # Cases that passed on config but failed another case_pos_list, case_labels_list = testres.case_type_sample( 1, with_success=True, min_success_diff=1) new_rows, new_cols, flat_case_labels = convert_case_pos_to_cfgx( case_pos_list, case_labels_list) sel_rows.extend(new_rows) sel_cols.extend(new_cols) if view_cases: case_pos_list, case_labels_list = testres.case_type_sample(1, with_success=False) new_rows, new_cols, flat_case_labels = convert_case_pos_to_cfgx( case_pos_list, case_labels_list) sel_rows.extend(new_rows) sel_cols.extend(new_cols) if view_hard2: # TODO handle returning case_pos_list #samplekw = ut.argparse_dict(dict(per_group=5)) samplekw = ut.argparse_dict(dict(per_group=None)) case_pos_list = testres.get_case_positions(mode='failure', samplekw=samplekw) failure_qx_list = ut.unique_ordered(case_pos_list.T[0]) sel_rows.extend(np.array(failure_qx_list).tolist()) sel_cols.extend(list(range(len(cfg_list)))) if view_easy: new_hard_qx_list = testres.get_new_hard_qx_list() new_easy_qx_list = np.setdiff1d(np.arange(len(qaids)), new_hard_qx_list).tolist() sel_rows.extend(new_easy_qx_list) sel_cols.extend(list(range(len(cfg_list)))) if view_interesting: interesting_qx_list = testres.get_interesting_ranks() sel_rows.extend(interesting_qx_list) # TODO: grab the best scoring and most interesting configs if len(sel_cols) == 0: sel_cols.extend(list(range(len(cfg_list)))) if hist_sample: # Careful if there is more than one config config_rand_bin_qxs = testres.get_rank_histogram_qx_sample(size=10) sel_rows = np.hstack(ut.flatten(config_rand_bin_qxs)) # TODO: grab the best scoring and most interesting configs if len(sel_cols) == 0: sel_cols.extend(list(range(len(cfg_list)))) if filt_cfg is not None: # NEW WAY OF SAMPLING verbose = kwargs.get('verbose', None) case_pos_list = testres.case_sample2(filt_cfg, verbose=verbose) new_rows, new_cols, flat_case_labels = convert_case_pos_to_cfgx(case_pos_list, None) sel_rows.extend(new_rows) sel_cols.extend(new_cols) pass sel_rows = ut.unique_ordered(sel_rows) sel_cols = ut.unique_ordered(sel_cols) sel_cols = list(sel_cols) sel_rows = list(sel_rows) sel_rowxs = ut.get_argval('-r', type_=list, default=None) sel_colxs = ut.get_argval('-c', type_=list, default=None) if sel_rowxs is not None: sel_rows = ut.take(sel_rows, sel_rowxs) print('sel_rows = %r' % (sel_rows,)) if sel_colxs is not None: sel_cols = ut.take(sel_cols, sel_colxs) if ut.NOT_QUIET: print('Returning Case Selection') print('len(sel_rows) = %r/%r' % (len(sel_rows), len(qaids))) print('len(sel_cols) = %r/%r' % (len(sel_cols), len(cfg_list))) return sel_rows, sel_cols, flat_case_labels