def possible_mana_combinations(land_list, deck=None):
"""
CommandLine:
python -m mtgmonte.mtgutils --test-possible_mana_combinations
Example:
>>> # ENABLE_DOCTEST
>>> from mtgmonte.mtgutils import * # NOQA
>>> from mtgmonte import mtgobjs
>>> deck = mtgobjs.Deck(mtgobjs.load_cards(['Tropical Island', 'Sunken Hollow', 'Island']))
>>> land_list = mtgobjs.load_cards(['Ancient Tomb', 'Island', 'Flooded Strand', 'Flooded Strand', 'Shivan Reef'])
>>> card = land_list[-1]
>>> mana_combos = possible_mana_combinations(land_list, deck)
>>> result = (ut.repr2(mana_combos, nl=1, strvals=True, nobraces=True))
>>> print(result)
({CC}, {U}, {G}, {U}, {C}),
({CC}, {U}, {G}, {B}, {C}),
({CC}, {U}, {U}, {U}, {C}),
({CC}, {U}, {U}, {B}, {C}),
({CC}, {U}, {G}, {U}, {R}),
({CC}, {U}, {G}, {B}, {R}),
({CC}, {U}, {U}, {U}, {R}),
({CC}, {U}, {U}, {B}, {R}),
"""
from mtgmonte import mtgobjs
avail_mana = [land.mana_potential2(deck=deck, recurse=False) for land in land_list]
avail_mana = filter(len, avail_mana)
mana_combos1 = list(ut.iprod(*avail_mana))
# Encode the idea that two fetches cant fetch the same land
non_class1 = [[c for c in co if not isinstance(c, six.string_types)] for co in mana_combos1]
flags = [len(co) == 0 or len(set(co)) == len(co) for co in non_class1]
mana_combos2 = ut.compress(mana_combos1, flags)
mana_combos3 = [
[[c] if isinstance(c, mtgobjs.ManaSet) else c.mana_potential2(deck=deck) for c in co] for co in mana_combos2
]
unflat_combos3 = [list(ut.iprod(*co)) for co in mana_combos3]
mana_combos4 = ut.flatten(unflat_combos3)
# mana_combos4 = [reduce(operator.add, m) for m in mana_combos4]
# z = reduce(operator.add, m)
# import utool
# utool.embed()
# avail_mana = [land.mana_potential(deck=deck) for land in land_list]
# avail_mana = filter(len, avail_mana)
# mana_combos4 = list(ut.iprod(*avail_mana))
combo_ids = [tuple(sorted(x)) for x in mana_combos4]
flags = ut.flag_unique_items(combo_ids)
mana_combos = ut.compress(mana_combos4, flags)
# mana_combos = list(map(tuple, [''.join(c) for c in mana_combos]))
return mana_combos
def postingest_tesdb1_func(ibs):
print('postingest_tesdb1_func')
# Adjust data as we see fit
import numpy as np
gid_list = np.array(ibs.get_valid_gids())
unixtimes_even = (gid_list[0::2] + 100).tolist()
unixtimes_odd = (gid_list[1::2] + 9001).tolist()
unixtime_list = unixtimes_even + unixtimes_odd
ibs.set_image_unixtime(gid_list, unixtime_list)
# Unname first aid in every name
aid_list = ibs.get_valid_aids()
nid_list = ibs.get_annot_nids(aid_list)
nid_list = [ (nid if nid > 0 else None) for nid in nid_list]
unique_flag = utool.flag_unique_items(nid_list)
unique_nids = utool.filter_items(nid_list, unique_flag)
none_nids = [ nid is not None for nid in nid_list]
flagged_nids = [nid for nid in unique_nids if nid_list.count(nid) > 1]
plural_flag = [nid in flagged_nids for nid in nid_list]
flag_list = map(all, izip(plural_flag, unique_flag, none_nids))
flagged_aids = utool.filter_items(aid_list, flag_list)
if utool.VERYVERBOSE:
def print2(*args):
print('[post_testdb1] ' + ', '.join(args))
print2('aid_list=%r' % aid_list)
print2('nid_list=%r' % nid_list)
print2('unique_flag=%r' % unique_flag)
print2('plural_flag=%r' % plural_flag)
print2('unique_nids=%r' % unique_nids)
print2('none_nids=%r' % none_nids)
print2('flag_list=%r' % flag_list)
print2('flagged_nids=%r' % flagged_nids)
print2('flagged_aids=%r' % flagged_aids)
# print2('new_nids=%r' % new_nids)
# Unname, some annotations for testing
delete_aids = utool.filter_items(aid_list, flag_list)
ibs.delete_annot_nids(delete_aids)
# Add all annotations with names as exemplars
from ibeis.control.IBEISControl import IBEISController
assert isinstance(ibs, IBEISController)
unflagged_aids = utool.get_dirty_items(aid_list, flag_list)
exemplar_flags = [True] * len(unflagged_aids)
ibs.set_annot_exemplar_flag(unflagged_aids, exemplar_flags)
return None
def add_cleanly(db, tblname, colnames, params_iter, get_rowid_from_superkey, superkey_paramx=(0,)):
""" ADDER Extra input:
the first item of params_iter must be a superkey (like a uuid), """
# ADD_CLEANLY_1: PREPROCESS INPUT
params_list = list(params_iter) # eagerly evaluate for superkeys
# Extract superkeys from the params list (requires eager eval)
superkey_lists = [[None if params is None else params[x]
for params in params_list]
for x in superkey_paramx]
# ADD_CLEANLY_2: PREFORM INPUT CHECKS
# check which parameters are valid
isvalid_list = [params is not None for params in params_list]
# Check for duplicate inputs
isunique_list = utool.flag_unique_items(list(zip(*superkey_lists)))
# Check to see if this already exists in the database
rowid_list_ = get_rowid_from_superkey(*superkey_lists)
isnew_list = [rowid is None for rowid in rowid_list_]
if VERBOSE and not all(isunique_list):
print('[WARNING]: duplicate inputs to db.add_cleanly')
# Flag each item that needs to added to the database
isdirty_list = list(map(all, zip(isvalid_list, isunique_list, isnew_list)))
# ADD_CLEANLY_3.1: EXIT IF CLEAN
if not any(isdirty_list):
return rowid_list_ # There is nothing to add. Return the rowids
# ADD_CLEANLY_3.2: PERFORM DIRTY ADDITIONS
dirty_params = utool.filter_items(params_list, isdirty_list)
if utool.VERBOSE:
print('[sql] adding %r/%r new %s' % (len(dirty_params), len(params_list), tblname))
# Add any unadded parameters to the database
try:
db._add(tblname, colnames, dirty_params)
except Exception as ex:
utool.printex(ex, key_list=['isdirty_list', 'superkey_lists', 'rowid_list_'])
raise
# TODO: We should only have to preform a subset of adds here
# (at the positions where rowid_list was None in the getter check)
rowid_list = get_rowid_from_superkey(*superkey_lists)
# ADD_CLEANLY_4: SANITY CHECK AND RETURN
assert len(rowid_list) == len(params_list), 'failed sanity check'
return rowid_list
def show_qres_analysis(ibs, cm, qreq_=None, **kwargs):
"""
Wrapper around show_qres.
KWARGS:
aid_list - show matches against aid_list (default top 3)
Args:
ibs (IBEISController): ibeis controller object
cm (ChipMatch): object of feature correspondences and scores
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
N, show_gt, show_query, aid_list, figtitle, viz_name_score, viz_name_score
CommandLine:
python -m ibeis.viz.viz_qres --exec-show_qres_analysis --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.viz.viz_qres import * # NOQA
>>> import ibeis
>>> species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> daids = ibs.get_valid_aids(species=species)
>>> qaids = ibs.get_valid_aids(species=species)
>>> qres_list, qreq_ = ibs.query_chips(
>>> [1], [2, 3, 4, 5, 6, 7, 8, 9],
>>> cfgdict=dict(), return_request=True)
>>> cm = qres_list[0]
>>> kwargs = dict(show_query=False, viz_name_score=True,
>>> show_timedelta=True, N=3, show_gf=True)
>>> show_qres_analysis(ibs, cm, qreq_, **kwargs)
>>> ut.show_if_requested()
"""
if ut.NOT_QUIET:
print('[show_qres] cm.show_analysis()')
# Parse arguments
N = kwargs.get('N', DEFAULT_NTOP)
show_gt = kwargs.pop('show_gt', True)
show_gf = kwargs.pop('show_gf', False)
show_query = kwargs.pop('show_query', True)
aid_list = kwargs.pop('aid_list', None)
figtitle = kwargs.pop('figtitle', None)
#viz_name_score = kwargs.get('viz_name_score', qreq_ is not None)
#viz_name_score = kwargs.get('viz_name_score', False)
viz_name_score = kwargs.get('viz_name_score', True)
failed_to_match = False
# Debug printing
#print('[analysis] noshow_gt = %r' % noshow_gt)
#print('[analysis] show_query = %r' % show_query)
#print('[analysis] aid_list = %r' % aid_list)
if aid_list is None:
# Compare to aid_list instead of using top ranks
#print('[analysis] showing top aids')
top_aids = cm.get_top_aids(N)
if len(top_aids) == 0:
failed_to_match = True
print('WARNING! No matches found for this query')
if figtitle is None:
if len(top_aids) == 0:
figtitle = 'WARNING: no matches found!' + ibsfuncs.aidstr(
cm.qaid)
else:
topscore = cm.get_annot_scores(top_aids)[0]
figtitle = ('q%s -- topscore=%r' %
(ibsfuncs.aidstr(cm.qaid), topscore))
else:
print('[analysis] showing a given list of aids')
top_aids = aid_list
if figtitle is None:
figtitle = 'comparing to ' + ibsfuncs.aidstr(top_aids) + figtitle
# Get any groundtruth if you are showing it
showgt_aids = []
if show_gt:
# Get the missed groundtruth annotations
#if isinstance(cm, chip_match.ChipMatch):
assert qreq_ is not None
matchable_aids = qreq_.daids
#matchable_aids = cm.daid_list
#else:
# matchable_aids = cm.daids
#matchable_aids = ibs.get_recognition_database_aids()
#matchable_aids = list(cm.aid2_fm.keys())
_gtaids = ibs.get_annot_groundtruth(cm.qaid, daid_list=matchable_aids)
if viz_name_score:
# Only look at the groundtruth if a name isnt in the top list
_gtnids = ibs.get_annot_name_rowids(_gtaids)
top_nids = ibs.get_annot_name_rowids(top_aids)
_valids = ~np.in1d(_gtnids, top_nids)
_gtaids = ut.compress(_gtaids, _valids)
# No need to display highly ranked groundtruth. It will already show up
_gtaids = np.setdiff1d(_gtaids, top_aids)
# Sort missed grountruth by score
_gtscores = cm.get_annot_scores(_gtaids)
_gtaids = ut.sortedby(_gtaids, _gtscores, reverse=True)
if viz_name_score:
if len(_gtaids) > 1:
_gtaids = _gtaids[0:1]
else:
if len(_gtaids) > 3:
# Hack to not show too many unmatched groundtruths
#_isexmp = ibs.get_annot_exemplar_flags(_gtaids)
_gtaids = _gtaids[0:3]
showgt_aids = _gtaids
if show_gf:
# Show only one top-scoring groundfalse example
top_nids = ibs.get_annot_name_rowids(top_aids)
is_groundfalse = top_nids != ibs.get_annot_name_rowids(cm.qaid)
gf_idxs = np.nonzero(is_groundfalse)[0]
if len(gf_idxs) > 0:
best_gf_idx = gf_idxs[0]
isvalid = ~is_groundfalse
isvalid[best_gf_idx] = True
# Filter so there is only one groundfalse
top_aids = top_aids.compress(isvalid)
else:
# seems like there were no results. Must be bad feature detections
# maybe too much spatial verification
top_aids = []
if len(showgt_aids) != 0:
# Hack to just include gtaids in normal list
top_aids = np.append(top_aids, showgt_aids)
showgt_aids = []
if viz_name_score:
# Make sure that there is only one of each name in the list
top_nids = ibs.get_annot_name_rowids(top_aids)
top_aids = ut.compress(top_aids, ut.flag_unique_items(top_nids))
return show_qres(ibs,
cm,
gt_aids=showgt_aids,
top_aids=top_aids,
figtitle=figtitle,
show_query=show_query,
qreq_=qreq_,
failed_to_match=failed_to_match,
**kwargs)
def get_pipecfg_list(test_cfg_name_list, ibs=None):
r"""
Builds a list of varied query configurations. Only custom configs depend on
an ibs object. The order of the output is not gaurenteed to aggree with
input order.
Args:
test_cfg_name_list (list): list of strs
ibs (IBEISController): ibeis controller object (optional)
Returns:
tuple: (cfg_list, cfgx2_lbl) -
cfg_list (list): list of config objects
cfgx2_lbl (list): denotes which parameters are being varied.
If there is just one config then nothing is varied
CommandLine:
python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:0
python -m ibeis.expt.experiment_helpers --exec-get_pipecfg_list:1 --db humpbacks
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> #test_cfg_name_list = ['best', 'custom', 'custom:sv_on=False']
>>> #test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> # execute function
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> # verify results
>>> assert pipecfg_list[0].sv_cfg.sv_on is True
>>> assert pipecfg_list[1].sv_cfg.sv_on is False
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls))
>>> print(result)
pipecfg_lbls = [
'default:',
'default:sv_on=False',
]
Example1:
>>> # DISABLE_DOCTEST
>>> import ibeis_flukematch.plugin
>>> from ibeis.expt.experiment_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='humpbacks')
>>> test_cfg_name_list = ['default:pipeline_root=BC_DTW,decision=average', 'default:K=[1,4]']
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.list_str(pipecfg_lbls))
>>> print(result)
>>> print_pipe_configs(pcfgdict_list, pipecfg_list)
"""
if ut.VERBOSE:
print('[expt_help.get_pipecfg_list] building pipecfg_list using: %s' %
test_cfg_name_list)
if isinstance(test_cfg_name_list, six.string_types):
test_cfg_name_list = [test_cfg_name_list]
_standard_cfg_names = []
_pcfgdict_list = []
# HACK: Parse out custom configs first
for test_cfg_name in test_cfg_name_list:
if test_cfg_name.startswith('custom:') or test_cfg_name == 'custom':
print('[expthelpers] Parsing nonstandard custom config')
if test_cfg_name.startswith('custom:'):
# parse out modifications to custom
cfgstr_list = ':'.join(test_cfg_name.split(':')[1:]).split(',')
augcfgdict = ut.parse_cfgstr_list(cfgstr_list, smartcast=True)
else:
augcfgdict = {}
# Take the configuration from the ibeis object
pipe_cfg = ibs.cfg.query_cfg.deepcopy()
# Update with augmented params
pipe_cfg.update_query_cfg(**augcfgdict)
# Parse out a standard cfgdict
cfgdict = dict(pipe_cfg.parse_items())
cfgdict['_cfgname'] = 'custom'
cfgdict['_cfgstr'] = test_cfg_name
_pcfgdict_list.append(cfgdict)
else:
_standard_cfg_names.append(test_cfg_name)
# Handle stanndard configs next
if len(_standard_cfg_names) > 0:
# Get parsing information
#cfg_default_dict = dict(Config.QueryConfig().parse_items())
#valid_keys = list(cfg_default_dict.keys())
cfgstr_list = _standard_cfg_names
named_defaults_dict = ut.dict_subset(
experiment_configs.__dict__, experiment_configs.TEST_NAMES)
alias_keys = experiment_configs.ALIAS_KEYS
# Parse standard pipeline cfgstrings
metadata = {'ibs': ibs}
dict_comb_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list,
named_defaults_dict,
cfgtype=None,
alias_keys=alias_keys,
# Hack out valid keys for humpbacks
#valid_keys=valid_keys,
strict=False,
metadata=metadata
)
# Get varied params (there may be duplicates)
_pcfgdict_list.extend(ut.flatten(dict_comb_list))
# TODO: respsect different algorithm parameters
# like flukes
# Expand cfgdicts into PipelineConfig config objects
if ibs is None:
configclass_list = [Config.QueryConfig] * len(_pcfgdict_list)
else:
root_to_config = ibs.depc.configclass_dict
configclass_list = [
root_to_config.get(_cfgdict.get('pipeline_root', 'vsmany'), Config.QueryConfig)
for _cfgdict in _pcfgdict_list]
_pipecfg_list = [cls(**_cfgdict) for cls, _cfgdict in zip(configclass_list, _pcfgdict_list)]
# Enforce rule that removes duplicate configs
# by using feasiblity from ibeis.algo.Config
# TODO: Move this unique finding code to its own function
# and then move it up one function level so even the custom
# configs can be uniquified
_flag_list = ut.flag_unique_items(_pipecfg_list)
cfgdict_list = ut.compress(_pcfgdict_list, _flag_list)
pipecfg_list = ut.compress(_pipecfg_list, _flag_list)
if ut.NOT_QUIET:
print('[harn.help] return %d / %d unique pipeline configs from: %r' %
(len(cfgdict_list), len(_pcfgdict_list), test_cfg_name_list))
if ut.get_argflag(('--pcfginfo', '--pinfo', '--pipecfginfo')):
import sys
ut.colorprint('Requested PcfgInfo for tests... ', 'red')
print_pipe_configs(cfgdict_list, pipecfg_list)
ut.colorprint('Finished Reporting PcfgInfo. Exiting', 'red')
sys.exit(1)
return (cfgdict_list, pipecfg_list)
def show_qres_analysis(ibs, cm, qreq_=None, **kwargs):
"""
Wrapper around show_qres.
KWARGS:
aid_list - show matches against aid_list (default top 3)
Args:
ibs (IBEISController): ibeis controller object
cm (ChipMatch): object of feature correspondences and scores
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
N, show_gt, show_query, aid_list, figtitle, viz_name_score, viz_name_score
CommandLine:
python -m ibeis.viz.viz_qres --exec-show_qres_analysis --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.viz.viz_qres import * # NOQA
>>> import ibeis
>>> species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> daids = ibs.get_valid_aids(species=species)
>>> qaids = ibs.get_valid_aids(species=species)
>>> qres_list, qreq_ = ibs.query_chips(
>>> [1], [2, 3, 4, 5, 6, 7, 8, 9],
>>> cfgdict=dict(), return_request=True)
>>> cm = qres_list[0]
>>> kwargs = dict(show_query=False, viz_name_score=True,
>>> show_timedelta=True, N=3, show_gf=True)
>>> show_qres_analysis(ibs, cm, qreq_, **kwargs)
>>> ut.show_if_requested()
"""
if ut.NOT_QUIET:
print('[show_qres] cm.show_analysis()')
# Parse arguments
N = kwargs.get('N', DEFAULT_NTOP)
show_gt = kwargs.pop('show_gt', True)
show_gf = kwargs.pop('show_gf', False)
show_query = kwargs.pop('show_query', True)
aid_list = kwargs.pop('aid_list', None)
figtitle = kwargs.pop('figtitle', None)
#viz_name_score = kwargs.get('viz_name_score', qreq_ is not None)
#viz_name_score = kwargs.get('viz_name_score', False)
viz_name_score = kwargs.get('viz_name_score', True)
failed_to_match = False
# Debug printing
#print('[analysis] noshow_gt = %r' % noshow_gt)
#print('[analysis] show_query = %r' % show_query)
#print('[analysis] aid_list = %r' % aid_list)
if aid_list is None:
# Compare to aid_list instead of using top ranks
#print('[analysis] showing top aids')
top_aids = cm.get_top_aids(N)
if len(top_aids) == 0:
failed_to_match = True
print('WARNING! No matches found for this query')
if figtitle is None:
if len(top_aids) == 0:
figtitle = 'WARNING: no matches found!' + ibsfuncs.aidstr(cm.qaid)
else:
topscore = cm.get_annot_scores(top_aids)[0]
figtitle = ('q%s -- topscore=%r' % (ibsfuncs.aidstr(cm.qaid), topscore))
else:
print('[analysis] showing a given list of aids')
top_aids = aid_list
if figtitle is None:
figtitle = 'comparing to ' + ibsfuncs.aidstr(top_aids) + figtitle
# Get any groundtruth if you are showing it
showgt_aids = []
if show_gt:
# Get the missed groundtruth annotations
#if isinstance(cm, chip_match.ChipMatch):
assert qreq_ is not None
matchable_aids = qreq_.daids
#matchable_aids = cm.daid_list
#else:
# matchable_aids = cm.daids
#matchable_aids = ibs.get_recognition_database_aids()
#matchable_aids = list(cm.aid2_fm.keys())
_gtaids = ibs.get_annot_groundtruth(cm.qaid, daid_list=matchable_aids)
if viz_name_score:
# Only look at the groundtruth if a name isnt in the top list
_gtnids = ibs.get_annot_name_rowids(_gtaids)
top_nids = ibs.get_annot_name_rowids(top_aids)
_valids = ~np.in1d(_gtnids, top_nids)
_gtaids = ut.compress(_gtaids, _valids)
# No need to display highly ranked groundtruth. It will already show up
_gtaids = np.setdiff1d(_gtaids, top_aids)
# Sort missed grountruth by score
_gtscores = cm.get_annot_scores(_gtaids)
_gtaids = ut.sortedby(_gtaids, _gtscores, reverse=True)
if viz_name_score:
if len(_gtaids) > 1:
_gtaids = _gtaids[0:1]
else:
if len(_gtaids) > 3:
# Hack to not show too many unmatched groundtruths
#_isexmp = ibs.get_annot_exemplar_flags(_gtaids)
_gtaids = _gtaids[0:3]
showgt_aids = _gtaids
if show_gf:
# Show only one top-scoring groundfalse example
top_nids = ibs.get_annot_name_rowids(top_aids)
is_groundfalse = top_nids != ibs.get_annot_name_rowids(cm.qaid)
gf_idxs = np.nonzero(is_groundfalse)[0]
if len(gf_idxs) > 0:
best_gf_idx = gf_idxs[0]
isvalid = ~is_groundfalse
isvalid[best_gf_idx] = True
# Filter so there is only one groundfalse
top_aids = top_aids.compress(isvalid)
else:
# seems like there were no results. Must be bad feature detections
# maybe too much spatial verification
top_aids = []
if len(showgt_aids) != 0:
# Hack to just include gtaids in normal list
top_aids = np.append(top_aids, showgt_aids)
showgt_aids = []
if viz_name_score:
# Make sure that there is only one of each name in the list
top_nids = ibs.get_annot_name_rowids(top_aids)
top_aids = ut.compress(top_aids, ut.flag_unique_items(top_nids))
return show_qres(ibs, cm, gt_aids=showgt_aids, top_aids=top_aids,
figtitle=figtitle, show_query=show_query, qreq_=qreq_,
failed_to_match=failed_to_match,
**kwargs)
def get_pipecfg_list(test_cfg_name_list, ibs=None, verbose=None):
r"""
Builds a list of varied query configurations. Only custom configs depend on
an ibs object. The order of the output is not gaurenteed to aggree with
input order.
FIXME:
This breaks if you proot=BC_DTW and ibs is None
Args:
test_cfg_name_list (list): list of strs
ibs (wbia.IBEISController): wbia controller object (optional)
Returns:
tuple: (cfg_list, cfgx2_lbl) -
cfg_list (list): list of config objects
cfgx2_lbl (list): denotes which parameters are being varied.
If there is just one config then nothing is varied
CommandLine:
python -m wbia get_pipecfg_list:0
python -m wbia get_pipecfg_list:1 --db humpbacks
python -m wbia get_pipecfg_list:2
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.expt.experiment_helpers import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='testdb1')
>>> #test_cfg_name_list = ['best', 'custom', 'custom:sv_on=False']
>>> #test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> # execute function
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> # verify results
>>> assert pipecfg_list[0].sv_cfg.sv_on is True
>>> assert pipecfg_list[1].sv_cfg.sv_on is False
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.repr2(pipecfg_lbls))
>>> print(result)
pipecfg_lbls = ['default:', 'default:sv_on=False']
Example1:
>>> # DISABLE_DOCTEST
>>> import wbia_flukematch.plugin
>>> from wbia.expt.experiment_helpers import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='humpbacks')
>>> test_cfg_name_list = ['default:pipeline_root=BC_DTW,decision=average,crop_dim_size=[960,500]', 'default:K=[1,4]']
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.repr2(pipecfg_lbls))
>>> print(result)
>>> print_pipe_configs(pcfgdict_list, pipecfg_list)
"""
if verbose is None:
verbose = ut.VERBOSE
if ut.VERBOSE:
logger.info(
'[expt_help.get_pipecfg_list] building pipecfg_list using: %s'
% test_cfg_name_list
)
if isinstance(test_cfg_name_list, six.string_types):
test_cfg_name_list = [test_cfg_name_list]
_standard_cfg_names = []
_pcfgdict_list = []
# HACK: Parse out custom configs first
for test_cfg_name in test_cfg_name_list:
if test_cfg_name.startswith('custom:') or test_cfg_name == 'custom':
logger.info('[expthelpers] Parsing nonstandard custom config')
assert False, 'custom is no longer supported'
# if test_cfg_name.startswith('custom:'):
# # parse out modifications to custom
# cfgstr_list = ':'.join(test_cfg_name.split(':')[1:]).split(',')
# augcfgdict = ut.parse_cfgstr_list(cfgstr_list, smartcast=True)
# else:
# augcfgdict = {}
# # Take the configuration from the wbia object
# pipe_cfg = ibs.--cfg.query_cfg.deepcopy()
# # Update with augmented params
# pipe_cfg.update_query_cfg(**augcfgdict)
# # Parse out a standard cfgdict
# cfgdict = dict(pipe_cfg.parse_items())
# cfgdict['_cfgname'] = 'custom'
# cfgdict['_cfgstr'] = test_cfg_name
# _pcfgdict_list.append(cfgdict)
else:
_standard_cfg_names.append(test_cfg_name)
# Handle stanndard configs next
if len(_standard_cfg_names) > 0:
# Get parsing information
# cfg_default_dict = dict(Config.QueryConfig().parse_items())
# valid_keys = list(cfg_default_dict.keys())
cfgstr_list = _standard_cfg_names
named_defaults_dict = ut.dict_subset(
experiment_configs.__dict__, experiment_configs.TEST_NAMES
)
alias_keys = experiment_configs.ALIAS_KEYS
# Parse standard pipeline cfgstrings
metadata = {'ibs': ibs}
dict_comb_list = cfghelpers.parse_cfgstr_list2(
cfgstr_list,
named_defaults_dict,
cfgtype=None,
alias_keys=alias_keys,
# Hack out valid keys for humpbacks
# valid_keys=valid_keys,
strict=False,
metadata=metadata,
)
# Get varied params (there may be duplicates)
_pcfgdict_list.extend(ut.flatten(dict_comb_list))
# Expand cfgdicts into PipelineConfig config objects
# TODO: respsect different algorithm parameters like flukes
if ibs is None:
configclass_list = [Config.QueryConfig] * len(_pcfgdict_list)
else:
root_to_config = ibs.depc_annot.configclass_dict.copy()
from wbia.algo.smk import smk_pipeline
root_to_config['smk'] = smk_pipeline.SMKRequestConfig
configclass_list = [
root_to_config.get(
_cfgdict.get('pipeline_root', _cfgdict.get('proot', 'vsmany')),
Config.QueryConfig,
)
for _cfgdict in _pcfgdict_list
]
_pipecfg_list = [
cls(**_cfgdict) for cls, _cfgdict in zip(configclass_list, _pcfgdict_list)
]
# Enforce rule that removes duplicate configs
# by using feasiblity from wbia.algo.Config
# TODO: Move this unique finding code to its own function
# and then move it up one function level so even the custom
# configs can be uniquified
_flag_list = ut.flag_unique_items(_pipecfg_list)
cfgdict_list = ut.compress(_pcfgdict_list, _flag_list)
pipecfg_list = ut.compress(_pipecfg_list, _flag_list)
if verbose:
# for cfg in _pipecfg_list:
# logger.info(cfg.get_cfgstr())
# logger.info(cfg)
logger.info(
'[harn.help] return %d / %d unique pipeline configs from: %r'
% (len(cfgdict_list), len(_pcfgdict_list), test_cfg_name_list)
)
if ut.get_argflag(('--pcfginfo', '--pinfo', '--pipecfginfo')):
ut.colorprint('Requested PcfgInfo for tests... ', 'red')
print_pipe_configs(cfgdict_list, pipecfg_list)
ut.colorprint('Finished Reporting PcfgInfo. Exiting', 'red')
sys.exit(0)
return (cfgdict_list, pipecfg_list)
def postingest_tesdb1_func(ibs):
import numpy as np
from ibeis import constants as const
print('postingest_tesdb1_func')
# Adjust data as we see fit
gid_list = np.array(ibs.get_valid_gids())
# Set image unixtimes
unixtimes_even = (gid_list[0::2] + 100).tolist()
unixtimes_odd = (gid_list[1::2] + 9001).tolist()
unixtime_list = unixtimes_even + unixtimes_odd
ibs.set_image_unixtime(gid_list, unixtime_list)
# Unname first aid in every name
aid_list = ibs.get_valid_aids()
nid_list = ibs.get_annot_name_rowids(aid_list)
nid_list = [ (nid if nid > 0 else None) for nid in nid_list]
unique_flag = ut.flag_unique_items(nid_list)
unique_nids = ut.compress(nid_list, unique_flag)
none_nids = [ nid is not None for nid in nid_list]
flagged_nids = [nid for nid in unique_nids if nid_list.count(nid) > 1]
plural_flag = [nid in flagged_nids for nid in nid_list]
flag_list = list(map(all, zip(plural_flag, unique_flag, none_nids)))
flagged_aids = ut.compress(aid_list, flag_list)
if ut.VERYVERBOSE:
def print2(*args):
print('[post_testdb1] ' + ', '.join(args))
print2('aid_list=%r' % aid_list)
print2('nid_list=%r' % nid_list)
print2('unique_flag=%r' % unique_flag)
print2('plural_flag=%r' % plural_flag)
print2('unique_nids=%r' % unique_nids)
print2('none_nids=%r' % none_nids)
print2('flag_list=%r' % flag_list)
print2('flagged_nids=%r' % flagged_nids)
print2('flagged_aids=%r' % flagged_aids)
# print2('new_nids=%r' % new_nids)
# Unname, some annotations for testing
unname_aids = ut.compress(aid_list, flag_list)
ibs.delete_annot_nids(unname_aids)
# Add all annotations with names as exemplars
#from ibeis.control.IBEISControl import IBEISController
#assert isinstance(ibs, IBEISController)
unflagged_aids = ut.get_dirty_items(aid_list, flag_list)
exemplar_flags = [True] * len(unflagged_aids)
ibs.set_annot_exemplar_flags(unflagged_aids, exemplar_flags)
# Set some test species labels
species_text_list = ibs.get_annot_species_texts(aid_list)
for ix in range(0, 6):
species_text_list[ix] = const.TEST_SPECIES.ZEB_PLAIN
# These are actually plains zebras.
for ix in range(8, 10):
species_text_list[ix] = const.TEST_SPECIES.ZEB_GREVY
for ix in range(10, 12):
species_text_list[ix] = const.TEST_SPECIES.BEAR_POLAR
ibs.set_annot_species(aid_list, species_text_list)
ibs.set_annot_notes(aid_list[8:10], ['this is actually a plains zebra'] * 2)
ibs.set_annot_notes(aid_list[0:1], ['aid 1 and 2 are correct matches'])
ibs.set_annot_notes(aid_list[6:7], ['very simple image to debug feature detector'])
ibs.set_annot_notes(aid_list[7:8], ['standard test image'])
# Set some randomish gps flags that are within nnp
unixtime_list = ibs.get_image_unixtime(gid_list)
valid_lat_min = -1.4446
valid_lat_max = -1.3271
valid_lon_min = 36.7619
valid_lon_max = 36.9622
valid_lat_range = valid_lat_max - valid_lat_min
valid_lon_range = valid_lon_max - valid_lon_min
randstate = np.random.RandomState(unixtime_list)
new_gps_list = randstate.rand(len(gid_list), 2)
new_gps_list[:, 0] = (new_gps_list[:, 0] * valid_lat_range) + valid_lat_min
new_gps_list[:, 1] = (new_gps_list[:, 1] * valid_lon_range) + valid_lon_min
new_gps_list[8, :] = [-1, -1]
ibs.set_image_gps(gid_list, new_gps_list)
# TODO: add a nan timestamp
ibs.append_annot_case_tags([2], ['error:bbox'])
ibs.append_annot_case_tags([4], ['quality:washedout'])
ibs.append_annot_case_tags([4], ['lighting'])
aidgroups = ibs.group_annots_by_name(
ibs.filter_annots_general(min_pername=2, verbose=True))[0]
aid1_list = ut.take_column(aidgroups, 0)
aid2_list = ut.take_column(aidgroups, 1)
annotmatch_rowids = ibs.add_annotmatch(aid1_list, aid2_list)
ibs.set_annotmatch_truth(annotmatch_rowids, [True] * len(annotmatch_rowids))
ibs.set_annotmatch_truth(annotmatch_rowids, [True] * len(annotmatch_rowids))
ibs.set_annotmatch_prop('photobomb', annotmatch_rowids, [True] * len(annotmatch_rowids))
for aids in aidgroups:
pass
return None