def execute_and_save(qreq_miss): # Iterate over vsone queries in chunks. total_chunks = ut.get_num_chunks(len(qreq_miss.qaids), qreq_miss.chunksize) qaid_chunk_iter = ut.ichunks(qreq_miss.qaids, qreq_miss.chunksize) _prog = ut.ProgPartial( length=total_chunks, freq=1, label='[mc5] query chunk: ', prog_hook=qreq_miss.prog_hook, bs=False, ) qaid_chunk_iter = iter(_prog(qaid_chunk_iter)) qaid_to_cm = {} for qaids in qaid_chunk_iter: sub_qreq = qreq_miss.shallowcopy(qaids=qaids) cm_batch = sub_qreq.execute_pipeline() assert len(cm_batch) == len(qaids), 'bad alignment' assert all([qaid == cm.qaid for qaid, cm in zip(qaids, cm_batch)]) # TODO: we already computed the fpaths # should be able to pass them in fpath_list = sub_qreq.get_chipmatch_fpaths(qaids) _prog = ut.ProgPartial( length=len(cm_batch), adjust=True, freq=1, label='saving chip matches', bs=True, ) for cm, fpath in _prog(zip(cm_batch, fpath_list)): cm.save_to_fpath(fpath, verbose=False) qaid_to_cm.update({cm.qaid: cm for cm in cm_batch}) return qaid_to_cm
def compute_gammas(inva, alpha, thresh): """ Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.smk.inverted_index import * # NOQA >>> qreq_, inva = testdata_inva() >>> inva.wx_to_weight = inva.compute_word_weights('uniform') >>> alpha = 3.0 >>> thresh = 0.0 >>> gamma_list = inva.compute_gammas(alpha, thresh) """ # TODO: sep wx_to_weight = inva.wx_to_weight _prog = ut.ProgPartial(length=len(inva.wx_lists), bs=True, lbl='gamma', adjust=True) _iter = zip(inva.wx_lists, inva.agg_rvecs, inva.agg_flags) gamma_list = [] for wx_list, phiX_list, flagsX_list in _prog(_iter): if inva.int_rvec: phiX_list = smk_funcs.uncast_residual_integer(phiX_list) weight_list = np.array(ut.take(wx_to_weight, wx_list)) gammaX = smk_funcs.gamma_agg(phiX_list, flagsX_list, weight_list, alpha, thresh) gamma_list.append(gammaX) return gamma_list
def get_patches(inva, wx, ibs, verbose=True): """ Loads the patches assigned to a particular word in this stack >>> inva.wx_to_aids = inva.compute_inverted_list() >>> verbose=True """ config = inva.config aid_list = inva.wx_to_aids[wx] X_list = [inva.get_annot(aid) for aid in aid_list] fxs_groups = [X.fxs(wx) for X in X_list] all_kpts_list = ibs.depc.d.get_feat_kpts(aid_list, config=config) sub_kpts_list = vt.ziptake(all_kpts_list, fxs_groups, axis=0) total_patches = sum(ut.lmap(len, fxs_groups)) chip_list = ibs.depc_annot.d.get_chips_img(aid_list, config=config) # convert to approprate colorspace #if colorspace is not None: # chip_list = vt.convert_image_list_colorspace(chip_list, colorspace) # ut.print_object_size(chip_list, 'chip_list') patch_size = 64 shape = (total_patches, patch_size, patch_size, 3) _prog = ut.ProgPartial(enabled=verbose, lbl='warping patches', bs=True) _patchiter = ut.iflatten([ vt.get_warped_patches(chip, kpts, patch_size=patch_size)[0] #vt.get_warped_patches(chip, kpts, patch_size=patch_size, use_cpp=True)[0] for chip, kpts in _prog(zip(chip_list, sub_kpts_list), length=len(aid_list)) ]) word_patches = vt.fromiter_nd(_patchiter, shape, dtype=np.uint8) return word_patches
def predict_matches(smk, qreq_, verbose=True): """ >>> from wbia.algo.smk.smk_pipeline import * # NOQA >>> ibs, smk, qreq_ = testdata_smk() >>> verbose = True """ logger.info('Predicting matches') # assert qreq_.qinva.vocab is qreq_.dinva.vocab # X_list = qreq_.qinva.inverted_annots(qreq_.qaids) # Y_list = qreq_.dinva.inverted_annots(qreq_.daids) # verbose = 2 _prog = ut.ProgPartial(lbl='smk query', bs=verbose <= 1, enabled=verbose) daids = np.array(qreq_.daids) cm_list = [ smk.match_single(qaid, daids, qreq_, verbose=verbose > 1) for qaid in _prog(qreq_.qaids) ] return cm_list
def match_single(smk, qaid, daids, qreq_, verbose=True): """ CommandLine: python -m wbia.algo.smk.smk_pipeline SMK.match_single --profile python -m wbia.algo.smk.smk_pipeline SMK.match_single --show python -m wbia SMK.match_single -a ctrl:qmingt=2 --profile --db PZ_Master1 python -m wbia SMK.match_single -a ctrl --profile --db GZ_ALL Example: >>> # FUTURE_ENABLE >>> from wbia.algo.smk.smk_pipeline import * # NOQA >>> import wbia >>> qreq_ = wbia.testdata_qreq_(defaultdb='PZ_MTEST') >>> ibs = qreq_.ibs >>> daids = qreq_.daids >>> #ibs, daids = wbia.testdata_aids(defaultdb='PZ_MTEST', default_set='dcfg') >>> qreq_ = SMKRequest(ibs, daids[0:1], daids, {'agg': True, >>> 'num_words': 1000, >>> 'sv_on': True}) >>> qreq_.ensure_data() >>> qaid = qreq_.qaids[0] >>> daids = qreq_.daids >>> daid = daids[1] >>> verbose = True >>> cm = qreq_.smk.match_single(qaid, daids, qreq_) >>> ut.quit_if_noshow() >>> ut.qtensure() >>> cm.ishow_analysis(qreq_) >>> ut.show_if_requested() """ from wbia.algo.hots import chip_match from wbia.algo.hots import pipeline alpha = qreq_.qparams['smk_alpha'] thresh = qreq_.qparams['smk_thresh'] agg = qreq_.qparams['agg'] # nAnnotPerName = qreq_.qparams.nAnnotPerNameSVER sv_on = qreq_.qparams.sv_on if sv_on: nNameShortList = qreq_.qparams.nNameShortlistSVER shortsize = nNameShortList else: shortsize = None X = qreq_.qinva.get_annot(qaid) # Determine which database annotations need to be checked # with ut.Timer('searching qaid=%r' % (qaid,), verbose=verbose): hit_inva_wxs = ut.take(qreq_.dinva.wx_to_aids, X.wx_list) hit_daids = np.array(list(set(ut.iflatten(hit_inva_wxs)))) # Mark impossible daids # with ut.Timer('checking impossible daids=%r' % (qaid,), verbose=verbose): valid_flags = check_can_match(qaid, hit_daids, qreq_) valid_daids = hit_daids.compress(valid_flags) shortlist = ut.Shortlist(shortsize) # gammaX = smk.gamma(X, wx_to_weight, agg, alpha, thresh) _prog = ut.ProgPartial(lbl='smk scoring qaid=%r' % (qaid, ), enabled=verbose, bs=True, adjust=True) wx_to_weight = qreq_.dinva.wx_to_weight debug = False if debug: qnid = qreq_.get_qreq_annot_nids([qaid])[0] daids = np.array(qreq_.daids) dnids = qreq_.get_qreq_annot_nids(daids) correct_aids = daids[np.where(dnids == qnid)[0]] daid = correct_aids[0] if agg: for daid in _prog(valid_daids): Y = qreq_.dinva.get_annot(daid) item = match_kernel_agg(X, Y, wx_to_weight, alpha, thresh) shortlist.insert(item) else: for daid in _prog(valid_daids): Y = qreq_.dinva.get_annot(daid) item = match_kernel_sep(X, Y, wx_to_weight, alpha, thresh) shortlist.insert(item) # Build chipmatches for the shortlist results # with ut.Timer('build cms', verbose=verbose): cm = chip_match.ChipMatch(qaid=qaid, fsv_col_lbls=['smk']) cm.daid_list = [] cm.fm_list = [] cm.fsv_list = [] _prog = ut.ProgPartial(lbl='smk build cm qaid=%r' % (qaid, ), enabled=verbose, bs=True, adjust=True) for item in _prog(shortlist): (score, score_list, Y, X_idx, Y_idx) = item X_fxs = ut.take(X.fxs_list, X_idx) Y_fxs = ut.take(Y.fxs_list, Y_idx) # Only build matches for those that sver will use if agg: X_maws = ut.take(X.maws_list, X_idx) Y_maws = ut.take(Y.maws_list, Y_idx) fm, fs = smk_funcs.build_matches_agg(X_fxs, Y_fxs, X_maws, Y_maws, score_list) else: fm, fs = smk_funcs.build_matches_sep(X_fxs, Y_fxs, score_list) if len(fm) > 0: # assert not np.any(np.isnan(fs)) daid = Y.aid fsv = fs[:, None] cm.daid_list.append(daid) cm.fm_list.append(fm) cm.fsv_list.append(fsv) cm._update_daid_index() cm.arraycast_self() cm.score_name_maxcsum(qreq_) # if False: # cm.assert_self(qreq_=qreq_, verbose=True) if sv_on: cm = pipeline.sver_single_chipmatch(qreq_, cm, verbose=verbose) cm.score_name_maxcsum(qreq_) return cm
def run_asmk_script(): with ut.embed_on_exception_context: # NOQA """ >>> from wbia.algo.smk.script_smk import * """ # NOQA # ============================================== # PREPROCESSING CONFIGURATION # ============================================== config = { # 'data_year': 2013, 'data_year': None, 'dtype': 'float32', # 'root_sift': True, 'root_sift': False, # 'centering': True, 'centering': False, 'num_words': 2**16, # 'num_words': 1E6 # 'num_words': 8000, 'kmeans_impl': 'sklearn.mini', 'extern_words': False, 'extern_assign': False, 'assign_algo': 'kdtree', 'checks': 1024, 'int_rvec': True, 'only_xy': False, } # Define which params are relevant for which operations relevance = {} relevance['feats'] = ['dtype', 'root_sift', 'centering', 'data_year'] relevance['words'] = relevance['feats'] + [ 'num_words', 'extern_words', 'kmeans_impl', ] relevance['assign'] = relevance['words'] + [ 'checks', 'extern_assign', 'assign_algo', ] # relevance['ydata'] = relevance['assign'] + ['int_rvec'] # relevance['xdata'] = relevance['assign'] + ['only_xy', 'int_rvec'] nAssign = 1 class SMKCacher(ut.Cacher): def __init__(self, fname, ext='.cPkl'): relevant_params = relevance[fname] relevant_cfg = ut.dict_subset(config, relevant_params) cfgstr = ut.get_cfg_lbl(relevant_cfg) dbdir = ut.truepath('/raid/work/Oxford/') super(SMKCacher, self).__init__(fname, cfgstr, cache_dir=dbdir, ext=ext) # ============================================== # LOAD DATASET, EXTRACT AND POSTPROCESS FEATURES # ============================================== if config['data_year'] == 2007: data = load_oxford_2007() elif config['data_year'] == 2013: data = load_oxford_2013() elif config['data_year'] is None: data = load_oxford_wbia() offset_list = data['offset_list'] all_kpts = data['all_kpts'] raw_vecs = data['all_vecs'] query_uri_order = data['query_uri_order'] data_uri_order = data['data_uri_order'] # del data # ================ # PRE-PROCESS # ================ import vtool as vt # Alias names to avoid errors in interactive sessions proc_vecs = raw_vecs del raw_vecs feats_cacher = SMKCacher('feats', ext='.npy') all_vecs = feats_cacher.tryload() if all_vecs is None: if config['dtype'] == 'float32': logger.info('Converting vecs to float32') proc_vecs = proc_vecs.astype(np.float32) else: proc_vecs = proc_vecs raise NotImplementedError('other dtype') if config['root_sift']: with ut.Timer('Apply root sift'): np.sqrt(proc_vecs, out=proc_vecs) vt.normalize(proc_vecs, ord=2, axis=1, out=proc_vecs) if config['centering']: with ut.Timer('Apply centering'): mean_vec = np.mean(proc_vecs, axis=0) # Center and then re-normalize np.subtract(proc_vecs, mean_vec[None, :], out=proc_vecs) vt.normalize(proc_vecs, ord=2, axis=1, out=proc_vecs) if config['dtype'] == 'int8': smk_funcs all_vecs = proc_vecs feats_cacher.save(all_vecs) del proc_vecs # ===================================== # BUILD VISUAL VOCABULARY # ===================================== if config['extern_words']: words = data['words'] assert config['num_words'] is None or len( words) == config['num_words'] else: word_cacher = SMKCacher('words') words = word_cacher.tryload() if words is None: with ut.embed_on_exception_context: if config['kmeans_impl'] == 'sklearn.mini': import sklearn.cluster rng = np.random.RandomState(13421421) # init_size = int(config['num_words'] * 8) init_size = int(config['num_words'] * 4) # converged after 26043 iterations clusterer = sklearn.cluster.MiniBatchKMeans( config['num_words'], init_size=init_size, batch_size=1000, compute_labels=False, max_iter=20, random_state=rng, n_init=1, verbose=1, ) clusterer.fit(all_vecs) words = clusterer.cluster_centers_ elif config['kmeans_impl'] == 'yael': from yael import ynumpy centroids, qerr, dis, assign, nassign = ynumpy.kmeans( all_vecs, config['num_words'], init='kmeans++', verbose=True, output='all', ) words = centroids word_cacher.save(words) # ===================================== # ASSIGN EACH VECTOR TO ITS NEAREST WORD # ===================================== if config['extern_assign']: assert config[ 'extern_words'], 'need extern cluster to extern assign' idx_to_wxs = vt.atleast_nd(data['idx_to_wx'], 2) idx_to_maws = np.ones(idx_to_wxs.shape, dtype=np.float32) idx_to_wxs = np.ma.array(idx_to_wxs) idx_to_maws = np.ma.array(idx_to_maws) else: from wbia.algo.smk import vocab_indexer vocab = vocab_indexer.VisualVocab(words) dassign_cacher = SMKCacher('assign') assign_tup = dassign_cacher.tryload() if assign_tup is None: vocab.flann_params['algorithm'] = config['assign_algo'] vocab.build() # Takes 12 minutes to assign jegous vecs to 2**16 vocab with ut.Timer('assign vocab neighbors'): _idx_to_wx, _idx_to_wdist = vocab.nn_index( all_vecs, nAssign, checks=config['checks']) if nAssign > 1: idx_to_wxs, idx_to_maws = smk_funcs.weight_multi_assigns( _idx_to_wx, _idx_to_wdist, massign_alpha=1.2, massign_sigma=80.0, massign_equal_weights=True, ) else: idx_to_wxs = np.ma.masked_array(_idx_to_wx, fill_value=-1) idx_to_maws = np.ma.ones(idx_to_wxs.shape, fill_value=-1, dtype=np.float32) idx_to_maws.mask = idx_to_wxs.mask assign_tup = (idx_to_wxs, idx_to_maws) dassign_cacher.save(assign_tup) idx_to_wxs, idx_to_maws = assign_tup # Breakup vectors, keypoints, and word assignments by annotation wx_lists = [ idx_to_wxs[left:right] for left, right in ut.itertwo(offset_list) ] maw_lists = [ idx_to_maws[left:right] for left, right in ut.itertwo(offset_list) ] vecs_list = [ all_vecs[left:right] for left, right in ut.itertwo(offset_list) ] kpts_list = [ all_kpts[left:right] for left, right in ut.itertwo(offset_list) ] # ======================= # FIND QUERY SUBREGIONS # ======================= ibs, query_annots, data_annots, qx_to_dx = load_ordered_annots( data_uri_order, query_uri_order) daids = data_annots.aids qaids = query_annots.aids query_super_kpts = ut.take(kpts_list, qx_to_dx) query_super_vecs = ut.take(vecs_list, qx_to_dx) query_super_wxs = ut.take(wx_lists, qx_to_dx) query_super_maws = ut.take(maw_lists, qx_to_dx) # Mark which keypoints are within the bbox of the query query_flags_list = [] only_xy = config['only_xy'] for kpts_, bbox in zip(query_super_kpts, query_annots.bboxes): flags = kpts_inside_bbox(kpts_, bbox, only_xy=only_xy) query_flags_list.append(flags) logger.info('Queries are crops of existing database images.') logger.info('Looking at average percents') percent_list = [ flags_.sum() / flags_.shape[0] for flags_ in query_flags_list ] percent_stats = ut.get_stats(percent_list) logger.info('percent_stats = %s' % (ut.repr4(percent_stats), )) import vtool as vt query_kpts = vt.zipcompress(query_super_kpts, query_flags_list, axis=0) query_vecs = vt.zipcompress(query_super_vecs, query_flags_list, axis=0) query_wxs = vt.zipcompress(query_super_wxs, query_flags_list, axis=0) query_maws = vt.zipcompress(query_super_maws, query_flags_list, axis=0) # ======================= # CONSTRUCT QUERY / DATABASE REPR # ======================= # int_rvec = not config['dtype'].startswith('float') int_rvec = config['int_rvec'] X_list = [] _prog = ut.ProgPartial(length=len(qaids), label='new X', bs=True, adjust=True) for aid, fx_to_wxs, fx_to_maws in _prog( zip(qaids, query_wxs, query_maws)): X = new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec) X_list.append(X) # ydata_cacher = SMKCacher('ydata') # Y_list = ydata_cacher.tryload() # if Y_list is None: Y_list = [] _prog = ut.ProgPartial(length=len(daids), label='new Y', bs=True, adjust=True) for aid, fx_to_wxs, fx_to_maws in _prog(zip(daids, wx_lists, maw_lists)): Y = new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec) Y_list.append(Y) # ydata_cacher.save(Y_list) # ====================== # Add in some groundtruth logger.info('Add in some groundtruth') for Y, nid in zip(Y_list, ibs.get_annot_nids(daids)): Y.nid = nid for X, nid in zip(X_list, ibs.get_annot_nids(qaids)): X.nid = nid for Y, qual in zip(Y_list, ibs.get_annot_quality_texts(daids)): Y.qual = qual # ====================== # Add in other properties for Y, vecs, kpts in zip(Y_list, vecs_list, kpts_list): Y.vecs = vecs Y.kpts = kpts imgdir = ut.truepath('/raid/work/Oxford/oxbuild_images') for Y, imgid in zip(Y_list, data_uri_order): gpath = ut.unixjoin(imgdir, imgid + '.jpg') Y.gpath = gpath for X, vecs, kpts in zip(X_list, query_vecs, query_kpts): X.kpts = kpts X.vecs = vecs # ====================== logger.info('Building inverted list') daids = [Y.aid for Y in Y_list] # wx_list = sorted(ut.list_union(*[Y.wx_list for Y in Y_list])) wx_list = sorted(set.union(*[Y.wx_set for Y in Y_list])) assert daids == data_annots.aids assert len(wx_list) <= config['num_words'] wx_to_aids = smk_funcs.invert_lists(daids, [Y.wx_list for Y in Y_list], all_wxs=wx_list) # Compute IDF weights logger.info('Compute IDF weights') ndocs_total = len(daids) # Use only the unique number of words ndocs_per_word = np.array([len(set(wx_to_aids[wx])) for wx in wx_list]) logger.info('ndocs_perword stats: ' + ut.repr4(ut.get_stats(ndocs_per_word))) idf_per_word = smk_funcs.inv_doc_freq(ndocs_total, ndocs_per_word) wx_to_weight = dict(zip(wx_list, idf_per_word)) logger.info('idf stats: ' + ut.repr4(ut.get_stats(wx_to_weight.values()))) # Filter junk Y_list_ = [Y for Y in Y_list if Y.qual != 'junk'] # ======================= # CHOOSE QUERY KERNEL # ======================= params = { 'asmk': dict(alpha=3.0, thresh=0.0), 'bow': dict(), 'bow2': dict(), } # method = 'bow' method = 'bow2' method = 'asmk' smk = SMK(wx_to_weight, method=method, **params[method]) # Specific info for the type of query if method == 'asmk': # Make residual vectors if True: # The stacked way is 50x faster # TODO: extend for multi-assignment and record fxs flat_query_vecs = np.vstack(query_vecs) flat_query_wxs = np.vstack(query_wxs) flat_query_offsets = np.array( [0] + ut.cumsum(ut.lmap(len, query_wxs))) flat_wxs_assign = flat_query_wxs flat_offsets = flat_query_offsets flat_vecs = flat_query_vecs tup = smk_funcs.compute_stacked_agg_rvecs( words, flat_wxs_assign, flat_vecs, flat_offsets) all_agg_vecs, all_error_flags, agg_offset_list = tup if int_rvec: all_agg_vecs = smk_funcs.cast_residual_integer( all_agg_vecs) agg_rvecs_list = [ all_agg_vecs[left:right] for left, right in ut.itertwo(agg_offset_list) ] agg_flags_list = [ all_error_flags[left:right] for left, right in ut.itertwo(agg_offset_list) ] for X, agg_rvecs, agg_flags in zip(X_list, agg_rvecs_list, agg_flags_list): X.agg_rvecs = agg_rvecs X.agg_flags = agg_flags[:, None] flat_wxs_assign = idx_to_wxs flat_offsets = offset_list flat_vecs = all_vecs tup = smk_funcs.compute_stacked_agg_rvecs( words, flat_wxs_assign, flat_vecs, flat_offsets) all_agg_vecs, all_error_flags, agg_offset_list = tup if int_rvec: all_agg_vecs = smk_funcs.cast_residual_integer( all_agg_vecs) agg_rvecs_list = [ all_agg_vecs[left:right] for left, right in ut.itertwo(agg_offset_list) ] agg_flags_list = [ all_error_flags[left:right] for left, right in ut.itertwo(agg_offset_list) ] for Y, agg_rvecs, agg_flags in zip(Y_list, agg_rvecs_list, agg_flags_list): Y.agg_rvecs = agg_rvecs Y.agg_flags = agg_flags[:, None] else: # This non-stacked way is about 500x slower _prog = ut.ProgPartial(label='agg Y rvecs', bs=True, adjust=True) for Y in _prog(Y_list_): make_agg_vecs(Y, words, Y.vecs) _prog = ut.ProgPartial(label='agg X rvecs', bs=True, adjust=True) for X in _prog(X_list): make_agg_vecs(X, words, X.vecs) elif method == 'bow2': # Hack for orig tf-idf bow vector nwords = len(words) for X in ut.ProgIter(X_list, label='make bow vector'): ensure_tf(X) bow_vector(X, wx_to_weight, nwords) for Y in ut.ProgIter(Y_list_, label='make bow vector'): ensure_tf(Y) bow_vector(Y, wx_to_weight, nwords) if method != 'bow2': for X in ut.ProgIter(X_list, 'compute X gamma'): X.gamma = smk.gamma(X) for Y in ut.ProgIter(Y_list_, 'compute Y gamma'): Y.gamma = smk.gamma(Y) # Execute matches (could go faster by enumerating candidates) scores_list = [] for X in ut.ProgIter(X_list, label='query %s' % (smk, )): scores = [smk.kernel(X, Y) for Y in Y_list_] scores = np.array(scores) scores = np.nan_to_num(scores) scores_list.append(scores) import sklearn.metrics avep_list = [] _iter = list(zip(scores_list, X_list)) _iter = ut.ProgIter(_iter, label='evaluate %s' % (smk, )) for scores, X in _iter: truth = [X.nid == Y.nid for Y in Y_list_] avep = sklearn.metrics.average_precision_score(truth, scores) avep_list.append(avep) avep_list = np.array(avep_list) mAP = np.mean(avep_list) logger.info('mAP = %r' % (mAP, ))
def run_expt( ibs, acfg_name_list, test_cfg_name_list, use_cache=None, qaid_override=None, daid_override=None, initial_aids=None, ): r""" Loops over annot configs. Try and use this function as a starting point to clean up this module. The code is getting too untenable. CommandLine: python -m wbia.expt.harness run_expt --acfginfo python -m wbia.expt.harness run_expt --pcfginfo python -m wbia.expt.harness run_expt Ignore: test_cfg_name_list = [p] Example: >>> # SLOW_DOCTEST >>> from wbia.expt.harness import * # NOQA >>> import wbia >>> ibs = wbia.opendb(defaultdb='PZ_MTEST') >>> default_acfgstrs = ['ctrl:qsize=20,dpername=1,dsize=10', >>> 'ctrl:qsize=20,dpername=10,dsize=20'] >>> acfg_name_list = default_acfgstrs >>> test_cfg_name_list = ['default:proot=smk', 'default'] >>> #test_cfg_name_list = ['custom', 'custom:fg_on=False'] >>> use_cache = False >>> testres_list = run_expt(ibs, acfg_name_list, test_cfg_name_list, use_cache) """ logger.info('[harn] run_expt') # Generate list of database annotation configurations if len(acfg_name_list) == 0: raise ValueError('must give acfg name list') acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list( ibs, acfg_name_list, qaid_override=qaid_override, daid_override=daid_override, initial_aids=initial_aids, use_cache=use_cache, ) # Generate list of query pipeline param configs cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list( test_cfg_name_list, ibs=ibs) cfgx2_lbl = experiment_helpers.get_varied_pipecfg_lbls(cfgdict_list) # NOTE: Can specify --pcfginfo or --acfginfo if ut.NOT_QUIET: ut.colorprint( textwrap.dedent(""" [harn]================ [harn] harness.test_configurations2()""").strip(), 'white', ) msg = '[harn] Running %s using %s and %s' % ( ut.quantstr('test', len(acfg_list) * len(cfgdict_list)), ut.quantstr('pipeline config', len(cfgdict_list)), ut.quantstr('annot config', len(acfg_list)), ) ut.colorprint(msg, 'white') testres_list = [] nAcfg = len(acfg_list) testnameid = ibs.get_dbname() + ' ' + str(test_cfg_name_list) + str( acfg_name_list) lbl = '[harn] TEST_CFG ' + str(test_cfg_name_list) + str(acfg_name_list) expanded_aids_iter = ut.ProgIter( expanded_aids_list, lbl='annot config', freq=1, autoadjust=False, enabled=ut.NOT_QUIET, ) for acfgx, (qaids, daids) in enumerate(expanded_aids_iter): assert len(qaids) != 0, '[harness] No query annots specified' assert len(daids) != 0, '[harness] No database annotas specified' acfg = acfg_list[acfgx] if ut.NOT_QUIET: ut.colorprint('\n---Annot config testnameid=%r' % (testnameid, ), 'brightcyan') subindexer_partial = ut.ProgPartial(parent_index=acfgx, parent_length=nAcfg, enabled=ut.NOT_QUIET) testres_ = make_single_testres( ibs, qaids, daids, pipecfg_list, cfgx2_lbl, cfgdict_list, lbl, testnameid, use_cache=use_cache, subindexer_partial=subindexer_partial, ) if DRY_RUN: continue testres_.acfg = acfg testres_.test_cfg_name_list = test_cfg_name_list testres_list.append(testres_) if DRY_RUN: logger.info('DRYRUN: Cannot continue past run_expt') sys.exit(0) testres = test_result.combine_testres_list(ibs, testres_list) # testres.print_results() logger.info('Returning Test Result') return testres