def _heuristic_weighting(nodes, avail_uv): avail_uv = np.array(avail_uv) weights = np.ones(len(avail_uv)) if 'view_weight' in enabled_heuristics: from vtool import _rhomb_dist view_edge = [(node_to_view[u], node_to_view[v]) for (u, v) in avail_uv] view_weight = np.array( [_rhomb_dist.VIEW_CODE_DIST[(v1, v2)] for (v1, v2) in view_edge] ) # Assume comparable by default and prefer undefined # more than probably not, but less than definately so. view_weight[np.isnan(view_weight)] = 1.5 # Prefer viewpoint 10x more than time weights += 10 * view_weight if 'time_weight' in enabled_heuristics: # Prefer linking annotations closer in time times = ut.take(node_to_time, nodes) maxtime = vt.safe_max(times, fill=1, nans=False) mintime = vt.safe_min(times, fill=0, nans=False) time_denom = maxtime - mintime # Try linking by time for lynx data time_delta = np.array( [abs(node_to_time[u] - node_to_time[v]) for u, v in avail_uv] ) time_weight = time_delta / time_denom weights += time_weight weights = np.array(weights) weights[np.isnan(weights)] = 1.0 avail = [(u, v, {'weight': w}) for (u, v), w in zip(avail_uv, weights)] return avail
def _make_lnbnn_scores(infr, edges): edge_to_data = infr._get_cm_edge_data(edges) edges = list(edge_to_data.keys()) edge_scores = list(ut.take_column(edge_to_data.values(), 'score')) edge_scores = ut.replace_nones(edge_scores, np.nan) edge_scores = np.array(edge_scores) # take the inf-norm normscores = edge_scores / vt.safe_max(edge_scores, nans=False) return normscores
def apply_match_scores(infr): """ Applies precomputed matching scores to edges that already exist in the graph. Typically you should run infr.apply_match_edges() before running this. CommandLine: python -m wbia.algo.graph.core apply_match_scores --show Example: >>> # xdoctest: +REQUIRES(--slow) >>> # ENABLE_DOCTEST >>> from wbia.algo.graph.core import * # NOQA >>> infr = testdata_infr('PZ_MTEST') >>> infr.exec_matching() >>> infr.apply_match_edges() >>> infr.apply_match_scores() >>> infr.get_edge_attrs('score') """ if infr.cm_list is None: infr.print('apply_match_scores - no scores to apply!') return infr.print('apply_match_scores', 1) edges = list(infr.graph.edges()) edge_to_data = infr._get_cm_edge_data(edges) # Remove existing attrs ut.nx_delete_edge_attr(infr.graph, 'score') ut.nx_delete_edge_attr(infr.graph, 'rank') ut.nx_delete_edge_attr(infr.graph, 'normscore') edges = list(edge_to_data.keys()) edge_scores = list(ut.take_column(edge_to_data.values(), 'score')) edge_scores = ut.replace_nones(edge_scores, np.nan) edge_scores = np.array(edge_scores) edge_ranks = np.array(ut.take_column(edge_to_data.values(), 'rank')) # take the inf-norm normscores = edge_scores / vt.safe_max(edge_scores, nans=False) # Add new attrs infr.set_edge_attrs('score', ut.dzip(edges, edge_scores)) infr.set_edge_attrs('rank', ut.dzip(edges, edge_ranks)) # Hack away zero probabilites # probs = np.vstack([p_nomatch, p_match, p_notcomp]).T + 1e-9 # probs = vt.normalize(probs, axis=1, ord=1, out=probs) # entropy = -(np.log2(probs) * probs).sum(axis=1) infr.set_edge_attrs('normscore', dict(zip(edges, normscores)))
def split_analysis(ibs): """ CommandLine: python -m wbia.other.dbinfo split_analysis --show python -m wbia split_analysis --show python -m wbia split_analysis --show --good Ignore: # mount sshfs -o idmap=user lev:/ ~/lev # unmount fusermount -u ~/lev Example: >>> # DISABLE_DOCTEST GGR >>> from wbia.other.dbinfo import * # NOQA >>> import wbia >>> dbdir = '/media/danger/GGR/GGR-IBEIS' >>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS') >>> ibs = wbia.opendb(dbdir=dbdir, allow_newdir=False) >>> import wbia.guitool as gt >>> gt.ensure_qtapp() >>> win = split_analysis(ibs) >>> ut.quit_if_noshow() >>> import wbia.plottool as pt >>> gt.qtapp_loop(qwin=win) >>> #ut.show_if_requested() """ # nid_list = ibs.get_valid_nids(filter_empty=True) import datetime day1 = datetime.date(2016, 1, 30) day2 = datetime.date(2016, 1, 31) filter_kw = { 'multiple': None, # 'view': ['right'], # 'minqual': 'good', 'is_known': True, 'min_pername': 1, } aids1 = ibs.filter_annots_general(filter_kw=ut.dict_union( filter_kw, { 'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)), 'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 1.0)), }, )) aids2 = ibs.filter_annots_general(filter_kw=ut.dict_union( filter_kw, { 'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 0.0)), 'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)), }, )) all_aids = aids1 + aids2 all_annots = ibs.annots(all_aids) logger.info('%d annots on day 1' % (len(aids1))) logger.info('%d annots on day 2' % (len(aids2))) logger.info('%d annots overall' % (len(all_annots))) logger.info('%d names overall' % (len(ut.unique(all_annots.nids)))) nid_list, annots_list = all_annots.group(all_annots.nids) REVIEWED_EDGES = True if REVIEWED_EDGES: aids_list = [annots.aids for annots in annots_list] # aid_pairs = [annots.get_am_aidpairs() for annots in annots_list] # Slower aid_pairs = ibs.get_unflat_am_aidpairs(aids_list) # Faster else: # ALL EDGES aid_pairs = [annots.get_aidpairs() for annots in annots_list] speeds_list = ibs.unflat_map(ibs.get_annotpair_speeds, aid_pairs) import vtool as vt max_speeds = np.array([vt.safe_max(s, nans=False) for s in speeds_list]) nan_idx = np.where(np.isnan(max_speeds))[0] inf_idx = np.where(np.isinf(max_speeds))[0] bad_idx = sorted(ut.unique(ut.flatten([inf_idx, nan_idx]))) ok_idx = ut.index_complement(bad_idx, len(max_speeds)) logger.info('#nan_idx = %r' % (len(nan_idx), )) logger.info('#inf_idx = %r' % (len(inf_idx), )) logger.info('#ok_idx = %r' % (len(ok_idx), )) ok_speeds = max_speeds[ok_idx] ok_nids = ut.take(nid_list, ok_idx) ok_annots = ut.take(annots_list, ok_idx) sortx = np.argsort(ok_speeds)[::-1] sorted_speeds = np.array(ut.take(ok_speeds, sortx)) sorted_annots = np.array(ut.take(ok_annots, sortx)) sorted_nids = np.array(ut.take(ok_nids, sortx)) # NOQA sorted_speeds = np.clip(sorted_speeds, 0, 100) # idx = vt.find_elbow_point(sorted_speeds) # EXCESSIVE_SPEED = sorted_speeds[idx] # http://www.infoplease.com/ipa/A0004737.html # http://www.speedofanimals.com/animals/zebra # ZEBRA_SPEED_MAX = 64 # km/h # ZEBRA_SPEED_RUN = 50 # km/h ZEBRA_SPEED_SLOW_RUN = 20 # km/h # ZEBRA_SPEED_FAST_WALK = 10 # km/h # ZEBRA_SPEED_WALK = 7 # km/h MAX_SPEED = ZEBRA_SPEED_SLOW_RUN # MAX_SPEED = ZEBRA_SPEED_WALK # MAX_SPEED = EXCESSIVE_SPEED flags = sorted_speeds > MAX_SPEED flagged_ok_annots = ut.compress(sorted_annots, flags) inf_annots = ut.take(annots_list, inf_idx) flagged_annots = inf_annots + flagged_ok_annots logger.info('MAX_SPEED = %r km/h' % (MAX_SPEED, )) logger.info('%d annots with infinite speed' % (len(inf_annots), )) logger.info('%d annots with large speed' % (len(flagged_ok_annots), )) logger.info( 'Marking all pairs of annots above the threshold as non-matching') from wbia.algo.graph import graph_iden import networkx as nx progkw = dict(freq=1, bs=True, est_window=len(flagged_annots)) bad_edges_list = [] good_edges_list = [] for annots in ut.ProgIter(flagged_annots, lbl='flag speeding names', **progkw): edge_to_speeds = annots.get_speeds() bad_edges = [ edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED ] good_edges = [ edge for edge, speed in edge_to_speeds.items() if speed <= MAX_SPEED ] bad_edges_list.append(bad_edges) good_edges_list.append(good_edges) all_bad_edges = ut.flatten(bad_edges_list) good_edges_list = ut.flatten(good_edges_list) logger.info('num_bad_edges = %r' % (len(ut.flatten(bad_edges_list)), )) logger.info('num_bad_edges = %r' % (len(ut.flatten(good_edges_list)), )) if 1: from wbia.viz import viz_graph2 import wbia.guitool as gt gt.ensure_qtapp() if ut.get_argflag('--good'): logger.info('Looking at GOOD (no speed problems) edges') aid_pairs = good_edges_list else: logger.info('Looking at BAD (speed problems) edges') aid_pairs = all_bad_edges aids = sorted(list(set(ut.flatten(aid_pairs)))) infr = graph_iden.AnnotInference(ibs, aids, verbose=False) infr.initialize_graph() # Use random scores to randomize sort order rng = np.random.RandomState(0) scores = (-rng.rand(len(aid_pairs)) * 10).tolist() infr.graph.add_edges_from(aid_pairs) if True: edge_sample_size = 250 pop_nids = ut.unique( ibs.get_annot_nids(ut.unique(ut.flatten(aid_pairs)))) sorted_pairs = ut.sortedby(aid_pairs, scores)[::-1][0:edge_sample_size] sorted_nids = ibs.get_annot_nids(ut.take_column(sorted_pairs, 0)) sample_size = len(ut.unique(sorted_nids)) am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(*zip( *sorted_pairs)) flags = ut.not_list(ut.flag_None_items(am_rowids)) # am_rowids = ut.compress(am_rowids, flags) positive_tags = ['SplitCase', 'Photobomb'] flags_list = [ ut.replace_nones(ibs.get_annotmatch_prop(tag, am_rowids), 0) for tag in positive_tags ] logger.info('edge_case_hist: ' + ut.repr3([ '%s %s' % (txt, sum(flags_)) for flags_, txt in zip(flags_list, positive_tags) ])) is_positive = ut.or_lists(*flags_list) num_positive = sum( ut.lmap(any, ut.group_items(is_positive, sorted_nids).values())) pop = len(pop_nids) logger.info('A positive is any edge flagged as a %s' % (ut.conj_phrase(positive_tags, 'or'), )) logger.info('--- Sampling wrt edges ---') logger.info('edge_sample_size = %r' % (edge_sample_size, )) logger.info('edge_population_size = %r' % (len(aid_pairs), )) logger.info('num_positive_edges = %r' % (sum(is_positive))) logger.info('--- Sampling wrt names ---') logger.info('name_population_size = %r' % (pop, )) vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level=0.95) nx.set_edge_attributes(infr.graph, name='score', values=dict(zip(aid_pairs, scores))) win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False, init_mode=None) win.populate_edge_model() win.show() return win # Make review interface for only bad edges infr_list = [] iter_ = list(zip(flagged_annots, bad_edges_list)) for annots, bad_edges in ut.ProgIter(iter_, lbl='creating inference', **progkw): aids = annots.aids nids = [1] * len(aids) infr = graph_iden.AnnotInference(ibs, aids, nids, verbose=False) infr.initialize_graph() infr.reset_feedback() infr_list.append(infr) # Check which ones are user defined as incorrect # num_positive = 0 # for infr in infr_list: # flag = np.any(infr.get_feedback_probs()[0] == 0) # num_positive += flag # logger.info('num_positive = %r' % (num_positive,)) # pop = len(infr_list) # logger.info('pop = %r' % (pop,)) iter_ = list(zip(infr_list, bad_edges_list)) for infr, bad_edges in ut.ProgIter(iter_, lbl='adding speed edges', **progkw): flipped_edges = [] for aid1, aid2 in bad_edges: if infr.graph.has_edge(aid1, aid2): flipped_edges.append((aid1, aid2)) infr.add_feedback((aid1, aid2), NEGTV) nx.set_edge_attributes(infr.graph, name='_speed_split', values='orig') nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'new' for edge in bad_edges}) nx.set_edge_attributes( infr.graph, name='_speed_split', values={edge: 'flip' for edge in flipped_edges}, ) # for infr in ut.ProgIter(infr_list, lbl='flagging speeding edges', **progkw): # annots = ibs.annots(infr.aids) # edge_to_speeds = annots.get_speeds() # bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED] def inference_stats(infr_list_): relabel_stats = [] for infr in infr_list_: num_ccs, num_inconsistent = infr.relabel_using_reviews() state_hist = ut.dict_hist( nx.get_edge_attributes(infr.graph, 'decision').values()) if POSTV not in state_hist: state_hist[POSTV] = 0 hist = ut.dict_hist( nx.get_edge_attributes(infr.graph, '_speed_split').values()) subgraphs = infr.positive_connected_compoments() subgraph_sizes = [len(g) for g in subgraphs] info = ut.odict([ ('num_nonmatch_edges', state_hist[NEGTV]), ('num_match_edges', state_hist[POSTV]), ( 'frac_nonmatch_edges', state_hist[NEGTV] / (state_hist[POSTV] + state_hist[NEGTV]), ), ('num_inconsistent', num_inconsistent), ('num_ccs', num_ccs), ('edges_flipped', hist.get('flip', 0)), ('edges_unchanged', hist.get('orig', 0)), ('bad_unreviewed_edges', hist.get('new', 0)), ('orig_size', len(infr.graph)), ('new_sizes', subgraph_sizes), ]) relabel_stats.append(info) return relabel_stats relabel_stats = inference_stats(infr_list) logger.info('\nAll Split Info:') lines = [] for key in relabel_stats[0].keys(): data = ut.take_column(relabel_stats, key) if key == 'new_sizes': data = ut.flatten(data) lines.append( 'stats(%s) = %s' % (key, ut.repr2(ut.get_stats(data, use_median=True), precision=2))) logger.info('\n'.join(ut.align_lines(lines, '='))) num_incon_list = np.array(ut.take_column(relabel_stats, 'num_inconsistent')) can_split_flags = num_incon_list == 0 logger.info('Can trivially split %d / %d' % (sum(can_split_flags), len(can_split_flags))) splittable_infrs = ut.compress(infr_list, can_split_flags) relabel_stats = inference_stats(splittable_infrs) logger.info('\nTrival Split Info:') lines = [] for key in relabel_stats[0].keys(): if key in ['num_inconsistent']: continue data = ut.take_column(relabel_stats, key) if key == 'new_sizes': data = ut.flatten(data) lines.append( 'stats(%s) = %s' % (key, ut.repr2(ut.get_stats(data, use_median=True), precision=2))) logger.info('\n'.join(ut.align_lines(lines, '='))) num_match_edges = np.array(ut.take_column(relabel_stats, 'num_match_edges')) num_nonmatch_edges = np.array( ut.take_column(relabel_stats, 'num_nonmatch_edges')) flags1 = np.logical_and(num_match_edges > num_nonmatch_edges, num_nonmatch_edges < 3) reasonable_infr = ut.compress(splittable_infrs, flags1) new_sizes_list = ut.take_column(relabel_stats, 'new_sizes') flags2 = [ len(sizes) == 2 and sum(sizes) > 4 and (min(sizes) / max(sizes)) > 0.3 for sizes in new_sizes_list ] reasonable_infr = ut.compress(splittable_infrs, flags2) logger.info('#reasonable_infr = %r' % (len(reasonable_infr), )) for infr in ut.InteractiveIter(reasonable_infr): annots = ibs.annots(infr.aids) edge_to_speeds = annots.get_speeds() logger.info('max_speed = %r' % (max(edge_to_speeds.values())), ) infr.initialize_visual_node_attrs() infr.show_graph(use_image=True, only_reviewed=True) rest = ~np.logical_or(flags1, flags2) nonreasonable_infr = ut.compress(splittable_infrs, rest) rng = np.random.RandomState(0) random_idx = ut.random_indexes(len(nonreasonable_infr) - 1, 15, rng=rng) random_infr = ut.take(nonreasonable_infr, random_idx) for infr in ut.InteractiveIter(random_infr): annots = ibs.annots(infr.aids) edge_to_speeds = annots.get_speeds() logger.info('max_speed = %r' % (max(edge_to_speeds.values())), ) infr.initialize_visual_node_attrs() infr.show_graph(use_image=True, only_reviewed=True) # import scipy.stats as st # conf_interval = .95 # st.norm.cdf(conf_interval) # view-source:http://www.surveysystem.com/sscalc.htm # zval = 1.96 # 95 percent confidence # zValC = 3.8416 # # zValC = 6.6564 # import statsmodels.stats.api as sms # es = sms.proportion_effectsize(0.5, 0.75) # sms.NormalIndPower().solve_power(es, power=0.9, alpha=0.05, ratio=1) pop = 279 num_positive = 3 sample_size = 15 conf_level = 0.95 # conf_level = .99 vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level) logger.info('---') vt.calc_error_bars_from_sample(sample_size + 38, num_positive, pop, conf_level) logger.info('---') vt.calc_error_bars_from_sample(sample_size + 38 / 3, num_positive, pop, conf_level) logger.info('---') vt.calc_error_bars_from_sample(15 + 38, num_positive=3, pop=675, conf_level=0.95) vt.calc_error_bars_from_sample(15, num_positive=3, pop=675, conf_level=0.95) pop = 279 # err_frac = .05 # 5% err_frac = 0.10 # 10% conf_level = 0.95 vt.calc_sample_from_error_bars(err_frac, pop, conf_level) pop = 675 vt.calc_sample_from_error_bars(err_frac, pop, conf_level) vt.calc_sample_from_error_bars(0.05, pop, conf_level=0.95, prior=0.1) vt.calc_sample_from_error_bars(0.05, pop, conf_level=0.68, prior=0.2) vt.calc_sample_from_error_bars(0.10, pop, conf_level=0.68) vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=0.95) vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=0.68)
def compute_occurrence_groups(ibs, gid_list, config={}, use_gps=False, verbose=None): r""" Args: ibs (IBEISController): wbia controller object gid_list (list): Returns: tuple: (None, None) CommandLine: python -m wbia compute_occurrence_groups Example: >>> # DISABLE_DOCTEST >>> from wbia.algo.preproc.preproc_occurrence import * # NOQA >>> import wbia >>> ibs = wbia.opendb(defaultdb='testdb1') >>> verbose = True >>> images = ibs.images() >>> gid_list = images.gids >>> config = {} # wbia.algo.Config.OccurrenceConfig().asdict() >>> tup = wbia_compute_occurrences(ibs, gid_list) >>> (flat_imgsetids, flat_gids) >>> aids_list = list(ut.group_items(aid_list_, flat_imgsetids).values()) >>> metric = list(map(len, aids_list)) >>> sortx = ut.list_argsort(metric)[::-1] >>> index = sortx[1] >>> aids = aids_list[index] >>> gids = list(set(ibs.get_annot_gids(aids))) """ if verbose is None: verbose = ut.NOT_QUIET # Config info gid_list = np.unique(gid_list) if verbose: logger.info('[occur] Computing occurrences on %r images.' % (len(gid_list))) logger.info('[occur] config = ' + ut.repr3(config)) use_gps = config['use_gps'] datas = prepare_X_data(ibs, gid_list, use_gps=use_gps) from wbia.algo.preproc import occurrence_blackbox cluster_algo = config.get('cluster_algo', 'agglomerative') km_per_sec = config.get('km_per_sec', occurrence_blackbox.KM_PER_SEC) thresh_sec = config.get('seconds_thresh', 30 * 60.0) min_imgs_per_occurence = config.get('min_imgs_per_occurence', 1) # 30 minutes = 3.6 kilometers # 5 minutes = 0.6 kilometers assert cluster_algo == 'agglomerative', 'only agglomerative is supported' # Group datas with different values separately all_gids = [] all_labels = [] for key in datas.keys(): val = datas[key] gids, latlons, posixtimes = val labels = occurrence_blackbox.cluster_timespace_sec( latlons, posixtimes, thresh_sec, km_per_sec=km_per_sec ) if labels is None: labels = np.zeros(len(gids), dtype=np.int) all_gids.append(gids) all_labels.append(labels) # Combine labels across different groups pads = [vt.safe_max(ys, fill=0) + 1 for ys in all_labels] offsets = np.array([0] + pads[:-1]).cumsum() all_labels_ = [ys + offset for ys, offset in zip(all_labels, offsets)] label_arr = np.array(ut.flatten(all_labels_)) gid_arr = np.array(ut.flatten(all_gids)) # Group images by unique label labels, label_gids = group_images_by_label(label_arr, gid_arr) # Remove occurrences less than the threshold occur_labels = labels occur_gids = label_gids occur_unixtimes = compute_occurrence_unixtime(ibs, occur_gids) occur_labels, occur_gids = filter_and_relabel( labels, label_gids, min_imgs_per_occurence, occur_unixtimes ) if verbose: logger.info('[occur] Found %d clusters.' % len(occur_labels)) if len(label_gids) > 0 and verbose: logger.info('[occur] Cluster image size stats:') ut.print_dict( ut.get_stats(list(map(len, occur_gids)), use_median=True, use_sum=True), 'occur image stats', ) return occur_labels, occur_gids