def simulate_user_feedback(infr): qreq_ = infr.qreq_ aid_pairs = np.array(ut.take_column(infr.needs_review_list, [0, 1])) nid_pairs = qreq_.ibs.get_annot_nids(aid_pairs) truth = nid_pairs.T[0] == nid_pairs.T[1] user_feedback = ut.odict([ ('aid1', aid_pairs.T[0]), ('aid2', aid_pairs.T[1]), ('p_match', truth.astype(np.float)), ('p_nomatch', 1.0 - truth), ('p_notcomp', np.array([0.0] * len(aid_pairs))), ]) return user_feedback
def random_case_set(): r""" Returns: tuple: (labels, pairwise_feats) CommandLine: python -m ibeis.algo.hots.testem random_case_set --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.testem import * # NOQA >>> (labels, pairwise_feats) = random_case_set() >>> result = ('(labels, pairwise_feats) = %s' % (ut.repr2((labels, pairwise_feats)),)) >>> print(result) """ rng = np.random.RandomState(0) case_params = dict(num_names=5, rng=rng) num_annots = 600 test_cases = [random_test_annot(**case_params) for _ in ut.ProgIter(range(num_annots), bs=1)] pairxs = list(ut.product_nonsame(range(num_annots), range(num_annots))) import utool utool.embed() test_pairs = list(ut.unflat_take(test_cases, pairxs)) cases1 = ut.make_instancelist(ut.take_column(test_pairs, 0), check=False) cases2 = ut.make_instancelist(ut.take_column(test_pairs, 1), check=False) # FIXME labels = labels1 = make_test_pairwise_labels2(cases1, cases2) # NOQA #labels = np.array([make_test_pairwise_labels(case1, case2) # for case1, case2 in ut.ProgIter(test_pairs, bs=1)]) pairwise_feats_ = [make_test_pairwise_fetaures(case1, case2, label, rng) for label, (case1, case2) in ut.ProgIter(list(zip(labels, test_pairs)), bs=1)] pairwise_feats = np.vstack(pairwise_feats_) print(ut.dict_hist(labels)) return labels, pairwise_feats
def get_rarity(card, rule='recent'): """ >>> from mtgmonte.mtgobjs import * # NOQA >>> card = lookup_card('Wasteland') >>> card = lookup_card('Wastes') >>> result = card.rarity 'Common' """ rarity_printings = ut.take_column(card.printings, 1) # Take the most recent (todo; allow for customization of this rule) if rule == 'recent': return rarity_printings[0] elif rule == 'first': return rarity_printings[-1] elif rule == 'greatest': pass elif rule == 'lowest': pass
def graph_info(graph, verbose=False): import utool as ut node_attrs = list(graph.node.values()) edge_attrs = list(ut.take_column(graph.edges(data=True), 2)) node_attr_hist = ut.dict_hist(ut.flatten([attr.keys() for attr in node_attrs])) edge_attr_hist = ut.dict_hist(ut.flatten([attr.keys() for attr in edge_attrs])) node_type_hist = ut.dict_hist(list(map(type, graph.nodes()))) info_dict = ut.odict([ ('directed', graph.is_directed()), ('multi', graph.is_multigraph()), ('num_nodes', len(graph)), ('num_edges', len(list(graph.edges()))), ('edge_attr_hist', ut.sort_dict(edge_attr_hist)), ('node_attr_hist', ut.sort_dict(node_attr_hist)), ('node_type_hist', ut.sort_dict(node_type_hist)), ('graph_attrs', graph.graph), ('graph_name', graph.name), ]) #unique_attrs = ut.map_dict_vals(ut.unique, ut.dict_accum(*node_attrs)) #ut.dict_isect_combine(*node_attrs)) #[list(attrs.keys())] if verbose: print(ut.repr3(info_dict)) return info_dict
def build_alias_map(regex_map, tag_vocab): """ Constructs explicit mapping. Order of items in regex map matters. Items at top are given preference. Example: >>> tags_list = [['t1', 't2'], [], ['t3'], ['t4', 't5']] >>> tag_vocab = ut.flat_unique(*tags_list) >>> regex_map = [('t[3-4]', 'A9'), ('t0', 'a0')] >>> unmapped = list(set(tag_vocab) - set(alias_map.keys())) """ import utool as ut import re alias_map = ut.odict([]) for pats, new_tag in reversed(regex_map): pats = ut.ensure_iterable(pats) for pat in pats: flags = [re.match(pat, t) for t in tag_vocab] for old_tag in ut.compress(tag_vocab, flags): alias_map[old_tag] = new_tag identity_map = ut.take_column(regex_map, 1) for tag in ut.filter_Nones(identity_map): alias_map[tag] = tag return alias_map
def make_inference(infr): cm_list = infr.cm_list unique_nids, prob_names = infr.make_prob_names() cluster_tuples = infr.make_clusters() # Make pair list for output if infr.user_feedback is not None: keys = list(zip(infr.user_feedback['aid1'], infr.user_feedback['aid2'])) feedback_lookup = ut.make_index_lookup(keys) user_feedback = infr.user_feedback p_bg = 0 user_feedback = ut.map_dict_vals(np.array, infr.user_feedback) part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp']) part2 = p_bg * user_feedback['p_notcomp'] p_same_list = part1 + part2 else: feedback_lookup = {} infr.user_feedback needs_review_list = [] num_top = 4 for cm, row in zip(cm_list, prob_names): # Find top scoring names for this chip match in the posterior distribution idxs = row.argsort()[::-1] top_idxs = idxs[:num_top] nids = ut.take(unique_nids, top_idxs) # Find the matched annotations in the pairwise prior distributions nidxs = ut.dict_take(cm.nid2_nidx, nids, None) name_groupxs = ut.take(cm.name_groupxs, ut.filter_Nones(nidxs)) daids_list = ut.take(cm.daid_list, name_groupxs) for daids in daids_list: ut.take(cm.score_list, ut.take(cm.daid2_idx, daids)) scores_all = cm.annot_score_list / cm.annot_score_list.sum() idxs = ut.take(cm.daid2_idx, daids) scores = scores_all.take(idxs) raw_scores = cm.score_list.take(idxs) scorex = scores.argmax() raw_score = raw_scores[scorex] daid = daids[scorex] import scipy.special # SUPER HACK: these are not probabilities # TODO: set a and b based on dbsize and param configuration # python -m plottool.draw_func2 --exec-plot_func --show --range=0,3 --func="lambda x: scipy.special.expit(2 * x - 2)" #a = 2.0 a = 1.5 b = 2 p_same = scipy.special.expit(b * raw_score - a) #confidence = scores[scorex] #p_diff = 1 - p_same #decision = 'same' if confidence > thresh else 'diff' #confidence = p_same if confidence > thresh else p_diff #tup = (cm.qaid, daid, decision, confidence, raw_score) confidence = (2 * np.abs(0.5 - p_same)) ** 2 #if infr.user_feedback is not None: # import utool # utool.embed( key = (cm.qaid, daid) fb_idx = feedback_lookup.get(key) if fb_idx is not None: confidence = p_same_list[fb_idx] tup = (cm.qaid, daid, p_same, confidence, raw_score) needs_review_list.append(tup) # Sort resulting list by confidence sortx = ut.argsort(ut.take_column(needs_review_list, 3)) needs_review_list = ut.take(needs_review_list, sortx) infr.needs_review_list = needs_review_list infr.cluster_tuples = cluster_tuples
def nx_agraph_layout(graph, orig_graph=None, inplace=False, verbose=None, **kwargs): r""" orig_graph = graph graph = layout_graph References: http://www.graphviz.org/content/attrs http://www.graphviz.org/doc/info/attrs.html """ import networkx as nx import pygraphviz kwargs = kwargs.copy() prog = kwargs.pop('prog', 'dot') if prog != 'dot': kwargs['overlap'] = kwargs.get('overlap', 'false') kwargs['splines'] = kwargs.get('splines', 'spline') kwargs['notranslate'] = 'true' # for neato postprocessing argparts = ['-G%s=%s' % (key, str(val)) for key, val in kwargs.items()] args = ' '.join(argparts) splines = kwargs['splines'] if verbose is None: verbose = ut.VERBOSE if verbose: print('args = %r' % (args,)) # Convert to agraph format graph_ = graph.copy() ut.nx_ensure_agraph_color(graph_) # Reduce size to be in inches not pixels # FIXME: make robust to param settings # Hack to make the w/h of the node take thae max instead of # dot which takes the minimum shaped_nodes = [n for n, d in graph_.nodes(data=True) if 'width' in d] node_attrs = ut.dict_take(graph_.node, shaped_nodes) width_px = np.array(ut.take_column(node_attrs, 'width')) height_px = np.array(ut.take_column(node_attrs, 'height')) scale = np.array(ut.dict_take_column(node_attrs, 'scale', default=1.0)) width_in = width_px / 72.0 * scale height_in = height_px / 72.0 * scale width_in_dict = dict(zip(shaped_nodes, width_in)) height_in_dict = dict(zip(shaped_nodes, height_in)) nx.set_node_attributes(graph_, 'width', width_in_dict) nx.set_node_attributes(graph_, 'height', height_in_dict) ut.nx_delete_node_attr(graph_, 'scale') # Check for any nodes with groupids node_to_groupid = nx.get_node_attributes(graph_, 'groupid') if node_to_groupid: groupid_to_nodes = ut.group_items(*zip(*node_to_groupid.items())) else: groupid_to_nodes = {} # Initialize agraph format #import utool #utool.embed() ut.nx_delete_None_edge_attr(graph_) agraph = nx.nx_agraph.to_agraph(graph_) # Add subgraphs labels # TODO: subgraph attrs group_attrs = graph.graph.get('groupattrs', {}) for groupid, nodes in groupid_to_nodes.items(): # subgraph_attrs = {} subgraph_attrs = group_attrs.get(groupid, {}).copy() cluster_flag = True # FIXME: make this more natural to specify if 'cluster' in subgraph_attrs: cluster_flag = subgraph_attrs['cluster'] del subgraph_attrs['cluster'] # subgraph_attrs = dict(rankdir='LR') # subgraph_attrs = dict(rankdir='LR') # subgraph_attrs['rank'] = 'min' # subgraph_attrs['rank'] = 'source' name = groupid if cluster_flag: # graphviz treast subgraphs labeld with cluster differently name = 'cluster_' + groupid else: name = groupid agraph.add_subgraph(nodes, name, **subgraph_attrs) for node in graph_.nodes(): # force pinning of node points anode = pygraphviz.Node(agraph, node) if anode.attr['pin'] == 'true': if anode.attr['pos'] is not None and len(anode.attr['pos']) > 0 and not anode.attr['pos'].endswith('!'): import re #utool.embed() ptstr_ = anode.attr['pos'] #print('ptstr_ = %r' % (ptstr_,)) ptstr = ptstr_.strip('[]').strip(' ').strip('()') #print('ptstr = %r' % (ptstr,)) ptstr_list = [x.rstrip(',') for x in re.split(r'\s+', ptstr)] #print('ptstr_list = %r' % (ptstr_list,)) pt_list = list(map(float, ptstr_list)) #print('pt_list = %r' % (pt_list,)) pt_arr = np.array(pt_list) / 72.0 #print('pt_arr = %r' % (pt_arr,)) new_ptstr_list = list(map(str, pt_arr)) new_ptstr = ','.join(new_ptstr_list) + '!' #print('new_ptstr = %r' % (new_ptstr,)) anode.attr['pos'] = new_ptstr # Run layout #print('prog = %r' % (prog,)) if ut.VERBOSE or verbose > 0: print('BEFORE LAYOUT\n' + str(agraph)) agraph.layout(prog=prog, args=args) agraph.draw(ut.truepath('~/test_graphviz_draw.png')) if ut.VERBOSE or verbose > 1: print('AFTER LAYOUT\n' + str(agraph)) # TODO: just replace with a single dict of attributes node_layout_attrs = ut.ddict(dict) edge_layout_attrs = ut.ddict(dict) #for node in agraph.nodes(): for node in graph_.nodes(): anode = pygraphviz.Node(agraph, node) node_attrs = parse_anode_layout_attrs(anode) for key, val in node_attrs.items(): node_layout_attrs[key][node] = val edges = list(ut.nx_edges(graph_, keys=True)) for edge in edges: aedge = pygraphviz.Edge(agraph, *edge) edge_attrs = parse_aedge_layout_attrs(aedge) for key, val in edge_attrs.items(): edge_layout_attrs[key][edge] = val if orig_graph is not None and kwargs.get('draw_implicit', True): # ADD IN IMPLICIT EDGES layout_edges = set(ut.nx_edges(graph_, keys=True)) orig_edges = set(ut.nx_edges(orig_graph, keys=True)) implicit_edges = list(orig_edges - layout_edges) #all_edges = list(set.union(orig_edges, layout_edges)) needs_implicit = len(implicit_edges) > 0 if needs_implicit: # Pin down positions for node in agraph.nodes(): anode = pygraphviz.Node(agraph, node) anode.attr['pin'] = 'true' anode.attr['pos'] += '!' # Add new edges to route for iedge in implicit_edges: data = orig_graph.get_edge_data(*iedge) agraph.add_edge(*iedge, **data) if ut.VERBOSE or verbose: print('BEFORE IMPLICIT LAYOUT\n' + str(agraph)) # Route the implicit edges (must use neato) control_node = pygraphviz.Node(agraph, node) #print('control_node = %r' % (control_node,)) node1_attr1 = parse_anode_layout_attrs(control_node) #print('node1_attr1 = %r' % (node1_attr1,)) implicit_kw = kwargs.copy() implicit_kw['overlap'] = 'true' #del implicit_kw['overlap'] # can cause node positions to change argparts = ['-G%s=%s' % (key, str(val)) for key, val in implicit_kw.items()] args = ' '.join(argparts) #print('args = %r' % (args,)) #import utool #utool.embed() agraph.layout(prog='neato', args='-n ' + args) agraph.draw(ut.truepath('~/implicit_test_graphviz_draw.png')) if ut.VERBOSE or verbose: print('AFTER IMPLICIT LAYOUT\n' + str(agraph)) control_node = pygraphviz.Node(agraph, node) print('control_node = %r' % (control_node,)) node1_attr2 = parse_anode_layout_attrs(control_node) print('node1_attr2 = %r' % (node1_attr2,)) # graph positions shifted # This is not the right place to divide by 72 translation = (node1_attr1['pos'] - node1_attr2['pos'] ) #print('translation = %r' % (translation,)) #translation = np.array([0, 0]) print('translation = %r' % (translation,)) #for iedge in all_edges: for iedge in implicit_edges: aedge = pygraphviz.Edge(agraph, *iedge) iedge_attrs = parse_aedge_layout_attrs(aedge, translation) for key, val in iedge_attrs.items(): edge_layout_attrs[key][iedge] = val graph_layout_attrs = dict( splines=splines ) layout_info = { 'graph': graph_layout_attrs, 'edge': dict(edge_layout_attrs), 'node': dict(node_layout_attrs), } if inplace: if orig_graph is not None: graph = orig_graph apply_graph_layout_attrs(graph, layout_info) return graph, layout_info
def commit_localization_results( ibs, gid_list, results_list, note=None, labeler_algo='pipeline', labeler_model_tag=None, use_labeler_species=False, orienter_algo=None, orienter_model_tag=None, update_json_log=True, **kwargs, ): global_gid_list = [] global_bbox_list = [] global_theta_list = [] global_class_list = [] global_conf_list = [] global_notes_list = [] zipped_list = list(zip(gid_list, results_list)) for gid, results in zipped_list: score, bbox_list, theta_list, conf_list, class_list = results num = len(bbox_list) gid_list_ = [gid] * num notes_list = [note] * num global_gid_list += list(gid_list_) global_bbox_list += list(bbox_list) global_theta_list += list(theta_list) global_class_list += list(class_list) global_conf_list += list(conf_list) global_notes_list += list(notes_list) assert len(global_gid_list) == len(global_bbox_list) assert len(global_gid_list) == len(global_theta_list) assert len(global_gid_list) == len(global_class_list) assert len(global_gid_list) == len(global_conf_list) assert len(global_gid_list) == len(global_notes_list) global_aid_list = ibs.add_annots( global_gid_list, global_bbox_list, global_theta_list, global_class_list, detect_confidence_list=global_conf_list, notes_list=global_notes_list, quiet_delete_thumbs=True, skip_cleaning=True, ) global_aid_set = set(global_aid_list) aids_list = ibs.get_image_aids(gid_list) aids_list = [[aid for aid in aid_list_ if aid in global_aid_set] for aid_list_ in aids_list] aid_list = ut.flatten(aids_list) if labeler_model_tag is not None: labeler_config = {} labeler_config['labeler_algo'] = labeler_algo labeler_config['labeler_weight_filepath'] = labeler_model_tag viewpoint_list = ibs.depc_annot.get_property('labeler', aid_list, 'viewpoint', config=labeler_config) ibs.set_annot_viewpoints(aid_list, viewpoint_list) if use_labeler_species: species_list = ibs.depc_annot.get_property('labeler', aid_list, 'species', config=labeler_config) ibs.set_annot_species(aid_list, species_list) if orienter_algo is not None: orienter_config = {} orienter_config['orienter_algo'] = orienter_algo orienter_config['orienter_weight_filepath'] = orienter_model_tag result_list = ibs.depc_annot.get_property('orienter', aid_list, None, config=orienter_config) xtl_list = list( map(int, map(np.around, ut.take_column(result_list, 0)))) ytl_list = list( map(int, map(np.around, ut.take_column(result_list, 1)))) w_list = list(map(int, map(np.around, ut.take_column(result_list, 2)))) h_list = list(map(int, map(np.around, ut.take_column(result_list, 3)))) theta_list = ut.take_column(result_list, 4) bbox_list = list(zip(xtl_list, ytl_list, w_list, h_list)) assert len(aid_list) == len(bbox_list) assert len(aid_list) == len(theta_list) if len(bbox_list) > 0: ibs.set_annot_bboxes(aid_list, bbox_list, theta_list=theta_list) ibs._clean_species() if update_json_log: ibs.log_detections(aid_list) return aids_list
def postingest_tesdb1_func(ibs): import numpy as np from ibeis import constants as const print('postingest_tesdb1_func') # Adjust data as we see fit gid_list = np.array(ibs.get_valid_gids()) # Set image unixtimes unixtimes_even = (gid_list[0::2] + 100).tolist() unixtimes_odd = (gid_list[1::2] + 9001).tolist() unixtime_list = unixtimes_even + unixtimes_odd ibs.set_image_unixtime(gid_list, unixtime_list) # Unname first aid in every name aid_list = ibs.get_valid_aids() nid_list = ibs.get_annot_name_rowids(aid_list) nid_list = [ (nid if nid > 0 else None) for nid in nid_list] unique_flag = ut.flag_unique_items(nid_list) unique_nids = ut.compress(nid_list, unique_flag) none_nids = [ nid is not None for nid in nid_list] flagged_nids = [nid for nid in unique_nids if nid_list.count(nid) > 1] plural_flag = [nid in flagged_nids for nid in nid_list] flag_list = list(map(all, zip(plural_flag, unique_flag, none_nids))) flagged_aids = ut.compress(aid_list, flag_list) if ut.VERYVERBOSE: def print2(*args): print('[post_testdb1] ' + ', '.join(args)) print2('aid_list=%r' % aid_list) print2('nid_list=%r' % nid_list) print2('unique_flag=%r' % unique_flag) print2('plural_flag=%r' % plural_flag) print2('unique_nids=%r' % unique_nids) print2('none_nids=%r' % none_nids) print2('flag_list=%r' % flag_list) print2('flagged_nids=%r' % flagged_nids) print2('flagged_aids=%r' % flagged_aids) # print2('new_nids=%r' % new_nids) # Unname, some annotations for testing unname_aids = ut.compress(aid_list, flag_list) ibs.delete_annot_nids(unname_aids) # Add all annotations with names as exemplars #from ibeis.control.IBEISControl import IBEISController #assert isinstance(ibs, IBEISController) unflagged_aids = ut.get_dirty_items(aid_list, flag_list) exemplar_flags = [True] * len(unflagged_aids) ibs.set_annot_exemplar_flags(unflagged_aids, exemplar_flags) # Set some test species labels species_text_list = ibs.get_annot_species_texts(aid_list) for ix in range(0, 6): species_text_list[ix] = const.TEST_SPECIES.ZEB_PLAIN # These are actually plains zebras. for ix in range(8, 10): species_text_list[ix] = const.TEST_SPECIES.ZEB_GREVY for ix in range(10, 12): species_text_list[ix] = const.TEST_SPECIES.BEAR_POLAR ibs.set_annot_species(aid_list, species_text_list) ibs.set_annot_notes(aid_list[8:10], ['this is actually a plains zebra'] * 2) ibs.set_annot_notes(aid_list[0:1], ['aid 1 and 2 are correct matches']) ibs.set_annot_notes(aid_list[6:7], ['very simple image to debug feature detector']) ibs.set_annot_notes(aid_list[7:8], ['standard test image']) # Set some randomish gps flags that are within nnp unixtime_list = ibs.get_image_unixtime(gid_list) valid_lat_min = -1.4446 valid_lat_max = -1.3271 valid_lon_min = 36.7619 valid_lon_max = 36.9622 valid_lat_range = valid_lat_max - valid_lat_min valid_lon_range = valid_lon_max - valid_lon_min randstate = np.random.RandomState(unixtime_list) new_gps_list = randstate.rand(len(gid_list), 2) new_gps_list[:, 0] = (new_gps_list[:, 0] * valid_lat_range) + valid_lat_min new_gps_list[:, 1] = (new_gps_list[:, 1] * valid_lon_range) + valid_lon_min new_gps_list[8, :] = [-1, -1] ibs.set_image_gps(gid_list, new_gps_list) # TODO: add a nan timestamp ibs.append_annot_case_tags([2], ['error:bbox']) ibs.append_annot_case_tags([4], ['quality:washedout']) ibs.append_annot_case_tags([4], ['lighting']) aidgroups = ibs.group_annots_by_name( ibs.filter_annots_general(min_pername=2, verbose=True))[0] aid1_list = ut.take_column(aidgroups, 0) aid2_list = ut.take_column(aidgroups, 1) annotmatch_rowids = ibs.add_annotmatch(aid1_list, aid2_list) ibs.set_annotmatch_truth(annotmatch_rowids, [True] * len(annotmatch_rowids)) ibs.set_annotmatch_truth(annotmatch_rowids, [True] * len(annotmatch_rowids)) ibs.set_annotmatch_prop('photobomb', annotmatch_rowids, [True] * len(annotmatch_rowids)) for aids in aidgroups: pass return None
def make_agraph(graph): # FIXME; use this in nx_agraph_layout instead to comparementalize more import networkx as nx import pygraphviz # Convert to agraph format graph_ = graph.copy() ut.nx_ensure_agraph_color(graph_) # Reduce size to be in inches not pixels # FIXME: make robust to param settings # Hack to make the w/h of the node take thae max instead of # dot which takes the minimum shaped_nodes = [n for n, d in graph_.nodes(data=True) if 'width' in d] node_attrs = ut.dict_take(graph_.node, shaped_nodes) width_px = np.array(ut.take_column(node_attrs, 'width')) height_px = np.array(ut.take_column(node_attrs, 'height')) scale = np.array(ut.dict_take_column(node_attrs, 'scale', default=1.0)) width_in = width_px / 72.0 * scale height_in = height_px / 72.0 * scale width_in_dict = dict(zip(shaped_nodes, width_in)) height_in_dict = dict(zip(shaped_nodes, height_in)) nx.set_node_attributes(graph_, 'width', width_in_dict) nx.set_node_attributes(graph_, 'height', height_in_dict) ut.nx_delete_node_attr(graph_, 'scale') # Check for any nodes with groupids node_to_groupid = nx.get_node_attributes(graph_, 'groupid') if node_to_groupid: groupid_to_nodes = ut.group_items(*zip(*node_to_groupid.items())) else: groupid_to_nodes = {} # Initialize agraph format #import utool #utool.embed() ut.nx_delete_None_edge_attr(graph_) agraph = nx.nx_agraph.to_agraph(graph_) # Add subgraphs labels # TODO: subgraph attrs for groupid, nodes in groupid_to_nodes.items(): subgraph_attrs = {} #subgraph_attrs = dict(rankdir='LR') #subgraph_attrs['rank'] = 'min' subgraph_attrs['rank'] = 'same' name = groupid name = 'cluster_' + groupid agraph.add_subgraph(nodes, name, **subgraph_attrs) for node in graph_.nodes(): # force pinning of node points anode = pygraphviz.Node(agraph, node) if anode.attr['pin'] == 'true': if anode.attr['pos'] is not None and not anode.attr['pos'].endswith('!'): import re #utool.embed() ptstr = anode.attr['pos'].strip('[]').strip(' ') ptstr_list = re.split(r'\s+', ptstr) pt_arr = np.array(list(map(float, ptstr_list))) / 72.0 #print('pt_arr = %r' % (pt_arr,)) new_ptstr_list = list(map(str, pt_arr)) new_ptstr = ','.join(new_ptstr_list) + '!' #print('new_ptstr = %r' % (new_ptstr,)) anode.attr['pos'] = new_ptstr return agraph
def compute_labels_localizations(depc, loc_id_list, config=None): r""" Extracts the detections for a given input image Args: depc (ibeis.depends_cache.DependencyCache): loc_id_list (list): list of localization rowids config (dict): (default = None) Yields: (float, str): tup CommandLine: ibeis compute_labels_localizations Example: >>> # DISABLE_DOCTEST >>> from ibeis.core_images import * # NOQA >>> import ibeis >>> defaultdb = 'PZ_MTEST' >>> ibs = ibeis.opendb(defaultdb=defaultdb) >>> depc = ibs.depc_image >>> gid_list = ibs.get_valid_gids()[0:100] >>> depc.delete_property('labeler', gid_list) >>> results = depc.get_property('labeler', gid_list, None) >>> results = depc.get_property('labeler', gid_list, 'species') >>> print(results) """ from ibeis.algo.detect.labeler.labeler import label_chip_list print('[ibs] Process Localization Labels') print('config = %r' % (config,)) # Get controller ibs = depc.controller depc = ibs.depc_image gid_list_ = depc.get_ancestor_rowids('localizations', loc_id_list, 'images') assert len(gid_list_) == len(loc_id_list) # Grab the localizations bboxes_list = depc.get_native('localizations', loc_id_list, 'bboxes') thetas_list = depc.get_native('localizations', loc_id_list, 'thetas') gids_list = [ np.array([gid] * len(bbox_list)) for gid, bbox_list in zip(gid_list_, bboxes_list) ] # Flatten all of these lists for efficiency bbox_list = ut.flatten(bboxes_list) theta_list = ut.flatten(thetas_list) gid_list = ut.flatten(gids_list) bbox_size_list = ut.take_column(bbox_list, [2, 3]) newsize_list = [(128, 128)] * len(bbox_list) # Checks invalid_flags = [w == 0 or h == 0 for (w, h) in bbox_size_list] invalid_bboxes = ut.compress(bbox_list, invalid_flags) assert len(invalid_bboxes) == 0, 'invalid bboxes=%r' % (invalid_bboxes,) # Build transformation from image to chip M_list = [ vt.get_image_to_chip_transform(bbox, new_size, theta) for bbox, theta, new_size in zip(bbox_list, theta_list, newsize_list) ] # Extract "chips" flags = cv2.INTER_LANCZOS4 borderMode = cv2.BORDER_CONSTANT warpkw = dict(flags=flags, borderMode=borderMode) last_gid = None chip_list = [] for gid, new_size, M in zip(gid_list, newsize_list, M_list): if gid != last_gid: img = ibs.get_images(gid) last_gid = gid chip = cv2.warpAffine(img, M[0:2], tuple(new_size), **warpkw) # cv2.imshow('', chip) # cv2.waitKey() assert chip.shape[0] == 128 and chip.shape[1] == 128 chip_list.append(chip) # Get the results from the algorithm result_list = label_chip_list(chip_list) assert len(gid_list) == len(result_list) # Group the results group_dict = {} for gid, result in zip(gid_list, result_list): if gid not in group_dict: group_dict[gid] = [] group_dict[gid].append(result) assert len(gid_list_) == len(group_dict.keys()) # Return the results for gid in gid_list_: result_list = group_dict[gid] zipped_list = zip(*result_list) ret_tuple = ( np.array(zipped_list[0]), np.array(zipped_list[1]), np.array(zipped_list[2]), np.array(zipped_list[3]), np.array(zipped_list[4]), list(zipped_list[5]), ) # print(ret_tuple[:-1]) # print('-------') yield ret_tuple
def merge_level_order(level_orders, topsort): """ Merge orders of individual subtrees into a total ordering for computation. >>> level_orders = { >>> 'multi_chip_multitest': [['dummy_annot'], ['chip'], ['multitest'], >>> ['multitest_score'], ], >>> 'multi_fgweight_multitest': [ ['dummy_annot'], ['chip', 'probchip'], >>> ['keypoint'], ['fgweight'], ['multitest'], ['multitest_score'], ], >>> 'multi_keypoint_nnindexer': [ ['dummy_annot'], ['chip'], ['keypoint'], >>> ['nnindexer'], ['multitest'], ['multitest_score'], ], >>> 'normal': [ ['dummy_annot'], ['chip', 'probchip'], ['keypoint'], >>> ['fgweight'], ['spam'], ['multitest'], ['multitest_score'], ], >>> 'nwise_notch_multitest_1': [ ['dummy_annot'], ['notch'], ['multitest'], >>> ['multitest_score'], ], >>> 'nwise_notch_multitest_2': [ ['dummy_annot'], ['notch'], ['multitest'], >>> ['multitest_score'], ], >>> 'nwise_notch_notchpair_1': [ ['dummy_annot'], ['notch'], ['notchpair'], >>> ['multitest'], ['multitest_score'], ], >>> 'nwise_notch_notchpair_2': [ ['dummy_annot'], ['notch'], ['notchpair'], >>> ['multitest'], ['multitest_score'], ], >>> } >>> topsort = [u'dummy_annot', u'notch', u'probchip', u'chip', u'keypoint', >>> u'fgweight', u'nnindexer', u'spam', u'notchpair', u'multitest', >>> u'multitest_score'] >>> print(ut.repr3(ut.merge_level_order(level_orders, topsort))) EG2: level_orders = {u'normal': [[u'dummy_annot'], [u'chip', u'probchip'], [u'keypoint'], [u'fgweight'], [u'spam']]} topsort = [u'dummy_annot', u'probchip', u'chip', u'keypoint', u'fgweight', u'spam'] """ import utool as ut if False: compute_order = [] level_orders = ut.map_dict_vals(ut.total_flatten, level_orders) level_sets = ut.map_dict_vals(set, level_orders) for tablekey in topsort: compute_order.append((tablekey, [groupkey for groupkey, set_ in level_sets.items() if tablekey in set_])) return compute_order else: # Do on common subgraph import itertools # Pointer to current level.: Start at the end and # then work your way up. main_ptr = len(topsort) - 1 stack = [] #from six.moves import zip_longest keys = list(level_orders.keys()) type_to_ptr = {key: -1 for key in keys} print('level_orders = %s' % (ut.repr3(level_orders),)) for count in itertools.count(0): print('----') print('count = %r' % (count,)) ptred_levels = [] for key in keys: levels = level_orders[key] ptr = type_to_ptr[key] try: level = tuple(levels[ptr]) except IndexError: level = None ptred_levels.append(level) print('ptred_levels = %r' % (ptred_levels,)) print('main_ptr = %r' % (main_ptr,)) # groupkeys, groupxs = ut.group_indices(ptred_levels) # Group keys are tablenames # They point to the (type) of the input # num_levelkeys = len(ut.total_flatten(ptred_levels)) groupkeys, groupxs = ut.group_indices(ptred_levels) main_idx = None while main_idx is None and main_ptr >= 0: target = topsort[main_ptr] print('main_ptr = %r' % (main_ptr,)) print('target = %r' % (target,)) # main_idx = ut.listfind(groupkeys, (target,)) # if main_idx is None: possible_idxs = [idx for idx, keytup in enumerate(groupkeys) if keytup is not None and target in keytup] if len(possible_idxs) == 1: main_idx = possible_idxs[0] else: main_idx = None if main_idx is None: main_ptr -= 1 if main_idx is None: print('break I') break found_groups = ut.apply_grouping(keys, groupxs)[main_idx] print('found_groups = %r' % (found_groups,)) stack.append((target, found_groups)) for k in found_groups: type_to_ptr[k] -= 1 if len(found_groups) == len(keys): main_ptr -= 1 if main_ptr < 0: print('break E') break print('stack = %s' % (ut.repr3(stack),)) print('have = %r' % (sorted(ut.take_column(stack, 0)),)) print('need = %s' % (sorted(ut.total_flatten(level_orders.values())),)) compute_order = stack[::-1] return compute_order
def draw_twoday_count(ibs, visit_info_list_): import copy visit_info_list = copy.deepcopy(visit_info_list_) aids_day1, aids_day2 = ut.take_column(visit_info_list_, 'aids') nids_day1, nids_day2 = ut.take_column(visit_info_list_, 'unique_nids') resight_nids = ut.isect(nids_day1, nids_day2) if False: # HACK REMOVE DATA TO MAKE THIS FASTER num = 20 for info in visit_info_list: non_resight_nids = list(set(info['unique_nids']) - set(resight_nids)) sample_nids2 = non_resight_nids[0:num] + resight_nids[:num] info['grouped_aids'] = ut.dict_subset(info['grouped_aids'], sample_nids2) info['unique_nids'] = sample_nids2 # Build a graph of matches if False: debug = False for info in visit_info_list: edges = [] grouped_aids = info['grouped_aids'] aids_list = list(grouped_aids.values()) ams_list = ibs.get_annotmatch_rowids_in_cliques(aids_list) aids1_list = ibs.unflat_map(ibs.get_annotmatch_aid1, ams_list) aids2_list = ibs.unflat_map(ibs.get_annotmatch_aid2, ams_list) for ams, aids, aids1, aids2 in zip(ams_list, aids_list, aids1_list, aids2_list): edge_nodes = set(aids1 + aids2) ##if len(edge_nodes) != len(set(aids)): # #print('--') # #print('aids = %r' % (aids,)) # #print('edge_nodes = %r' % (edge_nodes,)) bad_aids = edge_nodes - set(aids) if len(bad_aids) > 0: print('bad_aids = %r' % (bad_aids,)) unlinked_aids = set(aids) - edge_nodes mst_links = list(ut.itertwo(list(unlinked_aids) + list(edge_nodes)[:1])) bad_aids.add(None) user_links = [(u, v) for (u, v) in zip(aids1, aids2) if u not in bad_aids and v not in bad_aids] new_edges = mst_links + user_links new_edges = [(int(u), int(v)) for u, v in new_edges if u not in bad_aids and v not in bad_aids] edges += new_edges info['edges'] = edges # Add edges between days grouped_aids1, grouped_aids2 = ut.take_column(visit_info_list, 'grouped_aids') nids_day1, nids_day2 = ut.take_column(visit_info_list, 'unique_nids') resight_nids = ut.isect(nids_day1, nids_day2) resight_aids1 = ut.take(grouped_aids1, resight_nids) resight_aids2 = ut.take(grouped_aids2, resight_nids) #resight_aids3 = [list(aids1) + list(aids2) for aids1, aids2 in zip(resight_aids1, resight_aids2)] ams_list = ibs.get_annotmatch_rowids_between_groups(resight_aids1, resight_aids2) aids1_list = ibs.unflat_map(ibs.get_annotmatch_aid1, ams_list) aids2_list = ibs.unflat_map(ibs.get_annotmatch_aid2, ams_list) between_edges = [] for ams, aids1, aids2, rawaids1, rawaids2 in zip(ams_list, aids1_list, aids2_list, resight_aids1, resight_aids2): link_aids = aids1 + aids2 rawaids3 = rawaids1 + rawaids2 badaids = ut.setdiff(link_aids, rawaids3) assert not badaids user_links = [(int(u), int(v)) for (u, v) in zip(aids1, aids2) if u is not None and v is not None] # HACK THIS OFF user_links = [] if len(user_links) == 0: # Hack in an edge between_edges += [(rawaids1[0], rawaids2[0])] else: between_edges += user_links assert np.all(0 == np.diff(np.array(ibs.unflat_map(ibs.get_annot_nids, between_edges)), axis=1)) import plottool as pt import networkx as nx #pt.qt4ensure() #len(list(nx.connected_components(graph1))) #print(ut.graph_info(graph1)) # Layout graph layoutkw = dict( prog='neato', draw_implicit=False, splines='line', #splines='curved', #splines='spline', #sep=10 / 72, #prog='dot', rankdir='TB', ) def translate_graph_to_origin(graph): x, y, w, h = ut.get_graph_bounding_box(graph) ut.translate_graph(graph, (-x, -y)) def stack_graphs(graph_list, vert=False, pad=None): graph_list_ = [g.copy() for g in graph_list] for g in graph_list_: translate_graph_to_origin(g) bbox_list = [ut.get_graph_bounding_box(g) for g in graph_list_] if vert: dim1 = 3 dim2 = 2 else: dim1 = 2 dim2 = 3 dim1_list = np.array([bbox[dim1] for bbox in bbox_list]) dim2_list = np.array([bbox[dim2] for bbox in bbox_list]) if pad is None: pad = np.mean(dim1_list) / 2 offset1_list = ut.cumsum([0] + [d + pad for d in dim1_list[:-1]]) max_dim2 = max(dim2_list) offset2_list = [(max_dim2 - d2) / 2 for d2 in dim2_list] if vert: t_xy_list = [(d2, d1) for d1, d2 in zip(offset1_list, offset2_list)] else: t_xy_list = [(d1, d2) for d1, d2 in zip(offset1_list, offset2_list)] for g, t_xy in zip(graph_list_, t_xy_list): ut.translate_graph(g, t_xy) nx.set_node_attributes(g, name='pin', values='true') new_graph = nx.compose_all(graph_list_) #pt.show_nx(new_graph, layout='custom', node_labels=False, as_directed=False) # NOQA return new_graph # Construct graph for count, info in enumerate(visit_info_list): graph = nx.Graph() edges = [(int(u), int(v)) for u, v in info['edges'] if u is not None and v is not None] graph.add_edges_from(edges, attr_dict={'zorder': 10}) nx.set_node_attributes(graph, name='zorder', values=20) # Layout in neato _ = pt.nx_agraph_layout(graph, inplace=True, **layoutkw) # NOQA # Extract components and then flatten in nid ordering ccs = list(nx.connected_components(graph)) root_aids = [] cc_graphs = [] for cc_nodes in ccs: cc = graph.subgraph(cc_nodes) try: root_aids.append(list(ut.nx_source_nodes(cc.to_directed()))[0]) except nx.NetworkXUnfeasible: root_aids.append(list(cc.nodes())[0]) cc_graphs.append(cc) root_nids = ibs.get_annot_nids(root_aids) nid2_graph = dict(zip(root_nids, cc_graphs)) resight_nids_ = set(resight_nids).intersection(set(root_nids)) noresight_nids_ = set(root_nids) - resight_nids_ n_graph_list = ut.take(nid2_graph, sorted(noresight_nids_)) r_graph_list = ut.take(nid2_graph, sorted(resight_nids_)) if len(n_graph_list) > 0: n_graph = nx.compose_all(n_graph_list) _ = pt.nx_agraph_layout(n_graph, inplace=True, **layoutkw) # NOQA n_graphs = [n_graph] else: n_graphs = [] r_graphs = [stack_graphs(chunk) for chunk in ut.ichunks(r_graph_list, 100)] if count == 0: new_graph = stack_graphs(n_graphs + r_graphs, vert=True) else: new_graph = stack_graphs(r_graphs[::-1] + n_graphs, vert=True) #pt.show_nx(new_graph, layout='custom', node_labels=False, as_directed=False) # NOQA info['graph'] = new_graph graph1_, graph2_ = ut.take_column(visit_info_list, 'graph') if False: _ = pt.show_nx(graph1_, layout='custom', node_labels=False, as_directed=False) # NOQA _ = pt.show_nx(graph2_, layout='custom', node_labels=False, as_directed=False) # NOQA graph_list = [graph1_, graph2_] twoday_graph = stack_graphs(graph_list, vert=True, pad=None) nx.set_node_attributes(twoday_graph, name='pin', values='true') if debug: ut.nx_delete_None_edge_attr(twoday_graph) ut.nx_delete_None_node_attr(twoday_graph) print('twoday_graph(pre) info' + ut.repr3(ut.graph_info(twoday_graph), nl=2)) # Hack, no idea why there are nodes that dont exist here between_edges_ = [edge for edge in between_edges if twoday_graph.has_node(edge[0]) and twoday_graph.has_node(edge[1])] twoday_graph.add_edges_from(between_edges_, attr_dict={'alpha': .2, 'zorder': 0}) ut.nx_ensure_agraph_color(twoday_graph) layoutkw['splines'] = 'line' layoutkw['prog'] = 'neato' agraph = pt.nx_agraph_layout(twoday_graph, inplace=True, return_agraph=True, **layoutkw)[-1] # NOQA if False: fpath = ut.truepath('~/ggr_graph.png') agraph.draw(fpath) ut.startfile(fpath) if debug: print('twoday_graph(post) info' + ut.repr3(ut.graph_info(twoday_graph))) _ = pt.show_nx(twoday_graph, layout='custom', node_labels=False, as_directed=False) # NOQA
def grep_projects(tofind_list, user_profile=None, verbose=True, new=False, **kwargs): r""" Greps the projects defined in the current UserProfile Args: tofind_list (list): user_profile (None): (default = None) Kwargs: user_profile CommandLine: python -m utool --tf grep_projects grep_projects Example: >>> # DISABLE_DOCTEST >>> from utool.util_project import * # NOQA >>> import utool as ut >>> import sys >>> tofind_list = ut.get_argval('--find', type_=list, >>> default=[sys.argv[-1]]) >>> grep_projects(tofind_list) """ import utool as ut user_profile = ensure_user_profile(user_profile) kwargs = kwargs.copy() colored = kwargs.pop('colored', True) grepkw = {} grepkw['greater_exclude_dirs'] = user_profile.project_exclude_dirs grepkw['exclude_dirs'] = user_profile.project_exclude_dirs grepkw['dpath_list'] = user_profile.project_dpaths grepkw['include_patterns'] = user_profile.project_include_patterns grepkw['exclude_patterns'] = user_profile.project_exclude_patterns grepkw.update(kwargs) msg_list1 = [] msg_list2 = [] print_ = msg_list1.append print_('Greping Projects') print_('tofind_list = %s' % (ut.repr4(tofind_list, nl=True),)) #print_('grepkw = %s' % ut.repr4(grepkw, nl=True)) if verbose: print('\n'.join(msg_list1)) #with ut.Timer('greping', verbose=True): grep_result = ut.grep(tofind_list, **grepkw) found_fpath_list, found_lines_list, found_lxs_list = grep_result # HACK, duplicate behavior. TODO: write grep print result function reflags = grepkw.get('reflags', 0) _exprs_flags = [ut.extend_regex2(expr, reflags) for expr in tofind_list] extended_regex_list = ut.take_column(_exprs_flags, 0) reflags_list = ut.take_column(_exprs_flags, 1) # HACK # pat = ut.util_regex.regex_or(extended_regex_list) reflags = reflags_list[0] # from utool import util_regex resultstr = ut.make_grep_resultstr(grep_result, extended_regex_list, reflags, colored=colored) msg_list2.append(resultstr) print_ = msg_list2.append #for fpath, lines, lxs in zip(found_fpath_list, found_lines_list, # found_lxs_list): # print_('----------------------') # print_('found %d line(s) in %r: ' % (len(lines), fpath)) # name = split(fpath)[1] # max_line = len(lines) # ndigits = str(len(str(max_line))) # for (lx, line) in zip(lxs, lines): # line = line.replace('\n', '') # print_(('%s : %' + ndigits + 'd |%s') % (name, lx, line)) # iter_ = zip(found_fpath_list, found_lines_list, found_lxs_list) # for fpath, lines, lxs in iter_: # print_('----------------------') # print_('found %d line(s) in %r: ' % (len(lines), fpath)) # name = split(fpath)[1] # max_line = len(lines) # ndigits = str(len(str(max_line))) # for (lx, line) in zip(lxs, lines): # line = line.replace('\n', '') # colored_line = ut.highlight_regex( # line.rstrip('\n'), pat, reflags=reflags) # print_(('%s : %' + ndigits + 'd |%s') % (name, lx, colored_line)) print_('====================') print_('found_fpath_list = ' + ut.repr4(found_fpath_list)) print_('') #print_('gvim -o ' + ' '.join(found_fpath_list)) if verbose: print('\n'.join(msg_list2)) msg_list = msg_list1 + msg_list2 if new: return GrepResult(found_fpath_list, found_lines_list, found_lxs_list, extended_regex_list, reflags) else: return msg_list
def split_analysis(ibs): """ CommandLine: python -m ibeis.other.dbinfo split_analysis --show python -m ibeis split_analysis --show python -m ibeis split_analysis --show --good Ignore: # mount sshfs -o idmap=user lev:/ ~/lev # unmount fusermount -u ~/lev Example: >>> # DISABLE_DOCTEST GGR >>> from ibeis.other.dbinfo import * # NOQA >>> import ibeis >>> dbdir = '/media/danger/GGR/GGR-IBEIS' >>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS') >>> ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False) >>> import guitool as gt >>> gt.ensure_qtapp() >>> win = split_analysis(ibs) >>> ut.quit_if_noshow() >>> import plottool as pt >>> gt.qtapp_loop(qwin=win) >>> #ut.show_if_requested() """ #nid_list = ibs.get_valid_nids(filter_empty=True) import datetime day1 = datetime.date(2016, 1, 30) day2 = datetime.date(2016, 1, 31) filter_kw = { 'multiple': None, #'view': ['right'], #'minqual': 'good', 'is_known': True, 'min_pername': 1, } aids1 = ibs.filter_annots_general(filter_kw=ut.dict_union( filter_kw, { 'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)), 'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 1.0)), }) ) aids2 = ibs.filter_annots_general(filter_kw=ut.dict_union( filter_kw, { 'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 0.0)), 'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)), }) ) all_aids = aids1 + aids2 all_annots = ibs.annots(all_aids) print('%d annots on day 1' % (len(aids1)) ) print('%d annots on day 2' % (len(aids2)) ) print('%d annots overall' % (len(all_annots)) ) print('%d names overall' % (len(ut.unique(all_annots.nids))) ) nid_list, annots_list = all_annots.group(all_annots.nids) REVIEWED_EDGES = True if REVIEWED_EDGES: aids_list = [annots.aids for annots in annots_list] #aid_pairs = [annots.get_am_aidpairs() for annots in annots_list] # Slower aid_pairs = ibs.get_unflat_am_aidpairs(aids_list) # Faster else: # ALL EDGES aid_pairs = [annots.get_aidpairs() for annots in annots_list] speeds_list = ibs.unflat_map(ibs.get_annotpair_speeds, aid_pairs) import vtool as vt max_speeds = np.array([vt.safe_max(s, nans=False) for s in speeds_list]) nan_idx = np.where(np.isnan(max_speeds))[0] inf_idx = np.where(np.isinf(max_speeds))[0] bad_idx = sorted(ut.unique(ut.flatten([inf_idx, nan_idx]))) ok_idx = ut.index_complement(bad_idx, len(max_speeds)) print('#nan_idx = %r' % (len(nan_idx),)) print('#inf_idx = %r' % (len(inf_idx),)) print('#ok_idx = %r' % (len(ok_idx),)) ok_speeds = max_speeds[ok_idx] ok_nids = ut.take(nid_list, ok_idx) ok_annots = ut.take(annots_list, ok_idx) sortx = np.argsort(ok_speeds)[::-1] sorted_speeds = np.array(ut.take(ok_speeds, sortx)) sorted_annots = np.array(ut.take(ok_annots, sortx)) sorted_nids = np.array(ut.take(ok_nids, sortx)) # NOQA sorted_speeds = np.clip(sorted_speeds, 0, 100) #idx = vt.find_elbow_point(sorted_speeds) #EXCESSIVE_SPEED = sorted_speeds[idx] # http://www.infoplease.com/ipa/A0004737.html # http://www.speedofanimals.com/animals/zebra #ZEBRA_SPEED_MAX = 64 # km/h #ZEBRA_SPEED_RUN = 50 # km/h ZEBRA_SPEED_SLOW_RUN = 20 # km/h #ZEBRA_SPEED_FAST_WALK = 10 # km/h #ZEBRA_SPEED_WALK = 7 # km/h MAX_SPEED = ZEBRA_SPEED_SLOW_RUN #MAX_SPEED = ZEBRA_SPEED_WALK #MAX_SPEED = EXCESSIVE_SPEED flags = sorted_speeds > MAX_SPEED flagged_ok_annots = ut.compress(sorted_annots, flags) inf_annots = ut.take(annots_list, inf_idx) flagged_annots = inf_annots + flagged_ok_annots print('MAX_SPEED = %r km/h' % (MAX_SPEED,)) print('%d annots with infinite speed' % (len(inf_annots),)) print('%d annots with large speed' % (len(flagged_ok_annots),)) print('Marking all pairs of annots above the threshold as non-matching') from ibeis.algo.graph import graph_iden import networkx as nx progkw = dict(freq=1, bs=True, est_window=len(flagged_annots)) bad_edges_list = [] good_edges_list = [] for annots in ut.ProgIter(flagged_annots, lbl='flag speeding names', **progkw): edge_to_speeds = annots.get_speeds() bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED] good_edges = [edge for edge, speed in edge_to_speeds.items() if speed <= MAX_SPEED] bad_edges_list.append(bad_edges) good_edges_list.append(good_edges) all_bad_edges = ut.flatten(bad_edges_list) good_edges_list = ut.flatten(good_edges_list) print('num_bad_edges = %r' % (len(ut.flatten(bad_edges_list)),)) print('num_bad_edges = %r' % (len(ut.flatten(good_edges_list)),)) if 1: from ibeis.viz import viz_graph2 import guitool as gt gt.ensure_qtapp() if ut.get_argflag('--good'): print('Looking at GOOD (no speed problems) edges') aid_pairs = good_edges_list else: print('Looking at BAD (speed problems) edges') aid_pairs = all_bad_edges aids = sorted(list(set(ut.flatten(aid_pairs)))) infr = graph_iden.AnnotInference(ibs, aids, verbose=False) infr.initialize_graph() # Use random scores to randomize sort order rng = np.random.RandomState(0) scores = (-rng.rand(len(aid_pairs)) * 10).tolist() infr.graph.add_edges_from(aid_pairs) if True: edge_sample_size = 250 pop_nids = ut.unique(ibs.get_annot_nids(ut.unique(ut.flatten(aid_pairs)))) sorted_pairs = ut.sortedby(aid_pairs, scores)[::-1][0:edge_sample_size] sorted_nids = ibs.get_annot_nids(ut.take_column(sorted_pairs, 0)) sample_size = len(ut.unique(sorted_nids)) am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(*zip(*sorted_pairs)) flags = ut.not_list(ut.flag_None_items(am_rowids)) #am_rowids = ut.compress(am_rowids, flags) positive_tags = ['SplitCase', 'Photobomb'] flags_list = [ut.replace_nones(ibs.get_annotmatch_prop(tag, am_rowids), 0) for tag in positive_tags] print('edge_case_hist: ' + ut.repr3( ['%s %s' % (txt, sum(flags_)) for flags_, txt in zip(flags_list, positive_tags)])) is_positive = ut.or_lists(*flags_list) num_positive = sum(ut.lmap(any, ut.group_items(is_positive, sorted_nids).values())) pop = len(pop_nids) print('A positive is any edge flagged as a %s' % (ut.conj_phrase(positive_tags, 'or'),)) print('--- Sampling wrt edges ---') print('edge_sample_size = %r' % (edge_sample_size,)) print('edge_population_size = %r' % (len(aid_pairs),)) print('num_positive_edges = %r' % (sum(is_positive))) print('--- Sampling wrt names ---') print('name_population_size = %r' % (pop,)) vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level=.95) nx.set_edge_attributes(infr.graph, name='score', values=dict(zip(aid_pairs, scores))) win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False, init_mode=None) win.populate_edge_model() win.show() return win # Make review interface for only bad edges infr_list = [] iter_ = list(zip(flagged_annots, bad_edges_list)) for annots, bad_edges in ut.ProgIter(iter_, lbl='creating inference', **progkw): aids = annots.aids nids = [1] * len(aids) infr = graph_iden.AnnotInference(ibs, aids, nids, verbose=False) infr.initialize_graph() infr.reset_feedback() infr_list.append(infr) # Check which ones are user defined as incorrect #num_positive = 0 #for infr in infr_list: # flag = np.any(infr.get_feedback_probs()[0] == 0) # num_positive += flag #print('num_positive = %r' % (num_positive,)) #pop = len(infr_list) #print('pop = %r' % (pop,)) iter_ = list(zip(infr_list, bad_edges_list)) for infr, bad_edges in ut.ProgIter(iter_, lbl='adding speed edges', **progkw): flipped_edges = [] for aid1, aid2 in bad_edges: if infr.graph.has_edge(aid1, aid2): flipped_edges.append((aid1, aid2)) infr.add_feedback((aid1, aid2), NEGTV) nx.set_edge_attributes(infr.graph, name='_speed_split', values='orig') nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'new' for edge in bad_edges}) nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'flip' for edge in flipped_edges}) #for infr in ut.ProgIter(infr_list, lbl='flagging speeding edges', **progkw): # annots = ibs.annots(infr.aids) # edge_to_speeds = annots.get_speeds() # bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED] def inference_stats(infr_list_): relabel_stats = [] for infr in infr_list_: num_ccs, num_inconsistent = infr.relabel_using_reviews() state_hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, 'decision').values()) if POSTV not in state_hist: state_hist[POSTV] = 0 hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, '_speed_split').values()) subgraphs = infr.positive_connected_compoments() subgraph_sizes = [len(g) for g in subgraphs] info = ut.odict([ ('num_nonmatch_edges', state_hist[NEGTV]), ('num_match_edges', state_hist[POSTV]), ('frac_nonmatch_edges', state_hist[NEGTV] / (state_hist[POSTV] + state_hist[NEGTV])), ('num_inconsistent', num_inconsistent), ('num_ccs', num_ccs), ('edges_flipped', hist.get('flip', 0)), ('edges_unchanged', hist.get('orig', 0)), ('bad_unreviewed_edges', hist.get('new', 0)), ('orig_size', len(infr.graph)), ('new_sizes', subgraph_sizes), ]) relabel_stats.append(info) return relabel_stats relabel_stats = inference_stats(infr_list) print('\nAll Split Info:') lines = [] for key in relabel_stats[0].keys(): data = ut.take_column(relabel_stats, key) if key == 'new_sizes': data = ut.flatten(data) lines.append('stats(%s) = %s' % (key, ut.repr2(ut.get_stats(data, use_median=True), precision=2))) print('\n'.join(ut.align_lines(lines, '='))) num_incon_list = np.array(ut.take_column(relabel_stats, 'num_inconsistent')) can_split_flags = num_incon_list == 0 print('Can trivially split %d / %d' % (sum(can_split_flags), len(can_split_flags))) splittable_infrs = ut.compress(infr_list, can_split_flags) relabel_stats = inference_stats(splittable_infrs) print('\nTrival Split Info:') lines = [] for key in relabel_stats[0].keys(): if key in ['num_inconsistent']: continue data = ut.take_column(relabel_stats, key) if key == 'new_sizes': data = ut.flatten(data) lines.append('stats(%s) = %s' % ( key, ut.repr2(ut.get_stats(data, use_median=True), precision=2))) print('\n'.join(ut.align_lines(lines, '='))) num_match_edges = np.array(ut.take_column(relabel_stats, 'num_match_edges')) num_nonmatch_edges = np.array(ut.take_column(relabel_stats, 'num_nonmatch_edges')) flags1 = np.logical_and(num_match_edges > num_nonmatch_edges, num_nonmatch_edges < 3) reasonable_infr = ut.compress(splittable_infrs, flags1) new_sizes_list = ut.take_column(relabel_stats, 'new_sizes') flags2 = [len(sizes) == 2 and sum(sizes) > 4 and (min(sizes) / max(sizes)) > .3 for sizes in new_sizes_list] reasonable_infr = ut.compress(splittable_infrs, flags2) print('#reasonable_infr = %r' % (len(reasonable_infr),)) for infr in ut.InteractiveIter(reasonable_infr): annots = ibs.annots(infr.aids) edge_to_speeds = annots.get_speeds() print('max_speed = %r' % (max(edge_to_speeds.values())),) infr.initialize_visual_node_attrs() infr.show_graph(use_image=True, only_reviewed=True) rest = ~np.logical_or(flags1, flags2) nonreasonable_infr = ut.compress(splittable_infrs, rest) rng = np.random.RandomState(0) random_idx = ut.random_indexes(len(nonreasonable_infr) - 1, 15, rng=rng) random_infr = ut.take(nonreasonable_infr, random_idx) for infr in ut.InteractiveIter(random_infr): annots = ibs.annots(infr.aids) edge_to_speeds = annots.get_speeds() print('max_speed = %r' % (max(edge_to_speeds.values())),) infr.initialize_visual_node_attrs() infr.show_graph(use_image=True, only_reviewed=True) #import scipy.stats as st #conf_interval = .95 #st.norm.cdf(conf_interval) # view-source:http://www.surveysystem.com/sscalc.htm #zval = 1.96 # 95 percent confidence #zValC = 3.8416 # #zValC = 6.6564 #import statsmodels.stats.api as sms #es = sms.proportion_effectsize(0.5, 0.75) #sms.NormalIndPower().solve_power(es, power=0.9, alpha=0.05, ratio=1) pop = 279 num_positive = 3 sample_size = 15 conf_level = .95 #conf_level = .99 vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level) print('---') vt.calc_error_bars_from_sample(sample_size + 38, num_positive, pop, conf_level) print('---') vt.calc_error_bars_from_sample(sample_size + 38 / 3, num_positive, pop, conf_level) print('---') vt.calc_error_bars_from_sample(15 + 38, num_positive=3, pop=675, conf_level=.95) vt.calc_error_bars_from_sample(15, num_positive=3, pop=675, conf_level=.95) pop = 279 #err_frac = .05 # 5% err_frac = .10 # 10% conf_level = .95 vt.calc_sample_from_error_bars(err_frac, pop, conf_level) pop = 675 vt.calc_sample_from_error_bars(err_frac, pop, conf_level) vt.calc_sample_from_error_bars(.05, pop, conf_level=.95, prior=.1) vt.calc_sample_from_error_bars(.05, pop, conf_level=.68, prior=.2) vt.calc_sample_from_error_bars(.10, pop, conf_level=.68) vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.95) vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.68)
def estimate_twoday_count(ibs, day1, day2, filter_kw): #gid_list = ibs.get_valid_gids() all_images = ibs.images() dates = [dt.date() for dt in all_images.datetime] date_to_images = all_images.group_items(dates) date_to_images = ut.sort_dict(date_to_images) #date_hist = ut.map_dict_vals(len, date2_gids) #print('date_hist = %s' % (ut.repr2(date_hist, nl=2),)) verbose = 0 visit_dates = [day1, day2] visit_info_list_ = [] for day in visit_dates: images = date_to_images[day] aids = ut.flatten(images.aids) aids = ibs.filter_annots_general(aids, filter_kw=filter_kw, verbose=verbose) nids = ibs.get_annot_name_rowids(aids) grouped_aids = ut.group_items(aids, nids) unique_nids = ut.unique(list(grouped_aids.keys())) if False: aids_list = ut.take(grouped_aids, unique_nids) for aids in aids_list: if len(aids) > 30: break timedeltas_list = ibs.get_unflat_annots_timedelta_list(aids_list) # Do the five second rule marked_thresh = 5 flags = [] for nid, timedeltas in zip(unique_nids, timedeltas_list): flags.append(timedeltas.max() > marked_thresh) print('Unmarking %d names' % (len(flags) - sum(flags))) unique_nids = ut.compress(unique_nids, flags) grouped_aids = ut.dict_subset(grouped_aids, unique_nids) unique_aids = ut.flatten(list(grouped_aids.values())) info = { 'unique_nids': unique_nids, 'grouped_aids': grouped_aids, 'unique_aids': unique_aids, } visit_info_list_.append(info) # Estimate statistics from ibeis.other import dbinfo aids_day1, aids_day2 = ut.take_column(visit_info_list_, 'unique_aids') nids_day1, nids_day2 = ut.take_column(visit_info_list_, 'unique_nids') resight_nids = ut.isect(nids_day1, nids_day2) nsight1 = len(nids_day1) nsight2 = len(nids_day2) resight = len(resight_nids) lp_index, lp_error = dbinfo.sight_resight_count(nsight1, nsight2, resight) if False: from ibeis.other import dbinfo print('DAY 1 STATS:') _ = dbinfo.get_dbinfo(ibs, aid_list=aids_day1) # NOQA print('DAY 2 STATS:') _ = dbinfo.get_dbinfo(ibs, aid_list=aids_day2) # NOQA print('COMBINED STATS:') _ = dbinfo.get_dbinfo(ibs, aid_list=aids_day1 + aids_day2) # NOQA print('%d annots on day 1' % (len(aids_day1)) ) print('%d annots on day 2' % (len(aids_day2)) ) print('%d names on day 1' % (nsight1,)) print('%d names on day 2' % (nsight2,)) print('resight = %r' % (resight,)) print('lp_index = %r ± %r' % (lp_index, lp_error)) return nsight1, nsight2, resight, lp_index, lp_error
def mark_unreviewed_above_score_as_correct(qres_wgt): selected_qtindex_list = qres_wgt.selectedRows() if len(selected_qtindex_list) == 1: qtindex = selected_qtindex_list[0] #aid1, aid2 = qres_wgt.get_aidpair_from_qtindex(qtindex) thresh = qtindex.model().get_header_data('score', qtindex) print('thresh = %r' % (thresh, )) rows = qres_wgt.review_api.ider() scores_ = qres_wgt.review_api.get( qres_wgt.review_api.col_name_list.index('score'), rows) valid_rows = ut.compress(rows, scores_ >= thresh) aids1 = qres_wgt.review_api.get( qres_wgt.review_api.col_name_list.index('qaid'), valid_rows) aids2 = qres_wgt.review_api.get( qres_wgt.review_api.col_name_list.index('aid'), valid_rows) #ibs = qres_wgt.ibs ibs = qres_wgt.ibs am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey( aids1, aids2) reviewed = ibs.get_annotmatch_reviewed(am_rowids) unreviewed = ut.not_list(reviewed) valid_rows = ut.compress(valid_rows, unreviewed) aids1 = ut.compress(aids1, unreviewed) aids2 = ut.compress(aids2, unreviewed) import networkx as nx graph = nx.Graph() graph.add_edges_from(list(zip(aids1, aids2)), {'user_thresh_match': True}) review_groups = list(nx.connected_component_subgraphs(graph)) changing_aids = list(graph.nodes()) nids = ibs.get_annot_nids(changing_aids) nid2_aids = ut.group_items(changing_aids, nids) for nid, aids in nid2_aids.items(): # Connect all original names in the database to denote merges for u, v in ut.itertwo(aids): graph.add_edge(u, v) dbside_groups = list(nx.connected_component_subgraphs(graph)) options = [ 'Accept', #'Review More' ] msg = ut.codeblock(''' There are %d names and %d annotations in this mass review set. Mass review has discovered %d internal groups. Accepting will induce a database grouping of %d names. ''') % (len(nid2_aids), len(changing_aids), len(review_groups), len(dbside_groups)) reply = gt.user_option(msg=msg, options=options) if reply == options[0]: # This is not the smartest way to group names. # Ideally what will happen here, is that reviewed edges will go into # the new graph name inference algorithm. # then the chosen point will be used as the threshold. Then # the graph cut algorithm will be applied. logger = qres_wgt.logger logger.debug(msg) logger.info('START MASS_THRESHOLD_MERGE') logger.info('num_groups=%d thresh=%r' % ( len(dbside_groups), thresh, )) for count, subgraph in enumerate(dbside_groups): thresh_aid_pairs = [ edge for edge, flag in nx.get_edge_attributes( graph, 'user_thresh_match').items() if flag ] thresh_uuid_pairs = ibs.unflat_map(ibs.get_annot_uuids, thresh_aid_pairs) aids = list(subgraph.nodes()) nids = ibs.get_annot_name_rowids(aids) flags = ut.not_list(ibs.is_aid_unknown(aids)) previous_names = ibs.get_name_texts(nids) valid_nids = ut.compress(nids, flags) if len(valid_nids) == 0: merge_nid = ibs.make_next_nids(num=1)[0] type_ = 'new' else: merge_nid = min(valid_nids) type_ = 'existing' # Need to find other non-exemplar / query names that may # need merging other_aids = ibs.get_name_aids(valid_nids) other_aids = set(ut.flatten(other_aids)) - set(aids) other_auuids = ibs.get_annot_uuids(other_aids) other_previous_names = ibs.get_annot_names(other_aids) merge_name = ibs.get_name_texts(merge_nid) annot_uuids = ibs.get_annot_uuids(aids) ### # Set as reviewed (so we dont see them again), but mark it # with a different code to denote that it was a MASS review aid1_list = ut.take_column(thresh_aid_pairs, 0) aid2_list = ut.take_column(thresh_aid_pairs, 1) am_rowids = ibs.add_annotmatch_undirected( aid1_list, aid2_list) ibs.set_annotmatch_reviewer( am_rowids, ['algo:lnbnn_thresh'] * len(am_rowids)) logger.info('START GROUP %d' % (count, )) logger.info( 'GROUP BASED ON %d ANNOT_PAIRS WITH SCORE ABOVE (thresh=%r)' % ( len(thresh_uuid_pairs), thresh, )) logger.debug('(uuid_pairs=%r)' % (thresh_uuid_pairs)) logger.debug('(merge_name=%r)' % (merge_name)) logger.debug( 'CHANGE NAME OF %d (annot_uuids=%r) WITH (previous_names=%r) TO (%s) (merge_name=%r)' % (len(annot_uuids), annot_uuids, previous_names, type_, merge_name)) logger.debug( 'ADDITIONAL CHANGE NAME OF %d (annot_uuids=%r) WITH (previous_names=%r) TO (%s) (merge_name=%r)' % (len(other_auuids), other_auuids, other_previous_names, type_, merge_name)) logger.info('END GROUP %d' % (count, )) new_nids = [merge_nid] * len(aids) ibs.set_annot_name_rowids(aids, new_nids) logger.info('END MASS_THRESHOLD_MERGE') else: print('[context] Multiple %d selection' % (len(selected_qtindex_list), ))
def draw_feat_scoresep(testres, f=None, disttype=None): r""" SeeAlso: wbia.unstable.scorenorm.train_featscore_normalizer CommandLine: python -m wbia --tf TestResult.draw_feat_scoresep --show python -m wbia --tf TestResult.draw_feat_scoresep --show -t default:sv_on=[True,False] python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_Master1 python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_Master1 --disttype=L2_sift,fg python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_Master1 --disttype=L2_sift python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_MTEST -t best:lnbnn_on=True --namemode=True python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_MTEST -t best:lnbnn_on=True --namemode=False python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_MTEST --disttype=L2_sift python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_MTEST --disttype=L2_sift -t best:SV=False utprof.py -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_Master1 utprof.py -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_Master1 --fsvx=1:2 utprof.py -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_Master1 --fsvx=0:1 utprof.py -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_Master1 -t best:lnbnn_on=False,bar_l2_on=True --fsvx=0:1 # We want to query the oxford annots taged query # and we want the database to contain # K correct images per query, as well as the distractors python -m wbia --tf TestResult.draw_feat_scoresep --show --db Oxford -a default:qhas_any=\(query,\),dpername=1,exclude_reference=True,minqual=ok python -m wbia --tf TestResult.draw_feat_scoresep --show --db Oxford -a default:qhas_any=\(query,\),dpername=1,exclude_reference=True,minqual=good python -m wbia --tf get_annotcfg_list --db PZ_Master1 -a timectrl --acfginfo --verbtd --veryverbtd --nocache-aid python -m wbia --tf TestResult.draw_feat_scoresep --show --db PZ_MTEST --disttype=ratio Example: >>> # SCRIPT >>> from wbia.expt.test_result import * # NOQA >>> from wbia.init import main_helpers >>> disttype = ut.get_argval('--disttype', type_=list, default=None) >>> ibs, testres = main_helpers.testdata_expts( >>> defaultdb='PZ_MTEST', a=['timectrl'], t=['best']) >>> f = ut.get_argval(('--filt', '-f'), type_=list, default=['']) >>> testres.draw_feat_scoresep(f=f) >>> ut.show_if_requested() """ logger.info('[testres] draw_feat_scoresep') import wbia.plottool as pt def load_feat_scores(qreq_, qaids): import wbia # NOQA from os.path import dirname, join # NOQA # HACKY CACHE cfgstr = qreq_.get_cfgstr(with_input=True) cache_dir = join(dirname(dirname(wbia.__file__)), 'TMP_FEATSCORE_CACHE') namemode = ut.get_argval('--namemode', default=True) fsvx = ut.get_argval('--fsvx', type_='fuzzy_subset', default=slice(None, None, None)) threshx = ut.get_argval('--threshx', type_=int, default=None) thresh = ut.get_argval('--thresh', type_=float, default=0.9) num = ut.get_argval('--num', type_=int, default=1) cfg_components = [ cfgstr, disttype, namemode, fsvx, threshx, thresh, f, num ] cache_cfgstr = ','.join(ut.lmap(six.text_type, cfg_components)) cache_hashid = ut.hashstr27(cache_cfgstr + '_v1') cache_name = 'get_cfgx_feat_scores_' + cache_hashid @ut.cached_func(cache_name, cache_dir=cache_dir, key_argx=[], use_cache=True) def get_cfgx_feat_scores(qreq_, qaids): from wbia.algo.hots import scorenorm cm_list = qreq_.execute(qaids) # logger.info('Done loading cached chipmatches') tup = scorenorm.get_training_featscores(qreq_, cm_list, disttype, namemode, fsvx, threshx, thresh, num=num) # logger.info(ut.depth_profile(tup)) tp_scores, tn_scores, scorecfg = tup return tp_scores, tn_scores, scorecfg tp_scores, tn_scores, scorecfg = get_cfgx_feat_scores(qreq_, qaids) return tp_scores, tn_scores, scorecfg valid_case_pos = testres.case_sample2(filt_cfg=f, return_mask=False) cfgx2_valid_qxs = ut.group_items(valid_case_pos.T[0], valid_case_pos.T[1]) test_qaids = testres.get_test_qaids() cfgx2_valid_qaids = ut.map_dict_vals(ut.partial(ut.take, test_qaids), cfgx2_valid_qxs) join_acfgs = True # TODO: option to average over pipeline configurations if join_acfgs: groupxs = testres.get_cfgx_groupxs() else: groupxs = list(zip(range(len(testres.cfgx2_qreq_)))) grouped_qreqs = ut.apply_grouping(testres.cfgx2_qreq_, groupxs) grouped_scores = [] for cfgxs, qreq_group in zip(groupxs, grouped_qreqs): # testres.print_pcfg_info() score_group = [] for cfgx, qreq_ in zip(cfgxs, testres.cfgx2_qreq_): logger.info('Loading cached chipmatches') qaids = cfgx2_valid_qaids[cfgx] tp_scores, tn_scores, scorecfg = load_feat_scores(qreq_, qaids) score_group.append((tp_scores, tn_scores, scorecfg)) grouped_scores.append(score_group) cfgx2_shortlbl = testres.get_short_cfglbls(join_acfgs=join_acfgs) for score_group, lbl in zip(grouped_scores, cfgx2_shortlbl): tp_scores = np.hstack(ut.take_column(score_group, 0)) tn_scores = np.hstack(ut.take_column(score_group, 1)) scorecfg = '+++'.join(ut.unique(ut.take_column(score_group, 2))) score_group # TODO: learn this score normalizer as a model # encoder = vt.ScoreNormalizer(adjust=4, monotonize=False) encoder = vt.ScoreNormalizer(adjust=2, monotonize=True) encoder.fit_partitioned(tp_scores, tn_scores, verbose=False) figtitle = 'Feature Scores: %s, %s' % (scorecfg, lbl) fnum = None vizkw = {} sephack = ut.get_argflag('--sephack') if not sephack: vizkw['target_tpr'] = 0.95 vizkw['score_range'] = (0, 1.0) encoder.visualize( figtitle=figtitle, fnum=fnum, with_scores=False, # with_prebayes=True, with_prebayes=False, with_roc=True, with_postbayes=False, # with_postbayes=True, **vizkw, ) icon = testres.ibs.get_database_icon() if icon is not None: pt.overlay_icon(icon, coords=(1, 0), bbox_alignment=(1, 0)) if ut.get_argflag('--contextadjust'): pt.adjust_subplots(left=0.1, bottom=0.25, wspace=0.2, hspace=0.2) pt.adjust_subplots(use_argv=True) return encoder
def make_review_api(ibs, cm_list, review_cfg, qreq_=None): """ Builds columns which are displayable in a ColumnListTableWidget CommandLine: python -m wbia.gui.id_review_api --test-test_review_widget --show python -m wbia.gui.id_review_api --test-make_review_api Example: >>> # ENABLE_DOCTEST >>> from wbia.gui.id_review_api import * # NOQA >>> import wbia >>> import wbia.guitool as gt >>> from wbia.gui import id_review_api >>> cm_list, qreq_ = wbia.main_helpers.testdata_cmlist() >>> tblname = 'chipmatch' >>> name_scoring = False >>> ranks_top = 5 >>> review_cfg = dict(ranks_top=ranks_top, name_scoring=name_scoring) >>> review_api = make_review_api(qreq_.ibs, cm_list, review_cfg, qreq_=qreq_) >>> print('review_api = %r' % (review_api,)) """ # TODO: Add in timedelta to column info if ut.VERBOSE: logger.info('[inspect] make_review_api') review_edges = get_review_edges(cm_list, ibs=ibs, review_cfg=review_cfg) # Get extra info (qaids, daids, scores, ranks) = review_edges RES_THUMB_TEXT = 'ResThumb' # NOQA QUERY_THUMB_TEXT = 'querythumb' MATCH_THUMB_TEXT = 'MatchThumb' col_name_list = [ 'result_index', 'score', REVIEWED_STATUS_TEXT, ] if review_cfg.get('show_chips', True): col_name_list += [ MATCHED_STATUS_TEXT, QUERY_THUMB_TEXT, ] col_name_list += [ RES_THUMB_TEXT, 'qaid', 'aid', 'rank', 'timedelta', 'dnGt', 'qnGt', 'tags', 'qname', 'name', ] col_types_dict = dict([ ('qaid', int), ('aid', int), ('dnGt', int), ('qnGt', int), ('timedelta', float), # ('review', 'BUTTON'), (MATCHED_STATUS_TEXT, str), (REVIEWED_STATUS_TEXT, str), (QUERY_THUMB_TEXT, 'PIXMAP'), (RES_THUMB_TEXT, 'PIXMAP'), ('qname', str), ('name', str), ('score', float), ('rank', int), ('truth', bool), ('opt', int), ('result_index', int), ]) timedelta_list = np.array( ut.take_column( ibs.get_unflat_annots_timedelta_list(list(zip(qaids, daids))), 0)) # TODO: make a display role # timediff_list = [ut.get_posix_timedelta_str(t, year=True, approx=True) for t in (timedelta_list * 60 * 60)] def get_pair_tags(edge): aid1, aid2 = edge assert not ut.isiterable(aid1), 'aid1=%r, aid2=%r' % (aid1, aid2) assert not ut.isiterable(aid2), 'aid1=%r, aid2=%r' % (aid1, aid2) am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey([aid1], [aid2]) tag_text = ibs.get_annotmatch_tag_text(am_rowids)[0] if tag_text is None: tag_text = '' return str(tag_text) col_getter_dict = dict([ ('qaid', np.array(qaids)), ('aid', np.array(daids)), ('dnGt', ibs.get_annot_num_groundtruth), ('qnGt', ibs.get_annot_num_groundtruth), ('timedelta', np.array(timedelta_list)), # ('review', lambda rowid: get_buttontup), (MATCHED_STATUS_TEXT, partial(get_match_status, ibs)), (REVIEWED_STATUS_TEXT, partial(get_reviewed_status, ibs)), (QUERY_THUMB_TEXT, ibs.get_annot_chip_thumbtup), (RES_THUMB_TEXT, ibs.get_annot_chip_thumbtup), ('qname', ibs.get_annot_names), ('name', ibs.get_annot_names), ('score', np.array(scores)), ('rank', np.array(ranks)), ('result_index', np.arange(len(ranks))), ('tags', get_pair_tags), # lambda aid_pair: ibs.get_annotmatch_tag_text(ibs.get_annotmatch_rowid_from_undirected_superkey(ut.ensure_iterable(aid_pair[0]), ut.ensure_iterable(aid_pair[1])))[0]), # ('truth', truths), # ('opt', opts), ]) # default is 100 col_width_dict = { 'score': 75, REVIEWED_STATUS_TEXT: 75, MATCHED_STATUS_TEXT: 75, 'rank': 42, 'qaid': 42, 'aid': 42, 'result_index': 42, 'qname': 60, 'name': 60, 'dnGt': 50, 'timedelta': 75, 'tags': 75, 'qnGt': 50, } USE_MATCH_THUMBS = 1 if USE_MATCH_THUMBS: def get_match_thumbtup( ibs, qaid2_cm, qaids, daids, index, qreq_=None, thumbsize=(128, 128), match_thumbtup_cache={}, ): daid = daids[index] qaid = qaids[index] cm = qaid2_cm[qaid] assert cm.qaid == qaid, 'aids do not aggree' OLD = False if OLD: fpath = ensure_match_img( ibs, cm, daid, qreq_=qreq_, match_thumbtup_cache=match_thumbtup_cache) if isinstance(thumbsize, int): thumbsize = (thumbsize, thumbsize) thumbtup = ( ut.augpath(fpath, 'thumb_%d,%d' % thumbsize), fpath, thumbsize, [], [], ) return thumbtup else: # Hacky new way of drawing fpath, func, func2 = make_ensure_match_img_nosql_func( qreq_, cm, daid) # match_thumbdir = ibs.get_match_thumbdir() # match_thumb_fname = get_match_thumb_fname(cm, daid, qreq_) # fpath = ut.unixjoin(match_thumbdir, match_thumb_fname) thumbdat = { 'fpath': fpath, 'thread_func': func, 'main_func': func2, # 'args': (ibs, cm, daid), # 'kwargs': dict(qreq_=qreq_, # match_thumbtup_cache=match_thumbtup_cache) } return thumbdat col_name_list.insert(col_name_list.index('qaid'), MATCH_THUMB_TEXT) col_types_dict[MATCH_THUMB_TEXT] = 'PIXMAP' # col_types_dict[MATCH_THUMB_TEXT] = CustomMatchThumbDelegate qaid2_cm = {cm.qaid: cm for cm in cm_list} get_match_thumbtup_ = partial( get_match_thumbtup, ibs, qaid2_cm, qaids, daids, qreq_=qreq_, match_thumbtup_cache={}, ) col_getter_dict[MATCH_THUMB_TEXT] = get_match_thumbtup_ col_bgrole_dict = { MATCHED_STATUS_TEXT: partial(get_match_status_bgrole, ibs), REVIEWED_STATUS_TEXT: partial(get_reviewed_status_bgrole, ibs), } # TODO: remove ider dict. # it is massively unuseful col_ider_dict = { MATCHED_STATUS_TEXT: ('qaid', 'aid'), REVIEWED_STATUS_TEXT: ('qaid', 'aid'), 'tags': ('qaid', 'aid'), QUERY_THUMB_TEXT: ('qaid'), RES_THUMB_TEXT: ('aid'), 'dnGt': ('aid'), 'qnGt': ('qaid'), 'qname': ('qaid'), 'name': ('aid'), } col_setter_dict = { 'qname': ibs.set_annot_names, 'name': ibs.set_annot_names } editable_colnames = ['truth', 'notes', 'qname', 'name', 'opt'] sortby = 'score' def get_thumb_size(): return ibs.cfg.other_cfg.thumb_size col_display_role_func_dict = { 'timedelta': ut.partial(ut.get_posix_timedelta_str, year=True, approx=2), } if not review_cfg.get('show_chips', True): del col_getter_dict[QUERY_THUMB_TEXT] del col_getter_dict[RES_THUMB_TEXT] del col_types_dict[RES_THUMB_TEXT] del col_types_dict[QUERY_THUMB_TEXT] del col_ider_dict[RES_THUMB_TEXT] del col_ider_dict[QUERY_THUMB_TEXT] # del col_bgrole_dict[RES_THUMB_TEXT] # del col_bgrole_dict[QUERY_THUMB_TEXT] # Insert info into dict review_api = gt.CustomAPI( col_name_list=col_name_list, col_types_dict=col_types_dict, col_getter_dict=col_getter_dict, col_bgrole_dict=col_bgrole_dict, col_ider_dict=col_ider_dict, col_setter_dict=col_setter_dict, editable_colnames=editable_colnames, col_display_role_func_dict=col_display_role_func_dict, sortby=sortby, get_thumb_size=get_thumb_size, sort_reverse=True, col_width_dict=col_width_dict, ) # review_api.review_edges = review_edges return review_api
def take_column(self, colx, with_header=True): import utool as ut if with_header: return ut.take_column(self.row_data, colx) else: return ut.take_column(self.row_data[1:], colx)
def expand(sample, denc_per_name=[1], extra_dbsize_fracs=[0]): # Vary the number of database encounters in each sample target_daids_list = [] target_info_list_ = [] for num in denc_per_name: dname_encs_ = ut.take_column(sample.dname_encs, slice(0, num)) dnames_ = ut.lmap(ut.flatten, dname_encs_) daids_ = ut.total_flatten(dname_encs_) target_daids_list.append(daids_) name_lens = ut.lmap(len, dnames_) dpername = (name_lens[0] if ut.allsame(name_lens) else np.mean(name_lens)) target_info_list_.append(ut.odict([ ('qsize', len(sample.qaids)), ('t_n_names', len(dname_encs_)), ('t_dpername', dpername), ('t_denc_pername', num), ('t_dsize', len(daids_)), ])) # Append confusors to maintain a constant dbsize in each base sample dbsize_list = ut.lmap(len, target_daids_list) max_dsize = max(dbsize_list) n_need = max_dsize - min(dbsize_list) n_extra_avail = len(sample.confusor_pool) - n_need assert len(sample.confusor_pool) > n_need, 'not enough confusors' padded_daids_list = [] padded_info_list_ = [] for daids_, info_ in zip(target_daids_list, target_info_list_): num_take = max_dsize - len(daids_) pad_aids = sample.confusor_pool[:num_take] new_aids = daids_ + pad_aids info_ = info_.copy() info_['n_pad'] = len(pad_aids) info_['pad_dsize'] = len(new_aids) padded_info_list_.append(info_) padded_daids_list.append(new_aids) # Vary the dbsize by appending extra confusors if extra_dbsize_fracs is None: extra_dbsize_fracs = [1.] extra_fracs = np.array(extra_dbsize_fracs) n_extra_list = np.unique(extra_fracs * n_extra_avail).astype(np.int) daids_list = [] info_list = [] for n in n_extra_list: for daids_, info_ in zip(padded_daids_list, padded_info_list_): extra_aids = sample.confusor_pool[len(sample.confusor_pool) - n:] daids = sorted(daids_ + extra_aids) daids_list.append(daids) info = info_.copy() info['n_extra'] = len(extra_aids) info['dsize'] = len(daids) info_list.append(info) import pandas as pd verbose = 0 if verbose: print(pd.DataFrame.from_records(info_list)) print('#qaids = %r' % (len(sample.qaids),)) print('num_need = %r' % (n_need,)) print('max_dsize = %r' % (max_dsize,)) return sample.qaids, daids_list, info_list
def take_fuzzy_column(self, pat): import utool as ut colx = self.fuzzy_find_colx(pat) self.take_column(colx) return ut.take_column(self.row_data, colx)
def add_annotmatch( ibs, aid1_list, aid2_list, annotmatch_evidence_decision_list=None, annotmatch_meta_decision_list=None, annotmatch_confidence_list=None, annotmatch_tag_text_list=None, annotmatch_reviewer_list=None, annotmatch_posixtime_modified_list=None, anotmatch_count_list=None, ): r""" Returns: returns annotmatch_rowid_list of added (or already existing annotmatchs) TemplateInfo: Tadder_native tbl = annotmatch """ n_input = len(aid1_list) # Rectify inputs if annotmatch_evidence_decision_list is None: annotmatch_evidence_decision_list = [None] * n_input if annotmatch_meta_decision_list is None: annotmatch_meta_decision_list = [None] * n_input if annotmatch_confidence_list is None: annotmatch_confidence_list = [None] * n_input if annotmatch_tag_text_list is None: annotmatch_tag_text_list = [None] * n_input if annotmatch_reviewer_list is None: annotmatch_reviewer_list = [None] * n_input if annotmatch_posixtime_modified_list is None: annotmatch_posixtime_modified_list = [None] * n_input if anotmatch_count_list is None: anotmatch_count_list = [0] * n_input # Associate input lists with columns cols_to_lists = [ (ANNOT_ROWID1, aid1_list), (ANNOT_ROWID2, aid2_list), (ANNOTMATCH_EVIDENCE_DECISION, annotmatch_evidence_decision_list), (ANNOTMATCH_META_DECISION, annotmatch_meta_decision_list), (ANNOTMATCH_CONFIDENCE, annotmatch_confidence_list), (ANNOTMATCH_TAG_TEXT, annotmatch_tag_text_list), (ANNOTMATCH_REVIEWER, annotmatch_reviewer_list), (ANNOTMATCH_POSIXTIME_MODIFIED, annotmatch_posixtime_modified_list), (ANNOTMATCH_COUNT, anotmatch_count_list), ] colnames = ut.take_column(cols_to_lists, 0) params_iter = zip(*ut.take_column(cols_to_lists, 1)) get_rowid_from_superkey = ibs.get_annotmatch_rowid_from_superkey # FIXME: encode superkey paramx superkey_paramx = (0, 1) annotmatch_rowid_list = ibs.db.add_cleanly( const.ANNOTMATCH_TABLE, colnames, params_iter, get_rowid_from_superkey, superkey_paramx, ) return annotmatch_rowid_list
def start_identify_annots_query(ibs, query_annot_uuid_list=None, # query_annot_name_uuid_list=None, query_annot_name_list=None, database_annot_uuid_list=None, # database_annot_name_uuid_list=None, database_annot_name_list=None, matching_state_list=[], query_config_dict={}, echo_query_params=True, callback_url=None, callback_method=None): r""" REST: Method: GET URL: /api/engine/query/graph/ Args: query_annot_uuid_list (list) : specifies the query annotations to identify. query_annot_name_list (list) : specifies the query annotation names database_annot_uuid_list (list) : specifies the annotations that the algorithm is allowed to use for identification. If not specified all annotations are used. (default=None) database_annot_name_list (list) : specifies the database annotation names (default=None) matching_state_list (list of tuple) : the list of matching state 3-tuples corresponding to the query_annot_uuid_list (default=None) query_config_dict (dict) : dictionary of algorithmic configuration arguments. (default=None) echo_query_params (bool) : flag for if to return the original query parameters with the result CommandLine: # Normal mode python -m ibeis.web.apis_engine start_identify_annots_query # Split mode ibeis --web python -m ibeis.web.apis_engine start_identify_annots_query --show --domain=localhost Example: >>> # DISABLE_DOCTEST >>> from ibeis.web.apis_engine import * # NOQA >>> import ibeis >>> #domain = 'localhost' >>> domain = None >>> web_ibs = ibeis.opendb_bg_web('testdb1', domain=domain) # , domain='http://52.33.105.88') >>> aids = web_ibs.send_ibeis_request('/api/annot/', 'get')[0:3] >>> uuid_list = web_ibs.send_ibeis_request('/api/annot/uuids/', type_='get', aid_list=aids) >>> quuid_list = ut.get_argval('--quuids', type_=list, default=uuid_list)[0:1] >>> duuid_list = ut.get_argval('--duuids', type_=list, default=uuid_list) >>> query_config_dict = { >>> #'pipeline_root' : 'BC_DTW' >>> } >>> data = dict( >>> query_annot_uuid_list=quuid_list, database_annot_uuid_list=duuid_list, >>> query_config_dict=query_config_dict, >>> ) >>> jobid = web_ibs.send_ibeis_request('/api/engine/query/graph/', **data) >>> print('jobid = %r' % (jobid,)) >>> status_response = web_ibs.wait_for_results(jobid) >>> result_response = web_ibs.read_engine_results(jobid) >>> print('result_response = %s' % (ut.repr3(result_response),)) >>> inference_result = result_response['json_result'] >>> if isinstance(inference_result, six.string_types): >>> print(inference_result) >>> cm_dict = inference_result['cm_dict'] >>> quuid = quuid_list[0] >>> cm = cm_dict[str(quuid)] >>> web_ibs.terminate2() """ valid_states = { 'match': ['matched'], # ['match', 'matched'], 'nonmatch': ['notmatched'], # ['nonmatch', 'notmatched', 'nonmatched', 'notmatch', 'non-match', 'not-match'], 'notcomp' : ['notcomparable'], } prefered_states = ut.take_column(valid_states.values(), 0) flat_states = ut.flatten(valid_states.values()) def sanitize(state): state = state.strip().lower() state = ''.join(state.split()) assert state in flat_states, 'matching_state_list has unrecognized states. Should be one of %r' % (prefered_states,) return state # HACK # if query_annot_uuid_list is None: # if True: # query_annot_uuid_list = [] # else: # query_annot_uuid_list = ibs.get_annot_uuids(ibs.get_valid_aids()[0:1]) dname_list = database_annot_name_list qname_list = query_annot_name_list # Check inputs assert len(query_annot_uuid_list) == 1, 'Can only identify one query annotation at a time. Got %d ' % (len(query_annot_uuid_list),) if qname_list is not None: assert len(query_annot_uuid_list) == len(qname_list) if database_annot_uuid_list is not None and dname_list is not None: assert len(database_annot_uuid_list) == len(dname_list) # Check UUIDs ibs.web_check_uuids([], query_annot_uuid_list, database_annot_uuid_list) #import ibeis #from ibeis.web import apis_engine #ibs.load_plugin_module(apis_engine) def ensure_uuid_list(list_): if list_ is not None and len(list_) > 0 and isinstance(list_[0], six.string_types): list_ = list(map(uuid.UUID, list_)) return list_ qannot_uuid_list = ensure_uuid_list(query_annot_uuid_list) dannot_uuid_list = ensure_uuid_list(database_annot_uuid_list) # Ensure annotations qaid_list = ibs.get_annot_aids_from_uuid(qannot_uuid_list) if dannot_uuid_list is None or (len(dannot_uuid_list) == 1 and dannot_uuid_list[0] is None): # VERY HACK daid_list = ibs.get_valid_aids() else: daid_list = ibs.get_annot_aids_from_uuid(dannot_uuid_list) # Ensure names # FIXME: THE QREQ STRUCTURE SHOULD HANDLE NAMES. if qname_list is not None: # Set names for query annotations qnid_list = ibs.add_names(qname_list) ibs.set_annot_name_rowids(qaid_list, qnid_list) if dname_list is not None: # Set names for database annotations dnid_list = ibs.add_names(dname_list) ibs.set_annot_name_rowids(daid_list, dnid_list) # Convert annot UUIDs to aids for matching_state_list into user_feedback for query state_list = map(sanitize, ut.take_column(matching_state_list, 2)) {'aid1': [1], 'aid2': [2], 'p_match': [1.0], 'p_nomatch': [0.0], 'p_notcomp': [0.0]} user_feedback = { 'aid1' : ibs.get_annot_aids_from_uuid(ut.take_column(matching_state_list, 0)), 'aid2' : ibs.get_annot_aids_from_uuid(ut.take_column(matching_state_list, 1)), 'p_match' : [1.0 if state in valid_states['match'] else 0.0 for state in state_list], 'p_nomatch' : [1.0 if state in valid_states['nonmatch'] else 0.0 for state in state_list], 'p_notcomp' : [1.0 if state in valid_states['notcomp'] else 0.0 for state in state_list], } ibs.assert_valid_aids(qaid_list, msg='error in start_identify qaids', auuid_list=qannot_uuid_list) ibs.assert_valid_aids(daid_list, msg='error in start_identify daids', auuid_list=dannot_uuid_list) args = (qaid_list, daid_list, user_feedback, query_config_dict, echo_query_params) jobid = ibs.job_manager.jobiface.queue_job('query_chips_graph', callback_url, callback_method, *args) return jobid
def keys(self): return ut.take_column(self._items, 0)
def detect_sharks(ibs, gids): # import wbia # ibs = wbia.opendb('WS_ALL') config = { 'algo': 'yolo', 'sensitivity': 0.2, 'config_filepath': ut.truepath( '~/work/WS_ALL/localizer_backup/detect.yolo.2.cfg' ), 'weight_filepath': ut.truepath( '~/work/WS_ALL/localizer_backup/detect.yolo.2.39000.weights' ), 'class_filepath': ut.truepath( '~/work/WS_ALL/localizer_backup/detect.yolo.2.cfg.classes' ), } depc = ibs.depc_image # imgsets = ibs.imagesets(text='Injured Sharks') # images = ibs.images(imgsets.gids[0]) images = ibs.images(gids) images = images.compress([ext not in ['.gif'] for ext in images.exts]) gid_list = images.gids # result is a tuple: # (score, bbox_list, theta_list, conf_list, class_list) results_list = depc.get_property('localizations', gid_list, None, config=config) results_list2 = [] multi_gids = [] failed_gids = [] # ibs.set_image_imagesettext(failed_gids, ['Fixme'] * len(failed_gids)) ibs.set_image_imagesettext(multi_gids, ['Fixme2'] * len(multi_gids)) failed_gids for gid, res in zip(gid_list, results_list): score, bbox_list, theta_list, conf_list, class_list = res if len(bbox_list) == 0: failed_gids.append(gid) elif len(bbox_list) == 1: results_list2.append((gid, bbox_list, theta_list)) elif len(bbox_list) > 1: multi_gids.append(gid) idx = conf_list.argmax() res2 = (gid, bbox_list[idx : idx + 1], theta_list[idx : idx + 1]) results_list2.append(res2) ut.dict_hist(([t[1].shape[0] for t in results_list])) localized_imgs = ibs.images(ut.take_column(results_list2, 0)) assert all([len(a) == 1 for a in localized_imgs.aids]) old_annots = ibs.annots(ut.flatten(localized_imgs.aids)) # old_tags = old_annots.case_tags # Override old bboxes import numpy as np bboxes = np.array(ut.take_column(results_list2, 1))[:, 0, :] ibs.set_annot_bboxes(old_annots.aids, bboxes) if False: import wbia.plottool as pt pt.qt4ensure() inter = pt.MultiImageInteraction( ibs.get_image_paths(ut.take_column(results_list2, 0)), bboxes_list=ut.take_column(results_list2, 1), ) inter.dump_to_disk('shark_loc', num=50, prefix='shark_loc') inter.start() inter = pt.MultiImageInteraction(ibs.get_image_paths(failed_gids)) inter.start() inter = pt.MultiImageInteraction(ibs.get_image_paths(multi_gids)) inter.start()
def start_identify_annots_query( ibs, query_annot_uuid_list=None, # query_annot_name_uuid_list=None, query_annot_name_list=None, database_annot_uuid_list=None, # database_annot_name_uuid_list=None, database_annot_name_list=None, matching_state_list=[], query_config_dict={}, echo_query_params=True, callback_url=None, callback_method=None): r""" REST: Method: GET URL: /api/engine/query/graph/ Args: query_annot_uuid_list (list) : specifies the query annotations to identify. query_annot_name_list (list) : specifies the query annotation names database_annot_uuid_list (list) : specifies the annotations that the algorithm is allowed to use for identification. If not specified all annotations are used. (default=None) database_annot_name_list (list) : specifies the database annotation names (default=None) matching_state_list (list of tuple) : the list of matching state 3-tuples corresponding to the query_annot_uuid_list (default=None) query_config_dict (dict) : dictionary of algorithmic configuration arguments. (default=None) echo_query_params (bool) : flag for if to return the original query parameters with the result CommandLine: # Normal mode python -m ibeis.web.apis_engine start_identify_annots_query # Split mode ibeis --web python -m ibeis.web.apis_engine start_identify_annots_query --show --domain=localhost Example: >>> # DISABLE_DOCTEST >>> from ibeis.web.apis_engine import * # NOQA >>> import ibeis >>> #domain = 'localhost' >>> domain = None >>> web_ibs = ibeis.opendb_bg_web('testdb1', domain=domain) # , domain='http://52.33.105.88') >>> aids = web_ibs.send_ibeis_request('/api/annot/', 'get')[0:3] >>> uuid_list = web_ibs.send_ibeis_request('/api/annot/uuids/', type_='get', aid_list=aids) >>> quuid_list = ut.get_argval('--quuids', type_=list, default=uuid_list)[0:1] >>> duuid_list = ut.get_argval('--duuids', type_=list, default=uuid_list) >>> query_config_dict = { >>> #'pipeline_root' : 'BC_DTW' >>> } >>> data = dict( >>> query_annot_uuid_list=quuid_list, database_annot_uuid_list=duuid_list, >>> query_config_dict=query_config_dict, >>> ) >>> jobid = web_ibs.send_ibeis_request('/api/engine/query/graph/', **data) >>> print('jobid = %r' % (jobid,)) >>> status_response = web_ibs.wait_for_results(jobid) >>> result_response = web_ibs.read_engine_results(jobid) >>> print('result_response = %s' % (ut.repr3(result_response),)) >>> inference_result = result_response['json_result'] >>> if isinstance(inference_result, six.string_types): >>> print(inference_result) >>> cm_dict = inference_result['cm_dict'] >>> quuid = quuid_list[0] >>> cm = cm_dict[str(quuid)] >>> web_ibs.terminate2() """ valid_states = { 'match': ['matched'], # ['match', 'matched'], 'nonmatch': [ 'notmatched' ], # ['nonmatch', 'notmatched', 'nonmatched', 'notmatch', 'non-match', 'not-match'], 'notcomp': ['notcomparable'], } prefered_states = ut.take_column(valid_states.values(), 0) flat_states = ut.flatten(valid_states.values()) def sanitize(state): state = state.strip().lower() state = ''.join(state.split()) assert state in flat_states, 'matching_state_list has unrecognized states. Should be one of %r' % ( prefered_states, ) return state # HACK # if query_annot_uuid_list is None: # if True: # query_annot_uuid_list = [] # else: # query_annot_uuid_list = ibs.get_annot_uuids(ibs.get_valid_aids()[0:1]) dname_list = database_annot_name_list qname_list = query_annot_name_list # Check inputs assert len( query_annot_uuid_list ) == 1, 'Can only identify one query annotation at a time. Got %d ' % ( len(query_annot_uuid_list), ) if qname_list is not None: assert len(query_annot_uuid_list) == len(qname_list) if database_annot_uuid_list is not None and dname_list is not None: assert len(database_annot_uuid_list) == len(dname_list) # Check UUIDs ibs.web_check_uuids([], query_annot_uuid_list, database_annot_uuid_list) #import ibeis #from ibeis.web import apis_engine #ibs.load_plugin_module(apis_engine) def ensure_uuid_list(list_): if list_ is not None and len(list_) > 0 and isinstance( list_[0], six.string_types): list_ = list(map(uuid.UUID, list_)) return list_ qannot_uuid_list = ensure_uuid_list(query_annot_uuid_list) dannot_uuid_list = ensure_uuid_list(database_annot_uuid_list) # Ensure annotations qaid_list = ibs.get_annot_aids_from_uuid(qannot_uuid_list) if dannot_uuid_list is None or (len(dannot_uuid_list) == 1 and dannot_uuid_list[0] is None): # VERY HACK daid_list = ibs.get_valid_aids() else: daid_list = ibs.get_annot_aids_from_uuid(dannot_uuid_list) # Ensure names # FIXME: THE QREQ STRUCTURE SHOULD HANDLE NAMES. if qname_list is not None: # Set names for query annotations qnid_list = ibs.add_names(qname_list) ibs.set_annot_name_rowids(qaid_list, qnid_list) if dname_list is not None: # Set names for database annotations dnid_list = ibs.add_names(dname_list) ibs.set_annot_name_rowids(daid_list, dnid_list) # Convert annot UUIDs to aids for matching_state_list into user_feedback for query state_list = map(sanitize, ut.take_column(matching_state_list, 2)) { 'aid1': [1], 'aid2': [2], 'p_match': [1.0], 'p_nomatch': [0.0], 'p_notcomp': [0.0] } user_feedback = { 'aid1': ibs.get_annot_aids_from_uuid(ut.take_column(matching_state_list, 0)), 'aid2': ibs.get_annot_aids_from_uuid(ut.take_column(matching_state_list, 1)), 'p_match': [ 1.0 if state in valid_states['match'] else 0.0 for state in state_list ], 'p_nomatch': [ 1.0 if state in valid_states['nonmatch'] else 0.0 for state in state_list ], 'p_notcomp': [ 1.0 if state in valid_states['notcomp'] else 0.0 for state in state_list ], } ibs.assert_valid_aids(qaid_list, msg='error in start_identify qaids', auuid_list=qannot_uuid_list) ibs.assert_valid_aids(daid_list, msg='error in start_identify daids', auuid_list=dannot_uuid_list) args = (qaid_list, daid_list, user_feedback, query_config_dict, echo_query_params) jobid = ibs.job_manager.jobiface.queue_job('query_chips_graph', callback_url, callback_method, *args) return jobid
def get_injured_sharks(): """ >>> from wbia.scripts.getshark import * # NOQA """ import requests url = 'http://www.whaleshark.org/getKeywordImages.jsp' resp = requests.get(url) assert resp.status_code == 200 keywords = resp.json()['keywords'] key_list = ut.take_column(keywords, 'indexName') key_to_nice = {k['indexName']: k['readableName'] for k in keywords} injury_patterns = [ 'injury', 'net', 'hook', 'trunc', 'damage', 'scar', 'nicks', 'bite', ] injury_keys = [ key for key in key_list if any([pat in key for pat in injury_patterns]) ] noninjury_keys = ut.setdiff(key_list, injury_keys) injury_nice = ut.lmap(lambda k: key_to_nice[k], injury_keys) # NOQA noninjury_nice = ut.lmap(lambda k: key_to_nice[k], noninjury_keys) # NOQA key_list = injury_keys keyed_images = {} for key in ut.ProgIter(key_list, lbl='reading index', bs=True): key_url = url + '?indexName={indexName}'.format(indexName=key) key_resp = requests.get(key_url) assert key_resp.status_code == 200 key_imgs = key_resp.json()['images'] keyed_images[key] = key_imgs key_hist = {key: len(imgs) for key, imgs in keyed_images.items()} key_hist = ut.sort_dict(key_hist, 'vals') logger.info(ut.repr3(key_hist)) nice_key_hist = ut.map_dict_keys(lambda k: key_to_nice[k], key_hist) nice_key_hist = ut.sort_dict(nice_key_hist, 'vals') logger.info(ut.repr3(nice_key_hist)) key_to_urls = {key: ut.take_column(vals, 'url') for key, vals in keyed_images.items()} overlaps = {} import itertools overlap_img_list = [] for k1, k2 in itertools.combinations(key_to_urls.keys(), 2): overlap_imgs = ut.isect(key_to_urls[k1], key_to_urls[k2]) num_overlap = len(overlap_imgs) overlaps[(k1, k2)] = num_overlap overlaps[(k1, k1)] = len(key_to_urls[k1]) if num_overlap > 0: # logger.info('[%s][%s], overlap=%r' % (k1, k2, num_overlap)) overlap_img_list.extend(overlap_imgs) all_img_urls = list(set(ut.flatten(key_to_urls.values()))) num_all = len(all_img_urls) # NOQA logger.info('num_all = %r' % (num_all,)) # Determine super-categories categories = ['nicks', 'scar', 'trunc'] # Force these keys into these categories key_to_cat = {'scarbite': 'other_injury'} cat_to_keys = ut.ddict(list) for key in key_to_urls.keys(): flag = 1 if key in key_to_cat: cat = key_to_cat[key] cat_to_keys[cat].append(key) continue for cat in categories: if cat in key: cat_to_keys[cat].append(key) flag = 0 if flag: cat = 'other_injury' cat_to_keys[cat].append(key) cat_urls = ut.ddict(list) for cat, keys in cat_to_keys.items(): for key in keys: cat_urls[cat].extend(key_to_urls[key]) cat_hist = {} for cat in list(cat_urls.keys()): cat_urls[cat] = list(set(cat_urls[cat])) cat_hist[cat] = len(cat_urls[cat]) logger.info(ut.repr3(cat_to_keys)) logger.info(ut.repr3(cat_hist)) key_to_cat = dict([(val, key) for key, vals in cat_to_keys.items() for val in vals]) # ingestset = { # '__class__': 'ImageSet', # 'images': ut.ddict(dict) # } # for key, key_imgs in keyed_images.items(): # for imgdict in key_imgs: # url = imgdict['url'] # encid = imgdict['correspondingEncounterNumber'] # # Make structure # encdict = encounters[encid] # encdict['__class__'] = 'Encounter' # imgdict = ut.delete_keys(imgdict.copy(), ['correspondingEncounterNumber']) # imgdict['__class__'] = 'Image' # cat = key_to_cat[key] # annotdict = {'relative_bbox': [.01, .01, .98, .98], 'tags': [cat, key]} # annotdict['__class__'] = 'Annotation' # # Ensure structures exist # encdict['images'] = encdict.get('images', []) # imgdict['annots'] = imgdict.get('annots', []) # # Add an image to this encounter # encdict['images'].append(imgdict) # # Add an annotation to this image # imgdict['annots'].append(annotdict) # # http://springbreak.wildbook.org/rest/org.ecocean.Encounter/1111 # get_enc_url = 'http://www.whaleshark.org/rest/org.ecocean.Encounter/%s' % (encid,) # resp = requests.get(get_enc_url) # logger.info(ut.repr3(encdict)) # logger.info(ut.repr3(encounters)) # Download the files to the local disk # fpath_list = all_urls = ut.unique( ut.take_column( ut.flatten( ut.dict_subset(keyed_images, ut.flatten(cat_to_keys.values())).values() ), 'url', ) ) dldir = ut.truepath('~/tmpsharks') from os.path import commonprefix, basename # NOQA prefix = commonprefix(all_urls) suffix_list = [url_[len(prefix) :] for url_ in all_urls] fname_list = [suffix.replace('/', '--') for suffix in suffix_list] fpath_list = [] for url, fname in ut.ProgIter( zip(all_urls, fname_list), lbl='downloading imgs', freq=1 ): fpath = ut.grab_file_url(url, download_dir=dldir, fname=fname, verbose=False) fpath_list.append(fpath) # Make sure we keep orig info # url_to_keys = ut.ddict(list) url_to_info = ut.ddict(dict) for key, imgdict_list in keyed_images.items(): for imgdict in imgdict_list: url = imgdict['url'] info = url_to_info[url] for k, v in imgdict.items(): info[k] = info.get(k, []) info[k].append(v) info['keys'] = info.get('keys', []) info['keys'].append(key) # url_to_keys[url].append(key) info_list = ut.take(url_to_info, all_urls) for info in info_list: if len(set(info['correspondingEncounterNumber'])) > 1: assert False, 'url with two different encounter nums' # Combine duplicate tags hashid_list = [ ut.get_file_uuid(fpath_, stride=8) for fpath_ in ut.ProgIter(fpath_list, bs=True) ] groupxs = ut.group_indices(hashid_list)[1] # Group properties by duplicate images # groupxs = [g for g in groupxs if len(g) > 1] fpath_list_ = ut.take_column(ut.apply_grouping(fpath_list, groupxs), 0) url_list_ = ut.take_column(ut.apply_grouping(all_urls, groupxs), 0) info_list_ = [ ut.map_dict_vals(ut.flatten, ut.dict_accum(*info_)) for info_ in ut.apply_grouping(info_list, groupxs) ] encid_list_ = [ ut.unique(info_['correspondingEncounterNumber'])[0] for info_ in info_list_ ] keys_list_ = [ut.unique(info_['keys']) for info_ in info_list_] cats_list_ = [ut.unique(ut.take(key_to_cat, keys)) for keys in keys_list_] clist = ut.ColumnLists( { 'gpath': fpath_list_, 'url': url_list_, 'encid': encid_list_, 'key': keys_list_, 'cat': cats_list_, } ) # for info_ in ut.apply_grouping(info_list, groupxs): # info = ut.dict_accum(*info_) # info = ut.map_dict_vals(ut.flatten, info) # x = ut.unique(ut.flatten(ut.dict_accum(*info_)['correspondingEncounterNumber'])) # if len(x) > 1: # info = info.copy() # del info['keys'] # logger.info(ut.repr3(info)) flags = ut.lmap(ut.fpath_has_imgext, clist['gpath']) clist = clist.compress(flags) import wbia ibs = wbia.opendb('WS_Injury', allow_newdir=True) gid_list = ibs.add_images(clist['gpath']) clist['gid'] = gid_list failed_flags = ut.flag_None_items(clist['gid']) logger.info('# failed %s' % (sum(failed_flags)),) passed_flags = ut.not_list(failed_flags) clist = clist.compress(passed_flags) ut.assert_all_not_None(clist['gid']) # ibs.get_image_uris_original(clist['gid']) ibs.set_image_uris_original(clist['gid'], clist['url'], overwrite=True) # ut.zipflat(clist['cat'], clist['key']) if False: # Can run detection instead clist['tags'] = ut.zipflat(clist['cat']) aid_list = ibs.use_images_as_annotations( clist['gid'], adjust_percent=0.01, tags_list=clist['tags'] ) aid_list import wbia.plottool as pt from wbia import core_annots pt.qt4ensure() # annots = ibs.annots() # aids = [1, 2] # ibs.depc_annot.get('hog', aids , 'hog') # ibs.depc_annot.get('chip', aids, 'img') for aid in ut.InteractiveIter(ibs.get_valid_aids()): hogs = ibs.depc_annot.d.get_hog_hog([aid]) chips = ibs.depc_annot.d.get_chips_img([aid]) chip = chips[0] hogimg = core_annots.make_hog_block_image(hogs[0]) pt.clf() pt.imshow(hogimg, pnum=(1, 2, 1)) pt.imshow(chip, pnum=(1, 2, 2)) fig = pt.gcf() fig.show() fig.canvas.draw() # logger.info(len(groupxs)) # if False: # groupxs = ut.find_duplicate_items(ut.lmap(basename, suffix_list)).values() # logger.info(ut.repr3(ut.apply_grouping(all_urls, groupxs))) # # FIX # for fpath, fname in zip(fpath_list, fname_list): # if ut.checkpath(fpath): # ut.move(fpath, join(dirname(fpath), fname)) # logger.info('fpath = %r' % (fpath,)) # import wbia # from wbia.dbio import ingest_dataset # dbdir = wbia.sysres.lookup_dbdir('WS_ALL') # self = ingest_dataset.Ingestable2(dbdir) if False: # Show overlap matrix import wbia.plottool as pt import pandas as pd import numpy as np dict_ = overlaps s = pd.Series(dict_, index=pd.MultiIndex.from_tuples(overlaps)) df = s.unstack() lhs, rhs = df.align(df.T) df = lhs.add(rhs, fill_value=0).fillna(0) label_texts = df.columns.values def label_ticks(label_texts): import wbia.plottool as pt truncated_labels = [repr(lbl[0:100]) for lbl in label_texts] ax = pt.gca() ax.set_xticks(list(range(len(label_texts)))) ax.set_xticklabels(truncated_labels) [lbl.set_rotation(-55) for lbl in ax.get_xticklabels()] [lbl.set_horizontalalignment('left') for lbl in ax.get_xticklabels()] # xgrid, ygrid = np.meshgrid(range(len(label_texts)), range(len(label_texts))) # pt.plot_surface3d(xgrid, ygrid, disjoint_mat) ax.set_yticks(list(range(len(label_texts)))) ax.set_yticklabels(truncated_labels) [lbl.set_horizontalalignment('right') for lbl in ax.get_yticklabels()] [lbl.set_verticalalignment('center') for lbl in ax.get_yticklabels()] # [lbl.set_rotation(20) for lbl in ax.get_yticklabels()] # df = df.sort(axis=0) # df = df.sort(axis=1) sortx = np.argsort(df.sum(axis=1).values)[::-1] df = df.take(sortx, axis=0) df = df.take(sortx, axis=1) fig = pt.figure(fnum=1) fig.clf() mat = df.values.astype(np.int32) mat[np.diag_indices(len(mat))] = 0 vmax = mat[(1 - np.eye(len(mat))).astype(np.bool)].max() import matplotlib.colors norm = matplotlib.colors.Normalize(vmin=0, vmax=vmax, clip=True) pt.plt.imshow(mat, cmap='hot', norm=norm, interpolation='none') pt.plt.colorbar() pt.plt.grid('off') label_ticks(label_texts) fig.tight_layout() # overlap_df = pd.DataFrame.from_dict(overlap_img_list) class TmpImage(ut.NiceRepr): pass from skimage.feature import hog from skimage import data, color, exposure import wbia.plottool as pt image2 = color.rgb2gray(data.astronaut()) # NOQA fpath = './GOPR1120.JPG' import vtool as vt for fpath in [fpath]: """ http://scikit-image.org/docs/dev/auto_examples/plot_hog.html """ image = vt.imread(fpath, grayscale=True) image = pt.color_funcs.to_base01(image) fig = pt.figure(fnum=2) fd, hog_image = hog( image, orientations=8, pixels_per_cell=(16, 16), cells_per_block=(1, 1), visualise=True, ) fig, (ax1, ax2) = pt.plt.subplots(1, 2, figsize=(8, 4), sharex=True, sharey=True) ax1.axis('off') ax1.imshow(image, cmap=pt.plt.cm.gray) ax1.set_title('Input image') ax1.set_adjustable('box-forced') # Rescale histogram for better display hog_image_rescaled = exposure.rescale_intensity(hog_image, in_range=(0, 0.02)) ax2.axis('off') ax2.imshow(hog_image_rescaled, cmap=pt.plt.cm.gray) ax2.set_title('Histogram of Oriented Gradients') ax1.set_adjustable('box-forced') pt.plt.show()
def limited_power_toughness_histogram(): r""" CommandLine: python -m mtgmonte.stats --exec-limited_power_toughness_histogram --show Example: >>> # DISABLE_DOCTEST >>> from mtgmonte.stats import * # NOQA >>> result = limited_power_toughness_histogram() >>> print(result) >>> ut.show_if_requested() """ from mtgmonte import mtgobjs from mtglib.gatherer_request import SearchRequest from mtglib.card_extractor import CardExtractor # from mtglib.card_renderer import CardList request = SearchRequest({"set": "Oath of the Gatewatch"}) def add_page(url, page): parts = url.split("/") part1 = "/".join(parts[:-1]) part2 = "/Default.aspx?page=%d&" % (page,) part3 = parts[-1].replace("Default.aspx?", "") url2 = part1 + part2 + part3 return url2 card_list = [] for page in range(0, 10): url = request.url url2 = add_page(url, page) extract = CardExtractor(url2) card_list0 = extract.cards for card in card_list0: card2 = mtgobjs.Card2() card2.__dict__.update(card.__dict__) card_list.append(card2) if len(card_list0) != 100: break for c in card_list: c.nice_attrs += ["rarity"] creats = [_card2 for _card2 in card_list if "Creature" in card2.types] creats = [_card2 for _card2 in creats if _card2.rarity in ["Common", "Uncommon"]] powtough = [] for c in creats: try: powtough.append((int(c.power), int(c.toughness))) except ValueError: pass import plottool as pt pt.ensure_pylab_qt4() import numpy as np scores_list = np.array(list(zip(*powtough))) xdata = np.arange(0, np.max(scores_list) + 1) powhist = np.histogram(scores_list[0], bins=xdata)[0] toughist = np.histogram(scores_list[1], bins=xdata)[0] pt.multi_plot(xdata, [powhist, toughist], label_list=["power", "toughness"], kind="bar") bothhist = ut.dict_hist(powtough) xdata = np.arange(len(bothhist)) dat = sorted(bothhist.items()) xticklabels = ut.take_column(dat, 0) ydata = ut.take_column(dat, 1) pt.multi_plot(xdata, [ydata], xticklabels=xticklabels, kind="bar")