def consolodate_duplicates(self): fnames = map(basename, self.rel_fpath_list) duplicate_map = ut.find_duplicate_items(fnames) groups = [] for dupname, idxs in duplicate_map.items(): uuids = self.get_prop('uuids', idxs) unique_uuids, groupxs = ut.group_indices(uuids) groups.extend(ut.apply_grouping(idxs, groupxs)) multitons = [g for g in groups if len(g) > 1] # singletons = [g for g in groups if len(g) <= 1] ut.unflat_take(list(self.fpaths()), multitons)
def color_by_nids(graph, unique_nids=None, ibs=None, nid2_color_=None): """ Colors edges and nodes by nid """ # TODO use ut.color_nodes import plottool as pt ensure_graph_nid_labels(graph, unique_nids, ibs=ibs) node_to_nid = nx.get_node_attributes(graph, 'nid') unique_nids = ut.unique(node_to_nid.values()) ncolors = len(unique_nids) if (ncolors) == 1: unique_colors = [pt.UNKNOWN_PURP] else: if nid2_color_ is not None: unique_colors = pt.distinct_colors(ncolors + len(nid2_color_) * 2) else: unique_colors = pt.distinct_colors(ncolors) # Find edges and aids strictly between two nids nid_to_color = dict(zip(unique_nids, unique_colors)) if nid2_color_ is not None: # HACK NEED TO ENSURE COLORS ARE NOT REUSED nid_to_color.update(nid2_color_) edge_aids = list(graph.edges()) edge_nids = ut.unflat_take(node_to_nid, edge_aids) flags = [nids[0] == nids[1] for nids in edge_nids] flagged_edge_aids = ut.compress(edge_aids, flags) flagged_edge_nids = ut.compress(edge_nids, flags) flagged_edge_colors = [nid_to_color[nids[0]] for nids in flagged_edge_nids] edge_to_color = dict(zip(flagged_edge_aids, flagged_edge_colors)) node_to_color = ut.map_dict_vals(ut.partial(ut.take, nid_to_color), node_to_nid) nx.set_edge_attributes(graph, 'color', edge_to_color) nx.set_node_attributes(graph, 'color', node_to_color)
def _update_state_gco(model, weight_key='cut_weight', name_label_key='name_label'): import networkx as nx # Get nx graph properties external_nodes = sorted(list(model.graph.nodes())) external_edges = list(model.graph.edges()) edge_to_weights = nx.get_edge_attributes(model.graph, weight_key) node_to_labeling = nx.get_node_attributes(model.graph, name_label_key) edge_weights = ut.dict_take(edge_to_weights, external_edges, 0) external_labeling = [ node_to_labeling.get(node, -node) for node in external_nodes ] # Map to internal ids for pygco internal_nodes = ut.rebase_labels(external_nodes) extern2_intern = dict(zip(external_nodes, internal_nodes)) internal_edges = ut.unflat_take(extern2_intern, external_edges) internal_labeling = ut.rebase_labels(external_labeling) internal_labeling = np.array(internal_labeling) internal_edges = np.array(internal_edges) n_nodes = len(internal_nodes) # Model state model.n_nodes = n_nodes model.extern2_intern = extern2_intern model.intern2_extern = ut.invert_dict(extern2_intern) model.edges = internal_edges model.edge_weights = edge_weights # Model parameters model.labeling = np.zeros(model.n_nodes, dtype=np.int32) model._update_labels(labeling=internal_labeling) model._update_weights()
def nx_from_matrix(weight_matrix, nodes=None, remove_self=True): import networkx as nx import utool as ut import numpy as np if nodes is None: nodes = list(range(len(weight_matrix))) weight_list = weight_matrix.ravel() flat_idxs_ = np.arange(weight_matrix.size) multi_idxs_ = np.unravel_index(flat_idxs_, weight_matrix.shape) # Remove 0 weight edges flags = np.logical_not(np.isclose(weight_list, 0)) weight_list = ut.compress(weight_list, flags) multi_idxs = ut.compress(list(zip(*multi_idxs_)), flags) edge_list = ut.lmap(tuple, ut.unflat_take(nodes, multi_idxs)) if remove_self: flags = [e1 != e2 for e1, e2 in edge_list] edge_list = ut.compress(edge_list, flags) weight_list = ut.compress(weight_list, flags) graph = nx.Graph() graph.add_nodes_from(nodes) graph.add_edges_from(edge_list) label_list = ['%.2f' % w for w in weight_list] nx.set_edge_attributes(graph, 'weight', dict(zip(edge_list, weight_list))) nx.set_edge_attributes(graph, 'label', dict(zip(edge_list, label_list))) return graph
def get_cut_edges(model): extern_uv_list = np.array(list(model.graph.edges())) intern_uv_list = ut.unflat_take(model.extern2_intern, extern_uv_list) intern_uv_list = np.array(intern_uv_list) u_labels = model.labeling[intern_uv_list.T[0]] v_labels = model.labeling[intern_uv_list.T[1]] # Remove edges between all annotations with different labels cut_edges = extern_uv_list[u_labels != v_labels] cut_edges = [tuple(uv.tolist()) for uv in cut_edges] return cut_edges
def random_case_set(): r""" Returns: tuple: (labels, pairwise_feats) CommandLine: python -m ibeis.algo.hots.testem random_case_set --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.testem import * # NOQA >>> (labels, pairwise_feats) = random_case_set() >>> result = ('(labels, pairwise_feats) = %s' % (ut.repr2((labels, pairwise_feats)),)) >>> print(result) """ rng = np.random.RandomState(0) case_params = dict(num_names=5, rng=rng) num_annots = 600 test_cases = [ random_test_annot(**case_params) for _ in ut.ProgIter(range(num_annots), bs=1) ] pairxs = list(ut.product_nonsame(range(num_annots), range(num_annots))) import utool utool.embed() test_pairs = list(ut.unflat_take(test_cases, pairxs)) cases1 = ut.instancelist(ut.take_column(test_pairs, 0), check=False) cases2 = ut.instancelist(ut.take_column(test_pairs, 1), check=False) # FIXME labels = labels1 = make_test_pairwise_labels2(cases1, cases2) # NOQA #labels = np.array([make_test_pairwise_labels(case1, case2) # for case1, case2 in ut.ProgIter(test_pairs, bs=1)]) pairwise_feats_ = [ make_test_pairwise_fetaures(case1, case2, label, rng) for label, (case1, case2) in ut.ProgIter(list(zip(labels, test_pairs)), bs=1) ] pairwise_feats = np.vstack(pairwise_feats_) print(ut.dict_hist(labels)) return labels, pairwise_feats
def rectify_labels(G, labels): # Ensure labels are rebased and # are different between different connected compoments graph = G.copy() node_to_annot_idx = nx.get_node_attributes(graph, 'annot_idx') cut_edges = [] for u, v in graph.edges(): idx1 = node_to_annot_idx[u] idx2 = node_to_annot_idx[v] if labels[idx1] != labels[idx2]: cut_edges.append((u, v)) graph.remove_edges_from(cut_edges) ccs_nodes = list(nx.connected_components(graph)) ccs_idxs = ut.unflat_take(node_to_annot_idx, ccs_nodes) # Make consistent sorting ccs_idxs = [sorted(idxs) for idxs in ccs_idxs] ccs_idxs = ut.sortedby(ccs_idxs, ut.take_column(ccs_idxs, 0)) labels = ut.ungroup([[c] * len(x) for c, x in enumerate(ccs_idxs)], ccs_idxs) labels = np.array(labels) return labels
def random_case_set(): r""" Returns: tuple: (labels, pairwise_feats) CommandLine: python -m ibeis.algo.hots.testem random_case_set --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.testem import * # NOQA >>> (labels, pairwise_feats) = random_case_set() >>> result = ('(labels, pairwise_feats) = %s' % (ut.repr2((labels, pairwise_feats)),)) >>> print(result) """ rng = np.random.RandomState(0) case_params = dict(num_names=5, rng=rng) num_annots = 600 test_cases = [random_test_annot(**case_params) for _ in ut.ProgIter(range(num_annots), bs=1)] pairxs = list(ut.product_nonsame(range(num_annots), range(num_annots))) import utool utool.embed() test_pairs = list(ut.unflat_take(test_cases, pairxs)) cases1 = ut.make_instancelist(ut.take_column(test_pairs, 0), check=False) cases2 = ut.make_instancelist(ut.take_column(test_pairs, 1), check=False) # FIXME labels = labels1 = make_test_pairwise_labels2(cases1, cases2) # NOQA #labels = np.array([make_test_pairwise_labels(case1, case2) # for case1, case2 in ut.ProgIter(test_pairs, bs=1)]) pairwise_feats_ = [make_test_pairwise_fetaures(case1, case2, label, rng) for label, (case1, case2) in ut.ProgIter(list(zip(labels, test_pairs)), bs=1)] pairwise_feats = np.vstack(pairwise_feats_) print(ut.dict_hist(labels)) return labels, pairwise_feats
def make_name_model( num_annots, num_names=None, verbose=True, mode=1, num_scores=2, p_score_given_same=None, hack_score_only=False, score_basis=None, special_names=None, ): r""" CommandLine: python -m wbia.algo.hots.bayes --exec-make_name_model --no-cnn python -m wbia.algo.hots.bayes --exec-make_name_model --show --no-cnn python -m wbia.algo.hots.bayes --exec-make_name_model --num-annots=3 Example: >>> # DISABLE_DOCTEST >>> from wbia.algo.hots.bayes import * # NOQA >>> defaults = dict(num_annots=2, num_names=2, verbose=True) >>> modeltype = ut.get_argval('--modeltype', default='bayes') >>> kw = ut.argparse_funckw(make_name_model, defaults) >>> model = make_name_model(**kw) >>> ut.quit_if_noshow() >>> model.show_model(show_prior=False, show_title=False, modeltype=modeltype) >>> ut.show_if_requested() """ if special_names is None: special_names = SPECIAL_BASIS_POOL assert mode == 1, 'only can do mode 1' base = ut.get_argval('--base', type_=str, default='a') annots = ut.chr_range(num_annots, base=base) # The indexes of match CPDs will not change if another annotation is added upper_diag_idxs = ut.colwise_diag_idxs(num_annots, 2) if hack_score_only: upper_diag_idxs = upper_diag_idxs[-hack_score_only:] if num_names is None: num_names = num_annots # +--- Define CPD Templates and Instantiation --- cpd_list = [] # Name Factor name_cpd_t = pgm_ext.TemplateCPD(NAME_TTYPE, ('n', num_names), special_basis_pool=special_names) name_cpds = [name_cpd_t.new_cpd(parents=aid) for aid in annots] # name_cpds = [name_cpd_t.new_cpd(parents=aid, constrain_state=count) # for count, aid in enumerate(annots, start=1)] cpd_list.extend(name_cpds) # Match Factor def match_pmf(match_type, n1, n2): return { True: { 'same': 1.0, 'diff': 0.0 }, False: { 'same': 0.0, 'diff': 1.0 } }[n1 == n2][match_type] match_states = ['diff', 'same'] match_cpd_t = pgm_ext.TemplateCPD( MATCH_TTYPE, match_states, evidence_ttypes=[name_cpd_t, name_cpd_t], pmf_func=match_pmf, ) # match_cpd_t.varpref = 'S' namepair_cpds = ut.unflat_take(name_cpds, upper_diag_idxs) match_cpds = [match_cpd_t.new_cpd(parents=cpds) for cpds in namepair_cpds] cpd_list.extend(match_cpds) # Score Factor score_states = list(range(num_scores)) if score_basis is not None: score_states = ['%.2f' % (s, ) for s in score_basis] if p_score_given_same is None: tmp = np.arange(num_scores + 1)[1:] tmp = np.cumsum(tmp) tmp = tmp / tmp.sum() p_score_given_same = tmp def score_pmf(score_type, match_type): if isinstance(score_type, six.string_types): score_type = score_states.index(score_type) if match_type == 'same': return p_score_given_same[score_type] else: return p_score_given_same[-(score_type + 1)] score_cpd_t = pgm_ext.TemplateCPD(SCORE_TTYPE, score_states, evidence_ttypes=[match_cpd_t], pmf_func=score_pmf) # match_cpd_t.varpref = 'P' score_cpds = [ score_cpd_t.new_cpd(parents=cpds) for cpds in zip(match_cpds) ] cpd_list.extend(score_cpds) with_humans = False if with_humans: human_states = ['diff', 'same'] human_cpd_t = pgm_ext.TemplateCPD( 'human', human_states, evidence_ttypes=[match_cpd_t], pmf_func=[[0.9, 0.1], [0.1, 0.9]], ) human_cpds = [ human_cpd_t.new_cpd(parents=cpds) for cpds in zip(match_cpds) ] cpd_list.extend(human_cpds) with_rank = False # Rank depends on dependant scores if with_rank: rank_states = ['0', '1', '2', '3'] rank_cpd_t = pgm_ext.TemplateCPD('rank', rank_states, evidence_ttypes=[match_cpd_t], pmf_func=None) rank_cpds = [ rank_cpd_t.new_cpd(parents=cpds) for cpds in zip(match_cpds) ] cpd_list.extend(rank_cpds) # L___ End CPD Definitions ___ logger.info('score_cpds = %r' % (ut.list_getattr(score_cpds, 'variable'), )) # Make Model model = pgm_ext.define_model(cpd_list) model.num_names = num_names if verbose: model.print_templates(ignore_ttypes=[MATCH_TTYPE]) return model
def nx_transitive_reduction(G, mode=1): """ References: https://en.wikipedia.org/wiki/Transitive_reduction#Computing_the_reduction_using_the_closure http://dept-info.labri.fr/~thibault/tmp/0201008.pdf http://stackoverflow.com/questions/17078696/im-trying-to-perform-the-transitive-reduction-of-directed-graph-in-python CommandLine: python -m utool.util_graph nx_transitive_reduction --show Example: >>> # DISABLE_DOCTEST >>> from utool.util_graph import * # NOQA >>> import utool as ut >>> import networkx as nx >>> G = nx.DiGraph([('a', 'b'), ('a', 'c'), ('a', 'e'), >>> ('a', 'd'), ('b', 'd'), ('c', 'e'), >>> ('d', 'e'), ('c', 'e'), ('c', 'd')]) >>> G = testdata_graph()[1] >>> G_tr = nx_transitive_reduction(G, mode=1) >>> G_tr2 = nx_transitive_reduction(G, mode=1) >>> ut.quit_if_noshow() >>> import plottool as pt >>> G_ = nx.dag.transitive_closure(G) >>> pt.show_nx(G , pnum=(1, 5, 1), fnum=1) >>> pt.show_nx(G_tr , pnum=(1, 5, 2), fnum=1) >>> pt.show_nx(G_tr2 , pnum=(1, 5, 3), fnum=1) >>> pt.show_nx(G_ , pnum=(1, 5, 4), fnum=1) >>> pt.show_nx(nx.dag.transitive_closure(G_tr), pnum=(1, 5, 5), fnum=1) >>> ut.show_if_requested() """ import utool as ut import networkx as nx has_cycles = not nx.is_directed_acyclic_graph(G) if has_cycles: # FIXME: this does not work for cycle graphs. # Need to do algorithm on SCCs G_orig = G G = nx.condensation(G_orig) nodes = list(G.nodes()) node2_idx = ut.make_index_lookup(nodes) # For each node u, perform DFS consider its set of (non-self) children C. # For each descendant v, of a node in C, remove any edge from u to v. if mode == 1: G_tr = G.copy() for parent in G_tr.nodes(): # Remove self loops if G_tr.has_edge(parent, parent): G_tr.remove_edge(parent, parent) # For each child of the parent for child in list(G_tr.successors(parent)): # Preorder nodes includes its argument (no added complexity) for gchild in list(G_tr.successors(child)): # Remove all edges from parent to non-child descendants for descendant in nx.dfs_preorder_nodes(G_tr, gchild): if G_tr.has_edge(parent, descendant): G_tr.remove_edge(parent, descendant) if has_cycles: # Uncondense graph uncondensed_G_tr = G.__class__() mapping = G.graph['mapping'] uncondensed_G_tr.add_nodes_from(mapping.keys()) inv_mapping = ut.invert_dict(mapping, unique_vals=False) for u, v in G_tr.edges(): u_ = inv_mapping[u][0] v_ = inv_mapping[v][0] uncondensed_G_tr.add_edge(u_, v_) for key, path in inv_mapping.items(): if len(path) > 1: directed_cycle = list(ut.itertwo(path, wrap=True)) uncondensed_G_tr.add_edges_from(directed_cycle) G_tr = uncondensed_G_tr else: def make_adj_matrix(G): edges = list(G.edges()) edge2_idx = ut.partial(ut.dict_take, node2_idx) uv_list = ut.lmap(edge2_idx, edges) A = np.zeros((len(nodes), len(nodes))) A[tuple(np.array(uv_list).T)] = 1 return A G_ = nx.dag.transitive_closure(G) A = make_adj_matrix(G) B = make_adj_matrix(G_) #AB = A * B #AB = A.T.dot(B) AB = A.dot(B) #AB = A.dot(B.T) A_and_notAB = np.logical_and(A, np.logical_not(AB)) tr_uvs = np.where(A_and_notAB) #nodes = G.nodes() edges = list(zip(*ut.unflat_take(nodes, tr_uvs))) G_tr = G.__class__() G_tr.add_nodes_from(nodes) G_tr.add_edges_from(edges) if has_cycles: # Uncondense graph uncondensed_G_tr = G.__class__() mapping = G.graph['mapping'] uncondensed_G_tr.add_nodes_from(mapping.keys()) inv_mapping = ut.invert_dict(mapping, unique_vals=False) for u, v in G_tr.edges(): u_ = inv_mapping[u][0] v_ = inv_mapping[v][0] uncondensed_G_tr.add_edge(u_, v_) for key, path in inv_mapping.items(): if len(path) > 1: directed_cycle = list(ut.itertwo(path, wrap=True)) uncondensed_G_tr.add_edges_from(directed_cycle) G_tr = uncondensed_G_tr return G_tr