def consolodate_duplicates(self):
fnames = map(basename, self.rel_fpath_list)
duplicate_map = ut.find_duplicate_items(fnames)
groups = []
for dupname, idxs in duplicate_map.items():
uuids = self.get_prop('uuids', idxs)
unique_uuids, groupxs = ut.group_indices(uuids)
groups.extend(ut.apply_grouping(idxs, groupxs))
multitons = [g for g in groups if len(g) > 1]
# singletons = [g for g in groups if len(g) <= 1]
ut.unflat_take(list(self.fpaths()), multitons)
def consolodate_duplicates(self):
fnames = map(basename, self.rel_fpath_list)
duplicate_map = ut.find_duplicate_items(fnames)
groups = []
for dupname, idxs in duplicate_map.items():
uuids = self.get_prop('uuids', idxs)
unique_uuids, groupxs = ut.group_indices(uuids)
groups.extend(ut.apply_grouping(idxs, groupxs))
multitons = [g for g in groups if len(g) > 1]
# singletons = [g for g in groups if len(g) <= 1]
ut.unflat_take(list(self.fpaths()), multitons)
def color_by_nids(graph, unique_nids=None, ibs=None, nid2_color_=None):
""" Colors edges and nodes by nid """
# TODO use ut.color_nodes
import plottool as pt
ensure_graph_nid_labels(graph, unique_nids, ibs=ibs)
node_to_nid = nx.get_node_attributes(graph, 'nid')
unique_nids = ut.unique(node_to_nid.values())
ncolors = len(unique_nids)
if (ncolors) == 1:
unique_colors = [pt.UNKNOWN_PURP]
else:
if nid2_color_ is not None:
unique_colors = pt.distinct_colors(ncolors + len(nid2_color_) * 2)
else:
unique_colors = pt.distinct_colors(ncolors)
# Find edges and aids strictly between two nids
nid_to_color = dict(zip(unique_nids, unique_colors))
if nid2_color_ is not None:
# HACK NEED TO ENSURE COLORS ARE NOT REUSED
nid_to_color.update(nid2_color_)
edge_aids = list(graph.edges())
edge_nids = ut.unflat_take(node_to_nid, edge_aids)
flags = [nids[0] == nids[1] for nids in edge_nids]
flagged_edge_aids = ut.compress(edge_aids, flags)
flagged_edge_nids = ut.compress(edge_nids, flags)
flagged_edge_colors = [nid_to_color[nids[0]] for nids in flagged_edge_nids]
edge_to_color = dict(zip(flagged_edge_aids, flagged_edge_colors))
node_to_color = ut.map_dict_vals(ut.partial(ut.take, nid_to_color),
node_to_nid)
nx.set_edge_attributes(graph, 'color', edge_to_color)
nx.set_node_attributes(graph, 'color', node_to_color)
def _update_state_gco(model,
weight_key='cut_weight',
name_label_key='name_label'):
import networkx as nx
# Get nx graph properties
external_nodes = sorted(list(model.graph.nodes()))
external_edges = list(model.graph.edges())
edge_to_weights = nx.get_edge_attributes(model.graph, weight_key)
node_to_labeling = nx.get_node_attributes(model.graph, name_label_key)
edge_weights = ut.dict_take(edge_to_weights, external_edges, 0)
external_labeling = [
node_to_labeling.get(node, -node) for node in external_nodes
]
# Map to internal ids for pygco
internal_nodes = ut.rebase_labels(external_nodes)
extern2_intern = dict(zip(external_nodes, internal_nodes))
internal_edges = ut.unflat_take(extern2_intern, external_edges)
internal_labeling = ut.rebase_labels(external_labeling)
internal_labeling = np.array(internal_labeling)
internal_edges = np.array(internal_edges)
n_nodes = len(internal_nodes)
# Model state
model.n_nodes = n_nodes
model.extern2_intern = extern2_intern
model.intern2_extern = ut.invert_dict(extern2_intern)
model.edges = internal_edges
model.edge_weights = edge_weights
# Model parameters
model.labeling = np.zeros(model.n_nodes, dtype=np.int32)
model._update_labels(labeling=internal_labeling)
model._update_weights()
def nx_from_matrix(weight_matrix, nodes=None, remove_self=True):
import networkx as nx
import utool as ut
import numpy as np
if nodes is None:
nodes = list(range(len(weight_matrix)))
weight_list = weight_matrix.ravel()
flat_idxs_ = np.arange(weight_matrix.size)
multi_idxs_ = np.unravel_index(flat_idxs_, weight_matrix.shape)
# Remove 0 weight edges
flags = np.logical_not(np.isclose(weight_list, 0))
weight_list = ut.compress(weight_list, flags)
multi_idxs = ut.compress(list(zip(*multi_idxs_)), flags)
edge_list = ut.lmap(tuple, ut.unflat_take(nodes, multi_idxs))
if remove_self:
flags = [e1 != e2 for e1, e2 in edge_list]
edge_list = ut.compress(edge_list, flags)
weight_list = ut.compress(weight_list, flags)
graph = nx.Graph()
graph.add_nodes_from(nodes)
graph.add_edges_from(edge_list)
label_list = ['%.2f' % w for w in weight_list]
nx.set_edge_attributes(graph, 'weight', dict(zip(edge_list,
weight_list)))
nx.set_edge_attributes(graph, 'label', dict(zip(edge_list,
label_list)))
return graph
def color_by_nids(graph, unique_nids=None, ibs=None, nid2_color_=None):
""" Colors edges and nodes by nid """
# TODO use ut.color_nodes
import plottool as pt
ensure_graph_nid_labels(graph, unique_nids, ibs=ibs)
node_to_nid = nx.get_node_attributes(graph, 'nid')
unique_nids = ut.unique(node_to_nid.values())
ncolors = len(unique_nids)
if (ncolors) == 1:
unique_colors = [pt.UNKNOWN_PURP]
else:
if nid2_color_ is not None:
unique_colors = pt.distinct_colors(ncolors + len(nid2_color_) * 2)
else:
unique_colors = pt.distinct_colors(ncolors)
# Find edges and aids strictly between two nids
nid_to_color = dict(zip(unique_nids, unique_colors))
if nid2_color_ is not None:
# HACK NEED TO ENSURE COLORS ARE NOT REUSED
nid_to_color.update(nid2_color_)
edge_aids = list(graph.edges())
edge_nids = ut.unflat_take(node_to_nid, edge_aids)
flags = [nids[0] == nids[1] for nids in edge_nids]
flagged_edge_aids = ut.compress(edge_aids, flags)
flagged_edge_nids = ut.compress(edge_nids, flags)
flagged_edge_colors = [nid_to_color[nids[0]] for nids in flagged_edge_nids]
edge_to_color = dict(zip(flagged_edge_aids, flagged_edge_colors))
node_to_color = ut.map_dict_vals(ut.partial(ut.take, nid_to_color), node_to_nid)
nx.set_edge_attributes(graph, 'color', edge_to_color)
nx.set_node_attributes(graph, 'color', node_to_color)
def get_cut_edges(model):
extern_uv_list = np.array(list(model.graph.edges()))
intern_uv_list = ut.unflat_take(model.extern2_intern, extern_uv_list)
intern_uv_list = np.array(intern_uv_list)
u_labels = model.labeling[intern_uv_list.T[0]]
v_labels = model.labeling[intern_uv_list.T[1]]
# Remove edges between all annotations with different labels
cut_edges = extern_uv_list[u_labels != v_labels]
cut_edges = [tuple(uv.tolist()) for uv in cut_edges]
return cut_edges
def random_case_set():
r"""
Returns:
tuple: (labels, pairwise_feats)
CommandLine:
python -m ibeis.algo.hots.testem random_case_set --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.testem import * # NOQA
>>> (labels, pairwise_feats) = random_case_set()
>>> result = ('(labels, pairwise_feats) = %s' % (ut.repr2((labels, pairwise_feats)),))
>>> print(result)
"""
rng = np.random.RandomState(0)
case_params = dict(num_names=5, rng=rng)
num_annots = 600
test_cases = [
random_test_annot(**case_params)
for _ in ut.ProgIter(range(num_annots), bs=1)
]
pairxs = list(ut.product_nonsame(range(num_annots), range(num_annots)))
import utool
utool.embed()
test_pairs = list(ut.unflat_take(test_cases, pairxs))
cases1 = ut.instancelist(ut.take_column(test_pairs, 0), check=False)
cases2 = ut.instancelist(ut.take_column(test_pairs, 1), check=False)
# FIXME
labels = labels1 = make_test_pairwise_labels2(cases1, cases2) # NOQA
#labels = np.array([make_test_pairwise_labels(case1, case2)
# for case1, case2 in ut.ProgIter(test_pairs, bs=1)])
pairwise_feats_ = [
make_test_pairwise_fetaures(case1, case2, label, rng)
for label, (case1,
case2) in ut.ProgIter(list(zip(labels, test_pairs)), bs=1)
]
pairwise_feats = np.vstack(pairwise_feats_)
print(ut.dict_hist(labels))
return labels, pairwise_feats
def rectify_labels(G, labels):
# Ensure labels are rebased and
# are different between different connected compoments
graph = G.copy()
node_to_annot_idx = nx.get_node_attributes(graph, 'annot_idx')
cut_edges = []
for u, v in graph.edges():
idx1 = node_to_annot_idx[u]
idx2 = node_to_annot_idx[v]
if labels[idx1] != labels[idx2]:
cut_edges.append((u, v))
graph.remove_edges_from(cut_edges)
ccs_nodes = list(nx.connected_components(graph))
ccs_idxs = ut.unflat_take(node_to_annot_idx, ccs_nodes)
# Make consistent sorting
ccs_idxs = [sorted(idxs) for idxs in ccs_idxs]
ccs_idxs = ut.sortedby(ccs_idxs, ut.take_column(ccs_idxs, 0))
labels = ut.ungroup([[c] * len(x) for c, x in enumerate(ccs_idxs)], ccs_idxs)
labels = np.array(labels)
return labels
def random_case_set():
r"""
Returns:
tuple: (labels, pairwise_feats)
CommandLine:
python -m ibeis.algo.hots.testem random_case_set --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.testem import * # NOQA
>>> (labels, pairwise_feats) = random_case_set()
>>> result = ('(labels, pairwise_feats) = %s' % (ut.repr2((labels, pairwise_feats)),))
>>> print(result)
"""
rng = np.random.RandomState(0)
case_params = dict(num_names=5, rng=rng)
num_annots = 600
test_cases = [random_test_annot(**case_params) for _ in ut.ProgIter(range(num_annots), bs=1)]
pairxs = list(ut.product_nonsame(range(num_annots), range(num_annots)))
import utool
utool.embed()
test_pairs = list(ut.unflat_take(test_cases, pairxs))
cases1 = ut.make_instancelist(ut.take_column(test_pairs, 0), check=False)
cases2 = ut.make_instancelist(ut.take_column(test_pairs, 1), check=False)
# FIXME
labels = labels1 = make_test_pairwise_labels2(cases1, cases2) # NOQA
#labels = np.array([make_test_pairwise_labels(case1, case2)
# for case1, case2 in ut.ProgIter(test_pairs, bs=1)])
pairwise_feats_ = [make_test_pairwise_fetaures(case1, case2, label, rng)
for label, (case1, case2) in ut.ProgIter(list(zip(labels, test_pairs)), bs=1)]
pairwise_feats = np.vstack(pairwise_feats_)
print(ut.dict_hist(labels))
return labels, pairwise_feats
def make_name_model(
num_annots,
num_names=None,
verbose=True,
mode=1,
num_scores=2,
p_score_given_same=None,
hack_score_only=False,
score_basis=None,
special_names=None,
):
r"""
CommandLine:
python -m wbia.algo.hots.bayes --exec-make_name_model --no-cnn
python -m wbia.algo.hots.bayes --exec-make_name_model --show --no-cnn
python -m wbia.algo.hots.bayes --exec-make_name_model --num-annots=3
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.algo.hots.bayes import * # NOQA
>>> defaults = dict(num_annots=2, num_names=2, verbose=True)
>>> modeltype = ut.get_argval('--modeltype', default='bayes')
>>> kw = ut.argparse_funckw(make_name_model, defaults)
>>> model = make_name_model(**kw)
>>> ut.quit_if_noshow()
>>> model.show_model(show_prior=False, show_title=False, modeltype=modeltype)
>>> ut.show_if_requested()
"""
if special_names is None:
special_names = SPECIAL_BASIS_POOL
assert mode == 1, 'only can do mode 1'
base = ut.get_argval('--base', type_=str, default='a')
annots = ut.chr_range(num_annots, base=base)
# The indexes of match CPDs will not change if another annotation is added
upper_diag_idxs = ut.colwise_diag_idxs(num_annots, 2)
if hack_score_only:
upper_diag_idxs = upper_diag_idxs[-hack_score_only:]
if num_names is None:
num_names = num_annots
# +--- Define CPD Templates and Instantiation ---
cpd_list = []
# Name Factor
name_cpd_t = pgm_ext.TemplateCPD(NAME_TTYPE, ('n', num_names),
special_basis_pool=special_names)
name_cpds = [name_cpd_t.new_cpd(parents=aid) for aid in annots]
# name_cpds = [name_cpd_t.new_cpd(parents=aid, constrain_state=count)
# for count, aid in enumerate(annots, start=1)]
cpd_list.extend(name_cpds)
# Match Factor
def match_pmf(match_type, n1, n2):
return {
True: {
'same': 1.0,
'diff': 0.0
},
False: {
'same': 0.0,
'diff': 1.0
}
}[n1 == n2][match_type]
match_states = ['diff', 'same']
match_cpd_t = pgm_ext.TemplateCPD(
MATCH_TTYPE,
match_states,
evidence_ttypes=[name_cpd_t, name_cpd_t],
pmf_func=match_pmf,
)
# match_cpd_t.varpref = 'S'
namepair_cpds = ut.unflat_take(name_cpds, upper_diag_idxs)
match_cpds = [match_cpd_t.new_cpd(parents=cpds) for cpds in namepair_cpds]
cpd_list.extend(match_cpds)
# Score Factor
score_states = list(range(num_scores))
if score_basis is not None:
score_states = ['%.2f' % (s, ) for s in score_basis]
if p_score_given_same is None:
tmp = np.arange(num_scores + 1)[1:]
tmp = np.cumsum(tmp)
tmp = tmp / tmp.sum()
p_score_given_same = tmp
def score_pmf(score_type, match_type):
if isinstance(score_type, six.string_types):
score_type = score_states.index(score_type)
if match_type == 'same':
return p_score_given_same[score_type]
else:
return p_score_given_same[-(score_type + 1)]
score_cpd_t = pgm_ext.TemplateCPD(SCORE_TTYPE,
score_states,
evidence_ttypes=[match_cpd_t],
pmf_func=score_pmf)
# match_cpd_t.varpref = 'P'
score_cpds = [
score_cpd_t.new_cpd(parents=cpds) for cpds in zip(match_cpds)
]
cpd_list.extend(score_cpds)
with_humans = False
if with_humans:
human_states = ['diff', 'same']
human_cpd_t = pgm_ext.TemplateCPD(
'human',
human_states,
evidence_ttypes=[match_cpd_t],
pmf_func=[[0.9, 0.1], [0.1, 0.9]],
)
human_cpds = [
human_cpd_t.new_cpd(parents=cpds) for cpds in zip(match_cpds)
]
cpd_list.extend(human_cpds)
with_rank = False # Rank depends on dependant scores
if with_rank:
rank_states = ['0', '1', '2', '3']
rank_cpd_t = pgm_ext.TemplateCPD('rank',
rank_states,
evidence_ttypes=[match_cpd_t],
pmf_func=None)
rank_cpds = [
rank_cpd_t.new_cpd(parents=cpds) for cpds in zip(match_cpds)
]
cpd_list.extend(rank_cpds)
# L___ End CPD Definitions ___
logger.info('score_cpds = %r' %
(ut.list_getattr(score_cpds, 'variable'), ))
# Make Model
model = pgm_ext.define_model(cpd_list)
model.num_names = num_names
if verbose:
model.print_templates(ignore_ttypes=[MATCH_TTYPE])
return model
def nx_transitive_reduction(G, mode=1):
"""
References:
https://en.wikipedia.org/wiki/Transitive_reduction#Computing_the_reduction_using_the_closure
http://dept-info.labri.fr/~thibault/tmp/0201008.pdf
http://stackoverflow.com/questions/17078696/im-trying-to-perform-the-transitive-reduction-of-directed-graph-in-python
CommandLine:
python -m utool.util_graph nx_transitive_reduction --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_graph import * # NOQA
>>> import utool as ut
>>> import networkx as nx
>>> G = nx.DiGraph([('a', 'b'), ('a', 'c'), ('a', 'e'),
>>> ('a', 'd'), ('b', 'd'), ('c', 'e'),
>>> ('d', 'e'), ('c', 'e'), ('c', 'd')])
>>> G = testdata_graph()[1]
>>> G_tr = nx_transitive_reduction(G, mode=1)
>>> G_tr2 = nx_transitive_reduction(G, mode=1)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> G_ = nx.dag.transitive_closure(G)
>>> pt.show_nx(G , pnum=(1, 5, 1), fnum=1)
>>> pt.show_nx(G_tr , pnum=(1, 5, 2), fnum=1)
>>> pt.show_nx(G_tr2 , pnum=(1, 5, 3), fnum=1)
>>> pt.show_nx(G_ , pnum=(1, 5, 4), fnum=1)
>>> pt.show_nx(nx.dag.transitive_closure(G_tr), pnum=(1, 5, 5), fnum=1)
>>> ut.show_if_requested()
"""
import utool as ut
import networkx as nx
has_cycles = not nx.is_directed_acyclic_graph(G)
if has_cycles:
# FIXME: this does not work for cycle graphs.
# Need to do algorithm on SCCs
G_orig = G
G = nx.condensation(G_orig)
nodes = list(G.nodes())
node2_idx = ut.make_index_lookup(nodes)
# For each node u, perform DFS consider its set of (non-self) children C.
# For each descendant v, of a node in C, remove any edge from u to v.
if mode == 1:
G_tr = G.copy()
for parent in G_tr.nodes():
# Remove self loops
if G_tr.has_edge(parent, parent):
G_tr.remove_edge(parent, parent)
# For each child of the parent
for child in list(G_tr.successors(parent)):
# Preorder nodes includes its argument (no added complexity)
for gchild in list(G_tr.successors(child)):
# Remove all edges from parent to non-child descendants
for descendant in nx.dfs_preorder_nodes(G_tr, gchild):
if G_tr.has_edge(parent, descendant):
G_tr.remove_edge(parent, descendant)
if has_cycles:
# Uncondense graph
uncondensed_G_tr = G.__class__()
mapping = G.graph['mapping']
uncondensed_G_tr.add_nodes_from(mapping.keys())
inv_mapping = ut.invert_dict(mapping, unique_vals=False)
for u, v in G_tr.edges():
u_ = inv_mapping[u][0]
v_ = inv_mapping[v][0]
uncondensed_G_tr.add_edge(u_, v_)
for key, path in inv_mapping.items():
if len(path) > 1:
directed_cycle = list(ut.itertwo(path, wrap=True))
uncondensed_G_tr.add_edges_from(directed_cycle)
G_tr = uncondensed_G_tr
else:
def make_adj_matrix(G):
edges = list(G.edges())
edge2_idx = ut.partial(ut.dict_take, node2_idx)
uv_list = ut.lmap(edge2_idx, edges)
A = np.zeros((len(nodes), len(nodes)))
A[tuple(np.array(uv_list).T)] = 1
return A
G_ = nx.dag.transitive_closure(G)
A = make_adj_matrix(G)
B = make_adj_matrix(G_)
#AB = A * B
#AB = A.T.dot(B)
AB = A.dot(B)
#AB = A.dot(B.T)
A_and_notAB = np.logical_and(A, np.logical_not(AB))
tr_uvs = np.where(A_and_notAB)
#nodes = G.nodes()
edges = list(zip(*ut.unflat_take(nodes, tr_uvs)))
G_tr = G.__class__()
G_tr.add_nodes_from(nodes)
G_tr.add_edges_from(edges)
if has_cycles:
# Uncondense graph
uncondensed_G_tr = G.__class__()
mapping = G.graph['mapping']
uncondensed_G_tr.add_nodes_from(mapping.keys())
inv_mapping = ut.invert_dict(mapping, unique_vals=False)
for u, v in G_tr.edges():
u_ = inv_mapping[u][0]
v_ = inv_mapping[v][0]
uncondensed_G_tr.add_edge(u_, v_)
for key, path in inv_mapping.items():
if len(path) > 1:
directed_cycle = list(ut.itertwo(path, wrap=True))
uncondensed_G_tr.add_edges_from(directed_cycle)
G_tr = uncondensed_G_tr
return G_tr
