def parse_items(cfg): r""" Returns: list: param_list CommandLine: python -m dtool.base --exec-parse_items Example: >>> # ENABLE_DOCTEST >>> from dtool.base import * # NOQA >>> from dtool.example_depcache import DummyVsManyConfig >>> cfg = DummyVsManyConfig() >>> param_list = cfg.parse_items() >>> result = ('param_list = %s' % (ut.repr2(param_list, nl=1),)) >>> print(result) """ namespace_param_list = cfg.parse_namespace_config_items() param_names = ut.get_list_column(namespace_param_list, 1) needs_namespace_keys = ut.find_duplicate_items(param_names) param_list = ut.get_list_column(namespace_param_list, [1, 2]) # prepend namespaces to variables that need it for idx in ut.flatten(needs_namespace_keys.values()): name = namespace_param_list[idx][0] param_list[idx][0] = name + '_' + param_list[idx][0] duplicate_keys = ut.find_duplicate_items(ut.get_list_column(param_list, 0)) # hack to let version through #import utool #with utool.embed_on_exception_context: assert len(duplicate_keys) == 0, ( 'Configs have duplicate names: %r' % duplicate_keys) return param_list
def parse_items(cfg): r""" Returns: list: param_list CommandLine: python -m dtool.base --exec-parse_items Example: >>> # ENABLE_DOCTEST >>> from dtool.base import * # NOQA >>> from dtool.example_depcache import DummyVsManyConfig >>> cfg = DummyVsManyConfig() >>> param_list = cfg.parse_items() >>> result = ('param_list = %s' % (ut.repr2(param_list, nl=1),)) >>> print(result) """ namespace_param_list = cfg.parse_namespace_config_items() param_names = ut.get_list_column(namespace_param_list, 1) needs_namespace_keys = ut.find_duplicate_items(param_names) param_list = ut.get_list_column(namespace_param_list, [1, 2]) # prepend namespaces to variables that need it for idx in ut.flatten(needs_namespace_keys.values()): name = namespace_param_list[idx][0] param_list[idx][0] = name + '_' + param_list[idx][0] duplicate_keys = ut.find_duplicate_items(ut.get_list_column(param_list, 0)) # hack to let version through import utool with utool.embed_on_exception_context: assert len(duplicate_keys) == 0, ( 'Configs have duplicate names: %r' % duplicate_keys) return param_list
def web_check_uuids(ibs, image_uuid_list=[], qannot_uuid_list=[], dannot_uuid_list=[]): r""" Args: ibs (ibeis.IBEISController): image analysis api image_uuid_list (list): (default = []) qannot_uuid_list (list): (default = []) dannot_uuid_list (list): (default = []) CommandLine: python -m ibeis.web.apis_engine --exec-web_check_uuids --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.web.apis_engine import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> image_uuid_list = [] >>> qannot_uuid_list = ibs.get_annot_uuids([1, 1, 2, 3, 2, 4]) >>> dannot_uuid_list = ibs.get_annot_uuids([1, 2, 3]) >>> try: >>> web_check_uuids(ibs, image_uuid_list, qannot_uuid_list, >>> dannot_uuid_list) >>> except controller_inject.DuplicateUUIDException: >>> pass >>> else: >>> raise AssertionError('Should have gotten DuplicateUUIDException') >>> try: >>> web_check_uuids(ibs, [1, 2, 3], qannot_uuid_list, >>> dannot_uuid_list) >>> except controller_inject.WebMissingUUIDException as ex: >>> pass >>> else: >>> raise AssertionError('Should have gotten WebMissingUUIDException') >>> print('Successfully reported errors') """ # Unique list image_uuid_list = list(set(image_uuid_list)) if qannot_uuid_list is None: qannot_uuid_list = [] if dannot_uuid_list is None: dannot_uuid_list = [] annot_uuid_list = list(set(qannot_uuid_list + dannot_uuid_list)) # Check for all annot UUIDs exist missing_image_uuid_list = ibs.get_image_missing_uuid(image_uuid_list) missing_annot_uuid_list = ibs.get_annot_missing_uuid(annot_uuid_list) if len(missing_image_uuid_list) > 0 or len(missing_annot_uuid_list) > 0: kwargs = { 'missing_image_uuid_list': missing_image_uuid_list, 'missing_annot_uuid_list': missing_annot_uuid_list, } raise controller_inject.WebMissingUUIDException(**kwargs) qdup_pos_map = ut.find_duplicate_items(dannot_uuid_list) ddup_pos_map = ut.find_duplicate_items(qannot_uuid_list) if len(ddup_pos_map) + len(qdup_pos_map) > 0: raise controller_inject.DuplicateUUIDException(qdup_pos_map, qdup_pos_map)
def web_check_uuids(ibs, image_uuid_list=[], qannot_uuid_list=[], dannot_uuid_list=[]): r""" Args: ibs (ibeis.IBEISController): image analysis api image_uuid_list (list): (default = []) qannot_uuid_list (list): (default = []) dannot_uuid_list (list): (default = []) CommandLine: python -m ibeis.web.apis_engine --exec-web_check_uuids --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.web.apis_engine import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> image_uuid_list = [] >>> qannot_uuid_list = ibs.get_annot_uuids([1, 1, 2, 3, 2, 4]) >>> dannot_uuid_list = ibs.get_annot_uuids([1, 2, 3]) >>> try: >>> web_check_uuids(ibs, image_uuid_list, qannot_uuid_list, >>> dannot_uuid_list) >>> except controller_inject.DuplicateUUIDException: >>> pass >>> else: >>> raise AssertionError('Should have gotten DuplicateUUIDException') >>> try: >>> web_check_uuids(ibs, [1, 2, 3], qannot_uuid_list, >>> dannot_uuid_list) >>> except controller_inject.WebMissingUUIDException as ex: >>> pass >>> else: >>> raise AssertionError('Should have gotten WebMissingUUIDException') >>> print('Successfully reported errors') """ # Unique list image_uuid_list = list(set(image_uuid_list)) if qannot_uuid_list is None: qannot_uuid_list = [] if dannot_uuid_list is None: dannot_uuid_list = [] annot_uuid_list = list(set(qannot_uuid_list + dannot_uuid_list)) # Check for all annot UUIDs exist missing_image_uuid_list = ibs.get_image_missing_uuid(image_uuid_list) missing_annot_uuid_list = ibs.get_annot_missing_uuid(annot_uuid_list) if len(missing_image_uuid_list) > 0 or len(missing_annot_uuid_list) > 0: kwargs = { 'missing_image_uuid_list' : missing_image_uuid_list, 'missing_annot_uuid_list' : missing_annot_uuid_list, } raise controller_inject.WebMissingUUIDException(**kwargs) qdup_pos_map = ut.find_duplicate_items(dannot_uuid_list) ddup_pos_map = ut.find_duplicate_items(qannot_uuid_list) if len(ddup_pos_map) + len(qdup_pos_map) > 0: raise controller_inject.DuplicateUUIDException(qdup_pos_map, qdup_pos_map)
def find_internal_duplicates(self): # First find which files take up the same amount of space nbytes = self.get_prop('nbytes') dups = ut.find_duplicate_items(nbytes) # Now evaluate the hashes of these candidates cand_idxs = ut.flatten(dups.values()) data = ut.ColumnLists({ 'idx': cand_idxs, 'fname': self.get_prop('fname', cand_idxs), 'dname': self.get_prop('dname', cand_idxs), 'full_path': self.get_prop('full_path', cand_idxs), 'nbytes': self.get_prop('nbytes', cand_idxs), }) # print(ut.repr4(ut.group_items(fpaths, nbytes))) data.ignore = ['full_path', 'dname'] data.print(ignore=['full_path', 'dname']) data['hash'] = self.get_prop('md5', data['idx']) data.print(ignore=['full_path', 'hash']) data.print(ignore=['full_path', 'dname']) multis = data.get_multis('hash') multis.print(ignore=data.ignore) return multis
def duplicates(self): uuid_to_dupxs = ut.find_duplicate_items(self.uuids) dup_fpaths = [ ut.take(self.rel_fpath_list, idxs) for idxs in uuid_to_dupxs.values() ] return dup_fpaths
def convert_multigraph_to_graph(G): """ For each duplicate edge make a dummy node. TODO: preserve data, keys, and directedness """ import utool as ut edge_list = list(G.edges()) node_list = list(G.nodes()) dupitem_to_idx = ut.find_duplicate_items(edge_list) node_to_freq = ut.ddict(lambda: 0) remove_idxs = ut.flatten(dupitem_to_idx.values()) ut.delete_items_by_index(edge_list, remove_idxs) for dup_edge in dupitem_to_idx.keys(): freq = len(dupitem_to_idx[dup_edge]) u, v = dup_edge[0:2] pair_node = dup_edge pair_nodes = [pair_node + tuple([count]) for count in range(freq)] for pair_node in pair_nodes: node_list.append(pair_node) for node in dup_edge: node_to_freq[node] += freq edge_list.append((u, pair_node)) edge_list.append((pair_node, v)) import networkx as nx G2 = nx.DiGraph() G2.add_edges_from(edge_list) G2.add_nodes_from(node_list) return G2
def assert_unique(item_list, ignore=[], name='list', verbose=None): import utool as ut dups = ut.find_duplicate_items(item_list) ut.delete_dict_keys(dups, ignore) if len(dups) > 0: raise AssertionError('Found duplicate items in %s: %s' % (name, ut.repr4(dups))) if verbose: print('No duplicates found in %s' % (name, ))
def assert_testdb_annot_consistency(ibs_gt, ibs2, aid_list1, aid_list2): """ just tests uuids if anything goes wrong this should fix it: from ibeis.other import ibsfuncs aid_list1 = ibs_gt.get_valid_aids() ibs_gt.update_annot_visual_uuids(aid_list1) ibs2.update_annot_visual_uuids(aid_list2) ibsfuncs.fix_remove_visual_dupliate_annotations(ibs_gt) """ assert len(aid_list2) == len(aid_list1) visualtup1 = ibs_gt.get_annot_visual_uuid_info(aid_list1) visualtup2 = ibs2.get_annot_visual_uuid_info(aid_list2) _visual_uuid_list1 = [ut.augment_uuid(*tup) for tup in zip(*visualtup1)] _visual_uuid_list2 = [ut.augment_uuid(*tup) for tup in zip(*visualtup2)] assert ut.hashstr(visualtup1) == ut.hashstr(visualtup2) ut.assert_lists_eq(visualtup1[0], visualtup2[0]) ut.assert_lists_eq(visualtup1[1], visualtup2[1]) ut.assert_lists_eq(visualtup1[2], visualtup2[2]) #semantic_uuid_list1 = ibs_gt.get_annot_semantic_uuids(aid_list1) #semantic_uuid_list2 = ibs2.get_annot_semantic_uuids(aid_list2) visual_uuid_list1 = ibs_gt.get_annot_visual_uuids(aid_list1) visual_uuid_list2 = ibs2.get_annot_visual_uuids(aid_list2) # make sure visual uuids are still determenistic ut.assert_lists_eq(visual_uuid_list1, visual_uuid_list2) ut.assert_lists_eq(_visual_uuid_list1, visual_uuid_list1) ut.assert_lists_eq(_visual_uuid_list2, visual_uuid_list2) if ut.VERBOSE: ibs1_dup_annots = ut.debug_duplicate_items(visual_uuid_list1) ibs2_dup_annots = ut.debug_duplicate_items(visual_uuid_list2) else: ibs1_dup_annots = ut.find_duplicate_items(visual_uuid_list1) ibs2_dup_annots = ut.find_duplicate_items(visual_uuid_list2) # if these fail try ibsfuncs.fix_remove_visual_dupliate_annotations assert len(ibs1_dup_annots) == 0 assert len(ibs2_dup_annots) == 0
def assert_unique(item_list, ignore=[], name='list', verbose=None): import utool as ut dups = ut.find_duplicate_items(item_list) ut.delete_dict_keys(dups, ignore) if len(dups) > 0: raise AssertionError( 'Found duplicate items in %s: %s' % ( name, ut.repr4(dups))) if verbose: print('No duplicates found in %s' % (name,))
def consolodate_duplicates(self): fnames = map(basename, self.rel_fpath_list) duplicate_map = ut.find_duplicate_items(fnames) groups = [] for dupname, idxs in duplicate_map.items(): uuids = self.get_prop('uuids', idxs) unique_uuids, groupxs = ut.group_indices(uuids) groups.extend(ut.apply_grouping(idxs, groupxs)) multitons = [g for g in groups if len(g) > 1] # singletons = [g for g in groups if len(g) <= 1] ut.unflat_take(list(self.fpaths()), multitons)
def find_nonunique_names(self): fnames = map(basename, self.rel_fpath_list) duplicate_map = ut.find_duplicate_items(fnames) groups = [] for dupname, idxs in duplicate_map.items(): uuids = self.get_prop('uuids', idxs) fpaths = self.get_prop('abs', idxs) groups = ut.group_items(fpaths, uuids) if len(groups) > 1: if all(x == 1 for x in map(len, groups.values())): # All groups are different, this is an simpler case print(ut.repr2(groups, nl=3)) else: # Need to handle the multi-item groups first pass
def __init__(qparams, query_cfg=None, cfgdict=None): """ Rename to pipeline params Structure to store static query pipeline parameters parses nested config structure into this flat one Args: query_cfg (QueryConfig): query_config cfgdict (dict or None): dictionary to update query_cfg with CommandLine: python -m ibeis.algo.hots.query_params --test-__init__ Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.query_params import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('testdb1') >>> query_cfg = ibs.cfg.query_cfg >>> #query_cfg.pipeline_root = 'asmk' >>> cfgdict = {'pipeline_root': 'asmk', 'sv_on': False, 'fg_on': True} >>> qparams = QueryParams(query_cfg, cfgdict) >>> assert qparams.pipeline_root == 'smk' >>> assert qparams.fg_on is True >>> result = qparams.query_cfgstr >>> print(')_\n'.join(result.split(')_'))) Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.query_params import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('testdb1') >>> query_cfg = ibs.cfg.query_cfg >>> #query_cfg.pipeline_root = 'asmk' >>> cfgdict = dict(rotation_invariance=True) >>> qparams = QueryParams(query_cfg, cfgdict) >>> ut.assert_eq(qparams.hesaff_params['rotation_invariance'], True) _smk_SMK(agg=True,t=0.0,a=3.0,idf)_ VocabAssign(nAssign=10,a=1.2,s=None,eqw=T)_ VocabTrain(nWords=8000,init=akmeans++,nIters=128,taids=all)_ SV(OFF)_ FEATWEIGHT(ON,uselabel,rf)_ FEAT(hesaff+sift_)_ CHIP(sz450) """ # if given custom settings update the config and ensure feasibilty if query_cfg is None: query_cfg = Config.QueryConfig() if cfgdict is not None: query_cfg = query_cfg.deepcopy() query_cfg.update_query_cfg(**cfgdict) # Get flat item list param_list = Config.parse_config_items(query_cfg) # Assert that there are no config conflicts duplicate_keys = ut.find_duplicate_items( ut.get_list_column(param_list, 0)) assert len(duplicate_keys ) == 0, 'Configs have duplicate names: %r' % duplicate_keys # Set nexted config attributes as flat qparam properties for key, val in param_list: setattr(qparams, key, val) # Add params not implicitly represented in Config object pipeline_root = query_cfg.pipeline_root qparams.chip_cfg_dict = query_cfg._featweight_cfg._feat_cfg._chip_cfg.to_dict( ) qparams.flann_params = query_cfg.flann_cfg.get_flann_params() qparams.hesaff_params = query_cfg._featweight_cfg._feat_cfg.get_hesaff_params( ) qparams.pipeline_root = pipeline_root qparams.vsmany = pipeline_root == 'vsmany' qparams.vsone = pipeline_root == 'vsone' # Add custom strings to the mix as well # TODO; Find better way to specify config strings # FIXME: probchip is not in here qparams.probchip_cfgstr = query_cfg._featweight_cfg.get_cfgstr( use_feat=False, use_chip=False) qparams.featweight_cfgstr = query_cfg._featweight_cfg.get_cfgstr() qparams.chip_cfgstr = query_cfg._featweight_cfg._feat_cfg._chip_cfg.get_cfgstr( ) qparams.feat_cfgstr = query_cfg._featweight_cfg._feat_cfg.get_cfgstr() qparams.nn_cfgstr = query_cfg.nn_cfg.get_cfgstr() qparams.nnweight_cfgstr = query_cfg.nnweight_cfg.get_cfgstr() qparams.sv_cfgstr = query_cfg.sv_cfg.get_cfgstr() qparams.flann_cfgstr = query_cfg.flann_cfg.get_cfgstr() qparams.query_cfgstr = query_cfg.get_cfgstr() qparams.vocabtrain_cfgstr = query_cfg.smk_cfg.vocabtrain_cfg.get_cfgstr( ) qparams.rrvsone_cfgstr = query_cfg.rrvsone_cfg.get_cfgstr()
def show_arch_nx_graph(layers, fnum=None, fullinfo=True): r""" CommandLine: python -m ibeis_cnn.draw_net show_arch_nx_graph:0 --show python -m ibeis_cnn.draw_net show_arch_nx_graph:1 --show Example0: >>> # ENABLE_DOCTEST >>> from ibeis_cnn.draw_net import * # NOQA >>> from ibeis_cnn import models >>> model = models.mnist.MNISTModel(batch_size=128, output_dims=10, >>> data_shape=(24, 24, 3)) >>> model.init_arch() >>> layers = model.get_all_layers() >>> show_arch_nx_graph(layers) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() Example1: >>> # ENABLE_DOCTEST >>> from ibeis_cnn.draw_net import * # NOQA >>> from ibeis_cnn import models >>> model = models.SiameseCenterSurroundModel(autoinit=True) >>> layers = model.get_all_layers() >>> show_arch_nx_graph(layers) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() """ import networkx as nx import plottool as pt import ibeis_cnn.__LASAGNE__ as lasange #from matplotlib import offsetbox #import matplotlib as mpl REMOVE_BATCH_SIZE = True from ibeis_cnn import net_strs def get_hex_color(layer_type): if 'Input' in layer_type: return '#A2CECE' if 'Conv2D' in layer_type: return '#7C9ABB' if 'Dense' in layer_type: return '#6CCF8D' if 'Pool' in layer_type: return '#9D9DD2' if 'SoftMax' in layer_type: return '#7E9FD9' else: return '#{0:x}'.format(hash(layer_type + 'salt') % 2 ** 24) node_dict = {} edge_list = [] edge_attrs = ut.ddict(dict) # Make layer ids (ensure no duplicates) layer_to_id = { l: repr(l) if l.name is None else l.name for l in set(layers) } keys_ = layer_to_id.keys() dups = ut.find_duplicate_items(layer_to_id.values()) for dupval, dupidxs in dups.items(): newval_fmt = dupval + '_%d' for layer in ut.take(keys_, dupidxs): newid = ut.get_nonconflicting_string(newval_fmt, layer_to_id.values()) layer_to_id[layer] = newid def layerid(layer): return layer_to_id[layer] main_nodes = [] for i, layer in enumerate(layers): layer_info = net_strs.get_layer_info(layer) layer_type = layer_info['classalias'] key = layerid(layer) color = get_hex_color(layer_info['classalias']) # Make label lines = [] if layer_info['name'] is not None: lines.append(layer_info['name']) if fullinfo: lines.append(layer_info['classalias']) for attr, val in layer_info['layer_attrs'].items(): if attr == 'shape' and REMOVE_BATCH_SIZE: val = val[1:] if attr == 'output_shape' and REMOVE_BATCH_SIZE: val = val[1:] lines.append('{0}: {1}'.format(attr, val)) nonlinearity = layer_info.get('nonlinearity') if nonlinearity is not None: alias_map = { 'LeakyRectify': 'LReLU', } val = layer_info['nonlinearity']['type'] val = alias_map.get(val, val) lines.append('nonlinearity:\n{0}'.format(val)) label = '\n'.join(lines) # append node is_main_layer = len(layer.params) > 0 #is_main_layer = len(lasange.layers.get_all_params(layer, trainable=True)) > 0 if layer_info['classname'] in lasange.layers.normalization.__all__: is_main_layer = False if layer_info['classname'] in lasange.layers.special.__all__: is_main_layer = False if layer_info['classname'].startswith('BatchNorm'): is_main_layer = False if layer_info['classname'].startswith('ElemwiseSum'): is_main_layer = True if layer_type == 'Input': is_main_layer = True if hasattr(layer, '_is_main_layer'): is_main_layer = layer._is_main_layer #if getattr(layer, 'name', '') is not None and getattr(layer, 'name', '') .endswith('/sum'): # is_main_layer = True node_attr = dict(name=key, label=label, color=color, fillcolor=color, style='filled', is_main_layer=is_main_layer) node_attr['is_main_layer'] = is_main_layer if is_main_layer: main_nodes.append(key) node_attr['classalias'] = layer_info['classalias'] if is_main_layer or node_attr['classalias'].startswith('Conv'): if hasattr(layer, 'shape'): if len(layer.shape) == 3: node_attr['out_size'] = (layer.shape[2], layer.shape[1]) node_attr['depth'] = layer.output_shape[0] if hasattr(layer, 'output_shape'): if len(layer.output_shape) == 4: depth = layer.output_shape[1] width, height = (layer.output_shape[3], layer.output_shape[2]) xshift = -width * (.1 / (depth ** (1 / 3))) / 3 yshift = height * (.1 / (depth ** (1 / 3))) / 2 node_attr['depth'] = depth node_attr['xshift'] = xshift node_attr['yshift'] = yshift node_attr['out_size'] = (width, height) if len(layer.output_shape) == 2: node_attr['out_size'] = (1, layer.output_shape[1]) node_dict[key] = node_attr _input_layers = [] if hasattr(layer, 'input_layers'): _input_layers += layer.input_layers if hasattr(layer, 'input_layer'): _input_layers += [layer.input_layer] for input_layer in _input_layers: parent_key = layerid(input_layer) edge = (parent_key, key) edge_list.append(edge) main_size_ = np.array((100, 100)) * 4 sub_size = np.array((75, 50)) * 4 # Setup scaled width and heights out_size_list = [v['out_size'] for v in node_dict.values() if 'out_size' in v] out_size_list = np.array(out_size_list) #out_size_list = out_size_list[out_size_list.T[0] > 1] area_arr = np.prod(out_size_list, axis=1) main_outsize = np.array(out_size_list[area_arr.argmax()]) #main_outsize = np.array(out_size_list[area_arr.argmin()]) scale = main_size_ / main_outsize scale_dense_max = .25 scale_dense_min = 8 for k, v in node_dict.items(): if v['is_main_layer'] or v['classalias'].startswith('Conv'): if 'out_size' in v: # Make dense layers more visible if v['classalias'] == 'Dense': v['shape'] = 'rect' v['width'] = scale_dense_min if v['out_size'][1] > main_outsize[1]: v['height'] = v['out_size'][1] * scale[1] * scale_dense_max elif v['out_size'][1] < scale_dense_min: v['height'] = scale_dense_min * v['out_size'][1] else: v['height'] = v['out_size'][1] elif v['classalias'].startswith('Conv'): v['shape'] = 'stack' #v['shape'] = 'rect' v['width'] = v['out_size'][0] * scale[0] v['height'] = v['out_size'][1] * scale[1] else: v['shape'] = 'rect' v['width'] = v['out_size'][0] * scale[0] v['height'] = v['out_size'][1] * scale[1] else: v['shape'] = 'rect' v['width'] = main_size_[0] v['height'] = main_size_[1] else: #v['shape'] = 'ellipse' v['shape'] = 'rect' v['style'] = 'rounded' v['width'] = sub_size[0] v['height'] = sub_size[1] key_order = ut.take(layer_to_id, layers) node_dict = ut.dict_subset(node_dict, key_order) #print('node_dict = ' + ut.repr3(node_dict)) # Create the networkx graph structure G = nx.DiGraph() G.add_nodes_from(node_dict.items()) G.add_edges_from(edge_list) for key, val in edge_attrs.items(): nx.set_edge_attributes(G, key, val) # Add invisible structure #main_nodes = [key for key, val in # nx.get_node_attributes(G, 'is_main_layer').items() if val] main_children = ut.odict() #for n1, n2 in ut.itertwo(main_nodes): # print('n1, n2 = %r %r' % (n1, n2)) # import utool # utool.embed() # children = ut.nx_all_nodes_between(G, n1, n2) # if n1 in children: # children.remove(n1) # if n2 in children: # children.remove(n2) # main_children[n1] = children # #pass #main_children[main_nodes[-1]] = [] for n1 in main_nodes: main_children[n1] = [] # Main nodes only place constraints on nodes in the next main group. # Not their own next_main = None G.node[n1]['group'] = n1 for (_, n2) in nx.bfs_edges(G, n1): if next_main is None: if n2 in main_nodes: next_main = n2 else: G.node[n2]['group'] = n1 main_children[n1].append(n2) else: if n2 not in list(nx.descendants(G, next_main)): G.node[n2]['group'] = n1 main_children[n1].append(n2) # Custom positioning x = 0 y = 1000 #print('main_children = %s' % (ut.repr3(main_children),)) #main_nodes = ut.isect(list(nx.topological_sort(G)), main_nodes) xpad = main_size_[0] * .3 ypad = main_size_[1] * .3 # Draw each main node, and then put its children under it # Then move to the left and draw the next main node. cumwidth = 0 for n1 in main_nodes: cumheight = 0 maxwidth = G.node[n1]['width'] for n2 in main_children[n1]: maxwidth = max(maxwidth, G.node[n2]['width']) cumwidth += xpad cumwidth += maxwidth / 2 pos = np.array([x + cumwidth, y - cumheight]) G.node[n1]['pos'] = pos G.node[n1]['pin'] = 'true' height = G.node[n1]['height'] cumheight += height / 2 for n2 in main_children[n1]: height = G.node[n2]['height'] cumheight += ypad cumheight += height / 2 pos = np.array([x + cumwidth, y - cumheight]) G.node[n2]['pos'] = pos G.node[n2]['pin'] = 'true' cumheight += height / 2 cumwidth += maxwidth / 2 # Pin everybody nx.set_node_attributes(G, 'pin', 'true') layoutkw = dict(prog='neato', splines='line') #layoutkw = dict(prog='neato', splines='spline') layoutkw = dict(prog='neato', splines='ortho') G_ = G.copy() # delete lables for positioning _labels = nx.get_node_attributes(G_, 'label') ut.nx_delete_node_attr(G_, 'label') nx.set_node_attributes(G_, 'label', '') nolayout = False if nolayout: G_.remove_edges_from(list(G_.edges())) else: layout_info = pt.nx_agraph_layout(G_, inplace=True, **layoutkw) # NOQA # reset labels if not nolayout: nx.set_node_attributes(G_, 'label', _labels) _ = pt.show_nx(G_, fontsize=8, arrow_width=.3, layout='custom', fnum=fnum) # NOQA #pt.adjust_subplots(top=1, bot=0, left=0, right=1) pt.plt.tight_layout()
def __init__(qparams, query_cfg=None, cfgdict=None): """ Rename to pipeline params Structure to store static query pipeline parameters parses nested config structure into this flat one Args: query_cfg (QueryConfig): query_config cfgdict (dict or None): dictionary to update query_cfg with CommandLine: python -m ibeis.algo.hots.query_params --test-__init__ Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.query_params import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('testdb1') >>> query_cfg = ibs.cfg.query_cfg >>> #query_cfg.pipeline_root = 'asmk' >>> cfgdict = {'pipeline_root': 'asmk', 'sv_on': False, 'fg_on': True} >>> qparams = QueryParams(query_cfg, cfgdict) >>> assert qparams.pipeline_root == 'smk' >>> assert qparams.fg_on is True >>> result = qparams.query_cfgstr >>> print(')_\n'.join(result.split(')_'))) Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.query_params import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('testdb1') >>> query_cfg = ibs.cfg.query_cfg >>> #query_cfg.pipeline_root = 'asmk' >>> cfgdict = dict(rotation_invariance=True) >>> qparams = QueryParams(query_cfg, cfgdict) >>> ut.assert_eq(qparams.hesaff_params['rotation_invariance'], True) _smk_SMK(agg=True,t=0.0,a=3.0,idf)_ VocabAssign(nAssign=10,a=1.2,s=None,eqw=T)_ VocabTrain(nWords=8000,init=akmeans++,nIters=128,taids=all)_ SV(OFF)_ FEATWEIGHT(ON,uselabel,rf)_ FEAT(hesaff+sift_)_ CHIP(sz450) """ # if given custom settings update the config and ensure feasibilty if query_cfg is None: query_cfg = Config.QueryConfig() if cfgdict is not None: query_cfg = query_cfg.deepcopy() query_cfg.update_query_cfg(**cfgdict) # Get flat item list param_list = Config.parse_config_items(query_cfg) # Assert that there are no config conflicts duplicate_keys = ut.find_duplicate_items(ut.get_list_column(param_list, 0)) assert len(duplicate_keys) == 0, 'Configs have duplicate names: %r' % duplicate_keys # Set nexted config attributes as flat qparam properties for key, val in param_list: setattr(qparams, key, val) # Add params not implicitly represented in Config object pipeline_root = query_cfg.pipeline_root qparams.chip_cfg_dict = query_cfg._featweight_cfg._feat_cfg._chip_cfg.to_dict() qparams.flann_params = query_cfg.flann_cfg.get_flann_params() qparams.hesaff_params = query_cfg._featweight_cfg._feat_cfg.get_hesaff_params() qparams.pipeline_root = pipeline_root qparams.vsmany = pipeline_root == 'vsmany' qparams.vsone = pipeline_root == 'vsone' # Add custom strings to the mix as well # TODO; Find better way to specify config strings # FIXME: probchip is not in here qparams.probchip_cfgstr = query_cfg._featweight_cfg.get_cfgstr( use_feat=False, use_chip=False) qparams.featweight_cfgstr = query_cfg._featweight_cfg.get_cfgstr() qparams.chip_cfgstr = query_cfg._featweight_cfg._feat_cfg._chip_cfg.get_cfgstr() qparams.feat_cfgstr = query_cfg._featweight_cfg._feat_cfg.get_cfgstr() qparams.nn_cfgstr = query_cfg.nn_cfg.get_cfgstr() qparams.nnweight_cfgstr = query_cfg.nnweight_cfg.get_cfgstr() qparams.sv_cfgstr = query_cfg.sv_cfg.get_cfgstr() qparams.flann_cfgstr = query_cfg.flann_cfg.get_cfgstr() qparams.query_cfgstr = query_cfg.get_cfgstr() qparams.vocabtrain_cfgstr = query_cfg.smk_cfg.vocabtrain_cfg.get_cfgstr() qparams.rrvsone_cfgstr = query_cfg.rrvsone_cfg.get_cfgstr()
def duplicates(self): uuid_to_dupxs = ut.find_duplicate_items(self.uuids) dup_fpaths = [ut.take(self.rel_fpath_list, idxs) for idxs in uuid_to_dupxs.values()] return dup_fpaths
def web_check_uuids(ibs, image_uuid_list=[], qannot_uuid_list=[], dannot_uuid_list=[]): r""" Args: ibs (wbia.IBEISController): image analysis api image_uuid_list (list): (default = []) qannot_uuid_list (list): (default = []) dannot_uuid_list (list): (default = []) CommandLine: python -m wbia.web.apis_engine --exec-web_check_uuids --show Example: >>> # DISABLE_DOCTEST >>> from wbia.web.apis_engine import * # NOQA >>> import wbia >>> ibs = wbia.opendb(defaultdb='testdb1') >>> image_uuid_list = [] >>> qannot_uuid_list = ibs.get_annot_uuids([1, 1, 2, 3, 2, 4]) >>> dannot_uuid_list = ibs.get_annot_uuids([1, 2, 3]) >>> try: >>> web_check_uuids(ibs, image_uuid_list, qannot_uuid_list, >>> dannot_uuid_list) >>> except controller_inject.WebDuplicateUUIDException: >>> pass >>> else: >>> raise AssertionError('Should have gotten WebDuplicateUUIDException') >>> try: >>> web_check_uuids(ibs, [1, 2, 3], qannot_uuid_list, >>> dannot_uuid_list) >>> except controller_inject.WebMissingUUIDException as ex: >>> pass >>> else: >>> raise AssertionError('Should have gotten WebMissingUUIDException') >>> print('Successfully reported errors') """ import uuid # Unique list if qannot_uuid_list is None: qannot_uuid_list = [] if dannot_uuid_list is None: dannot_uuid_list = [] annot_uuid_list = qannot_uuid_list + dannot_uuid_list # UUID parse check invalid_image_uuid_list = [] for index, image_uuid in enumerate(image_uuid_list): try: assert isinstance(image_uuid, uuid.UUID) except Exception: value = ( index, image_uuid, ) invalid_image_uuid_list.append(value) invalid_annot_uuid_list = [] for index, annot_uuid in enumerate(annot_uuid_list): try: assert isinstance(annot_uuid, uuid.UUID) except Exception: value = ( index, annot_uuid, ) invalid_annot_uuid_list.append(value) if len(invalid_image_uuid_list) > 0 or len(invalid_annot_uuid_list) > 0: kwargs = { 'invalid_image_uuid_list': invalid_image_uuid_list, 'invalid_annot_uuid_list': invalid_annot_uuid_list, } raise controller_inject.WebInvalidUUIDException(**kwargs) image_uuid_list = list(set(image_uuid_list)) annot_uuid_list = list(set(annot_uuid_list)) # Check for all annot UUIDs exist missing_image_uuid_list = ibs.get_image_missing_uuid(image_uuid_list) missing_annot_uuid_list = ibs.get_annot_missing_uuid(annot_uuid_list) if len(missing_image_uuid_list) > 0 or len(missing_annot_uuid_list) > 0: kwargs = { 'missing_image_uuid_list': missing_image_uuid_list, 'missing_annot_uuid_list': missing_annot_uuid_list, } raise controller_inject.WebMissingUUIDException(**kwargs) ddup_pos_map = ut.find_duplicate_items(qannot_uuid_list) qdup_pos_map = ut.find_duplicate_items(dannot_uuid_list) if len(ddup_pos_map) + len(qdup_pos_map) > 0: raise controller_inject.WebDuplicateUUIDException( qdup_pos_map, qdup_pos_map)