def _test_indent_print(): # Indent test code doesnt work in doctest blocks. import utool as ut flag = ut.ensure_logging() print('Checking indent. Should have none') with ut.Indenter('[INDENT] '): print('Checking indent. Should be indented') print('Should no longer be indented') text = ut.get_current_log_text() # The last line might sometimes be empty or not. # Not sure. # New hack: had to put in stride. Seems like logs get written # with two line breaks now last_lines = text.split('\n')[-8::2] if last_lines[-1] != '': assert False, 'DEV ERROR. REMOVE FIRST LINE INSTEAD OF LAST' last_lines = last_lines[:-1] #print('last_lines = %r' % (ut.repr3(last_lines))) try: assert last_lines[0].find('[INDENT] ') == -1, last_lines[0] assert last_lines[1].find('[INDENT] ') >= 0, 'did not indent %r' % ( last_lines[1], ) assert last_lines[2].find('[INDENT] ') == -1, last_lines[2] except AssertionError: print('Error. Last 3 lines') print(ut.repr3(last_lines)) raise if not flag: ut.stop_logging()
def queue_loop(port_dict): iface1, iface2 = port_dict['collect_url1'], port_dict['collect_url2'] print = partial(ut.colorprint, color='green') update_proctitle(queue_name) with ut.Indenter('[%s] ' % (queue_name, )): if VERBOSE_JOBS: print('Init make_queue_loop: name=%r' % (name, )) # bind the client dealer to the queue router rout_sock = ctx.socket(zmq.ROUTER) rout_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'ROUTER') rout_sock.bind(iface1) if VERBOSE_JOBS: print('bind %s_url1 = %r' % ( name, iface1, )) # bind the server router to the queue dealer deal_sock = ctx.socket(zmq.DEALER) deal_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'DEALER') deal_sock.bind(iface2) if VERBOSE_JOBS: print('bind %s_url2 = %r' % ( name, iface2, )) try: if 1: # the remainder of this function can be entirely replaced with zmq.device(zmq.QUEUE, rout_sock, deal_sock) else: # but this shows what is really going on: poller = zmq.Poller() poller.register(rout_sock, zmq.POLLIN) poller.register(deal_sock, zmq.POLLIN) while True: evts = dict(poller.poll()) # poll() returns a list of tuples [(socket, evt), (socket, evt)] # dict(poll()) turns this into {socket:evt, socket:evt} if rout_sock in evts: msg = rout_sock.recv_multipart() # ROUTER sockets prepend the identity of the jobiface, # for routing replies if VERBOSE_JOBS: print('ROUTER relayed %r via DEALER' % (msg, )) deal_sock.send_multipart(msg) if deal_sock in evts: msg = deal_sock.recv_multipart() if VERBOSE_JOBS: print('DEALER relayed %r via ROUTER' % (msg, )) rout_sock.send_multipart(msg) except KeyboardInterrupt: print('Caught ctrl+c in collector loop. Gracefully exiting') if VERBOSE_JOBS: print('Exiting %s' % (loop_name, ))
def cut_step(G, nodes, edges, n_annots, n_names, lookup_annot_idx, edge_probs, pass_values, fail_values): # Create nodes in the graphical model. In this case there are <num_vars> # nodes and each node can be assigned to one of <num_vars> possible labels space = np.full((n_annots,), fill_value=n_names, dtype=opengm.index_type) gm = opengm.gm(space, operator='adder') # Use one potts function for each edge gm = build_factor_graph(G, nodes, edges , n_annots, n_names, lookup_annot_idx, use_unaries=False, edge_probs=edge_probs, operator='adder') with ut.Indenter('[CUTS]'): ut.cprint('Brute Force Labels: (energy minimization)', 'blue') infr = opengm.inference.Bruteforce(gm, accumulator='minimizer') infr.infer() labels = rectify_labels(G, infr.arg()) print(pd.DataFrame(labels, columns=['nid'], index=pd.Series(nodes)).T) print('value = %r' % (infr.value(),)) mc_params = opengm.InfParam(maximalNumberOfConstraintsPerRound=1000000, initializeWith3Cycles=True, edgeRoundingValue=1e-08, timeOut=36000000.0, cutUp=1e+75, reductionMode=3, numThreads=0, # allowCutsWithin=? # workflow=workflow verbose=False, verboseCPLEX=False) infr = opengm.inference.Multicut(gm, parameter=mc_params, accumulator='minimizer') infr.infer() labels = infr.arg() labels = rectify_labels(G, infr.arg()) ut.cprint('Multicut Labels: (energy minimization)', 'blue') print(pd.DataFrame(labels, columns=['nid'], index=pd.Series(nodes)).T) print('value = %r' % (infr.value(),)) if pass_values is not None: gotany = False for pval in pass_values: if all(labels == pval): gotany = True break if not gotany: ut.cprint('INCORRECT DID NOT GET PASS VALUES', 'red') print('pass_values = %r' % (pass_values,)) if fail_values is not None: for fail in fail_values: if all(labels == fail): ut.cprint('INCORRECT', 'red')
def get_job_result(jobiface, jobid): with ut.Indenter('[client %d] ' % (jobiface.id_)): if jobiface.verbose >= 1: print = partial(ut.colorprint, color='teal') print('----') print('Request result of jobid=%r' % (jobid, )) pair_msg = dict(action='job_result', jobid=jobid) # CALLER: collector_request_result jobiface.collect_deal_sock.send_json(pair_msg) if jobiface.verbose >= 3: print('... waiting for collector reply') reply = jobiface.collect_deal_sock.recv_json() if jobiface.verbose >= 2: print('got reply = %s' % (ut.repr2(reply, truncate=True), )) return reply
def debug_nnindexer(nnindexer): r""" Makes sure the indexer has valid SIFT descriptors """ # FIXME: they might not agree if data has been added / removed init_data, extra_data = nnindexer.flann.get_indexed_data() with ut.Indenter('[NNINDEX_DEBUG]'): print('extra_data = %r' % (extra_data,)) print('init_data = %r' % (init_data,)) print('nnindexer.max_distance_sqrd = %r' % (nnindexer.max_distance_sqrd,)) data_agrees = nnindexer.idx2_vec is nnindexer.flann.get_indexed_data()[0] if data_agrees: print('indexed_data agrees') assert vt.check_sift_validity(init_data), 'bad SIFT properties' assert data_agrees, 'indexed data does not agree'
def collector_loop(port_dict, dbdir, containerized): """ Service that stores completed algorithm results """ import ibeis update_proctitle('collector_loop') print = partial(ut.colorprint, color='yellow') with ut.Indenter('[collect] '): collect_rout_sock = ctx.socket(zmq.ROUTER) collect_rout_sock.setsockopt_string(zmq.IDENTITY, 'collect.ROUTER') collect_rout_sock.connect(port_dict['collect_url2']) if VERBOSE_JOBS: print('connect collect_url2 = %r' % (port_dict['collect_url2'], )) ibs = ibeis.opendb(dbdir=dbdir, use_cache=False, web=False) # shelve_path = join(ut.get_shelves_dir(appname='ibeis'), 'engine') shelve_path = ibs.get_shelves_path() ut.delete(shelve_path) ut.ensuredir(shelve_path) collecter_data = {} awaiting_data = {} try: while True: # several callers here # CALLER: collector_notify # CALLER: collector_store # CALLER: collector_request_status # CALLER: collector_request_result idents, collect_request = rcv_multipart_json(collect_rout_sock, print=print) try: reply = on_collect_request(collect_request, collecter_data, awaiting_data, shelve_path, containerized=containerized) except Exception as ex: print(ut.repr3(collect_request)) ut.printex(ex, 'ERROR in collection') send_multipart_json(collect_rout_sock, idents, reply) except KeyboardInterrupt: print('Caught ctrl+c in collector loop. Gracefully exiting') if VERBOSE_JOBS: print('Exiting collector')
def wrp_cache_invalidator(self, *args, **kwargs): # the class must have a table_cache property colscache_ = self.table_cache[tblname] colnames_ = list( six.iterkeys(colscache_)) if colnames is None else colnames if DEBUG_API_CACHE: indenter = ut.Indenter('[%s]' % (tblname, )) indenter.start() print('+------') print( 'INVALIDATING tblname=%r, colnames=%r, rowidx=%r, force=%r' % (tblname, colnames, rowidx, force)) print('self = %r' % (self, )) print('args = %r' % (args, )) print('kwargs = %r' % (kwargs, )) print('colscache_ = ' + ut.dict_str(colscache_, truncate=1)) # Clear the cache of any specified colname # when the invalidator is called if rowidx is None: for colname in colnames_: kwargs_cache_ = colscache_[colname] # We dont know the rowsids so clear everything for cache_ in six.itervalues(kwargs_cache_): cache_.clear() else: rowid_list = args[rowidx] for colname in colnames_: kwargs_cache_ = colscache_[colname] # We know the rowids to delete # iterate over all getter kwargs values for cache_ in six.itervalues(kwargs_cache_): ut.delete_dict_keys(cache_, rowid_list) # Preform set/delete action if DEBUG_API_CACHE: print('After:') print('colscache_ = ' + ut.dict_str(colscache_, truncate=1)) print('L__________') writer_result = writer_func(self, *args, **kwargs) if DEBUG_API_CACHE: indenter.stop() return writer_result
def queue_job(jobiface, action, callback_url=None, callback_method=None, *args, **kwargs): r""" IBEIS: This is just a function that lives in the main thread and ships off a job. FIXME: I do not like having callback_url and callback_method specified like this with args and kwargs. If these must be there then they should be specified first, or THE PREFERED OPTION IS args and kwargs should not be specified without the * syntax The client - sends messages, and receives replies after they have been processed by the """ # NAME: job_client with ut.Indenter('[client %d] ' % (jobiface.id_)): print = partial(ut.colorprint, color='blue') if jobiface.verbose >= 1: print('----') engine_request = { 'action': action, 'args': args, 'kwargs': kwargs, 'callback_url': callback_url, 'callback_method': callback_method } if jobiface.verbose >= 2: print('Queue job: %s' % (engine_request)) # Flow of information tags: # CALLS: engine_queue jobiface.engine_deal_sock.send_json(engine_request) if jobiface.verbose >= 3: print('..sent, waiting for response') # RETURNED FROM: job_client_return reply_notify = jobiface.engine_deal_sock.recv_json() if jobiface.verbose >= 2: print('Got reply: %s' % (reply_notify)) jobid = reply_notify['jobid'] return jobid
def find_offending_part(datetime_str_, timefmt, splitchar=' '): import utool as ut parts_list = datetime_str_.split(splitchar) fmt_list = timefmt.split(splitchar) if len(parts_list) == 1: return for part, fmt in zip(parts_list, fmt_list): print('Trying:') with ut.Indenter(' '): try: print('fmt = %r' % (fmt, )) print('part = %r' % (part, )) datetime.datetime.strptime(part, fmt) except ValueError: find_offending_part(part, fmt, '/') print('Failed') else: print('Passed')
def evaluate_training_data(ibs, qaids, daids_list, varydict, nDaids_basis, verbose=True): nError_list = [] nDaids_list = [] cfgdict_list2 = [] cfgdict_list = ut.all_dict_combinations(varydict) for daids in ut.ProgressIter(daids_list, lbl='testing dbsize'): nDaids = len(daids) print('\n---------------------------') with ut.Indenter('[nDaids=%r]' % (nDaids)): print('nDaids = %r' % nDaids) for cfgdict in ut.ProgressIter(cfgdict_list, lbl='testing cfgdict'): qreq_ = ibs.new_query_request(qaids, daids, cfgdict=cfgdict, verbose=verbose) qres_list = ibs.query_chips(qreq_=qreq_, verbose=verbose) gt_ranks_list = [ qres.get_gt_ranks(ibs=ibs) for qres in qres_list ] incorrect_list = [ len(gt_ranks) == 0 or min(gt_ranks) != 0 for gt_ranks in gt_ranks_list ] nErrors = sum(incorrect_list) nError_list.append(nErrors) nDaids_list.append(nDaids) cfgdict_list2.append(cfgdict.copy()) nError_list = np.array(nError_list) nDaids_list = np.array(nDaids_list) K_list = np.array([cfgdict['K'] for cfgdict in cfgdict_list2]) return nDaids_list, K_list, nError_list
def parse_cfgstr_list2(cfgstr_list, named_defaults_dict=None, cfgtype=None, alias_keys=None, valid_keys=None, expand_nested=True, strict=True, special_join_dict=None, is_nestedcfgtype=False, metadata=None): r""" Parses config strings. By looking up name in a dict of configs DEPRICATE Args: cfgstr_list (list): named_defaults_dict (dict): (default = None) cfgtype (None): (default = None) alias_keys (None): (default = None) valid_keys (None): (default = None) expand_nested (bool): (default = True) strict (bool): (default = True) is_nestedcfgtype - used for annot configs so special joins arent geometrically combined Note: Normal Case: --flag name Custom Arugment Cases: --flag name:custom_key1=custom_val1,custom_key2=custom_val2 Multiple Config Case: --flag name1:custom_args1 name2:custom_args2 Multiple Config (special join) Case: (here name2 and name3 have some special interaction) --flag name1:custom_args1 name2:custom_args2::name3:custom_args3 Varied Argument Case: --flag name:key1=[val1,val2] Returns: list: cfg_combos_list CommandLine: python -m ibeis.expt.cfghelpers --exec-parse_cfgstr_list2 python -m ibeis.expt.cfghelpers --test-parse_cfgstr_list2 Example: >>> # ENABLE_DOCTET >>> from ibeis.expt.cfghelpers import * # NOQA >>> cfgstr_list = ['name', 'name:f=1', 'name:b=[1,2]', 'name1:f=1::name2:f=1,b=2'] >>> #cfgstr_list = ['name', 'name1:f=1::name2:f=1,b=2'] >>> named_defaults_dict = None >>> cfgtype = None >>> alias_keys = None >>> valid_keys = None >>> expand_nested = True >>> strict = False >>> special_join_dict = {'joined': True} >>> cfg_combos_list = parse_cfgstr_list2(cfgstr_list, named_defaults_dict, >>> cfgtype, alias_keys, valid_keys, >>> expand_nested, strict, >>> special_join_dict) >>> print('cfg_combos_list = %s' % (ut.list_str(cfg_combos_list, nl=2),)) >>> print(ut.depth_profile(cfg_combos_list)) >>> cfg_list = ut.flatten(cfg_combos_list) >>> cfg_list = ut.flatten([cfg if isinstance(cfg, list) else [cfg] for cfg in cfg_list]) >>> result = ut.repr2(ut.get_varied_cfg_lbls(cfg_list)) >>> print(result) ['name:', 'name:f=1', 'name:b=1', 'name:b=2', 'name1:f=1,joined=True', 'name2:b=2,f=1,joined=True'] """ with ut.Indenter(' '): cfg_combos_list = [] for cfgstr in cfgstr_list: cfg_combos = [] # Parse special joined cfg case if cfgstr.find('::') > -1: special_cfgstr_list = cfgstr.split('::') special_combo_list = parse_cfgstr_list2( special_cfgstr_list, named_defaults_dict=named_defaults_dict, cfgtype=cfgtype, alias_keys=alias_keys, valid_keys=valid_keys, strict=strict, expand_nested=expand_nested, is_nestedcfgtype=False, metadata=metadata) OLD = False if OLD: special_combo = ut.flatten(special_combo_list) if special_join_dict is not None: for cfg in special_combo: cfg.update(special_join_dict) else: if special_join_dict is not None: for special_combo in special_combo_list: for cfg in special_combo: cfg.update(special_join_dict) if is_nestedcfgtype: cfg_combo = tuple([combo for combo in special_combo_list]) else: # not sure if this is right cfg_combo = special_combo_list # FIXME DUPLICATE CODE if expand_nested: cfg_combos.extend(cfg_combo) else: #print('Appending: ' + str(ut.depth_profile(cfg_combo))) #if ut.depth_profile(cfg_combo) == [1, 9]: # ut.embed() cfg_combos_list.append(cfg_combo) else: cfgname, cfgopt_strs, subx = ut.parse_cfgstr_name_options(cfgstr) # -- # Lookup named default settings try: base_cfg_list = ut.lookup_base_cfg_list(cfgname, named_defaults_dict, metadata=metadata) except Exception as ex: ut.printex(ex, keys=['cfgstr_list']) raise # -- for base_cfg in base_cfg_list: cfg_combo = customize_base_cfg( cfgname, cfgopt_strs, base_cfg, cfgtype, alias_keys, valid_keys, strict=strict, offset=len(cfg_combos)) if is_nestedcfgtype: cfg_combo = [cfg_combo] if expand_nested: cfg_combos.extend(cfg_combo) else: cfg_combos_list.append(cfg_combo) # SUBX Cannot work here because of acfg hackiness #if subx is not None: # cfg_combo = ut.take(cfg_combo, subx) if expand_nested: cfg_combos_list.append(cfg_combos) # print('Updated to: ' + str(ut.depth_profile(cfg_combos_list))) #print('Returning len(cfg_combos_list) = %r' % (len(cfg_combos_list),)) return cfg_combos_list
def get_ibeis_patch_siam_dataset(**kwargs): """ CommandLine: python -m ibeis_cnn.ingest_data --test-get_ibeis_patch_siam_dataset --show python -m ibeis_cnn.ingest_data --test-get_ibeis_patch_siam_dataset --show --db PZ_Master1 --acfg_name default python -m ibeis_cnn.ingest_data --test-get_ibeis_patch_siam_dataset --show --db PZ_Master1 --acfg_name timectrl python -m ibeis_cnn.ingest_data --test-get_ibeis_patch_siam_dataset --show --db PZ_MTEST --acfg_name unctrl --dryrun Example: >>> # ENABLE_DOCTEST >>> from ibeis_cnn.ingest_data import * # NOQA >>> from ibeis_cnn import draw_results >>> import ibeis >>> kwargs = {} # ut.argparse_dict({'max_examples': None, 'num_top': 3}) >>> dataset = get_ibeis_patch_siam_dataset(**kwargs) >>> ut.quit_if_noshow() >>> dataset.interact() >>> ut.show_if_requested() """ datakw = ut.argparse_dict( { #'db': 'PZ_MTEST', 'max_examples': None, #'num_top': 3, 'num_top': None, 'min_featweight': .8 if not ut.WIN32 else None, 'controlled': True, 'colorspace': 'gray', 'acfg_name': None, }, alias_dict={'acfg_name': ['acfg', 'a']}, verbose=True) datakw.update(kwargs) #ut.get_func_kwargs(ingest_ibeis.get_aidpairs_and_matches) if datakw['acfg_name'] is not None: del datakw['controlled'] if datakw['max_examples'] is None: del datakw['max_examples'] if datakw['num_top'] is None: del datakw['num_top'] with ut.Indenter('[LOAD IBEIS DB]'): import ibeis dbname = ut.get_argval('--db', default='PZ_MTEST') ibs = ibeis.opendb(dbname=dbname, defaultdb='PZ_MTEST') # Nets dir is the root dir for all training on this data training_dpath = ibs.get_neuralnet_dir() ut.ensuredir(training_dpath) print('\n\n[get_ibeis_patch_siam_dataset] START') #log_dir = join(training_dpath, 'logs') #ut.start_logging(log_dir=log_dir) alias_key = ibs.get_dbname() + ';' + ut.dict_str( datakw, nl=False, explicit=True) try: if NOCACHE_DATASET: raise Exception('forced cache off') # Try and short circut cached loading dataset = DataSet.from_alias_key(alias_key) dataset.setprop('ibs', lambda: ibeis.opendb(db=dbname)) return dataset except Exception as ex: ut.printex(ex, 'alias definitions have changed. alias_key=%r' % (alias_key, ), iswarning=True) with ut.Indenter('[BuildDS]'): # Get training data pairs colorspace = datakw.pop('colorspace') patchmatch_tup = ingest_ibeis.get_aidpairs_and_matches(ibs, **datakw) aid1_list, aid2_list, kpts1_m_list, kpts2_m_list, fm_list, metadata_lists = patchmatch_tup # Extract and cache the data # TODO: metadata if ut.get_argflag('--dryrun'): print('exiting due to dry run') import sys sys.exit(0) tup = ingest_ibeis.cached_patchmetric_training_data_fpaths( ibs, aid1_list, aid2_list, kpts1_m_list, kpts2_m_list, fm_list, metadata_lists, colorspace=colorspace) data_fpath, labels_fpath, metadata_fpath, training_dpath, data_shape = tup print('\n[get_ibeis_patch_siam_dataset] FINISH\n\n') # hack for caching num_labels labels = ut.load_data(labels_fpath) num_labels = len(labels) dataset = DataSet.new_training_set( alias_key=alias_key, data_fpath=data_fpath, labels_fpath=labels_fpath, metadata_fpath=metadata_fpath, training_dpath=training_dpath, data_shape=data_shape, data_per_label=2, output_dims=1, num_labels=num_labels, ) dataset.setprop('ibs', ibs) return dataset
def get_ibeis_part_siam_dataset(**kwargs): """ PARTS based network data CommandLine: python -m ibeis_cnn.ingest_data --test-get_ibeis_part_siam_dataset --show python -m ibeis_cnn.ingest_data --test-get_ibeis_part_siam_dataset --show --db PZ_Master1 --acfg_name timectrl python -m ibeis_cnn.ingest_data --test-get_ibeis_part_siam_dataset --show --db PZ_MTEST --acfg_name unctrl --dryrun Example: >>> # ENABLE_DOCTEST >>> from ibeis_cnn.ingest_data import * # NOQA >>> from ibeis_cnn import draw_results >>> import ibeis >>> kwargs = {} # ut.argparse_dict({'max_examples': None, 'num_top': 3}) >>> dataset = get_ibeis_part_siam_dataset(**kwargs) >>> ut.quit_if_noshow() >>> dataset.interact(ibs=dataset.getprop('ibs')) >>> ut.show_if_requested() """ import ibeis datakw = ut.argparse_dict( { 'colorspace': 'gray', 'acfg_name': 'ctrl', #'db': None, 'db': 'PZ_MTEST', }, alias_dict={'acfg_name': ['acfg']}, verbose=True) datakw.update(kwargs) print('\n\n[get_ibeis_part_siam_dataset] START') alias_key = ut.dict_str(datakw, nl=False, explicit=True) dbname = datakw.pop('db') try: if NOCACHE_DATASET: raise Exception('forced cache off') # Try and short circut cached loading dataset = DataSet.from_alias_key(alias_key) dataset.setprop('ibs', lambda: ibeis.opendb(db=dbname)) return dataset except Exception as ex: ut.printex(ex, 'alias definitions have changed. alias_key=%r' % (alias_key, ), iswarning=True) with ut.Indenter('[LOAD IBEIS DB]'): ibs = ibeis.opendb(db=dbname) # Nets dir is the root dir for all training on this data training_dpath = ibs.get_neuralnet_dir() ut.ensuredir(training_dpath) with ut.Indenter('[BuildDS]'): # Get training data pairs colorspace = datakw.pop('colorspace') (aid_pairs, label_list, flat_metadata) = ingest_ibeis.get_aidpairs_partmatch(ibs, **datakw) # Extract and cache the data, labels, and metadata if ut.get_argflag('--dryrun'): print('exiting due to dry run') import sys sys.exit(0) tup = ingest_ibeis.cached_part_match_training_data_fpaths( ibs, aid_pairs, label_list, flat_metadata, colorspace=colorspace) data_fpath, labels_fpath, metadata_fpath, training_dpath, data_shape = tup print('\n[get_ibeis_part_siam_dataset] FINISH\n\n') # hack for caching num_labels labels = ut.load_data(labels_fpath) num_labels = len(labels) dataset = DataSet.new_training_set( alias_key=alias_key, data_fpath=data_fpath, labels_fpath=labels_fpath, metadata_fpath=metadata_fpath, training_dpath=training_dpath, data_shape=data_shape, data_per_label=2, output_dims=1, num_labels=num_labels, ) dataset.setprop('ibs', ibs) return dataset
def get_query_components(ibs, qaids): r""" Args: ibs (IBEISController): ibeis controller object qaids (?): Returns: ?: CommandLine: python -m ibeis.algo.hots.query_helpers --test-get_query_components Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.query_helpers import * # NOQA >>> import ibeis >>> # build test data >>> ibs = ibeis.opendb('testdb1') >>> qaids = ibs.get_valid_aids() >>> # execute function >>> result = get_query_components(ibs, qaids) >>> # verify results >>> print(result) """ from ibeis.algo.hots import pipeline from ibeis.algo.hots import query_request daids = ibs.get_valid_aids() cfgdict = dict(with_metadata=True) qreq_ = query_request.new_ibeis_query_request(ibs, qaids, daids, cfgdict) qaid = qaids[0] assert len(daids) > 0, '!!! nothing to search' assert len(qaids) > 0, '!!! nothing to query' qreq_.lazy_load() pipeline_locals_ = pipeline.testrun_pipeline_upto(qreq_, None) qaid2_nns = pipeline_locals_['qaid2_nns'] qaid2_nnvalid0 = pipeline_locals_['qaid2_nnvalid0'] qaid2_filtweights = pipeline_locals_['qaid2_filtweights'] qaid2_nnfilts = pipeline_locals_['qaid2_nnfilts'] qaid2_chipmatch_FILT = pipeline_locals_['qaid2_chipmatch_FILT'] qaid2_chipmatch_SVER = pipeline_locals_['qaid2_chipmatch_SVER'] qaid2_svtups = qreq_.metadata['qaid2_svtups'] #--- qaid2_qres = pipeline.chipmatch_to_resdict(qreq_, qaid2_chipmatch_SVER) ##################### # Testing components ##################### with ut.Indenter('[components]'): qfx2_idx, qfx2_dist = qaid2_nns[qaid] qfx2_aid = qreq_.indexer.get_nn_aids(qfx2_idx) qfx2_fx = qreq_.indexer.get_nn_featxs(qfx2_idx) qfx2_gid = ibs.get_annot_gids(qfx2_aid) # NOQA qfx2_nid = ibs.get_annot_name_rowids(qfx2_aid) # NOQA filtkey_list, qfx2_scores, qfx2_valids = qaid2_nnfilts[qaid] qaid2_nnfilt_ORIG = pipeline.identity_filter(qreq_, qaid2_nns) qaid2_chipmatch_ORIG = pipeline.build_chipmatches(qreq_, qaid2_nns, qaid2_nnfilt_ORIG) qaid2_qres_ORIG = pipeline.chipmatch_to_resdict(qaid2_chipmatch_ORIG, qreq_) qaid2_qres_FILT = pipeline.chipmatch_to_resdict(qaid2_chipmatch_FILT, qreq_) qaid2_qres_SVER = qaid2_qres ##################### # Relevant components ##################### qaid = qaids[0] qres_ORIG = qaid2_qres_ORIG[qaid] qres_FILT = qaid2_qres_FILT[qaid] qres_SVER = qaid2_qres_SVER[qaid] return locals()
def bp_step(G, nodes, edges, n_annots, n_names, lookup_annot_idx): gm = build_factor_graph(G, nodes, edges, n_annots, n_names, lookup_annot_idx, use_unaries=False, edge_probs=None, operator='multiplier') with ut.Indenter('[BELIEF]'): ut.cprint('Brute Force Labels: (probability maximization)', 'blue') infr = opengm.inference.Bruteforce(gm, accumulator='maximizer') infr.infer() labels = rectify_labels(G, infr.arg()) print(pd.DataFrame(labels, columns=['nid'], index=pd.Series(nodes)).T) print('value = %r' % (infr.value(), )) lpb_parmas = opengm.InfParam( damping=0.00, steps=10000, # convergenceBound=0, isAcyclic=False) # http://www.andres.sc/publications/opengm-2.0.2-beta-manual.pdf # I believe multiplier + integrator = marginalization # Manual says multiplier + adder = marginalization # Manual says multiplier + maximizer = probability maximization # infr = opengm.inference.TreeReweightedBp( LBP_algorithm = opengm.inference.BeliefPropagation # LBP_algorithm = opengm.inference.TreeReweightedBp ut.cprint('Belief Propogation (maximization)', 'blue') infr = LBP_algorithm(gm, parameter=lpb_parmas, accumulator='maximizer') infr.infer() labels = rectify_labels(G, infr.arg()) pairwise_factor_idxs = gm.pairwise_factor_idxs factor_marginals = infr.factorMarginals(pairwise_factor_idxs) # print('factor_marginals =\n%r' % (factor_marginals,)) edge_marginals_same_diff_ = [(np.diag(f).sum(), f[~np.eye(f.shape[0], dtype=bool)].sum()) for f in factor_marginals] edge_marginals_same_diff_ = np.array(edge_marginals_same_diff_) edge_marginals_same_diff = edge_marginals_same_diff_.copy() edge_marginals_same_diff /= edge_marginals_same_diff.sum(axis=1, keepdims=True) print('Unnormalized Edge Marginals:') print( pd.DataFrame(edge_marginals_same_diff, columns=['same', 'diff'], index=pd.Series(edges))) # print('Edge marginals after Belief Propogation') # print(pd.DataFrame(edge_marginals_same_diff, columns=['same', 'diff'], index=pd.Series(edges))) print('Labels:') print(pd.DataFrame(labels, columns=['nid'], index=pd.Series(nodes)).T) print('value = %r' % (infr.value(), )) ut.cprint('Belief Propogation (marginalization)', 'blue') infr = LBP_algorithm(gm, parameter=lpb_parmas, accumulator='integrator') infr.infer() labels = rectify_labels(G, infr.arg()) pairwise_factor_idxs = gm.pairwise_factor_idxs factor_marginals = infr.factorMarginals(pairwise_factor_idxs) # print('factor_marginals =\n%r' % (factor_marginals,)) edge_marginals_same_diff_ = [(np.diag(f).sum(), f[~np.eye(f.shape[0], dtype=bool)].sum()) for f in factor_marginals] edge_marginals_same_diff_ = np.array(edge_marginals_same_diff_) edge_marginals_same_diff = edge_marginals_same_diff_.copy() edge_marginals_same_diff /= edge_marginals_same_diff.sum(axis=1, keepdims=True) print('Unnormalized Edge Marginals:') print( pd.DataFrame(edge_marginals_same_diff, columns=['same', 'diff'], index=pd.Series(edges))) # print('Edge marginals after Belief Propogation') # print(pd.DataFrame(edge_marginals_same_diff, columns=['same', 'diff'], index=pd.Series(edges))) print('Labels:') print(pd.DataFrame(labels, columns=['nid'], index=pd.Series(nodes)).T) print('value = %r' % (infr.value(), )) # import plottool as pt # viz_factor_graph(gm) # # _ = pt.show_nx(G) # print("SHOW") # pt.plt.show() # marginals = infr.marginals(annot_idxs) # print('node marginals are') # print(pd.DataFrame(marginals, index=pd.Series(nodes))) return edge_marginals_same_diff
def test_sver_wrapper(): """ Test to ensure cpp and python agree and that cpp is faster CommandLine: python -m vtool.sver_c_wrapper --test-test_sver_wrapper python -m vtool.sver_c_wrapper --test-test_sver_wrapper --rebuild-sver python -m vtool.sver_c_wrapper --test-test_sver_wrapper --show python -m vtool.sver_c_wrapper --test-test_sver_wrapper --show --dummy python -m vtool.sver_c_wrapper --test-test_sver_wrapper --show --fname1=easy1.png --fname2=easy2.png python -m vtool.sver_c_wrapper --test-test_sver_wrapper --show --fname1=easy1.png --fname2=hard3.png python -m vtool.sver_c_wrapper --test-test_sver_wrapper --show --fname1=carl.jpg --fname2=hard3.png Example: >>> # ENABLE_DOCTEST >>> from vtool.sver_c_wrapper import * # NOQA >>> test_sver_wrapper() Ignore: %timeit call_python_version(*args) %timeit get_affine_inliers_cpp(*args) """ import vtool.spatial_verification as sver import vtool.tests.dummy as dummy xy_thresh_sqrd = ktool.KPTS_DTYPE(.4) scale_thresh_sqrd = ktool.KPTS_DTYPE(2.0) ori_thresh = ktool.KPTS_DTYPE(TAU / 4.0) keys = 'xy_thresh_sqrd, scale_thresh_sqrd, ori_thresh'.split(', ') print(ut.dict_str(ut.dict_subset(locals(), keys))) def report_errors(): pass if ut.get_argflag('--dummy'): testtup = dummy.testdata_dummy_matches() (kpts1, kpts2, fm_input, fs_input, rchip1, rchip2) = testtup fm_input = fm_input.astype(fm_dtype) #fm_input = fm_input[0:10].astype(fm_dtype) #fs_input = fs_input[0:10].astype(np.float32) else: fname1 = ut.get_argval('--fname1', type_=str, default='easy1.png') fname2 = ut.get_argval('--fname2', type_=str, default='easy2.png') testtup = dummy.testdata_ratio_matches(fname1, fname2) (kpts1, kpts2, fm_input, fs_input, rchip1, rchip2) = testtup # pack up call to aff hypothesis import vtool as vt import scipy.stats.mstats scales1 = vt.get_scales(kpts1.take(fm_input.T[0], axis=0)) scales2 = vt.get_scales(kpts2.take(fm_input.T[1], axis=0)) #fs_input = 1 / scipy.stats.mstats.gmean(np.vstack((scales1, scales2))) fs_input = scipy.stats.mstats.gmean(np.vstack((scales1, scales2))) print('fs_input = ' + ut.numpy_str(fs_input)) #fs_input[0:-9] = 0 #fs_input = np.ones(len(fm_input), dtype=fs_dtype) #ut.embed() #fs_input = scales1 * scales2 args = (kpts1, kpts2, fm_input, fs_input, xy_thresh_sqrd, scale_thresh_sqrd, ori_thresh) ex_list = [] try: with ut.Indenter('[TEST1] '): inlier_tup = vt.compare_implementations( sver.get_affine_inliers, get_affine_inliers_cpp, args, lbl1='py', lbl2='c', output_lbl=('aff_inliers_list', 'aff_errors_list', 'Aff_mats')) out_inliers, out_errors, out_mats = inlier_tup except AssertionError as ex: ex_list.append(ex) raise try: import functools with ut.Indenter('[TEST2] '): bestinlier_tup = vt.compare_implementations( functools.partial(sver.get_best_affine_inliers, forcepy=True), get_best_affine_inliers_cpp, args, show_output=True, lbl1='py', lbl2='c', output_lbl=('bestinliers', 'besterror', 'bestmat')) bestinliers, besterror, bestmat = bestinlier_tup except AssertionError as ex: ex_list.append(ex) raise if len(ex_list) > 0: raise AssertionError('some tests failed. see previous stdout') #num_inliers_list = np.array(map(len, out_inliers_c)) #best_argx = num_inliers_list.argmax() ##best_inliers_py = out_inliers_py[best_argx] #best_inliers_c = out_inliers_c[best_argx] if ut.show_was_requested(): import plottool as pt fm_output = fm_input.take(bestinliers, axis=0) fnum = pt.next_fnum() pt.figure(fnum=fnum, doclf=True, docla=True) pt.show_chipmatch2(rchip1, rchip2, kpts1, kpts2, fm_input, ell_linewidth=5, fnum=fnum, pnum=(2, 1, 1)) pt.show_chipmatch2(rchip1, rchip2, kpts1, kpts2, fm_output, ell_linewidth=5, fnum=fnum, pnum=(2, 1, 2)) pt.show_if_requested()
def analyze(ibsmap, qreq_dict, species_dict, path_to_file_list, params): print('[analyze] Beginning Analyze') print('[analyze] Received %d file paths' % (len(path_to_file_list))) # decompose the filename to get the car/person to whom this image belongs info_tup_list = [preprocess_fpath(ibsmap, species_dict, path_to_file, params) for path_to_file in path_to_file_list] is_valid_list = [tup_ is not None for tup_ in info_tup_list] # get the ungrouped tuples that were not None valid_tup_list_ug = ut.filter_items(info_tup_list, is_valid_list) valid_path_list_ug = ut.filter_items(path_to_file_list, is_valid_list) # group by species valid_species_list_ug = ut.get_list_column(valid_tup_list_ug, 3) seen_species = {} def get_species_tmpid(txt): if txt in seen_species: return seen_species[txt] else: seen_species[txt] = len(seen_species) return get_species_tmpid(txt) species_tmpid_list = np.array([get_species_tmpid(txt) for txt in valid_species_list_ug]) #ibs.get_species_rowids_from_text(valid_species_list_ug) unique_species_rowids, groupxs = vt.group_indices(np.array(species_tmpid_list)) grouped_valid_tup_list = vt.apply_grouping(np.array(valid_tup_list_ug, dtype=object), groupxs) grouped_path_list = vt.apply_grouping(np.array(valid_path_list_ug, dtype=object), groupxs) print('[analyze] Created %d species groups' % (len(grouped_valid_tup_list))) print('[analyze] grouped_valid_tup_list = ' + ut.list_str(grouped_valid_tup_list)) print('[analyze] grouped_path_list = ' + ut.list_str(grouped_path_list)) assert len(grouped_valid_tup_list) == len(grouped_path_list), 'lengths must match for zip' for groupx, (tup, valid_path_list) in enumerate(zip(grouped_valid_tup_list, grouped_path_list)): car_list, person_list, animal_list, species_list, offset_list, contributor_row_id_list = zip(*tup) assert ut.list_allsame(species_list) animal = animal_list[0] species = species_list[0] ibs = ibsmap[animal] with ut.Indenter('[GROUP-%d-%s]' % (groupx, species)): assert ((animal == 'zebra' and species == species_dict['zebra']) or (animal == 'giraffe' and species == species_dict['giraffe'])), 'animal/species mismatch!' # Add image to database gid_list = ibs.add_images(valid_path_list, auto_localize=False) reported_time_list = list(map(vt.parse_exif_unixtime, valid_path_list)) actual_unixtime_list = [ reported_unixtime + offset for reported_unixtime, offset in zip(reported_time_list, offset_list) ] ibs.set_image_unixtime(gid_list, actual_unixtime_list, duplicate_behavior='filter') ibs.set_image_contributor_rowid(gid_list, contributor_row_id_list, duplicate_behavior='filter') print('[analyze] starting detection for %d images and species %s...' % (len(valid_path_list), species)) qaids_list = ibs.detect_random_forest(gid_list, species=species) qaid_list, reverse_list = ut.invertible_flatten2(qaids_list) print('\n[analyze] detected %d animals of species %s' % (len(qaid_list), species)) # if there were no detections, don't bother if not qaid_list: continue # because qreq_ is persistent we need only to update the qaid_list qreq_ = qreq_dict[animal] # there is a qreq_ for each species qaid_list_unique, unique_inverse = np.unique(qaid_list, return_inverse=True) qreq_.set_external_qaids(qaid_list_unique) qres_list_unique = ibs.query_chips(qreq_=qreq_, verbose=False) qres_list = ut.list_take(qres_list_unique, unique_inverse) # so that we can draw a new bounding box for each detection detection_bbox_list = ibs.get_annot_verts(qaid_list) detection_bboxes_list = ut.unflatten2(detection_bbox_list, reverse_list) qreses_list = ut.unflatten2(qres_list, reverse_list) with ut.Indenter('[POSTPROCESS]'): for _tup in zip(valid_path_list, detection_bboxes_list, qreses_list, car_list, person_list, animal_list, gid_list, qaids_list): postprocess_result(ibs, _tup, params) with ut.Indenter('[REVIEW_CHECK]'): for car, person in zip(car_list, person_list): check_if_need_review(person, car, params)
def precfg(ibs, acfg_name_list, test_cfg_name_list): r""" Helper to precompute information Args: ibs (IBEISController): ibeis controller object qaids (list): query annotation ids daids (list): database annotation ids test_cfg_name_list (list): CommandLine: python -m ibeis.expt.precomputer --exec-precfg -t custom --expt-preload python -m ibeis.expt.precomputer --exec-precfg -t candidacy -a default:qaids=allgt --preload python -m ibeis.expt.precomputer --exec-precfg -t candidacy_invariance -a default:qaids=allgt --preload python -m ibeis.expt.precomputer --exec-precfg --delete-nn-cache Example: >>> # DISABLE_DOCTEST >>> from ibeis.expt.precomputer import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='PZ_MTEST') >>> default_acfgstrs = ['default:qaids=allgt'] >>> acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list, default=default_acfgstrs) >>> test_cfg_name_list = ut.get_argval('-t', type_=list, default=['custom']) >>> result = precfg(ibs, acfg_name_list, test_cfg_name_list) >>> print(result) """ # Generate list of database annotation configurations acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(ibs, acfg_name_list) # Generate list of query pipeline param configs cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs) #cfgx2_lbl = experiment_helpers.get_varied_cfg_lbls(cfgdict_list) expanded_aids_iter = ut.ProgressIter(expanded_aids_list, lbl='annot config', freq=1, autoadjust=False) nAcfg = len(acfg_list) for acfgx, (qaids, daids) in enumerate(expanded_aids_iter): if len(qaids) == 0: print('[harness] WARNING No query annotations specified') continue if len(daids) == 0: print('[harness] WARNING No database annotations specified') continue ut.colorprint('\n---Annot config', 'turquoise') nCfg = len(pipecfg_list) # number of configurations (cols) dbname = ibs.get_dbname() cfgiter = ut.ProgressIter(pipecfg_list, lbl='query config', freq=1, autoadjust=False, parent_index=acfgx, parent_nTotal=nAcfg) flag = False if ut.get_argflag('--delete-nn-cache'): ibs.delete_neighbor_cache() flag = True for cfgx, query_cfg in enumerate(cfgiter): print('') ut.colorprint(query_cfg.get_cfgstr(), 'turquoise') verbose = True with ut.Indenter('[%s cfg %d/%d]' % (dbname, (acfgx * nCfg) + cfgx * + 1, nCfg * nAcfg)): qreq_ = ibs.new_query_request(qaids, daids, verbose=True, query_cfg=query_cfg) if ut.get_argflag('--preload'): qreq_.lazy_preload(verbose=verbose) flag = True if ut.get_argflag('--preload-chip'): qreq_.ensure_chips(verbose=verbose, extra_tries=1) flag = True if ut.get_argflag('--preload-feat'): qreq_.ensure_features(verbose=verbose) flag = True if ut.get_argflag('--preload-featweight'): qreq_.ensure_featweights(verbose=verbose) flag = True if ut.get_argflag('--preindex'): flag = True if qreq_.qparams.pipeline_root in ['vsone', 'vsmany']: qreq_.load_indexer(verbose=verbose) assert flag is True, 'no flag specified' assert flag is True, 'no flag specified'
def make_single_testres( ibs, qaids, daids, pipecfg_list, cfgx2_lbl, cfgdict_list, lbl, testnameid, use_cache=None, subindexer_partial=ut.ProgIter, ): """ CommandLine: python -m wbia run_expt """ cfgslice = None if cfgslice is not None: pipecfg_list = pipecfg_list[cfgslice] dbname = ibs.get_dbname() # if ut.NOT_QUIET: # logger.info('[harn] Make single testres') cfgx2_qreq_ = [ ibs.new_query_request(qaids, daids, verbose=False, query_cfg=pipe_cfg) for pipe_cfg in ut.ProgIter( pipecfg_list, lbl='Building qreq_', enabled=False) ] if use_cache is None: use_cache = USE_BIG_TEST_CACHE if use_cache: try: bt_cachedir = ut.ensuredir( (ibs.get_cachedir(), 'BULK_TEST_CACHE2')) cfgstr_list = [ qreq_.get_cfgstr(with_input=True) for qreq_ in cfgx2_qreq_ ] bt_cachestr = ut.hashstr_arr27(cfgstr_list, ibs.get_dbname() + '_cfgs') bt_cachename = 'BULKTESTCACHE2_v2' testres = ut.load_cache(bt_cachedir, bt_cachename, bt_cachestr) testres.cfgdict_list = cfgdict_list testres.cfgx2_lbl = cfgx2_lbl # hack override except IOError: pass else: if ut.NOT_QUIET: ut.colorprint('[harn] single testres cache hit... returning', 'brightcyan') return testres if ibs.table_cache: # HACK prev_feat_cfgstr = None cfgx2_cmsinfo = [] cfgiter = subindexer_partial(range(len(cfgx2_qreq_)), lbl='pipe config', freq=1, adjust=False) # Run each pipeline configuration for cfgx in cfgiter: qreq_ = cfgx2_qreq_[cfgx] cprint = ut.colorprint cprint('testnameid=%r' % (testnameid, ), 'green') cprint( 'annot_cfgstr = %s' % (qreq_.get_cfgstr(with_input=True, with_pipe=False), ), 'yellow', ) cprint('pipe_cfgstr= %s' % (qreq_.get_cfgstr(with_data=False), ), 'brightcyan') cprint('pipe_hashstr = %s' % (qreq_.get_pipe_hashid(), ), 'cyan') if DRY_RUN: continue indent_prefix = '[%s cfg %d/%d]' % ( dbname, # cfgiter.count (doesnt work when quiet) (cfgiter.parent_index * cfgiter.length) + cfgx, cfgiter.length * cfgiter.parent_length, ) with ut.Indenter(indent_prefix): # Run the test / read cache _need_compute = True if use_cache: # smaller cache for individual configuration runs st_cfgstr = qreq_.get_cfgstr(with_input=True) st_cachedir = ut.unixjoin(bt_cachedir, 'small_tests') st_cachename = 'smalltest' ut.ensuredir(st_cachedir) try: cmsinfo = ut.load_cache(st_cachedir, st_cachename, st_cfgstr) except IOError: _need_compute = True else: _need_compute = False if _need_compute: assert not ibs.table_cache if ibs.table_cache: if len(prev_feat_cfgstr is not None and prev_feat_cfgstr != qreq_.qparams.feat_cfgstr): # Clear features to preserve memory ibs.clear_table_cache() # qreq_.ibs.print_cachestats_str() cm_list = qreq_.execute() cmsinfo = test_result.build_cmsinfo(cm_list, qreq_) # record previous feature configuration if ibs.table_cache: prev_feat_cfgstr = qreq_.qparams.feat_cfgstr if use_cache: ut.save_cache(st_cachedir, st_cachename, st_cfgstr, cmsinfo) if not NOMEMORY: # Store the results cfgx2_cmsinfo.append(cmsinfo) else: cfgx2_qreq_[cfgx] = None if ut.NOT_QUIET: ut.colorprint('[harn] Completed running test configurations', 'white') if DRY_RUN: logger.info('ran tests dryrun mode.') return if NOMEMORY: logger.info('ran tests in memory savings mode. Cannot Print. exiting') return # Store all pipeline config results in a test result object testres = test_result.TestResult(pipecfg_list, cfgx2_lbl, cfgx2_cmsinfo, cfgx2_qreq_) testres.testnameid = testnameid testres.lbl = lbl testres.cfgdict_list = cfgdict_list testres.aidcfg = None if use_cache: try: ut.save_cache(bt_cachedir, bt_cachename, bt_cachestr, testres) except Exception as ex: ut.printex(ex, 'error saving testres cache', iswarning=True) if ut.SUPER_STRICT: raise return testres
def engine_loop(id_, port_dict, dbdir=None): r""" IBEIS: This will be part of a worker process with its own IBEISController instance. Needs to send where the results will go and then publish the results there. The engine_loop - receives messages, performs some action, and sends a reply, preserving the leading two message parts as routing identities """ # NAME: engine_ # CALLED_FROM: engine_queue import ibeis update_proctitle('engine_loop') #base_print = print # NOQA print = partial(ut.colorprint, color='darkred') with ut.Indenter('[engine %d] ' % (id_)): if VERBOSE_JOBS: print('Initializing engine') print('connect engine_url2 = %r' % (port_dict['engine_url2'], )) assert dbdir is not None #ibs = ibeis.opendb(dbname) ibs = ibeis.opendb(dbdir=dbdir, use_cache=False, web=False, force_serial=True) engine_rout_sock = ctx.socket(zmq.ROUTER) engine_rout_sock.connect(port_dict['engine_url2']) collect_deal_sock = ctx.socket(zmq.DEALER) collect_deal_sock.setsockopt_string(zmq.IDENTITY, 'engine.collect.DEALER') collect_deal_sock.connect(port_dict['collect_url1']) if VERBOSE_JOBS: print('connect collect_url1 = %r' % (port_dict['collect_url1'], )) print('engine is initialized') try: while True: idents, engine_request = rcv_multipart_json(engine_rout_sock, print=print) action = engine_request['action'] jobid = engine_request['jobid'] args = engine_request['args'] kwargs = engine_request['kwargs'] callback_url = engine_request['callback_url'] callback_method = engine_request['callback_method'] engine_result = on_engine_request(ibs, jobid, action, args, kwargs) # Store results in the collector collect_request = dict( idents=idents, action='store', jobid=jobid, engine_result=engine_result, callback_url=callback_url, callback_method=callback_method, ) if VERBOSE_JOBS: print('...done working. pushing result to collector') # CALLS: collector_store collect_deal_sock.send_json(collect_request) except KeyboardInterrupt: print('Caught ctrl+c in engine loop. Gracefully exiting') # ---- if VERBOSE_JOBS: print('Exiting engine loop')
def engine_queue_loop(port_dict): """ Specialized queue loop """ # Flow of information tags: # NAME: engine_queue update_proctitle('engine_queue_loop') iface1, iface2 = port_dict['engine_url1'], port_dict['engine_url2'] name = 'engine' queue_name = name + '_queue' loop_name = queue_name + '_loop' print = partial(ut.colorprint, color='red') with ut.Indenter('[%s] ' % (queue_name, )): print('Init specialized make_queue_loop: name=%r' % (name, )) # bind the client dealer to the queue router rout_sock = ctx.socket(zmq.ROUTER) rout_sock.setsockopt_string(zmq.IDENTITY, 'special_queue.' + name + '.' + 'ROUTER') rout_sock.bind(iface1) if VERBOSE_JOBS: print('bind %s_url2 = %r' % ( name, iface1, )) # bind the server router to the queue dealer deal_sock = ctx.socket(zmq.DEALER) deal_sock.setsockopt_string(zmq.IDENTITY, 'special_queue.' + name + '.' + 'DEALER') deal_sock.bind(iface2) if VERBOSE_JOBS: print('bind %s_url2 = %r' % ( name, iface2, )) collect_deal_sock = ctx.socket(zmq.DEALER) collect_deal_sock.setsockopt_string(zmq.IDENTITY, queue_name + '.collect.DEALER') collect_deal_sock.connect(port_dict['collect_url1']) if VERBOSE_JOBS: print('connect collect_url1 = %r' % (port_dict['collect_url1'], )) job_counter = 0 # but this shows what is really going on: poller = zmq.Poller() poller.register(rout_sock, zmq.POLLIN) poller.register(deal_sock, zmq.POLLIN) try: while True: evts = dict(poller.poll()) if rout_sock in evts: # HACK GET REQUEST FROM CLIENT job_counter += 1 # CALLER: job_client idents, engine_request = rcv_multipart_json(rout_sock, num=1, print=print) #jobid = 'result_%s' % (id_,) #jobid = 'result_%s' % (uuid.uuid4(),) jobid = 'jobid-%04d' % (job_counter, ) if VERBOSE_JOBS: print('Creating jobid %r' % (jobid, )) # Reply immediately with a new jobid reply_notify = { 'jobid': jobid, 'status': 'ok', 'text': 'job accepted', 'action': 'notification', } engine_request = engine_request engine_request['jobid'] = jobid if VERBOSE_JOBS: print('...notifying collector about new job') # CALLS: collector_notify collect_deal_sock.send_json(reply_notify) if VERBOSE_JOBS: print('... notifying client that job was accepted') # RETURNS: job_client_return send_multipart_json(rout_sock, idents, reply_notify) if VERBOSE_JOBS: print('... notifying backend engine to start') # CALL: engine_ send_multipart_json(deal_sock, idents, engine_request) if deal_sock in evts: pass except KeyboardInterrupt: print('Caught ctrl+c in %s queue. Gracefully exiting' % (loop_name, )) if VERBOSE_JOBS: print('Exiting %s queue' % (loop_name, ))
def make_single_testres(ibs, qaids, daids, pipecfg_list, cfgx2_lbl, cfgdict_list, lbl, testnameid, use_cache=None, subindexer_partial=ut.ProgressIter): """ CommandLine: python -m ibeis.expt.harness --exec-run_test_configurations2 """ cfgslice = None if cfgslice is not None: pipecfg_list = pipecfg_list[cfgslice] dbname = ibs.get_dbname() if ut.NOT_QUIET: print('[harn] Make single testres') cfgx2_qreq_ = [ ibs.new_query_request(qaids, daids, verbose=False, query_cfg=pipe_cfg) for pipe_cfg in ut.ProgressIter( pipecfg_list, lbl='Building qreq_', enabled=False) ] if use_cache is None: use_cache = USE_BIG_TEST_CACHE if use_cache: get_big_test_cache_info(ibs, cfgx2_qreq_) try: cachetup = get_big_test_cache_info(ibs, cfgx2_qreq_) testres = ut.load_cache(*cachetup) testres.cfgdict_list = cfgdict_list testres.cfgx2_lbl = cfgx2_lbl # hack override except IOError: pass else: if ut.NOT_QUIET: ut.colorprint('[harn] single testres cache hit... returning', 'turquoise') return testres if ibs.table_cache: # HACK prev_feat_cfgstr = None cfgx2_cfgresinfo = [] #nPipeCfg = len(pipecfg_list) cfgiter = subindexer_partial(range(len(cfgx2_qreq_)), lbl='query config', freq=1, adjust=False, separate=True) # Run each pipeline configuration for cfgx in cfgiter: qreq_ = cfgx2_qreq_[cfgx] ut.colorprint('testnameid=%r' % (testnameid, ), 'green') ut.colorprint( 'annot_cfgstr = %s' % (qreq_.get_cfgstr(with_input=True, with_pipe=False), ), 'yellow') ut.colorprint( 'pipe_cfgstr= %s' % (qreq_.get_cfgstr(with_data=False), ), 'turquoise') ut.colorprint('pipe_hashstr = %s' % (qreq_.get_pipe_hashid(), ), 'teal') if DRY_RUN: continue indent_prefix = '[%s cfg %d/%d]' % ( dbname, # cfgiter.count (doesnt work when quiet) (cfgiter.parent_index * cfgiter.nTotal) + cfgx, cfgiter.nTotal * cfgiter.parent_nTotal) with ut.Indenter(indent_prefix): # Run the test / read cache _need_compute = True if use_cache: # smaller cache for individual configuration runs st_cfgstr = qreq_.get_cfgstr(with_input=True) bt_cachedir = cachetup[0] st_cachedir = ut.unixjoin(bt_cachedir, 'small_tests') st_cachename = 'smalltest' ut.ensuredir(st_cachedir) try: cfgres_info = ut.load_cache(st_cachedir, st_cachename, st_cfgstr) except IOError: _need_compute = True else: _need_compute = False if _need_compute: assert not ibs.table_cache if ibs.table_cache: if (len(prev_feat_cfgstr is not None and prev_feat_cfgstr != qreq_.qparams.feat_cfgstr)): # Clear features to preserve memory ibs.clear_table_cache() #qreq_.ibs.print_cachestats_str() cfgres_info = get_query_result_info(qreq_) # record previous feature configuration if ibs.table_cache: prev_feat_cfgstr = qreq_.qparams.feat_cfgstr if use_cache: ut.save_cache(st_cachedir, st_cachename, st_cfgstr, cfgres_info) if not NOMEMORY: # Store the results cfgx2_cfgresinfo.append(cfgres_info) else: cfgx2_qreq_[cfgx] = None if ut.NOT_QUIET: ut.colorprint('[harn] Completed running test configurations', 'white') if DRY_RUN: print('ran tests dryrun mode.') return if NOMEMORY: print('ran tests in memory savings mode. Cannot Print. exiting') return # Store all pipeline config results in a test result object testres = test_result.TestResult(pipecfg_list, cfgx2_lbl, cfgx2_cfgresinfo, cfgx2_qreq_) testres.testnameid = testnameid testres.lbl = lbl testres.cfgdict_list = cfgdict_list testres.aidcfg = None if use_cache: try: ut.save_cache(*tuple(list(cachetup) + [testres])) except Exception as ex: ut.printex(ex, 'error saving testres cache', iswarning=True) if ut.SUPER_STRICT: raise return testres
def get_rowids(depc, tablename, root_rowids, config=None, ensure=True, eager=True, nInput=None, _debug=None, recompute=False, recompute_all=False): """ Returns the rowids of `tablename` that correspond to `root_rowids` using `config`. Ignore: tablename = 'nnindexer' multi_rowids = (1, 2, 3, 4, 5) root_rowids = [[multi_rowids]] import plottool as pt pt.ensureqt() from dtool.depcache_control import * # NOQA from dtool.example_depcache import testdata_depc depc = testdata_depc() exec(ut.execstr_funckw(depc.get_rowids), globals()) print(ut.depth_profile(root_rowids)) tablename = 'neighbs' table = depc[tablename] # NOQA import plottool as pt pt.ensureqt() _debug = depc._debug = True depc.get_rowids(tablename, root_rowids, config, _debug=_debug) pt.show_nx(depc.graph) for key, val in table.type_to_subgraph.items(): pt.show_nx(val) pt.set_title(key) CommandLine: python -m dtool.depcache_control --exec-get_rowids python -m dtool.depcache_control --dump-get_rowids python -m dtool.depcache_control --exec-get_rowids:0 GridParams: >>> param_grid = dict( >>> tablename=[ 'spam', 'neighbs'] # 'spam', 'multitest_score','keypoint'], >>> #tablename=['neighbs', 'keypoint', 'spam', 'multitest_score','keypoint'], >>> ) >>> flat_root_ids = [1, 2, 3] >>> combos = ut.all_dict_combinations(param_grid) >>> index = 0 >>> keys = 'tablename'.split(', ') >>> tablename, = ut.dict_take(combos[index], keys) Setup: >>> # DISABLE_GRID_DOCTEST >>> from dtool.depcache_control import * # NOQA >>> from dtool.example_depcache import testdata_depc >>> depc = testdata_depc() >>> exec(ut.execstr_funckw(depc.get_rowids), globals()) >>> import plottool as pt >>> pt.ensureqt() >>> #pt.show_nx(depc.graph) GridExample0: >>> table = depc[tablename] # NOQA >>> flat_root_ids = [1, 2, 3] >>> root_rowids = [flat_root_ids for _ in table.input_order] >>> print('root_rowids = %r' % (root_rowids,)) >>> #root_rowids = [[flat_root_ids], [(flat_root_ids,)]] >>> #root_rowids = [list(zip(flat_root_ids)), (flat_root_ids,)] >>> _debug = True >>> depc.get_rowids(tablename, root_rowids, config, _debug=_debug) >>> for key, val in table.type_to_subgraph.items(): >>> pt.show_nx(val) >>> pt.set_title(key) Example1: >>> # ENABLE_DOCTEST >>> from dtool.depcache_control import * # NOQA >>> from dtool.example_depcache import testdata_depc >>> depc = testdata_depc() >>> exec(ut.execstr_funckw(depc.get_rowids), globals()) >>> root_rowids = [1, 2, 3] >>> tablename = 'spam' >>> table = depc[tablename] >>> kp_rowids = depc.get_rowids(tablename, root_rowids) >>> #result = ('prop_list = %s' % (ut.repr2(prop_list),)) >>> #print(result) Example: >>> # ENABLE_DOCTEST >>> from dtool.depcache_control import * # NOQA >>> from dtool.example_depcache import testdata_depc >>> depc = testdata_depc() >>> exec(ut.execstr_funckw(depc.get_rowids), globals()) >>> flat_root_ids = [1, 2, 3] >>> kp_rowids = depc.get_rowids('keypoint', flat_root_ids) >>> root_rowids = [flat_root_ids] * 8 >>> _debug = True >>> tablename = 'nnindexer' >>> tablename = 'multitest_score' >>> table = depc[tablename] # NOQA >>> #result = ('prop_list = %s' % (ut.repr2(prop_list),)) >>> # print(result) """ _debug = depc._debug if _debug is None else _debug if _debug: print(' * root_rowids=%s' % (ut.trunc_repr(root_rowids),)) print(' * config = %r' % (config,)) table = depc[tablename] # NOQA INDEXER_VERSION = False if tablename == 'neighbor_index': """ python -m ibeis.core_annots --exec-compute_neighbor_index --show """ import utool utool.embed() if INDEXER_VERSION or tablename == 'neighbs': compute_order = table.compute_order depend_order = compute_order['depend_compute_ids'] input_order = compute_order['input_compute_ids'] if _debug: print(' * input_order = %s' % (ut.repr3(input_order, nl=1),)) print(' * depend_order = %s' % (ut.repr3(depend_order, nl=1),)) if len(input_order) > 1: assert ut.depth_atleast(root_rowids, 2), ( 'input_order = %r' % (input_order,)) with ut.Indenter('[GetRowID-%s]' % (tablename,), enabled=_debug): # New way to get rowids input_level = depend_order[0] mid_levels = depend_order[1:-1] output_level = depend_order[-1] # List that holds a mapping from input order to input "name" input_order_lookup = ut.make_index_lookup(input_order) # Dictionary that holds the rowids computed for each table # while tracing the dependencies. rowid_lookup = ut.odict([(key, ut.odict()) for key in input_order]) # Need to split each path into parts. # Each part represents another level of unflattening # (because root indicies are all flat) # Handle input level assert input_level[0] == depc.root for compute_id in input_order: # for name in input_names: argx = input_order_lookup[compute_id] rowid_lookup[compute_id] = root_rowids[argx] # HACK: Flatten to scalars # The inputs should just be given in the "correct" nesting. # TODO: determine what correct nesting is. for i in range(5): try: current = rowid_lookup[compute_id] rowid_lookup[compute_id] = ut.flatten(current) except Exception: pass level = 0 if _debug: print('input_order_lookup = %r' % (input_order_lookup,)) ut.printdict(rowid_lookup, 'rowid_lookup') def handle_level(compute_id, rowid_lookup, _recompute, level): print('+--- HANDLE LEVEL %d -------' % (level,)) tablekey = compute_id[0] input_suff = compute_id[1] config_ = depc._ensure_config(tablekey, config) table = depc[tablekey] lookupkeys = [(n, input_suff) for n in table.parent_id_tablenames] # ordering = ut.dict_take(input_order_lookup, input_names) # sortx = ut.argsort(ordering) # FIXME: get inputs for each table. # input_names = ut.take(input_names, sortx) # lookupkeys = list(ut.iprod(table.parent_id_tablenames, input_names)) # lookupkeys = list(zip(table.parent_id_tablenames, input_types)) if _debug: print('---- LOCALS ------') ut.print_locals(compute_id, tablekey, lookupkeys, table) print('L----------') # FIXME generalize _parent_ids = [rowid_lookup[tblkey] for tblkey in lookupkeys] if table.ismulti: parent_rowidsT = [[tuple(x)] for x in _parent_ids] else: parent_rowidsT = _parent_ids parent_rowidsT = np.broadcast_arrays(*parent_rowidsT) parent_rowids = list(zip(*parent_rowidsT)) # Probably not right for general multi-input import utool with utool.embed_on_exception_context: next_rowids = table.get_rowid( parent_rowids, config=config_, eager=eager, nInput=nInput, ensure=ensure, recompute=_recompute) rowid_lookup[compute_id] = next_rowids if _debug: ut.printdict(rowid_lookup, 'rowid_lookup') if _debug: print('L___ HANDLE LEVEL %d -------' % (level,)) return next_rowids # Handle mid levels _recompute = recompute_all for level, compute_id in enumerate(mid_levels, start=1): handle_level(compute_id, rowid_lookup, _recompute, level) level += 1 # Handel final (requested) level compute_id = output_level _recompute = recompute rowid_list = handle_level(compute_id, rowid_lookup, _recompute, level) else: with ut.Indenter('[GetRowID-%s]' % (tablename,), enabled=_debug): # TODO: Get nonself rowids first # THen get self rowids for debugging ease try: if False: recompute_ = recompute or recompute_all parent_rowids = depc._get_parent_input( tablename, root_rowids, config, ensure=True, _debug=None, recompute=False, recompute_all=False, eager=True, nInput=None) config_ = depc._ensure_config(tablename, config) #if onthefly: # pass table = depc[tablename] rowid_list = table.get_rowid( parent_rowids, config=config_, eager=eager, nInput=nInput, ensure=ensure, recompute=recompute_) else: # Compute everything from the root to the requested table rowid_dict = depc.get_all_descendant_rowids( tablename, root_rowids, config=config, ensure=ensure, eager=eager, nInput=nInput, recompute=recompute, recompute_all=recompute_all, _debug=ut.countdown_flag(_debug)) rowid_list = rowid_dict[tablename] except depcache_table.ExternalStorageException: print('EXTERNAL EXCEPTION One retry in get_rowids') rowid_dict = depc.get_all_descendant_rowids( tablename, root_rowids, config=config, ensure=ensure, eager=eager, nInput=nInput, recompute=recompute, recompute_all=recompute_all, _debug=ut.countdown_flag(_debug)) rowid_list = rowid_dict[tablename] if _debug: print(' * return rowid_list = %s' % (ut.trunc_repr(rowid_list),)) return rowid_list
def get_all_descendant_rowids(depc, tablename, root_rowids, config=None, ensure=True, eager=True, nInput=None, recompute=False, recompute_all=False, levels_up=None, _debug=False): r""" Connects `root_rowids` to rowids in `tablename`, and computes all values needed along the way. This is the main workhorse function for dependency computations. Args: tablename (str): table to compute dependencies to root_rowids (list): rowids for ``tablename`` config (dict): config applicable for all tables (default = None) ensure (bool): eager evaluation if True(default = True) eager (bool): (default = True) nInput (None): (default = None) recompute (bool): (default = False) recompute_all (bool): (default = False) levels_up (int): only partially compute dependencies (default = 0) _debug (bool): (default = False) CommandLine: python -m dtool.depcache_control --exec-get_all_descendant_rowids:0 python -m dtool.depcache_control --exec-get_all_descendant_rowids:1 Example: >>> # DISABLE_DOCTEST >>> from dtool.depcache_control import * # NOQA >>> from dtool.example_depcache import testdata_depc >>> depc = testdata_depc() >>> tablename = 'spam' >>> root_rowids = [1, 2] >>> config1 = {'dim_size': 500} >>> config2 = {'dim_size': 100} >>> config3 = {'dim_size': 500, 'adapt_shape': False} >>> ensure, eager, nInput = True, True, None >>> _debug = True >>> rowid_dict1 = depc.get_all_descendant_rowids( >>> tablename, root_rowids, config1, ensure, eager, nInput, _debug=_debug) >>> rowid_dict2 = depc.get_all_descendant_rowids( >>> tablename, root_rowids, config2, ensure, eager, nInput, _debug=_debug) >>> rowid_dict3 = depc.get_all_descendant_rowids( >>> tablename, root_rowids, config3, ensure, eager, nInput, _debug=_debug) >>> result1 = 'rowid_dict1 = ' + ut.repr3(rowid_dict1, nl=1) >>> result2 = 'rowid_dict2 = ' + ut.repr3(rowid_dict2, nl=1) >>> result3 = 'rowid_dict3 = ' + ut.repr3(rowid_dict3, nl=1) >>> result = '\n'.join([result1, result2, result3]) >>> print(result) rowid_dict1 = { 'chip': [1, 2], 'dummy_annot': [1, 2], 'fgweight': [1, 2], 'keypoint': [1, 2], 'probchip': [1, 2], 'spam': [1, 2], } rowid_dict2 = { 'chip': [3, 4], 'dummy_annot': [1, 2], 'fgweight': [3, 4], 'keypoint': [3, 4], 'probchip': [1, 2], 'spam': [3, 4], } rowid_dict3 = { 'chip': [1, 2], 'dummy_annot': [1, 2], 'fgweight': [5, 6], 'keypoint': [5, 6], 'probchip': [1, 2], 'spam': [5, 6], } Example: >>> # ENABLE_DOCTEST >>> from dtool.depcache_control import * # NOQA >>> from dtool.example_depcache import testdata_depc >>> depc = testdata_depc() >>> _debug = True >>> tablename = 'vsmany' >>> config = depc.configclass_dict['vsmany']() >>> root_rowids = [1, 2, 3] >>> ensure, eager, nInput = False, True, None >>> # Get rowids of algo ( should be None ) >>> rowid_dict = depc.get_all_descendant_rowids( >>> tablename, root_rowids, config, ensure, eager, nInput, >>> _debug=_debug) >>> result = ut.repr3(rowid_dict, nl=1) >>> print(result) { 'dummy_annot': [1, 2, 3], 'vsmany': [None, None, None], } Example: >>> # ENABLE_DOCTEST >>> from dtool.depcache_control import * # NOQA >>> from dtool.example_depcache import testdata_depc >>> # Make sure algo config can correctly get properites >>> depc = testdata_depc() >>> tablename = 'chip' >>> recompute = False >>> recompute_all = False >>> _debug = True >>> root_rowids = [1, 2] >>> configclass = depc.configclass_dict['chip'] >>> config_ = configclass() >>> config1 = depc.configclass_dict['vsmany'](dim_size=500) >>> config2 = depc.configclass_dict['vsmany'](dim_size=100) >>> config = config2 >>> prop_dicts1 = depc.get_all_descendant_rowids( >>> tablename, root_rowids, config=config1, _debug=_debug) >>> prop_dicts2 = depc.get_all_descendant_rowids( >>> tablename, root_rowids, config=config2, _debug=_debug) >>> print(prop_dicts2) >>> print(prop_dicts1) >>> assert prop_dicts1 != prop_dicts2 Example: >>> # ENABLE_DOCTEST >>> from dtool.depcache_control import * # NOQA >>> from dtool.example_depcache import testdata_depc >>> depc = testdata_depc() >>> exec(ut.execstr_funckw(depc.get_all_descendant_rowids), globals()) >>> _debug = True >>> qaids, daids = [1, 2, 4], [2, 3, 4] >>> root_rowids = list(zip(*ut.product(qaids, daids))) >>> request = depc.new_request('vsone', qaids, daids) >>> results = request.execute() >>> tablename = 'vsone' >>> rowid_dict = depc.get_all_descendant_rowids( >>> tablename, root_rowids, config=None, _debug=_debug) """ # TODO: Need to have a nice way of ensuring configs dont overlap # via namespaces. _debug = depc._debug if _debug is None else _debug indenter = ut.Indenter('[Descend-to-%s]' % (tablename,), enabled=_debug) if _debug: indenter.start() print(' * GET DESCENDANT ROWIDS %s ' % (tablename,)) print(' * config = %r' % (config,)) dependency_levels = depc.get_dependencies(tablename) if levels_up is not None: dependency_levels = dependency_levels[:-levels_up] configclass_levels = [ [depc.configclass_dict.get(tablekey, None) for tablekey in keys] for keys in dependency_levels ] if _debug: print('[depc] dependency_levels = %s' % ut.repr3(dependency_levels, nl=1)) print('[depc] config_levels = %s' % ut.repr3(configclass_levels, nl=1)) # TODO: better support for multi-edges if (len(root_rowids) > 0 and ut.isiterable(root_rowids[0]) and not depc[tablename].ismulti): rowid_dict = {} for colx, col in enumerate(root_rowids): rowid_dict[depc.root + '%d' % (colx + 1,)] = col rowid_dict[depc.root] = ut.unique_ordered(ut.flatten(root_rowids)) else: rowid_dict = {depc.root: root_rowids} # Ensure that each level ``tablename``'s dependencies have been computed for level_keys in dependency_levels[1:]: if _debug: print(' * level_keys %s ' % (level_keys,)) # For each table in the level for tablekey in level_keys: try: child_rowids = depc._expand_level_rowids( tablename, tablekey, rowid_dict, ensure, eager, nInput, config, recompute, recompute_all, _debug) except Exception as ex: table = depc[tablekey] # NOQA keys = ['tablename', 'tablekey', 'rowid_dict', 'config', 'table', 'dependency_levels'] ut.printex(ex, 'error expanding rowids', keys=keys) raise rowid_dict[tablekey] = child_rowids if _debug: print(' GOT DESCENDANT ROWIDS') indenter.stop() return rowid_dict