def ensure_mtest():
"""
CommandLine:
python dev.py -t mtest
"""
import ibeis
ibeis.ensure_pz_mtest()
def reset_testdbs(**kwargs):
# Step 0) Parse Args
default_args = {'reset_' + key: False
for key in six.iterkeys(TEST_DBNAMES_MAP)}
default_args['reset_all'] = False
default_args.update(kwargs)
argdict = ut.parse_dict_from_argv(default_args)
if not any(list(six.itervalues(argdict))):
# Default behavior is to reset the small dbs
argdict['reset_testdb0'] = True
argdict['reset_testdb1'] = True
argdict['reset_testdb_guiall'] = True
# Step 1) Delete DBs to be Reset
for key, dbname in six.iteritems(TEST_DBNAMES_MAP):
if argdict.get('reset_' + key, False) or argdict['reset_all']:
delete_dbdir(dbname)
# Step 3) Ensure DBs that dont exist
ensure_smaller_testingdbs()
workdir = ibeis.sysres.get_workdir()
if not ut.checkpath(join(workdir, 'PZ_MTEST'), verbose=True):
ibeis.ensure_pz_mtest()
if not ut.checkpath(join(workdir, 'NAUT_test'), verbose=True):
ibeis.ensure_nauts()
if not ut.checkpath(join(workdir, 'testdb2'), verbose=True):
ibeis.init.sysres.ensure_testdb2()
# Step 4) testdb1 becomes the main database
workdir = ibeis.sysres.get_workdir()
TESTDB1 = join(workdir, 'testdb1')
sysres.set_default_dbdir(TESTDB1)
def testdata_ibeis(**kwargs):
"""
DEPRICATE
Step 1
builds ibs for testing
Example:
>>> from ibeis.algo.hots.smk.smk_debug import * # NOQA
>>> kwargs = {}
"""
print(' === Test Data IBEIS ===')
print('kwargs = ' + ut.dict_str(kwargs))
print('[smk_debug] testdata_ibeis')
db = kwargs.get('db', ut.get_argval('--db', str, 'PZ_MTEST'))
#with ut.Indenter('ENSURE'):
if db == 'PZ_MTEST':
ibeis.ensure_pz_mtest()
ibs = ibeis.opendb(db=db)
ibs._default_config()
aggregate = kwargs.get('aggregate', ut.get_argflag(('--agg', '--aggregate')))
nWords = kwargs.get( 'nWords', ut.get_argval(('--nWords', '--nCentroids'), int, default=8E3))
nAssign = kwargs.get( 'nAssign', ut.get_argval(('--nAssign', '--K'), int, default=10))
# Configs
ibs.cfg.query_cfg.pipeline_root = 'smk'
ibs.cfg.query_cfg.smk_cfg.aggregate = aggregate
ibs.cfg.query_cfg.smk_cfg.smk_alpha = 3
ibs.cfg.query_cfg.smk_cfg.smk_thresh = 0
ibs.cfg.query_cfg.smk_cfg.vocabtrain_cfg.nWords = nWords
ibs.cfg.query_cfg.smk_cfg.vocabassign_cfg.nAssign = nAssign
if ut.VERYVERBOSE:
ibs.cfg.query_cfg.smk_cfg.printme3()
return ibs
def ensure_mtest():
"""
CommandLine:
python dev.py -t mtest
"""
import ibeis
ibeis.ensure_pz_mtest()
def ensure_testing_data():
from ibeis.tests import reset_testdbs
print('Making sure test data exists')
import ibeis
from os.path import join
ut.change_term_title('ENSURE IBEIS TETSDATA')
reset_testdbs.reset_testdbs()
workdir = ibeis.get_workdir()
if not ut.checkpath(join(workdir, 'PZ_MTEST')):
ibeis.ensure_pz_mtest()
if not ut.checkpath(join(workdir, 'NAUT_test')):
ibeis.ensure_nauts()
if not ut.checkpath(join(workdir, 'wd_peter2')):
ibeis.ensure_wilddogs()
def testdata():
ibeis.ensure_pz_mtest()
ibs = ibeis.opendb('PZ_MTEST')
# Pandas Annotation Dataframe
annots_df = make_annot_df(ibs)
valid_aids = annots_df.index
# Training set
taids = valid_aids[:]
# Database set
daids = valid_aids[1:]
# Search set
#qaids = valid_aids[0::2]
qaids = valid_aids[0:1]
#default = 1000
default = 2E4
#default=5) # default=95000)
nWords = utool.get_argval(('--nWords', '--nCentroids'), int, default=default)
return ibs, annots_df, taids, daids, qaids, nWords
def testdata():
ibeis.ensure_pz_mtest()
ibs = ibeis.opendb('PZ_MTEST')
# Pandas Annotation Dataframe
annots_df = make_annot_df(ibs)
valid_aids = annots_df.index
# Training set
taids = valid_aids[:]
# Database set
daids = valid_aids[1:]
# Search set
#qaids = valid_aids[0::2]
qaids = valid_aids[0:1]
#default = 1000
default = 2E4
#default=5) # default=95000)
nWords = utool.get_argval(('--nWords', '--nCentroids'),
int,
default=default)
return ibs, annots_df, taids, daids, qaids, nWords
def reset_testdbs(**kwargs):
# Step 0) Parse Args
import ibeis
ibeis.ENABLE_WILDBOOK_SIGNAL = False
default_args = {
'reset_' + key: False
for key in six.iterkeys(TEST_DBNAMES_MAP)
}
default_args['reset_all'] = False
default_args.update(kwargs)
argdict = ut.parse_dict_from_argv(default_args)
if not any(list(six.itervalues(argdict))):
# Default behavior is to reset the small dbs
argdict['reset_testdb0'] = True
argdict['reset_testdb1'] = True
argdict['reset_testdb_guiall'] = True
# Step 1) Delete DBs to be Reset
for key, dbname in six.iteritems(TEST_DBNAMES_MAP):
if argdict.get('reset_' + key, False) or argdict['reset_all']:
delete_dbdir(dbname)
# Step 3) Ensure DBs that dont exist
ensure_smaller_testingdbs()
workdir = ibeis.sysres.get_workdir()
if not ut.checkpath(join(workdir, 'PZ_MTEST'), verbose=True):
ibeis.ensure_pz_mtest()
if not ut.checkpath(join(workdir, 'NAUT_test'), verbose=True):
ibeis.ensure_nauts()
if not ut.checkpath(join(workdir, 'wd_peter2'), verbose=True):
ibeis.ensure_wilddogs()
if not ut.checkpath(join(workdir, 'testdb2'), verbose=True):
ibeis.init.sysres.ensure_testdb2()
# Step 4) testdb1 becomes the main database
workdir = ibeis.sysres.get_workdir()
TESTDB1 = join(workdir, 'testdb1')
sysres.set_default_dbdir(TESTDB1)
def testdata_ibeis(**kwargs):
"""
DEPRICATE
Step 1
builds ibs for testing
Example:
>>> from ibeis.algo.hots.smk.smk_debug import * # NOQA
>>> kwargs = {}
"""
print(' === Test Data IBEIS ===')
print('kwargs = ' + ut.dict_str(kwargs))
print('[smk_debug] testdata_ibeis')
db = kwargs.get('db', ut.get_argval('--db', str, 'PZ_MTEST'))
#with ut.Indenter('ENSURE'):
if db == 'PZ_MTEST':
ibeis.ensure_pz_mtest()
ibs = ibeis.opendb(db=db)
ibs._default_config()
aggregate = kwargs.get('aggregate', ut.get_argflag(
('--agg', '--aggregate')))
nWords = kwargs.get(
'nWords', ut.get_argval(('--nWords', '--nCentroids'), int,
default=8E3))
nAssign = kwargs.get('nAssign',
ut.get_argval(('--nAssign', '--K'), int, default=10))
# Configs
ibs.cfg.query_cfg.pipeline_root = 'smk'
ibs.cfg.query_cfg.smk_cfg.aggregate = aggregate
ibs.cfg.query_cfg.smk_cfg.smk_alpha = 3
ibs.cfg.query_cfg.smk_cfg.smk_thresh = 0
ibs.cfg.query_cfg.smk_cfg.vocabtrain_cfg.nWords = nWords
ibs.cfg.query_cfg.smk_cfg.vocabassign_cfg.nAssign = nAssign
if ut.VERYVERBOSE:
ibs.cfg.query_cfg.smk_cfg.printme3()
return ibs
def ensure_pz_mtest_mergesplit_test():
r"""
Make a test database for MERGE and SPLIT cases
CommandLine:
python -m ibeis.init.sysres --test-ensure_pz_mtest_mergesplit_test
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest_mergesplit_test()
"""
import ibeis
ibeis.ensure_pz_mtest()
workdir = ibeis.sysres.get_workdir()
mtest_dbpath = join(workdir, 'PZ_MTEST')
source_dbdir = mtest_dbpath
dest_dbdir = join(workdir, 'PZ_MERGESPLIT_MTEST')
if ut.get_argflag('--reset'):
ut.delete(dest_dbdir)
if ut.checkpath(dest_dbdir):
return
copy_ibeisdb(source_dbdir, dest_dbdir)
ibs = ibeis.opendb('PZ_MERGESPLIT_MTEST')
assert len(ibs.get_valid_aids()) == 119
assert len(ibs.get_valid_nids()) == 41
aid_list = ibs.get_valid_aids()
aids_list, nid_list = ibs.group_annots_by_name(aid_list)
num_aids = list(map(len, aids_list))
# num cases wanted
num_merge = 3
num_split = 1
num_combo = 1
# num inputs needed
num_merge_names = num_merge
num_split_names = num_split * 2
num_combo_names = num_combo * 3
total_names = num_merge_names + num_split_names + num_combo_names
modify_aids = ut.take(aids_list, ut.list_argsort(num_aids, reverse=True)[0:total_names])
merge_nids1 = ibs.make_next_nids(num_merge, location_text='XMERGE')
merge_nids2 = ibs.make_next_nids(num_merge, location_text='XMERGE')
split_nid = ibs.make_next_nids(num_split, location_text='XSPLIT')[0]
combo_nids = ibs.make_next_nids(num_combo * 2, location_text='XCOMBO')
# the first 3 become merge cases
#left = 0
#right = left + num_merge
for aids, nid1, nid2 in zip(modify_aids[0:3], merge_nids1, merge_nids2):
#ibs.get_annot_nids(aids)
aids_ = aids[::2]
ibs.set_annot_name_rowids(aids_, [nid1] * len(aids_))
ibs.set_annot_name_rowids(aids_, [nid2] * len(aids_))
# the next 2 become split cases
#left = right
#right = left + num_split_names
for aids in modify_aids[3:5]:
ibs.set_annot_name_rowids(aids, [split_nid] * len(aids))
#left = right
#right = left + num_combo_names
# The final 3 are a combination case
for aids in modify_aids[5:8]:
aids_even = aids[::2]
aids_odd = aids[1::2]
ibs.set_annot_name_rowids(aids_even, [combo_nids[0]] * len(aids_even))
ibs.set_annot_name_rowids(aids_odd, [combo_nids[1]] * len(aids_odd))
final_result = ibs.unflat_map(ibs.get_annot_nids, modify_aids)
print('final_result = %s' % (ut.list_str(final_result),))
def ensure_pz_mtest_batchworkflow_test():
r"""
CommandLine:
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test --reset
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test --reset
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest_batchworkflow_test()
"""
import ibeis
ibeis.ensure_pz_mtest()
workdir = ibeis.sysres.get_workdir()
mtest_dbpath = join(workdir, 'PZ_MTEST')
source_dbdir = mtest_dbpath
dest_dbdir = join(workdir, 'PZ_BATCH_WORKFLOW_MTEST')
if ut.get_argflag('--reset'):
ut.delete(dest_dbdir)
if ut.checkpath(dest_dbdir):
return
else:
copy_ibeisdb(source_dbdir, dest_dbdir)
ibs = ibeis.opendb('PZ_BATCH_WORKFLOW_MTEST')
assert len(ibs.get_valid_aids()) == 119
assert len(ibs.get_valid_nids()) == 41
ibs.delete_all_encounters()
aid_list = ibs.get_valid_aids()
unixtime_list = ibs.get_annot_image_unixtimes(aid_list)
untimed_aids = ut.compress(aid_list, [t == -1 for t in unixtime_list])
ibs.get_annot_groundtruth(untimed_aids, aid_list)
aids_list, nid_list = ibs.group_annots_by_name(aid_list)
hourdiffs_list = ibs.get_name_hourdiffs(nid_list)
encounter_aids_list = [[] for _ in range(4)]
encounter_idx = 0
for hourdiffs, aids in zip(hourdiffs_list, aids_list):
#import scipy.spatial.distance as spdist
if len(aids) == 1:
encounter_aids_list[encounter_idx].extend(aids)
encounter_idx = (encounter_idx + 1) % len(encounter_aids_list)
else:
for chunk in list(ut.ichunks(aids, 2)):
encounter_aids_list[encounter_idx].extend(chunk)
encounter_idx = (encounter_idx + 1) % len(encounter_aids_list)
#import vtool as vt
#import networkx as netx
#nodes = list(range(len(aids)))
#edges_pairs = vt.pdist_argsort(hourdiffs)
#edge_weights = -hourdiffs[hourdiffs.argsort()]
#netx_graph = make_netx_graph(edges_pairs, nodes, edge_weights)
#cut_edges = netx.minimum_edge_cut(netx_graph)
#netx_graph.remove_edges_from(cut_edges)
#components = list(netx.connected_components(netx_graph))
#components = ut.sortedby(components, list(map(len, components)), reverse=True)
#print(components)
#encounter_aids_list[0].extend(components[0])
#for compoment in components:
# TODO do max-nway cut
#day_diffs = spdist.squareform(hourdiffs) / 24.0
#print(ut.numpy_str(day_diffs, precision=2, suppress_small=True))
#import itertools
#compare_idxs = [(r, c) for r, c in itertools.product(range(len(aids)), range(len(aids))) if (c > r)]
#print(len(aids))
#def make_netx_graph(edges_pairs, nodes=None, edge_weights=None):
# import networkx as netx
# node_lbls = [('id_', 'int')]
# edge_lbls = [('weight', 'float')]
# edges = [(pair[0], pair[1], weight) for pair, weight in zip(edges_pairs, edge_weights)]
# print('make_netx_graph')
# # Make a graph between the chips
# netx_nodes = [(ntup[0], {key[0]: val for (key, val) in zip(node_lbls, ntup[1:])})
# for ntup in iter(zip(nodes))]
# netx_edges = [(etup[0], etup[1], {key[0]: val for (key, val) in zip(edge_lbls, etup[2:])})
# for etup in iter(edges)]
# netx_graph = netx.Graph()
# netx_graph.add_nodes_from(netx_nodes)
# netx_graph.add_edges_from(netx_edges)
# return netx_graph
# Group into encounters based on old names
gids_list = ibs.unflat_map(ibs.get_annot_image_rowids, encounter_aids_list)
eid_list = ibs.new_encounters_from_images(gids_list) # NOQA
# Remove all names
ibs.delete_annot_nids(aid_list)
def ensure_pz_mtest_mergesplit_test():
r"""
Make a test database for MERGE and SPLIT cases
CommandLine:
python -m ibeis.init.sysres --test-ensure_pz_mtest_mergesplit_test
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest_mergesplit_test()
"""
import ibeis
ibeis.ensure_pz_mtest()
workdir = ibeis.sysres.get_workdir()
mtest_dbpath = join(workdir, 'PZ_MTEST')
source_dbdir = mtest_dbpath
dest_dbdir = join(workdir, 'PZ_MERGESPLIT_MTEST')
if ut.get_argflag('--reset'):
ut.delete(dest_dbdir)
if ut.checkpath(dest_dbdir):
return
copy_ibeisdb(source_dbdir, dest_dbdir)
ibs = ibeis.opendb('PZ_MERGESPLIT_MTEST')
assert len(ibs.get_valid_aids()) == 119
assert len(ibs.get_valid_nids()) == 41
aid_list = ibs.get_valid_aids()
aids_list, nid_list = ibs.group_annots_by_name(aid_list)
num_aids = list(map(len, aids_list))
# num cases wanted
num_merge = 3
num_split = 1
num_combo = 1
# num inputs needed
num_merge_names = num_merge
num_split_names = num_split * 2
num_combo_names = num_combo * 3
total_names = num_merge_names + num_split_names + num_combo_names
modify_aids = ut.take(
aids_list,
ut.list_argsort(num_aids, reverse=True)[0:total_names])
merge_nids1 = ibs.make_next_nids(num_merge, location_text='XMERGE')
merge_nids2 = ibs.make_next_nids(num_merge, location_text='XMERGE')
split_nid = ibs.make_next_nids(num_split, location_text='XSPLIT')[0]
combo_nids = ibs.make_next_nids(num_combo * 2, location_text='XCOMBO')
# the first 3 become merge cases
#left = 0
#right = left + num_merge
for aids, nid1, nid2 in zip(modify_aids[0:3], merge_nids1, merge_nids2):
#ibs.get_annot_nids(aids)
aids_ = aids[::2]
ibs.set_annot_name_rowids(aids_, [nid1] * len(aids_))
ibs.set_annot_name_rowids(aids_, [nid2] * len(aids_))
# the next 2 become split cases
#left = right
#right = left + num_split_names
for aids in modify_aids[3:5]:
ibs.set_annot_name_rowids(aids, [split_nid] * len(aids))
#left = right
#right = left + num_combo_names
# The final 3 are a combination case
for aids in modify_aids[5:8]:
aids_even = aids[::2]
aids_odd = aids[1::2]
ibs.set_annot_name_rowids(aids_even, [combo_nids[0]] * len(aids_even))
ibs.set_annot_name_rowids(aids_odd, [combo_nids[1]] * len(aids_odd))
final_result = ibs.unflat_map(ibs.get_annot_nids, modify_aids)
print('final_result = %s' % (ut.list_str(final_result), ))
def ensure_pz_mtest_batchworkflow_test():
r"""
CommandLine:
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test --reset
python -m ibeis.init.sysres --test-ensure_pz_mtest_batchworkflow_test --reset
Example:
>>> # SCRIPT
>>> from ibeis.init.sysres import * # NOQA
>>> ensure_pz_mtest_batchworkflow_test()
"""
import ibeis
ibeis.ensure_pz_mtest()
workdir = ibeis.sysres.get_workdir()
mtest_dbpath = join(workdir, 'PZ_MTEST')
source_dbdir = mtest_dbpath
dest_dbdir = join(workdir, 'PZ_BATCH_WORKFLOW_MTEST')
if ut.get_argflag('--reset'):
ut.delete(dest_dbdir)
if ut.checkpath(dest_dbdir):
return
else:
copy_ibeisdb(source_dbdir, dest_dbdir)
ibs = ibeis.opendb('PZ_BATCH_WORKFLOW_MTEST')
assert len(ibs.get_valid_aids()) == 119
assert len(ibs.get_valid_nids()) == 41
ibs.delete_all_imagesets()
aid_list = ibs.get_valid_aids()
unixtime_list = ibs.get_annot_image_unixtimes(aid_list)
untimed_aids = ut.compress(aid_list, [t == -1 for t in unixtime_list])
ibs.get_annot_groundtruth(untimed_aids, aid_list)
aids_list, nid_list = ibs.group_annots_by_name(aid_list)
hourdiffs_list = ibs.get_name_hourdiffs(nid_list)
imageset_aids_list = [[] for _ in range(4)]
imageset_idx = 0
for hourdiffs, aids in zip(hourdiffs_list, aids_list):
#import scipy.spatial.distance as spdist
if len(aids) == 1:
imageset_aids_list[imageset_idx].extend(aids)
imageset_idx = (imageset_idx + 1) % len(imageset_aids_list)
else:
for chunk in list(ut.ichunks(aids, 2)):
imageset_aids_list[imageset_idx].extend(chunk)
imageset_idx = (imageset_idx + 1) % len(imageset_aids_list)
#import vtool as vt
#import networkx as netx
#nodes = list(range(len(aids)))
#edges_pairs = vt.pdist_argsort(hourdiffs)
#edge_weights = -hourdiffs[hourdiffs.argsort()]
#netx_graph = make_netx_graph(edges_pairs, nodes, edge_weights)
#cut_edges = netx.minimum_edge_cut(netx_graph)
#netx_graph.remove_edges_from(cut_edges)
#components = list(netx.connected_components(netx_graph))
#components = ut.sortedby(components, list(map(len, components)), reverse=True)
#print(components)
#imageset_aids_list[0].extend(components[0])
#for compoment in components:
# TODO do max-nway cut
#day_diffs = spdist.squareform(hourdiffs) / 24.0
#print(ut.numpy_str(day_diffs, precision=2, suppress_small=True))
#import itertools
#compare_idxs = [(r, c) for r, c in itertools.product(range(len(aids)), range(len(aids))) if (c > r)]
#print(len(aids))
#def make_netx_graph(edges_pairs, nodes=None, edge_weights=None):
# import networkx as netx
# node_lbls = [('id_', 'int')]
# edge_lbls = [('weight', 'float')]
# edges = [(pair[0], pair[1], weight) for pair, weight in zip(edges_pairs, edge_weights)]
# print('make_netx_graph')
# # Make a graph between the chips
# netx_nodes = [(ntup[0], {key[0]: val for (key, val) in zip(node_lbls, ntup[1:])})
# for ntup in iter(zip(nodes))]
# netx_edges = [(etup[0], etup[1], {key[0]: val for (key, val) in zip(edge_lbls, etup[2:])})
# for etup in iter(edges)]
# netx_graph = netx.Graph()
# netx_graph.add_nodes_from(netx_nodes)
# netx_graph.add_edges_from(netx_edges)
# return netx_graph
# Group into imagesets based on old names
gids_list = ibs.unflat_map(ibs.get_annot_image_rowids, imageset_aids_list)
imgsetid_list = ibs.new_imagesets_from_images(gids_list) # NOQA
# Remove all names
ibs.delete_annot_nids(aid_list)
import ibeis
import six
import vtool
import utool
import numpy as np
import numpy.linalg as npl # NOQA
import pandas as pd
from vtool import clustering2 as clustertool
from vtool import nearest_neighbors as nntool
from plottool import draw_func2 as df2
np.set_printoptions(precision=2)
pd.set_option('display.max_rows', 10)
pd.set_option('display.max_columns', 10)
pd.set_option('isplay.notebook_repr_html', True)
ibeis.ensure_pz_mtest()
#taids = ibs.get_valid_aids()
#tvecs_list = ibs.get_annot_vecs(taids)
#tkpts_list = ibs.get_annot_kpts(taids)
#tvec_list = np.vstack(tvecs_list)
#print(idx2_vec)
#labels, words = vtool.clustering.cached_akmeans(tvec_list, 1000, 30, cache_dir='.')
#tvecdf_list = [pd.DataFrame(vecs) for vecs in tvecs_list]
#tvecs_df = pd.DataFrame(tvecdf_list, index=taids)
#kpts_col = pd.DataFrame(tkpts_list, index=taids, columns=['kpts'])
#vecs_col = pd.DataFrame(tvecs_list, index=taids, columns=['vecs'])
#tvecs_dflist = [pd.DataFrame(vecs, index=np.arange(len(vecs))) for vecs in tvecs_list]
#pd.concat(tvecs_dflist)
## Bui
