def example_getter_methods(depc, tablename, root_rowids):
"""
example of different ways to get data
"""
import dtool
print('\n+---')
print('Running getter example')
print(' * tablename=%r' % (tablename))
print(' * root_rowids=%r' % (ut.trunc_repr(tablename)))
# You can get a reference to data rows using the "root" (dummy_annot) rowids
# By default, if the data has not been computed, then it will be computed
# for you. But if you specify ensure=False, None will be returned if the data
# has not been computed yet.
tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=False) # NOQA
print('tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=False)')
print('tbl_rowids = %s' % (ut.trunc_repr(tbl_rowids),))
#assert tbl_rowids[0] is None
# The default is for the data to be computed though. Manaual interactions will
# launch as necessary.
tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=True) # NOQA
print('tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=True)')
print('tbl_rowids = %s' % (ut.trunc_repr(tbl_rowids),))
assert tbl_rowids[0] is not None
# Now the data is cached and will not need to be computed again
tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=False) # NOQA
assert tbl_rowids[0] is not None
# Can lookup a table, which can access data directly. The rowids can be
# used to lookup data values directly. By default all data in a row is
# returned.
table = depc[tablename]
datas = table.get_row_data(tbl_rowids) # NOQA
# But you can also ask for a specific column
col1 = table.columns[0]
col1_data = table.get_row_data(tbl_rowids, col1) # NOQA
# In the case of external columns:
if len(table.extern_columns) > 0:
excol = table.extern_columns[0]
# you can lookup the value of the external data very simply
extern_data = table.get_row_data(tbl_rowids, (excol,)) # NOQA
print('extern_data = table.get_row_data(tbl_rowids, (excol,))')
print(ut.varinfo_str(extern_data, 'extern_data'))
# you can lookup the hidden paths as follows
extern_paths = table.get_row_data(tbl_rowids, (excol + dtool.depcache_table.EXTERN_SUFFIX,)) # NOQA
print('extern_paths = table.get_row_data(tbl_rowids, (excol + dtool.depcache_table.EXTERN_SUFFIX,))')
print(ut.varinfo_str(extern_paths, 'extern_paths'))
# But you can also just the root rowids directly. This is the simplest way
# to access data and really "all you need to know"
if len(table.columns) > 1:
col1, col2 = table.columns[0:2]
datas = depc.get_property(tablename, root_rowids, (col1, col2)) # NOQA
print('L__')
def _expand_level_rowids(depc, tablename, tablekey, rowid_dict, ensure,
eager, nInput, config, recompute, recompute_all,
_debug):
table = depc[tablekey]
config_ = depc._ensure_config(tablekey, config)
parent_rowids = depc._get_parent_rowids(table, rowid_dict)
if _debug:
print(' * tablekey = %r' % (tablekey,))
print(' * (ensured) config_ = %r' % (config_,))
print(' * config_rowid = %r' % (table.get_config_rowid(config_),))
print(' * parent_rowids = %s' % (ut.trunc_repr(parent_rowids),))
_recompute = recompute_all or (tablekey == tablename and recompute)
level_rowids = table.get_rowid(
parent_rowids, config=config_, eager=eager, nInput=nInput,
ensure=ensure, recompute=_recompute)
if _debug:
print(' * level_rowids = %s' % (ut.trunc_repr(level_rowids),))
r
def _wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as ex:
import utool as ut
ut.printex(ex)
import inspect # NOQA
trace = inspect.trace()
locals_ = trace[-1][0].f_locals
print('-- <TRACE LOCALS> --')
for level, t in enumerate(trace[1:]):
frame = t[0]
locals_ = frame.f_locals
local_repr_dict = {key: ut.trunc_repr(val)
for key, val in locals_.items()}
print('LOCALS LEVEL %d' % (level,))
print(ut.repr3(local_repr_dict, strvals=True, nl=1))
print('-- </TRACE LOCALS> --')
#import utool
#utool.embed()
raise
def example_getter_methods(depc, tablename, root_rowids):
"""
example of different ways to get data
"""
import dtool
print('\n+---')
print('Running getter example')
print(' * tablename=%r' % (tablename))
print(' * root_rowids=%r' % (ut.trunc_repr(tablename)))
# You can get a reference to data rows using the "root" (dummy_annot) rowids
# By default, if the data has not been computed, then it will be computed
# for you. But if you specify ensure=False, None will be returned if the data
# has not been computed yet.
tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=False) # NOQA
print('tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=False)')
print('tbl_rowids = %s' % (ut.trunc_repr(tbl_rowids), ))
#assert tbl_rowids[0] is None
# The default is for the data to be computed though. Manaual interactions will
# launch as necessary.
tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=True) # NOQA
print('tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=True)')
print('tbl_rowids = %s' % (ut.trunc_repr(tbl_rowids), ))
assert tbl_rowids[0] is not None
# Now the data is cached and will not need to be computed again
tbl_rowids = depc.get_rowids(tablename, root_rowids, ensure=False) # NOQA
assert tbl_rowids[0] is not None
# Can lookup a table, which can access data directly. The rowids can be
# used to lookup data values directly. By default all data in a row is
# returned.
table = depc[tablename]
datas = table.get_row_data(tbl_rowids) # NOQA
# But you can also ask for a specific column
col1 = table.columns[0]
col1_data = table.get_row_data(tbl_rowids, col1) # NOQA
# In the case of external columns:
if len(table.extern_columns) > 0:
excol = table.extern_columns[0]
# you can lookup the value of the external data very simply
extern_data = table.get_row_data(tbl_rowids, (excol, )) # NOQA
print('extern_data = table.get_row_data(tbl_rowids, (excol,))')
print(ut.varinfo_str(extern_data, 'extern_data'))
# you can lookup the hidden paths as follows
extern_paths = table.get_row_data(
tbl_rowids, (excol + dtool.depcache_table.EXTERN_SUFFIX, )) # NOQA
print(
'extern_paths = table.get_row_data(tbl_rowids, (excol + dtool.depcache_table.EXTERN_SUFFIX,))'
)
print(ut.varinfo_str(extern_paths, 'extern_paths'))
# But you can also just the root rowids directly. This is the simplest way
# to access data and really "all you need to know"
if len(table.columns) > 1:
col1, col2 = table.columns[0:2]
datas = depc.get_property(tablename, root_rowids, (col1, col2)) # NOQA
print('L__')
def get_rowids(depc, tablename, root_rowids, config=None, ensure=True,
eager=True, nInput=None, _debug=None, recompute=False,
recompute_all=False):
"""
Returns the rowids of `tablename` that correspond to `root_rowids`
using `config`.
Ignore:
tablename = 'nnindexer'
multi_rowids = (1, 2, 3, 4, 5)
root_rowids = [[multi_rowids]]
import plottool as pt
pt.ensureqt()
from dtool.depcache_control import * # NOQA
from dtool.example_depcache import testdata_depc
depc = testdata_depc()
exec(ut.execstr_funckw(depc.get_rowids), globals())
print(ut.depth_profile(root_rowids))
tablename = 'neighbs'
table = depc[tablename] # NOQA
import plottool as pt
pt.ensureqt()
_debug = depc._debug = True
depc.get_rowids(tablename, root_rowids, config, _debug=_debug)
pt.show_nx(depc.graph)
for key, val in table.type_to_subgraph.items():
pt.show_nx(val)
pt.set_title(key)
CommandLine:
python -m dtool.depcache_control --exec-get_rowids
python -m dtool.depcache_control --dump-get_rowids
python -m dtool.depcache_control --exec-get_rowids:0
GridParams:
>>> param_grid = dict(
>>> tablename=[ 'spam', 'neighbs'] # 'spam', 'multitest_score','keypoint'],
>>> #tablename=['neighbs', 'keypoint', 'spam', 'multitest_score','keypoint'],
>>> )
>>> flat_root_ids = [1, 2, 3]
>>> combos = ut.all_dict_combinations(param_grid)
>>> index = 0
>>> keys = 'tablename'.split(', ')
>>> tablename, = ut.dict_take(combos[index], keys)
Setup:
>>> # DISABLE_GRID_DOCTEST
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> depc = testdata_depc()
>>> exec(ut.execstr_funckw(depc.get_rowids), globals())
>>> import plottool as pt
>>> pt.ensureqt()
>>> #pt.show_nx(depc.graph)
GridExample0:
>>> table = depc[tablename] # NOQA
>>> flat_root_ids = [1, 2, 3]
>>> root_rowids = [flat_root_ids for _ in table.input_order]
>>> print('root_rowids = %r' % (root_rowids,))
>>> #root_rowids = [[flat_root_ids], [(flat_root_ids,)]]
>>> #root_rowids = [list(zip(flat_root_ids)), (flat_root_ids,)]
>>> _debug = True
>>> depc.get_rowids(tablename, root_rowids, config, _debug=_debug)
>>> for key, val in table.type_to_subgraph.items():
>>> pt.show_nx(val)
>>> pt.set_title(key)
Example1:
>>> # ENABLE_DOCTEST
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> depc = testdata_depc()
>>> exec(ut.execstr_funckw(depc.get_rowids), globals())
>>> root_rowids = [1, 2, 3]
>>> tablename = 'spam'
>>> table = depc[tablename]
>>> kp_rowids = depc.get_rowids(tablename, root_rowids)
>>> #result = ('prop_list = %s' % (ut.repr2(prop_list),))
>>> #print(result)
Example:
>>> # ENABLE_DOCTEST
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> depc = testdata_depc()
>>> exec(ut.execstr_funckw(depc.get_rowids), globals())
>>> flat_root_ids = [1, 2, 3]
>>> kp_rowids = depc.get_rowids('keypoint', flat_root_ids)
>>> root_rowids = [flat_root_ids] * 8
>>> _debug = True
>>> tablename = 'nnindexer'
>>> tablename = 'multitest_score'
>>> table = depc[tablename] # NOQA
>>> #result = ('prop_list = %s' % (ut.repr2(prop_list),))
>>> # print(result)
"""
_debug = depc._debug if _debug is None else _debug
if _debug:
print(' * root_rowids=%s' % (ut.trunc_repr(root_rowids),))
print(' * config = %r' % (config,))
table = depc[tablename] # NOQA
INDEXER_VERSION = False
if tablename == 'neighbor_index':
"""
python -m ibeis.core_annots --exec-compute_neighbor_index --show
"""
import utool
utool.embed()
if INDEXER_VERSION or tablename == 'neighbs':
compute_order = table.compute_order
depend_order = compute_order['depend_compute_ids']
input_order = compute_order['input_compute_ids']
if _debug:
print(' * input_order = %s' % (ut.repr3(input_order, nl=1),))
print(' * depend_order = %s' % (ut.repr3(depend_order, nl=1),))
if len(input_order) > 1:
assert ut.depth_atleast(root_rowids, 2), (
'input_order = %r' % (input_order,))
with ut.Indenter('[GetRowID-%s]' % (tablename,),
enabled=_debug):
# New way to get rowids
input_level = depend_order[0]
mid_levels = depend_order[1:-1]
output_level = depend_order[-1]
# List that holds a mapping from input order to input "name"
input_order_lookup = ut.make_index_lookup(input_order)
# Dictionary that holds the rowids computed for each table
# while tracing the dependencies.
rowid_lookup = ut.odict([(key, ut.odict()) for key in input_order])
# Need to split each path into parts.
# Each part represents another level of unflattening
# (because root indicies are all flat)
# Handle input level
assert input_level[0] == depc.root
for compute_id in input_order:
# for name in input_names:
argx = input_order_lookup[compute_id]
rowid_lookup[compute_id] = root_rowids[argx]
# HACK: Flatten to scalars
# The inputs should just be given in the "correct" nesting.
# TODO: determine what correct nesting is.
for i in range(5):
try:
current = rowid_lookup[compute_id]
rowid_lookup[compute_id] = ut.flatten(current)
except Exception:
pass
level = 0
if _debug:
print('input_order_lookup = %r' % (input_order_lookup,))
ut.printdict(rowid_lookup, 'rowid_lookup')
def handle_level(compute_id, rowid_lookup, _recompute, level):
print('+--- HANDLE LEVEL %d -------' % (level,))
tablekey = compute_id[0]
input_suff = compute_id[1]
config_ = depc._ensure_config(tablekey, config)
table = depc[tablekey]
lookupkeys = [(n, input_suff) for n in table.parent_id_tablenames]
# ordering = ut.dict_take(input_order_lookup, input_names)
# sortx = ut.argsort(ordering)
# FIXME: get inputs for each table.
# input_names = ut.take(input_names, sortx)
# lookupkeys = list(ut.iprod(table.parent_id_tablenames, input_names))
# lookupkeys = list(zip(table.parent_id_tablenames, input_types))
if _debug:
print('---- LOCALS ------')
ut.print_locals(compute_id, tablekey, lookupkeys, table)
print('L----------')
# FIXME generalize
_parent_ids = [rowid_lookup[tblkey] for tblkey in lookupkeys]
if table.ismulti:
parent_rowidsT = [[tuple(x)] for x in _parent_ids]
else:
parent_rowidsT = _parent_ids
parent_rowidsT = np.broadcast_arrays(*parent_rowidsT)
parent_rowids = list(zip(*parent_rowidsT))
# Probably not right for general multi-input
import utool
with utool.embed_on_exception_context:
next_rowids = table.get_rowid(
parent_rowids, config=config_, eager=eager, nInput=nInput,
ensure=ensure, recompute=_recompute)
rowid_lookup[compute_id] = next_rowids
if _debug:
ut.printdict(rowid_lookup, 'rowid_lookup')
if _debug:
print('L___ HANDLE LEVEL %d -------' % (level,))
return next_rowids
# Handle mid levels
_recompute = recompute_all
for level, compute_id in enumerate(mid_levels, start=1):
handle_level(compute_id, rowid_lookup, _recompute, level)
level += 1
# Handel final (requested) level
compute_id = output_level
_recompute = recompute
rowid_list = handle_level(compute_id, rowid_lookup,
_recompute, level)
else:
with ut.Indenter('[GetRowID-%s]' % (tablename,),
enabled=_debug):
# TODO: Get nonself rowids first
# THen get self rowids for debugging ease
try:
if False:
recompute_ = recompute or recompute_all
parent_rowids = depc._get_parent_input(
tablename, root_rowids, config, ensure=True, _debug=None,
recompute=False, recompute_all=False, eager=True,
nInput=None)
config_ = depc._ensure_config(tablename, config)
#if onthefly:
# pass
table = depc[tablename]
rowid_list = table.get_rowid(
parent_rowids, config=config_, eager=eager, nInput=nInput,
ensure=ensure, recompute=recompute_)
else:
# Compute everything from the root to the requested table
rowid_dict = depc.get_all_descendant_rowids(
tablename, root_rowids, config=config, ensure=ensure,
eager=eager, nInput=nInput, recompute=recompute,
recompute_all=recompute_all, _debug=ut.countdown_flag(_debug))
rowid_list = rowid_dict[tablename]
except depcache_table.ExternalStorageException:
print('EXTERNAL EXCEPTION One retry in get_rowids')
rowid_dict = depc.get_all_descendant_rowids(
tablename, root_rowids, config=config, ensure=ensure,
eager=eager, nInput=nInput, recompute=recompute,
recompute_all=recompute_all, _debug=ut.countdown_flag(_debug))
rowid_list = rowid_dict[tablename]
if _debug:
print(' * return rowid_list = %s' % (ut.trunc_repr(rowid_list),))
return rowid_list