def _debug_repr_cpd(cpd):
import re
import utool as ut
code_fmt = ut.codeblock(
'''
{variable} = pgmpy.factors.TabularCPD(
variable={variable_repr},
variable_card={variable_card_repr},
values={get_cpd_repr},
evidence={evidence_repr},
evidence_card={evidence_card_repr},
)
''')
keys = ['variable', 'variable_card', 'values', 'evidence', 'evidence_card']
dict_ = ut.odict(zip(keys, [getattr(cpd, key) for key in keys]))
# HACK
dict_['values'] = cpd.get_cpd()
r = ut.repr2(dict_, explicit=True, nobraces=True, nl=True)
print(r)
# Parse props that are needed for this fmtstr
fmt_keys = [match.groups()[0] for match in re.finditer('{(.*?)}', code_fmt)]
need_reprs = [key[:-5] for key in fmt_keys if key.endswith('_repr')]
need_keys = [key for key in fmt_keys if not key.endswith('_repr')]
# Get corresponding props
# Call methods if needbe
tmp = [(prop, getattr(cpd, prop)) for prop in need_reprs]
tmp = [(x, y()) if ut.is_funclike(y) else (x, y) for (x, y) in tmp]
fmtdict = dict(tmp)
fmtdict = ut.map_dict_vals(ut.repr2, fmtdict)
fmtdict = ut.map_dict_keys(lambda x: x + '_repr', fmtdict)
tmp2 = [(prop, getattr(cpd, prop)) for prop in need_keys]
fmtdict.update(dict(tmp2))
code = code_fmt.format(**fmtdict)
return code
def format_cell(cell):
if ut.is_funclike(cell):
header = '# ' + ut.to_title_caps(ut.get_funcname(cell))
code = (header, ut.get_func_sourcecode(cell, stripdef=True, stripret=True))
else:
code = (None, cell)
return generate_notebook.format_cells(code)
def format_cell(cell):
if ut.is_funclike(cell):
header = '# ' + ut.to_title_caps(ut.get_funcname(cell))
code = (header,
ut.get_func_sourcecode(cell, stripdef=True, stripret=True))
else:
code = (None, cell)
return generate_notebook.format_cells(code)
def default_decorator(input_):
if utool.is_funclike(input_):
func_ = input_
#return utool.indent_func(profile(func_))
return profile(func_)
#return func_
else:
def closure_default(func):
return utool.indent_func(input_)(profile(func))
return closure_default
def register_command(self, name):
import utool as ut
if ut.is_funclike(name):
func = name
name = ut.get_funcname(func)
self._register_command(name, func)
return func
else:
def _wrap(func):
self._register_command(name, func)
return _wrap
def _set_iders(model, iders=None):
""" sets iders """
if VERBOSE:
print('[APIItemModel] _set_iders')
if iders is None:
iders = []
if ut.USE_ASSERT:
assert ut.is_list(iders), 'bad type: %r' % type(iders)
for index, ider in enumerate(iders):
assert ut.is_funclike(ider), 'bad type at index %r: %r' % (index, type(ider))
#printDBG('NEW IDER')
model.iders = iders
def plot_image(self, index):
px = index - self.start_index
gpath = self.gpath_list[index]
if self.vizkw is None:
_vizkw = {}
else:
_vizkw = self.vizkw.copy()
_vizkw.update({
'fnum': self.fnum,
'pnum': self.pnum_(px),
})
if ut.is_funclike(gpath):
showfunc = gpath
# HACK
# override of plot image function
showfunc(**_vizkw)
import plottool as pt
ax = pt.gca()
else:
if isinstance(gpath, six.string_types):
img = vt.imread(gpath)
else:
img = gpath
bbox_list = self.bboxes_list[index]
#print('bbox_list %r in display for px: %r ' % (bbox_list, px))
theta_list = self.thetas_list[index]
label_list = [ix + 1 for ix in range(len(bbox_list))]
#Add true values for every bbox to display
sel_list = [True for ix in range(len(bbox_list))]
_vizkw.update({
#title should always be the image number
'title': str(index),
'bbox_list': bbox_list,
'theta_list': theta_list,
'sel_list': sel_list,
'label_list': label_list,
})
#print(utool.dict_str(_vizkw))
#print('vizkw = ' + utool.dict_str(_vizkw))
_, ax = viz_image2.show_image(img, **_vizkw)
if self.xlabel_list is not None:
import plottool as pt
pt.set_xlabel(self.xlabel_list[index])
#print(index)
ph.set_plotdat(ax, 'bbox_list', bbox_list)
ph.set_plotdat(ax, 'gpath', gpath)
ph.set_plotdat(ax, 'px', str(px))
ph.set_plotdat(ax, 'index', index)
def plot_image(self, index):
px = index - self.start_index
gpath = self.gpath_list[index]
if self.vizkw is None:
_vizkw = {}
else:
_vizkw = self.vizkw.copy()
_vizkw.update({
'fnum': self.fnum,
'pnum': self.pnum_(px),
})
if ut.is_funclike(gpath):
showfunc = gpath
# HACK
# override of plot image function
showfunc(**_vizkw)
import plottool as pt
ax = pt.gca()
else:
if isinstance(gpath, six.string_types):
img = vt.imread(gpath)
else:
img = gpath
bbox_list = self.bboxes_list[index]
#print('bbox_list %r in display for px: %r ' % (bbox_list, px))
theta_list = self.thetas_list[index]
label_list = [ix + 1 for ix in range(len(bbox_list))]
#Add true values for every bbox to display
sel_list = [True for ix in range(len(bbox_list))]
_vizkw.update({
#title should always be the image number
'title': str(index),
'bbox_list' : bbox_list,
'theta_list' : theta_list,
'sel_list' : sel_list,
'label_list' : label_list,
})
#print(utool.dict_str(_vizkw))
#print('vizkw = ' + utool.dict_str(_vizkw))
_, ax = viz_image2.show_image(img, **_vizkw)
if self.xlabel_list is not None:
import plottool as pt
pt.set_xlabel(self.xlabel_list[index])
#print(index)
ph.set_plotdat(ax, 'bbox_list', bbox_list)
ph.set_plotdat(ax, 'gpath', gpath)
ph.set_plotdat(ax, 'px', str(px))
ph.set_plotdat(ax, 'index', index)
def register_command(self, name):
import utool as ut
if ut.is_funclike(name):
func = name
name = ut.get_funcname(func)
self._register_command(name, func)
return func
else:
def _wrap(func):
self._register_command(name, func)
return _wrap
def on_click_inside(self, event, ax):
index = ph.get_plotdat(ax, 'index')
print('index = %r' % (index,))
if index is not None:
if self.MOUSE_BUTTONS[event.button] == 'right':
if self.context_option_funcs is not None:
# if event.button == 3:
options = self.context_option_funcs[index]()
self.show_popup_menu(options, event)
elif self.MOUSE_BUTTONS[event.button] == 'left':
# bbox_list = ph.get_plotdat(ax, 'bbox_list')
gpath = self.gpath_list[index]
if ut.is_funclike(gpath):
print('gpath_isfunklike')
print('gpath = %r' % (gpath,))
import wbia.plottool as pt
fnum = pt.next_fnum()
gpath(fnum=fnum)
df2.update()
else:
bbox_list = self.bboxes_list[index]
print('Bbox of figure: %r' % (bbox_list,))
theta_list = self.thetas_list[index]
print('theta_list = %r' % (theta_list,))
# img = mpimg.imread(gpath)
if isinstance(gpath, six.string_types):
img = vt.imread(gpath)
else:
img = gpath
fnum = df2.next_fnum()
mc = interact_annotations.AnnotationInteraction(
img,
index,
self.update_images,
bbox_list=bbox_list,
theta_list=theta_list,
fnum=fnum,
)
mc.start()
self.mc = mc
# """wait for accept
# have a flag to tell if a bbox has been changed, on the bbox
# list that is brought it" on accept: viz_image2.show_image
# callback
# """
df2.update()
print('Clicked: ax: num=%r' % index)
def on_click_inside(self, event, ax):
index = ph.get_plotdat(ax, 'index')
print('index = %r' % (index,))
if index is not None:
if self.MOUSE_BUTTONS[event.button] == 'right':
if self.context_option_funcs is not None:
#if event.button == 3:
options = self.context_option_funcs[index]()
self.show_popup_menu(options, event)
elif self.MOUSE_BUTTONS[event.button] == 'left':
#bbox_list = ph.get_plotdat(ax, 'bbox_list')
gpath = self.gpath_list[index]
if ut.is_funclike(gpath):
print('gpath_isfunklike')
print('gpath = %r' % (gpath,))
import plottool as pt
fnum = pt.next_fnum()
gpath(fnum=fnum)
df2.update()
else:
bbox_list = self.bboxes_list[index]
print('Bbox of figure: %r' % (bbox_list,))
theta_list = self.thetas_list[index]
print('theta_list = %r' % (theta_list,))
#img = mpimg.imread(gpath)
if isinstance(gpath, six.string_types):
img = vt.imread(gpath)
else:
img = gpath
fnum = df2.next_fnum()
mc = interact_annotations.ANNOTATIONInteraction(
img, index, self.update_images, bbox_list=bbox_list,
theta_list=theta_list, fnum=fnum)
mc.start()
self.mc = mc
# """wait for accept
# have a flag to tell if a bbox has been changed, on the bbox
# list that is brought it" on accept: viz_image2.show_image
# callback
# """
df2.update()
print('Clicked: ax: num=%r' % index)
def get_index_butkw(dgt, qtindex):
""" The model data for a button should be a (text, callback) tuple. OR
it could be a function which accepts an qtindex and returns a button """
data = qtindex.model().data(qtindex, QtCore.Qt.DisplayRole)
# Get info
if isinstance(data, tuple):
buttontup = data
elif utool.is_funclike(data):
func = data
buttontup = func(qtindex)
else:
raise AssertionError('bad type')
text, callback = buttontup[0:2]
butkw = {
#'parent': dgt.parent(),
'text': text,
'clicked': callback,
}
if len(buttontup) > 2:
butkw['bgcolor'] = buttontup[2]
butkw['fgcolor'] = (0, 0, 0)
return butkw
def _debug_repr_cpd(cpd):
import re
import utool as ut
code_fmt = ut.codeblock("""
{variable} = pgmpy.factors.TabularCPD(
variable={variable_repr},
variable_card={variable_card_repr},
values={get_cpd_repr},
evidence={evidence_repr},
evidence_card={evidence_card_repr},
)
""")
keys = ['variable', 'variable_card', 'values', 'evidence', 'evidence_card']
dict_ = ut.odict(zip(keys, [getattr(cpd, key) for key in keys]))
# HACK
dict_['values'] = cpd.get_cpd()
r = ut.repr2(dict_, explicit=True, nobraces=True, nl=True)
logger.info(r)
# Parse props that are needed for this fmtstr
fmt_keys = [
match.groups()[0] for match in re.finditer('{(.*?)}', code_fmt)
]
need_reprs = [key[:-5] for key in fmt_keys if key.endswith('_repr')]
need_keys = [key for key in fmt_keys if not key.endswith('_repr')]
# Get corresponding props
# Call methods if needbe
tmp = [(prop, getattr(cpd, prop)) for prop in need_reprs]
tmp = [(x, y()) if ut.is_funclike(y) else (x, y) for (x, y) in tmp]
fmtdict = dict(tmp)
fmtdict = ut.map_dict_vals(ut.repr2, fmtdict)
fmtdict = ut.map_dict_keys(lambda x: x + '_repr', fmtdict)
tmp2 = [(prop, getattr(cpd, prop)) for prop in need_keys]
fmtdict.update(dict(tmp2))
code = code_fmt.format(**fmtdict)
return code
def get_layer_info(layer):
r"""
Args:
layer (?):
Returns:
?: layer_info
CommandLine:
python -m ibeis_cnn.net_strs get_layer_info --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis_cnn.net_strs import * # NOQA
>>> from ibeis_cnn import models
>>> model = models.mnist.MNISTModel(batch_size=8, data_shape=(24, 24, 1), output_dims=10)
>>> model.init_arch()
>>> nn_layers = model.get_all_layers()
>>> for layer in nn_layers:
>>> layer_info = get_layer_info(layer)
>>> print(ut.repr3(layer_info, nl=1))
"""
import operator
import ibeis_cnn.__LASAGNE__ as lasagne
# Information that contributes to RAM usage
import numpy as np
# Get basic layer infos
output_shape = lasagne.layers.get_output_shape(layer)
input_shape = getattr(layer, 'input_shape', [])
# Get number of outputs ignoring the batch size
num_outputs = functools.reduce(operator.mul, output_shape[1:])
if len(input_shape):
num_inputs = functools.reduce(operator.mul, input_shape[1:])
else:
num_inputs = 0
# TODO: if we can ever support non float32 calculations this must change
#layer_type = 'float32'
layer_dtype = np.dtype('float32')
# Get individual param infos
param_infos = []
for param, tags in layer.params.items():
value = param.get_value()
pbasename = param_basename(layer, param)
param_info = ut.odict([
('name', param.name),
('basename', pbasename),
('tags', tags),
('shape', value.shape),
('size', value.size),
('itemsize', value.dtype.itemsize),
('dtype', str(value.dtype)),
('bytes', value.size * value.dtype.itemsize),
])
def initializer_info(initclass):
initclassname = initclass.__class__.__name__
if initclassname == 'Constant':
spec = initclass.val
else:
spec = ut.odict()
spec['type'] = initclassname
for key, val in initclass.__dict__.items():
if isinstance(val, lasagne.init.Initializer):
spec[key] = initializer_info(val)
elif isinstance(val, type) and issubclass(val, lasagne.init.Initializer):
spec[key] = val.__name__
#initializer_info(val())
else:
spec[key] = val
return spec
if hasattr(layer, '_initializers'):
#print('layer = %r' % (layer,))
initclass = layer._initializers[param]
spec = initializer_info(initclass)
param_info['init'] = spec
param_infos.append(param_info)
# Combine param infos
param_str = surround(', '.join(
[paramstr(layer, p, tags) for p, tags in layer.params.items()]), '[]')
param_type_str = surround(', '.join(
[repr(p.type) for p, tags in layer.params.items()]), '[]')
num_params = sum([info['size'] for info in param_infos])
classalias_map = {
'ElemwiseSumLayer': 'ElemwiseSum',
'Conv2DCCLayer' : 'Conv2D',
'Conv2DDNNLayer' : 'Conv2D',
'Conv2DLayer' : 'Conv2D',
'MaxPool2DLayer': 'MaxPool2D',
'MaxPool2DCCLayer' : 'MaxPool2D',
'MaxPool2DDNNLayer' : 'MaxPool2D',
'LeakyRectify' : 'LReLU',
'InputLayer' : 'Input',
'GaussianNoiseLayer': 'Noise',
'DropoutLayer' : 'Dropout',
'DenseLayer' : 'Dense',
'NonlinearityLayer' : 'Nonlinearity',
'FlattenLayer' : 'Flatten',
'L2NormalizeLayer' : 'L2Norm',
'BatchNormLayer' : 'BatchNorm',
'BatchNormLayer2' : 'BatchNorm',
}
layer_attrs_ignore_dict = {
'MaxPool2D' : ['mode', 'ignore_border'],
'Dropout' : ['rescale'],
'Conv2D' : ['convolution'],
'BatchNorm': ['epsilon', 'mean', 'inv_std', 'axes', 'beta', 'gamma'],
'BatchNorm2': ['epsilon', 'mean', 'inv_std', 'axes', 'beta', 'gamma'],
#'ElemwiseSum': ['merge_function', 'cropping'],
#'ElemwiseSum': [],
'FeaturePoolLayer': ['axis'],
}
layer_attrs_dict = {
#'ElemwiseSum': ['coeffs'],
#'ElemwiseSum': ['coeffs', 'merge_function', 'cropping'],
'Noise' : ['sigma'],
'Input' : ['shape'],
'Dropout' : ['p', 'shared_axes'],
'Conv2D' : ['num_filters', 'filter_size', 'stride', 'output_shape', 'num_groups'],
'MaxPool2D' : ['stride', 'pool_size', 'output_shape'], # 'mode'],
'Dense' : ['num_units', 'num_leading_axes'],
'SoftMax' : ['num_units', 'num_leading_axes'],
'L2Norm' : ['axis'],
'BatchNorm' : ['alpha'],
'BatchNorm2' : ['alpha'],
'FeaturePoolLayer': ['pool_size', 'pool_function']
}
#layer_attrs_dict = {}
all_ignore_attrs = ['nonlinearity', 'b', 'W', 'get_output_kwargs', 'name',
'input_shapes', 'input_layers', 'input_shape',
'input_layer', 'input_var', 'untie_biases',
'_initializers',
'flip_filters', 'pad', 'params', 'n', '_is_main_layer']
classname = layer.__class__.__name__
classalias = classalias_map.get(classname, classname)
#if classalias == 'FeaturePoolLayer' and ut.get_funcname(layer.pool_function) == 'max':
# classalias = 'MaxOut'
if classalias == 'Dense' and ut.get_funcname(layer.nonlinearity) == 'softmax':
classalias = 'SoftMax'
layer_attrs = ut.odict([
(key, getattr(layer, key))
for key in layer_attrs_dict.get(classalias, [])
])
ignore_attrs = (all_ignore_attrs +
layer_attrs_ignore_dict.get(classalias, []))
if classalias not in layer_attrs_dict or (classalias == classname and len(layer_attrs) == 0):
layer_attrs = layer.__dict__.copy()
ut.delete_dict_keys(layer_attrs, ignore_attrs)
for key in list(layer_attrs.keys()):
val = layer_attrs[key]
if ut.is_funclike(val):
layer_attrs[key] = ut.get_funcname(val)
attr_key_list = list(layer_attrs.keys())
missing_keys = (set(layer.__dict__.keys()) - set(ignore_attrs) - set(attr_key_list))
missing_keys = [k for k in missing_keys if not k.startswith('_')]
#if layer_type == 'Conv2DCCLayer':
# ut.embed()
DEBUG = True
if DEBUG and len(missing_keys) > 0:
print('---')
print(' * ' + classname)
print(' * missing keys: %r' % (missing_keys,))
print(' * has keys: %r' % (attr_key_list,))
if True:
#import utool
#with utool.embed_on_exception_context:
#raise AssertionError('MISSING KEYS')
pass
# handle None batch sizes
if output_shape[0] is None:
size = np.prod(output_shape[1:])
else:
size = np.prod(output_shape)
layer_info = ut.odict([
('name', layer.name),
('classname', classname),
('classalias', classalias),
('output_shape', output_shape),
('input_shape', input_shape),
('num_outputs', num_outputs),
('num_inputs', num_inputs),
('size', size),
('itemsize', layer_dtype.itemsize),
('dtype', str(layer_dtype)),
('num_params', num_params),
('param_infos', param_infos),
('param_str', param_str),
('param_type_str', param_type_str),
('layer_attrs', layer_attrs),
('nonlinearity', None),
])
if hasattr(layer, 'nonlinearity'):
try:
nonlinearity = layer.nonlinearity.__name__
except AttributeError:
nonlinearity = layer.nonlinearity.__class__.__name__
layer_info['nonlinearity'] = ut.odict([])
layer_info['nonlinearity']['type'] = nonlinearity
layer_info['nonlinearity'].update(layer.nonlinearity.__dict__)
#attr_str_list.append('nonlinearity={0}'.format(nonlinearity))
param_bytes = sum([info['bytes'] for info in param_infos])
layer_bytes = layer_info['size'] * layer_info['itemsize']
#if classname in ['BatchNormLayer', 'NonlinearityLayer']:
# layer_bytes = 0
layer_info['bytes'] = layer_bytes
layer_info['param_bytes'] = param_bytes
layer_info['total_bytes'] = layer_bytes + param_bytes
layer_info['total_memory'] = ut.byte_str2(layer_info['total_bytes'])
return layer_info
def _update_headers(model, **headers):
if VERBOSE_MODEL:
logger.info('[APIItemModel] _update_headers')
iders = headers.get('iders', None)
name = headers.get('name', None)
nice = headers.get('nice', None)
col_name_list = headers.get('col_name_list', None)
col_type_list = headers.get('col_type_list', None)
col_nice_list = headers.get('col_nice_list', None)
col_edit_list = headers.get('col_edit_list', None)
col_setter_list = headers.get('col_setter_list', None)
col_getter_list = headers.get('col_getter_list', None)
col_level_list = headers.get('col_level_list', None)
col_sort_index = headers.get('col_sort_index', 0)
col_sort_reverse = headers.get('col_sort_reverse', False)
# New for dynamically getting non-data roles for each row
col_bgrole_getter_list = headers.get('col_bgrole_getter_list', None)
col_visible_list = headers.get('col_visible_list', None)
#
if iders is None:
iders = []
if ut.USE_ASSERT:
assert ut.is_list(iders), 'bad type: %r' % type(iders)
for index, ider in enumerate(iders):
assert ut.is_funclike(ider), 'bad type at index %r: %r' % (
index,
type(ider),
)
if col_name_list is None:
col_name_list = []
if col_type_list is None:
col_type_list = []
if col_nice_list is None:
col_nice_list = col_name_list[:]
if col_edit_list is None:
col_edit_list = [False] * len(col_name_list)
if col_setter_list is None:
col_setter_list = []
if col_getter_list is None:
col_getter_list = []
if col_bgrole_getter_list is None:
col_bgrole_getter_list = [None] * len(col_name_list)
if col_visible_list is None:
col_visible_list = [True] * len(col_name_list)
if col_level_list is None:
col_level_list = [0] * len(col_name_list)
if True or ut.USE_ASSERT:
assert len(col_name_list) == len(
col_type_list), 'inconsistent colnametype'
assert len(col_name_list) == len(
col_nice_list), 'inconsistent colnice'
assert len(col_name_list) == len(
col_edit_list), 'inconsistent coledit'
assert len(col_name_list) == len(
col_setter_list), 'inconsistent colsetter'
assert len(col_bgrole_getter_list) == len(
col_name_list), 'inconsistent col_bgrole_getter_list'
assert len(col_name_list) == len(
col_getter_list), 'inconsistent colgetter'
assert len(col_visible_list) == len(
col_name_list), 'inconsistent col_visible_list'
assert len(col_name_list) == len(
col_level_list), 'inconsistent collevel'
for colname, flag, func in zip(col_name_list, col_edit_list,
col_setter_list):
if flag:
assert func is not None, 'column=%r is editable but func is None' % (
colname, )
model.clear_cache()
model.name = str(name)
model.nice = str(nice)
model.iders = iders
model.col_name_list = col_name_list
model.col_type_list = col_type_list
model.col_nice_list = col_nice_list
model.col_edit_list = col_edit_list
model.col_setter_list = col_setter_list
model.col_getter_list = col_getter_list
model.col_visible_list = col_visible_list
model.col_level_list = col_level_list
model.col_bgrole_getter_list = col_bgrole_getter_list
model.col_display_role_func_dict = headers.get(
'col_display_role_func_dict', None)
model.num_rows_loaded = 0
# model.num_cols_loaded = 0
model.num_rows_total = None
model.lazy_rows = True
# calls model._update_rows()
model._set_sort(col_sort_index,
col_sort_reverse,
rebuild_structure=True)