def show_internals(self, fnum=None):
import plottool as pt
pt.qtensure()
pnum_ = pt.make_pnum_nextgen(nRows=1, nCols=len(self.forests))
for level, forest in enumerate(self.forests):
pt.show_nx(forest.to_networkx(), title='level=%r' % (level,),
fnum=fnum, pnum=pnum_())
def start_qt_interface(infr, loop=True):
import guitool as gt
from ibeis.viz.viz_graph2 import AnnotGraphWidget
from plottool import abstract_interaction
import plottool as pt
pt.qtensure()
gt.ensure_qtapp()
# win = AnnotGraphWidget(infr=infr, use_image=False, init_mode='review')
win = AnnotGraphWidget(infr=infr, use_image=False, init_mode=None)
abstract_interaction.register_interaction(win)
if loop:
gt.qtapp_loop(qwin=win, freq=10)
else:
win.show()
return win
def find_opt_ratio(pblm):
"""
script to help find the correct value for the ratio threshold
>>> from ibeis.algo.verif.vsone import * # NOQA
>>> pblm = OneVsOneProblem.from_empty('PZ_PB_RF_TRAIN')
>>> pblm = OneVsOneProblem.from_empty('GZ_Master1')
"""
# Find best ratio threshold
pblm.load_samples()
infr = pblm.infr
edges = ut.emap(tuple, pblm.samples.aid_pairs.tolist())
task = pblm.samples['match_state']
pos_idx = task.class_names.tolist().index(POSTV)
config = {'ratio_thresh': 1.0, 'sv_on': False}
matches = infr._exec_pairwise_match(edges, config)
import plottool as pt
pt.qtensure()
thresholds = np.linspace(0, 1.0, 100)
pos_truth = task.y_bin.T[pos_idx]
ratio_fs = [m.local_measures['ratio'] for m in matches]
aucs = []
# Given the current correspondences: Find the optimal
# correspondence threshold.
for thresh in ut.ProgIter(thresholds, 'computing thresh'):
scores = np.array([fs[fs < thresh].sum() for fs in ratio_fs])
roc = sklearn.metrics.roc_auc_score(pos_truth, scores)
aucs.append(roc)
aucs = np.array(aucs)
opt_auc = aucs.max()
opt_thresh = thresholds[aucs.argmax()]
if True:
pt.plt.plot(thresholds, aucs, 'r-', label='')
pt.plt.plot(opt_thresh,
opt_auc,
'ro',
label='L opt=%r' % (opt_thresh, ))
pt.set_ylabel('auc')
pt.set_xlabel('ratio threshold')
pt.legend()
def get_iters_vs_miou(harn):
from pysseg.util import jsonutil
import pysseg.backend.iface_caffe as iface
harn.prepare_test_model()
test_weight_dpaths = harn.find_test_weights_dpaths()
# for test_weights_dpath in test_weight_dpaths:
# harn.test_weights_dpath = test_weights_dpath
# harn.test_weights_fpath = ub.readfrom(join(test_weights_dpath, 'test_weights.caffemodel.lnk'))
# # if not exists(join(harn.test_weights_dpath, 'pred')):
# results_fpath = join(harn.test_weights_dpath, 'results.json')
# if exists(results_fpath):
# results = json.load(results_fpath)
iter_results = {}
for test_weights_dpath in test_weight_dpaths:
results_fpath = join(test_weights_dpath, 'results.json')
if exists(results_fpath):
iterno = iface.snapshot_iterno(test_weights_dpath)
results = json.load(open(results_fpath, 'r'))
ious = eval(results['ious'])
iter_results[iterno] = ious
iter_df = pd.DataFrame(iter_results)
iter_df.columns.name = 'iterno'
iter_df.index.name = 'class'
fpath = join(harn.test_dpath, 'iter_ious.json')
jsonutil.write_json(fpath, iter_df)
iter_df = iter_df.drop([57], axis=1)
iter_df.drop(harn.task.ignore_classnames).mean(axis=0)
if False:
"""
ffmpeg -y -f image2 -i ~/aretha/store/data/work/camvid/arch/segnet_proper/test/input_nqmmrhd/weights_abvroyo_segnet_proper_None_xwfmwfo_00040000/blend_pred/%*.png -crf 25 -vcodec libx264 -vf "setpts=4*PTS" camvid-results.avi
ffmpeg -y -f image2 -i out_haul83/%*.png -vcodec mpeg4 -vf "setpts=10*PTS" haul83-results.avi
"""
# move to computer with plottool
iter_df = pd.read_json(
'/home/joncrall/aretha/store/data/work/camvid/arch/segnet_proper/test/input_nqmmrhd/iter_ious.json'
)
import plottool as pt
pt.qtensure()
from pysseg.tasks import CamVid
task = CamVid()
iter_miou = iter_df.drop(task.ignore_classnames).mean(axis=0)
iter_miou = iter_miou.sort_index()
_set_mpl_rcparams()
fig = pt.figure(fnum=1, pnum=(1, 1, 1))
ax = pt.gca()
iter_miou.plot(ax=ax)
ax.set_xlabel('train iters')
ax.set_ylabel('test mIoU')
ax.set_title('Reproduced CamVid Results (init using VGG)')
ub.ensuredir('result_plots')
from pysseg.draw import render_figure_to_image
import cv2
cv2.imwrite('result_plots/miou.png',
render_figure_to_image(fig, dpi=100, transparent=True))
fig = pt.figure(fnum=2, pnum=(1, 1, 1))
ax = pt.gca()
iter_iou = iter_df.drop(task.ignore_classnames).T.sort_index()
# sort by results
iter_iou = iter_iou[iter_iou.iloc[-1].sort_values().index[::-1]]
colors = [
tuple(np.array(v[::-1]) / 255) + (1, )
for v in ub.take(task.class_colors, iter_iou.columns)
]
iter_iou.plot(ax=ax, colors=colors, lw=4)
ax.set_xlabel('train iters')
ax.set_ylabel('test IoU')
ax.set_title('Reproduced CamVid Results (init using VGG)')
ub.ensuredir('result_plots')
from pysseg.draw import render_figure_to_image
cv2.imwrite('result_plots/perclass_iou.png',
render_figure_to_image(fig, dpi=100, transparent=True))
def ewma():
import plottool as pt
import ubelt as ub
import numpy as np
pt.qtensure()
# Investigate the span parameter
span = 20
alpha = 2 / (span + 1)
# how long does it take for the estimation to hit 0?
# (ie, it no longer cares about the initial 1?)
# about 93 iterations to get to 1e-4
# about 47 iterations to get to 1e-2
# about 24 iterations to get to 1e-1
# 20 iterations goes to .135
data = ([1] + [0] * 20 + [1] * 40 + [0] * 20 + [1] * 50 + [0] * 20 +
[1] * 60 + [0] * 20 + [1] * 165 + [0] * 20 + [0])
mave = []
iter_ = iter(data)
current = next(iter_)
mave += [current]
for x in iter_:
current = (alpha * x) + (1 - alpha) * current
mave += [current]
if False:
pt.figure(fnum=1, doclf=True)
pt.plot(data)
pt.plot(mave)
np.where(np.array(mave) < 1e-1)
import sympy as sym
# span, alpha, n = sym.symbols('span, alpha, n')
n = sym.symbols('n', integer=True, nonnegative=True, finite=True)
span = sym.symbols('span', integer=True, nonnegative=True, finite=True)
thresh = sym.symbols('thresh', real=True, nonnegative=True, finite=True)
# alpha = 2 / (span + 1)
a, b, c = sym.symbols('a, b, c', real=True, nonnegative=True, finite=True)
sym.solve(sym.Eq(b**a, c), a)
current = 1
x = 0
steps = []
for _ in range(10):
current = (alpha * x) + (1 - alpha) * current
steps.append(current)
alpha = sym.symbols('alpha', real=True, nonnegative=True, finite=True)
base = sym.symbols('base', real=True, finite=True)
alpha = 2 / (span + 1)
thresh_expr = (1 - alpha)**n
thresthresh_exprh_expr = base**n
n_expr = sym.ceiling(sym.log(thresh) / sym.log(1 - 2 / (span + 1)))
sym.pprint(sym.simplify(thresh_expr))
sym.pprint(sym.simplify(n_expr))
print(sym.latex(sym.simplify(n_expr)))
# def calc_n2(span, thresh):
# return np.log(thresh) / np.log(1 - 2 / (span + 1))
def calc_n(span, thresh):
return np.log(thresh) / np.log((span - 1) / (span + 1))
def calc_thresh_val(n, span):
alpha = 2 / (span + 1)
return (1 - alpha)**n
span = np.arange(2, 200)
n_frac = calc_n(span, thresh=.5)
n = np.ceil(n_frac)
calc_thresh_val(n, span)
pt.figure(fnum=1, doclf=True)
ydatas = ut.odict([('thresh=%f' % thresh,
np.ceil(calc_n(span, thresh=thresh)))
for thresh in [1e-3, .01, .1, .2, .3, .4, .5]])
pt.multi_plot(
span,
ydatas,
xlabel='span',
ylabel='n iters to acheive thresh',
marker='',
# num_xticks=len(span),
fnum=1)
pt.gca().set_aspect('equal')
def both_sides(eqn, func):
return sym.Eq(func(eqn.lhs), func(eqn.rhs))
eqn = sym.Eq(thresh_expr, thresh)
n_expr = sym.solve(eqn,
n)[0].subs(base,
(1 - alpha)).subs(alpha, (2 / (span + 1)))
eqn = both_sides(eqn, lambda x: sym.log(x, (1 - alpha)))
lhs = eqn.lhs
from sympy.solvers.inequalities import solve_univariate_inequality
def eval_expr(span_value, n_value):
return np.array(
[thresh_expr.subs(span, span_value).subs(n, n_) for n_ in n_value],
dtype=np.float)
eval_expr(20, np.arange(20))
def linear(x, a, b):
return a * x + b
def sigmoidal_4pl(x, a, b, c, d):
return d + (a - d) / (1 + (x / c)**b)
def exponential(x, a, b, c):
return a + b * np.exp(-c * x)
import scipy.optimize
# Determine how to choose span, such that you get to .01 from 1
# in n timesteps
thresh_to_span_to_n = []
thresh_to_n_to_span = []
for thresh_value in ub.ProgIter([.0001, .001, .01, .1, .2, .3, .4, .5]):
print('')
test_vals = sorted([2, 3, 4, 5, 6])
n_to_span = []
for n_value in ub.ProgIter(test_vals):
# In n iterations I want to choose a span that the expression go
# less than a threshold
constraint = thresh_expr.subs(n, n_value) < thresh_value
solution = solve_univariate_inequality(constraint, span)
try:
lowbound = np.ceil(float(solution.args[0].lhs))
highbound = np.floor(float(solution.args[1].rhs))
assert lowbound <= highbound
span_value = lowbound
except AttributeError:
span_value = np.floor(float(solution.rhs))
n_to_span.append((n_value, span_value))
# Given a threshold, find a minimum number of steps
# that brings you up to that threshold given a span
test_vals = sorted(set(list(range(2, 1000, 50)) + [2, 3, 4, 5, 6]))
span_to_n = []
for span_value in ub.ProgIter(test_vals):
constraint = thresh_expr.subs(span, span_value) < thresh_value
solution = solve_univariate_inequality(constraint, n)
n_value = solution.lhs
span_to_n.append((span_value, n_value))
thresh_to_n_to_span.append((thresh_value, n_to_span))
thresh_to_span_to_n.append((thresh_value, span_to_n))
thresh_to_params = []
for thresh_value, span_to_n in thresh_to_span_to_n:
xdata, ydata = [np.array(_, dtype=np.float) for _ in zip(*span_to_n)]
p0 = (1 / np.diff((ydata - ydata[0])[1:]).mean(), ydata[0])
func = linear
popt, pcov = scipy.optimize.curve_fit(func, xdata, ydata, p0)
# popt, pcov = scipy.optimize.curve_fit(exponential, xdata, ydata)
if False:
yhat = func(xdata, *popt)
pt.figure(fnum=1, doclf=True)
pt.plot(xdata, ydata, label='measured')
pt.plot(xdata, yhat, label='predicteed')
pt.legend()
# slope = np.diff(ydata).mean()
# pt.plot(d)
thresh_to_params.append((thresh_value, popt))
# pt.plt.plot(*zip(*thresh_to_slope), 'x-')
# for thresh_value=.01, we get a rough line with slop ~2.302,
# for thresh_value=.5, we get a line with slop ~34.66
# if we want to get to 0 in n timesteps, with a thresh_value of
# choose span=f(thresh_value) * (n + 2))
# f is some inverse exponential
# 0.0001, 460.551314197147
# 0.001, 345.413485647860,
# 0.01, 230.275657098573,
# 0.1, 115.137828549287,
# 0.2, 80.4778885203347,
# 0.3, 60.2031233261536,
# 0.4, 45.8179484913827,
# 0.5, 34.6599400289520
# Seems to be 4PL symetrical sigmoid
# f(x) = -66500.85 + (66515.88 - -66500.85) / (1 + (x/0.8604672)^0.001503716)
# f(x) = -66500.85 + (66515.88 - -66500.85)/(1 + (x/0.8604672)^0.001503716)
def f(x):
return -66500.85 + (66515.88 -
-66500.85) / (1 + (x / 0.8604672)**0.001503716)
# return (10000 * (-6.65 + (13.3015) / (1 + (x/0.86) ** 0.00150)))
# f(.5) * (n - 1)
# f(
solve_rational_inequalities(thresh_expr < .01, n)
def do_infr_test(ccs, edges, new_edges):
"""
Creates a graph with `ccs` + `edges` and then adds `new_edges`
"""
# import networkx as nx
import plottool as pt
infr = demo.make_demo_infr(ccs, edges)
if ut.show_was_requested():
pt.qtensure()
# Preshow
fnum = 1
if ut.show_was_requested():
infr.set_node_attrs('shape', 'circle')
infr.show(pnum=(2, 1, 1),
fnum=fnum,
show_unreviewed_edges=True,
show_reviewed_cuts=True,
splines='spline',
show_inferred_diff=True,
groupby='name_label',
show_labels=True,
pickable=True)
pt.set_title('pre-review')
pt.gca().set_aspect('equal')
infr.set_node_attrs('pin', 'true')
# fig1 = pt.gcf()
# fig1.canvas.mpl_connect('pick_event', ut.partial(on_pick, infr=infr))
infr1 = infr
infr2 = infr.copy()
for new_edge in new_edges:
aid1, aid2, data = new_edge
evidence_decision = data['evidence_decision']
infr2.add_feedback((aid1, aid2), evidence_decision)
infr2.relabel_using_reviews(rectify=False)
infr2.apply_nondynamic_update()
# Postshow
if ut.show_was_requested():
infr2.show(pnum=(2, 1, 2),
fnum=fnum,
show_unreviewed_edges=True,
show_inferred_diff=True,
show_labels=True)
pt.gca().set_aspect('equal')
pt.set_title('post-review')
# fig2 = pt.gcf()
# if fig2 is not fig1:
# fig2.canvas.mpl_connect('pick_event', ut.partial(on_pick, infr=infr2))
class Checker(object):
"""
Asserts pre and post test properties of the graph
"""
def __init__(self, infr1, infr2):
self._errors = []
self.infr1 = infr1
self.infr2 = infr2
def __call__(self, infr, u, v, key, val, msg):
data = infr.get_nonvisual_edge_data((u, v))
if data is None:
assert infr.graph.has_edge(u, v), ('uv=%r, %r does not exist' %
(u, v))
got = data.get(key)
if got != val:
msg1 = 'key=%s %r!=%r, ' % (key, got, val)
errmsg = ''.join([
msg1, msg, '\nedge=',
ut.repr2((u, v)), '\n',
infr.repr_edge_data(data)
])
self._errors.append(errmsg)
def custom_precheck(self, func):
try:
func(self.infr1)
except AssertionError as ex:
self._errors.append(str(ex))
def after(self, errors=[]):
"""
Delays error reporting until after visualization
prints errors, then shows you the graph, then
finally if any errors were discovered they are raised
"""
errors = errors + self._errors
if errors:
ut.cprint('PRINTING %d FAILURE' % (len(errors)), 'red')
for msg in errors:
print(msg)
ut.cprint('HAD %d FAILURE' % (len(errors)), 'red')
if ut.show_was_requested():
pt.all_figures_tile(percent_w=.5)
ut.show_if_requested()
if errors:
raise AssertionError('There were errors')
check = Checker(infr1, infr2)
return infr1, infr2, check
def _model_data_flow_to_networkx(model_info):
layers = model_info['layer']
import networkx as nx
G = nx.DiGraph()
prev = None
# Stores last node with the data for this layer in it
prev_map = {}
SHOW_LOOPS = False
for layer in layers:
name = layer.get('name')
print('name = {!r}'.format(name))
G.add_node(name)
bottom = set(layer.get('bottom', []))
top = set(layer.get('top', []))
both = top.intersection(bottom)
if both:
if prev is None:
prev = both
for b in both:
prev_map[b] = name
for b in prev:
print(' * b = {!r}'.format(b))
G.add_edge(b, name, constraint=False)
for b in both:
print(' * b = {!r}'.format(b))
kw = {}
if not G.has_edge(b, name):
kw['color'] = 'red'
G.add_edge(b, name, constraint=True, **kw)
prev = [name]
else:
prev = None
# for b in (bottom - both):
for b in bottom:
print(' * b = {!r}'.format(b))
constraint = True
G.add_edge(prev_map.get(b, b), name, constraint=constraint)
if SHOW_LOOPS:
G.add_edge(b, name)
# for t in (bottom - top):
for t in top:
print(' * t = {!r}'.format(t))
constraint = True
G.add_edge(name, prev_map.get(t, t), constraint=constraint)
if SHOW_LOOPS:
G.add_edge(name, t)
G.remove_edges_from(list(G.selfloop_edges()))
import plottool as pt
pt.qtensure()
pt.show_nx(G, arrow_width=1)
pt.adjust_subplots(left=0, right=1, top=1, bottom=0)
pt.pan_factory()
pt.zoom_factory()
list(nx.topological_sort(G))
def demo_classes(pblm):
r"""
CommandLine:
python -m ibeis.algo.verif.vsone demo_classes --saveparts --save=classes.png --clipwhite
python -m ibeis.algo.verif.vsone demo_classes --saveparts --save=figures/classes.png --clipwhite --dpath=~/latex/crall-iccv-2017
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.verif.vsone import * # NOQA
>>> pblm = OneVsOneProblem.from_empty(defaultdb='PZ_PB_RF_TRAIN')
>>> pblm.load_features()
>>> pblm.load_samples()
>>> pblm.build_feature_subsets()
>>> pblm.demo_classes()
>>> ut.show_if_requested()
"""
task_key = 'match_state'
labels = pblm.samples.subtasks[task_key]
pb_labels = pblm.samples.subtasks['photobomb_state']
classname_offset = {
POSTV: 0,
NEGTV: 0,
INCMP: 0,
}
class_name = POSTV
class_name = NEGTV
class_name = INCMP
feats = pblm.samples.X_dict['learn(sum,glob)']
offset = 0
class_to_edge = {}
for class_name in labels.class_names:
print('Find example of %r' % (class_name, ))
# Find an example of each class (that is not a photobomb)
pbflags = pb_labels.indicator_df['notpb']
flags = labels.indicator_df[class_name]
assert np.all(pbflags.index == flags.index)
flags = flags & pbflags
ratio = feats['sum(ratio)']
if class_name == INCMP:
# flags &= feats['global(delta_yaw)'] > 3
flags &= feats['global(delta_view)'] > 2
# flags &= feats['sum(ratio)'] > 0
if class_name == NEGTV:
low = ratio[flags].max()
flags &= feats['sum(ratio)'] >= low
if class_name == POSTV:
low = ratio[flags].median() / 2
high = ratio[flags].median()
flags &= feats['sum(ratio)'] < high
flags &= feats['sum(ratio)'] > low
# flags &= pblm.samples.simple_scores[flags]['score_lnbnn_1vM'] > 0
idxs = np.where(flags)[0]
print('Found %d candidates' % (len(idxs)))
offset = classname_offset[class_name]
idx = idxs[offset]
series = labels.indicator_df.iloc[idx]
assert series[class_name]
edge = series.name
class_to_edge[class_name] = edge
import plottool as pt
import guitool as gt
gt.ensure_qapp()
pt.qtensure()
fnum = 1
pt.figure(fnum=fnum, pnum=(1, 3, 1))
pnum_ = pt.make_pnum_nextgen(1, 3)
# classname_alias = {
# POSTV: 'positive',
# NEGTV: 'negative',
# INCMP: 'incomparable',
# }
ibs = pblm.infr.ibs
for class_name in class_to_edge.keys():
edge = class_to_edge[class_name]
aid1, aid2 = edge
# alias = classname_alias[class_name]
print('class_name = %r' % (class_name, ))
annot1 = ibs.annots([aid1])[0]._make_lazy_dict()
annot2 = ibs.annots([aid2])[0]._make_lazy_dict()
vt.matching.ensure_metadata_normxy(annot1)
vt.matching.ensure_metadata_normxy(annot2)
match = vt.PairwiseMatch(annot1, annot2)
cfgdict = pblm.hyper_params.vsone_match.asdict()
match.apply_all(cfgdict)
pt.figure(fnum=fnum, pnum=pnum_())
match.show(show_ell=False, show_ori=False)
def opt_crf():
from clab.torch.urban_pred import get_snapshot, urban_mapper_eval_dataset, PredictHarness # NOQA
from clab.torch.sseg_train import task_datasets, get_task, SSegInputsWrapper # NOQA
from clab import util
import ubelt as ub
# train_dpath = ub.truepath(
# '~/remote/aretha/data/work/urban_mapper/arch/unet/train/input_4214-yxalqwdk/solver_4214-yxalqwdk_unet_vgg_nttxoagf_a=1,n_ch=5,n_cl=3')
# load_path = get_snapshot(train_dpath, epoch=202)
datasets = task_datasets(get_task('urban_mapper_3d'))
test_dataset = datasets['test']
test_dataset.with_gt = False
test_dataset.inputs.make_dumpsafe_names()
test_dataset.center_inputs = test_dataset._original_urban_mapper_normalizer()
test_dataset.tag = 'test'
prob_folder = ub.truepath(
'~/remote/aretha/data/work/urban_mapper/test/input_4224-rwyxarza/solver_4214-yxalqwdk_unet_vgg_nttxoagf_a=1,n_ch=5,n_cl=3/_epoch_00000202/log_probs')
import glob
subset = slice(300, 310)
prob_paths = test_dataset.inputs.align(glob.glob(prob_folder + '/*.npz'))[subset]
gt_paths = test_dataset.inputs.gt_paths[subset]
im_paths = test_dataset.inputs.im_paths[subset]
import numpy as np
imgs = [util.imread(p) for p in ub.ProgIter(im_paths)]
probs = [np.load(p)['arr_0'] for p in ub.ProgIter(prob_paths)]
gts = [util.imread(p) for p in ub.ProgIter(gt_paths)]
from .torch import filters
# true = gts[4]
import optml
class_median_weights = test_dataset.class_weights()
class_weights = class_median_weights / class_median_weights.sum()
class CRFModel(optml.models.Model):
__model_module__ = 'sklearn' # hack
def __init__(self, **kwargs):
self.kwargs = kwargs
def get_params(self, deep=False):
return self.kwargs
def fit(self, X, y=None):
pass
def predict(self, X):
return [filters.crf_posterior(imgs[i], probs[i], **self.kwargs).argmax(axis=0) for i in ub.ProgIter(X, label='predicting')]
def clf_score(y_true, y_pred):
from .metrics import confusion_matrix, jaccard_score_from_confusion # NOQA
cfsn = np.zeros((3, 3))
for i, pred in zip(y_true, y_pred):
true = gts[i]
cfsn += confusion_matrix(true.ravel(), pred.ravel(), [0, 1, 2])
ious = jaccard_score_from_confusion(cfsn)
weighted_miou = ((ious * class_weights)[0:2]).sum()
return weighted_miou
from optml.bayesian_optimizer import BayesianOptimizer
model = CRFModel()
params = [
# w1 = kwargs.get('w1', 4)
# w2 = kwargs.get('w2', 3)
# sigma_alpha = kwargs.get('sigma_alpha', 100)
# sigma_beta = kwargs.get('sigma_beta', 3)
# sigma_gamma = kwargs.get('sigma_gamma', 3)
# n_iters = kwargs.get('n_iters', 10)
optml.Parameter(name='w1', param_type='integer', lower=1, upper=100),
optml.Parameter(name='sigma_alpha', param_type='integer', lower=1, upper=150),
optml.Parameter(name='sigma_beta', param_type='integer', lower=1, upper=150),
optml.Parameter(name='w2', param_type='integer', lower=3, upper=3),
optml.Parameter(name='sigma_gamma', param_type='integer', lower=3, upper=3),
optml.Parameter(name='n_iters', param_type='integer', lower=10, upper=10),
# optml.Parameter(name='param3', param_type='categorical', possible_values=['val1','val2','val3'])
]
optimizer = BayesianOptimizer(model=model,
hyperparams=params,
eval_func=clf_score)
# optimizer.model = model
X_train = np.arange(len(prob_paths)) # dummy
y_train = np.arange(len(X_train))
# Add some good values to help out initial model
seed_params = [
# Best known so far
{'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 4, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 2, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 5, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 3, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 105, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 95, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 101, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 99, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 61, 'sigma_beta': 11, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 139, 'sigma_beta': 1, 'w1': 50, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 139, 'sigma_beta': 3, 'w1': 50, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 10, 'sigma_alpha': 139, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
]
seed_params = [
# Best known so far
{'n_iters': 5, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
{'n_iters': 20, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 4, 'w2': 3, 'sigma_gamma': 3},
# {'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 4, 'w2': 1, 'sigma_gamma': 1},
# {'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 4, 'w2': 2, 'sigma_gamma': 2},
# {'n_iters': 10, 'sigma_alpha': 100, 'sigma_beta': 3, 'w1': 4, 'w2': 4, 'sigma_gamma': 4},
]
for seed in seed_params:
print('seed = {}'.format(ub.repr2(seed, nl=0, precision=2)))
print(optimizer._try_params(seed, X_train, y_train, X_train, y_train))
bayes_best_params, bayes_best_model = optimizer.fit(X_train=X_train,
y_train=y_train,
n_iters=10,
verbose=True)
names = [p.name for p in optimizer.hyperparams]
names = ['w1', 'sigma_alpha', 'sigma_beta']
xs = np.array([list(ub.take(params, names)) for score, params in optimizer.hyperparam_history])
ys = np.array([score for score, params in optimizer.hyperparam_history])
xs.T[0]
import plottool as pt
pt.qtensure()
pt.plt.clf()
for i in range(len(names)):
pt.plt.plot(xs.T[i], ys, 'o', label=names[i])
pt.plt.legend()
def mnist_demo():
"""
CommandLine:
python -m clab.torch.fit_harness mnist_demo
Example:
>>> mnist_demo()
"""
from clab.torch import models
from clab.torch import hyperparams
root = os.path.expanduser('~/data/mnist/')
transform = torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize((0.1307, ), (0.3081, ))
])
datasets = {
'train':
torchvision.datasets.MNIST(root,
transform=transform,
train=True,
download=True),
'test':
torchvision.datasets.MNIST(root,
transform=transform,
train=False,
download=True),
}
batch_size = 128
n_classes = 10
xpu = xpu_device.XPU.from_argv(min_memory=4000)
model = models.MnistNet(n_channels=1, n_classes=n_classes)
hyper = hyperparams.HyperParams(
optimizer=torch.optim.Adam,
criterion=torch.nn.CrossEntropyLoss,
other={
'n_classes': n_classes,
# 'n_channels': n_channels,
})
train_dpath = os.path.expanduser('~/data/work/mnist/harness/mnist-net')
harn = FitHarness(
model=model,
datasets=datasets,
batch_size=batch_size,
train_dpath=train_dpath,
xpu=xpu,
hyper=hyper,
)
labels = np.arange(n_classes)
def custom_metrics(harn, output, label):
# ignore_label = datasets['train'].ignore_label
# labels = datasets['train'].task.labels
metrics_dict = metrics._clf_metrics(output, label, labels=labels)
return metrics_dict
harn.add_metric_hook(custom_metrics)
if False:
import plottool as pt
pt.qtensure()
ims, gts = next(iter(harn.loaders['train']))
pic = im_loaders.rgb_tensor_to_imgs(ims, norm=False)[0]
pt.clf()
pt.imshow(pic, norm=True, cmap='viridis', data_colorbar=True)
with pt.RenderingContext() as render:
tensor_data = datasets['train'][0][0][None, :]
pic = im_loaders.rgb_tensor_to_imgs(tensor_data, norm=False)[0]
pt.figure(fnum=1, doclf=True)
pt.imshow(pic,
norm=True,
cmap='viridis',
data_colorbar=True,
fnum=1)
render.image
harn.run()
def demo_refresh():
r"""
CommandLine:
python -m ibeis.algo.graph.refresh demo_refresh \
--num_pccs=40 --size=2 --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.graph.refresh import * # NOQA
>>> demo_refresh()
>>> ut.show_if_requested()
"""
from ibeis.algo.graph import demo
demokw = ut.argparse_dict({'num_pccs': 50, 'size': 4})
refreshkw = ut.argparse_funckw(RefreshCriteria)
# make an inference object
infr = demo.demodata_infr(size_std=0, **demokw)
edges = list(infr.dummy_verif.find_candidate_edges(K=100))
scores = np.array(infr.dummy_verif.predict_edges(edges))
sortx = scores.argsort()[::-1]
edges = ut.take(edges, sortx)
scores = scores[sortx]
ys = infr.match_state_df(edges)[POSTV].values
y_remainsum = ys[::-1].cumsum()[::-1]
# Do oracle reviews and wait to converge
refresh = RefreshCriteria(**refreshkw)
xdata = []
pprob_any = []
rfrac_any = []
for count, (edge, y) in enumerate(zip(edges, ys)):
refresh.add(y, user_id='user:oracle')
rfrac_any.append(y_remainsum[count] / y_remainsum[0])
pprob_any.append(refresh.prob_any_remain())
xdata.append(count + 1)
if refresh.check():
break
xdata = xdata
ydatas = ut.odict([
('Est. probability any remain', pprob_any),
('Fraction remaining', rfrac_any),
])
ut.quit_if_noshow()
import plottool as pt
pt.qtensure()
from ibeis.scripts.thesis import TMP_RC
import matplotlib as mpl
mpl.rcParams.update(TMP_RC)
pt.multi_plot(
xdata, ydatas, xlabel='# manual reviews', rcParams=TMP_RC, marker='',
ylim=(0, 1), use_legend=False,
)
demokw = ut.map_keys({'num_pccs': '#PCC', 'size': 'PCC size'},
demokw)
thresh = refreshkw.pop('thresh')
refreshkw['span'] = refreshkw.pop('window')
pt.relative_text((.02, .58 + .0), ut.get_cfg_lbl(demokw, sep=' ')[1:],
valign='bottom')
pt.relative_text((.02, .68 + .0), ut.get_cfg_lbl(refreshkw, sep=' ')[1:],
valign='bottom')
legend = pt.gca().legend()
legend.get_frame().set_alpha(1.0)
pt.plt.plot([xdata[0], xdata[-1]], [thresh, thresh], 'g--', label='thresh')