def test_mcc():
import wbia.plottool as pt
import sklearn.metrics
num = 100
xdata = np.linspace(0, 1, num * 2)
ydata = np.linspace(1, -1, num * 2)
pt.plt.plot(xdata, ydata, '--k', label='linear')
y_true = [1] * num + [0] * num
y_pred = y_true[:]
xs = []
for i in range(0, len(y_true)):
y_pred[-i] = 1 - y_pred[-i]
xs.append(sklearn.metrics.matthews_corrcoef(y_true, y_pred))
pt.plot(xdata, xs, label='change one class at a time')
y_true = ut.flatten(zip([1] * num, [0] * num))
y_pred = y_true[:]
xs = []
for i in range(0, len(y_true)):
y_pred[-i] = 1 - y_pred[-i]
xs.append(sklearn.metrics.matthews_corrcoef(y_true, y_pred))
pt.plot(xdata, xs, label='change classes evenly')
pt.gca().legend()
def choose_thresh(self):
# prob_annots /= prob_annots.sum(axis=1)[:, None]
# Find connected components
# thresh = .25
# thresh = 1 / (1.2 * np.sqrt(prob_names.shape[1]))
unique_nids, prob_names = self.make_prob_names()
if len(unique_nids) <= 2:
return 0.5
nscores = np.sort(prob_names.flatten())
# x = np.gradient(nscores).argmax()
# x = (np.gradient(np.gradient(nscores)) ** 2).argmax()
# thresh = nscores[x]
curve = nscores
idx1 = vt.find_elbow_point(curve)
idx2 = vt.find_elbow_point(curve[idx1:]) + idx1
if False:
import wbia.plottool as pt
idx3 = vt.find_elbow_point(curve[idx1:idx2 + 1]) + idx1
pt.plot(curve)
pt.plot(idx1, curve[idx1], 'bo')
pt.plot(idx2, curve[idx2], 'ro')
pt.plot(idx3, curve[idx3], 'go')
thresh = nscores[idx2]
# logger.info('thresh = %r' % (thresh,))
# thresh = .999
# thresh = .1
return thresh
def gridsearch_ratio_thresh(matches):
import sklearn
import sklearn.metrics
import vtool as vt
# Param search for vsone
import wbia.plottool as pt
pt.qt4ensure()
skf = sklearn.model_selection.StratifiedKFold(n_splits=10, random_state=119372)
y = np.array([m.annot1['nid'] == m.annot2['nid'] for m in matches])
basis = {'ratio_thresh': np.linspace(0.6, 0.7, 50).tolist()}
grid = ut.all_dict_combinations(basis)
xdata = np.array(ut.take_column(grid, 'ratio_thresh'))
def _ratio_thresh(y_true, match_list):
# Try and find optional ratio threshold
auc_list = []
for cfgdict in ut.ProgIter(grid, lbl='gridsearch'):
y_score = [
match.fs.compress(match.ratio_test_flags(cfgdict)).sum()
for match in match_list
]
auc = sklearn.metrics.roc_auc_score(y_true, y_score)
auc_list.append(auc)
auc_list = np.array(auc_list)
return auc_list
auc_list = _ratio_thresh(y, matches)
pt.plot(xdata, auc_list)
subx, suby = vt.argsubmaxima(auc_list, xdata)
best_ratio_thresh = subx[suby.argmax()]
skf_results = []
y_true = y
for train_idx, test_idx in skf.split(matches, y):
match_list_ = ut.take(matches, train_idx)
y_true = y.take(train_idx)
auc_list = _ratio_thresh(y_true, match_list_)
subx, suby = vt.argsubmaxima(auc_list, xdata, maxima_thresh=0.8)
best_ratio_thresh = subx[suby.argmax()]
skf_results.append(best_ratio_thresh)
logger.info('skf_results.append = %r' % (np.mean(skf_results),))
import utool
utool.embed()
def compare_data(Y_list_):
import wbia
qreq_ = wbia.testdata_qreq_(
defaultdb='Oxford',
a='oxford',
p=
'smk:nWords=[64000],nAssign=[1],SV=[False],can_match_sameimg=True,dim_size=None',
)
qreq_.ensure_data()
gamma1s = []
gamma2s = []
logger.info(len(Y_list_))
logger.info(len(qreq_.daids))
dinva = qreq_.dinva
bady = []
for Y in Y_list_:
aid = Y.aid
gamma1 = Y.gamma
if aid in dinva.aid_to_idx:
idx = dinva.aid_to_idx[aid]
gamma2 = dinva.gamma_list[idx]
gamma1s.append(gamma1)
gamma2s.append(gamma2)
else:
bady += [Y]
logger.info(Y.nid)
# logger.info(Y.qual)
# ibs = qreq_.ibs
# z = ibs.annots([a.aid for a in bady])
import wbia.plottool as pt
ut.qtensure()
gamma1s = np.array(gamma1s)
gamma2s = np.array(gamma2s)
sortx = gamma1s.argsort()
pt.plot(gamma1s[sortx], label='script')
pt.plot(gamma2s[sortx], label='pipe')
pt.legend()
def _dev_iters_until_threshold():
"""
INTERACTIVE DEVELOPMENT FUNCTION
How many iterations of ewma until you hit the poisson / biniomal threshold
This establishes a principled way to choose the threshold for the refresh
criterion in my thesis. There are paramters --- moving parts --- that we
need to work with: `a` the patience, `s` the span, and `mu` our ewma.
`s` is a span paramter indicating how far we look back.
`mu` is the average number of label-changing reviews in roughly the last
`s` manual decisions.
These numbers are used to estimate the probability that any of the next `a`
manual decisions will be label-chanigng. When that probability falls below
a threshold we terminate. The goal is to choose `a`, `s`, and the threshold
`t`, such that the probability will fall below the threshold after a maximum
of `a` consecutive non-label-chaning reviews. IE we want to tie the patience
paramter (how far we look ahead) to how far we actually are willing to go.
"""
import numpy as np
import utool as ut
import sympy as sym
i = sym.symbols('i', integer=True, nonnegative=True, finite=True)
# mu_i = sym.symbols('mu_i', integer=True, nonnegative=True, finite=True)
s = sym.symbols('s', integer=True, nonnegative=True, finite=True) # NOQA
thresh = sym.symbols('tau', real=True, nonnegative=True, finite=True) # NOQA
alpha = sym.symbols('alpha', real=True, nonnegative=True, finite=True) # NOQA
c_alpha = sym.symbols('c_alpha', real=True, nonnegative=True, finite=True)
# patience
a = sym.symbols('a', real=True, nonnegative=True, finite=True)
available_subs = {
a: 20,
s: a,
alpha: 2 / (s + 1),
c_alpha: (1 - alpha),
}
def subs(expr, d=available_subs):
""" recursive expression substitution """
expr1 = expr.subs(d)
if expr == expr1:
return expr1
else:
return subs(expr1, d=d)
# mu is either the support for the poisson distribution
# or is is the p in the binomial distribution
# It is updated at timestep i based on ewma, assuming each incoming responce is 0
mu_0 = 1.0
mu_i = c_alpha ** i
# Estimate probability that any event will happen in the next `a` reviews
# at time `i`.
poisson_i = 1 - sym.exp(-mu_i * a)
binom_i = 1 - (1 - mu_i) ** a
# Expand probabilities to be a function of i, s, and a
part = ut.delete_dict_keys(available_subs.copy(), [a, s])
mu_i = subs(mu_i, d=part)
poisson_i = subs(poisson_i, d=part)
binom_i = subs(binom_i, d=part)
if True:
# ewma of mu at time i if review is always not label-changing (meaningful)
mu_1 = c_alpha * mu_0 # NOQA
mu_2 = c_alpha * mu_1 # NOQA
if True:
i_vals = np.arange(0, 100)
mu_vals = np.array([subs(mu_i).subs({i: i_}).evalf() for i_ in i_vals]) # NOQA
binom_vals = np.array(
[subs(binom_i).subs({i: i_}).evalf() for i_ in i_vals]
) # NOQA
poisson_vals = np.array(
[subs(poisson_i).subs({i: i_}).evalf() for i_ in i_vals]
) # NOQA
# Find how many iters it actually takes my expt to terminate
thesis_draft_thresh = np.exp(-2)
np.where(mu_vals < thesis_draft_thresh)[0]
np.where(binom_vals < thesis_draft_thresh)[0]
np.where(poisson_vals < thesis_draft_thresh)[0]
sym.pprint(sym.simplify(mu_i))
sym.pprint(sym.simplify(binom_i))
sym.pprint(sym.simplify(poisson_i))
# Find the thresholds that force termination after `a` reviews have passed
# do this by setting i=a
poisson_thresh = poisson_i.subs({i: a})
binom_thresh = binom_i.subs({i: a})
logger.info('Poisson thresh')
logger.info(sym.latex(sym.Eq(thresh, poisson_thresh)))
logger.info(sym.latex(sym.Eq(thresh, sym.simplify(poisson_thresh))))
poisson_thresh.subs({a: 115, s: 30}).evalf()
sym.pprint(sym.Eq(thresh, poisson_thresh))
sym.pprint(sym.Eq(thresh, sym.simplify(poisson_thresh)))
logger.info('Binomial thresh')
sym.pprint(sym.simplify(binom_thresh))
sym.pprint(sym.simplify(poisson_thresh.subs({s: a})))
def taud(coeff):
return coeff * 360
if 'poisson_cache' not in vars():
poisson_cache = {}
binom_cache = {}
S, A = np.meshgrid(np.arange(1, 150, 1), np.arange(0, 150, 1))
import wbia.plottool as pt
SA_coords = list(zip(S.ravel(), A.ravel()))
for sval, aval in ut.ProgIter(SA_coords):
if (sval, aval) not in poisson_cache:
poisson_cache[(sval, aval)] = float(
poisson_thresh.subs({a: aval, s: sval}).evalf()
)
poisson_zdata = np.array(
[poisson_cache[(sval, aval)] for sval, aval in SA_coords]
).reshape(A.shape)
fig = pt.figure(fnum=1, doclf=True)
pt.gca().set_axis_off()
pt.plot_surface3d(
S,
A,
poisson_zdata,
xlabel='s',
ylabel='a',
rstride=3,
cstride=3,
zlabel='poisson',
mode='wire',
contour=True,
title='poisson3d',
)
pt.gca().set_zlim(0, 1)
pt.gca().view_init(elev=taud(1 / 16), azim=taud(5 / 8))
fig.set_size_inches(10, 6)
fig.savefig(
'a-s-t-poisson3d.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
for sval, aval in ut.ProgIter(SA_coords):
if (sval, aval) not in binom_cache:
binom_cache[(sval, aval)] = float(
binom_thresh.subs({a: aval, s: sval}).evalf()
)
binom_zdata = np.array(
[binom_cache[(sval, aval)] for sval, aval in SA_coords]
).reshape(A.shape)
fig = pt.figure(fnum=2, doclf=True)
pt.gca().set_axis_off()
pt.plot_surface3d(
S,
A,
binom_zdata,
xlabel='s',
ylabel='a',
rstride=3,
cstride=3,
zlabel='binom',
mode='wire',
contour=True,
title='binom3d',
)
pt.gca().set_zlim(0, 1)
pt.gca().view_init(elev=taud(1 / 16), azim=taud(5 / 8))
fig.set_size_inches(10, 6)
fig.savefig(
'a-s-t-binom3d.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
# Find point on the surface that achieves a reasonable threshold
# Sympy can't solve this
# sym.solve(sym.Eq(binom_thresh.subs({s: 50}), .05))
# sym.solve(sym.Eq(poisson_thresh.subs({s: 50}), .05))
# Find a numerical solution
def solve_numeric(expr, target, want, fixed, method=None, bounds=None):
"""
Args:
expr (Expr): symbolic expression
target (float): numberic value
fixed (dict): fixed values of the symbol
expr = poisson_thresh
expr.free_symbols
fixed = {s: 10}
solve_numeric(poisson_thresh, .05, {s: 30}, method=None)
solve_numeric(poisson_thresh, .05, {s: 30}, method='Nelder-Mead')
solve_numeric(poisson_thresh, .05, {s: 30}, method='BFGS')
"""
import scipy.optimize
# Find the symbol you want to solve for
want_symbols = expr.free_symbols - set(fixed.keys())
# TODO: can probably extend this to multiple params
assert len(want_symbols) == 1, 'specify all but one var'
assert want == list(want_symbols)[0]
fixed_expr = expr.subs(fixed)
def func(a1):
expr_value = float(fixed_expr.subs({want: a1}).evalf())
return (expr_value - target) ** 2
# if method is None:
# method = 'Nelder-Mead'
# method = 'Newton-CG'
# method = 'BFGS'
# Use one of the other params the startin gpoing
a1 = list(fixed.values())[0]
result = scipy.optimize.minimize(func, x0=a1, method=method, bounds=bounds)
if not result.success:
logger.info('\n')
logger.info(result)
logger.info('\n')
return result
# Numeric measurments of thie line
thresh_vals = [0.001, 0.01, 0.05, 0.1, 0.135]
svals = np.arange(1, 100)
target_poisson_plots = {}
for target in ut.ProgIter(thresh_vals, bs=False, freq=1):
poisson_avals = []
for sval in ut.ProgIter(svals, 'poisson', freq=1):
expr = poisson_thresh
fixed = {s: sval}
want = a
aval = solve_numeric(expr, target, want, fixed, method='Nelder-Mead').x[0]
poisson_avals.append(aval)
target_poisson_plots[target] = (svals, poisson_avals)
fig = pt.figure(fnum=3)
for target, dat in target_poisson_plots.items():
pt.plt.plot(*dat, label='prob={}'.format(target))
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('a')
pt.legend()
pt.gca().set_title('poisson')
fig.set_size_inches(5, 3)
fig.savefig(
'a-vs-s-poisson.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
target_binom_plots = {}
for target in ut.ProgIter(thresh_vals, bs=False, freq=1):
binom_avals = []
for sval in ut.ProgIter(svals, 'binom', freq=1):
aval = solve_numeric(
binom_thresh, target, a, {s: sval}, method='Nelder-Mead'
).x[0]
binom_avals.append(aval)
target_binom_plots[target] = (svals, binom_avals)
fig = pt.figure(fnum=4)
for target, dat in target_binom_plots.items():
pt.plt.plot(*dat, label='prob={}'.format(target))
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('a')
pt.legend()
pt.gca().set_title('binom')
fig.set_size_inches(5, 3)
fig.savefig(
'a-vs-s-binom.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
# ----
if True:
fig = pt.figure(fnum=5, doclf=True)
s_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for sval in s_vals:
pp = poisson_thresh.subs({s: sval})
a_vals = np.arange(0, 200)
pp_vals = np.array(
[float(pp.subs({a: aval}).evalf()) for aval in a_vals]
) # NOQA
pt.plot(a_vals, pp_vals, label='s=%r' % (sval,))
pt.legend()
pt.gca().set_xlabel('a')
pt.gca().set_ylabel('poisson prob after a reviews')
fig.set_size_inches(5, 3)
fig.savefig(
'a-vs-thresh-poisson.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
fig = pt.figure(fnum=6, doclf=True)
s_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for sval in s_vals:
pp = binom_thresh.subs({s: sval})
a_vals = np.arange(0, 200)
pp_vals = np.array(
[float(pp.subs({a: aval}).evalf()) for aval in a_vals]
) # NOQA
pt.plot(a_vals, pp_vals, label='s=%r' % (sval,))
pt.legend()
pt.gca().set_xlabel('a')
pt.gca().set_ylabel('binom prob after a reviews')
fig.set_size_inches(5, 3)
fig.savefig(
'a-vs-thresh-binom.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
# -------
fig = pt.figure(fnum=5, doclf=True)
a_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for aval in a_vals:
pp = poisson_thresh.subs({a: aval})
s_vals = np.arange(1, 200)
pp_vals = np.array(
[float(pp.subs({s: sval}).evalf()) for sval in s_vals]
) # NOQA
pt.plot(s_vals, pp_vals, label='a=%r' % (aval,))
pt.legend()
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('poisson prob')
fig.set_size_inches(5, 3)
fig.savefig(
's-vs-thresh-poisson.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
fig = pt.figure(fnum=5, doclf=True)
a_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for aval in a_vals:
pp = binom_thresh.subs({a: aval})
s_vals = np.arange(1, 200)
pp_vals = np.array(
[float(pp.subs({s: sval}).evalf()) for sval in s_vals]
) # NOQA
pt.plot(s_vals, pp_vals, label='a=%r' % (aval,))
pt.legend()
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('binom prob')
fig.set_size_inches(5, 3)
fig.savefig(
's-vs-thresh-binom.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True),
)
# ---------------------
# Plot out a table
mu_i.subs({s: 75, a: 75}).evalf()
poisson_thresh.subs({s: 75, a: 75}).evalf()
sval = 50
for target, dat in target_poisson_plots.items():
slope = np.median(np.diff(dat[1]))
aval = int(np.ceil(sval * slope))
thresh = float(poisson_thresh.subs({s: sval, a: aval}).evalf())
logger.info(
'aval={}, sval={}, thresh={}, target={}'.format(aval, sval, thresh, target)
)
for target, dat in target_binom_plots.items():
slope = np.median(np.diff(dat[1]))
aval = int(np.ceil(sval * slope))