def show_score_probs(verif):
"""
CommandLine:
python -m ibeis.algo.graph.demo DummyVerif.show_score_probs --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.graph.demo import * # NOQA
>>> import ibeis
>>> infr = ibeis.AnnotInference(None)
>>> verif = DummyVerif(infr)
>>> verif.show_score_probs()
>>> ut.show_if_requested()
"""
import plottool as pt
dist = verif.score_dist
n = 100000
for key in verif.dummy_params.keys():
probs = dist(shape=[n],
rng=verif.rng,
a_max=1,
a_min=0,
**verif.dummy_params[key])
color = verif.infr._get_truth_colors()[key]
pt.plt.hist(probs, bins=100, label=key, alpha=.8, color=color)
pt.legend()
def find_opt_ratio(pblm):
"""
script to help find the correct value for the ratio threshold
>>> from ibeis.algo.verif.vsone import * # NOQA
>>> pblm = OneVsOneProblem.from_empty('PZ_PB_RF_TRAIN')
>>> pblm = OneVsOneProblem.from_empty('GZ_Master1')
"""
# Find best ratio threshold
pblm.load_samples()
infr = pblm.infr
edges = ut.emap(tuple, pblm.samples.aid_pairs.tolist())
task = pblm.samples['match_state']
pos_idx = task.class_names.tolist().index(POSTV)
config = {'ratio_thresh': 1.0, 'sv_on': False}
matches = infr._exec_pairwise_match(edges, config)
import plottool as pt
pt.qtensure()
thresholds = np.linspace(0, 1.0, 100)
pos_truth = task.y_bin.T[pos_idx]
ratio_fs = [m.local_measures['ratio'] for m in matches]
aucs = []
# Given the current correspondences: Find the optimal
# correspondence threshold.
for thresh in ut.ProgIter(thresholds, 'computing thresh'):
scores = np.array([fs[fs < thresh].sum() for fs in ratio_fs])
roc = sklearn.metrics.roc_auc_score(pos_truth, scores)
aucs.append(roc)
aucs = np.array(aucs)
opt_auc = aucs.max()
opt_thresh = thresholds[aucs.argmax()]
if True:
pt.plt.plot(thresholds, aucs, 'r-', label='')
pt.plt.plot(opt_thresh,
opt_auc,
'ro',
label='L opt=%r' % (opt_thresh, ))
pt.set_ylabel('auc')
pt.set_xlabel('ratio threshold')
pt.legend()
def compare_data(Y_list_):
import ibeis
qreq_ = ibeis.testdata_qreq_(
defaultdb='Oxford',
a='oxford',
p='smk:nWords=[64000],nAssign=[1],SV=[False],can_match_sameimg=True,dim_size=None'
)
qreq_.ensure_data()
gamma1s = []
gamma2s = []
print(len(Y_list_))
print(len(qreq_.daids))
dinva = qreq_.dinva
bady = []
for Y in Y_list_:
aid = Y.aid
gamma1 = Y.gamma
if aid in dinva.aid_to_idx:
idx = dinva.aid_to_idx[aid]
gamma2 = dinva.gamma_list[idx]
gamma1s.append(gamma1)
gamma2s.append(gamma2)
else:
bady += [Y]
print(Y.nid)
# print(Y.qual)
# ibs = qreq_.ibs
# z = ibs.annots([a.aid for a in bady])
import plottool as pt
ut.qtensure()
gamma1s = np.array(gamma1s)
gamma2s = np.array(gamma2s)
sortx = gamma1s.argsort()
pt.plot(gamma1s[sortx], label='script')
pt.plot(gamma2s[sortx], label='pipe')
pt.legend()
def ewma():
import plottool as pt
import ubelt as ub
import numpy as np
pt.qtensure()
# Investigate the span parameter
span = 20
alpha = 2 / (span + 1)
# how long does it take for the estimation to hit 0?
# (ie, it no longer cares about the initial 1?)
# about 93 iterations to get to 1e-4
# about 47 iterations to get to 1e-2
# about 24 iterations to get to 1e-1
# 20 iterations goes to .135
data = ([1] + [0] * 20 + [1] * 40 + [0] * 20 + [1] * 50 + [0] * 20 +
[1] * 60 + [0] * 20 + [1] * 165 + [0] * 20 + [0])
mave = []
iter_ = iter(data)
current = next(iter_)
mave += [current]
for x in iter_:
current = (alpha * x) + (1 - alpha) * current
mave += [current]
if False:
pt.figure(fnum=1, doclf=True)
pt.plot(data)
pt.plot(mave)
np.where(np.array(mave) < 1e-1)
import sympy as sym
# span, alpha, n = sym.symbols('span, alpha, n')
n = sym.symbols('n', integer=True, nonnegative=True, finite=True)
span = sym.symbols('span', integer=True, nonnegative=True, finite=True)
thresh = sym.symbols('thresh', real=True, nonnegative=True, finite=True)
# alpha = 2 / (span + 1)
a, b, c = sym.symbols('a, b, c', real=True, nonnegative=True, finite=True)
sym.solve(sym.Eq(b**a, c), a)
current = 1
x = 0
steps = []
for _ in range(10):
current = (alpha * x) + (1 - alpha) * current
steps.append(current)
alpha = sym.symbols('alpha', real=True, nonnegative=True, finite=True)
base = sym.symbols('base', real=True, finite=True)
alpha = 2 / (span + 1)
thresh_expr = (1 - alpha)**n
thresthresh_exprh_expr = base**n
n_expr = sym.ceiling(sym.log(thresh) / sym.log(1 - 2 / (span + 1)))
sym.pprint(sym.simplify(thresh_expr))
sym.pprint(sym.simplify(n_expr))
print(sym.latex(sym.simplify(n_expr)))
# def calc_n2(span, thresh):
# return np.log(thresh) / np.log(1 - 2 / (span + 1))
def calc_n(span, thresh):
return np.log(thresh) / np.log((span - 1) / (span + 1))
def calc_thresh_val(n, span):
alpha = 2 / (span + 1)
return (1 - alpha)**n
span = np.arange(2, 200)
n_frac = calc_n(span, thresh=.5)
n = np.ceil(n_frac)
calc_thresh_val(n, span)
pt.figure(fnum=1, doclf=True)
ydatas = ut.odict([('thresh=%f' % thresh,
np.ceil(calc_n(span, thresh=thresh)))
for thresh in [1e-3, .01, .1, .2, .3, .4, .5]])
pt.multi_plot(
span,
ydatas,
xlabel='span',
ylabel='n iters to acheive thresh',
marker='',
# num_xticks=len(span),
fnum=1)
pt.gca().set_aspect('equal')
def both_sides(eqn, func):
return sym.Eq(func(eqn.lhs), func(eqn.rhs))
eqn = sym.Eq(thresh_expr, thresh)
n_expr = sym.solve(eqn,
n)[0].subs(base,
(1 - alpha)).subs(alpha, (2 / (span + 1)))
eqn = both_sides(eqn, lambda x: sym.log(x, (1 - alpha)))
lhs = eqn.lhs
from sympy.solvers.inequalities import solve_univariate_inequality
def eval_expr(span_value, n_value):
return np.array(
[thresh_expr.subs(span, span_value).subs(n, n_) for n_ in n_value],
dtype=np.float)
eval_expr(20, np.arange(20))
def linear(x, a, b):
return a * x + b
def sigmoidal_4pl(x, a, b, c, d):
return d + (a - d) / (1 + (x / c)**b)
def exponential(x, a, b, c):
return a + b * np.exp(-c * x)
import scipy.optimize
# Determine how to choose span, such that you get to .01 from 1
# in n timesteps
thresh_to_span_to_n = []
thresh_to_n_to_span = []
for thresh_value in ub.ProgIter([.0001, .001, .01, .1, .2, .3, .4, .5]):
print('')
test_vals = sorted([2, 3, 4, 5, 6])
n_to_span = []
for n_value in ub.ProgIter(test_vals):
# In n iterations I want to choose a span that the expression go
# less than a threshold
constraint = thresh_expr.subs(n, n_value) < thresh_value
solution = solve_univariate_inequality(constraint, span)
try:
lowbound = np.ceil(float(solution.args[0].lhs))
highbound = np.floor(float(solution.args[1].rhs))
assert lowbound <= highbound
span_value = lowbound
except AttributeError:
span_value = np.floor(float(solution.rhs))
n_to_span.append((n_value, span_value))
# Given a threshold, find a minimum number of steps
# that brings you up to that threshold given a span
test_vals = sorted(set(list(range(2, 1000, 50)) + [2, 3, 4, 5, 6]))
span_to_n = []
for span_value in ub.ProgIter(test_vals):
constraint = thresh_expr.subs(span, span_value) < thresh_value
solution = solve_univariate_inequality(constraint, n)
n_value = solution.lhs
span_to_n.append((span_value, n_value))
thresh_to_n_to_span.append((thresh_value, n_to_span))
thresh_to_span_to_n.append((thresh_value, span_to_n))
thresh_to_params = []
for thresh_value, span_to_n in thresh_to_span_to_n:
xdata, ydata = [np.array(_, dtype=np.float) for _ in zip(*span_to_n)]
p0 = (1 / np.diff((ydata - ydata[0])[1:]).mean(), ydata[0])
func = linear
popt, pcov = scipy.optimize.curve_fit(func, xdata, ydata, p0)
# popt, pcov = scipy.optimize.curve_fit(exponential, xdata, ydata)
if False:
yhat = func(xdata, *popt)
pt.figure(fnum=1, doclf=True)
pt.plot(xdata, ydata, label='measured')
pt.plot(xdata, yhat, label='predicteed')
pt.legend()
# slope = np.diff(ydata).mean()
# pt.plot(d)
thresh_to_params.append((thresh_value, popt))
# pt.plt.plot(*zip(*thresh_to_slope), 'x-')
# for thresh_value=.01, we get a rough line with slop ~2.302,
# for thresh_value=.5, we get a line with slop ~34.66
# if we want to get to 0 in n timesteps, with a thresh_value of
# choose span=f(thresh_value) * (n + 2))
# f is some inverse exponential
# 0.0001, 460.551314197147
# 0.001, 345.413485647860,
# 0.01, 230.275657098573,
# 0.1, 115.137828549287,
# 0.2, 80.4778885203347,
# 0.3, 60.2031233261536,
# 0.4, 45.8179484913827,
# 0.5, 34.6599400289520
# Seems to be 4PL symetrical sigmoid
# f(x) = -66500.85 + (66515.88 - -66500.85) / (1 + (x/0.8604672)^0.001503716)
# f(x) = -66500.85 + (66515.88 - -66500.85)/(1 + (x/0.8604672)^0.001503716)
def f(x):
return -66500.85 + (66515.88 -
-66500.85) / (1 + (x / 0.8604672)**0.001503716)
# return (10000 * (-6.65 + (13.3015) / (1 + (x/0.86) ** 0.00150)))
# f(.5) * (n - 1)
# f(
solve_rational_inequalities(thresh_expr < .01, n)
def iters_until_threshold():
"""
How many iterations of ewma until you hit the poisson / biniomal threshold
This establishes a principled way to choose the threshold for the refresh
criterion in my thesis. There are paramters --- moving parts --- that we
need to work with: `a` the patience, `s` the span, and `mu` our ewma.
`s` is a span paramter indicating how far we look back.
`mu` is the average number of label-changing reviews in roughly the last
`s` manual decisions.
These numbers are used to estimate the probability that any of the next `a`
manual decisions will be label-chanigng. When that probability falls below
a threshold we terminate. The goal is to choose `a`, `s`, and the threshold
`t`, such that the probability will fall below the threshold after a maximum
of `a` consecutive non-label-chaning reviews. IE we want to tie the patience
paramter (how far we look ahead) to how far we actually are willing to go.
"""
import numpy as np
import utool as ut
import sympy as sym
i = sym.symbols('i', integer=True, nonnegative=True, finite=True)
# mu_i = sym.symbols('mu_i', integer=True, nonnegative=True, finite=True)
s = sym.symbols('s', integer=True, nonnegative=True, finite=True) # NOQA
thresh = sym.symbols('tau', real=True, nonnegative=True,
finite=True) # NOQA
alpha = sym.symbols('alpha', real=True, nonnegative=True,
finite=True) # NOQA
c_alpha = sym.symbols('c_alpha', real=True, nonnegative=True, finite=True)
# patience
a = sym.symbols('a', real=True, nonnegative=True, finite=True)
available_subs = {
a: 20,
s: a,
alpha: 2 / (s + 1),
c_alpha: (1 - alpha),
}
def dosubs(expr, d=available_subs):
""" recursive expression substitution """
expr1 = expr.subs(d)
if expr == expr1:
return expr1
else:
return dosubs(expr1, d=d)
# mu is either the support for the poisson distribution
# or is is the p in the binomial distribution
# It is updated at timestep i based on ewma, assuming each incoming responce is 0
mu_0 = 1.0
mu_i = c_alpha**i
# Estimate probability that any event will happen in the next `a` reviews
# at time `i`.
poisson_i = 1 - sym.exp(-mu_i * a)
binom_i = 1 - (1 - mu_i)**a
# Expand probabilities to be a function of i, s, and a
part = ut.delete_dict_keys(available_subs.copy(), [a, s])
mu_i = dosubs(mu_i, d=part)
poisson_i = dosubs(poisson_i, d=part)
binom_i = dosubs(binom_i, d=part)
if True:
# ewma of mu at time i if review is always not label-changing (meaningful)
mu_1 = c_alpha * mu_0 # NOQA
mu_2 = c_alpha * mu_1 # NOQA
if True:
i_vals = np.arange(0, 100)
mu_vals = np.array(
[dosubs(mu_i).subs({
i: i_
}).evalf() for i_ in i_vals]) # NOQA
binom_vals = np.array(
[dosubs(binom_i).subs({
i: i_
}).evalf() for i_ in i_vals]) # NOQA
poisson_vals = np.array(
[dosubs(poisson_i).subs({
i: i_
}).evalf() for i_ in i_vals]) # NOQA
# Find how many iters it actually takes my expt to terminate
thesis_draft_thresh = np.exp(-2)
np.where(mu_vals < thesis_draft_thresh)[0]
np.where(binom_vals < thesis_draft_thresh)[0]
np.where(poisson_vals < thesis_draft_thresh)[0]
sym.pprint(sym.simplify(mu_i))
sym.pprint(sym.simplify(binom_i))
sym.pprint(sym.simplify(poisson_i))
# Find the thresholds that force termination after `a` reviews have passed
# do this by setting i=a
poisson_thresh = poisson_i.subs({i: a})
binom_thresh = binom_i.subs({i: a})
print('Poisson thresh')
print(sym.latex(sym.Eq(thresh, poisson_thresh)))
print(sym.latex(sym.Eq(thresh, sym.simplify(poisson_thresh))))
poisson_thresh.subs({a: 115, s: 30}).evalf()
sym.pprint(sym.Eq(thresh, poisson_thresh))
sym.pprint(sym.Eq(thresh, sym.simplify(poisson_thresh)))
print('Binomial thresh')
sym.pprint(sym.simplify(binom_thresh))
sym.pprint(sym.simplify(poisson_thresh.subs({s: a})))
def taud(coeff):
return coeff * 360
if 'poisson_cache' not in vars():
poisson_cache = {}
binom_cache = {}
S, A = np.meshgrid(np.arange(1, 150, 1), np.arange(0, 150, 1))
import plottool as pt
SA_coords = list(zip(S.ravel(), A.ravel()))
for sval, aval in ut.ProgIter(SA_coords):
if (sval, aval) not in poisson_cache:
poisson_cache[(sval, aval)] = float(
poisson_thresh.subs({
a: aval,
s: sval
}).evalf())
poisson_zdata = np.array([
poisson_cache[(sval, aval)] for sval, aval in SA_coords
]).reshape(A.shape)
fig = pt.figure(fnum=1, doclf=True)
pt.gca().set_axis_off()
pt.plot_surface3d(S,
A,
poisson_zdata,
xlabel='s',
ylabel='a',
rstride=3,
cstride=3,
zlabel='poisson',
mode='wire',
contour=True,
title='poisson3d')
pt.gca().set_zlim(0, 1)
pt.gca().view_init(elev=taud(1 / 16), azim=taud(5 / 8))
fig.set_size_inches(10, 6)
fig.savefig('a-s-t-poisson3d.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
for sval, aval in ut.ProgIter(SA_coords):
if (sval, aval) not in binom_cache:
binom_cache[(sval, aval)] = float(
binom_thresh.subs({
a: aval,
s: sval
}).evalf())
binom_zdata = np.array([
binom_cache[(sval, aval)] for sval, aval in SA_coords
]).reshape(A.shape)
fig = pt.figure(fnum=2, doclf=True)
pt.gca().set_axis_off()
pt.plot_surface3d(S,
A,
binom_zdata,
xlabel='s',
ylabel='a',
rstride=3,
cstride=3,
zlabel='binom',
mode='wire',
contour=True,
title='binom3d')
pt.gca().set_zlim(0, 1)
pt.gca().view_init(elev=taud(1 / 16), azim=taud(5 / 8))
fig.set_size_inches(10, 6)
fig.savefig('a-s-t-binom3d.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
# Find point on the surface that achieves a reasonable threshold
# Sympy can't solve this
# sym.solve(sym.Eq(binom_thresh.subs({s: 50}), .05))
# sym.solve(sym.Eq(poisson_thresh.subs({s: 50}), .05))
# Find a numerical solution
def solve_numeric(expr,
target,
solve_for,
fixed={},
method=None,
bounds=None):
"""
Args:
expr (Expr): symbolic expression
target (float): numberic value
solve_for (sympy.Symbol): The symbol you care about
fixed (dict): fixed values of the symbol
solve_numeric(poisson_thresh, .05, {s: 30}, method=None)
solve_numeric(poisson_thresh, .05, {s: 30}, method='Nelder-Mead')
solve_numeric(poisson_thresh, .05, {s: 30}, method='BFGS')
"""
import scipy.optimize
# Find the symbol you want to solve for
want_symbols = expr.free_symbols - set(fixed.keys())
# TODO: can probably extend this to multiple params
assert len(want_symbols) == 1, 'specify all but one var'
assert solve_for == list(want_symbols)[0]
fixed_expr = expr.subs(fixed)
def func(a1):
expr_value = float(fixed_expr.subs({solve_for: a1}).evalf())
return (expr_value - target)**2
if not fixed:
a1 = 0
else:
a1 = list(fixed.values())[0]
# if method is None:
# method = 'Nelder-Mead'
# method = 'Newton-CG'
# method = 'BFGS'
result = scipy.optimize.minimize(func,
x0=a1,
method=method,
bounds=bounds)
if not result.success:
print('\n')
print(result)
print('\n')
return result
# Numeric measurments of thie line
thresh_vals = [.001, .01, .05, .1, .135]
svals = np.arange(1, 100)
target_poisson_plots = {}
for target in ut.ProgIter(thresh_vals, bs=False, freq=1):
poisson_avals = []
for sval in ut.ProgIter(svals, 'poisson', freq=1):
expr = poisson_thresh
fixed = {s: sval}
want = a
aval = solve_numeric(expr,
target,
want,
fixed,
method='Nelder-Mead').x[0]
poisson_avals.append(aval)
target_poisson_plots[target] = (svals, poisson_avals)
fig = pt.figure(fnum=3)
for target, dat in target_poisson_plots.items():
pt.plt.plot(*dat, label='prob={}'.format(target))
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('a')
pt.legend()
pt.gca().set_title('poisson')
fig.set_size_inches(5, 3)
fig.savefig('a-vs-s-poisson.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
target_binom_plots = {}
for target in ut.ProgIter(thresh_vals, bs=False, freq=1):
binom_avals = []
for sval in ut.ProgIter(svals, 'binom', freq=1):
aval = solve_numeric(binom_thresh,
target,
a, {
s: sval
},
method='Nelder-Mead').x[0]
binom_avals.append(aval)
target_binom_plots[target] = (svals, binom_avals)
fig = pt.figure(fnum=4)
for target, dat in target_binom_plots.items():
pt.plt.plot(*dat, label='prob={}'.format(target))
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('a')
pt.legend()
pt.gca().set_title('binom')
fig.set_size_inches(5, 3)
fig.savefig('a-vs-s-binom.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
# ----
if True:
fig = pt.figure(fnum=5, doclf=True)
s_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for sval in s_vals:
pp = poisson_thresh.subs({s: sval})
a_vals = np.arange(0, 200)
pp_vals = np.array(
[float(pp.subs({
a: aval
}).evalf()) for aval in a_vals]) # NOQA
pt.plot(a_vals, pp_vals, label='s=%r' % (sval, ))
pt.legend()
pt.gca().set_xlabel('a')
pt.gca().set_ylabel('poisson prob after a reviews')
fig.set_size_inches(5, 3)
fig.savefig('a-vs-thresh-poisson.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
fig = pt.figure(fnum=6, doclf=True)
s_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for sval in s_vals:
pp = binom_thresh.subs({s: sval})
a_vals = np.arange(0, 200)
pp_vals = np.array(
[float(pp.subs({
a: aval
}).evalf()) for aval in a_vals]) # NOQA
pt.plot(a_vals, pp_vals, label='s=%r' % (sval, ))
pt.legend()
pt.gca().set_xlabel('a')
pt.gca().set_ylabel('binom prob after a reviews')
fig.set_size_inches(5, 3)
fig.savefig('a-vs-thresh-binom.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
# -------
fig = pt.figure(fnum=5, doclf=True)
a_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for aval in a_vals:
pp = poisson_thresh.subs({a: aval})
s_vals = np.arange(1, 200)
pp_vals = np.array(
[float(pp.subs({
s: sval
}).evalf()) for sval in s_vals]) # NOQA
pt.plot(s_vals, pp_vals, label='a=%r' % (aval, ))
pt.legend()
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('poisson prob')
fig.set_size_inches(5, 3)
fig.savefig('s-vs-thresh-poisson.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
fig = pt.figure(fnum=5, doclf=True)
a_vals = [1, 2, 3, 10, 20, 30, 40, 50]
for aval in a_vals:
pp = binom_thresh.subs({a: aval})
s_vals = np.arange(1, 200)
pp_vals = np.array(
[float(pp.subs({
s: sval
}).evalf()) for sval in s_vals]) # NOQA
pt.plot(s_vals, pp_vals, label='a=%r' % (aval, ))
pt.legend()
pt.gca().set_xlabel('s')
pt.gca().set_ylabel('binom prob')
fig.set_size_inches(5, 3)
fig.savefig('s-vs-thresh-binom.png',
dpi=300,
bbox_inches=pt.extract_axes_extents(fig, combine=True))
#---------------------
# Plot out a table
mu_i.subs({s: 75, a: 75}).evalf()
poisson_thresh.subs({s: 75, a: 75}).evalf()
sval = 50
for target, dat in target_poisson_plots.items():
slope = np.median(np.diff(dat[1]))
aval = int(np.ceil(sval * slope))
thresh = float(poisson_thresh.subs({s: sval, a: aval}).evalf())
print('aval={}, sval={}, thresh={}, target={}'.format(
aval, sval, thresh, target))
for target, dat in target_binom_plots.items():
slope = np.median(np.diff(dat[1]))
aval = int(np.ceil(sval * slope))
pass
def flann_add_time_experiment():
"""
builds plot of number of annotations vs indexer build time.
TODO: time experiment
CommandLine:
python -m ibeis.algo.hots._neighbor_experiment --test-flann_add_time_experiment --db PZ_MTEST --show
python -m ibeis.algo.hots._neighbor_experiment --test-flann_add_time_experiment --db PZ_Master0 --show
utprof.py -m ibeis.algo.hots._neighbor_experiment --test-flann_add_time_experiment --show
valgrind --tool=memcheck --suppressions=valgrind-python.supp python -m ibeis.algo.hots._neighbor_experiment --test-flann_add_time_experiment --db PZ_MTEST --no-with-reindex
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots._neighbor_experiment import * # NOQA
>>> import ibeis
>>> #ibs = ibeis.opendb('PZ_MTEST')
>>> result = flann_add_time_experiment()
>>> # verify results
>>> print(result)
>>> ut.show_if_requested()
"""
import ibeis
import utool as ut
import numpy as np
import plottool as pt
def make_flann_index(vecs, flann_params):
flann = pyflann.FLANN()
flann.build_index(vecs, **flann_params)
return flann
db = ut.get_argval('--db')
ibs = ibeis.opendb(db=db)
# Input
if ibs.get_dbname() == 'PZ_MTEST':
initial = 1
reindex_stride = 16
addition_stride = 4
max_ceiling = 120
elif ibs.get_dbname() == 'PZ_Master0':
#ibs = ibeis.opendb(db='GZ_ALL')
initial = 32
reindex_stride = 32
addition_stride = 16
max_ceiling = 300001
else:
assert False
#max_ceiling = 32
all_daids = ibs.get_valid_aids()
max_num = min(max_ceiling, len(all_daids))
flann_params = ibs.cfg.query_cfg.flann_cfg.get_flann_params()
# Output
count_list, time_list_reindex = [], []
count_list2, time_list_addition = [], []
# Setup
#all_randomize_daids_ = ut.deterministic_shuffle(all_daids[:])
all_randomize_daids_ = all_daids
# ensure all features are computed
ibs.get_annot_vecs(all_randomize_daids_)
def reindex_step(count, count_list, time_list_reindex):
daids = all_randomize_daids_[0:count]
vecs = np.vstack(ibs.get_annot_vecs(daids))
with ut.Timer(verbose=False) as t:
flann = make_flann_index(vecs, flann_params) # NOQA
count_list.append(count)
time_list_reindex.append(t.ellapsed)
def addition_step(count, flann, count_list2, time_list_addition):
daids = all_randomize_daids_[count:count + 1]
vecs = np.vstack(ibs.get_annot_vecs(daids))
with ut.Timer(verbose=False) as t:
flann.add_points(vecs)
count_list2.append(count)
time_list_addition.append(t.ellapsed)
def make_initial_index(initial):
daids = all_randomize_daids_[0:initial + 1]
vecs = np.vstack(ibs.get_annot_vecs(daids))
flann = make_flann_index(vecs, flann_params)
return flann
WITH_REINDEX = not ut.get_argflag('--no-with-reindex')
if WITH_REINDEX:
# Reindex Part
reindex_lbl = 'Reindexing'
_reindex_iter = range(1, max_num, reindex_stride)
reindex_iter = ut.ProgressIter(_reindex_iter, lbl=reindex_lbl, freq=1)
for count in reindex_iter:
reindex_step(count, count_list, time_list_reindex)
# Add Part
flann = make_initial_index(initial)
addition_lbl = 'Addition'
_addition_iter = range(initial + 1, max_num, addition_stride)
addition_iter = ut.ProgressIter(_addition_iter, lbl=addition_lbl)
for count in addition_iter:
addition_step(count, flann, count_list2, time_list_addition)
print('---')
print('Reindex took time_list_reindex %.2s seconds' % sum(time_list_reindex))
print('Addition took time_list_reindex %.2s seconds' % sum(time_list_addition))
print('---')
statskw = dict(precision=2, newlines=True)
print('Reindex stats ' + ut.get_stats_str(time_list_reindex, **statskw))
print('Addition stats ' + ut.get_stats_str(time_list_addition, **statskw))
print('Plotting')
#with pt.FigureContext:
next_fnum = iter(range(0, 2)).next # python3 PY3
pt.figure(fnum=next_fnum())
if WITH_REINDEX:
pt.plot2(count_list, time_list_reindex, marker='-o', equal_aspect=False,
x_label='num_annotations', label=reindex_lbl + ' Time', dark=False)
#pt.figure(fnum=next_fnum())
pt.plot2(count_list2, time_list_addition, marker='-o', equal_aspect=False,
x_label='num_annotations', label=addition_lbl + ' Time')
pt
pt.legend()
def augment_nnindexer_experiment():
"""
References:
http://answers.opencv.org/question/44592/flann-index-training-fails-with-segfault/
CommandLine:
utprof.py -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_MTEST --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 --nosave-flann --show
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 --nosave-flann --show
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 --nosave-flann --no-api-cache --nocache-uuids
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_MTEST --show
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --show
# RUNS THE SEGFAULTING CASE
python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --show
# Debug it
gdb python
run -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --show
gdb python
run -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots._neighbor_experiment import * # NOQA
>>> # execute function
>>> augment_nnindexer_experiment()
>>> # verify results
>>> ut.show_if_requested()
"""
import ibeis
# build test data
#ibs = ibeis.opendb('PZ_MTEST')
ibs = ibeis.opendb(defaultdb='PZ_Master0')
if ibs.get_dbname() == 'PZ_MTEST':
initial = 1
addition_stride = 4
max_ceiling = 100
elif ibs.get_dbname() == 'PZ_Master0':
initial = 128
#addition_stride = 64
#addition_stride = 128
addition_stride = 256
max_ceiling = 10000
#max_ceiling = 4000
#max_ceiling = 2000
#max_ceiling = 600
else:
assert False
all_daids = ibs.get_valid_aids(species='zebra_plains')
qreq_ = ibs.new_query_request(all_daids, all_daids)
max_num = min(max_ceiling, len(all_daids))
# Clear Caches
ibs.delete_flann_cachedir()
neighbor_index_cache.clear_memcache()
neighbor_index_cache.clear_uuid_cache(qreq_)
# Setup
all_randomize_daids_ = ut.deterministic_shuffle(all_daids[:])
# ensure all features are computed
#ibs.get_annot_vecs(all_randomize_daids_, ensure=True)
#ibs.get_annot_fgweights(all_randomize_daids_, ensure=True)
nnindexer_list = []
addition_lbl = 'Addition'
_addition_iter = list(range(initial + 1, max_num, addition_stride))
addition_iter = iter(ut.ProgressIter(_addition_iter, lbl=addition_lbl,
freq=1, autoadjust=False))
time_list_addition = []
#time_list_reindex = []
addition_count_list = []
tmp_cfgstr_list = []
#for _ in range(80):
# next(addition_iter)
try:
memtrack = ut.MemoryTracker(disable=False)
for count in addition_iter:
aid_list_ = all_randomize_daids_[0:count]
# Request an indexer which could be an augmented version of an existing indexer.
with ut.Timer(verbose=False) as t:
memtrack.report('BEFORE AUGMENT')
nnindexer_ = neighbor_index_cache.request_augmented_ibeis_nnindexer(qreq_, aid_list_)
memtrack.report('AFTER AUGMENT')
nnindexer_list.append(nnindexer_)
addition_count_list.append(count)
time_list_addition.append(t.ellapsed)
tmp_cfgstr_list.append(nnindexer_.cfgstr)
print('===============\n\n')
print(ut.list_str(time_list_addition))
print(ut.list_str(list(map(id, nnindexer_list))))
print(ut.list_str(tmp_cfgstr_list))
print(ut.list_str(list([nnindxer.cfgstr for nnindxer in nnindexer_list])))
IS_SMALL = False
if IS_SMALL:
nnindexer_list = []
reindex_label = 'Reindex'
# go backwards for reindex
_reindex_iter = list(range(initial + 1, max_num, addition_stride))[::-1]
reindex_iter = ut.ProgressIter(_reindex_iter, lbl=reindex_label)
time_list_reindex = []
#time_list_reindex = []
reindex_count_list = []
for count in reindex_iter:
print('\n+===PREDONE====================\n')
# check only a single size for memory leaks
#count = max_num // 16 + ((x % 6) * 1)
#x += 1
aid_list_ = all_randomize_daids_[0:count]
# Call the same code, but force rebuilds
memtrack.report('BEFORE REINDEX')
with ut.Timer(verbose=False) as t:
nnindexer_ = neighbor_index_cache.request_augmented_ibeis_nnindexer(
qreq_, aid_list_, force_rebuild=True, memtrack=memtrack)
memtrack.report('AFTER REINDEX')
ibs.print_cachestats_str()
print('[nnindex.MEMCACHE] size(NEIGHBOR_CACHE) = %s' % (
ut.get_object_size_str(neighbor_index_cache.NEIGHBOR_CACHE.items()),))
print('[nnindex.MEMCACHE] len(NEIGHBOR_CACHE) = %s' % (
len(neighbor_index_cache.NEIGHBOR_CACHE.items()),))
print('[nnindex.MEMCACHE] size(UUID_MAP_CACHE) = %s' % (
ut.get_object_size_str(neighbor_index_cache.UUID_MAP_CACHE),))
print('totalsize(nnindexer) = ' + ut.get_object_size_str(nnindexer_))
memtrack.report_type(neighbor_index_cache.NeighborIndex)
ut.print_object_size_tree(nnindexer_, lbl='nnindexer_')
if IS_SMALL:
nnindexer_list.append(nnindexer_)
reindex_count_list.append(count)
time_list_reindex.append(t.ellapsed)
#import cv2
#import matplotlib as mpl
#print(mem_top.mem_top(limit=30, width=120,
# #exclude_refs=[cv2.__dict__, mpl.__dict__]
# ))
print('L___________________\n\n\n')
print(ut.list_str(time_list_reindex))
if IS_SMALL:
print(ut.list_str(list(map(id, nnindexer_list))))
print(ut.list_str(list([nnindxer.cfgstr for nnindxer in nnindexer_list])))
except KeyboardInterrupt:
print('\n[train] Caught CRTL+C')
resolution = ''
from six.moves import input
while not (resolution.isdigit()):
print('\n[train] What do you want to do?')
print('[train] 0 - Continue')
print('[train] 1 - Embed')
print('[train] ELSE - Stop network training')
resolution = input('[train] Resolution: ')
resolution = int(resolution)
# We have a resolution
if resolution == 0:
print('resuming training...')
elif resolution == 1:
ut.embed()
import plottool as pt
next_fnum = iter(range(0, 1)).next # python3 PY3
pt.figure(fnum=next_fnum())
if len(addition_count_list) > 0:
pt.plot2(addition_count_list, time_list_addition, marker='-o', equal_aspect=False,
x_label='num_annotations', label=addition_lbl + ' Time')
if len(reindex_count_list) > 0:
pt.plot2(reindex_count_list, time_list_reindex, marker='-o', equal_aspect=False,
x_label='num_annotations', label=reindex_label + ' Time')
pt.set_figtitle('Augmented indexer experiment')
pt.legend()
def in_depth_ellipse(kp):
"""
Makes sure that I understand how the ellipse is created form a keypoint
representation. Walks through the steps I took in coming to an
understanding.
CommandLine:
python -m pyhesaff.tests.test_ellipse --test-in_depth_ellipse --show --num-samples=12
Example:
>>> # SCRIPT
>>> from pyhesaff.tests.test_ellipse import * # NOQA
>>> import pyhesaff.tests.pyhestest as pyhestest
>>> test_data = pyhestest.load_test_data(short=True)
>>> kpts = test_data['kpts']
>>> kp = kpts[0]
>>> #kp = np.array([0, 0, 10, 10, 10, 0])
>>> test_locals = in_depth_ellipse(kp)
>>> ut.quit_if_noshow()
>>> ut.show_if_requested()
"""
import plottool as pt
#nSamples = 12
nSamples = ut.get_argval('--num-samples', type_=int, default=12)
kp = np.array(kp, dtype=np.float64)
#-----------------------
# SETUP
#-----------------------
np.set_printoptions(precision=3)
#pt.reset()
pt.figure(9003, docla=True, doclf=True)
ax = pt.gca()
ax.invert_yaxis()
def _plotpts(data, px, color=pt.BLUE, label='', marker='.', **kwargs):
#pt.figure(9003, docla=True, pnum=(1, 1, px))
pt.plot2(data.T[0],
data.T[1],
marker,
'',
color=color,
label=label,
**kwargs)
#pt.update()
def _plotarrow(x, y, dx, dy, color=pt.BLUE, label=''):
ax = pt.gca()
arrowargs = dict(head_width=.5, length_includes_head=True, label=label)
arrow = mpl.patches.FancyArrow(x, y, dx, dy, **arrowargs)
arrow.set_edgecolor(color)
arrow.set_facecolor(color)
ax.add_patch(arrow)
#pt.update()
#-----------------------
# INPUT
#-----------------------
print('kp = %s' % ut.repr2(kp, precision=3))
print('--------------------------------')
print('Let V = Perdoch.A')
print('Let Z = Perdoch.E')
print('Let invV = Perdoch.invA')
print('--------------------------------')
print('Input from Perdoch\'s detector: ')
# We are given the keypoint in invA format
if len(kp) == 5:
(ix, iy, iv11, iv21, iv22) = kp
iv12 = 0
elif len(kp) == 6:
(ix, iy, iv11, iv21, iv22, ori) = kp
iv12 = 0
invV = np.array([[iv11, iv12, ix], [iv21, iv22, iy], [0, 0, 1]])
V = np.linalg.inv(invV)
Z = (V.T).dot(V)
import vtool as vt
V_2x2 = V[0:2, 0:2]
Z_2x2 = Z[0:2, 0:2]
V_2x2_ = vt.decompose_Z_to_V_2x2(Z_2x2)
assert np.all(np.isclose(V_2x2, V_2x2_))
#C = np.linalg.cholesky(Z)
#np.isclose(C.dot(C.T), Z)
#Z
print('invV is a transform from points on a unit-circle to the ellipse')
ut.horiz_print('invV = ', invV)
print('--------------------------------')
print('V is a transformation from points on the ellipse to a unit circle')
ut.horiz_print('V = ', V)
print('--------------------------------')
print('An ellipse is a special case of a conic. For any ellipse:')
print(
'Points on the ellipse satisfy (x_ - x_0).T.dot(Z).dot(x_ - x_0) = 1')
print('where Z = (V.T).dot(V)')
ut.horiz_print('Z = ', Z)
# Define points on a unit circle
theta_list = np.linspace(0, TAU, nSamples)
cicrle_pts = np.array([(np.cos(t_), np.sin(t_), 1) for t_ in theta_list])
# Transform those points to the ellipse using invV
ellipse_pts1 = invV.dot(cicrle_pts.T).T
#Lets check our assertion: (x_ - x_0).T.dot(Z).dot(x_ - x_0) == 1
x_0 = np.array([ix, iy, 1])
checks = [(x_ - x_0).T.dot(Z).dot(x_ - x_0) for x_ in ellipse_pts1]
try:
# HELP: The phase is off here. in 3x3 version I'm not sure why
#assert all([almost_eq(1, check) for check in checks1])
is_almost_eq_pos1 = [ut.almost_eq(1, check) for check in checks]
is_almost_eq_neg1 = [ut.almost_eq(-1, check) for check in checks]
assert all(is_almost_eq_pos1)
except AssertionError as ex:
print('circle pts = %r ' % cicrle_pts)
print(ex)
print(checks)
print([ut.almost_eq(-1, check, 1E-9) for check in checks])
raise
else:
#assert all([abs(1 - check) < 1E-11 for check in checks2])
print('... all of our plotted points satisfy this')
#=======================
# THE CONIC SECTION
#=======================
# All of this was from the Perdoch paper, now lets move into conic sections
# We will use the notation from wikipedia
# References:
# http://en.wikipedia.org/wiki/Conic_section
# http://en.wikipedia.org/wiki/Matrix_representation_of_conic_sections
#-----------------------
# MATRIX REPRESENTATION
#-----------------------
# The matrix representation of a conic is:
#(A, B2, B2_, C) = Z.flatten()
#(D, E, F) = (0, 0, 1)
(A, B2, D2, B2_, C, E2, D2_, E2_, F) = Z.flatten()
B = B2 * 2
D = D2 * 2
E = E2 * 2
assert B2 == B2_, 'matrix should by symmetric'
assert D2 == D2_, 'matrix should by symmetric'
assert E2 == E2_, 'matrix should by symmetric'
print('--------------------------------')
print('Now, using wikipedia\' matrix representation of a conic.')
con = np.array(((' A', 'B / 2', 'D / 2'), ('B / 2', ' C', 'E / 2'),
('D / 2', 'E / 2', ' F')))
ut.horiz_print('A matrix A_Q = ', con)
# A_Q is our conic section (aka ellipse matrix)
A_Q = np.array(((A, B / 2, D / 2), (B / 2, C, E / 2), (D / 2, E / 2, F)))
ut.horiz_print('A_Q = ', A_Q)
#-----------------------
# DEGENERATE CONICS
# References:
# http://individual.utoronto.ca/somody/quiz.html
print('----------------------------------')
print('As long as det(A_Q) != it is not degenerate.')
print('If the conic is not degenerate, we can use the 2x2 minor: A_33')
print('det(A_Q) = %s' % str(np.linalg.det(A_Q)))
assert np.linalg.det(A_Q) != 0, 'degenerate conic'
A_33 = np.array(((A, B / 2), (B / 2, C)))
ut.horiz_print('A_33 = ', A_33)
#-----------------------
# CONIC CLASSIFICATION
#-----------------------
print('----------------------------------')
print('The determinant of the minor classifies the type of conic it is')
print('(det == 0): parabola, (det < 0): hyperbola, (det > 0): ellipse')
print('det(A_33) = %s' % str(np.linalg.det(A_33)))
assert np.linalg.det(A_33) > 0, 'conic is not an ellipse'
print('... this is indeed an ellipse')
#-----------------------
# CONIC CENTER
#-----------------------
print('----------------------------------')
print('the centers of the ellipse are obtained by: ')
print('x_center = (B * E - (2 * C * D)) / (4 * A * C - B ** 2)')
print('y_center = (D * B - (2 * A * E)) / (4 * A * C - B ** 2)')
# Centers are obtained by solving for where the gradient of the quadratic
# becomes 0. Without going through the derivation the calculation is...
# These should be 0, 0 if we are at the origin, or our original x, y
# coordinate specified by the keypoints. I'm doing the calculation just for
# shits and giggles
x_center = (B * E - (2 * C * D)) / (4 * A * C - B**2)
y_center = (D * B - (2 * A * E)) / (4 * A * C - B**2)
ut.horiz_print('x_center = ', x_center)
ut.horiz_print('y_center = ', y_center)
#-----------------------
# MAJOR AND MINOR AXES
#-----------------------
# Now we are going to determine the major and minor axis
# of this beast. It just the center augmented by the eigenvecs
print('----------------------------------')
# Plot ellipse axis
# !HELP! I DO NOT KNOW WHY I HAVE TO DIVIDE, SQUARE ROOT, AND NEGATE!!!
(evals, evecs) = np.linalg.eig(A_33)
l1, l2 = evals
# The major and minor axis lengths
b = 1 / np.sqrt(l1)
a = 1 / np.sqrt(l2)
v1, v2 = evecs
# Find the transformation to align the axis
nminor = v1
nmajor = v2
dx1, dy1 = (v1 * b)
dx2, dy2 = (v2 * a)
minor = np.array([dx1, -dy1])
major = np.array([dx2, -dy2])
x_axis = np.array([[1], [0]])
cosang = (x_axis.T.dot(nmajor)).T
# Rotation angle
theta = np.arccos(cosang)
print('a = ' + str(a))
print('b = ' + str(b))
print('theta = ' + str(theta[0] / TAU) + ' * 2pi')
# The warped eigenvects should have the same magintude
# As the axis lengths
assert ut.almost_eq(a, major.dot(ltool.rotation_mat2x2(theta))[0])
assert ut.almost_eq(b, minor.dot(ltool.rotation_mat2x2(theta))[1])
try:
# HACK
if len(theta) == 1:
theta = theta[0]
except Exception:
pass
#-----------------------
# ECCENTRICITY
#-----------------------
print('----------------------------------')
print('The eccentricity is determined by:')
print('')
print(' (2 * np.sqrt((A - C) ** 2 + B ** 2)) ')
print('ecc = -----------------------------------------------')
print(' (nu * (A + C) + np.sqrt((A - C) ** 2 + B ** 2))')
print('')
print('(nu is always 1 for ellipses)')
nu = 1
ecc_numer = (2 * np.sqrt((A - C)**2 + B**2))
ecc_denom = (nu * (A + C) + np.sqrt((A - C)**2 + B**2))
ecc = np.sqrt(ecc_numer / ecc_denom)
print('ecc = ' + str(ecc))
# Eccentricity is a little easier in axis aligned coordinates
# Make sure they aggree
ecc2 = np.sqrt(1 - (b**2) / (a**2))
assert ut.almost_eq(ecc, ecc2)
#-----------------------
# APPROXIMATE UNIFORM SAMPLING
#-----------------------
# We are given the keypoint in invA format
print('----------------------------------')
print('Approximate uniform points an inscribed polygon bondary')
#def next_xy(x, y, d):
# # References:
# # http://gamedev.stackexchange.com/questions/1692/what-is-a-simple-algorithm-for-calculating-evenly-distributed-points-on-an-ellip
# num = (b ** 2) * (x ** 2)
# den = ((a ** 2) * ((a ** 2) - (x ** 2)))
# dxdenom = np.sqrt(1 + (num / den))
# deltax = d / dxdenom
# x_ = x + deltax
# y_ = b * np.sqrt(1 - (x_ ** 2) / (a ** 2))
# return x_, y_
def xy_fn(t):
return np.array((a * np.cos(t), b * np.sin(t))).T
#nSamples = 16
#(ix, iy, iv11, iv21, iv22), iv12 = kp, 0
#invV = np.array([[iv11, iv12, ix],
# [iv21, iv22, iy],
# [ 0, 0, 1]])
#theta_list = np.linspace(0, TAU, nSamples)
#cicrle_pts = np.array([(np.cos(t_), np.sin(t_), 1) for t_ in theta_list])
uneven_points = invV.dot(cicrle_pts.T).T[:, 0:2]
#uneven_points2 = xy_fn(theta_list)
def circular_distance(arr):
dist_most_ = ((arr[0:-1] - arr[1:])**2).sum(1)
dist_end_ = ((arr[-1] - arr[0])**2).sum()
return np.sqrt(np.hstack((dist_most_, dist_end_)))
# Calculate the distance from each point on the ellipse to the next
dists = circular_distance(uneven_points)
total_dist = dists.sum()
# Get an even step size
multiplier = 1
step_size = total_dist / (nSamples * multiplier)
# Walk along edge
num_steps_list = []
offset_list = []
dist_walked = 0
total_dist = step_size
for count in range(len(dists)):
segment_len = dists[count]
# Find where your starting location is
offset_list.append(total_dist - dist_walked)
# How far can you possibly go?
total_dist += segment_len
# How many steps can you take?
num_steps = int((total_dist - dist_walked) // step_size)
num_steps_list.append(num_steps)
# Log how much further youve gotten
dist_walked += (num_steps * step_size)
#print('step_size = %r' % step_size)
#print(np.vstack((num_steps_list, dists, offset_list)).T)
# store the percent location at each line segment where
# the cut will be made
cut_list = []
for num, dist, offset in zip(num_steps_list, dists, offset_list):
if num == 0:
cut_list.append([])
continue
offset1 = (step_size - offset) / dist
offset2 = ((num * step_size) - offset) / dist
cut_locs = (np.linspace(offset1, offset2, num, endpoint=True))
cut_list.append(cut_locs)
#print(cut_locs)
# Cut the segments into new better segments
approx_pts = []
nPts = len(uneven_points)
for count, cut_locs in enumerate(cut_list):
for loc in cut_locs:
pt1 = uneven_points[count]
pt2 = uneven_points[(count + 1) % nPts]
# Linearly interpolate between points
new_loc = ((1 - loc) * pt1) + ((loc) * pt2)
approx_pts.append(new_loc)
approx_pts = np.array(approx_pts)
# Warp approx_pts to the unit circle
print('----------------------------------')
print('For each aproximate point, find the closet point on the ellipse')
#new_unit = V.dot(approx_pts.T).T
ones_ = np.ones(len(approx_pts))
new_hlocs = np.vstack((approx_pts.T, ones_))
new_unit = V.dot(new_hlocs).T
# normalize new_unit
new_mag = np.sqrt((new_unit**2).sum(1))
new_unorm_unit = new_unit / np.vstack([new_mag] * 3).T
new_norm_unit = new_unorm_unit / np.vstack([new_unorm_unit[:, 2]] * 3).T
# Get angle (might not be necessary)
x_axis = np.array([1, 0, 0])
arccos_list = x_axis.dot(new_norm_unit.T)
uniform_theta_list = np.arccos(arccos_list)
# Maybe this?
uniform_theta_list = np.arctan2(new_norm_unit[:, 1], new_norm_unit[:, 0])
#
unevn_cicrle_pts = np.array([(np.cos(t_), np.sin(t_), 1)
for t_ in uniform_theta_list])
# This is the output. Approximately uniform points sampled along an ellipse
uniform_ell_pts = invV.dot(unevn_cicrle_pts.T).T
#uniform_ell_pts = invV.dot(new_norm_unit.T).T
_plotpts(approx_pts, 0, pt.YELLOW, label='approx points', marker='o-')
_plotpts(uniform_ell_pts, 0, pt.RED, label='uniform points', marker='o-')
# Desired number of points
#ecc = np.sqrt(1 - (b ** 2) / (a ** 2))
# Total arclength
#total_arclen = ellipeinc(TAU, ecc)
#firstquad_arclen = total_arclen / 4
# Desired arclength between points
#d = firstquad_arclen / nSamples
# Initial point
#x, y = xy_fn(.001)
#uniform_points = []
#for count in range(nSamples):
# if np.isnan(x_) or np.isnan(y_):
# print('nan on count=%r' % count)
# break
# uniform_points.append((x_, y_))
# The angle between the major axis and our x axis is:
#-----------------------
# DRAWING
#-----------------------
print('----------------------------------')
# Draw the keypoint using the tried and true pt
# Other things should subsiquently align
#pt.draw_kpts2(np.array([kp]), ell_linewidth=4,
# ell_color=pt.DEEP_PINK, ell_alpha=1, arrow=True, rect=True)
# Plot ellipse points
_plotpts(ellipse_pts1,
0,
pt.LIGHT_BLUE,
label='invV.dot(cicrle_pts.T).T',
marker='o-')
_plotarrow(x_center,
y_center,
dx1,
-dy1,
color=pt.GRAY,
label='minor axis')
_plotarrow(x_center,
y_center,
dx2,
-dy2,
color=pt.GRAY,
label='major axis')
# Rotate the ellipse so it is axis aligned and plot that
rot = ltool.rotation_around_mat3x3(theta, ix, iy)
ellipse_pts3 = rot.dot(ellipse_pts1.T).T
#!_plotpts(ellipse_pts3, 0, pt.GREEN, label='axis aligned points')
# Plot ellipse orientation
ortho_basis = np.eye(3)[:, 0:2]
orient_axis = invV.dot(ortho_basis)
print(orient_axis)
_dx1, _dx2, _dy1, _dy2, _1, _2 = orient_axis.flatten()
#!_plotarrow(x_center, y_center, _dx1, _dy1, color=pt.BLUE, label='ellipse rotation')
#!_plotarrow(x_center, y_center, _dx2, _dy2, color=pt.BLUE)
#pt.plt.gca().set_xlim(400, 600)
#pt.plt.gca().set_ylim(300, 500)
xmin, ymin = ellipse_pts1.min(0)[0:2] - 1
xmax, ymax = ellipse_pts1.max(0)[0:2] + 1
pt.plt.gca().set_xlim(xmin, xmax)
pt.plt.gca().set_ylim(ymin, ymax)
pt.legend()
pt.dark_background(doubleit=3)
pt.gca().invert_yaxis()
# Hack in another view
# It seems like the even points are not actually that even.
# there must be a bug
pt.figure(fnum=9003 + 1, docla=True, doclf=True, pnum=(1, 3, 1))
_plotpts(ellipse_pts1,
0,
pt.LIGHT_BLUE,
label='invV.dot(cicrle_pts.T).T',
marker='o-',
title='even')
pt.plt.gca().set_xlim(xmin, xmax)
pt.plt.gca().set_ylim(ymin, ymax)
pt.dark_background(doubleit=3)
pt.gca().invert_yaxis()
pt.figure(fnum=9003 + 1, pnum=(1, 3, 2))
_plotpts(approx_pts,
0,
pt.YELLOW,
label='approx points',
marker='o-',
title='approx')
pt.plt.gca().set_xlim(xmin, xmax)
pt.plt.gca().set_ylim(ymin, ymax)
pt.dark_background(doubleit=3)
pt.gca().invert_yaxis()
pt.figure(fnum=9003 + 1, pnum=(1, 3, 3))
_plotpts(uniform_ell_pts,
0,
pt.RED,
label='uniform points',
marker='o-',
title='uniform')
pt.plt.gca().set_xlim(xmin, xmax)
pt.plt.gca().set_ylim(ymin, ymax)
pt.dark_background(doubleit=3)
pt.gca().invert_yaxis()
return locals()
def shadowform_probability():
""" its hearthstone, but whatev
probability of
raza + no shadowform on turn 5 +
probability of
raza + shadowform on turn 5 +
probability of
kazakus turn 4, raza turn 5, + no shadowform
"""
from scipy.stats import hypergeom
def p_badstuff_shadowform(turn=5, hand_size=3):
deck_size = 30
num_shadowform = 2
def prob_nohave_card_never_mulled(copies=2, hand_size=3):
deck_size = 30
prb = hypergeom(deck_size, copies, hand_size)
# P(initial_miss)
p_none_premul = prb.cdf(0)
# GIVEN that we mul our first 3 what is prob we still are unlucky
# P(miss_turn0 | initial_miss)
prb = hypergeom(deck_size - hand_size, copies, hand_size)
p_none_in_mul = prb.cdf(0)
# TODO: add constraints about 2 drops
# P(miss_turn0) = P(miss_turn0 | initial_miss) * P(initial_miss)
p_none_at_start = p_none_in_mul * p_none_premul
return p_none_at_start
def prob_nohave_card_always_mulled(copies=2, hand_size=3):
# probability of getting the card initially
p_none_premul = hypergeom(deck_size, copies, hand_size).cdf(0)
# probability of getting the card if everything is thrown away
# (TODO: factor in the probability that you need to keep something)
# for now its fine because if we keep shadowform the end calculation is fine
p_nohave_postmul_given_nohave = hypergeom(deck_size - hand_size, copies, hand_size).cdf(0)
# not necessary, but it shows the theory
p_nohave_postmul_given_had = 1
p_nohave_turn0 = (
p_nohave_postmul_given_nohave * p_none_premul + (1 - p_none_premul) * p_nohave_postmul_given_had
)
return p_nohave_turn0
def prob_nohave_by_turn(p_none_turn0, turn, copies, hand_size):
# P(miss_turnN | miss_mul)
p_none_turnN_given_mulmis = hypergeom(deck_size - hand_size, copies, turn).cdf(0)
# P(miss_turnN) = P(miss_turnN | miss_mul) P(miss_mul)
p_none_turnN = p_none_turnN_given_mulmis * p_none_turn0
return p_none_turnN
p_no_shadowform_on_turn0 = prob_nohave_card_never_mulled(copies=num_shadowform, hand_size=hand_size)
no_shadowform_turnN = prob_nohave_by_turn(p_no_shadowform_on_turn0, turn, num_shadowform, hand_size)
# Assume you always mul raza
p_noraza_initial = prob_nohave_card_always_mulled(copies=1, hand_size=hand_size)
p_noraza_turnN = prob_nohave_by_turn(p_noraza_initial, turn, copies=1, hand_size=hand_size)
p_raza_turnN = 1 - p_noraza_turnN
# probability that you have raza and no shadowform by turn 5
p_raza_and_noshadowform_turnN = p_raza_turnN * no_shadowform_turnN
return p_raza_and_noshadowform_turnN
import plottool as pt # NOQA
turns = list(range(0, 26))
probs = [p_badstuff_shadowform(turn, hand_size=3) for turn in turns]
pt.plot(turns, probs, label="on play")
probs = [p_badstuff_shadowform(turn, hand_size=4) for turn in turns]
pt.plot(turns, probs, label="with coin")
pt.set_xlabel("turn")
pt.set_ylabel("probability")
pt.set_title("Probability of Having Raza without a Shadowform")
pt.legend()
pt.gca().set_ylim(0, 1)