def test_evalb_unofficial():
"Compare evalb_unofficial to evalb (official)."
from ldp.parsing.util import binarize
from ldp.prune.example import Setup
from ldp.parse.benchmark import Parser
from ldp.parse import leftchild
from arsenal.iterview import iterview
for grammar in ['medium']:
s = Setup(grammar=grammar, maxlength=30, train=0, dev=50)
parser = Parser(leftchild, s.grammar, chomsky=0)
for e in iterview(s.dev):
m = e.mask
state = parser(e, m)
ucoarse = parser.decode(e, state.derivation)
# TODO: Technically, average evalb([x_i]) over sentences is
# *NOT* the same as evalb([x_1...x_n]) on the corpus.
#
# This is a "macro v. micro average" problem.
unofficial = lambda a, b: fpr(*evalb_unofficial(a, b))[0]
fb = unofficial(e.gold_unbinarized, binarize(ucoarse))
f = unofficial(e.gold_unbinarized, ucoarse)
h = evalb(e.gold_unbinarized, ucoarse)
assert abs(fb - f) < 1e-8, "binarization shouldn't affect scores."
assert abs(f - h) < 1e-4
print '[test/evalb unofficial] pass'
def quick_fdcheck(func, w, g, n_checks=20, eps=1e-5, verbose=1, progressbar=1):
"""
Check gradient along random directions (a faster alternative to axis-aligned directions).
Tim Vieira (2017) "How to test gradient implementations"
https://timvieira.github.io/blog/post/2017/04/21/how-to-test-gradient-implementations/
"""
keys = ['rand_%s' % i for i in range(n_checks)]
H = {}
G = {}
was = w.flatten()
w = np.asarray(w.flat)
g = np.asarray(g.flat)
dim = len(w)
for k in (iterview(keys) if progressbar else keys):
d = spherical(dim)
G[k] = g.dot(d)
w[:] = was + eps * d
b = func()
w[:] = was - eps * d
a = func()
w[:] = was
H[k] = (b - a) / (2 * eps)
return compare(H, G, verbose=verbose)
def quick_fdcheck(func, w, g, n_checks = 20, eps = 1e-5, verbose=1, progressbar=1):
"""
Check gradient along random directions (a faster alternative to axis-aligned directions).
Tim Vieira (2017) "How to test gradient implementations"
https://timvieira.github.io/blog/post/2017/04/21/how-to-test-gradient-implementations/
"""
keys = ['rand_%s' % i for i in range(n_checks)]
H = {}
G = {}
was = w.flatten()
w = np.asarray(w.flat)
g = np.asarray(g.flat)
dim = len(w)
for k in (iterview(keys) if progressbar else keys):
d = spherical(dim)
G[k] = g.dot(d)
w[:] = was + eps*d
b = func()
w[:] = was - eps*d
a = func()
w[:] = was
H[k] = (b-a) / (2*eps)
return compare(H, G, verbose=verbose)
def test():
a0,b0,c0 = range(1, 10), range(21, 30), range(81, 90)
test = zip(a0,b0,c0)
a, b, c = iunzip(test)
a, b, c = map(list, (a,b,c))
assert a == a0 and b == b0 and c == c0
recombined = zip(a, b, c)
assert recombined == test
def example_iterview():
for _ in iterview(xrange(400), every=20):
sleep(0.01)
#example_iterview()
X = [[0 for i in xrange(4)] for j in xrange(4)]
for (k,(i,j)) in enumerate(cross_lower_triangle(range(4))):
X[i][j] = k+1
target = [[0, 0, 0, 0],
[1, 0, 0, 0],
[2, 3, 0, 0],
[4, 5, 6, 0]]
assert X == target
import numpy as np
d = (np.random.rand(20, 3) - 0.5) * 100
A = np.average(d, axis=0)
a = last(rolling_average(d))
assert np.linalg.norm(A - a) < 1e-10 # roughly zero difference
import doctest; doctest.testmod()
for _ in iterview(range(100), 1):
sleep(.1)
def sample(self, data, num=1000):
""" sample """
samples = []
inside = 0
correct1, correct2, total = 0, 0, 0
for instance in iterview(data, colored('Sampling', 'green')):
psi = self.features.potentials_catchup(instance, self.updater.w, self.updater)
sr = instance.sr
dist = {}
for s in self.model.sample(sr, psi, num=num):
output = ""
for x, y in s:
output += y
if output not in dist:
dist[output] = 0
dist[output] += 1
count = dist[instance.ur_gold] if instance.ur_gold in dist else 0
decoded = self.decode(instance)[1]
if decoded != instance.ur_gold and count > 0:
inside += 1
if decoded == instance.ur_gold:
correct1 += 1
if instance.ur_gold in dist:
correct2 += 1
total += 1
samples.append(dist)
# TODO: put into log
#print ; print inside
#print correct1 / total, correct2 / total
return samples
def optimize_chunk(self, iterations):
""" optimize the model """
for i in xrange(iterations):
for tree in iterview(self.train,
colored('Pass %s' % (i + 1), 'blue')):
gt0, gt, ge = self.features.potentials_catchup(
tree, self.updater.w, self.updater)
dgt0, dgt, dge = zeros_like(gt0), zeros_like(gt), zeros_like(
ge)
self.segmenter.dll(tree, ge, gt0, gt, dge, dgt0, dgt)
self.features.update(tree, dge, dgt0, dgt, self.updater)
self.updater.step += 1
self.save_segmenter(self.weights, i)
self.decode(self.train, VITERBI)
self.decode(self.dev, VITERBI)
test_acc, test_f1 = self.decode(self.test, VITERBI)
logging.info("chunk epoch {0} train acc: {1}".format(*(i,
train_acc)))
logging.info("chunk epoch {0} dev acc: {1}".format(*(i, dev_acc)))
logging.info("chunk epoch {0} test acc: {1}".format(*(i,
test_acc)))
logging.info("chunk epoch {0} train f1: {1}".format(*(i,
train_f1)))
logging.info("chunk epoch {0} dev f1: {1}".format(*(i, dev_f1)))
logging.info("chunk epoch {0} test f1: {1}".format(*(i, dev_f1)))
def preprocess_bubs_format(bubs, output):
"""Convert grammar from bubs-parser into ldp-friendly csv format. The result is
an equivalent grammar, which is much faster to load because it has been
integerized.
Given a gzipped grammar from bubs-parser, e.g. `eng.M2.gr.gz`, this function
will generate four files:
- eng.M2.gr.csv: grammar rules
- eng.M2.lex.csv: lexical rules
- eng.M2.lex.alphabet: mapping from terminals to integers
- eng.M2.sym.alphabet: mapping from syms to integers
"""
sym = Alphabet()
lex = Alphabet()
import gzip
lines = gzip.open(bubs, 'rb').readlines()
reading_lex = False
l = []
f = []
for line in iterview(lines[1:]): # drop first line
if line.startswith('===== LEXICON'):
reading_lex = True
continue
x = line.strip().split()
if not x:
continue
lhs = x[0]
rhs = tuple(b for b in x[2:-1])
score = x[-1]
if len(rhs) == 1:
rhs = (rhs[0], '')
y, z = rhs
lhs = sym[lhs]
y = lex[y] if reading_lex else sym[y]
z = sym[z] if z else -1
if reading_lex:
l.append({'score': score, 'head': lhs, 'left': y})
else:
f.append({'score': score, 'head': lhs, 'left': y, 'right': z})
# non-gzipped loads faster.
#DataFrame(f).to_csv(gzip.open(output + '.gr.csv.gz', 'wb'))
#DataFrame(l).to_csv(gzip.open(output + '.lex.csv.gz', 'wb'))
DataFrame(f).to_csv(output + '.gr.csv')
DataFrame(l).to_csv(output + '.lex.csv')
sym.save(output + '.sym.alphabet')
lex.save(output + '.lex.alphabet')
def fdcheck(func,
w,
g,
keys=None,
eps=1e-5,
quiet=0,
verbose=1,
progressbar=1,
throw=True):
"""
Finite-difference check.
Returns `arsenal.math.compare` instance.
- `func`: zero argument function, which references `w` in caller's scope.
- `w`: parameters.
- `g`: gradient estimate to compare against
- `keys`: dimensions to check
- `eps`: perturbation size
"""
if quiet:
verbose = 0
progressbar = 0
if keys is None:
if hasattr(
w, 'keys'
): # support for sparse vectors represented as a dictionary-like object.
keys = list(w.keys())
d = {}
else:
# use flat views, if need be.
if len(w.shape) > 1: w = w.flat
if len(g.shape) > 1: g = g.flat
d = np.zeros_like(w)
keys = list(
range(len(w))
) # TODO: these keys have lost their names. So not good for debugging.
else:
d = {}
for k in (iterview(keys) if progressbar else keys):
was = w[k]
w[k] = was + eps
b = func()
w[k] = was - eps
a = func()
w[k] = was
d[k] = (b - a) / (2 * eps)
if throw and not np.allclose([d[k] for k in keys], [g[k] for k in keys]):
compare(d, g, verbose=True)
raise AssertionError('^^^^ see compare above')
return compare([d[k] for k in keys], [g[k] for k in keys], verbose=verbose)
def sample_ur(self, data, num_samples=1000):
""" samples the UR """
samples = self.transducer.sample(data, num=num_samples)
for tree, samples in iterview(zip(data, samples),
colored('Sampling', 'red')):
tree.ur_samples = []
viterbi_ur = self.transducer.decode(tree)[1]
tree.ur_samples.append(viterbi_ur)
for sample, count in samples.items():
tree.ur_samples.append(sample)
def perceptron(self, data, rate=0.01, iterations=20, validate=None):
""" Parameter estimation with the perceptron algorithm. """
W = self.W
for i in range(iterations):
for x in iterview(data, msg='Iteration %s' % i):
for k in self.path_features(x, self.argmax(x)):
W[k] -= rate
for k in x.target_features:
W[k] += rate
if validate:
validate(self, i)
def test_gradient(self, data, subsetsize=100):
def fd(x, i, eps=1e-5):
"""Compute `i`th component of the finite-difference approximation to the
gradient of log-likelihood at current parameters on example `x`.
"""
was = self.W[i] # record value
self.W[i] = was + eps
b = self.likelihood(x)
self.W[i] = was - eps
a = self.likelihood(x)
self.W[i] = was # restore original value
return (b - a) / 2 / eps
for x in iterview(data, msg='test grad'):
g = defaultdict(float)
for k, v in self.expectation(x).iteritems():
g[k] -= 1 * v
for k in x.target_features:
g[k] += 1
# pick a subset of features to test
d = np.random.choice(g.keys(), subsetsize, replace=0)
f = {}
for i in iterview(d, msg='fd approx'): # loop over active features
f[i] = fd(x, i)
from arsenal.math import compare
compare([f[k] for k in d], [g[k] for k in d],
name='test gradient %s' % x,
scatter=1,
show_regression=1)
import pylab as pl
pl.show()
def optimize(self, iterations=10, start=0):
""" optimize the model """
#np.random.shuffle(self.train)
for i in xrange(iterations):
for instance in iterview(self.train, colored('Pass %s' % (i+1+start), 'blue')):
psi = self.features.potentials_catchup(instance, self.updater.w, self.updater)
dpsi = zeros_like(psi)
x, y = instance.sr, instance.ur
#print "LL", self.model.ll(x, y, psi, minx=MINX, miny=MINY)
dpsi = self.model.dll(x, y, psi, minx=MINX, miny=MINY)
self.features.update(instance, dpsi, self.updater)
self.updater.step += 1
def perceptron(self, data, rate=0.01, iterations=20, validate=None):
""" Parameter estimation with the perceptron algorithm. """
self.preprocess(data)
W = self.W
for i in xrange(iterations):
for x in iterview(data, msg='Iteration %s' % i):
for k in self.path_features(x, self.argmax(x)):
W[k] -= rate
for k in x.target_features:
W[k] += rate
if validate:
validate(self, i)
def sgd(self, data, iterations=20, a0=10, validate=None):
""" Parameter estimation with stochastic gradient descent (sgd). """
W = self.W
for i in range(iterations):
rate = a0 / (np.sqrt(i) + 1)
for x in iterview(data, msg='Iteration %s' % i):
for k, v in self.expectation(x).iteritems():
W[k] -= rate * v
for k in x.target_features:
W[k] += rate
if validate:
validate(self, i)
def test_gradient(self, data, subsetsize=100):
def fd(x, i, eps=1e-5):
"""Compute `i`th component of the finite-difference approximation to the
gradient of log-likelihood at current parameters on example `x`.
"""
was = self.W[i] # record value
self.W[i] = was+eps
b = self.likelihood(x)
self.W[i] = was-eps
a = self.likelihood(x)
self.W[i] = was # restore original value
return (b - a) / 2 / eps
for x in iterview(data, msg='test grad'):
g = defaultdict(float)
for k, v in self.expectation(x).iteritems():
g[k] -= 1*v
for k in x.target_features:
g[k] += 1
# pick a subset of features to test
d = np.random.choice(g.keys(), subsetsize, replace=0)
f = {}
for i in iterview(d, msg='fd approx'): # loop over active features
f[i] = fd(x, i)
from arsenal.math import compare
compare([f[k] for k in d],
[g[k] for k in d], name='test gradient %s' % x, scatter=1, show_regression=1)
import pylab as pl
pl.show()
def sgd(self, data, iterations=20, a0=10, validate=None):
""" Parameter estimation with stochastic gradient descent (sgd). """
self.preprocess(data)
W = self.W
for i in xrange(iterations):
rate = a0 / (sqrt(i) + 1)
for x in iterview(data, msg='Iteration %s' % i):
for k, v in self.expectation(x).iteritems():
W[k] -= rate*v
for k in x.target_features:
W[k] += rate
if validate:
validate(self, i)
def fdcheck(func,
w,
g,
keys=None,
eps=1e-5,
quiet=0,
verbose=1,
progressbar=1):
"""
Finite-difference check.
Returns `arsenal.math.compare` instance.
- `func`: zero argument function, which references `w` in caller's scope.
- `w`: parameters.
- `g`: gradient estimate to compare against
- `keys`: dimensions to check
- `eps`: perturbation size
"""
if quiet:
verbose = 0
progressbar = 0
if keys is None:
if hasattr(w, 'keys'):
keys = w.keys()
d = {}
else:
d = np.zeros_like(w)
# use flat views, if need be.
if len(w.shape) > 1:
w = w.flat
if len(g.shape) > 1:
g = g.flat
if len(d.shape) > 1:
d = d.flat
keys = range(len(w))
for k in (iterview(keys) if progressbar else keys):
was = w[k]
w[k] = was + eps
b = func()
w[k] = was - eps
a = func()
w[k] = was
d[k] = (b - a) / (2 * eps)
return compare([d[k] for k in keys], [g[k] for k in keys], verbose=verbose)
def quick_fdcheck(func,
w,
g,
n_checks=20,
eps=1e-5,
quiet=True,
verbose=False,
progressbar=False,
throw=True):
"""
Check gradient along random directions (a faster alternative to axis-aligned directions).
Tim Vieira (2017) "How to test gradient implementations"
https://timvieira.github.io/blog/post/2017/04/21/how-to-test-gradient-implementations/
"""
if quiet:
verbose = 0
progressbar = 0
keys = ['rand_%s' % i for i in range(n_checks)]
H = {}
G = {}
was = w.flatten()
w = np.asarray(w.flat)
g = np.asarray(g.flat)
dim = len(w)
for k in (iterview(keys) if progressbar else keys):
d = spherical(dim)
G[k] = g.dot(d)
w[:] = was + eps * d
b = func()
w[:] = was - eps * d
a = func()
w[:] = was
H[k] = (b - a) / (2 * eps)
different = not np.allclose(list(H.values()), list(G.values()))
if verbose or different:
compare(H, G, verbose=True)
if different and throw:
raise AssertionError('^^^^ see compare above')
return compare(H, G, verbose=False)
def main():
from arsenal.iterview import iterview
from time import sleep
from numpy.random import uniform
t = Timer('test')
for i in iterview(range(1, 20)):
for _ in range(10):
with t(i=i):
c = 0.01
z = max(i**2 * 0.0001 + uniform(-c, c), 0.0)
sleep(z)
a = t.plot_feature('i')
print(a)
pl.show()
def main():
import pylab as pl
from arsenal.iterview import iterview
from time import sleep
from numpy.random import uniform
t = Timer('test')
for i in iterview(xrange(1, 20)):
for _ in xrange(10):
with t(i=i):
c = 0.01
z = max(i**2 * 0.0001 + uniform(-c, c), 0.0)
sleep(z)
a = t.plot_feature('i')
print a
pl.show()
def fdcheck(func, w, g, keys = None, eps = 1e-5, quiet=0, verbose=1, progressbar=1):
"""
Finite-difference check.
Returns `arsenal.math.compare` instance.
- `func`: zero argument function, which references `w` in caller's scope.
- `w`: parameters.
- `g`: gradient estimate to compare against
- `keys`: dimensions to check
- `eps`: perturbation size
"""
if quiet:
verbose = 0
progressbar = 0
if keys is None:
if hasattr(w, 'keys'):
keys = w.keys()
d = {}
else:
d = np.zeros_like(w)
# use flat views, if need be.
if len(w.shape) > 1:
w = w.flat
if len(g.shape) > 1:
g = g.flat
if len(d.shape) > 1:
d = d.flat
keys = range(len(w))
for k in (iterview(keys) if progressbar else keys):
was = w[k]
w[k] = was + eps
b = func()
w[k] = was - eps
a = func()
w[k] = was
d[k] = (b-a) / (2*eps)
return compare([d[k] for k in keys],
[g[k] for k in keys],
verbose=verbose)
def quick_fdcheck(func, w, g, n_checks, eps=1e-5, verbose=1, progressbar=1):
"Check gradient along random directions (a faster alternative to axis-aligned directions)."
keys = ['rand_%s' % i for i in range(n_checks)]
H = {}
G = {}
was = w.copy()
for k in (iterview(keys) if progressbar else keys):
d = spherical(w.shape[0])
G[k] = g.dot(d)
w[:] = was + eps * d
b = func()
w[:] = was - eps * d
a = func()
w[:] = was
H[k] = (b - a) / (2 * eps)
return compare(H, G, verbose=verbose)
def quick_fdcheck(func, w, g, n_checks, eps = 1e-5, verbose=1, progressbar=1):
"Check gradient along random directions (a faster alternative to axis-aligned directions)."
keys = ['rand_%s' % i for i in range(n_checks)]
H = {}
G = {}
was = w.copy()
for k in (iterview(keys) if progressbar else keys):
d = spherical(w.shape[0])
G[k] = g.dot(d)
w[:] = was + eps*d
b = func()
w[:] = was - eps*d
a = func()
w[:] = was
H[k] = (b-a) / (2*eps)
return compare(H, G, verbose=verbose)
def evaluate(self, data, maximum=100000000):
""" decode the model """
correct, total = 0, 0
counter = 0
for instance in iterview(data, colored('Decoding', 'red')):
if counter == maximum:
break
psi = self.features.potentials_catchup(instance, self.updater.w, self.updater)
ur1 = instance.ur
results = self.model.decode(instance.sr, psi, minx=MINX, miny=MINY)
ll = self.model.ll(instance.sr, ur1, psi, minx=MINX, miny=MINY)
score, ur2 = results[0], results[1]
if ur1 == ur2:
correct += 1
print ur1, ur2
total += 1
counter += 1
print
return float(correct) / total
def fdcheck(func, w, g, keys = None, eps = 1e-5, quiet=0, verbose=1, progressbar=1):
"""
Finite-difference check.
Returns `arsenal.math.compare` instance.
- `func`: zero argument function, which references `w` in caller's scope.
- `w`: parameters.
- `g`: gradient estimate to compare against
- `keys`: dimensions to check
- `eps`: perturbation size
"""
if quiet:
verbose = 0
progressbar = 0
if keys is None:
if hasattr(w, 'keys'): # support for sparse vectors represented as a dictionary-like object.
keys = list(w.keys())
d = {}
else:
# use flat views, if need be.
if len(w.shape) > 1: w = w.flat
if len(g.shape) > 1: g = g.flat
d = np.zeros_like(w)
keys = list(range(len(w))) # TODO: these keys have lost their names. So not good for debugging.
else:
d = {}
for k in (iterview(keys) if progressbar else keys):
was = w[k]
w[k] = was + eps
b = func()
w[k] = was - eps
a = func()
w[k] = was
d[k] = (b-a) / (2*eps)
return compare([d[k] for k in keys],
[g[k] for k in keys],
verbose=verbose)
def very_sgd(self, data, iterations=20, a0=10, R=1, validate=None):
"""Parameter estimation with stochastic gradient descent (sgd) where
expectations are estimated by sampling.
"""
assert R > 0
W = self.W
for i in range(iterations):
rate = a0 / (np.sqrt(i) + 1)
for x in iterview(data, msg='Iteration %s' % i):
r = 0
for y in sample(x):
r += 1
for k in self.path_features(x, y):
W[k] -= rate / R
if r >= R:
break
for k in x.target_features:
W[k] += rate
if validate:
validate(self, i)
def very_sgd(self, data, iterations=20, a0=10, R=1, validate=None):
"""Parameter estimation with stochastic gradient descent (sgd) where
expectations are estimated by sampling.
"""
assert R > 0
self.preprocess(data)
W = self.W
for i in xrange(iterations):
rate = a0 / (sqrt(i) + 1)
for x in iterview(data, msg='Iteration %s' % i):
r = 0
for y in self.sample(x):
r += 1
for k in self.path_features(x, y):
W[k] -= rate / R
if r >= R:
break
for k in x.target_features:
W[k] += rate
if validate:
validate(self, i)
def _test_sample_tree(example, grammar, N):
# gold = {(X,I,K) for (X,I,K) in example.gold_items if (I,K) in example.nodes}
print()
_forest = parse_forest(example, grammar)
# apply temperature to grammar rules
forest = Hypergraph()
forest.root = _forest.root
for e in _forest.edges:
c = LogVal.Zero()
c.logeq(e.weight)
forest.edge(c, e.head, *e.body)
# run inside-outside
B, A = sum_product(forest)
Z = B[forest.root]
# compute marginals and recall from samples
# sample_recall = 0.0
m = defaultdict(float)
for _ in iterview(range(N)):
t = sample(forest, B)
for s in t.subtrees():
x = s.label()
m[x] += 1.0 / N
# xx = rename(grammar, x)
# sample_recall += (xx in gold) * 1.0 / N
# convert node names and marginalize-out time index
IO = defaultdict(float)
for x in forest.incoming:
IO[x] += (B[x] * A[x] / Z).to_real()
# check marginals
threshold = 1e-4
for x in IO:
(I, K, X, T) = x
if K - I > 1:
a = IO[x]
b = m[x]
if a > threshold or b > threshold:
print('[%s %s %8s, %s] %7.3f %7.3f' \
% (I, K, X, T, a, b))
assert abs(a - b) < 0.05
def optimize_joint(self, iterations, num_samples=10, eta1=0.0, eta2=0.0):
""" optimize jointly using importance sampling """
# TODO: unit test
self.updater.eta = eta1
for i in xrange(iterations):
samples = self.transducer.sample(self.train, num=num_samples)
for tree, sample in iterview(zip(self.train, samples),
colored('Pass %s' % (i + 1), 'blue')):
# compute approximate partition function
logZ = NINF
strings, weights = [], []
for (ur, count) in sample.items():
score = self.transducer.ll(tree, ur)
if self.segmenter_type == CHUNK:
score += self.score_chunk(tree, ur)
elif self.segmenter_type == TREE:
score += self.score_tree(tree, ur)
# TODO: double check
logZ = logaddexp(logZ, score)
weights.append(score)
strings.append(ur)
#TODO: make more elegant
tmp = []
for weight in weights:
tmp.append(weight - logZ) # TODO: double check
weights = tmp
# take a tranducer weight gradient step with the importance sampling
self.transducer.step_is(tree, strings, weights, eta=eta2)
# take a segmenter weight gradient step with the importance sampling
for ur, weight in zip(sample, weights):
if self.segmenter_type == CHUNK:
self.is_chunk(tree, ur, weight)
elif self.segmenter_type == TREE:
self.is_tree(tree, ur, weight)
self.updater.step += 1
def check_gradient(f, grad, theta, alphabet=None, eps=1e-4, tol=0.01, skip_zero=True,
verbose=True, progress=True, keys=None, random_subset=None):
"""Check gradient that `f(theta) == grad` by centered-difference approximation.
Provides feedback on which dimensions differ
Arguments:
- `f`: function we are taking the gradient of.
- `grad`: What we think the gradient is at `theta`.
- `theta`:
- `alphabet` (optional): a bijective map from strings to integers. Expects
`arsenal.alphabet.Alphabet` instance. This is used to map integer-valued
dimensions to human-readable names (e.g., strings).
- `eps`: perturbation size
- `tol`: what is deem an error (we use relative error)
- `skip_zero`: good relative error is hard to get for values which are
really small (near zero).
- `random_subset`: number dimensions to probe in comparison (useful for
high dimensions because this test is linear in the dimensionality of
`theta`).
"""
import numpy as np
from numpy import zeros_like
from arsenal.terminal import green, red, yellow
from random import sample
from arsenal.math import cosine
fails = 0
grad = np.asarray(grad)
if keys is None:
if alphabet is not None:
keys = list(alphabet._flip.keys())
assert len(alphabet), 'Alphabet is empty.'
else:
keys = list(range(len(theta)))
if random_subset is not None:
if hasattr(random_subset, '__iter__'):
keys = list(random_subset)
else:
keys = sample(keys, min(len(keys), random_subset))
assert len(keys) > 0
fd = zeros_like(theta)
for i in (iterview(keys, msg='checkgrad') if progress else keys):
was = theta[i]
# perturb right
theta[i] = was + eps
right = f(theta)
# perturb left
theta[i] = was - eps
left = f(theta)
# reset
theta[i] = was
# centered difference
fd[i] = (right - left) / (2*eps)
w = max(list(map(len, list(alphabet.keys())))) if alphabet is not None else 0
nzeros = 0
for i in keys:
# check relative error
if skip_zero and abs(fd[i]) < 1e-10 and abs(grad[i]) < 1e-10: # both approximately zero
nzeros += 1
continue
relative_error = abs(fd[i] - grad[i]) / max(abs(fd[i]), abs(grad[i]))
if relative_error > tol:
name = alphabet.lookup(i) if alphabet is not None else i
fails += 1
if verbose:
print(red % 'dim = %s rel-err = %5.3f' % (('%%-%ss' % w) % (name,), relative_error), \
'want: %g; got: %g' % (fd[i], grad[i]))
else:
assert False, \
'dim = %s rel-err = %5.3f, want: %g; got: %g' \
% (('%%-%ss' % w) % (name,), relative_error, fd[i], grad[i])
if nzeros * 1.0 / len(keys) >= 0.75:
print(yellow % '[warning] checkgradient skipped a lot of approximately zero components ' \
'percentage= %g (%s/%s)' % (nzeros * 1.0 / len(keys), nzeros, len(keys)))
if verbose:
print('gradient:', end=' ')
if not fails:
print(green % 'OK', end=' ')
else:
print(red % 'failed %s of %s' % (fails, len(keys)), end=' ')
print('cosine similarity: %g' % cosine(grad[keys], fd[keys]))
def check_gradient(f, grad, theta, alphabet=None, eps=1e-4, tol=0.01, skip_zero=True,
verbose=True, progress=True, keys=None, random_subset=None):
"""Check gradient that `f(theta) == grad` by centered-difference approximation.
Provides feedback on which dimensions differ
Arguments:
- `f`: function we are taking the gradient of.
- `grad`: What we think the gradient is at `theta`.
- `theta`:
- `alphabet` (optional): a bijective map from strings to integers. Expects
`arsenal.alphabet.Alphabet` instance. This is used to map integer-valued
dimensions to human-readable names (e.g., strings).
- `eps`: perturbation size
- `tol`: what is deem an error (we use relative error)
- `skip_zero`: good relative error is hard to get for values which are
really small (near zero).
- `random_subset`: number dimensions to probe in comparison (useful for
high dimensions because this test is linear in the dimensionality of
`theta`).
"""
fails = 0
grad = np.asarray(grad)
if keys is None:
if alphabet is not None:
keys = alphabet._flip.keys()
assert len(alphabet), 'Alphabet is empty.'
else:
keys = range(len(theta))
if random_subset is not None:
if hasattr(random_subset, '__iter__'):
keys = list(random_subset)
else:
keys = sample(keys, min(len(keys), random_subset))
assert len(keys) > 0
fd = zeros_like(theta)
for i in (iterview(keys, msg='checkgrad') if progress else keys):
was = theta[i]
# perturb right
theta[i] = was + eps
right = f(theta)
# perturb left
theta[i] = was - eps
left = f(theta)
# reset
theta[i] = was
# centered difference
fd[i] = (right - left) / (2*eps)
w = max(map(len, alphabet.keys())) if alphabet is not None else 0
nzeros = 0
for i in keys:
# check relative error
if skip_zero and abs(fd[i]) < 1e-10 and abs(grad[i]) < 1e-10: # both approximately zero
nzeros += 1
continue
relative_error = abs(fd[i] - grad[i]) / max(abs(fd[i]), abs(grad[i]))
if relative_error > tol:
name = alphabet.lookup(i) if alphabet is not None else i
fails += 1
if verbose:
print red % 'dim = %s rel-err = %5.3f' % (('%%-%ss' % w) % (name,), relative_error), \
'want: %g; got: %g' % (fd[i], grad[i])
else:
assert False, \
'dim = %s rel-err = %5.3f, want: %g; got: %g' \
% (('%%-%ss' % w) % (name,), relative_error, fd[i], grad[i])
if nzeros * 1.0 / len(keys) >= 0.75:
print yellow % '[warning] checkgradient skipped a lot of approximately zero components ' \
'percentage= %g (%s/%s)' % (nzeros * 1.0 / len(keys), nzeros, len(keys))
if verbose:
print 'gradient:',
if not fails:
print green % 'OK',
else:
print red % 'failed %s of %s' % (fails, len(keys)),
print 'cosine similarity: %g' % cosine(grad[keys], fd[keys])
def example_iterview():
for _ in iterview(xrange(400), every=20):
sleep(0.01)
def decode(self, data, data_type, decode_type=None, sep=u"#"):
""" decode the chunker """
if decode_type is None:
decode_type = self.decode_type
if decode_type == ORACLE:
self.oracle_ur(data)
elif decode_type == BASELINE:
self.baseline_ur(data)
elif decode_type == VITERBI:
self.decode_ur(data)
elif decode_type == SAMPLE:
self.sample_ur(data)
else:
raise Exception('Illicit Decode Type')
ur_correct, ur_total = 0, 0
correct, f1, tree_f1, lev, total = 0, 0, 0, 0, 0
for tree in iterview(data, colored('Decoding', 'red')):
max_ur, max_score = None, NINF
counter = 0
for ur in tree.ur_samples:
tree.update_ur(ur)
counter += 1
score = 0.0
score = self.transducer.ll(tree, ur)
#print
#print "LL", self.transducer.ll(tree, ur)
if self.segmenter_type == CHUNK:
score += self.score_chunk(tree, ur)
#print "SCORE", self.score_chunk(tree, ur)
#print ur
#raw_input()
elif self.segmenter_type == TREE:
score += self.score_tree(tree, ur)
# take the best importance sample
if score >= max_score:
max_score = score
max_ur = ur
#print "counter", counter
if max_ur == tree.ur_gold:
ur_correct += 1
ur_total += 1
truth, guess, tree_f1_tmp = None, None, None
if self.segmenter_type == CHUNK:
truth, guess = self.decode_chunk(tree, max_ur)
elif self.segmenter_type == TREE:
truth, guess, tree_f1_tmp = self.decode_tree(tree, max_ur)
tree_f1 += tree_f1_tmp
# ACCURACY
if truth == guess:
correct += 1
# LEVENSHTEIN
lev += Levenshtein.distance(sep.join(truth), sep.join(guess))
# F1
set1, set2 = set(guess), set(truth)
p, r = 0, 0
for e in set1:
if e in set2:
p += 1
for e in set2:
if e in set1:
r += 1
p /= len(set1)
r /= len(set2)
if p + r > 0:
f1 += 2 * p * r / (p + r)
total += 1
logging.info("decoder type: {0}".format(decode_type))
logging.info("{0} ur acc: {1}".format(*(data_type,
ur_correct / total)))
logging.info("{0} seg acc: {1}".format(*(data_type, correct / total)))
logging.info("{0} f1: {1}".format(*(data_type, f1 / total)))
logging.info("{0} edit: {1}".format(*(data_type, lev / total)))
if self.segmenter_type == TREE:
logging.info("{0} tree f1: {1}".format(*(data_type,
tree_f1 / total)))
def oracle_ur(self, data):
""" uses the oracle UR """
for tree in iterview(data, colored('Updating Oracle UR', 'red')):
tree.ur_samples = []
tree.ur_samples.append(tree.ur_gold)
def decode_ur(self, data):
""" decodes the UR """
for tree in iterview(data, colored('Updating Viterbi UR', 'red')):
tree.ur_samples = []
viterbi_ur = self.transducer.decode(tree)[1]
tree.ur_samples.append(viterbi_ur)
def baseline_ur(self, data):
""" baseline ur """
for tree in iterview(data, colored('Updating Baseline UR', 'red')):
tree.ur_samples = []
tree.ur_samples.append(tree.sr)
def load_results(results, _args, filters=()):
jobs_running = [y.split()[0] for y in list(file('tmp/jobs'))[2:]]
data = []
jobs = []
msgs = []
for x in iterview(results.glob('*')):
# Extract name of the experiment, {YYYY-MM-DD}-{NAME}-{argument hash}
name = x.basename()[11:].split('-')[0]
if _args.jobids:
if (x / 'sge-jobid.txt').exists():
jobid = (x / 'sge-jobid.txt').text().strip()
if jobid not in _args.jobids:
continue
else:
continue
else:
if not any(p == name for p in filters): # substring match
continue
# TODO: the finish file doesn't get written if we call qdel, but the log
# files now contain some timestamps, so we should grab the last
# timestamp logged as the "finish time."
done = (x / 'finish').exists()
if (x / 'sge-jobid.txt').exists():
jobid = (x / 'sge-jobid.txt').text().strip()
else:
jobid = None
dump_exists = False
log_exists = False
args_exists = False
args = {}
d = x / 'dump'
if d.exists(): # job hasn't started (might've died)
dump_exists = True
else:
# Dump doesn't exist for reasons other than it failed to get
# scheduled. This might have to do with lack of permissions or a
# failure in the python code (e.g., ImportError).
assert jobid is None, x
# try again, but this time it's not nested.
d = x
if d.exists():
dump_exists = True
if dump_exists:
if (d / 'log.csv'
).exists(): # job hasn't produced first data point
log_exists = True
if (d / 'args.pkl').exists():
# load command-line arguments fro pickle and add prefix (`'args_'`)
# to avoid collisions with other column names.
args = load(d / 'args.pkl')
args = {'args_' + k: v for k, v in args.__dict__.items()}
args_exists = True
if not args_exists or not log_exists or not dump_exists:
miss = []
if jobid is None:
miss.append('jobid')
if not args_exists:
miss.append('args')
if not log_exists:
miss.append('log')
if not dump_exists:
miss.append('dump')
msgs.append('%s %s' %
(x, yellow % '(missing: %s)' % ' '.join(miss)))
# Note: two jobs /might/ get assigned the same job id, which might cause
# 'hash collision' style problems.
running = (jobid in jobs_running)
if args_exists:
start = to_datetime((x / 'start').text())
elapsed = (start.now() - start).total_seconds()
else:
continue
J = dict(directory=x,
jobid=jobid,
start=start,
elapsed=elapsed,
done=done,
log_exists=log_exists,
dump_exists=dump_exists,
args_exists=args_exists,
running=running)
# TODO: use dedicated CLI options for filtering jobs. The eval trick
# (below) is unnecessary and provides a clunky filter notation.
class __args:
pass
for k, v in args.items():
if k.startswith('args_'):
setattr(__args, k[5:], v) # drop 'args_' prefix
skip = 0
for ff in _args.filter:
fff = ff.replace('df.args_', '__args.')
if not eval(fff):
skip = 1
break
if skip:
continue
J.update(args)
jobs.append(J)
if not log_exists:
continue
log = get_log(d / 'log.csv', _args.accuracy, _args.runtime)
if log is None:
continue
for (_, e) in log.iterrows():
b = e.to_dict()
status = dict(name=name,
jobid=jobid,
running=running,
done=done,
log=d / 'log.csv')
b.update(J)
b.update(status)
b.update(args)
data.append(b)
# add column for time-elapsed (in days) since initial iteration.
el = log.datetime.map(to_datetime).tolist()
log['elapsed'] = [(x - el[0]).total_seconds() / (24 * 60 * 60)
for x in el]
# TODO: move to after filters are applied
if _args.lc: # enable to view learning curves super-imposed on one another.
learning_curve_handler(_args, args, log, jobid)
for msg in msgs:
print msg
return data, jobs
threshold=0.05,
R=10_000,
verbose=1):
"Pair permutation test"
def effect(xs, ys):
return np.abs(statistic(xs) - statistic(ys)) # two-sided: A != B
ra = statistic(xs)
rb = statistic(ys)
diff = effect(xs, ys) # observed difference
n = len(xs)
k = 0
reps = range(R)
if verbose: reps = iterview(reps, msg='perm test')
for _ in reps:
# randomly generate a vector of zeros and ones (uniformly).
swaps = np.random.randint(0, 2, n).astype(bool) # flip n coins
k += diff <= effect(
np.select([swaps, ~swaps], [xs, ys]), # swap elements accordingly
np.select([~swaps, swaps], [xs, ys]))
s = k / R
#threshold = 0.5*threshold
#s *= 0.5 # because we have a two-sided test.
if verbose:
# which system has higher reward? is it significant?
asig = (colors.red %
