def test_batched_shuf_1d(self):
# 1D array with fitting size
l = list(u.batched(3, np.arange(6), shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(6))
# 1D arrays with fitting size
l = list(u.batched(3, np.arange(6), np.arange(6), shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])),
np.arange(6))
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])),
np.arange(6))
# 1D array with leftover
l = list(u.batched(3, np.arange(7), shuf=True))
self.assertEqual(len(l), 3)
self.assertEqual(len(l[-1]), 1)
npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(7))
# 1D arrays with leftover
l = list(u.batched(3, np.arange(7), np.arange(7)))
self.assertEqual(len(l), 3)
self.assertEqual(len(l[-1][0]), 1)
self.assertEqual(len(l[-1][1]), 1)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(l[2][0], l[2][1])
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])),
np.arange(7))
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])),
np.arange(7))
def test_batched_1d(self):
# 1D array with fitting size
l = list(u.batched(3, np.arange(6)))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0], [0,1,2])
npt.assert_array_equal(l[1], [3,4,5])
# 1D arrays with fitting size
l = list(u.batched(3, np.arange(6), np.arange(6)))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], [0,1,2])
npt.assert_array_equal(l[0][1], [0,1,2])
npt.assert_array_equal(l[1][0], [3,4,5])
npt.assert_array_equal(l[1][1], [3,4,5])
# 1D array with leftover
l = list(u.batched(3, np.arange(7)))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0], [0,1,2])
npt.assert_array_equal(l[1], [3,4,5])
npt.assert_array_equal(l[2], [6])
# 1D arrays with leftover
l = list(u.batched(3, np.arange(7), np.arange(7)))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0][0], [0,1,2])
npt.assert_array_equal(l[0][1], [0,1,2])
npt.assert_array_equal(l[1][0], [3,4,5])
npt.assert_array_equal(l[1][1], [3,4,5])
npt.assert_array_equal(l[2][0], [6])
npt.assert_array_equal(l[2][1], [6])
def test_batched_shuf_1d(self):
# 1D array with fitting size
l = list(u.batched(3, np.arange(6), shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(6))
# 1D arrays with fitting size
l = list(u.batched(3, np.arange(6), np.arange(6), shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])), np.arange(6))
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])), np.arange(6))
# 1D array with leftover
l = list(u.batched(3, np.arange(7), shuf=True))
self.assertEqual(len(l), 3)
self.assertEqual(len(l[-1]), 1)
npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(7))
# 1D arrays with leftover
l = list(u.batched(3, np.arange(7), np.arange(7)))
self.assertEqual(len(l), 3)
self.assertEqual(len(l[-1][0]), 1)
self.assertEqual(len(l[-1][1]), 1)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(l[2][0], l[2][1])
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])), np.arange(7))
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])), np.arange(7))
def test_batched_1d(self):
# 1D array with fitting size
l = list(u.batched(3, np.arange(6)))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0], [0, 1, 2])
npt.assert_array_equal(l[1], [3, 4, 5])
# 1D arrays with fitting size
l = list(u.batched(3, np.arange(6), np.arange(6)))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], [0, 1, 2])
npt.assert_array_equal(l[0][1], [0, 1, 2])
npt.assert_array_equal(l[1][0], [3, 4, 5])
npt.assert_array_equal(l[1][1], [3, 4, 5])
# 1D array with leftover
l = list(u.batched(3, np.arange(7)))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0], [0, 1, 2])
npt.assert_array_equal(l[1], [3, 4, 5])
npt.assert_array_equal(l[2], [6])
# 1D arrays with leftover
l = list(u.batched(3, np.arange(7), np.arange(7)))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0][0], [0, 1, 2])
npt.assert_array_equal(l[0][1], [0, 1, 2])
npt.assert_array_equal(l[1][0], [3, 4, 5])
npt.assert_array_equal(l[1][1], [3, 4, 5])
npt.assert_array_equal(l[2][0], [6])
npt.assert_array_equal(l[2][1], [6])
def test_batched_2d(self):
# 2D array with fitting size
l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(6)])))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0], [[0,1],[2,3],[4,5]])
npt.assert_array_equal(l[1], [[6,7],[8,9],[10,11]])
# 2D arrays with fitting size
l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(6)]), np.arange(6)))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], [[0,1],[2,3],[4,5]])
npt.assert_array_equal(l[0][1], [0, 1, 2])
npt.assert_array_equal(l[1][0], [[6,7],[8,9],[10,11]])
npt.assert_array_equal(l[1][1], [3, 4, 5])
# 2D array with leftover
l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(7)])))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0], [[0,1],[2,3],[4,5]])
npt.assert_array_equal(l[1], [[6,7],[8,9],[10,11]])
npt.assert_array_equal(l[2], [[12,13]])
# 2D arrays with leftover
l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(7)]), np.arange(7)))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0][0], [[0,1],[2,3],[4,5]])
npt.assert_array_equal(l[0][1], [0, 1, 2])
npt.assert_array_equal(l[1][0], [[6,7],[8,9],[10,11]])
npt.assert_array_equal(l[1][1], [3, 4, 5])
npt.assert_array_equal(l[2][0], [[12,13]])
npt.assert_array_equal(l[2][1], [6])
def dotrain(model, crit, aug, Xtr, ytr, nepochs=50, batchsize=100, title=None):
opt = df.AdaDelta(rho=.95, eps=1e-7, lr=1)
# progress = IntProgress(value=0, min=0, max=nepochs, description='Training:')
# display(progress)
model.training()
costs = []
for e in range(nepochs):
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)
model.zero_grad_parameters()
cost = model.accumulate_gradients(Xb, yb, crit)
opt.update_parameters(model)
# print("batch cost: %f" % cost)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
print("mean batch cost: %f" % costs[-1])
# progress.value = e+1
# liveplot(plotcost, costs, title)
return costs
def calcPreRec(_X, _scans, _wcs, _was, _r, ts):
mylist1 = []
mylist2 = []
for Xb in lbu.batched(1, _X, shuf=False, droplast=True):
Xbt = torch.Tensor(Xb).cuda()
del Xb
pred_y_conf0, pred_y_offs0 = net(Xbt)
del Xbt
pred_y_conf, pred_y_offs = pred_y_conf0.detach().cpu().numpy(
), pred_y_offs0.detach().cpu().numpy()
del pred_y_offs0, pred_y_conf0
print(pred_y_conf.shape)
m = input()
mylist1.append(pred_y_offs)
mylist2.append(pred_y_conf)
del pred_y_conf, pred_y_offs
pred_y_conf_batches = np.array(mylist2)
pred_y_offs_batches = np.array(mylist1)
del mylist1, mylist2
alldets, (wcdets, wadets) = pred2det_comb(_scans,
pred_y_conf_batches,
pred_y_offs_batches,
thresh=ts,
out_rphi=False)
gt_all = [wcs + was for wcs, was in zip(_wcs, _was)]
precs, recs = u.precrec(alldets, gt_all, _r, pred_rphi=False, gt_rphi=True)
return precs, recs
def dostats(model, aug, Xtr, batchsize=100):
model.training()
for Xb in batched(batchsize, Xtr):
if aug is None:
model.accumulate_statistics(Xb)
else:
for Xb_aug in aug.augbatch_pred(Xb):
model.accumulate_statistics(Xb_aug)
def dostats(model, aug, Xtr, batchsize=128):
model.training()
for Xb in batched(batchsize, Xtr):
if aug is None:
model.accumulate_statistics(Xb)
else:
for Xb_aug in aug.augbatch_pred(Xb):
model.accumulate_statistics(Xb_aug)
def dopred(model, aug, X, ensembling, output2preds, batchsize=128):
model.evaluate()
y_preds = []
for Xb in batched(batchsize, X):
if aug is None:
p_y = model.forward(X)
else:
p_y = ensembling([model.forward(X) for X in aug.augbatch_pred(Xb)])
y_preds += list(output2preds(p_y))
return np.array(y_preds)
def dopred(model, aug, X, ensembling, output2preds, batchsize=100):
model.evaluate()
y_preds = []
for Xb in batched(batchsize, X):
if aug is None:
p_y = model.forward(X)
else:
p_y = ensembling([model.forward(X) for X in aug.augbatch_pred(Xb)])
y_preds += list(output2preds(p_y))
return np.array(y_preds)
def test_batched_shuf_2d(self):
# Here we mostly test for the length of the batches and whether they
# still got the same content, NOT their shapes.
# Testing their shapes has been done in `test_Batched_2d` and this
# should only test the shuffling.
a6 = np.array([[2 * i, 2 * i + 1] for i in range(6)])
a7 = np.array([[2 * i, 2 * i + 1] for i in range(7)])
# 2D array with fitting size
l = list(u.batched(3, a6, shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l])), a6.flatten())
# 2D arrays with fitting size
l = list(u.batched(3, a6, a6, shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
l1, l2 = zip(*l)
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l1])), a6.flatten())
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l2])), a6.flatten())
# 2D array with leftover
l = list(u.batched(3, a7, shuf=True))
self.assertEqual(len(l), 3)
self.assertEqual(l[-1].shape, (1, 2))
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l])), a7.flatten())
# 2D arrays with leftover
l = list(u.batched(3, a7, a7))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(l[2][0], l[2][1])
l1, l2 = zip(*l)
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l1])), a7.flatten())
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l2])), a7.flatten())
def test_batched_shuf_2d(self):
# Here we mostly test for the length of the batches and whether they
# still got the same content, NOT their shapes.
# Testing their shapes has been done in `test_Batched_2d` and this
# should only test the shuffling.
a6 = np.array([[2*i, 2*i+1] for i in range(6)])
a7 = np.array([[2*i, 2*i+1] for i in range(7)])
# 2D array with fitting size
l = list(u.batched(3, a6, shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l])), a6.flatten())
# 2D arrays with fitting size
l = list(u.batched(3, a6, a6, shuf=True))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
l1, l2 = zip(*l)
npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l1])), a6.flatten())
npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l2])), a6.flatten())
# 2D array with leftover
l = list(u.batched(3, a7, shuf=True))
self.assertEqual(len(l), 3)
self.assertEqual(l[-1].shape, (1,2))
npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l])), a7.flatten())
# 2D arrays with leftover
l = list(u.batched(3, a7, a7))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(l[2][0], l[2][1])
l1, l2 = zip(*l)
npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l1])), a7.flatten())
npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l2])), a7.flatten())
def test_batched_2d(self):
# 2D array with fitting size
l = list(u.batched(3,
np.array([[2 * i, 2 * i + 1] for i in range(6)])))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0], [[0, 1], [2, 3], [4, 5]])
npt.assert_array_equal(l[1], [[6, 7], [8, 9], [10, 11]])
# 2D arrays with fitting size
l = list(
u.batched(3, np.array([[2 * i, 2 * i + 1] for i in range(6)]),
np.arange(6)))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], [[0, 1], [2, 3], [4, 5]])
npt.assert_array_equal(l[0][1], [0, 1, 2])
npt.assert_array_equal(l[1][0], [[6, 7], [8, 9], [10, 11]])
npt.assert_array_equal(l[1][1], [3, 4, 5])
# 2D array with leftover
l = list(u.batched(3,
np.array([[2 * i, 2 * i + 1] for i in range(7)])))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0], [[0, 1], [2, 3], [4, 5]])
npt.assert_array_equal(l[1], [[6, 7], [8, 9], [10, 11]])
npt.assert_array_equal(l[2], [[12, 13]])
# 2D arrays with leftover
l = list(
u.batched(3, np.array([[2 * i, 2 * i + 1] for i in range(7)]),
np.arange(7)))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0][0], [[0, 1], [2, 3], [4, 5]])
npt.assert_array_equal(l[0][1], [0, 1, 2])
npt.assert_array_equal(l[1][0], [[6, 7], [8, 9], [10, 11]])
npt.assert_array_equal(l[1][1], [3, 4, 5])
npt.assert_array_equal(l[2][0], [[12, 13]])
npt.assert_array_equal(l[2][1], [6])
def test_batched_N_2d(self):
# Here we mostly test for the length of the batches and whether they
# still got the same content, NOT their shapes.
# Testing their shapes has been done in `test_Batched_2d` and this
# should only test the N.
a6 = np.array([[2 * i, 2 * i + 1] for i in range(6)])
a7 = np.array([[2 * i, 2 * i + 1] for i in range(7)])
a8 = np.array([[2 * i, 2 * i + 1] for i in range(8)])
# 2D array with fitting size and N
l = list(u.batched(3, a6, N=6))
self.assertEqual(len(l), 2)
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l])), a6.flatten())
# 2D array with fitting N but too large size
l = list(u.batched(3, a7, N=6))
self.assertEqual(len(l), 2)
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l])), a6.flatten())
# 2D array with fitting size but too large N
with self.assertRaises(AssertionError):
l = list(u.batched(3, a6, N=7))
# 2D arrays with fitting size
l = list(u.batched(3, a6, a6, N=6))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
l1, l2 = zip(*l)
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l1])), a6.flatten())
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l2])), a6.flatten())
# 2D array with leftover
l = list(u.batched(3, a8, N=7))
self.assertEqual(len(l), 3)
self.assertEqual(l[-1].shape, (1, 2))
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l])), a7.flatten())
# 2D arrays with leftover
l = list(u.batched(3, a8, a7, N=7))
self.assertEqual(len(l), 3)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(l[2][0], l[2][1])
l1, l2 = zip(*l)
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l1])), a7.flatten())
npt.assert_array_equal(
sorted(np.concatenate([i.flatten() for i in l2])), a7.flatten())
def dotrain(model, crit, aug, Xtr, ytr, nepochs=3, batchsize=128, title=None):
opt = df.AdaDelta(rho=0.95, eps=1e-7, lr=1)
model.training()
costs = []
print("Training in progress...")
for e in range(nepochs):
print("Current epoch: {0} out of {1}".format(e + 1, nepochs))
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)
model.zero_grad_parameters()
cost = model.accumulate_gradients(Xb, yb, crit)
opt.update_parameters(model)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
return costs
def dotrain(model, crit, aug, Xtr, ytr, nepochs=3, batchsize=128, title=None):
opt = df.AdaDelta(rho=.95, eps=1e-7, lr=1)
model.training()
costs = []
print("Training in progress...")
for e in range(nepochs):
print("Current epoch: {0} out of {1}".format(e + 1, nepochs))
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)
model.zero_grad_parameters()
cost = model.accumulate_gradients(Xb, yb, crit)
opt.update_parameters(model)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
return costs
def dotrain(model,
sess,
loss,
aug,
Xtr,
ytr,
nepochs=50,
batchsize=100,
title=None):
opt = tf.train.AdadeltaOptimizer(learning_rate=1, rho=0.95,
epsilon=1e-7).minimize(loss)
# noinspection PyInterpreter
progress = IntProgress(value=0,
min=0,
max=nepochs,
description='Training:')
display(progress)
#model.training()
costs = []
for e in range(nepochs):
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)[0]
sess.run([opt, loss], feed_dict={model.X: Xb, model.y: yb})
# model.zero_grad_parameters()
# cost = model.accumulate_gradients(Xb, yb, crit)
# opt.update_parameters(model)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
progress.value = e + 1
liveplot(plotcost, costs, title)
return costs
def dotrain(model, crit, aug, Xtr, ytr, nepochs=50, batchsize=100, title=None):
opt = df.AdaDelta(rho=0.95, eps=1e-7, lr=1)
progress = IntProgress(value=0, min=0, max=nepochs, description="Training:")
display(progress)
model.training()
costs = []
for e in range(nepochs):
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)
model.zero_grad_parameters()
cost = model.accumulate_gradients(Xb, yb, crit)
opt.update_parameters(model)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
progress.value = e + 1
liveplot(plotcost, costs, title)
return costs
def test_batched_N_1d(self):
# 1D array with fitting size and N
l = list(u.batched(3, np.arange(6), N=6))
self.assertEqual(len(l), 2)
npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(6))
# 1D array with fitting N but too large size
l = list(u.batched(3, np.arange(8), N=6))
self.assertEqual(len(l), 2)
npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(6))
# 1D array with fitting N but too small size
with self.assertRaises(AssertionError):
l = list(u.batched(3, np.arange(6), N=8))
# 1D arrays with fitting size
l = list(u.batched(3, np.arange(7), np.arange(8), N=6))
self.assertEqual(len(l), 2)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])),
np.arange(6))
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])),
np.arange(6))
# 1D array with leftover
l = list(u.batched(3, np.arange(8), N=7))
self.assertEqual(len(l), 3)
self.assertEqual(len(l[-1]), 1)
npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(7))
# 1D arrays with leftover
l = list(u.batched(3, np.arange(8), np.arange(9), N=7))
self.assertEqual(len(l), 3)
self.assertEqual(len(l[-1][0]), 1)
self.assertEqual(len(l[-1][1]), 1)
npt.assert_array_equal(l[0][0], l[0][1])
npt.assert_array_equal(l[1][0], l[1][1])
npt.assert_array_equal(l[2][0], l[2][1])
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])),
np.arange(7))
npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])),
np.arange(7))
