def test_batched_shuf_1d(self): # 1D array with fitting size l = list(u.batched(3, np.arange(6), shuf=True)) self.assertEqual(len(l), 2) npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(6)) # 1D arrays with fitting size l = list(u.batched(3, np.arange(6), np.arange(6), shuf=True)) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])), np.arange(6)) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])), np.arange(6)) # 1D array with leftover l = list(u.batched(3, np.arange(7), shuf=True)) self.assertEqual(len(l), 3) self.assertEqual(len(l[-1]), 1) npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(7)) # 1D arrays with leftover l = list(u.batched(3, np.arange(7), np.arange(7))) self.assertEqual(len(l), 3) self.assertEqual(len(l[-1][0]), 1) self.assertEqual(len(l[-1][1]), 1) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) npt.assert_array_equal(l[2][0], l[2][1]) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])), np.arange(7)) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])), np.arange(7))
def test_batched_1d(self): # 1D array with fitting size l = list(u.batched(3, np.arange(6))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0], [0,1,2]) npt.assert_array_equal(l[1], [3,4,5]) # 1D arrays with fitting size l = list(u.batched(3, np.arange(6), np.arange(6))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], [0,1,2]) npt.assert_array_equal(l[0][1], [0,1,2]) npt.assert_array_equal(l[1][0], [3,4,5]) npt.assert_array_equal(l[1][1], [3,4,5]) # 1D array with leftover l = list(u.batched(3, np.arange(7))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0], [0,1,2]) npt.assert_array_equal(l[1], [3,4,5]) npt.assert_array_equal(l[2], [6]) # 1D arrays with leftover l = list(u.batched(3, np.arange(7), np.arange(7))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0][0], [0,1,2]) npt.assert_array_equal(l[0][1], [0,1,2]) npt.assert_array_equal(l[1][0], [3,4,5]) npt.assert_array_equal(l[1][1], [3,4,5]) npt.assert_array_equal(l[2][0], [6]) npt.assert_array_equal(l[2][1], [6])
def test_batched_1d(self): # 1D array with fitting size l = list(u.batched(3, np.arange(6))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0], [0, 1, 2]) npt.assert_array_equal(l[1], [3, 4, 5]) # 1D arrays with fitting size l = list(u.batched(3, np.arange(6), np.arange(6))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], [0, 1, 2]) npt.assert_array_equal(l[0][1], [0, 1, 2]) npt.assert_array_equal(l[1][0], [3, 4, 5]) npt.assert_array_equal(l[1][1], [3, 4, 5]) # 1D array with leftover l = list(u.batched(3, np.arange(7))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0], [0, 1, 2]) npt.assert_array_equal(l[1], [3, 4, 5]) npt.assert_array_equal(l[2], [6]) # 1D arrays with leftover l = list(u.batched(3, np.arange(7), np.arange(7))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0][0], [0, 1, 2]) npt.assert_array_equal(l[0][1], [0, 1, 2]) npt.assert_array_equal(l[1][0], [3, 4, 5]) npt.assert_array_equal(l[1][1], [3, 4, 5]) npt.assert_array_equal(l[2][0], [6]) npt.assert_array_equal(l[2][1], [6])
def test_batched_2d(self): # 2D array with fitting size l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(6)]))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0], [[0,1],[2,3],[4,5]]) npt.assert_array_equal(l[1], [[6,7],[8,9],[10,11]]) # 2D arrays with fitting size l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(6)]), np.arange(6))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], [[0,1],[2,3],[4,5]]) npt.assert_array_equal(l[0][1], [0, 1, 2]) npt.assert_array_equal(l[1][0], [[6,7],[8,9],[10,11]]) npt.assert_array_equal(l[1][1], [3, 4, 5]) # 2D array with leftover l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(7)]))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0], [[0,1],[2,3],[4,5]]) npt.assert_array_equal(l[1], [[6,7],[8,9],[10,11]]) npt.assert_array_equal(l[2], [[12,13]]) # 2D arrays with leftover l = list(u.batched(3, np.array([[2*i, 2*i+1] for i in range(7)]), np.arange(7))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0][0], [[0,1],[2,3],[4,5]]) npt.assert_array_equal(l[0][1], [0, 1, 2]) npt.assert_array_equal(l[1][0], [[6,7],[8,9],[10,11]]) npt.assert_array_equal(l[1][1], [3, 4, 5]) npt.assert_array_equal(l[2][0], [[12,13]]) npt.assert_array_equal(l[2][1], [6])
def dotrain(model, crit, aug, Xtr, ytr, nepochs=50, batchsize=100, title=None): opt = df.AdaDelta(rho=.95, eps=1e-7, lr=1) # progress = IntProgress(value=0, min=0, max=nepochs, description='Training:') # display(progress) model.training() costs = [] for e in range(nepochs): batchcosts = [] for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True): if aug is not None: Xb, yb = aug.augbatch_train(Xb, yb) model.zero_grad_parameters() cost = model.accumulate_gradients(Xb, yb, crit) opt.update_parameters(model) # print("batch cost: %f" % cost) batchcosts.append(cost) costs.append(np.mean(batchcosts)) print("mean batch cost: %f" % costs[-1]) # progress.value = e+1 # liveplot(plotcost, costs, title) return costs
def calcPreRec(_X, _scans, _wcs, _was, _r, ts): mylist1 = [] mylist2 = [] for Xb in lbu.batched(1, _X, shuf=False, droplast=True): Xbt = torch.Tensor(Xb).cuda() del Xb pred_y_conf0, pred_y_offs0 = net(Xbt) del Xbt pred_y_conf, pred_y_offs = pred_y_conf0.detach().cpu().numpy( ), pred_y_offs0.detach().cpu().numpy() del pred_y_offs0, pred_y_conf0 print(pred_y_conf.shape) m = input() mylist1.append(pred_y_offs) mylist2.append(pred_y_conf) del pred_y_conf, pred_y_offs pred_y_conf_batches = np.array(mylist2) pred_y_offs_batches = np.array(mylist1) del mylist1, mylist2 alldets, (wcdets, wadets) = pred2det_comb(_scans, pred_y_conf_batches, pred_y_offs_batches, thresh=ts, out_rphi=False) gt_all = [wcs + was for wcs, was in zip(_wcs, _was)] precs, recs = u.precrec(alldets, gt_all, _r, pred_rphi=False, gt_rphi=True) return precs, recs
def dostats(model, aug, Xtr, batchsize=100): model.training() for Xb in batched(batchsize, Xtr): if aug is None: model.accumulate_statistics(Xb) else: for Xb_aug in aug.augbatch_pred(Xb): model.accumulate_statistics(Xb_aug)
def dostats(model, aug, Xtr, batchsize=128): model.training() for Xb in batched(batchsize, Xtr): if aug is None: model.accumulate_statistics(Xb) else: for Xb_aug in aug.augbatch_pred(Xb): model.accumulate_statistics(Xb_aug)
def dopred(model, aug, X, ensembling, output2preds, batchsize=128): model.evaluate() y_preds = [] for Xb in batched(batchsize, X): if aug is None: p_y = model.forward(X) else: p_y = ensembling([model.forward(X) for X in aug.augbatch_pred(Xb)]) y_preds += list(output2preds(p_y)) return np.array(y_preds)
def dopred(model, aug, X, ensembling, output2preds, batchsize=100): model.evaluate() y_preds = [] for Xb in batched(batchsize, X): if aug is None: p_y = model.forward(X) else: p_y = ensembling([model.forward(X) for X in aug.augbatch_pred(Xb)]) y_preds += list(output2preds(p_y)) return np.array(y_preds)
def test_batched_shuf_2d(self): # Here we mostly test for the length of the batches and whether they # still got the same content, NOT their shapes. # Testing their shapes has been done in `test_Batched_2d` and this # should only test the shuffling. a6 = np.array([[2 * i, 2 * i + 1] for i in range(6)]) a7 = np.array([[2 * i, 2 * i + 1] for i in range(7)]) # 2D array with fitting size l = list(u.batched(3, a6, shuf=True)) self.assertEqual(len(l), 2) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l])), a6.flatten()) # 2D arrays with fitting size l = list(u.batched(3, a6, a6, shuf=True)) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) l1, l2 = zip(*l) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l1])), a6.flatten()) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l2])), a6.flatten()) # 2D array with leftover l = list(u.batched(3, a7, shuf=True)) self.assertEqual(len(l), 3) self.assertEqual(l[-1].shape, (1, 2)) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l])), a7.flatten()) # 2D arrays with leftover l = list(u.batched(3, a7, a7)) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) npt.assert_array_equal(l[2][0], l[2][1]) l1, l2 = zip(*l) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l1])), a7.flatten()) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l2])), a7.flatten())
def test_batched_shuf_2d(self): # Here we mostly test for the length of the batches and whether they # still got the same content, NOT their shapes. # Testing their shapes has been done in `test_Batched_2d` and this # should only test the shuffling. a6 = np.array([[2*i, 2*i+1] for i in range(6)]) a7 = np.array([[2*i, 2*i+1] for i in range(7)]) # 2D array with fitting size l = list(u.batched(3, a6, shuf=True)) self.assertEqual(len(l), 2) npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l])), a6.flatten()) # 2D arrays with fitting size l = list(u.batched(3, a6, a6, shuf=True)) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) l1, l2 = zip(*l) npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l1])), a6.flatten()) npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l2])), a6.flatten()) # 2D array with leftover l = list(u.batched(3, a7, shuf=True)) self.assertEqual(len(l), 3) self.assertEqual(l[-1].shape, (1,2)) npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l])), a7.flatten()) # 2D arrays with leftover l = list(u.batched(3, a7, a7)) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) npt.assert_array_equal(l[2][0], l[2][1]) l1, l2 = zip(*l) npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l1])), a7.flatten()) npt.assert_array_equal(sorted(np.concatenate([i.flatten() for i in l2])), a7.flatten())
def test_batched_2d(self): # 2D array with fitting size l = list(u.batched(3, np.array([[2 * i, 2 * i + 1] for i in range(6)]))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0], [[0, 1], [2, 3], [4, 5]]) npt.assert_array_equal(l[1], [[6, 7], [8, 9], [10, 11]]) # 2D arrays with fitting size l = list( u.batched(3, np.array([[2 * i, 2 * i + 1] for i in range(6)]), np.arange(6))) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], [[0, 1], [2, 3], [4, 5]]) npt.assert_array_equal(l[0][1], [0, 1, 2]) npt.assert_array_equal(l[1][0], [[6, 7], [8, 9], [10, 11]]) npt.assert_array_equal(l[1][1], [3, 4, 5]) # 2D array with leftover l = list(u.batched(3, np.array([[2 * i, 2 * i + 1] for i in range(7)]))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0], [[0, 1], [2, 3], [4, 5]]) npt.assert_array_equal(l[1], [[6, 7], [8, 9], [10, 11]]) npt.assert_array_equal(l[2], [[12, 13]]) # 2D arrays with leftover l = list( u.batched(3, np.array([[2 * i, 2 * i + 1] for i in range(7)]), np.arange(7))) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0][0], [[0, 1], [2, 3], [4, 5]]) npt.assert_array_equal(l[0][1], [0, 1, 2]) npt.assert_array_equal(l[1][0], [[6, 7], [8, 9], [10, 11]]) npt.assert_array_equal(l[1][1], [3, 4, 5]) npt.assert_array_equal(l[2][0], [[12, 13]]) npt.assert_array_equal(l[2][1], [6])
def test_batched_N_2d(self): # Here we mostly test for the length of the batches and whether they # still got the same content, NOT their shapes. # Testing their shapes has been done in `test_Batched_2d` and this # should only test the N. a6 = np.array([[2 * i, 2 * i + 1] for i in range(6)]) a7 = np.array([[2 * i, 2 * i + 1] for i in range(7)]) a8 = np.array([[2 * i, 2 * i + 1] for i in range(8)]) # 2D array with fitting size and N l = list(u.batched(3, a6, N=6)) self.assertEqual(len(l), 2) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l])), a6.flatten()) # 2D array with fitting N but too large size l = list(u.batched(3, a7, N=6)) self.assertEqual(len(l), 2) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l])), a6.flatten()) # 2D array with fitting size but too large N with self.assertRaises(AssertionError): l = list(u.batched(3, a6, N=7)) # 2D arrays with fitting size l = list(u.batched(3, a6, a6, N=6)) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) l1, l2 = zip(*l) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l1])), a6.flatten()) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l2])), a6.flatten()) # 2D array with leftover l = list(u.batched(3, a8, N=7)) self.assertEqual(len(l), 3) self.assertEqual(l[-1].shape, (1, 2)) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l])), a7.flatten()) # 2D arrays with leftover l = list(u.batched(3, a8, a7, N=7)) self.assertEqual(len(l), 3) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) npt.assert_array_equal(l[2][0], l[2][1]) l1, l2 = zip(*l) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l1])), a7.flatten()) npt.assert_array_equal( sorted(np.concatenate([i.flatten() for i in l2])), a7.flatten())
def dotrain(model, crit, aug, Xtr, ytr, nepochs=3, batchsize=128, title=None): opt = df.AdaDelta(rho=0.95, eps=1e-7, lr=1) model.training() costs = [] print("Training in progress...") for e in range(nepochs): print("Current epoch: {0} out of {1}".format(e + 1, nepochs)) batchcosts = [] for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True): if aug is not None: Xb, yb = aug.augbatch_train(Xb, yb) model.zero_grad_parameters() cost = model.accumulate_gradients(Xb, yb, crit) opt.update_parameters(model) batchcosts.append(cost) costs.append(np.mean(batchcosts)) return costs
def dotrain(model, crit, aug, Xtr, ytr, nepochs=3, batchsize=128, title=None): opt = df.AdaDelta(rho=.95, eps=1e-7, lr=1) model.training() costs = [] print("Training in progress...") for e in range(nepochs): print("Current epoch: {0} out of {1}".format(e + 1, nepochs)) batchcosts = [] for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True): if aug is not None: Xb, yb = aug.augbatch_train(Xb, yb) model.zero_grad_parameters() cost = model.accumulate_gradients(Xb, yb, crit) opt.update_parameters(model) batchcosts.append(cost) costs.append(np.mean(batchcosts)) return costs
def dotrain(model, sess, loss, aug, Xtr, ytr, nepochs=50, batchsize=100, title=None): opt = tf.train.AdadeltaOptimizer(learning_rate=1, rho=0.95, epsilon=1e-7).minimize(loss) # noinspection PyInterpreter progress = IntProgress(value=0, min=0, max=nepochs, description='Training:') display(progress) #model.training() costs = [] for e in range(nepochs): batchcosts = [] for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True): if aug is not None: Xb, yb = aug.augbatch_train(Xb, yb)[0] sess.run([opt, loss], feed_dict={model.X: Xb, model.y: yb}) # model.zero_grad_parameters() # cost = model.accumulate_gradients(Xb, yb, crit) # opt.update_parameters(model) batchcosts.append(cost) costs.append(np.mean(batchcosts)) progress.value = e + 1 liveplot(plotcost, costs, title) return costs
def dotrain(model, crit, aug, Xtr, ytr, nepochs=50, batchsize=100, title=None): opt = df.AdaDelta(rho=0.95, eps=1e-7, lr=1) progress = IntProgress(value=0, min=0, max=nepochs, description="Training:") display(progress) model.training() costs = [] for e in range(nepochs): batchcosts = [] for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True): if aug is not None: Xb, yb = aug.augbatch_train(Xb, yb) model.zero_grad_parameters() cost = model.accumulate_gradients(Xb, yb, crit) opt.update_parameters(model) batchcosts.append(cost) costs.append(np.mean(batchcosts)) progress.value = e + 1 liveplot(plotcost, costs, title) return costs
def test_batched_N_1d(self): # 1D array with fitting size and N l = list(u.batched(3, np.arange(6), N=6)) self.assertEqual(len(l), 2) npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(6)) # 1D array with fitting N but too large size l = list(u.batched(3, np.arange(8), N=6)) self.assertEqual(len(l), 2) npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(6)) # 1D array with fitting N but too small size with self.assertRaises(AssertionError): l = list(u.batched(3, np.arange(6), N=8)) # 1D arrays with fitting size l = list(u.batched(3, np.arange(7), np.arange(8), N=6)) self.assertEqual(len(l), 2) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])), np.arange(6)) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])), np.arange(6)) # 1D array with leftover l = list(u.batched(3, np.arange(8), N=7)) self.assertEqual(len(l), 3) self.assertEqual(len(l[-1]), 1) npt.assert_array_equal(sorted(np.concatenate(l)), np.arange(7)) # 1D arrays with leftover l = list(u.batched(3, np.arange(8), np.arange(9), N=7)) self.assertEqual(len(l), 3) self.assertEqual(len(l[-1][0]), 1) self.assertEqual(len(l[-1][1]), 1) npt.assert_array_equal(l[0][0], l[0][1]) npt.assert_array_equal(l[1][0], l[1][1]) npt.assert_array_equal(l[2][0], l[2][1]) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[0])), np.arange(7)) npt.assert_array_equal(sorted(np.concatenate(list(zip(*l))[1])), np.arange(7))