def test_bfgs_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Bfgs(obj.pars, obj.f, obj.fprime, args=args)
for i, info in enumerate(opt):
if i > 50:
break
assert obj.solved(), 'did not find solution'
def test_xnes_lr():
obj = LogisticRegression(seed=10101)
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Xnes(obj.pars, obj.f, args=args)
for i, info in enumerate(opt):
if i > 100:
break
assert obj.solved(), "did not find solution"
def test_smd_lr():
obj = LogisticRegression(seed=10101)
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Smd(obj.pars, obj.f, obj.fprime, obj.f_Hp, args=args, eta0=0.1)
for i, info in enumerate(opt):
if i > 150:
break
assert obj.solved(), 'did not find solution'
def test_nesterov_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Nesterov(obj.pars, obj.fprime, steprate=0.1, args=args)
for i, info in enumerate(opt):
if i > 750:
break
assert obj.solved(), 'did not find solution'
def test_ncg_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = NonlinearConjugateGradient(obj.pars, obj.f, obj.fprime, args=args)
for i, info in enumerate(opt):
if i > 50:
break
assert obj.solved(), 'did not find solution'
def test_xnes_lr():
obj = LogisticRegression(seed=10101)
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Xnes(obj.pars, obj.f, args=args)
for i, info in enumerate(opt):
if i > 100:
break
assert obj.solved(), 'did not find solution'
def test_asgd_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Asgd(obj.pars, obj.fprime, eta0=0.2, lmbd=1e-2, t0=0.1, args=args)
for i, info in enumerate(opt):
if i > 3000:
break
assert obj.solved(0.15), 'did not find solution'
def test_asgd_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Asgd(obj.pars, obj.fprime, eta0=0.2, lmbd=1e-2, t0=0.1, args=args)
for i, info in enumerate(opt):
if i > 3000:
break
assert obj.solved(0.15), 'did not find solution'
def test_sbfgs_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Sbfgs(obj.pars, obj.f, obj.fprime, args=args)
for i, info in enumerate(opt):
if i > 50:
break
assert obj.solved(), 'did not find solution'
def test_ncg_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = NonlinearConjugateGradient(obj.pars, obj.f, obj.fprime, args=args)
for i, info in enumerate(opt):
if i > 50:
break
assert obj.solved(), 'did not find solution'
def test_gd_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = GradientDescent(
obj.pars, obj.fprime, step_rate=0.01, momentum=.9, args=args)
for i, info in enumerate(opt):
if i > 500:
break
assert obj.solved(), 'did not find solution'
def test_rprop_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Rprop(obj.pars, obj.f, obj.fprime, step_shrink=0.1, step_grow=1.2,
min_step=1e-6, max_step=0.1, args=args)
for i, info in enumerate(opt):
if i > 500:
break
assert obj.solved(), 'did not find solution'
def test_adadelta_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Adadelta(obj.pars, obj.fprime, 0.9, args=args)
for i, info in enumerate(opt):
print obj.f(opt.wrt, obj.X, obj.Z)
if i > 3000:
break
assert obj.solved(0.15), 'did not find solution'
def test_gd_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = GradientDescent(obj.pars,
obj.fprime,
step_rate=0.01,
momentum=.9,
args=args)
for i, info in enumerate(opt):
if i > 500:
break
assert obj.solved(), 'did not find solution'
def test_rprop_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Rprop(obj.pars,
obj.fprime,
step_shrink=0.1,
step_grow=1.2,
min_step=1e-6,
max_step=0.1,
args=args)
for i, info in enumerate(opt):
if i > 500:
break
assert obj.solved(), 'did not find solution'
def test_rmsprop_continue():
obj = LogisticRegression(n_inpt=2, n_classes=2)
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = RmsProp(
obj.pars, obj.fprime, step_rate=0.01, momentum=.9, decay=0.9,
args=args)
continuation(opt)
def test_gd_continue():
obj = LogisticRegression(n_inpt=2, n_classes=2)
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = GradientDescent(obj.pars,
obj.fprime,
step_rate=0.01,
momentum=.9,
momentum_type='nesterov',
args=args)
continuation(opt)
def test_rprop_continue():
obj = LogisticRegression(n_inpt=2, n_classes=2)
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Rprop(obj.pars,
obj.fprime,
step_shrink=0.1,
step_grow=1.2,
min_step=1e-6,
max_step=0.1,
args=args)
continuation(opt)
def test_adadelta_continue():
obj = LogisticRegression(n_inpt=2, n_classes=2)
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Adadelta(obj.pars, obj.fprime, 0.9, args=args)
continuation(opt)
def test_radagrad_lr():
seed = 54
random.seed(seed)
n_samples = 100
n_dim = 40
n_classes = 3
obj_rada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
print obj_rada.X.shape
obj_ada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
obj_dada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
obj_fada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
ridx = random.sample(xrange(obj_rada.X.shape[0]), obj_rada.X.shape[0])
# ridx = xrange(obj_rada.X.shape[0])
# print ridx
eta = 0.5
eta_rada = 0.5
delta = 0.001
# k = obj_rada.pars.shape[0]
k = int(np.sqrt(n_dim) + 1)
print n_dim, k
opt_rada = Radagrad(obj_rada.pars, obj_rada.fprime, eta_rada, 0.001, delta, k, n_classes=n_classes, args=itertools.repeat(((obj_rada.X[ridx], obj_rada.Z[ridx]), {})))
opt_ada = Adadelta(obj_ada.pars, obj_ada.fprime, 0.9, args=itertools.repeat(((obj_ada.X[ridx], obj_ada.Z[ridx]), {})))
opt_dada = Adagrad(obj_dada.pars, obj_dada.fprime, eta, delta, args=itertools.repeat(((obj_dada.X[ridx], obj_dada.Z[ridx]), {})))
opt_fada = AdagradFull(obj_fada.pars, obj_fada.fprime, eta, 0.001, delta, n_classes=n_classes, args=itertools.repeat(((obj_rada.X[ridx], obj_rada.Z[ridx]), {})))
rada_loss, ada_loss, dada_loss, fada_loss = [], [], [], []
# print opt_ada.wrt
# print obj_ada.pars
for i, info in enumerate(opt_ada):
# print info
ada_loss += [obj_ada.f(opt_ada.wrt, obj_ada.X, obj_ada.Z)]
if i > n_samples * 6:
break
# print opt_dada.wrt
# print obj_dada.pars
for i, info in enumerate(opt_dada):
# print info
dada_loss += [obj_dada.f(opt_dada.wrt, obj_dada.X, obj_dada.Z)]
if i > n_samples * 6:
break
# print opt_rada.wrt
# print obj_rada.pars
for i, info in enumerate(opt_rada):
# print info['args'][0][i]
rada_loss += [obj_rada.f(opt_rada.wrt, obj_rada.X, obj_rada.Z)]
if i > n_samples * 6:
break
for i, info in enumerate(opt_fada):
# print info['args'][0][i]
fada_loss += [obj_rada.f(opt_fada.wrt, obj_fada.X, obj_fada.Z)]
if i > n_samples * 6:
break
plt.plot(rada_loss, '-r')
plt.plot(ada_loss, '-b')
plt.plot(dada_loss, '-g')
plt.plot(fada_loss, '-k')
plt.show()