def test_adadelta_lr():
obj = LogisticRegression()
args = itertools.repeat(((obj.X, obj.Z), {}))
opt = Adadelta(obj.pars, obj.fprime, 0.9, args=args)
for i, info in enumerate(opt):
print obj.f(opt.wrt, obj.X, obj.Z)
if i > 3000:
break
assert obj.solved(0.15), 'did not find solution'
def test_radagrad_lr():
seed = 54
random.seed(seed)
n_samples = 100
n_dim = 40
n_classes = 3
obj_rada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
print obj_rada.X.shape
obj_ada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
obj_dada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
obj_fada = LogisticRegression(n_samples=n_samples, n_inpt=n_dim, n_classes=n_classes, seed=seed)
ridx = random.sample(xrange(obj_rada.X.shape[0]), obj_rada.X.shape[0])
# ridx = xrange(obj_rada.X.shape[0])
# print ridx
eta = 0.5
eta_rada = 0.5
delta = 0.001
# k = obj_rada.pars.shape[0]
k = int(np.sqrt(n_dim) + 1)
print n_dim, k
opt_rada = Radagrad(obj_rada.pars, obj_rada.fprime, eta_rada, 0.001, delta, k, n_classes=n_classes, args=itertools.repeat(((obj_rada.X[ridx], obj_rada.Z[ridx]), {})))
opt_ada = Adadelta(obj_ada.pars, obj_ada.fprime, 0.9, args=itertools.repeat(((obj_ada.X[ridx], obj_ada.Z[ridx]), {})))
opt_dada = Adagrad(obj_dada.pars, obj_dada.fprime, eta, delta, args=itertools.repeat(((obj_dada.X[ridx], obj_dada.Z[ridx]), {})))
opt_fada = AdagradFull(obj_fada.pars, obj_fada.fprime, eta, 0.001, delta, n_classes=n_classes, args=itertools.repeat(((obj_rada.X[ridx], obj_rada.Z[ridx]), {})))
rada_loss, ada_loss, dada_loss, fada_loss = [], [], [], []
# print opt_ada.wrt
# print obj_ada.pars
for i, info in enumerate(opt_ada):
# print info
ada_loss += [obj_ada.f(opt_ada.wrt, obj_ada.X, obj_ada.Z)]
if i > n_samples * 6:
break
# print opt_dada.wrt
# print obj_dada.pars
for i, info in enumerate(opt_dada):
# print info
dada_loss += [obj_dada.f(opt_dada.wrt, obj_dada.X, obj_dada.Z)]
if i > n_samples * 6:
break
# print opt_rada.wrt
# print obj_rada.pars
for i, info in enumerate(opt_rada):
# print info['args'][0][i]
rada_loss += [obj_rada.f(opt_rada.wrt, obj_rada.X, obj_rada.Z)]
if i > n_samples * 6:
break
for i, info in enumerate(opt_fada):
# print info['args'][0][i]
fada_loss += [obj_rada.f(opt_fada.wrt, obj_fada.X, obj_fada.Z)]
if i > n_samples * 6:
break
plt.plot(rada_loss, '-r')
plt.plot(ada_loss, '-b')
plt.plot(dada_loss, '-g')
plt.plot(fada_loss, '-k')
plt.show()
