def test_derivative(f, df):
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(
get_network(), squared_cost, rms_error, eta=eta, alpha=alpha, name='BP')
bp_learner.weight_norms = []
fas_learner = FASkipLearner(
get_network(), squared_cost, rms_error, eta=eta, alpha=alpha, name='FA')
fas_learner.Bs = directBs
learners = [bp_learner, fas_learner]
for learner in learners:
learner.train(1, batch_fn)
for learner in learners:
print(", ".join("||W%d|| = %0.3f" % (i, norm(w))
for i, w in enumerate(learner.network.weights)))
return learners
def test_derivative(f, df):
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
# fa_learner = FALearner(
# get_network(), squared_cost, rms_error, eta=eta, alpha=alpha)
# fa_learner.Bs = [initial_w((j, i), kind='ortho', scale=2)
# for i, j in zip(dhids, dhids[1:] + [dout])]
# fa_learner.bp_angles = []
# # fa_learner.pbp_angles = []
fas_learner = FASkipLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='FA')
genB = lambda shape: initial_w(
shape, kind='ortho', normkind='rightmean', scale=0.2)
fas_learner.Bs = [genB((dout, dhid)) for dhid in dhids]
# learners = [bp_learner, fa_learner]
learners = [bp_learner, fas_learner]
for learner in learners:
learner.train(epochs, batch_fn, test_set=test_set)
for learner in learners:
print(", ".join("||W%d|| = %0.3f" % (i, norm(w))
for i, w in enumerate(learner.network.weights)))
return learners
def objective(args):
sargs = arg_string(args)
w_kind = args['w_kind']
w_scale = args['w_scale']
b_kind = args['b_kind']
# b_scale = args['b_scale']
eta = args['eta']
# alpha = args['alpha']
alpha = 0
# eta = [args['eta0'], args['eta1'], args['eta2']]
# max_cost = -np.inf
costs = []
for _ in range(5):
f, df = static_f_df(tau_rc=0.05, **args['neuron_type'])
weights = initial_weights(sizes, kind=w_kind, scale=w_scale, rng=rng)
# --- learners
network = Network(weights, f=f, df=df, biases=None)
# network = Network(weights, f=f, df=df, biases=None, noise=1.)
learner = Learner(network, squared_cost, rms_error, eta=eta, alpha=alpha)
# learner.Bs = [initial_w((dout, dhid), kind=b_kind, scale=b_scale) for dhid in dhids]
learner.Bs = [initial_w((dout, dhid), kind=b_kind, normkind='rightmean') for dhid in dhids]
learner.train(1, batch_fn, verbose=0)
y = learner.network.predict(Xvalid)
cost = rms(y - Yvalid, axis=1).mean() / Yvalidrms
costs.append(cost)
costs = sorted(costs)[1:-1] # drop largest and smallest
cost = np.mean(costs)
status = hyperopt.STATUS_OK if np.isfinite(cost) else hyperopt.STATUS_FAIL
print("%s: %0.3e" % (sargs, cost))
return dict(loss=cost, status=status)
def test_derivative(f, df):
# batch_fn = make_flat_batch_fn(X, Y, n_per_batch)
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
fas_learner = FASkipLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='FA')
fas_learner.Bs = directBs
learners = [bp_learner, fas_learner]
for learner in learners:
batch_fn = make_random_batch_fn(
trainX,
trainY,
n_per_batch,
rng=np.random.RandomState(trial_seed))
learner.train(epochs, batch_fn, test_set=test_set)
# for learner in learners:
# print(", ".join("||W%d|| = %0.3f" % (i, norm(w))
# for i, w in enumerate(learner.network.weights)))
return learners
print("B norms: %s" % ", ".join("%0.3f" % norm(B) for B in Bs))
print("Bc norms: %s" % ", ".join("%0.3f" % norm(B)
for B in combine_Bs(Bs)))
print("Bd norms: %s" % ", ".join("%0.3f" % norm(B) for B in Bs_direct))
# --- nonlinearity
tau_rc = 0.05
# amp = 0.01
amp = 0.025
# amp = 0.0253
f, df = static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp)
# f, df = static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp)
# --- learners
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
fa_learner = FALearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='global FA')
# eta = 1e-2
eta = 4e-3
# eta = 1e-3
# eta = 5e-4
# eta = 2e-4
# eta = 1e-4
# eta = 1e-5
# eta = 0
# alpha = 0
alpha = 1e-6
# --- network
noise = 0
# noise = 1.
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=noise, **kwargs)
# --- learners
shallow_learner = ShallowLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha)
bp_learner = BPLearner(get_network(), cost, error, eta=eta, alpha=alpha)
# bp_learner.weight_norms = []
# fa_learner = FALearner(
# get_network(), cost, error, eta=eta, alpha=alpha)
# fa_learner.Bs = [initial_w((j, i), kind='ortho', scale=2)
# for i, j in zip(dhids, dhids[1:] + [dout])]
# fa_learner.bp_angles = []
# weights[2][:] = np.dot(decoder(dhids[1]), encoder(dhids[2]))
# weights[3][:] = decoder(dhids[2])
f, df = static_f_df('relu')
# eta = 5e-2
# eta = 2e-2
# eta = 1e-2
# eta = 5e-3
# eta = 2e-3
eta = 1e-3
# momentum = 0
momentum = 0.9
get_network = lambda: Network(weights, f=f, df=df)
# network = Network(weights, biases=biases, f=f, df=df)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
momentum=momentum,
name='BP')
bpl_learner = BPLocalLearner(get_network(),
cost,
error,
eta=0.5 * eta,
momentum=momentum,
name='BPLocal')
# eta = 5e-3
# eta = 2e-3
# eta = 1e-3
# eta = 5e-4
# eta = 2e-4
eta = 1e-4
# momentum = 0
# momentum = 0.5
momentum = 0.9
if CONSTANT_INPUT:
weights[0] = np.vstack((weights[0], biases[0]))
weight_masks[0] = np.vstack((weight_masks[0], np.ones(dhids[0], dtype=bool)))
get_network = lambda: Network(weights, f=f, df=df, weight_masks=weight_masks)
else:
# get_network = lambda: Network(weights, f=f, df=df)
# get_network = lambda: Network(weights, biases=biases, f=f, df=df)
get_network = lambda: Network(weights, biases=biases, f=f, df=df, weight_masks=weight_masks)
network0 = get_network()
bp_learner = BPLearner(get_network(), cost, error,
eta=eta, momentum=momentum, name='BP')
bpl_learner = BPLocalLearner(get_network(), cost, error,
# eta=eta, momentum=momentum, name='local BP')
eta=0.1*eta, momentum=momentum, name='local BP')
# eta=0.05*eta, momentum=momentum, name='local BP')
globals().update(filedata)
print("Pretraining solver: %s" % solver)
weights = [encoders0.T * gain0, encoders1.T * gain1, decoders]
biases = [bias0 - 1, bias1 - 1, np.zeros(n_out)]
cost = nll_cost_on_inds
error = class_error_on_inds
f, df = static_f_df('liflinear',
tau_rc=neuron_type.tau_rc,
tau_ref=neuron_type.tau_ref,
amplitude=neuron_type.amplitude)
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=biases, noise=0, **kwargs)
reg = solver.reg
# reg = 0
alpha = (reg * h1.max())**2
# alpha = 1e-5
# alpha = 1e-6
# alpha = 0.00525450331251
# alpha = 0
print("alpha: %s" % alpha)
# network = get_network()
learner = ShallowLearner(get_network(), cost, error, eta=0, alpha=alpha)
train_cost = cost(train_out, trainY)[0].mean()