def test_derivative(f, df):
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
squared_cost,
rms_error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
fas_learner = FASkipLearner(get_network(),
squared_cost,
rms_error,
eta=eta,
alpha=alpha,
name='DFA')
fas_learner.Bs = directBs
learners = [bp_learner, fas_learner]
for learner in learners:
learner.train(1, batch_fn)
for learner in learners:
print(", ".join("||W%d|| = %0.3f" % (i, norm(w))
for i, w in enumerate(learner.network.weights)))
return learners
def test_derivative(f, df):
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
# fa_learner = FALearner(
# get_network(), squared_cost, rms_error, eta=eta, alpha=alpha)
# fa_learner.Bs = [initial_w((j, i), kind='ortho', scale=2)
# for i, j in zip(dhids, dhids[1:] + [dout])]
# fa_learner.bp_angles = []
# # fa_learner.pbp_angles = []
fas_learner = FASkipLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='FA')
genB = lambda shape: initial_w(
shape, kind='ortho', normkind='rightmean', scale=0.2)
fas_learner.Bs = [genB((dout, dhid)) for dhid in dhids]
# learners = [bp_learner, fa_learner]
learners = [bp_learner, fas_learner]
for learner in learners:
learner.train(epochs, batch_fn, test_set=test_set)
for learner in learners:
print(", ".join("||W%d|| = %0.3f" % (i, norm(w))
for i, w in enumerate(learner.network.weights)))
return learners
def test_derivative(f, df):
# batch_fn = make_flat_batch_fn(X, Y, n_per_batch)
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
fas_learner = FASkipLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='FA')
fas_learner.Bs = directBs
learners = [bp_learner, fas_learner]
for learner in learners:
batch_fn = make_random_batch_fn(
trainX,
trainY,
n_per_batch,
rng=np.random.RandomState(trial_seed))
learner.train(epochs, batch_fn, test_set=test_set)
# for learner in learners:
# print(", ".join("||W%d|| = %0.3f" % (i, norm(w))
# for i, w in enumerate(learner.network.weights)))
return learners
# --- nonlinearity
tau_rc = 0.05
# amp = 0.01
amp = 0.025
# amp = 0.0253
f, df = static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp)
# f, df = static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp)
# --- learners
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
fa_learner = FALearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='global FA')
fa_learner.Bs = [np.array(B) for B in Bs]
fal_learner = FALocalLearner(get_network(),
cost,
error,
# alpha = 0
alpha = 1e-6
# --- network
noise = 0
# noise = 1.
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=noise, **kwargs)
# --- learners
shallow_learner = ShallowLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha)
bp_learner = BPLearner(get_network(), cost, error, eta=eta, alpha=alpha)
# bp_learner.weight_norms = []
# fa_learner = FALearner(
# get_network(), cost, error, eta=eta, alpha=alpha)
# fa_learner.Bs = [initial_w((j, i), kind='ortho', scale=2)
# for i, j in zip(dhids, dhids[1:] + [dout])]
# fa_learner.bp_angles = []
fas_learner = FASkipLearner(get_network(), cost, error, eta=eta, alpha=alpha)
fas_learner.Bs = [
initial_w((dout, dhid), kind='ortho', scale=2) for dhid in dhids
]
# epochs = 1
# epochs = 3
# eta = 5e-2
# eta = 2e-2
# eta = 1e-2
# eta = 5e-3
# eta = 2e-3
eta = 1e-3
# momentum = 0
momentum = 0.9
get_network = lambda: Network(weights, f=f, df=df)
# network = Network(weights, biases=biases, f=f, df=df)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
momentum=momentum,
name='BP')
bpl_learner = BPLocalLearner(get_network(),
cost,
error,
eta=0.5 * eta,
momentum=momentum,
name='BPLocal')
fas_learner = FASkipLearner(
get_network(),
cost,
error,
# eta=eta, momentum=momentum, name='FA')
# --- nonlinearity
tau_rc = 0.05
# amp = 0.01
amp = 0.025
# f, df = static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp)
f, df = static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp)
# --- learners
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
squared_cost,
rms_error,
eta=eta,
alpha=alpha,
name='BP')
# bp_learner.weight_norms = []
bp_learner.delta_norms = []
fa_learner = FALearner(get_network(),
squared_cost,
rms_error,
eta=eta,
alpha=alpha,
name='FA')
fa_learner.Bs = Bs
fa_learner.delta_norms = []
fa_learner.bp_angles = []
fa_learner.bpd_angles = []