def objective(args):
sargs = arg_string(args)
w_kind = args['w_kind']
w_scale = args['w_scale']
eta = args['eta']
# eta = [args['eta0'], args['eta1'], args['eta2']]
max_cost = -np.inf
for _ in range(5):
f, df = static_f_df(tau_rc=0.05, **args['neuron_type'])
weights = initial_weights(sizes, kind=w_kind, scale=w_scale, rng=rng)
# weights = initial_weights(sizes, kind='ortho', rng=rng)
# lsuv(X, weights, f, target_input=True, target_std=1, verbose=1)
# lsuv(X[:100], weights, f, target_input=True, target_std=1, verbose=1)
# --- learners
network = Network(weights, f=f, df=df, biases=None)
# network = Network(weights, f=f, df=df, biases=None, noise=1.)
learner = Learner(network,
squared_cost,
rms_error,
eta=eta,
alpha=alpha,
momentum=momentum)
learner.train(1, batch_fn, verbose=0)
y = learner.network.predict(Xvalid)
mean_cost = rms(y - Yvalid, axis=1).mean()
max_cost = max(max_cost, mean_cost)
print("%s: %0.3e" % (sargs, max_cost))
return max_cost
def objective(args):
sargs = arg_string(args)
w_kind = args['w_kind']
w_scale = args['w_scale']
b_kind = args['b_kind']
# b_scale = args['b_scale']
eta = args['eta']
# alpha = args['alpha']
alpha = 0
# eta = [args['eta0'], args['eta1'], args['eta2']]
# max_cost = -np.inf
costs = []
for _ in range(5):
f, df = static_f_df(tau_rc=0.05, **args['neuron_type'])
weights = initial_weights(sizes, kind=w_kind, scale=w_scale, rng=rng)
# --- learners
network = Network(weights, f=f, df=df, biases=None)
# network = Network(weights, f=f, df=df, biases=None, noise=1.)
learner = Learner(network, squared_cost, rms_error, eta=eta, alpha=alpha)
# learner.Bs = [initial_w((dout, dhid), kind=b_kind, scale=b_scale) for dhid in dhids]
learner.Bs = [initial_w((dout, dhid), kind=b_kind, normkind='rightmean') for dhid in dhids]
learner.train(1, batch_fn, verbose=0)
y = learner.network.predict(Xvalid)
cost = rms(y - Yvalid, axis=1).mean() / Yvalidrms
costs.append(cost)
costs = sorted(costs)[1:-1] # drop largest and smallest
cost = np.mean(costs)
status = hyperopt.STATUS_OK if np.isfinite(cost) else hyperopt.STATUS_FAIL
print("%s: %0.3e" % (sargs, cost))
return dict(loss=cost, status=status)
# --- problem dataset
T = orthogonalize(rng.normal(size=(din, dout)))
genX = lambda n: rng.normal(scale=1., size=(n, din))
genY = lambda X: np.dot(X, T)
Xref = genX(1000)
Yref = genY(Xref)
Yrms = rms(Yref, axis=1).mean()
# --- nonlinearity
tau_rc = 0.05
amp = 0.01
f_dfs = [
static_f_df('lifnone', tau_rc=tau_rc, amplitude=amp),
static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp),
static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp),
static_f_df('lifclip', tau_rc=tau_rc, amplitude=amp, clip=1),
static_f_df('lifsoftlif', tau_rc=tau_rc, amplitude=amp, sigma=0.146),
# static_f_df('lifclip', tau_rc=tau_rc, amplitude=amp, clip=2),
# static_f_df('lifsoftlif', tau_rc=tau_rc, amplitude=amp, sigma=0.02),
]
f_df_labels = ['none', 'step', 'linearized', 'clipped', 'softlif']
# f_df_labels = ['none', 'step', 'linearized', 'clipped', 'softlif', 'clipped', 'softlif']
assert len(f_dfs) == len(f_df_labels)
etas = [0.02, 0.05, 0.1, 0.2]
learner_names = None
errors = {} # [eta][itrial][ifdf][ilearner]
for B, Bc, Bd in zip(Bs, combine_Bs(Bs), Bs_direct):
B *= norm(B) / norm(Bc)
print("B norms: %s" % ", ".join("%0.3f" % norm(B) for B in Bs))
print("Bc norms: %s" % ", ".join("%0.3f" % norm(B)
for B in combine_Bs(Bs)))
print("Bd norms: %s" % ", ".join("%0.3f" % norm(B) for B in Bs_direct))
# --- nonlinearity
tau_rc = 0.05
# amp = 0.01
amp = 0.025
# amp = 0.0253
f, df = static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp)
# f, df = static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp)
# --- learners
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=None, noise=0, **kwargs)
bp_learner = BPLearner(get_network(),
cost,
error,
eta=eta,
alpha=alpha,
name='BP')
bp_learner.weight_norms = []
fa_learner = FALearner(get_network(),
def cost(y, yinds, pointers=pointers):
return pointer_squared_cost_on_inds(y, yinds, pointers)
def error(y, yinds, pointers=pointers):
return pointer_class_error_on_inds(y, yinds, pointers)
din = trainX.shape[1]
sizes = [din] + dhids + [dout]
batch_fn = make_flat_batch_fn(trainX, trainY, n_per_batch)
# weights = initial_weights(sizes, kind='uniform', scale=0.001, rng=rng)
weights = initial_weights(sizes, kind='ortho', scale=1, rng=rng)
f, df = static_f_df('liflinear', tau_rc=0.05, amplitude=0.024)
# eta = 1e-1
# eta = 5e-2
# eta = 2e-2
# eta = 1e-2
eta = 4e-3
# eta = 1e-3
# eta = 5e-4
# eta = 2e-4
# eta = 1e-4
# eta = 1e-5
# eta = 0
# alpha = 0
alpha = 1e-6
def encoder(n):
W = np.zeros((n // 2, n))
for i in range(n // 4):
W[2 * i:2 * (i + 1), 4 * i:4 * (i + 1)] = np.array([[1, 1], [-1, -1],
[1, -1], [-1,
1]]).T
return W
# weights[0][:] = encoder(dhids[0])
# weights[1][:] = np.dot(decoder(dhids[0]), encoder(dhids[1]))
# weights[2][:] = np.dot(decoder(dhids[1]), encoder(dhids[2]))
# weights[3][:] = decoder(dhids[2])
f, df = static_f_df('relu')
# eta = 5e-2
# eta = 2e-2
# eta = 1e-2
# eta = 5e-3
# eta = 2e-3
eta = 1e-3
# momentum = 0
momentum = 0.9
get_network = lambda: Network(weights, f=f, df=df)
# network = Network(weights, biases=biases, f=f, df=df)
bp_learner = BPLearner(get_network(),
cost,
Yrms = rms(Y, axis=1).mean()
Ytestrms = rms(Ytest, axis=1).mean()
cost = squared_cost
error = rms_error
# --- neuron
tau_rc = 0.05
# amp = 0.01
amp = 0.025
# amp = 0.0253
# f, df = static_f_df('lifsoftlif', tau_rc=tau_rc, amplitude=amp)
# f, df = static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp, damplitude=2)
f, df = static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp, damplitude=2)
x = np.linspace(-1, 1, 10001)
print("df max: %0.3f" % df(x).max())
# --- Experiment
n_trials = 5
# n_trials = 1
trial_weights = []
for i_trial in range(n_trials):
# --- initial weights
# weights = initial_weights(sizes, kind='gaussian', scale=1e-1, rng=rng)
weights = initial_weights(sizes, kind='gaussian', scale=5e-2, rng=rng)
# weights = initial_weights(sizes, kind='gaussian', scale=1e-2, rng=rng)
print("Saved %r" % filename)
else:
with open(filename, 'rb') as fh:
filedata = dill.load(fh)
globals().update(filedata)
print("Pretraining solver: %s" % solver)
weights = [encoders0.T * gain0, encoders1.T * gain1, decoders]
biases = [bias0 - 1, bias1 - 1, np.zeros(n_out)]
cost = nll_cost_on_inds
error = class_error_on_inds
f, df = static_f_df('liflinear',
tau_rc=neuron_type.tau_rc,
tau_ref=neuron_type.tau_ref,
amplitude=neuron_type.amplitude)
get_network = lambda **kwargs: Network(
weights, f=f, df=df, biases=biases, noise=0, **kwargs)
reg = solver.reg
# reg = 0
alpha = (reg * h1.max())**2
# alpha = 1e-5
# alpha = 1e-6
# alpha = 0.00525450331251
# alpha = 0
print("alpha: %s" % alpha)