def objective(args): sargs = arg_string(args) w_kind = args['w_kind'] w_scale = args['w_scale'] eta = args['eta'] # eta = [args['eta0'], args['eta1'], args['eta2']] max_cost = -np.inf for _ in range(5): f, df = static_f_df(tau_rc=0.05, **args['neuron_type']) weights = initial_weights(sizes, kind=w_kind, scale=w_scale, rng=rng) # weights = initial_weights(sizes, kind='ortho', rng=rng) # lsuv(X, weights, f, target_input=True, target_std=1, verbose=1) # lsuv(X[:100], weights, f, target_input=True, target_std=1, verbose=1) # --- learners network = Network(weights, f=f, df=df, biases=None) # network = Network(weights, f=f, df=df, biases=None, noise=1.) learner = Learner(network, squared_cost, rms_error, eta=eta, alpha=alpha, momentum=momentum) learner.train(1, batch_fn, verbose=0) y = learner.network.predict(Xvalid) mean_cost = rms(y - Yvalid, axis=1).mean() max_cost = max(max_cost, mean_cost) print("%s: %0.3e" % (sargs, max_cost)) return max_cost
def objective(args): sargs = arg_string(args) w_kind = args['w_kind'] w_scale = args['w_scale'] b_kind = args['b_kind'] # b_scale = args['b_scale'] eta = args['eta'] # alpha = args['alpha'] alpha = 0 # eta = [args['eta0'], args['eta1'], args['eta2']] # max_cost = -np.inf costs = [] for _ in range(5): f, df = static_f_df(tau_rc=0.05, **args['neuron_type']) weights = initial_weights(sizes, kind=w_kind, scale=w_scale, rng=rng) # --- learners network = Network(weights, f=f, df=df, biases=None) # network = Network(weights, f=f, df=df, biases=None, noise=1.) learner = Learner(network, squared_cost, rms_error, eta=eta, alpha=alpha) # learner.Bs = [initial_w((dout, dhid), kind=b_kind, scale=b_scale) for dhid in dhids] learner.Bs = [initial_w((dout, dhid), kind=b_kind, normkind='rightmean') for dhid in dhids] learner.train(1, batch_fn, verbose=0) y = learner.network.predict(Xvalid) cost = rms(y - Yvalid, axis=1).mean() / Yvalidrms costs.append(cost) costs = sorted(costs)[1:-1] # drop largest and smallest cost = np.mean(costs) status = hyperopt.STATUS_OK if np.isfinite(cost) else hyperopt.STATUS_FAIL print("%s: %0.3e" % (sargs, cost)) return dict(loss=cost, status=status)
# --- problem dataset T = orthogonalize(rng.normal(size=(din, dout))) genX = lambda n: rng.normal(scale=1., size=(n, din)) genY = lambda X: np.dot(X, T) Xref = genX(1000) Yref = genY(Xref) Yrms = rms(Yref, axis=1).mean() # --- nonlinearity tau_rc = 0.05 amp = 0.01 f_dfs = [ static_f_df('lifnone', tau_rc=tau_rc, amplitude=amp), static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp), static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp), static_f_df('lifclip', tau_rc=tau_rc, amplitude=amp, clip=1), static_f_df('lifsoftlif', tau_rc=tau_rc, amplitude=amp, sigma=0.146), # static_f_df('lifclip', tau_rc=tau_rc, amplitude=amp, clip=2), # static_f_df('lifsoftlif', tau_rc=tau_rc, amplitude=amp, sigma=0.02), ] f_df_labels = ['none', 'step', 'linearized', 'clipped', 'softlif'] # f_df_labels = ['none', 'step', 'linearized', 'clipped', 'softlif', 'clipped', 'softlif'] assert len(f_dfs) == len(f_df_labels) etas = [0.02, 0.05, 0.1, 0.2] learner_names = None errors = {} # [eta][itrial][ifdf][ilearner]
for B, Bc, Bd in zip(Bs, combine_Bs(Bs), Bs_direct): B *= norm(B) / norm(Bc) print("B norms: %s" % ", ".join("%0.3f" % norm(B) for B in Bs)) print("Bc norms: %s" % ", ".join("%0.3f" % norm(B) for B in combine_Bs(Bs))) print("Bd norms: %s" % ", ".join("%0.3f" % norm(B) for B in Bs_direct)) # --- nonlinearity tau_rc = 0.05 # amp = 0.01 amp = 0.025 # amp = 0.0253 f, df = static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp) # f, df = static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp) # --- learners get_network = lambda **kwargs: Network( weights, f=f, df=df, biases=None, noise=0, **kwargs) bp_learner = BPLearner(get_network(), cost, error, eta=eta, alpha=alpha, name='BP') bp_learner.weight_norms = [] fa_learner = FALearner(get_network(),
def cost(y, yinds, pointers=pointers): return pointer_squared_cost_on_inds(y, yinds, pointers) def error(y, yinds, pointers=pointers): return pointer_class_error_on_inds(y, yinds, pointers) din = trainX.shape[1] sizes = [din] + dhids + [dout] batch_fn = make_flat_batch_fn(trainX, trainY, n_per_batch) # weights = initial_weights(sizes, kind='uniform', scale=0.001, rng=rng) weights = initial_weights(sizes, kind='ortho', scale=1, rng=rng) f, df = static_f_df('liflinear', tau_rc=0.05, amplitude=0.024) # eta = 1e-1 # eta = 5e-2 # eta = 2e-2 # eta = 1e-2 eta = 4e-3 # eta = 1e-3 # eta = 5e-4 # eta = 2e-4 # eta = 1e-4 # eta = 1e-5 # eta = 0 # alpha = 0 alpha = 1e-6
def encoder(n): W = np.zeros((n // 2, n)) for i in range(n // 4): W[2 * i:2 * (i + 1), 4 * i:4 * (i + 1)] = np.array([[1, 1], [-1, -1], [1, -1], [-1, 1]]).T return W # weights[0][:] = encoder(dhids[0]) # weights[1][:] = np.dot(decoder(dhids[0]), encoder(dhids[1])) # weights[2][:] = np.dot(decoder(dhids[1]), encoder(dhids[2])) # weights[3][:] = decoder(dhids[2]) f, df = static_f_df('relu') # eta = 5e-2 # eta = 2e-2 # eta = 1e-2 # eta = 5e-3 # eta = 2e-3 eta = 1e-3 # momentum = 0 momentum = 0.9 get_network = lambda: Network(weights, f=f, df=df) # network = Network(weights, biases=biases, f=f, df=df) bp_learner = BPLearner(get_network(), cost,
Yrms = rms(Y, axis=1).mean() Ytestrms = rms(Ytest, axis=1).mean() cost = squared_cost error = rms_error # --- neuron tau_rc = 0.05 # amp = 0.01 amp = 0.025 # amp = 0.0253 # f, df = static_f_df('lifsoftlif', tau_rc=tau_rc, amplitude=amp) # f, df = static_f_df('liflinear', tau_rc=tau_rc, amplitude=amp, damplitude=2) f, df = static_f_df('lifstep', tau_rc=tau_rc, amplitude=amp, damplitude=2) x = np.linspace(-1, 1, 10001) print("df max: %0.3f" % df(x).max()) # --- Experiment n_trials = 5 # n_trials = 1 trial_weights = [] for i_trial in range(n_trials): # --- initial weights # weights = initial_weights(sizes, kind='gaussian', scale=1e-1, rng=rng) weights = initial_weights(sizes, kind='gaussian', scale=5e-2, rng=rng) # weights = initial_weights(sizes, kind='gaussian', scale=1e-2, rng=rng)
print("Saved %r" % filename) else: with open(filename, 'rb') as fh: filedata = dill.load(fh) globals().update(filedata) print("Pretraining solver: %s" % solver) weights = [encoders0.T * gain0, encoders1.T * gain1, decoders] biases = [bias0 - 1, bias1 - 1, np.zeros(n_out)] cost = nll_cost_on_inds error = class_error_on_inds f, df = static_f_df('liflinear', tau_rc=neuron_type.tau_rc, tau_ref=neuron_type.tau_ref, amplitude=neuron_type.amplitude) get_network = lambda **kwargs: Network( weights, f=f, df=df, biases=biases, noise=0, **kwargs) reg = solver.reg # reg = 0 alpha = (reg * h1.max())**2 # alpha = 1e-5 # alpha = 1e-6 # alpha = 0.00525450331251 # alpha = 0 print("alpha: %s" % alpha)