def main(): np.random.seed(1) xtrain, ytrain, params_true = make_data_linreg_1d() predict_fun = lambda x: LinregModel.prediction(params_true, x) plot_data_and_pred(xtrain, ytrain, predict_fun) loss_fun = lambda w0, w1: LinregModel.objective([w0, w1], xtrain, ytrain) plot_error_surface(loss_fun, params_true) plt.show()
def main(): for fun_type in ['linear', 'quad', 'sine']: np.random.seed(1) N = 20 Xtrain, Ytrain, params_true, true_fun, ttl = make_data_linreg_1d(N, fun_type) model = LinregModel(1, True) params_ols, loss_ols = model.ols_fit(Xtrain, Ytrain) print(ttl) print params_ols # Plot data predict_fun = lambda x: model.prediction(params_ols, x) ax = plot_data_and_predictions_1d(Xtrain, Ytrain, true_fun, predict_fun) ax.set_title(ttl) # Plot error surface loss_fun = lambda w0, w1: model.objective([w0, w1], Xtrain, Ytrain) ax = plot_error_surface_2d(loss_fun, params_ols, params_true, fun_type) ax.set_title(ttl) plt.show()
def main(): np.random.seed(1) folder = 'figures/linreg-sgd' N = 50 init_lr = 0.05 num_epochs = 50 #fun_type = 'linear' #fun_type = 'sine' fun_type = 'quad' #model_type = 'linear' model_type = 'mlp:1-10-1' configs = [] # BFGS has to be the first config, in order to compute loss_opt configs.append({'data_generator': fun_type, 'N': N, 'model': model_type, 'optimizer': 'BFGS'}) configs.append({'data_generator': fun_type, 'N': N, 'model': model_type, 'optimizer': 'SGD', 'batch_size': 10, 'num_epochs': num_epochs, 'init_lr': init_lr, 'lr_decay': 0.9, 'method': 'momentum', 'mass': 0}) configs.append({'data_generator': fun_type, 'N': N, 'model': model_type, 'optimizer': 'SGD', 'batch_size': 10, 'num_epochs': num_epochs, 'init_lr': init_lr, 'lr_decay': 0.9, 'method': 'momentum', 'mass': 0.9}) configs.append({'data_generator': fun_type, 'N': N, 'model': model_type, 'optimizer': 'SGD', 'batch_size': 10, 'num_epochs': num_epochs, 'init_lr': init_lr, 'lr_decay': 0.9, 'method': 'RMSprop', 'grad_sq_decay': 0.9}) configs.append({'data_generator': fun_type, 'N': N, 'model': model_type, 'optimizer': 'SGD', 'batch_size': 10, 'num_epochs': num_epochs, 'init_lr': init_lr, 'lr_decay': 0, 'method': 'ADAM', 'grad_decay': 0.9, 'grad_sq_decay': 0.999}) params_opt = None loss_opt = None for expt_num, config in enumerate(configs): np.random.seed(1) ttl = config_to_str(config) print '\nstarting experiment {}'.format(ttl) print config xtrain, Xtrain, ytrain, params_true, true_fun, fun_name = make_data_linreg_1d(config['N'], config['data_generator']) data_dim = Xtrain.shape[1] if model_type == 'linear': model = LinregModel(data_dim, add_ones=True) params_opt, loss_opt = model.ols_fit(Xtrain, ytrain) elif model_type[0:3] == 'mlp': _, layer_sizes = model_type.split(':') layer_sizes = [int(n) for n in layer_sizes.split('-')] model = MLP(layer_sizes, 'regression', L2_reg=0.001) else: raise ValueError('unknown model type {}'.format(model_type)) initial_params = model.init_params() obj_fun = model.objective grad_fun = autograd.grad(obj_fun) param_dim = len(initial_params) plot_data = (data_dim == 1) plot_params = (param_dim == 2) nplots = 2 if plot_data: nplots += 1 if plot_params: nplots += 1 plot_rows, plot_cols = util.nsubplots(nplots) if config['optimizer'] == 'BFGS': logger = MinimizeLogger(obj_fun, grad_fun, (Xtrain, ytrain), print_freq=1, store_params=True) params, loss = bfgs_fit(initial_params, obj_fun, grad_fun, (Xtrain, ytrain), logger.update) loss_avg = loss if params_opt is None: params_opt = params loss_opt = loss if config['optimizer'] == 'SGD': logger = sgd.SGDLogger(print_freq=20, store_params=True) lr_fun = lambda iter, epoch: sgd.lr_exp_decay(iter, config['init_lr'], config['lr_decay']) if config['method'] == 'momentum': sgd_updater = sgd.SGDMomentum(lr_fun, config['mass']) if config['method'] == 'RMSprop': sgd_updater = sgd.RMSprop(lr_fun, config['grad_sq_decay']) if config['method'] == 'ADAM': sgd_updater = sgd.ADAM(config['init_lr'], config['grad_decay'], config['grad_sq_decay']) params, params_avg, loss_on_batch = sgd.sgd_minimize(initial_params, obj_fun, grad_fun, Xtrain, ytrain, config['batch_size'], config['num_epochs'], sgd_updater, logger.update) loss = obj_fun(params, Xtrain, ytrain) loss_avg = obj_fun(params_avg, Xtrain, ytrain) fig = plt.figure() ax = fig.add_subplot(plot_rows, plot_cols, 1) plot_loss_trace(logger.obj_trace, loss_opt, ax) ax.set_title('final objective {:0.3f}, {:0.3f}'.format(loss, loss_avg)) ax = fig.add_subplot(plot_rows, plot_cols, 2) ax.plot(logger.grad_norm_trace) ax.set_title('gradient norm vs num updates') if plot_data: ax = fig.add_subplot(plot_rows, plot_cols, 3) predict_fun = lambda X: model.prediction(params, X) plot_data_and_predictions(xtrain, ytrain, true_fun, predict_fun, ax) if plot_params: ax = fig.add_subplot(plot_rows, plot_cols, 4) loss_fun = lambda w0, w1: model.objective([w0, w1], xtrain, ytrain) plot_error_surface(loss_fun, params_opt, params_true, fun_type, ax) plot_param_trace(logger.param_trace, ax) fig.suptitle(ttl) fname = os.path.join(folder, 'linreg_1d_sgd_{}.png'.format(ttl)) plt.savefig(fname) plt.show()