def compute_integrated_gradients(inp, baseline, net, target, n_steps=100): """Compute integrated gradients. Parameters ---------- inp : torch.Tensor Input image (single image batch) of shape `(1, 3, *, *)`. baseline : torch.Tensor Basline image of the same shape as the `inp`. net : torch.nn.Module Classifier network. target : int Imagenet ground truth label id. n_steps : int Number of steps between the `inp` and `baseline` tensors. Returns ------- ig : torch.Tensor Integrated gradients with the same shape as the `inp`. inp_grad : torch.Tensor Gradient with respect to the `inp` tensor. Same shape as `inp`. """ path = [baseline + a * (inp - baseline) for a in np.linspace(0, 1, n_steps)] grads = [compute_gradient(func, x, net=net, target=target) for x in path] ig = (inp - baseline) * torch.cat(grads[:-1]).mean(dim=0, keepdims=True) return ig, grads[-1]
def confine_vorticity(self): if self.vorticity != 0: w = utils.compute_curl(self.v) grad_abs_w = 2 * utils.compute_gradient(np.abs(w)) grad_abs_w /= np.linalg.norm(grad_abs_w, axis=-1, keepdims=True) + 1e-5 f_conf = self.dt * self.vorticity * w[:, :, np.newaxis] * grad_abs_w self.v[2:-2, 2:-2] += f_conf[2:-2, 2:-2]
def attack(tensor, net, eps=1e-3, n_iter=50): """Run the Fast Sign Gradient Method (FSGM) attack. Parameters ---------- tensor : torch.Tensor The input image of shape `(1, 3, 224, 224)`. net : torch.nn.Module Classifier network. eps : float Determines how much we modify the image in a single iteration. n_iter : int Number of iterations. Returns ------- new_tensor : torch.Tensor New image that is a modification of the input image that "fools" the classifier. """ new_tensor = tensor.detach().clone() orig_prediction = net(tensor).argmax() print(f"Original prediction: {orig_prediction.item()}") for i in range(n_iter): net.zero_grad() grad = compute_gradient(func, new_tensor, net=net, target=orig_prediction.item()) new_tensor = torch.clamp(new_tensor + eps * grad.sign(), -2, 2) new_prediction = net(new_tensor).argmax() if orig_prediction != new_prediction: print(f"We fooled the network after {i} iterations!") print(f"New prediction: {new_prediction.item()}") break return new_tensor, orig_prediction.item(), new_prediction.item()
nIter = 10000 ws_star = np.random.randn(d, n_node) #ws_star[:n_node,:n_node] = np.eye(n_node) ws_star_norm = np.linalg.norm(ws_star, axis=0) ratio = 0.5 ws0 = np.copy(ws_star) + np.random.randn(d, n_node) * ratio nIter = 1000 ws = np.copy(ws0) ws_all = np.zeros((d, n_node, nIter)) eta = 0.01 for t in range(nIter): ws_all[:,:,t] = ws grad = compute_gradient(ws, ws_star) ws += eta * grad errs_per_w = np.sum(np.power(ws_all - ws_star[:,:,None], 2), axis=0) errs = np.sum(errs_per_w, axis=0) plt.plot(errs_per_w.T) plt.show() plt.plot(errs) plt.show() print(errs[0]) print(errs[-1])
plot_data(X, Y, xlabel, ylabel, legend) # ============= Map data to Polynomial features ============== # # this maps the features to a polynomial of degree 6 X = map_features(X[:, 0], X[:, 1]) # ============= Run logistic regression ================ # initial_theta = np.zeros(X.shape[1]) # Regularization Parameter: This dictates how much the cost function is penalized initial_lambda = 1 print('Computing cost with initial theta...') cost = compute_cost(initial_theta, X, Y, regularized=True, lambda_=initial_lambda) grad = compute_gradient(initial_theta, X, Y, regularized=True, lambda_=initial_lambda) print(f'Cost with initial theta: {cost}') print(f'First 5 gradients with initial theta\n{grad[:5].reshape(-1, 1)}') test_theta = np.ones(X.shape[1]) test_lambda = 10 print('Computing cost with test theta...') cost = compute_cost(test_theta, X, Y, regularized=True, lambda_=test_lambda) grad = compute_gradient(test_theta, X, Y, regularized=True, lambda_=test_lambda) print(f'Cost with test theta: {cost}') print(f'First 5 gradients with test theta\n{grad[:5].reshape(-1, 1)}')
def PGIRL(demonstrations=None, model=None, grad_path=None, features_idx=None, normalize_f=False, save_grad=True, opt_iters=10, compute_jacobian=False, estimate_weights=None, num_episodes=-1, pickled=False, continuous=False, num_hidden=8, num_layers=0, agent_name=None): if features_idx is None: features_idx = [0, 1, 2] logger = {} # Read or Calculate Gradient if args.read_grads: if grad_path != '': print("Reading gradients from:", grad_path) estimated_gradients = np.load(grad_path, allow_pickle=True) else: estimated_gradients = np.load(gradient_path + "estimated_gradients.npy", allow_pickle=True) estimated_gradients = estimated_gradients[:, :, features_idx] if num_episodes > 0: estimated_gradients = estimated_gradients[:num_episodes, :, :] if args.filter_gradients: estimated_gradients = filter_grads(estimated_gradients, verbose=args.verbose) else: if pickled: states_data = np.load(demonstrations + 'real_states.pkl', allow_pickle=True) actions_data = np.load(demonstrations + 'actions.pkl', allow_pickle=True) reward_data = np.load(demonstrations + 'rewards.pkl', allow_pickle=True) X_dataset = states_data[agent_name] y_dataset = actions_data[agent_name] r_dataset = reward_data[agent_name] print(np.sum(np.array(y_dataset) == 1)) input() dones_dataset = None else: # read trajectories X_dataset, y_dataset, _, _, r_dataset, dones_dataset = \ read_trajectories(demonstrations, all_columns=True, fill_size=EPISODE_LENGTH, fix_goal=True, cont_actions=args.continuous or args.lqg) if num_episodes > 0: X_dataset = X_dataset[:EPISODE_LENGTH * num_episodes] y_dataset = y_dataset[:EPISODE_LENGTH * num_episodes] r_dataset = r_dataset[:EPISODE_LENGTH * num_episodes] if dones_dataset is not None: dones_dataset = dones_dataset[:EPISODE_LENGTH * num_episodes] X_dim = len(X_dataset[0]) if continuous: y_dim = len(y_dataset[0]) else: y_dim = 2 # Create Policy linear = 'gpomdp' in model policy_train = load_policy(X_dim=X_dim, model=model, continuous=continuous, num_actions=y_dim, n_bases=X_dim, trainable_variance=args.trainable_variance, init_logstd=args.init_logstd, linear=linear, num_hidden=num_hidden, num_layers=num_layers) print('Loading dataset... done') # compute gradient estimation estimated_gradients, _ = compute_gradient( policy_train, X_dataset, y_dataset, r_dataset, dones_dataset, EPISODE_LENGTH, GAMMA, features_idx, verbose=args.verbose, use_baseline=args.baseline, use_mask=args.mask, scale_features=args.scale_features, filter_gradients=args.filter_gradients, normalize_f=normalize_f) # ================================================================================================================== if save_grad: print("Saving gradients in ", gradient_path) np.save(gradient_path + 'estimated_gradients.npy', estimated_gradients) # solve PGIRL or Rank Approx PGIRL if args.girl: weights_girl, loss_girl = solve_PGIRL(estimated_gradients, verbose=args.verbose) estimate_weights = weights_girl if args.rank_approx: weights, loss, jacobian = solve_ra_PGIRL( estimated_gradients, verbose=args.verbose, cov_estimation=args.cov_estimation, diag=args.diag, identity=args.identity, num_iters=opt_iters, compute_jacobian=compute_jacobian, other_options=[False, False, args.masked_cov]) if estimate_weights is not None or args.girl: mu, sigma = estimate_distribution_params( estimated_gradients=estimated_gradients, diag=args.diag, identity=args.identity, cov_estimation=args.cov_estimation, girl=False, other_options=[False, False, args.masked_cov]) id_matrix = np.identity(estimated_gradients.shape[1]) lf = make_loss_function(mu, sigma, id_matrix) estimated_loss = lf(estimate_weights) if compute_jacobian: print("Jacobian Rank:") print(np.linalg.matrix_rank(jacobian)) print("Jacobian s:") _, s, _ = np.linalg.svd(jacobian) print(s) else: weights, loss = solve_PGIRL(estimated_gradients, verbose=args.verbose) print("Weights:", weights) print("Loss:", loss) if args.girl: print("Weights Girl:", weights_girl) print("Loss Girl:", loss_girl) if estimate_weights is not None or args.girl: print("Loss in weights given:", estimated_loss) return logger, weights
n_bases=X_dim, trainable_variance=args.trainable_variance, init_logstd=args.init_logstd, linear=linear, num_hidden=args.num_hidden, num_layers=args.num_layers) print('Loading dataset... done') # compute gradient estimation estimated_gradients, _ = compute_gradient( policy_train, X_dataset, y_dataset, r_dataset, None, args.ep_len, GAMMA, features_idx, verbose=args.verbose, use_baseline=args.baseline, use_mask=args.mask, scale_features=args.scale_features, filter_gradients=args.filter_gradients, normalize_f=False) estimated_gradients_all.append(estimated_gradients) # ================================================================================================================== if args.save_grad: print("Saving gradients in ", args.save_path) np.save(args.save_path + '/estimated_gradients.npy', estimated_gradients) mus = []
print('Remember to close the plot. Otherwise, the process does not continue') xlabel = 'Score: First Exam' ylabel = 'Score: Second Exam' legend = ['Admitted', 'Not admitted'] plot_data(X, Y, xlabel, ylabel, legend) # ============= Part 2: Compute cost and gradient ============== # print('Calculating cost and gradient...') m, n = X.shape X = np.concatenate((np.ones((m, 1)), X), axis=1) initial_theta = np.zeros((n + 1, 1)) cost = compute_cost(initial_theta, X, Y) grad = compute_gradient(initial_theta, X, Y) print(f'Cost with initial parameters (all zeros): {cost}') print(f'Gradients with initial parameters:\n{grad}') test_theta = np.array([[-24], [0.2], [0.2]]) cost = compute_cost(test_theta, X, Y) grad = compute_gradient(test_theta, X, Y) print(f'Cost with test parameters:\n{test_theta}\nCost:{cost}') print(f'Gradients with test parameters: \n{grad}') input('Press enter to continue...') # ================= Part 3: Optimizing ================== #
num_actions=2, trainable_variance=args.trainable_variance, init_logstd=args.init_logstd, linear=linear) if args.num_episodes > 0: states = states[:EPISODE_LENGTH * args.num_episodes] actions = actions[:EPISODE_LENGTH * args.num_episodes] features = features[:EPISODE_LENGTH * args.num_episodes] dones = dones[:EPISODE_LENGTH * args.num_episodes] estimated_gradients, _ = compute_gradient( pi, states, actions, features, dones, EPISODE_LENGTH, args.gamma, features_idx, verbose=args.verbose, use_baseline=args.baseline, use_mask=args.mask, filter_gradients=args.filter_gradients, normalize_f=False) if args.save_grad: if not os.path.exists(read_path + '/gradients'): os.makedirs(read_path + '/gradients') np.save(read_path + '/gradients/estimated_gradients.npy', estimated_gradients) num_episodes, num_parameters, num_objectives = estimated_gradients.shape[:] mu, sigma = estimate_distribution_params( estimated_gradients=estimated_gradients, diag=args.diag,
os.makedirs(out_dir) with open(out_dir + 'trajectories.csv', 'w+', newline='') as trajectories_file: dicti_writer = csv.DictWriter(trajectories_file, fieldnames=data_keys) dicti_writer.writeheader() for trajectory in trajectories: dicti_writer.writerows([sample for sample in trajectory]) if args.compute_gradient: X_dataset, y_dataset, _, _, r_dataset, dones_dataset = \ read_trajectories(out_dir + 'trajectories.csv', all_columns=True, fill_size=args.ep_len, cont_actions=True) X_dim = len(X_dataset[0]) y_dim = len(y_dataset[0]) estimated_gradients, _ = compute_gradient(pi, X_dataset, y_dataset, r_dataset, dones_dataset, args.ep_len, args.gamma, features_idx, use_baseline=True, filter_gradients=args.filter_gradients) if not os.path.exists(out_dir + '/gradients'): os.makedirs(out_dir + '/gradients') np.save(out_dir + '/gradients/estimated_gradients.npy', estimated_gradients) else: print("Play!") s = env.reset(rbf=True) env._render() while True: a = policy(s) print("Action:", a) if args.debug_model: s = env.get_state(rbf=True) if 'trpo' in model:
input('Press enter to continue...') # ========== Test Logistic Regression ============ # theta_t = np.array([-2, -1, 1, 2]) ones = np.ones((5, 1)) X_t = np.concatenate( [np.ones((5, 1)), np.arange(1, 16).reshape(5, 3, order='F') / 10], axis=1) y_t = np.array([1, 0, 1, 0, 1]) >= 0.5 lambda_t = 3 cost = compute_cost(theta_t, X_t, y_t, regularized=True, lambda_=lambda_t) grad = compute_gradient(theta_t, X_t, y_t, regularized=True, lambda_=lambda_t) print(f'cost with test parameters: {cost}') print(f'gradients with test parameters:\n{grad}') input('Press enter to continue...') # =============== Train One vs All =============== # lambda_ = 0.1 m = OneVsAll(X, Y, num_labels, lambda_) m.fit() # =============== Predict One vs All =============== # preds = m.predict()
def numeric_gradient_descent(particles: np.ndarray, values: np.ndarray, function: cocoex.Problem, gradient_step: float, computation_step: float) -> np.ndarray: gradients = utils.compute_gradient(function, particles, gradient_step) new_particles = particles - gradients * computation_step return np.maximum(-5, np.minimum(5, new_particles))
newline='') as trajectories_file: dicti_writer = csv.DictWriter(trajectories_file, fieldnames=save_keys) dicti_writer.writeheader() for trajectory in trajectories: dicti_writer.writerows([sample for sample in trajectory]) # read the expert demonstrations states, actions, next_states, _, features, dones = read_trajectories( dir_to_read + str(exp) + "_" + str(demonstrations) + '_trajectories.csv', all_columns=True, fill_size=args.horizon, cont_actions=True) # if we are employing gradient based irl compute the gradients if args.settings not in ['re_irl', 'csi']: if not args.load_gradients: grads, _ = compute_gradient(pi, states, actions, features, dones, args.horizon, args.gamma, features_idx, use_baseline=True, filter_gradients=True) if args.save_gradients: if not os.path.exists(out_dir): os.makedirs(out_dir) np.save(out_dir + '/estimated_gradients.npy', grads) else: grads = np.load(dir_to_read + '/estimated_gradients.npy') means = grads.mean(axis=0) _, s, v = np.linalg.svd(means) print("Mean Gradient:") print(means) print("Singular values") print(s) for num_samples in n_samples_irl: