def compute_integrated_gradients(inp, baseline, net, target, n_steps=100):
"""Compute integrated gradients.
Parameters
----------
inp : torch.Tensor
Input image (single image batch) of shape `(1, 3, *, *)`.
baseline : torch.Tensor
Basline image of the same shape as the `inp`.
net : torch.nn.Module
Classifier network.
target : int
Imagenet ground truth label id.
n_steps : int
Number of steps between the `inp` and `baseline` tensors.
Returns
-------
ig : torch.Tensor
Integrated gradients with the same shape as the `inp`.
inp_grad : torch.Tensor
Gradient with respect to the `inp` tensor. Same shape as `inp`.
"""
path = [baseline + a * (inp - baseline) for a in np.linspace(0, 1, n_steps)]
grads = [compute_gradient(func, x, net=net, target=target) for x in path]
ig = (inp - baseline) * torch.cat(grads[:-1]).mean(dim=0, keepdims=True)
return ig, grads[-1]
def confine_vorticity(self):
if self.vorticity != 0:
w = utils.compute_curl(self.v)
grad_abs_w = 2 * utils.compute_gradient(np.abs(w))
grad_abs_w /= np.linalg.norm(grad_abs_w, axis=-1,
keepdims=True) + 1e-5
f_conf = self.dt * self.vorticity * w[:, :,
np.newaxis] * grad_abs_w
self.v[2:-2, 2:-2] += f_conf[2:-2, 2:-2]
def attack(tensor, net, eps=1e-3, n_iter=50):
"""Run the Fast Sign Gradient Method (FSGM) attack.
Parameters
----------
tensor : torch.Tensor
The input image of shape `(1, 3, 224, 224)`.
net : torch.nn.Module
Classifier network.
eps : float
Determines how much we modify the image in a single iteration.
n_iter : int
Number of iterations.
Returns
-------
new_tensor : torch.Tensor
New image that is a modification of the input image that "fools"
the classifier.
"""
new_tensor = tensor.detach().clone()
orig_prediction = net(tensor).argmax()
print(f"Original prediction: {orig_prediction.item()}")
for i in range(n_iter):
net.zero_grad()
grad = compute_gradient(func,
new_tensor,
net=net,
target=orig_prediction.item())
new_tensor = torch.clamp(new_tensor + eps * grad.sign(), -2, 2)
new_prediction = net(new_tensor).argmax()
if orig_prediction != new_prediction:
print(f"We fooled the network after {i} iterations!")
print(f"New prediction: {new_prediction.item()}")
break
return new_tensor, orig_prediction.item(), new_prediction.item()
nIter = 10000
ws_star = np.random.randn(d, n_node)
#ws_star[:n_node,:n_node] = np.eye(n_node)
ws_star_norm = np.linalg.norm(ws_star, axis=0)
ratio = 0.5
ws0 = np.copy(ws_star) + np.random.randn(d, n_node) * ratio
nIter = 1000
ws = np.copy(ws0)
ws_all = np.zeros((d, n_node, nIter))
eta = 0.01
for t in range(nIter):
ws_all[:,:,t] = ws
grad = compute_gradient(ws, ws_star)
ws += eta * grad
errs_per_w = np.sum(np.power(ws_all - ws_star[:,:,None], 2), axis=0)
errs = np.sum(errs_per_w, axis=0)
plt.plot(errs_per_w.T)
plt.show()
plt.plot(errs)
plt.show()
print(errs[0])
print(errs[-1])
plot_data(X, Y, xlabel, ylabel, legend)
# ============= Map data to Polynomial features ============== #
# this maps the features to a polynomial of degree 6
X = map_features(X[:, 0], X[:, 1])
# ============= Run logistic regression ================ #
initial_theta = np.zeros(X.shape[1])
# Regularization Parameter: This dictates how much the cost function is penalized
initial_lambda = 1
print('Computing cost with initial theta...')
cost = compute_cost(initial_theta, X, Y, regularized=True, lambda_=initial_lambda)
grad = compute_gradient(initial_theta, X, Y, regularized=True, lambda_=initial_lambda)
print(f'Cost with initial theta: {cost}')
print(f'First 5 gradients with initial theta\n{grad[:5].reshape(-1, 1)}')
test_theta = np.ones(X.shape[1])
test_lambda = 10
print('Computing cost with test theta...')
cost = compute_cost(test_theta, X, Y, regularized=True, lambda_=test_lambda)
grad = compute_gradient(test_theta, X, Y, regularized=True, lambda_=test_lambda)
print(f'Cost with test theta: {cost}')
print(f'First 5 gradients with test theta\n{grad[:5].reshape(-1, 1)}')
def PGIRL(demonstrations=None,
model=None,
grad_path=None,
features_idx=None,
normalize_f=False,
save_grad=True,
opt_iters=10,
compute_jacobian=False,
estimate_weights=None,
num_episodes=-1,
pickled=False,
continuous=False,
num_hidden=8,
num_layers=0,
agent_name=None):
if features_idx is None:
features_idx = [0, 1, 2]
logger = {}
# Read or Calculate Gradient
if args.read_grads:
if grad_path != '':
print("Reading gradients from:", grad_path)
estimated_gradients = np.load(grad_path, allow_pickle=True)
else:
estimated_gradients = np.load(gradient_path +
"estimated_gradients.npy",
allow_pickle=True)
estimated_gradients = estimated_gradients[:, :, features_idx]
if num_episodes > 0:
estimated_gradients = estimated_gradients[:num_episodes, :, :]
if args.filter_gradients:
estimated_gradients = filter_grads(estimated_gradients,
verbose=args.verbose)
else:
if pickled:
states_data = np.load(demonstrations + 'real_states.pkl',
allow_pickle=True)
actions_data = np.load(demonstrations + 'actions.pkl',
allow_pickle=True)
reward_data = np.load(demonstrations + 'rewards.pkl',
allow_pickle=True)
X_dataset = states_data[agent_name]
y_dataset = actions_data[agent_name]
r_dataset = reward_data[agent_name]
print(np.sum(np.array(y_dataset) == 1))
input()
dones_dataset = None
else:
# read trajectories
X_dataset, y_dataset, _, _, r_dataset, dones_dataset = \
read_trajectories(demonstrations, all_columns=True,
fill_size=EPISODE_LENGTH,
fix_goal=True,
cont_actions=args.continuous or args.lqg)
if num_episodes > 0:
X_dataset = X_dataset[:EPISODE_LENGTH * num_episodes]
y_dataset = y_dataset[:EPISODE_LENGTH * num_episodes]
r_dataset = r_dataset[:EPISODE_LENGTH * num_episodes]
if dones_dataset is not None:
dones_dataset = dones_dataset[:EPISODE_LENGTH * num_episodes]
X_dim = len(X_dataset[0])
if continuous:
y_dim = len(y_dataset[0])
else:
y_dim = 2
# Create Policy
linear = 'gpomdp' in model
policy_train = load_policy(X_dim=X_dim,
model=model,
continuous=continuous,
num_actions=y_dim,
n_bases=X_dim,
trainable_variance=args.trainable_variance,
init_logstd=args.init_logstd,
linear=linear,
num_hidden=num_hidden,
num_layers=num_layers)
print('Loading dataset... done')
# compute gradient estimation
estimated_gradients, _ = compute_gradient(
policy_train,
X_dataset,
y_dataset,
r_dataset,
dones_dataset,
EPISODE_LENGTH,
GAMMA,
features_idx,
verbose=args.verbose,
use_baseline=args.baseline,
use_mask=args.mask,
scale_features=args.scale_features,
filter_gradients=args.filter_gradients,
normalize_f=normalize_f)
# ==================================================================================================================
if save_grad:
print("Saving gradients in ", gradient_path)
np.save(gradient_path + 'estimated_gradients.npy', estimated_gradients)
# solve PGIRL or Rank Approx PGIRL
if args.girl:
weights_girl, loss_girl = solve_PGIRL(estimated_gradients,
verbose=args.verbose)
estimate_weights = weights_girl
if args.rank_approx:
weights, loss, jacobian = solve_ra_PGIRL(
estimated_gradients,
verbose=args.verbose,
cov_estimation=args.cov_estimation,
diag=args.diag,
identity=args.identity,
num_iters=opt_iters,
compute_jacobian=compute_jacobian,
other_options=[False, False, args.masked_cov])
if estimate_weights is not None or args.girl:
mu, sigma = estimate_distribution_params(
estimated_gradients=estimated_gradients,
diag=args.diag,
identity=args.identity,
cov_estimation=args.cov_estimation,
girl=False,
other_options=[False, False, args.masked_cov])
id_matrix = np.identity(estimated_gradients.shape[1])
lf = make_loss_function(mu, sigma, id_matrix)
estimated_loss = lf(estimate_weights)
if compute_jacobian:
print("Jacobian Rank:")
print(np.linalg.matrix_rank(jacobian))
print("Jacobian s:")
_, s, _ = np.linalg.svd(jacobian)
print(s)
else:
weights, loss = solve_PGIRL(estimated_gradients, verbose=args.verbose)
print("Weights:", weights)
print("Loss:", loss)
if args.girl:
print("Weights Girl:", weights_girl)
print("Loss Girl:", loss_girl)
if estimate_weights is not None or args.girl:
print("Loss in weights given:", estimated_loss)
return logger, weights
n_bases=X_dim,
trainable_variance=args.trainable_variance,
init_logstd=args.init_logstd,
linear=linear,
num_hidden=args.num_hidden,
num_layers=args.num_layers)
print('Loading dataset... done')
# compute gradient estimation
estimated_gradients, _ = compute_gradient(
policy_train,
X_dataset,
y_dataset,
r_dataset,
None,
args.ep_len,
GAMMA,
features_idx,
verbose=args.verbose,
use_baseline=args.baseline,
use_mask=args.mask,
scale_features=args.scale_features,
filter_gradients=args.filter_gradients,
normalize_f=False)
estimated_gradients_all.append(estimated_gradients)
# ==================================================================================================================
if args.save_grad:
print("Saving gradients in ", args.save_path)
np.save(args.save_path + '/estimated_gradients.npy',
estimated_gradients)
mus = []
print('Remember to close the plot. Otherwise, the process does not continue')
xlabel = 'Score: First Exam'
ylabel = 'Score: Second Exam'
legend = ['Admitted', 'Not admitted']
plot_data(X, Y, xlabel, ylabel, legend)
# ============= Part 2: Compute cost and gradient ============== #
print('Calculating cost and gradient...')
m, n = X.shape
X = np.concatenate((np.ones((m, 1)), X), axis=1)
initial_theta = np.zeros((n + 1, 1))
cost = compute_cost(initial_theta, X, Y)
grad = compute_gradient(initial_theta, X, Y)
print(f'Cost with initial parameters (all zeros): {cost}')
print(f'Gradients with initial parameters:\n{grad}')
test_theta = np.array([[-24], [0.2], [0.2]])
cost = compute_cost(test_theta, X, Y)
grad = compute_gradient(test_theta, X, Y)
print(f'Cost with test parameters:\n{test_theta}\nCost:{cost}')
print(f'Gradients with test parameters: \n{grad}')
input('Press enter to continue...')
# ================= Part 3: Optimizing ================== #
num_actions=2,
trainable_variance=args.trainable_variance,
init_logstd=args.init_logstd,
linear=linear)
if args.num_episodes > 0:
states = states[:EPISODE_LENGTH * args.num_episodes]
actions = actions[:EPISODE_LENGTH * args.num_episodes]
features = features[:EPISODE_LENGTH * args.num_episodes]
dones = dones[:EPISODE_LENGTH * args.num_episodes]
estimated_gradients, _ = compute_gradient(
pi,
states,
actions,
features,
dones,
EPISODE_LENGTH,
args.gamma,
features_idx,
verbose=args.verbose,
use_baseline=args.baseline,
use_mask=args.mask,
filter_gradients=args.filter_gradients,
normalize_f=False)
if args.save_grad:
if not os.path.exists(read_path + '/gradients'):
os.makedirs(read_path + '/gradients')
np.save(read_path + '/gradients/estimated_gradients.npy',
estimated_gradients)
num_episodes, num_parameters, num_objectives = estimated_gradients.shape[:]
mu, sigma = estimate_distribution_params(
estimated_gradients=estimated_gradients,
diag=args.diag,
os.makedirs(out_dir)
with open(out_dir + 'trajectories.csv', 'w+',
newline='') as trajectories_file:
dicti_writer = csv.DictWriter(trajectories_file, fieldnames=data_keys)
dicti_writer.writeheader()
for trajectory in trajectories:
dicti_writer.writerows([sample for sample in trajectory])
if args.compute_gradient:
X_dataset, y_dataset, _, _, r_dataset, dones_dataset = \
read_trajectories(out_dir + 'trajectories.csv', all_columns=True,
fill_size=args.ep_len,
cont_actions=True)
X_dim = len(X_dataset[0])
y_dim = len(y_dataset[0])
estimated_gradients, _ = compute_gradient(pi, X_dataset, y_dataset, r_dataset, dones_dataset,
args.ep_len, args.gamma, features_idx,
use_baseline=True,
filter_gradients=args.filter_gradients)
if not os.path.exists(out_dir + '/gradients'):
os.makedirs(out_dir + '/gradients')
np.save(out_dir + '/gradients/estimated_gradients.npy', estimated_gradients)
else:
print("Play!")
s = env.reset(rbf=True)
env._render()
while True:
a = policy(s)
print("Action:", a)
if args.debug_model:
s = env.get_state(rbf=True)
if 'trpo' in model:
input('Press enter to continue...')
# ========== Test Logistic Regression ============ #
theta_t = np.array([-2, -1, 1, 2])
ones = np.ones((5, 1))
X_t = np.concatenate(
[np.ones((5, 1)),
np.arange(1, 16).reshape(5, 3, order='F') / 10], axis=1)
y_t = np.array([1, 0, 1, 0, 1]) >= 0.5
lambda_t = 3
cost = compute_cost(theta_t, X_t, y_t, regularized=True, lambda_=lambda_t)
grad = compute_gradient(theta_t, X_t, y_t, regularized=True, lambda_=lambda_t)
print(f'cost with test parameters: {cost}')
print(f'gradients with test parameters:\n{grad}')
input('Press enter to continue...')
# =============== Train One vs All =============== #
lambda_ = 0.1
m = OneVsAll(X, Y, num_labels, lambda_)
m.fit()
# =============== Predict One vs All =============== #
preds = m.predict()
def numeric_gradient_descent(particles: np.ndarray, values: np.ndarray,
function: cocoex.Problem, gradient_step: float,
computation_step: float) -> np.ndarray:
gradients = utils.compute_gradient(function, particles, gradient_step)
new_particles = particles - gradients * computation_step
return np.maximum(-5, np.minimum(5, new_particles))
newline='') as trajectories_file:
dicti_writer = csv.DictWriter(trajectories_file, fieldnames=save_keys)
dicti_writer.writeheader()
for trajectory in trajectories:
dicti_writer.writerows([sample for sample in trajectory])
# read the expert demonstrations
states, actions, next_states, _, features, dones = read_trajectories(
dir_to_read + str(exp) + "_" + str(demonstrations) + '_trajectories.csv',
all_columns=True,
fill_size=args.horizon,
cont_actions=True)
# if we are employing gradient based irl compute the gradients
if args.settings not in ['re_irl', 'csi']:
if not args.load_gradients:
grads, _ = compute_gradient(pi, states, actions, features,
dones, args.horizon, args.gamma, features_idx, use_baseline=True,
filter_gradients=True)
if args.save_gradients:
if not os.path.exists(out_dir):
os.makedirs(out_dir)
np.save(out_dir + '/estimated_gradients.npy', grads)
else:
grads = np.load(dir_to_read + '/estimated_gradients.npy')
means = grads.mean(axis=0)
_, s, v = np.linalg.svd(means)
print("Mean Gradient:")
print(means)
print("Singular values")
print(s)
for num_samples in n_samples_irl:
