def project_onto_l1_ball(x: ep.Tensor, eps: ep.Tensor) -> ep.Tensor: """Computes Euclidean projection onto the L1 ball for a batch. [#Duchi08]_ Adapted from the pytorch version by Tony Duan: https://gist.github.com/tonyduan/1329998205d88c566588e57e3e2c0c55 Args: x: Batch of arbitrary-size tensors to project, possibly on GPU eps: radius of l-1 ball to project onto References: ..[#Duchi08] Efficient Projections onto the l1-Ball for Learning in High Dimensions John Duchi, Shai Shalev-Shwartz, Yoram Singer, and Tushar Chandra. International Conference on Machine Learning (ICML 2008) """ original_shape = x.shape x = flatten(x) mask = (ep.norms.l1(x, axis=1) <= eps).astype(x.dtype).expand_dims(1) mu = ep.flip(ep.sort(ep.abs(x)), axis=-1).astype(x.dtype) cumsum = ep.cumsum(mu, axis=-1) arange = ep.arange(x, 1, x.shape[1] + 1).astype(x.dtype) rho = (ep.max( ((mu * arange > (cumsum - eps.expand_dims(1)))).astype(x.dtype) * arange, axis=-1, ) - 1) # samples already under norm will have to select rho = ep.maximum(rho, 0) theta = (cumsum[ep.arange(x, x.shape[0]), rho.astype(ep.arange(x, 1).dtype)] - eps) / (rho + 1.0) proj = (ep.abs(x) - theta.expand_dims(1)).clip(min_=0, max_=ep.inf) x = mask * x + (1 - mask) * proj * ep.sign(x) return x.reshape(original_shape)
def mid_points(self, x0: ep.Tensor, x1: ep.Tensor, epsilons: ep.Tensor, bounds) -> ep.Tensor: # returns a point between x0 and x1 where # epsilon = 0 returns x0 and epsilon = 1 # returns x1 # get epsilons in right shape for broadcasting epsilons = epsilons.reshape(epsilons.shape + (1, ) * (x0.ndim - 1)) return epsilons * x1 + (1 - epsilons) * x0
def project_onto_l1_ball(x: ep.Tensor, eps: ep.Tensor): """ Compute Euclidean projection onto the L1 ball for a batch. min ||x - u||_2 s.t. ||u||_1 <= eps Inspired by the corresponding numpy version by Adrien Gaidon. Adapted from the pytorch version by Tony Duan: https://gist.github.com/tonyduan/1329998205d88c566588e57e3e2c0c55 Parameters ---------- x: (batch_size, *) torch array batch of arbitrary-size tensors to project, possibly on GPU eps: float radius of l-1 ball to project onto Returns ------- u: (batch_size, *) torch array batch of projected tensors, reshaped to match the original Notes ----- The complexity of this algorithm is in O(dlogd) as it involves sorting x. References ---------- [1] Efficient Projections onto the l1-Ball for Learning in High Dimensions John Duchi, Shai Shalev-Shwartz, Yoram Singer, and Tushar Chandra. International Conference on Machine Learning (ICML 2008) """ original_shape = x.shape x = flatten(x) mask = (ep.norms.l1(x, axis=1) < eps).astype(x.dtype).expand_dims(1) mu = ep.flip(ep.sort(ep.abs(x)), axis=-1) cumsum = ep.cumsum(mu, axis=-1) arange = ep.arange(x, 1, x.shape[1] + 1) rho = ep.max( (mu * arange > (cumsum - eps.expand_dims(1))) * arange, axis=-1) - 1 theta = (cumsum[ep.arange(x, x.shape[0]), rho] - eps) / (rho + 1.0) proj = (ep.abs(x) - theta.expand_dims(1)).clip(min_=0, max_=ep.inf) x = mask * x + (1 - mask) * proj * ep.sign(x) return x.reshape(original_shape)
def mid_points( self, x0: ep.Tensor, x1: ep.Tensor, epsilons: ep.Tensor, bounds: Tuple[float, float], ): # returns a point between x0 and x1 where # epsilon = 0 returns x0 and epsilon = 1 delta = x1 - x0 min_, max_ = bounds s = max_ - min_ # get epsilons in right shape for broadcasting epsilons = epsilons.reshape(epsilons.shape + (1, ) * (x0.ndim - 1)) clipped_delta = ep.where(delta < -epsilons * s, -epsilons * s, delta) clipped_delta = ep.where(clipped_delta > epsilons * s, epsilons * s, clipped_delta) return x0 + clipped_delta
def mid_points( self, x0: ep.Tensor, x1: ep.Tensor, epsilons: ep.Tensor, bounds: Tuple[float, float], ) -> ep.Tensor: # returns a point between x0 and x1 where # epsilon = 0 returns x0 and epsilon = 1 # returns x1 # get epsilons in right shape for broadcasting epsilons = epsilons.reshape(epsilons.shape + (1, ) * (x0.ndim - 1)) threshold = (bounds[1] - bounds[0]) * (1 - epsilons) mask = (x1 - x0).abs() > threshold new_x = ep.where(mask, x0 + (x1 - x0).sign() * ((x1 - x0).abs() - threshold), x0) return new_x
def _binary_search( self, x_adv_flat: ep.Tensor, mask: Union[ep.Tensor, List[bool]], mask_indices: ep.Tensor, indices: Union[ep.Tensor, List[int]], adv_values: ep.Tensor, non_adv_values: ep.Tensor, original_shape: Tuple, is_adversarial: Callable, ) -> ep.Tensor: for i in range(10): next_values = (adv_values + non_adv_values) / 2 x_adv_flat = ep.index_update( x_adv_flat, (mask_indices, indices), next_values ) is_adv = is_adversarial(x_adv_flat.reshape(original_shape))[mask] adv_values = ep.where(is_adv, next_values, adv_values) non_adv_values = ep.where(is_adv, non_adv_values, next_values) return adv_values
def draw_proposals(bounds: Bounds, originals: ep.Tensor, perturbed: ep.Tensor, unnormalized_source_directions: ep.Tensor, source_directions: ep.Tensor, source_norms: ep.Tensor, spherical_steps: ep.Tensor, source_steps: ep.Tensor, surrogate_model: Model) -> Tuple[ep.Tensor, ep.Tensor]: # remember the actual shape shape = originals.shape assert perturbed.shape == shape assert unnormalized_source_directions.shape == shape assert source_directions.shape == shape # flatten everything to (batch, size) originals = flatten(originals) perturbed = flatten(perturbed) unnormalized_source_directions = flatten(unnormalized_source_directions) source_directions = flatten(source_directions) N, D = originals.shape assert source_norms.shape == (N, ) assert spherical_steps.shape == (N, ) assert source_steps.shape == (N, ) # draw from an iid Gaussian (we can share this across the whole batch) eta = ep.normal(perturbed, (D, 1)) # make orthogonal (source_directions are normalized) eta = eta.T - ep.matmul(source_directions, eta) * source_directions assert eta.shape == (N, D) pg_factor = 0.5 if not surrogate_model is None: device = surrogate_model.device projected_gradient = get_projected_gradients(perturbed.reshape(shape), originals.reshape(shape), 0, surrogate_model) projected_gradient = projected_gradient.reshape((N, D)) projected_gradient = torch.tensor(projected_gradient, device=device) projected_gradient, restore_type = ep.astensor_(projected_gradient) eta = (1. - pg_factor) * eta + pg_factor * projected_gradient # rescale norms = ep.norms.l2(eta, axis=-1) assert norms.shape == (N, ) eta = eta * atleast_kd(spherical_steps * source_norms / norms, eta.ndim) # project on the sphere using Pythagoras distances = atleast_kd((spherical_steps.square() + 1).sqrt(), eta.ndim) directions = eta - unnormalized_source_directions spherical_candidates = originals + directions / distances # clip min_, max_ = bounds spherical_candidates = spherical_candidates.clip(min_, max_) # step towards the original inputs new_source_directions = originals - spherical_candidates assert new_source_directions.ndim == 2 new_source_directions_norms = ep.norms.l2(flatten(new_source_directions), axis=-1) # length if spherical_candidates would be exactly on the sphere lengths = source_steps * source_norms # length including correction for numerical deviation from sphere lengths = lengths + new_source_directions_norms - source_norms # make sure the step size is positive lengths = ep.maximum(lengths, 0) # normalize the length lengths = lengths / new_source_directions_norms lengths = atleast_kd(lengths, new_source_directions.ndim) candidates = spherical_candidates + lengths * new_source_directions # clip candidates = candidates.clip(min_, max_) # restore shape candidates = candidates.reshape(shape) spherical_candidates = spherical_candidates.reshape(shape) return candidates, spherical_candidates
def atleast_kd(x: ep.Tensor, k) -> ep.Tensor: shape = x.shape + (1, ) * (k - x.ndim) return x.reshape(shape)
def flatten(x: ep.Tensor, keep=1) -> ep.Tensor: shape = x.shape[:keep] + (-1, ) return x.reshape(shape)