def __call__(self, inputs, labels, *, binary_search_steps=15): x = ep.astensor(inputs) y = ep.astensor(labels) assert x.shape[0] == y.shape[0] assert y.ndim == 1 min_, max_ = self.model.bounds() target = (max_ + min_) / 2 v = target - x N = len(x) x0 = x npdtype = x.numpy().dtype lower_bound = np.zeros((N,), dtype=npdtype) upper_bound = np.ones((N,), dtype=npdtype) epsilons = lower_bound for _ in range(binary_search_steps): x = x0 + atleast_kd(ep.from_numpy(x0, epsilons), x0.ndim) * v logits = ep.astensor(self.model.forward(x.tensor)) classes = logits.argmax(axis=-1) is_adv = (classes != labels).numpy() lower_bound = np.where(is_adv, lower_bound, epsilons) upper_bound = np.where(is_adv, epsilons, upper_bound) epsilons = (lower_bound + upper_bound) / 2 epsilons = upper_bound epsilons = ep.from_numpy(x0, epsilons) x = x0 + atleast_kd(epsilons, x0.ndim) * v return x.tensor
def samples( fmodel: Model, dataset: str = "imagenet", index: int = 0, batchsize: int = 1, shape: Tuple[int, int] = (224, 224), data_format: Optional[str] = None, bounds: Optional[Bounds] = None, vis=False ) -> Any: if hasattr(fmodel, "data_format"): if data_format is None: data_format = fmodel.data_format # type: ignore elif data_format != fmodel.data_format: # type: ignore raise ValueError( f"data_format ({data_format}) does not match model.data_format ({fmodel.data_format})" # type: ignore ) elif data_format is None: raise ValueError( "data_format could not be inferred, please specify it explicitly" ) if bounds is None: bounds = fmodel.bounds if vis: images, labels, file_names = _samples( dataset=dataset, index=index, batchsize=batchsize, shape=shape, data_format=data_format, bounds=bounds, vis=vis ) else: images, labels = _samples( dataset=dataset, index=index, batchsize=batchsize, shape=shape, data_format=data_format, bounds=bounds, ) if hasattr(fmodel, "dummy") and fmodel.dummy is not None: # type: ignore images = ep.from_numpy(fmodel.dummy, images).raw # type: ignore labels = ep.from_numpy(fmodel.dummy, labels).raw # type: ignore else: warnings.warn(f"unknown model type {type(fmodel)}, returning NumPy arrays") if vis: return images, labels, file_names return images, labels
def download_dataset(base_path, examples, data_format, bounds, dimension): images_path = base_path + os.sep + "subset" labels_path = base_path + os.sep + "labels.txt" images, labels = [], [] files = os.listdir(images_path) files.sort() labels_numbers = [] for idx in range(0, examples): # okresl nazwe pliku file = files[idx] labels_numbers.append(int(re.match(r".+val_0+(.+)\.", file).group(1))) # otworz plik path = os.path.join(images_path, file) image = Image.open(path) image = image.resize(dimension) image = np.asarray(image, dtype=np.float32) # jesli obraz jest zapisany w skali szarosci to dodawana jest trzecia os if image.ndim == 2: image = np.repeat(image[:, :, np.newaxis], 3, axis=2) # ewentualne przestawienie kanalow if data_format == "channels_first": image = np.transpose(image, (2, 0, 1)) # dodanie obrazu do listy obrazow images.append(image) # pobranie etykiet with open(labels_path, "r") as csv_file: reader = csv.reader(csv_file) for i, line in enumerate(reader, 1): if len(labels) == len(files): break if i in labels_numbers: labels.append(int(line[0])) # konwersja list do tablic numpy images = np.stack(images) labels = np.array(labels) # wartosci pikseli zdjec domyslnie zawieraja sie w przedziale od 0 do 255 # jesli ograniczenie wartosci pikseli dla modelu jest inny to nalezy skonwertowac zdjecia if bounds != (0, 255): images = images / 255 * (bounds[1] - bounds[0]) + bounds[0] images = ep.from_numpy(ep.torch.zeros(0, device="cpu"), images).raw labels = ep.from_numpy(ep.torch.zeros(0, device="cpu"), labels).raw return ep.astensor(images), ep.astensor(labels).astype(torch.long)
def __call__(self, inputs, labels, *, directions=1000, steps=1000): x = ep.astensor(inputs) min_, max_ = self.model.bounds() N = len(x) assert directions >= 1 for j in range(directions): # random noise inputs tend to be classified into the same class, # so we might need to make very many draws if the original class # is that one random_ = ep.uniform(x, x.shape, min_, max_) logits_ = self.model.forward(random_) classes_ = logits_.argmax(axis=-1) is_adv_ = atleast_kd(classes_ != labels, x.ndim) if j == 0: random = random_ is_adv = is_adv_ else: cond1 = is_adv.astype(x.dtype) cond2 = is_adv_.astype(x.dtype) random = cond1 * random + (1 - cond1) * cond2 * random_ is_adv = is_adv.logical_or(is_adv_) if is_adv.all(): break if not is_adv.all(): warnings.warn( f"{self.__class__.__name__} failed to draw sufficent random" " inputs that are adversarial ({is_adv.sum()} / {N}).") x0 = x npdtype = x.numpy().dtype epsilons = np.linspace(0, 1, num=steps + 1, dtype=npdtype) best = np.ones((N, ), dtype=npdtype) for epsilon in epsilons: x = (1 - epsilon) * x0 + epsilon * random # TODO: due to limited floating point precision, clipping can be required logits = self.model.forward(x) classes = logits.argmax(axis=-1) is_adv = (classes != labels).numpy() best = np.minimum( np.logical_not(is_adv).astype(npdtype) + is_adv.astype(npdtype) * epsilon, best, ) if (best < 1).all(): break best = ep.from_numpy(x0, best) best = atleast_kd(best, x0.ndim) x = (1 - best) * x0 + best * random return x.tensor
def __call__(self, inputs, labels, *, steps=1000): x = ep.astensor(inputs) y = ep.astensor(labels) assert x.shape[0] == y.shape[0] assert y.ndim == 1 assert x.ndim == 4 if self.channel_axis == 1: h, w = x.shape[2:4] elif self.channel_axis == 3: h, w = x.shape[1:3] else: raise ValueError( "expected 'channel_axis' to be 1 or 3, got {channel_axis}") size = max(h, w) min_, max_ = self.model.bounds() x0 = x x0np = x0.numpy() epsilons = np.linspace(0, 1, num=steps + 1)[1:] logits = ep.astensor(self.model.forward(x0.tensor)) classes = logits.argmax(axis=-1) is_adv = classes != labels found = is_adv result = x0 for epsilon in epsilons: # TODO: reduce the batch size to the ones that haven't been sucessful sigmas = [epsilon * size] * 4 sigmas[0] = 0 sigmas[self.channel_axis] = 0 # TODO: once we can implement gaussian_filter in eagerpy, avoid converting from numpy x = gaussian_filter(x0np, sigmas) x = np.clip(x, min_, max_) x = ep.from_numpy(x0, x) logits = ep.astensor(self.model.forward(x.tensor)) classes = logits.argmax(axis=-1) is_adv = classes != labels new_adv = ep.logical_and(is_adv, found.logical_not()) result = ep.where(atleast_kd(new_adv, x.ndim), x, result) found = ep.logical_or(new_adv, found) if found.all(): break return result.tensor
def to_tensor(x: Any) -> Optional[ep.Tensor]: if x is None: return None if isinstance(x, ep.Tensor): return x try: y = ep.astensor(x) # might raise ValueError if not isinstance(y, type(self._dummy)): raise ValueError return y except ValueError: return ep.from_numpy(self._dummy, x)
def __call__(self, model: Model, inputs, labels): x, y, restore = wrap(inputs, labels) del inputs, labels adv_samples = [] for k in range(len(y)): while True: idx = np.random.randint(len(self.labels)) if int(y[k].numpy()) != self.labels[idx]: adv_samples.append(ep.astensor(self.samples[idx]).numpy()) break x = ep.from_numpy(x, np.stack(adv_samples)) return restore(x)
def test_hsj_targeted_attack( request: Any, fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack_and_p: Tuple[fa.HopSkipJumpAttack, Union[int, float]], ) -> None: if request.config.option.skipslow: pytest.skip() (fmodel, x, y), real, _ = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() if not real: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) logits_np = fmodel(x).numpy() num_classes = logits_np.shape[-1] y_np = logits_np.argmax(-1) target_classes_np = (y_np + 1) % num_classes for i in range(len(target_classes_np)): while target_classes_np[i] not in y_np: target_classes_np[i] = (target_classes_np[i] + 1) % num_classes target_classes = ep.from_numpy(y, target_classes_np) criterion = fbn.TargetedMisclassification(target_classes) init_attack = fa.DatasetAttack() init_attack.feed(fmodel, x) init_advs = init_attack.run(fmodel, x, criterion) attack, p = attack_and_p advs = attack.run(fmodel, x, criterion, starting_points=init_advs) init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1) norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1) is_smaller = norms < init_norms assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, target_classes) > fbn.accuracy( fmodel, x, target_classes) assert fbn.accuracy(fmodel, advs, target_classes) >= fbn.accuracy( fmodel, init_advs, target_classes) assert is_smaller.any()
def test_rescale_axis(request: Any, dummy: ep.Tensor) -> None: backend = request.config.option.backend if backend == "numpy": pytest.skip() x_np = np.random.uniform(0.0, 1.0, size=(16, 3, 64, 64)) x_np_ep = ep.astensor(x_np) x_up_np_ep = rescale_images(x_np_ep, (16, 3, 128, 128), 1) x_up_np = x_up_np_ep.numpy() x = ep.from_numpy(dummy, x_np) x_ep = ep.astensor(x) x_up_ep = rescale_images(x_ep, (16, 3, 128, 128), 1) x_up = x_up_ep.numpy() assert np.allclose(x_up_np, x_up, atol=1e-5)
def __call__(self, inputs, labels): inputs = ep.astensor(inputs) labels = ep.astensor(labels) x = ep.zeros_like(inputs) adv_samples = [] for k in range(len(labels)): while True: idx = np.random.randint(len(self.labels)) if int(labels[k].numpy()) != self.labels[idx]: adv_samples.append(ep.astensor(self.samples[idx]).numpy()) break x = ep.from_numpy(x, np.stack(adv_samples)) return x.tensor
def normalize(self, gradients: ep.Tensor, *, x: ep.Tensor, bounds: Bounds) -> ep.Tensor: bad_pos = ep.logical_or( ep.logical_and(x == bounds.lower, gradients < 0), ep.logical_and(x == bounds.upper, gradients > 0), ) gradients = ep.where(bad_pos, ep.zeros_like(gradients), gradients) abs_gradients = gradients.abs() quantiles = np.quantile(flatten(abs_gradients).numpy(), q=self.quantile, axis=-1) keep = abs_gradients >= atleast_kd(ep.from_numpy(gradients, quantiles), gradients.ndim) e = ep.where(keep, gradients.sign(), ep.zeros_like(gradients)) return normalize_lp_norms(e, p=1)
def __call__(self, inputs, labels, *, steps=1000): x = ep.astensor(inputs) y = ep.astensor(labels) assert x.shape[0] == y.shape[0] assert y.ndim == 1 min_, max_ = self.model.bounds() target = (max_ + min_) / 2 v = target - x N = len(x) x0 = x npdtype = x.numpy().dtype epsilons = np.linspace(0, 1, num=steps + 1, dtype=npdtype) best = np.ones((N,), dtype=npdtype) for epsilon in epsilons: # TODO: reduce the batch size to the ones that haven't been sucessful x = x0 + epsilon * v logits = ep.astensor(self.model.forward(x.tensor)) classes = logits.argmax(axis=-1) is_adv = (classes != labels).numpy() best = np.minimum( np.logical_not(is_adv).astype(npdtype) + is_adv.astype(npdtype) * epsilon, best, ) if (best < 1).all(): break best = ep.from_numpy(x0, best) x = x0 + atleast_kd(best, x0.ndim) * v return x.tensor
def isfull(self) -> ep.Tensor: assert self.tensor is not None result = ~np.isnan(self.data).any(axis=0) return ep.from_numpy(self.tensor, result)
def mean(self): result = np.nanmean(self.data, axis=0) return ep.from_numpy(self.tensor, result)
def __call__( self, inputs, labels, *, target_classes=None, binary_search_steps=9, max_iterations=10000, confidence=0, learning_rate=1e-2, initial_const=1e-3, abort_early=True, ): x = ep.astensor(inputs) N = len(x) targeted = target_classes is not None if targeted: labels = None target_classes = ep.astensor(target_classes) assert target_classes.shape == (N, ) is_adv = partial(targeted_is_adv, target_classes=target_classes, confidence=confidence) else: labels = ep.astensor(labels) assert labels.shape == (N, ) is_adv = partial(untargeted_is_adv, labels=labels, confidence=confidence) bounds = self.model.bounds() to_attack_space = partial(_to_attack_space, bounds=bounds) to_model_space = partial(_to_model_space, bounds=bounds) x_attack = to_attack_space(x) reconstsructed_x = to_model_space(x_attack) rows = np.arange(N) def loss_fun(delta: ep.Tensor, consts: ep.Tensor) -> ep.Tensor: assert delta.shape == x_attack.shape assert consts.shape == (N, ) x = to_model_space(x_attack + delta) logits = ep.astensor(self.model.forward(x.tensor)) if targeted: c_minimize = best_other_classes(logits, target_classes) c_maximize = target_classes else: c_minimize = labels c_maximize = best_other_classes(logits, labels) is_adv_loss = logits[rows, c_minimize] - logits[rows, c_maximize] assert is_adv_loss.shape == (N, ) is_adv_loss = is_adv_loss + confidence is_adv_loss = ep.maximum(0, is_adv_loss) is_adv_loss = is_adv_loss * consts squared_norms = flatten(x - reconstsructed_x).square().sum(axis=-1) loss = is_adv_loss.sum() + squared_norms.sum() return loss, (x, logits) loss_aux_and_grad = ep.value_and_grad_fn(x, loss_fun, has_aux=True) consts = initial_const * np.ones((N, )) lower_bounds = np.zeros((N, )) upper_bounds = np.inf * np.ones((N, )) best_advs = ep.zeros_like(x) best_advs_norms = ep.ones(x, (N, )) * np.inf # the binary search searches for the smallest consts that produce adversarials for binary_search_step in range(binary_search_steps): if (binary_search_step == binary_search_steps - 1 and binary_search_steps >= 10): # in the last iteration, repeat the search once consts = np.minimum(upper_bounds, 1e10) # create a new optimizer find the delta that minimizes the loss delta = ep.zeros_like(x_attack) optimizer = AdamOptimizer(delta) found_advs = np.full( (N, ), fill_value=False) # found adv with the current consts loss_at_previous_check = np.inf consts_ = ep.from_numpy(x, consts.astype(np.float32)) for iteration in range(max_iterations): loss, (perturbed, logits), gradient = loss_aux_and_grad(delta, consts_) delta += optimizer(gradient, learning_rate) if abort_early and iteration % (np.ceil( max_iterations / 10)) == 0: # after each tenth of the iterations, check progress if not (loss <= 0.9999 * loss_at_previous_check): break # stop Adam if there has been no progress loss_at_previous_check = loss found_advs_iter = is_adv(logits) found_advs = np.logical_or(found_advs, found_advs_iter.numpy()) norms = flatten(perturbed - x).square().sum(axis=-1).sqrt() closer = norms < best_advs_norms new_best = closer.float32() * found_advs_iter.float32() best_advs = ( atleast_kd(new_best, best_advs.ndim) * perturbed + (1 - atleast_kd(new_best, best_advs.ndim)) * best_advs) best_advs_norms = new_best * norms + ( 1 - new_best) * best_advs_norms upper_bounds = np.where(found_advs, consts, upper_bounds) lower_bounds = np.where(found_advs, lower_bounds, consts) consts_exponential_search = consts * 10 consts_binary_search = (lower_bounds + upper_bounds) / 2 consts = np.where(np.isinf(upper_bounds), consts_exponential_search, consts_binary_search) return best_advs.tensor
def mean(self) -> ep.Tensor: assert self.tensor is not None result = np.nanmean(self.data, axis=0) return ep.from_numpy(self.tensor, result)
def run( self, model: Model, inputs: T, criterion: Union[Criterion, T], *, early_stop: Optional[float] = None, starting_points: Optional[T] = None, **kwargs: Any, ) -> T: raise_if_kwargs(kwargs) originals, restore_type = ep.astensor_(inputs) del inputs, kwargs verify_input_bounds(originals, model) criterion = get_criterion(criterion) is_adversarial = get_is_adversarial(criterion, model) if starting_points is None: init_attack: MinimizationAttack if self.init_attack is None: init_attack = LinearSearchBlendedUniformNoiseAttack(steps=50) logging.info( f"Neither starting_points nor init_attack given. Falling" f" back to {init_attack!r} for initialization.") else: init_attack = self.init_attack # TODO: use call and support all types of attacks (once early_stop is # possible in __call__) x_advs = init_attack.run(model, originals, criterion, early_stop=early_stop) else: x_advs = ep.astensor(starting_points) is_adv = is_adversarial(x_advs) if not is_adv.all(): failed = is_adv.logical_not().float32().sum() if starting_points is None: raise ValueError( f"init_attack failed for {failed} of {len(is_adv)} inputs") else: raise ValueError( f"{failed} of {len(is_adv)} starting_points are not adversarial" ) del starting_points tb = TensorBoard(logdir=self.tensorboard) # Project the initialization to the boundary. x_advs = self._binary_search(is_adversarial, originals, x_advs) assert ep.all(is_adversarial(x_advs)) distances = self.distance(originals, x_advs) for step in range(self.steps): delta = self.select_delta(originals, distances, step) # Choose number of gradient estimation steps. num_gradient_estimation_steps = int( min([ self.initial_num_evals * math.sqrt(step + 1), self.max_num_evals ])) gradients = self.approximate_gradients( is_adversarial, x_advs, num_gradient_estimation_steps, delta) if self.constraint == "linf": update = ep.sign(gradients) else: update = gradients if self.stepsize_search == "geometric_progression": # find step size. epsilons = distances / math.sqrt(step + 1) while True: x_advs_proposals = ep.clip( x_advs + atleast_kd(epsilons, x_advs.ndim) * update, 0, 1) success = is_adversarial(x_advs_proposals) epsilons = ep.where(success, epsilons, epsilons / 2.0) if ep.all(success): break # Update the sample. x_advs = ep.clip( x_advs + atleast_kd(epsilons, update.ndim) * update, 0, 1) assert ep.all(is_adversarial(x_advs)) # Binary search to return to the boundary. x_advs = self._binary_search(is_adversarial, originals, x_advs) assert ep.all(is_adversarial(x_advs)) elif self.stepsize_search == "grid_search": # Grid search for stepsize. epsilons_grid = ep.expand_dims( ep.from_numpy( distances, np.logspace( -4, 0, num=20, endpoint=True, dtype=np.float32), ), 1, ) * ep.expand_dims(distances, 0) proposals_list = [] for epsilons in epsilons_grid: x_advs_proposals = ( x_advs + atleast_kd(epsilons, update.ndim) * update) x_advs_proposals = ep.clip(x_advs_proposals, 0, 1) mask = is_adversarial(x_advs_proposals) x_advs_proposals = self._binary_search( is_adversarial, originals, x_advs_proposals) # only use new values where initial guess was already adversarial x_advs_proposals = ep.where(atleast_kd(mask, x_advs.ndim), x_advs_proposals, x_advs) proposals_list.append(x_advs_proposals) proposals = ep.stack(proposals_list, 0) proposals_distances = self.distance( ep.expand_dims(originals, 0), proposals) minimal_idx = ep.argmin(proposals_distances, 0) x_advs = proposals[minimal_idx] distances = self.distance(originals, x_advs) # log stats tb.histogram("norms", distances, step) return restore_type(x_advs)
def test_index_update_indices_scalar(dummy: Tensor) -> Tensor: x = ep.ones(dummy, (3, 4)) ind = ep.from_numpy(dummy, np.array([0, 1, 2, 1])) return ep.index_update(x, ep.index[ind, ep.arange(x, 4)], 33.0)
def run( self, model: Model, inputs: T, criterion: Union[Misclassification, TargetedMisclassification, T], *, early_stop: Optional[float] = None, **kwargs: Any, ) -> T: raise_if_kwargs(kwargs) x, restore_type = ep.astensor_(inputs) criterion_ = get_criterion(criterion) del inputs, criterion, kwargs N = len(x) if isinstance(criterion_, Misclassification): targeted = False classes = criterion_.labels change_classes_logits = self.confidence elif isinstance(criterion_, TargetedMisclassification): targeted = True classes = criterion_.target_classes change_classes_logits = -self.confidence else: raise ValueError("unsupported criterion") def is_adversarial(perturbed: ep.Tensor, logits: ep.Tensor) -> ep.Tensor: if change_classes_logits != 0: logits += ep.onehot_like(logits, classes, value=change_classes_logits) return criterion_(perturbed, logits) if classes.shape != (N, ): name = "target_classes" if targeted else "labels" raise ValueError( f"expected {name} to have shape ({N},), got {classes.shape}") bounds = model.bounds to_attack_space = partial(_to_attack_space, bounds=bounds) to_model_space = partial(_to_model_space, bounds=bounds) x_attack = to_attack_space(x) reconstsructed_x = to_model_space(x_attack) rows = range(N) def loss_fun( delta: ep.Tensor, consts: ep.Tensor ) -> Tuple[ep.Tensor, Tuple[ep.Tensor, ep.Tensor]]: assert delta.shape == x_attack.shape assert consts.shape == (N, ) x = to_model_space(x_attack + delta) logits = model(x) if targeted: c_minimize = best_other_classes(logits, classes) c_maximize = classes # target_classes else: c_minimize = classes # labels c_maximize = best_other_classes(logits, classes) is_adv_loss = logits[rows, c_minimize] - logits[rows, c_maximize] assert is_adv_loss.shape == (N, ) is_adv_loss = is_adv_loss + self.confidence is_adv_loss = ep.maximum(0, is_adv_loss) is_adv_loss = is_adv_loss * consts squared_norms = flatten(x - reconstsructed_x).square().sum(axis=-1) loss = is_adv_loss.sum() + squared_norms.sum() return loss, (x, logits) loss_aux_and_grad = ep.value_and_grad_fn(x, loss_fun, has_aux=True) consts = self.initial_const * np.ones((N, )) lower_bounds = np.zeros((N, )) upper_bounds = np.inf * np.ones((N, )) best_advs = ep.zeros_like(x) best_advs_norms = ep.full(x, (N, ), ep.inf) # the binary search searches for the smallest consts that produce adversarials for binary_search_step in range(self.binary_search_steps): if (binary_search_step == self.binary_search_steps - 1 and self.binary_search_steps >= 10): # in the last binary search step, repeat the search once consts = np.minimum(upper_bounds, 1e10) # create a new optimizer find the delta that minimizes the loss delta = ep.zeros_like(x_attack) optimizer = AdamOptimizer(delta) # tracks whether adv with the current consts was found found_advs = np.full((N, ), fill_value=False) loss_at_previous_check = np.inf consts_ = ep.from_numpy(x, consts.astype(np.float32)) for step in range(self.steps): loss, (perturbed, logits), gradient = loss_aux_and_grad(delta, consts_) delta += optimizer(gradient, self.stepsize) if self.abort_early and step % (np.ceil(self.steps / 10)) == 0: # after each tenth of the overall steps, check progress if not (loss <= 0.9999 * loss_at_previous_check): break # stop Adam if there has been no progress loss_at_previous_check = loss found_advs_iter = is_adversarial(perturbed, logits) found_advs = np.logical_or(found_advs, found_advs_iter.numpy()) norms = flatten(perturbed - x).norms.l2(axis=-1) closer = norms < best_advs_norms new_best = ep.logical_and(closer, found_advs_iter) new_best_ = atleast_kd(new_best, best_advs.ndim) best_advs = ep.where(new_best_, perturbed, best_advs) best_advs_norms = ep.where(new_best, norms, best_advs_norms) upper_bounds = np.where(found_advs, consts, upper_bounds) lower_bounds = np.where(found_advs, lower_bounds, consts) consts_exponential_search = consts * 10 consts_binary_search = (lower_bounds + upper_bounds) / 2 consts = np.where(np.isinf(upper_bounds), consts_exponential_search, consts_binary_search) return restore_type(best_advs)
def __call__(self, inputs: T) -> T: x, restore_type = ep.astensor_(inputs) y = self._model(x.numpy()) z = ep.from_numpy(x, y) return restore_type(z)
def run( self, model: Model, inputs: T, criterion: Union[Criterion, T], *, early_stop: Optional[float] = None, **kwargs: Any, ) -> T: raise_if_kwargs(kwargs) x, restore_type = ep.astensor_(inputs) del inputs, kwargs criterion = get_criterion(criterion) is_adversarial = get_is_adversarial(criterion, model) if x.ndim != 4: raise NotImplementedError( "only implemented for inputs with two spatial dimensions (and one channel and one batch dimension)" ) if self.channel_axis is None: channel_axis = get_channel_axis(model, x.ndim) else: channel_axis = self.channel_axis % x.ndim if channel_axis is None: raise ValueError( "cannot infer the data_format from the model, please specify" " channel_axis when initializing the attack" ) max_sigma: float if self.max_sigma is None: if channel_axis == 1: h, w = x.shape[2:4] elif channel_axis == 3: h, w = x.shape[1:3] else: raise ValueError( "expected 'channel_axis' to be 1 or 3, got {channel_axis}" ) max_sigma = max(h, w) else: max_sigma = self.max_sigma min_, max_ = model.bounds x0 = x x0_ = x0.numpy() result = x0 found = is_adversarial(x0) epsilon = 0.0 stepsize = 1.0 / self.steps for _ in range(self.steps): # TODO: reduce the batch size to the ones that haven't been sucessful epsilon += stepsize sigmas = [epsilon * max_sigma] * x0.ndim sigmas[0] = 0 sigmas[channel_axis] = 0 # TODO: once we can implement gaussian_filter in eagerpy, avoid converting from numpy x_ = gaussian_filter(x0_, sigmas) x_ = np.clip(x_, min_, max_) x = ep.from_numpy(x0, x_) is_adv = is_adversarial(x) new_adv = ep.logical_and(is_adv, found.logical_not()) result = ep.where(atleast_kd(new_adv, x.ndim), x, result) found = ep.logical_or(new_adv, found) if found.all(): break return restore_type(result)
def test_slogdet(dummy: Tensor, array: Tensor, output: str) -> Tensor: a = ep.from_numpy(dummy, array).float32() outputs = dict() outputs["sign"], outputs["logdet"] = ep.slogdet(a) return outputs[output]
def isfull(self): result = ~np.isnan(self.data).any(axis=0) return ep.from_numpy(self.tensor, result)
def __call__( self, inputs, labels, *, target_classes=None, binary_search_steps=9, max_iterations=10000, confidence=0, initial_learning_rate=1e-2, regularization=1e-2, initial_const=1e-3, abort_early=True, decision_rule="EN", ): x_0 = ep.astensor(inputs) N = len(x_0) assert decision_rule in ("EN", "L1") targeted = target_classes is not None if targeted: labels = None target_classes = ep.astensor(target_classes) assert target_classes.shape == (N, ) is_adv = partial(targeted_is_adv, target_classes=target_classes, confidence=confidence) else: labels = ep.astensor(labels) assert labels.shape == (N, ) is_adv = partial(untargeted_is_adv, labels=labels, confidence=confidence) min_, max_ = self.model.bounds() rows = np.arange(N) def loss_fun(y_k: ep.Tensor, consts: ep.Tensor) -> ep.Tensor: assert y_k.shape == x_0.shape assert consts.shape == (N, ) logits = ep.astensor(self.model.forward(y_k.tensor)) if targeted: c_minimize = best_other_classes(logits, target_classes) c_maximize = target_classes else: c_minimize = labels c_maximize = best_other_classes(logits, labels) is_adv_loss = logits[rows, c_minimize] - logits[rows, c_maximize] assert is_adv_loss.shape == (N, ) is_adv_loss = is_adv_loss + confidence is_adv_loss = ep.maximum(0, is_adv_loss) is_adv_loss = is_adv_loss * consts squared_norms = flatten(y_k - x_0).square().sum(axis=-1) loss = is_adv_loss.sum() + squared_norms.sum() return loss, (y_k, logits) loss_aux_and_grad = ep.value_and_grad_fn(x_0, loss_fun, has_aux=True) consts = initial_const * np.ones((N, )) lower_bounds = np.zeros((N, )) upper_bounds = np.inf * np.ones((N, )) best_advs = ep.zeros_like(x_0) best_advs_norms = ep.ones(x_0, (N, )) * np.inf # the binary search searches for the smallest consts that produce adversarials for binary_search_step in range(binary_search_steps): if (binary_search_step == binary_search_steps - 1 and binary_search_steps >= 10): # in the last iteration, repeat the search once consts = np.minimum(upper_bounds, 1e10) # create a new optimizer find the delta that minimizes the loss # TODO: rewrite this once eagerpy supports .copy() x_k = x_0 # ep.zeros_like(x_0) + x_0 y_k = x_0 # ep.zeros_like(x_0) + x_0 found_advs = np.full( (N, ), fill_value=False) # found adv with the current consts loss_at_previous_check = np.inf consts_ = ep.from_numpy(x_0, consts.astype(np.float32)) for iteration in range(max_iterations): # square-root learning rate decay learning_rate = (initial_learning_rate * (1.0 - iteration / max_iterations)**0.5) loss, (x, logits), gradient = loss_aux_and_grad(x_k, consts_) x_k_old = x_k x_k = project_shrinkage_thresholding( y_k - learning_rate * gradient, x_0, regularization, min_, max_) y_k = x_k + iteration / (iteration + 3) - (x_k - x_k_old) if abort_early and iteration % (np.ceil( max_iterations / 10)) == 0: # after each tenth of the iterations, check progress if not (loss <= 0.9999 * loss_at_previous_check): break # stop Adam if there has been no progress loss_at_previous_check = loss found_advs_iter = is_adv(logits) best_advs, best_advs_norms = apply_decision_rule( decision_rule, regularization, best_advs, best_advs_norms, x_k, x_0, found_advs_iter, ) found_advs = np.logical_or(found_advs, found_advs_iter.numpy()) upper_bounds = np.where(found_advs, consts, upper_bounds) lower_bounds = np.where(found_advs, lower_bounds, consts) consts_exponential_search = consts * 10 consts_binary_search = (lower_bounds + upper_bounds) / 2 consts = np.where(np.isinf(upper_bounds), consts_exponential_search, consts_binary_search) return best_advs.tensor
def run( self, model: Model, inputs: T, criterion: Union[Criterion, Any] = None, *, starting_points: Optional[ep.Tensor] = None, early_stop: Optional[float] = None, **kwargs: Any, ) -> T: raise_if_kwargs(kwargs) del kwargs x, restore_type = ep.astensor_(inputs) del inputs verify_input_bounds(x, model) criterion_ = get_criterion(criterion) del criterion is_adversarial = get_is_adversarial(criterion_, model) if starting_points is None: init_attack: MinimizationAttack if self.init_attack is None: init_attack = SaltAndPepperNoiseAttack() logging.info( f"Neither starting_points nor init_attack given. Falling" f" back to {init_attack!r} for initialization." ) else: init_attack = self.init_attack # TODO: use call and support all types of attacks (once early_stop is # possible in __call__) starting_points = init_attack.run(model, x, criterion_) x_adv = ep.astensor(starting_points) assert is_adversarial(x_adv).all() original_shape = x.shape N = len(x) x_flat = flatten(x) x_adv_flat = flatten(x_adv) # was there a pixel left in the samples to manipulate, # i.e. reset to the clean version? found_index_to_manipulate = ep.from_numpy(x, np.ones(N, dtype=bool)) while ep.any(found_index_to_manipulate): diff_mask = (ep.abs(x_flat - x_adv_flat) > 1e-8).numpy() diff_idxs = [z.nonzero()[0] for z in diff_mask] untouched_indices = [z.tolist() for z in diff_idxs] untouched_indices = [ np.random.permutation(it).tolist() for it in untouched_indices ] found_index_to_manipulate = ep.from_numpy(x, np.zeros(N, dtype=bool)) # since the number of pixels still left to manipulate might differ # across different samples we track each of them separately and # and manipulate the images until there is no pixel left for # any of the samples. to not update already finished samples, we mask # the updates such that only samples that still have pixels left to manipulate # will be updated i = 0 while i < max([len(it) for it in untouched_indices]): # mask all samples that still have pixels to manipulate left relevant_mask = [len(it) > i for it in untouched_indices] relevant_mask = np.array(relevant_mask, dtype=bool) relevant_mask_index = np.flatnonzero(relevant_mask) # for each image get the index of the next pixel we try out relevant_indices = [it[i] for it in untouched_indices if len(it) > i] old_values = x_adv_flat[relevant_mask_index, relevant_indices] new_values = x_flat[relevant_mask_index, relevant_indices] x_adv_flat = ep.index_update( x_adv_flat, (relevant_mask_index, relevant_indices), new_values ) # check if still adversarial is_adv = is_adversarial(x_adv_flat.reshape(original_shape)) found_index_to_manipulate = ep.index_update( found_index_to_manipulate, relevant_mask_index, ep.logical_or(found_index_to_manipulate, is_adv)[relevant_mask], ) # if not, undo change new_or_old_values = ep.where( is_adv[relevant_mask], new_values, old_values ) x_adv_flat = ep.index_update( x_adv_flat, (relevant_mask_index, relevant_indices), new_or_old_values, ) i += 1 if not ep.any(found_index_to_manipulate): break if self.l2_binary_search: while True: diff_mask = (ep.abs(x_flat - x_adv_flat) > 1e-12).numpy() diff_idxs = [z.nonzero()[0] for z in diff_mask] untouched_indices = [z.tolist() for z in diff_idxs] # draw random shuffling of all indices for all samples untouched_indices = [ np.random.permutation(it).tolist() for it in untouched_indices ] # whether that run through all values made any improvement improved = ep.from_numpy(x, np.zeros(N, dtype=bool)).astype(bool) logging.info("Starting new loop through all values") # use the same logic as above i = 0 while i < max([len(it) for it in untouched_indices]): # mask all samples that still have pixels to manipulate left relevant_mask = [len(it) > i for it in untouched_indices] relevant_mask = np.array(relevant_mask, dtype=bool) relevant_mask_index = np.flatnonzero(relevant_mask) # for each image get the index of the next pixel we try out relevant_indices = [ it[i] for it in untouched_indices if len(it) > i ] old_values = x_adv_flat[relevant_mask_index, relevant_indices] new_values = x_flat[relevant_mask_index, relevant_indices] x_adv_flat = ep.index_update( x_adv_flat, (relevant_mask_index, relevant_indices), new_values ) # check if still adversarial is_adv = is_adversarial(x_adv_flat.reshape(original_shape)) improved = ep.index_update( improved, relevant_mask_index, ep.logical_or(improved, is_adv)[relevant_mask], ) if not ep.all(is_adv): # run binary search for examples that became non-adversarial updated_new_values = self._binary_search( x_adv_flat, relevant_mask, relevant_mask_index, relevant_indices, old_values, new_values, (-1, *original_shape[1:]), is_adversarial, ) x_adv_flat = ep.index_update( x_adv_flat, (relevant_mask_index, relevant_indices), ep.where( is_adv[relevant_mask], new_values, updated_new_values ), ) improved = ep.index_update( improved, relevant_mask_index, ep.logical_or( old_values != updated_new_values, improved[relevant_mask], ), ) i += 1 if not ep.any(improved): # no improvement for any of the indices break x_adv = x_adv_flat.reshape(original_shape) return restore_type(x_adv)