def pointwise_ops(self):
a = torch.randn(4)
b = torch.randn(4)
t = torch.tensor([-1, -2, 3], dtype=torch.int8)
r = torch.tensor([0, 1, 10, 0], dtype=torch.int8)
t = torch.tensor([-1, -2, 3], dtype=torch.int8)
s = torch.tensor([4, 0, 1, 0], dtype=torch.int8)
f = torch.zeros(3)
g = torch.tensor([-1, 0, 1])
w = torch.tensor([0.3810, 1.2774, -0.2972, -0.3719, 0.4637])
return (
torch.abs(torch.tensor([-1, -2, 3])),
torch.absolute(torch.tensor([-1, -2, 3])),
torch.acos(a),
torch.arccos(a),
torch.acosh(a.uniform_(1.0, 2.0)),
torch.add(a, 20),
torch.add(a, torch.randn(4, 1), alpha=10),
torch.addcdiv(torch.randn(1, 3),
torch.randn(3, 1),
torch.randn(1, 3),
value=0.1),
torch.addcmul(torch.randn(1, 3),
torch.randn(3, 1),
torch.randn(1, 3),
value=0.1),
torch.angle(a),
torch.asin(a),
torch.arcsin(a),
torch.asinh(a),
torch.arcsinh(a),
torch.atan(a),
torch.arctan(a),
torch.atanh(a.uniform_(-1.0, 1.0)),
torch.arctanh(a.uniform_(-1.0, 1.0)),
torch.atan2(a, a),
torch.bitwise_not(t),
torch.bitwise_and(t, torch.tensor([1, 0, 3], dtype=torch.int8)),
torch.bitwise_or(t, torch.tensor([1, 0, 3], dtype=torch.int8)),
torch.bitwise_xor(t, torch.tensor([1, 0, 3], dtype=torch.int8)),
torch.ceil(a),
torch.clamp(a, min=-0.5, max=0.5),
torch.clamp(a, min=0.5),
torch.clamp(a, max=0.5),
torch.clip(a, min=-0.5, max=0.5),
torch.conj(a),
torch.copysign(a, 1),
torch.copysign(a, b),
torch.cos(a),
torch.cosh(a),
torch.deg2rad(
torch.tensor([[180.0, -180.0], [360.0, -360.0], [90.0,
-90.0]])),
torch.div(a, b),
torch.divide(a, b, rounding_mode="trunc"),
torch.divide(a, b, rounding_mode="floor"),
torch.digamma(torch.tensor([1.0, 0.5])),
torch.erf(torch.tensor([0.0, -1.0, 10.0])),
torch.erfc(torch.tensor([0.0, -1.0, 10.0])),
torch.erfinv(torch.tensor([0.0, 0.5, -1.0])),
torch.exp(torch.tensor([0.0, math.log(2.0)])),
torch.exp2(torch.tensor([0.0, math.log(2.0), 3.0, 4.0])),
torch.expm1(torch.tensor([0.0, math.log(2.0)])),
torch.fake_quantize_per_channel_affine(
torch.randn(2, 2, 2),
(torch.randn(2) + 1) * 0.05,
torch.zeros(2),
1,
0,
255,
),
torch.fake_quantize_per_tensor_affine(a, 0.1, 0, 0, 255),
torch.float_power(torch.randint(10, (4, )), 2),
torch.float_power(torch.arange(1, 5), torch.tensor([2, -3, 4,
-5])),
torch.floor(a),
# torch.floor_divide(torch.tensor([4.0, 3.0]), torch.tensor([2.0, 2.0])),
# torch.floor_divide(torch.tensor([4.0, 3.0]), 1.4),
torch.fmod(torch.tensor([-3, -2, -1, 1, 2, 3]), 2),
torch.fmod(torch.tensor([1, 2, 3, 4, 5]), 1.5),
torch.frac(torch.tensor([1.0, 2.5, -3.2])),
torch.randn(4, dtype=torch.cfloat).imag,
torch.ldexp(torch.tensor([1.0]), torch.tensor([1])),
torch.ldexp(torch.tensor([1.0]), torch.tensor([1, 2, 3, 4])),
torch.lerp(torch.arange(1.0, 5.0),
torch.empty(4).fill_(10), 0.5),
torch.lerp(
torch.arange(1.0, 5.0),
torch.empty(4).fill_(10),
torch.full_like(torch.arange(1.0, 5.0), 0.5),
),
torch.lgamma(torch.arange(0.5, 2, 0.5)),
torch.log(torch.arange(5) + 10),
torch.log10(torch.rand(5)),
torch.log1p(torch.randn(5)),
torch.log2(torch.rand(5)),
torch.logaddexp(torch.tensor([-1.0]), torch.tensor([-1, -2, -3])),
torch.logaddexp(torch.tensor([-100.0, -200.0, -300.0]),
torch.tensor([-1, -2, -3])),
torch.logaddexp(torch.tensor([1.0, 2000.0, 30000.0]),
torch.tensor([-1, -2, -3])),
torch.logaddexp2(torch.tensor([-1.0]), torch.tensor([-1, -2, -3])),
torch.logaddexp2(torch.tensor([-100.0, -200.0, -300.0]),
torch.tensor([-1, -2, -3])),
torch.logaddexp2(torch.tensor([1.0, 2000.0, 30000.0]),
torch.tensor([-1, -2, -3])),
torch.logical_and(r, s),
torch.logical_and(r.double(), s.double()),
torch.logical_and(r.double(), s),
torch.logical_and(r, s, out=torch.empty(4, dtype=torch.bool)),
torch.logical_not(torch.tensor([0, 1, -10], dtype=torch.int8)),
torch.logical_not(
torch.tensor([0.0, 1.5, -10.0], dtype=torch.double)),
torch.logical_not(
torch.tensor([0.0, 1.0, -10.0], dtype=torch.double),
out=torch.empty(3, dtype=torch.int16),
),
torch.logical_or(r, s),
torch.logical_or(r.double(), s.double()),
torch.logical_or(r.double(), s),
torch.logical_or(r, s, out=torch.empty(4, dtype=torch.bool)),
torch.logical_xor(r, s),
torch.logical_xor(r.double(), s.double()),
torch.logical_xor(r.double(), s),
torch.logical_xor(r, s, out=torch.empty(4, dtype=torch.bool)),
torch.logit(torch.rand(5), eps=1e-6),
torch.hypot(torch.tensor([4.0]), torch.tensor([3.0, 4.0, 5.0])),
torch.i0(torch.arange(5, dtype=torch.float32)),
torch.igamma(a, b),
torch.igammac(a, b),
torch.mul(torch.randn(3), 100),
torch.multiply(torch.randn(4, 1), torch.randn(1, 4)),
torch.mvlgamma(torch.empty(2, 3).uniform_(1.0, 2.0), 2),
torch.tensor([float("nan"),
float("inf"), -float("inf"), 3.14]),
torch.nan_to_num(w),
torch.nan_to_num(w, nan=2.0),
torch.nan_to_num(w, nan=2.0, posinf=1.0),
torch.neg(torch.randn(5)),
# torch.nextafter(torch.tensor([1, 2]), torch.tensor([2, 1])) == torch.tensor([eps + 1, 2 - eps]),
torch.polygamma(1, torch.tensor([1.0, 0.5])),
torch.polygamma(2, torch.tensor([1.0, 0.5])),
torch.polygamma(3, torch.tensor([1.0, 0.5])),
torch.polygamma(4, torch.tensor([1.0, 0.5])),
torch.pow(a, 2),
torch.pow(torch.arange(1.0, 5.0), torch.arange(1.0, 5.0)),
torch.rad2deg(
torch.tensor([[3.142, -3.142], [6.283, -6.283],
[1.570, -1.570]])),
torch.randn(4, dtype=torch.cfloat).real,
torch.reciprocal(a),
torch.remainder(torch.tensor([-3.0, -2.0]), 2),
torch.remainder(torch.tensor([1, 2, 3, 4, 5]), 1.5),
torch.round(a),
torch.rsqrt(a),
torch.sigmoid(a),
torch.sign(torch.tensor([0.7, -1.2, 0.0, 2.3])),
torch.sgn(a),
torch.signbit(torch.tensor([0.7, -1.2, 0.0, 2.3])),
torch.sin(a),
torch.sinc(a),
torch.sinh(a),
torch.sqrt(a),
torch.square(a),
torch.sub(torch.tensor((1, 2)), torch.tensor((0, 1)), alpha=2),
torch.tan(a),
torch.tanh(a),
torch.trunc(a),
torch.xlogy(f, g),
torch.xlogy(f, g),
torch.xlogy(f, 4),
torch.xlogy(2, g),
)
def carlini_wagner_l2(
model_fn,
x,
n_classes,
y=None,
targeted=False,
lr=5e-3,
confidence=0,
clip_min=0,
clip_max=1,
initial_const=1e-2,
binary_search_steps=5,
max_iterations=1000,
):
"""
This attack was originally proposed by Carlini and Wagner. It is an
iterative attack that finds adversarial examples on many defenses that
are robust to other attacks.
Paper link: https://arxiv.org/abs/1608.04644
At a high level, this attack is an iterative attack using Adam and
a specially-chosen loss function to find adversarial examples with
lower distortion than other attacks. This comes at the cost of speed,
as this attack is often much slower than others.
:param model_fn: a callable that takes an input tensor and returns
the model logits. The logits should be a tensor of shape
(n_examples, n_classes).
:param x: input tensor of shape (n_examples, ...), where ... can
be any arbitrary dimension that is compatible with
model_fn.
:param n_classes: the number of classes.
:param y: (optional) Tensor with true labels. If targeted is true,
then provide the target label. Otherwise, only provide
this parameter if you'd like to use true labels when
crafting adversarial samples. Otherwise, model predictions
are used as labels to avoid the "label leaking" effect
(explained in this paper:
https://arxiv.org/abs/1611.01236). If provide y, it
should be a 1D tensor of shape (n_examples, ).
Default is None.
:param targeted: (optional) bool. Is the attack targeted or
untargeted? Untargeted, the default, will try to make the
label incorrect. Targeted will instead try to move in the
direction of being more like y.
:param lr: (optional) float. The learning rate for the attack
algorithm. Default is 5e-3.
:param confidence: (optional) float. Confidence of adversarial
examples: higher produces examples with larger l2
distortion, but more strongly classified as adversarial.
Default is 0.
:param clip_min: (optional) float. Minimum float value for
adversarial example components. Default is 0.
:param clip_max: (optional) float. Maximum float value for
adversarial example components. Default is 1.
:param initial_const: The initial tradeoff-constant to use to tune the
relative importance of size of the perturbation and
confidence of classification. If binary_search_steps is
large, the initial constant is not important. A smaller
value of this constant gives lower distortion results.
Default is 1e-2.
:param binary_search_steps: (optional) int. The number of times we
perform binary search to find the optimal tradeoff-constant
between norm of the perturbation and confidence of the
classification. Default is 5.
:param max_iterations: (optional) int. The maximum number of
iterations. Setting this to a larger value will produce
lower distortion results. Using only a few iterations
requires a larger learning rate, and will produce larger
distortion results. Default is 1000.
"""
def compare(pred, label, is_logits=False):
"""
A helper function to compare prediction against a label.
Returns true if the attack is considered successful.
:param pred: can be either a 1D tensor of logits or a predicted
class (int).
:param label: int. A label to compare against.
:param is_logits: (optional) bool. If True, treat pred as an
array of logits. Default is False.
"""
# Convert logits to predicted class if necessary
if is_logits:
pred_copy = pred.clone().detach()
pred_copy[label] += -confidence if targeted else confidence
pred = torch.argmax(pred_copy)
return pred == label if targeted else pred != label
if y is None:
# Using model predictions as ground truth to avoid label leaking
pred = model_fn(x)
y = torch.argmax(pred, 1)
# Initialize some values needed for binary search on const
lower_bound = [0.0] * len(x)
upper_bound = [1e10] * len(x)
const = x.new_ones(len(x), 1) * initial_const
o_bestl2 = [INF] * len(x)
o_bestscore = [-1.0] * len(x)
x = torch.clamp(x, clip_min, clip_max)
ox = x.clone().detach() # save the original x
o_bestattack = x.clone().detach()
# Map images into the tanh-space
x = (x - clip_min) / (clip_max - clip_min)
x = torch.clamp(x, 0, 1)
x = x * 2 - 1
x = torch.arctanh(x * 0.999999)
# Prepare some variables
modifier = torch.zeros_like(x, requires_grad=True)
y_onehot = torch.nn.functional.one_hot(y, n_classes).to(torch.float)
# Define loss functions and optimizer
f_fn = lambda real, other, targeted: torch.max(
((other - real) if targeted else (real - other)) + confidence,
torch.tensor(0.0).to(real.device),
)
l2dist_fn = lambda x, y: torch.pow(x - y, 2).sum(
list(range(len(x.size())))[1:])
optimizer = torch.optim.Adam([modifier], lr=lr)
# Outer loop performing binary search on const
for outer_step in range(binary_search_steps):
# Initialize some values needed for the inner loop
bestl2 = [INF] * len(x)
bestscore = [-1.0] * len(x)
# Inner loop performing attack iterations
for i in range(max_iterations):
# One attack step
new_x = (torch.tanh(modifier + x) + 1) / 2
new_x = new_x * (clip_max - clip_min) + clip_min
logits = model_fn(new_x)
real = torch.sum(y_onehot * logits, 1)
other, _ = torch.max((1 - y_onehot) * logits - y_onehot * 1e4, 1)
optimizer.zero_grad()
f = f_fn(real, other, targeted)
l2 = l2dist_fn(new_x, ox)
loss = (const * f + l2).sum()
loss.backward()
optimizer.step()
# Update best results
for n, (l2_n, logits_n,
new_x_n) in enumerate(zip(l2, logits, new_x)):
y_n = y[n]
succeeded = compare(logits_n, y_n, is_logits=True)
if l2_n < o_bestl2[n] and succeeded:
pred_n = torch.argmax(logits_n)
o_bestl2[n] = l2_n
o_bestscore[n] = pred_n
o_bestattack[n] = new_x_n
# l2_n < o_bestl2[n] implies l2_n < bestl2[n] so we modify inner loop variables too
bestl2[n] = l2_n
bestscore[n] = pred_n
elif l2_n < bestl2[n] and succeeded:
bestl2[n] = l2_n
bestscore[n] = torch.argmax(logits_n)
# Binary search step
for n in range(len(x)):
y_n = y[n]
if compare(bestscore[n], y_n) and bestscore[n] != -1:
# Success, divide const by two
upper_bound[n] = min(upper_bound[n], const[n])
if upper_bound[n] < 1e9:
const[n] = (lower_bound[n] + upper_bound[n]) / 2
else:
# Failure, either multiply by 10 if no solution found yet
# or do binary search with the known upper bound
lower_bound[n] = max(lower_bound[n], const[n])
if upper_bound[n] < 1e9:
const[n] = (lower_bound[n] + upper_bound[n]) / 2
else:
const[n] *= 10
return o_bestattack.detach()
def arctanh(a: Numeric):
return torch.arctanh(a)
# asin/arcsin
torch.asin(a)
torch.arcsin(a)
# asinh/arcsinh
torch.asinh(a)
torch.arcsinh(a)
# atan/arctan
torch.atan(a)
torch.arctan(a)
# atanh/arctanh
torch.atanh(a.uniform_(-1, 1))
torch.arctanh(a.uniform_(-1, 1))
# atan2
torch.atan2(a, a)
# bitwise_not
torch.bitwise_not(t)
# bitwise_and
torch.bitwise_and(t, torch.tensor([1, 0, 3], dtype=torch.int8))
torch.bitwise_and(torch.tensor([True, True, False]),
torch.tensor([False, True, False]))
# bitwise_or
torch.bitwise_or(t, torch.tensor([1, 0, 3], dtype=torch.int8))
torch.bitwise_or(torch.tensor([True, True, False]),
def forward(self, x, log_df_dz):
log_det = torch.sum(torch.log(deriv_arctanh(x)), dim=1)
return torch.arctanh(x), log_df_dz + log_det
def backward(self, x, log_df_dz):
log_det = torch.log(deriv_arctanh(x))
log_det = torch.sum(log_det.view(x.size(0), -1), dim=1)
return torch.arctanh(x), log_df_dz + log_det