def _rgb2hsv(img):
r, g, b = img.unbind(dim=-3)
# Implementation is based on https://github.com/python-pillow/Pillow/blob/4174d4267616897df3746d315d5a2d0f82c656ee/
# src/libImaging/Convert.c#L330
maxc = torch.max(img, dim=-3).values
minc = torch.min(img, dim=-3).values
# The algorithm erases S and H channel where `maxc = minc`. This avoids NaN
# from happening in the results, because
# + S channel has division by `maxc`, which is zero only if `maxc = minc`
# + H channel has division by `(maxc - minc)`.
#
# Instead of overwriting NaN afterwards, we just prevent it from occuring so
# we don't need to deal with it in case we save the NaN in a buffer in
# backprop, if it is ever supported, but it doesn't hurt to do so.
eqc = maxc == minc
cr = maxc - minc
# Since `eqc => cr = 0`, replacing denominator with 1 when `eqc` is fine.
ones = torch.ones_like(maxc)
s = cr / torch.where(eqc, ones, maxc)
# Note that `eqc => maxc = minc = r = g = b`. So the following calculation
# of `h` would reduce to `bc - gc + 2 + rc - bc + 4 + rc - bc = 6` so it
# would not matter what values `rc`, `gc`, and `bc` have here, and thus
# replacing denominator with 1 when `eqc` is fine.
cr_divisor = torch.where(eqc, ones, cr)
rc = (maxc - r) / cr_divisor
gc = (maxc - g) / cr_divisor
bc = (maxc - b) / cr_divisor
hr = (maxc == r) * (bc - gc)
hg = ((maxc == g) & (maxc != r)) * (2.0 + rc - bc)
hb = ((maxc != g) & (maxc != r)) * (4.0 + gc - rc)
h = (hr + hg + hb)
h = torch.fmod((h / 6.0 + 1.0), 1.0)
return torch.stack((h, s, maxc), dim=-3)
def forward(self,
q1,
q2,
index,
tar_tri,
tar_tri_ema,
sour_labels,
uncer=None,
epoch=0):
batchSize = q1.shape[0]
# tar_tri = normalize(tar_tri, axis=-1)
q1 = normalize(q1, axis=-1)
q2 = normalize(q2, axis=-1)
loss_q1 = self.memo_circle_loss(index + self.sour_numclass, q1, uncer)
loss_q2 = self.memo_center_circle_loss(sour_labels, q2)
with torch.no_grad():
q1 = q1.detach()
out_ids = torch.arange(batchSize).cuda()
out_ids += self.index
out_ids = torch.fmod(out_ids, self.queueSize - self.sour_numclass)
out_ids = (out_ids + self.sour_numclass).long()
self.memory.index_copy_(0, out_ids, q1)
self.index_memory.index_copy_(0, out_ids,
index + self.sour_numclass)
if uncer is not None:
self.uncer.index_copy_(0, out_ids, uncer)
self.index = (self.index + batchSize) % (self.queueSize -
self.sour_numclass)
for x, y in zip(q2, sour_labels):
self.memory[y] = self.momentum * self.memory[y] + (
1. - self.momentum) * x
self.memory[y] /= self.memory[y].norm()
return loss_q1, loss_q2, None, None
def get_labels(kmag, kori, mag_max=25):
# for circular coherence of labels
kori = torch.fmod(kori, 180)
mask = create_mask(kori, 174, 177)
kori[mask] = 174
kori[kori > 177] = 0
# for magnitude
kmag[kmag > mag_max] = mag_max
# find the labels
inte_ori = mag_max // 2
inte_mag = 2
idxq = torch.round(kori / 6)
tmp = kmag - 1
tmp[tmp < 0] = 0
idxm = torch.round(tmp / inte_mag)
kori_q = idxq * inte_ori
kmag_q = 1 + idxm * inte_mag
labels = idxq * inte_ori + idxm + 1
labels[kmag_q == 1] = 1
return labels - 1
def patch_match(x, y, mask, patch_size=3, radius=3, stride=1):
batch, channels, height, width = x.size()
y_pad = F.pad(y, (radius // 2, radius // 2, radius // 2, radius // 2)) # Left, right, up, down
distance_all = []
for i in range(0, radius, stride): # Searching/matching in row-major order
for j in range(0, radius, stride):
distance_pix = torch.sum((y_pad[:, :, i:i + height, j:j + width] - x) ** 2, dim=1, keepdim=True)
distance_all += [F.avg_pool2d(distance_pix, patch_size, stride=1, padding=patch_size // 2)]
distance_all = torch.cat(distance_all, dim=1) # Thus this stack of distances will be in row major order
location_min = torch.argmin(distance_all, dim=1) # get the pixel/patch with the minimal distance
location_min = location_min * mask # Only need to match within the mask
distance_min_x = torch.fmod(location_min, radius) - radius // 2 # Need to adjust to take into account searching behind
distance_min_y = location_min / radius - radius // 2
grid_x = torch.arange(width).cuda().unsqueeze(0).unsqueeze(0) + distance_min_x.type(torch.float32)
grid_y = torch.arange(height).cuda().unsqueeze(1).unsqueeze(0) + distance_min_y.type(torch.float32)
grid_x = torch.clamp(grid_x.float() / width, 0, 1) * 2 - 1
grid_y = torch.clamp(grid_y.float() / height, 0, 1) * 2 - 1
grid = torch.stack([grid_x, grid_y], dim=3)
out = F.grid_sample(y, grid)
return out
def _colorize_subset(
_subset: Tuple[Tensor, Tensor],
_correlation: float,
_decorr_op: Literal["random", "shift"],
) -> RawDataTuple:
x, y = _subset
x = x.unsqueeze(1).expand(-1, 3, -1, -1) / 255.0
for aug in augs:
x = aug(x)
s = y.clone()
if _decorr_op == "random": # this is for context and test set
indexes = torch.rand(s.shape) > _correlation
s[indexes] = torch.randint_like(s[indexes],
low=0,
high=num_colors)
elif cfg.bias.missing_s: # this is one possibility for training set
s = torch.randint_like(s, low=0, high=num_colors)
for to_remove in cfg.bias.missing_s:
s[s == to_remove] = (to_remove + 1) % num_colors
else: # this is another possibility for training set
indexes = torch.rand(s.shape) > _correlation
s[indexes] = torch.fmod(s[indexes] + 1, num_colors)
x_col = colorizer(x, s)
return RawDataTuple(x=x_col, s=s, y=y)
def adjust_hue_raw(input: torch.Tensor,
hue_factor: Union[float, torch.Tensor]) -> torch.Tensor:
r"""Adjust hue of an image. Expecting input to be in hsv format already.
"""
if not isinstance(input, torch.Tensor):
raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}")
if not isinstance(hue_factor, (float, torch.Tensor)):
raise TypeError(
f"The hue_factor should be a float number or torch.Tensor in the range between"
f" [-PI, PI]. Got {type(hue_factor)}")
if isinstance(hue_factor, float):
hue_factor = torch.as_tensor(hue_factor)
hue_factor = hue_factor.to(input.device, input.dtype)
# TODO: find a proper way to check bound values in batched tensors.
# if ((hue_factor < -pi) | (hue_factor > pi)).any():
# raise ValueError(f"Hue-factor must be in the range [-PI, PI]. Got {hue_factor}")
for _ in input.shape[1:]:
hue_factor = torch.unsqueeze(hue_factor, dim=-1)
# unpack the hsv values
h, s, v = torch.chunk(input, chunks=3, dim=-3)
# transform the hue value and appl module
divisor: float = 2 * pi
h_out: torch.Tensor = torch.fmod(h + hue_factor, divisor)
# pack back back the corrected hue
out: torch.Tensor = torch.cat([h_out, s, v], dim=-3)
return out
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size,
stride=1,
padding=0,
dilation=1,
groups: int = 1,
bias: bool = True,
padding_mode: str = 'zeros', # TODO: refine this type
hyper_module=None,
z_dim=512,
base=32):
super(HyperConv2d, self).__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.kernel_size = kernel_size
self.stride = stride
self.padding = padding
self.dilation = dilation
self.groups = groups
self.padding_mode = padding_mode
self.z_dim = z_dim
self.base = base
self.hyper_module = hyper_module
self.layer_embed = nn.ParameterList()
for i in range(self.out_channels // self.base):
for j in range(max(1, self.in_channels // self.base)):
self.layer_embed.append(
nn.Parameter(torch.fmod(torch.randn(self.z_dim), 2)))
if bias:
self.bias = nn.Parameter(torch.zeros(out_channels))
else:
self.register_parameter('bias', None)
def forward(self, x): x = self.feature_model(x) y0 = self.fc(x) Pc = F.softmax(y0, dim=1) y1 = torch.stack([self.bin_models[i](x) for i in range(self.num_classes)]).permute(1, 2, 0) Pl = F.softmax(y1, dim=1) Plc = Pl * torch.unsqueeze(Pc, dim=1) ind = torch.argmax(Plc.view(x.size(0), -1), dim=1, keepdim=True) ip = ind/self.num_classes ic = torch.fmod(ind, self.num_classes) label = torch.zeros(ic.size(0), self.num_classes).scatter_(1, ic.data.cpu(), 1.0) label = Variable(label.unsqueeze(2).cuda()) y1 = torch.squeeze(torch.bmm(y1, label), 2) if not args.multires: y2 = torch.stack([self.res_models[i](x) for i in range(self.num_classes)]).permute(1, 2, 0) y2 = torch.squeeze(torch.bmm(y2, label), 2) else: y2 = torch.stack([self.res_models[i](x) for i in range(self.num_classes * self.num_clusters)]) y2 = y2.view(self.num_classes, self.num_clusters, -1, self.ndim).permute(1, 2, 3, 0) y2 = torch.squeeze(torch.matmul(y2, label), 3) pose_label = torch.zeros(ip.size(0), self.num_clusters).scatter_(1, ip.data.cpu(), 1.0) pose_label = Variable(pose_label.unsqueeze(2).cuda()) y2 = torch.squeeze(torch.bmm(y2.permute(1, 2, 0), pose_label), 2) return [y0, y1, y2, Plc] # cat, pose_bin, pose_delta
def decision_step(self, batch_size, score, hx):
"""
排序 从topk* num_tag中选取topk个最大值
:param batch_size:
:param score:
:param hx:
:return:
"""
topv, topi = th.topk(score, self.topk) # topk*num_tag ->topk
class_id = th.fmod(topi, self.num_tag).long() # 求余数 当前时刻的输出状态
beam_id = th.div(topi, self.num_tag).long() # 求整数 前一时刻输出的索引
x = class_id.contiguous().view(-1, 1)
batch_offset = (th.arange(batch_size) * self.topk).view(
batch_size, 1).to(self.device).long()
# 前一时刻的隐状态
state_id = batch_offset + beam_id
state_id = state_id.view(-1)
hx = th.index_select(hx, 1, state_id).view(-1, batch_size * self.topk,
self.hidden_size)
return x, hx, topv, class_id, beam_id
def flow_to_color(flow, mask=None, max_flow=None):
"""Converts flow to 3-channel color image.
Args:
flow: tensor of shape [num_batch, 2, height, width].
mask: flow validity mask of shape [num_batch, 1, height, width].
"""
n = 8
B, _, H, W = flow.size()
mask = torch.ones(B, 1, H, W, dtype=flow.dtype, device=flow.device) \
if mask is None else mask
flow_u, flow_v = torch.split(flow, 1, dim=1)
if max_flow is not None:
max_flow = torch.max(torch.tensor(max_flow), torch.tensor(1.0))
else:
max_flow = torch.max(torch.abs(flow * mask))
mag = torch.pow(torch.sum(torch.pow(flow, 2), dim=1, keepdim=True), 0.5)
angle = torch.atan2(flow_v, flow_u)
im_h = torch.fmod(angle / (2 * np.pi) + 1.0, 1.0)
im_s = torch.clamp(mag * n / max_flow, 0., 1.)
im_v = torch.clamp(n - im_s, 0., 1.)
im_hsv = torch.cat((im_h, im_s, im_v), dim=1)
im_hsv = im_hsv.permute(0, 2, 3, 1)
im_rgb = np.empty((B, H, W, 3))
for i in range(B):
im_rgb[i, :, :, :] = hsv2rgb(im_hsv[i, :, :, :].cpu().numpy())
return torch.tensor(im_rgb, dtype=im_hsv.dtype).permute(0, 3, 1, 2)
def ipd_ild_features(stft_data, sampling_rate=8000):
"""
Computes interphase difference (IPD) and interlevel difference (ILD) for a
stereo spectrogram.
Args:
stft_data (Torch tensor): Tensor of shape batch_size, time_frames, mag+phase, channels, sources
sampling_rate (int): The rate at which data is sampled. Default = 8000 Hz
Returns:
ipd (``Torch tensor`): Interphase difference between two channels
ild (``Torch tensor``): Interlevel difference between two channels
"""
# The mag and phase are concatenated together, so each of them will have half the dimensionality
mag_phase_dim = stft_data.shape[2] // 2
# Separate the data by channels
stft_ch_one = stft_data[:, :, :, 0]
stft_ch_two = stft_data[:, :, :, 1]
# Calculate ILD over the magnitudes
# Extract the magnitudes from the stft data
stft_ch_one_mag = stft_ch_one[:, :, 0:mag_phase_dim]
stft_ch_two_mag = stft_ch_two[:, :, 0:mag_phase_dim]
vol = torch.abs(stft_ch_one_mag) + torch.abs(stft_ch_two_mag)
ild = torch.abs(stft_ch_one_mag) / (torch.abs(stft_ch_two_mag) + 1e-4)
ild = 20 * torch.log10(ild + 1e-8)
# Extract the phase from the stft data
phase_ch_two = stft_ch_one[..., -mag_phase_dim:]
phase_ch_one = stft_ch_two[..., -mag_phase_dim:]
ipd = torch.fmod(phase_ch_two - phase_ch_one, np.pi)
# Output shape of ILD and IPD = [batch_size, time_frames, mag_phase_dim, sources]
return ipd, ild, vol
def acc_call(output, target, type="vanilla"):
if type is "vanilla":
pred = output.data.max(1)[1]
correct = pred.cpu().eq(target).sum().item()
acc = correct * 1.
# print(acc)
return acc
if type is "ensemble":
with torch.no_grad():
# print(output)
# assert output is list
# output0, output1 = output[0], output[1]
# pred0 = F.softmax(output0, dim=1)
# pred1 = F.softmax(output1, dim=1)
# pred = (pred0 + pred1).data.max(1)[1]
pred = []
for op in output:
op = F.softmax(op, dim=1)
pred.append(op)
pred = torch.fmod(torch.cat(pred, dim=1).data.max(1)[1], 10)
correct = pred.cpu().eq(target).sum().item()
acc = correct * 1.
return acc
return 0.
def spdz_mul(x, y, workers, mod=field):
if x.get_shape() != y.get_shape():
raise ValueError("Shapes must be identical in order to multiply them")
shape = x.get_shape()
triple = generate_mul_triple_communication(shape, workers)
a, b, c = triple
d = torch.fmod((x - a), mod)
e = torch.fmod((y - b), mod)
delta = torch.fmod(d.child.sum_get(), mod)
epsilon = torch.fmod(e.child.sum_get(), mod)
epsilon_delta = epsilon * delta
delta = delta.broadcast(workers)
epsilon = epsilon.broadcast(workers)
z = torch.fmod((c + torch.fmod((delta * b), mod) + torch.fmod(
(epsilon * a), mod)), mod)
z.child.public_add_(epsilon_delta)
return z
def forward(self, input):
weight1 = self.weight * self.scale + (
self.weight - self.weight * self.scale).detach()
weight = weight1 + (wage_quantizer.Q(weight1, self.wl_weight) -
weight1).detach()
outputOrignal = F.linear(input, weight, self.bias)
output = torch.zeros_like(outputOrignal)
bitWeight = int(self.wl_weight)
bitActivation = int(self.wl_input)
if self.inference == 1:
# retention
weight = wage_quantizer.Retention(weight, self.t, self.v,
self.detect, self.target)
# set parameters for Hardware Inference
onoffratio = self.onoffratio
upper = 1
lower = 1 / onoffratio
output = torch.zeros_like(outputOrignal)
cellRange = 2**self.cellBit # cell precision is 4
# Now consider on/off ratio
dummyP = torch.zeros_like(weight)
dummyP[:, :] = (cellRange - 1) * (upper + lower) / 2
# need to divide to different subArray
numSubArray = int(weight.shape[1] / self.subArray)
if numSubArray == 0:
mask = torch.zeros_like(weight)
mask[:, :] = 1
# quantize input into binary sequence
inputQ = torch.round((2**bitActivation - 1) / 1 * (input - 0) +
0)
outputIN = torch.zeros_like(outputOrignal)
for z in range(bitActivation):
inputB = torch.fmod(inputQ, 2)
inputQ = torch.round((inputQ - inputB) / 2)
# after get the spacial kernel, need to transfer floating weight [-1, 1] to binarized ones
X_decimal = torch.round((2**bitWeight - 1) / 2 *
(weight + 1) + 0) * mask
outputP = torch.zeros_like(outputOrignal)
outputD = torch.zeros_like(outputOrignal)
for k in range(int(bitWeight / self.cellBit)):
remainder = torch.fmod(X_decimal, cellRange) * mask
variation = np.random.normal(
0, self.vari,
list(weight.size())).astype(np.float32)
X_decimal = torch.round(
(X_decimal - remainder) / cellRange) * mask
# Now also consider weight has on/off ratio effects
# Here remainder is the weight mapped to Hardware, so we introduce on/off ratio in this value
# the range of remainder is [0, cellRange-1], we truncate it to [lower, upper]
remainderQ = (upper - lower) * (remainder - 0) + (
cellRange -
1) * lower # weight cannot map to 0, but to Gmin
remainderQ = remainderQ + remainderQ * torch.from_numpy(
variation).cuda()
outputPartial = F.linear(input, remainderQ * mask,
self.bias)
outputDummyPartial = F.linear(input, dummyP * mask,
self.bias)
# Add ADC quanization effects here !!!
outputPartialQ = wage_quantizer.LinearQuantizeOut(
outputPartial, self.ADCprecision)
outputDummyPartialQ = wage_quantizer.LinearQuantizeOut(
outputDummyPartial, self.ADCprecision)
scaler = cellRange**k
outputP = outputP + outputPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
outputD = outputD + outputDummyPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
scalerIN = 2**z
outputIN = outputIN + (outputP - outputD) * scalerIN
output = output + outputIN / (2**bitActivation)
else:
inputQ = torch.round((2**bitActivation - 1) / 1 * (input - 0) +
0)
outputIN = torch.zeros_like(outputOrignal)
for z in range(bitActivation):
inputB = torch.fmod(inputQ, 2)
inputQ = torch.round((inputQ - inputB) / 2)
outputP = torch.zeros_like(outputOrignal)
for s in range(numSubArray):
mask = torch.zeros_like(weight)
mask[:,
(s * self.subArray):(s + 1) * self.subArray] = 1
# after get the spacial kernel, need to transfer floating weight [-1, 1] to binarized ones
X_decimal = torch.round((2**bitWeight - 1) / 2 *
(weight + 1) + 0) * mask
outputSP = torch.zeros_like(outputOrignal)
outputD = torch.zeros_like(outputOrignal)
for k in range(int(bitWeight / self.cellBit)):
remainder = torch.fmod(X_decimal, cellRange) * mask
variation = np.random.normal(
0, self.vari,
list(remainder.size())).astype(np.float32)
X_decimal = torch.round(
(X_decimal - remainder) / cellRange) * mask
# Now also consider weight has on/off ratio effects
# Here remainder is the weight mapped to Hardware, so we introduce on/off ratio in this value
# the range of remainder is [0, cellRange-1], we truncate it to [lower, upper]*(cellRange-1)
remainderQ = (upper - lower) * (remainder - 0) + (
cellRange - 1
) * lower # weight cannot map to 0, but to Gmin
remainderQ = remainderQ + remainderQ * torch.from_numpy(
variation).cuda()
outputPartial = F.linear(input, remainderQ * mask,
self.bias)
outputDummyPartial = F.linear(
input, dummyP * mask, self.bias)
# Add ADC quanization effects here !!!
outputPartialQ = wage_quantizer.LinearQuantizeOut(
outputPartial, self.ADCprecision)
outputDummyPartialQ = wage_quantizer.LinearQuantizeOut(
outputDummyPartial, self.ADCprecision)
scaler = cellRange**k
outputSP = outputSP + outputPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
outputD = outputD + outputDummyPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
outputSP = outputSP - outputD # minus dummy column
outputP = outputP + outputSP
scalerIN = 2**z
outputIN = outputIN + outputP * scalerIN
output = output + outputIN / (2**bitActivation)
output = output / (2**bitWeight)
else:
# original WAGE QCov2d
weight1 = self.weight * self.scale + (
self.weight - self.weight * self.scale).detach()
weight = weight1 + (wage_quantizer.Q(weight1, self.wl_weight) -
weight1).detach()
output = F.linear(input, weight, self.bias)
output = output / self.scale
output = wage_quantizer.WAGEQuantizer_f(output, self.wl_activate,
self.wl_error)
return output
def forward(self, input):
weight1 = self.weight * self.scale + (
self.weight - self.weight * self.scale).detach()
weight = weight1 + (wage_quantizer.Q(weight1, self.wl_weight) -
weight1).detach()
outputOrignal = F.conv2d(input, weight, self.bias, self.stride,
self.padding, self.dilation, self.groups)
bitWeight = int(self.wl_weight)
bitActivation = int(self.wl_input)
if self.inference == 1:
# retention
weight = wage_quantizer.Retention(weight, self.t, self.v,
self.detect, self.target)
# set parameters for Hardware Inference
onoffratio = self.onoffratio
upper = 1
lower = 1 / onoffratio
output = torch.zeros_like(outputOrignal)
del outputOrignal
cellRange = 2**self.cellBit # cell precision is 4
# Now consider on/off ratio
dummyP = torch.zeros_like(weight)
dummyP[:, :, :, :] = (cellRange - 1) * (upper + lower) / 2
for i in range(3):
for j in range(3):
# need to divide to different subArray
numSubArray = int(weight.shape[1] / self.subArray)
# cut into different subArrays
if numSubArray == 0:
mask = torch.zeros_like(weight)
mask[:, :, i, j] = 1
if weight.shape[1] == 3:
# after get the spacial kernel, need to transfer floating weight [-1, 1] to binarized ones
X_decimal = torch.round((2**bitWeight - 1) / 2 *
(weight + 1) + 0) * mask
outputP = torch.zeros_like(output)
outputD = torch.zeros_like(output)
for k in range(int(bitWeight / self.cellBit)):
remainder = torch.fmod(X_decimal,
cellRange) * mask
variation = np.random.normal(
0, self.vari,
list(weight.size())).astype(np.float32)
X_decimal = torch.round(
(X_decimal - remainder) / cellRange) * mask
# Now also consider weight has on/off ratio effects
# Here remainder is the weight mapped to Hardware, so we introduce on/off ratio in this value
# the range of remainder is [0, cellRange-1], we truncate it to [lower, upper]
remainderQ = (upper - lower) * (
remainder - 0
) + (
cellRange - 1
) * lower # weight cannot map to 0, but to Gmin
remainderQ = remainderQ + remainderQ * torch.from_numpy(
variation).cuda()
outputPartial = F.conv2d(
input, remainderQ * mask, self.bias,
self.stride, self.padding, self.dilation,
self.groups)
outputDummyPartial = F.conv2d(
input, dummyP * mask, self.bias,
self.stride, self.padding, self.dilation,
self.groups)
scaler = cellRange**k
outputP = outputP + outputPartial * scaler * 2 / (
1 - 1 / onoffratio)
outputD = outputD + outputDummyPartial * scaler * 2 / (
1 - 1 / onoffratio)
outputP = outputP - outputD
output = output + outputP
else:
# quantize input into binary sequence
inputQ = torch.round((2**bitActivation - 1) / 1 *
(input - 0) + 0)
outputIN = torch.zeros_like(output)
for z in range(bitActivation):
inputB = torch.fmod(inputQ, 2)
inputQ = torch.round((inputQ - inputB) / 2)
outputP = torch.zeros_like(output)
# after get the spacial kernel, need to transfer floating weight [-1, 1] to binarized ones
X_decimal = torch.round(
(2**bitWeight - 1) / 2 *
(weight + 1) + 0) * mask
outputD = torch.zeros_like(output)
for k in range(int(bitWeight / self.cellBit)):
remainder = torch.fmod(
X_decimal, cellRange) * mask
variation = np.random.normal(
0, self.vari,
list(weight.size())).astype(np.float32)
X_decimal = torch.round(
(X_decimal - remainder) /
cellRange) * mask
# Now also consider weight has on/off ratio effects
# Here remainder is the weight mapped to Hardware, so we introduce on/off ratio in this value
# the range of remainder is [0, cellRange-1], we truncate it to [lower, upper]
remainderQ = (upper - lower) * (
remainder - 0
) + (
cellRange - 1
) * lower # weight cannot map to 0, but to Gmin
remainderQ = remainderQ + remainderQ * torch.from_numpy(
variation).cuda()
outputPartial = F.conv2d(
input, remainderQ * mask, self.bias,
self.stride, self.padding,
self.dilation, self.groups)
outputDummyPartial = F.conv2d(
input, dummyP * mask, self.bias,
self.stride, self.padding,
self.dilation, self.groups)
# Add ADC quanization effects here !!!
outputPartialQ = wage_quantizer.LinearQuantizeOut(
outputPartial, self.ADCprecision)
outputDummyPartialQ = wage_quantizer.LinearQuantizeOut(
outputDummyPartial, self.ADCprecision)
scaler = cellRange**k
outputP = outputP + outputPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
outputD = outputD + outputDummyPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
scalerIN = 2**z
outputIN = outputIN + (outputP -
outputD) * scalerIN
output = output + outputIN / (2**bitActivation)
else:
# quantize input into binary sequence
inputQ = torch.round((2**bitActivation - 1) / 1 *
(input - 0) + 0)
outputIN = torch.zeros_like(output)
for z in range(bitActivation):
inputB = torch.fmod(inputQ, 2)
inputQ = torch.round((inputQ - inputB) / 2)
outputP = torch.zeros_like(output)
for s in range(numSubArray):
mask = torch.zeros_like(weight)
mask[:, (s * self.subArray):(s + 1) *
self.subArray, i, j] = 1
# after get the spacial kernel, need to transfer floating weight [-1, 1] to binarized ones
X_decimal = torch.round(
(2**bitWeight - 1) / 2 *
(weight + 1) + 0) * mask
outputSP = torch.zeros_like(output)
outputD = torch.zeros_like(output)
for k in range(int(bitWeight / self.cellBit)):
remainder = torch.fmod(
X_decimal, cellRange) * mask
variation = np.random.normal(
0, self.vari,
list(weight.size())).astype(np.float32)
X_decimal = torch.round(
(X_decimal - remainder) /
cellRange) * mask
# Now also consider weight has on/off ratio effects
# Here remainder is the weight mapped to Hardware, so we introduce on/off ratio in this value
# the range of remainder is [0, cellRange-1], we truncate it to [lower, upper]*(cellRange-1)
remainderQ = (upper - lower) * (
remainder - 0
) + (
cellRange - 1
) * lower # weight cannot map to 0, but to Gmin
remainderQ = remainderQ + remainderQ * torch.from_numpy(
variation).cuda()
outputPartial = F.conv2d(
inputB, remainderQ * mask, self.bias,
self.stride, self.padding,
self.dilation, self.groups)
outputDummyPartial = F.conv2d(
inputB, dummyP * mask, self.bias,
self.stride, self.padding,
self.dilation, self.groups)
# Add ADC quanization effects here !!!
outputPartialQ = wage_quantizer.LinearQuantizeOut(
outputPartial, self.ADCprecision)
outputDummyPartialQ = wage_quantizer.LinearQuantizeOut(
outputDummyPartial, self.ADCprecision)
scaler = cellRange**k
outputSP = outputSP + outputPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
outputD = outputD + outputDummyPartialQ * scaler * 2 / (
1 - 1 / onoffratio)
if (weight.shape[0]
== 256) & (weight.shape[1] == 128):
weightMatrix = (
remainderQ *
mask).cpu().data.numpy()
weight_file_name = './layer_record/weightForLayer3_subarray' + str(
s) + '_weightBitNo_' + str(
k) + ".csv"
cout = weightMatrix.shape[0]
weight_matrix = weightMatrix.reshape(
cout, -1).transpose()
np.savetxt(weight_file_name,
weight_matrix,
delimiter=",",
fmt='%10.5f')
# !!! Important !!! the dummy need to be multiplied by a ratio
outputSP = outputSP - outputD # minus dummy column
outputP = outputP + outputSP
scalerIN = 2**z
outputIN = outputIN + outputP * scalerIN
output = output + outputIN / (2**bitActivation)
output = output / (
2**bitWeight
) # since weight range was convert from [-1, 1] to [-256, 256]
else:
# original WAGE QCov2d
weight1 = self.weight * self.scale + (
self.weight - self.weight * self.scale).detach()
weight = weight1 + (wage_quantizer.Q(weight1, self.wl_weight) -
weight1).detach()
output = F.conv2d(input, weight, self.bias, self.stride,
self.padding, self.dilation, self.groups)
output = output / self.scale
output = wage_quantizer.WAGEQuantizer_f(output, self.wl_activate,
self.wl_error)
return output
def attack_all_images(self, args, arch_name, target_model, surrogate_model,
result_dump_path):
for batch_idx, data_tuple in enumerate(self.dataset_loader):
if args.dataset == "ImageNet":
if target_model.input_size[-1] >= 299:
images, true_labels = data_tuple[1], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[1]
if images.size(-1) != target_model.input_size[-1]:
images = F.interpolate(images,
size=target_model.input_size[-1],
mode='bilinear',
align_corners=True)
images = images.cuda()
true_labels = true_labels.cuda()
if self.targeted:
if self.target_type == 'random':
target_labels = torch.randint(
low=0,
high=CLASS_NUM[args.dataset],
size=true_labels.size()).long().cuda()
invalid_target_index = target_labels.eq(true_labels)
while invalid_target_index.sum().item() > 0:
target_labels[invalid_target_index] = torch.randint(
low=0,
high=CLASS_NUM[args.dataset],
size=target_labels[invalid_target_index].shape
).long().cuda()
invalid_target_index = target_labels.eq(true_labels)
elif args.target_type == 'least_likely':
logits = target_model(images)
target_labels = logits.argmin(dim=1)
elif args.target_type == "increment":
target_labels = torch.fmod(true_labels + 1,
CLASS_NUM[args.dataset])
else:
raise NotImplementedError('Unknown target_type: {}'.format(
args.target_type))
else:
target_labels = None
self.attack_images(batch_idx, images, true_labels, target_labels,
target_model, surrogate_model, args)
self.not_done_all[(self.query_all > args.max_queries).byte()] = 1
self.success_all[(self.query_all > args.max_queries).byte()] = 0
log.info('{} is attacked finished ({} images)'.format(
arch_name, self.total_images))
log.info(' avg correct: {:.4f}'.format(
self.correct_all.mean().item()))
log.info(' avg not_done: {:.4f}'.format(
self.not_done_all.mean().item())) # 有多少图没做完
if self.success_all.sum().item() > 0:
log.info(' avg mean_query: {:.4f}'.format(
self.success_query_all[self.success_all.byte()].mean().item()))
log.info(' avg median_query: {:.4f}'.format(
self.success_query_all[
self.success_all.byte()].median().item()))
log.info(' max query: {}'.format(
self.success_query_all[self.success_all.byte()].max().item()))
if self.not_done_all.sum().item() > 0:
log.info(' avg not_done_prob: {:.4f}'.format(
self.not_done_prob_all[
self.not_done_all.byte()].mean().item()))
log.info('Saving results to {}'.format(result_dump_path))
meta_info_dict = {
"avg_correct":
self.correct_all.mean().item(),
"avg_not_done":
self.not_done_all[self.correct_all.byte()].mean().item(),
"mean_query":
self.success_query_all[self.success_all.byte()].mean().item(),
"median_query":
self.success_query_all[self.success_all.byte()].median().item(),
"max_query":
self.success_query_all[self.success_all.byte()].max().item(),
"correct_all":
self.correct_all.detach().cpu().numpy().astype(np.int32).tolist(),
"not_done_all":
self.not_done_all.detach().cpu().numpy().astype(np.int32).tolist(),
"query_all":
self.query_all.detach().cpu().numpy().astype(np.int32).tolist(),
"not_done_prob":
self.not_done_prob_all[self.not_done_all.byte()].mean().item(),
# "gradient_cos_similarity": self.cos_similarity_all.detach().cpu().numpy().astype(np.float32).tolist(),
"args":
vars(args)
}
with open(result_dump_path, "w") as result_file_obj:
json.dump(meta_info_dict, result_file_obj, sort_keys=True)
log.info("done, write stats info to {}".format(result_dump_path))
def attack_all_images(self, args, model_data_dict, save_dir):
for (arch_name,
target_model), image_label_list in model_data_dict.items():
all_image_list = []
all_logits_list = []
target_model.cuda()
targeted_str = "untargeted" if not args.targeted else "targeted"
save_path_prefix = "{}/dataset_{}@arch_{}@norm_{}@loss_{}@{}".format(
save_dir, args.dataset, arch_name, args.norm, args.loss,
targeted_str)
images_path = "{}@images.npy".format(save_path_prefix)
shape_path = "{}@shape.txt".format(save_path_prefix)
logits_path = "{}@logits.npy".format(save_path_prefix)
log.info("Begin attack {}, the images will be saved to {}".format(
arch_name, images_path))
for batch_idx, (images,
true_labels) in enumerate(image_label_list):
images = images.cuda()
true_labels = true_labels.cuda()
if self.targeted:
if self.target_type == 'random':
target_labels = torch.randint(
low=0,
high=CLASS_NUM[args.dataset],
size=true_labels.size()).long().cuda()
invalid_target_index = target_labels.eq(true_labels)
while invalid_target_index.sum().item() > 0:
target_labels[
invalid_target_index] = torch.randint(
low=0,
high=CLASS_NUM[args.dataset],
size=target_labels[invalid_target_index].
shape).long().cuda()
invalid_target_index = target_labels.eq(
true_labels)
elif args.target_type == 'least_likely':
logits = target_model(images)
target_labels = logits.argmin(dim=1)
elif args.target_type == "increment":
target_labels = torch.fmod(true_labels + 1,
CLASS_NUM[args.dataset])
else:
raise NotImplementedError(
'Unknown target_type: {}'.format(args.target_type))
else:
target_labels = None
loss_type = "cw_loss" if not self.targeted else "xent_loss"
labels = true_labels if not self.targeted else target_labels
if self.norm == "l2":
saved_images, saved_logits = self.square_attack_l2(
target_model,
images.detach().cpu().numpy(),
labels.detach().cpu().numpy(), args.epsilon,
args.max_queries, args.p, loss_type)
elif self.norm == "linf":
saved_images, saved_logits = self.square_attack_linf(
target_model,
images.detach().cpu().numpy(),
labels.detach().cpu().numpy(), args.epsilon,
args.max_queries, args.p, loss_type)
all_image_list.extend(saved_images) # B,T,C,H,W
all_logits_list.extend(saved_logits) # B,T,#classes
all_image_list = np.stack(all_image_list) # B,T,C,H,W
all_logits_list = np.stack(all_logits_list)
store_shape = str(all_image_list.shape)
with open(shape_path, "w") as file_shape:
file_shape.write(store_shape)
file_shape.flush()
fp = np.memmap(images_path,
dtype='float32',
mode='w+',
shape=all_image_list.shape)
fp[:, :, :, :, :] = all_image_list[:, :, :, :, :]
del fp
np.save(logits_path, all_logits_list)
log.info('{} is attacked finished, save to {}'.format(
arch_name, images_path))
target_model.cpu()
def test_fmod(x, y):
c = torch.fmod(torch.add(x, y), 2)
return c
def sample_truncated_normal(num_seeds, truncate=2.0):
""" Truncate the values using fmod. fmod applies mod to floating point """
z = torch.randn((num_seeds, 128)).float().cuda()
if truncate is not False or truncate is not None:
z = torch.fmod(z, truncate)
return z
def recognize_beam_batch(self,
h,
hlens,
lpz,
recog_args,
char_list,
rnnlm=None,
normalize_score=True,
strm_idx=0,
tgt_lang_ids=None):
logging.info('input lengths: ' + str(h.size(1)))
att_idx = min(strm_idx, len(self.att) - 1)
h = mask_by_length(h, hlens, 0.0)
# search params
batch = len(hlens)
beam = recog_args.beam_size
penalty = recog_args.penalty
ctc_weight = recog_args.ctc_weight
att_weight = 1.0 - ctc_weight
n_bb = batch * beam
n_bo = beam * self.odim
n_bbo = n_bb * self.odim
pad_b = to_device(
self,
torch.LongTensor([i * beam
for i in six.moves.range(batch)]).view(-1, 1))
pad_bo = to_device(
self,
torch.LongTensor([i * n_bo
for i in six.moves.range(batch)]).view(-1, 1))
pad_o = to_device(
self,
torch.LongTensor([i * self.odim
for i in six.moves.range(n_bb)]).view(-1, 1))
max_hlen = int(max(hlens))
if recog_args.maxlenratio == 0:
maxlen = max_hlen
else:
maxlen = max(1, int(recog_args.maxlenratio * max_hlen))
minlen = int(recog_args.minlenratio * max_hlen)
logging.info('max output length: ' + str(maxlen))
logging.info('min output length: ' + str(minlen))
# initialization
c_prev = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
z_prev = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
c_list = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
z_list = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
vscores = to_device(self, torch.zeros(batch, beam))
a_prev = None
rnnlm_prev = None
self.att[att_idx].reset() # reset pre-computation of h
if self.replace_sos and recog_args.tgt_lang:
logging.info('<sos> index: ' +
str(char_list.index(recog_args.tgt_lang)))
logging.info('<sos> mark: ' + recog_args.tgt_lang)
yseq = [[char_list.index(recog_args.tgt_lang)]
for _ in six.moves.range(n_bb)]
elif tgt_lang_ids is not None:
# NOTE: used for evaluation during training
yseq = [[tgt_lang_ids[b // recog_args.beam_size]]
for b in six.moves.range(n_bb)]
else:
logging.info('<sos> index: ' + str(self.sos))
logging.info('<sos> mark: ' + char_list[self.sos])
yseq = [[self.sos] for _ in six.moves.range(n_bb)]
accum_odim_ids = [self.sos for _ in six.moves.range(n_bb)]
stop_search = [False for _ in six.moves.range(batch)]
nbest_hyps = [[] for _ in six.moves.range(batch)]
ended_hyps = [[] for _ in range(batch)]
exp_hlens = hlens.repeat(beam).view(beam,
batch).transpose(0,
1).contiguous()
exp_hlens = exp_hlens.view(-1).tolist()
exp_h = h.unsqueeze(1).repeat(1, beam, 1, 1).contiguous()
exp_h = exp_h.view(n_bb, h.size()[1], h.size()[2])
if lpz is not None:
device_id = torch.cuda.device_of(next(self.parameters()).data).idx
ctc_prefix_score = CTCPrefixScoreTH(lpz, 0, self.eos, beam,
exp_hlens, device_id)
ctc_states_prev = ctc_prefix_score.initial_state()
ctc_scores_prev = to_device(self, torch.zeros(batch, n_bo))
for i in six.moves.range(maxlen):
logging.debug('position ' + str(i))
vy = to_device(self, torch.LongTensor(self._get_last_yseq(yseq)))
ey = self.dropout_emb(self.embed(vy))
att_c, att_w = self.att[att_idx](exp_h, exp_hlens,
self.dropout_dec[0](z_prev[0]),
a_prev)
ey = torch.cat((ey, att_c), dim=1)
# attention decoder
z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_prev,
c_prev)
if self.context_residual:
logits = self.output(
torch.cat((self.dropout_dec[-1](z_list[-1]), att_c),
dim=-1))
else:
logits = self.output(self.dropout_dec[-1](z_list[-1]))
local_scores = att_weight * F.log_softmax(logits, dim=1)
# rnnlm
if rnnlm:
rnnlm_state, local_lm_scores = rnnlm.buff_predict(
rnnlm_prev, vy, n_bb)
local_scores = local_scores + recog_args.lm_weight * local_lm_scores
local_scores = local_scores.view(batch, n_bo)
# ctc
if lpz is not None:
ctc_scores, ctc_states = ctc_prefix_score(
yseq, ctc_states_prev, accum_odim_ids)
ctc_scores = ctc_scores.view(batch, n_bo)
local_scores = local_scores + ctc_weight * (ctc_scores -
ctc_scores_prev)
local_scores = local_scores.view(batch, beam, self.odim)
if i == 0:
local_scores[:, 1:, :] = self.logzero
local_best_scores, local_best_odims = torch.topk(
local_scores.view(batch, beam, self.odim), beam, 2)
# local pruning (via xp)
local_scores = np.full((n_bbo, ), self.logzero)
_best_odims = local_best_odims.view(n_bb, beam) + pad_o
_best_odims = _best_odims.view(-1).cpu().numpy()
_best_score = local_best_scores.view(-1).cpu().detach().numpy()
local_scores[_best_odims] = _best_score
local_scores = to_device(
self,
torch.from_numpy(local_scores).float()).view(
batch, beam, self.odim)
# (or indexing)
# local_scores = to_device(self, torch.full((batch, beam, self.odim), self.logzero))
# _best_odims = local_best_odims
# _best_score = local_best_scores
# for si in six.moves.range(batch):
# for bj in six.moves.range(beam):
# for bk in six.moves.range(beam):
# local_scores[si, bj, _best_odims[si, bj, bk]] = _best_score[si, bj, bk]
eos_vscores = local_scores[:, :, self.eos] + vscores
vscores = vscores.view(batch, beam, 1).repeat(1, 1, self.odim)
vscores[:, :, self.eos] = self.logzero
vscores = (vscores + local_scores).view(batch, n_bo)
# global pruning
accum_best_scores, accum_best_ids = torch.topk(vscores, beam, 1)
accum_odim_ids = torch.fmod(
accum_best_ids, self.odim).view(-1).data.cpu().tolist()
accum_padded_odim_ids = (torch.fmod(accum_best_ids, n_bo) +
pad_bo).view(-1).data.cpu().tolist()
accum_padded_beam_ids = (torch.div(accum_best_ids, self.odim) +
pad_b).view(-1).data.cpu().tolist()
y_prev = yseq[:][:]
yseq = self._index_select_list(yseq, accum_padded_beam_ids)
yseq = self._append_ids(yseq, accum_odim_ids)
vscores = accum_best_scores
vidx = to_device(self, torch.LongTensor(accum_padded_beam_ids))
if isinstance(att_w, torch.Tensor):
a_prev = torch.index_select(att_w.view(n_bb, *att_w.shape[1:]),
0, vidx)
elif isinstance(att_w, list):
# handle the case of multi-head attention
a_prev = [
torch.index_select(att_w_one.view(n_bb, -1), 0, vidx)
for att_w_one in att_w
]
else:
# handle the case of location_recurrent when return is a tuple
a_prev_ = torch.index_select(att_w[0].view(n_bb, -1), 0, vidx)
h_prev_ = torch.index_select(att_w[1][0].view(n_bb, -1), 0,
vidx)
c_prev_ = torch.index_select(att_w[1][1].view(n_bb, -1), 0,
vidx)
a_prev = (a_prev_, (h_prev_, c_prev_))
z_prev = [
torch.index_select(z_list[li].view(n_bb, -1), 0, vidx)
for li in range(self.dlayers)
]
c_prev = [
torch.index_select(c_list[li].view(n_bb, -1), 0, vidx)
for li in range(self.dlayers)
]
if rnnlm:
rnnlm_prev = self._index_select_lm_state(rnnlm_state, 0, vidx)
if lpz is not None:
ctc_vidx = to_device(self,
torch.LongTensor(accum_padded_odim_ids))
ctc_scores_prev = torch.index_select(ctc_scores.view(-1), 0,
ctc_vidx)
ctc_scores_prev = ctc_scores_prev.view(-1, 1).repeat(
1, self.odim).view(batch, n_bo)
ctc_states = torch.transpose(ctc_states, 1, 3).contiguous()
ctc_states = ctc_states.view(n_bbo, 2, -1)
ctc_states_prev = torch.index_select(ctc_states, 0,
ctc_vidx).view(
n_bb, 2, -1)
ctc_states_prev = torch.transpose(ctc_states_prev, 1, 2)
# pick ended hyps
if i > minlen:
k = 0
penalty_i = (i + 1) * penalty
thr = accum_best_scores[:, -1]
for samp_i in six.moves.range(batch):
if stop_search[samp_i]:
k = k + beam
continue
for beam_j in six.moves.range(beam):
if eos_vscores[samp_i, beam_j] > thr[samp_i]:
yk = y_prev[k][:]
yk.append(self.eos)
if len(yk) < hlens[samp_i]:
_vscore = eos_vscores[samp_i][
beam_j] + penalty_i
if normalize_score:
_vscore = _vscore / len(yk)
_score = _vscore.data.cpu().numpy()
ended_hyps[samp_i].append({
'yseq': yk,
'vscore': _vscore,
'score': _score
})
k = k + 1
# end detection
stop_search = [
stop_search[samp_i] or end_detect(ended_hyps[samp_i], i)
for samp_i in six.moves.range(batch)
]
stop_search_summary = list(set(stop_search))
if len(stop_search_summary) == 1 and stop_search_summary[0]:
break
torch.cuda.empty_cache()
dummy_hyps = [{
'yseq': [self.sos, self.eos],
'score': np.array([-float('inf')])
}]
ended_hyps = [
ended_hyps[samp_i] if len(ended_hyps[samp_i]) != 0 else dummy_hyps
for samp_i in six.moves.range(batch)
]
nbest_hyps = [
sorted(
ended_hyps[samp_i], key=lambda x: x['score'],
reverse=True)[:min(len(ended_hyps[samp_i]), recog_args.nbest)]
for samp_i in six.moves.range(batch)
]
return nbest_hyps
def test_fmod(self):
x = torch.randn(2, 3, 4)
y = torch.randn(2, 1, 4)
self.assertONNX(lambda x, y: torch.fmod(x, y), (x, y),
opset_version=10)
def truncated_normal(size, epsilon):
values = torch.fmod(torch.randn(size), 2) * epsilon
return values
def pointwise_ops(self):
a = torch.randn(4)
b = torch.randn(4)
t = torch.tensor([-1, -2, 3], dtype=torch.int8)
r = torch.tensor([0, 1, 10, 0], dtype=torch.int8)
t = torch.tensor([-1, -2, 3], dtype=torch.int8)
s = torch.tensor([4, 0, 1, 0], dtype=torch.int8)
f = torch.zeros(3)
g = torch.tensor([-1, 0, 1])
w = torch.tensor([0.3810, 1.2774, -0.2972, -0.3719, 0.4637])
return (
torch.abs(torch.tensor([-1, -2, 3])),
torch.absolute(torch.tensor([-1, -2, 3])),
torch.acos(a),
torch.arccos(a),
torch.acosh(a.uniform_(1.0, 2.0)),
torch.add(a, 20),
torch.add(a, torch.randn(4, 1), alpha=10),
torch.addcdiv(torch.randn(1, 3),
torch.randn(3, 1),
torch.randn(1, 3),
value=0.1),
torch.addcmul(torch.randn(1, 3),
torch.randn(3, 1),
torch.randn(1, 3),
value=0.1),
torch.angle(a),
torch.asin(a),
torch.arcsin(a),
torch.asinh(a),
torch.arcsinh(a),
torch.atan(a),
torch.arctan(a),
torch.atanh(a.uniform_(-1.0, 1.0)),
torch.arctanh(a.uniform_(-1.0, 1.0)),
torch.atan2(a, a),
torch.bitwise_not(t),
torch.bitwise_and(t, torch.tensor([1, 0, 3], dtype=torch.int8)),
torch.bitwise_or(t, torch.tensor([1, 0, 3], dtype=torch.int8)),
torch.bitwise_xor(t, torch.tensor([1, 0, 3], dtype=torch.int8)),
torch.ceil(a),
torch.clamp(a, min=-0.5, max=0.5),
torch.clamp(a, min=0.5),
torch.clamp(a, max=0.5),
torch.clip(a, min=-0.5, max=0.5),
torch.conj(a),
torch.copysign(a, 1),
torch.copysign(a, b),
torch.cos(a),
torch.cosh(a),
torch.deg2rad(
torch.tensor([[180.0, -180.0], [360.0, -360.0], [90.0,
-90.0]])),
torch.div(a, b),
torch.divide(a, b, rounding_mode="trunc"),
torch.divide(a, b, rounding_mode="floor"),
torch.digamma(torch.tensor([1.0, 0.5])),
torch.erf(torch.tensor([0.0, -1.0, 10.0])),
torch.erfc(torch.tensor([0.0, -1.0, 10.0])),
torch.erfinv(torch.tensor([0.0, 0.5, -1.0])),
torch.exp(torch.tensor([0.0, math.log(2.0)])),
torch.exp2(torch.tensor([0.0, math.log(2.0), 3.0, 4.0])),
torch.expm1(torch.tensor([0.0, math.log(2.0)])),
torch.fake_quantize_per_channel_affine(
torch.randn(2, 2, 2),
(torch.randn(2) + 1) * 0.05,
torch.zeros(2),
1,
0,
255,
),
torch.fake_quantize_per_tensor_affine(a, 0.1, 0, 0, 255),
torch.float_power(torch.randint(10, (4, )), 2),
torch.float_power(torch.arange(1, 5), torch.tensor([2, -3, 4,
-5])),
torch.floor(a),
# torch.floor_divide(torch.tensor([4.0, 3.0]), torch.tensor([2.0, 2.0])),
# torch.floor_divide(torch.tensor([4.0, 3.0]), 1.4),
torch.fmod(torch.tensor([-3, -2, -1, 1, 2, 3]), 2),
torch.fmod(torch.tensor([1, 2, 3, 4, 5]), 1.5),
torch.frac(torch.tensor([1.0, 2.5, -3.2])),
torch.randn(4, dtype=torch.cfloat).imag,
torch.ldexp(torch.tensor([1.0]), torch.tensor([1])),
torch.ldexp(torch.tensor([1.0]), torch.tensor([1, 2, 3, 4])),
torch.lerp(torch.arange(1.0, 5.0),
torch.empty(4).fill_(10), 0.5),
torch.lerp(
torch.arange(1.0, 5.0),
torch.empty(4).fill_(10),
torch.full_like(torch.arange(1.0, 5.0), 0.5),
),
torch.lgamma(torch.arange(0.5, 2, 0.5)),
torch.log(torch.arange(5) + 10),
torch.log10(torch.rand(5)),
torch.log1p(torch.randn(5)),
torch.log2(torch.rand(5)),
torch.logaddexp(torch.tensor([-1.0]), torch.tensor([-1, -2, -3])),
torch.logaddexp(torch.tensor([-100.0, -200.0, -300.0]),
torch.tensor([-1, -2, -3])),
torch.logaddexp(torch.tensor([1.0, 2000.0, 30000.0]),
torch.tensor([-1, -2, -3])),
torch.logaddexp2(torch.tensor([-1.0]), torch.tensor([-1, -2, -3])),
torch.logaddexp2(torch.tensor([-100.0, -200.0, -300.0]),
torch.tensor([-1, -2, -3])),
torch.logaddexp2(torch.tensor([1.0, 2000.0, 30000.0]),
torch.tensor([-1, -2, -3])),
torch.logical_and(r, s),
torch.logical_and(r.double(), s.double()),
torch.logical_and(r.double(), s),
torch.logical_and(r, s, out=torch.empty(4, dtype=torch.bool)),
torch.logical_not(torch.tensor([0, 1, -10], dtype=torch.int8)),
torch.logical_not(
torch.tensor([0.0, 1.5, -10.0], dtype=torch.double)),
torch.logical_not(
torch.tensor([0.0, 1.0, -10.0], dtype=torch.double),
out=torch.empty(3, dtype=torch.int16),
),
torch.logical_or(r, s),
torch.logical_or(r.double(), s.double()),
torch.logical_or(r.double(), s),
torch.logical_or(r, s, out=torch.empty(4, dtype=torch.bool)),
torch.logical_xor(r, s),
torch.logical_xor(r.double(), s.double()),
torch.logical_xor(r.double(), s),
torch.logical_xor(r, s, out=torch.empty(4, dtype=torch.bool)),
torch.logit(torch.rand(5), eps=1e-6),
torch.hypot(torch.tensor([4.0]), torch.tensor([3.0, 4.0, 5.0])),
torch.i0(torch.arange(5, dtype=torch.float32)),
torch.igamma(a, b),
torch.igammac(a, b),
torch.mul(torch.randn(3), 100),
torch.multiply(torch.randn(4, 1), torch.randn(1, 4)),
torch.mvlgamma(torch.empty(2, 3).uniform_(1.0, 2.0), 2),
torch.tensor([float("nan"),
float("inf"), -float("inf"), 3.14]),
torch.nan_to_num(w),
torch.nan_to_num(w, nan=2.0),
torch.nan_to_num(w, nan=2.0, posinf=1.0),
torch.neg(torch.randn(5)),
# torch.nextafter(torch.tensor([1, 2]), torch.tensor([2, 1])) == torch.tensor([eps + 1, 2 - eps]),
torch.polygamma(1, torch.tensor([1.0, 0.5])),
torch.polygamma(2, torch.tensor([1.0, 0.5])),
torch.polygamma(3, torch.tensor([1.0, 0.5])),
torch.polygamma(4, torch.tensor([1.0, 0.5])),
torch.pow(a, 2),
torch.pow(torch.arange(1.0, 5.0), torch.arange(1.0, 5.0)),
torch.rad2deg(
torch.tensor([[3.142, -3.142], [6.283, -6.283],
[1.570, -1.570]])),
torch.randn(4, dtype=torch.cfloat).real,
torch.reciprocal(a),
torch.remainder(torch.tensor([-3.0, -2.0]), 2),
torch.remainder(torch.tensor([1, 2, 3, 4, 5]), 1.5),
torch.round(a),
torch.rsqrt(a),
torch.sigmoid(a),
torch.sign(torch.tensor([0.7, -1.2, 0.0, 2.3])),
torch.sgn(a),
torch.signbit(torch.tensor([0.7, -1.2, 0.0, 2.3])),
torch.sin(a),
torch.sinc(a),
torch.sinh(a),
torch.sqrt(a),
torch.square(a),
torch.sub(torch.tensor((1, 2)), torch.tensor((0, 1)), alpha=2),
torch.tan(a),
torch.tanh(a),
torch.trunc(a),
torch.xlogy(f, g),
torch.xlogy(f, g),
torch.xlogy(f, 4),
torch.xlogy(2, g),
)
def attack_dataset(self, args, arch, result_dump_path):
success = 0
queries = []
not_done = []
correct_all = []
total = 0
for batch_idx, data_tuple in enumerate(self.data_loader):
if args.dataset == "ImageNet":
if self.model.input_size[-1] >= 299:
images, true_labels = data_tuple[1], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[1]
if images.size(-1) != self.model.input_size[-1]:
images = F.interpolate(images, size=self.model.input_size[-1], mode='bilinear', align_corners=True)
self.image_height = images.size(2)
self.image_width = images.size(3)
eps = args.epsilon
if args.norm == 'l2':
# epsilon = 1e-3
# eps = np.sqrt(epsilon * model.input_size[-1] * model.input_size[-1] * self.in_channels) # 1.752
learning_rate = 2.0 / np.sqrt(self.image_height * self.image_width * self.in_channels)
else:
learning_rate = 0.005
images = images.cuda()
true_labels = true_labels.cuda()
with torch.no_grad():
logits = self.model(images)
pred = logits.argmax(dim=1)
correct = pred.eq(true_labels).detach().cpu().numpy().astype(np.int32)
correct_all.append(correct)
if correct[0].item() == 0:
queries.append(0)
not_done.append(1)
log.info("The {}-th image is already classified incorrectly.")
continue
if self.targeted:
if self.target_type == 'random':
target_labels = torch.randint(low=0, high=CLASS_NUM[args.dataset],
size=true_labels.size()).long().cuda()
invalid_target_index = target_labels.eq(true_labels)
while invalid_target_index.sum().item() > 0:
target_labels[invalid_target_index] = torch.randint(low=0, high=logits.shape[1],
size=target_labels[invalid_target_index].shape).long().cuda()
invalid_target_index = target_labels.eq(true_labels)
elif args.target_type == 'least_likely':
target_labels = logits.argmin(dim=1)
elif args.target_type == "increment":
target_labels = torch.fmod(true_labels + 1, CLASS_NUM[args.dataset])
else:
raise NotImplementedError('Unknown target_type: {}'.format(args.target_type))
else:
target_labels = None
total += images.size(0)
sigma = args.sigma
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed(0)
adv_images = images.clone().cuda()
assert images.size(0) == 1
logits_real_images = self.model(images)
l = self.xent_loss(logits_real_images, true_labels, target_labels) # 按照元素论文来写的，好奇怪
lr = float(learning_rate)
total_q = 0
ite = 0
while total_q <= args.max_queries:
total_q += 1
# true = torch.squeeze(self.get_grad(self.model, adv_images, true_labels, target_labels)) # C,H,W, # 其实没啥用，只是为了看看估计的准不准
# log.info("Grad norm : {:.3f}".format(torch.sqrt(torch.sum(true * true)).item()))
if ite % 2 == 0 and sigma != args.sigma:
log.info("checking if sigma could be set to be 1e-4")
rand = torch.randn_like(adv_images)
rand = torch.div(rand, torch.clamp(torch.sqrt(torch.mean(torch.mul(rand, rand))), min=1e-12))
logits_1 = self.model(adv_images + args.sigma * rand)
rand_loss = self.xent_loss(logits_1, true_labels, target_labels) # shape = (batch_size,)
total_q += 1
rand = torch.randn_like(adv_images)
rand = torch.div(rand, torch.clamp(torch.sqrt(torch.mean(torch.mul(rand, rand))), min=1e-12))
logits_2 = self.model(adv_images + args.sigma * rand)
rand_loss2= self.xent_loss(logits_2, true_labels, target_labels) # shape = (batch_size,)
total_q += 1
if (rand_loss - l)[0].item() != 0 and (rand_loss2 - l)[0].item() != 0:
sigma = args.sigma
log.info("set sigma back to 1e-4, sigma={:.4f}".format(sigma))
if args.method != "uniform":
prior = torch.squeeze(self.get_grad(self.surrogate_model, adv_images, true_labels, target_labels)) # C,H,W
# 下面求得余弦值
# alpha = torch.sum(true * prior) / torch.clamp(torch.sqrt(torch.sum(true * true) * torch.sum(prior * prior)), min=1e-12) # 这个alpha仅仅用来看看梯度对不对，后续会更新
# log.info("alpha = {:.3}".format(alpha))
prior = prior / torch.clamp(torch.sqrt(torch.mean(torch.mul(prior, prior))),min=1e-12)
if args.method == "biased":
start_iter = 3 # 是只有start_iter=3的时候算一下gradient norm
if ite % 10 == 0 or ite == start_iter:
# Estimate norm of true gradient
s = 10
# pert shape = 10,C,H,W
pert = torch.randn(size=(s, adv_images.size(1), adv_images.size(2), adv_images.size(3)))
for i in range(s):
pert[i] = pert[i] / torch.clamp(torch.sqrt(torch.mean(torch.mul(pert[i], pert[i]))), min=1e-12)
pert = pert.cuda()
# pert = (10,C,H,W), adv_images = (1,C,H,W)
eval_points = adv_images + sigma * pert # broadcast, because tensor shape doesn't match exactly
# eval_points shape = (10,C,H,W) reshape to (10*1, C, H, W)
eval_points = eval_points.view(-1, adv_images.size(1), adv_images.size(2), adv_images.size(3))
target_labels_s = None
if target_labels is not None:
target_labels_s = target_labels.repeat(s)
losses = self.xent_loss(self.model(eval_points), true_labels.repeat(s), target_labels_s) # shape = (10*B,)
total_q += s
norm_square = torch.mean(((losses - l) / sigma) ** 2) # scalar
while True:
logits_for_prior_loss = self.model(adv_images + sigma* prior) # prior may be C,H,W
prior_loss = self.xent_loss(logits_for_prior_loss, true_labels, target_labels) # shape = (batch_size,)
total_q += 1
diff_prior = (prior_loss - l)[0].item()
if diff_prior == 0:
sigma *= 2
log.info("sigma={:.4f}, multiply sigma by 2".format(sigma))
else:
break
est_alpha = diff_prior / sigma / torch.clamp(torch.sqrt(torch.sum(torch.mul(prior,prior)) * norm_square), min=1e-12)
est_alpha = est_alpha.item()
log.info("Estimated alpha = {:.3f}".format(est_alpha))
alpha = est_alpha # alpha描述了替代模型的梯度是否有用，alpha越大λ也越大，λ=1表示相信这个prior
if alpha < 0: # 夹角大于90度，cos变成负数
prior = -prior # v = -v , negative the transfer gradient,
alpha = -alpha
q = args.samples_per_draw
n = self.image_height * self.image_width * self.in_channels
d = 50 * 50 * self.in_channels
gamma = 3.5
A_square = d / n * gamma
return_prior = False
if args.method == 'biased':
if args.dataprior:
best_lambda = A_square * (A_square - alpha ** 2 * (d + 2 * q - 2)) / (
A_square ** 2 + alpha ** 4 * d ** 2 - 2 * A_square * alpha ** 2 * (q + d * q - 1))
else:
best_lambda = (1 - alpha ** 2) * (1 - alpha ** 2 * (n + 2 * q - 2)) / (
alpha ** 4 * n * (n + 2 * q - 2) - 2 * alpha ** 2 * n * q + 1)
log.info("best_lambda = {:.4f}".format(best_lambda))
if best_lambda < 1 and best_lambda > 0:
lmda = best_lambda
else:
if alpha ** 2 * (n + 2 * q - 2) < 1:
lmda = 0
else:
lmda = 1
if abs(alpha) >= 1:
lmda = 1
log.info("lambda = {:.3f}".format(lmda))
if lmda == 1:
return_prior = True # lmda =1, we trust this prior as true gradient
elif args.method == "fixed_biased":
lmda = 0.5
if not return_prior:
if args.dataprior:
upsample = nn.UpsamplingNearest2d(size=(adv_images.size(-2), adv_images.size(-1))) # H, W of original image
pert = torch.randn(size=(q, self.in_channels, 50, 50))
pert = upsample(pert)
else:
pert = torch.randn(size=(q, adv_images.size(-3), adv_images.size(-2), adv_images.size(-1))) # q,C,H,W
pert = pert.cuda()
for i in range(q):
if args.method == 'biased' or args.method == 'fixed_biased':
angle_prior = torch.sum(pert[i] * prior) / \
torch.clamp(torch.sqrt(torch.sum(pert[i] * pert[i]) * torch.sum(prior * prior)),min=1e-12) # C,H,W x B,C,H,W
pert[i] = pert[i] - angle_prior * prior # prior = B,C,H,W so pert[i] = B,C,H,W # FIXME 这里不支持batch模式
pert[i] = pert[i] / torch.clamp(torch.sqrt(torch.mean(torch.mul(pert[i], pert[i]))), min=1e-12)
# pert[i]就是论文算法1的第九行第二项的最右边的一串
pert[i] = np.sqrt(1-lmda) * pert[i] + np.sqrt(lmda) * prior # paper's Algorithm 1: line 9
else:
pert[i] = pert[i] / torch.clamp(torch.sqrt(torch.mean(torch.mul(pert[i], pert[i]))),min=1e-12)
while True:
eval_points = adv_images + sigma * pert # (1,C,H,W) pert=(q,C,H,W)
logits_ = self.model(eval_points)
target_labels_q = None
if target_labels is not None:
target_labels_q = target_labels.repeat(q)
losses = self.xent_loss(logits_, true_labels.repeat(q), target_labels_q) # shape = (q,)
total_q += q
grad = (losses - l).view(-1, 1, 1, 1) * pert # (q,1,1,1) * (q,C,H,W)
grad = torch.mean(grad,dim=0,keepdim=True) # 1,C,H,W
norm_grad = torch.sqrt(torch.mean(torch.mul(grad,grad)))
if norm_grad.item() == 0:
sigma *= 5
log.info("estimated grad == 0, multiply sigma by 5. Now sigma={:.4f}".format(sigma))
else:
break
grad = grad / torch.clamp(torch.sqrt(torch.mean(torch.mul(grad,grad))), min=1e-12)
def print_loss(model, direction):
length = [1e-4, 1e-3]
les = []
for ss in length:
logits_p = model(adv_images + ss * direction)
loss_p = self.xent_loss(logits_p, true_labels, target_labels)
les.append((loss_p - l)[0].item())
log.info("losses: ".format(les))
if args.show_loss:
if args.method == 'biased' or args.method == 'fixed_biased':
show_input = adv_images + lr * prior
logits_show = self.model(show_input)
lprior = self.xent_loss(logits_show, true_labels, target_labels) - l
print_loss(self.model, prior)
show_input_2 = adv_images + lr * grad
logits_show2 = self.model(show_input_2)
lgrad = self.xent_loss(logits_show2, true_labels, target_labels) - l
print_loss(self.model, grad)
log.info(lprior, lgrad)
else:
grad = prior
# log.info("angle = {:.4f}".format(torch.sum(true*grad) /
# torch.clamp(torch.sqrt(torch.sum(true*true) * torch.sum(grad*grad)),min=1e-12)))
if args.norm == "l2":
# Bandits版本
adv_images = adv_images + lr * grad / torch.clamp(torch.sqrt(torch.mean(torch.mul(grad,grad))),min=1e-12)
adv_images = self.l2_proj_step(images, eps, adv_images)
# Below is the original author's L2 norm projection-based update
# adv_images = adv_images + lr * grad / torch.clamp(torch.sqrt(torch.mean(torch.mul(grad,grad))),min=1e-12)
# norm = torch.clamp(torch.norm(adv_images - images),min=1e-12).item()
# factor = min(1, eps / norm)
# adv_images = images + (adv_images - images) * factor
else:
if grad.dim() == 3:
grad = grad.unsqueeze(0)
adv_images = adv_images + lr * torch.sign(grad)
adv_images = torch.min(torch.max(adv_images, images - eps), images + eps)
adv_images = torch.clamp(adv_images, self.clip_min, self.clip_max)
adv_labels = self.get_pred(self.model, adv_images)
logits_ = self.model(adv_images)
l = self.xent_loss(logits_, true_labels, target_labels)
log.info('queries:', total_q, 'loss:', l, 'learning rate:', lr, 'sigma:', sigma, 'prediction:', adv_labels,
'distortion:', torch.max(torch.abs(adv_images - images)).item(), torch.norm((adv_images - images).view(images.size(0),-1)).item())
ite += 1
if (self.targeted and adv_labels[0].item() == target_labels[0].item()) \
or ((not self.targeted) and adv_labels[0].item() != true_labels[0].item()):
log.info("Stop at queries : {}".format(total_q))
success += 1
not_done.append(0)
queries.append(total_q)
break
else:
not_done.append(1)
queries.append(args.max_queries) # 因此不能用np.mean(queries)来计算，平均query次数
log.info('Attack {} success rate: {:.3f} Queries_mean: {:.3f} Queries_median: {:.3f}'.format(arch, success/total,
np.mean(queries), np.median(queries)))
correct_all = np.concatenate(correct_all, axis=0).astype(np.int32)
query_all = np.array(queries).astype(np.int32)
not_done_all = np.array(not_done).astype(np.int32)
success = (1 - not_done_all) * correct_all
success_query = success * query_all
meta_info_dict = {"query_all":query_all.tolist(),"not_done_all":not_done_all.tolist(),
"correct_all":correct_all.tolist(),
"mean_query": np.mean(success_query[np.nonzero(success)[0]]).item(),
"max_query":np.max(success_query[np.nonzero(success)[0]]).item(),
"median_query": np.median(success_query[np.nonzero(success)[0]]).item(),
"avg_not_done": np.mean(not_done_all[np.nonzero(correct_all)[0]].astype(np.float32)).item(),
"args": vars(args)}
with open(result_dump_path, "w") as result_file_obj:
json.dump(meta_info_dict, result_file_obj, sort_keys=True)
log.info("done, write stats info to {}".format(result_dump_path))
def forward_beam(self,z,num_steps,temperature):
if num_steps > self.max_seq_len:
raise ValueError("num_steps ({}) must be less or equal to max_seq_len ({})".format(num_steps,self.max_seq_len))
predictions = []
batch_size = z.size(0)
z = self.batchnorm1(self.activation(self.z2h(z))).repeat(self.beam_width,1)
hx = z # initialize the hidden state
hm = z # initialize the hidden state for memory
memory = self.memory
memory = memory[:num_steps,:]
memory = memory.expand(z.size(0),-1,-1) # copy the memory for each batch position
previous_output = z.new_zeros(size=(batch_size*self.beam_width,),dtype=torch.long)
previous_output[:] = self.SOS_TOKEN # <sos> token
# a table for storing the scores
scores = z.new_zeros(size=(batch_size*self.beam_width,self.output_size))
# an array of numbers for displacement ie. if batch_size is 2 and beam_width is 3 then this is [0,0,0,3,3,3]. This is used later for indexing
beam_displacement = torch.arange(start=0,end=batch_size*self.beam_width,step=self.beam_width,dtype=torch.long,device=z.device).view(-1,1).repeat(1,self.beam_width).view(-1)
for i in range(num_steps):
input = self.activation(self.embedding(previous_output))
step_input = torch.cat([input,z],dim=1)
step_input = self.batchnorm2(step_input)
step_mem = memory[:,i,:] # step_mem is of size [batch,step_input_size]
out,hx,hm = self.memcell(step_input,step_mem,hx=hx,hm=hm)
out = self.activation(out)
out = self.batchnorm3(out)
out = self.h2o(out)
out = self.last_activation(out,temperature)
# compute new scores
next_scores = scores + torch.log(out+1e-8)
# select top-k scores where k is the beam width
score,outputs = next_scores.view(batch_size,-1).topk(self.beam_width,dim=-1)
# flatten the output
outputs = outputs.view(-1)
# get the indices in the original onehot output by finding the module of the vocab size
indices = torch.fmod(outputs,self.output_size)
# find the index in the beam/batch for the onehot output. Add beam displacement to get correct index
beam_indices = torch.div(outputs,self.output_size) + beam_displacement
# check if some elements/words are repeated
res = torch.eq(previous_output,indices).nonzero()
# some elements/words is repeated
retries = 0
while res.shape[0] > 0:
mask = torch.ones(size=(batch_size*self.beam_width,self.output_size),requires_grad=False,device=z.device)
# set the mask to be zero when an option is non selectable
mask[beam_indices[res],indices[res]] = 0
# apply the mask
out = out * mask
# set the score for the repeated elements to be low
next_scores = scores + torch.log(out+1e-8)
# select top-k scores where k is the beam width
score,outputs = next_scores.view(batch_size,-1).topk(self.beam_width,dim=-1)
# flatten the output
outputs = outputs.view(-1)
# get the indices in the original onehot output by finding the module of the vocab size
indices = torch.fmod(outputs,self.output_size)
# find the index in the beam/batch for the onehot output. Add beam displacement to get correct index
beam_indices = torch.div(outputs,self.output_size) + beam_displacement
# check if some elements/words are repeated
res = torch.eq(previous_output,indices).nonzero()
if retries > 10:
break
retries += 1
# copy the score for each selected candidate
scores = score.view(-1,1).repeat(1,self.output_size)
# renormalize the output
out = out/out.sum(-1).view(-1,1).repeat(1,self.output_size)
# append the prediction to output
predictions.append(out[beam_indices,:])
# detach the output such that we don't backpropagate through timesteps
previous_output = indices.detach()
output = torch.stack(predictions).transpose(1,0)
# initialize an output_mask such that we can filter out sentences
output_mask = torch.zeros_like(output)
# set the selected sentences output_mask to 1
output_mask[scores[:,0].view(batch_size,-1).argmax(dim=-1) + beam_displacement.view(batch_size,-1)[:,0]] = 1
# collect the best prediction for each sample in batch
output = (output*output_mask).view(batch_size,self.beam_width,num_steps,self.output_size)
# sum the beam sentences. Since the sentences that is not selected is zero this doesn't change the actual sentences
output = output.sum(1)
return output
def __init__(self, d, k=10, kl=None, bn=True, vq_coef=1, commit_coef=0.5,
in_chns=3, colour_space='rgb', out_chns=None, task=None,
cos_distance=False, use_decor_loss=0, **kwargs):
super(VQ_CVAE, self).__init__()
if out_chns is None:
out_chns = in_chns
self.out_chns = out_chns
if task == 'segmentation':
out_chns = d
self.use_decor_loss = use_decor_loss
if self.use_decor_loss != 0:
self.decor_loss = torch.zeros(1)
self.d = d
self.k = k
if kl is None:
kl = d
self.kl = kl
self.emb = nearest_embed.NearestEmbed(k, kl, cos_distance)
self.colour_space = colour_space
self.task = task
self.hue_loss = HueLoss()
self.encoder = nn.Sequential(
nn.Conv2d(in_chns, d, kernel_size=4, stride=2, padding=1),
nn.BatchNorm2d(d),
nn.ReLU(inplace=True),
nn.Conv2d(d, d, kernel_size=4, stride=2, padding=1),
nn.BatchNorm2d(d),
nn.ReLU(inplace=True),
ResBlock(d, d, bn=True),
nn.BatchNorm2d(d),
ResBlock(d, kl, bn=True),
nn.BatchNorm2d(kl),
)
self.decoder = nn.Sequential(
ResBlock(kl, d),
nn.BatchNorm2d(d),
ResBlock(d, d),
nn.ConvTranspose2d(d, d, kernel_size=4, stride=2, padding=1),
nn.BatchNorm2d(d),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(d, out_chns, kernel_size=4, stride=2, padding=1)
)
if self.task == 'segmentation':
self.fc = nn.Sequential(
nn.BatchNorm2d(d),
nn.ReLU(),
nn.Conv2d(d, self.out_chns, 1)
)
self.vq_coef = vq_coef
self.commit_coef = commit_coef
self.mse = 0
self.vq_loss = torch.zeros(1)
self.commit_loss = 0
for l in self.modules():
if isinstance(l, nn.Linear) or isinstance(l, nn.Conv2d):
l.weight.detach().normal_(0, 0.02)
torch.fmod(l.weight, 0.04)
nn.init.constant_(l.bias, 0)
self.encoder[-1].weight.detach().fill_(1 / 40)
self.emb.weight.detach().normal_(0, 0.02)
torch.fmod(self.emb.weight, 0.04)
def recognize_beam_batch(
self,
h,
hlens,
lpz,
recog_args,
char_list,
rnnlm=None,
normalize_score=True,
strm_idx=0,
lang_ids=None,
):
# to support mutiple encoder asr mode, in single encoder mode,
# convert torch.Tensor to List of torch.Tensor
if self.num_encs == 1:
h = [h]
hlens = [hlens]
lpz = [lpz]
if self.num_encs > 1 and lpz is None:
lpz = [lpz] * self.num_encs
att_idx = min(strm_idx, len(self.att) - 1)
for idx in range(self.num_encs):
logging.info(
"Number of Encoder:{}; enc{}: input lengths: {}.".format(
self.num_encs, idx + 1, h[idx].size(1)))
h[idx] = mask_by_length(h[idx], hlens[idx], 0.0)
# search params
batch = len(hlens[0])
beam = recog_args.beam_size
penalty = recog_args.penalty
ctc_weight = getattr(recog_args, "ctc_weight", 0) # for NMT
att_weight = 1.0 - ctc_weight
ctc_margin = getattr(recog_args, "ctc_window_margin",
0) # use getattr to keep compatibility
# weights-ctc,
# e.g. ctc_loss = w_1*ctc_1_loss + w_2 * ctc_2_loss + w_N * ctc_N_loss
if lpz[0] is not None and self.num_encs > 1:
weights_ctc_dec = recog_args.weights_ctc_dec / np.sum(
recog_args.weights_ctc_dec) # normalize
logging.info("ctc weights (decoding): " +
" ".join([str(x) for x in weights_ctc_dec]))
else:
weights_ctc_dec = [1.0]
n_bb = batch * beam
pad_b = to_device(self, torch.arange(batch) * beam).view(-1, 1)
max_hlen = np.amin([max(hlens[idx]) for idx in range(self.num_encs)])
if recog_args.maxlenratio == 0:
maxlen = max_hlen
else:
maxlen = max(1, int(recog_args.maxlenratio * max_hlen))
minlen = int(recog_args.minlenratio * max_hlen)
logging.info("max output length: " + str(maxlen))
logging.info("min output length: " + str(minlen))
# initialization
c_prev = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
z_prev = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
c_list = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
z_list = [
to_device(self, torch.zeros(n_bb, self.dunits))
for _ in range(self.dlayers)
]
vscores = to_device(self, torch.zeros(batch, beam))
rnnlm_state = None
if self.num_encs == 1:
a_prev = [None]
att_w_list, ctc_scorer, ctc_state = [None], [None], [None]
self.att[att_idx].reset() # reset pre-computation of h
else:
a_prev = [None] * (self.num_encs + 1) # atts + han
att_w_list = [None] * (self.num_encs + 1) # atts + han
att_c_list = [None] * (self.num_encs) # atts
ctc_scorer, ctc_state = [None] * (self.num_encs), [None] * (
self.num_encs)
for idx in range(self.num_encs + 1):
self.att[idx].reset(
) # reset pre-computation of h in atts and han
if self.replace_sos and recog_args.tgt_lang:
logging.info("<sos> index: " +
str(char_list.index(recog_args.tgt_lang)))
logging.info("<sos> mark: " + recog_args.tgt_lang)
yseq = [[char_list.index(recog_args.tgt_lang)]
for _ in six.moves.range(n_bb)]
elif lang_ids is not None:
# NOTE: used for evaluation during training
yseq = [[lang_ids[b // recog_args.beam_size]]
for b in six.moves.range(n_bb)]
else:
logging.info("<sos> index: " + str(self.sos))
logging.info("<sos> mark: " + char_list[self.sos])
yseq = [[self.sos] for _ in six.moves.range(n_bb)]
accum_odim_ids = [self.sos for _ in six.moves.range(n_bb)]
stop_search = [False for _ in six.moves.range(batch)]
nbest_hyps = [[] for _ in six.moves.range(batch)]
ended_hyps = [[] for _ in range(batch)]
exp_hlens = [
hlens[idx].repeat(beam).view(beam,
batch).transpose(0, 1).contiguous()
for idx in range(self.num_encs)
]
exp_hlens = [
exp_hlens[idx].view(-1).tolist() for idx in range(self.num_encs)
]
exp_h = [
h[idx].unsqueeze(1).repeat(1, beam, 1, 1).contiguous()
for idx in range(self.num_encs)
]
exp_h = [
exp_h[idx].view(n_bb, h[idx].size()[1], h[idx].size()[2])
for idx in range(self.num_encs)
]
if lpz[0] is not None:
scoring_ratio = (CTC_SCORING_RATIO
if att_weight > 0.0 and not lpz[0].is_cuda else 0)
ctc_scorer = [
CTCPrefixScoreTH(
lpz[idx],
hlens[idx],
0,
self.eos,
beam,
scoring_ratio,
margin=ctc_margin,
) for idx in range(self.num_encs)
]
for i in six.moves.range(maxlen):
logging.debug("position " + str(i))
vy = to_device(self, torch.LongTensor(self._get_last_yseq(yseq)))
ey = self.dropout_emb(self.embed(vy))
if self.num_encs == 1:
att_c, att_w = self.att[att_idx](exp_h[0], exp_hlens[0],
self.dropout_dec[0](
z_prev[0]), a_prev[0])
att_w_list = [att_w]
else:
for idx in range(self.num_encs):
att_c_list[idx], att_w_list[idx] = self.att[idx](
exp_h[idx],
exp_hlens[idx],
self.dropout_dec[0](z_prev[0]),
a_prev[idx],
)
exp_h_han = torch.stack(att_c_list, dim=1)
att_c, att_w_list[self.num_encs] = self.att[self.num_encs](
exp_h_han,
[self.num_encs] * n_bb,
self.dropout_dec[0](z_prev[0]),
a_prev[self.num_encs],
)
ey = torch.cat((ey, att_c), dim=1)
# attention decoder
z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_prev,
c_prev)
if self.context_residual:
logits = self.output(
torch.cat((self.dropout_dec[-1](z_list[-1]), att_c),
dim=-1))
else:
logits = self.output(self.dropout_dec[-1](z_list[-1]))
local_scores = att_weight * F.log_softmax(logits, dim=1)
# rnnlm
if rnnlm:
rnnlm_state, local_lm_scores = rnnlm.buff_predict(
rnnlm_state, vy, n_bb)
local_scores = local_scores + recog_args.lm_weight * local_lm_scores
# ctc
if ctc_scorer[0]:
for idx in range(self.num_encs):
att_w = att_w_list[idx]
att_w_ = att_w if isinstance(att_w,
torch.Tensor) else att_w[0]
ctc_state[idx], local_ctc_scores = ctc_scorer[idx](
yseq, ctc_state[idx], local_scores, att_w_)
local_scores = (
local_scores +
ctc_weight * weights_ctc_dec[idx] * local_ctc_scores)
local_scores = local_scores.view(batch, beam, self.odim)
if i == 0:
local_scores[:, 1:, :] = self.logzero
# accumulate scores
eos_vscores = local_scores[:, :, self.eos] + vscores
vscores = vscores.view(batch, beam, 1).repeat(1, 1, self.odim)
vscores[:, :, self.eos] = self.logzero
vscores = (vscores + local_scores).view(batch, -1)
# global pruning
accum_best_scores, accum_best_ids = torch.topk(vscores, beam, 1)
accum_odim_ids = (torch.fmod(
accum_best_ids, self.odim).view(-1).data.cpu().tolist())
accum_padded_beam_ids = ((torch.div(accum_best_ids, self.odim) +
pad_b).view(-1).data.cpu().tolist())
y_prev = yseq[:][:]
yseq = self._index_select_list(yseq, accum_padded_beam_ids)
yseq = self._append_ids(yseq, accum_odim_ids)
vscores = accum_best_scores
vidx = to_device(self, torch.LongTensor(accum_padded_beam_ids))
a_prev = []
num_atts = self.num_encs if self.num_encs == 1 else self.num_encs + 1
for idx in range(num_atts):
if isinstance(att_w_list[idx], torch.Tensor):
_a_prev = torch.index_select(
att_w_list[idx].view(n_bb, *att_w_list[idx].shape[1:]),
0, vidx)
elif isinstance(att_w_list[idx], list):
# handle the case of multi-head attention
_a_prev = [
torch.index_select(att_w_one.view(n_bb, -1), 0, vidx)
for att_w_one in att_w_list[idx]
]
else:
# handle the case of location_recurrent when return is a tuple
_a_prev_ = torch.index_select(
att_w_list[idx][0].view(n_bb, -1), 0, vidx)
_h_prev_ = torch.index_select(
att_w_list[idx][1][0].view(n_bb, -1), 0, vidx)
_c_prev_ = torch.index_select(
att_w_list[idx][1][1].view(n_bb, -1), 0, vidx)
_a_prev = (_a_prev_, (_h_prev_, _c_prev_))
a_prev.append(_a_prev)
z_prev = [
torch.index_select(z_list[li].view(n_bb, -1), 0, vidx)
for li in range(self.dlayers)
]
c_prev = [
torch.index_select(c_list[li].view(n_bb, -1), 0, vidx)
for li in range(self.dlayers)
]
# pick ended hyps
if i >= minlen:
k = 0
penalty_i = (i + 1) * penalty
thr = accum_best_scores[:, -1]
for samp_i in six.moves.range(batch):
if stop_search[samp_i]:
k = k + beam
continue
for beam_j in six.moves.range(beam):
_vscore = None
if eos_vscores[samp_i, beam_j] > thr[samp_i]:
yk = y_prev[k][:]
if len(yk) <= min(hlens[idx][samp_i]
for idx in range(self.num_encs)):
_vscore = eos_vscores[samp_i][
beam_j] + penalty_i
elif i == maxlen - 1:
yk = yseq[k][:]
_vscore = vscores[samp_i][beam_j] + penalty_i
if _vscore:
yk.append(self.eos)
if rnnlm:
_vscore += recog_args.lm_weight * rnnlm.final(
rnnlm_state, index=k)
_score = _vscore.data.cpu().numpy()
ended_hyps[samp_i].append({
"yseq": yk,
"vscore": _vscore,
"score": _score
})
k = k + 1
# end detection
stop_search = [
stop_search[samp_i] or end_detect(ended_hyps[samp_i], i)
for samp_i in six.moves.range(batch)
]
stop_search_summary = list(set(stop_search))
if len(stop_search_summary) == 1 and stop_search_summary[0]:
break
if rnnlm:
rnnlm_state = self._index_select_lm_state(rnnlm_state, 0, vidx)
if ctc_scorer[0]:
for idx in range(self.num_encs):
ctc_state[idx] = ctc_scorer[idx].index_select_state(
ctc_state[idx], accum_best_ids)
torch.cuda.empty_cache()
dummy_hyps = [{
"yseq": [self.sos, self.eos],
"score": np.array([-float("inf")])
}]
ended_hyps = [
ended_hyps[samp_i] if len(ended_hyps[samp_i]) != 0 else dummy_hyps
for samp_i in six.moves.range(batch)
]
if normalize_score:
for samp_i in six.moves.range(batch):
for x in ended_hyps[samp_i]:
x["score"] /= len(x["yseq"])
nbest_hyps = [
sorted(
ended_hyps[samp_i], key=lambda x: x["score"],
reverse=True)[:min(len(ended_hyps[samp_i]), recog_args.nbest)]
for samp_i in six.moves.range(batch)
]
return nbest_hyps
def forward_beam(self,z,num_steps,temperature): predictions = [] batch_size = z.size(0) next_input = z.new_zeros(size=(batch_size*self.beam_width,num_steps),dtype=torch.long,requires_grad=False) next_input[:,:] = self.PAD_TOKEN next_input[:,0] = self.SOS_TOKEN # <sos> token z = self.activation(self.z2h(z)).view(batch_size,1,-1).repeat(self.beam_width,num_steps,1) previous_output = z.new_zeros(size=(batch_size*self.beam_width,),dtype=torch.long) previous_output[:] = self.SOS_TOKEN # <sos> token # a table for storing the scores scores = z.new_zeros(size=(batch_size*self.beam_width,self.output_size)) # an array of numbers for displacement ie. if batch_size is 2 and beam_width is 3 then this is [0,0,0,3,3,3]. This is used later for indexing beam_displacement = torch.arange(start=0,end=batch_size*self.beam_width,step=self.beam_width,dtype=torch.long,device=z.device).view(-1,1).repeat(1,self.beam_width).view(-1) for i in range(num_steps): input = next_input.detach() step_input = self.embedding(input) step_input = self.pos_embedding(step_input) step_input = torch.cat([step_input,z],dim=2) # step_input is of size [batch,seq_len,step_input_size] step_input = self.activation(self.s2h(step_input)) non_pad_mask = get_non_pad_mask(input,self.PAD_TOKEN) slf_attn_mask_subseq = get_subsequent_mask(input) slf_attn_mask_keypad = get_attn_key_pad_mask(input,self.PAD_TOKEN) attn_mask = (slf_attn_mask_keypad + slf_attn_mask_subseq).gt(0) out = self.transformer(step_input,non_pad_mask=non_pad_mask,attn_mask=attn_mask) out = out[:,i,:] out = self.activation(out) out = self.h2o(out) out = self.last_activation(out,temperature) # compute new scores next_scores = scores + torch.log(out+1e-8) # select top-k scores where k is the beam width score,outputs = next_scores.view(batch_size,-1).topk(self.beam_width,dim=-1) # flatten the output outputs = outputs.view(-1) # get the indices in the original onehot output by finding the module of the vocab size indices = torch.fmod(outputs,self.output_size) # find the index in the beam/batch for the onehot output. Add beam displacement to get correct index beam_indices = torch.div(outputs,self.output_size) + beam_displacement # check if some elements/words are repeated res = torch.eq(previous_output,indices).nonzero() # some elements/words is repeated retries = 0 while res.shape[0] > 0: mask = torch.ones(size=(batch_size*self.beam_width,self.output_size),requires_grad=False,device=z.device) # set the mask to be zero when an option is non selectable mask[beam_indices[res],indices[res]] = 0 # apply the mask out = out * mask # set the score for the repeated elements to be low next_scores = scores + torch.log(out+1e-8) # select top-k scores where k is the beam width score,outputs = next_scores.view(batch_size,-1).topk(self.beam_width,dim=-1) # flatten the output outputs = outputs.view(-1) # get the indices in the original onehot output by finding the module of the vocab size indices = torch.fmod(outputs,self.output_size) # find the index in the beam/batch for the onehot output. Add beam displacement to get correct index beam_indices = torch.div(outputs,self.output_size) + beam_displacement # check if some elements/words are repeated res = torch.eq(previous_output,indices).nonzero() if retries > 10: break retries += 1 # copy the score for each selected candidate scores = score.view(-1,1).repeat(1,self.output_size) # renormalize the output out = out/out.sum(-1).view(-1,1).repeat(1,self.output_size) # append the prediction to output predictions.append(out[beam_indices,:]) # detach the output such that we don't backpropagate through timesteps previous_output = indices.detach() next_input = torch.cat([input[:,:i+1],previous_output.view(-1,1),input[:,i+2:]],dim=1) output = torch.stack(predictions).transpose(1,0) # initialize an output_mask such that we can filter out sentences output_mask = torch.zeros_like(output) # set the selected sentences output_mask to 1 output_mask[scores[:,0].view(batch_size,-1).argmax(dim=-1) + beam_displacement.view(batch_size,-1)[:,0]] = 1 # collect the best prediction for each sample in batch output = (output*output_mask).view(batch_size,self.beam_width,num_steps,self.output_size) # sum the beam sentences. Since the sentences that is not selected is zero this doesn't change the actual sentences output = output.sum(1) return output
def batch_fmod(data, mask, dims, other_):
other = int(other_)
data = torch.fmod(data, other)
return data, mask, dims
def __init__(self, block_type, num_blocks, latent_dim=512,
in_channels=3, out_channels=None, num_kernels=64):
super(WaveNet, self).__init__()
self.in_channels = in_channels
if out_channels is None:
out_channels = in_channels
self.out_channels = out_channels
self.current_planes = num_kernels
self.resolution = 128
self.loss = 0
self.recons_loss = 0
self.kld_loss = 0
# prior to the systematic layers
self.preprocess = self._preprocess_layer()
# start of the systematic convolutional layers
self.layer1c = self._make_layer(
block_type, num_kernels, 1 * num_blocks[0], stride=2
)
self.layer2c = self._make_layer(
block_type, num_kernels * 2, num_blocks[1], stride=2
)
self.layer3c = self._make_layer(
block_type, num_kernels * 4, num_blocks[2], stride=2
)
self.layer4c = self._make_layer(
block_type, num_kernels * 8, num_blocks[3], stride=2
)
self.encoder = nn.Sequential(
self.layer1c, self.layer2c, self.layer3c, self.layer4c
)
latent_spatial = int(self.resolution / 16)
latent_in_size = self.current_planes * (latent_spatial ** 2)
# the latent space
# self.fc_mu = nn.Linear(latent_in_size, latent_dim)
# self.fc_var = nn.Linear(latent_in_size, latent_dim)
self.emb = NearestEmbed(latent_dim, self.current_planes)
self.vq_coef = 1
self.commit_coef = 0.5
self.emb.weight.detach().normal_(0, 0.02)
torch.fmod(self.emb.weight, 0.04)
self.mse = 0
self.vq_loss = torch.zeros(1)
self.commit_loss = 0
transpose_block_type = BottleneckBlockTranspose
self.latent_kernels = int(latent_dim / 64)
# self.current_planes = self.latent_kernels
# start of the systematic transpose layers
self.layer1t = self._make_layer_transpose(
transpose_block_type, num_kernels * 8, num_blocks[3], stride=2
)
self.layer2t = self._make_layer_transpose(
transpose_block_type, num_kernels * 4, num_blocks[2], stride=2
)
self.layer3t = self._make_layer_transpose(
transpose_block_type, num_kernels * 2, num_blocks[1], stride=2
)
self.layer4t = self._make_layer_transpose(
transpose_block_type, num_kernels * 1, num_blocks[0], stride=2
)
self.decoder = nn.Sequential(
self.layer1t, self.layer2t, self.layer3t, self.layer4t
)
# posterior to systematic layers
self.postprocess = self._postprocess_layer()
# initialisation
for m in self.modules():
if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
m.weight.detach().normal_(0, 0.02)
torch.fmod(m.weight, 0.04)
nn.init.constant_(m.bias, 0)
def __init__(self, d, k=10, kl=None, bn=True, vq_coef=1, commit_coef=0.5,
in_chns=3, colour_space='rgb', out_chns=None, task=None,
cos_distance=False, use_decor_loss=0, backbone=None, **kwargs):
super(Backbone_VQ_VAE, self).__init__()
self.backbone_encoder = pretrained_features.ResNetIntermediate(
**backbone)
if out_chns is None:
out_chns = in_chns
self.out_chns = out_chns
if task == 'segmentation':
out_chns = d
self.use_decor_loss = use_decor_loss
if self.use_decor_loss != 0:
self.decor_loss = torch.zeros(1)
self.d = d
self.k = k
if kl is None:
kl = d
self.kl = kl
self.emb = nearest_embed.NearestEmbed(k, kl, cos_distance)
self.colour_space = colour_space
self.task = task
self.hue_loss = HueLoss()
self.encoder = nn.Sequential(
self.backbone_encoder,
ResBlock(self.backbone_encoder.get_num_kernels(), kl, bn=True),
nn.BatchNorm2d(kl),
)
conv_transposes = []
num_conv_transpose = int(self.backbone_encoder.spatial_ratio / 2)
for i in range(int(np.log2(num_conv_transpose))):
conv_transposes.append(
nn.ConvTranspose2d(d, d, kernel_size=4, stride=2, padding=1)
)
conv_transposes.append(nn.BatchNorm2d(d))
conv_transposes.append(nn.ReLU(inplace=True))
self.decoder = nn.Sequential(
ResBlock(kl, d),
nn.BatchNorm2d(d),
ResBlock(d, d),
*conv_transposes,
nn.ConvTranspose2d(d, out_chns, kernel_size=4, stride=2, padding=1)
)
if self.task == 'segmentation':
self.fc = nn.Sequential(
nn.BatchNorm2d(d),
nn.ReLU(),
nn.Conv2d(d, self.out_chns, 1)
)
self.vq_coef = vq_coef
self.commit_coef = commit_coef
self.mse = 0
self.vq_loss = torch.zeros(1)
self.commit_loss = 0
for l in self.modules():
if (
isinstance(l, pretrained_features.ResNetIntermediate) or
l in self.backbone_encoder.modules()
):
continue
if isinstance(l, nn.Linear) or isinstance(l, nn.Conv2d):
l.weight.detach().normal_(0, 0.02)
torch.fmod(l.weight, 0.04)
nn.init.constant_(l.bias, 0)
self.encoder[-1].weight.detach().fill_(1 / 40)
self.emb.weight.detach().normal_(0, 0.02)
torch.fmod(self.emb.weight, 0.04)
def attack_all_images(self, args, arch_name, target_model,
result_dump_path):
for batch_idx, data_tuple in enumerate(self.dataset_loader):
if args.dataset == "ImageNet":
if target_model.input_size[-1] >= 299:
images, true_labels = data_tuple[1], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[1]
if images.size(-1) != target_model.input_size[-1]:
images = F.interpolate(images,
size=target_model.input_size[-1],
mode='bilinear',
align_corners=True)
images = images.cuda()
true_labels = true_labels.cuda()
selected = torch.arange(
batch_idx * args.batch_size,
min((batch_idx + 1) * args.batch_size, self.total_images))
if self.targeted:
if self.target_type == 'random':
target_labels = torch.randint(
low=0,
high=CLASS_NUM[args.dataset],
size=true_labels.size()).long().cuda()
invalid_target_index = target_labels.eq(true_labels)
while invalid_target_index.sum().item() > 0:
target_labels[invalid_target_index] = torch.randint(
low=0,
high=CLASS_NUM[args.dataset],
size=target_labels[invalid_target_index].shape
).long().cuda()
invalid_target_index = target_labels.eq(true_labels)
elif args.target_type == 'least_likely':
logits = target_model(images)
target_labels = logits.argmin(dim=1)
elif args.target_type == "increment":
target_labels = torch.fmod(true_labels + 1,
CLASS_NUM[args.dataset])
else:
raise NotImplementedError('Unknown target_type: {}'.format(
args.target_type))
else:
target_labels = None
with torch.no_grad():
logit = target_model(images)
pred = logit.argmax(dim=1)
correct = pred.eq(true_labels).float()
# correct_np = correct.detach().cpu().numpy()
# correct_indexes = np.nonzero(correct_np)[0]
loss_type = "cw_loss" if not self.targeted else "xent_loss"
labels = true_labels if not self.targeted else target_labels
if self.norm == "l2":
query, query_success_times, query_fail_times, adv_images = self.square_attack_l2(
target_model,
images.detach().cpu().numpy(),
labels.detach().cpu().numpy(), args.epsilon,
args.max_queries, args.p, loss_type)
elif self.norm == "linf":
query, query_success_times, query_fail_times, adv_images = self.square_attack_linf(
target_model,
images.detach().cpu().numpy(),
labels.detach().cpu().numpy(), args.epsilon,
args.max_queries, args.p, loss_type)
query = torch.from_numpy(query).float().cuda()
query_success_times = torch.from_numpy(query_success_times)
query_fail_times = torch.from_numpy(query_fail_times)
adv_images = torch.from_numpy(adv_images).float().cuda()
with torch.no_grad():
adv_logit = target_model(adv_images)
adv_prob = F.softmax(adv_logit, dim=1)
adv_pred = adv_logit.argmax(dim=1)
if args.targeted:
not_done = (1 - adv_pred.eq(target_labels).float()).float(
) # not_done初始化为 correct, shape = (batch_size,)
else:
not_done = adv_pred.eq(true_labels).float()
success = (1 - not_done) * correct
success_query = success * query
not_done_prob = adv_prob[torch.arange(args.batch_size),
true_labels] * not_done
for key in [
'query', 'query_success_times', 'query_fail_times',
'correct', 'not_done', 'success', 'success_query',
'not_done_prob'
]:
value_all = getattr(self, key + "_all")
value = eval(key)
value_all[selected] = value.detach().float().cpu(
) # 由于value_all是全部图片都放在一个数组里，当前batch选择出来
log.info(
"{}-th batch (size={}), current batch success rate:{:.3f}".
format(batch_idx, adv_images.size(0),
success.mean().item()))
log.info('{} is attacked finished ({} images)'.format(
arch_name, self.total_images))
log.info(' avg correct: {:.4f}'.format(
self.correct_all.mean().item()))
log.info(' avg not_done: {:.4f}'.format(
self.not_done_all.mean().item())) # 有多少图没做完
if self.success_all.sum().item() > 0:
log.info(' avg mean_query: {:.4f}'.format(
self.success_query_all[self.success_all.byte()].mean().item()))
log.info(' avg median_query: {:.4f}'.format(
self.success_query_all[
self.success_all.byte()].median().item()))
log.info(' max query: {}'.format(
self.success_query_all[self.success_all.byte()].max().item()))
if self.not_done_all.sum().item() > 0:
log.info(' avg not_done_prob: {:.4f}'.format(
self.not_done_prob_all[
self.not_done_all.byte()].mean().item()))
log.info('Saving results to {}'.format(result_dump_path))
meta_info_dict = {
"avg_correct":
self.correct_all.mean().item(),
"avg_not_done":
self.not_done_all[self.correct_all.byte()].mean().item(),
"mean_query_success_times":
self.query_success_times_all[
self.success_all.byte()].mean().item(),
"mean_query_fail_times":
self.query_fail_times_all[self.success_all.byte()].mean().item(),
"mean_query":
self.success_query_all[self.success_all.byte()].mean().item(),
"median_query":
self.success_query_all[self.success_all.byte()].median().item(),
"max_query":
self.success_query_all[self.success_all.byte()].max().item(),
"correct_all":
self.correct_all.detach().cpu().numpy().astype(np.int32).tolist(),
"not_done_all":
self.not_done_all.detach().cpu().numpy().astype(np.int32).tolist(),
"query_all":
self.query_all.detach().cpu().numpy().astype(np.int32).tolist(),
"query_success_times_all":
self.query_success_times_all.detach().cpu().numpy().astype(
np.int32).tolist(),
"query_fail_times_all":
self.query_fail_times_all.detach().cpu().numpy().astype(
np.int32).tolist(),
"not_done_prob":
self.not_done_prob_all[self.not_done_all.byte()].mean().item(),
"args":
vars(args)
}
with open(result_dump_path, "w") as result_file_obj:
json.dump(meta_info_dict, result_file_obj, sort_keys=True)
log.info("done, write stats info to {}".format(result_dump_path))
