def forward(self, inputs, targets, attack=False):
if not attack:
return self.basic_net(inputs)
x = inputs.detach()
if self.rand:
# x = x + torch.zeros_like(x).uniform_(-self.epsilon, self.epsilon)
noise = torch.zeros_like(x).normal_(0, self.step_size).view(
x.size(0), -1)
x = x + torch.renorm(noise, 2, 0, self.epsilon).view(x.size())
for _ in range(self.num_steps):
x.requires_grad_()
with torch.enable_grad():
logits = self.basic_net(x)
loss = F.cross_entropy(logits, targets, reduction='sum')
grad = torch.autograd.grad(loss, x)[0].detach()
grad_norm = grad.view(x.size(0), -1).norm(2, 1)
delta = self.step_size * grad / grad_norm.view(x.size(0), 1, 1, 1)
x = x.detach() + delta
# x = torch.min(torch.max(x, inputs - self.epsilon),
# inputs + self.epsilon)
delta = torch.renorm((x - inputs.detach()).view(x.size(0), -1), 2,
0, self.epsilon).view(x.size())
x = torch.clamp(inputs.detach() + delta, 0, 1)
return self.basic_net(x)
def normalizeEmbedding(self):
self.entityEmbedding.weight.data.copy_(torch.renorm(input=self.entityEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1))
self.relationEmbedding.weight.data.copy_(torch.renorm(input=self.relationEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1))
def apply(self, model):
last_weight = None
for name, module in list(model.named_children()):
if hasattr(module, 'weight') and (
not module.__class__.__name__.startswith('BatchNorm')):
module.weight.data = th.renorm(module.weight.data,2,0,maxnorm=2)
last_weight = module.weight
if last_weight is not None:
last_weight.data = th.renorm(last_weight.data,2,0,maxnorm=0.5)
def normalizeEmbedding(self):
'''
normalize the embedding and change the embedding.
:return:
'''
self.entEmbedding.weight.data.copy_(
torch.renorm(input=self.entMapEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1))
self.relEmbedding.weight.data.copy_(
torch.renorm(input=self.relEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1))
def forward(self, x):
if self.doWeightNorm:
self.weight.data = torch.renorm(self.weight.data,
p=2,
dim=0,
maxnorm=self.max_norm)
return super(LinearWithConstraint, self).forward(x)
def forward(self, x_t, f_t, h_m_t, c_m_t, h_w_t, c_w_t, last_goal, real_goal, t, temperature):
'''
args:
x_t([]):the current word
f_t([]):the feature vector come from the discriminator
real_goal([]):
temperature([]):as said in paper,this parameter is to control the generation entropy.
output:
last_goal_temp([]):输出作为last_goal
'''
sub_goal, h_m_tp1, c_m_tp1 = self.manager(f_t, h_m_t, c_m_t) #sub_goal = [batch_size,goal_out_size]
output, h_w_tp1, c_w_tp1 = self.worker(x_t, h_w_t, c_w_t) # output = [batch_size, vocab_size, goal_size] (as the O_t in the paper)
last_goal_temp = last_goal + sub_goal # 就单用上一个last_goal加上用f_t产生的sub_goal？？
w_t = torch.matmul( #矩阵乘法 []
real_goal, self.worker.goal_change # worker.goal_change[goal_out_size, goal_size]
)
# 主要问题就在上面，last_goal_temp后续没作用，意味着Manager这次的信息根本没到后续的w_t中，也就没到logits中
w_t = torch.renorm(w_t, 2, 0, 1.0) # []
w_t = torch.unsqueeze(w_t, -1)
logits = torch.squeeze(torch.matmul(output, w_t)) #logits for words
probs = F.softmax(temperature * logits, dim=1)
x_tp1 = Categorical(probs).sample()
return x_tp1, h_m_tp1, c_m_tp1, h_w_tp1, c_w_tp1,\
last_goal_temp, real_goal, sub_goal, probs, t + 1
def batchwise_lp_project(x, lp, lp_bound, dim=0):
""" Projects x (a N-by-(...) TENSOR) to be a N-by-(...) TENSOR into the
provided lp ball
ARGS:
x : Tensor (N-by-(...)) - arbitrary style
lp : 'inf' or int - which style of lp we use
lp_bound : float - size of lp ball we project into
dim : int - if not 0 is the dimension we keep and project onto
RETURNS:
None
"""
assert isinstance(lp, int) or lp == 'inf'
if lp == 'inf':
return torch.clamp(x, -lp_bound, lp_bound)
needs_squeeze = False
if len(x.shape) == 1:
x = x.unsqueeze(1)
needs_squeeze = True
output = torch.renorm(x, lp, dim, lp_bound)
if needs_squeeze:
return output.squeeze()
return output
def random_from_lp_ball(tensorlike, lp, lp_bound, dim=0):
""" Returns a new object of the same type/shape as tensorlike that is
randomly samples from the unit ball.
NOTE THIS IS NOT A UNIFORM SAMPLING METHOD!
(that's hard to implement, https://mathoverflow.net/a/9192/123034)
ARGS:
tensorlike : Tensor - reference object for which we generate
a new object of same shape/memory_location
lp : int or 'inf' - which style of lp we use
lp_bound : float - size of the L
dim : int - which dimension is the 'keep dimension'
RETURNS:
new tensorlike where each slice across dim is uniform across the
lp ball of size lp_bound
"""
assert isinstance(lp, int) or lp == 'inf'
rand_direction = torch.rand(tensorlike.shape).type(tensorlike.type())
if lp == 'inf':
return rand_direction * (2 * lp_bound) - lp_bound
else:
rand_direction = rand_direction - 0.5 # allow for sign swapping
# first magnify such that each element is above the ball
min_norm = torch.min(batchwise_norm(rand_direction.abs(), lp, dim=dim))
rand_direction = rand_direction / (min_norm + 1e-6)
rand_magnitudes = torch.rand(tensorlike.shape[dim]).type(
tensorlike.type())
rand_magnitudes = rand_magnitudes.unsqueeze(1)
rand_magnitudes = rand_magnitudes.expand(*rand_direction.shape)
return torch.renorm(rand_direction, lp, dim,
lp_bound) * rand_magnitudes
def attack(image, label, attack_name):
fmodel = foolbox.models.PyTorchModel(
model.eval().cuda(), bounds=(0, 1),
num_classes=10) #, preprocessing=(mean, std)
criterion1 = Misclassification()
distance = Linfinity #MeanAbsoluteDistance
attacker = attackers[attack_name](fmodel,
criterion=criterion1,
distance=distance)
image = image.cpu().numpy()
label = label.cpu().numpy()
adversarials = image.copy()
for i in tqdm.tqdm(range(args.batch_size), ncols=80):
adv = attacker(
image[i], label[i]
) # , unpack=True, steps=self.max_iter, subsample=self.subsample)
if adv is not None:
adv = torch.renorm(
torch.from_numpy(adv - image[i]), p=2, dim=0,
maxnorm=1).numpy() + image[i]
adversarials[i] = adv
adversarials = torch.from_numpy(adversarials).to(DEVICE)
return adversarials
def forward(self, x):
x = self.forward_features(x)
if self.drop_rate > 0.:
x = F.dropout(x, p=self.drop_rate, training=self.training)
x = self.fc(x)
if self.output_norm:
x = torch.renorm(x, self.output_norm, 0, 1.0)
return x
def on_batch_end(self, net, training, *args, **kwargs):
if training:
model = net.module_
last_weight = None
for name, module in list(model.named_children()):
if hasattr(module, "weight") and (
not module.__class__.__name__.startswith("BatchNorm")):
module.weight.data = torch.renorm(module.weight.data,
2,
0,
maxnorm=2)
last_weight = module.weight
if last_weight is not None:
last_weight.data = torch.renorm(last_weight.data,
2,
0,
maxnorm=0.5)
def _clip(inputs: Tensor, outputs: Tensor) -> Tensor:
diff = outputs - inputs
if norm == "Linf":
return inputs + torch.clamp(diff, -radius, radius)
elif norm == "L2":
return inputs + torch.renorm(diff, 2, 0, radius)
else:
raise AssertionError("Norm constraint must be L2 or Linf.")
def set_dim0(x):
x = torch.renorm(x, p=2, dim=0,
maxnorm=1e2) # otherwise leaves will explode
# NOTE: the paper does not mention the square part of the equation but if
# you try to derive it you get a square term in the equation
dim0 = torch.sqrt(1 + (x[:, 1:]**2).sum(dim=1))
x[:, 0] = dim0
return x
def __call__(self, module):
if self.lagrangian:
w = module.weight
norm_diff = th.norm(w, 2, self.axis).sub(self.value)
return self.scale * th.sum(norm_diff.gt(0).float().mul(norm_diff))
else:
module.weight.data = th.renorm(module.weight.data, 2, self.axis,
self.value)
def load_word2vec_format(filename, word_idx, binary=False, normalize=False,
encoding='utf8', unicode_errors='ignore'):
"""
refer to gensim
load Word Embeddings
If you trained the C model using non-utf8 encoding for words, specify that
encoding in `encoding`.
:param filename :
:param word_idx :
:param binary : a boolean indicating whether the data is in binary word2vec format.
:param normalize:
:param encoding :
:param unicode_errors: errors can be 'strict', 'replace' or 'ignore' and defaults to 'strict'.
"""
vocab = set()
print("loading word embedding from %s" % filename)
with open(filename, 'rb') as fin:
# header = to_unicode(fin.readline(), encoding=encoding)
# vocab_size, vector_size = map(int, header.split()) # throws for invalid file format
vocab_size = 1917494
vector_size = 300
word_matrix = torch.zeros(len(word_idx), vector_size)
def add_word(_word, _weights):
if _word not in word_idx:
return
vocab.add(_word)
word_matrix[word_idx[_word]] = _weights
if binary:
binary_len = np.dtype(np.float32).itemsize * vector_size
for _ in range(vocab_size):
# mixed text and binary: read text first, then binary
word = []
while True:
ch = fin.read(1)
if ch == b' ':
break
if ch != b'\n': # ignore newlines in front of words (some binary files have)
word.append(ch)
word = to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)
weights = torch.from_numpy(np.fromstring(fin.read(binary_len), dtype=REAL))
add_word(word, weights)
else:
for line_no, line in enumerate(fin):
parts = to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(" ")
if len(parts) != vector_size + 1:
raise ValueError("invalid vector on line %s (is this really the text format?)" % line_no)
word, weights = parts[0], list(map(float, parts[1:]))
weights = torch.Tensor(weights)
add_word(word, weights)
if word_idx is not None:
assert (len(word_idx), vector_size) == word_matrix.size()
if normalize:
# each row normalize to 1
word_matrix = torch.renorm(word_matrix, 2, 0, 1)
print("loaded %d words pre-trained from %s with %d" % (len(vocab), filename, vector_size))
return word_matrix, vector_size, vocab
def normalizeEmbedding(self):
self.entityEmbedding.weight.data.copy_(
torch.renorm(input=self.entityEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1.0))
self.relationEmbedding.weight.data.copy_(
torch.renorm(input=self.relationEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1.0))
self.entityCovar.weight.data.copy_(
torch.clamp(input=self.entityCovar.weight.detach().cpu(),
min=self.vmin,
max=self.vmax))
self.relationCovar.weight.data.copy_(
torch.clamp(input=self.relationCovar.weight.detach().cpu(),
min=self.vmin,
max=self.vmax))
def forward(self, x):
x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
x = self.pool(x)
x = F.relu(self.conv3(x))
x = F.relu(self.conv4(x))
x = self.pool2(x)
x = F.relu(self.conv5(x))
x = F.relu(self.conv6(x))
x = self.pool3(x)
x = self.pool4(x)
x = x.view(-1, self.cnn_output)
for ix, fc in enumerate(self.fcs):
x = fc(x)
if self.neural_noise is not None and ix == 0 and self.training:
mean, std = self.neural_noise
noise = torch.zeros_like(x, device=dev)
noise = noise.log_normal_(mean=mean, std=std)
x = x * noise
x = F.relu(x)
if self.excite and ix == 1 and self.n_new and self.training:
idx = self.idx_control if self.control else self.idx
excite_mask = torch.ones_like(x)
excite_mask[:, idx] = self.excite
excite_mask.to(dev)
x = x * excite_mask
if self.dropout:
x = F.dropout(x, p=self.dropout, training=self.training)
x = torch.renorm(x, 1, 1, 3) # max norm
# for ablation experiments
if self.ablate:
activation_size = x.size()[1]
ablate_size = int(self.ablation_prop * activation_size)
if self.ablation_mode == "random":
indices = np.random.choice(
range(activation_size),
size=size,
replace=False,
)
if self.ablation_mode == "targetted":
importance = torch.norm(fc.weight.detach().clone(),
p=1,
dim=1)
indices = torch.argsort(importance,
descending=True)[:ablate_size]
x[:, indices] = 0
x = self.fc3(x)
return x
def proj(x, eps=0.000001):
#norm = torch.norm(x, 2, 0)
#ones = torch.ones_like(x)
# 1 if ||x|| >= 1, 0 otherwise
#cmp = torch.ge(norm, ge)
#normalised = torch.div(x, norm)
# normalize x in rows dim
return torch.renorm(x, 2, 0, 1.)
def project_perturbation(perturbation, eps, p):
if p in ['inf', 'linf', 'Linf']:
pert_normalized = torch.clamp(perturbation, -eps, eps)
return pert_normalized
elif p in [2, 2.0, 'l2', 'L2', '2']:
pert_normalized = torch.renorm(perturbation, p=2, dim=0, maxnorm=eps)
return pert_normalized
elif p in [1, 1.0, 'l1', 'L1', '1']:
pert_normalized = project_onto_l1_ball(perturbation, eps)
return pert_normalized
else:
raise NotImplementedError('Projection only supports l1, l2 and inf norm')
def calculateNorm(self):
norm = torch.cross(self.vec1, self.vec2, dim=0)
norm = torch.renorm(torch.unsqueeze(norm, 0), p=2, dim=0, maxnorm=1)
norm = torch.squeeze(norm)
#centre = torch.tensor([278.0, 274.4, 279.6]) - self.meshGrid[0]
centre = torch.tensor([278.0, 273.0, -600.0]) - self.meshGrid[0]
centreDot = norm[0] * centre[0] + norm[1] * centre[1] + norm[
2] * centre[2]
if (centreDot < 0):
norm = -1.0 * norm
return norm
def forward(self, x):
"""Returns the linear transformation of input tensor.
Arguments
---------
x : torch.Tensor
Input to transform linearly.
"""
self.w.weight.data = torch.renorm(
self.w.weight.data, p=2, dim=0, maxnorm=self.max_norm
)
return super(LinearWithConstraint, self).forward(x)
def normalize(self, w):
"""
Normalize vector such that it is located on the hyperboloid
Args:
w: [batch_size, d + 1]
"""
d = w.size(-1) - 1
narrowed = w.narrow(-1, 1, d)
if self.max_norm:
narrowed = th.renorm(narrowed.view(-1, d), 2, 0, self.max_norm)
first = 1 + th.sum(th.pow(narrowed, 2), dim=-1, keepdim=True)
first = th.sqrt(first)
return th.cat((first, narrowed), dim=1)
def forward(self, x_t, f_t, h_m_t, c_m_t, h_w_t, c_w_t, last_goal,
real_goal, t, temperature):
sub_goal, h_m_tp1, c_m_tp1 = self.manager(f_t, h_m_t, c_m_t)
output, h_w_tp1, c_w_tp1 = self.worker(x_t, h_w_t, c_w_t)
last_goal_temp = last_goal + sub_goal
w_t = torch.matmul(real_goal, self.worker.goal_change)
w_t = torch.renorm(w_t, 2, 0, 1.0)
w_t = torch.unsqueeze(w_t, -1)
logits = torch.squeeze(torch.matmul(output, w_t))
probs = F.softmax(temperature * logits, dim=1)
x_tp1 = Categorical(probs).sample()
return x_tp1, h_m_tp1, c_m_tp1, h_w_tp1, c_w_tp1,\
last_goal_temp, real_goal, sub_goal, probs, t + 1
def normalize(self, p, c):
"""
Normalize vector to confirm it is located on the hyperboloid
:param p: [nodes, features(d + 1)]
:param c: parameter of curvature
"""
d = p.size(-1) - 1
narrowed = p.narrow(-1, 1, d)
if self.max_norm:
narrowed = torch.renorm(narrowed.view(-1, d), 2, 0, self.max_norm)
first = c + torch.sum(torch.pow(narrowed, 2), dim=-1, keepdim=True)
first = torch.sqrt(first)
return torch.cat((first, narrowed), dim=1)
def renorm_vector(self, concat_represntentions):
if self.renorm_method != linear:
return torch.renorm(concat_represntentions, 2, 0, 1)
# return self.relation_layer_norm(concat_represntentions)
x = self.first_liner_layer(concat_represntentions)
x = self.tanh(x)
x = self.drop_layer(x)
x = self.second_liner_layer(x)
if self.do_skip_connection:
x = x + concat_represntentions
return x
def forward(self, x):
"""Returns the output of the convolution.
Arguments
---------
x : torch.Tensor (batch, time, channel)
input to convolve. 2d or 4d tensors are expected.
"""
self.conv.weight.data = torch.renorm(
self.conv.weight.data, p=2, dim=0, maxnorm=self.max_norm
)
return super(Conv2dWithConstraint, self).forward(x)
def normalizeEmbedding(self):
'''
Normalize the embedding.
:return:
'''
self.entEmbedding.weight.data.copy_(
torch.renorm(input=self.entEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1.0))
self.relEmbedding.weight.data.copy_(
torch.renorm(input=self.relEmbedding.weight.detach().cpu(),
p=2,
dim=0,
maxnorm=1.0))
self.entCovar.weight.data.copy_(
torch.clamp(input=self.entCovar.weight.detach().cpu(),
min=self.vmin,
max=self.vmax))
self.relCovar.weight.data.copy_(
torch.clamp(input=self.relCovar.weight.detach().cpu(),
min=self.vmin,
max=self.vmax))
def forward(self, f_t, h_m_t, c_m_t):
"""
f_t = feature of CNN from discriminator leaked at time t, it is input into LSTM
h_m_t = ouput of previous LSTMCell
c_m_t = previous cell state
"""
#print("H_M size: {}".format(h_m_t.size()))
#print("C_M size: {}".format(c_m_t.size()))
#print("F_t size: {}".format(f_t.size()))
h_m_tp1, c_m_tp1 = self.recurrent_unit(f_t, (h_m_t, c_m_t))
sub_goal = self.fc(h_m_tp1)
sub_goal = torch.renorm(
sub_goal, 2, 0, 1.0
) #Returns a tensor where each sub-tensor of input along dimension dim is normalized such that the p-norm of the sub-tensor is lower than the value maxnorm
return sub_goal, h_m_tp1, c_m_tp1
def attack(image, label):
# mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
# std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))
fmodel = foolbox.models.PyTorchModel(model.eval().cuda(), bounds=(0, 1),num_classes=110)#, preprocessing=(mean, std)
criterion1 = Misclassification()
distance = Linfinity#MeanAbsoluteDistance
# advs=Adversarial(fmodel, criterion1, image, label,distance=distance)
#adversarial= attackers[args.attack_name](advs)
attacker = attackers[args.attack_name](fmodel,criterion=criterion1,distance=distance)
adversarial = attacker(image, label)
#l2_norms = (adversarial - image).view(1, -1).norm(2, 1)
#mean_norm = l2_norms.mean()
if adversarial is not None:
adversarial= torch.renorm(torch.from_numpy(adversarial - image), p=2, dim=0, maxnorm=args.max_norm).numpy() + image
return adversarial
def forward(self, input, offsets, ref=None):
'''
:param input: a 1-dim tensor of indices
:param offset: a 1-dim tensor of offsets
:param ref: a 2-dim tensor of ref feats, typically the features of ads
:return:
'''
assert (ref is None and not self.atten) or (ref is not None and self.atten)
# add 1 dim for Embedding
input = input.view(1,-1)
# return 1, n_word, n_dim
embedding = self.embedder(input)
#print(embedding)
size = embedding.size()
# n_word, n_dim
embedding = embedding.view(size[1],size[2])
if self.atten:
size = embedding.size()
# replicate ref n_word, n_dim
ref = replicate(ref,offsets,size[0])
#print(ref)
# calculate the attention
#todo
diff = ref-embedding
feat_for_atten = torch.cat([embedding,diff,ref],dim=1)
atten = self.linear1(feat_for_atten)
atten = self.activation(atten)
atten = self.linear2(atten)
# n_word, 1
atten = self.sigmoid(atten)
# print(atten)
embedding = embedding * atten
#print(embedding)
# n_sample, n_dim
res = reduce(embedding,offsets,self.mode)
# following lines constrain the max norm of embedding.
size = res.size()
# n_sample, n_field, n_dim//n_field
res = res.view(size[0]*self.n_field,size[1]//self.n_field)
renorm_res = torch.renorm(res,p=self.norm_type,dim=0,maxnorm=self.max_norm)
renorm_res = renorm_res.contiguous()
# res = F.normalize(res,p=self.norm_type,dim=2)*self.max_norm
res = renorm_res.view(size[0],size[1])
return res
def __call__(self, module):
w = module.weight.data
module.weight.data = th.renorm(w, 2, self.axis, self.value)
