def forward(self, inputs, targets, attack=False): if not attack: return self.basic_net(inputs) x = inputs.detach() if self.rand: # x = x + torch.zeros_like(x).uniform_(-self.epsilon, self.epsilon) noise = torch.zeros_like(x).normal_(0, self.step_size).view( x.size(0), -1) x = x + torch.renorm(noise, 2, 0, self.epsilon).view(x.size()) for _ in range(self.num_steps): x.requires_grad_() with torch.enable_grad(): logits = self.basic_net(x) loss = F.cross_entropy(logits, targets, reduction='sum') grad = torch.autograd.grad(loss, x)[0].detach() grad_norm = grad.view(x.size(0), -1).norm(2, 1) delta = self.step_size * grad / grad_norm.view(x.size(0), 1, 1, 1) x = x.detach() + delta # x = torch.min(torch.max(x, inputs - self.epsilon), # inputs + self.epsilon) delta = torch.renorm((x - inputs.detach()).view(x.size(0), -1), 2, 0, self.epsilon).view(x.size()) x = torch.clamp(inputs.detach() + delta, 0, 1) return self.basic_net(x)
def normalizeEmbedding(self): self.entityEmbedding.weight.data.copy_(torch.renorm(input=self.entityEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1)) self.relationEmbedding.weight.data.copy_(torch.renorm(input=self.relationEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1))
def apply(self, model): last_weight = None for name, module in list(model.named_children()): if hasattr(module, 'weight') and ( not module.__class__.__name__.startswith('BatchNorm')): module.weight.data = th.renorm(module.weight.data,2,0,maxnorm=2) last_weight = module.weight if last_weight is not None: last_weight.data = th.renorm(last_weight.data,2,0,maxnorm=0.5)
def normalizeEmbedding(self): ''' normalize the embedding and change the embedding. :return: ''' self.entEmbedding.weight.data.copy_( torch.renorm(input=self.entMapEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1)) self.relEmbedding.weight.data.copy_( torch.renorm(input=self.relEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1))
def forward(self, x): if self.doWeightNorm: self.weight.data = torch.renorm(self.weight.data, p=2, dim=0, maxnorm=self.max_norm) return super(LinearWithConstraint, self).forward(x)
def forward(self, x_t, f_t, h_m_t, c_m_t, h_w_t, c_w_t, last_goal, real_goal, t, temperature): ''' args: x_t([]):the current word f_t([]):the feature vector come from the discriminator real_goal([]): temperature([]):as said in paper,this parameter is to control the generation entropy. output: last_goal_temp([]):输出作为last_goal ''' sub_goal, h_m_tp1, c_m_tp1 = self.manager(f_t, h_m_t, c_m_t) #sub_goal = [batch_size,goal_out_size] output, h_w_tp1, c_w_tp1 = self.worker(x_t, h_w_t, c_w_t) # output = [batch_size, vocab_size, goal_size] (as the O_t in the paper) last_goal_temp = last_goal + sub_goal # 就单用上一个last_goal加上用f_t产生的sub_goal?? w_t = torch.matmul( #矩阵乘法 [] real_goal, self.worker.goal_change # worker.goal_change[goal_out_size, goal_size] ) # 主要问题就在上面,last_goal_temp后续没作用,意味着Manager这次的信息根本没到后续的w_t中,也就没到logits中 w_t = torch.renorm(w_t, 2, 0, 1.0) # [] w_t = torch.unsqueeze(w_t, -1) logits = torch.squeeze(torch.matmul(output, w_t)) #logits for words probs = F.softmax(temperature * logits, dim=1) x_tp1 = Categorical(probs).sample() return x_tp1, h_m_tp1, c_m_tp1, h_w_tp1, c_w_tp1,\ last_goal_temp, real_goal, sub_goal, probs, t + 1
def batchwise_lp_project(x, lp, lp_bound, dim=0): """ Projects x (a N-by-(...) TENSOR) to be a N-by-(...) TENSOR into the provided lp ball ARGS: x : Tensor (N-by-(...)) - arbitrary style lp : 'inf' or int - which style of lp we use lp_bound : float - size of lp ball we project into dim : int - if not 0 is the dimension we keep and project onto RETURNS: None """ assert isinstance(lp, int) or lp == 'inf' if lp == 'inf': return torch.clamp(x, -lp_bound, lp_bound) needs_squeeze = False if len(x.shape) == 1: x = x.unsqueeze(1) needs_squeeze = True output = torch.renorm(x, lp, dim, lp_bound) if needs_squeeze: return output.squeeze() return output
def random_from_lp_ball(tensorlike, lp, lp_bound, dim=0): """ Returns a new object of the same type/shape as tensorlike that is randomly samples from the unit ball. NOTE THIS IS NOT A UNIFORM SAMPLING METHOD! (that's hard to implement, https://mathoverflow.net/a/9192/123034) ARGS: tensorlike : Tensor - reference object for which we generate a new object of same shape/memory_location lp : int or 'inf' - which style of lp we use lp_bound : float - size of the L dim : int - which dimension is the 'keep dimension' RETURNS: new tensorlike where each slice across dim is uniform across the lp ball of size lp_bound """ assert isinstance(lp, int) or lp == 'inf' rand_direction = torch.rand(tensorlike.shape).type(tensorlike.type()) if lp == 'inf': return rand_direction * (2 * lp_bound) - lp_bound else: rand_direction = rand_direction - 0.5 # allow for sign swapping # first magnify such that each element is above the ball min_norm = torch.min(batchwise_norm(rand_direction.abs(), lp, dim=dim)) rand_direction = rand_direction / (min_norm + 1e-6) rand_magnitudes = torch.rand(tensorlike.shape[dim]).type( tensorlike.type()) rand_magnitudes = rand_magnitudes.unsqueeze(1) rand_magnitudes = rand_magnitudes.expand(*rand_direction.shape) return torch.renorm(rand_direction, lp, dim, lp_bound) * rand_magnitudes
def attack(image, label, attack_name): fmodel = foolbox.models.PyTorchModel( model.eval().cuda(), bounds=(0, 1), num_classes=10) #, preprocessing=(mean, std) criterion1 = Misclassification() distance = Linfinity #MeanAbsoluteDistance attacker = attackers[attack_name](fmodel, criterion=criterion1, distance=distance) image = image.cpu().numpy() label = label.cpu().numpy() adversarials = image.copy() for i in tqdm.tqdm(range(args.batch_size), ncols=80): adv = attacker( image[i], label[i] ) # , unpack=True, steps=self.max_iter, subsample=self.subsample) if adv is not None: adv = torch.renorm( torch.from_numpy(adv - image[i]), p=2, dim=0, maxnorm=1).numpy() + image[i] adversarials[i] = adv adversarials = torch.from_numpy(adversarials).to(DEVICE) return adversarials
def forward(self, x): x = self.forward_features(x) if self.drop_rate > 0.: x = F.dropout(x, p=self.drop_rate, training=self.training) x = self.fc(x) if self.output_norm: x = torch.renorm(x, self.output_norm, 0, 1.0) return x
def on_batch_end(self, net, training, *args, **kwargs): if training: model = net.module_ last_weight = None for name, module in list(model.named_children()): if hasattr(module, "weight") and ( not module.__class__.__name__.startswith("BatchNorm")): module.weight.data = torch.renorm(module.weight.data, 2, 0, maxnorm=2) last_weight = module.weight if last_weight is not None: last_weight.data = torch.renorm(last_weight.data, 2, 0, maxnorm=0.5)
def _clip(inputs: Tensor, outputs: Tensor) -> Tensor: diff = outputs - inputs if norm == "Linf": return inputs + torch.clamp(diff, -radius, radius) elif norm == "L2": return inputs + torch.renorm(diff, 2, 0, radius) else: raise AssertionError("Norm constraint must be L2 or Linf.")
def set_dim0(x): x = torch.renorm(x, p=2, dim=0, maxnorm=1e2) # otherwise leaves will explode # NOTE: the paper does not mention the square part of the equation but if # you try to derive it you get a square term in the equation dim0 = torch.sqrt(1 + (x[:, 1:]**2).sum(dim=1)) x[:, 0] = dim0 return x
def __call__(self, module): if self.lagrangian: w = module.weight norm_diff = th.norm(w, 2, self.axis).sub(self.value) return self.scale * th.sum(norm_diff.gt(0).float().mul(norm_diff)) else: module.weight.data = th.renorm(module.weight.data, 2, self.axis, self.value)
def load_word2vec_format(filename, word_idx, binary=False, normalize=False, encoding='utf8', unicode_errors='ignore'): """ refer to gensim load Word Embeddings If you trained the C model using non-utf8 encoding for words, specify that encoding in `encoding`. :param filename : :param word_idx : :param binary : a boolean indicating whether the data is in binary word2vec format. :param normalize: :param encoding : :param unicode_errors: errors can be 'strict', 'replace' or 'ignore' and defaults to 'strict'. """ vocab = set() print("loading word embedding from %s" % filename) with open(filename, 'rb') as fin: # header = to_unicode(fin.readline(), encoding=encoding) # vocab_size, vector_size = map(int, header.split()) # throws for invalid file format vocab_size = 1917494 vector_size = 300 word_matrix = torch.zeros(len(word_idx), vector_size) def add_word(_word, _weights): if _word not in word_idx: return vocab.add(_word) word_matrix[word_idx[_word]] = _weights if binary: binary_len = np.dtype(np.float32).itemsize * vector_size for _ in range(vocab_size): # mixed text and binary: read text first, then binary word = [] while True: ch = fin.read(1) if ch == b' ': break if ch != b'\n': # ignore newlines in front of words (some binary files have) word.append(ch) word = to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors) weights = torch.from_numpy(np.fromstring(fin.read(binary_len), dtype=REAL)) add_word(word, weights) else: for line_no, line in enumerate(fin): parts = to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(" ") if len(parts) != vector_size + 1: raise ValueError("invalid vector on line %s (is this really the text format?)" % line_no) word, weights = parts[0], list(map(float, parts[1:])) weights = torch.Tensor(weights) add_word(word, weights) if word_idx is not None: assert (len(word_idx), vector_size) == word_matrix.size() if normalize: # each row normalize to 1 word_matrix = torch.renorm(word_matrix, 2, 0, 1) print("loaded %d words pre-trained from %s with %d" % (len(vocab), filename, vector_size)) return word_matrix, vector_size, vocab
def normalizeEmbedding(self): self.entityEmbedding.weight.data.copy_( torch.renorm(input=self.entityEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1.0)) self.relationEmbedding.weight.data.copy_( torch.renorm(input=self.relationEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1.0)) self.entityCovar.weight.data.copy_( torch.clamp(input=self.entityCovar.weight.detach().cpu(), min=self.vmin, max=self.vmax)) self.relationCovar.weight.data.copy_( torch.clamp(input=self.relationCovar.weight.detach().cpu(), min=self.vmin, max=self.vmax))
def forward(self, x): x = F.relu(self.conv1(x)) x = F.relu(self.conv2(x)) x = self.pool(x) x = F.relu(self.conv3(x)) x = F.relu(self.conv4(x)) x = self.pool2(x) x = F.relu(self.conv5(x)) x = F.relu(self.conv6(x)) x = self.pool3(x) x = self.pool4(x) x = x.view(-1, self.cnn_output) for ix, fc in enumerate(self.fcs): x = fc(x) if self.neural_noise is not None and ix == 0 and self.training: mean, std = self.neural_noise noise = torch.zeros_like(x, device=dev) noise = noise.log_normal_(mean=mean, std=std) x = x * noise x = F.relu(x) if self.excite and ix == 1 and self.n_new and self.training: idx = self.idx_control if self.control else self.idx excite_mask = torch.ones_like(x) excite_mask[:, idx] = self.excite excite_mask.to(dev) x = x * excite_mask if self.dropout: x = F.dropout(x, p=self.dropout, training=self.training) x = torch.renorm(x, 1, 1, 3) # max norm # for ablation experiments if self.ablate: activation_size = x.size()[1] ablate_size = int(self.ablation_prop * activation_size) if self.ablation_mode == "random": indices = np.random.choice( range(activation_size), size=size, replace=False, ) if self.ablation_mode == "targetted": importance = torch.norm(fc.weight.detach().clone(), p=1, dim=1) indices = torch.argsort(importance, descending=True)[:ablate_size] x[:, indices] = 0 x = self.fc3(x) return x
def proj(x, eps=0.000001): #norm = torch.norm(x, 2, 0) #ones = torch.ones_like(x) # 1 if ||x|| >= 1, 0 otherwise #cmp = torch.ge(norm, ge) #normalised = torch.div(x, norm) # normalize x in rows dim return torch.renorm(x, 2, 0, 1.)
def project_perturbation(perturbation, eps, p): if p in ['inf', 'linf', 'Linf']: pert_normalized = torch.clamp(perturbation, -eps, eps) return pert_normalized elif p in [2, 2.0, 'l2', 'L2', '2']: pert_normalized = torch.renorm(perturbation, p=2, dim=0, maxnorm=eps) return pert_normalized elif p in [1, 1.0, 'l1', 'L1', '1']: pert_normalized = project_onto_l1_ball(perturbation, eps) return pert_normalized else: raise NotImplementedError('Projection only supports l1, l2 and inf norm')
def calculateNorm(self): norm = torch.cross(self.vec1, self.vec2, dim=0) norm = torch.renorm(torch.unsqueeze(norm, 0), p=2, dim=0, maxnorm=1) norm = torch.squeeze(norm) #centre = torch.tensor([278.0, 274.4, 279.6]) - self.meshGrid[0] centre = torch.tensor([278.0, 273.0, -600.0]) - self.meshGrid[0] centreDot = norm[0] * centre[0] + norm[1] * centre[1] + norm[ 2] * centre[2] if (centreDot < 0): norm = -1.0 * norm return norm
def forward(self, x): """Returns the linear transformation of input tensor. Arguments --------- x : torch.Tensor Input to transform linearly. """ self.w.weight.data = torch.renorm( self.w.weight.data, p=2, dim=0, maxnorm=self.max_norm ) return super(LinearWithConstraint, self).forward(x)
def normalize(self, w): """ Normalize vector such that it is located on the hyperboloid Args: w: [batch_size, d + 1] """ d = w.size(-1) - 1 narrowed = w.narrow(-1, 1, d) if self.max_norm: narrowed = th.renorm(narrowed.view(-1, d), 2, 0, self.max_norm) first = 1 + th.sum(th.pow(narrowed, 2), dim=-1, keepdim=True) first = th.sqrt(first) return th.cat((first, narrowed), dim=1)
def forward(self, x_t, f_t, h_m_t, c_m_t, h_w_t, c_w_t, last_goal, real_goal, t, temperature): sub_goal, h_m_tp1, c_m_tp1 = self.manager(f_t, h_m_t, c_m_t) output, h_w_tp1, c_w_tp1 = self.worker(x_t, h_w_t, c_w_t) last_goal_temp = last_goal + sub_goal w_t = torch.matmul(real_goal, self.worker.goal_change) w_t = torch.renorm(w_t, 2, 0, 1.0) w_t = torch.unsqueeze(w_t, -1) logits = torch.squeeze(torch.matmul(output, w_t)) probs = F.softmax(temperature * logits, dim=1) x_tp1 = Categorical(probs).sample() return x_tp1, h_m_tp1, c_m_tp1, h_w_tp1, c_w_tp1,\ last_goal_temp, real_goal, sub_goal, probs, t + 1
def normalize(self, p, c): """ Normalize vector to confirm it is located on the hyperboloid :param p: [nodes, features(d + 1)] :param c: parameter of curvature """ d = p.size(-1) - 1 narrowed = p.narrow(-1, 1, d) if self.max_norm: narrowed = torch.renorm(narrowed.view(-1, d), 2, 0, self.max_norm) first = c + torch.sum(torch.pow(narrowed, 2), dim=-1, keepdim=True) first = torch.sqrt(first) return torch.cat((first, narrowed), dim=1)
def renorm_vector(self, concat_represntentions): if self.renorm_method != linear: return torch.renorm(concat_represntentions, 2, 0, 1) # return self.relation_layer_norm(concat_represntentions) x = self.first_liner_layer(concat_represntentions) x = self.tanh(x) x = self.drop_layer(x) x = self.second_liner_layer(x) if self.do_skip_connection: x = x + concat_represntentions return x
def forward(self, x): """Returns the output of the convolution. Arguments --------- x : torch.Tensor (batch, time, channel) input to convolve. 2d or 4d tensors are expected. """ self.conv.weight.data = torch.renorm( self.conv.weight.data, p=2, dim=0, maxnorm=self.max_norm ) return super(Conv2dWithConstraint, self).forward(x)
def normalizeEmbedding(self): ''' Normalize the embedding. :return: ''' self.entEmbedding.weight.data.copy_( torch.renorm(input=self.entEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1.0)) self.relEmbedding.weight.data.copy_( torch.renorm(input=self.relEmbedding.weight.detach().cpu(), p=2, dim=0, maxnorm=1.0)) self.entCovar.weight.data.copy_( torch.clamp(input=self.entCovar.weight.detach().cpu(), min=self.vmin, max=self.vmax)) self.relCovar.weight.data.copy_( torch.clamp(input=self.relCovar.weight.detach().cpu(), min=self.vmin, max=self.vmax))
def forward(self, f_t, h_m_t, c_m_t): """ f_t = feature of CNN from discriminator leaked at time t, it is input into LSTM h_m_t = ouput of previous LSTMCell c_m_t = previous cell state """ #print("H_M size: {}".format(h_m_t.size())) #print("C_M size: {}".format(c_m_t.size())) #print("F_t size: {}".format(f_t.size())) h_m_tp1, c_m_tp1 = self.recurrent_unit(f_t, (h_m_t, c_m_t)) sub_goal = self.fc(h_m_tp1) sub_goal = torch.renorm( sub_goal, 2, 0, 1.0 ) #Returns a tensor where each sub-tensor of input along dimension dim is normalized such that the p-norm of the sub-tensor is lower than the value maxnorm return sub_goal, h_m_tp1, c_m_tp1
def attack(image, label): # mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1)) # std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1)) fmodel = foolbox.models.PyTorchModel(model.eval().cuda(), bounds=(0, 1),num_classes=110)#, preprocessing=(mean, std) criterion1 = Misclassification() distance = Linfinity#MeanAbsoluteDistance # advs=Adversarial(fmodel, criterion1, image, label,distance=distance) #adversarial= attackers[args.attack_name](advs) attacker = attackers[args.attack_name](fmodel,criterion=criterion1,distance=distance) adversarial = attacker(image, label) #l2_norms = (adversarial - image).view(1, -1).norm(2, 1) #mean_norm = l2_norms.mean() if adversarial is not None: adversarial= torch.renorm(torch.from_numpy(adversarial - image), p=2, dim=0, maxnorm=args.max_norm).numpy() + image return adversarial
def forward(self, input, offsets, ref=None): ''' :param input: a 1-dim tensor of indices :param offset: a 1-dim tensor of offsets :param ref: a 2-dim tensor of ref feats, typically the features of ads :return: ''' assert (ref is None and not self.atten) or (ref is not None and self.atten) # add 1 dim for Embedding input = input.view(1,-1) # return 1, n_word, n_dim embedding = self.embedder(input) #print(embedding) size = embedding.size() # n_word, n_dim embedding = embedding.view(size[1],size[2]) if self.atten: size = embedding.size() # replicate ref n_word, n_dim ref = replicate(ref,offsets,size[0]) #print(ref) # calculate the attention #todo diff = ref-embedding feat_for_atten = torch.cat([embedding,diff,ref],dim=1) atten = self.linear1(feat_for_atten) atten = self.activation(atten) atten = self.linear2(atten) # n_word, 1 atten = self.sigmoid(atten) # print(atten) embedding = embedding * atten #print(embedding) # n_sample, n_dim res = reduce(embedding,offsets,self.mode) # following lines constrain the max norm of embedding. size = res.size() # n_sample, n_field, n_dim//n_field res = res.view(size[0]*self.n_field,size[1]//self.n_field) renorm_res = torch.renorm(res,p=self.norm_type,dim=0,maxnorm=self.max_norm) renorm_res = renorm_res.contiguous() # res = F.normalize(res,p=self.norm_type,dim=2)*self.max_norm res = renorm_res.view(size[0],size[1]) return res
def __call__(self, module): w = module.weight.data module.weight.data = th.renorm(w, 2, self.axis, self.value)