def sample(x_gen): n_comps = 10 logits = x_gen[:, 0:n_comps, :, :] sel = torch.argmax(logits, # - # torch.log(- torch.log(self.float_tensor(logits.size()).uniform_(1e-5, 1-1e-5))), dim=1, keepdim=True) one_hot = torch.zeros(logits.size()) if torch.cuda.is_available(): one_hot = one_hot.cuda() one_hot.scatter_(1, sel, 1.0) mean_x_r = torch.sum(x_gen[:, n_comps:2 * n_comps, :, :] * one_hot, 1, keepdim=True) # u_r = self.float_tensor(mean_x_r.size()).uniform_(1e-5, 1 - 1e-5) x_r = F.hardtanh(mean_x_r, # + torch.exp(log_scale_r) * (torch.log(u_r) - torch.log(1. - u_r)), min_val=0., max_val=1.) mean_x_g = torch.sum(x_gen[:, 2 * n_comps:3 * n_comps, :, :] * one_hot, 1, keepdim=True) + \ torch.tanh(torch.sum(x_gen[:, 3 * n_comps:4 * n_comps] * one_hot, 1, keepdim=True)) * x_r # u_g = self.float_tensor(mean_x_g.size()).uniform_(1e-5, 1 - 1e-5) x_g = F.hardtanh(mean_x_g, # + torch.exp(log_scale_g) * (torch.log(u_g) - torch.log(1. - u_g)), min_val=0., max_val=1.) mean_x_b = torch.sum(x_gen[:, 4 * n_comps:5 * n_comps, :, :] * one_hot, 1, keepdim=True) + \ torch.tanh(torch.sum(x_gen[:, 5 * n_comps:6 * n_comps] * one_hot, 1, keepdim=True)) * x_r + \ torch.tanh( torch.sum(x_gen[:, 6 * n_comps:7 * n_comps, :, :] * one_hot, 1, keepdim=True)) * x_g # u_b = self.float_tensor(mean_x_b.size()).uniform_(1e-5, 1 - 1e-5) x_b = F.hardtanh(mean_x_b, # + torch.exp(log_scale_b) * (torch.log(u_b) - torch.log(1. - u_b)), min_val=0., max_val=1.) sample = torch.cat([x_r, x_g, x_b], 1) return sample
def get_mask(self, log_alpha, re_sample=True): if self.training: if self.u is None or re_sample: self.u = u = torch.rand(log_alpha.size(), device=self.args.device) else: u = self.u s_u = torch.log(u + self.constant_eps) - torch.log(1 - u + self.constant_eps) s = torch.sigmoid((s_u + log_alpha + self.constant_eps) / self.beta) s_bar = s * (self.zeta - self.gamma) + self.gamma mask = nfunc.hardtanh(s_bar, min_val=0, max_val=1) else: s = torch.sigmoid(log_alpha / self.beta) s_bar = s * (self.zeta - self.gamma) + self.gamma mask = nfunc.hardtanh(s_bar, min_val=0, max_val=1) if self.args.dataset in ["image", 'imagenet']: mask = nfunc.interpolate(mask, size=(224, 224), mode='nearest') if self.args.dataset in ["image64", 'imagenet64']: if 'pool' in self.args.layer: mask = nfunc.interpolate(mask, size=(64, 64), mode='nearest') if 'caltech' in self.args.dataset: if 'pool' in self.args.layer: if "incept" == self.args.arch: mask = nfunc.interpolate(mask, size=(299, 299), mode='nearest') else: mask = nfunc.interpolate(mask, size=(256, 256), mode='nearest') return mask
def forward(self, input): if self.wbit == 32: input_q = input else: n_lv = 2**self.wbit scale = n_lv // 2 - 1 # gradient friendly a = F.softplus(self.a) # keep the learnable value positive c = F.softplus(self.c) if self.channel_wise == 1: input = input.div(a[:, None, None, None]) input = F.hardtanh(input, -1, 1) scale = torch.ones_like(self.a.data).mul(scale) input_q = WeightQuant.apply(input, scale) input_q = input_q.mul(c[:, None, None, None]) else: input = input.div(a) input = F.hardtanh(input, -1, 1) input_q = RoundQuant.apply(input, scale) input_q = input_q.mul(c) return input_q
def forward(self, mono_feature, non_mono_feature): y = self.non_mono_fc_in(non_mono_feature) y = F.relu(y) if self.compress_non_mono: non_mono_feature = self.non_mono_feature_extractor( non_mono_feature) non_mono_feature = F.hardtanh(non_mono_feature, min_val=0.0, max_val=1.0) x = self.mono_fc_in(torch.cat([mono_feature, non_mono_feature], dim=1)) x = F.relu(x) for i in range(int(len(self.mono_submods_out))): x = self.mono_submods_out[i](x) x = F.hardtanh(x, min_val=0.0, max_val=1.0) y = self.non_mono_submods_out[i](y) y = F.hardtanh(y, min_val=0.0, max_val=1.0) x = self.mono_submods_in[i](torch.cat([x, y], dim=1)) x = F.relu(x) y = self.non_mono_submods_in[i](y) y = F.relu(y) x = self.mono_fc_last(x) y = self.non_mono_fc_last(y) out = x + y if self.normalize_regression: out = F.sigmoid(out) return out
def forward(self, x): out = self.bn1(self.conv1(x)) out = F.hardtanh(out) out = self.bn2(self.conv2(out)) out += self.shortcut(x) out = F.hardtanh(out) return out
def closure_wrapper(): loss = closure() for group in optimizer.param_groups: for p in group['params']: if p.grad is not None: hardtanh(p.grad, min, max, inplace=True) return loss
def forward(self, x): if self.alpha is None: return x if self.training and self.init_state == 0: Qp = 2**(self.nbits - 1) - 1 self.xmax.data.copy_(x.abs().max()) self.alpha.data.copy_(self.xmax / Qp) # self.alpha[self.index].data.copy_(2 * x.abs().mean() / math.sqrt(Qp)) # self.xmax[self.index].data.copy_(self.alpha[self.index] * Qp) self.init_state.fill_(1) self.xmax.data.copy_(self.xmax.clamp(self.qmin, self.qmax)) self.alpha.data.copy_(self.alpha.clamp(self.dmin, self.dmax)) Qp = (self.xmax / self.alpha).item() g = 1.0 / math.sqrt(x.numel() * Qp) alpha = grad_scale(self.alpha, g) xmax = grad_scale(self.xmax, g) if self.signed: x = F.hardtanh(x / xmax.abs(), -1, 1) * xmax.abs() # x = round_pass((torch.clamp(x/xmax, -1, 1)*xmax)/alpha) * alpha else: x = F.hardtanh(x / xmax.abs(), 0, 1) * xmax.abs() # x = round_pass((torch.clamp(x/xmax, 0, 1)*xmax)/alpha) * alpha x = x / alpha.abs() x = round_pass(x) * alpha.abs() return x
def forward(self, agent_state: torch.FloatTensor, other_agent_states: torch.FloatTensor, other_agent_actions: Optional[torch.FloatTensor] = None, action: Optional[torch.FloatTensor] = None, min_std=0.05, scale=1,): assert min_std > 0 and scale >= 0, (min_std, scale) if self.continuous_actions: action_mean = self.actor(agent_state) std = F.hardtanh(self.std, min_val=min_std, max_val=scale) dist = torch.distributions.Normal(action_mean, std) else: action_probs = self.actor(agent_state) dist = torch.distributions.Categorical(probs=action_probs) if action is None: action = dist.sample().to(device) if action.ndim > 1: action = action.squeeze().to(device) if other_agent_actions is None: if self.continuous_actions: other_agent_action_mean = self.actor(agent_state) std = F.hardtanh(self.std, min_val=min_std, max_val=scale) other_agent_dist = torch.distributions.Normal(other_agent_action_mean, std) else: other_action_probs = self.actor(other_agent_states) other_agent_dist = torch.distributions.Categorical(probs=other_action_probs) other_agent_actions = other_agent_dist.sample().to(device) critic_value = self.critic(agent_state, other_agent_states, other_agent_actions, action) log_probs = dist.log_prob(action) dist_entropy = dist.entropy().mean() return action, log_probs, dist_entropy, critic_value
def calc_targets(feat, label, noise, predict, opts=None): clip_low = opts["value_low"] clip_high = opts["value_high"] compress_function = opts["compress_function"] if clip_low and clip_high and predict: predict = F.hardtanh(predict, clip_low, clip_high) if compress_function and predict: predict = compress_function(predict) with torch.no_grad(): feat = feat.sqrt() label = label.sqrt() target = label / feat target[torch.isinf(target)] = 0. target[torch.isnan(target)] = 0. if clip_low and clip_high: target = F.hardtanh(target, clip_low, clip_high) if compress_function: target = compress_function(target) ideal = target * feat if predict is None: predict = target enhanced = feat * predict return {"target": target, "predict": predict, "enhanced": enhanced, "ideal": ideal, "real_data": ideal, "fake_data": enhanced}
def forward(self, data): pos = data.pos # xyz # ref: https://github.com/yanx27/Pointnet_Pointnet2_pytorch/blob/master/data_utils/S3DISDataLoader.py rgb = data.x # rgb and 3 additional features batch = torch.max(data.batch) + 1 pos_list = [] rgb_list = [] for i in range(batch): pos_list.append(pos[data.batch == i]) rgb_list.append(rgb[data.batch == i]) pos = torch.stack(pos_list).permute(0, 2, 1).contiguous() rgb = torch.stack(rgb_list).permute(0, 2, 1).contiguous() if self.more_features: point_cloud = torch.cat([pos, rgb], dim=1) else: point_cloud = pos x, trans, trans_feat = self.feat(point_cloud) x = F.hardtanh(self.bn1(self.conv1(x))) x = F.hardtanh(self.bn2(self.conv2(x))) x = F.hardtanh(self.bn3(self.conv3(x))) x = self.conv4(x) x = x.transpose(2,1).contiguous() x = F.log_softmax(x.view(-1,self.k), dim=-1) x = x.view(-1, self.k) return { 'out': x, 'trans': trans, 'trans_feat': trans_feat, }
def cpt_gate(self, semantic_score: T) -> Tuple[T, T]: assert semantic_score.size()[0] > 4 score = semantic_score[1:-1] # (num_score - 2) fwd_score = torch.cat([torch.zeros(score.size(0)), score], dim=0) bwd_score = torch.cat([score, torch.zeros(score.size(0))], dim=0) fwd_score_hat = torch.stack([ fwd_score[i:i + score.size()[0]] for i in range(score.size()[0] - 1, 0, -1) ], dim=0) bwd_score_hat = torch.stack([ bwd_score[i:i + score.size(0)] for i in range(1, score.size()[0]) ], dim=0) if self.hard: fwd_gate = (F.hardtanh( (fwd_score_hat - score[None, :]) / self.resolution * 2 + 1) + 1) / 2 bwd_gate = (F.hardtanh( (bwd_score_hat - score[None, :]) / self.resolution * 2 + 1) + 1) / 2 else: fwd_gate = F.sigmoid((fwd_score_hat - score[None, :]) / self.resolution * 10 + 5) bwd_gate = F.sigmoid((bwd_score_hat - score[None, :]) / self.resolution * 10 + 5) fwd_gate = torch.cumprod(fwd_gate, dim=0) # seq x seq - 1 bwd_gate = torch.cumprod(bwd_gate, dim=0) # seq x seq - 1 return (fwd_gate, bwd_gate)
def forward(self, input): # 1. input data and weight quantization with torch.no_grad(): self.delta_w = self.weight.abs().max() / self.h_lvl_w * self.scaler_dw if self.training: self.counter.data += 1 self.delta_x = input.abs().max() / self.h_lvl_i self.delta_in_sum.data += self.delta_x else: self.delta_x = self.delta_in_sum.data / self.counter.data input_clip = F.hardtanh(input, min_val=-self.h_lvl_i * self.delta_x.item(), max_val=self.h_lvl_i * self.delta_x.item()) input_quan = quantize_input(input_clip, self.delta_x) # * self.delta_x # convert to voltage weight_quan = quantize_weight(self.weight, self.delta_w) # * self.delta_w if self.bias is not None: bias_quan = quantize_weight(self.bias, self.delta_x) else: bias_quan = None output_crxb = F.linear(input, weight_quan, bias_quan) with torch.no_grad(): if self.training: self.delta_i = output_crxb.abs().max() / self.h_lvl_a self.delta_out_sum.data += self.delta_i else: self.delta_i = self.delta_out_sum.data / self.counter.data self.delta_y = self.delta_w * self.delta_x * self.delta_i # print('adc LSB ration:', self.delta_i/self.max_i_LSB) output_clip = F.hardtanh(output_crxb, min_val=-self.h_lvl_a * self.delta_i.item(), max_val=self.h_lvl_a * self.delta_i.item()) output_adc = adc(output_clip, self.delta_i, 1.) return output_adc
def forward(self, x): out = F.hardtanh(self.conv1(x)) out = F.hardtanh(self.conv2(out)) out = F.hardtanh(self.conv3(out)) out = out.view(-1, 16) out = F.hardtanh(self.fc1(out)) out = self.fc2(out) return out
def forward(self, x): out = self.bn1(self.conv1(x)) out += self.shortcut(x) out = F.hardtanh(out, inplace=True) x1 = out out = self.bn2(self.conv2(out)) out += x1 out = F.hardtanh(out, inplace=True) return out
def sample_z(self, batch_size, sample=True): """Sample the hard-concrete gates for training and use a deterministic value for testing""" if sample: eps = self.get_eps(self.floatTensor(batch_size, self.dim_z)) z = self.quantile_concrete(eps).view(batch_size, self.dim_z, 1, 1) return F.hardtanh(z, min_val=0, max_val=1) else: # mode pi = F.sigmoid(self.qz_loga).view(1, self.dim_z, 1, 1) return F.hardtanh(pi * (limit_b - limit_a) + limit_a, min_val=0, max_val=1)
def forward(self, x): if self.act_func: x = x * (F.hardtanh(x + 3, 0, 6) / 6) if len(self.bits) == 1 and self.bits[0] == 32: return x else: x = F.hardtanh(x, 0, 1) x = RoundQuant.apply(x, self.n_lvs) return x
def forward(self, belief, state): x = F.hardtanh(self.belief1(belief)) x = F.hardtanh(self.belief2(x)) x = F.hardtanh(self.belief_interm(x)) x = torch.cat((x, state), 0) x = F.hardtanh(self.bs1(x)) x = F.hardtanh(self.bs2(x)) x = F.softmax(self.bs4(self.bs3(x))) return x
def forward(self, x): # x = f.hardtanh(self.bn1(self.fc1(x))) # x = f.hardtanh(self.bn2(self.fc2(x))) # x = self.bn3(self.fc3(x)) x = f.hardtanh(self.fc1(x)) x = f.hardtanh(self.fc2(x)) x = self.fc3(x) return x
def update(self, inputs: Tensor, denominator: Tensor) -> Tensor: # Step 5: Apply denominator # inputs has shape (m, R) if self.flow == 'source_to_target' else (n, R) # denominator has shape (m) if self.flow == 'source_to_target' else (n) inputs /= denominator.unsqueeze(-1) if self.flow == 'source_to_target': return F.hardtanh(inputs + self.vertex_translate) else: return F.hardtanh(inputs + self.edge_translate)
def forward(self, x): B, D, N = x.size() if self.tnet: trans = self.stn(x) else: trans = None x = x.transpose(2, 1) if D == 6: x, feature = x.split(3, dim=2) elif D == 9: x, feature = x.split([3, 6], dim=2) if self.tnet: x = torch.bmm(x, trans) if D > 3: x = torch.cat([x, feature], dim=2) x = x.transpose(2, 1) if self.use_bn: x = F.hardtanh(self.bn1(self.conv1(x))) else: x = F.hardtanh(self.conv1(x)) if self.tnet and self.feature_transform: trans_feat = self.fstn(x) x = x.transpose(2, 1) x = torch.bmm(x, trans_feat) x = x.transpose(2, 1) else: trans_feat = None pointfeat = x if self.use_bn: x = F.hardtanh(self.bn2(self.conv2(x))) x = self.bn3(self.conv3(x)) else: x = F.hardtanh(self.conv2(x)) x = self.conv3(x) if self.pool == 'max': x = torch.max(x, 2, keepdim=True)[0] elif self.pool == 'mean': x = torch.mean(x, 2, keepdim=True) elif self.pool == 'ema-max': if self.use_bn: x = torch.max(x, 2, keepdim=True)[0] + offset_map[N] else: x = torch.max(x, 2, keepdim=True)[0] - 0.3 x = x.view(-1, 1024) x = x.view(-1, 1024) if self.global_feat: return x, trans, trans_feat else: x = x.view(-1, 1024, 1).repeat(1, 1, N) return torch.cat([x, pointfeat], 1), trans, trans_feat
def features(self, x): x = F.max_pool2d(F.hardtanh(self.conv1(x)), 2) x = F.max_pool2d(F.hardtanh(self.conv2(x)), 2) x = F.max_pool2d(F.hardtanh(self.conv3(x)), 2) x_tanh = self.conv4(x) x = F.hardtanh(x_tanh) x = x.view(-1, 256) x = self.dropout(x) return x
def sample_z(self, batch_size, sample=True): # Sample the hard-concrete gates for training and use a deterministic value for testing # training if sample: eps = self.get_eps(self.floatTensor(batch_size, self.feature)) z = self.quantile_concrete(eps) return F.hardtanh(z, min_val=0, max_val=1) # testing else: pi = torch.sigmoid(self.qz_loga).view(1, self.feature).expand(batch_size, self.feature) return F.hardtanh(pi * (limit_b - limit_a) + limit_a, min_val=0, max_val=1)
def sample_weight(self): if self.training: z = self.quantile_concrete( self.get_eps(self.floatTensor(self.loga.size()))) mask = F.hardtanh(z, min_val=0, max_val=1) else: pi = torch.sigmoid(self.loga) mask = F.hardtanh(pi * (LIMIT_B - LIMIT_A) + LIMIT_A, min_val=0, max_val=1) return mask * self.weight
def forward(self, x): if self.act_func: x = x * (F.hardtanh(x + 3, 0, 6) / 6) if self.n_lv == 0: return x else: a = F.softplus(self.a) x = x - self.b x = F.hardtanh(x / a, 0, 1) x = RoundQuant.apply(x, self.n_lv) return x
def forward(self, predict, target): num = 0 for i in range(1, len(target)): for j in range(i): #num += torch.sign(predict[i][0] - predict[j][0]) * torch.sign(target[i] - target[j]) num += F.hardtanh(predict[i] - predict[j]) * F.hardtanh(target[i] - target[j]) x = 1 - 2 * num / (len(target) * (len(target) - 1)) #print(torch.autograd.grad(x, predict)) return x
def forward(self, x, blur_sigma=None, epsilon=None): """ Parameters ---------- x : torch.Tensor [n,H,W] pre-conv image probabilities blur_sigma : float | None amount of blur. 'None' means use value from __init__ call Returns ------- x : torch.Tensor [n,H,W] post-conv image probabilities """ if self.is_cuda: x = x.cuda() if blur_sigma is None: H_blur = self.H_blur blur_sigma = self.blur_sigma else: blur_sigma = check_float_tesnor(blur_sigma, self.device) H_blur = blur_filter(self.blur_fsize, blur_sigma, device=self.device) if epsilon is None: epsilon = self.epsilon else: epsilon = check_float_tesnor(epsilon, self.device) # unsqueeze x = x.unsqueeze(1) # apply broaden for i in range(self.nbroad): x = F.conv2d(x, self.H_broaden, padding=1) x = F.hardtanh(x, 0., 1.) # return if no blur if blur_sigma == 0: x = x.squeeze(1) return x # apply blur for i in range(2): x = F.conv2d(x, H_blur, padding=self.blur_pad) x = F.hardtanh(x, 0., 1.) # apply pixel noise if epsilon > 0: x = (1 - epsilon) * x + epsilon * (1 - x) # squeeze x = x.squeeze(1) return x
def effective_W(self): if self.training: z = self.quantile_concrete( self.get_eps( self.floatTensor(self.in_features, self.out_features))) mask = F.hardtanh(z, min_val=0, max_val=1) else: pi = F.sigmoid(self.qz_loga) mask = F.hardtanh(pi * (self.limit_b - self.limit_a) + self.limit_a, min_val=0, max_val=1) return mask * self.weight
def forward(self, input): #print('input shape is: ' + str(input.shape)) (10, 40960) # generate a random matrix #print(self.loss_prob) r = torch.rand((input.shape[0], self.pieces)) > self.loss_prob #print(r) # then extend it to the block random # u = np.concatenate((np.repeat(r.numpy(),self.block_size_y,axis = 1),np.ones((self.pieces,1))),axis = 1) u = torch.tensor(np.repeat(r.numpy(), self.block_size_x, axis=1)).float().cuda() input = input * u (input0, input1, input2, input3, input4) = torch.chunk(input, 5, 1) #print('shape of input0 is ' + str(input0.shape)) s0 = F.linear(input0, self.weight0, self.bias) s1 = F.linear(input1, self.weight1, self.bias) s2 = F.linear(input2, self.weight2, self.bias) s3 = F.linear(input3, self.weight3, self.bias) s4 = F.linear(input4, self.weight4, self.bias) s0 = F.hardtanh(s0, self.lower_bound, self.upper_bound) s1 = F.hardtanh(s1, self.lower_bound, self.upper_bound) s2 = F.hardtanh(s2, self.lower_bound, self.upper_bound) s3 = F.hardtanh(s3, self.lower_bound, self.upper_bound) s4 = F.hardtanh(s4, self.lower_bound, self.upper_bound) s0 = torch.round((s0 - self.lower_bound) / self.delta) * self.delta + self.lower_bound s1 = torch.round((s1 - self.lower_bound) / self.delta) * self.delta + self.lower_bound s2 = torch.round((s2 - self.lower_bound) / self.delta) * self.delta + self.lower_bound s3 = torch.round((s3 - self.lower_bound) / self.delta) * self.delta + self.lower_bound s4 = torch.round((s4 - self.lower_bound) / self.delta) * self.delta + self.lower_bound ''' print(torch.max(s0)) print(torch.min(s0)) print(torch.max(s1)) print(torch.min(s1)) print(torch.max(s2)) print(torch.min(s2)) print(torch.max(s3)) print(torch.min(s3)) print(torch.max(s4)) print(torch.min(s4)) ''' #print('mask shape is ' + str(self.weight.shape)) #print(self.weight.shape) #print(self.block_size_y) #print(self.block_size_x) return s0 + s1 + s2 + s3 + s4
def forward(self, x, meta_net): # out = binary_modules.BinActive().apply(x) out = self.conv1(x) out = self.bn1(out) out = F.hardtanh(out) # out = binary_modules.BinActive().apply(out) out = self.conv2(out) out = self.bn2(out) out += self.shortcut(x) out = F.hardtanh(out) return out
def forward(self, x): if self.act_func: x = x * (F.hardtanh(x + 3, 0, 6) / 6) if len(self.bits) == 1 and self.bits[0] == 32: return x else: a = F.softplus(self.a) c = F.softplus(self.c) x = x + self.b x = F.hardtanh(x / a, 0, 1) x = RoundQuant.apply(x, self.n_lvs) * c x = x + self.d return x
def forward(self, input, lengths=None): "See :obj:`onmt.modules.EncoderBase.forward()`" # (batch_size, 1, nfft, t) # layer 1 input = self.batch_norm1(self.layer1(input[:, :, :, :])) # (batch_size, 32, nfft/2, t/2) input = F.hardtanh(input, 0, 20, inplace=True) # (batch_size, 32, nfft/2/2, t/2) # layer 2 input = self.batch_norm2(self.layer2(input)) # (batch_size, 32, nfft/2/2, t/2) input = F.hardtanh(input, 0, 20, inplace=True) batch_size = input.size(0) length = input.size(3) input = input.view(batch_size, -1, length) input = input.transpose(0, 2).transpose(1, 2) output, hidden = self.rnn(input) return hidden, output