def linearity_induced_loss(y_pred, y, alpha=[1, 1], detach=False): """linearity-induced loss, actually MSE loss with z-score normalization""" if y_pred.size(0) > 1: # z-score normalization: (x-m(x))/sigma(x). sigma_hat, m_hat = torch.std_mean( y_pred.detach(), unbiased=False) if detach else torch.std_mean( y_pred, unbiased=False) y_pred = (y_pred - m_hat) / (sigma_hat + eps) sigma, m = torch.std_mean(y, unbiased=False) y = (y - m) / (sigma + eps) scale = 4 loss0, loss1 = 0, 0 if alpha[0] > 0: loss0 = F.mse_loss(y_pred, y) / scale # ~ 1 - rho, rho is PLCC if alpha[1] > 0: rho = torch.mean(y_pred * y) loss1 = F.mse_loss( rho * y_pred, y ) / scale # 1 - rho ** 2 = 1 - R^2, R^2 is Coefficient of determination # loss0 = (1 - torch.cosine_similarity(y_pred.t() - torch.mean(y_pred), y.t() - torch.mean(y))[0]) / 2 # yp = y_pred.detach() if detach else y_pred # ones = torch.ones_like(yp.detach()) # yp1 = torch.cat((yp, ones), dim=1) # h = torch.mm(torch.inverse(torch.mm(yp1.t(), yp1)), torch.mm(yp1.t(), y)) # err = torch.pow(torch.mm(torch.cat((y_pred, ones), dim=1), h) - y, 2) # # normalization = 1 + torch.max(err.detach()) if detach else 1 + torch.max(err) # loss1 = torch.mean(err) / normalization return (alpha[0] * loss0 + alpha[1] * loss1) / (alpha[0] + alpha[1]) else: return F.l1_loss(y_pred, y_pred.detach()) # 0 for batch with single sample.
def forward(self, x): assert x.ndim == 3, "Input needs to be tensor of shape B x T x D" n_batch, timedim, featdim = x.shape with torch.no_grad(): # Too small utterance, just normalize per time-step if timedim < self.kernel_size: v, m = torch.std_mean(x, dim=1, keepdims=True) return (x - m) / (v + self.eps) else: sliding_window = F.pad( x.transpose(1, 2), (self.kernel_size // 2, self.kernel_size // 2 - 1), mode='reflect').unfold(-1, self.kernel_size, 1).transpose(1, 2) if self.with_mean and self.with_std: v, m = torch.std_mean(sliding_window, dim=-1) return (x - m) / (v + self.eps) elif self.with_mean: m = sliding_window.mean(-1) return (x - m) elif self.with_std: v = sliding_window.std(-1) return x / (v + self.eps) else: return x # no normalization done
def model_stats(model): weight_array = None bias_array = None for name, parameter in model.named_parameters(): if not parameter.requires_grad: continue param = parameter.numel() array = torch.flatten(parameter) if name.split('.')[-1] == 'weight': if weight_array is None: weight_array = array else: weight_array = torch.cat((weight_array, array)) elif name.split('.')[-1] == 'bias': if bias_array is None: bias_array = array else: bias_array = torch.cat((bias_array, array)) weight_std, weight_mean = torch.std_mean(weight_array, unbiased=False) bias_std, bias_mean = torch.std_mean(bias_array, unbiased=False) return weight_std, weight_mean, bias_std, bias_mean
def preprocess_poses(cls, poses: tuple): """Generates (N, 6) vector of absolute poses Args: Tuple of batched rotations (N, 3, 3) and translations (N, 3) in Pytorch3d view-to-world coordinates. usually returned from a call to RenderManager._trajectory More information about Pytorch3D's coordinate system: https://github.com/facebookresearch/pytorch3d/blob/master/docs/notes/cameras.md 1. Computes rotation and translation matrices in view-to-world coordinates. 2. Generates unit quaternion from R and computes log q repr 3. Normalizes translation according to mean and stdev Returns: (N, 6) vector: [t1, t2, t3, logq1, logq2, logq3] """ R, T = poses cam_wvt = get_world_to_view_transform(R=R, T=T) pose_transform = cam_wvt.inverse().get_matrix() T = pose_transform[:, 3, :3] R = pose_transform[:, :3, :3] # Compute pose stats std_R, mean_R = torch.std_mean(R) std_T, mean_T = torch.std_mean(T) q = rc.matrix_to_quaternion(R) # q /= torch.norm(q) # q *= torch.sign(q[0]) # hemisphere constraint # logq = qlog(q) T -= mean_T T /= std_T return torch.cat((T, q), dim=1)
def get_mean_std(data): r = data[:, 0, :, :] g = data[:, 1, :, :] b = data[:, 2, :, :] r_std, r_mean = torch.std_mean(r) g_std, g_mean = torch.std_mean(g) b_std, b_mean = torch.std_mean(b) return (r_mean, g_mean, b_mean), (r_std, g_std, b_std)
def _loss(self, x: Tensor, target: Tensor): assert target.shape[2] % self.bin_size == 0, "The number of columns in the image can not be evenly divided into bins of size {}!".format( self.bin_size) bins_truth = torch.stack(target.split(self.bin_size, dim=-1)) bins_x = torch.stack(target.split(self.bin_size, dim=-1)) std_truth, mean_truth = torch.std_mean(bins_truth, dim=(1, 2, 3)) std_x, mean_x = torch.std_mean(bins_x, dim=(1, 2, 3)) return mse_loss(mean_x, mean_truth)
def forward(self, x, y): px_std, px_avg = torch.std_mean(self.att0(x), dim=(2, 3), keepdim=True) py_std, py_avg = torch.std_mean(self.att0(x), dim=(2, 3), keepdim=True) px = self.std_avg(torch.cat([px_std, px_avg], dim=2)) py = self.std_avg(torch.cat([py_std, py_avg], dim=2)) pf = torch.cat([py, px, py, px], dim=2) pw = self.conv(pf) * 2 fea_map = torch.cat([x.unsqueeze(2), y.unsqueeze(2)], dim=2) * pw.unsqueeze(-1) return torch.sum(fea_map, dim=2)
def cascading_randomization(intr, method, forward_function, target, inp_image, inp_transform_flag, transform_func, uncertainity_mode, visualize_method, sign, show_colorbar, title, device, library, output_path, input_file, is_3d, isDepthFirst, patcher_flag, batch_dim_present, **kwargs): forward_func = copy.deepcopy(forward_function) layers = list(forward_func.children()) if len(layers) == 1: layers = [l for l in layers[0].children()] layers.reverse() idx = 1 uncertainity_metrics = [] for layer in layers: for name, param in (layer.named_parameters()): forward_func.eval() std = torch.std_mean(param)[0].item() avg = torch.std_mean(param)[1].item() if np.isnan(std): std = 0 if np.isnan(avg): avg = 0 torch.nn.init.uniform_(param, avg - std, avg + std) forward_func.eval() if "nt_type" in kwargs.keys(): kwargs["nt_type"] = False uncertain_metric = getattr(intr, method)( forward_func, target, inp_image, inp_transform_flag, transform_func, True if uncertainity_mode == 2 else False, visualize_method, sign, show_colorbar, title, device, library, output_path, input_file, is_3d, isDepthFirst, patcher_flag, batch_dim_present, **kwargs, name_tag="_CasRand" + "_" + str(idx)) uncertainity_metrics.append(uncertain_metric) idx += 1 return uncertainity_metrics
def forward(self, input): x = self.projection_layer(input) x = nn.functional.softsign(x) x = torch.mean(x, dim=2, keepdim=True) x = self.weight_layer(x) weights = torch.sigmoid(x) self.counter += 1 if self.counter % 200 == 0: print('GA1', weights.shape, [x.item() for x in torch.std_mean(weights)]) print('GA2', [x.item() for x in torch.std_mean(torch.mean(weights, dim=(0, 2)))]) return input * weights
def _loss(self, batch): state, value, variance, policy, weight = batch if type(state) is not torch.Tensor: state = torch.from_numpy(batch[0]) value = torch.from_numpy(batch[1]) variance = torch.from_numpy(batch[2]) policy = torch.from_numpy(batch[3]) weight = torch.from_numpy(batch[4]) if self.use_cuda: state = state.cuda() value = value.cuda() variance = variance.cuda() policy = policy.cuda() weight = weight.cuda() _v, _var, _p = self.model(state) #print(_var.min().item(), variance.min().item(), flush=True) if self.loss_type == 'mle': loss = torch.std_mean(weight * self.l_func(_var, _v, variance, value)) return defaultdict(float, loss=loss[1], loss_std=loss[0]) elif self.loss_type == 'kldiv': loss = torch.std_mean(self.l_func(_var, _v, variance, value)) return defaultdict(float, loss=loss[1], loss_std=loss[0]) else: loss_v = self.l_func(_v, value) if self.use_variance: loss_var = self.l_func(_var, variance) else: loss_var = torch.FloatTensor([0]) if self.weighted: loss_v = weight * loss_v loss_var = weight * loss_var loss_v = loss_v.mean() loss_var = loss_var.mean() if self.use_policy: loss_p = F.kl_div(torch.log(_p), policy) else: loss_p = torch.FloatTensor([0]) loss = loss_v + loss_var + loss_p return defaultdict(float, loss=loss, loss_v=loss_v, loss_var=loss_var, loss_p=loss_p)
def projection_criterion(train_res, test_res, round): std_train, mean_train = torch.std_mean(train_res, dim=0) std_test, mean_test = torch.std_mean(test_res, dim=0) diff_mean = torch.dist(mean_train, mean_test) diff_std = torch.dist(std_train, std_test, p=1) if round % 60 == 0: print("projection criterion loss") print(diff_mean.cpu().data.numpy()) print(diff_std.cpu().data.numpy()) return 10 * diff_mean + diff_std
def zncc(images, templates): assert len( images.shape ) == 4, 'Expecting images with shape (idx, channels, height, width)' assert images.shape == templates.shape, 'images.shape must match templates.shape' istd, imean = torch.std_mean(images, (2, 3)) tstd, tmean = torch.std_mean(templates, (2, 3)) total = 0 for i_idx, (img, tmpl) in enumerate(zip(images, templates)): for c_idx, (i_chan, t_chan) in enumerate(zip(img, tmpl)): result = (i_chan - imean[i_idx][c_idx]) * (t_chan - tmean[i_idx][c_idx]) result = result.sum() / (istd[i_idx][c_idx] * tstd[i_idx][c_idx]) total = total + result return total / (reduce(lambda acc, val: acc * val, images.shape))
def _update_recurrent(self, dataset): """Update both the policy and the value function.""" device = self.device flat_dataset = list(itertools.chain.from_iterable(dataset)) if self.obs_normalizer: self._update_obs_normalizer(flat_dataset) assert "state" in flat_dataset[0] assert "v_teacher" in flat_dataset[0] if self.standardize_advantages: all_advs = torch.tensor([b["adv"] for b in flat_dataset], device=device) std_advs, mean_advs = torch.std_mean(all_advs, unbiased=False) else: mean_advs = None std_advs = None for _ in range(self.epochs): random.shuffle(dataset) for minibatch in _yield_subset_of_sequences_with_fixed_number_of_items( dataset, self.minibatch_size): self._update_once_recurrent(minibatch, mean_advs, std_advs)
def extract_patches(pyramid_layer, slice_indices, slice_size=7, unfold_batch_size=128, device="cpu"): assert pyramid_layer.ndim == 4 n = pyramid_layer.size(0) // unfold_batch_size + np.sign( pyramid_layer.size(0) % unfold_batch_size) # random slice 7x7 p_slice = [] for i in range(n): # [unfold_batch_size, ch, n_slices, slice_size, slice_size] ind_start = i * unfold_batch_size ind_end = min((i + 1) * unfold_batch_size, pyramid_layer.size(0)) x = pyramid_layer[ind_start:ind_end].unfold(2, slice_size, 1).unfold( 3, slice_size, 1).reshape(ind_end - ind_start, pyramid_layer.size(1), -1, slice_size, slice_size) # [unfold_batch_size, ch, n_descriptors, slice_size, slice_size] x = x[:, :, slice_indices, :, :] # [unfold_batch_size, n_descriptors, ch, slice_size, slice_size] p_slice.append(x.permute([0, 2, 1, 3, 4])) # sliced tensor per layer [batch, n_descriptors, ch, slice_size, slice_size] x = torch.cat(p_slice, dim=0) # normalize along ch std, mean = torch.std_mean(x, dim=(0, 1, 3, 4), keepdim=True) x = (x - mean) / (std + 1e-8) # reshape to 2rank x = x.reshape(-1, 3 * slice_size * slice_size) return x
def compress(tensor, name=None, ratio=0.05, tb=None, i_ratio=0.25, stages=1, ec_grad_w=1.0, ec_mem_w=0.0): with torch.no_grad(): numel = tensor.numel() k = max(int(numel * ratio), 1) t_norm = tensor.norm(2) GaussianKSGDCompressorEC.norm = t_norm norm_tensor = tensor / t_norm abs_norm_tensor = norm_tensor.abs() abs_norm_tensor = tensor # TODO:REMOVE if torch.__version__ < '1.3.0': t_std = torch.std(abs_norm_tensor) t_mean = torch.mean(abs_norm_tensor) else: t_std, t_mean = torch.std_mean(abs_norm_tensor) left_thres, right_thres = utils.gen_threshold_from_normal_distribution(1 - ratio, float(t_mean), float(t_std)) loops = 0 while loops < 5: one_indexes = abs_norm_tensor > right_thres indexes = one_indexes.nonzero().data.squeeze().view(-1) if indexes.numel() < 2 * k / 3: right_thres *= 0.5 elif indexes.numel() > 4 * k / 3: right_thres *= 1.5 else: break loops += 1 values = tensor.data[indexes] return tensor, indexes, values
def forward(self, content, gamma, beta): """ Args: content (torch.tensor): (b, c, h, w) gamma (torch.tensor): (b, c1, h, w) beta (torch.tensor): (b, c2, h, w) Returns: """ b, c, h, w = content.size() # (b, 1, h, w) content_std, content_mean = torch.std_mean(content, dim=1, keepdim=True) normalized_content = (content - content_mean) / (content_std + self.eps) stylized_content = (normalized_content * gamma) + beta if self.style_fact != 1.0: output = (1 - self.style_fact ) * content + self.style_fact * stylized_content else: output = stylized_content return output
def get_weight(self): if self.use_layernorm: weight = self.scale * F.layer_norm( self.weight, self.weight.shape[1:], eps=self.eps) else: std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False) weight = self.scale * (self.weight - mean) / (std + self.eps) scaling_factor = torch.mean(torch.mean(torch.mean(abs(weight), dim=3, keepdim=True), dim=2, keepdim=True), dim=1, keepdim=True) scaling_factor = scaling_factor.detach() binary_weights_no_grad = scaling_factor * torch.sign(weight) cliped_weights = torch.clamp(weight, -1.0, 1.0) binary_weights = binary_weights_no_grad.detach( ) - cliped_weights.detach() + cliped_weights return self.gain * binary_weights
def get_item_from_index(self, index): seq_file, label = self.sequence_files[index].strip().split(" ") with open(seq_file, "r") as file: images = file.readlines() temp = PIL.Image.open(images[0].strip()) temp = transforms.Resize(150)(temp) temp = np.array(temp) tensor_size = (self.z, ) + (temp.shape[-1], ) + (temp.shape[0:-1]) img_sequence = torch.empty(tensor_size, dtype=torch.float) for i in range(self.z): img = PIL.Image.open(images[i].strip()) img = transforms.Resize(150)(img) img_tensor = transforms.ToTensor()(img) # std, mean = torch.std_mean(img_tensor, dim=[1, 2]) # img_tensor = transforms.Normalize(mean, std)(img_tensor) img_sequence[i] = img_tensor img_sequence = img_sequence.permute(1, 2, 3, 0) std, mean = torch.std_mean(img_sequence) normalize = ms.Normalize(mean, std) img_sequence = normalize(img_sequence) # -1 for label as our labels are 1-35 # but we want 0-34 label = float(label) - 1. return img_sequence, label
def parse_audio(self, audio_path, mask=False): if isinstance(audio_path, str) and not os.path.exists(path=audio_path): raise Exception("音频路径不存在 " + audio_path) y, sr = torchaudio.load(audio_path) # # dither y += 1e-5 * torch.randn_like(y) # do preemphasis y = torch.cat( (y[:, 0].unsqueeze(1), y[:, 1:] - 0.97 * y[:, :-1]), dim=1, ) if mask: y = self.sub_secquence(y, weight=0.98) spec = self.mel_transformer(y) y = self.amplitudeToDB(spec) # F-T mask if mask: # y = self.sample_aug(y, 0.2) y = self.spec_augment(y, freq_mask=27, time_mask=0.07) # y = self.audio_f_mask(y) # y = self.audio_t_mask(y) # cutout # y = self.cutout(y) # 归一化 std, mean = torch.std_mean(y) y = torch.div((y - mean), std) return y # (1,64,T)
def _update_policy(self, batch: TrainingBatch, adv: torch.Tensor): if self.standardize_advantages: std_adv, mean_adv = torch.std_mean(adv, unbiased=False) adv = (adv - mean_adv) / (std_adv + 1e-8) action_distrib: distributions.Distribution = self.policy(batch.state) # Distribution to compute KL div against with torch.no_grad(): # torch.distributions.Distribution cannot be deepcopied action_distrib_old: distributions.Distribution = self.policy( batch.state) log_prob_old = action_distrib_old.log_prob(batch.action) gain = self._compute_gain( log_prob=action_distrib.log_prob(batch.action), log_prob_old=log_prob_old, entropy=action_distrib.entropy(), adv=adv, ) full_step = self._compute_kl_constrained_step( action_distrib=action_distrib, action_distrib_old=action_distrib_old, gain=gain, ) self._line_search( full_step=full_step, batch=batch, adv=adv, action_distrib_old=action_distrib_old, log_prob_old=log_prob_old, gain=gain, )
def get_agreement(samples, labels, topk=1, max_err=False): """Get Top-k agreement of samples on some samples.""" reduced = get_matching(samples, labels, topk, max_err) # has shape (num_f, num_eps) std, mean = torch.std_mean(reduced, dim=-1) return mean, std
def __getitem__(self,idx): if idx>=self.__len__(): raise IndexError imgid=idx%self.imgfilelength imgxname=self.fns[imgid] image = Image.open(imgxname).convert(self.imgmode) #toto randomsample is not a clever it should guided the postion to crop is more efficient I think maxstd=0 tmpsample=None for i in range(random.randint(1,30)): sampleimg=transforms.RandomCrop(self.patchsize)(image) std,mean=tc.std_mean(transforms.ToTensor()(sampleimg)) if std>maxstd: maxstd=std tmpsample=sampleimg sampleimg=tmpsample if tmpsample else sampleimg #do random rotation rotationtypes=['0',Image.ROTATE_90,Image.ROTATE_180,Image.ROTATE_270] transtype=random.choice(rotationtypes) if transtype !='0': patchx = sampleimg.transpose(transtype) else: patchx=sampleimg patchxtensor=transforms.ToTensor()(patchx) #to tensor c x h x w -0.5 --- 0.5 patchy = transforms.RandomVerticalFlip(p=0.5)(sampleimg) patchy = transforms.RandomHorizontalFlip(p=0.5)(patchy) patchytensor=transforms.ToTensor()(patchy) return patchxtensor,patchytensor #returns a random patch
def get_wiki_cs(transform=RowFeatNormalizer(subtract_min=True)): dataset = WikiCSDataset(transform=transform) g = dataset[0] std, mean = torch.std_mean(g.ndata['feat'], dim=0, unbiased=False) g.ndata['feat'] = (g.ndata['feat'] - mean) / std return [g]
def get_weight(self): if self.use_layernorm: weight = self.scale * F.layer_norm(self.weight, self.weight.shape[1:], eps=self.eps) else: std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False) weight = self.scale * (self.weight - mean) / (std + self.eps) return self.gain * weight
def calc_mean_and_std(mt_df, data_dir, num_batches, device): train_data = dataset.CovidDataset(mt_df, root_dir=data_dir, target_channel=None, for_data_statistics_calc=True) batch_size = int(len(train_data) / num_batches) train_loader = DataLoader(train_data, batch_size=batch_size) num_channels = len(data_layer.channels.Channels) mean = torch.zeros(num_channels).to(device) std = torch.zeros(num_channels).to(device) max_p = 0 min_p = 65535 for images in train_loader: images = images.to(device) # TODO: Divide by maximum value was removed batch_mean, batch_std = torch.std_mean(images.float(), dim=(0, 1, 2)) max_p = max(torch.max(images.float()), max_p) min_p = min(torch.min(images).float(), min_p) mean += batch_mean std += batch_std mean /= num_batches std /= num_batches print('mean of train data is ' + str(mean.tolist())) print('std of train data is ' + str(std.tolist())) print('maximum of train data is ' + str(max_p.tolist())) print('minimum of train data is ' + str(min_p.tolist())) return mean.tolist(), std.tolist()
def get_weight(module): std, mean = torch.std_mean(module.weight, dim=[1, 2, 3], keepdim=True, unbiased=False) weight = (module.weight - mean) / (std + module.eps) return weight
def reduction_ops(self): a = torch.randn(4) b = torch.randn(4) return ( torch.argmax(a), torch.argmin(a), torch.amax(a), torch.amin(a), torch.aminmax(a), torch.all(a), torch.any(a), torch.max(a), torch.min(a), torch.dist(a, b), torch.logsumexp(a, 0), torch.mean(a), torch.nanmean(a), torch.median(a), torch.nanmedian(a), torch.mode(a), torch.norm(a), torch.nansum(a), torch.prod(a), torch.quantile(a, torch.tensor([0.25, 0.5, 0.75])), torch.nanquantile(a, torch.tensor([0.25, 0.5, 0.75])), torch.std(a), torch.std_mean(a), torch.sum(a), torch.unique(a), torch.unique_consecutive(a), torch.var(a), torch.var_mean(a), torch.count_nonzero(a), )
def get_weight(self): std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False) weight = (self.weight - mean) / (std + self.eps) return weight
def _normalize_input(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: "Utility function that normalizes the input by batch." "" sp, mp = torch.std_mean(x, dim=(-3, -2, -1), keepdim=True) # WARNING: we need to .detach() input, otherwise the gradients produced by # the patches extractor with F.grid_sample are very noisy, making the detector # training totally unstable. return (x - mp.detach()) / (sp.detach() + eps)
def adain_normalization(features): epsilon = 1e-5 colorization_kernels, mean_features = torch.std_mean(features, (2, 3), keepdim=True) normalized_features = torch.div(features - mean_features, epsilon + colorization_kernels) return normalized_features, colorization_kernels, mean_features