def linearity_induced_loss(y_pred, y, alpha=[1, 1], detach=False):
"""linearity-induced loss, actually MSE loss with z-score normalization"""
if y_pred.size(0) > 1: # z-score normalization: (x-m(x))/sigma(x).
sigma_hat, m_hat = torch.std_mean(
y_pred.detach(), unbiased=False) if detach else torch.std_mean(
y_pred, unbiased=False)
y_pred = (y_pred - m_hat) / (sigma_hat + eps)
sigma, m = torch.std_mean(y, unbiased=False)
y = (y - m) / (sigma + eps)
scale = 4
loss0, loss1 = 0, 0
if alpha[0] > 0:
loss0 = F.mse_loss(y_pred, y) / scale # ~ 1 - rho, rho is PLCC
if alpha[1] > 0:
rho = torch.mean(y_pred * y)
loss1 = F.mse_loss(
rho * y_pred, y
) / scale # 1 - rho ** 2 = 1 - R^2, R^2 is Coefficient of determination
# loss0 = (1 - torch.cosine_similarity(y_pred.t() - torch.mean(y_pred), y.t() - torch.mean(y))[0]) / 2
# yp = y_pred.detach() if detach else y_pred
# ones = torch.ones_like(yp.detach())
# yp1 = torch.cat((yp, ones), dim=1)
# h = torch.mm(torch.inverse(torch.mm(yp1.t(), yp1)), torch.mm(yp1.t(), y))
# err = torch.pow(torch.mm(torch.cat((y_pred, ones), dim=1), h) - y, 2) #
# normalization = 1 + torch.max(err.detach()) if detach else 1 + torch.max(err)
# loss1 = torch.mean(err) / normalization
return (alpha[0] * loss0 + alpha[1] * loss1) / (alpha[0] + alpha[1])
else:
return F.l1_loss(y_pred,
y_pred.detach()) # 0 for batch with single sample.
def forward(self, x):
assert x.ndim == 3, "Input needs to be tensor of shape B x T x D"
n_batch, timedim, featdim = x.shape
with torch.no_grad():
# Too small utterance, just normalize per time-step
if timedim < self.kernel_size:
v, m = torch.std_mean(x, dim=1, keepdims=True)
return (x - m) / (v + self.eps)
else:
sliding_window = F.pad(
x.transpose(1, 2),
(self.kernel_size // 2, self.kernel_size // 2 - 1),
mode='reflect').unfold(-1, self.kernel_size,
1).transpose(1, 2)
if self.with_mean and self.with_std:
v, m = torch.std_mean(sliding_window, dim=-1)
return (x - m) / (v + self.eps)
elif self.with_mean:
m = sliding_window.mean(-1)
return (x - m)
elif self.with_std:
v = sliding_window.std(-1)
return x / (v + self.eps)
else:
return x # no normalization done
def model_stats(model):
weight_array = None
bias_array = None
for name, parameter in model.named_parameters():
if not parameter.requires_grad: continue
param = parameter.numel()
array = torch.flatten(parameter)
if name.split('.')[-1] == 'weight':
if weight_array is None:
weight_array = array
else:
weight_array = torch.cat((weight_array, array))
elif name.split('.')[-1] == 'bias':
if bias_array is None:
bias_array = array
else:
bias_array = torch.cat((bias_array, array))
weight_std, weight_mean = torch.std_mean(weight_array, unbiased=False)
bias_std, bias_mean = torch.std_mean(bias_array, unbiased=False)
return weight_std, weight_mean, bias_std, bias_mean
def preprocess_poses(cls, poses: tuple):
"""Generates (N, 6) vector of absolute poses
Args:
Tuple of batched rotations (N, 3, 3) and translations (N, 3) in Pytorch3d view-to-world coordinates. usually returned from a call to RenderManager._trajectory
More information about Pytorch3D's coordinate system: https://github.com/facebookresearch/pytorch3d/blob/master/docs/notes/cameras.md
1. Computes rotation and translation matrices in view-to-world coordinates.
2. Generates unit quaternion from R and computes log q repr
3. Normalizes translation according to mean and stdev
Returns:
(N, 6) vector: [t1, t2, t3, logq1, logq2, logq3]
"""
R, T = poses
cam_wvt = get_world_to_view_transform(R=R, T=T)
pose_transform = cam_wvt.inverse().get_matrix()
T = pose_transform[:, 3, :3]
R = pose_transform[:, :3, :3]
# Compute pose stats
std_R, mean_R = torch.std_mean(R)
std_T, mean_T = torch.std_mean(T)
q = rc.matrix_to_quaternion(R)
# q /= torch.norm(q)
# q *= torch.sign(q[0]) # hemisphere constraint
# logq = qlog(q)
T -= mean_T
T /= std_T
return torch.cat((T, q), dim=1)
def get_mean_std(data):
r = data[:, 0, :, :]
g = data[:, 1, :, :]
b = data[:, 2, :, :]
r_std, r_mean = torch.std_mean(r)
g_std, g_mean = torch.std_mean(g)
b_std, b_mean = torch.std_mean(b)
return (r_mean, g_mean, b_mean), (r_std, g_std, b_std)
def _loss(self, x: Tensor, target: Tensor):
assert target.shape[2] % self.bin_size == 0, "The number of columns in the image can not be evenly divided into bins of size {}!".format(
self.bin_size)
bins_truth = torch.stack(target.split(self.bin_size, dim=-1))
bins_x = torch.stack(target.split(self.bin_size, dim=-1))
std_truth, mean_truth = torch.std_mean(bins_truth, dim=(1, 2, 3))
std_x, mean_x = torch.std_mean(bins_x, dim=(1, 2, 3))
return mse_loss(mean_x, mean_truth)
def forward(self, x, y):
px_std, px_avg = torch.std_mean(self.att0(x), dim=(2, 3), keepdim=True)
py_std, py_avg = torch.std_mean(self.att0(x), dim=(2, 3), keepdim=True)
px = self.std_avg(torch.cat([px_std, px_avg], dim=2))
py = self.std_avg(torch.cat([py_std, py_avg], dim=2))
pf = torch.cat([py, px, py, px], dim=2)
pw = self.conv(pf) * 2
fea_map = torch.cat([x.unsqueeze(2), y.unsqueeze(2)],
dim=2) * pw.unsqueeze(-1)
return torch.sum(fea_map, dim=2)
def cascading_randomization(intr, method, forward_function, target, inp_image,
inp_transform_flag, transform_func,
uncertainity_mode, visualize_method, sign,
show_colorbar, title, device, library, output_path,
input_file, is_3d, isDepthFirst, patcher_flag,
batch_dim_present, **kwargs):
forward_func = copy.deepcopy(forward_function)
layers = list(forward_func.children())
if len(layers) == 1:
layers = [l for l in layers[0].children()]
layers.reverse()
idx = 1
uncertainity_metrics = []
for layer in layers:
for name, param in (layer.named_parameters()):
forward_func.eval()
std = torch.std_mean(param)[0].item()
avg = torch.std_mean(param)[1].item()
if np.isnan(std): std = 0
if np.isnan(avg): avg = 0
torch.nn.init.uniform_(param, avg - std, avg + std)
forward_func.eval()
if "nt_type" in kwargs.keys():
kwargs["nt_type"] = False
uncertain_metric = getattr(intr, method)(
forward_func,
target,
inp_image,
inp_transform_flag,
transform_func,
True if uncertainity_mode == 2 else False,
visualize_method,
sign,
show_colorbar,
title,
device,
library,
output_path,
input_file,
is_3d,
isDepthFirst,
patcher_flag,
batch_dim_present,
**kwargs,
name_tag="_CasRand" + "_" + str(idx))
uncertainity_metrics.append(uncertain_metric)
idx += 1
return uncertainity_metrics
def forward(self, input):
x = self.projection_layer(input)
x = nn.functional.softsign(x)
x = torch.mean(x, dim=2, keepdim=True)
x = self.weight_layer(x)
weights = torch.sigmoid(x)
self.counter += 1
if self.counter % 200 == 0:
print('GA1', weights.shape, [x.item() for x in torch.std_mean(weights)])
print('GA2', [x.item() for x in torch.std_mean(torch.mean(weights, dim=(0, 2)))])
return input * weights
def _loss(self, batch):
state, value, variance, policy, weight = batch
if type(state) is not torch.Tensor:
state = torch.from_numpy(batch[0])
value = torch.from_numpy(batch[1])
variance = torch.from_numpy(batch[2])
policy = torch.from_numpy(batch[3])
weight = torch.from_numpy(batch[4])
if self.use_cuda:
state = state.cuda()
value = value.cuda()
variance = variance.cuda()
policy = policy.cuda()
weight = weight.cuda()
_v, _var, _p = self.model(state)
#print(_var.min().item(), variance.min().item(), flush=True)
if self.loss_type == 'mle':
loss = torch.std_mean(weight *
self.l_func(_var, _v, variance, value))
return defaultdict(float, loss=loss[1], loss_std=loss[0])
elif self.loss_type == 'kldiv':
loss = torch.std_mean(self.l_func(_var, _v, variance, value))
return defaultdict(float, loss=loss[1], loss_std=loss[0])
else:
loss_v = self.l_func(_v, value)
if self.use_variance:
loss_var = self.l_func(_var, variance)
else:
loss_var = torch.FloatTensor([0])
if self.weighted:
loss_v = weight * loss_v
loss_var = weight * loss_var
loss_v = loss_v.mean()
loss_var = loss_var.mean()
if self.use_policy:
loss_p = F.kl_div(torch.log(_p), policy)
else:
loss_p = torch.FloatTensor([0])
loss = loss_v + loss_var + loss_p
return defaultdict(float,
loss=loss,
loss_v=loss_v,
loss_var=loss_var,
loss_p=loss_p)
def projection_criterion(train_res, test_res, round):
std_train, mean_train = torch.std_mean(train_res, dim=0)
std_test, mean_test = torch.std_mean(test_res, dim=0)
diff_mean = torch.dist(mean_train, mean_test)
diff_std = torch.dist(std_train, std_test, p=1)
if round % 60 == 0:
print("projection criterion loss")
print(diff_mean.cpu().data.numpy())
print(diff_std.cpu().data.numpy())
return 10 * diff_mean + diff_std
def zncc(images, templates):
assert len(
images.shape
) == 4, 'Expecting images with shape (idx, channels, height, width)'
assert images.shape == templates.shape, 'images.shape must match templates.shape'
istd, imean = torch.std_mean(images, (2, 3))
tstd, tmean = torch.std_mean(templates, (2, 3))
total = 0
for i_idx, (img, tmpl) in enumerate(zip(images, templates)):
for c_idx, (i_chan, t_chan) in enumerate(zip(img, tmpl)):
result = (i_chan - imean[i_idx][c_idx]) * (t_chan -
tmean[i_idx][c_idx])
result = result.sum() / (istd[i_idx][c_idx] * tstd[i_idx][c_idx])
total = total + result
return total / (reduce(lambda acc, val: acc * val, images.shape))
def _update_recurrent(self, dataset):
"""Update both the policy and the value function."""
device = self.device
flat_dataset = list(itertools.chain.from_iterable(dataset))
if self.obs_normalizer:
self._update_obs_normalizer(flat_dataset)
assert "state" in flat_dataset[0]
assert "v_teacher" in flat_dataset[0]
if self.standardize_advantages:
all_advs = torch.tensor([b["adv"] for b in flat_dataset],
device=device)
std_advs, mean_advs = torch.std_mean(all_advs, unbiased=False)
else:
mean_advs = None
std_advs = None
for _ in range(self.epochs):
random.shuffle(dataset)
for minibatch in _yield_subset_of_sequences_with_fixed_number_of_items(
dataset, self.minibatch_size):
self._update_once_recurrent(minibatch, mean_advs, std_advs)
def extract_patches(pyramid_layer,
slice_indices,
slice_size=7,
unfold_batch_size=128,
device="cpu"):
assert pyramid_layer.ndim == 4
n = pyramid_layer.size(0) // unfold_batch_size + np.sign(
pyramid_layer.size(0) % unfold_batch_size)
# random slice 7x7
p_slice = []
for i in range(n):
# [unfold_batch_size, ch, n_slices, slice_size, slice_size]
ind_start = i * unfold_batch_size
ind_end = min((i + 1) * unfold_batch_size, pyramid_layer.size(0))
x = pyramid_layer[ind_start:ind_end].unfold(2, slice_size, 1).unfold(
3, slice_size,
1).reshape(ind_end - ind_start, pyramid_layer.size(1), -1,
slice_size, slice_size)
# [unfold_batch_size, ch, n_descriptors, slice_size, slice_size]
x = x[:, :, slice_indices, :, :]
# [unfold_batch_size, n_descriptors, ch, slice_size, slice_size]
p_slice.append(x.permute([0, 2, 1, 3, 4]))
# sliced tensor per layer [batch, n_descriptors, ch, slice_size, slice_size]
x = torch.cat(p_slice, dim=0)
# normalize along ch
std, mean = torch.std_mean(x, dim=(0, 1, 3, 4), keepdim=True)
x = (x - mean) / (std + 1e-8)
# reshape to 2rank
x = x.reshape(-1, 3 * slice_size * slice_size)
return x
def compress(tensor, name=None, ratio=0.05, tb=None, i_ratio=0.25, stages=1, ec_grad_w=1.0, ec_mem_w=0.0):
with torch.no_grad():
numel = tensor.numel()
k = max(int(numel * ratio), 1)
t_norm = tensor.norm(2)
GaussianKSGDCompressorEC.norm = t_norm
norm_tensor = tensor / t_norm
abs_norm_tensor = norm_tensor.abs()
abs_norm_tensor = tensor # TODO:REMOVE
if torch.__version__ < '1.3.0':
t_std = torch.std(abs_norm_tensor)
t_mean = torch.mean(abs_norm_tensor)
else:
t_std, t_mean = torch.std_mean(abs_norm_tensor)
left_thres, right_thres = utils.gen_threshold_from_normal_distribution(1 - ratio, float(t_mean), float(t_std))
loops = 0
while loops < 5:
one_indexes = abs_norm_tensor > right_thres
indexes = one_indexes.nonzero().data.squeeze().view(-1)
if indexes.numel() < 2 * k / 3:
right_thres *= 0.5
elif indexes.numel() > 4 * k / 3:
right_thres *= 1.5
else:
break
loops += 1
values = tensor.data[indexes]
return tensor, indexes, values
def forward(self, content, gamma, beta):
"""
Args:
content (torch.tensor): (b, c, h, w)
gamma (torch.tensor): (b, c1, h, w)
beta (torch.tensor): (b, c2, h, w)
Returns:
"""
b, c, h, w = content.size()
# (b, 1, h, w)
content_std, content_mean = torch.std_mean(content,
dim=1,
keepdim=True)
normalized_content = (content - content_mean) / (content_std +
self.eps)
stylized_content = (normalized_content * gamma) + beta
if self.style_fact != 1.0:
output = (1 - self.style_fact
) * content + self.style_fact * stylized_content
else:
output = stylized_content
return output
def get_weight(self):
if self.use_layernorm:
weight = self.scale * F.layer_norm(
self.weight, self.weight.shape[1:], eps=self.eps)
else:
std, mean = torch.std_mean(self.weight,
dim=[1, 2, 3],
keepdim=True,
unbiased=False)
weight = self.scale * (self.weight - mean) / (std + self.eps)
scaling_factor = torch.mean(torch.mean(torch.mean(abs(weight),
dim=3,
keepdim=True),
dim=2,
keepdim=True),
dim=1,
keepdim=True)
scaling_factor = scaling_factor.detach()
binary_weights_no_grad = scaling_factor * torch.sign(weight)
cliped_weights = torch.clamp(weight, -1.0, 1.0)
binary_weights = binary_weights_no_grad.detach(
) - cliped_weights.detach() + cliped_weights
return self.gain * binary_weights
def get_item_from_index(self, index):
seq_file, label = self.sequence_files[index].strip().split(" ")
with open(seq_file, "r") as file:
images = file.readlines()
temp = PIL.Image.open(images[0].strip())
temp = transforms.Resize(150)(temp)
temp = np.array(temp)
tensor_size = (self.z, ) + (temp.shape[-1], ) + (temp.shape[0:-1])
img_sequence = torch.empty(tensor_size, dtype=torch.float)
for i in range(self.z):
img = PIL.Image.open(images[i].strip())
img = transforms.Resize(150)(img)
img_tensor = transforms.ToTensor()(img)
# std, mean = torch.std_mean(img_tensor, dim=[1, 2])
# img_tensor = transforms.Normalize(mean, std)(img_tensor)
img_sequence[i] = img_tensor
img_sequence = img_sequence.permute(1, 2, 3, 0)
std, mean = torch.std_mean(img_sequence)
normalize = ms.Normalize(mean, std)
img_sequence = normalize(img_sequence)
# -1 for label as our labels are 1-35
# but we want 0-34
label = float(label) - 1.
return img_sequence, label
def parse_audio(self, audio_path, mask=False):
if isinstance(audio_path, str) and not os.path.exists(path=audio_path):
raise Exception("音频路径不存在 " + audio_path)
y, sr = torchaudio.load(audio_path)
# # dither
y += 1e-5 * torch.randn_like(y)
# do preemphasis
y = torch.cat(
(y[:, 0].unsqueeze(1), y[:, 1:] - 0.97 * y[:, :-1]),
dim=1,
)
if mask:
y = self.sub_secquence(y, weight=0.98)
spec = self.mel_transformer(y)
y = self.amplitudeToDB(spec)
# F-T mask
if mask:
# y = self.sample_aug(y, 0.2)
y = self.spec_augment(y, freq_mask=27, time_mask=0.07)
# y = self.audio_f_mask(y)
# y = self.audio_t_mask(y)
# cutout
# y = self.cutout(y)
# 归一化
std, mean = torch.std_mean(y)
y = torch.div((y - mean), std)
return y # (1,64,T)
def _update_policy(self, batch: TrainingBatch, adv: torch.Tensor):
if self.standardize_advantages:
std_adv, mean_adv = torch.std_mean(adv, unbiased=False)
adv = (adv - mean_adv) / (std_adv + 1e-8)
action_distrib: distributions.Distribution = self.policy(batch.state)
# Distribution to compute KL div against
with torch.no_grad():
# torch.distributions.Distribution cannot be deepcopied
action_distrib_old: distributions.Distribution = self.policy(
batch.state)
log_prob_old = action_distrib_old.log_prob(batch.action)
gain = self._compute_gain(
log_prob=action_distrib.log_prob(batch.action),
log_prob_old=log_prob_old,
entropy=action_distrib.entropy(),
adv=adv,
)
full_step = self._compute_kl_constrained_step(
action_distrib=action_distrib,
action_distrib_old=action_distrib_old,
gain=gain,
)
self._line_search(
full_step=full_step,
batch=batch,
adv=adv,
action_distrib_old=action_distrib_old,
log_prob_old=log_prob_old,
gain=gain,
)
def get_agreement(samples, labels, topk=1, max_err=False):
"""Get Top-k agreement of samples on some samples."""
reduced = get_matching(samples, labels, topk, max_err)
# has shape (num_f, num_eps)
std, mean = torch.std_mean(reduced, dim=-1)
return mean, std
def __getitem__(self,idx):
if idx>=self.__len__():
raise IndexError
imgid=idx%self.imgfilelength
imgxname=self.fns[imgid]
image = Image.open(imgxname).convert(self.imgmode)
#toto randomsample is not a clever it should guided the postion to crop is more efficient I think
maxstd=0
tmpsample=None
for i in range(random.randint(1,30)):
sampleimg=transforms.RandomCrop(self.patchsize)(image)
std,mean=tc.std_mean(transforms.ToTensor()(sampleimg))
if std>maxstd:
maxstd=std
tmpsample=sampleimg
sampleimg=tmpsample if tmpsample else sampleimg
#do random rotation
rotationtypes=['0',Image.ROTATE_90,Image.ROTATE_180,Image.ROTATE_270]
transtype=random.choice(rotationtypes)
if transtype !='0':
patchx = sampleimg.transpose(transtype)
else:
patchx=sampleimg
patchxtensor=transforms.ToTensor()(patchx) #to tensor c x h x w -0.5 --- 0.5
patchy = transforms.RandomVerticalFlip(p=0.5)(sampleimg)
patchy = transforms.RandomHorizontalFlip(p=0.5)(patchy)
patchytensor=transforms.ToTensor()(patchy)
return patchxtensor,patchytensor #returns a random patch
def get_wiki_cs(transform=RowFeatNormalizer(subtract_min=True)):
dataset = WikiCSDataset(transform=transform)
g = dataset[0]
std, mean = torch.std_mean(g.ndata['feat'], dim=0, unbiased=False)
g.ndata['feat'] = (g.ndata['feat'] - mean) / std
return [g]
def get_weight(self):
if self.use_layernorm:
weight = self.scale * F.layer_norm(self.weight, self.weight.shape[1:], eps=self.eps)
else:
std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False)
weight = self.scale * (self.weight - mean) / (std + self.eps)
return self.gain * weight
def calc_mean_and_std(mt_df, data_dir, num_batches, device):
train_data = dataset.CovidDataset(mt_df,
root_dir=data_dir,
target_channel=None,
for_data_statistics_calc=True)
batch_size = int(len(train_data) / num_batches)
train_loader = DataLoader(train_data, batch_size=batch_size)
num_channels = len(data_layer.channels.Channels)
mean = torch.zeros(num_channels).to(device)
std = torch.zeros(num_channels).to(device)
max_p = 0
min_p = 65535
for images in train_loader:
images = images.to(device)
# TODO: Divide by maximum value was removed
batch_mean, batch_std = torch.std_mean(images.float(), dim=(0, 1, 2))
max_p = max(torch.max(images.float()), max_p)
min_p = min(torch.min(images).float(), min_p)
mean += batch_mean
std += batch_std
mean /= num_batches
std /= num_batches
print('mean of train data is ' + str(mean.tolist()))
print('std of train data is ' + str(std.tolist()))
print('maximum of train data is ' + str(max_p.tolist()))
print('minimum of train data is ' + str(min_p.tolist()))
return mean.tolist(), std.tolist()
def get_weight(module):
std, mean = torch.std_mean(module.weight,
dim=[1, 2, 3],
keepdim=True,
unbiased=False)
weight = (module.weight - mean) / (std + module.eps)
return weight
def reduction_ops(self):
a = torch.randn(4)
b = torch.randn(4)
return (
torch.argmax(a),
torch.argmin(a),
torch.amax(a),
torch.amin(a),
torch.aminmax(a),
torch.all(a),
torch.any(a),
torch.max(a),
torch.min(a),
torch.dist(a, b),
torch.logsumexp(a, 0),
torch.mean(a),
torch.nanmean(a),
torch.median(a),
torch.nanmedian(a),
torch.mode(a),
torch.norm(a),
torch.nansum(a),
torch.prod(a),
torch.quantile(a, torch.tensor([0.25, 0.5, 0.75])),
torch.nanquantile(a, torch.tensor([0.25, 0.5, 0.75])),
torch.std(a),
torch.std_mean(a),
torch.sum(a),
torch.unique(a),
torch.unique_consecutive(a),
torch.var(a),
torch.var_mean(a),
torch.count_nonzero(a),
)
def get_weight(self):
std, mean = torch.std_mean(self.weight,
dim=[1, 2, 3],
keepdim=True,
unbiased=False)
weight = (self.weight - mean) / (std + self.eps)
return weight
def _normalize_input(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:
"Utility function that normalizes the input by batch." ""
sp, mp = torch.std_mean(x, dim=(-3, -2, -1), keepdim=True)
# WARNING: we need to .detach() input, otherwise the gradients produced by
# the patches extractor with F.grid_sample are very noisy, making the detector
# training totally unstable.
return (x - mp.detach()) / (sp.detach() + eps)
def adain_normalization(features):
epsilon = 1e-5
colorization_kernels, mean_features = torch.std_mean(features, (2, 3),
keepdim=True)
normalized_features = torch.div(features - mean_features,
epsilon + colorization_kernels)
return normalized_features, colorization_kernels, mean_features
