def update(self,
param_results: ParameterSamplingResult,
ret_avg_curr: float = None):
# Average the return values over the rollouts
rets_avg_ros = to.from_numpy(param_results.mean_returns).to(
to.get_default_dtype())
# Descending sort according to return values and the importance samples a.k.a. elites (see [1, p.12])
idcs_dcs = to.argsort(rets_avg_ros, descending=True)
idcs_dcs = idcs_dcs[:self.num_is_samples]
rets_avg_is = rets_avg_ros[idcs_dcs]
params_is = param_results.parameters[idcs_dcs, :]
# Update the policy parameters from the mean importance samples
self._policy.param_values = to.mean(params_is, dim=0)
# Update the exploration covariance from the empirical variance of the importance samples
if isinstance(self._expl_strat.noise, DiagNormalNoise):
std_is = to.std(params_is, dim=0)
extra_expl_std = self.extra_expl_std_init * max(
1.0 - self._curr_iter / self.extra_expl_decay_iter,
0 # see [2, p.4]
)
self._expl_strat.noise.adapt(std=std_is + extra_expl_std)
elif isinstance(self._expl_strat.noise, FullNormalNoise):
cov_is = cov(params_is, data_along_rows=True)
extra_expl_cov = to.pow(self.extra_expl_std_init, 2) * max(
1.0 - self._curr_iter / self.extra_expl_decay_iter,
0 # see [2, p.4]
)
self._expl_strat.noise.adapt(cov=cov_is + extra_expl_cov)
# Logging
self.logger.add_value("median imp samp return", to.median(rets_avg_is),
4)
self.logger.add_value("min imp samp return", to.min(rets_avg_is), 4)
self.logger.add_value("min expl strat std",
to.min(self._expl_strat.std), 4)
self.logger.add_value("avg expl strat std",
to.mean(self._expl_strat.std), 4)
self.logger.add_value("max expl strat std",
to.max(self._expl_strat.std), 4)
self.logger.add_value("expl strat entropy",
self._expl_strat.get_entropy(), 4)
def forward(self, x):
#print(f"Activation Function {np.shape(x)}")
# x should be of shape batchSize x dimNodeSignals x N
batchSize = x.shape[0]
dimNodeSignals = x.shape[1]
assert x.shape[2] == self.N
xK = x # xK is a tensor aggregating the 0-hop (x), 1-hop, ..., K-hop
# max's
# It is initialized with the 0-hop neigh. (x itself)
xK = xK.unsqueeze(3) # extra dimension added for concatenation ahead
#x = x.unsqueeze(3) # B x F x N x 1
for k in range(1, self.K + 1):
kHopNeighborhood = self.neighborhood[k - 1]
# Fetching k-hop neighborhoods of all nodes
kHopMedian = torch.empty(0)
# Initializing the vector that will contain the k-hop median for
# every node
for n in range(self.N):
# Iterating over the nodes
# This step is necessary because here the neighborhoods are
# lists of lists. It is impossible to pad them and feed them as
# a matrix, as this would impact the outcome of the median
# operation
nodeNeighborhood = torch.tensor(np.array(kHopNeighborhood[n]))
neighborhoodLen = len(nodeNeighborhood)
gatherNode = nodeNeighborhood.reshape([1, 1, neighborhoodLen])
gatherNode = gatherNode.repeat([batchSize, dimNodeSignals, 1])
# Reshaping the node neighborhood for the gather operation
xNodeNeighbors = torch.gather(x, 2, gatherNode.long().cuda())
# Gathering signal values in the node neighborhood
nodeMedian, _ = torch.median(xNodeNeighbors,
dim=2,
keepdim=True)
# Computing the median in the neighborhood
kHopMedian = torch.cat(
[kHopMedian.cuda(), nodeMedian.cuda()], 2)
# Concatenating k-hop medians node by node
kHopMedian = kHopMedian.unsqueeze(3) # Extra dimension for
# concatenation with the previous (k-1)-hop median tensor
xK = torch.cat([xK, kHopMedian], 3)
out = torch.matmul(xK, self.weight.unsqueeze(2))
# Multiplying each k-hop median by corresponding trainable weight
out = out.reshape([batchSize, dimNodeSignals, self.N])
return out
def reduction_ops(self):
a = torch.randn(4)
b = torch.randn(4)
c = torch.tensor(0.5)
return (
torch.argmax(a),
torch.argmin(a),
torch.amax(a),
torch.amin(a),
torch.aminmax(a),
torch.all(a),
torch.any(a),
torch.max(a),
a.max(a),
torch.max(a, 0),
torch.min(a),
a.min(a),
torch.min(a, 0),
torch.dist(a, b),
torch.logsumexp(a, 0),
torch.mean(a),
torch.mean(a, 0),
torch.nanmean(a),
torch.median(a),
torch.nanmedian(a),
torch.mode(a),
torch.norm(a),
a.norm(2),
torch.norm(a, dim=0),
torch.norm(c, torch.tensor(2)),
torch.nansum(a),
torch.prod(a),
torch.quantile(a, torch.tensor([0.25, 0.5, 0.75])),
torch.quantile(a, 0.5),
torch.nanquantile(a, torch.tensor([0.25, 0.5, 0.75])),
torch.std(a),
torch.std_mean(a),
torch.sum(a),
torch.unique(a),
torch.unique_consecutive(a),
torch.var(a),
torch.var_mean(a),
torch.count_nonzero(a),
)
def _test_one_batch(self, src, tgt, rotation_ab, translation_ab, temp):
batch_size = src.size(0)
identity = torch.eye(3, device=src.device).unsqueeze(0).repeat(
batch_size, 1, 1)
rotation_ab_pred = torch.eye(3, device=src.device,
dtype=torch.float32).view(1, 3, 3).repeat(
batch_size, 1, 1)
translation_ab_pred = torch.zeros(3,
device=src.device,
dtype=torch.float32).view(
1, 3).repeat(batch_size, 1)
total_loss = 0
with torch.no_grad():
distance = pairwise_distance(src, src)
sort_distance, _ = torch.sort(distance, dim=-1)
nearest_distance = sort_distance[:, :, 1].squeeze()
median = torch.median(nearest_distance, dim=-1)[0]
meanDist = torch.mean(median)
sigmal_ = meanDist * self.sigma_times
sigmal_ = sigmal_.repeat(batch_size)
temp = torch.tensor(temp).cuda().repeat(batch_size)
res_rotation_ab = rotation_ab
res_translation_ab = translation_ab
for i in range(self.num_iters):
sigmal_ = sigmal_ * self.DecayPram
rotation_ab_pred_i, translation_ab_pred_i = self.forward(
src, tgt, temp, i, sigmal_)
rotation_ab_pred = torch.matmul(rotation_ab_pred_i,
rotation_ab_pred)
translation_ab_pred = torch.matmul(rotation_ab_pred_i, translation_ab_pred.unsqueeze(2)).squeeze(2) \
+ translation_ab_pred_i
mcc_loss = self.mcc_loss(src, rotation_ab_pred_i,
translation_ab_pred_i, res_rotation_ab,
res_translation_ab, sigmal_)
total_loss = total_loss + mcc_loss * self.discount_factor**i
res_rotation_ab = torch.matmul(rotation_ab,
rotation_ab_pred.transpose(2, 1))
res_translation_ab = translation_ab - torch.matmul(
res_rotation_ab, translation_ab_pred.unsqueeze(2)).squeeze(2)
src = transform_point_cloud(src, rotation_ab_pred_i,
translation_ab_pred_i)
return total_loss.item(), rotation_ab_pred, translation_ab_pred
def train(extractor_model, estimator_model, extractor_optimizer,
estimator_optimizer, training_epoch):
start = time.time()
train_sq_errs = []
train_abs_errs = []
for batch_idx, (data, target) in enumerate(train_loader):
if cuda:
data, target = data.cuda(async=True), target.cuda(async=True)
data, target = Variable(data), Variable(target)
extractor_optimizer.zero_grad()
estimator_optimizer.zero_grad()
Fs, regularization_factor = train_ds.get_fps_and_regularization_factor(
batch_idx * int(args['--batch-size']))
regularization_factor = 1.0 / regularization_factor
ext_output = extractor_model(data).squeeze().unsqueeze(
dim=0).unsqueeze(dim=0)
output = estimator_model(ext_output).squeeze()
target = torch.median(target * 60.0).type(torch.FloatTensor).cuda()
train_abs_err = (torch.abs(output - target)).view(1)
train_sq_err = ((output - target)**2).view(1)
if len(train_sq_errs) == 0:
train_sq_errs = train_sq_err
train_abs_errs = train_abs_err
else:
train_sq_errs = torch.cat((train_sq_errs, train_sq_err), dim=0)
train_abs_errs = torch.cat((train_abs_errs, train_abs_err), dim=0)
(regularization_factor * train_abs_err).backward()
if training_epoch % 2 == 0:
extractor_optimizer.step()
else:
estimator_optimizer.step()
end = time.time()
return train_sq_errs.data.cpu().numpy(), train_abs_errs.data.cpu().numpy(
), end - start
def forward(self, lrs, alphas):
'''
Super resolves a batch of low-resolution images.
Args:
lrs : tensor (B, L, W, H), low-resolution images
alphas : tensor (B, L), boolean indicator (0 if padded low-res view, 1 otherwise)
Returns:
srs: tensor (B, C_out, W, H), super-resolved images
'''
###################### Encode #########################
batch_size, num_low_res, height, width = lrs.shape
# Extend channel dimension
lrs = lrs.view(batch_size, num_low_res, 1, height, width)
# create a reference view based on median statistics
refs, _ = torch.median(lrs[:, :9], 1)
alphas = alphas.view(-1, num_low_res, 1, 1, 1)
# encode LR views
lrs = lrs.view(batch_size * num_low_res, 1, height, width)
lrs = self.unit1(lrs)
lrs = lrs.view(batch_size, num_low_res, -1, height, width)
# encode ref view
refs = self.unit1(refs)
refs = refs.unsqueeze(1)
refs = refs.repeat(1, num_low_res, 1, 1, 1)
# Co-register ref encoded features with LR encoded features
out = torch.cat([lrs, refs], 2)
out = out.view(batch_size * num_low_res, -1, height, width)
out = self.unit2(out)
out = out.view(batch_size, num_low_res, -1, height,
width) # tensor (b, l, c, w, h)
###################### Fuse #########################
out = self.fuse(out, alphas)
###################### Decode #########################
out = self.decode(out)
return out
def _median_smoothing(indices: Tensor, win_length: int) -> Tensor:
r"""
Apply median smoothing to the 1D tensor over the given window.
"""
# Centered windowed
pad_length = (win_length - 1) // 2
# "replicate" padding in any dimension
indices = torch.nn.functional.pad(indices, (pad_length, 0),
mode="constant",
value=0.)
indices[..., :pad_length] = torch.cat(
pad_length * [indices[..., pad_length].unsqueeze(-1)], dim=-1)
roll = indices.unfold(-1, win_length, 1)
values, _ = torch.median(roll, -1)
return values
def median_trick(self):
'''
Compute the bandwidth for kernel based on the median trick
Reference: Section 2.2 in https://arxiv.org/pdf/1707.07269.pdf
:param X: The data of shape (num_data, dims) or (num_data, dim1, dim2)
dist_func: The distance function for pairwise distances
:return:
'''
# Get pairwise distances
dists = self.kernel.get_pairwise_distances(self.X, self.X).reshape(-1)
# Compute median of the dists
h = torch.median(dists)
# Get bandwidth
nu = torch.sqrt(h / 2.0)
return nu
def estimate_ratio_compute_mmd(x_de, x_nu, σs):
dsq_dede, dsq_denu, dsq_nunu = prepare(x_de, x_nu)
if len(σs) == 0:
# A heuristic is to use the median of pairwise distances as σ, suggested by Sugiyama's book
sigma = torch.sqrt(
torch.median(
torch.cat([
dsq_dede.squeeze(),
dsq_denu.squeeze(),
dsq_nunu.squeeze()
], 1))).item()
if not opt.nowandb:
wandb.log({"heuristic_sigma": sigma})
elif opt.monitor_heuristic:
print("heuristic sigma: ", sigma)
# Use [sigma / 5, sigma / 3, sigma, sigma * 3, sigma * 5] if nothing provided
if len(σs) == 0:
σs.append(sigma)
σs.append(sigma * 0.333)
σs.append(sigma * 0.2)
σs.append(sigma / 0.2)
σs.append(sigma / 0.333)
is_first = True
ratio = None
mmdsq = None
for σ in σs:
Kdede = gaussian_gramian(dsq_dede, σ)
Kdenu = gaussian_gramian(dsq_denu, σ)
Knunu = gaussian_gramian(dsq_nunu, σ)
if is_first:
ratio = kmm_ratios(Kdede, Kdenu, opt.eps_ratio)
mmdsq = mmdsq_of(Kdede, Kdenu, Knunu)
is_first = False
else:
ratio += kmm_ratios(Kdede, Kdenu, opt.eps_ratio)
mmdsq += mmdsq_of(Kdede, Kdenu, Knunu)
ratio = ratio / len(σs)
ratio = torch.relu(ratio) if opt.clip_ratio else ratio
mmd = torch.sqrt(torch.relu(mmdsq))
return ratio, mmd
def median(cls, csvecs):
# make sure all CSVecs match
d = csvecs[0].d
c = csvecs[0].c
r = csvecs[0].r
device = csvecs[0].device
numBlocks = csvecs[0].numBlocks
for csvec in csvecs:
assert (csvec.d == d)
assert (csvec.c == c)
assert (csvec.r == r)
assert (csvec.device == device)
assert (csvec.numBlocks == numBlocks)
tables = [csvec.table for csvec in csvecs]
med = torch.median(torch.stack(tables), dim=0)[0]
returnCSVec = copy.deepcopy(csvecs[0])
returnCSVec.table = med
return returnCSVec
def get_binary_code(self, train, test):
train_zy = [(self.encode(xb.to(self.device))[0], yb) for xb, yb in train]
train_z, train_y = zip(*train_zy)
train_z = torch.cat(train_z, dim=0)
train_y = torch.cat(train_y, dim=0)
test_zy = [(self.encode(xb.to(self.device))[0], yb) for xb, yb in test]
test_z, test_y = zip(*test_zy)
test_z = torch.cat(test_z, dim=0)
test_y = torch.cat(test_y, dim=0)
mid_val, _ = torch.median(train_z, dim=0)
train_b = (train_z > mid_val).type(torch.cuda.ByteTensor)
test_b = (test_z > mid_val).type(torch.cuda.ByteTensor)
del train_z
del test_z
return train_b, test_b, train_y, test_y
def heuristic_sigma(self, x: Tensor, xm: Tensor) -> Tensor:
"""
Uses the median-heuristic for selecting the
appropriate sigma for the RBF kernel based
on the given samples.
The kernel width is set to the media of the
pairwise distances between x and xm.
:param x: (Tensor) [N x D]
:param xm: (Tensor) [M x D]
:return:
"""
with torch.no_grad():
x1 = x.unsqueeze(-2) # Make it into a column tensor
x2 = xm.unsqueeze(-3) # Make it into a row tensor
pdist_mat = torch.sqrt(((x1 - x2)**2).sum(dim=-1)) # [N x M]
kernel_width = torch.median(torch.flatten(pdist_mat))
return kernel_width
def rbf_kernel(x, y, bandwidth=None):
"""
Compute the value of the rbf kernel between every row of x and every row of y
:param x: Data with dimensions number of examples x number of features.
:param y: Data with dimensions number of examples x number of features.
:param bandwidth: Bandwidth for the RBF kernel. If None, the median pairwise distance rule of thumb is used.
:return: gram: The gram matrix between x and y.
:return: bandwidth: The bandwidth used for the kernel.
"""
norm = squared_l2_norm(x, y)
if bandwidth is None:
bandwidth = torch.median(
torch.sqrt(torch.max(norm,
torch.DoubleTensor([1e-6]).to(DEVICE))))
print('Using rule of thumb bandwidth:', bandwidth.item())
gram = torch.exp(-1 / (2 * bandwidth**2) * norm)
return gram, bandwidth
def _clip_and_add_noise(self, grads, has_grads, shapes=None):
shared = torch.stack(has_grads).prod(0).bool()
pc_grad, num_task = copy.deepcopy(grads), len(grads)
merged_grad = torch.zeros_like(grads[0]).to(grads[0].device)
stacked_grads = torch.stack([g[shared] for g in grads])
merged_grad[shared] = torch.median(stacked_grads, dim=0)[0]
u = torch.rand(merged_grad.shape)
top_quantile = np.minimum((num_task + 1) / (2 * num_task), 1.0)
bottom_quantile = np.maximum((num_task - 1) / (2 * num_task), 0.0)
noise_max = torch.quantile(stacked_grads.abs(), top_quantile, dim=0) - merged_grad
noise_min = merged_grad - torch.quantile(stacked_grads.abs(), bottom_quantile, dim=0)
noise = (u * (noise_max - noise_min) + noise_min) * self._noise_ratio
merged_grad += noise
merged_grad[~shared] = torch.stack([g[~shared]
for g in pc_grad]).sum(dim=0)
return merged_grad
def forward(self, feats, labels):
# normalize weights
W = F.normalize(self.W)
# dot product
logits = F.linear(feats, W)
# feature re-scale
theta = torch.acos(torch.clamp(logits, -1.0 + 1e-7, 1.0 - 1e-7))
one_hot = torch.zeros_like(logits)
one_hot.scatter_(1, labels.view(-1, 1).long(), 1)
with torch.no_grad():
B_avg = torch.where(one_hot < 1, torch.exp(self.scale * logits),
torch.zeros_like(logits))
B_avg = torch.sum(B_avg) / feats.size(0)
# print(B_avg)
theta_med = torch.median(theta[one_hot == 1])
self.scale = torch.log(B_avg) / torch.cos(
torch.min(math.pi / 4 * torch.ones_like(theta_med), theta_med))
output = self.scale * logits
return output
def __medianModels(self, models):
client1 = self.clients[0]
model = models[client1]
modelCopy = copy.deepcopy(model)
params = model.named_parameters()
for name1, param1 in params:
m = []
for client2 in self.clients:
params2 = models[client2].named_parameters()
dictParams2 = dict(params2)
m.append(dictParams2[name1].data.view(-1).to("cpu").numpy())
# logPrint("Size: ", dictParams2[name1].data.size())
m = torch.tensor(m)
med = torch.median(m, dim=0)[0]
dictParamsm = dict(modelCopy.named_parameters())
dictParamsm[name1].data.copy_(
med.view(dictParamsm[name1].data.size()))
# logPrint("Median computed, size: ", med.size())
return modelCopy.to(self.device)
def mae(input, target, Fs, regularization_factor, cuda=True):
input = input.view(1, -1)
target = target.view(1, -1)
bpm_range = torch.arange(40, 240)
hr_target = Variable((torch.median(target).data * 60.0) -
bpm_range[0]).type(torch.LongTensor)
complex_absolute = TorchLossComputer.complex_absolute(
input, Fs, bpm_range, cuda)
whole_max_val, whole_max_idx = complex_absolute.view(-1).max(0)
regularization_factor = (1.0 / regularization_factor)
return torch.abs(
Variable(
torch.FloatTensor([
regularization_factor *
(hr_target.data[0] - whole_max_idx.data[0])
])))
def test(self, x, v_batch, id_batch, hidden, cell, sampling=False,n_samples=None):
if n_samples is None:
n_samples = self.params.sample_times
batch_size = x.shape[1]
if sampling:
samples = torch.zeros(n_samples, batch_size, self.params.predict_steps,
device=self.params.device)
for j in range(n_samples):
decoder_hidden = hidden
decoder_cell = cell
for t in range(self.params.predict_steps):
mu_de, sigma_de, decoder_hidden, decoder_cell = \
self(x[self.params.predict_start + t].unsqueeze(0),
id_batch, decoder_hidden, decoder_cell)
mean = torch.zeros(mu_de.shape,device=self.params.device)
dev = torch.ones(mu_de.shape,device=self.params.device)
gaussian = torch.distributions.normal.Normal(mean, dev)
pred = mu_de + sigma_de*gaussian.sample() # not scaled
samples[j, :, t] = pred * v_batch[:, 0] + v_batch[:, 1]
if t < (self.params.predict_steps - 1):
x[self.params.predict_start + t + 1, :, 0] = pred
sample_mu = torch.median(samples, dim=0)[0]
sample_sigma = samples.std(dim=0)
return samples, sample_mu, sample_sigma
else:
decoder_hidden = hidden
decoder_cell = cell
sample_mu = torch.zeros(batch_size, self.params.predict_steps, device=self.params.device)
sample_sigma = torch.zeros(batch_size, self.params.predict_steps, device=self.params.device)
for t in range(self.params.predict_steps):
mu_de, sigma_de, decoder_hidden, decoder_cell = self(x[self.params.predict_start + t].unsqueeze(0),
id_batch, decoder_hidden, decoder_cell)
sample_mu[:, t] = mu_de * v_batch[:, 0] + v_batch[:, 1]
sample_sigma[:, t] = sigma_de * v_batch[:, 0]
if t < (self.params.predict_steps - 1):
x[self.params.predict_start + t + 1, :, 0] = mu_de
return sample_mu, sample_sigma
def median_blur(input: torch.Tensor, kernel_size: Tuple[int,
int]) -> torch.Tensor:
r"""Blurs an image using the median filter.
Args:
input (torch.Tensor): the input image with shape :math:`(B,C,H,W)`.
kernel_size (Tuple[int, int]): the blurring kernel size.
Returns:
torch.Tensor: the blurred input tensor with shape :math:`(B,C,H,W)`.
Example:
>>> input = torch.rand(2, 4, 5, 7)
>>> output = median_blur(input, (3, 3))
>>> output.shape
torch.Size([2, 4, 5, 7])
"""
if not isinstance(input, torch.Tensor):
raise TypeError("Input type is not a torch.Tensor. Got {}".format(
type(input)))
if not len(input.shape) == 4:
raise ValueError(
"Invalid input shape, we expect BxCxHxW. Got: {}".format(
input.shape))
padding: Tuple[int, int] = _compute_zero_padding(kernel_size)
# prepare kernel
kernel: torch.Tensor = get_binary_kernel2d(kernel_size).to(input)
b, c, h, w = input.shape
# map the local window to single vector
features: torch.Tensor = F.conv2d(input.reshape(b * c, 1, h, w),
kernel,
padding=padding,
stride=1)
features = features.view(b, c, -1, h, w) # BxCx(K_h * K_w)xHxW
# compute the median along the feature axis
median: torch.Tensor = torch.median(features, dim=2)[0]
return median
def _test_one_batch(self, src, tgt, rotation_ab, translation_ab, temp,
epoch):
batch_size = src.size(0)
identity = torch.eye(3, device=src.device).unsqueeze(0).repeat(
batch_size, 1, 1)
rotation_ab_pred = torch.eye(3, device=src.device,
dtype=torch.float32).view(1, 3, 3).repeat(
batch_size, 1, 1)
translation_ab_pred = torch.zeros(3,
device=src.device,
dtype=torch.float32).view(
1, 3).repeat(batch_size, 1)
total_loss = 0
temp = torch.tensor(temp).cuda().repeat(batch_size)
# 求点集内点的密集程度，求其最近点的平均距离
distance = pairwise_distance(src, src)
sort_distance, _ = torch.sort(distance, dim=-1)
nearest_distance = sort_distance[:, :, 1].squeeze(-1)
median = torch.median(nearest_distance, dim=-1)[0]
meanDist = torch.mean(median)
sigmal_ = meanDist * self.sigma_times
self.sigma_ = sigmal_.item()
sigmal_ = sigmal_.repeat(batch_size)
gamma_ = epoch / self.epochs
for i in range(self.num_iters):
sigmal_ = sigmal_ * self.DecayPram
rotation_ab_pred_i, translation_ab_pred_i = self.forward(
src, tgt, temp, i, sigmal_)
rotation_ab_pred = torch.matmul(rotation_ab_pred_i,
rotation_ab_pred)
translation_ab_pred = torch.matmul(rotation_ab_pred_i, translation_ab_pred.unsqueeze(2)).squeeze(2) \
+ translation_ab_pred_i
mcc_loss = self.mcc_loss(src, rotation_ab_pred,
translation_ab_pred, rotation_ab,
translation_ab, sigmal_)
mse_loss = (F.mse_loss(torch.matmul(rotation_ab_pred.transpose(2, 1), rotation_ab), identity) \
+ F.mse_loss(translation_ab_pred, translation_ab))
loss = mcc_loss * gamma_ + mse_loss * (1 - gamma_)
total_loss = total_loss + loss * self.discount_factor**i
src = transform_point_cloud(src, rotation_ab_pred_i,
translation_ab_pred_i)
return total_loss.item(), rotation_ab_pred, translation_ab_pred
def learn_sde(points, dists, nf=1024, wstd=3.0, lam=1.5):
#
if type(points)!=torch.Tensor:
points = torch.from_numpy(points)
if type(dists)!=torch.Tensor:
dists = torch.from_numpy(dists)
#
freqs = wstd*torch.randn(3, nf)
def _encode(x):
return encode(x, freqs)
#
inputs = _encode(points)
targets = dists.reshape(-1, 1)
#
W = torch.lstsq(targets, inputs)[0][0:(2*nf)]
#
# compute training error:
errs = torch.abs(dists - torch.matmul(inputs, W).reshape(-1))
print("* learning error statistics:")
print(" - mean: %f" % torch.mean(errs))
print(" - median: %f" % torch.median(errs))
print(" - max: %f" % torch.max(errs))
#
def sdefn(points):
#
if type(points)!=torch.Tensor:
points = torch.from_numpy(points)
wasnumpy = True
else:
wasnumpy = False
#
inputs = _encode(points)
vals = torch.matmul(inputs, W).reshape(-1)
#
vals = vals/lam
#
if wasnumpy:
return vals.numpy()
else:
return vals
#
return sdefn
def median_blur(input: torch.Tensor, kernel_size: Tuple[int, int]) -> torch.Tensor:
r"""Blur an image using the median filter.
.. image:: _static/img/median_blur.png
Args:
input: the input image with shape :math:`(B,C,H,W)`.
kernel_size: the blurring kernel size.
Returns:
the blurred input tensor with shape :math:`(B,C,H,W)`.
.. note::
See a working example `here <https://kornia-tutorials.readthedocs.io/en/latest/
filtering_operators.html>`__.
Example:
>>> input = torch.rand(2, 4, 5, 7)
>>> output = median_blur(input, (3, 3))
>>> output.shape
torch.Size([2, 4, 5, 7])
"""
if not isinstance(input, torch.Tensor):
raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}")
if not len(input.shape) == 4:
raise ValueError(f"Invalid input shape, we expect BxCxHxW. Got: {input.shape}")
padding: Tuple[int, int] = _compute_zero_padding(kernel_size)
# prepare kernel
kernel: torch.Tensor = get_binary_kernel2d(kernel_size).to(input)
b, c, h, w = input.shape
# map the local window to single vector
features: torch.Tensor = F.conv2d(input.reshape(b * c, 1, h, w), kernel, padding=padding, stride=1)
features = features.view(b, c, -1, h, w) # BxCx(K_h * K_w)xHxW
# compute the median along the feature axis
median: torch.Tensor = torch.median(features, dim=2)[0]
return median
def do_bench(fn, warmup=25, rep=100, grad_to_none=None, percentiles=[0.2, 0.8]):
# Estimate the runtime of the function
fn()
torch.cuda.synchronize()
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
start_event.record()
for _ in range(5):
fn()
end_event.record()
torch.cuda.synchronize()
estimate_ms = start_event.elapsed_time(end_event) / 5
# We maintain a buffer of 256 MB that we clear
# before each kernel call to make sure that the L2
# doesn't contain any input data before the run
start_event = [torch.cuda.Event(enable_timing=True) for i in range(rep)]
end_event = [torch.cuda.Event(enable_timing=True) for i in range(rep)]
cache = torch.empty(int(256e6), dtype=torch.int8, device='cuda')
# Warm-up
for _ in range(int(warmup / estimate_ms)):
fn()
# Benchmark
for i in range(rep):
# we don't want `fn` to accumulate gradient values
# if it contains a backward pass. So we clear the
# provided gradients
if grad_to_none is not None:
grad_to_none.grad = None
# we clear the L2 cache before each run
cache.zero_()
# record time of `fn`
start_event[i].record()
fn()
end_event[i].record()
torch.cuda.synchronize()
times = torch.tensor([s.elapsed_time(e) for s, e in zip(start_event, end_event)])
percentiles = torch.quantile(times, torch.tensor(percentiles)).tolist()
med_ms = torch.median(times).item()
if percentiles:
return tuple([med_ms] + percentiles)
else:
return med_ms
def get_kernel(particle_tensor):
"""
Compute the RBF kernel for the input particles
Input: particles = tensor of shape (N, M)
Output: kernel_matrix = tensor of shape (N, N)
"""
num_particles = particle_tensor.size(0)
pairwise_d_matrix = get_pairwise_distance_matrix(particle_tensor)
median_dist = torch.median(
pairwise_d_matrix) # tf.reduce_mean(euclidean_dists) ** 2
h = median_dist / np.log(num_particles)
kernel_matrix = torch.exp(-pairwise_d_matrix / h)
kernel_sum = torch.sum(input=kernel_matrix, dim=1, keepdim=True)
grad_kernel = -torch.matmul(kernel_matrix, particle_tensor)
grad_kernel += particle_tensor * kernel_sum
grad_kernel /= h
return kernel_matrix, grad_kernel, h
def make_tree(self, x, indices, depth):
if depth == self.big_leaf_depth: # add to big_leaves if depth=5
self.big_leaves.append(int(indices[0]))
if x.shape[0] > self.min_size:
v = self.choose_rule(x)
distances = torch.tensordot(
x, v, dims=1) # create list of projection distances
median = torch.median(distances)
left_bool = (
distances <= median
) # create boolean array where entries are true if distance <= median
right_bool = ~left_bool # inverse of left_bool
left_indices = indices[left_bool]
right_indices = indices[right_bool]
self.make_tree(x[left_bool, :], left_indices, depth + 1)
self.make_tree(x[right_bool, :], right_indices, depth + 1)
elif x.shape[0] != 0:
self.leaves.append(indices.tolist())
self.sizes.append(x.shape[0])
return
def forward(self, input: torch.Tensor): # type: ignore
if not torch.is_tensor(input):
raise TypeError("Input type is not a torch.Tensor. Got {}"
.format(type(input)))
if not len(input.shape) == 4:
raise ValueError("Invalid input shape, we expect BxCxHxW. Got: {}"
.format(input.shape))
# prepare kernel
b, c, h, w = input.shape
tmp_kernel: torch.Tensor = self.kernel.to(input.device).to(input.dtype)
kernel: torch.Tensor = tmp_kernel.repeat(c, 1, 1, 1)
# map the local window to single vector
features: torch.Tensor = F.conv2d(
input, kernel, padding=self.padding, stride=1, groups=c)
features = features.view(b, c, -1, h, w) # BxCx(K_h * K_w)xHxW
# compute the median along the feature axis
median: torch.Tensor = torch.median(features, dim=2)[0]
return median
def _loss(weights_matrix, max_ratio, min_norm):
num_filters = weights_matrix.size(0)
if num_filters <= 1:
return 0.0
weights_matrix = weights_matrix.view(num_filters, -1)
norms = torch.sqrt(torch.sum(weights_matrix**2, dim=-1))
with torch.no_grad():
norm_ratio = torch.max(norms) / torch.min(norms)
median_norm = torch.median(norms)
if norm_ratio < max_ratio and median_norm > min_norm:
return 0.0
trg_norm = max(min_norm, median_norm)
losses = (norms - trg_norm)**2
loss = losses.mean()
return loss
def fval(self, nu=None, nu_prev=None):
if nu_prev is None:
if self.I_empty:
return 0, 0
n = torch.median(self.nus[-1].abs(), 1)[0]
no = self.nu_ones[-1]
# From notes:
# \sum_i l_i[nu_i]_+ \approx (-n + no)/2
# which is the negative of the term for the upper bound
# for the lower bound, use -nu and negate the output, so
# (n - no)/2 since the no term flips twice and the l1 term
# flips only once.
zl = (-n - no) / 2
zu = (n - no) / 2
return zl, zu
else:
return DualReLU.fval(self, nu=nu, nu_prev=nu_prev)
def SVGD_kernal(self, x, h=-1):
init_dist = pdist(x.cpu())
pairwise_dists = torch.tensor(squareform(init_dist),
dtype=torch.float).to(x.device)
if h < 0: # if h < 0, using median trick
h = torch.median(pairwise_dists)
h = h**2 / torch.tensor(np.log(x.shape[0] + 1), dtype=torch.float)
h = h.to(x.device)
#print(h)
kernal_xj_xi = torch.exp(-pairwise_dists**2 / h)
d_kernal_xi = torch.zeros(x.shape).to(x.device)
for i_index in range(x.shape[0]):
d_kernal_xi[i_index] = torch.matmul(kernal_xj_xi[i_index],
x[i_index] - x) * 2 / h
return kernal_xj_xi, d_kernal_xi
def _optimal_rank(self, s: pt.Tensor) -> int:
"""Compute the optimal singular value hard threshold.
This function implements the svht_ rank estimation.
.. _svht: https://doi.org/10.1109/TIT.2014.2323359
:param s: sorted singular values
:type s: pt.Tensor
:return: optimal rank for truncation
:rtype: int
"""
beta = min(self._rows, self._cols) / max(self._rows, self._cols)
omega = 0.56 * beta**3 - 0.95 * beta**2 + 1.82 * beta + 1.43
tau_star = omega * pt.median(s)
closest = pt.argmin((s - tau_star).abs()).item()
if s[closest] > tau_star:
return closest + 1
else:
return closest
