def bbox_iou(box1, box2, x1y1x2y2=True):
"""
Returns the IoU of two bounding boxes
"""
if not x1y1x2y2:
# Transform from center and width to exact coordinates
b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2
b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2
b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2
b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2
else:
# Get the coordinates of bounding boxes
b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3]
b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3]
# get the corrdinates of the intersection rectangle
inter_rect_x1 = torch.max(b1_x1, b2_x1)
inter_rect_y1 = torch.max(b1_y1, b2_y1)
inter_rect_x2 = torch.min(b1_x2, b2_x2)
inter_rect_y2 = torch.min(b1_y2, b2_y2)
# Intersection area
inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp(
inter_rect_y2 - inter_rect_y1 + 1, min=0
)
# Union Area
b1_area = (b1_x2 - b1_x1 + 1) * (b1_y2 - b1_y1 + 1)
b2_area = (b2_x2 - b2_x1 + 1) * (b2_y2 - b2_y1 + 1)
iou = inter_area / (b1_area + b2_area - inter_area + 1e-16)
return iou
def F_bilinear_interp2d(input, coords):
"""
bilinear interpolation of 2d torch.autograd.Variable
"""
x = torch.clamp(coords[:,:,0], 0, input.size(1)-2)
x0 = x.floor()
x1 = x0 + 1
y = torch.clamp(coords[:,:,1], 0, input.size(2)-2)
y0 = y.floor()
y1 = y0 + 1
stride = torch.LongTensor(input.stride())
x0_ix = x0.mul(stride[1]).long()
x1_ix = x1.mul(stride[1]).long()
y0_ix = y0.mul(stride[2]).long()
y1_ix = y1.mul(stride[2]).long()
input_flat = input.view(input.size(0),-1).contiguous()
vals_00 = input_flat.gather(1, x0_ix.add(y0_ix).detach())
vals_10 = input_flat.gather(1, x1_ix.add(y0_ix).detach())
vals_01 = input_flat.gather(1, x0_ix.add(y1_ix).detach())
vals_11 = input_flat.gather(1, x1_ix.add(y1_ix).detach())
xd = x - x0
yd = y - y0
xm = 1 - xd
ym = 1 - yd
x_mapped = (vals_00.mul(xm).mul(ym) +
vals_10.mul(xd).mul(ym) +
vals_01.mul(xm).mul(yd) +
vals_11.mul(xd).mul(yd))
return x_mapped.view_as(input)
def bbox_iou(box1, box2):
"""
Returns the IoU of two bounding boxes
"""
#Get the coordinates of bounding boxes
b1_x1, b1_y1, b1_x2, b1_y2 = box1[:,0], box1[:,1], box1[:,2], box1[:,3]
b2_x1, b2_y1, b2_x2, b2_y2 = box2[:,0], box2[:,1], box2[:,2], box2[:,3]
#get the corrdinates of the intersection rectangle
inter_rect_x1 = torch.max(b1_x1, b2_x1)
inter_rect_y1 = torch.max(b1_y1, b2_y1)
inter_rect_x2 = torch.min(b1_x2, b2_x2)
inter_rect_y2 = torch.min(b1_y2, b2_y2)
#Intersection area
inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp(inter_rect_y2 - inter_rect_y1 + 1, min=0)
#Union Area
b1_area = (b1_x2 - b1_x1 + 1)*(b1_y2 - b1_y1 + 1)
b2_area = (b2_x2 - b2_x1 + 1)*(b2_y2 - b2_y1 + 1)
iou = inter_area / (b1_area + b2_area - inter_area)
return iou
def sample_from_discretized_mix_logistic_1d(l, nr_mix):
# Pytorch ordering
l = l.permute(0, 2, 3, 1)
ls = [int(y) for y in l.size()]
xs = ls[:-1] + [1] #[3]
# unpack parameters
logit_probs = l[:, :, :, :nr_mix]
l = l[:, :, :, nr_mix:].contiguous().view(xs + [nr_mix * 2]) # for mean, scale
# sample mixture indicator from softmax
temp = torch.FloatTensor(logit_probs.size())
if l.is_cuda : temp = temp.cuda()
temp.uniform_(1e-5, 1. - 1e-5)
temp = logit_probs.data - torch.log(- torch.log(temp))
_, argmax = temp.max(dim=3)
one_hot = to_one_hot(argmax, nr_mix)
sel = one_hot.view(xs[:-1] + [1, nr_mix])
# select logistic parameters
means = torch.sum(l[:, :, :, :, :nr_mix] * sel, dim=4)
log_scales = torch.clamp(torch.sum(
l[:, :, :, :, nr_mix:2 * nr_mix] * sel, dim=4), min=-7.)
u = torch.FloatTensor(means.size())
if l.is_cuda : u = u.cuda()
u.uniform_(1e-5, 1. - 1e-5)
u = Variable(u)
x = means + torch.exp(log_scales) * (torch.log(u) - torch.log(1. - u))
x0 = torch.clamp(torch.clamp(x[:, :, :, 0], min=-1.), max=1.)
out = x0.unsqueeze(1)
return out
def forward(self, input) -> torch.FloatTensor:
""" Preprocess the input matrix
:param input tensor
"""
if isinstance(input, np.ndarray):
input = torch.from_numpy(input).type(self.dtype)
if isinstance(input, rlt.FeatureVector):
input = input.float_features.type(self.dtype)
# ONNX doesn't support != yet
not_missing_input = (
self.one_tensor.float() - (input == self.missing_tensor).float()
)
feature_starts = self._get_type_boundaries()
outputs = []
for i, feature_type in enumerate(FEATURE_TYPES):
begin_index = feature_starts[i]
if (i + 1) == len(FEATURE_TYPES):
end_index = len(self.normalization_parameters)
else:
end_index = feature_starts[i + 1]
if begin_index == end_index:
continue # No features of this type
if feature_type == ENUM:
# Process one-at-a-time
for j in range(begin_index, end_index):
norm_params = self.normalization_parameters[self.sorted_features[j]]
new_output = self._preprocess_feature_single_column(
j, input[:, j : j + 1], norm_params
)
new_output *= not_missing_input[:, j : j + 1]
self._check_preprocessing_output(new_output, [norm_params])
outputs.append(new_output)
else:
norm_params = []
for f in self.sorted_features[begin_index:end_index]:
norm_params.append(self.normalization_parameters[f])
new_output = self._preprocess_feature_multi_column(
begin_index, input[:, begin_index:end_index], norm_params
)
new_output *= not_missing_input[:, begin_index:end_index]
self._check_preprocessing_output(new_output, norm_params)
outputs.append(new_output)
def wrap(output):
if self.typed_output:
return rlt.FeatureVector(float_features=output)
else:
return output
if len(outputs) == 1:
return wrap(torch.clamp(outputs[0], MIN_FEATURE_VALUE, MAX_FEATURE_VALUE))
return wrap(
torch.clamp(torch.cat(outputs, dim=1), MIN_FEATURE_VALUE, MAX_FEATURE_VALUE)
)
def train_actor_critic(actor, critic, memory, actor_optim, critic_optim, args):
memory = np.array(memory)
states = np.vstack(memory[:, 0])
actions = list(memory[:, 1])
rewards = list(memory[:, 2])
masks = list(memory[:, 3])
old_values = critic(torch.Tensor(states))
returns, advants = get_gae(rewards, masks, old_values, args)
mu, std = actor(torch.Tensor(states))
old_policy = log_prob_density(torch.Tensor(actions), mu, std)
criterion = torch.nn.MSELoss()
n = len(states)
arr = np.arange(n)
for _ in range(args.ppo_update_num):
np.random.shuffle(arr)
for i in range(n // args.batch_size):
batch_index = arr[args.batch_size * i : args.batch_size * (i + 1)]
batch_index = torch.LongTensor(batch_index)
inputs = torch.Tensor(states)[batch_index]
actions_samples = torch.Tensor(actions)[batch_index]
returns_samples = returns.unsqueeze(1)[batch_index]
advants_samples = advants.unsqueeze(1)[batch_index]
oldvalue_samples = old_values[batch_index].detach()
values = critic(inputs)
clipped_values = oldvalue_samples + \
torch.clamp(values - oldvalue_samples,
-args.clip_param,
args.clip_param)
critic_loss1 = criterion(clipped_values, returns_samples)
critic_loss2 = criterion(values, returns_samples)
critic_loss = torch.max(critic_loss1, critic_loss2).mean()
loss, ratio, entropy = surrogate_loss(actor, advants_samples, inputs,
old_policy.detach(), actions_samples,
batch_index)
clipped_ratio = torch.clamp(ratio,
1.0 - args.clip_param,
1.0 + args.clip_param)
clipped_loss = clipped_ratio * advants_samples
actor_loss = -torch.min(loss, clipped_loss).mean()
loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy
critic_optim.zero_grad()
loss.backward(retain_graph=True)
critic_optim.step()
actor_optim.zero_grad()
loss.backward()
actor_optim.step()
def forward(self, x=None, warmup=1., inf_net=None): #, k=1): #, marginf_type=0):
outputs = {}
B = x.shape[0]
if inf_net is None:
# mu, logvar = self.inference_net(x)
z, logits = self.q.sample(x)
else:
# mu, logvar = inf_net.inference_net(x)
z, logqz = inf_net.sample(x)
# print (z[0])
# b = harden(z)
# print (b[0])
# logpz = torch.sum( self.prior.log_prob(b), dim=1)
# print (logpz[0])
# print (logpz.shape)
# fdasf
probs_q = torch.sigmoid(logits)
probs_q = torch.clamp(probs_q, min=.00000001, max=.9999999)
probs_p = torch.ones(B, self.z_size).cuda() *.5
KL = probs_q*torch.log(probs_q/probs_p) + (1-probs_q)*torch.log((1-probs_q)/(1-probs_p))
KL = torch.sum(KL, dim=1)
# print (z.shape)
# Decode Image
x_hat = self.generator.forward(z)
alpha = torch.sigmoid(x_hat)
beta = Beta(alpha*self.beta_scale, (1.-alpha)*self.beta_scale)
x_noise = torch.clamp(x + torch.FloatTensor(x.shape).uniform_(0., 1./256.).cuda(), min=1e-5, max=1-1e-5)
logpx = beta.log_prob(x_noise) #[120,3,112,112] # add uniform noise here
logpx = torch.sum(logpx.view(B, -1),1) # [PB] * self.w_logpx
# print (logpx.shape,logpz.shape,logqz.shape)
# fsdfda
log_ws = logpx - KL #+ logpz - logqz
outputs['logpx'] = torch.mean(logpx)
outputs['x_recon'] = alpha
# outputs['welbo'] = torch.mean(logpx + warmup*( logpz - logqz))
outputs['welbo'] = torch.mean(logpx + warmup*(KL))
outputs['elbo'] = torch.mean(log_ws)
outputs['logws'] = log_ws
outputs['z'] = z
outputs['logpz'] = torch.zeros(1) #torch.mean(logpz)
outputs['logqz'] = torch.mean(KL)
# outputs['logvar'] = logvar
return outputs
def F_trilinear_interp3d(input, coords):
"""
trilinear interpolation of 3D image
"""
# take clamp then floor/ceil of x coords
x = torch.clamp(coords[:,0], 0, input.size(1)-2)
x0 = x.floor()
x1 = x0 + 1
# take clamp then floor/ceil of y coords
y = torch.clamp(coords[:,1], 0, input.size(2)-2)
y0 = y.floor()
y1 = y0 + 1
# take clamp then floor/ceil of z coords
z = torch.clamp(coords[:,2], 0, input.size(3)-2)
z0 = z.floor()
z1 = z0 + 1
stride = torch.LongTensor(input.stride())[1:]
x0_ix = x0.mul(stride[0]).long()
x1_ix = x1.mul(stride[0]).long()
y0_ix = y0.mul(stride[1]).long()
y1_ix = y1.mul(stride[1]).long()
z0_ix = z0.mul(stride[2]).long()
z1_ix = z1.mul(stride[2]).long()
input_flat = th_flatten(input)
vals_000 = input_flat[x0_ix.add(y0_ix).add(z0_ix).detach()]
vals_100 = input_flat[x1_ix.add(y0_ix).add(z0_ix).detach()]
vals_010 = input_flat[x0_ix.add(y1_ix).add(z0_ix).detach()]
vals_001 = input_flat[x0_ix.add(y0_ix).add(z1_ix).detach()]
vals_101 = input_flat[x1_ix.add(y0_ix).add(z1_ix).detach()]
vals_011 = input_flat[x0_ix.add(y1_ix).add(z1_ix).detach()]
vals_110 = input_flat[x1_ix.add(y1_ix).add(z0_ix).detach()]
vals_111 = input_flat[x1_ix.add(y1_ix).add(z1_ix).detach()]
xd = x - x0
yd = y - y0
zd = z - z0
xm = 1 - xd
ym = 1 - yd
zm = 1 - zd
x_mapped = (vals_000.mul(xm).mul(ym).mul(zm) +
vals_100.mul(xd).mul(ym).mul(zm) +
vals_010.mul(xm).mul(yd).mul(zm) +
vals_001.mul(xm).mul(ym).mul(zd) +
vals_101.mul(xd).mul(ym).mul(zd) +
vals_011.mul(xm).mul(yd).mul(zd) +
vals_110.mul(xd).mul(yd).mul(zm) +
vals_111.mul(xd).mul(yd).mul(zd))
return x_mapped.view_as(input)
def F_batch_trilinear_interp3d(input, coords):
"""
input : torch.Tensor
size = (N,H,W,C)
coords : torch.Tensor
size = (N,H*W*C,2)
"""
x = torch.clamp(coords[:,:,0], 0, input.size(2)-2)
x0 = x.floor()
x1 = x0 + 1
y = torch.clamp(coords[:,:,1], 0, input.size(3)-2)
y0 = y.floor()
y1 = y0 + 1
z = torch.clamp(coords[:,:,2], 0, input.size(4)-2)
z0 = z.floor()
z1 = z0 + 1
stride = torch.LongTensor(input.stride())
x0_ix = x0.mul(stride[2]).long()
x1_ix = x1.mul(stride[2]).long()
y0_ix = y0.mul(stride[3]).long()
y1_ix = y1.mul(stride[3]).long()
z0_ix = z0.mul(stride[4]).long()
z1_ix = z1.mul(stride[4]).long()
input_flat = input.contiguous().view(input.size(0),-1)
vals_000 = input_flat.gather(1,x0_ix.add(y0_ix).add(z0_ix).detach())
vals_100 = input_flat.gather(1,x1_ix.add(y0_ix).add(z0_ix).detach())
vals_010 = input_flat.gather(1,x0_ix.add(y1_ix).add(z0_ix).detach())
vals_001 = input_flat.gather(1,x0_ix.add(y0_ix).add(z1_ix).detach())
vals_101 = input_flat.gather(1,x1_ix.add(y0_ix).add(z1_ix).detach())
vals_011 = input_flat.gather(1,x0_ix.add(y1_ix).add(z1_ix).detach())
vals_110 = input_flat.gather(1,x1_ix.add(y1_ix).add(z0_ix).detach())
vals_111 = input_flat.gather(1,x1_ix.add(y1_ix).add(z1_ix).detach())
xd = x - x0
yd = y - y0
zd = z - z0
xm = 1 - xd
ym = 1 - yd
zm = 1 - zd
x_mapped = (vals_000.mul(xm).mul(ym).mul(zm) +
vals_100.mul(xd).mul(ym).mul(zm) +
vals_010.mul(xm).mul(yd).mul(zm) +
vals_001.mul(xm).mul(ym).mul(zd) +
vals_101.mul(xd).mul(ym).mul(zd) +
vals_011.mul(xm).mul(yd).mul(zd) +
vals_110.mul(xd).mul(yd).mul(zm) +
vals_111.mul(xd).mul(yd).mul(zd))
return x_mapped.view_as(input)
def calculate_variance_term(pred, gt, means, n_objects, delta_v, norm=2):
"""pred: bs, height * width, n_filters
gt: bs, height * width, n_instances
means: bs, n_instances, n_filters"""
bs, n_loc, n_filters = pred.size()
n_instances = gt.size(2)
# bs, n_loc, n_instances, n_filters
means = means.unsqueeze(1).expand(bs, n_loc, n_instances, n_filters)
# bs, n_loc, n_instances, n_filters
pred = pred.unsqueeze(2).expand(bs, n_loc, n_instances, n_filters)
# bs, n_loc, n_instances, n_filters
gt = gt.unsqueeze(3).expand(bs, n_loc, n_instances, n_filters)
_var = (torch.clamp(torch.norm((pred - means), norm, 3) -
delta_v, min=0.0) ** 2) * gt[:, :, :, 0]
var_term = 0.0
for i in range(bs):
_var_sample = _var[i, :, :n_objects[i]] # n_loc, n_objects
_gt_sample = gt[i, :, :n_objects[i], 0] # n_loc, n_objects
var_term += torch.sum(_var_sample) / torch.sum(_gt_sample)
var_term = var_term / bs
return var_term
def calculate_distance_term(means, n_objects, delta_d, norm=2, usegpu=True):
"""means: bs, n_instances, n_filters"""
bs, n_instances, n_filters = means.size()
dist_term = 0.0
for i in range(bs):
_n_objects_sample = n_objects[i]
if _n_objects_sample <= 1:
continue
_mean_sample = means[i, : _n_objects_sample, :] # n_objects, n_filters
means_1 = _mean_sample.unsqueeze(1).expand(
_n_objects_sample, _n_objects_sample, n_filters)
means_2 = means_1.permute(1, 0, 2)
diff = means_1 - means_2 # n_objects, n_objects, n_filters
_norm = torch.norm(diff, norm, 2)
margin = 2 * delta_d * (1.0 - torch.eye(_n_objects_sample))
if usegpu:
margin = margin.cuda()
margin = Variable(margin)
_dist_term_sample = torch.sum(
torch.clamp(margin - _norm, min=0.0) ** 2)
_dist_term_sample = _dist_term_sample / \
(_n_objects_sample * (_n_objects_sample - 1))
dist_term += _dist_term_sample
dist_term = dist_term / bs
return dist_term
def encode(self, x):
# x = x.view(-1, 1, self.x_size, self.x_size)
# print (x.shape)
x = self.act_func(self.conv1(x))
# print (x.shape)
x = self.act_func(self.conv2(x))
x = self.act_func(self.conv3(x))
# print (x.size())
x = x.view(-1, self.intermediate_size)
h1 = self.act_func(self.fc1(x))
h2 = self.fc2(h1)
mean = h2[:,:self.z_size]
logvar = h2[:,self.z_size:]
#this solves the nan grad problem.
logvar = torch.clamp(logvar, min=-20.)
self.mean = mean
self.logvar = logvar
return mean, logvar
def inference_net(self, x):
mean_logvar = self.image_encoder2(x)
mean = mean_logvar[:,:6]
logvar = mean_logvar[:,6:6*2]
xenc = mean_logvar[:,6*2:]
logvar = torch.clamp(logvar, min=-15., max=10.)
return mean, logvar, xenc
def _get_state_cost(self, worlds: List[WikiTablesWorld], state: CoverageState) -> torch.Tensor:
if not state.is_finished():
raise RuntimeError("_get_state_cost() is not defined for unfinished states!")
world = worlds[state.batch_indices[0]]
# Our checklist cost is a sum of squared error from where we want to be, making sure we
# take into account the mask. We clamp the lower limit of the balance at 0 to avoid
# penalizing agenda actions produced multiple times.
checklist_balance = torch.clamp(state.checklist_state[0].get_balance(), min=0.0)
checklist_cost = torch.sum((checklist_balance) ** 2)
# This is the number of items on the agenda that we want to see in the decoded sequence.
# We use this as the denotation cost if the path is incorrect.
denotation_cost = torch.sum(state.checklist_state[0].checklist_target.float())
checklist_cost = self._checklist_cost_weight * checklist_cost
action_history = state.action_history[0]
batch_index = state.batch_indices[0]
action_strings = [state.possible_actions[batch_index][i][0] for i in action_history]
logical_form = world.get_logical_form(action_strings)
lisp_string = state.extras[batch_index]
if self._executor.evaluate_logical_form(logical_form, lisp_string):
cost = checklist_cost
else:
cost = checklist_cost + (1 - self._checklist_cost_weight) * denotation_cost
return cost
def log_Bernoulli(x, mean, average=False, dim=None):
probs = torch.clamp( mean, min=min_epsilon, max=max_epsilon )
log_bernoulli = x * torch.log( probs ) + (1. - x ) * torch.log( 1. - probs )
if average:
return torch.mean( log_bernoulli, dim )
else:
return torch.sum( log_bernoulli, dim )
def pdist(self, fX):
"""Compute pdist à-la scipy.spatial.distance.pdist
Parameters
----------
fX : (n, d) torch.Tensor
Embeddings.
Returns
-------
distances : (n * (n-1) / 2,) torch.Tensor
Condensed pairwise distance matrix
"""
n_sequences, _ = fX.size()
distances = []
for i in range(n_sequences - 1):
if self.metric in ('cosine', 'angular'):
d = 1. - F.cosine_similarity(
fX[i, :].expand(n_sequences - 1 - i, -1),
fX[i+1:, :], dim=1, eps=1e-8)
if self.metric == 'angular':
d = torch.acos(torch.clamp(1. - d, -1 + 1e-6, 1 - 1e-6))
elif self.metric == 'euclidean':
d = F.pairwise_distance(
fX[i, :].expand(n_sequences - 1 - i, -1),
fX[i+1:, :], p=2, eps=1e-06).view(-1)
distances.append(d)
return torch.cat(distances)
def get_triplet_loss(image_a_pred, image_b_pred, matches_a, matches_b, non_matches_a, non_matches_b, alpha):
"""
Computes the loss function
\sum_{triplets} ||D(I_a, u_a, I_b, u_{b,match})||_2^2 - ||D(I_a, u_a, I_b, u_{b,non-match)||_2^2 + alpha
"""
num_matches = matches_a.size()[0]
num_non_matches = non_matches_a.size()[0]
multiplier = num_non_matches / num_matches
## non_matches_a is already replicated up to be the right size
## non_matches_b is also that side
## matches_a is just a smaller version of non_matches_a
## matches_b is the only thing that needs to be replicated up in size
matches_b_long = torch.t(matches_b.repeat(multiplier, 1)).contiguous().view(-1)
matches_a_descriptors = torch.index_select(image_a_pred, 1, non_matches_a)
matches_b_descriptors = torch.index_select(image_b_pred, 1, matches_b_long)
non_matches_b_descriptors = torch.index_select(image_b_pred, 1, non_matches_b)
triplet_losses = (matches_a_descriptors - matches_b_descriptors).pow(2) - (matches_a_descriptors - non_matches_b_descriptors).pow(2) + alpha
triplet_loss = 1.0 / num_non_matches * torch.clamp(triplet_losses, min=0).sum()
return triplet_loss
def l2_pixel_loss(self, matches_b, non_matches_b, M_pixel=None):
"""
Apply l2 loss in pixel space.
This weights non-matches more if they are "far away" in pixel space.
:param matches_b: A torch.LongTensor with shape torch.Shape([num_matches])
:param non_matches_b: A torch.LongTensor with shape torch.Shape([num_non_matches])
:return l2 loss per sample: A torch.FloatTensorof with shape torch.Shape([num_matches])
"""
if M_pixel is None:
M_pixel = self._config['M_pixel']
num_non_matches_per_match = len(non_matches_b)/len(matches_b)
ground_truth_pixels_for_non_matches_b = torch.t(matches_b.repeat(num_non_matches_per_match,1)).contiguous().view(-1,1)
ground_truth_u_v_b = self.flattened_pixel_locations_to_u_v(ground_truth_pixels_for_non_matches_b)
sampled_u_v_b = self.flattened_pixel_locations_to_u_v(non_matches_b.unsqueeze(1))
# each element is always within [0,1], you have 1 if you are at least M_pixel away in
# L2 norm in pixel space
norm_degree = 2
squared_l2_pixel_loss = 1.0/M_pixel * torch.clamp((ground_truth_u_v_b - sampled_u_v_b).float().norm(norm_degree,1), max=M_pixel)
return squared_l2_pixel_loss, ground_truth_u_v_b, sampled_u_v_b
def hinge_loss(positive_predictions, negative_predictions, mask=None):
"""
Hinge pairwise loss function.
Parameters
----------
positive_predictions: tensor
Tensor containing predictions for known positive items.
negative_predictions: tensor
Tensor containing predictions for sampled negative items.
mask: tensor, optional
A binary tensor used to zero the loss from some entries
of the loss tensor.
Returns
-------
loss, float
The mean value of the loss function.
"""
loss = torch.clamp(negative_predictions -
positive_predictions +
1.0, 0.0)
if mask is not None:
mask = mask.float()
loss = loss * mask
return loss.sum() / mask.sum()
return loss.mean()
def project_to_2d(X, camera_params):
"""
Project 3D points to 2D using the Human3.6M camera projection function.
This is a differentiable and batched reimplementation of the original MATLAB script.
Arguments:
X -- 3D points in *camera space* to transform (N, *, 3)
camera_params -- intrinsic parameteres (N, 2+2+3+2=9)
"""
assert X.shape[-1] == 3
assert len(camera_params.shape) == 2
assert camera_params.shape[-1] == 9
assert X.shape[0] == camera_params.shape[0]
while len(camera_params.shape) < len(X.shape):
camera_params = camera_params.unsqueeze(1)
f = camera_params[..., :2]
c = camera_params[..., 2:4]
k = camera_params[..., 4:7]
p = camera_params[..., 7:]
XX = torch.clamp(X[..., :2] / X[..., 2:], min=-1, max=1)
r2 = torch.sum(XX[..., :2]**2, dim=len(XX.shape)-1, keepdim=True)
radial = 1 + torch.sum(k * torch.cat((r2, r2**2, r2**3), dim=len(r2.shape)-1), dim=len(r2.shape)-1, keepdim=True)
tan = torch.sum(p*XX, dim=len(XX.shape)-1, keepdim=True)
XXX = XX*(radial + tan) + p*r2
return f*XXX + c
def discretized_mix_logistic_loss_1d(x, l):
""" log-likelihood for mixture of discretized logistics, assumes the data has been rescaled to [-1,1] interval """
# Pytorch ordering
x = x.permute(0, 2, 3, 1)
l = l.permute(0, 2, 3, 1)
xs = [int(y) for y in x.size()]
ls = [int(y) for y in l.size()]
# here and below: unpacking the params of the mixture of logistics
nr_mix = int(ls[-1] / 3)
logit_probs = l[:, :, :, :nr_mix]
l = l[:, :, :, nr_mix:].contiguous().view(xs + [nr_mix * 2]) # 2 for mean, scale
means = l[:, :, :, :, :nr_mix]
log_scales = torch.clamp(l[:, :, :, :, nr_mix:2 * nr_mix], min=-7.)
# here and below: getting the means and adjusting them based on preceding
# sub-pixels
x = x.contiguous()
x = x.unsqueeze(-1) + Variable(torch.zeros(xs + [nr_mix]).cuda(), requires_grad=False)
# means = torch.cat((means[:, :, :, 0, :].unsqueeze(3), m2, m3), dim=3)
centered_x = x - means
inv_stdv = torch.exp(-log_scales)
plus_in = inv_stdv * (centered_x + 1. / 255.)
cdf_plus = F.sigmoid(plus_in)
min_in = inv_stdv * (centered_x - 1. / 255.)
cdf_min = F.sigmoid(min_in)
# log probability for edge case of 0 (before scaling)
log_cdf_plus = plus_in - F.softplus(plus_in)
# log probability for edge case of 255 (before scaling)
log_one_minus_cdf_min = -F.softplus(min_in)
cdf_delta = cdf_plus - cdf_min # probability for all other cases
mid_in = inv_stdv * centered_x
# log probability in the center of the bin, to be used in extreme cases
# (not actually used in our code)
log_pdf_mid = mid_in - log_scales - 2. * F.softplus(mid_in)
inner_inner_cond = (cdf_delta > 1e-5).float()
inner_inner_out = inner_inner_cond * torch.log(torch.clamp(cdf_delta, min=1e-12)) + (1. - inner_inner_cond) * (log_pdf_mid - np.log(127.5))
inner_cond = (x > 0.999).float()
inner_out = inner_cond * log_one_minus_cdf_min + (1. - inner_cond) * inner_inner_out
cond = (x < -0.999).float()
log_probs = cond * log_cdf_plus + (1. - cond) * inner_out
log_probs = torch.sum(log_probs, dim=3) + log_prob_from_logits(logit_probs)
#Don't sum over batch dimension
lse = log_sum_exp(log_probs)
return -torch.sum(lse.view(lse.size(0), -1), dim=1)
def feed(self, Q, pi_s, actions, stats, old_pi_s=dict()):
"""One iteration of policy gradient.
rho nabla_w log p_w(a|s) Q + entropy_ratio * nabla H(pi(.|s))
Args:
Q(tensor): estimated return
actions(tensor): action
pi_s(variable): policy
old_pi_s(tensor, optional): old policy, in order to
get importance factor.
If you specify multiple policies, then all the log prob of these
policies are added, and their importance factors are multiplied.
Feed to stats: policy error and nll error
"""
# We need to set it beforehand.
# Note that the samples we collect might be off-policy, so we need
# to do importance sampling.
pg_weights = Q.clone()
policy_err = None
entropy_err = None
log_pi_s = []
for pi_node, a_node in self.policy_action_nodes:
pi = pi_s[pi_node]
a = actions[a_node].squeeze()
if pi_node in old_pi_s:
old_pi = old_pi_s[pi_node].squeeze()
# Cap it.
clamped_ratios = torch.clamp(
pi.data.div(old_pi), max=self.options.ratio_clamp)
coeff = clamped_ratios.gather(1, a.view(-1, 1)).squeeze()
pg_weights.mul_(coeff)
# There is another term (to compensate clamping), but we omit
# it for now.
# Compute policy gradient error:
errs = self._compute_policy_entropy_err(pi, Variable(a))
policy_err = add_err(policy_err, errs["policy_err"])
entropy_err = add_err(entropy_err, errs["entropy_err"])
log_pi_s.append(errs["logpi"])
stats["nll_" + pi_node].feed(errs["policy_err"].data[0])
stats["entropy_" + pi_node].feed(errs["entropy_err"].data[0])
for log_pi in log_pi_s:
self._reg_backward(log_pi, Variable(pg_weights))
if len(self.policy_action_nodes) > 1:
stats["total_nll"].feed(policy_err.data[0])
stats["total_entropy"].feed(entropy_err.data[0])
return policy_err + entropy_err * self.options.entropy_ratio
def __call__(self, *inputs):
outputs = []
for idx, _input in enumerate(inputs):
channel_means = _input.mean(1).mean(2)
channel_means = channel_means.expand_as(_input)
_input = th.clamp((_input - channel_means) * self.value + channel_means,0,1)
outputs.append(_input)
return outputs if idx > 1 else outputs[0]
def th_nearest_interp3d(input, coords):
"""
2d nearest neighbor interpolation th.Tensor
"""
# take clamp of coords so they're in the image bounds
coords[:,0] = th.clamp(coords[:,0], 0, input.size(1)-1).round()
coords[:,1] = th.clamp(coords[:,1], 0, input.size(2)-1).round()
coords[:,2] = th.clamp(coords[:,2], 0, input.size(3)-1).round()
stride = th.LongTensor(input.stride())[1:].float()
idx = coords.mv(stride).long()
input_flat = th_flatten(input)
mapped_vals = input_flat[idx]
return mapped_vals.view_as(input)
def forward(self, x, y, xidx=None, yidx=None):
K = torch.sqrt(l2_distance(x, y))
u, v = self._get_uv(x, y, xidx, yidx)
if self.regularization == 'entropy':
return torch.exp((u[:, None] + v[None, :] - K) / self.alpha)
else:
return torch.clamp((u[:, None] + v[None, :] - K),
min=0) / (2 * self.alpha)
def __call__(self, *inputs):
outputs = []
for idx, _input in enumerate(inputs):
_in_gs = Grayscale(keep_channels=True)(_input)
alpha = 1.0 + self.value
_in = th.clamp(_blend(_input, _in_gs, alpha), 0, 1)
outputs.append(_in)
return outputs if idx > 1 else outputs[0]
def th_nearest_interp2d(input, coords):
"""
2d nearest neighbor interpolation th.Tensor
"""
# take clamp of coords so they're in the image bounds
x = th.clamp(coords[:,:,0], 0, input.size(1)-1).round()
y = th.clamp(coords[:,:,1], 0, input.size(2)-1).round()
stride = th.LongTensor(input.stride())
x_ix = x.mul(stride[1]).long()
y_ix = y.mul(stride[2]).long()
input_flat = input.view(input.size(0),-1)
mapped_vals = input_flat.gather(1, x_ix.add(y_ix))
return mapped_vals.view_as(input)
def batch_iou_pair(yx_min1, yx_max1, yx_min2, yx_max2, min=float(np.finfo(np.float32).eps)):
"""
Pairwisely calculates the IoU of two lists (at the same size M) of bounding boxes for N independent batches.
:author 申瑞珉 (Ruimin Shen)
:param yx_min1: The top left coordinates (y, x) of the first lists (size [N, M, 2]) of bounding boxes.
:param yx_max1: The bottom right coordinates (y, x) of the first lists (size [N, M, 2]) of bounding boxes.
:param yx_min2: The top left coordinates (y, x) of the second lists (size [N, M, 2]) of bounding boxes.
:param yx_max2: The bottom right coordinates (y, x) of the second lists (size [N, M, 2]) of bounding boxes.
:return: The lists (size [N, M]) of the IoU.
"""
yx_min = torch.max(yx_min1, yx_min2)
yx_max = torch.min(yx_max1, yx_max2)
size = torch.clamp(yx_max - yx_min, min=0)
intersect_area = torch.prod(size, -1)
area1 = torch.prod(yx_max1 - yx_min1, -1)
area2 = torch.prod(yx_max2 - yx_min2, -1)
union_area = torch.clamp(area1 + area2 - intersect_area, min=min)
return intersect_area / union_area
def part_loss(pred_part, gt_seg_part, gt_seg_object, object_label, valid):
mask_object = (gt_seg_object == object_label)
loss = F.nll_loss(pred_part, gt_seg_part * mask_object.long(), reduction='none')
loss = loss * mask_object.float()
loss = torch.sum(loss.view(loss.size(0), -1), dim=1)
nr_pixel = torch.sum(mask_object.view(mask_object.shape[0], -1), dim=1)
sum_pixel = (nr_pixel * valid).sum()
loss = (loss * valid.float()).sum() / torch.clamp(sum_pixel, 1).float()
return loss
def forward(self, anchor, positive, negative):
sim1 = torch.sum(anchor*positive,1)
sim2 = torch.sum(anchor*negative,1)
#print sim1
#print sim2
dist = sim2-sim1+self.margin
dist_hinge = torch.clamp(dist, min=0.0)
loss = torch.mean(dist_hinge)
return loss
def convert_to_multimnist(d, t, t_len=14):
d = d.permute(1, 0)
d = d.reshape((64, 1, t_len, t_len))
perm = torch.randperm(d.shape[0])
drand = d[perm]
#save_image(d.data.cpu(), 'multimnist_0.png')
#save_image(drand.data.cpu(), 'multimnist_1_preshift.png')
for i in range(0, 64):
dr = drand[i]
rshiftx1 = random.randint(0, 3)
rshiftx2 = random.randint(0, 3)
rshifty1 = random.randint(0, 3)
rshifty2 = random.randint(0, 3)
dr = dr[:, rshiftx1:t_len - rshiftx2, rshifty1:t_len - rshifty2]
if random.uniform(0, 1) < 0.5:
padl = rshiftx1 + rshiftx2
padr = 0
else:
padl = 0
padr = rshiftx1 + rshiftx2
if random.uniform(0, 1) < 0.5:
padt = rshifty1 + rshifty2
padb = 0
else:
padt = 0
padb = rshifty1 + rshifty2
dr = torch.cat([
torch.zeros(1, padl, dr.shape[2]).long(), dr,
torch.zeros(1, padr, dr.shape[2]).long()
], 1)
dr = torch.cat([
torch.zeros(1, dr.shape[1], padt).long(), dr,
torch.zeros(1, dr.shape[1], padb).long()
], 2)
drand[i] = dr * 1.0
#save_image(drand.data.cpu(), 'multimnist_1.png')
d = torch.clamp((d + drand), 0, 1)
tr = t[perm]
#print(t[0:5], tr[0:5])
new_target = t * 0.0
for i in range(t.shape[0]):
if t[i] >= tr[i]:
nt = int(str(t[i].item()) + str(tr[i].item()))
else:
nt = int(str(tr[i].item()) + str(t[i].item()))
new_target[i] += nt
#print('new target i', new_target[0:5])
#save_image(d.data.cpu(), 'multimnist.png')
#raise Exception('done')
d = d.reshape((64, t_len * t_len))
d = d.permute(1, 0)
return d, new_target
def _do_paste_mask(masks, boxes, img_h, img_w, skip_empty=True):
"""Paste instance masks acoording to boxes.
This implementation is modified from
https://github.com/facebookresearch/detectron2/
Args:
masks (Tensor): N, 1, H, W
boxes (Tensor): N, 4
img_h (int): Height of the image to be pasted.
img_w (int): Width of the image to be pasted.
skip_empty (bool): Only paste masks within the region that
tightly bound all boxes, and returns the results this region only.
An important optimization for CPU.
Returns:
tuple: (Tensor, tuple). The first item is mask tensor, the second one
is the slice object.
If skip_empty == False, the whole image will be pasted. It will
return a mask of shape (N, img_h, img_w) and an empty tuple.
If skip_empty == True, only area around the mask will be pasted.
A mask of shape (N, h', w') and its start and end coordinates
in the original image will be returned.
"""
# On GPU, paste all masks together (up to chunk size)
# by using the entire image to sample the masks
# Compared to pasting them one by one,
# this has more operations but is faster on COCO-scale dataset.
device = masks.device
if skip_empty:
x0_int, y0_int = torch.clamp(
boxes.min(dim=0).values.floor()[:2] - 1,
min=0).to(dtype=torch.int32)
x1_int = torch.clamp(
boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32)
y1_int = torch.clamp(
boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32)
else:
x0_int, y0_int = 0, 0
x1_int, y1_int = img_w, img_h
x0, y0, x1, y1 = torch.split(boxes, 1, dim=1) # each is Nx1
N = masks.shape[0]
img_y = torch.arange(
y0_int, y1_int, device=device, dtype=torch.float32) + 0.5
img_x = torch.arange(
x0_int, x1_int, device=device, dtype=torch.float32) + 0.5
img_y = (img_y - y0) / (y1 - y0) * 2 - 1
img_x = (img_x - x0) / (x1 - x0) * 2 - 1
# img_x, img_y have shapes (N, w), (N, h)
if torch.isinf(img_x).any():
inds = torch.where(torch.isinf(img_x))
img_x[inds] = 0
if torch.isinf(img_y).any():
inds = torch.where(torch.isinf(img_y))
img_y[inds] = 0
gx = img_x[:, None, :].expand(N, img_y.size(1), img_x.size(1))
gy = img_y[:, :, None].expand(N, img_y.size(1), img_x.size(1))
grid = torch.stack([gx, gy], dim=3)
img_masks = F.grid_sample(
masks.to(dtype=torch.float32), grid, align_corners=False)
if skip_empty:
return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int))
else:
return img_masks[:, 0], ()
def find_dynamic_lmk_idx_and_bcoords(vertices, pose, dynamic_lmk_faces_idx,
dynamic_lmk_b_coords,
neck_kin_chain, dtype=torch.float32):
''' Compute the faces, barycentric coordinates for the dynamic landmarks
To do so, we first compute the rotation of the neck around the y-axis
and then use a pre-computed look-up table to find the faces and the
barycentric coordinates that will be used.
Special thanks to Soubhik Sanyal ([email protected])
for providing the original TensorFlow implementation and for the LUT.
Parameters
----------
vertices: torch.tensor BxVx3, dtype = torch.float32
The tensor of input vertices
pose: torch.tensor Bx(Jx3), dtype = torch.float32
The current pose of the body model
dynamic_lmk_faces_idx: torch.tensor L, dtype = torch.long
The look-up table from neck rotation to faces
dynamic_lmk_b_coords: torch.tensor Lx3, dtype = torch.float32
The look-up table from neck rotation to barycentric coordinates
neck_kin_chain: list
A python list that contains the indices of the joints that form the
kinematic chain of the neck.
dtype: torch.dtype, optional
Returns
-------
dyn_lmk_faces_idx: torch.tensor, dtype = torch.long
A tensor of size BxL that contains the indices of the faces that
will be used to compute the current dynamic landmarks.
dyn_lmk_b_coords: torch.tensor, dtype = torch.float32
A tensor of size BxL that contains the indices of the faces that
will be used to compute the current dynamic landmarks.
'''
batch_size = vertices.shape[0]
aa_pose = torch.index_select(pose.view(batch_size, -1, 3), 1,
neck_kin_chain)
rot_mats = batch_rodrigues(
aa_pose.view(-1, 3), dtype=dtype).view(batch_size, -1, 3, 3)
rel_rot_mat = torch.eye(3, device=vertices.device,
dtype=dtype).unsqueeze_(dim=0)
for idx in range(len(neck_kin_chain)):
rel_rot_mat = torch.bmm(rot_mats[:, idx], rel_rot_mat)
y_rot_angle = torch.round(
torch.clamp(-rot_mat_to_euler(rel_rot_mat) * 180.0 / np.pi,
max=39)).to(dtype=torch.long)
neg_mask = y_rot_angle.lt(0).to(dtype=torch.long)
mask = y_rot_angle.lt(-39).to(dtype=torch.long)
neg_vals = mask * 78 + (1 - mask) * (39 - y_rot_angle)
y_rot_angle = (neg_mask * neg_vals +
(1 - neg_mask) * y_rot_angle)
dyn_lmk_faces_idx = torch.index_select(dynamic_lmk_faces_idx,
0, y_rot_angle)
dyn_lmk_b_coords = torch.index_select(dynamic_lmk_b_coords,
0, y_rot_angle)
return dyn_lmk_faces_idx, dyn_lmk_b_coords
def amp_to_db(x):
o = 20 * torch.log10(torch.clamp(x, min=1e-5))
return o
def funcInf(a, b):
return torch.clamp(a + b, min=0, max=float('inf'))
def build_targets(pred_boxes, pred_conf, pred_cls, target, anchor_wh, nA, nC, nG, batch_report):
"""
returns nT, nCorrect, tx, ty, tw, th, tconf, tcls
"""
nG_w = nG[0]
nG_h = nG[1]
nB = len(target) # number of images in batch
nT = [len(x) for x in target] # torch.argmin(target[:, :, 4], 1) # targets per image
tx = torch.zeros(nB, nA, nG_h, nG_w) # batch size (4), number of anchors (3), number of grid points (13)
ty = torch.zeros(nB, nA, nG_h, nG_w)
tw = torch.zeros(nB, nA, nG_h, nG_w)
th = torch.zeros(nB, nA, nG_h, nG_w)
tconf = torch.ByteTensor(nB, nA, nG_h, nG_w).fill_(0)
tcls = torch.ByteTensor(nB, nA, nG_h, nG_w, nC).fill_(0) # nC = number of classes
TP = torch.ByteTensor(nB, max(nT)).fill_(0)
FP = torch.ByteTensor(nB, max(nT)).fill_(0)
FN = torch.ByteTensor(nB, max(nT)).fill_(0)
TC = torch.ShortTensor(nB, max(nT)).fill_(-1) # target category
for b in range(nB):
nTb = nT[b] # number of targets
if nTb == 0:
continue
t = target[b]
# Convert to position relative to box
"Mijenjano"
TC[b, :nTb], gx, gy, gw, gh = t[:, 0].long(), t[:, 1] * nG_w, t[:, 2] * nG_h, t[:, 3] * nG_w, t[:, 4] * nG_h
# Get grid box indices and prevent overflows (i.e. 13.01 on 13 anchors)
gi = torch.clamp(gx.long(), min=0, max=nG_w - 1)
gj = torch.clamp(gy.long(), min=0, max=nG_h - 1)
# iou of targets-anchors (using wh only)
"Mijenjano"
box1_1 = (t[:, 3] * nG_w).view(-1,1)
box1_2 = (t[:, 4] * nG_h).view(-1,1)
box1 = torch.cat((box1_1, box1_2), 1)
box2 = anchor_wh.unsqueeze(1)
inter_area = torch.min(box1, box2).prod(2)
iou = inter_area / (gw * gh + box2.prod(2) - inter_area + 1e-16)
# Select best iou_pred and anchor
iou_best, a = iou.max(0) # best anchor [0-2] for each target
# Select best unique target-anchor combinations
if nTb > 1:
iou_order = torch.argsort(-iou_best) # best to worst
# Unique anchor selection
u = torch.cat((gi, gj, a), 0).view(3, -1)
_, first_unique = np.unique(u[:, iou_order], axis=1, return_index=True) # first unique indices
# _, first_unique = torch.unique(u[:, iou_order], dim=1, return_inverse=True) # different than numpy?
i = iou_order[first_unique]
# best anchor must share significant commonality (iou) with target
i = i[iou_best[i] > 0.10]
if len(i) == 0:
continue
a, gj, gi, t = a[i], gj[i], gi[i], t[i]
if len(t.shape) == 1:
t = t.view(1, 5)
else:
if iou_best < 0.10:
continue
i = 0
tc, gx, gy, gw, gh = t[:, 0].long(), t[:, 1] * nG_w, t[:, 2] * nG_h, t[:, 3] * nG_w, t[:, 4] * nG_h
# Coordinates
tx[b, a, gj, gi] = gx - gi.float()
ty[b, a, gj, gi] = gy - gj.float()
# Width and height (yolo method)
tw[b, a, gj, gi] = torch.log(gw / anchor_wh[a, 0])
th[b, a, gj, gi] = torch.log(gh / anchor_wh[a, 1])
# Width and height (power method)
# tw[b, a, gj, gi] = torch.sqrt(gw / anchor_wh[a, 0]) / 2
# th[b, a, gj, gi] = torch.sqrt(gh / anchor_wh[a, 1]) / 2
# One-hot encoding of label
tcls[b, a, gj, gi, tc] = 1
tconf[b, a, gj, gi] = 1
if batch_report:
# predicted classes and confidence
tb = torch.cat((gx - gw / 2, gy - gh / 2, gx + gw / 2, gy + gh / 2)).view(4, -1).t() # target boxes
pcls = torch.argmax(pred_cls[b, a, gj, gi], 1).cpu()
pconf = torch.sigmoid(pred_conf[b, a, gj, gi]).cpu()
iou_pred = bbox_iou(tb, pred_boxes[b, a, gj, gi].cpu())
TP[b, i] = (pconf > 0.5) & (iou_pred > 0.5) & (pcls == tc)
FP[b, i] = (pconf > 0.5) & (TP[b, i] == 0) # coordinates or class are wrong
FN[b, i] = pconf <= 0.5 # confidence score is too low (set to zero)
return tx, ty, tw, th, tconf, tcls, TP, FP, FN, TC
if args.norm_adv:
mb_advantages = (mb_advantages - mb_advantages.mean()) / (
mb_advantages.std() + 1e-8)
_, newlogproba, entropy = agent.get_action(
b_obs[minibatch_ind],
b_actions.long()[minibatch_ind].T,
b_invalid_action_masks[minibatch_ind])
ratio = (newlogproba - b_logprobs[minibatch_ind]).exp()
# Stats
approx_kl = (b_logprobs[minibatch_ind] - newlogproba).mean()
# Policy loss
pg_loss1 = -mb_advantages * ratio
pg_loss2 = -mb_advantages * torch.clamp(ratio, 1 - args.clip_coef,
1 + args.clip_coef)
pg_loss = torch.max(pg_loss1, pg_loss2).mean()
entropy_loss = entropy.mean()
# Value loss
new_values = agent.get_value(b_obs[minibatch_ind]).view(-1)
if args.clip_vloss:
v_loss_unclipped = ((new_values - b_returns[minibatch_ind])**2)
v_clipped = b_values[minibatch_ind] + torch.clamp(
new_values - b_values[minibatch_ind], -args.clip_coef,
args.clip_coef)
v_loss_clipped = (v_clipped - b_returns[minibatch_ind])**2
v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped)
v_loss = 0.5 * v_loss_max.mean()
else:
v_loss = 0.5 * ((new_values - b_returns[minibatch_ind])**2)
def sampleIsotropic(self, w):
theta = 2 * self.PI * self.random()
phi = torch.acos(torch.clamp(1 - 2 * self.random(), -1, 1))
return self.vec(
torch.sin(phi) * torch.cos(theta),
torch.sin(phi) * torch.sin(theta), torch.cos(phi))
def disagg_fold(dataset, fold_num, lr, p):
train, test = get_train_test(dataset, num_folds=num_folds, fold_num=fold_num)
valid = train[int(0.8 * len(train)):].copy()
train = train[:int(0.8 * len(train))].copy()
train_aggregate = train[:, 0, :, :].reshape(train.shape[0], 1, -1, 24)
valid_aggregate = valid[:, 0, :, :].reshape(valid.shape[0], 1, -1, 24)
test_aggregate = test[:, 0, :, :].reshape(test.shape[0], 1, -1, 24)
out_train, out_valid, out_test = preprocess(train, valid, test)
loss_func = nn.L1Loss()
model = AppliancesCNN(len(ORDER))
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
if cuda_av:
model = model.cuda()
loss_func = loss_func.cuda()
inp = Variable(torch.Tensor(train_aggregate), requires_grad=False)
valid_inp = Variable(torch.Tensor(valid_aggregate), requires_grad=False)
test_inp = Variable(torch.Tensor(test_aggregate), requires_grad=False)
if cuda_av:
inp = inp.cuda()
valid_inp = valid_inp.cuda()
test_inp = test_inp.cuda()
valid_out = torch.cat([out_valid[appliance_num] for appliance_num, appliance in enumerate(ORDER)])
test_out = torch.cat([out_test[appliance_num] for appliance_num, appliance in enumerate(ORDER)])
train_out = torch.cat([out_train[appliance_num] for appliance_num, appliance in enumerate(ORDER)])
valid_pred = {}
train_pred = {}
test_pred = {}
train_losses = {}
test_losses = {}
valid_losses = {}
params = [inp, p]
for a_num, appliance in enumerate(ORDER):
params.append(out_train[a_num])
if cuda_av:
train_out = train_out.cuda()
for t in range(1, num_iterations + 1):
pred = model(*params)
optimizer.zero_grad()
loss = loss_func(pred, train_out)
if t % 500 == 0:
if cuda_av:
valid_inp = valid_inp.cuda()
valid_params = [valid_inp, -2]
for i in range(len(ORDER)):
valid_params.append(None)
valid_pr = model(*valid_params)
valid_loss = loss_func(valid_pr, valid_out)
if cuda_av:
test_inp = test_inp.cuda()
test_params = [test_inp, -2]
for i in range(len(ORDER)):
test_params.append(None)
test_pr = model(*test_params)
test_loss = loss_func(test_pr, test_out)
test_losses[t] = test_loss.data[0]
valid_losses[t] = valid_loss.data[0]
train_losses[t] = loss.data[0]
# np.save("./baseline/p_50_loss")
if t % 1000 == 0:
valid_pr = torch.clamp(valid_pr, min=0.)
valid_pred[t] = valid_pr
test_pr = torch.clamp(test_pr, min=0.)
test_pred[t] = test_pr
train_pr = pred
train_pr = torch.clamp(train_pr, min=0.)
train_pred[t] = train_pr
print("Round:", t, "Training Error:", loss.data[0], "Validation Error:", valid_loss.data[0], "Test Error:", test_loss.data[0])
loss.backward()
optimizer.step()
train_fold = [None for x in range(len(ORDER))]
train_fold = {}
for t in range(1000, num_iterations + 1, 1000):
train_pred[t] = torch.split(train_pred[t], train_aggregate.shape[0])
train_fold[t] = [None for x in range(len(ORDER))]
if cuda_av:
for appliance_num, appliance in enumerate(ORDER):
train_fold[t][appliance_num] = train_pred[t][appliance_num].cpu().data.numpy().reshape(-1, 24)
else:
for appliance_num, appliance in enumerate(ORDER):
train_fold[t][appliance_num] = train_pred[t][appliance_num].data.numpy().reshape(-1, 24)
valid_fold = {}
for t in range(1000, num_iterations + 1, 1000):
valid_pred[t] = torch.split(valid_pred[t], valid_aggregate.shape[0])
valid_fold[t] = [None for x in range(len(ORDER))]
if cuda_av:
for appliance_num, appliance in enumerate(ORDER):
valid_fold[t][appliance_num] = valid_pred[t][appliance_num].cpu().data.numpy().reshape(-1, 24)
else:
for appliance_num, appliance in enumerate(ORDER):
valid_fold[t][appliance_num] = valid_pred[t][appliance_num].data.numpy().reshape(-1, 24)
test_fold = {}
for t in range(1000, num_iterations + 1, 1000):
test_pred[t] = torch.split(test_pred[t], test_aggregate.shape[0])
test_fold[t] = [None for x in range(len(ORDER))]
if cuda_av:
for appliance_num, appliance in enumerate(ORDER):
test_fold[t][appliance_num] = test_pred[t][appliance_num].cpu().data.numpy().reshape(-1, 24)
else:
for appliance_num, appliance in enumerate(ORDER):
test_fold[t][appliance_num] = test_pred[t][appliance_num].data.numpy().reshape(-1, 24)
# store ground truth of validation set
train_gt_fold = [None for x in range(len(ORDER))]
for appliance_num, appliance in enumerate(ORDER):
train_gt_fold[appliance_num] = train[:, APPLIANCE_ORDER.index(appliance), :, :].reshape(
train_aggregate.shape[0],
-1, 1).reshape(-1, 24)
valid_gt_fold = [None for x in range(len(ORDER))]
for appliance_num, appliance in enumerate(ORDER):
valid_gt_fold[appliance_num] = valid[:, APPLIANCE_ORDER.index(appliance), :, :].reshape(
valid_aggregate.shape[0],
-1, 1).reshape(-1, 24)
test_gt_fold = [None for x in range(len(ORDER))]
for appliance_num, appliance in enumerate(ORDER):
test_gt_fold[appliance_num] = test[:, APPLIANCE_ORDER.index(appliance), :, :].reshape(
test_aggregate.shape[0],
-1, 1).reshape(-1, 24)
# calcualte the error of validation set
train_error = {}
for t in range(1000, num_iterations + 1, 1000):
train_error[t] = {}
for appliance_num, appliance in enumerate(ORDER):
train_error[t][appliance] = mean_absolute_error(train_fold[t][appliance_num], train_gt_fold[appliance_num])
valid_error = {}
for t in range(1000, num_iterations + 1, 1000):
valid_error[t] = {}
for appliance_num, appliance in enumerate(ORDER):
valid_error[t][appliance] = mean_absolute_error(valid_fold[t][appliance_num], valid_gt_fold[appliance_num])
test_error = {}
for t in range(1000, num_iterations + 1, 1000):
test_error[t] = {}
for appliance_num, appliance in enumerate(ORDER):
test_error[t][appliance] = mean_absolute_error(test_fold[t][appliance_num], test_gt_fold[appliance_num])
return train_fold, valid_fold, test_fold, train_error, valid_error, test_error, train_losses, valid_losses, test_losses
def _pgd_whitebox(model,
X,
y,
args):
device = X.device
out = model(X)
acc = (out.data.max(1)[1] == y.data).float().mean()
X_pgd = Variable(X.data, requires_grad=True)
batch_size = X.shape[0]
if args.pgd_loss == "baseline":
criterion_ce = nn.CrossEntropyLoss().to(device)
else:
criterion_ce = nn.CrossEntropyLoss(reduction='none').to(device)
max_loss_pgd = torch.zeros(y.shape[0]).cuda()
max_delta = torch.zeros_like(X).cuda()
for zz in range(10):
if args.random_init:
random_noise = torch.FloatTensor(*X_pgd.shape).uniform_(-args.epsilon_eval, args.epsilon_eval).to(device)
X_pgd = Variable(X_pgd.data + random_noise, requires_grad=True)
# ####################################################################
# ########################### PGD LOSS ###############################
# ####################################################################
for _ in range(args.num_steps_eval):
opt = optim.SGD([X_pgd], lr=1e-3)
opt.zero_grad()
perturbation = X_pgd - X
with torch.enable_grad():
outputs_adv = model(X_pgd)
outputs = model(X)
loss_pgd, loss_individual_pgd = pgd_loss_ce(args, outputs_adv, outputs, y, perturbation, criterion_ce)
loss_pgd.backward()
eta = args.step_size_eval * X_pgd.grad.data.sign()
X_pgd = Variable(X_pgd.data + eta, requires_grad=True)
eta = torch.clamp(X_pgd.data - X.data, -args.epsilon_eval, args.epsilon_eval)
X_pgd = Variable(X.data + eta, requires_grad=True)
X_pgd = Variable(torch.clamp(X_pgd, 0, 1.0), requires_grad=True)
"""
margin loss, if enabled
"""
# outputs_adv = model(X_pgd)
# label_mask = nn.functional.one_hot(y, 10).to(torch.bool)
# label_logit = outputs_adv[label_mask]
# others = outputs_adv[~label_mask].reshape(-1, 9)
# top_other_logit, _ = torch.max(others, dim=1)
# all_loss = (top_other_logit - label_logit).detach()
"""
cross entropy loss, if enabled
"""
all_loss = F.cross_entropy(model(X_pgd), y, reduction='none').detach()
delta = X_pgd - X
max_delta[all_loss >= max_loss_pgd] = delta.detach()[all_loss >= max_loss_pgd]
max_loss_pgd = torch.max(max_loss_pgd, all_loss)
outputs_adv = model(X + max_delta)
one_hot_adv = (outputs_adv.data.max(1)[1] == y.data).float().detach().cpu().numpy()
"""
margin loss, if enabled
"""
# label_mask = nn.functional.one_hot(y, 10).to(torch.bool)
# label_logit = outputs_adv[label_mask]
# others = outputs_adv[~label_mask].reshape(-1, 9)
# top_other_logit, _ = torch.max(others, dim=1)
# loss_individual_pgd = (top_other_logit - label_logit).detach().cpu().numpy()
"""
cross entropy loss, if enabled
"""
loss_individual_pgd = F.cross_entropy(outputs_adv, y, reduction='none').detach().cpu().numpy()
outputs = model(X)
one_hot = (outputs.data.max(1)[1] == y.data).float().detach().cpu().numpy()
"""
margin loss, if enabled
"""
# label_mask = nn.functional.one_hot(y, 10).to(torch.bool)
# label_logit = outputs[label_mask]
# others = outputs[~label_mask].reshape(-1, 9)
# top_other_logit, _ = torch.max(others, dim=1)
# loss_individual = (top_other_logit - label_logit).detach().cpu().numpy()
"""
cross entropy loss, if enabled
"""
loss_individual = F.cross_entropy(outputs, y, reduction='none').detach().cpu().numpy()
perturbation = max_delta
margin, weights_margin = compute_weights(args, outputs_adv, perturbation, y)
acc = np.mean(one_hot)
return margin, weights_margin, one_hot, one_hot_adv, acc, loss_individual, loss_individual_pgd
def test_model_fit(model_fit: bool):
"""
Test that controls that scVI inferred distributions make sense on a non-trivial synthetic
dataset.
We define technical zeros of the synthetic dataset as the zeros that result from
highly expressed genes (relatively to the considered cell) and the biological zeros as the
rest of the zeros
:return: None
"""
print("model_fit set to : ", model_fit)
folder = "/tmp/scVI_zeros_test"
print("Saving graphs in : {}".format(folder))
if not os.path.exists(folder):
os.makedirs(folder)
n_epochs = 150 if model_fit else 1
n_mc_sim_total = 100 if model_fit else 1
n_cells_cluster = 1000 if model_fit else 100
torch.manual_seed(seed=42)
synth_data = ZISyntheticDatasetCorr(
n_clusters=8,
n_genes_high=15,
n_overlap=8,
lam_0=320,
n_cells_cluster=n_cells_cluster,
weight_high=1.714286,
weight_low=1,
dropout_coef_low=0.08,
dropout_coef_high=0.05,
)
is_high = synth_data.is_highly_exp.squeeze()
poisson_params_gt = synth_data.exprs_param.squeeze()
# Step 2: Training scVI model
mdl = VAE(
n_input=synth_data.nb_genes,
n_batch=synth_data.n_batches,
reconstruction_loss="zinb",
n_latent=5,
)
trainer = UnsupervisedTrainer(
model=mdl, gene_dataset=synth_data, use_cuda=True, train_size=1.0
)
trainer.train(n_epochs=n_epochs, lr=1e-3)
full = trainer.create_posterior(
trainer.model, synth_data, indices=np.arange(len(synth_data))
)
# Step 3: Inference
poisson_params = []
p_dropout_infered = []
latent_reps = []
bio_zero_p = []
tech_zero_p = []
with torch.no_grad():
for tensors in full.sequential():
# TODO: Properly sample posterior
sample_batch, _, _, batch_index, labels = tensors
px_scale, px_dispersion, px_rate, px_dropout, qz_m, qz_v, z, ql_m, ql_v, library = mdl.inference(
sample_batch, batch_index
)
p_zero = 1.0 / (1.0 + torch.exp(-px_dropout))
p_dropout_infered.append(p_zero.cpu().numpy())
l_train_batch = torch.zeros(
(sample_batch.size(0), sample_batch.size(1), n_mc_sim_total),
device=sample_batch.device,
)
for n_mc_sim in range(n_mc_sim_total):
p = px_rate / (px_rate + px_dispersion)
r = px_dispersion
l_train = torch.distributions.Gamma(
concentration=r, rate=(1 - p) / p
).sample()
l_train = torch.clamp(l_train, max=1e18)
X = torch.distributions.Poisson(l_train).sample()
l_train_batch[:, :, n_mc_sim] = l_train
p_zero = 1.0 / (1.0 + torch.exp(-px_dropout))
random_prob = torch.rand_like(p_zero)
X[random_prob <= p_zero] = 0
l_train_batch = torch.mean(l_train_batch, dim=(-1))
bio_zero_prob_batch = torch.exp(-l_train_batch)
tech_zero_prob_batch = p_zero
bio_zero_p.append(bio_zero_prob_batch.cpu().numpy())
tech_zero_p.append(tech_zero_prob_batch.cpu().numpy())
latent_reps.append(z.cpu().numpy())
poisson_params.append(l_train_batch.cpu().numpy())
latent_reps = np.concatenate(latent_reps)
bio_zero_p = np.concatenate(bio_zero_p)
tech_zero_p = np.concatenate(tech_zero_p)
bio_zero_tech_no = bio_zero_p * (1.0 - tech_zero_p)
tech_zero_bio_no = (1.0 - bio_zero_p) * tech_zero_p
# Final Step: Checking predictions
# Dropout checks
p_dropout_infered_all = np.concatenate(p_dropout_infered)
p_dropout_gt = synth_data.p_dropout.squeeze()
vmin = 0.0
vmax = 2.0 * p_dropout_gt.max()
fig, axes = plt.subplots(ncols=2, nrows=2, figsize=(10, 10))
sns.heatmap(p_dropout_infered_all, vmin=vmin, vmax=vmax, ax=axes[0, 1])
axes[0, 1].set_title("Dropout Rate Predicted")
sns.heatmap(p_dropout_gt, vmin=vmin, vmax=vmax, ax=axes[0, 0])
axes[0, 0].set_title("Dropout Rate GT")
# Poisson Params checks
poisson_params = np.concatenate(poisson_params)
vmin = min(poisson_params_gt.min(), poisson_params.min())
vmax = max(poisson_params_gt.max(), poisson_params.max())
sns.heatmap(poisson_params, vmin=vmin, vmax=vmax, ax=axes[1, 1])
axes[1, 1].set_title("Poisson Distribution Parameter Predicted")
sns.heatmap(poisson_params_gt, vmin=vmin, vmax=vmax, ax=axes[1, 0])
axes[1, 0].set_title("Poisson Distribution Parameter GT")
plt.savefig(os.path.join(folder, "params_comparison.png"))
plt.close()
# TODO: Decrease test tolerances
l1_poisson = np.abs(poisson_params - poisson_params_gt).mean()
if model_fit:
print("Average Poisson L1 error: ", l1_poisson)
assert l1_poisson <= 0.75, "High Error on Poisson parameter inference"
l1_dropout = np.abs(p_dropout_infered_all - synth_data.p_dropout).mean()
print("Average Dropout L1 error: ", l1_dropout)
assert l1_dropout <= 5e-2, "High Error on Dropout parameter inference"
# tSNE plot
print("Computing tSNE rep ...")
x_rep = TSNE(n_components=2).fit_transform(latent_reps)
print("Done!")
pos = np.random.permutation(len(x_rep))[:1000]
labels = ["c_{}".format(idx) for idx in synth_data.labels[pos].squeeze()]
sns.scatterplot(x=x_rep[pos, 0], y=x_rep[pos, 1], hue=labels, palette="Set2")
plt.title("Synthetic Dataset latent space")
plt.savefig(os.path.join(folder, "t_sne.png"))
plt.close()
# Tech/Bio Classif checks
# --For high expressed genes
# ---Poisson nul and ZI non null
print(bio_zero_tech_no[is_high].mean(), synth_data.probas_zero_bio_tech_high[1, 0])
# ---Poisson non nul and .
print(tech_zero_bio_no[is_high].mean(), synth_data.probas_zero_bio_tech_high[0, 1])
# --Low expressed expressend
# ---Poisson nul and ZI non null
print(bio_zero_tech_no[~is_high].mean(), synth_data.probas_zero_bio_tech_low[1, 0])
# ---Poisson non nul and .
print(tech_zero_bio_no[~is_high].mean(), synth_data.probas_zero_bio_tech_low[0, 1])
diff1 = np.abs(
bio_zero_tech_no[is_high].mean() - synth_data.probas_zero_bio_tech_high[1, 0]
)
diff2 = np.abs(
tech_zero_bio_no[is_high].mean() - synth_data.probas_zero_bio_tech_high[0, 1]
)
diff3 = np.abs(
bio_zero_tech_no[~is_high].mean() - synth_data.probas_zero_bio_tech_low[1, 0]
)
diff4 = np.abs(
tech_zero_bio_no[~is_high].mean() - synth_data.probas_zero_bio_tech_low[0, 1]
)
if model_fit:
assert diff1 <= 2e-2
assert diff2 <= 2e-2
assert diff3 <= 2e-2
assert diff4 <= 2e-2
def nms(boxes, scores, overlap=0.5, top_k=200):
"""Apply non-maximum suppression at test time to avoid detecting too many
overlapping bounding boxes for a given object.
Args:
boxes: (tensor) The location preds for the img, Shape: [num_priors,4].
scores: (tensor) The class predscores for the img, Shape:[num_priors].
overlap: (float) The overlap thresh for suppressing unnecessary boxes.
top_k: (int) The Maximum number of box preds to consider.
Return:
The indices of the kept boxes with respect to num_priors.
"""
keep = scores.new(scores.size(0)).zero_().long()
if boxes.numel() == 0:
return keep
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]
area = torch.mul(x2 - x1, y2 - y1)
v, idx = scores.sort(0) # sort in ascending order
# I = I[v >= 0.01]
idx = idx[-top_k:] # indices of the top-k largest vals
xx1 = boxes.new()
yy1 = boxes.new()
xx2 = boxes.new()
yy2 = boxes.new()
w = boxes.new()
h = boxes.new()
# keep = torch.Tensor()
count = 0
while idx.numel() > 0:
i = idx[-1] # index of current largest val
# keep.append(i)
keep[count] = i
count += 1
if idx.size(0) == 1:
break
idx = idx[:-1] # remove kept element from view
# load bboxes of next highest vals
torch.index_select(x1, 0, idx, out=xx1)
torch.index_select(y1, 0, idx, out=yy1)
torch.index_select(x2, 0, idx, out=xx2)
torch.index_select(y2, 0, idx, out=yy2)
# store element-wise max with next highest score
xx1 = torch.clamp(xx1, min=x1[i])
yy1 = torch.clamp(yy1, min=y1[i])
xx2 = torch.clamp(xx2, max=x2[i])
yy2 = torch.clamp(yy2, max=y2[i])
w.resize_as_(xx2)
h.resize_as_(yy2)
w = xx2 - xx1
h = yy2 - yy1
# check sizes of xx1 and xx2.. after each iteration
w = torch.clamp(w, min=0.0)
h = torch.clamp(h, min=0.0)
inter = w * h
# IoU = i / (area(a) + area(b) - i)
rem_areas = torch.index_select(area, 0, idx) # load remaining areas)
union = (rem_areas - inter) + area[i]
IoU = inter / union # store result in iou
# keep only elements with an IoU <= overlap
idx = idx[IoU.le(overlap)]
return keep, count
optimizer.zero_grad()
#Prepare sample and target
image = torch.autograd.Variable(sample_batched['image'].cpu())
depth = torch.autograd.Variable(sample_batched['depth'].cpu())
# print("input", image.shape)
# Normalize depth
depth_n = DepthNorm( depth )
# Predict
output = model(image)
# print("prediction output", output.shape)
# print("GT", depth_n.shape)
# Compute the loss
l_depth = l1_criterion(output, depth_n)
l_ssim = torch.clamp((1 - ssim(output, depth_n, val_range = 1000.0 / 10.0)) * 0.5, 0, 1)
loss = (1.0 * l_ssim.mean().item()) + (0.1 * l_depth)
# Update step
losses.update(loss.data.item(), image.size(0))
loss.backward()
optimizer.step()
# Measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
eta = str(datetime.timedelta(seconds=int(batch_time.val*(N - i))))
# Log progress
def funcOptMax(a, b):
return torch.clamp(a + b, min=0)
def detect(self, sample):
input_pair = []
detection_pair = []
camera = sample[30][0].cuda()
for indexOffset in [0, ]:
images, image_metas, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, gt_parameters, gt_depth, extrinsics, planes, gt_segmentation = sample[indexOffset + 0].cuda(), sample[indexOffset + 1].numpy(), sample[indexOffset + 2].cuda(), sample[indexOffset + 3].cuda(), sample[indexOffset + 4].cuda(), sample[indexOffset + 5].cuda(), sample[indexOffset + 6].cuda(), sample[indexOffset + 7].cuda(), sample[indexOffset + 8].cuda(), sample[indexOffset + 9].cuda(), sample[indexOffset + 10].cuda(), sample[indexOffset + 11].cuda()
rpn_class_logits, rpn_pred_bbox, target_class_ids, mrcnn_class_logits, target_deltas, mrcnn_bbox, target_mask, mrcnn_mask, target_parameters, mrcnn_parameters, detections, detection_masks, detection_gt_parameters, detection_gt_masks, rpn_rois, roi_features, roi_indices, depth_np_pred = self.model.predict([images, image_metas, gt_class_ids, gt_boxes, gt_masks, gt_parameters, camera], mode='inference_detection', use_nms=2, use_refinement=True)
if len(detections) > 0:
detections, detection_masks = unmoldDetections(self.config, camera, detections, detection_masks, depth_np_pred, debug=False)
pass
XYZ_pred, detection_mask, plane_XYZ = calcXYZModule(self.config, camera, detections, detection_masks, depth_np_pred, return_individual=True)
detection_mask = detection_mask.unsqueeze(0)
input_pair.append({'image': images, 'depth': gt_depth, 'mask': gt_masks, 'bbox': gt_boxes, 'extrinsics': extrinsics, 'segmentation': gt_segmentation, 'camera': camera})
if 'nyu_dorn_only' in self.options.dataset:
XYZ_pred[1:2] = sample[27].cuda()
pass
detection_pair.append({'XYZ': XYZ_pred, 'depth': XYZ_pred[1:2], 'mask': detection_mask, 'detection': detections, 'masks': detection_masks, 'depth_np': depth_np_pred, 'plane_XYZ': plane_XYZ})
continue
if ('refine' in self.modelType or 'refine' in self.options.suffix):
pose = sample[26][0].cuda()
pose = torch.cat([pose[0:3], pose[3:6] * pose[6]], dim=0)
pose_gt = torch.cat([pose[0:1], -pose[2:3], pose[1:2], pose[3:4], -pose[5:6], pose[4:5]], dim=0).unsqueeze(0)
camera = camera.unsqueeze(0)
for c in range(1):
detection_dict, input_dict = detection_pair[c], input_pair[c]
new_input_dict = {k: v for k, v in input_dict.items()}
new_input_dict['image'] = (input_dict['image'] + self.config.MEAN_PIXEL_TENSOR.view((-1, 1, 1))) / 255.0 - 0.5
new_input_dict['image_2'] = (sample[13].cuda() + self.config.MEAN_PIXEL_TENSOR.view((-1, 1, 1))) / 255.0 - 0.5
detections = detection_dict['detection']
detection_masks = detection_dict['masks']
depth_np = detection_dict['depth_np']
image = new_input_dict['image']
image_2 = new_input_dict['image_2']
depth_gt = new_input_dict['depth'].unsqueeze(1)
masks_inp = torch.cat([detection_masks.unsqueeze(1), detection_dict['plane_XYZ']], dim=1)
segmentation = new_input_dict['segmentation']
detection_masks = torch.nn.functional.interpolate(detection_masks[:, 80:560].unsqueeze(1), size=(192, 256), mode='nearest').squeeze(1)
image = torch.nn.functional.interpolate(image[:, :, 80:560], size=(192, 256), mode='bilinear')
image_2 = torch.nn.functional.interpolate(image_2[:, :, 80:560], size=(192, 256), mode='bilinear')
masks_inp = torch.nn.functional.interpolate(masks_inp[:, :, 80:560], size=(192, 256), mode='bilinear')
depth_np = torch.nn.functional.interpolate(depth_np[:, 80:560].unsqueeze(1), size=(192, 256), mode='bilinear').squeeze(1)
plane_depth = torch.nn.functional.interpolate(detection_dict['depth'][:, 80:560].unsqueeze(1), size=(192, 256), mode='bilinear').squeeze(1)
segmentation = torch.nn.functional.interpolate(segmentation[:, 80:560].unsqueeze(1).float(), size=(192, 256), mode='nearest').squeeze().long()
new_input_dict['image'] = image
new_input_dict['image_2'] = image_2
results = self.refine_model(image, image_2, camera, masks_inp, detection_dict['detection'][:, 6:9], plane_depth, depth_np)
masks = results[-1]['mask'].squeeze(1)
all_masks = torch.softmax(masks, dim=0)
masks_small = all_masks[1:]
all_masks = torch.nn.functional.interpolate(all_masks.unsqueeze(1), size=(480, 640), mode='bilinear').squeeze(1)
all_masks = (all_masks.max(0, keepdim=True)[1] == torch.arange(len(all_masks)).cuda().long().view((-1, 1, 1))).float()
masks = all_masks[1:]
detection_masks = torch.zeros(detection_dict['masks'].shape).cuda()
detection_masks[:, 80:560] = masks
detection_dict['masks'] = detection_masks
detection_dict['depth_ori'] = detection_dict['depth'].clone()
detection_dict['mask'][:, 80:560] = (masks.max(0, keepdim=True)[0] > (1 - masks.sum(0, keepdim=True))).float()
if self.options.modelType == 'fitting':
masks_cropped = masks_small
ranges = self.config.getRanges(camera).transpose(1, 2).transpose(0, 1)
XYZ = torch.nn.functional.interpolate(ranges.unsqueeze(1), size=(192, 256), mode='bilinear').squeeze(1) * results[-1]['depth'].squeeze(1)
detection_areas = masks_cropped.sum(-1).sum(-1)
A = masks_cropped.unsqueeze(1) * XYZ
b = masks_cropped
Ab = (A * b.unsqueeze(1)).sum(-1).sum(-1)
AA = (A.unsqueeze(2) * A.unsqueeze(1)).sum(-1).sum(-1)
plane_parameters = torch.stack([torch.matmul(torch.inverse(AA[planeIndex]), Ab[planeIndex]) if detection_areas[planeIndex] else detection_dict['detection'][planeIndex, 6:9] for planeIndex in range(len(AA))], dim=0)
plane_offsets = torch.norm(plane_parameters, dim=-1, keepdim=True)
plane_parameters = plane_parameters / torch.clamp(torch.pow(plane_offsets, 2), 1e-4)
detection_dict['detection'][:, 6:9] = plane_parameters
XYZ_pred, detection_mask, plane_XYZ = calcXYZModule(self.config, camera, detection_dict['detection'], detection_masks, detection_dict['depth'], return_individual=True)
detection_dict['depth'] = XYZ_pred[1:2]
pass
continue
pass
return detection_pair
def postprocess(im, invert=False):
im = th.clamp((im + 1.0) / 2.0, 0, 1)
if invert:
im = (1.0 - im)
im = ttools.tensor2image(im)
return im
def forward(self, x, target=None):
# backbone
c2, c3, c4, c5 = self.backbone(x)
B = c5.size(0)
# bottom-up
p5 = self.spp(c5)
p4 = self.deconv5(p5)
p3 = self.deconv4(p4)
p2 = self.deconv3(p3)
# head
cls_pred = self.cls_pred(p2)
txty_pred = self.txty_pred(p2)
twth_pred = self.twth_pred(p2)
# train
if self.trainable:
# [B, H*W, num_classes]
cls_pred = cls_pred.permute(0, 2, 3, 1).contiguous().view(B, -1, self.num_classes)
# [B, H*W, 2]
txty_pred = txty_pred.permute(0, 2, 3, 1).contiguous().view(B, -1, 2)
# [B, H*W, 2]
twth_pred = twth_pred.permute(0, 2, 3, 1).contiguous().view(B, -1, 2)
# compute loss
cls_loss, txty_loss, twth_loss, total_loss = tools.loss(pred_cls=cls_pred,
pred_txty=txty_pred,
pred_twth=twth_pred,
label=target,
num_classes=self.num_classes
)
return cls_loss, txty_loss, twth_loss, total_loss
# test
else:
with torch.no_grad():
# batch_size = 1
cls_pred = torch.sigmoid(cls_pred)
# simple nms
hmax = F.max_pool2d(cls_pred, kernel_size=5, padding=2, stride=1)
keep = (hmax == cls_pred).float()
cls_pred *= keep
# decode box
txtytwth_pred = torch.cat([txty_pred, twth_pred], dim=1).permute(0, 2, 3, 1).contiguous().view(B, -1, 4)
# [B, H*W, 4] -> [H*W, 4]
bbox_pred = torch.clamp((self.decode_boxes(txtytwth_pred) / self.scale_torch)[0], 0., 1.)
# topk
topk_scores, topk_inds, topk_clses = self._topk(cls_pred)
topk_scores = topk_scores[0].cpu().numpy()
topk_ind = topk_clses[0].cpu().numpy()
topk_bbox_pred = bbox_pred[topk_inds[0]].cpu().numpy()
if self.use_nms:
# nms
keep = np.zeros(len(topk_bbox_pred), dtype=np.int)
for i in range(self.num_classes):
inds = np.where(topk_ind == i)[0]
if len(inds) == 0:
continue
c_bboxes = topk_bbox_pred[inds]
c_scores = topk_scores[inds]
c_keep = self.nms(c_bboxes, c_scores)
keep[inds[c_keep]] = 1
keep = np.where(keep > 0)
topk_bbox_pred = topk_bbox_pred[keep]
topk_scores = topk_scores[keep]
topk_ind = topk_ind[keep]
return topk_bbox_pred, topk_scores, topk_ind
def earlystop(model,
data,
target,
step_size,
epsilon,
perturb_steps,
tau,
randominit_type,
loss_fn,
rand_init=True,
omega=0):
'''
The implematation of early-stopped PGD
Following the Alg.1 in our FAT paper <https://arxiv.org/abs/2002.11242>
:param step_size: the PGD step size
:param epsilon: the perturbation bound
:param perturb_steps: the maximum PGD step
:param tau: the step controlling how early we should stop interations when wrong adv data is found
:param randominit_type: To decide the type of random inirialization (random start for searching adv data)
:param rand_init: To decide whether to initialize adversarial sample with random noise (random start for searching adv data)
:param omega: random sample parameter for adv data generation (this is for escaping the local minimum.)
:return: output_adv (friendly adversarial data) output_target (targets), output_natural (the corresponding natrual data), count (average backword propagations count)
'''
model.eval()
K = perturb_steps
count = 0
output_target = []
output_adv = []
output_natural = []
control = (torch.ones(len(target)) * tau).cuda()
# Initialize the adversarial data with random noise
if rand_init:
if randominit_type == "normal_distribution_randominit":
iter_adv = data.detach() + 0.001 * torch.randn(
data.shape).cuda().detach()
iter_adv = torch.clamp(iter_adv, 0.0, 1.0)
if randominit_type == "uniform_randominit":
iter_adv = data.detach() + torch.from_numpy(
np.random.uniform(-epsilon, epsilon,
data.shape)).float().cuda()
iter_adv = torch.clamp(iter_adv, 0.0, 1.0)
else:
iter_adv = data.cuda().detach()
iter_clean_data = data.cuda().detach()
iter_target = target.cuda().detach()
output_iter_clean_data = model(data)
while K > 0:
iter_adv.requires_grad_()
output = model(iter_adv)
pred = output.max(1, keepdim=True)[1]
output_index = []
iter_index = []
# Calculate the indexes of adversarial data those still needs to be iterated
for idx in range(len(pred)):
if pred[idx] != target[idx]:
if control[idx] == 0:
output_index.append(idx)
else:
control[idx] -= 1
iter_index.append(idx)
else:
iter_index.append(idx)
# Add adversarial data those do not need any more iteration into set output_adv
if len(output_index) != 0:
if len(output_target) == 0:
# incorrect adv data should not keep iterated
output_adv = iter_adv[output_index].reshape(-1, 3, 32,
32).cuda()
output_natural = iter_clean_data[output_index].reshape(
-1, 3, 32, 32).cuda()
output_target = iter_target[output_index].reshape(-1).cuda()
else:
# incorrect adv data should not keep iterated
output_adv = torch.cat(
(output_adv, iter_adv[output_index].reshape(-1, 3, 32,
32).cuda()),
dim=0)
output_natural = torch.cat(
(output_natural, iter_clean_data[output_index].reshape(
-1, 3, 32, 32).cuda()),
dim=0)
output_target = torch.cat(
(output_target,
iter_target[output_index].reshape(-1).cuda()),
dim=0)
# calculate gradient
model.zero_grad()
with torch.enable_grad():
if loss_fn == "cent":
loss_adv = nn.CrossEntropyLoss(reduction='mean')(output,
iter_target)
if loss_fn == "kl":
criterion_kl = nn.KLDivLoss(size_average=False).cuda()
loss_adv = criterion_kl(
F.log_softmax(output, dim=1),
F.softmax(output_iter_clean_data, dim=1))
loss_adv.backward(retain_graph=True)
grad = iter_adv.grad
# update iter adv
if len(iter_index) != 0:
control = control[iter_index]
iter_adv = iter_adv[iter_index]
iter_clean_data = iter_clean_data[iter_index]
iter_target = iter_target[iter_index]
output_iter_clean_data = output_iter_clean_data[iter_index]
grad = grad[iter_index]
eta = step_size * grad.sign()
iter_adv = iter_adv.detach() + eta + omega * torch.randn(
iter_adv.shape).detach().cuda()
iter_adv = torch.min(
torch.max(iter_adv, iter_clean_data - epsilon),
iter_clean_data + epsilon)
iter_adv = torch.clamp(iter_adv, 0, 1)
count += len(iter_target)
else:
return output_adv, output_target, output_natural, count
K = K - 1
if len(output_target) == 0:
output_target = iter_target.reshape(-1).squeeze().cuda()
output_adv = iter_adv.reshape(-1, 3, 32, 32).cuda()
output_natural = iter_clean_data.reshape(-1, 3, 32, 32).cuda()
else:
output_adv = torch.cat((output_adv, iter_adv.reshape(-1, 3, 32, 32)),
dim=0).cuda()
output_target = torch.cat((output_target, iter_target.reshape(-1)),
dim=0).squeeze().cuda()
output_natural = torch.cat(
(output_natural, iter_clean_data.reshape(-1, 3, 32, 32).cuda()),
dim=0).cuda()
return output_adv, output_target, output_natural, count
def update_policy(args,
get_loss,
get_kl,
policy_net,
policy_optimizer=None,
value_net=None,
likelihood=None):
"""Updates parameters of policy network according to optimization scheme."""
# This method is used for updating the policy parameters based on the selected policy gradient setting.
grads = torch.autograd.grad(get_loss(), policy_net.parameters())
loss_grad = torch.cat([grad.view(-1) for grad in grads]).data
step_size = None
def Fvp(v):
# Computes Fisher vector product as a Hessian vector product of KL divergence (same as the trick used in TRPO)
kl = get_kl()
kl = kl.mean()
grads = torch.autograd.grad(kl,
policy_net.parameters(),
create_graph=True)
flat_grad_kl = torch.cat([grad.view(-1) for grad in grads])
kl_v = (flat_grad_kl * Variable(v)).sum()
grads = torch.autograd.grad(kl_v, policy_net.parameters())
flat_grad_grad_kl = torch.cat(
[grad.contiguous().view(-1) for grad in grads]).data
return flat_grad_grad_kl + v * args.damping # damping ensures numerical stability and faster convergence
if args.pg_algorithm == "vanilla": # Computes conventional policy gradient
if args.pg_estimator == 'BQ' and args.UAPG_flag:
# Computes the low-rank SVD of the covariance matrix for vanilla PG, without constructing it in the first place.
# Specifically, we utilize fast covariance vector products for fast low-rank SVD computation.
u_cov, s_cov, v_cov = fast_svd.pca_Cov(args, conjugate_gradients,
Fvp, policy_net, value_net,
likelihood)
# Lowering the step-size of the gradient components with high estimation uncertainty.
new_s_cov = 1 - torch.sqrt(s_cov.min()) / torch.sqrt(s_cov)
# Final UAPG update for vanilla policy gradient
loss_grad = loss_grad - torch.matmul(
u_cov,
torch.matmul(torch.diag(new_s_cov),
torch.matmul(v_cov, loss_grad)))
policy_optimizer.zero_grad()
set_flat_grad_to(policy_net, loss_grad)
policy_optimizer.step()
else: # Computes natural policy gradient, for NPG and TRPO
neg_stepdir = conjugate_gradients(Fvp,
loss_grad,
50,
device=args.device)
if args.pg_estimator == 'BQ' and args.UAPG_flag:
# Computes the low-rank SVD of the inverse Covariance matrix for the natural gradient.
u_cov, s_cov, v_cov = fast_svd.pca_NPG_InvCov(
args, conjugate_gradients, Fvp, policy_net, value_net,
likelihood)
# Increasing the step-size of the gradient components with low estimation uncertainty (most confident directions).
new_s_cov = torch.clamp(torch.sqrt(s_cov / s_cov.min()), 1,
args.UAPG_epsilon) - 1
# Final UAPG update for natural policy gradient
neg_stepdir = neg_stepdir + torch.matmul(
u_cov,
torch.matmul(torch.diag(new_s_cov),
torch.matmul(v_cov, neg_stepdir)))
if args.pg_algorithm == "NPG": # NPG update
old_params = get_flat_params_from(policy_net)
policy_optimizer.zero_grad()
set_flat_grad_to(policy_net, neg_stepdir)
policy_optimizer.step()
new_params = get_flat_params_from(policy_net)
else: # TRPO update
stepdir = -neg_stepdir # Search direction after solving the constrained optimization problem, same as natural gradient
shs = 0.5 * (stepdir * Fvp(stepdir)).sum(0, keepdim=True) ## Important, here is the TRPO magic
lm = torch.sqrt(shs / args.max_kl) # One over largest step size/ trust region size
beta = torch.sqrt(args.max_kl/shs)
step_size = 1 / lm
fullstep = stepdir / lm[
0] # Naive trust region based update corresponding to the largest step size
# print(fullstep)
neggdotstepdir = (stepdir * Fvp(stepdir)).sum(0, keepdim=True)
# print(neggdotstepdir)
prev_params = get_flat_params_from(policy_net)
# Line search avoids large policy steps that result in catastrophic performance degradation.
success, new_params = linesearch(policy_net, get_loss, prev_params,
fullstep, neggdotstepdir / lm[0])
set_flat_params_to(policy_net, new_params)
return step_size
def run(self, frame, frames):
if self.skip_flag == 0:
inp_dim = int(self.model.net_info["height"])
assert inp_dim % 32 == 0
assert inp_dim > 32
self.model.eval()
img, orig_im, dim = prep_image(frame, inp_dim)
im_dim = torch.FloatTensor(dim).repeat(1, 2)
if self.CUDA:
im_dim = im_dim.cuda()
img = img.cuda()
with torch.no_grad():
output = self.model(Variable(img), self.CUDA)
output = write_results(output,
self.confidence,
self.num_classes,
nms=True,
nms_conf=self.nms_thesh)
if type(output) == int:
return frame
im_dim = im_dim.repeat(output.size(0), 1)
scaling_factor = torch.min(inp_dim / im_dim, 1)[0].view(-1, 1)
output[:,
[1, 3]] -= (inp_dim -
scaling_factor * im_dim[:, 0].view(-1, 1)) / 2
output[:,
[2, 4]] -= (inp_dim -
scaling_factor * im_dim[:, 1].view(-1, 1)) / 2
output[:, 1:5] /= scaling_factor
for i in range(output.shape[0]):
output[i, [1, 3]] = torch.clamp(output[i, [1, 3]], 0.0,
im_dim[i, 0])
output[i, [2, 4]] = torch.clamp(output[i, [2, 4]], 0.0,
im_dim[i, 1])
classes = load_classes('data/coco.names')
colors = pkl.load(open("pallete", "rb"))
#print('output: {}'.format(output.shape[0]))
for i in range(output.shape[0]):
data_list = np.array([[
frames,
int(output[i, 1]),
int(output[i, 3]),
int(output[i, 2]),
int(output[i, 4]), classes[int(output[i, 7])]
]])
self.data = np.vstack([self.data, data_list])
#print(self.data)
#print(self.data)
list(
map(lambda x: write(
x,
orig_im,
classes,
colors,
frames,
), output))
return orig_im
def forward(self,
article_sentences,
article_sentences_lengths,
num_codes,
codes=None,
code_description=None,
code_description_length=None,
linearized_codes=None,
linearized_codes_lengths=None,
linearized_descriptions=None,
linearized_descriptions_lengths=None):
nq = num_codes.max()
nq_temp = self.codes_per_checkpoint
scores, word_level_attentions, traceback_word_level_attentions, sentence_level_scores = [], [], [], []
for offset in range(0, nq, nq_temp):
if codes is not None:
codes_temp = codes[:, offset:offset + nq_temp]
else:
codes_temp = torch.zeros(0)
if code_description is not None:
code_description_temp = code_description[:, offset:offset +
nq_temp]
code_description_length_temp = code_description_length[:,
offset:
offset +
nq_temp]
else:
code_description_temp = torch.zeros(0)
code_description_length_temp = torch.zeros(0)
if linearized_codes is not None:
linearized_codes_temp = linearized_codes[:, offset:offset +
nq_temp]
linearized_codes_lengths_temp = linearized_codes_lengths[:,
offset:
offset
+
nq_temp]
else:
linearized_codes_temp = torch.zeros(0)
linearized_codes_lengths_temp = torch.zeros(0)
num_codes_temp = torch.clamp(num_codes - offset, 0, nq_temp)
scores_temp, word_level_attentions_temp, traceback_word_level_attentions_temp, sentence_level_scores_temp = checkpoint(
self.inner_forward, article_sentences,
article_sentences_lengths, num_codes_temp, codes_temp,
code_description_temp, code_description_length_temp,
linearized_codes_temp, linearized_codes_lengths_temp,
*self.parameters())
scores.append(scores_temp)
word_level_attentions.append(word_level_attentions_temp)
traceback_word_level_attentions.append(
traceback_word_level_attentions_temp)
sentence_level_scores.append(sentence_level_scores_temp)
scores = torch.cat(scores, 1)
word_level_attentions = torch.cat(word_level_attentions, 1)
traceback_word_level_attentions = torch.cat(
traceback_word_level_attentions, 1)
sentence_level_scores = torch.cat(sentence_level_scores, 1)
return_dict = dict(
scores=scores,
num_codes=num_codes,
word_level_attentions=word_level_attentions,
traceback_word_level_attentions=traceback_word_level_attentions,
sentence_level_scores=sentence_level_scores,
article_sentences_lengths=article_sentences_lengths)
if codes is not None:
return_dict['codes'] = codes
return return_dict
def clip(self, x, clipping_value):
return torch.clamp(x, min=-clipping_value, max=clipping_value)
def funcOptMin(a, b):
return torch.clamp(a + b, max=2)
def f_p(self, p):
# Calculate activation for inferred grounded location, using a leaky relu for sparsity. Either apply to full multi-frequency grounded location or single frequency module
activation = [utils.leaky_relu(torch.clamp(p_f, min=-1, max=1)) for p_f in p] if type(p) is list else utils.leaky_relu(torch.clamp(p, min=-1, max=1))
return activation
def func2(a, b):
return torch.clamp(a + b, min=0, max=2)
def f_g_clamp(self, g):
# Calculate activation for abstract location, thresholding between -1 and 1
activation = [torch.clamp(g_f, min=-1, max=1) for g_f in g]
return activation
def acquire_scores(base_cfg, samples_to_score, all_samples, model_file, depth_ifp_w=0, verbose=False):
calc_depth_distances = depth_ifp_w > 0
depth_lambda = base_cfg["label_selection"]["depth_lambda"]
entropy_lambda = base_cfg["label_selection"]["entropy_lambda"]
dist_bias_weight = base_cfg["label_selection"]["bias_weight"]
ifp_args = base_cfg["label_selection"]["ifp_args"]
if not verbose:
if isinstance(depth_lambda, list):
for dl, el in zip(depth_lambda, entropy_lambda):
assert dl + el > 0
else:
assert depth_lambda + entropy_lambda > 0 or calc_depth_distances
if calc_depth_distances and ifp_args["m"] in ["aspp", "u4", "u3", "bn"]:
depth_teacher = build_depth_trainer_model(base_cfg)
cfg = deepcopy(base_cfg)
cfg['data']['augmentations'] = {}
cfg['monodepth_options'].pop('crop_h')
cfg['monodepth_options'].pop('crop_w')
cfg['training']['batch_size'] = 1
cfg['data']['shuffle_trainset'] = False
restrict_subset = all_samples if calc_depth_distances else samples_to_score
cfg['data']['restrict_to_subset'] = {
"mode": "fixed",
"n_subset": len(restrict_subset),
"subset": restrict_subset,
}
cfg["training"]["resume"] = model_file
trainer = build_trainer(cfg, "label_selection_scoring")
if cfg["training"]["resume"] is not None:
trainer.load_resume(strict=True, load_model_only=True)
else:
print("LABEL_SELECTION: Warning - Evaluated model is None. This might happen when using ifp.")
scores = []
all_depth_features = []
dist_i_to_img_idx = {}
img_idx_to_dist_i = {}
dist_bias = []
trainer.model.eval()
with torch.no_grad():
depth_loss_mask = None
for inputs in tqdm(trainer.train_data_loader):
for k, v in inputs.items():
cuda_tensor_names = [("color_aug", 0, 0), "pseudo_depth"]
if verbose:
cuda_tensor_names.extend(["lbl", ("color", 0, 0)])
if torch.is_tensor(v) and k in cuda_tensor_names:
inputs[k] = v.to(trainer.device, non_blocking=True)
if calc_depth_distances:
if ifp_args["pool"] == "avg":
pool_fn = torch.nn.functional.adaptive_avg_pool2d
elif ifp_args["pool"] == "max":
pool_fn = torch.nn.functional.adaptive_max_pool2d
else:
raise NotImplementedError(ifp_args["pool"])
if ifp_args["m"] in ["aspp", "u3", "u4", "bn"]:
teacher_outputs = depth_teacher(inputs)
if ifp_args["m"] == "u3":
depth_features = teacher_outputs[("upconv", 3)]
elif ifp_args["m"] == "u4":
depth_features = teacher_outputs[("upconv", 4)]
elif ifp_args["m"] == "bn":
depth_features = teacher_outputs["bottleneck"]
else:
raise NotImplementedError(ifp_args["m"])
depth_features = pool_fn(depth_features, (ifp_args["h"], 2 * ifp_args["h"]))
elif ifp_args["m"] == "logdepth":
depth_features = inputs["pseudo_depth"][0]
depth_features = torch.log(torch.clamp(1 / depth_features, 0.1, 80))
depth_features = pool_fn(depth_features, (ifp_args["h"], 2 * ifp_args["h"]))
depth_features.unsqueeze_(0)
elif ifp_args["m"] == "depth":
depth_features = inputs["pseudo_depth"][0]
depth_features = torch.clamp(1 / depth_features, 0.1, 80)
depth_features = pool_fn(depth_features, (ifp_args["h"], 2 * ifp_args["h"]))
depth_features.unsqueeze_(0)
else:
raise NotImplementedError(ifp_args["m"])
assert depth_features.shape[0] == 1
dist_i_to_img_idx[len(all_depth_features)] = inputs["idx"].item()
img_idx_to_dist_i[inputs["idx"].item()] = len(all_depth_features)
all_depth_features.append(depth_features.detach())
if not verbose and dist_bias_weight == 0:
scores.append({
"idx": inputs["idx"],
"label_criterion": [0],
"depth_error": [0],
"entropy_mean": 0,
})
continue
with autocast(enabled=trainer.cfg["training"]["amp"]):
outputs = trainer.model(inputs)
if inputs["idx"] not in samples_to_score:
dist_bias.append(0)
continue
entropy_imgs = pixel_wise_entropy(outputs["semantics"])
disp_pred = outputs["disp", 0][0][0]
disp_pseudo = inputs["pseudo_depth"][0][0]
depth_error_maps = []
depth_errors = []
depth_error_types = cfg["label_selection"].get("depth_error_types", "abs")
if not isinstance(depth_error_types, list):
depth_error_types = [depth_error_types]
for depth_error_type in depth_error_types:
if depth_error_type == "abs":
depth_error_map = torch.abs(disp_pred - disp_pseudo)
elif depth_error_type == "abs_inv_log":
depth_pred = torch.log(torch.clamp(1 / disp_pred, 0.1, 80))
depth_pseudo = torch.log(torch.clamp(1 / disp_pseudo, 0.1, 80))
depth_error_map = torch.abs(depth_pseudo - depth_pred)
elif depth_error_type == "abs_inv":
depth_pred = torch.clamp(1 / disp_pred, 0.1, 80)
depth_pseudo = torch.clamp(1 / disp_pseudo, 0.1, 80)
depth_error_map = torch.abs(depth_pseudo - depth_pred)
elif depth_error_type == "sq":
depth_error_map = (disp_pred - disp_pseudo) ** 2
elif depth_error_type == "abs_rel":
depth_error_map = torch.abs(disp_pred - disp_pseudo) / (disp_pseudo + 1e-1)
elif depth_error_type == "sq_rel":
depth_error_map = ((disp_pred - disp_pseudo) ** 2) / (disp_pseudo + 1e-1)
elif depth_error_type == "abs_log":
depth_error_map = torch.abs(torch.log(1 + disp_pred) - torch.log(1 + disp_pseudo))
else:
raise NotImplementedError(depth_error_type)
# Mask out cars moving in front with very small disparity
mask = dilate((disp_pseudo < 0.07).float(), 7, 3)
depth_error_map *= (1 - mask)
# Mask out own car
depth_error_map[int(0.87 * depth_error_map.shape[0]):, :] = 0
depth_error = torch.mean(depth_error_map)
depth_error_maps.append(depth_error_map.detach())
depth_errors.append(depth_error.detach())
entropy_mean = torch.mean(entropy_imgs[0])
assert not (isinstance(depth_lambda, list) and len(depth_error_types) > 1)
if isinstance(depth_lambda, list):
label_criterion = []
for dl, el in zip(depth_lambda, entropy_lambda):
label_criterion.append((dl * depth_error + el * entropy_mean).detach())
depth_error_maps.append(depth_error_map)
depth_errors.append(depth_error)
elif isinstance(depth_error_types, list):
label_criterion = []
for depth_error in depth_errors:
label_criterion.append((depth_lambda * depth_error + entropy_lambda * entropy_mean).detach())
else:
label_criterion = (depth_lambda * depth_error + entropy_lambda * entropy_mean).detach()
if dist_bias_weight > 0:
assert len(label_criterion) == 1
dist_bias.append(dist_bias_weight * label_criterion[0])
scores.append({
"idx": inputs["idx"],
"label_criterion": label_criterion,
"depth_error": depth_errors,
"entropy_mean": entropy_mean.detach(),
})
if verbose:
segmentation_loss = trainer.loss_fn(
input=outputs["semantics"], target=inputs["lbl"],
pixel_weights=None
)
preds = outputs["semantics"].data.max(1)[1].cpu().numpy()
gts = inputs["lbl"].data.cpu().numpy()
for k, v in outputs.items():
if "depth" in k or "cam_T_cam" in k:
outputs[k] = v.to(torch.float32)
# trainer.monodepth_loss_calculator_train.generate_images_pred(all_inputs, outputs)
# mono_losses = trainer.monodepth_loss_calculator_train.compute_losses(all_inputs, outputs)
# mono_loss = mono_losses["loss"]
mono_loss = torch.tensor([0])
mono_outputs = trainer.model.predict_test_disp(inputs)
trainer.monodepth_loss_calculator_val.generate_depth_test_pred(mono_outputs)
# Crop away bottom of image with own car
if depth_loss_mask is None:
depth_loss_mask = torch.ones(outputs["disp", 0].shape, device=trainer.device)
depth_loss_mask[:, :, int(outputs["disp", 0].shape[2] * 0.9):, :] = 0
pseudo_depth_loss = berhu(outputs["disp", 0], inputs["pseudo_depth"], depth_loss_mask)
running_metrics_val = runningScore(trainer.n_classes)
running_metrics_val.update(gts, preds)
score, class_iou = running_metrics_val.get_scores()
scores[-1].update({
"image": inputs["color_aug", 0, 0][0].detach().cpu(),
"segmentation_entropy": entropy_imgs[0].detach().cpu(),
"disparity": torch.log(torch.clamp(1 / outputs["disp", 0][0], 0.1, 80)).detach().cpu(),
"teacher_depth": torch.log(torch.clamp(1 / inputs["pseudo_depth"][0], 0.1, 80)).detach().cpu(),
"depth_error_map": depth_error_maps,
"mIoU": score["Mean IoU : \t"],
"fwAcc": score["FreqW Acc : \t"],
"mAcc": score["Mean Acc : \t"],
"tAcc": score["Overall Acc: \t"],
"cIoU": class_iou,
"segmentation_loss": segmentation_loss.item(),
"mono_loss": mono_loss.item(),
"pseudo_depth_loss": pseudo_depth_loss.item(),
"segmentation_pred": preds[0],
"segmentation_gt": gts[0],
# "reprojection_error_map": outputs["to_optimise/0"][0].detach().cpu(),
})
depth_feature_distances = 0
if calc_depth_distances:
depth_feature_distances = _calc_feature_distance(all_depth_features, dist_bias, dist_bias_weight,
p=ifp_args["p"],
normalize_features=ifp_args.get("norm", False),
patch_wise=ifp_args.get("pw", False))
feature_distances = depth_ifp_w * depth_feature_distances
return scores, {'distances': feature_distances, 'dist_i_to_img_idx': dist_i_to_img_idx,
'img_idx_to_dist_i': img_idx_to_dist_i}
def backward(self, sample_):
sample_["logp"] = self.pd.log_prob(self.action)
sample_["value"] = self.Q
self.replay_buffer.push(sample_)
self.running_step += 1
"""""" """""" ""
"training part"
"in each step, we train for batch batch_training_times"
"""""" """""" ""
if self.step > self.learning_starts:
if self.running_step % self.run_step == 0 and self.training_step == 0:
" sample advantage generate "
with torch.no_grad():
sample = self.replay_buffer.recent_step_sample(
self.running_step)
last_value = self.value_model.forward(sample["s_"][-1])
self.record_sample = gae(sample, last_value, self.gamma,
self.lam)
self.running_step = 0
if self.training_step < self.sample_training_step and self.record_sample is not None:
pg_loss_re = 0
entropy_re = 0
vf_loss_re = 0
loss_re = 0
for _ in range(self.batch_training_round):
index = self.train_ticks[self.training_step]
S = self.record_sample["s"][index].detach()
A = self.record_sample["a"][index].detach()
old_log = self.record_sample["logp"][index].detach()
advs = self.record_sample["advs"][index].detach()
value = self.record_sample["value"][index].detach()
returns = self.record_sample["return"][index].detach()
# generate Policy gradient loss
outcome = self.run_policy.forward(S)
new_policy = self.dist(outcome)
new_lop = new_policy.log_prob(A)
ratio = torch.exp(new_lop - old_log)
pg_loss1 = advs * ratio
pg_loss2 = advs * torch.clamp(ratio, 1.0 - self.cliprange,
1.0 + self.cliprange)
pg_loss = -.5 * torch.min(pg_loss1, pg_loss2).mean()
# value loss
value_now = self.run_value.forward(S)
value_clip = value + torch.clamp(
value_now - value,
min=-self.cliprange,
max=self.cliprange) # Clipped value
vf_loss1 = self.loss_cal(value_now,
returns) # Unclipped loss
vf_loss2 = self.loss_cal(value_clip,
returns) # clipped loss
vf_loss = .5 * torch.max(vf_loss1, vf_loss2)
# vf_loss = 0.5 * vf_loss1
# entropy
entropy = new_policy.entropy().mean()
loss = pg_loss - entropy * self.ent_coef + vf_loss * self.vf_coef
# approxkl = self.loss_cal(neg_log_pac, self.record_sample["neglogp"])
# self.cliprange = torch.gt(torch.abs(ratio - 1.0).mean(), self.cliprange)
self.value_model_optim.zero_grad()
loss.backward(retain_graph=True)
self.value_model_optim.step()
self.policy_model_optim.zero_grad()
loss.backward()
self.policy_model_optim.step()
self.training_step += 1
pg_loss_re += pg_loss.data.numpy()
entropy_re += entropy.data.numpy()
vf_loss_re += vf_loss.data.numpy()
loss_re += loss.data.numpy()
if self.training_step == self.sample_training_step:
print("the" + str(self.episode) +
" round have training finished")
self.run_policy.load_state_dict(
self.policy_model.state_dict())
self.run_value.load_state_dict(
self.value_model.state_dict())
self.training_step = 0
self.record_sample = None
return loss_re, {
"pg_loss": pg_loss_re,
"entropy": entropy_re,
"vf_loss": vf_loss_re
}
return 0, {"pg_loss": 0, "entropy": 0, "vf_loss": 0}
init_states[:, n_link + which] = mesh[1].reshape(-1)
n_steps = 2000
ROApoints = []
grads = []
for n in range(n_samples):
print("Sample # {}".format(n))
x = init_states[n]
Trj = [x.copy()]
running_V = 100.
inROA = True
for t in range(n_steps):
x_torch = numpy2torch(x.copy())
with torch.no_grad():
_, u, V = net(x_torch.unsqueeze(0))
u = torch.clamp(u, -10., 10.) # Limit the amount of control input
u = torch2numpy(u[0])
times = np.arange(2) * dt
x_orig = x.copy()
x_orig[:n_link] = wrapToPi(x_orig[:n_link] + target)
y = odeint(system.gradient, x_orig, times, args=(u, ))
y = y[1]
y[:n_link] = wrapToPi(y[:n_link] - target)
if t == 0:
grads.append(system.gradient(x_orig, times, u))
within = (y.copy() <= domain).all()
within = within & (y.copy() >= -domain).all()
def safe_log(x, eps=1e-6):
""" Prevents :math:`log(0)` by using :math:`log(max(x, eps))`."""
return th.log(th.clamp(x, min=eps))
