def optimize_cgd(self):
# self.w = self.w / 1000
# self.v = self.v / 1000
for i in range(self.n_iterations):
if self.w.grad is not None:
self.w.grad.zero_()
if self.v.grad is not None:
self.v.grad.zero_()
self.w.requires_grad = True
self.v.requires_grad = False
L_w = self.objective_function()
L_w.backward()
w_grad = self.w.grad.clone().detach()
self.w.requires_grad = False
self.v.requires_grad = True
L_v = -self.objective_function()
L_v.backward()
v_grad = self.v.grad.clone().detach()
#* update
self.v.requires_grad = False
self.w = self.w - self.lr_w / (1+self.lr_w**2*16) * (w_grad - self.lr_w*4*v_grad)
self.v = self.v - self.lr_v / (1+self.lr_v**2*16) * (v_grad + self.lr_v*4*w_grad)
torch.clamp_(self.w, min=-1, max=1)
torch.clamp_(self.v, min=-1, max=1)
print('\n')
print('Iteration :', i)
print('current w: ', self.w)
print('current v: ', self.v)
if i %1000 == 0:
pdb.set_trace()
def aggregate(self, inputs: Tensor, index: Tensor,
dim_size: Optional[int] = None) -> Tensor:
if self.aggr == 'softmax':
out = scatter_softmax(inputs * self.t, index, dim=self.node_dim)
return scatter(inputs * out, index, dim=self.node_dim,
dim_size=dim_size, reduce='sum')
elif self.aggr == 'softmax_sg':
out = scatter_softmax(inputs * self.t, index,
dim=self.node_dim).detach()
return scatter(inputs * out, index, dim=self.node_dim,
dim_size=dim_size, reduce='sum')
elif self.aggr == 'power':
min_value, max_value = 1e-7, 1e1
torch.clamp_(inputs, min_value, max_value)
out = scatter(torch.pow(inputs, self.p), index, dim=self.node_dim,
dim_size=dim_size, reduce='mean')
torch.clamp_(out, min_value, max_value)
return torch.pow(out, 1 / self.p)
else:
return scatter(inputs, index, dim=self.node_dim, dim_size=dim_size,
reduce=self.aggr)
def solve(self, w, c1, c2):
"""
Runs the PSO on opt_sunc within constraints to perform opt_task
Inputs:
param w : weight of velocity of particle
param c1 : weight of velocity of particle towards its pbest
param c2 : weight of velocity of particle towards gbest
Outputs:
2-tuple : (optimized solution or position of best particle, fitness of best particle)
"""
iteration = 1
while (
(self.particle_pos[0] == self.particle_pos).all().item() == False):
self._get_fitness()
self._update_bests(iteration)
#Updating velocities of particles
self.particle_vels = w * self.particle_vels + c1 * torch.rand(
1).to(self.device) * (
self.particle_pbests -
self.particle_pos) + c2 * torch.rand(1).to(
self.device) * (self.gbest - self.particle_pos)
#Updating positions of particles
self.particle_pos = self.particle_pos + self.particle_vels
# Clamping positions of particles if they exit the constrained search space.
for dim in range(self.num_dims):
torch.clamp_(self.particle_pos[:, dim],
min=self.constraints[0, dim],
max=self.constraints[1, dim])
# Gathering values for plotting
if self.plot:
self.gbest_fitness_lst.append(self.best_gbest_fitness)
# Printing per iteration log
print(
f"Iteration {iteration} : Gbest = {self.gbest}, Gbest_fitness : {self.best_gbest_fitness}"
)
print(
"__________________________________________________________________________________"
)
iteration += 1
# PLotting gbest fitness vs iteration number
if self.plot:
plt.plot(range(iteration - 1), self.gbest_fitness_lst)
plt.xlabel("Iteration Number")
plt.ylabel("Fitness Value")
plt.xscale("log")
plt.savefig("new_plot.png")
# Animating the best particle
if self.animate:
self._animate(iteration - 1)
return self.gbest, self.opt_func(
[self.gbest[dim] for dim in range(self.num_dims)])
def con_joint_limit(p):
global joint_limit_cost, var_p_limit, latest_p_limit
var_p_limit = pre_process(p)
latest_p_limit = var_p_limit.data[1:-1].numpy().reshape(-1)
joint_limit_cost = -torch.sum(torch.clamp_(robot.limits[:, 0]-var_p_limit, min=0)\
+ torch.clamp_(var_p_limit-robot.limits[:, 1], min=0))
return joint_limit_cost.data.numpy()
def forward(self, g, node_feats, edge_feats):
with g.local_scope():
# Node and edge feature dimension need to match.
g.ndata['h'] = node_feats
g.edata['h'] = self.edge_encoder(edge_feats)
g.apply_edges(fn.u_add_e('h', 'h', 'm'))
if self.aggr == 'softmax':
g.edata['m'] = F.relu(g.edata['m']) + self.eps
g.edata['a'] = edge_softmax(g, g.edata['m'] * self.beta)
g.update_all(
lambda edge: {'x': edge.data['m'] * edge.data['a']},
fn.sum('x', 'm'))
elif self.aggr == 'power':
minv, maxv = 1e-7, 1e1
torch.clamp_(g.edata['m'], minv, maxv)
g.update_all(
lambda edge: {'x': torch.pow(edge.data['m'], self.p)},
fn.mean('x', 'm'))
torch.clamp_(g.ndata['m'], minv, maxv)
g.ndata['m'] = torch.pow(g.ndata['m'], self.p)
else:
raise NotImplementedError(
f'Aggregator {self.aggr} is not supported.')
if self.msg_norm is not None:
g.ndata['m'] = self.msg_norm(node_feats, g.ndata['m'])
feats = node_feats + g.ndata['m']
return self.mlp(feats)
def projection(self):
"""
projection on the simplex (projected gradient)
"""
torch.clamp_(self.fc1.weight.data, min=-1, max=1)
self.fc2.weight.data = train.simplex_proj(
self.fc2.weight.data.flatten(), device=self.device).view(-1, 1)
def __call__(self, img):
"""
Args:
img: torch tensor data to extract the contour, with shape: [batch_size, channels, height, width[, depth]]
Returns:
A torch tensor with the same shape as img, note:
1. it's the binary classification result of whether a pixel is edge or not.
2. in order to keep the original shape of mask image, we use padding as default.
3. the edge detection is just approximate because it defects inherent to Laplace kernel,
ideally the edge should be thin enough, but now it has a thickness.
"""
channels = img.shape[1]
if img.ndim == 4:
kernel = torch.tensor([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]], dtype=torch.float32, device=img.device)
kernel = kernel.repeat(channels, 1, 1, 1)
contour_img = F.conv2d(img, kernel, bias=None, stride=1, padding=1, dilation=1, groups=channels)
elif img.ndim == 5:
kernel = -1 * torch.ones(3, 3, 3, dtype=torch.float32, device=img.device)
kernel[1, 1, 1] = 26
kernel = kernel.repeat(channels, 1, 1, 1, 1)
contour_img = F.conv3d(img, kernel, bias=None, stride=1, padding=1, dilation=1, groups=channels)
else:
raise RuntimeError("the dimensions of img should be 4 or 5.")
torch.clamp_(contour_img, min=0.0, max=1.0)
return contour_img
def forward(self, x, batch, bsize=None):
r"""Args:
x (Tensor): Node feature matrix
:math:`\mathbf{X} \in \mathbb{R}^{(N_1 + \ldots + N_B) \times F}`.
batch (LongTensor): Batch vector :math:`\mathbf{b} \in {\{ 0, \ldots,
B-1\}}^N`, which assigns each node to a specific example.
size (int, optional): Batch-size :math:`B`.
Automatically calculated if not given. (default: :obj:`None`)
:rtype: :class:`Tensor`
"""
bsize = int(batch.max().item() + 1) if bsize is None else bsize
n_nodes = scatter_add(torch.ones_like(x), batch, dim=0, dim_size=bsize)
if self.family == "softmax":
out = scatter_softmax(self.p * x.detach(), batch, dim=0)
return scatter_add(x * out,
batch, dim=0, dim_size=bsize)*n_nodes / (1+self.beta*(n_nodes-1))
elif self.family == "power":
# numerical stability - avoid powers of large numbers or negative ones
min_x, max_x = 1e-7, 1e+3
torch.clamp_(x, min_x, max_x)
out = scatter_add(torch.pow(x, self.p),
batch, dim=0, dim_size=bsize) / (1+self.beta*(n_nodes-1))
torch.clamp_(out, min_x, max_x)
return torch.pow(out, 1 / self.p)
def aggregate(self,
inputs: Tensor,
index: Tensor,
dim_size: Optional[int] = None) -> Tensor:
if self.aggr == 'softmax':
out = scatter_softmax(inputs * self.t, index, dim=self.node_dim)
return scatter(inputs * out,
index,
dim=self.node_dim,
dim_size=dim_size,
reduce='sum')
elif self.aggr == 'softmax_sg':
out = scatter_softmax(inputs * self.t, index,
dim=self.node_dim).detach()
return scatter(inputs * out,
index,
dim=self.node_dim,
dim_size=dim_size,
reduce='sum')
elif self.aggr == 'stat':
_mean = scatter_mean(inputs,
index,
dim=self.node_dim,
dim_size=dim_size)
_std = scatter_std(inputs,
index,
dim=self.node_dim,
dim_size=dim_size).detach()
_min = scatter_min(inputs,
index,
dim=self.node_dim,
dim_size=dim_size)[0]
_max = scatter_max(inputs,
index,
dim=self.node_dim,
dim_size=dim_size)[0]
_mean = _mean.unsqueeze(dim=-1)
_std = _std.unsqueeze(dim=-1)
_min = _min.unsqueeze(dim=-1)
_max = _max.unsqueeze(dim=-1)
stat = torch.cat([_mean, _std, _min, _max], dim=-1)
stat = self.lin_stat(stat)
stat = stat.squeeze(dim=-1)
return stat
else:
min_value, max_value = 1e-7, 1e1
torch.clamp_(inputs, min_value, max_value)
out = scatter(torch.pow(inputs, self.p),
index,
dim=self.node_dim,
dim_size=dim_size,
reduce='mean')
torch.clamp_(out, min_value, max_value)
return torch.pow(out, 1 / self.p)
def _optimize(self, loss):
self._optimizer.zero_grad()
loss.backward()
for var in self._network.parameters():
torch.clamp_(var.grad.data, -self._config_dict['gradient_clip'],
self._config_dict['gradient_clip'])
self._optimizer.step()
def projection(self):
torch.clamp_(self.fc1.weight.data, min=-1, max=1)
gamma = self.fc2.weight.clone().view(-1, self.K)
# projection to guarantee marginale beta
for k in range(self.K):
gamma[:, k] = train.simplex_proj(gamma[:, k],
self.beta[k],
device=self.device)
self.fc2.weight.data = gamma.view(-1, 1)
def optimize_no_regret(self):
self.v_bounds = tuple( [ ( self.param.v_bound[0], self.param.v_bound[1]) for _ in range(len(self.v)) ] )
for i in range(1, self.n_iterations):
max_v = self.maximize_v()
self.v = torch.tensor(max_v.x, dtype = dtype)
self.v_avg = (1-1/i)*self.v_avg + (1/i)*self.v.clone().detach()
self.v_tail.append(self.v.clone().detach())
copy = self.w.clone().detach()
self.optimizer_w.zero_grad()
self.w.requires_grad = True
L = self.objective_function()
L.backward()
lr = self.lr_w /(math.sqrt(i))
# lr = self.lr_w
for param_group in self.optimizer_w.param_groups:
param_group['lr'] = lr
w_grad_term = self.optimizer_w.step()
self.w.requires_grad = False
torch.clamp_(self.w, min=-1, max = 1)
w_grad_norm = torch.norm(self.w-copy,p=1)/(lr)
self.w_avg = (1-1/i)*self.w_avg + (1/i)*self.w.clone().detach()
self.w_tail.append(self.w.clone().detach())
while len(self.w_tail) > self.tail_fraction*i and len(self.w_tail)>1:
self.w_tail.popleft()
self.v_tail.popleft()
print('kick one out')
# self.w_avg = (1-(eta+1)/(i+eta))*w_avg + (eta+1)/(i+eta)*w.clone().detach()
if i %10 == 0:
print('\n')
print('Iteration :', i)
print('Current learning rate:', lr)
print('grad norm w: ', w_grad_norm)
print('current w: ', self.w)
print('current v: ', self.v)
print('average w: ', self.w_avg)
print('average v: ', self.v_avg)
print('tail average w: ', sum(self.w_tail) / len(self.w_tail))
print('tail average v: ', sum(self.v_tail) / len(self.v_tail))
print('objective:', L)
if logging:
self.writer.add_scalar('Summary/1.current w', self.w,i)
self.writer.add_scalar('Summary/2.current v', self.v,i)
self.writer.add_scalar('Summary/3.tail average w', sum(self.w_tail) / len(self.w_tail), i)
self.writer.add_scalar('Summary/4.tail average v', sum(self.v_tail) / len(self.v_tail), i)
self.writer.add_scalar('Summary/5.average w', self.w_avg,i)
self.writer.add_scalar('Summary/6.average v', self.v_avg,i)
# self.writer.add_scalar('Summary/5.squared error wrt true v pi', (sum(self.w_collection)/len(self.w_collection)-v_star)**2, i)
# self.writer.add_scalar('Summary/6.l1 norm w - w mu hat', torch.norm(self.w - w_star),i)
# self.writer.add_scalar('Summary/7.grad norm w', torch.norm(self.w.grad),i)
# self.writer.add_scalar('Summary/8.td error v', self.td_error(self.v),i)
if i % 1000 == 0:
pdb.set_trace()
pass
def DetectionInference(detector, data_loader, dataset, idx_to_class, thresh=0.8, nms_thresh=0.3, output_dir=None, dtype=torch.float32, device='cpu'):
# ship model to GPU
detector.to(dtype=dtype, device=device)
detector.eval()
start_t = time.time()
if output_dir is not None:
det_dir = 'mAP/input/detection-results'
gt_dir = 'mAP/input/ground-truth'
if os.path.exists(det_dir):
shutil.rmtree(det_dir)
os.mkdir(det_dir)
if os.path.exists(gt_dir):
shutil.rmtree(gt_dir)
os.mkdir(gt_dir)
for iter_num, data_batch in enumerate(data_loader):
images, boxes, w_batch, h_batch, img_ids = data_batch
images = images.to(dtype=dtype, device=device)
final_proposals, final_conf_scores, final_class = detector.inference(images, thresh=thresh, nms_thresh=nms_thresh)
# clamp on the proposal coordinates
batch_size = len(images)
for idx in range(batch_size):
torch.clamp_(final_proposals[idx][:, 0::2], min=0, max=w_batch[idx])
torch.clamp_(final_proposals[idx][:, 1::2], min=0, max=h_batch[idx])
# visualization
# get the original image
# hack to get the original image so we don't have to load from local again...
i = batch_size*iter_num + idx
img, _ = dataset.__getitem__(i)
valid_box = sum([1 if j != -1 else 0 for j in boxes[idx][:, 0]])
final_all = torch.cat((final_proposals[idx], \
final_class[idx].float(), final_conf_scores[idx]), dim=-1).cpu()
resized_proposals = coord_trans(final_all, w_batch[idx], h_batch[idx])
# write results to file for evaluation (use mAP API https://github.com/Cartucho/mAP for now...)
if output_dir is not None:
file_name = img_ids[idx].replace('.jpg', '.txt')
with open(os.path.join(det_dir, file_name), 'w') as f_det, \
open(os.path.join(gt_dir, file_name), 'w') as f_gt:
# print('{}: {} GT bboxes and {} proposals'.format(img_ids[idx], valid_box, resized_proposals.shape[0]))
for b in boxes[idx][:valid_box]:
f_gt.write('{} {:.2f} {:.2f} {:.2f} {:.2f}\n'.format(idx_to_class[b[4].item()], b[0], b[1], b[2], b[3]))
for b in resized_proposals:
f_det.write('{} {:.6f} {:.2f} {:.2f} {:.2f} {:.2f}\n'.format(idx_to_class[b[4].item()], b[5], b[0], b[1], b[2], b[3]))
else:
eecs598.vis.detection_visualizer(img, idx_to_class, boxes[idx][:valid_box], resized_proposals)
end_t = time.time()
print('Total inference time: {:.1f}s'.format(end_t-start_t))
def aggregate(self, inputs, index, ptr=None, dim_size=None):
if self.aggr in ["add", "mean", "max", None]:
return super(GenMessagePassing,
self).aggregate(inputs, index, ptr, dim_size)
elif self.aggr in ["softmax_sg", "softmax", "softmax_sum"]:
if self.learn_t:
out = scatter_softmax(inputs * self.t,
index,
dim=self.node_dim)
else:
with torch.no_grad():
out = scatter_softmax(inputs * self.t,
index,
dim=self.node_dim)
out = scatter(inputs * out,
index,
dim=self.node_dim,
dim_size=dim_size,
reduce="sum")
if self.aggr == "softmax_sum":
self.sigmoid_y = torch.sigmoid(self.y)
degrees = degree(index, num_nodes=dim_size).unsqueeze(1)
out = torch.pow(degrees, self.sigmoid_y) * out
return out
elif self.aggr in ["power", "power_sum"]:
min_value, max_value = 1e-7, 1e1
torch.clamp_(inputs, min_value, max_value)
out = scatter(
torch.pow(inputs, self.p),
index,
dim=self.node_dim,
dim_size=dim_size,
reduce="mean",
)
torch.clamp_(out, min_value, max_value)
out = torch.pow(out, 1 / self.p)
# torch.clamp(out, min_value, max_value)
if self.aggr == "power_sum":
self.sigmoid_y = torch.sigmoid(self.y)
degrees = degree(index, num_nodes=dim_size).unsqueeze(1)
out = torch.pow(degrees, self.sigmoid_y) * out
return out
else:
raise NotImplementedError("To be implemented")
def get_target_action_t(self, obs_t: 'Tensor') -> 'Tensor':
"""
in the PID controller, it just calculates the output like always, but as a tensor
"""
err_der_int_t = T.split(obs_t, 3, dim=1)[0] #we only need the first three columns of the obs_t tensor
mu_t = err_der_int_t * self.pid_tensor_t
mu_t = T.sum(mu_t, dim=1)
T.clamp_(mu_t, -1, 1) #to have PID-output in the range of [-1,1]
mu_t = mu_t * self.act_scale_t + self.act_shift_t #to have PID-output in the range of the action_space
return mu_t.unsqueeze(1) #to get a Tesnor of shape ([batch_size, 1]) instead of ([batch_size])
def recon_upsample(embed, labels, idx_train, adj=None, portion=1.0, im_class_num=3):
c_largest = labels.max().item()
avg_number = int(idx_train.shape[0]/(c_largest+1))
#ipdb.set_trace()
adj_new = None
for i in range(im_class_num):
chosen = idx_train[(labels==(c_largest-i))[idx_train]]
num = int(chosen.shape[0]*portion)
if portion == 0:
c_portion = int(avg_number/chosen.shape[0])
num = chosen.shape[0]
else:
c_portion = 1
for j in range(c_portion):
chosen = chosen[:num]
chosen_embed = embed[chosen,:]
distance = squareform(pdist(chosen_embed.detach()))
np.fill_diagonal(distance,distance.max()+100)
idx_neighbor = distance.argmin(axis=-1)
interp_place = random.random()
new_embed = embed[chosen,:] + (embed[idx_neighbor,:]-embed[chosen,:])*interp_place
new_labels = labels.new(torch.Size((chosen.shape[0],1))).reshape(-1).fill_(c_largest-i)
idx_new = np.arange(embed.shape[0], embed.shape[0]+chosen.shape[0])
idx_train_append = idx_train.new(idx_new)
embed = torch.cat((embed,new_embed), 0)
labels = torch.cat((labels,new_labels), 0)
idx_train = torch.cat((idx_train,idx_train_append), 0)
if adj is not None:
if adj_new is None:
adj_new = adj.new(torch.clamp_(adj[chosen,:] + adj[idx_neighbor,:], min=0.0, max = 1.0))
else:
temp = adj.new(torch.clamp_(adj[chosen,:] + adj[idx_neighbor,:], min=0.0, max = 1.0))
adj_new = torch.cat((adj_new, temp), 0)
if adj is not None:
add_num = adj_new.shape[0]
new_adj = adj.new(torch.Size((adj.shape[0]+add_num, adj.shape[0]+add_num))).fill_(0.0)
new_adj[:adj.shape[0], :adj.shape[0]] = adj[:,:]
new_adj[adj.shape[0]:, :adj.shape[0]] = adj_new[:,:]
new_adj[:adj.shape[0], adj.shape[0]:] = torch.transpose(adj_new, 0, 1)[:,:]
return embed, labels, idx_train, new_adj.detach()
else:
return embed, labels, idx_train
def denoise(self, noise_im) -> torch.Tensor:
"""
input:
noise_im: [b, c, h, w]
clean_im: [b, c, h, w]
return:
restored_im: [b, c, h, w]
"""
# half quadratic split
restored_im = noise_im.clone()
for beta in self.betas:
for t in range(self.num_iters):
restored_imcol = im2col(
restored_im, 8, 8, self.stride
) # matlab style im2col, output shape = [batch, c, patch_size**2, num_patches],
# GMM prior
p_y_z, mean_noise_imcol, GMM_noisy_covs = self.prior(
noise_imcol=restored_imcol, noise_sd=beta**(-0.5))
# weiner filtering: update z in Equa. 5
max_index = torch.argmax(p_y_z, dim=1)
Xhat = torch.zeros_like(restored_imcol)
for b in range(Xhat.shape[0]):
for i in range(self.GMM["nmodels"]):
index = torch.nonzero((max_index[b] - i) == 0,
as_tuple=False)[:, 0]
B = torch.matmul(self.GMM["covs"][i],
restored_imcol[b, :, :, index])
solution = torch.matmul(GMM_noisy_covs[i].inverse(), B)
Xhat[b, :, :, index] = solution
Xhat += mean_noise_imcol
restored_imcol = Xhat
I1 = torch.zeros_like(restored_im)
for b in range(I1.shape[0]):
I1[b] = avg_col2im(restored_imcol[b], noise_im.shape[2],
noise_im.shape[3], self.stride)
# update Xhat in Equa. 4
restored_im = noise_im * self.lamb / (
self.lamb + beta * 8**2) + (beta * 8**2 /
(self.lamb + beta * 8**2)) * I1
psnr1 = 10 * torch.log10(1 / torch.mean(
(restored_im - self.clean_im)**2))
print(f"[beta={beta:.3f}, iter={t}] PSNR={psnr1.item():.3f}")
torch.clamp_(restored_im, min=0, max=1)
return restored_im
def train_epoch(epoch, args, model, train_dataloader, device, n_gpu, optimizer, scheduler, global_step, local_rank=0):
global logger
torch.cuda.empty_cache()
model.train()
log_step = args.n_display
start_time = time.time()
total_loss = 0
for step, batch in enumerate(train_dataloader):
if n_gpu == 1:
# multi-gpu does scattering it-self
batch = tuple(t.to(device=device, non_blocking=True) for t in batch)
input_ids, input_mask, segment_ids, video, video_mask = batch
loss = model(input_ids, segment_ids, input_mask, video, video_mask)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
total_loss += float(loss)
if (step + 1) % args.gradient_accumulation_steps == 0:
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
if scheduler is not None:
scheduler.step() # Update learning rate schedule
optimizer.step()
optimizer.zero_grad()
# https://github.com/openai/CLIP/issues/46
if hasattr(model, 'module'):
torch.clamp_(model.module.clip.logit_scale.data, max=np.log(100))
else:
torch.clamp_(model.clip.logit_scale.data, max=np.log(100))
global_step += 1
if global_step % log_step == 0 and local_rank == 0:
logger.info("Epoch: %d/%s, Step: %d/%d, Lr: %s, Loss: %f, Time/step: %f", epoch + 1,
args.epochs, step + 1,
len(train_dataloader), "-".join([str('%.9f'%itm) for itm in sorted(list(set(optimizer.get_lr())))]),
float(loss),
(time.time() - start_time) / (log_step * args.gradient_accumulation_steps))
start_time = time.time()
total_loss = total_loss / len(train_dataloader)
return total_loss, global_step
def con_max_move(p):
global max_move_cost, var_p_max_move, latest_p_max_move
var_p_max_move = pre_process(p)
latest_p_max_move = var_p_max_move.data[1:-1].numpy().reshape(-1)
control_points = robot.fkine(var_p_max_move)
max_move_cost = -torch.clamp_((control_points[1:]-control_points[:-1]).pow(2).sum(dim=2)-1.5**2, min=0).sum()
return max_move_cost.data.numpy()
def Gquant_(self):
self.w_max = min(self.weight.max().item(), self.G_max)
self.w_min = max(self.weight.min().item(), self.G_min)
self.lsb = (self.w_max - self.w_min) / math.pow(2, self.q_bit)
self.weight.data = torch.clamp_(self.weight, self.G_min, self.G_max)
self.weight.data = ((self.weight - self.w_min) /
self.lsb).round() * self.lsb + self.w_min
def forward(self, x):
"""Forward pass through network."""
# turn input into stacked real and imaginary spectograms
specs = self.stft(x)
real = specs[:, :self.fft_len // 2 + 1]
imag = specs[:, self.fft_len // 2 + 1:]
out = torch.stack([real, imag], 1)[:, :, 1:]
# pass through encoder
encoder_out, out = self.encoder(out)
# hidden layers
out = self.enhance(out)
# pass through the decoder
out = self.decoder(out, encoder_out)
# mask
real, imag = self.mask(out, real, imag)
# prepare output (time or freq domain)
y_pred = torch.cat([real, imag], 1)
if not self.forward_output_is_spec:
y_pred = self.istft(y_pred)
y_pred = torch.squeeze(y_pred, 1)
y_pred = torch.clamp_(y_pred, -1, 1)
return y_pred
def con_collision_free(p):
global cnt_check, collision_cost, var_p_collision, latest_p_collision
var_p_collision = pre_process(p)
latest_p_collision = var_p_collision.data[1:-1].numpy().reshape(-1)
cnt_check += len(p)
collision_cost = torch.sum(-torch.clamp_(dist_est(var_p_collision[1:-1])-safety_margin, min=0))
return collision_cost.data.numpy()
def refine_actions(model, actions, single_observarion, learning_rate,
num_updates, batch_size, refine_loss):
observations = torch.tensor(single_observarion,
device=model.device).unsqueeze(0)
actions = torch.tensor(actions)
refined_actions = []
model.eval()
preprocessed = model.preprocess(observations)
preprocessed = {k: v.detach() for k, v in preprocessed.items()}
for start in range(0, len(actions), batch_size):
action_batch = actions[start:][:batch_size].to(model.device)
action_batch = torch.nn.Parameter(action_batch)
optimizer = torch.optim.Adam([action_batch], lr=learning_rate)
losses = []
for _ in range(num_updates):
optimizer.zero_grad()
logits = model(None, action_batch, preprocessed=preprocessed)
if refine_loss == 'ce':
loss = model.ce_loss(logits, actions.new_ones(len(logits)))
elif refine_loss == 'linear':
loss = -logits.sum()
else:
raise ValueError(f'Unknown loss: {refine_loss}')
loss.backward()
losses.append(loss.item())
optimizer.step()
action_batch = torch.clamp_(action_batch.data, 0, 1)
refined_actions.append(action_batch.cpu().numpy())
refined_actions = np.concatenate(refined_actions, 0).tolist()
return refined_actions
def make_image_under_env(self):
image_env = self.render_layer.forward_env(self.albedo, self.normal,
self.rough, self.mask,
self.SH) + self.image_bg
image_in = 2 * image_env - 1
image_in = torch.clamp_(image_in, -1, 1)
# image under env with bg
return image_in
def forward(self, input):
if self.activation == 'exu':
input = input - self.bias
h = F.linear(input, torch.exp(self.weight))
return relu_n(h)
h = F.linear(input, self.weight, self.bias)
return torch.clamp_(h, min=0.) # relu
def optimize_acgd(self):
for i in range(self.n_iterations):
self.w.requires_grad = True
self.v.requires_grad = True
loss = self.objective_function()
self.optimizer.zero_grad()
self.optimizer.step(loss=loss)
self.w.requires_grad = False
self.v.requires_grad = False
torch.clamp_(self.w, min=-1, max=1)
torch.clamp_(self.v, min=-1, max=1)
print('\n')
print('Iteration :', i)
print('current w: ', self.w)
print('current v: ', self.v)
if i %1000 == 0:
pdb.set_trace()
def _quantile_encode_approx(tensor: torch.Tensor,
n_bits: int) -> Tuple[torch.Tensor, torch.Tensor]:
n_bins = 2**n_bits
borders = torch.as_tensor(
_quantile_qq_approximation(tensor.numpy(), n_bins + 1)[1:-1])
quant_weight = torch.clamp_(torch.bucketize(tensor, borders), 0,
n_bins - 1)
lookup = average_buckets(tensor, quant_weight, n_bins)
return quant_weight, lookup
def optimize_optimistic(self):
for i in range(self.n_iterations):
self.optimizer_w.zero_grad()
self.optimizer_v.zero_grad()
self.v.requires_grad = True
L = -self.objective_function()
L.backward()
copy = self.v.clone().detach()
if self.param.lr_decay:
lr_v = self.lr_v /(math.sqrt(i+1))
for param_group in self.optimizer_v.param_groups:
param_group['lr'] = lr_v
v_grad_term = self.optimizer_v.step()
self.v.requires_grad = False
torch.clamp_(self.v, min=-1, max = 1)
v_grad_norm = torch.norm(self.v-copy,p=1)/(self.lr_v)
self.w.requires_grad = True
L = self.objective_function()
L.backward()
copy = self.w.clone().detach()
if self.param.lr_decay:
lr_w = self.lr_w / (math.sqrt(i+1))
for param_group in self.optimizer_w.param_groups:
param_group['lr'] = lr_w
w_grad_term = self.optimizer_w.step()
self.w.requires_grad = False
torch.clamp_(self.w, min=-1, max = 1)
w_grad_norm = torch.norm(self.w-copy,p=1)/(self.lr_w)
print('\n')
print('Iteration :', i)
# print('Current learning rate:', lr_w)
print('grad norm w: ', w_grad_norm)
print('grad norm v: ', v_grad_norm)
print('current w: ', self.w)
print('current v: ', self.v)
# print('average w: ', self.w_avg)
# print('average v: ', self.v_avg)
# print('tail average w: ', sum(self.w_tail) / len(self.w_tail))
# print('tail average v: ', sum(self.v_tail) / len(self.v_tail))
if i % 100 == 0:
pdb.set_trace()
def forward(self):
"""forward: generate all data and GT"""
input = torch.cat(
[self.image_s_pe, self.image_s_pe * self.mask, self.mask], 1)
# encoder
feat = self.encoder(input)
self.env_pred = self.env_predictor(feat[-1])
brdf_feat, brdf_pred = self.decoder_brdf(feat)
self.albedo_pred, self.normal_pred, \
self.rough_pred, self.depth_pred = brdf_pred
light_t = torch.cat(
[self.light_t,
self.env_pred.view(self.env_pred.size(0), -1)], 1)
self.relit_pred = self.decoder_render(feat, brdf_feat,
light_t) * self.mask
# render
image_pt = self.render_layer.forward_batch(self.albedo_pred,
self.normal_pred,
self.rough_pred,
self.depth_pred, self.mask,
self.light_t)
image_env = self.render_layer.forward_env(self.albedo_pred,
self.normal_pred,
self.rough_pred, self.mask,
self.env_pred)
relit_render = 2 * (image_pt + image_env) - 1
self.relit_render = torch.clamp_(relit_render, -1, 1) * self.mask
# ----------------- Cas1 -----------------
cas_input = torch.cat([self.image_s_pe, \
self.relit_pred * self.mask, \
self.relit_render * self.mask, \
self.albedo_pred * self.mask, \
self.normal_pred * self.mask, \
self.rough_pred * self.mask, \
self.depth_pred * self.mask, \
self.mask, \
], dim=1)
feat_cas = self.encoderRef(cas_input)
brdf_feat_cas, brdf_pred_cas = self.decoderRef_brdf(feat_cas)
self.albedo_pred_cas, self.normal_pred_cas, \
self.rough_pred_cas, self.depth_pred_cas = brdf_pred_cas
self.env_pred_cas = self.env_caspredictor(feat_cas[-1], self.env_pred)
light_t = torch.cat([
self.light_t,
self.env_pred_cas.view(self.env_pred_cas.size(0), -1)
], 1)
self.relit_caspred = self.decoderRef_render(feat_cas, brdf_feat_cas,
light_t) * self.mask
def make_image_under_pt_and_env(self, light):
image_pt = self.render_layer.forward_batch(
self.albedo, self.normal, self.rough, self.depth, self.mask,
light) * self.mask
image_env = self.render_layer.forward_env(self.albedo, self.normal,
self.rough, self.mask,
self.SH) + self.image_bg
image_pe = 2 * (image_pt + image_env) - 1
image_pe = torch.clamp_(image_pe, -1, 1)
# image under point and env with bg
return image_pe
