def fov(self, value): self._fov = value fov_factor = 1.0 / torch.tan(transform.radians(0.5 * self._fov)) o = torch.ones([1], dtype=torch.float32) diag = torch.cat([fov_factor, fov_factor, o], 0) self._cam_to_ndc = torch.diag(diag) self.ndc_to_cam = torch.inverse(self._cam_to_ndc)
def _batch_mahalanobis(L, x): r""" Computes the squared Mahalanobis distance :math:`\mathbf{x}^\top\mathbf{M}^{-1}\mathbf{x}` for a factored :math:`\mathbf{M} = \mathbf{L}\mathbf{L}^\top`. Accepts batches for both L and x. """ # TODO: use `torch.potrs` or similar once a backwards pass is implemented. flat_L = L.unsqueeze(0).reshape((-1,) + L.shape[-2:]) L_inv = torch.stack([torch.inverse(Li.t()) for Li in flat_L]).view(L.shape) return (x.unsqueeze(-1) * L_inv).sum(-2).pow(2.0).sum(-1)
def high_dimension_gaussain_energy(x): u = High_mu Sigma = High_Sigma Sigma = torch.inverse(Sigma) if isinstance(x, Variable): u = Variable(u, requires_grad=True) Sigma = Variable(Sigma, requires_grad=True) diff = x - u temp = 0.5 * torch.matmul(torch.matmul(diff, Sigma), diff.t()) return temp
def anomalyScore(args, model, dataset, mean, cov, channel_idx=0, score_predictor=None): predictions = [] rearranged = [] errors = [] hiddens = [] predicted_scores = [] with torch.no_grad(): # Turn on evaluation mode which disables dropout. model.eval() pasthidden = model.init_hidden(1) for t in range(len(dataset)): out, hidden = model.forward(dataset[t].unsqueeze(0), pasthidden) predictions.append([]) rearranged.append([]) errors.append([]) hiddens.append(model.extract_hidden(hidden)) if score_predictor is not None: predicted_scores.append(score_predictor.predict(model.extract_hidden(hidden).numpy())) predictions[t].append(out.data.cpu()[0][0][channel_idx]) pasthidden = model.repackage_hidden(hidden) for prediction_step in range(1, args.prediction_window_size): out, hidden = model.forward(out, hidden) predictions[t].append(out.data.cpu()[0][0][channel_idx]) if t >= args.prediction_window_size: for step in range(args.prediction_window_size): rearranged[t].append( predictions[step + t - args.prediction_window_size][args.prediction_window_size - 1 - step]) rearranged[t] =torch.FloatTensor(rearranged[t]).to(args.device).unsqueeze(0) errors[t] = rearranged[t] - dataset[t][0][channel_idx] else: rearranged[t] = torch.zeros(1,args.prediction_window_size).to(args.device) errors[t] = torch.zeros(1, args.prediction_window_size).to(args.device) predicted_scores = np.array(predicted_scores) scores = [] for error in errors: mult1 = error-mean.unsqueeze(0) # [ 1 * prediction_window_size ] mult2 = torch.inverse(cov) # [ prediction_window_size * prediction_window_size ] mult3 = mult1.t() # [ prediction_window_size * 1 ] score = torch.mm(mult1,torch.mm(mult2,mult3)) scores.append(score[0][0]) scores = torch.stack(scores) rearranged = torch.cat(rearranged,dim=0) errors = torch.cat(errors,dim=0) return scores, rearranged, errors, hiddens, predicted_scores
def transform(point, center, scale, resolution, invert=False): _pt = torch.ones(3) _pt[0] = point[0] _pt[1] = point[1] h = 200.0 * scale t = torch.eye(3) t[0, 0] = resolution / h t[1, 1] = resolution / h t[0, 2] = resolution * (-center[0] / h + 0.5) t[1, 2] = resolution * (-center[1] / h + 0.5) if invert: t = torch.inverse(t) new_point = (torch.matmul(t, _pt))[0:2] return new_point.int()
def compute_L_inverse(self,X,Y): N = X.size()[0] # num of points (along dim 0) # construct matrix K Xmat = X.expand(N,N) Ymat = Y.expand(N,N) P_dist_squared = torch.pow(Xmat-Xmat.transpose(0,1),2)+torch.pow(Ymat-Ymat.transpose(0,1),2) P_dist_squared[P_dist_squared==0]=1 # make diagonal 1 to avoid NaN in log computation K = torch.mul(P_dist_squared,torch.log(P_dist_squared)) if self.reg_factor != 0: K+=torch.eye(K.size(0),K.size(1))*self.reg_factor # construct matrix L O = torch.FloatTensor(N,1).fill_(1) Z = torch.FloatTensor(3,3).fill_(0) P = torch.cat((O,X,Y),1) L = torch.cat((torch.cat((K,P),1),torch.cat((P.transpose(0,1),Z),1)),0) Li = torch.inverse(L) if self.use_cuda: Li = Li.cuda() return Li
def NormalEnergy(x): # x = Variable(x, requires_grad=True) # u = torch.zeros(1, 2) # u[0, :] = torch.FloatTensor([2, 2]) # Sigma = torch.FloatTensor([[1.0, 0.8], [0.8, 1.0]]) u = Gaussian_u Sigma = Gaussian_Sigma Sigma = torch.inverse(Sigma) # Sigma = Sigma.t() if isinstance(x, Variable): u = Variable(u, requires_grad=True) Sigma = Variable(Sigma, requires_grad=True) diff = x - u temp = 0.5 * torch.matmul(torch.matmul(diff, Sigma), diff.t()) return temp
def __init__(self, values, env_to_world = torch.eye(4, 4)): # Convert to constant texture if necessary if isinstance(values, torch.Tensor): values = pyredner.Texture(values) assert(values.texels.is_contiguous()) assert(values.texels.dtype == torch.float32) if pyredner.get_use_gpu(): assert(values.texels.is_cuda) else: assert(not values.texels.is_cuda) assert(env_to_world.dtype == torch.float32) # Build sampling table luminance = 0.212671 * values.texels[:, :, 0] + \ 0.715160 * values.texels[:, :, 1] + \ 0.072169 * values.texels[:, :, 2] # For each y, compute CDF over x sample_cdf_xs_ = torch.cumsum(luminance, dim = 1) y_weight = torch.sin(\ math.pi * (torch.arange(luminance.shape[0], dtype = torch.float32, device = luminance.device) + 0.5) \ / float(luminance.shape[0])) # Compute CDF for x sample_cdf_ys_ = torch.cumsum(sample_cdf_xs_[:, -1] * y_weight, dim = 0) pdf_norm = (luminance.shape[0] * luminance.shape[1]) / \ (sample_cdf_ys_[-1].item() * (2 * math.pi * math.pi)) # Normalize to [0, 1) sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \ torch.max(sample_cdf_xs_[:, (luminance.shape[1] - 1):luminance.shape[1]], 1e-8 * torch.ones(sample_cdf_xs_.shape[0], 1, device = sample_cdf_ys_.device)) sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \ torch.max(sample_cdf_ys_[-1], torch.tensor([1e-8], device = sample_cdf_ys_.device)) self.values = values self.env_to_world = env_to_world self.world_to_env = torch.inverse(env_to_world).contiguous() self.sample_cdf_ys = sample_cdf_ys.contiguous() self.sample_cdf_xs = sample_cdf_xs.contiguous() self.pdf_norm = pdf_norm
def __init__(self, position, look_at, up, fov, clip_near, resolution, cam_to_ndc = None, fisheye = False): assert(position.dtype == torch.float32) assert(len(position.shape) == 1 and position.shape[0] == 3) assert(look_at.dtype == torch.float32) assert(len(look_at.shape) == 1 and look_at.shape[0] == 3) assert(up.dtype == torch.float32) assert(len(up.shape) == 1 and up.shape[0] == 3) if fov is not None: assert(fov.dtype == torch.float32) assert(len(fov.shape) == 1 and fov.shape[0] == 1) assert(isinstance(clip_near, float)) self.position = position self.look_at = look_at self.up = up self._fov = fov # self.cam_to_world = transform.gen_look_at_matrix(position, look_at, up) # self.world_to_cam = torch.inverse(self.cam_to_world).contiguous() if cam_to_ndc is None: fov_factor = 1.0 / torch.tan(transform.radians(0.5 * fov)) o = torch.ones([1], dtype=torch.float32) diag = torch.cat([fov_factor, fov_factor, o], 0) self._cam_to_ndc = torch.diag(diag) else: self._cam_to_ndc = cam_to_ndc self.ndc_to_cam = torch.inverse(self.cam_to_ndc) self.clip_near = clip_near self.resolution = resolution self.fisheye = fisheye
def regress(self, img_original, hand_bbox_list, add_margin=False, debug=True, viz_path="tmp.png", K=None): """ args: img_original: original raw image (BGR order by using cv2.imread) hand_bbox_list: [ dict( left_hand = [x0, y0, w, h] or None right_hand = [x0, y0, w, h] or None ) ... ] add_margin: whether to do add_margin given the hand bbox outputs: To be filled Note: Output element can be None. This is to keep the same output size with input bbox """ pred_output_list = list() hand_bbox_list_processed = list() for hand_bboxes in hand_bbox_list: if hand_bboxes is None: # Should keep the same size with bbox size pred_output_list.append(None) hand_bbox_list_processed.append(None) continue pred_output = dict(left_hand=None, right_hand=None) hand_bboxes_processed = dict(left_hand=None, right_hand=None) for hand_type in hand_bboxes: bbox = hand_bboxes[hand_type] if bbox is None: continue else: img_cropped, norm_img, bbox_scale_ratio, bbox_processed = \ self.__process_hand_bbox(img_original, hand_bboxes[hand_type], hand_type, add_margin) hand_bboxes_processed[hand_type] = bbox_processed with torch.no_grad(): # pred_rotmat, pred_betas, pred_camera = self.model_regressor(norm_img.to(self.device)) self.model_regressor.set_input_imgonly( {'img': norm_img.unsqueeze(0)}) self.model_regressor.test() pred_res = self.model_regressor.get_pred_result() ##Output cam = pred_res['cams'][0, :] #scale, tranX, tranY pred_verts_origin = pred_res['pred_verts'][0] faces = self.model_regressor.right_hand_faces_local pred_pose = pred_res['pred_pose_params'].copy() pred_joints = pred_res['pred_joints_3d'].copy()[0] if hand_type == 'left_hand': cam[1] *= -1 pred_verts_origin[:, 0] *= -1 faces = faces[:, ::-1] pred_pose[:, 1::3] *= -1 pred_pose[:, 2::3] *= -1 pred_joints[:, 0] *= -1 pred_output[hand_type] = dict() pred_output[hand_type][ 'pred_vertices_smpl'] = pred_verts_origin # SMPL-X hand vertex in bbox space pred_output[hand_type][ 'pred_joints_smpl'] = pred_joints pred_output[hand_type]['faces'] = faces pred_output[hand_type][ 'bbox_scale_ratio'] = bbox_scale_ratio pred_output[hand_type]['bbox_top_left'] = np.array( bbox_processed[:2]) pred_output[hand_type]['pred_camera'] = cam pred_output[hand_type]['img_cropped'] = img_cropped # Get global camera global_cams = local_to_global_cam( bbox_wh_to_xy( np.array([list(bbox_processed) ]).astype(np.float)), cam[None, :], max(img_original.shape)) pred_output[hand_type]["global_cams"] = global_cams # pred hand pose & shape params & hand joints 3d pred_output[hand_type][ 'pred_hand_pose'] = pred_pose # (1, 48): (1, 3) for hand rotation, (1, 45) for finger pose. # Recover PCA components if hand_type == "right_hand": comps = torch.Tensor(self.mano_model.rh_mano_pca. full_hand_components) elif hand_type == "left_hand": comps = torch.Tensor(self.mano_model.lh_mano_pca. full_hand_components) pca_comps = torch.einsum('bi,ij->bj', [ torch.Tensor(pred_pose[:, 3:]), torch.inverse(comps) ]) pred_output[hand_type]['pred_hand_betas'] = pred_res[ 'pred_shape_params'] # (1, 10) pred_output[hand_type][ 'pred_pca_pose'] = pca_comps.numpy() hand_side = hand_type.split("_")[0] pred_output[hand_type]["hand_side"] = hand_side pred_output[hand_type][ 'mano_trans'] = self.mano_model.get_mano_trans( mano_pose=pred_pose[:, 3:][0], rot=pred_pose[:, :3][0], betas=pred_res["pred_shape_params"][0], ref_verts=pred_verts_origin, side=hand_side) #Convert vertices into bbox & image space cam_scale = cam[0] cam_trans = cam[1:] vert_smplcoord = pred_verts_origin.copy() joints_smplcoord = pred_joints.copy() vert_bboxcoord = convert_smpl_to_bbox( vert_smplcoord, cam_scale, cam_trans, bAppTransFirst=True) # SMPL space -> bbox space joints_bboxcoord = convert_smpl_to_bbox( joints_smplcoord, cam_scale, cam_trans, bAppTransFirst=True) # SMPL space -> bbox space hand_boxScale_o2n = pred_output[hand_type][ 'bbox_scale_ratio'] hand_bboxTopLeft = pred_output[hand_type][ 'bbox_top_left'] vert_imgcoord = convert_bbox_to_oriIm( vert_bboxcoord, hand_boxScale_o2n, hand_bboxTopLeft, img_original.shape[1], img_original.shape[0]) pred_output[hand_type][ 'pred_vertices_img'] = vert_imgcoord joints_imgcoord = convert_bbox_to_oriIm( joints_bboxcoord, hand_boxScale_o2n, hand_bboxTopLeft, img_original.shape[1], img_original.shape[0]) pred_output[hand_type][ 'pred_joints_img'] = joints_imgcoord if K is not None: K = npt.numpify(K) K_viz = K.copy() # r_hand, _ = cv2.Rodrigues(r_vec) K_viz[:2] = K_viz[:2] / max(img_original.shape) ortho_scale_pixels = global_cams[0, 0] / 2 * max( img_original.shape ) # scale of verts_pixel / pred_verts_origin ortho_trans_pixels = ( global_cams[0, 1:] + 1 / global_cams[0, 0]) * ortho_scale_pixels t_vec = camconvs.weakcam2persptrans( np.concatenate([ np.array([ortho_scale_pixels]), ortho_trans_pixels ], 0) / max(img_original.shape), K_viz)[:, None] r_hand = np.eye(3) # Sanity check nptype = np.float32 proj2d, camverts = project.proj2d( pred_verts_origin.astype(nptype), K.astype(nptype), rot=r_hand.astype(nptype), trans=t_vec[:, 0].astype(nptype)) pred_output[hand_type]['camverts'] = camverts pred_output[hand_type][ 'perspective_trans'] = t_vec.transpose() pred_output[hand_type]['perspective_rot'] = r_hand pred_output_list.append(pred_output) hand_bbox_list_processed.append(hand_bboxes_processed) assert len(hand_bbox_list_processed) == len(hand_bbox_list) return pred_output_list
def unmoldDetections(config, camera, detections, detection_masks, detection_masks_var, depth_np, unmold_masks=True, debug=False): """Reformats the detections of one image from the format of the neural network output to a format suitable for use in the rest of the application. detections: [N, (y1, x1, y2, x2, class_id, score)] mrcnn_mask: [N, height, width, num_classes] image_shape: [height, width, depth] Original size of the image before resizing window: [y1, x1, y2, x2] Box in the image where the real image is excluding the padding. Returns: boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels class_ids: [N] Integer class IDs for each bounding box scores: [N] Float probability scores of the class_id masks: [height, width, num_instances] Instance masks """ if config.GLOBAL_MASK: masks = detection_masks[ torch.arange(len(detection_masks)).cuda().long(), 0, :, :] else: masks = detection_masks[ torch.arange(len(detection_masks)).cuda().long(), detections[:, 4].long(), :, :] pass final_masks = [] for detectionIndex in range(len(detections)): box = detections[detectionIndex][:4].long() if (box[2] - box[0]) * (box[3] - box[1]) <= 0: continue mask = masks[detectionIndex] mask = mask.unsqueeze(0).unsqueeze(0) mask = F.upsample(mask, size=(box[2] - box[0], box[3] - box[1]), mode='bilinear') mask = mask.squeeze(0).squeeze(0) final_mask = torch.zeros(config.IMAGE_MAX_DIM, config.IMAGE_MAX_DIM).cuda() final_mask[box[0]:box[2], box[1]:box[3]] = mask final_masks.append(final_mask) continue final_masks = torch.stack(final_masks, dim=0) if config.NUM_PARAMETER_CHANNELS > 0: ## We could potentially predict depth and/or normals for each instance (not being used) parameters_array = detection_masks[ torch.arange(len(detection_masks)).cuda().long(), -config.NUM_PARAMETER_CHANNELS:, :, :] final_parameters_array = [] for detectionIndex in range(len(detections)): box = detections[detectionIndex][:4].long() if (box[2] - box[0]) * (box[3] - box[1]) <= 0: continue parameters = F.upsample( parameters_array[detectionIndex].unsqueeze(0), size=(box[2] - box[0], box[3] - box[1]), mode='bilinear').squeeze(0) final_parameters = torch.zeros(config.NUM_PARAMETER_CHANNELS, config.IMAGE_MAX_DIM, config.IMAGE_MAX_DIM).cuda() final_parameters[:, box[0]:box[2], box[1]:box[3]] = parameters final_parameters_array.append(final_parameters) continue final_parameters = torch.stack(final_parameters_array, dim=0) final_masks = torch.cat([final_masks.unsqueeze(1), final_parameters], dim=1) pass masks = final_masks if 'normal' in config.ANCHOR_TYPE: ## Compute offset based normal prediction and depthmap prediction ranges = config.getRanges(camera).transpose(1, 2).transpose(0, 1) zeros = torch.zeros(3, (config.IMAGE_MAX_DIM - config.IMAGE_MIN_DIM) // 2, config.IMAGE_MAX_DIM).cuda() ranges = torch.cat([zeros, ranges, zeros], dim=1) if config.NUM_PARAMETER_CHANNELS == 4: ## If we predict depthmap and normal map for each instance, we compute normals again (not used) masks_cropped = masks[:, 0:1, 80:560] mask_sum = masks_cropped.sum(-1).sum(-1) plane_normals = (masks[:, 2:5, 80:560] * masks_cropped).sum(-1).sum(-1) / mask_sum plane_normals = plane_normals / torch.clamp( torch.norm(plane_normals, dim=-1, keepdim=True), min=1e-4) XYZ_np_cropped = (ranges * masks[:, 1:2])[:, :, 80:560] offsets = ((plane_normals.view(-1, 3, 1, 1) * XYZ_np_cropped).sum( 1, keepdim=True) * masks_cropped).sum(-1).sum(-1) / mask_sum plane_parameters = plane_normals * offsets.view((-1, 1)) masks = masks[:, 0] else: if config.NUM_PARAMETER_CHANNELS > 0: ## If we predict depthmap independently for each instance, we use the individual depthmap instead of the global depth map (not used) if config.OCCLUSION: XYZ_np = ranges * depth_np XYZ_np_cropped = XYZ_np[:, 80:560] masks_cropped = masks[:, 1, 80:560] masks = masks[:, 0] else: XYZ_np_cropped = (ranges * masks[:, 1:2])[:, :, 80:560] masks = masks[:, 0] masks_cropped = masks[:, 80:560] pass else: ## We use the global depthmap prediction to compute plane offsets # print('ranges', type(ranges),len(ranges),ranges[1].size()) # print('depth', type(depth_np),len(depth_np),depth_np[0].size()) # print('we come here') XYZ_np = ranges * depth_np[0] XYZ_np_cropped = XYZ_np[:, 80:560] masks_cropped = masks[:, 80:560] pass if config.FITTING_TYPE % 2 == 1: ## We fit all plane parameters using depthmap prediction (not used) A = masks_cropped.unsqueeze(1) * XYZ_np_cropped b = masks_cropped Ab = (A * b.unsqueeze(1)).sum(-1).sum(-1) AA = (A.unsqueeze(2) * A.unsqueeze(1)).sum(-1).sum(-1) plane_parameters = torch.stack([ torch.matmul(torch.inverse(AA[planeIndex]), Ab[planeIndex]) for planeIndex in range(len(AA)) ], dim=0) plane_offsets = torch.norm(plane_parameters, dim=-1, keepdim=True) plane_parameters = plane_parameters / torch.clamp( torch.pow(plane_offsets, 2), 1e-4) else: ## We compute only plane offset using depthmap prediction # print('and here') plane_parameters = detections[:, 6:9] plane_parameter_uncertainitiy = detections[:, 9: 12] #uncertainity from mrcnn_parameter uncertainity plane_normals = plane_parameters / torch.clamp( torch.norm(plane_parameters, dim=-1, keepdim=True), 1e-4) plane_param_uncert_normal_temp = plane_parameter_uncertainitiy / torch.clamp( torch.norm(plane_parameters, dim=-1, keepdim=True), 1e-4) offsets = ( (plane_normals.view(-1, 3, 1, 1) * XYZ_np_cropped).sum(1) * masks_cropped).sum(-1).sum(-1) / torch.clamp( masks_cropped.sum(-1).sum(-1), min=1e-4) plane_parameters = plane_normals * offsets.view((-1, 1)) plane_parameter_uncertainity = plane_param_uncert_normal_temp * offsets.view( (-1, 1)) pass pass detections = torch.cat([ detections[:, :6], plane_parameters, plane_parameter_uncertainity ], dim=-1) # print(detections) # uncertainities are quite large after this step # revisit pass return detections, masks
def recursiveNystrom(K, s, correction=True, minEig=1e-16, expand_eigs=True, eps=1e-16, accelerated=False): device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") n = K.size()[0] if accelerated: sLevel = np.ceil(np.sqrt(n * s + s ^ 3) / (4 * n)) else: sLevel = s # start of the algorithm oversamp = np.math.log(sLevel) k = np.ceil(sLevel / (4 * oversamp)).astype(int) nLevels = np.int(np.ceil(np.math.log2(n / sLevel))) # random permutation for successful uniform samples perm = torch.randperm(n).to(device) # set up sizes for recursive levels lSize = np.zeros(nLevels + 1) lSize[0] = n for i in range(1, nLevels + 1): lSize[i] = np.ceil(lSize[i - 1] / 2) pass lSize = lSize.astype(int) # rInd: indices of points selected at previous level of recursion # at the base level it is just a uniform sample of ~sLevel points samp = list(range(0, lSize[-1])) rInd = perm[samp] weights = torch.ones([len(rInd), 1], dtype=torch.float64).to(device) # diagonal of the whole matrix is np.diag(K) # main recursion, unrolled for efficiency for l in range(nLevels - 1, -1, -1): # indices of current uniform samples rIndCurr = perm[0:lSize[l]] # sampled kernel KS = K[np.ix_(rIndCurr.detach().cpu().numpy(), rInd.detach().cpu().numpy())] KS = KS.to(device) SKS = KS[samp] SKS = SKS.to(device) # print("checking for loops:", SKS[0,0], rIndCurr[0], rInd[0]) SKSn = SKS.size()[0] ################### START MIN EIG CORRECTION ############################################### if correction == True: if expand_eigs == False: # compute local minEig minEig = compute_minEig(SKS, eps=eps) # correct using precomputed minEig cols = list(range(SKSn)) KS[samp, cols] = KS[samp, cols] - minEig SKS[cols, cols] = SKS[cols, cols] - minEig ########################## END MIN EIG CORRECTION ########################################## # print("is SkS PSD:", is_pos_def(SKS)) # optimal lambda for taking O(klogk) samples if k >= SKSn: # for the rare chance we take less than k samples in a round lambda_ = 10e-6 # don't set to zero for stability issues else: lambda_ = ( torch.sum(torch.diag(SKS) * torch.pow(weights, 2)) - \ torch.sum(torch.abs(torch.real( get_top_k_eigenvalues(SKS*torch.pow(weights, 2), k) ))) ) / k # for the case when lambda may be set to zero if lambda_ < 10e-6: lambda_ = 10e-6 # print(np.diag(SKS)) # print("sum first:", np.sum(np.diag(SKS)*weights**2), \ # "sum second:", np.sum(np.abs(np.real( eigs(SKS*(weights**2), k)[0] ))), "lambda:", lambda_) pass # compute and sample by lambda ridge leverage scores if l != 0: # on intermediate levels, we independently sample each column # by its leverage score. the sample size is sLevel in expectation R = torch.inverse(SKS + torch.diag(lambda_ * torch.pow(weights, -2))).to( dtype=KS.dtype).to(device) # max(0,.) helps avoid numerical issues, unnecessary in theory levs = torch.minimum( torch.ones_like(rIndCurr), oversamp*(1/lambda_)*\ torch.maximum(torch.zeros_like(rIndCurr), torch.diag(K)[rIndCurr]- \ torch.sum(torch.matmul(KS,R)*KS, dim=1)) ) levs = rescale_levs(levs) samp = np.where( np.random.random(lSize[l]) < levs.detach().cpu().numpy())[0] # with very low probability, we could accidentally sample no # columns. In this case, just take a fixed size uniform sample. if len(samp) == 0: levs[:] = sLevel / lSize[l] samp = random.sample(range(lSize[l]), sLevel) pass weights = torch.sqrt(1 / levs[samp]) else: # on the top level, we sample exactly s landmark points without replacement R = torch.inverse(SKS + torch.diag(lambda_ * torch.pow(weights, -2))).to( dtype=KS.dtype).to(device) levs = torch.minimum(torch.ones_like(rIndCurr), (1/lambda_)*\ torch.maximum(torch.zeros_like(rIndCurr), torch.diag(K)[rIndCurr]- \ torch.sum(torch.matmul(KS,R)*KS, dim=1)) ) levs = rescale_levs(levs) samp = np.random.choice(n, size=s, replace=False, \ p=levs.detach().cpu().numpy()/sum(levs.detach().cpu().numpy())) pass rInd = perm[samp] C = K[np.ix_(list(range(len(K))), rInd.detach().cpu().numpy())] SKS = C[rInd] # correct SKS for min Eig SKSn = SKS.size()[0] indices = range(SKSn) SKS[indices, indices] = SKS[indices, indices] - minEig + eps # print("is SKS PSD:", is_pos_def(SKS)) W = torch.inverse(SKS + (10e-6) * torch.eye(s).to(device)) error = torch.norm(K - torch.matmul(torch.matmul( C, W), torch.transpose(C, 0, 1))) / torch.norm(K) return C, W, error, minEig
def render_scene(vertex_positions: InputTensor, view_projection_matrix: InputTensor = None, image_size: Tuple[int, int] = (256, 256), normals: InputTensor = None, tex_coords: InputTensor = None, material_ids: InputTensor = None, diffuse_coefficients: InputTensor = None, diffuse_textures: InputTensor = None, diffuse_texture_indices: InputTensor = None, specular_coefficient: InputTensor = None, ambient_coefficients: InputTensor = None, cull_back_facing=True, light_position: InputTensor = None, light_color: InputTensor = (1.0, 1.0, 1.0), ambient_light_color: InputTensor = (0.2, 0.2, 0.2), clear_color: InputTensor = (0, 0, 0, 1), output_type=t.uint8, vertex_shader=None, geometry_shader=None, fragment_shader=None, debug_io_buffer=None, return_rgb=True, cuda_device=None): """Renders the given scene. Args: vertex_positions: The triangle geometry, specified through the triangle vertex positions, float32[num_triangles, 3, 3] view_projection_matrix: The view projection matrix, float32[4, 4] image_size: Desired output image size, (height, width), normals: Per-vertex shading normals, float32[num_triangles, 3, 3]. If set to None, normals will be computed from the vertex positions. tex_coords: Texture coordinate, float32[num_triangles, 3, 2]. If set to None, all texture coordinates will be 0. material_ids: Per-triangle material indices used to index in the various coefficient tensors below, int32[num_triangles]. If set to None, all triangles will have the same default material. diffuse_coefficients: The diffuse coefficients, one per material, float32[num_materials, 3]. Cannot be None if material_ids is not None. Must be None if material_ids is None. diffuse_textures: uint8[num_textures, height, width, 3]. Can be None if there are no textures used in the mesh. diffuse_texture_indices: Diffuse texture indices, one per material, int32[num_materials]. If set to None, the texture indices for all materials will be -1. specular_coefficient: Specular coefficients, one per material, float32[num_materials, 4]. The first 3 channels are the R, G, and B specular coefficients, the last channel is the specular power. If set to None, R, G, and B will be 0 for all materials and power will be 2048. ambient_coefficients: float32[num_materials, 3]. The ambient coefficients. If None, all ambient coefficient will be 0.05. cull_back_facing: whether to cull backfacing triangles. light_position: float32[3], the light position. If set to None, the light will be placed at the camera origin. light_color: The light diffuse RGB color, float32[3] ambient_light_color: The light ambient RGB color, float32[3] clear_color: The RGB color to use when clearing the image, float32[3] output_type: The desired output type. Either tf.uint8 or tf.float32. vertex_shader: The vertex shader to use. If empty, uses a default shader. geometry_shader: The geometry shader. If empty, uses a default shader. fragment_shader: The fragment shader. If empty, uses a default shader. debug_io_buffer: Aids debugging of shaders. Shaders can communicate with host programs through OpenGL input/output buffers. Any tensor passed in this argument will be forwarded to the shaders as buffer with name "debug_io". return_rgb: If true, returns a 3 channel image, otherwise returns a 4 channel image. cuda_device: The index of the GPU to use, given as CUDA device Returns: The rendered image, dt[height, width, c] where dt is either float32 or uint8 depending on the value of output_type and c is either 3 or 4, depending on return_rgb. If the debug_io_buffer argument was not None, returns a tuple containing the rendered image, and the shader output from the "debug_io" buffer. The second element of the tuple has the same shape and type as debug_io_buffer. """ height, width = image_size vertex_positions = util.to_tensor(vertex_positions, t.float32, "cpu") assert (len(vertex_positions.shape) == 3 and vertex_positions.shape[1:] == (3, 3)) num_triangles = vertex_positions.shape[0] if view_projection_matrix is None: view_projection_matrix = camera_util.get_default_camera_for_mesh( vertex_positions) view_projection_matrix = util.to_tensor(view_projection_matrix, t.float32, "cpu") assert view_projection_matrix.shape == (4, 4) has_normals = True if normals is None: normals = t.zeros_like(vertex_positions) has_normals = False normals = util.to_tensor(normals, t.float32, "cpu") assert normals.shape == (num_triangles, 3, 3) if tex_coords is None: tex_coords = t.zeros([num_triangles, 3, 2], dtype=t.float32) tex_coords = util.to_tensor(tex_coords, t.float32, "cpu") assert tex_coords.shape == (num_triangles, 3, 2) if material_ids is None: material_ids = t.zeros([num_triangles], dtype=t.int32) material_ids = util.to_tensor(material_ids, t.int32, "cpu") assert material_ids.shape == (num_triangles, ) num_used_materials = material_ids.max().cpu().numpy() + 1 # type: int def create_coefficient_array(cur_tensor: InputTensor, num_channels, default_value): arr = cur_tensor if arr is None: arr = ( t.ones([num_used_materials, num_channels], dtype=t.float32) * t.tensor(default_value)) arr = util.to_tensor(arr, t.float32, "cpu") assert len(arr.shape) == 2 arr = arr[:num_used_materials] assert arr.shape == (num_used_materials, num_channels) return arr diffuse_coefficients = create_coefficient_array(diffuse_coefficients, 3, 0.8) ambient_coefficients = create_coefficient_array(ambient_coefficients, 3, 0.05) specular_coefficient = create_coefficient_array(specular_coefficient, 4, (0, 0, 0, 2048.0)) if diffuse_texture_indices is None: diffuse_texture_indices = t.ones([num_used_materials], dtype=t.int32) * -1 diffuse_texture_indices = util.to_tensor(diffuse_texture_indices, t.int32, "cpu") assert len(diffuse_texture_indices.shape) == 1 diffuse_texture_indices = diffuse_texture_indices[:num_used_materials] assert diffuse_texture_indices.shape == (num_used_materials, ) num_used_textures = diffuse_texture_indices.max().cpu().numpy() + 1 num_used_textures = max(num_used_textures, 1) if diffuse_textures is None: diffuse_textures = t.ones([num_used_textures, 1, 1, 3], dtype=t.uint8) diffuse_textures = util.to_tensor(diffuse_textures, t.uint8, "cpu") assert len(diffuse_textures.shape) == 4 diffuse_textures = diffuse_textures[:num_used_textures] assert (diffuse_textures.shape[0] == num_used_textures and diffuse_textures.shape[3] == 3) camera_position = t.mv(t.inverse(view_projection_matrix), t.tensor([0, 0, -1, 1], dtype=t.float32)) camera_position = camera_position[:3] / camera_position[3] if light_position is None: light_position = camera_position light_position = util.to_tensor(light_position, t.float32, "cpu") assert light_position.shape == (3, ) light_color = util.to_tensor(light_color, t.float32, "cpu") assert light_color.shape == (3, ) ambient_light_color = util.to_tensor(ambient_light_color, t.float32, "cpu") assert ambient_light_color.shape == (3, ) ambient_coefficients = t.constant_pad_nd(ambient_coefficients, [0, 1]) diffuse_coefficients = t.cat([ diffuse_coefficients, diffuse_texture_indices.to(t.float32)[:, np.newaxis] ], -1) materials = t.cat( [ambient_coefficients, diffuse_coefficients, specular_coefficient], dim=-1) render_args = [ rasterizer.Uniform("view_projection_matrix", view_projection_matrix), rasterizer.Uniform("light_position", light_position), rasterizer.Uniform("has_normals", has_normals), rasterizer.Uniform("has_texcoords", True), rasterizer.Buffer(0, vertex_positions.reshape([-1])), rasterizer.Buffer(1, normals.reshape([-1])), rasterizer.Buffer(2, tex_coords.reshape([-1])), rasterizer.Buffer(3, material_ids.reshape([-1])), rasterizer.Buffer(4, materials.reshape([-1])), rasterizer.Texture("textures", diffuse_textures, bind_as_array=True), rasterizer.Uniform("light_color", light_color), rasterizer.Uniform("camera_position", camera_position), rasterizer.Uniform("ambient_light_color", ambient_light_color), rasterizer.Uniform("cull_backfacing", cull_back_facing), ] if debug_io_buffer is not None: render_args.append(rasterizer.Buffer(5, debug_io_buffer, is_io=True)) if not geometry_shader: geometry_shader = resources.read_text(shaders, "triangle_renderer.geom") if not vertex_shader: vertex_shader = resources.read_text(shaders, "noop.vert") if not fragment_shader: fragment_shader = resources.read_text(shaders, "point_light_illumination.frag") result = rasterizer.gl_simple_render(rasterizer.RenderInput( num_points=num_triangles, arguments=render_args, output_resolution=(height, width), clear_color=clear_color, output_type=output_type, vertex_shader=vertex_shader, geometry_shader=geometry_shader, fragment_shader=fragment_shader), cuda_device=cuda_device) c = 3 if return_rgb else 4 if debug_io_buffer is None: return result[..., :c] else: return result[..., :c], render_args[-1].value
def inverse(self, regul=1e-8): inv_tensor = torch.inverse(self.data + regul * torch.eye(self.size(0), device=self.data.device)) return PMatDense(generator=self.generator, data=inv_tensor)
def BatchInverse(tensor): batch_size = tensor.size()[0] tensor_inverse = [] for i in range(batch_size): tensor_inverse += [torch.inverse(tensor[i]).unsqueeze(0)] return torch.cat(tensor_inverse, 0)
def batch_global_rigid_transformation(Rs, Js, parent, rotate_base=False, betas_extra=None, device="cuda"): """ Computes absolute joint locations given pose. rotate_base: if True, rotates the global rotation by 90 deg in x axis. if False, this is the original SMPL coordinate. Args: Rs: N x 24 x 3 x 3 rotation vector of K joints Js: N x 24 x 3, joint locations before posing parent: 24 holding the parent id for each index Returns new_J : `Tensor`: N x 24 x 3 location of absolute joints A : `Tensor`: N x 24 4 x 4 relative joint transformations for LBS. """ # Now Js is N x 24 x 3 x 1 Js = Js.unsqueeze(-1) N = Rs.shape[0] if rotate_base: print('Flipping the SMPL coordinate frame!!!!') rot_x = torch.Tensor([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) rot_x = torch.reshape(torch.repeat(rot_x, [N, 1]), [N, 3, 3]) # In tf it was tile root_rotation = torch.matmul(Rs[:, 0, :, :], rot_x) else: root_rotation = Rs[:, 0, :, :] Js_orig = Js.clone() scaling_factors = torch.ones(N, parent.shape[0], 3).to(device) if betas_extra is not None: scaling_factors = betas_extra.reshape(-1, 35, 3) # debug_only # scaling_factors[:, 25:32, 0] = 0.2 # scaling_factors[:, 7, 2] = 2.0 scale_factors_3x3 = torch.diag_embed(scaling_factors, dim1=-2, dim2=-1) def make_A(R, t): # Rs is N x 3 x 3, ts is N x 3 x 1 R_homo = torch.nn.functional.pad(R, (0, 0, 0, 1, 0, 0)) t_homo = torch.cat([t, torch.ones([N, 1, 1]).to(device)], 1) return torch.cat([R_homo, t_homo], 2) A0 = make_A(root_rotation, Js[:, 0]) results = [A0] for i in range(1, parent.shape[0]): j_here = Js[:, i] - Js[:, parent[i]] s_par_inv = torch.inverse(scale_factors_3x3[:, parent[i]]) rot = Rs[:, i] s = scale_factors_3x3[:, i] rot_new = s_par_inv @ rot @ s A_here = make_A(rot_new, j_here) res_here = torch.matmul(results[parent[i]], A_here) results.append(res_here) # 10 x 24 x 4 x 4 results = torch.stack(results, dim=1) # scale updates new_J = results[:, :, :3, 3] # --- Compute relative A: Skinning is based on # how much the bone moved (not the final location of the bone) # but (final_bone - init_bone) # --- Js_w0 = torch.cat([Js_orig, torch.zeros([N, 35, 1, 1]).to(device)], 2) init_bone = torch.matmul(results, Js_w0) # Append empty 4 x 3: init_bone = torch.nn.functional.pad(init_bone, (3, 0, 0, 0, 0, 0, 0, 0)) A = results - init_bone return new_J, A
def get_right_inverse(w): return torch.mm(w.t(), torch.inverse(w.mm(w.t())))
def warp_affine( src: torch.Tensor, M: torch.Tensor, dsize: Tuple[int, int], flags: str = "bilinear", padding_mode: str = "zeros", align_corners: bool = False, ) -> torch.Tensor: r"""Applies an affine transformation to a tensor. The function warp_affine transforms the source tensor using the specified matrix: .. math:: \text{dst}(x, y) = \text{src} \left( M_{11} x + M_{12} y + M_{13} , M_{21} x + M_{22} y + M_{23} \right ) Args: src (torch.Tensor): input tensor of shape :math:`(B, C, H, W)`. M (torch.Tensor): affine transformation of shape :math:`(B, 2, 3)`. dsize (Tuple[int, int]): size of the output image (height, width). mode (str): interpolation mode to calculate output values 'bilinear' | 'nearest'. Default: 'bilinear'. padding_mode (str): padding mode for outside grid values 'zeros' | 'border' | 'reflection'. Default: 'zeros'. align_corners (bool): mode for grid_generation. Default: False. See https://pytorch.org/docs/stable/nn.functional.html#torch.nn.functional.interpolate for details Returns: torch.Tensor: the warped tensor. Shape: - Output: :math:`(B, C, H, W)` .. note:: See a working example `here <https://kornia.readthedocs.io/en/latest/ tutorials/warp_affine.html>`__. """ if not torch.is_tensor(src): raise TypeError("Input src type is not a torch.Tensor. Got {}".format( type(src))) if not torch.is_tensor(M): raise TypeError("Input M type is not a torch.Tensor. Got {}".format( type(M))) if not len(src.shape) == 4: raise ValueError("Input src must be a BxCxHxW tensor. Got {}".format( src.shape)) if not (len(M.shape) == 3 or M.shape[-2:] == (2, 3)): raise ValueError("Input M must be a Bx2x3 tensor. Got {}".format( M.shape)) B, C, H, W = src.size() dsize_src = (H, W) out_size = dsize # we generate a 3x3 transformation matrix from 2x3 affine M_3x3: torch.Tensor = convert_affinematrix_to_homography(M) dst_norm_trans_src_norm: torch.Tensor = normalize_homography( M_3x3, dsize_src, out_size) src_norm_trans_dst_norm = torch.inverse(dst_norm_trans_src_norm) grid = F.affine_grid( src_norm_trans_dst_norm[:, :2, :], [B, C, out_size[0], out_size[1]], align_corners=align_corners, ) return F.grid_sample(src, grid, align_corners=align_corners, mode=flags, padding_mode=padding_mode)
def ransac_voting_layer(mask, vertex, round_hyp_num, inlier_thresh=0.999, confidence=0.99, max_iter=20, min_num=5, max_num=30000): ''' :param mask: [b,h,w] :param vertex: [b,h,w,vn,2] :param round_hyp_num: :param inlier_thresh: :return: [b,vn,2] ''' b, h, w, vn, _ = vertex.shape batch_win_pts = [] for bi in range(b): hyp_num = 0 cur_mask = (mask[bi]).byte() foreground_num = torch.sum(cur_mask) # if too few points, just skip it if foreground_num < min_num: win_pts = torch.zeros([1, vn, 2], dtype=torch.float32, device=mask.device) batch_win_pts.append(win_pts) # [1,vn,2] continue # if too many inliers, we randomly down sample it if foreground_num > max_num: selection = torch.zeros(cur_mask.shape, dtype=torch.float32, device=mask.device).uniform_(0, 1) selected_mask = (selection < (max_num / foreground_num.float())) cur_mask *= selected_mask coords = torch.nonzero(cur_mask).float() # [tn,2] coords = coords[:, [1, 0]] direct = vertex[bi].masked_select(torch.unsqueeze(torch.unsqueeze(cur_mask, 2), 3)) # [tn,vn,2] direct = direct.view([coords.shape[0], vn, 2]) tn = coords.shape[0] idxs = torch.zeros([round_hyp_num, vn, 2], dtype=torch.int32, device=mask.device).random_(0, direct.shape[0]) all_win_ratio = torch.zeros([vn], dtype=torch.float32, device=mask.device) all_win_pts = torch.zeros([vn, 2], dtype=torch.float32, device=mask.device) cur_iter = 0 while True: # generate hypothesis cur_hyp_pts = ransac_voting.generate_hypothesis(direct, coords, idxs) # [hn,vn,2] # voting for hypothesis cur_inlier = torch.zeros([round_hyp_num, vn, tn], dtype=torch.uint8, device=mask.device) ransac_voting.voting_for_hypothesis(direct, coords, cur_hyp_pts, cur_inlier, inlier_thresh) # [hn,vn,tn] # find max cur_inlier_counts = torch.sum(cur_inlier, 2) # [hn,vn] cur_win_counts, cur_win_idx = torch.max(cur_inlier_counts, 0) # [vn] cur_win_pts = cur_hyp_pts[cur_win_idx, torch.arange(vn)] cur_win_ratio = cur_win_counts.float() / tn # update best point larger_mask = all_win_ratio < cur_win_ratio all_win_pts[larger_mask,:] = cur_win_pts[larger_mask,:] all_win_ratio[larger_mask] = cur_win_ratio[larger_mask] # check confidence hyp_num += round_hyp_num cur_iter += 1 cur_min_ratio = torch.min(all_win_ratio) if (1 - (1 - cur_min_ratio ** 2) ** hyp_num) > confidence or cur_iter > max_iter: break # compute mean intersection again normal = torch.zeros_like(direct) # [tn,vn,2] normal[:,:, 0] = direct[:,:, 1] normal[:,:, 1] = -direct[:,:, 0] all_inlier = torch.zeros([1, vn, tn], dtype=torch.uint8, device=mask.device) all_win_pts = torch.unsqueeze(all_win_pts, 0) # [1,vn,2] ransac_voting.voting_for_hypothesis(direct, coords, all_win_pts, all_inlier, inlier_thresh) # [1,vn,tn] # coords [tn,2] normal [vn,tn,2] all_inlier = torch.squeeze(all_inlier.float(), 0) # [vn,tn] normal = normal.permute(1, 0, 2) # [vn,tn,2] normal = normal*torch.unsqueeze(all_inlier, 2) # [vn,tn,2] outlier is all zero b = torch.sum(normal*torch.unsqueeze(coords, 0), 2) # [vn,tn] ATA = torch.matmul(normal.permute(0, 2, 1), normal) # [vn,2,2] ATb = torch.sum(normal*torch.unsqueeze(b, 2), 1) # [vn,2] try: all_win_pts = torch.matmul(torch.inverse(ATA), torch.unsqueeze(ATb, 2)) # [vn,2,1] batch_win_pts.append(all_win_pts[None,:,:, 0]) except: all_win_pts = torch.zeros([1, ATA.size(0), 2]).to(ATA.device) batch_win_pts.append(all_win_pts) batch_win_pts = torch.cat(batch_win_pts) return batch_win_pts
def forward(x, xref, uref, _lambda, verbose=False, acc=False, detach=False): # x: bs x n x 1 bs = x.shape[0] W = W_func(x) M = torch.inverse(W) f = f_func(x) B = B_func(x) DfDx = Jacobian(f, x) DBDx = torch.zeros(bs, num_dim_x, num_dim_x, num_dim_control).type(x.type()) for i in range(num_dim_control): DBDx[:, :, :, i] = Jacobian(B[:, :, i].unsqueeze(-1), x) _Bbot = Bbot_func(x) u = u_func(x, x - xref, uref) # u: bs x m x 1 # TODO: x - xref K = Jacobian(u, x) A = DfDx + sum([ u[:, i, 0].unsqueeze(-1).unsqueeze(-1) * DBDx[:, :, :, i] for i in range(num_dim_control) ]) dot_x = f + B.matmul(u) dot_M = weighted_gradients(M, dot_x, x, detach=detach) # DMDt dot_W = weighted_gradients(W, dot_x, x, detach=detach) # DWDt if detach: Contraction = dot_M + (A + B.matmul(K)).transpose(1, 2).matmul( M.detach()) + M.detach().matmul( A + B.matmul(K)) + 2 * _lambda * M.detach() else: Contraction = dot_M + (A + B.matmul(K)).transpose( 1, 2).matmul(M) + M.matmul(A + B.matmul(K)) + 2 * _lambda * M # C1 C1_inner = -weighted_gradients(W, f, x) + DfDx.matmul(W) + W.matmul( DfDx.transpose(1, 2)) + 2 * _lambda * W C1_LHS_1 = _Bbot.transpose(1, 2).matmul(C1_inner).matmul( _Bbot) # this has to be a negative definite matrix # C2 C2_inners = [] C2s = [] for j in range(num_dim_control): C2_inner = weighted_gradients(W, B[:, :, j].unsqueeze(-1), x) - ( DBDx[:, :, :, j].matmul(W) + W.matmul(DBDx[:, :, :, j].transpose(1, 2))) C2 = _Bbot.transpose(1, 2).matmul(C2_inner).matmul(_Bbot) C2_inners.append(C2_inner) C2s.append(C2) loss = 0 loss += loss_pos_matrix_random_sampling( -Contraction - epsilon * torch.eye(Contraction.shape[-1]).unsqueeze(0).type(x.type())) loss += loss_pos_matrix_random_sampling( -C1_LHS_1 - epsilon * torch.eye(C1_LHS_1.shape[-1]).unsqueeze(0).type(x.type())) loss += loss_pos_matrix_random_sampling( args.w_ub * torch.eye(W.shape[-1]).unsqueeze(0).type(x.type()) - W) loss += 1. * sum( [1. * (C2**2).reshape(bs, -1).sum(dim=1).mean() for C2 in C2s]) if verbose: print( torch.symeig(Contraction)[0].min(dim=1)[0].mean(), torch.symeig(Contraction)[0].max(dim=1)[0].mean(), torch.symeig(Contraction)[0].mean()) if acc: return loss, ((torch.symeig(Contraction)[0] >= 0).sum( dim=1) == 0).cpu().detach().numpy(), (( torch.symeig(C1_LHS_1)[0] >= 0).sum( dim=1) == 0).cpu().detach().numpy(), sum([ 1. * (C2**2).reshape(bs, -1).sum(dim=1).mean() for C2 in C2s ]).item() else: return loss, None, None, None
def inverse(self, z, log_jacobian): w = torch.inverse(self.w) x = F.conv2d(z, w.unsqueeze(-1).unsqueeze(-1)) log_jacobian = log_jacobian + (torch.slogdet(w)[1] / z.size(1)) return x, log_jacobian
def _calc_integrals(): ######################## short-range integrals ######################## ############# 3-centre 2-electron integral ############# _basisw, _fusew = intor.LibcintWrapper.concatenate( self._wrapper, fuse_aux_wrapper) # (nkpts_ij, nao, nao, nxao+nxcao) j3c_short_f = intor.pbc_coul3c(_basisw, other2=_fusew, kpts_ij=kpts_ij, options=self._lattsum_opt) j3c_short = j3c_short_f[..., :nxao] - j3c_short_f[ ..., nxao:] # (nkpts_ij, nao, nao, nxao) ############# 2-centre 2-electron integrals ############# # (nkpts_unique, nxao+nxcao, nxao+nxcao) j2c_short_f = intor.pbc_coul2c(fuse_aux_wrapper, kpts=kpts_reduce, options=self._lattsum_opt) # j2c_short: (nkpts_unique, nxao, nxao) j2c_short = j2c_short_f[..., :nxao, :nxao] + j2c_short_f[..., nxao:, nxao:] \ - j2c_short_f[..., :nxao, nxao:] - j2c_short_f[..., nxao:, :nxao] ######################## long-range integrals ######################## # only use the compensating wrapper as the gcut gcut = get_gcut(self._lattsum_opt.precision, [aux_comp_wrapper]) # gvgrids: (ngv, ndim), gvweights: (ngv,) gvgrids, gvweights = self._lattice.get_gvgrids(gcut) ngv = gvgrids.shape[0] gvk = gvgrids.unsqueeze(-2) + kpts_reduce # (ngv, nkpts_ij, ndim) gvk = gvk.view(-1, gvk.shape[-1]) # (ngv * nkpts_ij, ndim) # get the fourier transform variables # TODO: iterate over ngv axis # ft of the compensating basis comp_ft = intor.eval_gto_ft(aux_comp_wrapper, gvk) # (nxcao, ngv * nkpts_ij) comp_ft = comp_ft.view(-1, ngv, nkpts_ij) # (nxcao, ngv, nkpts_ij) # ft of the auxiliary basis auxb_ft_c = intor.eval_gto_ft(aux_wrapper, gvk) # (nxao, ngv * nkpts_ij) auxb_ft_c = auxb_ft_c.view(-1, ngv, nkpts_ij) # (nxao, ngv, nkpts_ij) auxb_ft = auxb_ft_c - comp_ft # (nxao, ngv, nkpts_ij) # ft of the overlap integral of the basis (nkpts_ij, nao, nao, ngv) aoao_ft = self._get_pbc_overlap_with_kpts_ij( gvgrids, kpts_reduce, kpts_j) # ft of the coulomb kernel coul_ft = unweighted_coul_ft(gvk) # (ngv * nkpts_ij,) coul_ft = coul_ft.to(comp_ft.dtype).view( ngv, nkpts_ij) * gvweights.unsqueeze(-1) # (ngv, nkpts_ij) # 1: (nkpts_ij, nxao, nxao) pattern = "gi,xgi,ygi->ixy" j2c_long = torch.einsum(pattern, coul_ft, comp_ft.conj(), auxb_ft) # 2: (nkpts_ij, nxao, nxao) j2c_long += torch.einsum(pattern, coul_ft, auxb_ft.conj(), comp_ft) # 3: (nkpts_ij, nxao, nxao) j2c_long += torch.einsum(pattern, coul_ft, comp_ft.conj(), comp_ft) # calculate the j3c long-range patternj3 = "gi,xgi,iyzg->iyzx" # (nkpts_ij, nao, nao, nxao) j3c_long = torch.einsum(patternj3, coul_ft, comp_ft.conj(), aoao_ft) # get the average potential auxbar_f = self._auxbar( kpts_reduce, fuse_aux_wrapper) # (nkpts_ij, nxao + nxcao) auxbar = auxbar_f[:, :nxao] - auxbar_f[:, nxao:] # (nkpts_ij, nxao) auxbar = auxbar.reshape(nkpts, nkpts, auxbar.shape[-1]) # (nkpts, nkpts, nxao) olp_mat = intor.pbc_overlap( self._wrapper, kpts=self._kpts, options=self._lattsum_opt) # (nkpts, nao, nao) j3c_bar = auxbar[:, :, None, None, :] * olp_mat[ ..., None] # (nkpts, nkpts, nao, nao, nxao) j3c_bar = j3c_bar.reshape( -1, *j3c_bar.shape[2:]) # (nkpts_ij, nao, nao, nxao) ######################## combining integrals ######################## j2c = j2c_short + j2c_long # (nkpts_ij, nxao, nxao) j3c = j3c_short + j3c_long - j3c_bar # (nkpts_ij, nao, nao, nxao) el_mat = torch.einsum("kxy,kaby->kabx", torch.inverse(j2c), j3c) # (nkpts_ij, nao, nao, nxao) return j2c, j3c, el_mat
def ray_sampling(Ks, Ts, image_size, masks=None, mask_threshold=0.5, images=None): h = image_size[0] w = image_size[1] M = Ks.size(0) x = torch.linspace(0, h - 1, steps=h, device=Ks.device) y = torch.linspace(0, w - 1, steps=w, device=Ks.device) grid_x, grid_y = torch.meshgrid(x, y) coordinates = torch.stack([grid_y, grid_x]).unsqueeze(0).repeat(M, 1, 1, 1) #(M,2,H,W) coordinates = torch.cat([ coordinates, torch.ones(coordinates.size(0), 1, coordinates.size(2), coordinates.size(3), device=Ks.device) ], dim=1).permute(0, 2, 3, 1).unsqueeze(-1) inv_Ks = torch.inverse(Ks) dirs = torch.matmul(inv_Ks, coordinates) #(M,H,W,3,1) dirs = dirs / torch.norm(dirs, dim=3, keepdim=True) dirs = torch.cat([ dirs, torch.zeros(dirs.size(0), coordinates.size(1), coordinates.size(2), 1, 1, device=Ks.device) ], dim=3) #(M,H,W,4,1) dirs = torch.matmul(Ts, dirs) #(M,H,W,4,1) dirs = dirs[:, :, :, 0:3, 0] #(M,H,W,3) pos = Ts[:, 0:3, 3] #(M,3) pos = pos.unsqueeze(1).unsqueeze(1).repeat(1, h, w, 1) rays = torch.cat([dirs, pos], dim=3) #(M,H,W,6) if images is not None: rgbs = images.permute(0, 2, 3, 1) #(M,H,W,C) else: rgbs = None if masks is not None: rays = rays[masks > mask_threshold, :] if rgbs is not None: rgbs = rgbs[masks > mask_threshold, :] else: rays = rays.reshape((-1, rays.size(3))) if rgbs is not None: rgbs = rgbs.reshape((-1, rgbs.size(3))) return rays, rgbs
def se(self, X, Y, batch_size=25, parallel=0, sampling=100, each_species=True): dataLoader = self._get_DataLoader(X, Y, batch_size=batch_size, shuffle=False) loss_func = self.__build_loss_function(train=True) se = [] y_dim = np.size(self.weights_numpy[0][0], 1) weights = self.weights[0][0] _ = sys.stdout.write("\nCalculating standard errors...\n") if each_species: for i in range(y_dim): _ = sys.stdout.write("\rSpecies: {}/{} ".format(i + 1, y_dim)) sys.stdout.flush() weights = torch.tensor(self.weights_numpy[0][0][:, i].reshape( [-1, 1]), device=self.device, dtype=self.dtype, requires_grad=True).to(self.device) if i == 0: constants = torch.tensor(self.weights_numpy[0][0][:, (i + 1):], device=self.device, dtype=self.dtype).to(self.device) w = torch.cat([weights, constants], dim=1) elif i < y_dim: w = torch.cat([ torch.tensor(self.weights_numpy[0][0][:, 0:i], device=self.device, dtype=self.dtype).to(self.device), weights, torch.tensor(self.weights_numpy[0][0][:, (i + 1):], device=self.device, dtype=self.dtype).to(self.device) ], dim=1) else: constants = torch.tensor(self.weights_numpy[0][0][:, 0:i], device=self.device, dtype=self.dtype).to(self.device) w = torch.cat([constants, weights], dim=1) for step, (x, y) in enumerate(dataLoader): x = x.to(self.device, non_blocking=True) y = y.to(self.device, non_blocking=True) mu = torch.nn.functional.linear(x, w.t()) loss = loss_func(mu, y, x.shape[0], sampling) loss = torch.sum(loss) first_gradients = torch.autograd.grad(loss, weights, retain_graph=True, create_graph=True, allow_unused=True) second = [] for j in range(self.input_shape): second.append( torch.autograd.grad(first_gradients[0][j, 0], inputs=weights, retain_graph=True, create_graph=False, allow_unused=False)[0]) hessian = torch.cat(second, dim=1) if step < 1: hessian_out = hessian else: hessian_out += hessian se.append( torch.sqrt(torch.diag( torch.inverse(hessian_out))).data.cpu().numpy()) return se else: for step, (x, y) in enumerate(dataLoader): x = x.to(self.device, non_blocking=True) y = y.to(self.device, non_blocking=True) mu = self.layers[0](x) loss = torch.sum(loss_func(mu, y, x.shape[0], sampling)) first_gradients = torch.autograd.grad( loss, weights, retain_graph=True, create_graph=True, allow_unused=True)[0].reshape([-1]) hessian = [] for j in range(first_gradients.shape[0]): hessian.append( torch.autograd.grad( first_gradients[j], inputs=weights, retain_graph=True, create_graph=False, allow_unused=False)[0].reshape([-1]).reshape( [y_dim * self.input_shape, 1])) hessian = torch.cat(hessian, dim=1) if step < 1: hessian_out = hessian else: hessian_out += hessian return hessian_out.data.cpu().numpy()
import torch from torchvision import transforms mat_rgb2lab = torch.tensor([[0.412453, 0.357580, 0.180423], [0.212671, 0.715160, 0.072169], [0.019334, 0.119193, 0.950227]]) mat_lab2rgb = torch.inverse(mat_rgb2lab) d65_2 = torch.tensor([0.95047, 1.00000, 1.08883]) # CIE Standard Illuminant D65, 2° observer def clip(x): ''' Project a tensor's values on [0, 1] (inplace). Parameters ---------- x : torch.Tensor Returns ------- x : torch.Tensor ''' x[x > 1.] = 1. x[x < 0.] = 0. return x def rgb2lab(img_rgb):
def l1_ll(): ss = torch.matmul(self.sigma, self.sigma.t()) if inverse: ss = torch.inverse(ss) return torch.mul( l1, torch.sum(torch.abs(torch.triu(ss, diag))))
import torch import torch.optim as optim from anvil.models import RealNVP from anvil.train import shifted_kl L = 2 # very small system N_UNITS = L**2 np.random.seed(seed=0) A_NP = np.array(np.random.rand(N_UNITS, N_UNITS), dtype=np.float32) A = torch.from_numpy(A_NP) # just make some positive definite matrix COV_TARGET = [email protected] INV = torch.inverse(COV_TARGET) def target_distribution_s(phi): r"""Returns action S(\phi) for a stack of states \phi, shape (N_states, D). The action corresponds to a toy training example of a multigaussian distribution. """ return 0.5*((phi@INV)*phi).sum(axis=-1, keepdim=True) def split_to_flat_index(length): r"""Returns a 1D tensor of size length^2. If used to index a split state such that \phi = (\phi_a, \phi_b) then \phi -> flattened state with \phi_a and \phi_b interlaced in a checkerboard pattern You only need to use this function if the action is defined to accept the flattened state geometry. If the action is defined to act just on a split
def cam_to_ndc(self, value): self._cam_to_ndc = value self.ndc_to_cam = torch.inverse(self._cam_to_ndc)
def compute_projection(self, camera_to_world): # compute projection by voxels -> image world_to_camera = torch.inverse(camera_to_world) voxel_bounds_min, voxel_bounds_max = self.compute_frustum_bounds( camera_to_world) return voxel_bounds_min, voxel_bounds_max, world_to_camera
env.env.state = np.array([x0, 0.0, q0, 0.0], dtype=np.float32) obs = env.env._get_obs() y = torch.tensor([obs[0], obs[1], obs[2], obs[3], obs[4], u10, u20], requires_grad=True, device=device, dtype=torch.float32).view(1, 7) t_eval = torch.linspace(t_span[0], t_span[1], n_eval).to(device) y_traj = [] y_traj.append(y) frames = [] for i in range(len(t_eval)-1): frames.append(env.render(mode='rgb_array')) x_cos_q_sin_q, x_dot_q_dot, u = torch.split(y, [3, 2, 2], dim=1) M_q_inv = symoden_ode_struct_model.M_net(x_cos_q_sin_q) x_dot_q_dot_aug = torch.unsqueeze(x_dot_q_dot, dim=2) p = torch.squeeze(torch.matmul(torch.inverse(M_q_inv), x_dot_q_dot_aug), dim=2) x_cos_q_sin_q_p = torch.cat((x_cos_q_sin_q, p), dim=1) x_cos_q_sin_q, p = torch.split(x_cos_q_sin_q_p, [3, 2], dim=1) M_q_inv = symoden_ode_struct_model.M_net(x_cos_q_sin_q) _, cos_q, sin_q = torch.chunk(x_cos_q_sin_q, 3,dim=1) V_q = symoden_ode_struct_model.V_net(x_cos_q_sin_q) p_aug = torch.unsqueeze(p, dim=2) H = torch.squeeze(torch.matmul(torch.transpose(p_aug, 1, 2), torch.matmul(M_q_inv, p_aug)))/2.0 + torch.squeeze(V_q) dH = torch.autograd.grad(H.sum(), x_cos_q_sin_q_p, create_graph=True)[0] dHdx, dHdcos_q, dHdsin_q, dHdp= torch.split(dH, [1, 1, 1, 2], dim=1) dV = torch.autograd.grad(V_q, x_cos_q_sin_q)[0] dVdx, dVdcos_q, dVdsin_q= torch.chunk(dV, 3, dim=1) dV_q = - dVdcos_q * sin_q + dVdsin_q * cos_q g_xq = symoden_ode_struct_model.g_net(x_cos_q_sin_q) g_xq_T = torch.transpose(g_xq, 1, 2) inv_g_g_T = torch.inverse(torch.matmul(g_xq, g_xq_T))
y = xy[:, :, 1] mtm = construct_pred_MtM(bSize, Pw, x, y) xtmtmx = torch.bmm( torch.bmm(gCc.view(-1, 1, 12), mtm.view(-1, 12, 12)), gCc.view(-1, 12, 1)).view(-1) efLoss = 1e7 * xtmtmx.sum() if False: for n in range(len(Pi)): uv = Pi[n].view(-1, 2) cc = gCc[n].view(-1, 1) # normalize the uv uv1 = torch.cat((uv, torch.ones(len(uv), 1.0).double()), 1) uv = torch.inverse(K).mm(uv1.t()).t() uv = uv[:, :2] Cw, Alpha, M = PnP.PrepareData(Pw, uv) mtm = M.t().mm(M) eigenFreeLoss = cc.t().mm(mtm).mm(cc) efLoss = efLoss + eigenFreeLoss # get RT from M Km = PnP.GetKernel(M) vK = Km[:, 3] # take the last column: the one with smallest eigenvalues vK = vK.view(4, 3) m = vK[:, 2].mean() if (m <= 0).data.numpy(): vK = -vK R, T, r = PnP.Procrustes(Cw, vK)
def differentiable_nms(scores_unsorted, iou_unsorted, nms_threshold=0.4, pruning_method="linear", temperature=0.01, valid_box_prob_threshold=0.3, return_sorted_prob=False, sorting_method="hard", sorting_temperature=None, group_boxes=True, mask_group_boxes=True, group_size=100, debug=False): """ GrooMeD-NMS: Grouped Mathematical Differentiable NMS. Abhinav Kumar :param scores_unsorted: Unsorted scores of the boxes Tensor (N, ) :param iou_unsorted: Overlap matrix of the boxes Tensor (N, N) :param nms_threshold: NMS threshold :param pruning_method: Pruning method/function- Can be one of linear/soft_nms (exponential)/sigmoidal :param temperature: Temperature of the pruning function. If pruning function is linear, this is not used :param valid_box_prob_threshold: Used to decide if the box is valid after re-scoring. :param return_sorted_prob: Whether should we return sorted scores or not :param sorting_method: Sorting should be hard or soft :param sorting_temperature: Sorting temperature for soft sorting. If not supplied, sorting_temperature = temperature :param group_boxes: Grouping should be carried out or not :param mask_group_boxes: Masking should be carried out or not :param group_size: Maximum Group size of the group :param debug: Only for debugging :return: valid_boxes_index: Valid box indices after NMS (Variable tensor of shape between 1 and N) invalid_boxes_index: Invalid box indices after NMS (Variable tensor of shape between N-1 and 0) non_suppression_prob: Re-scores or updated-scores of the boxes after NMS Tensor (N, ) """ if type(scores_unsorted) == np.ndarray: scores_unsorted = torch.from_numpy(scores_unsorted).float() iou_unsorted = torch.from_numpy(iou_unsorted).float() #====================================================================== # Do the sorting of the scores and the corresponding IoU matrix #====================================================================== _, indices = torch.sort(scores_unsorted, descending=True) if sorting_method == "soft": if sorting_temperature is None: sorting_temperature = temperature scores, convex_comb_matrix, iou = soft_sort( scores_unsorted, full_matrix=iou_unsorted, temperature=sorting_temperature) else: scores = scores_unsorted[indices] iou = iou_unsorted[indices][:, indices] if debug: print("\nInside diff NMS... After sorting") print(scores) print(iou) num_boxes = scores.shape[0] mask = torch.eye(num_boxes).byte() identity = torch.eye(num_boxes) ones = torch.ones(num_boxes, ) mask = cast_to_cpu_cuda_tensor(mask, iou) identity = cast_to_cpu_cuda_tensor(identity, iou) ones = cast_to_cpu_cuda_tensor(ones, iou) if group_boxes: inversion_matrix = torch.zeros(iou.shape) inversion_matrix = cast_to_cpu_cuda_tensor(inversion_matrix, iou) #====================================================================== # Prune Matrix #====================================================================== overlap_probabilities = pruning_function(iou, nms_threshold=nms_threshold, temperature=temperature, pruning_method=pruning_method) overlap_probabilities = torch.tril(overlap_probabilities) overlap_probabilities.masked_fill_(mask, 0) phi_matrix = overlap_probabilities #====================================================================== # Inversion or Subtraction #====================================================================== if group_boxes: #================================================================== # Grouping #================================================================== groups = get_groups(iou_unsorted=iou, group_threshold=nms_threshold, group_size=group_size, scores_unsorted=scores) if debug: print("Groups") print(groups) num_groups = len(groups) #================================================================== # Masking #================================================================== if mask_group_boxes: mask = mask.float() mask[:, :] = 0 for j in range(num_groups): mask[groups[j], groups[j][0]] = 1 phi_matrix = phi_matrix * mask # Do groupwise inversion for j in range(num_groups): if mask_group_boxes: temp_store = identity[groups[j]][:, groups[j]] - phi_matrix[ groups[j]][:, groups[j]] else: temp_store = torch.inverse(identity[groups[j]][:, groups[j]] + phi_matrix[groups[j]][:, groups[j]]) inversion_matrix = indices_copy(A=inversion_matrix, B=temp_store, indA=groups[j]) else: inversion_matrix = torch.inverse(identity + phi_matrix) non_suppression_prob_unsort = torch.clamp(torch.matmul( inversion_matrix, scores), min=0, max=1) # non_suppression_prob_unsort = torch.min(non_suppression_prob_unsort, scores) non_suppression_prob_unsort_2 = non_suppression_prob_unsort.clone() non_suppression_prob_unsort[ non_suppression_prob_unsort < valid_box_prob_threshold] = 0 if return_sorted_prob: non_suppression_prob, sorted_indices = torch.sort( non_suppression_prob_unsort, descending=True) valid_boxes_index = indices[sorted_indices[ non_suppression_prob >= valid_box_prob_threshold]] invalid_boxes_index = indices[sorted_indices[ non_suppression_prob < valid_box_prob_threshold]] else: dummy, sorted_indices = torch.sort(non_suppression_prob_unsort, descending=True) valid_boxes_index = indices[sorted_indices[ dummy >= valid_box_prob_threshold]] invalid_boxes_index = indices[sorted_indices[ dummy < valid_box_prob_threshold]] if group_boxes: non_suppression_prob = non_suppression_prob_unsort_2 else: non_suppression_prob = non_suppression_prob_unsort return valid_boxes_index, invalid_boxes_index, non_suppression_prob
def inverse(self, x): return torch.inverse(x)
def QuantizedWeight(x, n, nbit=2, training=False): """ Quantize weight. Args: x (torch.Tensor): a 4D tensor. [K x K x iC x oC] -> [oC x iC x K x K] Must have known number of channels, but can have other unknown dimensions. n (int or double): variance of weight initialization. nbit (int): number of bits of quantized weight. Defaults to 2. training (bool): Returns: torch.Tensor with attribute `variables`. Variable Names: * ``basis``: basis of quantized weight. Note: About multi-GPU training: moving averages across GPUs are not aggregated. Batch statistics are computed by main training tower. This is consistent with most frameworks. """ oc, ic, k1, k2 = x.shape device = x.device init_basis = [] base = NORM_PPF_0_75 * ((2. / n) ** 0.5) / (2 ** (nbit - 1)) for j in range(nbit): init_basis.append([(2 ** j) * base for i in range(oc)]) num_levels = 2 ** nbit delta = EPS # initialize level multiplier # binary code of each level: # shape: [num_levels, nbit] init_level_multiplier = [] for i in range(num_levels): level_multiplier_i = [0. for j in range(nbit)] level_number = i for j in range(nbit): binary_code = level_number % 2 if binary_code == 0: binary_code = -1 level_multiplier_i[j] = float(binary_code) level_number = level_number // 2 init_level_multiplier.append(level_multiplier_i) # initialize threshold multiplier # shape: [num_levels-1, num_levels] # [[0,0,0,0,0,0,0.5,0.5] # [0,0,0,0,0,0.5,0.5,0,] # [0,0,0,0,0.5,0.5,0,0,] # ... # [0.5,0.5,0,0,0,0,0,0,]] init_thrs_multiplier = [] for i in range(1, num_levels): thrs_multiplier_i = [0. for j in range(num_levels)] thrs_multiplier_i[i - 1] = 0.5 thrs_multiplier_i[i] = 0.5 init_thrs_multiplier.append(thrs_multiplier_i) # [nbit, oc] basis = torch.tensor(init_basis, dtype=torch.float32, requires_grad=False) # [2**nbit, nbit] or [num_levels, nbit] level_codes = torch.tensor(init_level_multiplier) # [num_levels-1, num_levels] thrs_multiplier = torch.tensor(init_thrs_multiplier) # [oC x iC x K x K] -> [K x K x iC x oC] xp = x.permute((3, 2, 1, 0)) N = 3 # training if training: for _ in torch.arange(N): # calculate levels and sort [2**nbit, oc] or [num_levels, oc] levels = torch.matmul(level_codes, basis) levels, sort_id = torch.sort(levels, 0) # calculate threshold # [num_levels-1, oc] thrs = torch.matmul(thrs_multiplier, levels) # calculate level codes per channel # ix:sort_id [num_levels, oc], iy: torch.arange(nbit) # level_codes = [num_levels, nbit] # level_codes_channelwise [num_levels, oc, nbit] for oc_idx in torch.arange(oc): if oc_idx == 0: level_codes_t = level_codes[ torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1) level_codes_channelwise = level_codes_t else: level_codes_t = level_codes[ torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1) level_codes_channelwise = torch.cat((level_codes_channelwise, level_codes_t), 1) # calculate output y and its binary code # y [K, K, iC, oC] # bits_y [K x K x iC, oC, nbit] reshape_x = torch.reshape(xp, [-1, oc]) y = torch.zeros_like(xp) + levels[0] # output zero_y = torch.zeros_like(xp) bits_y = torch.full([reshape_x.shape[0], oc, nbit], -1., device=device) zero_bits_y = torch.zeros_like(bits_y) # [K x K x iC x oC] [1, oC] for i in torch.arange(num_levels - 1): g = torch.ge(xp, thrs[i]) # [K, K, iC, oC] + [1, oC], [K, K, iC, oC] => [K, K, iC, oC] y = torch.where(g, zero_y + levels[i + 1], y) # [K x K x iC, oC, nbit] bits_y = torch.where(g.view(-1, oc, 1), zero_bits_y + level_codes_channelwise[i + 1], bits_y) # calculate BTxB # [oC, nbit, K x K x iC] x [oC, K x K x iC, nbit] => [oC, nbit, nbit] BTxB = torch.matmul(bits_y.permute(1, 2, 0), bits_y.permute(1, 0, 2)) + delta * torch.eye(nbit, device=device) # calculate inverse of BTxB # [oC, nbit, nbit] if nbit > 2: BTxB_inv = torch.inverse(BTxB) elif nbit == 2: det = torch.det(BTxB) BTxB_inv = torch.stack((BTxB[:, 1, 1], -BTxB[:, 0, 1], -BTxB[:, 1, 0], BTxB[:, 0, 0]), 1).view(OC, nbit, nbit) / det.unsqueeze(-1).unsqueeze(-1) elif nbit == 1: BTxB_inv = 1 / BTxB else: BTxB_inv = None # calculate BTxX # bits_y [K x K x iC, oc, nbit] reshape_x [K x K x iC, oC] # [oC, nbit, K x K x iC] [oC, K x K x iC, 1] => [oC, nbit, 1] BTxX = torch.matmul(bits_y.permute(1, 2, 0), reshape_x.permute(1, 0).unsqueeze(-1)) BTxX = BTxX + (delta * basis.permute(1, 0).unsqueeze(-1)) # + basis # calculate new basis # BTxB_inv: [oC, nbit, nbit] BTxX: [oC, nbit, 1] # [oC, nbit, nbit] x [oC, nbit, 1] => [oC, nbit, 1] => [nbit, oC] new_basis = torch.matmul(BTxB_inv, BTxX).squeeze(-1).permute(1, 0) # print(BTxB_inv.shape,BTxX.shape) # print(new_basis.shape) # create moving averages op basis -= (1 - MOVING_AVERAGES_FACTOR) * (basis - new_basis) # print("\nbasis:\n", basis) # calculate levels and sort [2**nbit, oc] or [num_levels, oc] levels = torch.matmul(level_codes, basis) levels, sort_id = torch.sort(levels, 0) # calculate threshold # [num_levels-1, oc] thrs = torch.matmul(thrs_multiplier, levels) # calculate level codes per channel # ix:sort_id [num_levels, oc], iy: torch.arange(nbit) # level_codes = [num_levels, nbit] # level_codes_channelwise [num_levels, oc, nbit] for oc_idx in torch.arange(oc): if oc_idx == 0: level_codes_t = level_codes[ torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1) level_codes_channelwise = level_codes_t else: level_codes_t = level_codes[ torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1) level_codes_channelwise = torch.cat((level_codes_channelwise, level_codes_t), 1) # calculate output y and its binary code # y [K, K, iC, oC] # bits_y [K x K x iC, oC, nbit] reshape_x = torch.reshape(xp, [-1, oc]) y = torch.zeros_like(xp) + levels[0] # output zero_y = torch.zeros_like(xp) bits_y = torch.full([reshape_x.shape[0], oc, nbit], -1., device=device) zero_bits_y = torch.zeros_like(bits_y) # [K x K x iC x oC] [1, oC] for i in torch.arange(num_levels - 1): g = torch.ge(xp, thrs[i]) # [K, K, iC, oC] + [1, oC], [K, K, iC, oC] => [K, K, iC, oC] y = torch.where(g, zero_y + levels[i + 1], y) # [K x K x iC, oC, nbit] bits_y = torch.where(g.view(-1, oc, 1), zero_bits_y + level_codes_channelwise[i + 1], bits_y) return y.permute(3, 2, 1, 0), levels.permute(1, 0), thrs.permute(1, 0)
def smoothness_norm(T, theta, lambda_smooth=0.5, lambda_var=0.1, print_info=False): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") D, d = T.get_basis().shape B = T.get_basis() if isinstance(B, np.ndarray): B = torch.from_numpy(B).float() B = B.to(device) nC = d + 1 # = Tess size n = 1 # num sampples? # Convert type #B = tf.cast(B, tf.float32) #theta = tf.cast(theta, tf.float32) theta_T = torch.transpose(theta, 0, 1) # for plotting, {"title": item_to_plot/show} covariance_to_plot = {} items_to_plot = {} # Distance between centers centers = torch.linspace(-1., 1., D).to(device) # from 0 to 1 with nC steps # calculate the distance dists = torch_dist_mat(centers) # DxD # # scale the distance # for x>0, e^(-x^2) decays very fast from 1 to 0 cov_avees = torch.exp(-(dists / lambda_smooth)) cov_avees *= (cov_avees * (lambda_var * D)**2) B_T = torch.transpose(B, 0, 1) cov_cpa = torch.matmul(B_T, torch.matmul(cov_avees, B)) precision_theta = torch.inverse(cov_cpa) if print_info: print("Info: ") print(" Distance between centers shape:", dists.shape) print(" B shape:", B.shape) print(" D shape:", D) print(" d shape:", d) print(" cov_avess shape:", cov_avees.shape) print(" cov_cpa shape:", cov_cpa.shape) print(" theta shape:", theta.shape) # plot velocity field plot_velocity_field = False if plot_velocity_field: nb_points = 1000 points = T.uniform_meshgrid([nb_points for i in range(T._ndim)]) vector_field = T.calc_vectorfield( points, theta_T.eval()) # tf.transpose(theta) items_to_plot["CPA Velocity Field for theta with prior"] = vector_field items_to_plot["Theta values for: theta with prior"] = theta_T # Calculate smoothness norm theta = torch.squeeze(theta) theta_T = torch.transpose(theta, 0, 1) smooth_norm = torch.matmul(theta, torch.matmul(precision_theta, theta_T)) smooth_norm = torch.mean(smooth_norm) return smooth_norm
def closed_form_ls(F_t, y_t): # Returns min norm least squares solution w_t = F_t.T @ torch.inverse(F_t @ F_t.T) @ y_t return w_t
def l2_ll(): ss = torch.matmul(self.sigma, self.sigma.t()) if inverse: ss = torch.inverse(ss) return torch.mul( l2, torch.sum(torch.pow(torch.triu(ss, diag), 2.0)))
def compute_gploss(self, zy_in, imgid, batch_id, label_flg=0): tensor_mat = zy_in Sg_Pred = torch.zeros([self.train_batch_size, 1]) Sg_Pred = Sg_Pred.cuda() LSg_Pred = torch.zeros([self.train_batch_size, 1]) LSg_Pred = LSg_Pred.cuda() sq_err = 0 for i in range(tensor_mat.shape[0]): tmp_i = self.dict_unlbl[ imgid[i]] if label_flg == 0 else self.dict_lbl[ imgid[i]] # imag_id in the dictionary tensor = tensor_mat[i, :, :, :] # z tensor tensor_vec = tensor.view(-1, self.z_height * self.z_width) # z tensor to a vector ker_UU = self.kernel_comp( tensor_vec, tensor_vec, self.z_height, self.z_width, self.z_numchnls, self.z_numchnls) # k(z,z), i.e kernel value for z,z nearest_vl = self.ker_unlbl[ tmp_i, :] if label_flg == 0 else self.ker_lbl[ tmp_i, :] #kernel values are used to get neighbors tp32_vec = np.array( sorted(range(len(nearest_vl)), key=lambda k: nearest_vl[k])[-1 * self.num_nearest:]) lt32_vec = np.array( sorted(range(len(nearest_vl)), key=lambda k: nearest_vl[k])[:self.num_nearest]) # Nearest neighbor latent space labeled vectors near_dic_lbl = np.zeros((self.num_nearest, self.z_numchnls, self.z_height, self.z_width)) for j in range(self.num_nearest): near_dic_lbl[j, :] = self.Fz_lbl[tp32_vec[j], :, :, :] near_vec_lbl = np.reshape(near_dic_lbl, (self.num_nearest * self.z_numchnls, self.z_height * self.z_width)) far_dic_lbl = np.zeros((self.num_nearest, self.z_numchnls, self.z_height, self.z_width)) for j in range(self.num_nearest): far_dic_lbl[j, :] = self.Fz_lbl[lt32_vec[j], :, :, :] far_vec_lbl = np.reshape(far_dic_lbl, (self.num_nearest * self.z_numchnls, self.z_height * self.z_width)) # computing kernel matrix of nearest label vectors # and then computing (K_L+sig^2I)^(-1) ker_LL = cosine_similarity(near_vec_lbl, near_vec_lbl) inv_ker = inv(ker_LL + 1.0 * np.eye(self.num_nearest * self.z_numchnls)) farker_LL = cosine_similarity(far_vec_lbl, far_vec_lbl) farinv_ker = inv(farker_LL + 1.0 * np.eye(self.num_nearest * self.z_numchnls)) mn_pre = np.matmul( inv_ker, near_vec_lbl) # used for computing mean prediction mn_pre = mn_pre.astype(np.float32) #converting require variables to cuda tensors near_vec_lbl = torch.from_numpy(near_vec_lbl.astype(np.float32)) far_vec_lbl = torch.from_numpy(far_vec_lbl.astype(np.float32)) inv_ker = torch.from_numpy(inv_ker.astype(np.float32)) farinv_ker = torch.from_numpy(farinv_ker.astype(np.float32)) inv_ker = inv_ker.cuda() farinv_ker = farinv_ker.cuda() near_vec_lbl = near_vec_lbl.cuda() far_vec_lbl = far_vec_lbl.cuda() mn_pre = torch.from_numpy( mn_pre) # used for mean prediction (mu) or z_pseudo mn_pre = mn_pre.cuda() # Identity matrix Eye = torch.eye(self.z_numchnls) Eye = Eye.cuda() # computing sigma or variance between nearest labeled vectors and unlabeled vector ker_UL = self.kernel_comp(tensor_vec, near_vec_lbl, self.z_height, self.z_width, self.z_numchnls, self.z_numchnls * self.num_nearest) sigma_est = ker_UU - torch.matmul( ker_UL, torch.matmul(inv_ker, ker_UL.t())) + Eye # computing variance between farthest labeled vectors and unlabeled vector Farker_UL = self.kernel_comp(tensor_vec, far_vec_lbl, self.z_height, self.z_width, self.z_numchnls, self.z_numchnls * self.num_nearest) far_sigma_est = ker_UU - torch.matmul( Farker_UL, torch.matmul(farinv_ker, Farker_UL.t())) + Eye # computing mean prediction mean_pred = torch.matmul(ker_UL, mn_pre) #mean prediction (mu) or z_pseudo inv_sigma = torch.inverse(sigma_est) sq_err += torch.mean( torch.matmul((tensor_vec - mean_pred).t(), torch.matmul(inv_sigma, (tensor_vec - mean_pred))) ) + 1.0 * self.lambda_var * torch.log( torch.det(sigma_est)) - 0.000001 * self.lambda_var * torch.log( torch.det(far_sigma_est)) Sg_Pred[i, :] = torch.log(torch.det(sigma_est)) LSg_Pred[i, :] = torch.log(torch.det(far_sigma_est)) if not (batch_id % 100): print(LSg_Pred.max().item(), Sg_Pred.max().item(), sq_err.mean().item() / self.train_batch_size, Sg_Pred.mean().item()) gp_loss = ((1.0 * sq_err / self.train_batch_size)) return gp_loss
def compute_projection(self, depth, camera_to_world, world_to_grid): # compute projection by voxels -> image #print 'camera_to_world', camera_to_world #print 'intrinsic', self.intrinsic #print(world_to_grid) world_to_camera = torch.inverse(camera_to_world) grid_to_world = torch.inverse(world_to_grid) voxel_bounds_min, voxel_bounds_max = self.compute_frustum_bounds(world_to_grid, camera_to_world) voxel_bounds_min = np.maximum(voxel_bounds_min, 0).cuda().float() if depth.is_cuda else np.maximum(voxel_bounds_min, 0).cpu().float() voxel_bounds_max = np.minimum(voxel_bounds_max, self.volume_dims).cuda().float() if depth.is_cuda else np.minimum(voxel_bounds_max, self.volume_dims).cpu().float() # coordinates within frustum bounds # TODO python opt for this part instead of lua/torch opt? lin_ind_volume = torch.arange(0, self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2], out=torch.LongTensor()) lin_ind_volume = lin_ind_volume.cuda() if depth.is_cuda else lin_ind_volume.cpu() coords = camera_to_world.new(4, lin_ind_volume.size(0)) coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1]) tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long() coords[1] = tmp / self.volume_dims[0] coords[0] = torch.remainder(tmp, self.volume_dims[0]) coords[3].fill_(1) mask_frustum_bounds = torch.ge(coords[0], voxel_bounds_min[0]) * torch.ge(coords[1], voxel_bounds_min[1]) * torch.ge(coords[2], voxel_bounds_min[2]) mask_frustum_bounds = mask_frustum_bounds * torch.lt(coords[0], voxel_bounds_max[0]) * torch.lt(coords[1], voxel_bounds_max[1]) * torch.lt(coords[2], voxel_bounds_max[2]) if not mask_frustum_bounds.any(): print('error: nothing in frustum bounds') return None lin_ind_volume = lin_ind_volume[mask_frustum_bounds] coords = coords.resize_(4, lin_ind_volume.size(0)) coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1]) tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long() coords[1] = tmp / self.volume_dims[0] coords[0] = torch.remainder(tmp, self.volume_dims[0]) coords[3].fill_(1) # transform to current frame p = torch.mm(world_to_camera, torch.mm(grid_to_world, coords)) # project into image p[0] = (p[0] * self.intrinsic[0][0]) / p[2] + self.intrinsic[0][2] p[1] = (p[1] * self.intrinsic[1][1]) / p[2] + self.intrinsic[1][2] pi = torch.round(p).long() valid_ind_mask = torch.ge(pi[0], 0) * torch.ge(pi[1], 0) * torch.lt(pi[0], self.image_dims[0]) * torch.lt(pi[1], self.image_dims[1]) if not valid_ind_mask.any(): print('error: no valid image indices') return None valid_image_ind_x = pi[0][valid_ind_mask] valid_image_ind_y = pi[1][valid_ind_mask] valid_image_ind_lin = valid_image_ind_y * self.image_dims[0] + valid_image_ind_x depth_vals = torch.index_select(depth.view(-1), 0, valid_image_ind_lin) depth_mask = depth_vals.ge(self.depth_min) * depth_vals.le(self.depth_max) * torch.abs(depth_vals - p[2][valid_ind_mask]).le(self.voxel_size) if not depth_mask.any(): print('error: no valid depths') return None lin_ind_update = lin_ind_volume[valid_ind_mask] lin_ind_update = lin_ind_update[depth_mask] lin_indices_3d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size) lin_indices_2d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size) lin_indices_3d[0] = lin_ind_update.shape[0] lin_indices_2d[0] = lin_ind_update.shape[0] lin_indices_3d[1:1+lin_indices_3d[0]] = lin_ind_update lin_indices_2d[1:1+lin_indices_2d[0]] = torch.index_select(valid_image_ind_lin, 0, torch.nonzero(depth_mask)[:,0]) num_ind = lin_indices_3d[0] #print '[proj] #ind = ', lin_indices_3d[0] #print '2d', torch.min(lin_indices_2d[1:1+num_ind]), torch.max(lin_indices_2d[1:1+num_ind]) #print '3d', torch.min(lin_indices_3d[1:1+num_ind]), torch.max(lin_indices_3d[1:1+num_ind]) return lin_indices_3d, lin_indices_2d
def gevd(a, b=None): """This method computes the eigenvectors and the eigenvalues of complex Hermitian matrices. The method finds a solution to the problem AV = BVD where V are the eigenvectors and D are the eigenvalues. The eigenvectors returned by the method (vs) are stored in a tensor with the following format (*,C,C,2). The eigenvalues returned by the method (ds) are stored in a tensor with the following format (*,C,C,2). Arguments --------- a : tensor A first input matrix. It is equivalent to the matrix A in the equation in the description above. The tensor must have the following format: (*,2,C+P). b : tensor A second input matrix. It is equivalent tot the matrix B in the equation in the description above. The tensor must have the following format: (*,2,C+P). This argument is optional and its default value is None. If b == None, then b is replaced by the identity matrix in the computations. Example ------- Suppose we would like to compute eigenvalues/eigenvectors on the following complex Hermitian matrix: A = [ 52 34 + 37j 16 + j28 ; 34 - 37j 125 41 + j3 ; 16 - 28j 41 - j3 62 ] >>> a = torch.FloatTensor([[52,34,16,125,41,62],[0,37,28,0,3,0]]) >>> vs, ds = gevd(a) This corresponds to: D = [ 20.9513 0 0 ; 0 43.9420 0 ; 0 0 174.1067 ] V = [ 0.085976 - 0.85184j -0.24620 + 0.12244j -0.24868 - 0.35991j ; -0.16006 + 0.20244j 0.37084 + 0.40173j -0.79175 - 0.087312j ; -0.43990 + 0.082884j -0.36724 - 0.70045j -0.41728 + 0 j ] where A = VDV^-1 """ # Dimensions D = a.dim() P = a.shape[D - 1] C = int(round(((1 + 8 * P)**0.5 - 1) / 2)) # Converting the input matrices to block matrices ash = f(a) if b is None: b = torch.zeros(a.shape, dtype=a.dtype, device=a.device) ids = torch.triu_indices(C, C) b[..., 0, ids[0] == ids[1]] = 1.0 bsh = f(b) # Performing the Cholesky decomposition lsh = torch.cholesky(bsh) lsh_inv = torch.inverse(lsh) lsh_inv_T = torch.transpose(lsh_inv, D - 2, D - 1) # Computing the matrix C csh = torch.matmul(lsh_inv, torch.matmul(ash, lsh_inv_T)) # Performing the eigenvalue decomposition es, ysh = torch.symeig(csh, eigenvectors=True) # Collecting the eigenvalues dsh = torch.zeros( a.shape[slice(0, D - 2)] + (2 * C, 2 * C), dtype=a.dtype, device=a.device, ) dsh[..., range(0, 2 * C), range(0, 2 * C)] = es # Collecting the eigenvectors vsh = torch.matmul(lsh_inv_T, ysh) # Converting the block matrices to full complex matrices vs = ginv(vsh) ds = ginv(dsh) return vs, ds
def forward(ctx, input): inverse = torch.inverse(input) ctx.save_for_backward(inverse) return inverse
def inv(x): """Inverse Hermitian Matrix. This method finds the inverse of a complex Hermitian matrix represented by its upper triangular part. The result will have the following format: (*, C, C, 2). Arguments --------- x : tensor An input matrix to work with. The tensor must have the following format: (*, 2, C+P) Example ------- >>> import torch >>> >>> from speechbrain.dataio.dataio import read_audio >>> from speechbrain.processing.features import STFT >>> from speechbrain.processing.multi_mic import Covariance >>> from speechbrain.processing.decomposition import inv >>> >>> xs_speech = read_audio( ... 'samples/audio_samples/multi_mic/speech_-0.82918_0.55279_-0.082918.flac' ... ) >>> xs_noise = read_audio('samples/audio_samples/multi_mic/noise_0.70225_-0.70225_0.11704.flac') >>> xs = xs_speech + 0.05 * xs_noise >>> xs = xs.unsqueeze(0).float() >>> >>> stft = STFT(sample_rate=16000) >>> cov = Covariance() >>> >>> Xs = stft(xs) >>> XXs = cov(Xs) >>> XXs_inv = inv(XXs) """ # Dimensions d = x.dim() p = x.shape[-1] n_channels = int(round(((1 + 8 * p)**0.5 - 1) / 2)) # Output matrix ash = f(pos_def(x)) ash_inv = torch.inverse(ash) as_inv = finv(ash_inv) indices = torch.triu_indices(n_channels, n_channels) x_inv = torch.zeros( x.shape[slice(0, d - 2)] + (n_channels, n_channels, 2), dtype=x.dtype, device=x.device, ) x_inv[..., indices[1], indices[0], 0] = as_inv[..., 0, :] x_inv[..., indices[1], indices[0], 1] = -1 * as_inv[..., 1, :] x_inv[..., indices[0], indices[1], 0] = as_inv[..., 0, :] x_inv[..., indices[0], indices[1], 1] = as_inv[..., 1, :] return x_inv