def fov(self, value):
self._fov = value
fov_factor = 1.0 / torch.tan(transform.radians(0.5 * self._fov))
o = torch.ones([1], dtype=torch.float32)
diag = torch.cat([fov_factor, fov_factor, o], 0)
self._cam_to_ndc = torch.diag(diag)
self.ndc_to_cam = torch.inverse(self._cam_to_ndc)
def _batch_mahalanobis(L, x):
r"""
Computes the squared Mahalanobis distance :math:`\mathbf{x}^\top\mathbf{M}^{-1}\mathbf{x}`
for a factored :math:`\mathbf{M} = \mathbf{L}\mathbf{L}^\top`.
Accepts batches for both L and x.
"""
# TODO: use `torch.potrs` or similar once a backwards pass is implemented.
flat_L = L.unsqueeze(0).reshape((-1,) + L.shape[-2:])
L_inv = torch.stack([torch.inverse(Li.t()) for Li in flat_L]).view(L.shape)
return (x.unsqueeze(-1) * L_inv).sum(-2).pow(2.0).sum(-1)
def high_dimension_gaussain_energy(x):
u = High_mu
Sigma = High_Sigma
Sigma = torch.inverse(Sigma)
if isinstance(x, Variable):
u = Variable(u, requires_grad=True)
Sigma = Variable(Sigma, requires_grad=True)
diff = x - u
temp = 0.5 * torch.matmul(torch.matmul(diff, Sigma), diff.t())
return temp
def anomalyScore(args, model, dataset, mean, cov, channel_idx=0, score_predictor=None):
predictions = []
rearranged = []
errors = []
hiddens = []
predicted_scores = []
with torch.no_grad():
# Turn on evaluation mode which disables dropout.
model.eval()
pasthidden = model.init_hidden(1)
for t in range(len(dataset)):
out, hidden = model.forward(dataset[t].unsqueeze(0), pasthidden)
predictions.append([])
rearranged.append([])
errors.append([])
hiddens.append(model.extract_hidden(hidden))
if score_predictor is not None:
predicted_scores.append(score_predictor.predict(model.extract_hidden(hidden).numpy()))
predictions[t].append(out.data.cpu()[0][0][channel_idx])
pasthidden = model.repackage_hidden(hidden)
for prediction_step in range(1, args.prediction_window_size):
out, hidden = model.forward(out, hidden)
predictions[t].append(out.data.cpu()[0][0][channel_idx])
if t >= args.prediction_window_size:
for step in range(args.prediction_window_size):
rearranged[t].append(
predictions[step + t - args.prediction_window_size][args.prediction_window_size - 1 - step])
rearranged[t] =torch.FloatTensor(rearranged[t]).to(args.device).unsqueeze(0)
errors[t] = rearranged[t] - dataset[t][0][channel_idx]
else:
rearranged[t] = torch.zeros(1,args.prediction_window_size).to(args.device)
errors[t] = torch.zeros(1, args.prediction_window_size).to(args.device)
predicted_scores = np.array(predicted_scores)
scores = []
for error in errors:
mult1 = error-mean.unsqueeze(0) # [ 1 * prediction_window_size ]
mult2 = torch.inverse(cov) # [ prediction_window_size * prediction_window_size ]
mult3 = mult1.t() # [ prediction_window_size * 1 ]
score = torch.mm(mult1,torch.mm(mult2,mult3))
scores.append(score[0][0])
scores = torch.stack(scores)
rearranged = torch.cat(rearranged,dim=0)
errors = torch.cat(errors,dim=0)
return scores, rearranged, errors, hiddens, predicted_scores
def transform(point, center, scale, resolution, invert=False):
_pt = torch.ones(3)
_pt[0] = point[0]
_pt[1] = point[1]
h = 200.0 * scale
t = torch.eye(3)
t[0, 0] = resolution / h
t[1, 1] = resolution / h
t[0, 2] = resolution * (-center[0] / h + 0.5)
t[1, 2] = resolution * (-center[1] / h + 0.5)
if invert:
t = torch.inverse(t)
new_point = (torch.matmul(t, _pt))[0:2]
return new_point.int()
def compute_L_inverse(self,X,Y):
N = X.size()[0] # num of points (along dim 0)
# construct matrix K
Xmat = X.expand(N,N)
Ymat = Y.expand(N,N)
P_dist_squared = torch.pow(Xmat-Xmat.transpose(0,1),2)+torch.pow(Ymat-Ymat.transpose(0,1),2)
P_dist_squared[P_dist_squared==0]=1 # make diagonal 1 to avoid NaN in log computation
K = torch.mul(P_dist_squared,torch.log(P_dist_squared))
if self.reg_factor != 0:
K+=torch.eye(K.size(0),K.size(1))*self.reg_factor
# construct matrix L
O = torch.FloatTensor(N,1).fill_(1)
Z = torch.FloatTensor(3,3).fill_(0)
P = torch.cat((O,X,Y),1)
L = torch.cat((torch.cat((K,P),1),torch.cat((P.transpose(0,1),Z),1)),0)
Li = torch.inverse(L)
if self.use_cuda:
Li = Li.cuda()
return Li
def NormalEnergy(x):
# x = Variable(x, requires_grad=True)
# u = torch.zeros(1, 2)
# u[0, :] = torch.FloatTensor([2, 2])
# Sigma = torch.FloatTensor([[1.0, 0.8], [0.8, 1.0]])
u = Gaussian_u
Sigma = Gaussian_Sigma
Sigma = torch.inverse(Sigma)
# Sigma = Sigma.t()
if isinstance(x, Variable):
u = Variable(u, requires_grad=True)
Sigma = Variable(Sigma, requires_grad=True)
diff = x - u
temp = 0.5 * torch.matmul(torch.matmul(diff, Sigma), diff.t())
return temp
def __init__(self, values, env_to_world = torch.eye(4, 4)):
# Convert to constant texture if necessary
if isinstance(values, torch.Tensor):
values = pyredner.Texture(values)
assert(values.texels.is_contiguous())
assert(values.texels.dtype == torch.float32)
if pyredner.get_use_gpu():
assert(values.texels.is_cuda)
else:
assert(not values.texels.is_cuda)
assert(env_to_world.dtype == torch.float32)
# Build sampling table
luminance = 0.212671 * values.texels[:, :, 0] + \
0.715160 * values.texels[:, :, 1] + \
0.072169 * values.texels[:, :, 2]
# For each y, compute CDF over x
sample_cdf_xs_ = torch.cumsum(luminance, dim = 1)
y_weight = torch.sin(\
math.pi * (torch.arange(luminance.shape[0],
dtype = torch.float32, device = luminance.device) + 0.5) \
/ float(luminance.shape[0]))
# Compute CDF for x
sample_cdf_ys_ = torch.cumsum(sample_cdf_xs_[:, -1] * y_weight, dim = 0)
pdf_norm = (luminance.shape[0] * luminance.shape[1]) / \
(sample_cdf_ys_[-1].item() * (2 * math.pi * math.pi))
# Normalize to [0, 1)
sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \
torch.max(sample_cdf_xs_[:, (luminance.shape[1] - 1):luminance.shape[1]],
1e-8 * torch.ones(sample_cdf_xs_.shape[0], 1, device = sample_cdf_ys_.device))
sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \
torch.max(sample_cdf_ys_[-1], torch.tensor([1e-8], device = sample_cdf_ys_.device))
self.values = values
self.env_to_world = env_to_world
self.world_to_env = torch.inverse(env_to_world).contiguous()
self.sample_cdf_ys = sample_cdf_ys.contiguous()
self.sample_cdf_xs = sample_cdf_xs.contiguous()
self.pdf_norm = pdf_norm
def __init__(self,
position,
look_at,
up,
fov,
clip_near,
resolution,
cam_to_ndc = None,
fisheye = False):
assert(position.dtype == torch.float32)
assert(len(position.shape) == 1 and position.shape[0] == 3)
assert(look_at.dtype == torch.float32)
assert(len(look_at.shape) == 1 and look_at.shape[0] == 3)
assert(up.dtype == torch.float32)
assert(len(up.shape) == 1 and up.shape[0] == 3)
if fov is not None:
assert(fov.dtype == torch.float32)
assert(len(fov.shape) == 1 and fov.shape[0] == 1)
assert(isinstance(clip_near, float))
self.position = position
self.look_at = look_at
self.up = up
self._fov = fov
# self.cam_to_world = transform.gen_look_at_matrix(position, look_at, up)
# self.world_to_cam = torch.inverse(self.cam_to_world).contiguous()
if cam_to_ndc is None:
fov_factor = 1.0 / torch.tan(transform.radians(0.5 * fov))
o = torch.ones([1], dtype=torch.float32)
diag = torch.cat([fov_factor, fov_factor, o], 0)
self._cam_to_ndc = torch.diag(diag)
else:
self._cam_to_ndc = cam_to_ndc
self.ndc_to_cam = torch.inverse(self.cam_to_ndc)
self.clip_near = clip_near
self.resolution = resolution
self.fisheye = fisheye
def regress(self,
img_original,
hand_bbox_list,
add_margin=False,
debug=True,
viz_path="tmp.png",
K=None):
"""
args:
img_original: original raw image (BGR order by using cv2.imread)
hand_bbox_list: [
dict(
left_hand = [x0, y0, w, h] or None
right_hand = [x0, y0, w, h] or None
)
...
]
add_margin: whether to do add_margin given the hand bbox
outputs:
To be filled
Note:
Output element can be None. This is to keep the same output size with input bbox
"""
pred_output_list = list()
hand_bbox_list_processed = list()
for hand_bboxes in hand_bbox_list:
if hand_bboxes is None: # Should keep the same size with bbox size
pred_output_list.append(None)
hand_bbox_list_processed.append(None)
continue
pred_output = dict(left_hand=None, right_hand=None)
hand_bboxes_processed = dict(left_hand=None, right_hand=None)
for hand_type in hand_bboxes:
bbox = hand_bboxes[hand_type]
if bbox is None:
continue
else:
img_cropped, norm_img, bbox_scale_ratio, bbox_processed = \
self.__process_hand_bbox(img_original, hand_bboxes[hand_type], hand_type, add_margin)
hand_bboxes_processed[hand_type] = bbox_processed
with torch.no_grad():
# pred_rotmat, pred_betas, pred_camera = self.model_regressor(norm_img.to(self.device))
self.model_regressor.set_input_imgonly(
{'img': norm_img.unsqueeze(0)})
self.model_regressor.test()
pred_res = self.model_regressor.get_pred_result()
##Output
cam = pred_res['cams'][0, :] #scale, tranX, tranY
pred_verts_origin = pred_res['pred_verts'][0]
faces = self.model_regressor.right_hand_faces_local
pred_pose = pred_res['pred_pose_params'].copy()
pred_joints = pred_res['pred_joints_3d'].copy()[0]
if hand_type == 'left_hand':
cam[1] *= -1
pred_verts_origin[:, 0] *= -1
faces = faces[:, ::-1]
pred_pose[:, 1::3] *= -1
pred_pose[:, 2::3] *= -1
pred_joints[:, 0] *= -1
pred_output[hand_type] = dict()
pred_output[hand_type][
'pred_vertices_smpl'] = pred_verts_origin # SMPL-X hand vertex in bbox space
pred_output[hand_type][
'pred_joints_smpl'] = pred_joints
pred_output[hand_type]['faces'] = faces
pred_output[hand_type][
'bbox_scale_ratio'] = bbox_scale_ratio
pred_output[hand_type]['bbox_top_left'] = np.array(
bbox_processed[:2])
pred_output[hand_type]['pred_camera'] = cam
pred_output[hand_type]['img_cropped'] = img_cropped
# Get global camera
global_cams = local_to_global_cam(
bbox_wh_to_xy(
np.array([list(bbox_processed)
]).astype(np.float)), cam[None, :],
max(img_original.shape))
pred_output[hand_type]["global_cams"] = global_cams
# pred hand pose & shape params & hand joints 3d
pred_output[hand_type][
'pred_hand_pose'] = pred_pose # (1, 48): (1, 3) for hand rotation, (1, 45) for finger pose.
# Recover PCA components
if hand_type == "right_hand":
comps = torch.Tensor(self.mano_model.rh_mano_pca.
full_hand_components)
elif hand_type == "left_hand":
comps = torch.Tensor(self.mano_model.lh_mano_pca.
full_hand_components)
pca_comps = torch.einsum('bi,ij->bj', [
torch.Tensor(pred_pose[:, 3:]),
torch.inverse(comps)
])
pred_output[hand_type]['pred_hand_betas'] = pred_res[
'pred_shape_params'] # (1, 10)
pred_output[hand_type][
'pred_pca_pose'] = pca_comps.numpy()
hand_side = hand_type.split("_")[0]
pred_output[hand_type]["hand_side"] = hand_side
pred_output[hand_type][
'mano_trans'] = self.mano_model.get_mano_trans(
mano_pose=pred_pose[:, 3:][0],
rot=pred_pose[:, :3][0],
betas=pred_res["pred_shape_params"][0],
ref_verts=pred_verts_origin,
side=hand_side)
#Convert vertices into bbox & image space
cam_scale = cam[0]
cam_trans = cam[1:]
vert_smplcoord = pred_verts_origin.copy()
joints_smplcoord = pred_joints.copy()
vert_bboxcoord = convert_smpl_to_bbox(
vert_smplcoord,
cam_scale,
cam_trans,
bAppTransFirst=True) # SMPL space -> bbox space
joints_bboxcoord = convert_smpl_to_bbox(
joints_smplcoord,
cam_scale,
cam_trans,
bAppTransFirst=True) # SMPL space -> bbox space
hand_boxScale_o2n = pred_output[hand_type][
'bbox_scale_ratio']
hand_bboxTopLeft = pred_output[hand_type][
'bbox_top_left']
vert_imgcoord = convert_bbox_to_oriIm(
vert_bboxcoord, hand_boxScale_o2n,
hand_bboxTopLeft, img_original.shape[1],
img_original.shape[0])
pred_output[hand_type][
'pred_vertices_img'] = vert_imgcoord
joints_imgcoord = convert_bbox_to_oriIm(
joints_bboxcoord, hand_boxScale_o2n,
hand_bboxTopLeft, img_original.shape[1],
img_original.shape[0])
pred_output[hand_type][
'pred_joints_img'] = joints_imgcoord
if K is not None:
K = npt.numpify(K)
K_viz = K.copy()
# r_hand, _ = cv2.Rodrigues(r_vec)
K_viz[:2] = K_viz[:2] / max(img_original.shape)
ortho_scale_pixels = global_cams[0, 0] / 2 * max(
img_original.shape
) # scale of verts_pixel / pred_verts_origin
ortho_trans_pixels = (
global_cams[0, 1:] +
1 / global_cams[0, 0]) * ortho_scale_pixels
t_vec = camconvs.weakcam2persptrans(
np.concatenate([
np.array([ortho_scale_pixels]),
ortho_trans_pixels
], 0) / max(img_original.shape), K_viz)[:,
None]
r_hand = np.eye(3)
# Sanity check
nptype = np.float32
proj2d, camverts = project.proj2d(
pred_verts_origin.astype(nptype),
K.astype(nptype),
rot=r_hand.astype(nptype),
trans=t_vec[:, 0].astype(nptype))
pred_output[hand_type]['camverts'] = camverts
pred_output[hand_type][
'perspective_trans'] = t_vec.transpose()
pred_output[hand_type]['perspective_rot'] = r_hand
pred_output_list.append(pred_output)
hand_bbox_list_processed.append(hand_bboxes_processed)
assert len(hand_bbox_list_processed) == len(hand_bbox_list)
return pred_output_list
def unmoldDetections(config,
camera,
detections,
detection_masks,
detection_masks_var,
depth_np,
unmold_masks=True,
debug=False):
"""Reformats the detections of one image from the format of the neural
network output to a format suitable for use in the rest of the
application.
detections: [N, (y1, x1, y2, x2, class_id, score)]
mrcnn_mask: [N, height, width, num_classes]
image_shape: [height, width, depth] Original size of the image before resizing
window: [y1, x1, y2, x2] Box in the image where the real image is
excluding the padding.
Returns:
boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels
class_ids: [N] Integer class IDs for each bounding box
scores: [N] Float probability scores of the class_id
masks: [height, width, num_instances] Instance masks
"""
if config.GLOBAL_MASK:
masks = detection_masks[
torch.arange(len(detection_masks)).cuda().long(), 0, :, :]
else:
masks = detection_masks[
torch.arange(len(detection_masks)).cuda().long(),
detections[:, 4].long(), :, :]
pass
final_masks = []
for detectionIndex in range(len(detections)):
box = detections[detectionIndex][:4].long()
if (box[2] - box[0]) * (box[3] - box[1]) <= 0:
continue
mask = masks[detectionIndex]
mask = mask.unsqueeze(0).unsqueeze(0)
mask = F.upsample(mask,
size=(box[2] - box[0], box[3] - box[1]),
mode='bilinear')
mask = mask.squeeze(0).squeeze(0)
final_mask = torch.zeros(config.IMAGE_MAX_DIM,
config.IMAGE_MAX_DIM).cuda()
final_mask[box[0]:box[2], box[1]:box[3]] = mask
final_masks.append(final_mask)
continue
final_masks = torch.stack(final_masks, dim=0)
if config.NUM_PARAMETER_CHANNELS > 0:
## We could potentially predict depth and/or normals for each instance (not being used)
parameters_array = detection_masks[
torch.arange(len(detection_masks)).cuda().long(),
-config.NUM_PARAMETER_CHANNELS:, :, :]
final_parameters_array = []
for detectionIndex in range(len(detections)):
box = detections[detectionIndex][:4].long()
if (box[2] - box[0]) * (box[3] - box[1]) <= 0:
continue
parameters = F.upsample(
parameters_array[detectionIndex].unsqueeze(0),
size=(box[2] - box[0], box[3] - box[1]),
mode='bilinear').squeeze(0)
final_parameters = torch.zeros(config.NUM_PARAMETER_CHANNELS,
config.IMAGE_MAX_DIM,
config.IMAGE_MAX_DIM).cuda()
final_parameters[:, box[0]:box[2], box[1]:box[3]] = parameters
final_parameters_array.append(final_parameters)
continue
final_parameters = torch.stack(final_parameters_array, dim=0)
final_masks = torch.cat([final_masks.unsqueeze(1), final_parameters],
dim=1)
pass
masks = final_masks
if 'normal' in config.ANCHOR_TYPE:
## Compute offset based normal prediction and depthmap prediction
ranges = config.getRanges(camera).transpose(1, 2).transpose(0, 1)
zeros = torch.zeros(3,
(config.IMAGE_MAX_DIM - config.IMAGE_MIN_DIM) // 2,
config.IMAGE_MAX_DIM).cuda()
ranges = torch.cat([zeros, ranges, zeros], dim=1)
if config.NUM_PARAMETER_CHANNELS == 4:
## If we predict depthmap and normal map for each instance, we compute normals again (not used)
masks_cropped = masks[:, 0:1, 80:560]
mask_sum = masks_cropped.sum(-1).sum(-1)
plane_normals = (masks[:, 2:5, 80:560] *
masks_cropped).sum(-1).sum(-1) / mask_sum
plane_normals = plane_normals / torch.clamp(
torch.norm(plane_normals, dim=-1, keepdim=True), min=1e-4)
XYZ_np_cropped = (ranges * masks[:, 1:2])[:, :, 80:560]
offsets = ((plane_normals.view(-1, 3, 1, 1) * XYZ_np_cropped).sum(
1, keepdim=True) * masks_cropped).sum(-1).sum(-1) / mask_sum
plane_parameters = plane_normals * offsets.view((-1, 1))
masks = masks[:, 0]
else:
if config.NUM_PARAMETER_CHANNELS > 0:
## If we predict depthmap independently for each instance, we use the individual depthmap instead of the global depth map (not used)
if config.OCCLUSION:
XYZ_np = ranges * depth_np
XYZ_np_cropped = XYZ_np[:, 80:560]
masks_cropped = masks[:, 1, 80:560]
masks = masks[:, 0]
else:
XYZ_np_cropped = (ranges * masks[:, 1:2])[:, :, 80:560]
masks = masks[:, 0]
masks_cropped = masks[:, 80:560]
pass
else:
## We use the global depthmap prediction to compute plane offsets
# print('ranges', type(ranges),len(ranges),ranges[1].size())
# print('depth', type(depth_np),len(depth_np),depth_np[0].size())
# print('we come here')
XYZ_np = ranges * depth_np[0]
XYZ_np_cropped = XYZ_np[:, 80:560]
masks_cropped = masks[:, 80:560]
pass
if config.FITTING_TYPE % 2 == 1:
## We fit all plane parameters using depthmap prediction (not used)
A = masks_cropped.unsqueeze(1) * XYZ_np_cropped
b = masks_cropped
Ab = (A * b.unsqueeze(1)).sum(-1).sum(-1)
AA = (A.unsqueeze(2) * A.unsqueeze(1)).sum(-1).sum(-1)
plane_parameters = torch.stack([
torch.matmul(torch.inverse(AA[planeIndex]), Ab[planeIndex])
for planeIndex in range(len(AA))
],
dim=0)
plane_offsets = torch.norm(plane_parameters,
dim=-1,
keepdim=True)
plane_parameters = plane_parameters / torch.clamp(
torch.pow(plane_offsets, 2), 1e-4)
else:
## We compute only plane offset using depthmap prediction
# print('and here')
plane_parameters = detections[:, 6:9]
plane_parameter_uncertainitiy = detections[:, 9:
12] #uncertainity from mrcnn_parameter uncertainity
plane_normals = plane_parameters / torch.clamp(
torch.norm(plane_parameters, dim=-1, keepdim=True), 1e-4)
plane_param_uncert_normal_temp = plane_parameter_uncertainitiy / torch.clamp(
torch.norm(plane_parameters, dim=-1, keepdim=True), 1e-4)
offsets = (
(plane_normals.view(-1, 3, 1, 1) * XYZ_np_cropped).sum(1) *
masks_cropped).sum(-1).sum(-1) / torch.clamp(
masks_cropped.sum(-1).sum(-1), min=1e-4)
plane_parameters = plane_normals * offsets.view((-1, 1))
plane_parameter_uncertainity = plane_param_uncert_normal_temp * offsets.view(
(-1, 1))
pass
pass
detections = torch.cat([
detections[:, :6], plane_parameters, plane_parameter_uncertainity
],
dim=-1)
# print(detections)
# uncertainities are quite large after this step
# revisit
pass
return detections, masks
def recursiveNystrom(K,
s,
correction=True,
minEig=1e-16,
expand_eigs=True,
eps=1e-16,
accelerated=False):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
n = K.size()[0]
if accelerated:
sLevel = np.ceil(np.sqrt(n * s + s ^ 3) / (4 * n))
else:
sLevel = s
# start of the algorithm
oversamp = np.math.log(sLevel)
k = np.ceil(sLevel / (4 * oversamp)).astype(int)
nLevels = np.int(np.ceil(np.math.log2(n / sLevel)))
# random permutation for successful uniform samples
perm = torch.randperm(n).to(device)
# set up sizes for recursive levels
lSize = np.zeros(nLevels + 1)
lSize[0] = n
for i in range(1, nLevels + 1):
lSize[i] = np.ceil(lSize[i - 1] / 2)
pass
lSize = lSize.astype(int)
# rInd: indices of points selected at previous level of recursion
# at the base level it is just a uniform sample of ~sLevel points
samp = list(range(0, lSize[-1]))
rInd = perm[samp]
weights = torch.ones([len(rInd), 1], dtype=torch.float64).to(device)
# diagonal of the whole matrix is np.diag(K)
# main recursion, unrolled for efficiency
for l in range(nLevels - 1, -1, -1):
# indices of current uniform samples
rIndCurr = perm[0:lSize[l]]
# sampled kernel
KS = K[np.ix_(rIndCurr.detach().cpu().numpy(),
rInd.detach().cpu().numpy())]
KS = KS.to(device)
SKS = KS[samp]
SKS = SKS.to(device)
# print("checking for loops:", SKS[0,0], rIndCurr[0], rInd[0])
SKSn = SKS.size()[0]
################### START MIN EIG CORRECTION ###############################################
if correction == True:
if expand_eigs == False:
# compute local minEig
minEig = compute_minEig(SKS, eps=eps)
# correct using precomputed minEig
cols = list(range(SKSn))
KS[samp, cols] = KS[samp, cols] - minEig
SKS[cols, cols] = SKS[cols, cols] - minEig
########################## END MIN EIG CORRECTION ##########################################
# print("is SkS PSD:", is_pos_def(SKS))
# optimal lambda for taking O(klogk) samples
if k >= SKSn:
# for the rare chance we take less than k samples in a round
lambda_ = 10e-6
# don't set to zero for stability issues
else:
lambda_ = ( torch.sum(torch.diag(SKS) * torch.pow(weights, 2)) - \
torch.sum(torch.abs(torch.real( get_top_k_eigenvalues(SKS*torch.pow(weights, 2), k) ))) ) / k
# for the case when lambda may be set to zero
if lambda_ < 10e-6:
lambda_ = 10e-6
# print(np.diag(SKS))
# print("sum first:", np.sum(np.diag(SKS)*weights**2), \
# "sum second:", np.sum(np.abs(np.real( eigs(SKS*(weights**2), k)[0] ))), "lambda:", lambda_)
pass
# compute and sample by lambda ridge leverage scores
if l != 0:
# on intermediate levels, we independently sample each column
# by its leverage score. the sample size is sLevel in expectation
R = torch.inverse(SKS +
torch.diag(lambda_ * torch.pow(weights, -2))).to(
dtype=KS.dtype).to(device)
# max(0,.) helps avoid numerical issues, unnecessary in theory
levs = torch.minimum( torch.ones_like(rIndCurr), oversamp*(1/lambda_)*\
torch.maximum(torch.zeros_like(rIndCurr), torch.diag(K)[rIndCurr]- \
torch.sum(torch.matmul(KS,R)*KS, dim=1)) )
levs = rescale_levs(levs)
samp = np.where(
np.random.random(lSize[l]) < levs.detach().cpu().numpy())[0]
# with very low probability, we could accidentally sample no
# columns. In this case, just take a fixed size uniform sample.
if len(samp) == 0:
levs[:] = sLevel / lSize[l]
samp = random.sample(range(lSize[l]), sLevel)
pass
weights = torch.sqrt(1 / levs[samp])
else:
# on the top level, we sample exactly s landmark points without replacement
R = torch.inverse(SKS +
torch.diag(lambda_ * torch.pow(weights, -2))).to(
dtype=KS.dtype).to(device)
levs = torch.minimum(torch.ones_like(rIndCurr), (1/lambda_)*\
torch.maximum(torch.zeros_like(rIndCurr), torch.diag(K)[rIndCurr]- \
torch.sum(torch.matmul(KS,R)*KS, dim=1)) )
levs = rescale_levs(levs)
samp = np.random.choice(n, size=s, replace=False, \
p=levs.detach().cpu().numpy()/sum(levs.detach().cpu().numpy()))
pass
rInd = perm[samp]
C = K[np.ix_(list(range(len(K))), rInd.detach().cpu().numpy())]
SKS = C[rInd]
# correct SKS for min Eig
SKSn = SKS.size()[0]
indices = range(SKSn)
SKS[indices, indices] = SKS[indices, indices] - minEig + eps
# print("is SKS PSD:", is_pos_def(SKS))
W = torch.inverse(SKS + (10e-6) * torch.eye(s).to(device))
error = torch.norm(K - torch.matmul(torch.matmul(
C, W), torch.transpose(C, 0, 1))) / torch.norm(K)
return C, W, error, minEig
def render_scene(vertex_positions: InputTensor,
view_projection_matrix: InputTensor = None,
image_size: Tuple[int, int] = (256, 256),
normals: InputTensor = None,
tex_coords: InputTensor = None,
material_ids: InputTensor = None,
diffuse_coefficients: InputTensor = None,
diffuse_textures: InputTensor = None,
diffuse_texture_indices: InputTensor = None,
specular_coefficient: InputTensor = None,
ambient_coefficients: InputTensor = None,
cull_back_facing=True,
light_position: InputTensor = None,
light_color: InputTensor = (1.0, 1.0, 1.0),
ambient_light_color: InputTensor = (0.2, 0.2, 0.2),
clear_color: InputTensor = (0, 0, 0, 1),
output_type=t.uint8,
vertex_shader=None,
geometry_shader=None,
fragment_shader=None,
debug_io_buffer=None,
return_rgb=True,
cuda_device=None):
"""Renders the given scene.
Args:
vertex_positions: The triangle geometry, specified through the triangle
vertex positions, float32[num_triangles, 3, 3]
view_projection_matrix: The view projection matrix, float32[4, 4]
image_size: Desired output image size, (height, width),
normals: Per-vertex shading normals, float32[num_triangles, 3, 3]. If set to
None, normals will be computed from the vertex positions.
tex_coords: Texture coordinate, float32[num_triangles, 3, 2]. If set to
None, all texture coordinates will be 0.
material_ids: Per-triangle material indices used to index in the various
coefficient tensors below, int32[num_triangles]. If set to None, all
triangles will have the same default material.
diffuse_coefficients: The diffuse coefficients, one per material,
float32[num_materials, 3]. Cannot be None if material_ids is not None.
Must be None if material_ids is None.
diffuse_textures: uint8[num_textures, height, width, 3]. Can be None if
there are no textures used in the mesh.
diffuse_texture_indices: Diffuse texture indices, one per material,
int32[num_materials]. If set to None, the texture indices for all
materials will be -1.
specular_coefficient: Specular coefficients, one per material,
float32[num_materials, 4]. The first 3 channels are the R, G, and B
specular coefficients, the last channel is the specular power. If set to
None, R, G, and B will be 0 for all materials and power will be 2048.
ambient_coefficients: float32[num_materials, 3]. The ambient coefficients.
If None, all ambient coefficient will be 0.05.
cull_back_facing: whether to cull backfacing triangles.
light_position: float32[3], the light position. If set to None, the light
will be placed at the camera origin.
light_color: The light diffuse RGB color, float32[3]
ambient_light_color: The light ambient RGB color, float32[3]
clear_color: The RGB color to use when clearing the image, float32[3]
output_type: The desired output type. Either tf.uint8 or tf.float32.
vertex_shader: The vertex shader to use. If empty, uses a default shader.
geometry_shader: The geometry shader. If empty, uses a default shader.
fragment_shader: The fragment shader. If empty, uses a default shader.
debug_io_buffer: Aids debugging of shaders. Shaders can communicate with
host programs through OpenGL input/output buffers. Any tensor passed in
this argument will be forwarded to the shaders as buffer with name
"debug_io".
return_rgb: If true, returns a 3 channel image, otherwise returns a 4
channel image.
cuda_device: The index of the GPU to use, given as CUDA device
Returns:
The rendered image, dt[height, width, c] where dt is either float32 or uint8
depending on the value of output_type and c is either 3 or 4, depending on
return_rgb. If the debug_io_buffer argument was not None, returns a
tuple containing the rendered image, and the shader output from the
"debug_io" buffer. The second element of the tuple has the same shape
and type as debug_io_buffer.
"""
height, width = image_size
vertex_positions = util.to_tensor(vertex_positions, t.float32, "cpu")
assert (len(vertex_positions.shape) == 3
and vertex_positions.shape[1:] == (3, 3))
num_triangles = vertex_positions.shape[0]
if view_projection_matrix is None:
view_projection_matrix = camera_util.get_default_camera_for_mesh(
vertex_positions)
view_projection_matrix = util.to_tensor(view_projection_matrix, t.float32,
"cpu")
assert view_projection_matrix.shape == (4, 4)
has_normals = True
if normals is None:
normals = t.zeros_like(vertex_positions)
has_normals = False
normals = util.to_tensor(normals, t.float32, "cpu")
assert normals.shape == (num_triangles, 3, 3)
if tex_coords is None:
tex_coords = t.zeros([num_triangles, 3, 2], dtype=t.float32)
tex_coords = util.to_tensor(tex_coords, t.float32, "cpu")
assert tex_coords.shape == (num_triangles, 3, 2)
if material_ids is None:
material_ids = t.zeros([num_triangles], dtype=t.int32)
material_ids = util.to_tensor(material_ids, t.int32, "cpu")
assert material_ids.shape == (num_triangles, )
num_used_materials = material_ids.max().cpu().numpy() + 1 # type: int
def create_coefficient_array(cur_tensor: InputTensor, num_channels,
default_value):
arr = cur_tensor
if arr is None:
arr = (
t.ones([num_used_materials, num_channels], dtype=t.float32) *
t.tensor(default_value))
arr = util.to_tensor(arr, t.float32, "cpu")
assert len(arr.shape) == 2
arr = arr[:num_used_materials]
assert arr.shape == (num_used_materials, num_channels)
return arr
diffuse_coefficients = create_coefficient_array(diffuse_coefficients, 3,
0.8)
ambient_coefficients = create_coefficient_array(ambient_coefficients, 3,
0.05)
specular_coefficient = create_coefficient_array(specular_coefficient, 4,
(0, 0, 0, 2048.0))
if diffuse_texture_indices is None:
diffuse_texture_indices = t.ones([num_used_materials],
dtype=t.int32) * -1
diffuse_texture_indices = util.to_tensor(diffuse_texture_indices, t.int32,
"cpu")
assert len(diffuse_texture_indices.shape) == 1
diffuse_texture_indices = diffuse_texture_indices[:num_used_materials]
assert diffuse_texture_indices.shape == (num_used_materials, )
num_used_textures = diffuse_texture_indices.max().cpu().numpy() + 1
num_used_textures = max(num_used_textures, 1)
if diffuse_textures is None:
diffuse_textures = t.ones([num_used_textures, 1, 1, 3], dtype=t.uint8)
diffuse_textures = util.to_tensor(diffuse_textures, t.uint8, "cpu")
assert len(diffuse_textures.shape) == 4
diffuse_textures = diffuse_textures[:num_used_textures]
assert (diffuse_textures.shape[0] == num_used_textures
and diffuse_textures.shape[3] == 3)
camera_position = t.mv(t.inverse(view_projection_matrix),
t.tensor([0, 0, -1, 1], dtype=t.float32))
camera_position = camera_position[:3] / camera_position[3]
if light_position is None:
light_position = camera_position
light_position = util.to_tensor(light_position, t.float32, "cpu")
assert light_position.shape == (3, )
light_color = util.to_tensor(light_color, t.float32, "cpu")
assert light_color.shape == (3, )
ambient_light_color = util.to_tensor(ambient_light_color, t.float32, "cpu")
assert ambient_light_color.shape == (3, )
ambient_coefficients = t.constant_pad_nd(ambient_coefficients, [0, 1])
diffuse_coefficients = t.cat([
diffuse_coefficients,
diffuse_texture_indices.to(t.float32)[:, np.newaxis]
], -1)
materials = t.cat(
[ambient_coefficients, diffuse_coefficients, specular_coefficient],
dim=-1)
render_args = [
rasterizer.Uniform("view_projection_matrix", view_projection_matrix),
rasterizer.Uniform("light_position", light_position),
rasterizer.Uniform("has_normals", has_normals),
rasterizer.Uniform("has_texcoords", True),
rasterizer.Buffer(0, vertex_positions.reshape([-1])),
rasterizer.Buffer(1, normals.reshape([-1])),
rasterizer.Buffer(2, tex_coords.reshape([-1])),
rasterizer.Buffer(3, material_ids.reshape([-1])),
rasterizer.Buffer(4, materials.reshape([-1])),
rasterizer.Texture("textures", diffuse_textures, bind_as_array=True),
rasterizer.Uniform("light_color", light_color),
rasterizer.Uniform("camera_position", camera_position),
rasterizer.Uniform("ambient_light_color", ambient_light_color),
rasterizer.Uniform("cull_backfacing", cull_back_facing),
]
if debug_io_buffer is not None:
render_args.append(rasterizer.Buffer(5, debug_io_buffer, is_io=True))
if not geometry_shader:
geometry_shader = resources.read_text(shaders,
"triangle_renderer.geom")
if not vertex_shader:
vertex_shader = resources.read_text(shaders, "noop.vert")
if not fragment_shader:
fragment_shader = resources.read_text(shaders,
"point_light_illumination.frag")
result = rasterizer.gl_simple_render(rasterizer.RenderInput(
num_points=num_triangles,
arguments=render_args,
output_resolution=(height, width),
clear_color=clear_color,
output_type=output_type,
vertex_shader=vertex_shader,
geometry_shader=geometry_shader,
fragment_shader=fragment_shader),
cuda_device=cuda_device)
c = 3 if return_rgb else 4
if debug_io_buffer is None:
return result[..., :c]
else:
return result[..., :c], render_args[-1].value
def inverse(self, regul=1e-8):
inv_tensor = torch.inverse(self.data +
regul * torch.eye(self.size(0),
device=self.data.device))
return PMatDense(generator=self.generator,
data=inv_tensor)
def BatchInverse(tensor):
batch_size = tensor.size()[0]
tensor_inverse = []
for i in range(batch_size):
tensor_inverse += [torch.inverse(tensor[i]).unsqueeze(0)]
return torch.cat(tensor_inverse, 0)
def batch_global_rigid_transformation(Rs,
Js,
parent,
rotate_base=False,
betas_extra=None,
device="cuda"):
"""
Computes absolute joint locations given pose.
rotate_base: if True, rotates the global rotation by 90 deg in x axis.
if False, this is the original SMPL coordinate.
Args:
Rs: N x 24 x 3 x 3 rotation vector of K joints
Js: N x 24 x 3, joint locations before posing
parent: 24 holding the parent id for each index
Returns
new_J : `Tensor`: N x 24 x 3 location of absolute joints
A : `Tensor`: N x 24 4 x 4 relative joint transformations for LBS.
"""
# Now Js is N x 24 x 3 x 1
Js = Js.unsqueeze(-1)
N = Rs.shape[0]
if rotate_base:
print('Flipping the SMPL coordinate frame!!!!')
rot_x = torch.Tensor([[1, 0, 0], [0, -1, 0], [0, 0, -1]])
rot_x = torch.reshape(torch.repeat(rot_x, [N, 1]),
[N, 3, 3]) # In tf it was tile
root_rotation = torch.matmul(Rs[:, 0, :, :], rot_x)
else:
root_rotation = Rs[:, 0, :, :]
Js_orig = Js.clone()
scaling_factors = torch.ones(N, parent.shape[0], 3).to(device)
if betas_extra is not None:
scaling_factors = betas_extra.reshape(-1, 35, 3)
# debug_only
# scaling_factors[:, 25:32, 0] = 0.2
# scaling_factors[:, 7, 2] = 2.0
scale_factors_3x3 = torch.diag_embed(scaling_factors, dim1=-2, dim2=-1)
def make_A(R, t):
# Rs is N x 3 x 3, ts is N x 3 x 1
R_homo = torch.nn.functional.pad(R, (0, 0, 0, 1, 0, 0))
t_homo = torch.cat([t, torch.ones([N, 1, 1]).to(device)], 1)
return torch.cat([R_homo, t_homo], 2)
A0 = make_A(root_rotation, Js[:, 0])
results = [A0]
for i in range(1, parent.shape[0]):
j_here = Js[:, i] - Js[:, parent[i]]
s_par_inv = torch.inverse(scale_factors_3x3[:, parent[i]])
rot = Rs[:, i]
s = scale_factors_3x3[:, i]
rot_new = s_par_inv @ rot @ s
A_here = make_A(rot_new, j_here)
res_here = torch.matmul(results[parent[i]], A_here)
results.append(res_here)
# 10 x 24 x 4 x 4
results = torch.stack(results, dim=1)
# scale updates
new_J = results[:, :, :3, 3]
# --- Compute relative A: Skinning is based on
# how much the bone moved (not the final location of the bone)
# but (final_bone - init_bone)
# ---
Js_w0 = torch.cat([Js_orig, torch.zeros([N, 35, 1, 1]).to(device)], 2)
init_bone = torch.matmul(results, Js_w0)
# Append empty 4 x 3:
init_bone = torch.nn.functional.pad(init_bone, (3, 0, 0, 0, 0, 0, 0, 0))
A = results - init_bone
return new_J, A
def get_right_inverse(w):
return torch.mm(w.t(), torch.inverse(w.mm(w.t())))
def warp_affine(
src: torch.Tensor,
M: torch.Tensor,
dsize: Tuple[int, int],
flags: str = "bilinear",
padding_mode: str = "zeros",
align_corners: bool = False,
) -> torch.Tensor:
r"""Applies an affine transformation to a tensor.
The function warp_affine transforms the source tensor using
the specified matrix:
.. math::
\text{dst}(x, y) = \text{src} \left( M_{11} x + M_{12} y + M_{13} ,
M_{21} x + M_{22} y + M_{23} \right )
Args:
src (torch.Tensor): input tensor of shape :math:`(B, C, H, W)`.
M (torch.Tensor): affine transformation of shape :math:`(B, 2, 3)`.
dsize (Tuple[int, int]): size of the output image (height, width).
mode (str): interpolation mode to calculate output values
'bilinear' | 'nearest'. Default: 'bilinear'.
padding_mode (str): padding mode for outside grid values
'zeros' | 'border' | 'reflection'. Default: 'zeros'.
align_corners (bool): mode for grid_generation. Default: False. See
https://pytorch.org/docs/stable/nn.functional.html#torch.nn.functional.interpolate for details
Returns:
torch.Tensor: the warped tensor.
Shape:
- Output: :math:`(B, C, H, W)`
.. note::
See a working example `here <https://kornia.readthedocs.io/en/latest/
tutorials/warp_affine.html>`__.
"""
if not torch.is_tensor(src):
raise TypeError("Input src type is not a torch.Tensor. Got {}".format(
type(src)))
if not torch.is_tensor(M):
raise TypeError("Input M type is not a torch.Tensor. Got {}".format(
type(M)))
if not len(src.shape) == 4:
raise ValueError("Input src must be a BxCxHxW tensor. Got {}".format(
src.shape))
if not (len(M.shape) == 3 or M.shape[-2:] == (2, 3)):
raise ValueError("Input M must be a Bx2x3 tensor. Got {}".format(
M.shape))
B, C, H, W = src.size()
dsize_src = (H, W)
out_size = dsize
# we generate a 3x3 transformation matrix from 2x3 affine
M_3x3: torch.Tensor = convert_affinematrix_to_homography(M)
dst_norm_trans_src_norm: torch.Tensor = normalize_homography(
M_3x3, dsize_src, out_size)
src_norm_trans_dst_norm = torch.inverse(dst_norm_trans_src_norm)
grid = F.affine_grid(
src_norm_trans_dst_norm[:, :2, :],
[B, C, out_size[0], out_size[1]],
align_corners=align_corners,
)
return F.grid_sample(src,
grid,
align_corners=align_corners,
mode=flags,
padding_mode=padding_mode)
def ransac_voting_layer(mask, vertex, round_hyp_num, inlier_thresh=0.999, confidence=0.99, max_iter=20,
min_num=5, max_num=30000):
'''
:param mask: [b,h,w]
:param vertex: [b,h,w,vn,2]
:param round_hyp_num:
:param inlier_thresh:
:return: [b,vn,2]
'''
b, h, w, vn, _ = vertex.shape
batch_win_pts = []
for bi in range(b):
hyp_num = 0
cur_mask = (mask[bi]).byte()
foreground_num = torch.sum(cur_mask)
# if too few points, just skip it
if foreground_num < min_num:
win_pts = torch.zeros([1, vn, 2], dtype=torch.float32, device=mask.device)
batch_win_pts.append(win_pts) # [1,vn,2]
continue
# if too many inliers, we randomly down sample it
if foreground_num > max_num:
selection = torch.zeros(cur_mask.shape, dtype=torch.float32, device=mask.device).uniform_(0, 1)
selected_mask = (selection < (max_num / foreground_num.float()))
cur_mask *= selected_mask
coords = torch.nonzero(cur_mask).float() # [tn,2]
coords = coords[:, [1, 0]]
direct = vertex[bi].masked_select(torch.unsqueeze(torch.unsqueeze(cur_mask, 2), 3)) # [tn,vn,2]
direct = direct.view([coords.shape[0], vn, 2])
tn = coords.shape[0]
idxs = torch.zeros([round_hyp_num, vn, 2], dtype=torch.int32, device=mask.device).random_(0, direct.shape[0])
all_win_ratio = torch.zeros([vn], dtype=torch.float32, device=mask.device)
all_win_pts = torch.zeros([vn, 2], dtype=torch.float32, device=mask.device)
cur_iter = 0
while True:
# generate hypothesis
cur_hyp_pts = ransac_voting.generate_hypothesis(direct, coords, idxs) # [hn,vn,2]
# voting for hypothesis
cur_inlier = torch.zeros([round_hyp_num, vn, tn], dtype=torch.uint8, device=mask.device)
ransac_voting.voting_for_hypothesis(direct, coords, cur_hyp_pts, cur_inlier, inlier_thresh) # [hn,vn,tn]
# find max
cur_inlier_counts = torch.sum(cur_inlier, 2) # [hn,vn]
cur_win_counts, cur_win_idx = torch.max(cur_inlier_counts, 0) # [vn]
cur_win_pts = cur_hyp_pts[cur_win_idx, torch.arange(vn)]
cur_win_ratio = cur_win_counts.float() / tn
# update best point
larger_mask = all_win_ratio < cur_win_ratio
all_win_pts[larger_mask,:] = cur_win_pts[larger_mask,:]
all_win_ratio[larger_mask] = cur_win_ratio[larger_mask]
# check confidence
hyp_num += round_hyp_num
cur_iter += 1
cur_min_ratio = torch.min(all_win_ratio)
if (1 - (1 - cur_min_ratio ** 2) ** hyp_num) > confidence or cur_iter > max_iter:
break
# compute mean intersection again
normal = torch.zeros_like(direct) # [tn,vn,2]
normal[:,:, 0] = direct[:,:, 1]
normal[:,:, 1] = -direct[:,:, 0]
all_inlier = torch.zeros([1, vn, tn], dtype=torch.uint8, device=mask.device)
all_win_pts = torch.unsqueeze(all_win_pts, 0) # [1,vn,2]
ransac_voting.voting_for_hypothesis(direct, coords, all_win_pts, all_inlier, inlier_thresh) # [1,vn,tn]
# coords [tn,2] normal [vn,tn,2]
all_inlier = torch.squeeze(all_inlier.float(), 0) # [vn,tn]
normal = normal.permute(1, 0, 2) # [vn,tn,2]
normal = normal*torch.unsqueeze(all_inlier, 2) # [vn,tn,2] outlier is all zero
b = torch.sum(normal*torch.unsqueeze(coords, 0), 2) # [vn,tn]
ATA = torch.matmul(normal.permute(0, 2, 1), normal) # [vn,2,2]
ATb = torch.sum(normal*torch.unsqueeze(b, 2), 1) # [vn,2]
try:
all_win_pts = torch.matmul(torch.inverse(ATA), torch.unsqueeze(ATb, 2)) # [vn,2,1]
batch_win_pts.append(all_win_pts[None,:,:, 0])
except:
all_win_pts = torch.zeros([1, ATA.size(0), 2]).to(ATA.device)
batch_win_pts.append(all_win_pts)
batch_win_pts = torch.cat(batch_win_pts)
return batch_win_pts
def forward(x, xref, uref, _lambda, verbose=False, acc=False, detach=False):
# x: bs x n x 1
bs = x.shape[0]
W = W_func(x)
M = torch.inverse(W)
f = f_func(x)
B = B_func(x)
DfDx = Jacobian(f, x)
DBDx = torch.zeros(bs, num_dim_x, num_dim_x,
num_dim_control).type(x.type())
for i in range(num_dim_control):
DBDx[:, :, :, i] = Jacobian(B[:, :, i].unsqueeze(-1), x)
_Bbot = Bbot_func(x)
u = u_func(x, x - xref, uref) # u: bs x m x 1 # TODO: x - xref
K = Jacobian(u, x)
A = DfDx + sum([
u[:, i, 0].unsqueeze(-1).unsqueeze(-1) * DBDx[:, :, :, i]
for i in range(num_dim_control)
])
dot_x = f + B.matmul(u)
dot_M = weighted_gradients(M, dot_x, x, detach=detach) # DMDt
dot_W = weighted_gradients(W, dot_x, x, detach=detach) # DWDt
if detach:
Contraction = dot_M + (A + B.matmul(K)).transpose(1, 2).matmul(
M.detach()) + M.detach().matmul(
A + B.matmul(K)) + 2 * _lambda * M.detach()
else:
Contraction = dot_M + (A + B.matmul(K)).transpose(
1, 2).matmul(M) + M.matmul(A + B.matmul(K)) + 2 * _lambda * M
# C1
C1_inner = -weighted_gradients(W, f, x) + DfDx.matmul(W) + W.matmul(
DfDx.transpose(1, 2)) + 2 * _lambda * W
C1_LHS_1 = _Bbot.transpose(1, 2).matmul(C1_inner).matmul(
_Bbot) # this has to be a negative definite matrix
# C2
C2_inners = []
C2s = []
for j in range(num_dim_control):
C2_inner = weighted_gradients(W, B[:, :, j].unsqueeze(-1), x) - (
DBDx[:, :, :, j].matmul(W) +
W.matmul(DBDx[:, :, :, j].transpose(1, 2)))
C2 = _Bbot.transpose(1, 2).matmul(C2_inner).matmul(_Bbot)
C2_inners.append(C2_inner)
C2s.append(C2)
loss = 0
loss += loss_pos_matrix_random_sampling(
-Contraction -
epsilon * torch.eye(Contraction.shape[-1]).unsqueeze(0).type(x.type()))
loss += loss_pos_matrix_random_sampling(
-C1_LHS_1 -
epsilon * torch.eye(C1_LHS_1.shape[-1]).unsqueeze(0).type(x.type()))
loss += loss_pos_matrix_random_sampling(
args.w_ub * torch.eye(W.shape[-1]).unsqueeze(0).type(x.type()) - W)
loss += 1. * sum(
[1. * (C2**2).reshape(bs, -1).sum(dim=1).mean() for C2 in C2s])
if verbose:
print(
torch.symeig(Contraction)[0].min(dim=1)[0].mean(),
torch.symeig(Contraction)[0].max(dim=1)[0].mean(),
torch.symeig(Contraction)[0].mean())
if acc:
return loss, ((torch.symeig(Contraction)[0] >= 0).sum(
dim=1) == 0).cpu().detach().numpy(), ((
torch.symeig(C1_LHS_1)[0] >= 0).sum(
dim=1) == 0).cpu().detach().numpy(), sum([
1. * (C2**2).reshape(bs, -1).sum(dim=1).mean()
for C2 in C2s
]).item()
else:
return loss, None, None, None
def inverse(self, z, log_jacobian):
w = torch.inverse(self.w)
x = F.conv2d(z, w.unsqueeze(-1).unsqueeze(-1))
log_jacobian = log_jacobian + (torch.slogdet(w)[1] / z.size(1))
return x, log_jacobian
def _calc_integrals():
######################## short-range integrals ########################
############# 3-centre 2-electron integral #############
_basisw, _fusew = intor.LibcintWrapper.concatenate(
self._wrapper, fuse_aux_wrapper)
# (nkpts_ij, nao, nao, nxao+nxcao)
j3c_short_f = intor.pbc_coul3c(_basisw,
other2=_fusew,
kpts_ij=kpts_ij,
options=self._lattsum_opt)
j3c_short = j3c_short_f[..., :nxao] - j3c_short_f[
..., nxao:] # (nkpts_ij, nao, nao, nxao)
############# 2-centre 2-electron integrals #############
# (nkpts_unique, nxao+nxcao, nxao+nxcao)
j2c_short_f = intor.pbc_coul2c(fuse_aux_wrapper,
kpts=kpts_reduce,
options=self._lattsum_opt)
# j2c_short: (nkpts_unique, nxao, nxao)
j2c_short = j2c_short_f[..., :nxao, :nxao] + j2c_short_f[..., nxao:, nxao:] \
- j2c_short_f[..., :nxao, nxao:] - j2c_short_f[..., nxao:, :nxao]
######################## long-range integrals ########################
# only use the compensating wrapper as the gcut
gcut = get_gcut(self._lattsum_opt.precision, [aux_comp_wrapper])
# gvgrids: (ngv, ndim), gvweights: (ngv,)
gvgrids, gvweights = self._lattice.get_gvgrids(gcut)
ngv = gvgrids.shape[0]
gvk = gvgrids.unsqueeze(-2) + kpts_reduce # (ngv, nkpts_ij, ndim)
gvk = gvk.view(-1, gvk.shape[-1]) # (ngv * nkpts_ij, ndim)
# get the fourier transform variables
# TODO: iterate over ngv axis
# ft of the compensating basis
comp_ft = intor.eval_gto_ft(aux_comp_wrapper,
gvk) # (nxcao, ngv * nkpts_ij)
comp_ft = comp_ft.view(-1, ngv, nkpts_ij) # (nxcao, ngv, nkpts_ij)
# ft of the auxiliary basis
auxb_ft_c = intor.eval_gto_ft(aux_wrapper,
gvk) # (nxao, ngv * nkpts_ij)
auxb_ft_c = auxb_ft_c.view(-1, ngv,
nkpts_ij) # (nxao, ngv, nkpts_ij)
auxb_ft = auxb_ft_c - comp_ft # (nxao, ngv, nkpts_ij)
# ft of the overlap integral of the basis (nkpts_ij, nao, nao, ngv)
aoao_ft = self._get_pbc_overlap_with_kpts_ij(
gvgrids, kpts_reduce, kpts_j)
# ft of the coulomb kernel
coul_ft = unweighted_coul_ft(gvk) # (ngv * nkpts_ij,)
coul_ft = coul_ft.to(comp_ft.dtype).view(
ngv, nkpts_ij) * gvweights.unsqueeze(-1) # (ngv, nkpts_ij)
# 1: (nkpts_ij, nxao, nxao)
pattern = "gi,xgi,ygi->ixy"
j2c_long = torch.einsum(pattern, coul_ft, comp_ft.conj(), auxb_ft)
# 2: (nkpts_ij, nxao, nxao)
j2c_long += torch.einsum(pattern, coul_ft, auxb_ft.conj(), comp_ft)
# 3: (nkpts_ij, nxao, nxao)
j2c_long += torch.einsum(pattern, coul_ft, comp_ft.conj(), comp_ft)
# calculate the j3c long-range
patternj3 = "gi,xgi,iyzg->iyzx"
# (nkpts_ij, nao, nao, nxao)
j3c_long = torch.einsum(patternj3, coul_ft, comp_ft.conj(),
aoao_ft)
# get the average potential
auxbar_f = self._auxbar(
kpts_reduce, fuse_aux_wrapper) # (nkpts_ij, nxao + nxcao)
auxbar = auxbar_f[:, :nxao] - auxbar_f[:,
nxao:] # (nkpts_ij, nxao)
auxbar = auxbar.reshape(nkpts, nkpts,
auxbar.shape[-1]) # (nkpts, nkpts, nxao)
olp_mat = intor.pbc_overlap(
self._wrapper, kpts=self._kpts,
options=self._lattsum_opt) # (nkpts, nao, nao)
j3c_bar = auxbar[:, :, None, None, :] * olp_mat[
..., None] # (nkpts, nkpts, nao, nao, nxao)
j3c_bar = j3c_bar.reshape(
-1, *j3c_bar.shape[2:]) # (nkpts_ij, nao, nao, nxao)
######################## combining integrals ########################
j2c = j2c_short + j2c_long # (nkpts_ij, nxao, nxao)
j3c = j3c_short + j3c_long - j3c_bar # (nkpts_ij, nao, nao, nxao)
el_mat = torch.einsum("kxy,kaby->kabx", torch.inverse(j2c),
j3c) # (nkpts_ij, nao, nao, nxao)
return j2c, j3c, el_mat
def ray_sampling(Ks,
Ts,
image_size,
masks=None,
mask_threshold=0.5,
images=None):
h = image_size[0]
w = image_size[1]
M = Ks.size(0)
x = torch.linspace(0, h - 1, steps=h, device=Ks.device)
y = torch.linspace(0, w - 1, steps=w, device=Ks.device)
grid_x, grid_y = torch.meshgrid(x, y)
coordinates = torch.stack([grid_y,
grid_x]).unsqueeze(0).repeat(M, 1, 1,
1) #(M,2,H,W)
coordinates = torch.cat([
coordinates,
torch.ones(coordinates.size(0),
1,
coordinates.size(2),
coordinates.size(3),
device=Ks.device)
],
dim=1).permute(0, 2, 3, 1).unsqueeze(-1)
inv_Ks = torch.inverse(Ks)
dirs = torch.matmul(inv_Ks, coordinates) #(M,H,W,3,1)
dirs = dirs / torch.norm(dirs, dim=3, keepdim=True)
dirs = torch.cat([
dirs,
torch.zeros(dirs.size(0),
coordinates.size(1),
coordinates.size(2),
1,
1,
device=Ks.device)
],
dim=3) #(M,H,W,4,1)
dirs = torch.matmul(Ts, dirs) #(M,H,W,4,1)
dirs = dirs[:, :, :, 0:3, 0] #(M,H,W,3)
pos = Ts[:, 0:3, 3] #(M,3)
pos = pos.unsqueeze(1).unsqueeze(1).repeat(1, h, w, 1)
rays = torch.cat([dirs, pos], dim=3) #(M,H,W,6)
if images is not None:
rgbs = images.permute(0, 2, 3, 1) #(M,H,W,C)
else:
rgbs = None
if masks is not None:
rays = rays[masks > mask_threshold, :]
if rgbs is not None:
rgbs = rgbs[masks > mask_threshold, :]
else:
rays = rays.reshape((-1, rays.size(3)))
if rgbs is not None:
rgbs = rgbs.reshape((-1, rgbs.size(3)))
return rays, rgbs
def se(self,
X,
Y,
batch_size=25,
parallel=0,
sampling=100,
each_species=True):
dataLoader = self._get_DataLoader(X,
Y,
batch_size=batch_size,
shuffle=False)
loss_func = self.__build_loss_function(train=True)
se = []
y_dim = np.size(self.weights_numpy[0][0], 1)
weights = self.weights[0][0]
_ = sys.stdout.write("\nCalculating standard errors...\n")
if each_species:
for i in range(y_dim):
_ = sys.stdout.write("\rSpecies: {}/{} ".format(i + 1, y_dim))
sys.stdout.flush()
weights = torch.tensor(self.weights_numpy[0][0][:, i].reshape(
[-1, 1]),
device=self.device,
dtype=self.dtype,
requires_grad=True).to(self.device)
if i == 0:
constants = torch.tensor(self.weights_numpy[0][0][:, (i +
1):],
device=self.device,
dtype=self.dtype).to(self.device)
w = torch.cat([weights, constants], dim=1)
elif i < y_dim:
w = torch.cat([
torch.tensor(self.weights_numpy[0][0][:, 0:i],
device=self.device,
dtype=self.dtype).to(self.device),
weights,
torch.tensor(self.weights_numpy[0][0][:, (i + 1):],
device=self.device,
dtype=self.dtype).to(self.device)
],
dim=1)
else:
constants = torch.tensor(self.weights_numpy[0][0][:, 0:i],
device=self.device,
dtype=self.dtype).to(self.device)
w = torch.cat([constants, weights], dim=1)
for step, (x, y) in enumerate(dataLoader):
x = x.to(self.device, non_blocking=True)
y = y.to(self.device, non_blocking=True)
mu = torch.nn.functional.linear(x, w.t())
loss = loss_func(mu, y, x.shape[0], sampling)
loss = torch.sum(loss)
first_gradients = torch.autograd.grad(loss,
weights,
retain_graph=True,
create_graph=True,
allow_unused=True)
second = []
for j in range(self.input_shape):
second.append(
torch.autograd.grad(first_gradients[0][j, 0],
inputs=weights,
retain_graph=True,
create_graph=False,
allow_unused=False)[0])
hessian = torch.cat(second, dim=1)
if step < 1:
hessian_out = hessian
else:
hessian_out += hessian
se.append(
torch.sqrt(torch.diag(
torch.inverse(hessian_out))).data.cpu().numpy())
return se
else:
for step, (x, y) in enumerate(dataLoader):
x = x.to(self.device, non_blocking=True)
y = y.to(self.device, non_blocking=True)
mu = self.layers[0](x)
loss = torch.sum(loss_func(mu, y, x.shape[0], sampling))
first_gradients = torch.autograd.grad(
loss,
weights,
retain_graph=True,
create_graph=True,
allow_unused=True)[0].reshape([-1])
hessian = []
for j in range(first_gradients.shape[0]):
hessian.append(
torch.autograd.grad(
first_gradients[j],
inputs=weights,
retain_graph=True,
create_graph=False,
allow_unused=False)[0].reshape([-1]).reshape(
[y_dim * self.input_shape, 1]))
hessian = torch.cat(hessian, dim=1)
if step < 1:
hessian_out = hessian
else:
hessian_out += hessian
return hessian_out.data.cpu().numpy()
import torch
from torchvision import transforms
mat_rgb2lab = torch.tensor([[0.412453, 0.357580, 0.180423],
[0.212671, 0.715160, 0.072169],
[0.019334, 0.119193, 0.950227]])
mat_lab2rgb = torch.inverse(mat_rgb2lab)
d65_2 = torch.tensor([0.95047, 1.00000,
1.08883]) # CIE Standard Illuminant D65, 2° observer
def clip(x):
'''
Project a tensor's values on [0, 1] (inplace).
Parameters
----------
x : torch.Tensor
Returns
-------
x : torch.Tensor
'''
x[x > 1.] = 1.
x[x < 0.] = 0.
return x
def rgb2lab(img_rgb):
def l1_ll():
ss = torch.matmul(self.sigma, self.sigma.t())
if inverse:
ss = torch.inverse(ss)
return torch.mul(
l1, torch.sum(torch.abs(torch.triu(ss, diag))))
import torch import torch.optim as optim from anvil.models import RealNVP from anvil.train import shifted_kl L = 2 # very small system N_UNITS = L**2 np.random.seed(seed=0) A_NP = np.array(np.random.rand(N_UNITS, N_UNITS), dtype=np.float32) A = torch.from_numpy(A_NP) # just make some positive definite matrix COV_TARGET = [email protected] INV = torch.inverse(COV_TARGET) def target_distribution_s(phi): r"""Returns action S(\phi) for a stack of states \phi, shape (N_states, D). The action corresponds to a toy training example of a multigaussian distribution. """ return 0.5*((phi@INV)*phi).sum(axis=-1, keepdim=True) def split_to_flat_index(length): r"""Returns a 1D tensor of size length^2. If used to index a split state such that \phi = (\phi_a, \phi_b) then \phi -> flattened state with \phi_a and \phi_b interlaced in a checkerboard pattern You only need to use this function if the action is defined to accept the flattened state geometry. If the action is defined to act just on a split
def cam_to_ndc(self, value):
self._cam_to_ndc = value
self.ndc_to_cam = torch.inverse(self._cam_to_ndc)
def compute_projection(self, camera_to_world):
# compute projection by voxels -> image
world_to_camera = torch.inverse(camera_to_world)
voxel_bounds_min, voxel_bounds_max = self.compute_frustum_bounds(
camera_to_world)
return voxel_bounds_min, voxel_bounds_max, world_to_camera
env.env.state = np.array([x0, 0.0, q0, 0.0], dtype=np.float32)
obs = env.env._get_obs()
y = torch.tensor([obs[0], obs[1], obs[2], obs[3], obs[4], u10, u20], requires_grad=True, device=device, dtype=torch.float32).view(1, 7)
t_eval = torch.linspace(t_span[0], t_span[1], n_eval).to(device)
y_traj = []
y_traj.append(y)
frames = []
for i in range(len(t_eval)-1):
frames.append(env.render(mode='rgb_array'))
x_cos_q_sin_q, x_dot_q_dot, u = torch.split(y, [3, 2, 2], dim=1)
M_q_inv = symoden_ode_struct_model.M_net(x_cos_q_sin_q)
x_dot_q_dot_aug = torch.unsqueeze(x_dot_q_dot, dim=2)
p = torch.squeeze(torch.matmul(torch.inverse(M_q_inv), x_dot_q_dot_aug), dim=2)
x_cos_q_sin_q_p = torch.cat((x_cos_q_sin_q, p), dim=1)
x_cos_q_sin_q, p = torch.split(x_cos_q_sin_q_p, [3, 2], dim=1)
M_q_inv = symoden_ode_struct_model.M_net(x_cos_q_sin_q)
_, cos_q, sin_q = torch.chunk(x_cos_q_sin_q, 3,dim=1)
V_q = symoden_ode_struct_model.V_net(x_cos_q_sin_q)
p_aug = torch.unsqueeze(p, dim=2)
H = torch.squeeze(torch.matmul(torch.transpose(p_aug, 1, 2), torch.matmul(M_q_inv, p_aug)))/2.0 + torch.squeeze(V_q)
dH = torch.autograd.grad(H.sum(), x_cos_q_sin_q_p, create_graph=True)[0]
dHdx, dHdcos_q, dHdsin_q, dHdp= torch.split(dH, [1, 1, 1, 2], dim=1)
dV = torch.autograd.grad(V_q, x_cos_q_sin_q)[0]
dVdx, dVdcos_q, dVdsin_q= torch.chunk(dV, 3, dim=1)
dV_q = - dVdcos_q * sin_q + dVdsin_q * cos_q
g_xq = symoden_ode_struct_model.g_net(x_cos_q_sin_q)
g_xq_T = torch.transpose(g_xq, 1, 2)
inv_g_g_T = torch.inverse(torch.matmul(g_xq, g_xq_T))
y = xy[:, :, 1]
mtm = construct_pred_MtM(bSize, Pw, x, y)
xtmtmx = torch.bmm(
torch.bmm(gCc.view(-1, 1, 12), mtm.view(-1, 12, 12)),
gCc.view(-1, 12, 1)).view(-1)
efLoss = 1e7 * xtmtmx.sum()
if False:
for n in range(len(Pi)):
uv = Pi[n].view(-1, 2)
cc = gCc[n].view(-1, 1)
# normalize the uv
uv1 = torch.cat((uv, torch.ones(len(uv), 1.0).double()), 1)
uv = torch.inverse(K).mm(uv1.t()).t()
uv = uv[:, :2]
Cw, Alpha, M = PnP.PrepareData(Pw, uv)
mtm = M.t().mm(M)
eigenFreeLoss = cc.t().mm(mtm).mm(cc)
efLoss = efLoss + eigenFreeLoss
# get RT from M
Km = PnP.GetKernel(M)
vK = Km[:,
3] # take the last column: the one with smallest eigenvalues
vK = vK.view(4, 3)
m = vK[:, 2].mean()
if (m <= 0).data.numpy():
vK = -vK
R, T, r = PnP.Procrustes(Cw, vK)
def differentiable_nms(scores_unsorted,
iou_unsorted,
nms_threshold=0.4,
pruning_method="linear",
temperature=0.01,
valid_box_prob_threshold=0.3,
return_sorted_prob=False,
sorting_method="hard",
sorting_temperature=None,
group_boxes=True,
mask_group_boxes=True,
group_size=100,
debug=False):
"""
GrooMeD-NMS: Grouped Mathematical Differentiable NMS.
Abhinav Kumar
:param scores_unsorted: Unsorted scores of the boxes Tensor (N, )
:param iou_unsorted: Overlap matrix of the boxes Tensor (N, N)
:param nms_threshold: NMS threshold
:param pruning_method: Pruning method/function- Can be one of linear/soft_nms (exponential)/sigmoidal
:param temperature: Temperature of the pruning function. If pruning function is linear, this is not used
:param valid_box_prob_threshold: Used to decide if the box is valid after re-scoring.
:param return_sorted_prob: Whether should we return sorted scores or not
:param sorting_method: Sorting should be hard or soft
:param sorting_temperature: Sorting temperature for soft sorting. If not supplied, sorting_temperature = temperature
:param group_boxes: Grouping should be carried out or not
:param mask_group_boxes: Masking should be carried out or not
:param group_size: Maximum Group size of the group
:param debug: Only for debugging
:return:
valid_boxes_index: Valid box indices after NMS (Variable tensor of shape between 1 and N)
invalid_boxes_index: Invalid box indices after NMS (Variable tensor of shape between N-1 and 0)
non_suppression_prob: Re-scores or updated-scores of the boxes after NMS Tensor (N, )
"""
if type(scores_unsorted) == np.ndarray:
scores_unsorted = torch.from_numpy(scores_unsorted).float()
iou_unsorted = torch.from_numpy(iou_unsorted).float()
#======================================================================
# Do the sorting of the scores and the corresponding IoU matrix
#======================================================================
_, indices = torch.sort(scores_unsorted, descending=True)
if sorting_method == "soft":
if sorting_temperature is None:
sorting_temperature = temperature
scores, convex_comb_matrix, iou = soft_sort(
scores_unsorted,
full_matrix=iou_unsorted,
temperature=sorting_temperature)
else:
scores = scores_unsorted[indices]
iou = iou_unsorted[indices][:, indices]
if debug:
print("\nInside diff NMS... After sorting")
print(scores)
print(iou)
num_boxes = scores.shape[0]
mask = torch.eye(num_boxes).byte()
identity = torch.eye(num_boxes)
ones = torch.ones(num_boxes, )
mask = cast_to_cpu_cuda_tensor(mask, iou)
identity = cast_to_cpu_cuda_tensor(identity, iou)
ones = cast_to_cpu_cuda_tensor(ones, iou)
if group_boxes:
inversion_matrix = torch.zeros(iou.shape)
inversion_matrix = cast_to_cpu_cuda_tensor(inversion_matrix, iou)
#======================================================================
# Prune Matrix
#======================================================================
overlap_probabilities = pruning_function(iou,
nms_threshold=nms_threshold,
temperature=temperature,
pruning_method=pruning_method)
overlap_probabilities = torch.tril(overlap_probabilities)
overlap_probabilities.masked_fill_(mask, 0)
phi_matrix = overlap_probabilities
#======================================================================
# Inversion or Subtraction
#======================================================================
if group_boxes:
#==================================================================
# Grouping
#==================================================================
groups = get_groups(iou_unsorted=iou,
group_threshold=nms_threshold,
group_size=group_size,
scores_unsorted=scores)
if debug:
print("Groups")
print(groups)
num_groups = len(groups)
#==================================================================
# Masking
#==================================================================
if mask_group_boxes:
mask = mask.float()
mask[:, :] = 0
for j in range(num_groups):
mask[groups[j], groups[j][0]] = 1
phi_matrix = phi_matrix * mask
# Do groupwise inversion
for j in range(num_groups):
if mask_group_boxes:
temp_store = identity[groups[j]][:, groups[j]] - phi_matrix[
groups[j]][:, groups[j]]
else:
temp_store = torch.inverse(identity[groups[j]][:, groups[j]] +
phi_matrix[groups[j]][:, groups[j]])
inversion_matrix = indices_copy(A=inversion_matrix,
B=temp_store,
indA=groups[j])
else:
inversion_matrix = torch.inverse(identity + phi_matrix)
non_suppression_prob_unsort = torch.clamp(torch.matmul(
inversion_matrix, scores),
min=0,
max=1)
# non_suppression_prob_unsort = torch.min(non_suppression_prob_unsort, scores)
non_suppression_prob_unsort_2 = non_suppression_prob_unsort.clone()
non_suppression_prob_unsort[
non_suppression_prob_unsort < valid_box_prob_threshold] = 0
if return_sorted_prob:
non_suppression_prob, sorted_indices = torch.sort(
non_suppression_prob_unsort, descending=True)
valid_boxes_index = indices[sorted_indices[
non_suppression_prob >= valid_box_prob_threshold]]
invalid_boxes_index = indices[sorted_indices[
non_suppression_prob < valid_box_prob_threshold]]
else:
dummy, sorted_indices = torch.sort(non_suppression_prob_unsort,
descending=True)
valid_boxes_index = indices[sorted_indices[
dummy >= valid_box_prob_threshold]]
invalid_boxes_index = indices[sorted_indices[
dummy < valid_box_prob_threshold]]
if group_boxes:
non_suppression_prob = non_suppression_prob_unsort_2
else:
non_suppression_prob = non_suppression_prob_unsort
return valid_boxes_index, invalid_boxes_index, non_suppression_prob
def inverse(self, x):
return torch.inverse(x)
def QuantizedWeight(x, n, nbit=2, training=False):
"""
Quantize weight.
Args:
x (torch.Tensor): a 4D tensor. [K x K x iC x oC] -> [oC x iC x K x K]
Must have known number of channels, but can have other unknown dimensions.
n (int or double): variance of weight initialization.
nbit (int): number of bits of quantized weight. Defaults to 2.
training (bool):
Returns:
torch.Tensor with attribute `variables`.
Variable Names:
* ``basis``: basis of quantized weight.
Note:
About multi-GPU training: moving averages across GPUs are not aggregated.
Batch statistics are computed by main training tower. This is consistent with most frameworks.
"""
oc, ic, k1, k2 = x.shape
device = x.device
init_basis = []
base = NORM_PPF_0_75 * ((2. / n) ** 0.5) / (2 ** (nbit - 1))
for j in range(nbit):
init_basis.append([(2 ** j) * base for i in range(oc)])
num_levels = 2 ** nbit
delta = EPS
# initialize level multiplier
# binary code of each level:
# shape: [num_levels, nbit]
init_level_multiplier = []
for i in range(num_levels):
level_multiplier_i = [0. for j in range(nbit)]
level_number = i
for j in range(nbit):
binary_code = level_number % 2
if binary_code == 0:
binary_code = -1
level_multiplier_i[j] = float(binary_code)
level_number = level_number // 2
init_level_multiplier.append(level_multiplier_i)
# initialize threshold multiplier
# shape: [num_levels-1, num_levels]
# [[0,0,0,0,0,0,0.5,0.5]
# [0,0,0,0,0,0.5,0.5,0,]
# [0,0,0,0,0.5,0.5,0,0,]
# ...
# [0.5,0.5,0,0,0,0,0,0,]]
init_thrs_multiplier = []
for i in range(1, num_levels):
thrs_multiplier_i = [0. for j in range(num_levels)]
thrs_multiplier_i[i - 1] = 0.5
thrs_multiplier_i[i] = 0.5
init_thrs_multiplier.append(thrs_multiplier_i)
# [nbit, oc]
basis = torch.tensor(init_basis, dtype=torch.float32, requires_grad=False)
# [2**nbit, nbit] or [num_levels, nbit]
level_codes = torch.tensor(init_level_multiplier)
# [num_levels-1, num_levels]
thrs_multiplier = torch.tensor(init_thrs_multiplier)
# [oC x iC x K x K] -> [K x K x iC x oC]
xp = x.permute((3, 2, 1, 0))
N = 3
# training
if training:
for _ in torch.arange(N):
# calculate levels and sort [2**nbit, oc] or [num_levels, oc]
levels = torch.matmul(level_codes, basis)
levels, sort_id = torch.sort(levels, 0)
# calculate threshold
# [num_levels-1, oc]
thrs = torch.matmul(thrs_multiplier, levels)
# calculate level codes per channel
# ix:sort_id [num_levels, oc], iy: torch.arange(nbit)
# level_codes = [num_levels, nbit]
# level_codes_channelwise [num_levels, oc, nbit]
for oc_idx in torch.arange(oc):
if oc_idx == 0:
level_codes_t = level_codes[
torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1)
level_codes_channelwise = level_codes_t
else:
level_codes_t = level_codes[
torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1)
level_codes_channelwise = torch.cat((level_codes_channelwise, level_codes_t), 1)
# calculate output y and its binary code
# y [K, K, iC, oC]
# bits_y [K x K x iC, oC, nbit]
reshape_x = torch.reshape(xp, [-1, oc])
y = torch.zeros_like(xp) + levels[0] # output
zero_y = torch.zeros_like(xp)
bits_y = torch.full([reshape_x.shape[0], oc, nbit], -1., device=device)
zero_bits_y = torch.zeros_like(bits_y)
# [K x K x iC x oC] [1, oC]
for i in torch.arange(num_levels - 1):
g = torch.ge(xp, thrs[i])
# [K, K, iC, oC] + [1, oC], [K, K, iC, oC] => [K, K, iC, oC]
y = torch.where(g, zero_y + levels[i + 1], y)
# [K x K x iC, oC, nbit]
bits_y = torch.where(g.view(-1, oc, 1), zero_bits_y + level_codes_channelwise[i + 1], bits_y)
# calculate BTxB
# [oC, nbit, K x K x iC] x [oC, K x K x iC, nbit] => [oC, nbit, nbit]
BTxB = torch.matmul(bits_y.permute(1, 2, 0), bits_y.permute(1, 0, 2)) + delta * torch.eye(nbit,
device=device)
# calculate inverse of BTxB
# [oC, nbit, nbit]
if nbit > 2:
BTxB_inv = torch.inverse(BTxB)
elif nbit == 2:
det = torch.det(BTxB)
BTxB_inv = torch.stack((BTxB[:, 1, 1], -BTxB[:, 0, 1], -BTxB[:, 1, 0], BTxB[:, 0, 0]),
1).view(OC, nbit, nbit) / det.unsqueeze(-1).unsqueeze(-1)
elif nbit == 1:
BTxB_inv = 1 / BTxB
else:
BTxB_inv = None
# calculate BTxX
# bits_y [K x K x iC, oc, nbit] reshape_x [K x K x iC, oC]
# [oC, nbit, K x K x iC] [oC, K x K x iC, 1] => [oC, nbit, 1]
BTxX = torch.matmul(bits_y.permute(1, 2, 0), reshape_x.permute(1, 0).unsqueeze(-1))
BTxX = BTxX + (delta * basis.permute(1, 0).unsqueeze(-1)) # + basis
# calculate new basis
# BTxB_inv: [oC, nbit, nbit] BTxX: [oC, nbit, 1]
# [oC, nbit, nbit] x [oC, nbit, 1] => [oC, nbit, 1] => [nbit, oC]
new_basis = torch.matmul(BTxB_inv, BTxX).squeeze(-1).permute(1, 0)
# print(BTxB_inv.shape,BTxX.shape)
# print(new_basis.shape)
# create moving averages op
basis -= (1 - MOVING_AVERAGES_FACTOR) * (basis - new_basis)
# print("\nbasis:\n", basis)
# calculate levels and sort [2**nbit, oc] or [num_levels, oc]
levels = torch.matmul(level_codes, basis)
levels, sort_id = torch.sort(levels, 0)
# calculate threshold
# [num_levels-1, oc]
thrs = torch.matmul(thrs_multiplier, levels)
# calculate level codes per channel
# ix:sort_id [num_levels, oc], iy: torch.arange(nbit)
# level_codes = [num_levels, nbit]
# level_codes_channelwise [num_levels, oc, nbit]
for oc_idx in torch.arange(oc):
if oc_idx == 0:
level_codes_t = level_codes[
torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1)
level_codes_channelwise = level_codes_t
else:
level_codes_t = level_codes[
torch.meshgrid(sort_id[:, oc_idx], torch.arange(nbit, device=sort_id.device))].unsqueeze(1)
level_codes_channelwise = torch.cat((level_codes_channelwise, level_codes_t), 1)
# calculate output y and its binary code
# y [K, K, iC, oC]
# bits_y [K x K x iC, oC, nbit]
reshape_x = torch.reshape(xp, [-1, oc])
y = torch.zeros_like(xp) + levels[0] # output
zero_y = torch.zeros_like(xp)
bits_y = torch.full([reshape_x.shape[0], oc, nbit], -1., device=device)
zero_bits_y = torch.zeros_like(bits_y)
# [K x K x iC x oC] [1, oC]
for i in torch.arange(num_levels - 1):
g = torch.ge(xp, thrs[i])
# [K, K, iC, oC] + [1, oC], [K, K, iC, oC] => [K, K, iC, oC]
y = torch.where(g, zero_y + levels[i + 1], y)
# [K x K x iC, oC, nbit]
bits_y = torch.where(g.view(-1, oc, 1), zero_bits_y + level_codes_channelwise[i + 1], bits_y)
return y.permute(3, 2, 1, 0), levels.permute(1, 0), thrs.permute(1, 0)
def smoothness_norm(T,
theta,
lambda_smooth=0.5,
lambda_var=0.1,
print_info=False):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
D, d = T.get_basis().shape
B = T.get_basis()
if isinstance(B, np.ndarray):
B = torch.from_numpy(B).float()
B = B.to(device)
nC = d + 1 # = Tess size
n = 1 # num sampples?
# Convert type
#B = tf.cast(B, tf.float32)
#theta = tf.cast(theta, tf.float32)
theta_T = torch.transpose(theta, 0, 1)
# for plotting, {"title": item_to_plot/show}
covariance_to_plot = {}
items_to_plot = {}
# Distance between centers
centers = torch.linspace(-1., 1.,
D).to(device) # from 0 to 1 with nC steps
# calculate the distance
dists = torch_dist_mat(centers) # DxD
# # scale the distance
# for x>0, e^(-x^2) decays very fast from 1 to 0
cov_avees = torch.exp(-(dists / lambda_smooth))
cov_avees *= (cov_avees * (lambda_var * D)**2)
B_T = torch.transpose(B, 0, 1)
cov_cpa = torch.matmul(B_T, torch.matmul(cov_avees, B))
precision_theta = torch.inverse(cov_cpa)
if print_info:
print("Info: ")
print(" Distance between centers shape:", dists.shape)
print(" B shape:", B.shape)
print(" D shape:", D)
print(" d shape:", d)
print(" cov_avess shape:", cov_avees.shape)
print(" cov_cpa shape:", cov_cpa.shape)
print(" theta shape:", theta.shape)
# plot velocity field
plot_velocity_field = False
if plot_velocity_field:
nb_points = 1000
points = T.uniform_meshgrid([nb_points for i in range(T._ndim)])
vector_field = T.calc_vectorfield(
points, theta_T.eval()) # tf.transpose(theta)
items_to_plot["CPA Velocity Field for theta with prior"] = vector_field
items_to_plot["Theta values for: theta with prior"] = theta_T
# Calculate smoothness norm
theta = torch.squeeze(theta)
theta_T = torch.transpose(theta, 0, 1)
smooth_norm = torch.matmul(theta, torch.matmul(precision_theta, theta_T))
smooth_norm = torch.mean(smooth_norm)
return smooth_norm
def closed_form_ls(F_t, y_t):
# Returns min norm least squares solution
w_t = F_t.T @ torch.inverse(F_t @ F_t.T) @ y_t
return w_t
def l2_ll():
ss = torch.matmul(self.sigma, self.sigma.t())
if inverse:
ss = torch.inverse(ss)
return torch.mul(
l2, torch.sum(torch.pow(torch.triu(ss, diag), 2.0)))
def compute_gploss(self, zy_in, imgid, batch_id, label_flg=0):
tensor_mat = zy_in
Sg_Pred = torch.zeros([self.train_batch_size, 1])
Sg_Pred = Sg_Pred.cuda()
LSg_Pred = torch.zeros([self.train_batch_size, 1])
LSg_Pred = LSg_Pred.cuda()
sq_err = 0
for i in range(tensor_mat.shape[0]):
tmp_i = self.dict_unlbl[
imgid[i]] if label_flg == 0 else self.dict_lbl[
imgid[i]] # imag_id in the dictionary
tensor = tensor_mat[i, :, :, :] # z tensor
tensor_vec = tensor.view(-1, self.z_height *
self.z_width) # z tensor to a vector
ker_UU = self.kernel_comp(
tensor_vec, tensor_vec, self.z_height, self.z_width,
self.z_numchnls,
self.z_numchnls) # k(z,z), i.e kernel value for z,z
nearest_vl = self.ker_unlbl[
tmp_i, :] if label_flg == 0 else self.ker_lbl[
tmp_i, :] #kernel values are used to get neighbors
tp32_vec = np.array(
sorted(range(len(nearest_vl)),
key=lambda k: nearest_vl[k])[-1 * self.num_nearest:])
lt32_vec = np.array(
sorted(range(len(nearest_vl)),
key=lambda k: nearest_vl[k])[:self.num_nearest])
# Nearest neighbor latent space labeled vectors
near_dic_lbl = np.zeros((self.num_nearest, self.z_numchnls,
self.z_height, self.z_width))
for j in range(self.num_nearest):
near_dic_lbl[j, :] = self.Fz_lbl[tp32_vec[j], :, :, :]
near_vec_lbl = np.reshape(near_dic_lbl,
(self.num_nearest * self.z_numchnls,
self.z_height * self.z_width))
far_dic_lbl = np.zeros((self.num_nearest, self.z_numchnls,
self.z_height, self.z_width))
for j in range(self.num_nearest):
far_dic_lbl[j, :] = self.Fz_lbl[lt32_vec[j], :, :, :]
far_vec_lbl = np.reshape(far_dic_lbl,
(self.num_nearest * self.z_numchnls,
self.z_height * self.z_width))
# computing kernel matrix of nearest label vectors
# and then computing (K_L+sig^2I)^(-1)
ker_LL = cosine_similarity(near_vec_lbl, near_vec_lbl)
inv_ker = inv(ker_LL +
1.0 * np.eye(self.num_nearest * self.z_numchnls))
farker_LL = cosine_similarity(far_vec_lbl, far_vec_lbl)
farinv_ker = inv(farker_LL +
1.0 * np.eye(self.num_nearest * self.z_numchnls))
mn_pre = np.matmul(
inv_ker, near_vec_lbl) # used for computing mean prediction
mn_pre = mn_pre.astype(np.float32)
#converting require variables to cuda tensors
near_vec_lbl = torch.from_numpy(near_vec_lbl.astype(np.float32))
far_vec_lbl = torch.from_numpy(far_vec_lbl.astype(np.float32))
inv_ker = torch.from_numpy(inv_ker.astype(np.float32))
farinv_ker = torch.from_numpy(farinv_ker.astype(np.float32))
inv_ker = inv_ker.cuda()
farinv_ker = farinv_ker.cuda()
near_vec_lbl = near_vec_lbl.cuda()
far_vec_lbl = far_vec_lbl.cuda()
mn_pre = torch.from_numpy(
mn_pre) # used for mean prediction (mu) or z_pseudo
mn_pre = mn_pre.cuda()
# Identity matrix
Eye = torch.eye(self.z_numchnls)
Eye = Eye.cuda()
# computing sigma or variance between nearest labeled vectors and unlabeled vector
ker_UL = self.kernel_comp(tensor_vec, near_vec_lbl, self.z_height,
self.z_width, self.z_numchnls,
self.z_numchnls * self.num_nearest)
sigma_est = ker_UU - torch.matmul(
ker_UL, torch.matmul(inv_ker, ker_UL.t())) + Eye
# computing variance between farthest labeled vectors and unlabeled vector
Farker_UL = self.kernel_comp(tensor_vec, far_vec_lbl,
self.z_height, self.z_width,
self.z_numchnls,
self.z_numchnls * self.num_nearest)
far_sigma_est = ker_UU - torch.matmul(
Farker_UL, torch.matmul(farinv_ker, Farker_UL.t())) + Eye
# computing mean prediction
mean_pred = torch.matmul(ker_UL,
mn_pre) #mean prediction (mu) or z_pseudo
inv_sigma = torch.inverse(sigma_est)
sq_err += torch.mean(
torch.matmul((tensor_vec - mean_pred).t(),
torch.matmul(inv_sigma, (tensor_vec - mean_pred)))
) + 1.0 * self.lambda_var * torch.log(
torch.det(sigma_est)) - 0.000001 * self.lambda_var * torch.log(
torch.det(far_sigma_est))
Sg_Pred[i, :] = torch.log(torch.det(sigma_est))
LSg_Pred[i, :] = torch.log(torch.det(far_sigma_est))
if not (batch_id % 100):
print(LSg_Pred.max().item(),
Sg_Pred.max().item(),
sq_err.mean().item() / self.train_batch_size,
Sg_Pred.mean().item())
gp_loss = ((1.0 * sq_err / self.train_batch_size))
return gp_loss
def compute_projection(self, depth, camera_to_world, world_to_grid):
# compute projection by voxels -> image
#print 'camera_to_world', camera_to_world
#print 'intrinsic', self.intrinsic
#print(world_to_grid)
world_to_camera = torch.inverse(camera_to_world)
grid_to_world = torch.inverse(world_to_grid)
voxel_bounds_min, voxel_bounds_max = self.compute_frustum_bounds(world_to_grid, camera_to_world)
voxel_bounds_min = np.maximum(voxel_bounds_min, 0).cuda().float() if depth.is_cuda else np.maximum(voxel_bounds_min, 0).cpu().float()
voxel_bounds_max = np.minimum(voxel_bounds_max, self.volume_dims).cuda().float() if depth.is_cuda else np.minimum(voxel_bounds_max, self.volume_dims).cpu().float()
# coordinates within frustum bounds
# TODO python opt for this part instead of lua/torch opt?
lin_ind_volume = torch.arange(0, self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2], out=torch.LongTensor())
lin_ind_volume = lin_ind_volume.cuda() if depth.is_cuda else lin_ind_volume.cpu()
coords = camera_to_world.new(4, lin_ind_volume.size(0))
coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1])
tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long()
coords[1] = tmp / self.volume_dims[0]
coords[0] = torch.remainder(tmp, self.volume_dims[0])
coords[3].fill_(1)
mask_frustum_bounds = torch.ge(coords[0], voxel_bounds_min[0]) * torch.ge(coords[1], voxel_bounds_min[1]) * torch.ge(coords[2], voxel_bounds_min[2])
mask_frustum_bounds = mask_frustum_bounds * torch.lt(coords[0], voxel_bounds_max[0]) * torch.lt(coords[1], voxel_bounds_max[1]) * torch.lt(coords[2], voxel_bounds_max[2])
if not mask_frustum_bounds.any():
print('error: nothing in frustum bounds')
return None
lin_ind_volume = lin_ind_volume[mask_frustum_bounds]
coords = coords.resize_(4, lin_ind_volume.size(0))
coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1])
tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long()
coords[1] = tmp / self.volume_dims[0]
coords[0] = torch.remainder(tmp, self.volume_dims[0])
coords[3].fill_(1)
# transform to current frame
p = torch.mm(world_to_camera, torch.mm(grid_to_world, coords))
# project into image
p[0] = (p[0] * self.intrinsic[0][0]) / p[2] + self.intrinsic[0][2]
p[1] = (p[1] * self.intrinsic[1][1]) / p[2] + self.intrinsic[1][2]
pi = torch.round(p).long()
valid_ind_mask = torch.ge(pi[0], 0) * torch.ge(pi[1], 0) * torch.lt(pi[0], self.image_dims[0]) * torch.lt(pi[1], self.image_dims[1])
if not valid_ind_mask.any():
print('error: no valid image indices')
return None
valid_image_ind_x = pi[0][valid_ind_mask]
valid_image_ind_y = pi[1][valid_ind_mask]
valid_image_ind_lin = valid_image_ind_y * self.image_dims[0] + valid_image_ind_x
depth_vals = torch.index_select(depth.view(-1), 0, valid_image_ind_lin)
depth_mask = depth_vals.ge(self.depth_min) * depth_vals.le(self.depth_max) * torch.abs(depth_vals - p[2][valid_ind_mask]).le(self.voxel_size)
if not depth_mask.any():
print('error: no valid depths')
return None
lin_ind_update = lin_ind_volume[valid_ind_mask]
lin_ind_update = lin_ind_update[depth_mask]
lin_indices_3d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size)
lin_indices_2d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size)
lin_indices_3d[0] = lin_ind_update.shape[0]
lin_indices_2d[0] = lin_ind_update.shape[0]
lin_indices_3d[1:1+lin_indices_3d[0]] = lin_ind_update
lin_indices_2d[1:1+lin_indices_2d[0]] = torch.index_select(valid_image_ind_lin, 0, torch.nonzero(depth_mask)[:,0])
num_ind = lin_indices_3d[0]
#print '[proj] #ind = ', lin_indices_3d[0]
#print '2d', torch.min(lin_indices_2d[1:1+num_ind]), torch.max(lin_indices_2d[1:1+num_ind])
#print '3d', torch.min(lin_indices_3d[1:1+num_ind]), torch.max(lin_indices_3d[1:1+num_ind])
return lin_indices_3d, lin_indices_2d
def gevd(a, b=None):
"""This method computes the eigenvectors and the eigenvalues
of complex Hermitian matrices. The method finds a solution to
the problem AV = BVD where V are the eigenvectors and D are
the eigenvalues.
The eigenvectors returned by the method (vs) are stored in a tensor
with the following format (*,C,C,2).
The eigenvalues returned by the method (ds) are stored in a tensor
with the following format (*,C,C,2).
Arguments
---------
a : tensor
A first input matrix. It is equivalent to the matrix A in the
equation in the description above. The tensor must have the
following format: (*,2,C+P).
b : tensor
A second input matrix. It is equivalent tot the matrix B in the
equation in the description above. The tensor must have the
following format: (*,2,C+P).
This argument is optional and its default value is None. If
b == None, then b is replaced by the identity matrix in the
computations.
Example
-------
Suppose we would like to compute eigenvalues/eigenvectors on the
following complex Hermitian matrix:
A = [ 52 34 + 37j 16 + j28 ;
34 - 37j 125 41 + j3 ;
16 - 28j 41 - j3 62 ]
>>> a = torch.FloatTensor([[52,34,16,125,41,62],[0,37,28,0,3,0]])
>>> vs, ds = gevd(a)
This corresponds to:
D = [ 20.9513 0 0 ;
0 43.9420 0 ;
0 0 174.1067 ]
V = [ 0.085976 - 0.85184j -0.24620 + 0.12244j -0.24868 - 0.35991j ;
-0.16006 + 0.20244j 0.37084 + 0.40173j -0.79175 - 0.087312j ;
-0.43990 + 0.082884j -0.36724 - 0.70045j -0.41728 + 0 j ]
where
A = VDV^-1
"""
# Dimensions
D = a.dim()
P = a.shape[D - 1]
C = int(round(((1 + 8 * P)**0.5 - 1) / 2))
# Converting the input matrices to block matrices
ash = f(a)
if b is None:
b = torch.zeros(a.shape, dtype=a.dtype, device=a.device)
ids = torch.triu_indices(C, C)
b[..., 0, ids[0] == ids[1]] = 1.0
bsh = f(b)
# Performing the Cholesky decomposition
lsh = torch.cholesky(bsh)
lsh_inv = torch.inverse(lsh)
lsh_inv_T = torch.transpose(lsh_inv, D - 2, D - 1)
# Computing the matrix C
csh = torch.matmul(lsh_inv, torch.matmul(ash, lsh_inv_T))
# Performing the eigenvalue decomposition
es, ysh = torch.symeig(csh, eigenvectors=True)
# Collecting the eigenvalues
dsh = torch.zeros(
a.shape[slice(0, D - 2)] + (2 * C, 2 * C),
dtype=a.dtype,
device=a.device,
)
dsh[..., range(0, 2 * C), range(0, 2 * C)] = es
# Collecting the eigenvectors
vsh = torch.matmul(lsh_inv_T, ysh)
# Converting the block matrices to full complex matrices
vs = ginv(vsh)
ds = ginv(dsh)
return vs, ds
def forward(ctx, input):
inverse = torch.inverse(input)
ctx.save_for_backward(inverse)
return inverse
def inv(x):
"""Inverse Hermitian Matrix.
This method finds the inverse of a complex Hermitian matrix
represented by its upper triangular part. The result will have
the following format: (*, C, C, 2).
Arguments
---------
x : tensor
An input matrix to work with. The tensor must have the
following format: (*, 2, C+P)
Example
-------
>>> import torch
>>>
>>> from speechbrain.dataio.dataio import read_audio
>>> from speechbrain.processing.features import STFT
>>> from speechbrain.processing.multi_mic import Covariance
>>> from speechbrain.processing.decomposition import inv
>>>
>>> xs_speech = read_audio(
... 'samples/audio_samples/multi_mic/speech_-0.82918_0.55279_-0.082918.flac'
... )
>>> xs_noise = read_audio('samples/audio_samples/multi_mic/noise_0.70225_-0.70225_0.11704.flac')
>>> xs = xs_speech + 0.05 * xs_noise
>>> xs = xs.unsqueeze(0).float()
>>>
>>> stft = STFT(sample_rate=16000)
>>> cov = Covariance()
>>>
>>> Xs = stft(xs)
>>> XXs = cov(Xs)
>>> XXs_inv = inv(XXs)
"""
# Dimensions
d = x.dim()
p = x.shape[-1]
n_channels = int(round(((1 + 8 * p)**0.5 - 1) / 2))
# Output matrix
ash = f(pos_def(x))
ash_inv = torch.inverse(ash)
as_inv = finv(ash_inv)
indices = torch.triu_indices(n_channels, n_channels)
x_inv = torch.zeros(
x.shape[slice(0, d - 2)] + (n_channels, n_channels, 2),
dtype=x.dtype,
device=x.device,
)
x_inv[..., indices[1], indices[0], 0] = as_inv[..., 0, :]
x_inv[..., indices[1], indices[0], 1] = -1 * as_inv[..., 1, :]
x_inv[..., indices[0], indices[1], 0] = as_inv[..., 0, :]
x_inv[..., indices[0], indices[1], 1] = as_inv[..., 1, :]
return x_inv
