def validate(test_loader, query_loader, model, score_fun):
logger.info('>>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>')
iter_time = AverageMeter()
data_time = AverageMeter()
mAP = RetrievalMAP(compute_on_step=False)
mRR = RetrievalMRR(compute_on_step=False)
pAt10 = RetrievalPrecision(compute_on_step=False, k=10)
model.eval()
end = time.time()
# To process each image only once, we store query info in memory (it's enough with just queries,
# and they are relatively few compared to catalog, so less memory required)
processed_queries = []
for (query_input, query_target, query_index) in query_loader:
query_input = query_input.cuda(non_blocking=True)
query_target = query_target.cuda(non_blocking=True)
query_index = query_index.cuda(non_blocking=True)
query_embeding = model(query_input, getFeatVec=True)
processed_queries.append((query_embeding, query_target, query_index))
for i, (test_input, test_target) in enumerate(test_loader):
data_time.update(time.time() - end)
test_input = test_input.cuda(non_blocking=True)
test_target = test_target.cuda(non_blocking=True)
test_embeding = model(test_input, getFeatVec=True)
for query_embeding, query_target, query_index in processed_queries:
scores = score_fun(test_embeding, query_embeding)
indices = torch.broadcast_to(query_index.unsqueeze(0),
scores.size())
target = test_target.unsqueeze(1) == query_target.unsqueeze(0)
mAP(scores, target, indices)
mRR(scores, target, indices)
pAt10(scores, target, indices)
iter_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.print_freq == 0:
logger.info(
'Test: [{}/{}] '
'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
'Iter {iter_time.val:.3f} ({iter_time.avg:.3f})'.format(
i + 1,
len(test_loader),
data_time=data_time,
iter_time=iter_time))
map_value = mAP.compute()
mrr_value = mRR.compute()
pAt10_value = pAt10.compute()
logger.info('Val result: mAP/mRR/P@10 {:.4f}/{:.4f}/{:.4f}.'.format(
map_value, mrr_value, pAt10_value))
logger.info('<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<')
return map_value, mrr_value, pAt10_value
def __getitem__(self, index):
img = PIL.Image.open(self.files[index][0])
label = self.files[index][1]
img = self.resize(img)
img = self.tranlationTransform(img)
img = self.rotationalTransformation(img)
#img = np.array(img, dtype=np.float32)
img = self.tensorTransform(img) #Will read it into 3xheightxwidth
#Standard normalize images
ravel = img.view(3, -1)
means = torch.mean(ravel, dim=1)
ravel -= torch.transpose(
torch.broadcast_to(means, (ravel.shape[1], 3)), 0, 1)
std = torch.std(ravel, dim=1)
ravel /= torch.transpose(torch.broadcast_to(std, (ravel.shape[1], 3)),
0, 1)
img = ravel.view(3, self.imgSize, self.imgSize)
return img, label
def _input_mask(input: Tensor, *args, **kwargs) -> Tensor:
"""Return canonical input mask.
Canonical input mask is a boolean tensor with the same shape as
input and with (broadcasted) content of mask, if specified.
"""
mask = kwargs.get('mask')
if mask is None:
inmask = input.new_ones(input.shape, dtype=torch.bool)
elif mask.ndim < input.ndim:
inmask = torch.broadcast_to(mask.clone(),
input.shape).to(dtype=torch.bool)
elif mask.ndim > input.ndim:
raise IndexError(
"_input_mask expected broadcastable mask (got mask dimensionality higher than of the input)"
)
elif mask.shape != input.shape:
inmask = torch.broadcast_to(mask.clone(),
input.shape).to(dtype=torch.bool)
else:
inmask = mask.to(dtype=torch.bool)
return inmask
def retrieval_validate(image_loader, sketch_loader, model, score_fun):
if main_process():
logger.info('>>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>')
iter_time = AverageMeter()
data_time = AverageMeter()
mAP = RetrievalMAP(compute_on_step=False)
model.eval()
end = time.time()
# To process each image only once, we store query info in memory (it's enough with just queries,
# and they are relatively few compared to catalog, so less memory required)
processed_queries = []
for (sketch_input, sketch_target, sketch_index) in sketch_loader:
sketch_input = sketch_input.cuda(non_blocking=True)
sketch_target = sketch_target.cuda(non_blocking=True)
sketch_index = sketch_index.cuda(non_blocking=True)
sketchFeatVec, _ = model(sketch_input, mode='sketch')
processed_queries.append((sketchFeatVec, sketch_target, sketch_index))
for i, (image_input, image_target) in enumerate(image_loader):
data_time.update(time.time() - end)
image_input = image_input.cuda(non_blocking=True)
image_target = image_target.cuda(non_blocking=True)
imageFeatVec, _ = model(image_input, mode='image')
for sketchFeatVec, sketch_target, sketch_index in processed_queries:
scores = score_fun(imageFeatVec, sketchFeatVec)
indices = torch.broadcast_to(sketch_index.unsqueeze(0),
scores.size())
target = image_target.unsqueeze(1) == sketch_target.unsqueeze(0)
mAP(scores, target, indices)
iter_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.test_print_freq == 0 and main_process():
logger.info(
'Test batch: [{}/{}] '
'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
'Iter {iter_time.val:.3f} ({iter_time.avg:.3f})'.format(
i + 1,
len(image_loader),
data_time=data_time,
iter_time=iter_time))
map_value = mAP.compute() # computes across all distributed processes
if main_process():
logger.info('Val result: mAP {:.4f}.'.format(map_value))
logger.info('<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<')
return map_value
def make_causal_mask(x: Tensor,
device: str = 'cuda:0',
dtype: torch.dtype = torch.float32) -> Tensor:
"""Make a causal mask for self-attention.
Args:
x: input array of shape `[batch..., len]`
dtype: mask return dtype
Returns:
A `[batch..., len, len]` shaped causal attention mask.
"""
idxs = torch.broadcast_to(
torch.arange(x.shape[-1], dtype=torch.int32, device=device), x.shape)
return make_attention_mask(idxs, idxs, torch.greater_equal, dtype=dtype)
def forward(ctx, weight, alpha, g, Qn, Qp, per_channel):
# assert alpha > 0, "alpha={}".format(alpha)
ctx.save_for_backward(weight, alpha)
ctx.other = g, Qn, Qp, per_channel
if per_channel:
sizes = weight.size()
weight = weight.contiguous().view(weight.size()[0], -1)
weight = torch.transpose(weight, 0, 1)
alpha = torch.broadcast_to(alpha, weight.size())
w_q = Round.apply(torch.div(weight, alpha)).clamp(Qn, Qp)
w_q = w_q * alpha
w_q = torch.transpose(w_q, 0, 1)
w_q = w_q.contiguous().view(sizes)
else:
w_q = Round.apply(torch.div(weight, alpha)).clamp(Qn, Qp)
w_q = w_q * alpha
return w_q
def sys_data_sampler(self, sys_data, Ndots_per_image):
u, images = sys_data.u, sys_data.y
#images has shape (Ns, C, H, W) for (Ns, H, W)
if len(images.shape)==4:
Ns, C, W, H = images.shape
elif len(images.shape)==3:
Ns, W, H = images.shape
C = None
else:
assert False, 'check images.shape'
sampleselector = torch.broadcast_to(torch.arange(Ns)[:,None],(Ns,Ndots_per_image))
h = np.random.randint(low=0, high=H, size=(Ns,Ndots_per_image))
w = np.random.randint(low=0, high=W, size=(Ns,Ndots_per_image))
images_shots = images[sampleselector,:,h,w] if C!=None else images[sampleselector,h,w] #what shape does this have? I hope it has (Ns, Nshots, C)
sys_data = System_data(u=u, y=images_shots, x=images)
sys_data.h, sys_data.w = h, w
return sys_data
def extract_into_tensor(arr, timesteps, broadcast_shape):
"""Extract values from a 1-D numpy array for a batch of indices.
Args:
arr: the 1-D numpy array.
timesteps: a tensor of indices into the array to extract.
broadcast_shape: a larger shape of K dimensions with the batch dimension
equal to the length of timesteps.
Returns:
res: a tensor of shape [batch_size, 1, ...] where the shape has K dims.
"""
device = timesteps.device
assert arr.device == device
res = arr[timesteps].float()
new_dims = [1] * (len(broadcast_shape) - res.ndim)
res = res.view(*res.shape, *new_dims)
return torch.broadcast_to(res, broadcast_shape)
def search(self, tgt, ctx, i=None):
cfg = self.cfg
unk = torch.equal(tgt, cfg.UNK)
prior = torch.one_hot(tgt, cfg.num_toks, 0.0, utils.big_neg)
if i is not None:
unk = unk[:, i]
prior = prior[:, i, :]
if torch.reduce_all(unk) is True:
logi = prior
else:
y = self.decode(tgt, ctx)
if i is not None:
y = y[:, i, :]
sh = y.shape # torch.int_shape(y)
y = torch.reshape(y, (-1, sh[-1]))
y = self.logits(y)
y = torch.reshape(y, sh[:-1] + y.shape[-1:])
u = torch.expand_dims(unk, axis=2)
u = torch.broadcast_to(u, y.shape)
logi = torch.where(u, y, prior)
logp = y - torch.reduce_logsumexp(y, axis=-1, keepdims=True)
return logp, logi, unk
def other_ops(self):
a = torch.randn(4)
b = torch.randn(4)
c = torch.randint(0, 8, (5, ), dtype=torch.int64)
e = torch.randn(4, 3)
f = torch.randn(4, 4, 4)
size = [0, 1]
dims = [0, 1]
return (
torch.atleast_1d(a),
torch.atleast_2d(a),
torch.atleast_3d(a),
torch.bincount(c),
torch.block_diag(a),
torch.broadcast_tensors(a),
torch.broadcast_to(a, (4)),
# torch.broadcast_shapes(a),
torch.bucketize(a, b),
torch.cartesian_prod(a),
torch.cdist(e, e),
torch.clone(a),
torch.combinations(a),
torch.corrcoef(a),
# torch.cov(a),
torch.cross(e, e),
torch.cummax(a, 0),
torch.cummin(a, 0),
torch.cumprod(a, 0),
torch.cumsum(a, 0),
torch.diag(a),
torch.diag_embed(a),
torch.diagflat(a),
torch.diagonal(e),
torch.diff(a),
torch.einsum("iii", f),
torch.flatten(a),
torch.flip(e, dims),
torch.fliplr(e),
torch.flipud(e),
torch.kron(a, b),
torch.rot90(e),
torch.gcd(c, c),
torch.histc(a),
torch.histogram(a),
torch.meshgrid(a),
torch.lcm(c, c),
torch.logcumsumexp(a, 0),
torch.ravel(a),
torch.renorm(e, 1, 0, 5),
torch.repeat_interleave(c),
torch.roll(a, 1, 0),
torch.searchsorted(a, b),
torch.tensordot(e, e),
torch.trace(e),
torch.tril(e),
torch.tril_indices(3, 3),
torch.triu(e),
torch.triu_indices(3, 3),
torch.vander(a),
torch.view_as_real(torch.randn(4, dtype=torch.cfloat)),
torch.view_as_complex(torch.randn(4, 2)),
torch.resolve_conj(a),
torch.resolve_neg(a),
)
def inference(self, images, thresh=0.5, nms_thresh=0.7, mode='RPN'):
"""
Inference-time forward pass for the Region Proposal Network.
Inputs:
- images: Tensor of shape (B, 3, H, W) giving input images
- thresh: Threshold value on confidence scores. Proposals with a predicted
object probability above thresh should be kept. HINT: You can convert the
object score to an object probability using a sigmoid nonlinearity.
- nms_thresh: IoU threshold for non-maximum suppression
- mode: One of 'RPN' or 'FasterRCNN' to determine the outputs.
The region proposal network can output a variable number of region proposals
per input image. We assume that the input image images[i] gives rise to
P_i final propsals after thresholding and NMS.
NOTE: NMS is performed independently per-image!
Outputs:
- final_proposals: List of length B, where final_proposals[i] is a Tensor
of shape (P_i, 4) giving the coordinates of the predicted region proposals
for the input image images[i].
- final_conf_probs: List of length B, where final_conf_probs[i] is a
Tensor of shape (P_i,) giving the predicted object probabilities for each
predicted region proposal for images[i]. Note that these are
*probabilities*, not scores, so they should be between 0 and 1.
- features: Tensor of shape (B, D, H', W') giving the image features
predicted by the backbone network for each element of images.
If mode is "RPN" then this is a dummy list of zeros instead.
"""
assert mode in ('RPN', 'FasterRCNN'), 'invalid inference mode!'
features, final_conf_probs, final_proposals = None, None, None
##############################################################################
# TODO: Predicting the RPN proposal coordinates `final_proposals` and #
# confidence scores `final_conf_probs`. #
# The overall steps are similar to the forward pass but now you do not need #
# to decide the activated nor negative anchors. #
# HINT: Threshold the conf_scores based on the threshold value `thresh`. #
# Then, apply NMS to the filtered proposals given the threshold `nms_thresh`.#
# HINT: Use `torch.no_grad` as context to speed up the computation. #
##############################################################################
# Replace "pass" statement with your code
batch_size = images.shape[0]
with torch.no_grad(
): # "no_grad" context used to speed up the computation.
# Image feature extraction (using the RPN).
features = self.feat_extractor(images)
# Grid and anchor generation.
grid = GenerateGrid(batch_size)
anchors = GenerateAnchor(self.anchor_list, grid)
B, A, Hp, Wp, _ = anchors.shape
# Pass 'features' through the RPN network (with "inference" mode).
conf_scores, offsets = self.prop_module(features)
# Reshape 'conf_scores' from (B, A, 2, H', W') to (B, A, H', W', 2)
conf_scores = torch.transpose(conf_scores, 2, 4)
# Reshape 'conf_scores' from (B, A, H', W', 2) to (B, A*H'*W', 2)
conf_scores = torch.flatten(conf_scores, start_dim=1, end_dim=-2)
# Transform 'conf_scores' into probabilities, by squashing them into (0,1) range.
conf_probs = torch.sigmoid(conf_scores)
# Retrieve only object's probability, we don't care about the 'background' prob.
# Now, 'conf_probs' will have a shape of (B, A*H'*W')
conf_probs = conf_probs[..., 0]
# Reshape 'offsets' from (B, A, 4, H', W') to (B, A, H', W', 4)
offsets = torch.transpose(offsets, 2, 4)
proposals = GenerateProposal(anchors, offsets, method='FasterRCNN')
# Reshape 'proposals' from (B, A, H', W', 4) to (B, A*H'*W', 4)
proposals = torch.flatten(proposals, start_dim=1, end_dim=-2)
final_conf_probs, final_proposals = [], []
for idx in range(batch_size):
# Get current image's proposals and conf_probs.
cr_proposal = proposals[idx] # Tensor's shape: (A*H'*W', 4)
cr_conf_probs = conf_probs[idx] # Tensor's shape: (A*H'*W',)
# Define a boolean mask which indicates indexes to delete.
del_idx_mask = ~(cr_conf_probs < thresh)
# Apply the mask on current proposals, conf_probs and class_indices.
cr_conf_probs = cr_conf_probs[del_idx_mask]
# Get the number of 'cr_conf_probs' that have been [K]ept.
K = cr_conf_probs.shape[0]
# Reshape 'del_idx_mask' from (A*H'*W',) to (A*H'*W', 1)
del_idx_mask = del_idx_mask.unsqueeze(1)
# Reshape via broadcasting 'del_idx_mask' from (A*H'*W', 1) to (A*H'*W', 4)
del_idx_mask = torch.broadcast_to(del_idx_mask, (A * Hp * Wp, 4))
# 'cr_proposal' will have a shape of (K, 4)
cr_proposal = cr_proposal[del_idx_mask].reshape(K, 4)
# Get indices of kept proposals (using NMS).
icr_proposal = torchvision.ops.nms(cr_proposal, cr_conf_probs,
nms_thresh)
final_conf_probs.append(cr_conf_probs[icr_proposal].unsqueeze(1))
final_proposals.append(cr_proposal[icr_proposal])
##############################################################################
# END OF YOUR CODE #
##############################################################################
if mode == 'RPN':
features = [torch.zeros_like(i)
for i in final_conf_probs] # dummy class
return final_proposals, final_conf_probs, features
def multivariate_normal_cdf(value,loc=0.0,covariance_matrix=None,diagonality_tolerance=0.0):
"""Compute orthant probabilities ``P(Z_i < value_i, i = 1,...,d)`` for a multivariate normal random vector Z.
Closed-form backward differentiation with respect to `value`, `loc` or `covariance_matrix` is supported.
Parameters
----------
value : torch.Tensor,
upper integration limits. It can have batch shape.
The last dimension must be equal to d, the dimension of the
Gaussian vector.
loc : torch.Tensor, optional
Mean of the Gaussian vector. Default is zeros. Can have batch
shape. Last dimension must be equal to d, the dimension of the
Gaussian vector. If a float is provided, the value is repeated
for all the d components.
covariance_matrix : torch.Tensor, optional
Covariance matrix of the Gaussian vector.
Can have batch shape. The two last dimensions must be equal
to d. Identity matrix by default.
diagonality_tolerance=0.0 : float, optional
Avoid expensive numerical integration if the maximum of all
off-diagonal values is below this tolerance (in absolute value),
as the covariance is considered diagonal. If there is a batch of
covariances (e.g. `covariance_matrix` has shape [N,d,d]), then
the numerical integrations are avoided only if *all* covariances
are considered diagonal. Diagonality check can be avoided with
a negative value.
Returns
-------
probability : torch.Tensor
The probability of the event ``Y < value``. Its shape is the
the broadcasted batch shape (just a scalar if the batchshape is []).
Closed form derivative are implemented if `value` `loc`,
`covariance_matrix` require a gradient.
Notes
-------
Parameters `value` and `covariance_matrix`, as
well as the returned probability tensor are broadcasted to their
common batch shape. See PyTorch' `broadcasting semantics
<https://pytorch.org/docs/stable/notes/broadcasting.html#broadcasting-semantics>`_.
The integration is performed with Scipy's impementation of A. Genz method [1]_.
Partial derivative are computed using closed form formula, see e.g. Marmin et al. [2]_, p 13.
References
----------
.. [1] Alan Genz and Frank Bretz, "Comparison of Methods for the Computation of Multivariate
t-Probabilities", Journal of Computational and Graphical Statistics 11, pp. 950-971, 2002. `Source code <http://www.math.wsu.edu/faculty/genz/software/fort77/mvtdstpack.f>`_.
.. [2] Sébastien Marmin, Clément Chevalier and David Ginsbourger, "Differentiating the multipoint Expected Improvement for optimal batch design", International Workshop on Machine learning, Optimization and big Data, Taormina, Italy, 2015. `PDF <https://hal.archives-ouvertes.fr/hal-01133220v4/document>`_.
Examples
--------
>>> import torch
>>> from torch.autograd import grad
>>> from mvnorm import multivariate_normal_cdf as Phi
>>> n = 4
>>> x = 1 + torch.randn(n)
>>> x.requires_grad = True
>>> # Make a positive semi-definite matrix
>>> A = torch.randn(n,n)
>>> C = 1/n*torch.matmul(A,A.t())
>>> p = Phi(x,covariance_matrix=C)
>>> p
tensor(0.3721, grad_fn=<PhiHighDimBackward>)
>>> grad(p,(x,))[0]
tensor([0.0085, 0.2510, 0.1272, 0.0332])
"""
m = loc-value # actually do P(Y-value<0)
m_shape = m.shape
d = m_shape[-1]
if covariance_matrix is None:
covariance_matrix = eye(d)
off_diag = -0.0
else:
if diagonality_tolerance>=0:
if d>=2:
off_diag = tril(covariance_matrix.detach(),diagonal = -1).abs().max()
else:
off_diag = -0.0
else: # diagonality check forbidden by user
off_diag = diagonality_tolerance + 1
if off_diag<=diagonality_tolerance: # assumed diagonal
D = diagonal(covariance_matrix,dim1 = -2, dim2 = -1)
z = -m/D.sqrt()
return PhiDiagonal(z)
cov_shape = covariance_matrix.shape[-2:]
if len(cov_shape) < 2:
raise ValueError("covariance_matrix must have at last " \
"two dimensions when not diagonal.")
if cov_shape[-2] != d or cov_shape[-1] != d:
raise ValueError("Covariance matrix must have the last two " \
"dimensions equal to d. Here it's "+str(list(cov_shape[-2:])))
batch_shape = broadcast_shape(m.shape[:-1],cov_shape[:-2])
vector_shape = batch_shape + [d]
matrix_shape = batch_shape + [d,d]
m_b = broadcast_to(m,vector_shape)
c_b = broadcast_to(covariance_matrix,matrix_shape)
return Phi(m_b,c_b)
("argsort", MF.argsort, torch.argsort, [(1000, )], [
(1000, 1000),
], True, 1000),
(
"avg_pool2d",
lambda x: MF.avg_pool2d(x, 2),
lambda x: TF.avg_pool2d(x, 2),
[(2, 32, 16, 16)],
[(64, 512, 16, 16)],
True,
1000,
),
(
"broadcast",
lambda x: MF.broadcast_to(x, (5, ) + x.shape),
lambda x: torch.broadcast_to(x, (5, ) + x.shape),
[(100, 100)],
[(64, 512, 16, 16)],
True,
1000,
),
(
"batchedmatmul",
MF.matmul,
torch.matmul,
[(8, 64, 32), (8, 32, 64)],
[(8, 2048, 512), (8, 512, 2048)],
True,
1000,
),
(
def _input_mask(input: Tensor, *args, **kwargs) -> Tensor:
"""Return canonical input mask.
A canonical input mask is defined as a boolean mask tensor that
shape and layout matches with the shape and the layout of the
input.
The canonical input mask is computed from the :attr:`mask` tensor
content to meet the following criteria:
1. The shape of the canonical input mask is the same as the shape
of :attr:`input` tensor. If the mask tensor has a smaller shape
than the shape of the :attr:`input`, broadcasting rules will be
applied. Downcasting of mask is not supported.
2. The layout of the canonical input mask is the same as the
layout of the :attr:`input` tensor. If the mask has different
layout, it will be converted to the expected layout. In the
case of sparse COO layout, the canonical input mask will be
coalesced.
3. The dtype of the canonical input mask is torch.bool. If the
mask dtype is not bool then it will be converted to bool dtype
using `.to(dtype=bool)` method call.
4. The elements of the canonical input mask have boolean values
copied from the content of the :attr:`mask` tensor (after
possible broadcasting and dtype conversion transforms). In
general, the sparsity pattern of the sparse canonical input
mask need not to be the same as the sparsity pattern of the
sparse :attr:`input` tensor.
"""
if input.layout not in {torch.strided, torch.sparse_coo}:
raise ValueError(
f'_input_mask expects strided or sparse COO tensor but got {input.layout}'
)
mask = kwargs.get('mask')
# default mask
if mask is None:
raise ValueError('_input_mask requires explicit mask')
# mask shape must match with input shape
if mask.shape != input.shape:
if mask.ndim > input.ndim:
raise IndexError(
"_input_mask expected broadcastable mask (got mask dimensionality higher than of the input)"
)
if mask.layout == torch.strided:
mask = torch.broadcast_to(mask.clone(),
input.shape).to(dtype=torch.bool)
else:
mask = torch._sparse_broadcast_to(mask, input.shape)
# mask layout must match with input layout
if mask.layout != input.layout:
if input.layout == torch.strided:
mask = mask.to_dense()
else:
mask = mask.to_sparse(input.sparse_dim())
# sparse mask must be coalesced
if mask.layout == torch.sparse_coo:
mask = mask.coalesce()
# mask is a boolean tensor
mask = mask.to(dtype=torch.bool)
return mask
def inference(self, images, thresh=0.5, nms_thresh=0.7):
""""
Inference-time forward pass for the single stage detector.
Inputs:
- images: Input images
- thresh: Threshold value on confidence scores
- nms_thresh: Threshold value on NMS
Outputs:
- final_propsals: Keeped proposals after confidence score thresholding and NMS,
a list of B (*x4) tensors
- final_conf_scores: Corresponding confidence scores, a list of B (*x1) tensors
- final_class: Corresponding class predictions, a list of B (*x1) tensors
"""
final_proposals, final_conf_scores, final_class = [], [], []
##############################################################################
# TODO: Predicting the final proposal coordinates `final_proposals`, #
# confidence scores `final_conf_scores`, and the class index `final_class`. #
# The overall steps are similar to the forward pass but now you do not need #
# to decide the activated nor negative anchors. #
# HINT: Thresholding the conf_scores based on the threshold value `thresh`. #
# Then, apply NMS (torchvision.ops.nms) to the filtered proposals given the #
# threshold `nms_thresh`. #
# The class index is determined by the class with the maximal probability. #
# Note that `final_propsals`, `final_conf_scores`, and `final_class` are all #
# lists of B 2-D tensors (you may need to unsqueeze dim=1 for the last two). #
##############################################################################
# Replace "pass" statement with your code
# Note that in the code below, there is a lot of tensor's reshapes.
# These operations are -unfortunately- needed to match different functions'
# input tensors shapes criteria.
batch_size = images.shape[0]
# Image feature extraction (using the backbone CNN network).
features = self.feat_extractor(images)
# Grid and anchor generation.
grid = GenerateGrid(batch_size)
anchors = GenerateAnchor(self.anchor_list, grid)
B, A, Hp, Wp, _ = anchors.shape
# Pass 'features' through the prediction network (with inference mode).
conf_scores, offsets, class_scores = self.pred_network(features)
# Reshape 'conf_scores' from (B, A, H', W') to (B, A*H'*W')
conf_scores = torch.flatten(conf_scores, start_dim=1)
# Reshape 'offsets' from (B, A, 4, H', W') to (B, A, H', W', 4)
offsets = torch.transpose(offsets, 2, 4)
# Get the indices of maximums within 'class_scores'.
# 'class_scores' has shape of (B, C, H', W')
# 'class_indices' (the output) has shape of (B, H', W')
class_indices = torch.max(class_scores, dim=1)[1]
# Reshape 'class_indices' from (B, H', W') to (B, 1, H', W')
class_indices = class_indices.unsqueeze(1)
# Reshape via broadcasting 'class_indices' from (B, 1, H', W') to (B, A, H', W')
class_indices = torch.broadcast_to(class_indices, (B, A, Hp, Wp))
# Reshape 'class_indices' from (B, A, H', W') to (B, A*H'*W')
class_indices = torch.flatten(class_indices, start_dim=1)
proposals = GenerateProposal(anchors, offsets, method='YOLO')
# Reshape 'proposals' from (B, A, H', W', 4) to (B, A*H'*W', 4)
proposals = torch.flatten(proposals, start_dim=1, end_dim=-2)
final_proposals, final_conf_scores, final_class = [], [], []
for idx in range(batch_size):
# Get current image's proposals, conf_scores and class_indices.
cr_proposal = proposals[idx] # Tensor's shape: (A*H'*W', 4)
cr_conf_scores = conf_scores[idx] # Tensor's shape: (A*H'*W',)
cr_classes = class_indices[idx] # Tensor's shape: (A*H'*W',)
# Define a boolean mask which indicates indexes to delete.
del_idx_mask = ~(cr_conf_scores < thresh)
# Apply the mask on current proposals, conf_scores and class_indices.
cr_conf_scores = cr_conf_scores[del_idx_mask]
# Get the number of 'cr_conf_scores' that have been [K]ept.
K = cr_conf_scores.shape[0]
# 'cr_classes' will have a shape of (K, 1)
cr_classes = cr_classes[del_idx_mask].unsqueeze(1)
# Reshape 'del_idx_mask' from (A*H'*W',) to (A*H'*W', 1)
del_idx_mask = del_idx_mask.unsqueeze(1)
# Reshape via broadcasting 'del_idx_mask' from (A*H'*W', 1) to (A*H'*W', 4)
del_idx_mask = torch.broadcast_to(del_idx_mask, (A * Hp * Wp, 4))
# 'cr_proposal' will have a shape of (K, 4)
cr_proposal = cr_proposal[del_idx_mask].reshape(K, 4)
# Get indices of kept proposals (using NMS).
icr_proposal = torchvision.ops.nms(cr_proposal, cr_conf_scores,
nms_thresh)
final_proposals.append(cr_proposal[icr_proposal])
final_conf_scores.append(cr_conf_scores[icr_proposal].unsqueeze(1))
final_class.append(cr_classes[icr_proposal])
##############################################################################
# END OF YOUR CODE #
##############################################################################
return final_proposals, final_conf_scores, final_class
def unconstrained_rational_quadratic_spline(inputs,
unnormalized_widths,
unnormalized_heights,
unnormalized_derivatives,
inverse=False,
tails='linear',
tail_bound=1.,
min_bin_width=DEFAULT_MIN_BIN_WIDTH,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
min_derivative=DEFAULT_MIN_DERIVATIVE):
inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
outside_interval_mask = ~inside_interval_mask
outputs = torch.zeros_like(inputs)
logabsdet = torch.zeros_like(inputs)
if tails == 'linear':
unnormalized_derivatives_ = F.pad(unnormalized_derivatives, pad=(1, 1))
constant = np.log(np.exp(1 - min_derivative) - 1)
unnormalized_derivatives_[..., 0] = constant
unnormalized_derivatives_[..., -1] = constant
outputs[outside_interval_mask] = inputs[outside_interval_mask]
logabsdet[outside_interval_mask] = 0
elif tails == 'circular':
unnormalized_derivatives_ = F.pad(unnormalized_derivatives, pad=(0, 1))
unnormalized_derivatives_[..., -1] = unnormalized_derivatives_[..., 0]
outputs[outside_interval_mask] = inputs[outside_interval_mask]
logabsdet[outside_interval_mask] = 0
elif isinstance(tails, list) or isinstance(tails, tuple):
unnormalized_derivatives_ = unnormalized_derivatives.clone()
ind_lin = [t == 'linear' for t in tails]
ind_circ = [t == 'circular' for t in tails]
constant = np.log(np.exp(1 - min_derivative) - 1)
unnormalized_derivatives_[..., ind_lin, 0] = constant
unnormalized_derivatives_[..., ind_lin, -1] = constant
unnormalized_derivatives_[..., ind_circ, -1] = unnormalized_derivatives_[..., ind_circ, 0]
else:
raise RuntimeError('{} tails are not implemented.'.format(tails))
if torch.is_tensor(tail_bound):
tail_bound_ = torch.broadcast_to(tail_bound, inputs.shape)
left = -tail_bound_[inside_interval_mask]
right = tail_bound_[inside_interval_mask]
bottom = -tail_bound_[inside_interval_mask]
top = tail_bound_[inside_interval_mask]
else:
left = -tail_bound
right = tail_bound
bottom = -tail_bound
top = tail_bound
outputs[inside_interval_mask], logabsdet[inside_interval_mask] = rational_quadratic_spline(
inputs=inputs[inside_interval_mask],
unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
unnormalized_derivatives=unnormalized_derivatives_[inside_interval_mask, :],
inverse=inverse,
left=left, right=right, bottom=bottom, top=top,
min_bin_width=min_bin_width,
min_bin_height=min_bin_height,
min_derivative=min_derivative
)
return outputs, logabsdet
pos.requires_grad = False
n_batches = K // 2
divpoints = array_split_divpoints_ntotal(K, n_batches)
z = a * th.eth(1j * th.angle(Psi_model))
z_tmp = th.zeros_like(z)
Lambda = th.zeros_like(z)
AtA = th.zeros(S_model.shape, dtype=th.float32)
u = S_model
u_tmp = th.zeros_like(u)
psi = psi_model.copy()
psi_tmp = th.zeros_like(psi)
psi_denom = th.zeros(psi.shape, dtype=th.float32)
psi_bc = th.broadcast_to(psi[None, ...], (K, MY, MX))
print(f"psi_bc norm: {th.norm(psi_bc)**2}")
AtA = Qoverlap_real(r, th.abs(psi_bc)**2, AtA)
plot(AtA.get(), 'AtA ptycho')
plotAbsAngle(z[0].get(), 'z[0] ptycho')
# init
u = At(z, psi, r, u)
plotAbsAngle(u[MY // 2:-MY // 2, MX // 2:-MX // 2].get(),
'At(z, psi, r, u) ptycho')
u /= AtA
plotAbsAngle(u[MY // 2:-MY // 2, MX // 2:-MX // 2].get(),
'At(z, psi, r, u)/AtA ptycho')
def sgn(x):
"""Signum (sign) function"""
def log_prob(self, z):
log_p_u = torch.broadcast_to(-torch.log(self.scale[self.ind]), (len(z), -1))
log_p_g = - 0.5 * np.log(2 * np.pi) - torch.log(self.scale[self.ind_]) \
- 0.5 * torch.pow(z[..., self.ind_] / self.scale[self.ind_], 2)
return torch.sum(log_p_u, -1) + torch.sum(log_p_g, -1)
def broadcast_to(x, new_shape):
return torch.broadcast_to(x, new_shape)
def log_prob(self, node_heights: torch.Tensor):
taxa_shape = node_heights.shape[:-1] + (int(
(node_heights.shape[-1] + 1) / 2), )
tip_heights = node_heights[..., :taxa_shape[-1]]
serially_sampled = torch.any(tip_heights > 0.0)
m = self.mu.shape[-1]
if self.times is None:
dtimes = (self.origin / m).expand(self.origin.shape[:-1] + (m, ))
times = torch.cat(
(torch.zeros(dtimes.shape[:-1] +
(1, ), dtype=dtimes.dtype), dtimes),
-1).cumsum(-1)
else:
times = self.times
times = torch.broadcast_to(times,
self.mu.shape[:-1] + times.shape[-1:])
if self.relative_times and self.times is not None:
times = times * self.origin
p, A, B = self.log_p(times[..., 1:], times[..., :-1])
# first term
e = torch.exp(-A[..., 0] * times[..., 1])
q0 = 4.0 * e / torch.pow(e * (1.0 - B[..., 0]) + (1.0 + B[..., 0]), 2)
log_p = torch.log(q0)
# condition on sampling at least one individual
if self.survival:
log_p -= torch.log(1.0 - p[..., 0])
# calculate l(x) with l(t)=1 iff t_{i-1} <= t < t_i
x = times[..., -1:] - node_heights[..., taxa_shape[-1]:]
indices_x = torch.max(times.unsqueeze(-2) >= x.unsqueeze(-1),
dim=-1)[1] - 1
log_p += (torch.log(self.lambda_.gather(-1, indices_x)) + self.log_q(
A.gather(-1, indices_x),
B.gather(-1, indices_x),
x,
torch.gather(times[..., 1:], -1, indices_x),
)).sum(-1)
y = times[..., -1:] - tip_heights
if serially_sampled:
indices_y = torch.max(times.unsqueeze(-2) >= y.unsqueeze(-1),
dim=-1)[1] - 1
log_p += (torch.log(self.psi.gather(-1, indices_y)) - self.log_q(
A.gather(-1, indices_y),
B.gather(-1, indices_y),
y,
times.gather(-1, indices_y + 1),
)).sum(-1)
# last term
if m > 1:
# number of degree 2 vertices
ni = (
torch.sum(x.unsqueeze(-2) < times[..., 1:].unsqueeze(-1), -1) -
torch.sum(y.unsqueeze(-2) <= times[..., 1:].unsqueeze(-1),
-1))[..., :-1] + 1.0
# contemporenaous term
log_p += (ni * (self.log_q(A[..., 1:], B[..., 1:],
times[..., 1:-1], times[..., 2:]) +
torch.log(1.0 - self.rho[..., :-1]))).sum(-1)
N = torch.sum(
times[..., 1:].unsqueeze(-2) == torch.unsqueeze(
times[..., -1:] - tip_heights, -1),
-2,
)
mask = (N > 0).logical_and(self.rho > 0.0)
if torch.any(mask):
p = torch.masked_select(N, mask) * torch.masked_select(
self.rho, mask).log()
log_p += p.squeeze() if log_p.dim() == 0 else p
return log_p
def reduce_indices(self):
return torch.broadcast_to(
torch.repeat_interleave(torch.arange(len(self.sizes)).to(self.device), torch.tensor(self.sizes).to(self.device)),
self.shape)
def broadcast_to(a: Numeric, *shape: Int):
return torch.broadcast_to(a, shape)
