def fcn_scoring_graph(input, config, mode):
in_heatmap, pr_scores = input
detections_per_image = pr_scores.shape[2]
rois_per_image = KB.int_shape(pr_scores)[2]
img_h, img_w = config.IMAGE_SHAPE[:2]
batch_size = config.BATCH_SIZE
num_classes = config.NUM_CLASSES
heatmap_scale = config.HEATMAP_SCALE_FACTOR
class_column = 4
score_column = 5
if mode == 'training':
sequence_column = 6
norm_score_column = 7
else:
dt_type_column = 6
sequence_column = 7
norm_score_column = 8
print('\n ')
print('----------------------')
print('>>> FCN Scoring Layer - mode:', mode)
print('----------------------')
logt('in_heatmap.shape ', in_heatmap)
logt('pr_hm_scores.shape', pr_scores)
# rois per image is determined by size of input tensor
# detection mode: config.TRAIN_ROIS_PER_IMAGE
# ground_truth : config.DETECTION_MAX_INSTANCES
print(' detctions_per_image : ', detections_per_image,
'pr_scores shape', pr_scores.shape)
print(' rois_per_image : ', rois_per_image)
print(' config.DETECTION_MAX_INSTANCES : ',
config.DETECTION_MAX_INSTANCES)
print(' config.DETECTIONS_PER_CLASS : ',
config.DETECTION_PER_CLASS)
print(' sequence_column : ', sequence_column)
print(' norm_score_column : ', norm_score_column)
##---------------------------------------------------------------------------------------------
## Stack non_zero bboxes from PR_SCORES into pt2_dense
##---------------------------------------------------------------------------------------------
# pt2_ind shape : [?, 3] : [ {image_index, class_index , roi row_index }]
# pt2_dense shape: [?, 11] :
# pt2_dense[0:3] roi coordinates
# pt2_dense[4] is class id
# pt2_dense[5] is score from mrcnn
# pt2_dense[6] is bbox sequence id
# pt2_dense[7] is normalized score (per class)
#-----------------------------------------------------------------------------
pt2_sum = tf.reduce_sum(tf.abs(pr_scores[:, :, :, :class_column]), axis=-1)
pt2_ind = tf.where(pt2_sum > 0)
pt2_dense = tf.gather_nd(pr_scores, pt2_ind)
logt('in_heatmap ', in_heatmap)
logt('pr_scores.shape ', pr_scores)
logt('pt2_sum shape ', pt2_sum)
logt('pt2_ind shape ', pt2_ind)
logt('pt2_dense shape ', pt2_dense)
##---------------------------------------------------------------------------------------------
## Build mean and convariance tensors for bounding boxes
##---------------------------------------------------------------------------------------------
# bboxes_scaled = tf.to_int32(tf.round(pt2_dense[...,0:4])) / heatmap_scale
bboxes_scaled = pt2_dense[..., 0:class_column] / heatmap_scale
width = bboxes_scaled[:, 3] - bboxes_scaled[:, 1] # x2 - x1
height = bboxes_scaled[:, 2] - bboxes_scaled[:, 0]
cx = bboxes_scaled[:, 1] + (width / 2.0)
cy = bboxes_scaled[:, 0] + (height / 2.0)
# means = tf.stack((cx,cy),axis = -1)
covar = tf.stack((width * 0.5, height * 0.5), axis=-1)
covar = tf.sqrt(covar)
##---------------------------------------------------------------------------------------------
## build indices and extract heatmaps corresponding to each bounding boxes' class id
##---------------------------------------------------------------------------------------------
hm_indices = tf.cast(pt2_ind[:, :2], dtype=tf.int32)
logt('hm_indices ', hm_indices)
pt2_heatmaps = tf.transpose(in_heatmap, [0, 3, 1, 2])
logt('pt2_heatmaps', pt2_heatmaps)
pt2_heatmaps = tf.gather_nd(pt2_heatmaps, hm_indices)
logt('pt2_heatmaps', pt2_heatmaps)
##--------------------------------------------------------------------------------------------
## (0) Generate scores using prob_grid and pt2_dense
##--------------------------------------------------------------------------------------------
old_style_scores = tf.map_fn(
build_hm_score_v2,
[pt2_heatmaps, bboxes_scaled, pt2_dense[:, norm_score_column]],
dtype=tf.float32,
swap_memory=True)
logt('old_style_scores', old_style_scores)
# old_style_scores = tf.scatter_nd(pt2_ind, old_style_scores,
# [batch_size, num_classes, rois_per_image, KB.int_shape(old_style_scores)[-1]],
# name = 'scores_scattered')
# print(' old_style_scores :', old_style_scores.get_shape(), KB.int_shape(old_style_scores))
##---------------------------------------------------------------------------------------------
## generate score based on gaussian using bounding box masks
##---------------------------------------------------------------------------------------------
alt_scores_1 = tf.map_fn(build_hm_score_v3, [pt2_heatmaps, cy, cx, covar],
dtype=tf.float32)
logt('alt_scores_1 ', alt_scores_1)
##---------------------------------------------------------------------------------------------
## Scatter back to per-class tensor / normalize by class
##---------------------------------------------------------------------------------------------
alt_scores_1_norm = tf.scatter_nd(
pt2_ind,
alt_scores_1, [
batch_size, num_classes, detections_per_image,
KB.int_shape(alt_scores_1)[-1]
],
name='alt_scores_1_norm')
logt('alt_scores_1_scattered', alt_scores_1_norm)
alt_scores_1_norm = normalize_scores(alt_scores_1_norm)
logt('alt_scores_1_norm(by_class)', alt_scores_1_norm)
alt_scores_1_norm = tf.gather_nd(alt_scores_1_norm, pt2_ind)
logt('alt_scores_1_norm(by_image)', alt_scores_1_norm)
##---------------------------------------------------------------------------------------------
## Normalize input heatmap normalization (per class) to calculate alt_score_2
##--------------------------------------------------------------------------------------------
print(
'\n Normalize heatmap within each class !-------------------------------------'
)
in_heatmap_norm = tf.transpose(in_heatmap, [0, 3, 1, 2])
print(' in_heatmap_norm : ', in_heatmap_norm.get_shape(),
'Keras tensor ', KB.is_keras_tensor(in_heatmap_norm))
## normalize in class
normalizer = tf.reduce_max(in_heatmap_norm, axis=[-2, -1], keepdims=True)
normalizer = tf.where(normalizer < 1.0e-15, tf.ones_like(normalizer),
normalizer)
in_heatmap_norm = in_heatmap_norm / normalizer
# gauss_heatmap_sum_normalized = gauss_heatmap_sum / normalizer
print(' normalizer shape : ', normalizer.shape)
print(' normalized heatmap : ', in_heatmap_norm.shape, ' Keras tensor ',
KB.is_keras_tensor(in_heatmap_norm))
##---------------------------------------------------------------------------------------------
## build indices and extract heatmaps corresponding to each bounding boxes' class id
## build alternative scores# based on normalized/sclaked clipped heatmap
##---------------------------------------------------------------------------------------------
hm_indices = tf.cast(pt2_ind[:, :2], dtype=tf.int32)
logt('hm_indices shape', hm_indices)
pt2_heatmaps = tf.gather_nd(in_heatmap_norm, hm_indices)
logt('pt2_heatmaps', pt2_heatmaps)
alt_scores_2 = tf.map_fn(build_hm_score_v3, [pt2_heatmaps, cy, cx, covar],
dtype=tf.float32)
logt('alt_scores_2', alt_scores_2)
alt_scores_2_norm = tf.scatter_nd(
pt2_ind,
alt_scores_2, [
batch_size, num_classes, rois_per_image,
KB.int_shape(alt_scores_2)[-1]
],
name='alt_scores_2')
logt('alt_scores_2(scattered)', alt_scores_2_norm)
alt_scores_2_norm = normalize_scores(alt_scores_2_norm)
logt('alt_scores_2_norm(by_class)', alt_scores_2_norm)
alt_scores_2_norm = tf.gather_nd(alt_scores_2_norm, pt2_ind)
logt('alt_scores_2_norm(by_image)', alt_scores_2_norm)
####################################################################################################################
##--------------------------------------------------------------------------------------------
## Append alt_scores_1, alt_scores_1_norm to yield fcn_scores_dense
##--------------------------------------------------------------------------------------------
fcn_scores_dense = tf.concat([
pt2_dense[:, :norm_score_column + 1], old_style_scores, alt_scores_1,
alt_scores_1_norm, alt_scores_2, alt_scores_2_norm
],
axis=-1,
name='fcn_scores_dense')
logt('fcn_scores_dense ', fcn_scores_dense)
##---------------------------------------------------------------------------------------------
## Scatter back to per-image tensor
##---------------------------------------------------------------------------------------------
seq_ids = tf.to_int32(rois_per_image - pt2_dense[:, sequence_column])
scatter_ind = tf.stack([hm_indices[:, 0], seq_ids],
axis=-1,
name='scatter_ind')
fcn_scores_by_class = tf.scatter_nd(
pt2_ind,
fcn_scores_dense, [
batch_size, num_classes, detections_per_image,
fcn_scores_dense.shape[-1]
],
name='fcn_hm_scores')
# fcn_scores_by_image = tf.scatter_nd(scatter_ind, fcn_scores_dense,
# [batch_size, detections_per_image, fcn_scores_dense.shape[-1]], name='fcn_hm_scores_by_image')
logt('seq_ids ', seq_ids)
logt('sscatter_ids ', scatter_ind)
logt('fcn_scores_by_class ', fcn_scores_by_class)
# logt('fcn_scores_by_image ', fcn_scores_by_image)
logt('complete')
return fcn_scores_by_class
def build_heatmap_inference(in_tensor, config, names = None):
'''
input:
-------
pred_tensor: [ Bsz, Num_Classes, 200, 9 : {y1,x1,y2,x2, class, score, det_type, sequence_id, normalized_score}]
output:
-------
pr_heatmap (None, Heatmap-height, Heatmap_width, num_classes)
pr_scores (None, num_classes, 200, 24)
[batchSz, Detection_Max_instance, (y1,x1,y2,x2, class, score, det_type, sequence_id, normalized_score,
scores-0: gaussian_sum, bbox_area, weighted_norm_sum
scores-1: score, mask_sum, score/mask_sum, (score, mask_sum, score/mask_sum) normalized by class
scores-2: score, mask_sum, score/mask_sum, (score, mask_sum, score/mask_sum) normalized by class ]
'''
verbose = config.VERBOSE
num_detections = config.DETECTION_MAX_INSTANCES
img_h, img_w = config.IMAGE_SHAPE[:2]
batch_size = config.BATCH_SIZE
num_classes = config.NUM_CLASSES
heatmap_scale = config.HEATMAP_SCALE_FACTOR
grid_h, grid_w = config.IMAGE_SHAPE[:2] // heatmap_scale
# rois_per_image = config.DETECTION_PER_CLASS
rois_per_image = (in_tensor.shape)[2]
CLASS_COLUMN = 4
SCORE_COLUMN = 5
DT_TYPE_COLUMN = 6
SEQUENCE_COLUMN = 7
NORM_SCORE_COLUMN = 8
if verbose:
print('\n ')
print(' > build_inference_heatmap() for ', names )
print(' in_tensor shape : ', in_tensor.shape)
print(' num bboxes per class : ', rois_per_image )
print(' heatmap scale : ', heatmap_scale, 'Dimensions: w:', grid_w,' h:', grid_h)
##-----------------------------------------------------------------------------
## Stack non_zero bboxes from in_tensor into pt2_dense
##-----------------------------------------------------------------------------
# pt2_ind shape is [?, 3]. pt2_dense shape is [?, 7]
# pt2_ind[0] corresponds to image_index pt2_dense[0:3] roi coordinates
# pt2_ind[1] corresponds to class_index pt2_dense[4] class id
# pt2_ind[2] corresponds to roi row_index pt2_dense[5] score from mrcnn
# pt2_dense[6] bbox sequence id
# pt2_dense[7] per-class normalized score
#-----------------------------------------------------------------------------
pt2_sum = tf.reduce_sum(tf.abs(in_tensor[:,:,:,:4]), axis=-1)
pt2_ind = tf.where(pt2_sum > 0)
pt2_dense = tf.gather_nd( in_tensor, pt2_ind)
logt('pt2_sum ', pt2_sum, verbose = verbose)
logt('pt2_ind ', pt2_ind, verbose = verbose)
logt('pt2_dense ', pt2_dense, verbose = verbose)
##-----------------------------------------------------------------------------
## Build mesh-grid to hold pixel coordinates
##-----------------------------------------------------------------------------
X = tf.range(grid_w, dtype=tf.int32)
Y = tf.range(grid_h, dtype=tf.int32)
X, Y = tf.meshgrid(X, Y)
# duplicate (repeat) X and Y into a batch_size x rois_per_image tensor
ones = tf.ones([tf.shape(pt2_dense)[0] , 1, 1], dtype = tf.int32)
rep_X = ones * X
rep_Y = ones * Y
if verbose:
print(' X/Y shapes :', X.get_shape(), Y.get_shape())
print(' Ones: ', ones.shape)
print(' ones_exp * X', ones.shape, '*', X.shape, '= ',rep_X.shape)
print(' ones_exp * Y', ones.shape, '*', Y.shape, '= ',rep_Y.shape)
# # stack the X and Y grids
pos_grid = tf.to_float(tf.stack([rep_X,rep_Y], axis = -1))
logt('pos_grid before transpse ', pos_grid, verbose = verbose)
pos_grid = tf.transpose(pos_grid,[1,2,0,3])
logt('pos_grid after transpose ', pos_grid, verbose = verbose)
##-----------------------------------------------------------------------------
## Build mean and convariance tensors for Multivariate Normal Distribution
##-----------------------------------------------------------------------------
bboxes_scaled = pt2_dense[:,:4]/heatmap_scale
width = bboxes_scaled[:,3] - bboxes_scaled[:,1] # x2 - x1
height = bboxes_scaled[:,2] - bboxes_scaled[:,0]
cx = bboxes_scaled[:,1] + ( width / 2.0)
cy = bboxes_scaled[:,0] + ( height / 2.0)
means = tf.stack((cx,cy),axis = -1)
covar = tf.stack((width * 0.5 , height * 0.5), axis = -1)
covar = tf.sqrt(covar)
## Added 2019-05-12 to prevent NaN when bounding box is extremely small
## resulting in width or height being equal to zero
covar = tf.where(covar < 1.0e-15, tf.ones_like(covar), covar)
##-----------------------------------------------------------------------------
## Compute Normal Distribution for bounding boxes
##-----------------------------------------------------------------------------
tfd = tf.contrib.distributions
mvn = tfd.MultivariateNormalDiag(loc = means, scale_diag = covar)
prob_grid = mvn.prob(pos_grid)
logt('Input to MVN.PROB: pos_grid (meshgrid) ', pos_grid, verbose = verbose)
logt('Prob_grid shape from mvn.probe ',prob_grid, verbose = verbose)
prob_grid = tf.transpose(prob_grid,[2,0,1])
logt('Prob_grid shape after tanspose ', prob_grid, verbose = verbose)
logt('Output probabilities shape ' , prob_grid, verbose = verbose)
##--------------------------------------------------------------------------------------------
## (0) Generate scores using prob_grid and pt2_dense - (NEW METHOD added 09-21-2018)
##--------------------------------------------------------------------------------------------
old_style_scores = tf.map_fn(build_hm_score_v2, [prob_grid, bboxes_scaled, pt2_dense[ :, NORM_SCORE_COLUMN ] ],
dtype = tf.float32, swap_memory = True)
old_style_scores = tf.scatter_nd(pt2_ind, old_style_scores,
[batch_size, num_classes, rois_per_image, KB.int_shape(old_style_scores)[-1]],
name = 'scores_scattered')
logt('old_style_scores :', old_style_scores, verbose = verbose)
##----------------------------------------------------------------------------------------------------
## Generate scores using same method as FCN, over the prob_grid
## using (prob_grid_clipped) as input is superfluous == RETURNS EXACT SAME Results AS prob_grid above
##----------------------------------------------------------------------------------------------------
# alt_scores_0 = tf.map_fn(build_hm_score_v3, [prob_grid, cy, cx,covar], dtype=tf.float32)
# print(' alt_scores_0 : ', KB.int_shape(alt_scores_0), ' Keras tensor ', KB.is_keras_tensor(alt_scores_0) )
# alt_scores_0 = tf.scatter_nd(pt2_ind, alt_scores_0,
# [batch_size, num_classes, rois_per_image, KB.int_shape(alt_scores_0)[-1]], name = 'alt_scores_0')
##---------------------------------------------------------------------------------------------
## (NEW STEP - Clipped heatmaps)
## (1) Clip heatmap to region surrounding Cy,Cx and Covar X, Y
## Similar ro what is being done for gt_heatmap in CHMLayerTarget
##---------------------------------------------------------------------------------------------
prob_grid_clipped = tf.map_fn(clip_heatmap, [prob_grid, cy,cx, covar], dtype = tf.float32, swap_memory = True)
logt(' prob_grid_clipped : ', prob_grid_clipped, verbose = verbose)
##---------------------------------------------------------------------------------------------
## (2) apply normalization per bbox heatmap instance --> move to [0,1] range
##---------------------------------------------------------------------------------------------
logt('\n normalization ------------------------------------------------------', verbose = verbose)
normalizer = tf.reduce_max(prob_grid_clipped, axis=[-2,-1], keepdims = True)
normalizer = tf.where(normalizer < 1.0e-15, tf.ones_like(normalizer), normalizer)
logt(' normalizer : ', normalizer, verbose = verbose)
prob_grid_cns = prob_grid_clipped / normalizer
logt(' prob_grid_cns: clipped/normed/scaled : ', prob_grid_cns, verbose = verbose)
## replace above lines with lines below
## x_max = tf.reduce_max(prob_grid_clipped, axis=[-2,-1], keepdims = True)
## x_min = tf.reduce_min(prob_grid_clipped, axis=[-2,-1], keepdims = True)
##logt(' Reduce Max Shape: ', x_max, verbose = verbose)
##logt(' Reduce Min Shape: ', x_min, verbose = verbose)
## prob_grid_cns = (prob_grid_clipped - x_min) / (x_max - x_min)
## logt(' prob_grid_cns: clipped/normed/scaled : ', prob_grid_cns, verbose = verbose)
##---------------------------------------------------------------------------------------------
## (3) multiply normalized heatmap by normalized score in in_tensor/ (pt2_dense NORM_SCORE_COLUMN)
## broadcasting : https://stackoverflow.com/questions/49705831/automatic-broadcasting-in-tensorflow
##---------------------------------------------------------------------------------------------
prob_grid_cns = tf.transpose(tf.transpose(prob_grid_cns) * pt2_dense[ :, NORM_SCORE_COLUMN ])
logt(' prob_grid_cns: clipped/normed/scaled : ', prob_grid_cns, verbose = verbose)
##---------------------------------------------------------------------------------------------
## - Build alternative scores based on normalized/scaled/clipped heatmap
##---------------------------------------------------------------------------------------------
alt_scores_1 = tf.map_fn(build_hm_score_v3, [prob_grid_cns, cy, cx,covar], dtype=tf.float32)
logt('alt_scores_1 ', alt_scores_1, verbose = verbose)
alt_scores_1 = tf.scatter_nd(pt2_ind, alt_scores_1,
[batch_size, num_classes, rois_per_image, KB.int_shape(alt_scores_1)[-1]], name = 'alt_scores_1')
logt('alt_scores_1(by class) ', alt_scores_1, verbose = verbose)
alt_scores_1_norm = normalize_scores(alt_scores_1)
logt('alt_scores_1_norm(by_class) ', alt_scores_1_norm, verbose = verbose)
# alt_scores_1_norm = tf.gather_nd(alt_scores_1_norm, pt2_ind)
# print(' alt_scores_1_norm(by_image) : ', alt_scores_1_norm.shape, KB.int_shape(alt_scores_1_norm))
##-------------------------------------------------------------------------------------
## (3) scatter out the probability distributions based on class
##-------------------------------------------------------------------------------------
gauss_heatmap = tf.scatter_nd(pt2_ind, prob_grid_cns,
[batch_size, num_classes, rois_per_image, grid_w, grid_h], name = 'gauss_scatter')
logt('\n Scatter out the probability distributions based on class --------------', verbose = verbose)
logt('pt2_ind shape ', pt2_ind , verbose = verbose)
logt('prob_grid_clippped ', prob_grid_cns, verbose = verbose)
logt('gauss_heatmap ', gauss_heatmap, verbose = verbose) # batch_sz , num_classes, num_rois, image_h, image_w
##-------------------------------------------------------------------------------------
## Construction of Gaussian Heatmap output using Reduce SUM
##
## (4) SUM : Reduce and sum up gauss_heatmaps by class
## (5) heatmap normalization (per class)
## (6) Transpose heatmap to shape required for FCN
##-------------------------------------------------------------------------------------
gauss_heatmap_sum = tf.reduce_sum(gauss_heatmap, axis=2, name='gauss_heatmap_sum')
logt('\n Reduce SUM based on class and normalize within each class -----------------------', verbose = verbose)
logt('gaussian_heatmap_sum ', gauss_heatmap_sum , verbose = verbose)
## normalize in class
normalizer = tf.reduce_max(gauss_heatmap_sum, axis=[-2,-1], keepdims = True)
normalizer = tf.where(normalizer < 1.0e-15, tf.ones_like(normalizer), normalizer)
gauss_heatmap_sum = gauss_heatmap_sum / normalizer
logt('normalizer shape : ', normalizer, verbose = verbose)
logt('normalized heatmap : ', gauss_heatmap_sum, verbose = verbose)
## replaced above with following two lines::: 5-30-19
## gauss_heatmap_sum = tf.transpose(gauss_heatmap_sum, [0,2,3,1])
## gauss_heatmap_sum = normalize_heatmaps(gauss_heatmap_sum)
## logt('normalized heatmap : ', gauss_heatmap_sum, verbose = verbose)
##---------------------------------------------------------------------------------------------
## Score on reduced sum heatmaps.
##
## build indices and extract heatmaps corresponding to each bounding boxes' class id
## build alternative scores# based on normalized/sclaked clipped heatmap
##---------------------------------------------------------------------------------------------
hm_indices = tf.cast(pt2_ind[:, :2],dtype=tf.int32)
logt('hm_indices ', hm_indices, verbose = verbose)
pt2_heatmaps = tf.gather_nd(gauss_heatmap_sum, hm_indices )
## added5-30-2019 to replace above line
## pt2_heatmaps = tf.transpose(gauss_heatmap_sum, [0,3,1,2])
## pt2_heatmaps = tf.gather_nd(pt2_heatmaps, hm_indices )
logt('pt2_heatmaps ', pt2_heatmaps, verbose = verbose)
alt_scores_2 = tf.map_fn(build_hm_score_v3, [pt2_heatmaps, cy, cx,covar], dtype=tf.float32)
logt(' alt_scores_2 : ', alt_scores_2, verbose = verbose)
alt_scores_2 = tf.scatter_nd(pt2_ind, alt_scores_2,
[batch_size, num_classes, rois_per_image, KB.int_shape(alt_scores_2)[-1]], name = 'alt_scores_2')
logt('alt_scores_2(scattered) ', alt_scores_2, verbose = verbose)
alt_scores_2_norm = normalize_scores(alt_scores_2)
logt('alt_scores_2_norm(by_class) ', alt_scores_2_norm, verbose = verbose)
##---------------------------------------------------------------------------------------------
## (6) Transpose heatmaps to shape required for FCN [batchsize , width, height, num_classes]
##---------------------------------------------------------------------------------------------
gauss_heatmap_sum = tf.transpose(gauss_heatmap_sum ,[0,2,3,1], name = names[0])
logt(' gauss_heatmap_sum (final) ', gauss_heatmap_sum, verbose = verbose)
# gauss_heatmap_sum_normalized = tf.transpose(gauss_heatmap_sum_normalized,[0,2,3,1], name = names[0]+'_norm')
# print(' reshaped heatmap normalized : ', gauss_heatmap_sum_normalized.shape,' Keras tensor ', KB.is_keras_tensor(gauss_heatmap_sum_normalized) )
# gauss_heatmap_max = tf.transpose(gauss_heatmap_max ,[0,2,3,1], name = names[0]+'_max')
# print(' reshaped heatmap_max : ', gauss_heatmap_max.shape,' Keras tensor ', KB.is_keras_tensor(gauss_heatmap_max) )
# gauss_heatmap_max_normalized = tf.transpose(gauss_heatmap_max_normalized,[0,2,3,1], name = names[0]+'_max_norm')
# print(' reshaped heatmap_max normalized: ', gauss_heatmap_max_normalized.shape,' Keras tensor ', KB.is_keras_tensor(gauss_heatmap_max_normalized) )
##--------------------------------------------------------------------------------------------
## APPEND ALL SCORES TO input score tensor TO YIELD output scores tensor
##--------------------------------------------------------------------------------------------
gauss_scores = tf.concat([in_tensor, old_style_scores, alt_scores_1, alt_scores_1_norm, alt_scores_2, alt_scores_2_norm],
axis = -1,name = names[0]+'_scores')
logt(' gauss_scores : ', gauss_scores, verbose = verbose)
logt(' complete', verbose = verbose)
return gauss_heatmap_sum, gauss_scores
def build_gt_heatmap(in_tensor, config, names=None):
verbose = config.VERBOSE
num_detections = config.DETECTION_MAX_INSTANCES
img_h, img_w = config.IMAGE_SHAPE[:2]
batch_size = config.BATCH_SIZE
num_classes = config.NUM_CLASSES
heatmap_scale = config.HEATMAP_SCALE_FACTOR
grid_h, grid_w = config.IMAGE_SHAPE[:2] // heatmap_scale
# rois per image is determined by size of input tensor
# detection mode: config.TRAIN_ROIS_PER_IMAGE
# ground_truth : config.DETECTION_MAX_INSTANCES
# strt_cls = 0 if rois_per_image == 32 else 1
# rois_per_image = config.DETECTION_PER_CLASS
rois_per_image = (in_tensor.shape)[2]
if verbose:
print('\n ')
print(' > build_heatmap() for ', names)
print(' in_tensor shape : ', in_tensor.shape)
print(' num bboxes per class : ', rois_per_image)
print(' heatmap scale : ', heatmap_scale, 'Dimensions: w:',
grid_w, ' h:', grid_h)
##-----------------------------------------------------------------------------
## Stack non_zero bboxes from in_tensor into pt2_dense
##-----------------------------------------------------------------------------
# pt2_ind shape is [?, 3].
# pt2_ind[0] corresponds to image_index
# pt2_ind[1] corresponds to class_index
# pt2_ind[2] corresponds to roi row_index
# pt2_dense shape is [?, 7]
# pt2_dense[0:3] roi coordinates
# pt2_dense[4] is class id
# pt2_dense[5] is score from mrcnn
# pt2_dense[6] is bbox sequence id
# pt2_dense[7] is normalized score (per class)
#-----------------------------------------------------------------------------
pt2_sum = tf.reduce_sum(tf.abs(in_tensor[:, :, :, :4]), axis=-1)
pt2_ind = tf.where(pt2_sum > 0)
pt2_dense = tf.gather_nd(in_tensor, pt2_ind)
logt('pt2_sum ', pt2_sum, verbose=verbose)
logt('pt2_ind ', pt2_ind, verbose=verbose)
logt('pt2_dense ', pt2_dense, verbose=verbose)
##-----------------------------------------------------------------------------
## Build mesh-grid to hold pixel coordinates
##-----------------------------------------------------------------------------
# X = tf.range(grid_w, dtype=tf.int32)
# Y = tf.range(grid_h, dtype=tf.int32)
# X, Y = tf.meshgrid(X, Y)
# duplicate (repeat) X and Y into a batch_size x rois_per_image tensor
# print(' X/Y shapes :', X.get_shape(), Y.get_shape())
# ones = tf.ones([tf.shape(pt2_dense)[0] , 1, 1], dtype = tf.int32)
# rep_X = ones * X
# rep_Y = ones * Y
# print(' Ones: ', ones.shape)
# print(' ones_exp * X', ones.shape, '*', X.shape, '= ',rep_X.shape)
# print(' ones_exp * Y', ones.shape, '*', Y.shape, '= ',rep_Y.shape)
# # stack the X and Y grids
# pos_grid = tf.to_float(tf.stack([rep_X,rep_Y], axis = -1))
# print(' pos_grid before transpose : ', pos_grid.get_shape())
# pos_grid = tf.transpose(pos_grid,[1,2,0,3])
# print(' pos_grid after transpose : ', pos_grid.get_shape())
##-----------------------------------------------------------------------------
## Build mean and convariance tensors for Multivariate Normal Distribution
##-----------------------------------------------------------------------------
pt2_dense_scaled = pt2_dense[:, :4] / heatmap_scale
width = pt2_dense_scaled[:, 3] - pt2_dense_scaled[:, 1] # x2 - x1
height = pt2_dense_scaled[:, 2] - pt2_dense_scaled[:, 0]
cx = pt2_dense_scaled[:, 1] + (width / 2.0)
cy = pt2_dense_scaled[:, 0] + (height / 2.0)
means = tf.stack((cx, cy), axis=-1)
covar = tf.stack((width * 0.5, height * 0.5), axis=-1)
covar = tf.sqrt(covar)
##-----------------------------------------------------------------------------
## Compute Normal Distribution for bounding boxes
##-----------------------------------------------------------------------------
prob_grid = tf.ones([tf.shape(pt2_dense)[0], grid_h, grid_w],
dtype=tf.float32)
logt('Prob_grid ', prob_grid, verbose=verbose)
# tfd = tf.contrib.distributions
# mvn = tfd.MultivariateNormalDiag(loc = means, scale_diag = covar)
# prob_grid = mvn.prob(pos_grid)
# print(' >> input to MVN.PROB: pos_grid (meshgrid) shape: ', pos_grid.shape)
# print(' box_dims: ', box_dims.shape)
# print(' Prob_grid shape from mvn.probe: ', prob_grid.shape)
# prob_grid = tf.transpose(prob_grid,[2,0,1])
# print(' Prob_grid shape after tanspose: ', prob_grid.shape)
# print(' << output probabilities shape : ', prob_grid.shape)
#--------------------------------------------------------------------------------
# Kill distributions of NaN boxes (resulting from bboxes with height/width of zero
# which cause singular sigma cov matrices
#--------------------------------------------------------------------------------
# prob_grid = tf.where(tf.is_nan(prob_grid), tf.zeros_like(prob_grid), prob_grid)
#---------------------------------------------------------------------------------------------
# (1) apply normalization per bbox heatmap instance
#---------------------------------------------------------------------------------------------
# print('\n normalization ------------------------------------------------------')
# normalizer = tf.reduce_max(prob_grid, axis=[-2,-1], keepdims = True)
# normalizer = tf.where(normalizer < 1.0e-15, tf.ones_like(normalizer), normalizer)
# print(' normalizer : ', normalizer.shape)
# prob_grid_norm = prob_grid / normalizer
#---------------------------------------------------------------------------------------------
# (2) multiply normalized heatmap by normalized score in i n_tensor/ (pt2_dense column 7)
# broadcasting : https://stackoverflow.com/questions/49705831/automatic-broadcasting-in-tensorflow
#---------------------------------------------------------------------------------------------
# prob_grid_norm_scaled = tf.transpose(tf.transpose(prob_grid_norm) * pt2_dense[:,7])
# print(' prob_grid_norm_scaled : ', prob_grid_norm_scaled.shape)
##---------------------------------------------------------------------------------------------
## (NEW STEP) Clip heatmap to region surrounding Cy,Cx and Covar X, Y
##---------------------------------------------------------------------------------------------
prob_grid_clipped = tf.map_fn(clip_heatmap, [prob_grid, cy, cx, covar],
dtype=tf.float32,
swap_memory=True)
logt('prob_grid_clipped ', prob_grid_clipped, verbose=verbose)
##--------------------------------------------------------------------------------------------
## (0) Generate scores using prob_grid and pt2_dense - (NEW METHOD added 09-21-2018)
## pt2_dense[:,7] is the per-class-normalized score from in_tensor
##
## 11-27-2018: (note - here, build_hm_score_v2 is being applied to prob_grid_clipped,
## unlilke chm_layer) - Changed to prob_grid to make it consistent with chm_layer.py
##
## When using prob_grid:
## [ 1.0000 1.0000 138.0000 1.0000 4615.0000 4531.1250 4615.0000
## [ 3.0000 1.0000 179.0000 1.0000 570.0000 547.5000 570.0000
##
## When using prob_grid_clipped:
## [ 1.0000 1.0000 138.0000 1.0000 144.0000 4531.1250 144.0000
## [ 3.0000 1.0000 179.0000 1.0000 56.0000 547.5000 56.0000
##--------------------------------------------------------------------------------------------
old_style_scores = tf.map_fn(
build_hm_score_v2, [prob_grid, pt2_dense_scaled, pt2_dense[:, 7]],
dtype=tf.float32,
swap_memory=True)
old_style_scores = tf.scatter_nd(
pt2_ind,
old_style_scores, [batch_size, num_classes, rois_per_image, 3],
name='scores_scattered')
logt('old_style_scores ', old_style_scores, verbose=verbose)
##---------------------------------------------------------------------------------------------
## - Build alternative scores based on normalized/scaled/clipped heatmap
##---------------------------------------------------------------------------------------------
alt_scores_1 = tf.map_fn(build_hm_score_v3,
[prob_grid_clipped, cy, cx, covar],
dtype=tf.float32)
logt('alt_scores_1 ', alt_scores_1, verbose=verbose)
alt_scores_1 = tf.scatter_nd(pt2_ind,
alt_scores_1, [
batch_size, num_classes, rois_per_image,
KB.int_shape(alt_scores_1)[-1]
],
name='alt_scores_1')
alt_scores_1_norm = normalize_scores(alt_scores_1)
logt('alt_scores_1(by class) ', alt_scores_1, verbose=verbose)
logt('alt_scores_1_norm(by_class) ', alt_scores_1_norm, verbose=verbose)
##-------------------------------------------------------------------------------------
## (3) scatter out the probability distribution heatmaps based on class
##-------------------------------------------------------------------------------------
gauss_heatmap = tf.scatter_nd(
pt2_ind,
prob_grid_clipped,
[batch_size, num_classes, rois_per_image, grid_w, grid_h],
name='gauss_heatmap')
logt(
'\n Scatter out the probability distributions based on class --------------'
)
logt('pt2_ind ', pt2_ind, verbose=verbose)
logt('prob_grid ', prob_grid, verbose=verbose)
logt('gauss_heatmap ', gauss_heatmap,
verbose=verbose) # batch_sz , num_classes, num_rois, image_h, image_w
##-------------------------------------------------------------------------------------
## (4) MAX : Reduce_MAX up gauss_heatmaps by class
## Since all values are set to '1' in the 'heatmap', there is no need to
## sum or normalize. We Reduce_max on the class axis, and as a result the
## correspoding areas in the heatmap are set to '1'
##-------------------------------------------------------------------------------------
gauss_heatmap = tf.reduce_max(gauss_heatmap, axis=2, name='gauss_heatmap')
logt(
'\n Reduce MAX based on class -------------------------------------',
verbose=verbose)
logt(' gaussian_heatmap : ', gauss_heatmap, verbose=verbose)
#---------------------------------------------------------------------------------------------
# (5) heatmap normalization
# normalizer is set to one when the max of class is zero
# this prevents elements of gauss_heatmap_norm computing to nan
#---------------------------------------------------------------------------------------------
# print('\n normalization ------------------------------------------------------')
# normalizer = tf.reduce_max(gauss_heatmap, axis=[-2,-1], keepdims = True)
# normalizer = tf.where(normalizer < 1.0e-15, tf.ones_like(normalizer), normalizer)
# gauss_heatmap_norm = gauss_heatmap / normalizer
# print(' normalizer shape : ', normalizer.shape)
# print(' gauss norm : ', gauss_heatmap_norm.shape ,' Keras tensor ', KB.is_keras_tensor(gauss_heatmap_norm) )
##---------------------------------------------------------------------------------------------
## build indices and extract heatmaps corresponding to each bounding boxes' class id
## build alternative scores# based on normalized/sclaked clipped heatmap
##---------------------------------------------------------------------------------------------
hm_indices = tf.cast(pt2_ind[:, :2], dtype=tf.int32)
pt2_heatmaps = tf.gather_nd(gauss_heatmap, hm_indices)
logt('hm_indices ', hm_indices, verbose=verbose)
logt('pt2_heatmaps ', pt2_heatmaps, verbose=verbose)
alt_scores_2 = tf.map_fn(build_hm_score_v3, [pt2_heatmaps, cy, cx, covar],
dtype=tf.float32)
logt('alt_scores_2 ', alt_scores_2, verbose=verbose)
alt_scores_2 = tf.scatter_nd(pt2_ind,
alt_scores_2, [
batch_size, num_classes, rois_per_image,
KB.int_shape(alt_scores_2)[-1]
],
name='alt_scores_2')
alt_scores_2_norm = normalize_scores(alt_scores_2)
logt('alt_scores_2(by class) : ', alt_scores_2, verbose=verbose)
logt('alt_scores_2_norm(by_class) : ', alt_scores_2_norm, verbose=verbose)
##--------------------------------------------------------------------------------------------
## Transpose tensor to [BatchSz, Height, Width, Num_Classes]
##--------------------------------------------------------------------------------------------
gauss_heatmap = tf.transpose(gauss_heatmap, [0, 2, 3, 1], name=names[0])
# gauss_heatmap_norm = tf.transpose(gauss_heatmap_norm,[0,2,3,1], name = names[0]+'_norm')
# print(' gauss_heatmap_norm : ', gauss_heatmap_norm.shape,' Keras tensor ', KB.is_keras_tensor(gauss_heatmap_norm) )
# print(' complete')
##--------------------------------------------------------------------------------------------
## APPEND ALL SCORES TO input score tensor TO YIELD output scores tensor
##--------------------------------------------------------------------------------------------
gauss_scores = tf.concat([
in_tensor, old_style_scores, alt_scores_1, alt_scores_1_norm,
alt_scores_2, alt_scores_2_norm
],
axis=-1,
name=names[0] + '_scores')
# alt_scores_2[...,:3], alt_scores_3],
logt('gauss_heatmap ', gauss_heatmap, verbose=verbose)
logt('gauss_scores', gauss_scores, verbose=verbose)
logt('complete ', verbose=verbose)
return gauss_heatmap, gauss_scores