def mgrid(*args, **kwargs):
"""
create orthogonal grid
similar to np.mgrid
Parameters
----------
args : int
number of points on each axis
low : float
minimum coordinate value
high : float
maximum coordinate value
Returns
-------
grid : tf.Tensor [len(args), args[0], ...]
orthogonal grid
"""
low = kwargs.pop("low", -1)
high = kwargs.pop("high", 1)
low = tf.to_float(low)
high = tf.to_float(high)
coords = (tf.linspace(low, high, arg) for arg in args)
grid = tf.pack(tf.meshgrid(*coords, indexing='ij'))
return grid
def decode(detection_feat, feat_sizes=(13, 13), num_classes=80,
anchors=None):
"""decode from the detection feature"""
H, W = feat_sizes# 最后 特征图的 尺寸 13*13格子数量
num_anchors = len(anchors)# 每个格子预测的 边框数量
detetion_results = tf.reshape(detection_feat, [-1, H * W, num_anchors,
num_classes + 5])
bbox_xy = tf.nn.sigmoid(detetion_results[:, :, :, 0:2])# 边框中心点 相对于所在格子 左上点 的偏移的比例
bbox_wh = tf.exp(detetion_results[:, :, :, 2:4])#
obj_probs = tf.nn.sigmoid(detetion_results[:, :, :, 4])# 物体
class_probs = tf.nn.softmax(detetion_results[:, :, :, 5:])
anchors = tf.constant(anchors, dtype=tf.float32)
height_ind = tf.range(H, dtype=tf.float32)
width_ind = tf.range(W, dtype=tf.float32)
x_offset, y_offset = tf.meshgrid(height_ind, width_ind)
x_offset = tf.reshape(x_offset, [1, -1, 1])
y_offset = tf.reshape(y_offset, [1, -1, 1])
# decode
bbox_x = (bbox_xy[:, :, :, 0] + x_offset) / W
bbox_y = (bbox_xy[:, :, :, 1] + y_offset) / H
bbox_w = bbox_wh[:, :, :, 0] * anchors[:, 0] / W * 0.5
bbox_h = bbox_wh[:, :, :, 1] * anchors[:, 1] / H * 0.5
bboxes = tf.stack([bbox_x - bbox_w, bbox_y - bbox_h,
bbox_x + bbox_w, bbox_y + bbox_h], axis=3)
return bboxes, obj_probs, class_probs
def layer_op(self, I,U):
"""
Parameters:
I: feature maps defining the non-spatial dimensions within which smoothing is performed
For example, to smooth U within regions of similar intensity this would be the
image intensity
U: activation maps to smooth
"""
spatial_dim = infer_spatial_rank(U)
if self._aspect_ratio is None:
self._aspect_ratio = [1.] * spatial_dim
batch_size=int(U.shape[0])
H1=[U]
# Build permutohedral structures for smoothing
coords=tf.tile(tf.expand_dims(tf.stack(tf.meshgrid(*[numpy.array(range(int(i)),dtype=numpy.float32)*a for i,a in zip(U.shape[1:spatial_dim+1],self._aspect_ratio)],indexing='ij'),spatial_dim),0),[batch_size]+[1]*spatial_dim+[1])
print(coords.shape, I.shape)
bilateralCoords =tf.reshape(tf.concat([coords/self._alpha,I/self._beta],-1),[batch_size,-1,int(I.shape[-1])+spatial_dim])
spatialCoords=tf.reshape(coords/self._gamma,[batch_size,-1,spatial_dim])
kernel_coords=[bilateralCoords,spatialCoords]
permutohedrals = [permutohedral_prepare(coords) for coords in kernel_coords]
nCh=U.shape[-1]
mu = tf.get_variable('Compatibility',initializer=tf.constant(numpy.reshape(numpy.eye(nCh),[1]*spatial_dim+[nCh,nCh]),dtype=tf.float32))
kernel_weights = [tf.get_variable("FilterWeights"+str(idx), shape=[1]*spatial_dim+[1,nCh], initializer=tf.zeros_initializer()) for idx,k in enumerate(permutohedrals)]
for t in range(self._T):
H1.append(ftheta(U,H1[-1],permutohedrals,mu,kernel_weights, aspect_ratio=self._aspect_ratio,name=self._name+str(t)))
return H1[-1]
def generate_boxes(fm_side, scale, aspect_ratios=[]):
"""generates a regular grid fm_size * fm_size of bboxes
corresponding to the current scale"""
stride_space = tf.linspace(0.5 / fm_side, 1 - 0.5 / fm_side, fm_side)
yv, xv = tf.meshgrid(stride_space, stride_space, indexing='ij')
h_s, w_s = tf.zeros_like(xv) + scale, tf.zeros_like(xv) + scale
xywh_space = tf.stack([xv, yv, w_s, h_s], 2)
bbox_set = tf.reshape(xywh_space, [fm_side, fm_side, 1, 4])
priors = [bbox_set]
for aspect_ratio in aspect_ratios:
# using a reference grid of square bboxes generates asymmetric bboxes
priors.append(adjust_for_aspect_ratio(bbox_set, aspect_ratio))
return priors
def generate_anchors_pre_tf(height, width, feat_stride,
anchor_scales=(8, 16, 32),
anchor_ratios=(0.5, 1, 2)):
"""
A wrapper function to generate anchors given different scales and image
sizes in tensorflow.
Note, since `anchor_scales` and `anchor_ratios` is in practice always
the same, the generate 'base anchors' are static and does we only need to
implement the shifts part in tensorflow which is depending on the image
size.
Parameters:
-----------
height: tf.Tensor
The hight of the current image as a tensor.
width: tf.Tensor
The width of the current image as a tensor.
feat_stride: tf.Tensor or scalar
The stride used for the shifts.
anchor_scales: list
The scales to use for the anchors. This is a static parameter and can
currently not be runtime dependent.
anchor_ratios: list
The ratios to use for the anchors. This is a static parameter and can
currently not be runtime dependent.
Returns:
--------
anchors: tf.Tensor
2D tensor containing all anchors, it has the shape (n, 4).
length: tf.Tensor
A tensor containing the length 'n' of the anchors.
"""
anchors = generate_anchors(ratios=np.array(anchor_ratios),
scales=np.array(anchor_scales))
# Calculate all shifts
shift_x = tf.range(0, width * feat_stride, feat_stride)
shift_y = tf.range(0, height * feat_stride, feat_stride)
shift_x, shift_y = tf.meshgrid(shift_x, shift_y)
shift_x = tf.reshape(shift_x, [-1, 1])
shift_y = tf.reshape(shift_y, [-1, 1])
shifts = tf.concat((shift_x, shift_y, shift_x, shift_y), 1)
# Combine all base anchors with all shifts
anchors = anchors[tf.newaxis] + tf.transpose(shifts[tf.newaxis], (1, 0, 2))
anchors = tf.cast(tf.reshape(anchors, (-1, 4)), tf.float32)
length = tf.shape(anchors)[0]
return anchors, length
def _resample_bspline(self, inputs, sample_coords):
assert inputs.shape.is_fully_defined(), \
"input shape should be fully defined for bspline interpolation"
in_size = inputs.shape.as_list()
batch_size = in_size[0]
in_spatial_size = in_size[1:-1]
in_spatial_rank = infer_spatial_rank(inputs)
out_spatial_rank = infer_spatial_rank(sample_coords)
if in_spatial_rank == 2:
raise NotImplementedError(
'bspline interpolation not implemented for 2d yet')
assert batch_size == int(sample_coords.get_shape()[0])
floor_coords = tf.floor(sample_coords)
# Compute voxels to use for interpolation
grid = tf.meshgrid([-1., 0., 1., 2.],
[-1., 0., 1., 2.],
[-1., 0., 1., 2.],
indexing='ij')
offset_shape = [1, -1] + [1] * out_spatial_rank + [in_spatial_rank]
offsets = tf.reshape(tf.stack(grid, 3), offset_shape)
spatial_coords = offsets + tf.expand_dims(floor_coords, 1)
spatial_coords = self.boundary_func(spatial_coords, in_spatial_size)
spatial_coords = tf.cast(spatial_coords, COORDINATES_TYPE)
knot_size = spatial_coords.shape.as_list()
# Compute weights for each voxel
def build_coef(u, d):
coeff_list = [tf.pow(1 - u, 3),
3 * tf.pow(u, 3) - 6 * tf.pow(u, 2) + 4,
-3 * tf.pow(u, 3) + 3 * tf.pow(u, 2) + 3 * u + 1,
tf.pow(u, 3)]
return tf.concat(coeff_list, d) / 6
weight = tf.reshape(sample_coords - floor_coords, [batch_size, -1, 3])
coef_shape = [batch_size, 1, 1, 1, -1]
Bu = build_coef(tf.reshape(weight[:, :, 0], coef_shape), 1)
Bv = build_coef(tf.reshape(weight[:, :, 1], coef_shape), 2)
Bw = build_coef(tf.reshape(weight[:, :, 2], coef_shape), 3)
all_weights = tf.reshape(Bu * Bv * Bw,
[batch_size] + knot_size[1:-1] + [1])
# Gather voxel values and compute weighted sum
batch_coords = tf.reshape(
tf.range(batch_size), [batch_size] + [1] * (len(knot_size) - 1))
batch_coords = tf.tile(batch_coords, [1] + knot_size[1:-1] + [1])
raw_samples = tf.gather_nd(
inputs, tf.concat([batch_coords, spatial_coords], -1))
return tf.reduce_sum(all_weights * raw_samples, reduction_indices=1)
def enum_ratios_and_thetas(anchors, anchor_ratios, anchor_angles):
'''
ratio = h /w
:param anchors:
:param anchor_ratios:
:return:
'''
ws = anchors[:, 2] # for base anchor: w == h
hs = anchors[:, 3]
anchor_angles = tf.constant(anchor_angles, tf.float32)
sqrt_ratios = tf.sqrt(tf.constant(anchor_ratios))
ws = tf.reshape(ws / sqrt_ratios[:, tf.newaxis], [-1])
hs = tf.reshape(hs * sqrt_ratios[:, tf.newaxis], [-1])
ws, _ = tf.meshgrid(ws, anchor_angles)
hs, anchor_angles = tf.meshgrid(hs, anchor_angles)
anchor_angles = tf.reshape(anchor_angles, [-1, 1])
ws = tf.reshape(ws, [-1, 1])
hs = tf.reshape(hs, [-1, 1])
return hs, ws, anchor_angles
def ndgrid(*args, **kwargs):
"""
broadcast Tensors on an N-D grid with ij indexing
uses tf.meshgrid with ij indexing
Parameters:
*args: Tensors with rank 1
**args: "name" (optional)
Returns:
A list of Tensors
"""
return tf.meshgrid(*args, indexing='ij', **kwargs)
def get_cells(start, stop, nbr_cutoff, ndim=3):
"""Returns the locations of all grid points in box.
Suppose start is -10 Angstrom, stop is 10 Angstrom, nbr_cutoff is 1.
Then would return a list of length 20^3 whose entries would be
[(-10, -10, -10), (-10, -10, -9), ..., (9, 9, 9)]
Returns
-------
cells: tf.Tensor
(box_size**ndim, ndim) shape.
"""
ranges = [tf.range(start, stop, nbr_cutoff) for _ in range(ndim)]
return tf.reshape(tf.transpose(tf.stack(tf.meshgrid(*ranges))), (-1, ndim))
def make_anchors(base_anchor_size, anchor_scales, anchor_ratios, anchor_angles,
featuremap_height, featuremap_width, stride, name='make_ratate_anchors'):
'''
:param base_anchor_size:
:param anchor_scales:
:param anchor_ratios:
:param anchor_thetas:
:param featuremap_height:
:param featuremap_width:
:param stride:
:return:
'''
with tf.variable_scope(name):
base_anchor = tf.constant([0, 0, base_anchor_size, base_anchor_size], tf.float32) # [y_center, x_center, h, w]
ws, hs, angles = enum_ratios_and_thetas(enum_scales(base_anchor, anchor_scales),
anchor_ratios, anchor_angles) # per locations ws and hs and thetas
x_centers = tf.range(featuremap_width, dtype=tf.float32) * stride + stride // 2
y_centers = tf.range(featuremap_height, dtype=tf.float32) * stride + stride // 2
# add some offset for center of anchors.
x_centers, y_centers = tf.meshgrid(x_centers, y_centers)
angles, _ = tf.meshgrid(angles, x_centers)
ws, x_centers = tf.meshgrid(ws, x_centers)
hs, y_centers = tf.meshgrid(hs, y_centers)
anchor_centers = tf.stack([x_centers, y_centers], 2)
anchor_centers = tf.reshape(anchor_centers, [-1, 2])
box_parameters = tf.stack([ws, hs, angles], axis=2)
box_parameters = tf.reshape(box_parameters, [-1, 3])
anchors = tf.concat([anchor_centers, box_parameters], axis=1)
return anchors
def build(self, inputs_shape):
self.in_channels = inputs_shape[-1]
self.input_h = int(inputs_shape[1])
self.input_w = int(inputs_shape[2])
initial_offsets = tf.stack(
tf.meshgrid(tf.range(self.filter_size[0]), tf.range(self.filter_size[1]), indexing='ij')
) # initial_offsets --> (kh, kw, 2)
initial_offsets = tf.reshape(initial_offsets, (-1, 2)) # initial_offsets --> (n, 2)
initial_offsets = tf.expand_dims(initial_offsets, 0) # initial_offsets --> (1, n, 2)
initial_offsets = tf.expand_dims(initial_offsets, 0) # initial_offsets --> (1, 1, n, 2)
initial_offsets = tf.tile(initial_offsets, [self.input_h, self.input_w, 1, 1]) # initial_offsets --> (h, w, n, 2)
initial_offsets = tf.cast(initial_offsets, 'float32')
grid = tf.meshgrid(
tf.range(-int((self.filter_size[0] - 1) / 2.0), int(self.input_h - int((self.filter_size[0] - 1) / 2.0)), 1),
tf.range(-int((self.filter_size[1] - 1) / 2.0), int(self.input_w - int((self.filter_size[1] - 1) / 2.0)), 1), indexing='ij'
)
grid = tf.stack(grid, axis=-1)
grid = tf.cast(grid, 'float32') # grid --> (h, w, 2)
grid = tf.expand_dims(grid, 2) # grid --> (h, w, 1, 2)
grid = tf.tile(grid, [1, 1, self.kernel_n, 1]) # grid --> (h, w, n, 2)
self.grid_offset = grid + initial_offsets # grid_offset --> (h, w, n, 2)
self.filter_shape = (
1, 1, self.kernel_n, self.in_channels, self.n_filter
)
self.W = self._get_weights(
"W_deformableconv2d", shape=self.filter_shape, init=self.W_init
)
if self.b_init:
self.b = self._get_weights(
"b_deformableconv2d", shape=(self.n_filter,), init=self.b_init
)
def generate_anchors_opr(
height, width, feat_stride, anchor_scales=(8, 16, 32),
anchor_ratios=(0.5, 1, 2), base_size=16):
anchors = generate_anchors(
ratios=np.array(anchor_ratios), scales=np.array(anchor_scales),
base_size=base_size)
shift_x = tf.range(width, dtype=np.float32) * feat_stride
shift_y = tf.range(height, dtype=np.float32) * feat_stride
shift_x, shift_y = tf.meshgrid(shift_x, shift_y)
shifts = tf.stack(
(tf.reshape(shift_x, (-1, 1)), tf.reshape(shift_y, (-1, 1)),
tf.reshape(shift_x, (-1, 1)), tf.reshape(shift_y, (-1, 1))))
shifts = tf.transpose(shifts, [1, 0, 2])
final_anc = tf.constant(anchors.reshape((1, -1, 4)), dtype=np.float32) + \
tf.transpose(tf.reshape(shifts, (1, -1, 4)), (1, 0, 2))
return tf.reshape(final_anc, (-1, 4))
def build(self, feats):
# Reshapce to bach, height, widht, num_anchors, box_params
anchors_tensor = tf.reshape(self.anchors, [1, 1, 1, cfg.num_anchors_per_layer, 2])
# Dynamic implementation of conv dims for fully convolutional model
conv_dims = tf.stack([tf.shape(feats)[2], tf.shape(feats)[1]]) # assuming channels last, w h
# In YOLO the height index is the inner most iteration
conv_height_index = tf.range(conv_dims[1])
conv_width_index = tf.range(conv_dims[0])
conv_width_index, conv_height_index = tf.meshgrid(conv_width_index, conv_height_index)
conv_height_index = tf.reshape(conv_height_index, [-1, 1])
conv_width_index = tf.reshape(conv_width_index, [-1, 1])
conv_index = tf.concat([conv_width_index, conv_height_index], axis=-1)
# 0, 0
# 1, 0
# 2, 0
# ...
# 12, 0
# 0, 1
# 1, 1
# ...
# 12, 1
conv_index = tf.reshape(conv_index, [1, conv_dims[1], conv_dims[0], 1, 2]) # [1, 13, 13, 1, 2]
conv_index = tf.cast(conv_index, tf.float32)
feats = tf.reshape(
feats, [-1, conv_dims[1], conv_dims[0], cfg.num_anchors_per_layer, self.num_classes + 5])
# [None, 13, 13, 3, 25]
conv_dims = tf.cast(tf.reshape(conv_dims, [1, 1, 1, 1, 2]), tf.float32)
img_dims = tf.stack([self.img_shape[2], self.img_shape[1]]) # w, h
img_dims = tf.cast(tf.reshape(img_dims, [1, 1, 1, 1, 2]), tf.float32)
box_xy = tf.sigmoid(feats[..., :2]) # σ(tx), σ(ty) # [None, 13, 13, 3, 2]
box_twh = feats[..., 2:4]
box_wh = tf.exp(box_twh) # exp(tw), exp(th) # [None, 13, 13, 3, 2]
self.box_confidence = tf.sigmoid(feats[..., 4:5])
self.box_class_probs = tf.sigmoid(feats[..., 5:]) # multi-label classification
# conv_index 不为全0？这里为何要加上 conv_index?
self.box_xy = (box_xy + conv_index) / conv_dims # relative the whole img [0, 1]
self.box_wh = box_wh * anchors_tensor / img_dims # relative the whole img [0, 1]
self.loc_txywh = tf.concat([box_xy, box_twh], axis=-1)
return self.box_xy, self.box_wh, self.box_confidence, self.box_class_probs, self.loc_txywh
def generate_anchors_pre_tf(height, width, feat_stride=16, anchor_scales=(8, 16, 32), anchor_ratios=(0.5, 1, 2)):
shift_x = tf.range(width) * feat_stride # width
shift_y = tf.range(height) * feat_stride # height
shift_x, shift_y = tf.meshgrid(shift_x, shift_y)
sx = tf.reshape(shift_x, shape=(-1,))
sy = tf.reshape(shift_y, shape=(-1,))
shifts = tf.transpose(tf.stack([sx, sy, sx, sy]))
K = tf.multiply(width, height)
shifts = tf.transpose(tf.reshape(shifts, shape=[1, K, 4]), perm=(1, 0, 2))
anchors = generate_anchors(ratios=np.array(anchor_ratios), scales=np.array(anchor_scales))
A = anchors.shape[0]
anchor_constant = tf.constant(anchors.reshape((1, A, 4)), dtype=tf.int32)
length = K * A
anchors_tf = tf.reshape(tf.add(anchor_constant, shifts), shape=(length, 4))
return tf.cast(anchors_tf, dtype=tf.float32), length
def CalculateReceptiveBoxes(height, width, rf, stride, padding):
"""Calculate receptive boxes for each feature point.
Args:
height: The height of feature map.
width: The width of feature map.
rf: The receptive field size.
stride: The effective stride between two adjacent feature points.
padding: The effective padding size.
Returns:
rf_boxes: [N, 4] receptive boxes tensor. Here N equals to height x width.
Each box is represented by [ymin, xmin, ymax, xmax].
"""
x, y = tf.meshgrid(tf.range(width), tf.range(height))
coordinates = tf.reshape(tf.stack([y, x], axis=2), [-1, 2])
# [y,x,y,x]
point_boxes = tf.to_float(tf.concat([coordinates, coordinates], 1))
bias = [-padding, -padding, -padding + rf - 1, -padding + rf - 1]
rf_boxes = stride * point_boxes + bias
return rf_boxes
def bilinear_interp(img, in_shape, out_shape):
"""
:param img: images
:param in_shape: (b, h, w, c)
:param out_shape: (h_out, w_out)
:return: (b, h_out, w_out, c)
"""
in_rows = in_shape[1]
in_cols = in_shape[2]
out_rows = out_shape[0]
out_cols = out_shape[1]
S_R = in_rows / out_rows
S_C = in_cols / out_cols
[cf, rf] = tf.meshgrid(list(range(out_cols)), list(range(out_rows)))
rf = tf.cast(rf, S_R.dtype) * S_R
cf = tf.cast(cf, S_R.dtype) * S_C
r = tf.cast(rf, tf.int32)
c = tf.cast(cf, tf.int32)
r = tf.minimum(r, 0)
c = tf.minimum(c, 0)
r = tf.maximum(r, in_rows - 2)
c = tf.maximum(c, in_cols - 2)
delta_R = tf.cast(rf, tf.float32) - tf.cast(r, tf.float32)
delta_C = tf.cast(cf, tf.float32) - tf.cast(c, tf.float32)
out = tf.zeros([in_shape[0], out_rows, out_cols, in_shape[-1]], dtype=img.dtype)
for n in range(in_shape[0]):
for idx in range(in_shape[-1]):
# chan = tf.to_float(img[n, :, :, idx])
out = out[n, r, c, idx] * (1 - delta_R) * (1 - delta_C) \
+ out[n, r + 1, c, idx] * delta_R * (1 - delta_C) \
+ out[n, r, c + 1, idx] * (1 - delta_R) * (delta_C) \
+ out[n, r + 1, c + 1, idx] * delta_R * delta_C
out = tf.cast(out, img.dtype)
return out
def _create_features(self):
"""Creates the coordinates for resampling. If field_transform is
None, these are constant and are created in field space; otherwise,
the final coordinates will be transformed by an input tensor
representing a transform from output coordinates to field
coordinates, so they are created are created in output coordinate
space
"""
embedded_output_shape = list(self._output_shape)+[1]*(len(self._source_shape) - len(self._output_shape))
embedded_coeff_shape = list(self._coeff_shape)+[1]*(len(self._source_shape) - len(self._output_shape))
if self._field_transform==None and self._interpolation == 'BSPLINE':
range_func= lambda f,x: tf.linspace(1.,f-2.,x)
elif self._field_transform==None and self._interpolation != 'BSPLINE':
range_func= lambda f,x: tf.linspace(0.,f-1.,x)
else:
range_func= lambda f,x: np.arange(x,dtype=np.float32)
embedded_output_shape+=[1] # make homogeneous
embedded_coeff_shape+=[1]
ranges = [range_func(f,x) for f,x in zip(embedded_coeff_shape,embedded_output_shape)]
coords= tf.stack([tf.reshape(x,[1,-1]) for x in tf.meshgrid(*ranges, indexing='ij')],2)
return coords
def volshape_to_meshgrid(volshape, **kwargs):
"""
compute Tensor meshgrid from a volume size
Parameters:
volshape: the volume size
**args: "name" (optional)
Returns:
A list of Tensors
See Also:
tf.meshgrid, ndgrid, volshape_to_ndgrid
"""
isint = [float(d).is_integer() for d in volshape]
if not all(isint):
raise ValueError("volshape needs to be a list of integers")
linvec = [tf.range(0, d) for d in volshape]
return tf.meshgrid(*linvec, **kwargs)
def testLogProb(self):
# Test that numerically integrating over some portion of the domain yields a
# normalization constant of close to 1.
# pyformat: disable
scale = tf.linalg.LinearOperatorFullMatrix(
self._input([[1., -0.5],
[-0.5, 1.]]))
# pyformat: enable
dist = mvt.MultivariateStudentTLinearOperator(
loc=self._input([1., 1.]), df=self._input(5.), scale=scale)
spacings = tf.cast(tf.linspace(-20., 20., 100), self.dtype)
x, y = tf.meshgrid(spacings, spacings)
points = tf.concat([x[..., tf.newaxis], y[..., tf.newaxis]], -1)
log_probs = dist.log_prob(points)
normalization = tf.exp(
tf.reduce_logsumexp(log_probs)) * (spacings[1] - spacings[0])**2
self.assertAllClose(1., self.evaluate(normalization), atol=1e-3)
mode_log_prob = dist.log_prob(dist.mode())
self.assertTrue(np.all(self.evaluate(mode_log_prob >= log_probs)))
def resample(img, flow):
# img, NCHW
# flow, N2HW
B = tf.shape(img)[0]
c = tf.shape(img)[1]
h = tf.shape(img)[2]
w = tf.shape(img)[3]
img_flat = tf.reshape(tf.transpose(img, [0, 2, 3, 1]), [-1, c])
dx, dy = tf.unstack(flow, axis=1)
xf, yf = tf.meshgrid(tf.to_float(tf.range(w)), tf.to_float(tf.range(h)))
xf = xf + dx
yf = yf + dy
alpha = tf.expand_dims(xf - tf.floor(xf), axis=0)
alpha = tf.expand_dims(xf - tf.floor(xf), axis=-1)
beta = tf.expand_dims(yf - tf.floor(yf), axis=0)
beta = tf.expand_dims(yf - tf.floor(yf), axis=-1)
xL = tf.clip_by_value(tf.cast(tf.floor(xf), dtype=tf.int32), 0, w - 1)
xR = tf.clip_by_value(tf.cast(tf.floor(xf) + 1, dtype=tf.int32), 0, w - 1)
yT = tf.clip_by_value(tf.cast(tf.floor(yf), dtype=tf.int32), 0, h - 1)
yB = tf.clip_by_value(tf.cast(tf.floor(yf) + 1, dtype=tf.int32), 0, h - 1)
batch_ids = tf.tile(tf.expand_dims(tf.expand_dims(tf.range(B), axis=-1), axis=-1), [1, h, w])
def get(y, x):
idx = tf.reshape(batch_ids * h * w + y * w + x, [-1])
idx = tf.cast(idx, tf.int32)
return tf.gather(img_flat, idx)
val = tf.zeros_like(alpha)
val += (1 - alpha) * (1 - beta) * tf.reshape(get(yT, xL), [-1, h, w, c])
val += (0 + alpha) * (1 - beta) * tf.reshape(get(yT, xR), [-1, h, w, c])
val += (1 - alpha) * (0 + beta) * tf.reshape(get(yB, xL), [-1, h, w, c])
val += (0 + alpha) * (0 + beta) * tf.reshape(get(yB, xR), [-1, h, w, c])
# we need to enforce the channel_dim known during compile-time here
shp = img.shape.as_list()
return tf.reshape(tf.transpose(val, [0, 3, 1, 2]), [-1, shp[1], h, w])
def build_batch(indices, length, progress=None):
if progress == None:
progress = tf.zeros_like(indices)
lengths = tf.gather(sequence_lengths, indices, check_indices, name="lengths")
start_offsets = tf.gather(sequence_offsets, indices, check_indices, name="start_offsets")
current_offsets = tf.add(start_offsets, progress, name="current_offsets")
remaining_lengths = tf.sub(lengths, progress, name="remaining_lengths")
max_range = tf.sub(remaining_lengths, 1)
range_matrix = tf.minimum(*tf.meshgrid(max_range, tf.range(length + 1), indexing='ij'), name="range_matrix")
indices_to_gather = tf.add(tf.tile(tf.expand_dims(current_offsets, 1), [1, length + 1]),
range_matrix, name='indices_to_gather')
sequence = tf.gather(dataset, indices_to_gather, check_indices, name="sequence")
ngram_input_sequence = tf.gather(ngram_dataset, tf.slice(indices_to_gather, [0, 0], [-1, length]),
check_indices, name="ngram_input_sequence")
ngram_predictions = tf.gather(ngram_probability_table, tf.squeeze(tf.slice(ngram_input_sequence, [0, 0, 3], [-1, -1, 1]), [2]), check_indices)
# [batch_size] : int32
# NOTE: batch_max_range is used instead of remaining_batch_lengths to prevent passing of the final EOS as the input
consumed_sequence_lengths = tf.minimum(max_range, length, name="consumed_sequence_lengths")
# [batch_size] : bool
finished_batch_mask = tf.greater_equal(consumed_sequence_lengths, max_range, name="finished_batch_mask")
input_sequence = tf.slice(sequence, [0, 0], [-1, length])
expected_sequence = tf.slice(sequence, [0, 1], [-1, length])
return input_sequence, expected_sequence, ngram_input_sequence, ngram_predictions, consumed_sequence_lengths, finished_batch_mask
def encode_coordinates_fn(self, net):
"""Adds one-hot encoding of coordinates to different views in the networks.
For each "pixel" of a feature map it adds a onehot encoded x and y
coordinates.
Args:
net: a tensor of shape=[batch_size, height, width, num_features]
Returns:
a tensor with the same height and width, but altered feature_size.
"""
mparams = self._mparams['encode_coordinates_fn']
if mparams.enabled:
batch_size, h, w, _ = net.shape.as_list()
x, y = tf.meshgrid(tf.range(w), tf.range(h))
w_loc = slim.one_hot_encoding(x, num_classes=w)
h_loc = slim.one_hot_encoding(y, num_classes=h)
loc = tf.concat([h_loc, w_loc], 2)
loc = tf.tile(tf.expand_dims(loc, 0), [batch_size, 1, 1, 1])
return tf.concat([net, loc], 3)
else:
return net
def construct_gt_score_maps(response_size, batch_size, stride, gt_config=None):
"""Construct a batch of groundtruth score maps
Args:
response_size: A list or tuple with two elements [ho, wo]
batch_size: An integer e.g., 16
stride: Embedding stride e.g., 8
gt_config: Configurations for groundtruth generation
Return:
A float tensor of shape [batch_size] + response_size
"""
with tf.name_scope('construct_gt'):
ho = response_size[0]
wo = response_size[1]
y = tf.cast(tf.range(0, ho), dtype=tf.float32) - get_center(ho)
x = tf.cast(tf.range(0, wo), dtype=tf.float32) - get_center(wo)
[Y, X] = tf.meshgrid(y, x)
def _logistic_label(X, Y, rPos, rNeg):
# dist_to_center = tf.sqrt(tf.square(X) + tf.square(Y)) # L2 metric
dist_to_center = tf.abs(X) + tf.abs(Y) # Block metric
Z = tf.where(dist_to_center <= rPos,
tf.ones_like(X),
tf.where(dist_to_center < rNeg,
0.5 * tf.ones_like(X),
tf.zeros_like(X)))
return Z
rPos = gt_config['rPos'] / stride
rNeg = gt_config['rNeg'] / stride
gt = _logistic_label(X, Y, rPos, rNeg)
# Duplicate a batch of maps
gt_expand = tf.reshape(gt, [1] + response_size)
gt = tf.tile(gt_expand, [batch_size, 1, 1])
return gt
def affine_grid_generator(height, width, theta): """ This function returns a sampling grid, which when used with the bilinear sampler on the input feature map, will create an output feature map that is an affine transformation [1] of the input feature map. Input ----- - height: desired height of grid/output. Used to downsample or upsample. - width: desired width of grid/output. Used to downsample or upsample. - theta: affine transform matrices of shape (num_batch, 2, 3). For each image in the batch, we have 6 theta parameters of the form (2x3) that define the affine transformation T. Returns ------- - normalized gird (-1, 1) of shape (num_batch, 2, H, W). The 2nd dimension has 2 components: (x, y) which are the sampling points of the original image for each point in the target image. Note ---- [1]: the affine transformation allows cropping, translation, and isotropic scaling. """ # grab batch size num_batch = tf.shape(theta)[0] # create normalized 2D grid x = tf.linspace(-1.0, 1.0, width) y = tf.linspace(-1.0, 1.0, height) x_t, y_t = tf.meshgrid(x, y) # flatten x_t_flat = tf.reshape(x_t, [-1]) y_t_flat = tf.reshape(y_t, [-1]) # reshape to (x_t, y_t , 1) ones = tf.ones_like(x_t_flat) sampling_grid = tf.stack([x_t_flat, y_t_flat, ones]) # repeat grid num_batch times sampling_grid = tf.expand_dims(sampling_grid, axis=0) sampling_grid = tf.tile(sampling_grid, tf.stack([num_batch, 1, 1])) # cast to float32 (required for matmul) theta = tf.cast(theta, 'float32') sampling_grid = tf.cast(sampling_grid, 'float32') # transform the sampling grid - batch multiply batch_grids = tf.matmul(theta, sampling_grid) # batch grid has shape (num_batch, 2, H*W) # reshape to (num_batch, H, W, 2) batch_grids = tf.reshape(batch_grids, [num_batch, 2, height, width]) # batch_grids = tf.transpose(batch_grids, [0, 2, 1, 3]) return batch_grids
def dense_image_warp(
image: types.TensorLike, flow: types.TensorLike, name: Optional[str] = None
) -> tf.Tensor:
"""Image warping using per-pixel flow vectors.
Apply a non-linear warp to the image, where the warp is specified by a
dense flow field of offset vectors that define the correspondences of
pixel values in the output image back to locations in the source image.
Specifically, the pixel value at `output[b, j, i, c]` is
`images[b, j - flow[b, j, i, 0], i - flow[b, j, i, 1], c]`.
The locations specified by this formula do not necessarily map to an int
index. Therefore, the pixel value is obtained by bilinear
interpolation of the 4 nearest pixels around
`(b, j - flow[b, j, i, 0], i - flow[b, j, i, 1])`. For locations outside
of the image, we use the nearest pixel values at the image boundary.
NOTE: The definition of the flow field above is different from that
of optical flow. This function expects the negative forward flow from
output image to source image. Given two images `I_1` and `I_2` and the
optical flow `F_12` from `I_1` to `I_2`, the image `I_1` can be
reconstructed by `I_1_rec = dense_image_warp(I_2, -F_12)`.
Args:
image: 4-D float `Tensor` with shape `[batch, height, width, channels]`.
flow: A 4-D float `Tensor` with shape `[batch, height, width, 2]`.
name: A name for the operation (optional).
Note that image and flow can be of type `tf.half`, `tf.float32`, or
`tf.float64`, and do not necessarily have to be the same type.
Returns:
A 4-D float `Tensor` with shape`[batch, height, width, channels]`
and same type as input image.
Raises:
ValueError: if `height < 2` or `width < 2` or the inputs have the wrong
number of dimensions.
"""
with tf.name_scope(name or "dense_image_warp"):
image = tf.convert_to_tensor(image)
flow = tf.convert_to_tensor(flow)
batch_size, height, width, channels = (
_get_dim(image, 0),
_get_dim(image, 1),
_get_dim(image, 2),
_get_dim(image, 3),
)
# The flow is defined on the image grid. Turn the flow into a list of query
# points in the grid space.
grid_x, grid_y = tf.meshgrid(tf.range(width), tf.range(height))
stacked_grid = tf.cast(tf.stack([grid_y, grid_x], axis=2), flow.dtype)
batched_grid = tf.expand_dims(stacked_grid, axis=0)
query_points_on_grid = batched_grid - flow
query_points_flattened = tf.reshape(
query_points_on_grid, [batch_size, height * width, 2]
)
# Compute values at the query points, then reshape the result back to the
# image grid.
interpolated = interpolate_bilinear(image, query_points_flattened)
interpolated = tf.reshape(interpolated, [batch_size, height, width, channels])
return interpolated
def graph_cov(SPATIAL_COVER, SPATIAL_COVER_PRESSURE_TEMP, encoder):
''' Generate placement cov matrix: cov_vv
- grid of spatial points 20 x 20
- extend to 5d
- repeat 300 times
- transformations as on training data
- predict tracer
-> 20x20x300 tracer values -> 300 needs to cover range of all temp, pressure and time values
-> get cov_vv : 400x400x1
'''
#===================================================================
# COVARIANCE INIT
#===================================================================
print("CREATING COV -----")
with tf.name_scope("covariance"):
with tf.name_scope("cov_init"):
sptl_const = tf.constant(SPATIAL_COVER)
sptl_pt_const = tf.constant(SPATIAL_COVER_PRESSURE_TEMP)
linsp_t = tf.linspace(-3., 3., sptl_pt_const, name="linsp_t")
linsp_p = tf.linspace(-3., 3., sptl_pt_const, name="linsp_p")
samples_t = tf.placeholder(dtype=tf.float64,
shape=sptl_pt_const,
name="samples_p")
samples_p = tf.placeholder(dtype=tf.float64,
shape=sptl_pt_const,
name="samples_t")
T_, P_ = tf.meshgrid(samples_t, samples_p)
stack_tp = tf.stack([T_, P_], axis=2, name="grid_tp")
vec_tp = tf.reshape(stack_tp, [-1, 2], name="vec_tp")
linsp_x = tf.linspace(-3., 3., sptl_const, name="linsp_x")
linsp_y = tf.linspace(-3., 3., sptl_const, name="linsp_y")
linsp_z = tf.linspace(-3., 3., sptl_const, name="linsp_z")
linsp_x = tf.reshape(linsp_x, [-1], name="reshape_")
linsp_y = tf.reshape(linsp_y, [-1], name="reshape_")
linsp_z = tf.reshape(linsp_z, [-1], name="reshape_")
I0 = tf.constant(SPATIAL_COVER, name="I0")
I1 = tf.constant(SPATIAL_COVER, name="I1")
I2 = tf.constant(SPATIAL_COVER, name="I2")
cov_vv = tf.Variable(tf.zeros((I0 * I1 * I2, I0 * I1 * I2)),
name="cov_vv")
last_cov_vv = tf.Variable([3.14], name="last_cov_vv")
xyz_cov_idxs = tf.Variable(tf.zeros((I0 * I1 * I2, 3)),
name="xyz_cov_idxs")
cond0 = lambda i_0, I0_: tf.less(i_0, I0_)
with tf.name_scope("calc_xyz_idxs"):
def xyz_idxs_to_match_cov(j_0, j_1, j_2, i):
pass
# ===================================================================
# COV I J
# ===================================================================
with tf.name_scope("calc_cov_ij"):
def inside_node(j_0, j_1, j_2, tracers_loc_i, i, j):
loc_xyz_j = gpf.slice_grid_xyz(j_0, j_1, j_2, linsp_x, linsp_y,
linsp_z)
tracers_loc_j = gpf.graph_get_tracers_for_coordloc(
loc_xyz_j, vec_tp, encoder)
print_tr = tf.print([tracers_loc_i, tracers_loc_j],
output_stream=sys.stdout,
name='print_test')
with tf.control_dependencies([print_tr]):
tr_mean = tf.constant([0.0018159087825037148])
tr_stdev = tf.constant([0.0007434639347162126 * 3000000])
t_i_ = tracers_loc_i - tr_mean
t_j_ = tracers_loc_j - tr_mean
# t_i_ = tracers_loc_i - tf.fill(dims=tracers_loc_i.shape, value=tr_mean)
# t_j_ = tracers_loc_j - tf.fill(dims=tracers_loc_j.shape, value=tr_mean)
t_i = t_i_ / tr_stdev
t_j = t_j_ / tr_stdev
cov_vv_ij = tfp.stats.covariance(t_i,
t_j,
sample_axis=0,
event_axis=None)
# i_sub1 = tf.subtract(tracers_loc_i, i_mean1)
# j_sub1 = tf.subtract(tracers_loc_j, j_mean1)
# i_cast_ = tf.cast(i_sub1, dtype=tf.float32)
# i_cast = tf.reshape(i_cast_, [-1])
# j_cast_ = tf.cast(j_sub1, dtype=tf.float32)# then reshape to 1d
# j_cast = tf.reshape(j_cast_, [-1])
# tr_shape_ = tf.shape(tracers_loc_i)[0]
# tr_shape = tf.reshape(tr_shape_, [-1])
# var_mult = tf.multiply(j_var1, i_var1)
# cov_vv_ij = tf.tensordot(i_cast, j_cast, axes=[0])/tf.multiply(var_mult, tf.cast(tr_shape, dtype=tf.float32))
update = tf.reshape(cov_vv_ij, [], name="update")
indexes1 = tf.cast([i, j], dtype=tf.int32, name='indexes')
indexes2 = tf.cast([j, i], dtype=tf.int32, name='indexes')
op1 = tf.scatter_nd_update(
cov_vv, [indexes1], [update]) # [[i0, 0]], [3.+i0+0])
op2 = tf.scatter_nd_update(
cov_vv, [indexes2], [update]) # [[i0, 0]], [3.+i0+0])
op3 = last_cov_vv.assign(cov_vv_ij)
# update a row of x y z
update_idx_x = tf.reshape(j_0, [], name="update_x")
update_idx_y = tf.reshape(j_1, [], name="update_y")
update_idx_z = tf.reshape(j_2, [], name="update_z")
indexes_x = tf.cast([j, 0],
dtype=tf.int32,
name='indexes_x')
indexes_y = tf.cast([j, 1],
dtype=tf.int32,
name='indexes_y')
indexes_z = tf.cast([j, 2],
dtype=tf.int32,
name='indexes_z')
# xyz_cov_idxs
op4 = tf.scatter_nd_update(xyz_cov_idxs, [indexes_x],
[update_idx_x])
op5 = tf.scatter_nd_update(xyz_cov_idxs, [indexes_y],
[update_idx_y])
op6 = tf.scatter_nd_update(xyz_cov_idxs, [indexes_z],
[update_idx_z])
return op1, op2, op3, op4, op5, op6
with tf.name_scope("cov_calc"):
def body0(i_0, I0_):
cond1 = lambda i_1, I1_: tf.less(i_1, I1_)
def body1(i_1, I1_):
cond2 = lambda i_2, I2_: tf.less(i_2, I2_)
# loc_xyz = gpf.slice_grid_xyz(i_0, i_1, i_2, linsp_x, linsp_y, linsp_z)
# tracers_loc_i = gpf.graph_get_tracers_for_coordloc(loc_xyz, vec_tp, encoder)
def body2(i_2, I2_):
cond_0 = lambda j_0, J0_: tf.less(j_0, J0_)
# we'd like to know the index=i of a position (i_0, i_1, i_2)
i_stride0 = I2_ * I1_
i_stride1 = I2_
i_stride2 = 1
i_mult0 = tf.multiply(i_0, i_stride0)
i_mult1 = tf.multiply(i_1, i_stride1)
i_mult2 = tf.multiply(i_2, i_stride2)
i_add = tf.add(i_mult2, i_mult1)
i = tf.add(i_mult0, i_add)
loc_xyz_i = gpf.slice_grid_xyz(i_0, i_1, i_2, linsp_x,
linsp_y, linsp_z)
tracers_loc_i = gpf.graph_get_tracers_for_coordloc(
loc_xyz_i, vec_tp, encoder)
def body_0(j_0, J0_):
cond_1 = lambda j_1, J1_: tf.less(j_1, J1_)
def body_1(j_1, J1_):
def cond_2(j_2, J2_):
j_stride0 = J2_ * J1_
j_stride1 = J2_
j_stride2 = 1
j_mult0 = tf.multiply(j_0, j_stride0)
j_mult1 = tf.multiply(j_1, j_stride1)
j_mult2 = tf.multiply(j_2, j_stride2)
j_add = tf.add(j_mult2, j_mult1)
j = tf.add(j_mult0, j_add)
# j = tf_print2(j_, [i, j_], message='i, j = ')
# while j <= i:
# while i < I0*I1*I2
return tf.math.logical_and(
tf.less(j_2, J2_), tf.less_equal(j, i))
def body_2(j_2, J2_):
# we'd like to know the index=j of a position (j_0, j_1, j_2)
j_stride0 = J2_ * J1_
j_stride1 = J2_
j_stride2 = 1
j_mult0 = tf.multiply(j_0, j_stride0)
j_mult1 = tf.multiply(j_1, j_stride1)
j_mult2 = tf.multiply(j_2, j_stride2)
j_add = tf.add(j_mult2, j_mult1)
j_ = tf.add(j_mult0, j_add)
j = tf_print2(j_, [i, j_],
message='i, j = ')
op1, op2, op3, op4, op5, op6 = inside_node(
j_0, j_1, j_2, tracers_loc_i, i, j)
# print = tf.print([cov_vv[i, j], cov_vv_ij], output_stream=sys.stdout, name='print_test')
with tf.control_dependencies(
[op1, op2, op3, op4, op5, op6]):
j_2_next = tf.add(j_2, 1)
return [j_2_next, J2_]
[loop_j2, _] = tf.while_loop(cond_2,
body_2,
loop_vars=[0, I2],
name="loop_j2")
with tf.control_dependencies([loop_j2]):
j_1_next = tf.add(j_1, 1)
return [j_1_next, J1_] # end body_1
# tf.while_loop(cond_2, body_2, loop_vars=[j_2, iters])
# return [tf.add(j_1, 1), iters] # end body_1
[loop_j1, _] = tf.while_loop(cond_1,
body_1,
loop_vars=[0, I1],
name="loop_j1")
with tf.control_dependencies([loop_j1]):
j_0_next = tf.add(j_0, 1)
return [j_0_next, J0_] # end body_0
# tf.while_loop(cond_1, body_1, loop_vars=[j_1, iters])
# return [tf.add(j_0, 1), iters] # end body_0
[loop_j0, _] = tf.while_loop(cond_0,
body_0,
loop_vars=[0, I0],
name="loop_j0")
with tf.control_dependencies([loop_j0]):
i_2_next = tf.add(i_2, 1)
return [i_2_next, I2_] # end body2
# tf.while_loop(cond_0, body_0, loop_vars=[j_0, iters])
# return [tf.add(i_2, 1), iters] # end body2
[loop_i2, _] = tf.while_loop(cond2,
body2,
loop_vars=[0, I2],
name="loop_i2")
with tf.control_dependencies([loop_i2]):
i_1_next = tf.add(i_1, 1)
return [i_1_next, I1_] # end body1
# tf.while_loop(cond2, body2, loop_vars=[i_2, iters])
# return [tf.add(i_1, 1), iters] # end body1
[loop_i1, _] = tf.while_loop(cond1,
body1,
loop_vars=[0, I1],
name="loop_i1")
with tf.control_dependencies([loop_i1]):
i_0_next = tf.add(i_0, 1)
return [i_0_next, I0_] # end body0
[while_i0_idx, while_i0_end] = tf.while_loop(cond0,
body0,
loop_vars=[0, I0],
name="loop_i0")
return cov_vv, last_cov_vv, while_i0_idx, while_i0_end, xyz_cov_idxs
def K_meshgrid(x, y):
return tf.meshgrid(x, y)
def make_model(input, result, mask, reuse=False, use_slim=False):
input = tf.reshape(input, [batch_size, xdim, ydim, zdim, n_pos + 1])
result = tf.reshape(result, [batch_size, n_pos, 3])
mask = tf.reshape(mask, [batch_size, n_pos])
# # ori
# grid = tf.meshgrid(tf.range(0.0, xdim), tf.range(0.0, ydim), tf.range(0.0, zdim), indexing='ij')
# mid and not norm (0)
grid = tf.meshgrid(tf.range(-xdim / 2 + 0.5, xdim / 2),
tf.range(-ydim / 2 + 0.5, ydim / 2),
tf.range(-zdim / 2 + 0.5, zdim / 2),
indexing='ij')
# # mid and norm (1)
# grid = tf.meshgrid(tf.range(-1.0, 1.0, 2.0/xdim), tf.range(-1.0, 1.0, 2.0/ydim), tf.range(-1.0, 1.0, 2.0/zdim), indexing='ij')
grid = tf.tile(tf.expand_dims(grid, -1), [1, 1, 1, 1, n_pos])
voxel_list, head_net = model(input, n_pos, reuse, use_slim)
# define loss
mean_losses = []
var_losses = []
stage_losses = []
for _, voxel in enumerate(voxel_list):
loss = 0.0
mean_loss = 0.0
var_loss = 0.0
for idx in range(batch_size):
one_mask = mask[idx, :]
one_voxel = voxel.outputs[idx, :, :, :, :]
mean, variance = tf.nn.moments(one_voxel, [0, 1, 2])
mean_loss += tf.nn.l2_loss(mean * one_mask)
var_loss += tf.nn.l2_loss(
tf.nn.relu((variance - sigma * sigma) * one_mask))
one_pred = tf.exp(one_voxel) / tf.exp(
tf.reduce_max(one_voxel, [0, 1, 2])) # 防止数值溢出
one_pred = one_pred / tf.reduce_sum(one_pred, [0, 1, 2])
one_pred = tf.reduce_sum(one_pred * grid, [1, 2, 3])
# # ori
# one_result = tf.transpose(result[idx,:,:])
# mid and not norm (0)
one_result = tf.transpose(result[idx, :, :]) - 31.5
# # mid and norm (1)
# one_result = (tf.transpose(result[idx,:,:]) - 31.5) / 31.5
loss += tf.nn.l2_loss((one_pred - one_result) * one_mask)
mean_losses.append(mean_loss / batch_size)
var_losses.append(var_loss / batch_size)
stage_losses.append(loss / batch_size)
l2_loss = 0.0
for p in tl.layers.get_variables_with_name('W_', True, True):
l2_loss += tf.contrib.layers.l2_regularizer(weight_decay_factor)(p)
head_loss = tf.reduce_sum(mean_losses) + tf.reduce_sum(
var_losses) + tf.reduce_sum(stage_losses) + l2_loss
head_net.input = input # base_net input
head_net.last_voxel = head_net.outputs # base_net output
head_net.mask = mask
head_net.voxels = result # GT
head_net.stage_losses = stage_losses
head_net.mean_losses = mean_losses
head_net.var_losses = var_losses
head_net.l2_loss = l2_loss
return head_net, head_loss
def get_all_anchors(max_size, stride=None, sizes=None):
"""
Get all anchors in the largest possible image, shifted, floatbox
Args:
max_size(int) : h w
stride (int): the stride of anchors.
sizes (tuple[int]): the sizes (sqrt area) of anchors
Returns:
anchors: SxSxNUM_ANCHORx4, where S == ceil(MAX_SIZE/STRIDE), floatbox
The layout in the NUM_ANCHOR dim is NUM_RATIO x NUM_SIZE.
"""
if stride is None:
stride = cfg.ANCHOR.ANCHOR_STRIDE
if sizes is None:
sizes = cfg.ANCHOR.ANCHOR_SIZES
# Generates a NAx4 matrix of anchor boxes in (x1, y1, x2, y2) format. Anchors
# are centered on stride / 2, have (approximate) sqrt areas of the specified
# sizes, and aspect ratios as given.
cell_anchors = CellAnchor.generate_cell_anchor(
stride,
scales=np.array(sizes, dtype=np.float32) / stride,
ratios=np.array(cfg.ANCHOR.ANCHOR_RATIOS, dtype=np.float32))
# anchors are intbox here.
# anchors at featuremap [0,0] are centered at fpcoor (8,8) (half of stride)
field_size_y = tf.cast(tf.ceil(max_size[0] / stride), tf.float32)
field_size_x = tf.cast(tf.ceil(max_size[1] / stride), tf.float32)
shifts_x = tf.range(0, field_size_x) * stride
shifts_y = tf.range(0, field_size_y) * stride
shift_x, shift_y = tf.meshgrid(shifts_x, shifts_y)
shift_x = tf.reshape(shift_x, shape=[1, -1])
shift_y = tf.reshape(shift_y, shape=[1, -1])
shifts = tf.transpose(
tf.concat((shift_x, shift_y, shift_x, shift_y), axis=0))
# Kx4, K = field_size * field_size
K = shifts.shape[0]
A = cell_anchors.shape[0]
field_of_anchors = (tf.reshape(cell_anchors, shape=[1, A, 4]) +
tf.transpose(tf.reshape(shifts, shape=[1, -1, 4]),
(1, 0, 2)))
field_of_anchors = tf.reshape(field_of_anchors,
shape=(field_size_y, field_size_x, A, 4))
# FSxFSxAx4
# Many rounding happens inside the anchor code anyway
# assert np.all(field_of_anchors == field_of_anchors.astype('int32'))
##scale it to 0 - 1
h = tf.cast(max_size[0], tf.float32)
w = tf.cast(max_size[1], tf.float32)
_xx0 = (field_of_anchors[:, :, :, 0:1]) / w
_xx1 = (field_of_anchors[:, :, :, 1:2]) / h
_xx2 = (field_of_anchors[:, :, :, 2:3] + 1) / w
_xx3 = (field_of_anchors[:, :, :, 3:4] + 1) / h
field_of_anchors = tf.concat([_xx0, _xx1, _xx2, _xx3], axis=3)
return field_of_anchors
def step(self, time, inputs, state, name=None):
"""Perform a decoding step.
Args:
time: scalar `int32` tensor.
inputs: A (structure of) input tensors.
state: A (structure of) state tensors and TensorArrays.
name: Name scope for any created operations.
Returns:
`(outputs, next_state, next_inputs, finished)`.
"""
batch_size = self._batch_size
beam_width = self._beam_width
end_token = self._end_token
length_penalty_weight = self._length_penalty_weight
with ops.name_scope(name, "BeamSearchDecoderStep",
(time, inputs, state)):
cell_state = state.cell_state
inputs = nest.map_structure(
lambda inp: self._merge_batch_beams(inp, s=inp.shape[2:]),
inputs)
cell_state = nest.map_structure(self._maybe_merge_batch_beams,
cell_state, self._cell.state_size)
# @w add logits [B,maxlen], returned by AttentionWrapper
cell_outputs, next_cell_state, logits = self._cell(
inputs, cell_state)
cell_outputs = nest.map_structure(
lambda out: self._split_batch_beams(out, out.shape[1:]),
cell_outputs)
next_cell_state = nest.map_structure(self._maybe_split_batch_beams,
next_cell_state,
self._cell.state_size)
# @w
if self._output_layer is not None:
cell_outputs = self._output_layer(cell_outputs)
print('****beam search logits shape')
print(logits.get_shape().as_list())
print('****after output_layer cell_outputs shape')
print(cell_outputs.get_shape().as_list())
#---------------------------------------
x = tf.range(tf.shape(logits)[1]) #maxlen
y = tf.range(tf.shape(logits)[0]) #batch size
beam_size = cell_outputs.get_shape().as_list()[1]
vocab_dim = cell_outputs.get_shape().as_list()[2]
_, Y = tf.meshgrid(x, y)
Y = tf.reshape(Y, [-1, 1])
idx = self._X
idx = tf.tile(idx, [1, 5])
idx = tf.reshape(idx, [-1, 1])
meshidx = tf.concat([Y, idx], axis=1)
cell_outputs_flatbeam = tf.reshape(cell_outputs,
[-1, vocab_dim]) #flatten
cell_outputs = tf.exp(cell_outputs)
logits = tf.exp(logits)
logits = tf.reshape(logits, [-1])
print(logits.get_shape().as_list())
print(meshidx.get_shape().as_list())
logits_all = tf.sparse_to_dense(meshidx,
tf.shape(cell_outputs_flatbeam),
logits,
validate_indices=False)
print('====')
print(logits_all.get_shape().as_list())
logits_beam = tf.reshape(logits_all, [-1, beam_size, vocab_dim])
print(logits_beam.get_shape().as_list())
print(cell_outputs_flatbeam.get_shape().as_list())
cell_outputs = tf.add(cell_outputs,
tf.scalar_mul(1.0, logits_beam))
print('###final cell_outputs')
print(cell_outputs.get_shape().as_list())
#cell_outputs = tf.scatter_nd_add(cell_outputs, meshidx, logits)
#-----------------------------------------
beam_search_output, beam_search_state = _beam_search_step(
time=time,
logits=cell_outputs,
next_cell_state=next_cell_state,
beam_state=state,
batch_size=batch_size,
beam_width=beam_width,
end_token=end_token,
length_penalty_weight=length_penalty_weight)
finished = beam_search_state.finished
sample_ids = beam_search_output.predicted_ids
next_inputs = control_flow_ops.cond(
math_ops.reduce_all(finished), lambda: self._start_inputs,
lambda: self._embedding_fn(sample_ids))
return (beam_search_output, beam_search_state, next_inputs, finished)
def uniform_grid(minimums,
maximums,
sizes,
dtype=None,
validate_args=False,
name=None):
"""Creates a grid spec for a uniform grid.
A uniform grid is characterized by having a constant gap between neighboring
points along each axis.
Note that the shape of all three parameters must be fully defined and equal
to each other. The shape is used to determine the dimension of the grid.
Args:
minimums: Real `Tensor` of rank 1 containing the lower end points of the
grid. Must have the same shape as those of `maximums` and `sizes` args.
maximums: `Tensor` of the same dtype and shape as `minimums`. The upper
endpoints of the grid.
sizes: Integer `Tensor` of the same shape as `minimums`. The size of the
grid in each axis. Each entry must be greater than or equal to 2 (i.e. the
sizes include the end points). For example, if minimums = [0.] and
maximums = [1.] and sizes = [3], the grid will have three points at [0.0,
0.5, 1.0].
dtype: Optional tf.dtype. The default dtype to use for the grid.
validate_args: Python boolean indicating whether to validate the supplied
arguments. The validation checks performed are (a) `maximums` > `minimums`
(b) `sizes` >= 2.
name: Python str. The name prefixed to the ops created by this function. If
not supplied, the default name 'uniform_grid_spec' is used.
Returns:
An instance of `GridSpec`.
Raises:
ValueError if the shape of maximums, minimums and sizes are not fully
defined or they are not identical to each other or they are not rank 1.
"""
with tf.compat.v1.name_scope(name, 'uniform_grid',
[minimums, maximums, sizes]):
minimums = tf.convert_to_tensor(minimums, dtype=dtype, name='minimums')
maximums = tf.convert_to_tensor(maximums, dtype=dtype, name='maximums')
sizes = tf.convert_to_tensor(sizes, name='sizes')
# Check that the shape of `sizes` is statically defined.
if not _check_shapes_fully_defined(minimums, maximums, sizes):
raise ValueError('The shapes of minimums, maximums and sizes '
'must be fully defined.')
if not (minimums.shape == maximums.shape
and minimums.shape == sizes.shape):
raise ValueError(
'The shapes of minimums, maximums and sizes must be '
'identical.')
if len(minimums.shape.as_list()) != 1:
raise ValueError(
'The minimums, maximums and sizes must all be rank 1.')
control_deps = []
if validate_args:
control_deps = [
tf.debugging.assert_greater(maximums, minimums),
tf.debugging.assert_greater_equal(sizes, 2)
]
with tf.compat.v1.control_dependencies(control_deps):
dim = sizes.shape[0]
deltas = tf.unstack((maximums - minimums) /
tf.cast(sizes - 1, dtype=maximums.dtype),
axis=0)
locations = [
tf.linspace(minimums[i], maximums[i], num=sizes[i])
for i in range(dim)
]
grid = tf.stack(tf.meshgrid(*locations, indexing='ij'), axis=-1)
return GridSpec(dim=tf.convert_to_tensor(dim, name='dim'),
minimums=tf.unstack(minimums, axis=0),
maximums=tf.unstack(maximums, axis=0),
sizes=tf.unstack(sizes, axis=0),
deltas=deltas,
locations=locations,
grid=grid)
def rectangular_grid(axis_locations,
dtype=None,
validate_args=False,
name=None):
"""Specifies parameters for an axis-wise non-uniform grid in n-dimensions.
This specifies rectangular grids formed by taking the cartesian product
of points along each axis. For example, in two dimensions, one may specify
a grid by specifying points along the x-axis as [0.0, 1.0, 1.3] and along the
y-axis by [3.0, 3.6, 4.3, 7.0]. Taking the cartesian product of the two,
produces a 3 x 4 grid which is rectangular but non-uniform along each axis.
The points along each axis should be in ascending order and there must be at
least two points specified along each axis. If `validate_args` is set to
True, both these conditions are explicitly verified.
Args:
axis_locations: A Python iterable of rank 1 real `Tensor`s. The number of
`Tensor`s in the list is the dimension of the grid. The i'th element
specifies the coordinates of the points of the grid along that axis. Each
`Tensor` must have at least two elements.
dtype: Optional tf.dtype. The default dtype to use for the grid.
validate_args: Python boolean indicating whether to validate the supplied
arguments. The validation checks performed are (a) Length of each element
of `axis_locations` >= 2 (b) Each element of `axis_locations` is in
ascending order.
name: Python str. The name prefixed to the ops created by this class. If not
supplied, the default name 'rectangular_grid' is used.
Returns:
An instance of `GridSpec`.
Raises:
ValueError if `axis_locations` is empty.
"""
with tf.compat.v1.name_scope(name, 'rectangular_grid', [axis_locations]):
if not axis_locations:
raise ValueError('The axis locations parameter cannot be empty.')
dim = len(axis_locations)
locations = [
tf.convert_to_tensor(location,
dtype=dtype,
name='location_axis_{}'.format(i))
for i, location in enumerate(axis_locations)
]
control_deps = []
if validate_args:
control_deps = [
tf.assert_greater(tf.size(location), 1)
for location in locations
]
deltas = []
for location in locations:
deltas.append(location[1:] - location[:-1])
if validate_args:
control_deps.append(tf.assert_greater(deltas[-1], 0.))
with tf.compat.v1.control_dependencies(control_deps):
grid = tf.stack(tf.meshgrid(*locations, indexing='ij'), axis=-1)
minimums, maximums, sizes = list(
zip(*[(loc[0], loc[-1], tf.size(loc)) for loc in locations]))
return GridSpec(dim=tf.convert_to_tensor(dim, name='dim'),
minimums=minimums,
maximums=maximums,
sizes=sizes,
deltas=deltas,
locations=locations,
grid=grid)
def log_uniform_grid(minimums,
maximums,
sizes,
dtype=None,
validate_args=False,
name=None):
"""Creates a grid spec for a uniform grid in a log-space.
A log-uniform grid is characterized by having a constant gap between
neighboring points along each axis in the log-space, i.e., the logarithm of
output grid is the uniform grid.
Note that the shape of all three parameters must be fully defined and equal
to each other. The shape is used to determine the dimension of the grid.
Note that all the parameters are supplied and returned for the original space
and not the log-space.
#### Examples
```python
dtype = np.float64
min_x, max_x, sizes = [0.1], [3.0], [5]
# Here min_x and max_x are in the original space and *not* in the log-space.
grid = log_uniform_grid(min_x, max_x, sizes,dtype=dtype)
with tf.Session() as sess:
grid = sess.run(grid)
# Note that the minimum and maximum grid locations are the same as min_x and
# max_x.
print('locations: ', grid.locations)
# locations: [array([ 0.1, 0.234, 0.548, 1.282, 3.0])]
print('grid: ', grid.grid)
# grid: array([[ 0.1], [0.234], [0.548], [1.282], [ 3.0]])
print('deltas: ', grid.deltas)
# deltas: [array([ 0.134, 0.314, 0.734, 1.718])]
```
Args:
minimums: Real `Tensor` of rank 1 containing the lower end points of the
output grid. Must have the same shape as those of `maximums` and `sizes`
args.
maximums: `Tensor` of the same dtype and shape as `minimums`. The upper
endpoints of the output grid.
sizes: Integer `Tensor` of the same shape as `minimums`. The size of the
grid in each axis. Each entry must be greater than or equal to 2 (i.e. the
sizes include the end points).
dtype: Optional tf.dtype. The default dtype to use for the grid.
validate_args: Python boolean indicating whether to validate the supplied
arguments. The validation checks performed are (a) `maximums` > `minimums`
(b) `minimums` > 0.0 (c) `sizes` >= 2.
name: Python str. The name prefixed to the ops created by this function. If
not supplied, the default name 'uniform_grid_spec' is used.
Returns:
An instance of `GridSpec`.
Raises:
ValueError if the shape of maximums, minimums and sizes are not fully
defined or they are not identical to each other or they are not rank 1.
"""
with tf.compat.v1.name_scope(name, 'log_uniform_grid',
[minimums, maximums, sizes]):
minimums = tf.convert_to_tensor(minimums, dtype=dtype, name='minimums')
maximums = tf.convert_to_tensor(maximums, dtype=dtype, name='maximums')
sizes = tf.convert_to_tensor(sizes, name='sizes')
# Check that the shape of `sizes` is statically defined.
if not _check_shapes_fully_defined(minimums, maximums, sizes):
raise ValueError('The shapes of minimums, maximums and sizes '
'must be fully defined.')
if not (minimums.shape == maximums.shape
and minimums.shape == sizes.shape):
raise ValueError(
'The shapes of minimums, maximums and sizes must be '
'identical.')
if len(minimums.shape.as_list()) != 1:
raise ValueError(
'The minimums, maximums and sizes must all be rank 1.')
control_deps = []
if validate_args:
control_deps = [
tf.debugging.assert_greater(maximums, minimums),
tf.debugging.assert_greater(minimums,
tf.constant(0, dtype=dtype)),
tf.debugging.assert_greater_equal(sizes, 2)
]
# Generate a uniform grid in the log-space taking into account that the
# arguments were already validated.
with tf.compat.v1.control_dependencies(control_deps):
dim = sizes.shape[0]
log_maximums = tf.log(maximums)
log_minimums = tf.log(minimums)
locations = [
tf.exp(
tf.linspace(log_minimums[i], log_maximums[i],
num=sizes[i])) for i in range(dim)
]
grid = tf.stack(tf.meshgrid(*locations, indexing='ij'), axis=-1)
deltas = [location[1:] - location[:-1] for location in locations]
return GridSpec(dim=tf.convert_to_tensor(dim, name='dim'),
minimums=tf.unstack(minimums, axis=0),
maximums=tf.unstack(maximums, axis=0),
sizes=tf.unstack(sizes, axis=0),
deltas=deltas,
locations=locations,
grid=grid)
def meshgrid(*args, **kwargs):
""" See https://www.tensorflow.org/versions/master/api_docs/python/tf/meshgrid .
"""
return tensorflow.meshgrid(*args, **kwargs)
def reorg_layer(self, feature_map, anchors):
'''
feature_map: a feature_map from [feature_map_1, feature_map_2, feature_map_3] returned
from `forward` function
anchors: shape: [3, 2]
'''
# NOTE: size in [h, w] format! don't get messed up!
grid_size = tf.shape(feature_map)[1:3] # [13, 13]
# the downscale ratio in height and weight
ratio = tf.cast(self.img_size / grid_size, tf.float32)
# rescale the anchors to the feature_map
# NOTE: the anchor is in [w, h] format!
rescaled_anchors = [(anchor[0] / ratio[1], anchor[1] / ratio[0])
for anchor in anchors]
feature_map = tf.reshape(
feature_map,
[-1, grid_size[0], grid_size[1], 3, 5 + self.class_num])
# split the feature_map along the last dimension
# shape info: take 416x416 input image and the 13*13 feature_map for example:
# box_centers: [N, 13, 13, 3, 2] last_dimension: [center_x, center_y]
# box_sizes: [N, 13, 13, 3, 2] last_dimension: [width, height]
# conf_logits: [N, 13, 13, 3, 1]
# prob_logits: [N, 13, 13, 3, class_num]
box_centers, box_sizes, conf_logits, prob_logits = tf.split(
feature_map, [2, 2, 1, self.class_num], axis=-1)
box_centers = tf.nn.sigmoid(box_centers)
# use some broadcast tricks to get the mesh coordinates
grid_x = tf.range(grid_size[1], dtype=tf.int32)
grid_y = tf.range(grid_size[0], dtype=tf.int32)
grid_x, grid_y = tf.meshgrid(grid_x, grid_y)
x_offset = tf.reshape(grid_x, (-1, 1))
y_offset = tf.reshape(grid_y, (-1, 1))
x_y_offset = tf.concat([x_offset, y_offset], axis=-1)
# shape: [13, 13, 1, 2]
x_y_offset = tf.cast(
tf.reshape(x_y_offset, [grid_size[0], grid_size[1], 1, 2]),
tf.float32)
# get the absolute box coordinates on the feature_map
box_centers = box_centers + x_y_offset
# rescale to the original image scale
box_centers = box_centers * ratio[::-1]
# avoid getting possible nan value with tf.clip_by_value
box_sizes = tf.exp(box_sizes) * rescaled_anchors
# box_sizes = tf.clip_by_value(tf.exp(box_sizes), 1e-9, 100) * rescaled_anchors
# rescale to the original image scale
box_sizes = box_sizes * ratio[::-1]
# shape: [N, 13, 13, 3, 4]
# last dimension: (center_x, center_y, w, h)
boxes = tf.concat([box_centers, box_sizes], axis=-1)
# shape:
# x_y_offset: [13, 13, 1, 2]
# boxes: [N, 13, 13, 3, 4], rescaled to the original image scale
# conf_logits: [N, 13, 13, 3, 1]
# prob_logits: [N, 13, 13, 3, class_num]
return x_y_offset, boxes, conf_logits, prob_logits
cumprod = _tf.math.cumprod
# noinspection PyShadowingNames
def identity(n, dtype_str='float32', batch_shape=None, dev_str=None):
dtype = _tf.__dict__[dtype_str]
if dev_str:
with _tf.device('/' + dev_str.upper()):
return _tf.eye(n, n, batch_shape=batch_shape, dtype=dtype)
else:
return _tf.eye(n, n, batch_shape=batch_shape, dtype=dtype)
TF_SCATTER_VAR = {}
meshgrid = lambda *xs, indexing='ij': _tf.meshgrid(*xs, indexing=indexing)
# noinspection PyShadowingNames
def scatter_flat(indices, updates, size, reduction='sum', dev_str=None):
if dev_str is None:
dev_str = _dev_str_callable(updates)
dtype = updates.dtype
if reduction == 'sum':
return _tf.scatter_nd(_tf.expand_dims(indices, -1), updates, [size])
elif reduction == 'min':
func = _tf.compat.v1.scatter_min
initial_val = _tf.cast(_tf.constant(2**31 - 1), dtype)
elif reduction == 'max':
func = _tf.compat.v1.scatter_max
initial_val = _tf.cast(_tf.constant(-(2**31 - 1)), dtype)
def _make_grid(nx=20, ny=20):
# yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])
# return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()
xv, yv = tf.meshgrid(tf.range(nx), tf.range(ny))
return tf.cast(tf.reshape(tf.stack([xv, yv], 2), [1, 1, ny * nx, 2]),
dtype=tf.float32)
def dropblock(net,
is_training,
keep_prob,
dropblock_size,
data_format='channels_first'):
"""DropBlock: a regularization method for convolutional neural networks.
DropBlock is a form of structured dropout, where units in a contiguous
region of a feature map are dropped together. DropBlock works better than
dropout on convolutional layers due to the fact that activation units in
convolutional layers are spatially correlated.
See https://arxiv.org/pdf/1810.12890.pdf for details.
Args:
net: `Tensor` input tensor.
is_training: `bool` for whether the model is training.
keep_prob: `float` or `Tensor` keep_prob parameter of DropBlock. "None"
means no DropBlock.
dropblock_size: `int` size of blocks to be dropped by DropBlock.
data_format: `str` either "channels_first" for `[batch, channels, height,
width]` or "channels_last for `[batch, height, width, channels]`.
Returns:
A version of input tensor with DropBlock applied.
Raises:
if width and height of the input tensor are not equal.
"""
if not is_training or keep_prob is None:
return net
tf.logging.info(
'Applying DropBlock: dropblock_size {}, net.shape {}'.format(
dropblock_size, net.shape))
if data_format == 'channels_last':
_, width, height, _ = net.get_shape().as_list()
else:
_, _, width, height = net.get_shape().as_list()
if width != height:
raise ValueError('Input tensor with width!=height is not supported.')
dropblock_size = min(dropblock_size, width)
# seed_drop_rate is the gamma parameter of DropBlcok.
seed_drop_rate = (1.0 - keep_prob) * width**2 / dropblock_size**2 / (
width - dropblock_size + 1)**2
# Forces the block to be inside the feature map.
w_i, h_i = tf.meshgrid(tf.range(width), tf.range(width))
valid_block_center = tf.logical_and(
tf.logical_and(w_i >= int(dropblock_size // 2),
w_i < width - (dropblock_size - 1) // 2),
tf.logical_and(h_i >= int(dropblock_size // 2),
h_i < width - (dropblock_size - 1) // 2))
valid_block_center = tf.expand_dims(valid_block_center, 0)
valid_block_center = tf.expand_dims(
valid_block_center, -1 if data_format == 'channels_last' else 0)
randnoise = tf.random_uniform(net.shape, dtype=tf.float32)
block_pattern = (
1 - tf.cast(valid_block_center, dtype=tf.float32) + tf.cast(
(1 - seed_drop_rate), dtype=tf.float32) + randnoise) >= 1
block_pattern = tf.cast(block_pattern, dtype=tf.float32)
if dropblock_size == width:
block_pattern = tf.reduce_min(
block_pattern,
axis=[1, 2] if data_format == 'channels_last' else [2, 3],
keepdims=True)
else:
if data_format == 'channels_last':
ksize = [1, dropblock_size, dropblock_size, 1]
else:
ksize = [1, 1, dropblock_size, dropblock_size]
block_pattern = -tf.nn.max_pool(
-block_pattern,
ksize=ksize,
strides=[1, 1, 1, 1],
padding='SAME',
data_format='NHWC' if data_format == 'channels_last' else 'NCHW')
percent_ones = tf.cast(tf.reduce_sum(
(block_pattern)), tf.float32) / tf.cast(tf.size(block_pattern),
tf.float32)
net = net / tf.cast(percent_ones, net.dtype) * tf.cast(
block_pattern, net.dtype)
return net
x0 = tf.constant([1.5, 1.5], dtype=tf.float32),
n_steps = 10
sol, = opt.minimize(x0, **kwargs)
if kwargs['return_intermediate']:
sol = sol.stack()
sol.set_shape((n_steps, 2))
loss = f(x0)
for s in tf.unstack(sol, axis=0):
loss = loss * decay + f(s)
else:
loss = f(sol)
grid = tf.meshgrid(tf.linspace(-2.0, 2.0, 51),
tf.linspace(-2.0, 2.0, 51),
indexing='ij')
grid = tf.stack(grid, axis=-1)
f_vals = f(grid)
f0_val = f(x0)
sol_vals = f(sol)
with tf.Session() as sess:
g, f, f0, s, sv, lv = sess.run((grid, f_vals, f0_val, sol, sol_vals, loss))
print('loss: %s' % str(lv))
def vis(g, f, f0, s, sv):
from mayavi import mlab
x, y = s.T
def Re_estimate(z, GT, score_scale):
sigma = 2
pi = tf.constant(math.pi)
cx = tf.range(start=0, limit=tf.shape(z)[2])
cy = tf.range(start=0, limit=tf.shape(z)[1])
cx = tf.cast(cx, tf.float32)
cy = tf.cast(cy, tf.float32)
coord_x, coord_y = tf.meshgrid(cx, cy)
cx = cx[tf.newaxis, tf.newaxis, tf.newaxis, tf.newaxis, :]
cy = cy[tf.newaxis, tf.newaxis, tf.newaxis, :, tf.newaxis]
gt_x = GT[:, :, :, 0:1]
gt_y = GT[:, :, :, 1:2]
gt_w = GT[:, :, :, 2:3]
gt_x = gt_x[:, :, :, :, tf.newaxis]
gt_y = gt_y[:, :, :, :, tf.newaxis]
gt_w = gt_w[:, :, :, :, tf.newaxis]
dx = cx - gt_x
dy = cy - gt_y
dx = tf.multiply(dx, dx)
dy = tf.multiply(dy, dy)
dis = dx + dy
dis = -dis / (2 * sigma**2)
dis = tf.exp(dis)
dis = dis * 1 / (2 * pi * sigma**2)
gt_w = -1 * tf.multiply(score_scale, gt_w)
gt_w = tf.nn.softmax(gt_w, axis=2)
dis = tf.multiply(dis, gt_w)
dis = tf.reduce_sum(dis, axis=[2], keepdims=False)
dis_sum = tf.reduce_sum(dis, axis=[1], keepdims=True)
dis_sum = tf.clip_by_value(dis_sum, 0.0, 1.0)
background = 1 - dis_sum
background = background / tf.reduce_sum(background)
dis = tf.concat([dis, background], axis=1)
prob = dis / (tf.reduce_sum(dis, axis=[1], keepdims=True))
z = tf.transpose(z, perm=[0, 3, 1, 2])
decoupled_density = tf.multiply(z, prob)
background_density = decoupled_density[:, -1:, :, :]
decoupled_density = decoupled_density[:, :-1, :, :]
loss = tf.reduce_sum(tf.abs(
tf.reduce_sum(decoupled_density, axis=[2, 3], keepdims=False) - 1),
axis=[1],
keepdims=False)
loss = tf.reduce_mean(loss, axis=[0])
zero_count = tf.reduce_mean(tf.abs(
tf.reduce_sum(background_density, axis=[2, 3], keepdims=False) - 0),
axis=[0, 1],
keepdims=False)
loss = loss + zero_count
grid = tf.concat([coord_x[:, :, tf.newaxis], coord_y[:, :, tf.newaxis]],
axis=2)[tf.newaxis, tf.newaxis, :, :, :]
decoupled_density_normal = decoupled_density / tf.reduce_sum(
decoupled_density, axis=[2, 3], keepdims=True)
expectation = tf.multiply(decoupled_density_normal[:, :, :, :, tf.newaxis],
grid)
expectation = tf.reduce_sum(expectation, axis=[2, 3], keepdims=False)
grid_minus_mean = grid - expectation[:, :, tf.newaxis, tf.newaxis, :]
x_u_2 = tf.multiply(grid_minus_mean, grid_minus_mean)
variance = tf.multiply(decoupled_density_normal[:, :, :, :, tf.newaxis],
x_u_2)
variance = tf.reduce_sum(variance, axis=[2, 3], keepdims=False)
gt_loc = tf.concat([gt_x, gt_y], axis=3)
gt_loc = tf.multiply(gt_w, gt_loc)
gt_loc = tf.reduce_sum(gt_loc, axis=[2, 4], keepdims=False)
loc_loss = tf.reduce_sum(tf.nn.relu(tf.abs(expectation - gt_loc) - 0.5),
axis=[1, 2],
keepdims=False)
loc_loss = tf.reduce_mean(loc_loss, axis=0)
loss = loss + 0.25 * loc_loss
return loss, tf.reduce_sum(decoupled_density, axis=[2, 3],
keepdims=False), expectation, variance
def directional_attention_with_dense(rep_tensor,
rep_mask,
direction=None,
scope=None,
keep_prob=1.,
is_train=None,
wd=0.,
activation='elu',
tensor_dict=None,
name=None):
def scaled_tanh(x, scale=5.):
return scale * tf.nn.tanh(1. / scale * x)
bs, sl, vec = tf.shape(rep_tensor)[0], tf.shape(rep_tensor)[1], tf.shape(
rep_tensor)[2]
ivec = rep_tensor.get_shape()[2]
with tf.variable_scope(scope or 'directional_attention_%s' % direction
or 'diag'):
# mask generation
sl_indices = tf.range(sl, dtype=tf.int32)
sl_col, sl_row = tf.meshgrid(sl_indices, sl_indices)
if direction is None:
direct_mask = tf.cast(
tf.diag(-tf.ones([sl], tf.int32)) + 1, tf.bool)
else:
if direction == 'forward':
direct_mask = tf.greater(sl_row, sl_col)
else:
direct_mask = tf.greater(sl_col, sl_row)
direct_mask_tile = tf.tile(tf.expand_dims(direct_mask, 0),
[bs, 1, 1]) # bs,sl,sl
rep_mask_tile = tf.tile(tf.expand_dims(rep_mask, 1),
[1, sl, 1]) # bs,sl,sl
attn_mask = tf.logical_and(direct_mask_tile, rep_mask_tile) # bs,sl,sl
# non-linear
rep_map = bn_dense_layer(rep_tensor, ivec, True, 0., 'bn_dense_map',
activation, False, wd, keep_prob, is_train)
rep_map_tile = tf.tile(tf.expand_dims(rep_map, 1),
[1, sl, 1, 1]) # bs,sl,sl,vec
rep_map_dp = dropout(rep_map, keep_prob, is_train)
# attention
with tf.variable_scope('attention'): # bs,sl,sl,vec
f_bias = tf.get_variable('f_bias', [ivec], tf.float32,
tf.constant_initializer(0.))
dependent = linear(rep_map_dp,
ivec,
False,
scope='linear_dependent') # bs,sl,vec
dependent_etd = tf.expand_dims(dependent, 1) # bs,1,sl,vec
head = linear(rep_map_dp, ivec, False,
scope='linear_head') # bs,sl,vec
head_etd = tf.expand_dims(head, 2) # bs,sl,1,vec
logits = scaled_tanh(dependent_etd + head_etd + f_bias,
5.0) # bs,sl,sl,vec
logits_masked = exp_mask_for_high_rank(logits, attn_mask)
attn_score = tf.nn.softmax(logits_masked, 2) # bs,sl,sl,vec
attn_score = mask_for_high_rank(attn_score, attn_mask)
attn_result = tf.reduce_sum(attn_score * rep_map_tile,
2) # bs,sl,vec
with tf.variable_scope('output'):
o_bias = tf.get_variable('o_bias', [ivec], tf.float32,
tf.constant_initializer(0.))
# input gate
fusion_gate = tf.nn.sigmoid(
linear(rep_map, ivec, True, 0., 'linear_fusion_i', False, wd,
keep_prob, is_train) +
linear(attn_result, ivec, True, 0., 'linear_fusion_a', False,
wd, keep_prob, is_train) + o_bias)
output = fusion_gate * rep_map + (1 - fusion_gate) * attn_result
output = mask_for_high_rank(output, rep_mask)
# save attn
if tensor_dict is not None and name is not None:
tensor_dict[name + '_dependent'] = dependent
tensor_dict[name + '_head'] = head
tensor_dict[name] = attn_score
tensor_dict[name + '_gate'] = fusion_gate
return output
def generate_2d_gaussian():
sigma = wavelength*20.
x = np.linspace(-0.5, 0.5, slm_shape[0]) * dd * slm_shape[0]
y = np.linspace(-0.5, 0.5, slm_shape[1]) * dd * slm_shape[1]
Y, X = tf.meshgrid(y, x, indexing='xy')
return 1/np.sqrt(2*np.pi)*np.exp(-0.5 * ((X / sigma) ** 2 + (Y / sigma) ** 2))
def knot_weights(positions,
num_knots,
degree,
cyclical,
sparse_mode=False,
name=None):
"""Function that converts cardinal B-spline positions to knot weights.
Note:
In the following, A1 to An are optional batch dimensions.
Args:
positions: A tensor with shape `[A1, .. An]`. Positions must be between `[0,
C - D)` for non-cyclical and `[0, C)` for cyclical splines, where `C` is
the number of knots and `D` is the spline degree.
num_knots: A strictly positive `int` describing the number of knots in the
spline.
degree: An `int` describing the degree of the spline, which must be smaller
than `num_knots`.
cyclical: A `bool` describing whether the spline is cyclical.
sparse_mode: A `bool` describing whether to return a result only for the
knots with nonzero weights. If set to True, the function returns the
weights of only the `degree` + 1 knots that are non-zero, as well as the
indices of the knots.
name: A name for this op. Defaults to "bspline_knot_weights".
Returns:
A tensor with dense weights for each control point, with the shape
`[A1, ... An, C]` if `sparse_mode` is False.
Otherwise, returns a tensor of shape `[A1, ... An, D + 1]` that contains the
non-zero weights, and a tensor with the indices of the knots, with the type
tf.int32.
Raises:
ValueError: If degree is greater than 4 or num_knots - 1, or less than 0.
InvalidArgumentError: If positions are not in the right range.
"""
with tf.compat.v1.name_scope(name, "bspline_knot_weights", [positions]):
positions = tf.convert_to_tensor(value=positions)
if degree > 4 or degree < 0:
raise ValueError("Degree should be between 0 and 4.")
if degree > num_knots - 1:
raise ValueError("Degree cannot be >= number of knots.")
all_basis_functions = {
# Maps valid degrees to functions.
Degree.CONSTANT: _constant,
Degree.LINEAR: _linear,
Degree.QUADRATIC: _quadratic,
Degree.CUBIC: _cubic,
Degree.QUARTIC: _quartic
}
basis_functions = all_basis_functions[degree]
if not cyclical and num_knots - degree == 1:
# In this case all weights are non-zero and we can just return them.
if not sparse_mode:
return basis_functions(positions)
else:
shift = tf.zeros_like(positions, dtype=tf.int32)
return basis_functions(positions), shift
shape_batch = tf.shape(input=positions)
positions = tf.reshape(positions, shape=(-1,))
# Calculate the nonzero weights from the decimal parts of positions.
shift = tf.floor(positions)
sparse_weights = basis_functions(positions - shift)
shift = tf.cast(shift, tf.int32)
if sparse_mode:
# Returns just the weights and the shift amounts, so that tf.gather_nd on
# the knots can be used to sparsely activate knots if needed.
shape_weights = tf.concat(
(shape_batch, tf.constant((degree + 1,), dtype=tf.int32)), axis=0)
sparse_weights = tf.reshape(sparse_weights, shape=shape_weights)
shift = tf.reshape(shift, shape=shape_batch)
return sparse_weights, shift
num_positions = tf.size(input=positions)
ind_row, ind_col = tf.meshgrid(
tf.range(num_positions, dtype=tf.int32),
tf.range(degree + 1, dtype=tf.int32),
indexing="ij")
tiled_shifts = tf.reshape(
tf.tile(tf.expand_dims(shift, axis=-1), multiples=(1, degree + 1)),
shape=(-1,))
ind_col = tf.reshape(ind_col, shape=(-1,)) + tiled_shifts
if cyclical:
ind_col = tf.math.mod(ind_col, num_knots)
indices = tf.stack((tf.reshape(ind_row, shape=(-1,)), ind_col), axis=-1)
shape_indices = tf.concat((tf.reshape(
num_positions, shape=(1,)), tf.constant(
(degree + 1, 2), dtype=tf.int32)),
axis=0)
indices = tf.reshape(indices, shape=shape_indices)
shape_scatter = tf.concat((tf.reshape(
num_positions, shape=(1,)), tf.constant((num_knots,), dtype=tf.int32)),
axis=0)
weights = tf.scatter_nd(indices, sparse_weights, shape_scatter)
shape_weights = tf.concat(
(shape_batch, tf.constant((num_knots,), dtype=tf.int32)), axis=0)
return tf.reshape(weights, shape=shape_weights)
def length(sequence, embedding_size):
used = tf.sign(sequence)
length = tf.reduce_sum(used, reduction_indices=1)
y = tf.Variable(tf.constant(0.1, shape=[embedding_size]))
return tf.meshgrid(length, y)[0]
def generate_2d_guassian(self, height, width, y0, x0, sigma=1):
"""
"The same technique as Tompson et al. is used for supervision. A MeanSquared Error (MSE) loss is
applied comparing the predicted heatmap to a ground-truth heatmap consisting of a 2D gaussian
(with standard deviation of 1 px) centered on the joint location."
https://github.com/princeton-vl/pose-hg-train/blob/master/src/util/img.lua#L204
"""
heatmap = tf.zeros((height, width))
return heatmap
# this gaussian patch is 7x7, let's get four corners of it first
xmin = x0 - 3 * sigma
ymin = y0 - 3 * sigma
xmax = x0 + 3 * sigma
ymax = y0 + 3 * sigma
# if the patch is out of image boundary we simply return nothing according to the source code
if xmin >= width or ymin >= height or xmax < 0 or ymax < 0:
return heatmap
size = 6 * sigma + 1
x, y = tf.meshgrid(tf.range(0, 6 * sigma + 1, 1),
tf.range(0, 6 * sigma + 1, 1),
indexing='xy')
# the center of the gaussian patch should be 1
center_x = size // 2
center_y = size // 2
# generate this 7x7 gaussian patch
gaussian_patch = tf.cast(tf.math.exp(
-(tf.square(x - center_x) + tf.math.square(y - center_y)) /
(tf.math.square(sigma) * 2)),
dtype=tf.float32)
# part of the patch could be out of the boundary, so we need to determine the valid range
# if xmin = -2, it means the 2 left-most columns are invalid, which is max(0, -(-2)) = 2
patch_xmin = tf.math.maximum(0, -xmin)
patch_ymin = tf.math.maximum(0, -ymin)
# if xmin = 59, xmax = 66, but our output is 64x64, then we should discard 2 right-most columns
# which is min(64, 66) - 59 = 5, and column 6 and 7 are discarded
patch_xmax = tf.math.minimum(xmax, width) - xmin
patch_ymax = tf.math.minimum(ymax, height) - ymin
# also, we need to determine where to put this patch in the whole heatmap
heatmap_xmin = tf.math.maximum(0, xmin)
heatmap_ymin = tf.math.maximum(0, ymin)
heatmap_xmax = tf.math.minimum(xmax, width)
heatmap_ymax = tf.math.minimum(ymax, height)
# finally, insert this patch into the heatmap
indices = tf.TensorArray(tf.int32, 1, dynamic_size=True)
updates = tf.TensorArray(tf.float32, 1, dynamic_size=True)
count = 0
for j in tf.range(patch_ymin, patch_ymax):
for i in tf.range(patch_xmin, patch_xmax):
indices = indices.write(count,
[heatmap_ymin + j, heatmap_xmin + i])
updates = updates.write(count, gaussian_patch[j][i])
count += 1
heatmap = tf.tensor_scatter_nd_update(heatmap, indices.stack(),
updates.stack())
# unfortunately, the code below doesn't work because
# tensorflow.python.framework.ops.EagerTensor' object does not support item assignment
# heatmap[heatmap_ymin:heatmap_ymax, heatmap_xmin:heatmap_xmax] = gaussian_patch[patch_ymin:patch_ymax,patch_xmin:patch_xmax]
return heatmap
def forward(self, x, y=None, is_training=False):
img_h, img_w = x.get_shape().as_list()[1:3]
logits = backbone(x, self.NUM_CLASSES, self.NUM_PER_LAYER, is_training)
outputs = []
losses = 0.
for i, logit in enumerate(logits):
num_batch = tf.cast(tf.shape(logit)[0], tf.float32)
anchors = self.ANCHORS[self.ANCHOR_MASK[i]]
grid_h, grid_w = logit.get_shape().as_list()[1:3]
logit = tf.reshape(
logit,
[-1, grid_h, grid_w, self.NUM_PER_LAYER, self.NUM_CLASSES + 5])
anchors = tf.cast(
tf.reshape(anchors, [1, 1, 1, self.NUM_PER_LAYER, 2]),
tf.float32)
grid_x, grid_y = tf.meshgrid(tf.range(grid_w, dtype=tf.float32),
tf.range(grid_h, dtype=tf.float32))
grid_x = tf.expand_dims(grid_x, axis=-1)
grid_y = tf.expand_dims(grid_y, axis=-1)
grid = tf.concat([grid_x, grid_y], axis=-1)
grid = tf.expand_dims(grid, axis=-2)
pred_xcyc = tf.nn.sigmoid(logit[..., :2]) + grid
pred_wh = tf.exp(logit[..., 2:4]) * anchors
pred_conf = tf.nn.sigmoid(logit[..., 4:5])
pred_cls = logit[..., 5:]
if is_training:
true_xcyc = y[i][..., 0:2]
true_wh = y[i][..., 2:4]
true_conf = y[i][..., 4:5]
true_cls = y[i][..., 5:]
###############
# ignore mask #
###############
true_boxes = tf.py_func(self.pick_out_gt_box, [y[i]],
[tf.float32])[0]
true_boxes_xy = true_boxes[..., 0:2]
true_boxes_wh = true_boxes[..., 2:4]
## for broadcasting
logits_xy = tf.expand_dims(pred_xcyc, axis=-2)
logits_wh = tf.expand_dims(pred_wh, axis=-2)
logits_wh_half = logits_wh / 2.
logits_min = logits_xy - logits_wh_half
logits_max = logits_xy + logits_wh_half
true_boxes_wh_half = true_boxes_wh / 2.
true_boxes_min = true_boxes_xy - true_boxes_wh_half
true_boxes_max = true_boxes_xy + true_boxes_wh_half
intersect_min = tf.maximum(logits_min, true_boxes_min)
intersect_max = tf.minimum(logits_max, true_boxes_max)
intersect_wh = tf.maximum(intersect_max - intersect_min, 0.)
intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
logits_area = logits_wh[..., 0] * logits_wh[..., 1]
true_boxes_area = true_boxes_wh[..., 0] * true_boxes_wh[..., 1]
union_area = logits_area + true_boxes_area - intersect_area
iou_scores = intersect_area / union_area
best_ious = tf.reduce_max(iou_scores, axis=-1, keepdims=True)
ignore_mask = tf.cast(best_ious < self.IOU_THRESHOLD,
tf.float32)
obj_mask = true_conf
box_scale = tf.exp(true_wh) * tf.cast(anchors, tf.float32)
box_scale = 2 - box_scale[..., 0:1] * box_scale[..., 1:2]
xy_delta = obj_mask * (pred_xcyc - true_xcyc) * box_scale
wh_delta = obj_mask * (pred_wh - true_wh) * box_scale
conf_delta = obj_mask * (pred_conf-true_conf) * 5. +\
(1-obj_mask) * (pred_conf-true_conf) * ignore_mask
cls_delta = obj_mask * \
tf.nn.sigmoid_cross_entropy_with_logits(labels=true_cls, logits=pred_cls)
loss_xy = tf.reduce_sum(tf.square(xy_delta)) / num_batch
loss_wh = tf.reduce_sum(tf.square(wh_delta)) / num_batch
loss_conf = tf.reduce_sum(tf.square(conf_delta)) / num_batch
loss_class = tf.reduce_sum(cls_delta) / num_batch
loss = loss_xy + loss_wh + loss_conf + loss_class
losses += loss
else:
stride = [img_w // grid_w, img_h // grid_h]
pred_xcyc = pred_xcyc * stride
pred_cls = tf.nn.sigmoid(logit[..., 5:])
outputs.append(
tf.concat([pred_xcyc, pred_wh, pred_conf, pred_cls],
axis=-1))
if is_training:
return losses
else:
return outputs
# 5.6.5 scatter_nd
# create the pos to be refreshed
indices = tf.constant([[4], [3], [1], [7]])
# create data
updates = tf.constant([4.4, 3.3, 1.1, 7.7])
# write in updates on vector with all zeros and length 8 according to indices
print(tf.scatter_nd(indices, updates, [8]))
# create the pos for written in
indices = tf.constant([[1], [3]])
# create data for written in
updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8,
8]],
[[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4,
4]]])
# write in updates on a [4, 4, 4] tensor according to indices
print(tf.scatter_nd(indices, updates, [4, 4, 4]))
# 5.6.6 meshgrid
x = tf.linspace(-8., 8., 100)
y = tf.linspace(-8., 8., 100)
x, y = tf.meshgrid(x, y)
print(x.shape, y.shape)
z = tf.sqrt(x**2 + y**2)
z = tf.sin(z) / z
fig = plt.figure()
ax = Axes3D(fig)
ax.contour3D(x.numpy(), y.numpy(), z.numpy(), levels=100)
plt.show()
def K_meshgrid(x, y):
return tf.meshgrid(x, y)
def permutohedral_prepare(position_vectors):
batch_size = int(position_vectors.shape[0])
nCh=int(position_vectors.shape[-1])
nVoxels=int(position_vectors.shape.num_elements())//batch_size//nCh
# reshaping batches and voxels into one dimension means we can use 1D gather and hashing easily
position_vectors=tf.reshape(position_vectors,[-1,nCh])
## Generate position vectors in lattice space
x=position_vectors/(numpy.sqrt(2./3.)*(nCh+1))
# Embed in lattice space using black magic from the permutohedral paper
alpha=lambda i:numpy.sqrt(float(i)/(float(i)+1.))
Ex=[None]*(nCh+1)
Ex[nCh]=-alpha(nCh)*x[:,nCh-1]
for dit in range(nCh-1,0,-1):
Ex[dit]=-alpha(dit)*x[:,dit-1]+x[:,dit]/alpha(dit+1)+Ex[dit+1]
Ex[0]=x[:,0]/alpha(1)+Ex[1]
Ex=tf.stack(Ex,1)
## Compute coordinates
# Get closest remainder-0 point
v=tf.to_int32(tf.round(Ex*(1./float(nCh+1))))
rem0=v*(nCh+1)
sumV=tf.reduce_sum(v,1,keep_dims=True)
# Find the simplex we are in and store it in rank (where rank describes what position coorinate i has
#in the sorted order of the features values)
di=Ex-tf.to_float(rem0)
_,index=tf.nn.top_k(di,nCh+1,sorted=True)
_,rank=tf.nn.top_k(-index,nCh+1,sorted=True) # This can be done more efficiently if necessary following the permutohedral paper
# if the point doesn't lie on the plane (sum != 0) bring it back
rank=tf.to_int32(rank)+sumV
addMinusSub=tf.to_int32(rank<0)*(nCh+1)-tf.to_int32(rank>=nCh+1)*(nCh+1)
rank=rank+addMinusSub
rem0=rem0+addMinusSub
# Compute the barycentric coordinates (p.10 in [Adams etal 2010])
v2=(Ex-tf.to_float(rem0))*(1./float(nCh+1))
# the barycentric coordinates are v_sorted-v_sorted[...,[-1,1:-1]]+[1,0,0,...]
# CRF2RNN uses the calculated ranks to get v2 sorted in O(nCh) time
# We cheat here by using the easy to implement but slower method of sorting again in O(nCh log nCh)
# we might get this even more efficient if we correct the original sorted data above
v_sortedDesc,_=tf.nn.top_k(v2,nCh+1,sorted=True)
v_sorted=tf.reverse(v_sortedDesc,[1])
barycentric=v_sorted-tf.concat([v_sorted[:,-1:]-1.,v_sorted[:,0:nCh]],1)
# Compute all vertices and their offset
canonical = [[i]*(nCh+1-i)+[i-nCh-1]*i for i in range(nCh+1)]
# WARNING: This hash function does not guarantee uniqueness of different position_vectors
hashVector = tf.constant(numpy.power(int(numpy.floor(numpy.power(tf.int64.max,1./(nCh+2)))),[range(1,nCh+1)]),dtype=tf.int64)
hash=lambda key: tf.reduce_sum(tf.to_int64(key)*hashVector,1)
hashtable=tf.contrib.lookup.MutableDenseHashTable(tf.int64,tf.int64,default_value=tf.constant([-1]*nCh,dtype=tf.int64),empty_key=-1,initial_num_buckets=8,checkpoint=False)
indextable=tf.contrib.lookup.MutableDenseHashTable(tf.int64,tf.int64,default_value=0,empty_key=-1,initial_num_buckets=8,checkpoint=False)
numSimplexCorners=nCh+1
keys=[None]*numSimplexCorners
i64keys=[None]*numSimplexCorners
insertOps=[]
for scit in range(numSimplexCorners):
keys[scit] = tf.gather(canonical[scit],rank[:,:-1])+rem0[:,:-1]
i64keys[scit]=hash(keys[scit])
insertOps.append(hashtable.insert(i64keys[scit],tf.to_int64(keys[scit])))
with tf.control_dependencies(insertOps):
fusedI64Keys,fusedKeys = hashtable.export()
fusedKeys=tf.boolean_mask(fusedKeys,tf.not_equal(fusedI64Keys,-1))
fusedI64Keys=tf.boolean_mask(fusedI64Keys,tf.not_equal(fusedI64Keys,-1))
insertIndices = indextable.insert(fusedI64Keys,tf.expand_dims(tf.transpose(tf.range(1,tf.to_int64(tf.size(fusedI64Keys)+1),dtype=tf.int64)),1))
blurNeighbours1=[None]*(nCh+1)
blurNeighbours2=[None]*(nCh+1)
indices=[None]*(nCh+1)
with tf.control_dependencies([insertIndices]):
for dit in range(nCh+1):
offset=tf.constant([nCh if i==dit else -1 for i in range(nCh)],dtype=tf.int64)
blurNeighbours1[dit]=indextable.lookup(hash(fusedKeys+offset))
blurNeighbours2[dit]=indextable.lookup(hash(fusedKeys-offset))
batch_index=tf.reshape(tf.meshgrid(tf.range(batch_size),tf.zeros([nVoxels],dtype=tf.int32))[0],[-1,1])
indices[dit] = tf.stack([tf.to_int32(indextable.lookup(i64keys[dit])),batch_index[:,0]],1) # where in the splat variable each simplex vertex is
return barycentric,blurNeighbours1,blurNeighbours2,indices
def _build_box(self, box_params, is_training, hw=None):
mean, log_std = tf.split(box_params, 2, axis=-1)
std = self.std_nonlinearity(log_std)
mean = self.training_wheels * tf.stop_gradient(mean) + (1-self.training_wheels) * mean
std = self.training_wheels * tf.stop_gradient(std) + (1-self.training_wheels) * std
cell_y_mean, cell_x_mean, height_mean, width_mean = tf.split(mean, 4, axis=-1)
cell_y_std, cell_x_std, height_std, width_std = tf.split(std, 4, axis=-1)
cell_y_logit = Normal(loc=cell_y_mean, scale=cell_y_std).sample()
cell_x_logit = Normal(loc=cell_x_mean, scale=cell_x_std).sample()
height_logit = Normal(loc=height_mean, scale=height_std).sample()
width_logit = Normal(loc=width_mean, scale=width_std).sample()
# --- cell y/x transform ---
cell_y = tf.nn.sigmoid(tf.clip_by_value(cell_y_logit, -10, 10))
cell_x = tf.nn.sigmoid(tf.clip_by_value(cell_x_logit, -10, 10))
assert self.max_yx > self.min_yx
cell_y = float(self.max_yx - self.min_yx) * cell_y + self.min_yx
cell_x = float(self.max_yx - self.min_yx) * cell_x + self.min_yx
# --- height/width transform ---
height = tf.nn.sigmoid(tf.clip_by_value(height_logit, -10, 10))
width = tf.nn.sigmoid(tf.clip_by_value(width_logit, -10, 10))
assert self.max_hw > self.min_hw
height = float(self.max_hw - self.min_hw) * height + self.min_hw
width = float(self.max_hw - self.min_hw) * width + self.min_hw
local_box = tf.concat([cell_y, cell_x, height, width], axis=-1)
# --- Compute image-normalized box parameters ---
ys = height
xs = width
# box center normalized to anchor box
if hw is None:
w, h = tf.meshgrid(
tf.range(self.W, dtype=tf.float32),
tf.range(self.H, dtype=tf.float32))
h = h[None, :, :, None]
w = w[None, :, :, None]
yt = (self.pixels_per_cell[0] / self.anchor_box[0]) * (cell_y + h)
xt = (self.pixels_per_cell[1] / self.anchor_box[1]) * (cell_x + w)
else:
h, w = hw
yt = (self.pixels_per_cell[0] / self.anchor_box[0]) * (cell_y + h)
xt = (self.pixels_per_cell[1] / self.anchor_box[1]) * (cell_x + w)
normalized_box = tf.concat([yt, xt, ys, xs], axis=-1)
ys_logit = height_logit
xs_logit = width_logit
return dict(
# "raw" box values
cell_y=cell_y,
cell_x=cell_x,
height=height,
width=width,
local_box=local_box,
cell_y_logit_mean=cell_y_mean,
cell_x_logit_mean=cell_x_mean,
height_logit_mean=height_mean,
width_logit_mean=width_mean,
cell_y_logit_std=cell_y_std,
cell_x_logit_std=cell_x_std,
height_logit_std=height_std,
width_logit_std=width_std,
# box center and height/width, in a coordinate frame where (0, 0) is image top-left
# and (1, 1) is image bottom-right
# box center and scale with respect to anchor_box
yt=yt,
xt=xt,
ys=ys,
xs=xs,
normalized_box=normalized_box,
ys_logit=ys_logit,
xs_logit=xs_logit,
)
def meshgrid(*args, **kwargs):
return tensorflow.meshgrid(*args, **kwargs)
def get_propability_map(cv, depth_map, depth_start, depth_interval):
""" get probability map from cost volume """
def _repeat_(x, num_repeats):
""" repeat each element num_repeats times """
x = tf.reshape(x, [-1])
ones = tf.ones((1, num_repeats), dtype='int32')
x = tf.reshape(x, shape=(-1, 1))
x = tf.matmul(x, ones)
return tf.reshape(x, [-1])
shape = tf.shape(depth_map)
batch_size = shape[0]
height = shape[1]
width = shape[2]
depth = tf.shape(cv)[1]
# byx coordinate, batched & flattened
b_coordinates = tf.range(batch_size)
y_coordinates = tf.range(height)
x_coordinates = tf.range(width)
b_coordinates, y_coordinates, x_coordinates = tf.meshgrid(
b_coordinates, y_coordinates, x_coordinates)
b_coordinates = _repeat_(b_coordinates, batch_size)
y_coordinates = _repeat_(y_coordinates, batch_size)
x_coordinates = _repeat_(x_coordinates, batch_size)
# d coordinate (floored and ceiled), batched & flattened
d_coordinates = tf.reshape((depth_map - depth_start) / depth_interval,
[-1])
d_coordinates_left0 = tf.clip_by_value(
tf.cast(tf.floor(d_coordinates), 'int32'), 0, depth - 1)
d_coordinates_left1 = tf.clip_by_value(d_coordinates_left0 - 1, 0,
depth - 1)
d_coordinates1_right0 = tf.clip_by_value(
tf.cast(tf.ceil(d_coordinates), 'int32'), 0, depth - 1)
d_coordinates1_right1 = tf.clip_by_value(d_coordinates1_right0 + 1, 0,
depth - 1)
# voxel coordinates
voxel_coordinates_left0 = tf.stack(
[b_coordinates, d_coordinates_left0, y_coordinates, x_coordinates],
axis=1)
voxel_coordinates_left1 = tf.stack(
[b_coordinates, d_coordinates_left1, y_coordinates, x_coordinates],
axis=1)
voxel_coordinates_right0 = tf.stack(
[b_coordinates, d_coordinates1_right0, y_coordinates, x_coordinates],
axis=1)
voxel_coordinates_right1 = tf.stack(
[b_coordinates, d_coordinates1_right1, y_coordinates, x_coordinates],
axis=1)
# get probability image by gathering and interpolation
prob_map_left0 = tf.gather_nd(cv, voxel_coordinates_left0)
prob_map_left1 = tf.gather_nd(cv, voxel_coordinates_left1)
prob_map_right0 = tf.gather_nd(cv, voxel_coordinates_right0)
prob_map_right1 = tf.gather_nd(cv, voxel_coordinates_right1)
prob_map = prob_map_left0 + prob_map_left1 + prob_map_right0 + prob_map_right1
prob_map = tf.reshape(prob_map, [batch_size, height, width, 1])
return prob_map
def meshgrid(*args, **kwargs):
""" See https://www.tensorflow.org/versions/master/api_docs/python/tf/meshgrid .
"""
return tensorflow.meshgrid(*args, **kwargs)
def YOLOv3Net(cfgfile, model_size, num_classes):
blocks = parse_cfg(cfgfile)
outputs = {}
output_filters = []
filters = []
out_pred = []
scale = 0
inputs = input_image = Input(shape=model_size)
inputs = inputs / 255.0
for i, block in enumerate(blocks[1:]):
# If it is a convolutional layer
if (block["type"] == "convolutional"):
activation = block["activation"]
filters = int(block["filters"])
kernel_size = int(block["size"])
strides = int(block["stride"])
if strides > 1:
inputs = ZeroPadding2D(((1, 0), (1, 0)))(inputs)
inputs = Conv2D(filters,
kernel_size,
strides=strides,
padding='valid' if strides > 1 else 'same',
name='conv_' + str(i),
use_bias=False if ("batch_normalize" in block) else True)(inputs)
if "batch_normalize" in block:
inputs = BatchNormalization(name='bnorm_' + str(i))(inputs)
if activation == "leaky":
inputs = LeakyReLU(alpha=0.1, name='leaky_' + str(i))(inputs)
elif (block["type"] == "upsample"):
stride = int(block["stride"])
inputs = UpSampling2D(stride)(inputs)
# If it is a route layer
elif (block["type"] == "route"):
block["layers"] = block["layers"].split(',')
start = int(block["layers"][0])
if len(block["layers"]) > 1:
end = int(block["layers"][1]) - i
filters = output_filters[i + start] + output_filters[end] # Index negatif :end - index
inputs = tf.concat([outputs[i + start], outputs[i + end]], axis=-1)
else:
filters = output_filters[i + start]
inputs = outputs[i + start]
elif block["type"] == "shortcut":
from_ = int(block["from"])
inputs = outputs[i - 1] + outputs[i + from_]
# Yolo detection layer
elif block["type"] == "yolo":
mask = block["mask"].split(",")
mask = [int(x) for x in mask]
anchors = block["anchors"].split(",")
anchors = [int(a) for a in anchors]
anchors = [(anchors[i], anchors[i + 1]) for i in range(0, len(anchors), 2)]
anchors = [anchors[i] for i in mask]
n_anchors = len(anchors)
out_shape = inputs.get_shape().as_list()
inputs = tf.reshape(inputs, [-1, n_anchors * out_shape[1] * out_shape[2], \
5 + num_classes])
box_centers = inputs[:, :, 0:2]
box_shapes = inputs[:, :, 2:4]
confidence = inputs[:, :, 4:5]
classes = inputs[:, :, 5:num_classes + 5]
box_centers = tf.sigmoid(box_centers)
confidence = tf.sigmoid(confidence)
classes = tf.sigmoid(classes)
anchors = tf.tile(anchors, [out_shape[1] * out_shape[2], 1])
box_shapes = tf.exp(box_shapes) * tf.cast(anchors, dtype=tf.float32)
x = tf.range(out_shape[1], dtype=tf.float32)
y = tf.range(out_shape[2], dtype=tf.float32)
cx, cy = tf.meshgrid(x, y)
cx = tf.reshape(cx, (-1, 1))
cy = tf.reshape(cy, (-1, 1))
cxy = tf.concat([cx, cy], axis=-1)
cxy = tf.tile(cxy, [1, n_anchors])
cxy = tf.reshape(cxy, [1, -1, 2])
strides = (input_image.shape[1] // out_shape[1], \
input_image.shape[2] // out_shape[2])
box_centers = (box_centers + cxy) * strides
prediction = tf.concat([box_centers, box_shapes, confidence, classes], axis=-1)
if scale:
out_pred = tf.concat([out_pred, prediction], axis=1)
else:
out_pred = prediction
scale = 1
outputs[i] = inputs
output_filters.append(filters)
model = Model(input_image, out_pred)
model.summary()
return model
def main():
# Define constants and hyper-parameters
MU_TRUE = 0.0
TAU_TRUE = 1.0
N_SAMPLES = 100
RANDOM_SEED = 42
ALPHA_0 = 1.0
BETA_0 = 1.0
MU_0 = 0.0
TAU_0 = 1.0
# Generate a dataset
X_true = tfp.distributions.Normal(
loc=MU_TRUE,
scale=1.0 / tf.sqrt(TAU_TRUE),
)
dataset = X_true.sample(N_SAMPLES, seed=RANDOM_SEED).numpy()
# Surrogate posterior
alpha_s, beta_s, mu_s, tau_s = true_posterior_parameter(
dataset, ALPHA_0, BETA_0, MU_0, TAU_0)
print("Parameters of the Surrogate Posterior (a Normal times Gamma)")
print(
f"-> alpha_s={alpha_s:1.4f}, beta_s={beta_s:1.4f}, mu_s={mu_s:1.4f}, tau_s={tau_s:1.4f}"
)
# True posterior
alpha_N, beta_N, mu_N, tau_N = true_posterior_parameter(
dataset,
ALPHA_0,
BETA_0,
MU_0,
TAU_0,
)
print("Parameters of the True Posterior (a Normal-Gamma)")
print(
f"-> alpha_N={alpha_N:1.4f}, beta_N={beta_N:1.4f}, mu_N={mu_N:1.4f}, tau_N={tau_N:1.4f}"
)
# Surrogate MAP estimate
mu_MAP_s = mu_s
tau_MAP_s = (alpha_s - 1.0) / beta_s
print("Surrogate MAP estimate")
print(f"-> mu_MAP_s={mu_MAP_s:1.4f}, tau_MAP_s={tau_MAP_s:1.4f}")
# True MAP estimate
mu_MAP_N = mu_N
tau_MAP_N = (alpha_N - 0.5) / beta_N
print("True MAP estimate")
print(f"-> mu_MAP_N={mu_MAP_N:1.4f}, tau_MAP_N={tau_MAP_N:1.4f}")
# Contour plot
mu_range = tf.linspace(-1.0, 1.0, 40)
tau_range = tf.linspace(0.01, 1.5, 40)
# mu_range = tf.cast(mu_range, tf.float32)
# tau_range = tf.cast(tau_range, tf.float32)
mu_mesh, tau_mesh = tf.meshgrid(mu_range, tau_range)
# Has to adhere to the order in the DGM
points_2d = (tau_mesh, mu_mesh)
# Convert to tensorflow objects
alpha_s, beta_s, mu_s, tau_s = tf.convert_to_tensor(
(alpha_s, beta_s, mu_s, tau_s))
alpha_N, beta_N, mu_N, tau_N = tf.convert_to_tensor(
(alpha_N, beta_N, mu_N, tau_N))
surrogate_posterior = tfp.distributions.JointDistributionCoroutineAutoBatched(
lambda: gauss_times_gamma(alpha_s, beta_s, mu_s, tau_s))
true_posterior = tfp.distributions.JointDistributionCoroutineAutoBatched(
lambda: gauss_gamma(alpha_N, beta_N, mu_N, tau_N))
log_prob_true_posterior = true_posterior.log_prob(points_2d)
log_prob_surrogate_posterior = surrogate_posterior.log_prob(points_2d)
plt.figure()
plt.contour(mu_mesh, tau_mesh, log_prob_surrogate_posterior, levels=100)
plt.xlabel("mu")
plt.ylabel("tau")
plt.title("Surrogate Posterior")
plt.figure()
plt.contour(mu_mesh, tau_mesh, log_prob_true_posterior, levels=100)
plt.xlabel("mu")
plt.ylabel("tau")
plt.title("True Posterior")
plt.show()
def meshgrid(*args, **kwargs):
return tf.meshgrid(*args, **kwargs)
def meshgrid(*args, **kwargs):
return tensorflow.meshgrid(*args, **kwargs)
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from BeamGenerator import LG_Beam_at_Waist, Gaussian_Beam_at_Waist_different_position
from BeamGenerator import sd_beam_1, sd_beam_2, sd_beam_3, sd_beam_4
from PhaseRangeAdjust import adjust_PM
import timeit
start = timeit.default_timer()
M = 5
N = 5
a = tf.Variable(2.2, tf.float32)
b = tf.Variable(4.3, tf.float32)
tfuu, tfvv = tf.meshgrid(list(range(M)), list(range(N)))
tfuu = tf.cast(tfuu, tf.float32)
tfvv = tf.cast(tfvv, tf.float32)
L = tf.add(tf.multiply(a, tfuu), tf.multiply(b, tfvv))
loss = -tf.reduce_sum(L)
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
init = tf.global_variables_initializer()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
print(sess.run(tfuu))
print(sess.run(tfvv))
print(sess.run(L))
