def dense_blk(name, l, ch, ksize, count, split=1, padding='valid'):
with tf.variable_scope(name):
for i in range(0, count):
with tf.variable_scope('blk/' + str(i)):
x = BNReLU('preact_bna', l)
x = Conv2D('conv1',
x,
ch[0],
ksize[0],
padding=padding,
activation=BNReLU)
x = Conv2D('conv2',
x,
ch[1],
ksize[1],
padding=padding,
split=split)
##
if padding == 'valid':
x_shape = x.get_shape().as_list()
l_shape = l.get_shape().as_list()
l = crop_op(
l, (l_shape[2] - x_shape[2], l_shape[3] - x_shape[3]))
l = tf.concat([l, x], axis=1)
l = BNReLU('blk_bna', l)
return l
def apply_preactivation(l, preact):
"""
'no_preact' for the first resblock only, because the input is activated already.
'both_preact' for the first block in each group, due to the projection shotcut.
'default' for all the non-first blocks, where identity mapping is preserved on shortcut path.
"""
if preact == 'both_preact':
l = BNReLU('preact', l)
shortcut = l
elif preact == 'default':
shortcut = l
l = BNReLU('preact', l)
else:
shortcut = l
return l, shortcut
def apply_preactivation(x, preact):
if preact == 'bnrelu':
shortcut = x # preserve identity mapping
x = BNReLU('preact', x)
else:
shortcut = x
return x, shortcut
def apply_preactivation(l, preact):
if preact == 'bnrelu':
shortcut = l # preserve identity mapping
l = BNReLU('preact', l)
else:
shortcut = l
return l, shortcut
def get_logits(image, num_classes=1000):
#
with ssdnet_argscope():
# dropblock
if get_current_tower_context().is_training:
dropblock_keep_prob = tf.get_variable('dropblock_keep_prob', (),
dtype=tf.float32,
trainable=False)
else:
dropblock_keep_prob = None
l = image #tf.transpose(image, perm=[0, 2, 3, 1])
# conv1
l = Conv2D('conv1',
l,
16,
4,
strides=2,
activation=None,
padding='SAME')
with tf.variable_scope('conv1'):
l = BNReLU(tf.concat([l, -l], axis=-1))
l = MaxPooling('pool1', l, 2)
# conv2
l = LinearBottleneck('conv2', l, 48, 24, 5, t=1, use_ab=True)
l = l + LinearBottleneck('conv3', l, 24, 24, 5, t=2, use_ab=True)
ch_all = [48, 72, 96]
iters = [2, 4, 4]
mults = [3, 4, 6]
bsize = [3, 3, 3]
hlist = []
for ii, (ch, it, mu, bs) in enumerate(zip(ch_all, iters, mults,
bsize)):
use_ab = (ii < 2)
for jj in range(it):
name = 'inc{}/{}'.format(ii, jj)
stride = 2 if jj == 0 else 1
swap_block = True if jj % 2 == 1 else False
l = inception(name,
l,
ch,
stride,
t=mu,
swap_block=swap_block,
use_ab=use_ab)
l = DropBlock('inc{}/drop'.format(ii),
l,
keep_prob=dropblock_keep_prob,
block_size=bs)
l = Conv2D('convf', l, 96 * 6, 1, activation=None)
l = BatchNorm('convf/bn', l)
l = tf.nn.relu(l)
l = GlobalAvgPooling('poolf', l)
fc = FullyConnected('fc', l, 1280, activation=BNReLU)
fc = Dropout(fc, keep_prob=0.9)
logits = FullyConnected('linear', fc, num_classes, use_bias=True)
return logits
def ssdnet_backbone(image, **kwargs):
#
with ssdnet_argscope():
l = image #tf.transpose(image, perm=[0, 2, 3, 1])
with argscope([BatchNorm], training=False):
# conv1
l = Conv2D('conv1', l, 24, 4, strides=2, activation=None, padding='SAME')
with tf.variable_scope('conv1'):
l = BNReLU(tf.concat([l, -l], axis=-1))
l = MaxPooling('pool1', l, 2)
l = tf.stop_gradient(l)
with argscope([BatchNorm], training=None):
# conv2
l = LinearBottleneck('conv2', l, 48, 24, 3, t=1, use_ab=True)
l = l + LinearBottleneck('conv3', l, 24, 24, 5, t=2, use_ab=True)
ch_all = [48, 72, 96]
iters = [2, 4, 4]
mults = [3, 4, 6]
hlist = []
for ii, (ch, it, mu) in enumerate(zip(ch_all, iters, mults)):
use_ab = (ii < 2)
for jj in range(it):
name = 'inc{}/{}'.format(ii, jj)
stride = 2 if jj == 0 else 1
k = 3 if jj < (it // 2) else 5
swap_block = True if jj % 2 == 1 else False
l = inception(name, l, ch, k, stride, t=mu, swap_block=swap_block, use_ab=use_ab)
hlist.append(l)
return hlist
def resnet_bottleneck(l, ch_out, stride, stride_first=False):
"""
stride_first: original resnet put stride on first conv. fb.resnet.torch put stride on second conv.
"""
shortcut = l
l = Conv2D('conv1', l, ch_out, 1, strides=stride if stride_first else 1, activation=BNReLU)
l_3x3 = Conv2D('conv2', l, ch_out, 3, strides=1 if stride_first else stride, activation=tf.identity)
shape = l_3x3.get_shape().as_list()
l_gap = GlobalAvgPooling('gap', l)
l_gap = FullyConnected('fc1', l_gap, ch_out, activation=tf.nn.relu)
l_gap = FullyConnected('fc2', l_gap, ch_out, activation=tf.identity)
l_gap = tf.reshape(l_gap, [-1, ch_out, 1, 1])
l_gap = tf.tile(l_gap, [1, 1, shape[2], shape[3]])
l_concat = tf.concat([l_3x3, l_gap], axis = 1)
l_concat = Conv2D('conv_c1', l_concat, ch_out, 1, strides=1, activation=tf.nn.relu)
l_concat = Conv2D('conv_c2', l_concat, ch_out, 1, strides=1, activation=tf.identity)
l_concat = tf.sigmoid(l_concat)
l = l_3x3 + l_gap * l_concat
l = BNReLU('conv2',l)
l = Conv2D('conv3', l, ch_out * 4, 1, activation=get_bn(zero_init=True))
return l + resnet_shortcut(shortcut, ch_out * 4, stride, activation=get_bn(zero_init=False))
def residual_layer(name, l, out_filters, strides, data_format):
ch_out = out_filters
data_format = get_data_format(data_format, keras_mode=False)
ch_dim = 3 if data_format == 'NHWC' else 1
ch_in = _get_dim(l, ch_dim)
l_in = l
with tf.variable_scope('{}.0'.format(name)):
l = BNReLU(l)
l = SeparableConv2D('conv1', l, ch_out, 3, strides=strides, activation=BNReLU)
l = SeparableConv2D('conv2', l, ch_out, 3)
# The second conv need to be BN before addition.
l = BatchNorm('bn2', l)
shortcut = l_in
if strides > 1:
shortcut = AvgPooling('pool', shortcut, 2)
if ch_in < ch_out:
pad_paddings = [[0, 0], [0, 0], [0, 0], [0, 0]]
pad_width = (ch_out - ch_in)
pad_paddings[ch_dim] = [0, pad_width]
shortcut = tf.pad(shortcut, pad_paddings)
elif ch_in > ch_out:
if data_format == 'NHWC':
shortcut1 = shortcut[:, :, :, :ch_out]
shortcut2 = shortcut[:, :, :, ch_out:]
else:
shortcut1 = shortcut[:, :ch_out, :, :]
shortcut2 = shortcut[:, ch_out:, :, :]
shortcut2 = Conv2D('conv_short', shortcut2, ch_out, 1, strides=strides)
shortcut2 = BatchNorm('bn_short', shortcut2)
shortcut = shortcut1 + shortcut2
l += shortcut
return l
def res_blk(name, l, ch, ksize, count, split=1, strides=1, freeze=False):
ch_in = l.get_shape().as_list()
with tf.variable_scope(name):
for i in range(0, count):
with tf.variable_scope('block' + str(i)):
x = l if i == 0 else BNReLU('preact', l)
x = Conv2D('conv1', x, ch[0], ksize[0], activation=BNReLU)
x = Conv2D('conv2', x, ch[1], ksize[1], split=split,
strides=strides if i == 0 else 1, activation=BNReLU)
x = Conv2D('conv3', x, ch[2], ksize[2], activation=tf.identity)
if (strides != 1 or ch_in[1] != ch[2]) and i == 0:
l = Conv2D('convshortcut', l, ch[2], 1, strides=strides)
x = tf.stop_gradient(x) if freeze else x
l = l + x
# end of each group need an extra activation
l = BNReLU('bnlast',l)
return l
def preresnet_group(name, l, block_func, features, count, stride):
with tf.variable_scope(name):
for i in range(0, count):
with tf.variable_scope('block{}'.format(i)):
# first block doesn't need activation
l = block_func(l, features, stride if i == 0 else 1,
'no_preact' if i == 0 else 'bnrelu')
# end of each group need an extra activation
l = BNReLU('bnlast', l)
return l
def apply_preactivation(l: tf.Tensor, preact: str):
if preact == "bnrelu":
shortcut = l # preserve identity mapping
l = BNReLU("preact", l)
if preact == "inrelu":
shortcut = l
l = INReLU("preact", l)
else:
shortcut = l
return l, shortcut
def res_se_blk(name, l, ch, ksize, count, split=1, strides=1, freeze=False):
ch_in = l.get_shape().as_list()
with tf.variable_scope(name):
for i in range(0, count):
with tf.variable_scope('block' + str(i)):
if i != 0:
strides = 1
l = se_bottleneck(l, ch, strides)
l = BNReLU('bnlast',l)
return l
def apply_preactivation(l, preact):
"""
'no_preact' for the first resblock in each group only, because the input is activated already.
'bnrelu' for all the non-first blocks, where identity mapping is preserved on shortcut path.
"""
if preact == 'bnrelu':
shortcut = l # preserve identity mapping
l = BNReLU('preact', l)
else:
shortcut = l
return l, shortcut
def preresnet_group(l, name, block_func, features, count, stride, rate):
with tf.variable_scope(name):
if rate != 1:
multi_grid_rate = [m * rate for m in cfg.multi_grid]
else:
multi_grid_rate = [1] * count
for i in range(0, count):
with tf.variable_scope('block{}'.format(i)):
# first block doesn't need activation
l = block_func(l, features, stride if i == 0 else 1,
multi_grid_rate[i],
'no_preact' if i == 0 else 'bnrelu')
# end of each group need an extra activation
l = BNReLU('bnlast', l)
return l
def preresnet_group(x, name, block_func, features, count, stride, bn=True):
with tf.compat.v1.variable_scope(name):
for i in range(0, count):
with tf.compat.v1.variable_scope('block{}'.format(i)):
# first block doesn't need activation
x = block_func(x,
features,
stride if i == 0 else 1,
'no_preact' if i == 0 else 'bnrelu',
bn=bn)
# end of each group need an extra activation
if bn:
x = BNReLU('bnlast', x)
else:
x = tf.nn.relu(x, 'relulast')
return x
def preresnet_group(name, l, block_func, features, count, stride, option):
if features == 64: k = 10
elif features == 128: k = 20
elif features == 256: k = 30
elif features == 512: k = 40
with tf.variable_scope(name):
for i in range(0, count):
with tf.variable_scope('block{}'.format(i)):
# first block doesn't need activation
l = block_func(option,
l,
features,
stride if i == 0 else 1,
'no_preact' if i == 0 else 'bnrelu',
count=k)
# end of each group need an extra activation
l = BNReLU('bnlast', l)
return l
def DWConv(x, kernel, padding='SAME', stride=1, w_init=None, active=True, data_format='NHWC'):
'''
Depthwise conv + BN + (optional) ReLU.
We do not use channel multiplier here (fixed as 1).
'''
assert data_format in ('NHWC', 'channels_last')
channel = x.get_shape().as_list()[-1]
if not isinstance(kernel, (list, tuple)):
kernel = [kernel, kernel]
filter_shape = [kernel[0], kernel[1], channel, 1]
if w_init is None:
w_init = tf.variance_scaling_initializer(2.0)
W = tf.get_variable('W', filter_shape, initializer=w_init)
out = tf.nn.depthwise_conv2d(x, W, [1, stride, stride, 1], padding=padding, data_format=data_format)
if active is None:
return out
out = BNReLU(out) if active else BatchNorm('bn', out)
return out
def _build_graph(self, inputs):
images, truemap_coded = inputs
orig_imgs = images
o_true_np = truemap_coded[..., 0]
o_true_np = tf.cast(o_true_np, tf.int32)
o_true_np = tf.identity(o_true_np, name='truemap-np')
o_one_np = tf.one_hot(o_true_np, 2, axis=-1)
o_true_np = tf.expand_dims(o_true_np, axis=-1)
o_true_dist = truemap_coded[..., 1:]
o_true_dist = tf.identity(o_true_dist, name='truemap-dist')
####
with argscope(Conv2D, activation=tf.identity, use_bias=False, # K.he initializer
W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')), \
argscope([Conv2D, BatchNorm], data_format=self.data_format):
i = tf.transpose(images, [0, 3, 1, 2])
i = i if not self.input_norm else i / 255.0
####
d = encoder(i, self.freeze)
d[0] = crop_op(d[0], (184, 184))
d[1] = crop_op(d[1], (72, 72))
####
o_np = decoder('np', d)
o_np = BNReLU('preact_out_np', o_np)
o_dist = decoder('dst', d)
o_dist = BNReLU('preact_out_dist', o_dist)
####
o_logi_np = Conv2D('conv_out_np',
o_np,
2,
1,
use_bias=True,
activation=tf.identity)
o_logi_np = tf.transpose(o_logi_np, [0, 2, 3, 1])
o_soft_np = tf.nn.softmax(o_logi_np, axis=-1)
o_prob_np = tf.identity(o_soft_np[..., 1], name='predmap-prob-np')
o_prob_np = tf.expand_dims(o_prob_np, axis=-1)
o_pred_np = tf.argmax(o_soft_np, axis=-1, name='predmap-np')
o_pred_np = tf.expand_dims(tf.cast(o_pred_np, tf.float32), axis=-1)
####
o_logi_dist = Conv2D('conv_out_dist',
o_dist,
1,
1,
use_bias=True,
activation=tf.identity)
o_logi_dist = tf.transpose(o_logi_dist, [0, 2, 3, 1])
o_prob_dist = tf.identity(o_logi_dist, name='predmap-prob-dist')
o_pred_dist = tf.identity(o_logi_dist, name='predmap-dist')
# encoded so that inference can extract all output at once
predmap_coded = tf.concat([o_prob_np, o_pred_dist],
axis=-1,
name='predmap-coded')
####
####
if get_current_tower_context().is_training:
######## LOSS
### Distance regression loss
loss_mse = o_pred_dist - o_true_dist
loss_mse = loss_mse * loss_mse
loss_mse = tf.reduce_mean(loss_mse, name='loss-mse')
add_moving_summary(loss_mse)
### Nuclei Blob classification loss
loss_bce = categorical_crossentropy(o_soft_np, o_one_np)
loss_bce = tf.reduce_mean(loss_bce, name='loss-bce')
add_moving_summary(loss_bce)
### combine the loss into single cost function
self.cost = tf.identity(loss_mse + loss_bce, name='overall-loss')
add_moving_summary(self.cost)
####
add_param_summary(('.*/W', ['histogram'])) # monitor W
#### logging visual sthg
orig_imgs = tf.cast(orig_imgs, tf.uint8)
tf.summary.image('input', orig_imgs, max_outputs=1)
orig_imgs = crop_op(orig_imgs, (190, 190), "NHWC")
o_pred_np = colorize(o_prob_np[..., 0], cmap='jet')
o_true_np = colorize(o_true_np[..., 0], cmap='jet')
o_pred_np = tf.cast(o_pred_np * 255, tf.uint8)
o_true_np = tf.cast(o_true_np * 255, tf.uint8)
o_pred_dist = colorize(o_prob_dist[..., 0], cmap='jet')
o_true_dist = colorize(o_true_dist[..., 0], cmap='jet')
o_pred_dist = tf.cast(o_pred_dist * 255, tf.uint8)
o_true_dist = tf.cast(o_true_dist * 255, tf.uint8)
viz = tf.concat([
orig_imgs,
o_true_np,
o_pred_np,
o_true_dist,
o_pred_dist,
], 2)
tf.summary.image('output', viz, max_outputs=1)
return
def _build_graph(self, inputs):
images, truemap_coded = inputs
orig_imgs = images
o_true_np = truemap_coded[..., 0]
o_true_np = tf.cast(o_true_np, tf.int32)
o_true_np = tf.identity(o_true_np, name='truemap-np')
o_one_np = tf.one_hot(o_true_np, 2, axis=-1)
o_true_np = tf.expand_dims(o_true_np, axis=-1)
o_true_xy = truemap_coded[..., 1:]
o_true_xy = tf.identity(o_true_xy, name='truemap-xy')
####
with argscope(Conv2D, activation=tf.identity, use_bias=False, # K.he initializer
W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')), \
argscope([Conv2D, BatchNorm], data_format=self.data_format):
i = tf.transpose(images, [0, 3, 1, 2])
i = i if not self.input_norm else i / 255.0
####
d = encoder(i, self.freeze)
d[0] = crop_op(d[0], (184, 184))
d[1] = crop_op(d[1], (72, 72))
####
o_np = decoder('np', d)
o_np = BNReLU('preact_out_np', o_np)
o_xy = decoder('xy', d)
o_xy = BNReLU('preact_out_xy', o_xy)
####
o_logi_np = Conv2D('conv_out_np',
o_np,
2,
1,
use_bias=True,
activation=tf.identity)
o_logi_np = tf.transpose(o_logi_np, [0, 2, 3, 1])
o_soft_np = tf.nn.softmax(o_logi_np, axis=-1)
o_prob_np = tf.identity(o_soft_np[..., 1], name='predmap-prob-np')
o_prob_np = tf.expand_dims(o_prob_np, axis=-1)
o_pred_np = tf.argmax(o_soft_np, axis=-1, name='predmap-np')
o_pred_np = tf.expand_dims(tf.cast(o_pred_np, tf.float32), axis=-1)
o_logi_xy = Conv2D('conv_out_xy',
o_xy,
2,
1,
use_bias=True,
activation=tf.identity)
o_logi_xy = tf.transpose(o_logi_xy, [0, 2, 3, 1])
o_prob_xy = tf.identity(o_logi_xy, name='predmap-prob-xy')
o_pred_xy = tf.identity(o_logi_xy, name='predmap-xy')
# encoded so that inference can extract all output at once
predmap_coded = tf.concat([o_prob_np, o_pred_xy],
axis=-1,
name='predmap-coded')
####
####
if get_current_tower_context().is_training:
#---- LOSS ----#
### XY regression loss
loss_mse = o_pred_xy - o_true_xy
loss_mse = loss_mse * loss_mse
loss_mse = tf.reduce_mean(loss_mse, name='loss-mse')
add_moving_summary(loss_mse)
loss_xy = loss_mse
if 'msge' in self.loss_term:
nuclear = truemap_coded[..., 0]
nuclear = tf.stack([nuclear, nuclear], axis=-1)
pred_grad = get_gradient_xy(o_pred_xy, 1, 0)
true_grad = get_gradient_xy(o_true_xy, 1, 0)
loss_msge = pred_grad - true_grad
loss_msge = nuclear * (loss_msge * loss_msge)
# artificial reduce_mean with focus region
loss_msge = tf.reduce_sum(loss_msge)
loss_msge = loss_msge / tf.reduce_sum(nuclear)
loss_msge = tf.identity(loss_msge, name='loss-msge')
add_moving_summary(loss_msge)
loss_xy = 2 * loss_mse + loss_msge
loss_xy = tf.identity(loss_xy, name='overall-xy')
### Nuclei Blob classification loss
loss_bce = categorical_crossentropy(o_soft_np, o_one_np)
loss_bce = tf.reduce_mean(loss_bce, name='loss-bce')
add_moving_summary(loss_bce)
loss_np = loss_bce
if 'dice' in self.loss_term:
loss_dice = dice_loss(o_prob_np, o_true_np)
loss_dice = tf.identity(loss_dice, name='loss-dice')
add_moving_summary(loss_dice)
loss_np = loss_bce + loss_dice
loss_np = tf.identity(loss_np, name='overall-np')
### combine the loss into single cost function
self.cost = tf.identity(loss_xy + loss_np, name='overall-loss')
add_moving_summary(self.cost, loss_xy, loss_np)
####
add_param_summary(('.*/W', ['histogram'])) # monitor W
### logging visual sthg
orig_imgs = tf.cast(orig_imgs, tf.uint8)
tf.summary.image('input', orig_imgs, max_outputs=1)
orig_imgs = crop_op(orig_imgs, (190, 190), "NHWC")
o_pred = colorize(o_prob_np[..., 0], cmap='jet')
o_true = colorize(o_true_np[..., 0], cmap='jet')
o_pred = tf.cast(o_pred * 255, tf.uint8)
o_true = tf.cast(o_true * 255, tf.uint8)
pred_x = colorize(o_prob_xy[..., 0], vmin=-1, vmax=1, cmap='jet')
pred_y = colorize(o_prob_xy[..., 1], vmin=-1, vmax=1, cmap='jet')
true_x = colorize(o_true_xy[..., 0], vmin=-1, vmax=1, cmap='jet')
true_y = colorize(o_true_xy[..., 1], vmin=-1, vmax=1, cmap='jet')
pred_x = tf.cast(pred_x * 255, tf.uint8)
pred_y = tf.cast(pred_y * 255, tf.uint8)
true_x = tf.cast(true_x * 255, tf.uint8)
true_y = tf.cast(true_y * 255, tf.uint8)
viz = tf.concat(
[orig_imgs, pred_x, pred_y, o_pred, true_x, true_y, o_true], 2)
tf.summary.image('output', viz, max_outputs=1)
return
def _build_graph(self, inputs):
images, truemap_coded = inputs
orig_imgs = images
true_np = truemap_coded[...,0]
true_np = tf.cast(true_np, tf.int32)
true_np = tf.identity(true_np, name='truemap-np')
one_np = tf.one_hot(true_np, 2, axis=-1)
true_np = tf.expand_dims(true_np, axis=-1)
true_dist = truemap_coded[...,1:]
true_dist = tf.identity(true_dist, name='truemap-dist')
####
with argscope(Conv2D, activation=tf.identity, use_bias=False, # K.he initializer
W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')), \
argscope([Conv2D, BatchNorm], data_format=self.data_format):
i = tf.transpose(images, [0, 3, 1, 2])
i = i if not self.input_norm else i / 255.0
####
d = encoder(i, self.freeze)
d[0] = crop_op(d[0], (184, 184))
d[1] = crop_op(d[1], (72, 72))
####
np_feat = decoder('np', d)
np = BNReLU('preact_out_np', np_feat[-1])
dist_feat = decoder('dst', d)
dist = BNReLU('preact_out_dist', dist_feat[-1])
#### Initialise weights prior distn
logi_np = Conv2D('conv_out_np', npx, 2, 1, use_bias=True, activation=tf.identity, bias_initializer=tf.constant_initializer(value=-np.log((1 - 0.05) / 0.05)))
logi_np = tf.transpose(logi_np, [0, 2, 3, 1])
soft_np = tf.nn.sigmoid(logi_np) #Change to softmax if not using focal
prob_np = tf.identity(soft_np[...,1], name='predmap-prob-np')
prob_np = tf.expand_dims(prob_np, axis=-1)
pred_np = tf.argmax(soft_np, axis=-1, name='predmap-np')
pred_np = tf.expand_dims(tf.cast(pred_np, tf.float32), axis=-1)
####
logi_dist = Conv2D('conv_out_dist', dist, 1, 1, use_bias=True, activation=tf.identity)
logi_dist = tf.transpose(logi_dist, [0, 2, 3, 1])
prob_dist = tf.identity(logi_dist, name='predmap-prob-dist')
pred_dist = tf.identity(logi_dist, name='predmap-dist')
# encoded so that inference can extract all output at once
predmap_coded = tf.concat([prob_np, pred_dist], axis=-1, name='predmap-coded')
####
####
if get_current_tower_context().is_training:
######## LOSS
### Distance regression loss
alpha = 0.5
delta = 1
res = true - pred
if res == alpha:
g1 = 2*alpha + tf.math.log(0.5*(1+tf.math.exp(delta*(alpha)))**(-2/delta))
g2 = 0
elif res == -alpha:
g1 = 0
g2 = tf.math.log(0.5*(1+tf.math.exp(delta*(-alpha)))**(-2/delta))
else:
g1 = 0
g2 = 0
f1 = alpha*(res)**2
f2 = alpha*(tf.math.log(0.5*(1+tf.math.exp(delta*(res)))**(2/delta))-(res) + g1 + g2)
loss_mse = tf.reduce_mean(tf.where((tf.math.abs(res)) <= alpha, f1, f2 ,name='loss_mse'))
add_moving_summary(loss_mse)
### Nuclei Blob classification loss
loss_bce = categorical_crossentropy(soft_np, one_np)
loss_bce = tf.reduce_mean(loss_bce, name='loss-bce')
add_moving_summary(loss_bce)
### combine the loss into single cost function
self.cost = tf.identity(loss_mse + loss_bce, name='overall-loss')
add_moving_summary(self.cost)
####
#add_param_summary(('.*/W', ['histogram'])) # monitor W
#### logging visual sthg
orig_imgs = tf.cast(orig_imgs , tf.uint8)
tf.summary.image('input', orig_imgs, max_outputs=1)
orig_imgs = crop_op(orig_imgs, (190, 190), "NHWC")
pred_np = colorize(prob_np[...,0], cmap='jet')
true_np = colorize(true_np[...,0], cmap='jet')
pred_dist = colorize(prob_dist[...,0], cmap='jet')
true_dist = colorize(true_dist[...,0], cmap='jet')
viz = tf.concat([orig_imgs,
true_np, pred_np,
true_dist, pred_dist,], 2)
tf.summary.image('output', viz, max_outputs=1)
return
def _build_graph(self, inputs):
images, truemap_coded = inputs
orig_imgs = images
if hasattr(self, 'type_classification') and self.type_classification:
true_type = truemap_coded[...,1]
true_type = tf.cast(true_type, tf.int32)
true_type = tf.identity(true_type, name='truemap-type')
one_type = tf.one_hot(true_type, self.nr_types, axis=-1)
true_type = tf.expand_dims(true_type, axis=-1)
true_np = tf.cast(true_type > 0, tf.int32) # ? sanity this
true_np = tf.identity(true_np, name='truemap-np')
one_np = tf.one_hot(tf.squeeze(true_np), 2, axis=-1)
else:
true_np = truemap_coded[...,0]
true_np = tf.cast(true_np, tf.int32)
true_np = tf.identity(true_np, name='truemap-np')
one_np = tf.one_hot(true_np, 2, axis=-1)
true_np = tf.expand_dims(true_np, axis=-1)
true_hv = truemap_coded[...,-2:]
true_hv = tf.identity(true_hv, name='truemap-hv')
####
with argscope(Conv2D, activation=tf.identity, use_bias=False, # K.he initializer
W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')), \
argscope([Conv2D, BatchNorm, MaxPooling, AvgPooling], data_format=self.data_format):
i = tf.transpose(images, [0, 3, 1, 2])
i = i if not self.input_norm else i / 255.0
#i = images #IF non local
####
#d = encoder(i, self.freeze)
if self.encoder_name == 'inception':
d = inception_encoder(i, self.freeze)
elif self.encoder_name == 'densenet':
d = densenet_encoder(i, self.freeze)
else:
d = encoder(i, self.freeze)
d[0] = crop_op(d[0], (184, 184))
d[1] = crop_op(d[1], (72, 72))
####
np_feat = decoder('np', d, self.use_assp)
npx = BNReLU('preact_out_np', np_feat[-1])
hv_feat = decoder('hv', d, self.use_assp)
hv = BNReLU('preact_out_hv', hv_feat[-1])
if self.type_classification:
tp_feat = decoder('tp', d, self.use_assp)
tp = BNReLU('preact_out_tp', tp_feat[-1])
# Nuclei Type Pixels (TP)
logi_class = Conv2D('conv_out_tp', tp, self.nr_types, 1, use_bias=True, activation=tf.identity)
logi_class = tf.transpose(logi_class, [0, 2, 3, 1])
soft_class = tf.nn.softmax(logi_class, axis=-1)
#### Nuclei Pixels (NP)
logi_np = Conv2D('conv_out_np', npx, 2, 1, use_bias=True, activation=tf.identity)
logi_np = tf.transpose(logi_np, [0, 2, 3, 1])
soft_np = tf.nn.softmax(logi_np, axis=-1)
prob_np = tf.identity(soft_np[...,1], name='predmap-prob-np')
prob_np = tf.expand_dims(prob_np, axis=-1)
#### Horizontal-Vertival (HV)
logi_hv = Conv2D('conv_out_hv', hv, 2, 1, use_bias=True, activation=tf.identity)
logi_hv = tf.transpose(logi_hv, [0, 2, 3, 1])
prob_hv = tf.identity(logi_hv, name='predmap-prob-hv')
pred_hv = tf.identity(logi_hv, name='predmap-hv')
# * channel ordering: type-map, segmentation map
# encoded so that inference can extract all output at once
if self.type_classification:
predmap_coded = tf.concat([soft_class, prob_np, pred_hv], axis=-1, name='predmap-coded')
else:
predmap_coded = tf.concat([prob_np, pred_hv], axis=-1, name='predmap-coded')
####
def get_gradient_hv(l, h_ch, v_ch):
def get_sobel_kernel(size):
assert size % 2 == 1, 'Must be odd, get size=%d' % size
h_range = np.arange(-size//2+1, size//2+1, dtype=np.float32)
v_range = np.arange(-size//2+1, size//2+1, dtype=np.float32)
h, v = np.meshgrid(h_range, v_range)
kernel_h = h / (h * h + v * v + 1.0e-15)
kernel_v = v / (h * h + v * v + 1.0e-15)
return kernel_h, kernel_v
mh, mv = get_sobel_kernel(5)
mh = tf.constant(mh, dtype=tf.float32)
mv = tf.constant(mv, dtype=tf.float32)
mh = tf.reshape(mh, [5, 5, 1, 1])
mv = tf.reshape(mv, [5, 5, 1, 1])
# central difference to get gradient, ignore the boundary problem
h = tf.expand_dims(l[...,h_ch], axis=-1)
v = tf.expand_dims(l[...,v_ch], axis=-1)
dh = tf.nn.conv2d(h, mh, strides=[1, 1, 1, 1], padding='SAME')
dv = tf.nn.conv2d(v, mv, strides=[1, 1, 1, 1], padding='SAME')
output = tf.concat([dh, dv], axis=-1)
return output
#######################################################
# LOSS FUNCTIONS
#######################################################
def loss_mse(true,pred,name=None):
###########################
#Standard MSE:
###########################
res = pred - true
loss = res**2
loss = tf.reduce_mean(loss, name=name)
return loss
def loss_huber(true,pred,name=None):
###########################
#Standard Huber Loss:
###########################
alpha = 0.5
res = true - pred
f1= alpha*(res)**2
f2 = alpha*tf.math.abs(res) - 0.5*(alpha**2)
loss = tf.where((tf.math.abs(res)) <= alpha, f1, f2 ,name=name)
return tf.reduce_mean(loss)
def pseudo_huber(true,pred,name=None):
###########################
#Standard Huber Loss:
###########################
delta=0.5
res = true - pred
loss = delta**2 * (sqrt(1 + ((res)/delta)**2) - 1)
return tf.reduce_mean(loss)
def loss_huber_smooth(true,pred,name=None):
###########################
#Smoother Huber Loss:
###########################
alpha = 0.5 #Determines how much you want to penalise outliers in your dataset.
delta = 1 #When delta -> 0 it becomes smoother. Try to pick delta > 1e-5 otherwise 2/delta becomes too large.
res = true - pred
#If code is too slow, then set g_j(delta) = 0 as continuity doesn't make too much difference to results.
if res == alpha:
g1 = 2*alpha + tf.math.log(0.5*(1+tf.math.exp(delta*(alpha)))**(-2/delta))
g2 = 0
elif res == -alpha:
g1 = 0
g2 = tf.math.log(0.5*(1+tf.math.exp(delta*(-alpha)))**(-2/delta))
else:
g1 = 0
g2 = 0
f1 = alpha*(res)**2
f2 = alpha*((2/delta)*tf.math.log(0.5*(1+tf.math.exp(delta*(res))))-(res) + g1 + g2)
loss = tf.where((tf.math.abs(res)) <= alpha, f1, f2 ,name=name)
return tf.reduce_mean(loss)
def loss_msge(true, pred, focus, name=None):
###########################
#MSE of gradients:
###########################
focus = tf.stack([focus, focus], axis=-1)
pred_grad = get_gradient_hv(pred, 1, 0)
true_grad = get_gradient_hv(true, 1, 0)
loss = pred_grad - true_grad
loss = focus * (loss * loss)
# artificial reduce_mean with focus region
loss = tf.reduce_sum(loss) / (tf.reduce_sum(focus) + 1.0e-8)
loss = tf.identity(loss, name=name)
return loss
####
if get_current_tower_context().is_training:
#---- LOSS ----#
loss = 0
for term, weight in self.loss_term.items():
if term == 'mse':
term_loss = loss_mse(true_hv, pred_hv, name='loss-mse')
elif term == 'msge':
focus = truemap_coded[...,0]
term_loss = loss_msge(true_hv, pred_hv, focus, name='loss-msge')
elif term == 'bce':
term_loss = categorical_crossentropy(soft_np, one_np)
term_loss = tf.reduce_mean(term_loss, name='loss-bce')
elif 'dice' in self.loss_term:
term_loss = dice_loss(soft_np[...,0], one_np[...,0]) \
+ dice_loss(soft_np[...,1], one_np[...,1])
term_loss = tf.identity(term_loss, name='loss-dice')
else:
assert False, 'Not support loss term: %s' % term
add_moving_summary(term_loss)
loss += term_loss * weight
if self.type_classification:
term_loss = categorical_crossentropy(soft_class, one_type)
term_loss = tf.reduce_mean(term_loss, name='loss-xentropy-class')
add_moving_summary(term_loss)
loss = loss + term_loss
term_loss = 0
for type_id in range(self.nr_types):
term_loss += dice_loss(soft_class[...,type_id],
one_type[...,type_id])
term_loss = tf.identity(term_loss, name='loss-dice-class')
add_moving_summary(term_loss)
loss = loss + term_loss
### combine the loss into single cost function
self.cost = tf.identity(loss, name='overall-loss')
add_moving_summary(self.cost)
####
#add_param_summary(('.*/W', ['histogram'])) # monitor W
### logging visual sthg
orig_imgs = tf.cast(orig_imgs , tf.uint8)
tf.summary.image('input', orig_imgs, max_outputs=1)
orig_imgs = crop_op(orig_imgs, (190, 190), "NHWC")
pred_np = colorize(prob_np[...,0], cmap='jet')
true_np = colorize(true_np[...,0], cmap='jet')
pred_h = colorize(prob_hv[...,0], vmin=-1, vmax=1, cmap='jet')
pred_v = colorize(prob_hv[...,1], vmin=-1, vmax=1, cmap='jet')
true_h = colorize(true_hv[...,0], vmin=-1, vmax=1, cmap='jet')
true_v = colorize(true_hv[...,1], vmin=-1, vmax=1, cmap='jet')
if not self.type_classification:
viz = tf.concat([orig_imgs,
pred_h, pred_v, pred_np,
true_h, true_v, true_np], 2)
else:
pred_type = tf.transpose(soft_class, (0, 1, 3, 2))
pred_type = tf.reshape(pred_type, [-1, 80, 80 * self.nr_types])
true_type = tf.cast(true_type[...,0] / self.nr_classes, tf.float32)
true_type = colorize(true_type, vmin=0, vmax=1, cmap='jet')
pred_type = colorize(pred_type, vmin=0, vmax=1, cmap='jet')
viz = tf.concat([orig_imgs,
pred_h, pred_v, pred_np, pred_type,
true_h, true_v, true_np, true_type,], 2)
viz = tf.concat([viz[0], viz[-1]], axis=0)
viz = tf.expand_dims(viz, axis=0)
#print(viz.name)
viz_size = (240, 160)
tf.summary.image('output', viz, max_outputs=1)
return
def feature_to_prediction_and_loss(scope_name,
l,
label,
num_classes,
prediction_feature,
ch_dim,
label_smoothing=0,
dense_dropout_keep_prob=1.0,
is_last=True):
"""
Given the feature l at scope_name, compute a classifier.
"""
with tf.variable_scope(scope_name):
n_dim = len(l.get_shape().as_list())
if n_dim == 4 and not is_last:
with tf.variable_scope('aux_preprocess'):
l = tf.nn.relu(l)
l = AvgPooling('pool',
l,
pool_size=5,
strides=3,
padding='valid')
l = Conv2D('conv_proj',
l,
128,
1,
strides=1,
activation=BNReLU)
shape = l.get_shape().as_list()
if ch_dim != 1:
shape = shape[1:3]
else:
shape = shape[2:4]
l = Conv2D('conv_flat',
l,
768,
shape,
strides=1,
padding='valid',
activation=BNReLU)
l = tf.layers.flatten(l)
else:
l = BNReLU('bnrelu_pred', l)
ch_in = _get_dim(l, ch_dim)
if prediction_feature == '1x1':
ch_out = ch_in
if n_dim == 4:
l = Conv2D('conv1x1', l, ch_out, 1)
else:
assert n_dim == 2, n_dim
l = FullyConnected('fc1x1',
l,
ch_out,
activation=tf.identity)
l = BNReLU('bnrelu1x1', l)
elif prediction_feature == 'msdense':
assert n_dim == 2, n_dim
ch_inter = ch_in
l = Conv2D('conv1x1_0', l, ch_inter, 3, strides=2)
l = BNReLU('bnrelu1x1_0', l)
l = Conv2D('conv1x1_1', l, ch_inter, 3, strides=2)
l = BNReLU('bnrelu1x1_1', l)
elif prediction_feature == 'bn':
l = BatchNorm('bn', l)
else:
# Do nothing to the input feature
pass
if n_dim > 2:
l = GlobalAvgPooling('gap', l)
variables = []
if num_classes > 0:
if is_last:
l = Dropout('drop_pre_fc',
l,
keep_prob=dense_dropout_keep_prob)
logits = FullyConnected('linear',
l,
num_classes,
activation=tf.identity)
variables.append(logits.variables.W)
variables.append(logits.variables.b)
tf.nn.softmax(logits, name='preds')
## local cost/error_rate
if label_smoothing > 0:
one_hot_labels = tf.one_hot(label, num_classes)
cost = tf.losses.softmax_cross_entropy(\
onehot_labels=one_hot_labels, logits=logits,
label_smoothing=label_smoothing)
else:
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(\
logits=logits, labels=label)
cost = tf.reduce_mean(cost, name='cross_entropy_loss')
add_moving_summary(cost)
def prediction_incorrect(logits,
label,
topk=1,
name='incorrect_vector'):
return tf.cast(tf.logical_not(
tf.nn.in_top_k(logits, label, topk)),
tf.float32,
name=name)
wrong = prediction_incorrect(logits, label, 1, name='wrong-top1')
add_moving_summary(tf.reduce_mean(wrong, name='train_error'))
wrong5 = prediction_incorrect(logits, label, 5, name='wrong-top5')
add_moving_summary(tf.reduce_mean(wrong5, name='train-error-top5'))
else:
# for regression:
pred = FullyConnected('linear', l, 1, activation=tf.identity)
variables.append(pred.variables.W)
variables.append(pred.variables.b)
pred = tf.nn.relu(pred)
tf.identity(pred, name='preds')
cost = tf.reduce_mean(0.5 * (pred - label)**2,
name='mean_square_error')
add_moving_summary(cost)
return cost, variables
def _build_graph(self, inputs):
images, truemap_coded = inputs
####
with argscope(Conv2D, activation=tf.identity, use_bias=False, # K.he initializer
W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')), \
argscope([Conv2D, BatchNorm], data_format=self.data_format):
i = tf.transpose(images, [0, 3, 1, 2])
i = i if not self.input_norm else i / 255.0
####
d = encoder(i)
d[0] = crop_op(d[0], (92, 92))
d[1] = crop_op(d[1], (36, 36))
####
np_feat = decoder('np', d)
npx = BNReLU('preact_out_np', np_feat[-1])
hv_feat = decoder('hv', d)
hv = BNReLU('preact_out_hv', hv_feat[-1])
tp_feat = decoder('tp', d)
tp = BNReLU('preact_out_tp', tp_feat[-1])
# Nuclei Type Pixels (TP)
logi_class = Conv2D('conv_out_tp',
tp,
self.nr_types,
1,
use_bias=True,
activation=tf.identity)
logi_class = tf.transpose(logi_class, [0, 2, 3, 1])
soft_class = tf.nn.softmax(logi_class, axis=-1)
#### Nuclei Pixels (NP)
logi_np = Conv2D('conv_out_np',
npx,
2,
1,
use_bias=True,
activation=tf.identity)
logi_np = tf.transpose(logi_np, [0, 2, 3, 1])
soft_np = tf.nn.softmax(logi_np, axis=-1)
prob_np = tf.identity(soft_np[..., 1], name='predmap-prob-np')
prob_np = tf.expand_dims(prob_np, axis=-1)
#### Horizontal-Vertival (HV)
logi_hv = Conv2D('conv_out_hv',
hv,
2,
1,
use_bias=True,
activation=tf.identity)
logi_hv = tf.transpose(logi_hv, [0, 2, 3, 1])
prob_hv = tf.identity(logi_hv, name='predmap-prob-hv')
pred_hv = tf.identity(logi_hv, name='predmap-hv')
# * channel ordering: type-map, segmentation map
# encoded so that inference can extract all output at once
predmap_coded = tf.concat([soft_class, prob_np, pred_hv],
axis=-1,
name='predmap-coded')
return
def Grconv(
x,
filters,
kernel_size,
strides=(1, 1),
padding='same',
data_format='channels_last', #'channels_last'默认 表示输入中维度的顺序。 channels_last 对应输入尺寸为 (batch, height, width, channels)， channels_first 对应输入尺寸为 (batch, channels, height, width)。
dilation_rate=(
1, 1
), # 一个整数或 2 个整数的元组或列表,为所有空间维度指定相同的值。 当前，指定任何 dilation_rate 值 != 1 与 指定 stride 值 != 1 两者不兼容。
activation=None,
use_bias=True, # 布尔值，该层是否使用偏置向量。
kernel_initializer=tf.contrib.layers.variance_scaling_initializer(
2.0), # 一种更好的方差初始化的方法，调整初始随机权重的方差
bias_initializer=tf.zeros_initializer(), #初始化为0
kernel_regularizer=None, # 运用到 kernel 权值矩阵的正则化函数
bias_regularizer=None, # 运用到偏置向量的正则化函数
activity_regularizer=None, # 运用到层输出（它的激活值）的正则化函数
split=1,
glocal=False):
z3 = Conv2D('z3',
x,
filters=filters,
kernel_size=3,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=tf.identity,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
split=1) # tf.identity 线性操作
out_shape = z3.get_shape().as_list() # 输出z3的维度并以列表的方式显示
z1 = Conv2D('z1',
x,
filters=filters,
kernel_size=1,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=tf.identity,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
split=1)
zf = fc(x,
out_shape,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=tf.identity,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer) # zf为经过池化及全连接后的网络
p = tf.get_variable('PPP', [3, 1, out_shape[2], out_shape[3]],
initializer=tf.ones_initializer(),
trainable=True)
# 创建一个名为'ppp' ,形状为，用initializer初始化的变量，并将变量添加到图形集合（trainable）
p = tf.nn.softmax(p, 0)
z = p[0:1, :, :, :] * z3 + p[1:2, :, :, :] * z1 + p[
2:3, :, :, :] * zf # 存疑 三个样本
z = BNReLU('sum', z) #这是一个其他函数
return z
def _build_graph(self, inputs):
images, truemap_coded = inputs
orig_imgs = images
if hasattr(self, 'type_classification') and self.type_classification:
true_type = truemap_coded[...,1]
true_type = tf.cast(true_type, tf.int32)
true_type = tf.identity(true_type, name='truemap-type')
one_type = tf.one_hot(true_type, self.nr_types, axis=-1)
true_type = tf.expand_dims(true_type, axis=-1)
true_np = tf.cast(true_type > 0, tf.int32) # ? sanity this
true_np = tf.identity(true_np, name='truemap-np')
one_np = tf.one_hot(tf.squeeze(true_np), 2, axis=-1)
else:
true_np = truemap_coded[...,0]
true_np = tf.cast(true_np, tf.int32)
true_np = tf.identity(true_np, name='truemap-np')
one_np = tf.one_hot(true_np, 2, axis=-1)
true_np = tf.expand_dims(true_np, axis=-1)
true_hv = truemap_coded[...,-2:]
true_hv = tf.identity(true_hv, name='truemap-hv')
####
with argscope(Conv2D, activation=tf.identity, use_bias=False, # K.he initializer
W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')), \
argscope([Conv2D, BatchNorm], data_format=self.data_format):
i = tf.transpose(images, [0, 3, 1, 2])
i = i if not self.input_norm else i / 255.0
####
d = encoder(i, self.freeze)
d[0] = crop_op(d[0], (92, 92))
d[1] = crop_op(d[1], (36, 36))
####
np_feat = decoder('np', d)
npx = BNReLU('preact_out_np', np_feat[-1])
hv_feat = decoder('hv', d)
hv = BNReLU('preact_out_hv', hv_feat[-1])
if self.type_classification:
tp_feat = decoder('tp', d)
tp = BNReLU('preact_out_tp', tp_feat[-1])
# Nuclei Type Pixels (TP)
logi_class = Conv2D('conv_out_tp', tp, self.nr_types, 1, use_bias=True, activation=tf.identity)
logi_class = tf.transpose(logi_class, [0, 2, 3, 1])
soft_class = tf.nn.softmax(logi_class, axis=-1)
#### Nuclei Pixels (NP)
logi_np = Conv2D('conv_out_np', npx, 2, 1, use_bias=True, activation=tf.identity)
logi_np = tf.transpose(logi_np, [0, 2, 3, 1])
soft_np = tf.nn.softmax(logi_np, axis=-1)
prob_np = tf.identity(soft_np[...,1], name='predmap-prob-np')
prob_np = tf.expand_dims(prob_np, axis=-1)
#### Horizontal-Vertival (HV)
logi_hv = Conv2D('conv_out_hv', hv, 2, 1, use_bias=True, activation=tf.identity)
logi_hv = tf.transpose(logi_hv, [0, 2, 3, 1])
prob_hv = tf.identity(logi_hv, name='predmap-prob-hv')
pred_hv = tf.identity(logi_hv, name='predmap-hv')
# * channel ordering: type-map, segmentation map
# encoded so that inference can extract all output at once
if self.type_classification:
predmap_coded = tf.concat([soft_class, prob_np, pred_hv], axis=-1, name='predmap-coded')
else:
predmap_coded = tf.concat([prob_np, pred_hv], axis=-1, name='predmap-coded')
####
def get_gradient_hv(l, h_ch, v_ch):
"""
Calculate the horizontal partial differentiation for horizontal channel
and the vertical partial differentiation for vertical channel.
The partial differentiation is approximated by calculating the central differnce
which is obtained by using Sobel kernel of size 5x5. The boundary is zero-padded
when channel is convolved with the Sobel kernel.
Args:
l (tensor): tensor of shape NHWC with C should be 2 (1 channel for horizonal
and 1 channel for vertical)
h_ch(int) : index within C axis of `l` that corresponds to horizontal channel
v_ch(int) : index within C axis of `l` that corresponds to vertical channel
"""
def get_sobel_kernel(size):
assert size % 2 == 1, 'Must be odd, get size=%d' % size
h_range = np.arange(-size//2+1, size//2+1, dtype=np.float32)
v_range = np.arange(-size//2+1, size//2+1, dtype=np.float32)
h, v = np.meshgrid(h_range, v_range)
kernel_h = h / (h * h + v * v + 1.0e-15)
kernel_v = v / (h * h + v * v + 1.0e-15)
return kernel_h, kernel_v
mh, mv = get_sobel_kernel(5)
mh = tf.constant(mh, dtype=tf.float32)
mv = tf.constant(mv, dtype=tf.float32)
mh = tf.reshape(mh, [5, 5, 1, 1])
mv = tf.reshape(mv, [5, 5, 1, 1])
# central difference to get gradient, ignore the boundary problem
h = tf.expand_dims(l[...,h_ch], axis=-1)
v = tf.expand_dims(l[...,v_ch], axis=-1)
dh = tf.nn.conv2d(h, mh, strides=[1, 1, 1, 1], padding='SAME')
dv = tf.nn.conv2d(v, mv, strides=[1, 1, 1, 1], padding='SAME')
output = tf.concat([dh, dv], axis=-1)
return output
def loss_mse(true, pred, name=None):
### regression loss
loss = pred - true
loss = tf.reduce_mean(loss * loss, name=name)
return loss
def loss_msge(true, pred, focus, name=None):
focus = tf.stack([focus, focus], axis=-1)
pred_grad = get_gradient_hv(pred, 1, 0)
true_grad = get_gradient_hv(true, 1, 0)
loss = pred_grad - true_grad
loss = focus * (loss * loss)
# artificial reduce_mean with focus region
loss = tf.reduce_sum(loss) / (tf.reduce_sum(focus) + 1.0e-8)
loss = tf.identity(loss, name=name)
return loss
####
if get_current_tower_context().is_training:
#---- LOSS ----#
loss = 0
for term, weight in self.loss_term.items():
if term == 'mse':
term_loss = loss_mse(true_hv, pred_hv, name='loss-mse')
elif term == 'msge':
focus = truemap_coded[...,0]
term_loss = loss_msge(true_hv, pred_hv, focus, name='loss-msge')
elif term == 'bce':
term_loss = categorical_crossentropy(soft_np, one_np)
term_loss = tf.reduce_mean(term_loss, name='loss-bce')
elif 'dice' in self.loss_term:
term_loss = dice_loss(soft_np[...,0], one_np[...,0]) \
+ dice_loss(soft_np[...,1], one_np[...,1])
term_loss = tf.identity(term_loss, name='loss-dice')
#elif term == 'focal':
# term_loss = focal_loss(soft_np, one_np,alpha=0.25,gamma=2)
# term_loss = tf.reduce_mean(term_loss, name='loss-focal')
else:
assert False, 'Not support loss term: %s' % term
add_moving_summary(term_loss)
loss += term_loss * weight
if self.type_classification:
term_loss = categorical_crossentropy(soft_class, one_type) #focal_loss(soft_class, one_type,alpha=0.25,gamma=2)
term_loss = tf.reduce_mean(term_loss, name='loss-xentropy-class') # tf.reduce_mean(term_loss, name='loss-focal-class')
add_moving_summary(term_loss)
loss = loss + term_loss
term_loss = 0
for type_id in range(self.nr_types):
term_loss += dice_loss(soft_class[...,type_id],
one_type[...,type_id])
term_loss = tf.identity(term_loss, name='loss-dice-class')
add_moving_summary(term_loss)
loss = loss + term_loss
### combine the loss into single cost function
self.cost = tf.identity(loss, name='overall-loss')
add_moving_summary(self.cost)
####
add_param_summary(('.*/W', ['histogram'])) # monitor W
### logging visual sthg
orig_imgs = tf.cast(orig_imgs , tf.uint8)
tf.summary.image('input', orig_imgs, max_outputs=1)
orig_imgs = crop_op(orig_imgs, (92, 92), "NHWC")
pred_np = colorize(prob_np[...,0], cmap='jet')
true_np = colorize(true_np[...,0], cmap='jet')
pred_h = colorize(prob_hv[...,0], vmin=-1, vmax=1, cmap='jet')
pred_v = colorize(prob_hv[...,1], vmin=-1, vmax=1, cmap='jet')
true_h = colorize(true_hv[...,0], vmin=-1, vmax=1, cmap='jet')
true_v = colorize(true_hv[...,1], vmin=-1, vmax=1, cmap='jet')
if not self.type_classification:
viz = tf.concat([orig_imgs,
pred_h, pred_v, pred_np,
true_h, true_v, true_np], 2)
else:
pred_type = tf.transpose(soft_class, (0, 1, 3, 2))
pred_type = tf.reshape(pred_type, [-1, 164, 164 * self.nr_types])
true_type = tf.cast(true_type[...,0] / self.nr_classes, tf.float32)
true_type = colorize(true_type, vmin=0, vmax=1, cmap='jet')
pred_type = colorize(pred_type, vmin=0, vmax=1, cmap='jet')
viz = tf.concat([orig_imgs,
pred_h, pred_v, pred_np, pred_type,
true_h, true_v, true_np, true_type,], 2)
viz = tf.concat([viz[0], viz[-1]], axis=0)
viz = tf.expand_dims(viz, axis=0)
tf.summary.image('output', viz, max_outputs=1)
return
def dense_blk(name, l, ch, ksize, count, split=1, padding='valid'):
with tf.variable_scope(name):
for i in range(0, count):
with tf.variable_scope('blk/' + str(i)):
x = BNReLU('preact_bna', l)
x = Conv2D('conv1',
x,
ch[0],
ksize[0],
padding=padding,
activation=BNReLU)
x = Conv2D('conv2',
x,
ch[1],
ksize[1],
padding=padding,
split=split)
##
if padding == 'valid':
x_shape = x.get_shape().as_list()
l_shape = l.get_shape().as_list()
l = crop_op(
l, (l_shape[2] - x_shape[2], l_shape[3] - x_shape[3]))
l = Conv2D('conv2forl',
l,
ch[1],
ksize[0],
padding='same',
split=split)
with tf.variable_scope('CBAM', reuse=tf.AUTO_REUSE):
residual = x
ratio = 8
###CBAM module
kernel_initializer = tf.contrib.layers.variance_scaling_initializer(
)
bias_initializer = tf.constant_initializer(value=0.0)
channel = x.shape[1]
print(channel)
# plt.pause(3.0)
avg_pool = tf.reduce_mean(x, axis=[2, 3], keepdims=True)
# avg_pool = GlobalAvgPooling('globalAveragepooling',x,data_format='channels_first')
# avg_pool = tf.reshape(avg_pool,[-1,channel,1,1],name='reshape_avgpool')
print(avg_pool)
# plt.pause(3.0)
assert avg_pool.shape[1:] == (channel, 1, 1)
# avg_pool = tf.compat.v1.layers.Dense(
# units=channel//ratio,
# activation=tf.nn.relu,
# kernel_initializer=kernel_initializer,
# bias_initializer=bias_initializer,
# name='mlp_0')(avg_pool)
avg_pool = FullyConnected(
'fullyconnected1',
avg_pool,
units=channel // ratio,
activation=tf.nn.relu,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)
print(avg_pool)
# plt.pause(3.0)
avg_pool = tf.reshape(avg_pool,
[-1, channel // ratio, 1, 1],
name='reshape_avgpoolB')
assert avg_pool.shape[1:] == (channel // ratio, 1, 1)
print(avg_pool)
# plt.pause(3.0)
avg_pool = FullyConnected(
'fullyconnectedB',
avg_pool,
units=channel,
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)
avg_pool = tf.reshape(avg_pool, [-1, channel, 1, 1],
name='reshape_avgpoolC')
assert avg_pool.get_shape()[1:] == (channel, 1, 1)
print(avg_pool)
# plt.pause(3.0)
max_pool = tf.reduce_max(x, axis=[2, 3], keepdims=True)
assert max_pool.get_shape()[1:] == (channel, 1, 1)
max_pool = FullyConnected('fullyconnected1',
max_pool,
units=channel // ratio,
activation=tf.nn.relu)
max_pool = tf.reshape(max_pool,
[-1, channel // ratio, 1, 1],
name='reshape_maxpoolA')
assert max_pool.get_shape()[1:] == (channel // ratio, 1, 1)
max_pool = FullyConnected('fullyconnectedB',
max_pool,
units=channel,
activation=None)
max_pool = tf.reshape(max_pool, [-1, channel, 1, 1],
name='reshape_maxpoolB')
assert max_pool.get_shape()[1:] == (channel, 1, 1)
scale = tf.sigmoid(avg_pool + max_pool, 'sigmoid')
channel_map = tf.math.multiply(residual,
scale,
name='merge_Channel')
#spatial_attention
kernel_size = 7
avg_pool_spatial = tf.reduce_mean(channel_map,
axis=[1],
keepdims=True)
print("Spatial averge pooling : ", avg_pool_spatial)
# plt.pause(3.0)
assert avg_pool_spatial.get_shape()[1] == 1
max_pool_spatial = tf.reduce_max(channel_map,
axis=[1],
keepdims=True)
assert max_pool_spatial.get_shape()[1] == 1
print("Spatial max pooling : ", max_pool_spatial)
# plt.pause(3.0)
concat = tf.concat([avg_pool_spatial, max_pool_spatial], 1)
assert concat.get_shape()[1] == 2
print(concat.get_shape())
concat = tp.models.Conv2D(
'spatialconvolution',
concat,
1,
kernel_size,
kernel_initializer=kernel_initializer,
use_bias=False)
assert concat.get_shape()[1] == 1
print("concat convolution shape : ", concat.get_shape())
concat = tf.sigmoid(concat, 'sigmoid')
concat = concat * channel_map
l = l + concat
# l = tf.concat([l, concat], axis=1)
l = BNReLU('blk_bna', l)
return l
def preresnet_group(
name: str,
l: tf.Tensor,
block_func: Callable[[tf.Tensor, int, int, str, bool, int, bool],
tf.Tensor],
features: int,
count: int,
stride: int,
isDownsampling: bool,
activation_function: str = "inrelu",
dilation: int = 1,
withDropout: bool = False,
addLongSkip: Optional[Tuple[int, tf.Tensor]] = None,
getLongSkipFrom: Optional[int] = None,
) -> Union[tf.Tensor, Tuple[tf.Tensor, tf.Tensor]]:
if addLongSkip and getLongSkipFrom:
assert addLongSkip[0] != getLongSkipFrom
if stride != 1:
assert dilation == 1
if dilation != 1:
assert stride == 1
with tf.variable_scope(name):
if addLongSkip is not None:
addSkipAt, skipConn = addLongSkip
else:
addSkipAt, skipConn = -1, None
for i in range(0, count):
with tf.variable_scope("block%d" % i):
# first block doesn't need activation
l = block_func(
l,
features,
stride if i == 0 else 1,
"no_preact" if i == 0 else activation_function,
isDownsampling if i == 0 else True,
dilation,
withDropout,
)
if getLongSkipFrom is not None:
if i == getLongSkipFrom:
skipConnection = l
if i == addSkipAt:
with tf.variable_scope("long_shortcut"):
changed_shortcut = resnet_shortcut(
skipConn, l.shape[-1], 1, True)
l = l + changed_shortcut
# end of each group need an extra activation
if activation_function == "bnrelu":
l = BNReLU("bnlast", l)
if activation_function == "inrelu":
l = INReLU(l)
if getLongSkipFrom is not None:
return l, skipConnection
else:
return l
def _build_graph(self, inputs):
images, truemap_coded = inputs
orig_imgs = images
true_np = truemap_coded[..., 0]
true_np = tf.cast(true_np, tf.int32)
true_np = tf.identity(true_np, name="truemap-np")
one_np = tf.one_hot(true_np, 2, axis=-1)
true_np = tf.expand_dims(true_np, axis=-1)
true_dist = truemap_coded[..., 1:]
true_dist = tf.identity(true_dist, name="truemap-dist")
####
with argscope(
Conv2D,
activation=tf.identity,
use_bias=False, # K.he initializer
W_init=tf.variance_scaling_initializer(scale=2.0, mode="fan_out"),
), argscope([Conv2D, BatchNorm], data_format=self.data_format):
i = tf.transpose(images, [0, 3, 1, 2])
i = i if not self.input_norm else i / 255.0
####
d = encoder(i, self.freeze)
d[0] = crop_op(d[0], (184, 184))
d[1] = crop_op(d[1], (72, 72))
####
np_feat = decoder("np", d)
np = BNReLU("preact_out_np", np_feat[-1])
dist_feat = decoder("dst", d)
dist = BNReLU("preact_out_dist", dist_feat[-1])
####
logi_np = Conv2D(
"conv_out_np", np, 2, 1, use_bias=True, activation=tf.identity
)
logi_np = tf.transpose(logi_np, [0, 2, 3, 1])
soft_np = tf.nn.softmax(logi_np, axis=-1)
prob_np = tf.identity(soft_np[..., 1], name="predmap-prob-np")
prob_np = tf.expand_dims(prob_np, axis=-1)
pred_np = tf.argmax(soft_np, axis=-1, name="predmap-np")
pred_np = tf.expand_dims(tf.cast(pred_np, tf.float32), axis=-1)
####
logi_dist = Conv2D(
"conv_out_dist", dist, 1, 1, use_bias=True, activation=tf.identity
)
logi_dist = tf.transpose(logi_dist, [0, 2, 3, 1])
prob_dist = tf.identity(logi_dist, name="predmap-prob-dist")
pred_dist = tf.identity(logi_dist, name="predmap-dist")
# encoded so that inference can extract all output at once
predmap_coded = tf.concat(
[prob_np, pred_dist], axis=-1, name="predmap-coded"
)
####
####
if get_current_tower_context().is_training:
######## LOSS
### Distance regression loss
loss_mse = pred_dist - true_dist
loss_mse = loss_mse * loss_mse
loss_mse = tf.reduce_mean(loss_mse, name="loss-mse")
add_moving_summary(loss_mse)
### Nuclei Blob classification loss
loss_bce = categorical_crossentropy(soft_np, one_np)
loss_bce = tf.reduce_mean(loss_bce, name="loss-bce")
add_moving_summary(loss_bce)
### combine the loss into single cost function
self.cost = tf.identity(loss_mse + loss_bce, name="overall-loss")
add_moving_summary(self.cost)
####
add_param_summary((".*/W", ["histogram"])) # monitor W
#### logging visual sthg
orig_imgs = tf.cast(orig_imgs, tf.uint8)
tf.summary.image("input", orig_imgs, max_outputs=1)
orig_imgs = crop_op(orig_imgs, (190, 190), "NHWC")
pred_np = colorize(prob_np[..., 0], cmap="jet")
true_np = colorize(true_np[..., 0], cmap="jet")
pred_dist = colorize(prob_dist[..., 0], cmap="jet")
true_dist = colorize(true_dist[..., 0], cmap="jet")
viz = tf.concat([orig_imgs, true_np, pred_np, true_dist, pred_dist,], 2)
tf.summary.image("output", viz, max_outputs=1)
return
def get_logits(image, num_classes=1000):
#
with ssdnet_argscope():
# dropblock
if get_current_tower_context().is_training:
dropblock_keep_prob = tf.get_variable('dropblock_keep_prob', (),
dtype=tf.float32,
trainable=False)
else:
dropblock_keep_prob = None
l = image #tf.transpose(image, perm=[0, 2, 3, 1])
# conv1
l = Conv2D('conv1',
l,
32,
4,
strides=2,
activation=None,
padding='SAME')
with tf.variable_scope('conv1'):
l = BNReLU(tf.concat([l, -l], axis=-1))
l = MaxPooling('pool1', l, 2)
# conv2
l = LinearBottleneck('conv2', l, 32, 32, 5, t=3, use_ab=True)
# l = l + LinearBottleneck('conv3', l, 24, 24, 5, t=3, use_ab=True)
ch_all = [48, 80, 112]
iters = [2, 4, 4]
mults = [4, 5, 6]
bsize = [3, 3, 3]
strides = [2, 2, 2]
hlist = []
for ii, (ch, it, mu, bs,
ss) in enumerate(zip(ch_all, iters, mults, bsize, strides)):
use_ab = (ii < 2)
for jj in range(it):
name = 'inc{}/{}'.format(ii, jj)
stride = ss if jj == 0 else 1
k = 3 if (jj < it // 2) else 5
swap_block = True if jj % 2 == 1 else False
l = inception(name,
l,
ch,
k,
stride,
t=mu,
swap_block=swap_block,
use_ab=use_ab)
# l = DropBlock('inc{}/drop'.format(ii), l, keep_prob=dropblock_keep_prob, block_size=bs)
nch = ch_all[-1] * mults[-1]
l = Conv2D('convf', l, nch, 1, activation=BNReLU)
l = GlobalAvgPooling('poolf', l)
### position sensitve fc ---
# hh, ww = l.get_shape().as_list()[1:3]
# l = tf.reshape(l, [-1, hh*ww, ch_all[-1]])
#
# ll = tf.split(l, hh*ww, axis=1)
# ll = [tf.layers.flatten(li) for li in ll]
# ll = [FullyConnected('psfc{:02d}'.format(ii), li, 24, activation=BNReLU) for ii, li in enumerate(ll)]
# l = tf.concat(ll, axis=-1)
### --- position sensitve fc
fc = FullyConnected('fc', l, 1280, activation=BNReLU)
fc = Dropout(fc, keep_prob=0.8)
logits = FullyConnected('linear', fc, num_classes, use_bias=True)
return logits
def Grconv(
x,
filters,
kernel_size,
strides=(1, 1),
padding='same',
data_format='channels_last',
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer=tf.contrib.layers.variance_scaling_initializer(2.0),
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
split=1,
glocal=False):
z3 = Conv2D('z3',
x,
filters=filters,
kernel_size=3,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=tf.identity,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
split=1)
out_shape = z3.get_shape().as_list()
z1 = Conv2D('z1',
x,
filters=filters,
kernel_size=1,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=tf.identity,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
split=1)
zf = fc(x,
out_shape,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=tf.identity,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer)
p = tf.get_variable('PPP', [3, 1, out_shape[2], out_shape[3]],
initializer=tf.ones_initializer(),
trainable=True)
p = tf.nn.softmax(p, 0)
z = p[0:1, :, :, :] * z3 + p[1:2, :, :, :] * z1 + p[2:3, :, :, :] * zf
z = BNReLU('sum', z)
return z
