class Resnet18(object):
def __init__(self, method='baseline'):
self.nn = NnKits()
self.method = method
def forward(self, input, prefix=''):
method = self.method
with tf.compat.v1.variable_scope('encoder'):
conv1 = self.nn.conv(input, 64, 7, 2, \
normalizer_fn=tf.keras.layers.BatchNormalization, \
activation_fn=tf.nn.relu) # H/2 - 64D
pool1 = self.nn.maxpool(conv1, 3) # H/4 - 64D
conv2 = self.nn.res33block(pool1, 64,
method=method) # H/4 - 64D
conv3 = self.nn.res33block(conv2, 128,
method=method) # H/8 - 128D
conv4 = self.nn.res33block(conv3, 256,
method=method) # H/16 - 256D
self.enc_feat = conv4
with tf.compat.v1.variable_scope('skips'):
self.skip1 = conv1
self.skip2 = pool1
self.skip3 = conv2
self.skip4 = conv3
def __init__(self, img):
self.nn = NnKits()
self.imgproc = ImgProc()
self.img = img
# init
if img is not None:
self.geo_reg_init()
def __init__(self, stages=5, up_type='conv', do_duo=False, upconv='conv'):
assert stages >= 4 or stages <= 6, '>>> UNet decoder stages must be within 4 or 6!!'
self.up_type = up_type
self.upconv = upconv
self.do_duo = do_duo
self.stages = stages
self.nn = NnKits()
self.mdeproc = MdeProc(None)
class ResASPP(object):
def __init__(self, aspp_type='aspp'):
self.nn = NnKits()
self.aspp = ASPP(aspp_type)
def forward(self, input):
with tf.compat.v1.variable_scope('encoder'):
conv1 = self.nn.conv(input, 64, 7, 2) # H/2 - 64D
pool1 = self.nn.maxpool(conv1, 3) # H/4 - 64D
conv2 = self.nn.resblock(pool1, 64, 3) # H/8 - 64D
conv3 = self.nn.resblock(conv2, 128, 4) # H/16 - 128D
pool3 = self.nn.maxpool(conv3, 3) # H/32 - 128D
self.enc_feat = self.aspp.enc(pool3) # H/32
with tf.compat.v1.variable_scope('skips'):
self.skip1 = conv1
self.skip2 = pool1
self.skip3 = conv2
self.skip4 = conv3
class Resnet(object):
def __init__(self, aspp_type='aspp'):
self.nn = NnKits()
def forward(self, input):
with tf.compat.v1.variable_scope('encoder'):
self.conv1 = self.nn.conv(input, 64, 7, 2) # H/2 - 64D
self.pool1 = self.nn.maxpool(self.conv1, 3) # H/4 - 64D
self.conv2 = self.nn.resblock(self.pool1, 64, 3) # H/8 - 256D
self.conv3 = self.nn.resblock(self.conv2, 128, 4) # H/16 - 512D
self.conv4 = self.nn.resblock(self.conv3, 256, 6) # H/32 - 1024D
self.enc_feat = self.nn.resblock(self.conv4, 512,
3) # H/64 - 2048D
with tf.compat.v1.variable_scope('skips'):
self.skip1 = self.conv1
self.skip2 = self.pool1
self.skip3 = self.conv2
self.skip4 = self.conv3
self.skip5 = self.conv4
class Vgg(object):
def __init__(self, aspp_type='aspp'):
self.nn = NnKits()
def forward(self, input):
with tf.compat.v1.variable_scope('encoder'):
self.conv1 = self.nn.conv_block(input, 32, 7) # H/2
self.conv2 = self.nn.conv_block(self.conv1, 64, 5) # H/4
self.conv3 = self.nn.conv_block(self.conv2, 128, 3) # H/8
self.conv4 = self.nn.conv_block(self.conv3, 256, 3) # H/16
self.conv5 = self.nn.conv_block(self.conv4, 512, 3) # H/32
self.conv6 = self.nn.conv_block(self.conv5, 512, 3) # H/64
self.enc_feat = self.nn.conv_block(self.conv6, 512, 3) # H/128
with tf.compat.v1.variable_scope('skips'):
self.skip1 = self.conv1
self.skip2 = self.conv2
self.skip3 = self.conv3
self.skip4 = self.conv4
self.skip5 = self.conv5
self.skip6 = self.conv6
def __init__(self,
params,
mode,
left,
right,
reuse_variables=None,
model_index=0):
self.params = params
self.mode = mode
self.left = left
self.right = right
self.model_collection = ['model_' + str(model_index)]
self.reuse_variables = reuse_variables
# init toolkits
self.nn = NnKits()
self.imgproc = ImgProc()
self.mdeproc = MdeProc(self.left)
self.mde_loss = MdeLoss()
# init codec
self.encoder(self.params.encoder)
self.decoder()
# init postproc
if not self.mode == 'train':
self.postproc = MdePostproc(self.left)
# run model/output
self.build_model()
self.build_outputs()
if not self.mode == 'train':
return
# run loss
self.build_losses()
def __init__(self, method='baseline'):
self.nn = NnKits()
self.method = method
def __init__(self, aspp_type='aspp'):
self.nn = NnKits()
def __init__(self, aspp_type='aspp'):
self.nn = NnKits()
self.aspp = ASPP(aspp_type)
class MdeProc(object):
"""utils for monodepth"""
def __init__(self, img):
self.nn = NnKits()
self.imgproc = ImgProc()
self.img = img
# init
if img is not None:
self.geo_reg_init()
# Regularization
def geo_reg_init(self):
b = self.img.get_shape().as_list()[0]
h = self.img.get_shape().as_list()[1]
w = self.img.get_shape().as_list()[2]
m, l = np.meshgrid(np.linspace(0, 1, w), np.linspace(0, 1, h))
l_mask = np.clip(2*(0.5 - l), 0, 1)
m_mask = 1. - np.clip(2.*abs(0.5 - m), 0, 1)
geo_reg_mask = l_mask * m_mask
geo_reg_mask = np.repeat(geo_reg_mask[np.newaxis, :, :], b, axis=0)
geo_reg_mask = np.repeat(geo_reg_mask[:, :, :, np.newaxis], 2, axis=3)
self.geo_reg_mask = self.imgproc.scale_pyramid(geo_reg_mask, 4)
# Disparity Generation
def get_disp(self, x, do_due=False):
if do_due:
disp = 0.3 * self.nn.conv(x, 4, 3, 1, tf.nn.sigmoid)
else:
disp = 0.3 * self.nn.conv(x, 2, 3, 1, tf.nn.sigmoid)
return disp
# Image Conversion by Disparity
def generate_image_left(self, img, disp):
return bilinear_sampler_1d_h(img, -disp)
def generate_image_right(self, img, disp):
return bilinear_sampler_1d_h(img, disp)
# Smoothness
def get_disparity_smoothness(self, disp, pyramid):
disp_gradients_x = [self.imgproc.gradient_x(d) for d in disp]
disp_gradients_y = [self.imgproc.gradient_y(d) for d in disp]
image_gradients_x = [self.imgproc.gradient_x(img) for img in pyramid]
image_gradients_y = [self.imgproc.gradient_y(img) for img in pyramid]
weights_x = [tf.exp(-tf.reduce_mean(tf.abs(g), 3, keepdims=True)) for g in image_gradients_x]
weights_y = [tf.exp(-tf.reduce_mean(tf.abs(g), 3, keepdims=True)) for g in image_gradients_y]
smoothness_x = [disp_gradients_x[i] * weights_x[i] for i in range(4)]
smoothness_y = [disp_gradients_y[i] * weights_y[i] for i in range(4)]
return smoothness_x + smoothness_y
def get_disparity_edge(self, disp, pyramid):
disp_edge = [tf.image.sobel_edges(d) for d in disp]
image_edge = [tf.image.sobel_edges(img) for img in pyramid]
disp_edge_x = [tf.abs(d[:,:,:,:,0]) for d in disp_edge]
image_edge_x = [tf.abs(img[:,:,:,:,0]) for img in image_edge]
weights_x = [tf.exp(-tf.reduce_mean(tf.abs(g), 3, keep_dims=True)) for g in image_edge_x]
edgeness_x = [disp_edge_x[i] * weights_x[i] for i in range(4)]
return edgeness_x
# occlusion detection
def get_disp_occ(self, disp, kernel_size=21):
p = np.floor((kernel_size - 1) / 2).astype(np.int32)
disp_p = tf.pad(disp, [[0, 0], [1, 1], [p, p], [0, 0]])
disp_p_ap = tf.nn.avg_pool2d(disp_p, ksize=[1, 3, p, 1], strides=[1, 1, 1, 1], padding='SAME')
gx = disp_p_ap[:,1:-1,:(-2*p),:] - disp_p_ap[:,1:-1,(p+1):(-p+1),:]
gx_max, gx_min = tf.reduce_max(gx), tf.reduce_min(gx)
edgeness = tf.sigmoid((((gx - gx_min) / ( gx_max - gx_min))-0.5)*32)
occ = tf.nn.relu(0.5 - edgeness)
return occ
def __init__(self, type='aspp'):
self.nn = NnKits()
self.enc = self.aspp
def __init__(self):
self.nn = NnKits()