def deconv2d_layer_concat(x,
name,
W_s,
concat_x,
output_shape=None,
stride=2,
stddev=0.02,
if_relu=False):
'''
Deconv2d operator for U-Net concat.
Args:
x: inputs
W_s: shape of weight
output_shape: shape after deconv2d
'''
if output_shape == None:
x_shape = tf.shape(x)
output_shape = tf.stack(
[x_shape[0], x_shape[1] * 2, x_shape[2] * 2, x_shape[3] // 2])
W_t = utils.weight_variable(W_s, stddev=stddev, name='W_' + name)
b_t = utils.bias_variable([W_s[2]], name='b_' + name)
#conv_t = utils.conv2d_transpose_strided_valid(x, W_t, b_t, output_shape, stride)
conv_t = utils.conv2d_transpose_strided(x, W_t, b_t, output_shape, stride)
if if_relu:
conv_t = tf.nn.relu(conv_t, name=name + '_relu')
conv_concat = utils.crop_and_concat(concat_x, conv_t)
return conv_concat
def conv2d_layer(x,
name,
W_s,
pool_,
if_relu=False,
stride=2,
stddev=0.02,
if_dropout=False,
keep_prob_=1):
'''Conv2d operator
Args:
pool_: if pool_==0:not pooling else pooling
'''
W = utils.weight_variable(W_s, stddev=stddev, name='W' + name)
b = utils.bias_variable([W_s[3]], name='b' + name)
#conv = utils.conv2d_strided_valid(x, W, b, stride)
conv = utils.conv2d_strided(x, W, b, stride)
print('shape after conv: ', conv.shape)
print('--------------------------------')
if if_dropout:
conv = tf.nn.dropout(conv, keep_prob_)
if if_relu:
conv = tf.nn.relu(conv, name=name + '_relu')
if pool_:
conv = utils.max_pool(conv, pool_, 2)
print('shape after pool: ', conv.shape)
return conv
def deconv2d_layer(x, name, W_s, output_shape=None, stride=2):
'''Deconv2d operator
Args:
x: inputs
W_s: shape of weight
output_shape: shape after deconv2d
'''
W_t = utils.weight_variable(W_s, name='W_' + name)
b_t = utils.bias_variable([W_s[2]], name='b_' + name)
conv_t = utils.conv2d_transpose_strided(x, W_t, b_t, output_shape, stride)
print('conv_%s: ' % name, conv_t.get_shape())
return conv_t
def u_net_inference(self, images, inference_name, channel, keep_prob):
"""
Semantic segmentation network definition
:param image: input image. Should have values in range 0-255
:param keep_prob:
:return:
"""
print("setting up resnet101 initialized conv layers ...")
#mean_pixel = np.mean(mean, axis=(0, 1))
processed_images = utils.process_image(images, cfgs.mean_pixel)
#processed_images = tf.nn.dropout(processed_images, self.input_keep_prob)
with tf.variable_scope(inference_name):
#U-Net
logits = self.u_net.u_net_op(x=processed_images,
keep_prob_=keep_prob,
channels=channel,
n_class=cfgs.n_class,
layers=cfgs.layers,
features_root=cfgs.features_root,
filter_size=cfgs.filter_size,
pool_size=cfgs.pool_size)
self.unet_pro = tf.nn.softmax(logits)[:, 2:cfgs.IMAGE_SIZE[0] +
2, :, :]
anno_pred = tf.argmax(tf.nn.softmax(logits),
dimension=3,
name="prediction")
#pdb.set_trace()
print('logits shape', logits.shape)
return logits[:, 2:cfgs.IMAGE_SIZE[0] +
2, :, :], anno_pred[0, 2:cfgs.IMAGE_SIZE[0] + 2, :]
def valid_one_epoch(self, sess, data_loader, data_num, epoch, step):
'''
Generate once of image which remove instruments.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
total_loss, total_im_warp_loss, total_flow_loss = 0, 0, 0
total_l1, total_im_l1, total_l2, total_im_l2, total_psnr = 0, 0, 0, 0, 0
count = 0
t0 = time.time()
im_warp_loss, flow_loss = 0, 0
#True number to loss
t_count = 0
for count in range(1, data_num):
step += 1
images, images_da, mask, fn, flag, rect_param = data_loader.get_next_sequence_valid(
)
if flag == False:
#print(fn)
# Can't find prev data.
continue
# Origin optical flow
for frame in range(1, cfgs.seq_frames):
im, last_im = images[frame], images[frame - 1]
flow = sess.run(self.flow_tensor,
feed_dict={
self.flow_img0: im,
self.flow_img1: last_im
})
# Da optical flow
for frame in range(1, cfgs.seq_frames):
im, last_im = images_da[frame], images_da[frame - 1]
flow_da = sess.run(self.flow_tensor,
feed_dict={
self.flow_img0: im,
self.flow_img1: last_im
})
normal_flow, normal_max_v = normal_data(flow)
inpt_input, flag_ = concat_data(normal_flow, mask, cfgs.grid)
#epd_sd_mask, epd_rect_param = expand_sdm(rect_param, cfgs.gamma, cfgs.epd_ratio)
sd_mask = sdm(rect_param, cfgs.gamma)
if flag_ == False:
print(fn)
#After grid, area of mask = 0
continue
t_count += 1
loss, im_warp_loss, flow_loss,\
l_inpt_p, \
l1_e, l2_e, im_l1_e, im_l2_e, psnr,\
l_ori_im, l_warped_im,\
inpt_warped_im, inpt_flow, pred_flow \
=sess.run([self.loss, self.im_warp_loss, self.flow_loss_,\
self.local_patch_inpt_flow, \
self.l1_e, self.l2_e, self.w_l1_e, self.w_l2_e, self.w_psnr,\
self.local_ori_im, self.local_warped_im,\
self.inpt_warped_im, self.inpt_pred_flow, self.pred_complete_flow],\
feed_dict={self.flow_img1: last_im,
self.flow_img0: im,
self.ori_cur_im: images[1],
self.ori_flow: flow,
self.max_v: normal_max_v,
self.inpt_data: inpt_input,
#self.epd_sd_mask: epd_sd_mask,
#self.epd_rect_param: epd_rect_param,
self.sd_mask: sd_mask,
self.rect_param: rect_param})
#if count % 10 == 0:
if False:
self.view(np.expand_dims(fn, 0),
inpt_warped_im,
images_da[cfgs.seq_frames - 2],
images_da[cfgs.seq_frames - 1],
step,
f_path='valid')
self.view_patch_one(l_ori_im[0],
fn,
step,
'l_ori',
f_path='valid')
self.view_patch_one(l_warped_im[0],
fn,
step,
'l_warped',
f_path='valid')
self.view_flow_patch_one(l_inpt_p[0],
fn,
step,
'l_inpt_p',
f_path='valid')
self.view_flow_one(flow[0], fn, step, f_path='valid')
self.view_flow_one(inpt_flow[0], fn, step, '_inpt', 'valid')
self.view_flow_one(flow_da[0], fn, step, '_da', 'valid')
self.view_flow_one(pred_flow[0],
fn,
step,
'complete',
f_path='valid')
#3. calculate loss
total_loss += loss
total_im_warp_loss += im_warp_loss
total_flow_loss += flow_loss
total_l1 += l1_e
total_im_l1 += im_l1_e
total_l2 += l2_e
total_im_l2 += im_l2_e
total_psnr += psnr
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
#line = 'Valid epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t m_imW_loss = %.4f\t m_f_loss = %.4f\t time = %.2f' % (epoch, cfgs.inpt_lr, step, count, loss, (total_loss/t_count), (total_im_warp_loss/t_count), (total_flow_loss/t_count), time_per_batch)
line = 'Valid epoch %2d\t step = %4d\t count = %4d\t m_imW_loss = %.2f\t m_f_loss = %.2f\t l1_e = %.3f\t im_l1_e = %.3f\t l2_e = %.3f\t im_l2_e = %.3f\t psnr = %4f\t' % (
epoch, step, count, (total_im_warp_loss / t_count),
(total_flow_loss / t_count), (total_l1 / count),
(total_im_l1 / count), (total_l2 / count),
(total_im_l2 / count), (total_psnr / count))
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t m_imW_loss = %.4f\t m_f_loss = %.4f\t '
% (epoch, cfgs.inpt_lr, (total_loss / t_count),
(total_im_warp_loss / t_count), (total_flow_loss / t_count)))
print('Take time %3.1f' % (time.time() - t0))
return step
def valid_one_epoch(self, sess, data_loader, data_num, epoch, step):
'''
Generate once of image which remove instruments.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
sum_acc, sum_acc_iou, sum_acc_ellip, total_loss = 0, 0, 0, 0
count = 0
t0 = time.time()
mean_acc, mean_acc_iou, mean_acc_label, mean_acc_ellip = 0, 0, 0, 0
self.ellip_acc, self.accu, self.accu_iou, loss = 0, 0, 0, 0
for count in range(1, data_num):
step += 1
images, images_da, mask, fn, flag, mask_sum = data_loader.get_next_sequence(
)
if flag == False:
print(fn)
# Can't find prev data.
continue
# Origin optical flow
for frame in range(1, cfgs.seq_frames):
im, last_im = images[frame], images[frame - 1]
flow = sess.run(self.flow_tensor,
feed_dict={
self.flow_img0: im,
self.flow_img1: last_im
})
# Da optical flow
for frame in range(1, cfgs.seq_frames):
im, last_im = images_da[frame], images_da[frame - 1]
flow_da = sess.run(self.flow_tensor,
feed_dict={
self.flow_img0: im,
self.flow_img1: last_im
})
normal_flow, normal_max_v = normal_data(flow)
inpt_input, flag_ = concat_data(normal_flow, mask, cfgs.grid)
if flag_ == False:
print(fn)
#After grid, area of mask = 0
continue
loss, inpt_warped_im, inpt_flow, pred_flow =sess.run([self.loss, \
self.inpt_warped_im, self.inpt_pred_flow, self.pred_complete_flow],\
feed_dict={self.flow_img1: last_im,
self.flow_img0: im,
self.ori_flow: flow,
self.max_v: normal_max_v,
self.inpt_data: inpt_input})
#if count % 20 == 0:
if True:
self.view(np.expand_dims(fn, 0),
inpt_warped_im,
images_da[cfgs.seq_frames - 2],
images_da[cfgs.seq_frames - 1],
step,
f_path='valid')
self.view_flow_one(flow[0], fn, step, f_path='valid')
self.view_flow_one(inpt_flow[0], fn, step, '_inpt', 'valid')
self.view_flow_one(flow_da[0], fn, step, '_da', 'valid')
self.view_flow_one(pred_flow[0],
fn,
step,
'complete',
f_path='valid')
#2. calculate accurary
self.ellip_acc = 0
sum_acc += self.accu
sum_acc_iou += self.accu_iou
sum_acc_ellip += self.ellip_acc
mean_acc = sum_acc / count
mean_acc_iou = sum_acc_iou / count
mean_acc_ellip = sum_acc_ellip / count
#3. calculate loss
total_loss += loss
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
line = 'Valid epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t max_v = %.2f\t fn = %s\t time = %.2f' % (
epoch, cfgs.inpt_lr, step, count, loss,
(total_loss / count), normal_max_v, fn, time_per_batch)
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t train_acc = %.2f%%\t train_iou_acc = %.2f%%\t train_ellip_acc = %.2f'
% (epoch, cfgs.inpt_lr, (total_loss / count), (sum_acc / count),
(sum_acc_iou / count), (sum_acc_ellip / count)))
print('Take time %3.1f' % (time.time() - t0))
return step
def valid_once(self, sess, data_loader, data_num, epoch, step):
'''
Generate once training or validation.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
sum_acc = 0
sum_acc_iou = 0
sum_acc_ellip = 0
count = 0
total_loss = 0
t0 = time.time()
mean_acc = 0
mean_acc_iou = 0
mean_acc_label = 0
mean_acc_ellip = 0
for count in range(1, data_num):
step += 1
images, images_pad, gt, fn = data_loader.get_next_sequence()
# Optical flow
optflow = []
for frame in range(1, cfgs.seq_frames):
im, last_im = images[frame], images[frame - 1]
flow = sess.run(self.flow_tensor,
feed_dict={
self.flow_img0: im,
self.flow_img1: last_im
})
optflow.append(flow)
# Static segmentation(logits)
static_segm = []
for frame in range(cfgs.seq_frames):
im = images_pad[frame]
x_logits = sess.run(self.static_output,
feed_dict={self.unet_images: im})
static_segm.append(x_logits)
# GRFP
rnn_input = {
self.gru_lr: cfgs.grfp_lr,
self.gru_input_images_tensor: np.stack(images),
self.gru_input_flow_tensor: np.stack(optflow),
self.gru_input_seg_tensor: np.stack(static_segm),
self.gru_targets: gt
}
loss, pred, pred_pro,\
self.accu, self.accu_iou \
= sess.run([self.gru_loss, self.gru_pred, self.gru_pred_pro,
self.accu_tensor, self.accu_iou_tensor],
feed_dict=rnn_input)
#self.view(fn, pred, pred_pro, images[cfgs.seq_frames], step)
#2. calculate accurary
self.ellip_acc = 0
#self.calculate_acc(images[cfgs.seq_frames, 2:cfgs.ANNO_IMAGE_SIZE[0]+2, :, :].copy(), fn, pred, pred_pro, gt, ellip_infos_, if_epoch=if_epoch)
sum_acc += self.accu
sum_acc_iou += self.accu_iou
sum_acc_ellip += self.ellip_acc
mean_acc = sum_acc / count
mean_acc_iou = sum_acc_iou / count
mean_acc_ellip = sum_acc_ellip / count
#3. calculate loss
total_loss += loss
pdb.set_trace()
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
line = 'Valid epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t acc = %.2f%%\t iou_acc = %.2f%%\t ellip_acc = %.2f\t time = %.2f' % (
epoch, self.learning_rate, step, count, loss,
(total_loss / count), mean_acc, mean_acc_iou, mean_acc_ellip,
time_per_batch)
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t train_acc = %.2f%%\t train_iou_acc = %.2f%%\t train_ellip_acc = %.2f'
% (epoch, self.learning_rate, (total_loss / count),
(sum_acc / count), (sum_acc_iou / count),
(sum_acc_ellip / count)))
print('Take time %3.1f' % (time.time() - t0))
return step
def train_one_epoch(self, sess, data_loader, data_num, epoch, step):
'''
Generate once training or validation.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
sum_acc = 0
sum_acc_iou = 0
sum_acc_ellip = 0
count = 0
total_loss = 0
t0 = time.time()
mean_acc = 0
mean_acc_iou = 0
mean_acc_label = 0
mean_acc_ellip = 0
self.ellip_acc, self.accu, self.accu_iou, loss = 0, 0, 0, 0
for count in range(1, data_num):
step += 1
images, images_pad, gt, fn = data_loader.get_next_sequence()
# Optical flow
optflow = []
for frame in range(1, cfgs.seq_frames):
im, last_im = images[frame], images[frame - 1]
flow = sess.run(self.flow_tensor,
feed_dict={
self.flow_img0: im,
self.flow_img1: last_im
})
optflow.append(flow)
# Static segmentation(logits)
static_segm = []
for frame in range(cfgs.seq_frames):
im = images_pad[frame]
x_logits, x, x_pro = sess.run(
[self.static_output, self.static_anno_pred, self.unet_pro],
feed_dict={self.unet_images: im})
static_segm.append(x_logits)
# GRFP
rnn_input = {
self.gru_lr: cfgs.grfp_lr,
self.gru_input_images_tensor: np.stack(images),
self.gru_input_flow_tensor: np.stack(optflow),
self.gru_input_seg_tensor: np.stack(static_segm),
self.gru_targets: gt
}
prev_warped, I_diff, pro_prev_warped, anno_prev_warped, \
cur_pro = \
sess.run([self.gru_prev_warped, self.gru_I_diff, self.gru_pro_prev_warped, self.gru_anno_prev_warped, \
self.gru_pred_pro],
feed_dict=rnn_input)
#pdb.set_trace()
self.view(np.expand_dims(fn, 0), anno_prev_warped,
images[cfgs.seq_frames - 2], images[cfgs.seq_frames - 1],
step)
#self.view(np.expand_dims(fn, 0), prev_warping, images[cfgs.seq_frames-2], images[cfgs.seq_frames-1], step)
filter_input = {
self.inst_mask_pro: pro_prev_warped,
self.corn_mask_pro: cur_pro
}
persist_im = sess.run(self.persist_im, feed_dict=filter_input)
cv2.imwrite(os.path.join('persist_inter1', '%s_persist.bmp' % fn),
persist_im[0] * 255)
#pdb.set_trace()
#2. calculate accurary
self.ellip_acc = 0
#self.acc, self.acc_iou = 0
#self.calculate_acc(images[cfgs.seq_frames, 2:cfgs.ANNO_IMAGE_SIZE[0]+2, :, :].copy(), fn, pred, pred_pro, gt, ellip_infos_, if_epoch=if_epoch)
sum_acc += self.accu
sum_acc_iou += self.accu_iou
sum_acc_ellip += self.ellip_acc
mean_acc = sum_acc / count
mean_acc_iou = sum_acc_iou / count
mean_acc_ellip = sum_acc_ellip / count
#3. calculate loss
total_loss += loss
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
line = 'Train epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t acc = %.2f%%\t iou_acc = %.2f%%\t ellip_acc = %.2f\t time = %.2f' % (
epoch, self.learning_rate, step, count, loss,
(total_loss / count), mean_acc, mean_acc_iou, mean_acc_ellip,
time_per_batch)
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t train_acc = %.2f%%\t train_iou_acc = %.2f%%\t train_ellip_acc = %.2f'
% (epoch, self.learning_rate, (total_loss / count),
(sum_acc / count), (sum_acc_iou / count),
(sum_acc_ellip / count)))
print('Take time %3.1f' % (time.time() - t0))
return step
def train_one_epoch_warp_inpt_flow(self, sess, data_loader, data_num,
epoch, step):
'''
Generate one epoch of images using inpatinting flow.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
sum_acc, sum_acc_iou, sum_acc_ellip, count, total_loss = 0, 0, 0, 0, 0
t0 = time.time()
mean_acc, mean_acc_iou, mean_acc_label, mean_acc_ellip = 0, 0, 0, 0
self.ellip_acc, self.accu, self.accu_iou, loss = 0, 0, 0, 0
for count in range(1, data_num):
step += 1
images, images_pad, gt, fn = data_loader.get_next_sequence()
# Static segmentation(logits)
static_segm = []
static_anno = []
static_corn_mask = []
static_corn_mask_im = []
for frame in range(cfgs.seq_frames):
im = images_pad[frame]
im_ori = images[frame]
x_logits, x, x_pro, corn_mask, corn_mask_im, inst_mask = sess.run(
[
self.static_output, self.static_anno_pred,
self.unet_pro, self.corn_mask, self.corn_mask_im,
self.no_inst_mask
],
feed_dict={
self.unet_images: im,
self.persist_im: im_ori
})
x = np.expand_dims(x, 2)
x3 = x
x3 = np.append(x3, x, axis=2)
x3 = np.expand_dims(np.append(x3, x, axis=2), axis=0)
static_segm.append(x_logits)
static_anno.append(x3)
static_corn_mask.append(corn_mask)
static_corn_mask_im.append(corn_mask_im)
#flow inpainting
path = os.path.join(cfgs.inpt_flow_pickle, '%s.pickle' % fn)
if os.path.exists(path):
flow_inpt = self.load_pickle(path)
flow_inpt = np.pad(flow_inpt, ((0, 0), (0, 4), (0, 0), (0, 0)),
'constant')
else:
continue
# Optical flow
optflow = []
optflow_no_inst = []
for frame in range(1, cfgs.seq_frames):
#Origin Image
im, last_im = images[frame], images[frame - 1]
#Segmentation
segm, last_segm = static_segm[frame], static_segm[frame - 1]
#Anno
#im, last_im = static_anno[frame], static_anno[frame-1]
flow, warped, flow_no_inst, flow_comb, flow_insect,\
warped_no_inst, in_area,\
warped_inpt, warped_insect_inpt = sess.run(\
[self.flow_tensor, self.warped_im, \
self.no_inst_flow, self.flow_comb, self.pred_corn_flow, \
self.no_inst_warped_segm, self.insect_area,\
self.result_warped, self.insect_result_warped],
feed_dict={self.flow_img0: im,
self.flow_img1: last_im,
self.flow_inst_mask: inst_mask,
self.flow_corn_mask: corn_mask,
self.flow_prev_corn_mask: static_corn_mask[cfgs.seq_frames-2],
self.flow_segm: last_segm,
self.flow_warp_img1: last_segm,
self.flow_warp_img0: segm,
self.warp_flow: flow_inpt})
optflow.append(flow_comb)
optflow_no_inst.append(flow_no_inst)
#self.view(np.expand_dims(fn, 0), warped_no_inst, images[cfgs.seq_frames-2], images[cfgs.seq_frames-1], step, '_no_inst')
#self.view(np.expand_dims(fn, 0), warped_no_inst*127, (static_anno[cfgs.seq_frames-2]*127).astype(np.uint8), (static_anno[cfgs.seq_frames-1]*127).astype(np.uint8), step)
#self.view(np.expand_dims(fn, 0), warped_inpt*127, (static_anno[cfgs.seq_frames-2]*127).astype(np.uint8), (static_anno[cfgs.seq_frames-1]*127).astype(np.uint8), step, '_inpt', if_more=False)
self.view(
np.expand_dims(fn, 0),
warped_insect_inpt * 255,
(static_anno[cfgs.seq_frames - 2] * 127).astype(np.uint8),
(static_anno[cfgs.seq_frames - 1] * 127).astype(np.uint8),
step,
'_insect_inpt',
if_more=True)
self.view_flow_one(flow_inpt[0], fn, step, '_inpt')
#self.view_flow_one(flow_no_inst[0], fn, step, '_no_inst')
#self.view_flow_one(flow_insect[0], fn, step, '_insect')
self.view_flow_one(flow[0], fn, step)
#self.view_inst_mask_one((255-inst_mask*255).astype(np.uint8), fn, step)
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
line = 'Train epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t acc = %.2f%%\t iou_acc = %.2f%%\t ellip_acc = %.2f\t time = %.2f' % (
epoch, self.learning_rate, step, count, loss,
(total_loss / count), mean_acc, mean_acc_iou, mean_acc_ellip,
time_per_batch)
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t train_acc = %.2f%%\t train_iou_acc = %.2f%%\t train_ellip_acc = %.2f'
% (epoch, self.learning_rate, (total_loss / count),
(sum_acc / count), (sum_acc_iou / count),
(sum_acc_ellip / count)))
print('Take time %3.1f' % (time.time() - t0))
return step
def train_one_epoch_remv_occ_segm(self, sess, data_loader, data_num, epoch,
step):
'''
Generate once of image which remove instruments.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
sum_acc = 0
sum_acc_iou = 0
sum_acc_ellip = 0
count = 0
total_loss = 0
t0 = time.time()
mean_acc = 0
mean_acc_iou = 0
mean_acc_label = 0
mean_acc_ellip = 0
self.ellip_acc, self.accu, self.accu_iou, loss = 0, 0, 0, 0
for count in range(1, data_num):
step += 1
images, images_pad, gt, fn = data_loader.get_next_sequence()
# Static segmentation(logits)
static_segm = []
static_anno = []
static_corn_mask = []
static_corn_mask_im = []
for frame in range(cfgs.seq_frames):
im = images_pad[frame]
im_ori = images[frame]
x_logits, x, x_pro, corn_mask, corn_mask_im, inst_mask = sess.run(
[
self.static_output, self.static_anno_pred,
self.unet_pro, self.corn_mask, self.corn_mask_im,
self.no_inst_mask
],
feed_dict={
self.unet_images: im,
self.persist_im: im_ori
})
x = np.expand_dims(x, 2)
x3 = x
x3 = np.append(x3, x, axis=2)
x3 = np.expand_dims(np.append(x3, x, axis=2), axis=0)
if cfgs.if_dilate:
inst_mask = self.dilate_(inst_mask)
static_segm.append(x_logits)
static_anno.append(x3)
static_corn_mask.append(corn_mask)
static_corn_mask_im.append(corn_mask_im)
# Optical flow
optflow = []
optflow_no_inst = []
for frame in range(1, cfgs.seq_frames):
#Origin Image
im, last_im = images[frame], images[frame - 1]
#Segmentation
segm, last_segm = static_segm[frame], static_segm[frame - 1]
#Anno
#im, last_im = static_anno[frame], static_anno[frame-1]
flow, warped, flow_no_inst, flow_comb, flow_insect,\
warped_no_inst, in_area,\
flow_mean = sess.run(\
[self.flow_tensor, self.warped_im, \
self.no_inst_flow, self.flow_comb, self.pred_corn_flow, \
self.no_inst_warped_segm, self.insect_area,\
self.corn_flow_mean],
feed_dict={self.flow_img0: im,
self.flow_img1: last_im,
self.flow_inst_mask: inst_mask,
self.flow_corn_mask: corn_mask,
self.flow_prev_corn_mask: static_corn_mask[cfgs.seq_frames-2],
self.flow_segm: last_segm})
optflow.append(flow_comb)
optflow_no_inst.append(flow_no_inst)
#self.view(np.expand_dims(fn, 0), warped_no_inst, images[cfgs.seq_frames-2], images[cfgs.seq_frames-1], step, '_no_inst')
#self.view(np.expand_dims(fn, 0), warped_no_inst*127, (static_anno[cfgs.seq_frames-2]*127).astype(np.uint8), (static_anno[cfgs.seq_frames-1]*127).astype(np.uint8), step)
#self.view(np.expand_dims(fn, 0), warped*127, (static_anno[cfgs.seq_frames-2]*127).astype(np.uint8), (static_anno[cfgs.seq_frames-1]*127).astype(np.uint8), step)
#self.view_flow_one(flow_comb[0], fn, step, 'comb')
self.view_flow_one(flow_no_inst[0], fn, step, '_no_inst')
#self.view_flow_one(flow_insect[0], fn, step, '_insect')
self.view_flow_one(flow[0], fn, step)
self.view_inst_mask_one((255 - inst_mask * 255).astype(np.uint8),
fn, step)
self.pickle_f(flow_no_inst[0], fn)
#2. calculate accurary
self.ellip_acc = 0
#self.acc, self.acc_iou = 0
#self.calculate_acc(images[cfgs.seq_frames, 2:cfgs.ANNO_IMAGE_SIZE[0]+2, :, :].copy(), fn, pred, pred_pro, gt, ellip_infos_, if_epoch=if_epoch)
sum_acc += self.accu
sum_acc_iou += self.accu_iou
sum_acc_ellip += self.ellip_acc
mean_acc = sum_acc / count
mean_acc_iou = sum_acc_iou / count
mean_acc_ellip = sum_acc_ellip / count
#3. calculate loss
total_loss += loss
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
line = 'Train epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t acc = %.2f%%\t iou_acc = %.2f%%\t ellip_acc = %.2f\t time = %.2f' % (
epoch, self.learning_rate, step, count, loss,
(total_loss / count), mean_acc, mean_acc_iou, mean_acc_ellip,
time_per_batch)
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t train_acc = %.2f%%\t train_iou_acc = %.2f%%\t train_ellip_acc = %.2f'
% (epoch, self.learning_rate, (total_loss / count),
(sum_acc / count), (sum_acc_iou / count),
(sum_acc_ellip / count)))
print('Take time %3.1f' % (time.time() - t0))
return step
def valid_one_epoch_warped_im(self, sess, data_loader, data_num, epoch,
step):
'''
Using warped prev image to do new segmentation.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
total_loss, total_im_warp_loss, total_flow_loss = 0, 0, 0
count = 0
t0 = time.time()
im_warp_loss, flow_loss, loss = 0, 0, 0
#True number to loss
t_count = 0
for count in range(1, data_num):
step += 1
#1. Load data.
images_pad, mask, fn, flag = data_loader.get_next_sequence()
if flag == False:
print(fn)
# Can't find prev data.
continue
feed_dict = {
self.prev_img: images_pad[0],
self.cur_img: images_pad[1],
self.cur_mask: mask
}
inpt_flow, inpt_warped_im,\
flow,\
cur_anno, reconst_anno, warped_anno = sess.run(\
[self.inpt_pred_flow_re, self.warped_prev_im,\
self.flow_tensor_re,\
self.cur_static_anno_pred, self.reconst_cur_anno, self.warped_static_anno_pred],\
feed_dict=feed_dict)
#reconst_anno = sess.run(self.reconst_cur_anno, feed_dict=feed_dict)
#cur_anno = sess.run(self.cur_static_anno_pred, feed_dict=feed_dict)
#
self.view_flow_one(inpt_flow[0], fn, step, 'flow_inpt', 'valid')
self.view_flow_one(flow[0], fn, step, 'flow', 'valid')
#self.view_patch_one(inpt_warped_im[0], fn, step, 'im', 'valid')
self.view_im_one(cur_anno[0] * 127, fn, step, 'anno_cur', 'valid')
self.view_im_one(warped_anno[0] * 127, fn, step, 'anno_warped',
'valid')
self.view_im_one(reconst_anno[0] * 255, fn, step, 'anno_reconst',
'valid')
#self.view_im_cvt_one(images_pad[0][0]+inpt_warped_im[0], fn, step, 'im_prev', 'valid')
#self.view_im_cvt_one(images_pad[1][0]+inpt_warped_im[0], fn, step, 'im_cur', 'valid')
self.view_im_cvt_one(images_pad[0][0], fn, step, 'im_prev',
'valid')
self.view_im_cvt_one(images_pad[1][0], fn, step, 'im_cur', 'valid')
self.view_im_cvt_one(inpt_warped_im[0], fn, step, 'im_warped',
'valid')
#self.view_im_one(cur_anno[0]*127, fn, step, 'anno_cur', 'valid')
t_count += 1
#3. calculate loss
total_loss += loss
total_im_warp_loss += im_warp_loss
total_flow_loss += flow_loss
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
#line = 'none'
line = 'Train epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t m_imW_loss = %.4f\t m_f_loss = %.4f\t time = %.2f' % (
epoch, cfgs.inpt_lr, step, count, loss, (total_loss / t_count),
(total_im_warp_loss / t_count),
(total_flow_loss / t_count), time_per_batch)
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t m_imW_loss = %.4f\t m_f_loss = %.4f\t '
% (epoch, cfgs.inpt_lr, (total_loss / t_count),
(total_im_warp_loss / t_count), (total_flow_loss / t_count)))
print('Take time %3.1f' % (time.time() - t0))
return step
def u_net_op(self,
x,
keep_prob_,
channels,
n_class,
layers=3,
features_root=16,
filter_size=3,
pool_size=2,
summaries=True):
'''
Args:
x: input data
keep_prob: dropout probability
channels: number of channels of input image
n_class: number of output labels
layers: number of layers in the net
features_root: number of features in the first layer
pool_size: size of max pooling
summaries: Flag if summaries should be created
'''
#1. down layers
dw_h_convs = {}
for layer in range(0, layers):
out_ch = 2**layer * features_root
stddev = np.sqrt(2 / (filter_size**2 * out_ch))
if layer == 0:
in_ch = channels
else:
#// exact division
in_ch = out_ch // 2
name = 'down_conv_%s' % str(layer)
x = self.down_layer_unit(x,
filter_size,
in_ch,
out_ch,
stddev,
keep_prob_,
if_dropout=True,
name=str(layer))
dw_h_convs[name] = x
if layer < layers - 1:
x = utils.max_pool_valid(x,
kernel_size=pool_size,
stride=pool_size)
x = dw_h_convs[name]
#2. up layers
up_h_convs = {}
for layer in range(layers - 2, -1, -1):
features = 2**(layer + 1) * features_root
stddev = np.sqrt(2 / (filter_size**2 * features))
concat_x = dw_h_convs['down_conv_%s' % str(layer)]
x = self.up_layer_unit(x,
concat_x,
filter_size,
pool_size,
features,
stddev,
keep_prob_,
if_dropout=True,
name=str(layer))
name = 'up_conv_%s' % str(layer)
#if cfgs.if_pad[layer]:
#paddings = tf.constant([[0,0], [0,0], [1,0], [0,0]])
#x = tf.pad(x, paddings, 'CONSTANT')
up_h_convs[name] = x
#3. output map
with tf.name_scope('output_map'):
with tf.variable_scope('out_map', reuse=tf.AUTO_REUSE):
w_shape = [1, 1, features_root, n_class]
name = 'output'
output_map = conv2d_layer(x,
name,
w_shape,
pool_=0,
if_relu=True,
stride=1,
stddev=stddev,
if_dropout=True,
keep_prob_=keep_prob_)
up_h_convs['out'] = output_map
return output_map
def u_net_op(self, x, keep_prob_, channels, n_class, layers=3, class_convs_num=1, features_root=16, filter_size=3, pool_size=2, summaries=True):
'''
Args:
x: input data
keep_prob: dropout probability
channels: number of channels of input image
n_class: number of output labels
layers: number of layers in the u-net
class_convs_num: number of conv operator after u-net down layers operator.
features_root: number of features in the first layer
pool_size: size of max pooling
summaries: Flag if summaries should be created
'''
#1. down layers
dw_h_convs = {}
for layer in range(0, layers):
out_ch = 2** layer * features_root
stddev = np.sqrt(2 / (filter_size**2 * out_ch))
if layer == 0:
in_ch = channels
else:
#// exact division
in_ch = out_ch // 2
name = 'down_conv_%s' % str(layer)
x = self.down_layer_unit(x, filter_size, in_ch, out_ch, stddev, keep_prob_, if_dropout=True, name=str(layer))
dw_h_convs[name] = x
if layer < layers-1:
x = utils.max_pool_valid(x, kernel_size=pool_size, stride=pool_size)
x = dw_h_convs[name]
#2. label occlusion
print('--------label occlusion-------------')
x_class = x
ch_class = out_ch
for i in range(class_convs_num):
print('class conv %d' % i)
scope_name= 'class_conv_%s' % str(i)
with tf.name_scope(scope_name):
var_name = 'class_conv_var_%s' % str(i)
with tf.variable_scope(var_name):
w_class_s = [filter_size, filter_size, ch_class, ch_class]
x_class = conv2d_layer(x_class, str(i), w_class_s, pool_=2, if_relu=True, stride=2)
sz_class = tf.shape(x_class)
p_k_sz = cfgs.p_k_sz
x_class = utils.avg_pool_diff(x_class, p_k_sz[0], p_k_sz[1], stride=1)
with tf.name_scope('full_conn'):
with tf.variable_scope('full_conn_var'):
fc_w = utils.weight_variable([1, 1, ch_class, 2], name="fc_w")
fc_b = utils.bias_variable([2], name="fc_b")
x_class = utils.conv2d_basic(x_class, fc_w, fc_b)
class_logits = tf.squeeze(x_class, [1,2])
print('--------label occlusion end----------')
#3. up layers
up_h_convs = {}
for layer in range(layers-2, -1, -1):
features = 2 ** (layer + 1) * features_root
stddev = np.sqrt(2 / (filter_size ** 2 * features))
concat_x = dw_h_convs['down_conv_%s' % str(layer)]
x = self.up_layer_unit(x, concat_x, filter_size, pool_size, features, stddev, keep_prob_, if_dropout=True, name=str(layer))
name = 'up_conv_%s' % str(layer)
#if cfgs.if_pad[layer]:
#paddings = tf.constant([[0,0], [0,0], [1,0], [0,0]])
#x = tf.pad(x, paddings, 'CONSTANT')
up_h_convs[name] = x
#4. output map
with tf.name_scope('output_map'):
with tf.variable_scope('out_map'):
w_shape = [1, 1, features_root, n_class]
name = 'output'
output_map = conv2d_layer(x, name, w_shape, pool_=0, if_relu=True, stride=1, stddev=stddev, if_dropout=True, keep_prob_=keep_prob_)
up_h_convs['out'] = output_map
return output_map, class_logits
def valid_one_epoch_warped_im(self, sess, data_loader, data_num, epoch,
step):
'''
Using warped prev image to do new segmentation.
Args:
data_loader: training or validation data_loader.
data_num: number of data.
'''
total_loss, total_im_warp_loss, total_flow_loss = 0, 0, 0
count = 0
t0 = time.time()
im_warp_loss, flow_loss, loss = 0, 0, 0
#True number to loss
t_count = 0
if_epoch = cfgs.test_accu
for count in range(1, data_num):
step += 1
#1. Load data.
cur_im, prev_im, ellip_info, fn = data_loader.get_next_sequence_search_key(
)
feed_dict = {
self.prev_img: prev_im,
self.cur_img: cur_im,
}
#reconst_anno = sess.run(self.reconst_cur_anno, feed_dict=feed_dict)
#_ = sess.run(self.cur_static_anno_pred, feed_dict=feed_dict)
reconst_anno, warped_anno, warped_im, inpt_flow, flow = sess.run(
[
self.reconst_cur_anno, self.warped_static_anno_pred,
self.warped_prev_im, self.inpt_pred_flow_re,
self.inpt_pred_flow
],
feed_dict=feed_dict)
self.view_im_cvt_one(warped_im[0], fn, step, 'warped_im', 'valid')
self.view_im_one(reconst_anno[0, :, :, 1] * 127, fn, step,
'anno_reconst', 'valid')
self.view_im_one(warped_anno[0, :, :, 1] * 127, fn, step,
'anno_warp', 'valid')
self.view_flow_one(inpt_flow[0], fn, step, 'flow', 'valid')
self.view_flow_one(flow[0], fn, step, 'flow_no', 'valid')
self.calculate_acc(cur_im.copy(), [fn], reconst_anno[:, :, :, 0],
[ellip_info], True, if_epoch)
t_count += 1
#3. calculate loss
total_loss += loss
total_im_warp_loss += im_warp_loss
total_flow_loss += flow_loss
#4. time consume
time_consumed = time.time() - t0
time_per_batch = time_consumed / count
#5. print
#line = 'none'
line = 'Train epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t m_imW_loss = %.4f\t m_f_loss = %.4f\t time = %.3f' % (
epoch, cfgs.inpt_lr, step, count, loss, (total_loss / t_count),
(total_im_warp_loss / t_count),
(total_flow_loss / t_count), time_per_batch)
utils.clear_line(len(line))
print('\r' + line, end='')
#End one epoch
#count -= 1
print(
'\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t m_imW_loss = %.4f\t m_f_loss = %.4f\t '
% (epoch, cfgs.inpt_lr, (total_loss / t_count),
(total_im_warp_loss / t_count), (total_flow_loss / t_count)))
print('Take time %3.1f' % (time.time() - t0))
return step
