def __init__(self, camera_info, topic_to, weights_path):
# Create publisher
self._lanetracker_publisher = rospy.Publisher(topic_to,
Image,
queue_size=1)
self._bridge = CvBridge()
self._CFG = global_config.cfg
self._VGG_MEAN = [103.939, 116.779, 123.68]
self._input_tensor = tf.compat.v1.placeholder(
dtype=tf.float32,
shape=[
1, self._CFG.TRAIN.IMG_HEIGHT, self._CFG.TRAIN.IMG_WIDTH, 3
],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet()
self._binary_seg_ret, self._instance_seg_ret = net.test_inference(
self._input_tensor, phase_tensor, 'lanenet_loss')
initial_var = tf.compat.v1.global_variables()
final_var = initial_var[:-1]
saver = tf.compat.v1.train.Saver(final_var)
# saver = tf.compat.v1.train.Saver()
# Configure GPU usage
sess_config = tf.compat.v1.ConfigProto(device_count={'GPU': 1})
sess_config.gpu_options.per_process_gpu_memory_fraction = self._CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = self._CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
# Create session and load weights
self._sess = tf.compat.v1.Session(config=sess_config)
saver.restore(sess=self._sess, save_path=weights_path)
def init_lanenet(self):
'''
initlize the tensorflow model
'''
self.input_tensor = tf.placeholder(dtype=tf.float32, shape=[1, 256, 512, 3], name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
self.binary_seg_ret, self.instance_seg_ret = net.inference(input_tensor=self.input_tensor, name='lanenet_model')
self.cluster = lanenet_cluster.LaneNetCluster()
self.postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if self.use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
self.sess = tf.Session(config=sess_config)
saver.restore(sess=self.sess, save_path=self.weight_path)
def test_lanenet(image_path, weights_path, use_gpu, image_list, batch_size, save_dir):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
test_dataset = lanenet_data_processor_test.DataSet(image_path, batch_size)
input_tensor = tf.placeholder(dtype=tf.string, shape=[None], name='input_tensor')
imgs = tf.map_fn(test_dataset.process_img, input_tensor, dtype=tf.float32)
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet()
binary_seg_ret, instance_seg_ret = net.test_inference(imgs, phase_tensor, 'lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
sess.run(tf.global_variables_initializer())
import IPython
IPython.embed()
saver.restore(sess=sess, save_path=weights_path)
for i in range(math.ceil(len(image_list) / batch_size)):
print(i)
paths = test_dataset.next_batch()
instance_seg_image, existence_output = sess.run([binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: paths})
for cnt, image_name in enumerate(paths):
print(image_name)
parent_path = os.path.dirname(image_name)
directory = os.path.join(save_dir, 'vgg_SCNN_DULR_w9', parent_path)
if not os.path.exists(directory):
os.makedirs(directory)
file_exist = open(os.path.join(directory, os.path.basename(image_name)[:-3] + 'exist.txt'), 'w')
for cnt_img in range(4):
cv2.imwrite(os.path.join(directory, os.path.basename(image_name)[:-4] + '_' + str(cnt_img + 1) + '_avg.png'),
(instance_seg_image[cnt, :, :, cnt_img + 1] * 255).astype(int))
if existence_output[cnt, cnt_img] > 0.5:
file_exist.write('1 ')
else:
file_exist.write('0 ')
file_exist.close()
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu, image_list, batch_size):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 288, 800, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet()
binary_seg_ret, instance_seg_ret = net.test_inference(
input_tensor, phase_tensor, 'lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver.restore(sess=sess, save_path=weights_path)
img = np.zeros((1, 288, 800, 3))
instance_seg_image, existence_output = sess.run(
[binary_seg_ret, instance_seg_ret], feed_dict={input_tensor: img})
print(instance_seg_image, existence_output)
saver.save(sess, './models/tmp/tmp.ckpt')
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
log.info('开始读取图像数据并进行预处理')
t_start = time.time()
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256))
orig = image
image = image - VGG_MEAN
log.info('图像读取完毕, 耗时: {:.5f}s'.format(time.time() - t_start))
input_tensor = tf.placeholder(dtype=tf.float32, shape=[1, 256, 512, 3], name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor, name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run([binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
binary_seg_image[0] = postprocessor.postprocess(binary_seg_image[0])
mask_image = cluster.get_lane_mask(binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
# for i in range(4):
# instance_seg_image[0][:, :, i] = minmax_scale(instance_seg_image[0][:, :, i])
# embedding_image = np.array(instance_seg_image[0], np.uint8)
# plt.figure('mask_image')
# plt.imshow(mask_image[:, :, (2, 1, 0)])
splits = image_path.split("/")
fileName = splits[-1]
fileName = fileName.split(".")[0]
print fileName[0:-4]
path = ""
for i in range(len(image_path.split("/")) - 1):
path += str(splits[i]) + "/"
print path
np.save(os.path.join(path, fileName + "_output.npy"), mask_image[:, :, (2, 1, 0)])
# plt.figure('src_image')
# plt.imshow(image_vis[:, :, (2, 1, 0)])
# plt.figure('instance_image')
# plt.imshow(embedding_image[:, :, (2, 1, 0)])
# plt.figure('binary_image')
# plt.imshow(binary_seg_image[0] * 255, cmap='gray')
#plt.show()
orig = cv2.resize(image_vis[:, :, (2, 1, 0)], (512, 256))
binary = binary_seg_image[0] * 255
for x in range(orig.shape[0]):
for y in range(orig.shape[1]):
if binary[x][y] > 0:
orig[x][y] = (0,0,255)
# plt.figure('output')
# plt.imshow(orig)
scipy.misc.imsave(os.path.join(path, fileName + "_output.jpg"), orig)
# plt.show()
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
log.info('开始读取图像数据并进行预处理')
t_start = time.time()
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256), interpolation=cv2.INTER_LINEAR)
image = image - VGG_MEAN
log.info('图像读取完毕, 耗时: {:.5f}s'.format(time.time() - t_start))
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('train', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_loss')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
binary_seg_image[0] = postprocessor.postprocess(binary_seg_image[0])
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
# mask_image = cluster.get_lane_mask_v2(instance_seg_ret=instance_seg_image[0])
# mask_image = cv2.resize(mask_image, (image_vis.shape[1], image_vis.shape[0]),
# interpolation=cv2.INTER_LINEAR)
ele_mex = np.max(instance_seg_image[0], axis=(0, 1))
for i in range(3):
if ele_mex[i] == 0:
scale = 1
else:
scale = 255 / ele_mex[i]
instance_seg_image[0][:, :, i] *= int(scale)
embedding_image = np.array(instance_seg_image[0], np.uint8)
# cv2.imwrite('embedding_mask.png', embedding_image)
# mask_image = cluster.get_lane_mask_v2(instance_seg_ret=embedding_image)
# mask_image = cv2.resize(mask_image, (image_vis.shape[1], image_vis.shape[0]),
# interpolation=cv2.INTER_LINEAR)
cv2.imwrite('binary_ret.png', binary_seg_image[0] * 255)
cv2.imwrite('instance_ret.png', embedding_image)
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis[:, :, (2, 1, 0)])
plt.figure('instance_image')
plt.imshow(embedding_image[:, :, (2, 1, 0)])
plt.figure('binary_image')
plt.imshow(binary_seg_image[0] * 255, cmap='gray')
plt.show()
sess.close()
return
def train_net(
dataset_dir,
weights_path=None,
net_flag='vgg',
save_dir="./logs/train/lanenet",
tboard_save_path="./tboard/lanenet",
ignore_labels_path="/media/remus/datasets/AVMSnapshots/AVM/ignore_labels.png",
my_checkpoint="true"):
"""
:param save_dir:
:param ignore_labels_path:
:param tboard_save_path:
:param dataset_dir:
:param net_flag: choose which base network to use
:param weights_path:
:return:
"""
train_dataset_file = ops.join(dataset_dir, 'train.txt')
val_dataset_file = ops.join(dataset_dir, 'val.txt')
assert ops.exists(train_dataset_file)
# tf.enable_eager_execution()
train_dataset = lanenet_data_processor.DataSet(train_dataset_file)
val_dataset = lanenet_data_processor.DataSet(val_dataset_file)
with tf.device('/gpu:1'):
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[
CFG.TRAIN.BATCH_SIZE,
CFG.TRAIN.IMG_HEIGHT,
CFG.TRAIN.IMG_WIDTH, 3
],
name='input_tensor')
binary_label_tensor = tf.placeholder(dtype=tf.int64,
shape=[
CFG.TRAIN.BATCH_SIZE,
CFG.TRAIN.IMG_HEIGHT,
CFG.TRAIN.IMG_WIDTH, 1
],
name='binary_input_label')
instance_label_tensor = tf.placeholder(dtype=tf.float32,
shape=[
CFG.TRAIN.BATCH_SIZE,
CFG.TRAIN.IMG_HEIGHT,
CFG.TRAIN.IMG_WIDTH
],
name='instance_input_label')
phase = tf.placeholder(dtype=tf.string, shape=None, name='net_phase')
net = lanenet_merge_model.LaneNet(net_flag=net_flag, phase=phase)
# calculate the loss
compute_ret = net.compute_loss(input_tensor=input_tensor,
binary_label=binary_label_tensor,
instance_label=instance_label_tensor,
ignore_label=255,
name='lanenet_model')
total_loss = compute_ret['total_loss']
binary_seg_loss = compute_ret['binary_seg_loss']
disc_loss = compute_ret['discriminative_loss']
pix_embedding = compute_ret['instance_seg_logits']
# calculate the accuracy
out_logits = compute_ret['binary_seg_logits']
out_logits = tf.nn.softmax(logits=out_logits)
out_logits_out = tf.argmax(out_logits, axis=-1)
out = tf.argmax(out_logits, axis=-1)
out = tf.expand_dims(out, axis=-1)
idx = tf.where(tf.equal(binary_label_tensor, 1))
pix_cls_ret = tf.gather_nd(out, idx)
accuracy = tf.count_nonzero(pix_cls_ret)
accuracy = tf.divide(accuracy,
tf.cast(tf.shape(pix_cls_ret)[0], tf.int64))
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(CFG.TRAIN.LEARNING_RATE,
global_step,
CFG.TRAIN.LR_DECAY_STEPS,
CFG.TRAIN.LR_DECAY_RATE,
staircase=True)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.MomentumOptimizer(
learning_rate=learning_rate,
momentum=0.9).minimize(loss=total_loss,
var_list=tf.trainable_variables(),
global_step=global_step)
# update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# Set tf saver
if my_checkpoint == "true":
init_saver = tf.train.Saver()
else:
from correct_path_saver import restore_from_classification_checkpoint_fn, get_variables_available_in_checkpoint
if weights_path is not None:
# var_map = restore_from_classification_checkpoint_fn("lanenet_model/inference")
available_var_map = (get_variables_available_in_checkpoint(
tf.global_variables(), weights_path,
include_global_step=False))
init_saver = tf.train.Saver(available_var_map)
else:
init_saver = tf.train.Saver()
if not ops.exists(save_dir):
os.makedirs(save_dir)
train_start_time = time.strftime('%Y-%m-%d-%H-%M-%S',
time.localtime(time.time()))
model_name = '{:s}_lanenet_{:s}.ckpt'.format(net_flag,
str(train_start_time))
model_save_path = ops.join(save_dir, model_name)
# Set tf summary
if not ops.exists(tboard_save_path):
os.makedirs(tboard_save_path)
train_cost_scalar = tf.summary.scalar(name='train_cost', tensor=total_loss)
val_cost_scalar = tf.summary.scalar(name='val_cost', tensor=total_loss)
train_accuracy_scalar = tf.summary.scalar(name='train_accuracy',
tensor=accuracy)
val_accuracy_scalar = tf.summary.scalar(name='val_accuracy',
tensor=accuracy)
train_binary_seg_loss_scalar = tf.summary.scalar(
name='train_binary_seg_loss', tensor=binary_seg_loss)
val_binary_seg_loss_scalar = tf.summary.scalar(name='val_binary_seg_loss',
tensor=binary_seg_loss)
train_instance_seg_loss_scalar = tf.summary.scalar(
name='train_instance_seg_loss', tensor=disc_loss)
val_instance_seg_loss_scalar = tf.summary.scalar(
name='val_instance_seg_loss', tensor=disc_loss)
learning_rate_scalar = tf.summary.scalar(name='learning_rate',
tensor=learning_rate)
train_merge_summary_op = tf.summary.merge([
train_accuracy_scalar, train_cost_scalar, learning_rate_scalar,
train_binary_seg_loss_scalar, train_instance_seg_loss_scalar
])
val_merge_summary_op = tf.summary.merge([
val_accuracy_scalar, val_cost_scalar, val_binary_seg_loss_scalar,
val_instance_seg_loss_scalar
])
# Set sess configuration
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
# sess_config.device_count = {'GPU': 0}
sess = tf.Session(config=sess_config)
# sess = tf_debug.TensorBoardDebugWrapperSession(sess=sess,
# grpc_debug_server_addresses="remusm-pc:7000",
# send_traceback_and_source_code=False)
summary_writer = tf.summary.FileWriter(tboard_save_path)
summary_writer.add_graph(sess.graph)
# Set the training parameters
train_epochs = CFG.TRAIN.EPOCHS
tf.logging.info('Global configuration is as follows:')
tf.logging.info(CFG)
iter_saver = tf.train.Saver(max_to_keep=10)
best_saver = tf.train.Saver(max_to_keep=3)
with sess.as_default():
sess.run(tf.global_variables_initializer())
tf.train.write_graph(graph_or_graph_def=sess.graph,
logdir='',
name='{:s}/lanenet_model.pb'.format(save_dir))
if weights_path is None:
tf.logging.info('Training from scratch')
init = tf.global_variables_initializer()
sess.run(init)
else:
tf.logging.info(
'Restore model from last model checkpoint {:s}'.format(
weights_path))
init_saver.restore(sess=sess, save_path=weights_path)
assign_op = global_step.assign(0)
sess.run(assign_op)
# 加载预训练参数
if net_flag == 'vgg' and weights_path is None:
pretrained_weights = np.load('./data/vgg16.npy',
encoding='latin1').item()
for vv in tf.trainable_variables():
weights_key = vv.name.split('/')[-3]
try:
weights = pretrained_weights[weights_key][0]
_op = tf.assign(vv, weights)
sess.run(_op)
except Exception as e:
continue
train_cost_time_mean = []
val_cost_time_mean = []
ignore_label_mask = cv2.imread(ignore_labels_path)
last_c = 100000
for epoch in range(train_epochs):
# training part
t_start = time.time()
with tf.device('/cpu:0'):
gt_imgs, binary_gt_labels, instance_gt_labels = train_dataset.next_batch(
CFG.TRAIN.BATCH_SIZE,
ignore_label_mask=ignore_label_mask,
ignore_label=255)
# gt_imgs = [tmp - VGG_MEAN for tmp in gt_imgs]
gt_imgs = [tmp / 128.0 - 1.0 for tmp in gt_imgs]
binary_gt_labels = [
np.expand_dims(tmp, axis=-1) for tmp in binary_gt_labels
]
phase_train = 'train'
_, c, train_accuracy, train_summary, binary_loss, instance_loss, embedding, binary_seg_img, g_step = \
sess.run([optimizer, total_loss,
accuracy,
train_merge_summary_op,
binary_seg_loss,
disc_loss,
pix_embedding,
out_logits_out,
global_step],
feed_dict={input_tensor: gt_imgs,
binary_label_tensor: binary_gt_labels,
instance_label_tensor: instance_gt_labels,
phase: phase_train})
# if epoch % 10 == 0:
# tf.logging.info("Epoch {}."
# "Total loss: {}. Train acc: {}."
# " Binary loss: {}. Instance loss: {}".format(epoch, c, train_accuracy,
# binary_loss, instance_loss))
if math.isnan(c) or math.isnan(binary_loss) or math.isnan(
instance_loss):
tf.logging.error('cost is: {:.5f}'.format(c))
tf.logging.error('binary cost is: {:.5f}'.format(binary_loss))
tf.logging.error(
'instance cost is: {:.5f}'.format(instance_loss))
# cv2.imwrite('nan_image.png', gt_imgs[0] + VGG_MEAN)
cv2.imwrite('nan_image.png', (gt_imgs[0] + 1.0) * 128)
cv2.imwrite('nan_instance_label.png', instance_gt_labels[0])
cv2.imwrite('nan_binary_label.png', binary_gt_labels[0] * 255)
return
if epoch % 100 == 0:
# cv2.imwrite('nan_image.png', gt_imgs[0] + VGG_MEAN)
cv2.imwrite('image.png', (gt_imgs[0] + 1.0) * 128)
cv2.imwrite('binary_label.png', binary_gt_labels[0] * 255)
cv2.imwrite('instance_label.png', instance_gt_labels[0])
cv2.imwrite('binary_seg_img.png', binary_seg_img[0] * 255)
for i in range(4):
embedding[0][:, :, i] = minmax_scale(embedding[0][:, :, i])
embedding_image = np.array(embedding[0], np.uint8)
cv2.imwrite('embedding.png', embedding_image[:, :, :-1])
cost_time = time.time() - t_start
train_cost_time_mean.append(cost_time)
summary_writer.add_summary(summary=train_summary,
global_step=epoch)
# validation part
with tf.device('/cpu:0'):
gt_imgs_val, binary_gt_labels_val, instance_gt_labels_val \
= val_dataset.next_batch(CFG.TRAIN.VAL_BATCH_SIZE, ignore_label_mask=ignore_label_mask)
# gt_imgs_val = [tmp - VGG_MEAN for tmp in gt_imgs_val]
gt_imgs_val = [tmp / 128.0 - 1.0 for tmp in gt_imgs_val]
binary_gt_labels_val = [
np.expand_dims(tmp, axis=-1)
for tmp in binary_gt_labels_val
]
phase_val = 'test'
t_start_val = time.time()
c_val, val_summary, val_accuracy, val_binary_seg_loss, val_instance_seg_loss = \
sess.run([total_loss, val_merge_summary_op, accuracy, binary_seg_loss, disc_loss],
feed_dict={input_tensor: gt_imgs_val,
binary_label_tensor: binary_gt_labels_val,
instance_label_tensor: instance_gt_labels_val,
phase: phase_val})
if epoch % 100 == 0:
# cv2.imwrite('test_image.png', gt_imgs_val[0] + VGG_MEAN)
cv2.imwrite('test_image.png', (gt_imgs_val[0] + 1.0) * 128)
summary_writer.add_summary(val_summary, global_step=epoch)
cost_time_val = time.time() - t_start_val
val_cost_time_mean.append(cost_time_val)
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
tf.logging.info(
'Step: {:d} total_loss= {:6f} binary_seg_loss= {:6f} instance_seg_loss= {:6f} accuracy= {:6f}'
' mean_cost_time= {:5f}s '.format(
epoch + 1, c, binary_loss, instance_loss,
train_accuracy, np.mean(train_cost_time_mean)))
train_cost_time_mean.clear()
if epoch % CFG.TRAIN.TEST_DISPLAY_STEP == 0:
tf.logging.info(
'Step_Val: {:d} total_loss= {:6f} binary_seg_loss= {:6f} '
'instance_seg_loss= {:6f} accuracy= {:6f} '
'mean_cost_time= {:5f}s '.format(
epoch + 1, c_val, val_binary_seg_loss,
val_instance_seg_loss, val_accuracy,
np.mean(val_cost_time_mean)))
val_cost_time_mean.clear()
if epoch % 2000 == 0:
iter_saver.save(sess=sess,
save_path=model_save_path,
global_step=epoch)
if c < last_c:
last_c = c
save_dir_best = save_dir + "/best"
if not ops.exists(save_dir_best):
os.makedirs(save_dir_best)
best_model_save_path = ops.join(save_dir_best, model_name)
best_saver.save(sess=sess,
save_path=best_model_save_path,
global_step=epoch)
sess.close()
return
def test_lanenet(self, image_path, weights_path, use_gpu):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
log.info('Start reading image data and pre-processing')
t_start = time.time()
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256), interpolation=cv2.INTER_LINEAR)
image = image - VGG_MEAN
log.info('Image is read, time taken {:.5f}s'.format(time.time() -
t_start))
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(
input_tensor=input_tensor, name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
log.info('GPU detected, processing on GPU now..')
else:
sess_config = tf.ConfigProto(device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=self.weights_path)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info(
'Single image lane line prediction time consuming: {:.5f}s'.
format(t_cost))
binary_seg_image[0] = postprocessor.postprocess(
binary_seg_image[0])
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
for i in range(4):
instance_seg_image[0][:, :, i] = self.minmax_scale(
instance_seg_image[0][:, :, i])
embedding_image = np.array(instance_seg_image[0], np.uint8)
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis[:, :, (2, 1, 0)])
plt.figure('instance_image')
plt.imshow(embedding_image[:, :, (2, 1, 0)])
plt.figure('binary_image')
plt.imshow(binary_seg_image[0] * 255, cmap='gray')
plt.show()
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu, image_list, batch_size):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
print('image_path:', image_path)
#frame = cv2.imread('obj.jpg')
cap = cv2.VideoCapture('harder_challenge_video.mp4')
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 288, 800, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet()
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
phase_tensor=phase_tensor,
name='lanenet_loss')
#binary_seg_ret, instance_seg_ret = net.test_inference(input_tensor=input_tensor, phase_tensor, 'lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver.restore(sess=sess, save_path=weights_path)
while (True):
ret, frame = cap.read()
image = process(frame)
instance_seg_image, existence_output = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: image})
for cnt_img in range(4):
print(existence_output[0, cnt_img])
if existence_output[0, cnt_img] > 0.5:
obj_mask = (instance_seg_image[0, :, :, cnt_img + 1] *
255).astype(int)
h_scale = frame.shape[0] / obj_mask.shape[0]
w_scale = frame.shape[1] / obj_mask.shape[1]
coordinate_tmp = np.zeros((1, 30))
img = (instance_seg_image[cnt, :, :, cnt_img + 1] *
255).astype(int)
#print(w, h)
for i in range(30):
lineId = math.ceil(288 - i * 20 / w * 288) - 1
img_line = img[lineId]
value = np.max(img_line)
id = np.where(img_line == value)
if (value / 255 > 0.3):
coordinate_tmp[0][i] = id[0][0]
if np.sum(coordinate_tmp > 0) < 2:
coordinate_tmp = np.zeros((1, 30))
for i in range(30):
if coordinate_tmp[0][i] > 0:
cv2.circle(src_image,
(int(coordinate_tmp[0][i] * h / 800),
int(w - i * 20)), 6, (0, 0, 255), -1)
#cv2.imwrite('obj.jpg', frame)
cv2.imshow('obj', frame)
cv2.waitKey(1)
sess.close()
return
def train_net(dataset_dir, weights_path=None, net_flag='vgg'):
train_dataset_file = ops.join(dataset_dir, 'train_gt.txt')
val_dataset_file = ops.join(dataset_dir, 'val_gt.txt')
assert ops.exists(train_dataset_file)
phase = tf.placeholder(dtype=tf.string, shape=None, name='net_phase')
train_dataset = lanenet_data_processor.DataSet(train_dataset_file)
val_dataset = lanenet_data_processor.DataSet(val_dataset_file)
net = lanenet_merge_model.LaneNet()
tower_grads = []
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.polynomial_decay(CFG.TRAIN.LEARNING_RATE,
global_step,
CFG.TRAIN.EPOCHS,
power=0.9)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=0.9)
img, label_instance, label_existence = train_dataset.next_batch(
CFG.TRAIN.BATCH_SIZE)
batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue(
[img, label_instance, label_existence],
capacity=2 * CFG.TRAIN.GPU_NUM,
num_threads=CFG.TRAIN.CPU_NUM)
val_img, val_label_instance, val_label_existence = val_dataset.next_batch(
CFG.TRAIN.BATCH_SIZE)
val_batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue(
[val_img, val_label_instance, val_label_existence],
capacity=2 * CFG.TRAIN.GPU_NUM,
num_threads=CFG.TRAIN.CPU_NUM)
with tf.variable_scope(tf.get_variable_scope()):
for i in range(CFG.TRAIN.GPU_NUM):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i):
total_loss, instance_loss, existence_loss, accuracy, accuracy_back, _, out_logits_out, \
grad = forward(batch_queue, net, phase, optimizer)
tower_grads.append(grad)
val_op_total_loss, val_op_instance_loss, val_op_existence_loss, val_op_accuracy, \
val_op_accuracy_back, val_op_IoU, _, _ = forward(val_batch_queue, net, phase)
grads = average_gradients(tower_grads)
train_op = optimizer.apply_gradients(grads, global_step=global_step)
train_cost_time_mean = []
train_instance_loss_mean = []
train_existence_loss_mean = []
train_accuracy_mean = []
train_accuracy_back_mean = []
saver = tf.train.Saver()
model_save_dir = 'model/culane_lanenet/culane_scnn'
if not ops.exists(model_save_dir):
os.makedirs(model_save_dir)
train_start_time = time.strftime('%Y-%m-%d-%H-%M-%S',
time.localtime(time.time()))
model_name = 'culane_lanenet_{:s}_{:s}.ckpt'.format(
net_flag, str(train_start_time))
model_save_path = ops.join(model_save_dir, model_name)
sess_config = tf.ConfigProto(device_count={'GPU': CFG.TRAIN.GPU_NUM},
allow_soft_placement=True)
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
with tf.Session(config=sess_config) as sess:
with sess.as_default():
if weights_path is None:
log.info('Training from scratch')
init = tf.global_variables_initializer()
sess.run(init)
else:
log.info(
'Restore model from last model checkpoint {:s}'.format(
weights_path))
saver.restore(sess=sess, save_path=weights_path)
# 加载预训练参数
if net_flag == 'vgg' and weights_path is None:
pretrained_weights = np.load('./data/vgg16.npy',
encoding='latin1').item()
for vv in tf.trainable_variables():
weights = vv.name.split('/')
if len(weights) >= 3 and weights[-3] in pretrained_weights:
try:
weights_key = weights[-3]
weights = pretrained_weights[weights_key][0]
_op = tf.assign(vv, weights)
sess.run(_op)
except Exception as e:
continue
tf.train.start_queue_runners(sess=sess)
for epoch in range(CFG.TRAIN.EPOCHS):
t_start = time.time()
_, c, train_accuracy, train_accuracy_back, train_instance_loss, train_existence_loss, binary_seg_img = \
sess.run([train_op, total_loss, accuracy, accuracy_back, instance_loss, existence_loss, out_logits_out],
feed_dict={phase: 'train'})
cost_time = time.time() - t_start
train_cost_time_mean.append(cost_time)
train_instance_loss_mean.append(train_instance_loss)
train_existence_loss_mean.append(train_existence_loss)
train_accuracy_mean.append(train_accuracy)
train_accuracy_back_mean.append(train_accuracy_back)
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
print(
'Epoch: {:d} loss_ins= {:6f} ({:6f}) loss_ext= {:6f} ({:6f}) accuracy= {:6f} ({:6f}) '
'accuracy_back= {:6f} ({:6f}) mean_time= {:5f}s '.format(
epoch + 1, train_instance_loss,
np.mean(train_instance_loss_mean),
train_existence_loss,
np.mean(train_existence_loss_mean), train_accuracy,
np.mean(train_accuracy_mean), train_accuracy_back,
np.mean(train_accuracy_back_mean),
np.mean(train_cost_time_mean)))
if epoch % 500 == 0:
train_cost_time_mean.clear()
train_instance_loss_mean.clear()
train_existence_loss_mean.clear()
train_accuracy_mean.clear()
train_accuracy_back_mean.clear()
if epoch % 1000 == 0:
saver.save(sess=sess,
save_path=model_save_path,
global_step=epoch)
if epoch % 10000 != 0 or epoch == 0:
continue
val_cost_time_mean = []
val_instance_loss_mean = []
val_existence_loss_mean = []
val_accuracy_mean = []
val_accuracy_back_mean = []
val_IoU_mean = []
for epoch_val in range(
int(
len(val_dataset) / CFG.TRAIN.VAL_BATCH_SIZE /
CFG.TRAIN.GPU_NUM)):
t_start_val = time.time()
c_val, val_accuracy, val_accuracy_back, val_IoU, val_instance_loss, val_existence_loss = \
sess.run(
[val_op_total_loss, val_op_accuracy, val_op_accuracy_back,
val_op_IoU, val_op_instance_loss, val_op_existence_loss],
feed_dict={phase: 'test'})
cost_time_val = time.time() - t_start_val
val_cost_time_mean.append(cost_time_val)
val_instance_loss_mean.append(val_instance_loss)
val_existence_loss_mean.append(val_existence_loss)
val_accuracy_mean.append(val_accuracy)
val_accuracy_back_mean.append(val_accuracy_back)
val_IoU_mean.append(val_IoU)
if epoch_val % 1 == 0:
print(
'Epoch_Val: {:d} loss_ins= {:6f} ({:6f}) '
'loss_ext= {:6f} ({:6f}) accuracy= {:6f} ({:6f}) accuracy_back= {:6f} ({:6f}) '
'mIoU= {:6f} ({:6f}) mean_time= {:5f}s '.format(
epoch_val + 1, val_instance_loss,
np.mean(val_instance_loss_mean),
val_existence_loss,
np.mean(val_existence_loss_mean), val_accuracy,
np.mean(val_accuracy_mean), val_accuracy_back,
np.mean(val_accuracy_back_mean), val_IoU,
np.mean(val_IoU_mean),
np.mean(val_cost_time_mean)))
val_cost_time_mean.clear()
val_instance_loss_mean.clear()
val_existence_loss_mean.clear()
val_accuracy_mean.clear()
val_accuracy_back_mean.clear()
val_IoU_mean.clear()
def test_lanenet(image_path, weights_path, use_gpu, save_dir):
"""
:param save_dir:
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
log.info('开始读取图像数据并进行预处理')
t_start = time.time()
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256), interpolation=cv2.INTER_LINEAR)
image = image / 128.0 - 1.0
log.info('图像读取完毕, 耗时: {:.5f}s'.format(time.time() - t_start))
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='mobilenet')
binary_seg_ret, instance_seg_ret, _ = net.inference(
input_tensor=input_tensor, name='lanenet_model')
binary_seg_ret_32 = tf.cast(binary_seg_ret, tf.int32, name="binary_seg")
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
# Set tf saver
# if weights_path is not None:
# var_map = restore_from_classification_checkpoint_fn("lanenet_model/inference")
# available_var_map = (get_variables_available_in_checkpoint(
# var_map, weights_path, include_global_step=False))
#
# saver = tf.train.Saver(available_var_map)
saver = tf.train.Saver()
iter_saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 1})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret_32, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
binary_seg_image[0] = postprocessor.postprocess(binary_seg_image[0])
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
for i in range(4):
instance_seg_image[0][:, :,
i] = minmax_scale(instance_seg_image[0][:, :,
i])
embedding_image = np.array(instance_seg_image[0], np.uint8)
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis[:, :, (2, 1, 0)])
plt.figure('instance_image')
plt.imshow(embedding_image[:, :, (3, 1, 0)])
plt.figure('binary_image')
plt.imshow(binary_seg_image[0] * 255, cmap='gray')
plt.show()
mask_image = mask_image[:, :, (2, 1, 0)]
image_name = ops.split(image_path)[1]
image_save_path = ops.join(save_dir, image_name)
cv2.imwrite(image_save_path, mask_image)
iter_saver.save(sess=sess,
save_path=save_dir + "inference_models/model20.ckpt")
tf.train.write_graph(sess.graph.as_graph_def(),
save_dir + "inference_models/", "graph20.pb")
# tf.train.write_graph(graph_or_graph_def=sess.graph, logdir='', name='{:s}/lanenet_model.pb'.format(save_dir))
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu):
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
# 将原图保存为image_vis，并resize成分辨率512x256
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256), interpolation=cv2.INTER_LINEAR)
image = image - VGG_MEAN
# Tensorflow的创建Graph过程
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
# 实例化LaneNet网络
net = lanenet_merge_model.LaneNet(phase=phase_tensor)
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_model')
# 实例化聚类对象以及后处理对象
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# 设置会话Session的全局配置
if use_gpu:
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
device_count={'GPU': 0})
else:
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TEST.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
# Tensorflow的打开Session过程
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
t_start = time.time()
# 对原图进行二值分割以及实例分割
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info('Predict a single image: cost_time {:.5f}s'.format(t_cost))
# 对掩模结果进行聚类以及后处理
t_start = time.time()
binary_seg_image[0] = postprocessor.postprocess(binary_seg_image[0])
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
t_cluster = time.time() - t_start
log.info('Cluster a single image: cost_time {:.5f}s'.format(t_cluster))
# 显示原图src_image和预测掩模结果图make_image
plt.figure('src_image')
plt.imshow(image_vis[:, :, (2, 1, 0)])
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.show()
# 关闭会话Session
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu, image_list, batch_size, save_dir):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
print("saving to "+save_dir)
test_dataset = lanenet_data_processor_test.DataSet(image_path, batch_size)
input_tensor = tf.placeholder(dtype=tf.string, shape=[None], name='input_tensor')
imgs = tf.map_fn(test_dataset.process_img, input_tensor, dtype=tf.float32)
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet()
binary_seg_ret, instance_seg_ret = net.test_inference(imgs, phase_tensor, 'lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TEST.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
out = cv2.VideoWriter(os.path.join(save_dir,"result.avi"),fourcc,30.0, (1920,1080))
print( "OPEN VIDEO FILE", out.isOpened())
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver.restore(sess=sess, save_path=weights_path)
for i in range(math.ceil(len(image_list) / batch_size)):
print(i)
paths = test_dataset.next_batch()
instance_seg_image, existence_output = sess.run([binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: paths})
for cnt, image_name in enumerate(paths):
print(image_name)
parent_path = os.path.dirname(image_name)
directory = os.path.join(save_dir, 'vgg_SCNN_DULR_w9', parent_path)
if not os.path.exists(directory):
os.makedirs(directory)
file_exist = open(os.path.join(directory, os.path.basename(image_name)[:-3] + 'exist.txt'), 'w')
ori_img = cv2.imread(os.path.join(directory, os.path.basename(image_name)),1)
for cnt_img in range(4):
cv2.imwrite(os.path.join(directory, os.path.basename(image_name)[:-4] + '_' + str(cnt_img + 1) + '_avg.png'),
(instance_seg_image[cnt, :, :, cnt_img + 1] * 255).astype('uint8'))
ori_size_lane = (cv2.resize(instance_seg_image[cnt, :, :, cnt_img + 1], ( ori_img.shape[1], ori_img.shape[0])) * 255).astype('uint8')
mean = np.mean(ori_size_lane)
std = np.std(ori_size_lane)
ori_size_lane = np.where(ori_size_lane>(mean - std), ori_size_lane,0)
if existence_output[cnt, cnt_img] > 0.5:
file_exist.write('1 ')
for channel,color in enumerate(LANE_COLOR[cnt_img]):
if color is 1:
#ori_img[:,:,channel] = np.where(ori_size_lane == 1, ori_img[:,:,channel] ,ori_size_lane )
ori_img[:,:,channel] += (ori_size_lane*255).astype('uint8')
np.where(ori_img[:,:,channel] > 255, ori_img[:,:,channel] ,255)
else:
file_exist.write('0 ')
file_exist.close()
out.write(ori_img)
cv2.imwrite(os.path.join(directory, os.path.basename(image_name)[:-4] + '_lane.png'),ori_img)
sess.close()
out.release()
return
def test_lanenet_batch(image_dir,
weights_path,
batch_size,
use_gpu,
save_dir=None):
"""
:param image_dir:
:param weights_path:
:param batch_size:
:param use_gpu:
:param save_dir:
:return:
"""
tf.reset_default_graph() # So we can run this multiple times
print("save_dir in function: ", save_dir)
assert ops.exists(image_dir), '{:s} not exist'.format(image_dir)
log.info('开始获取图像文件路径...')
image_path_list = glob.glob('{:s}/**/*.jpg'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.png'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.jpeg'.format(image_dir), recursive=True)
print("NUMBER OF IMAGES IN IMAGE_PATH: ", len(image_path_list))
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[None, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
print("SUCCESSFULLY RESTORED FROM SESSION")
epoch_nums = int(math.ceil(len(image_path_list) / batch_size))
print("EPOCH_NUMS = ", epoch_nums)
for epoch in range(epoch_nums):
log.info('[Epoch:{:d}] 开始图像读取和预处理...'.format(epoch))
t_start = time.time()
image_path_epoch = image_path_list[epoch * batch_size:(epoch + 1) *
batch_size]
image_list_epoch = [
cv2.imread(tmp, cv2.IMREAD_COLOR) for tmp in image_path_epoch
]
image_vis_list = image_list_epoch
image_list_epoch = [
cv2.resize(tmp, (512, 256), interpolation=cv2.INTER_LINEAR)
for tmp in image_list_epoch
]
image_list_epoch = [tmp - VGG_MEAN for tmp in image_list_epoch]
t_cost = time.time() - t_start
log.info(
'[Epoch:{:d}] 预处理{:d}张图像, 共耗时: {:.5f}s, 平均每张耗时: {:.5f}'.format(
epoch, len(image_path_epoch), t_cost,
t_cost / len(image_path_epoch)))
t_start = time.time()
binary_seg_images, instance_seg_images = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: image_list_epoch})
t_cost = time.time() - t_start
log.info(
'[Epoch:{:d}] 预测{:d}张图像车道线, 共耗时: {:.5f}s, 平均每张耗时: {:.5f}s'.
format(epoch, len(image_path_epoch), t_cost,
t_cost / len(image_path_epoch)))
cluster_time = []
for index, binary_seg_image in enumerate(binary_seg_images):
t_start = time.time()
binary_seg_image = postprocessor.postprocess(binary_seg_image)
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image,
instance_seg_ret=instance_seg_images[index])
cluster_time.append(time.time() - t_start)
mask_image = cv2.resize(mask_image,
(image_vis_list[index].shape[1],
image_vis_list[index].shape[0]),
interpolation=cv2.INTER_LINEAR)
if save_dir is None:
plt.ion()
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis_list[index][:, :, (2, 1, 0)])
plt.pause(3.0)
plt.show()
plt.ioff()
if save_dir is not None:
# Mask is the predicted lane line pixels with colors on a black background
if DO_POST_PROCESS == True:
mask_image = post_process_output(mask_image)
# After this line it will layer the lines on the actual image
# Comment it out to save just the predictions
mask_image = cv2.addWeighted(image_vis_list[index], 1.0,
mask_image, 1.0, 0)
image_name = ops.split(image_path_epoch[index])[1]
image_save_path = ops.join(save_dir, image_name)
cv2.imwrite(image_save_path, mask_image)
print("SAVED MASK IMAGE TO ", image_save_path)
log.info(
'[Epoch:{:d}] 进行{:d}张图像车道线聚类, 共耗时: {:.5f}s, 平均每张耗时: {:.5f}'.
format(epoch, len(image_path_epoch), np.sum(cluster_time),
np.mean(cluster_time)))
sess.close()
return
def test_lanenet_batch(image_dir, weights_path, batch_size, use_gpu, save_dir):
assert ops.exists(image_dir), '{:s} not exist'.format(image_dir)
# 读取image_dir目录下的所有图片
log.info('Reading images...')
image_path_list = glob.glob('{:s}/**/*.jpg'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.png'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.jpeg'.format(image_dir), recursive=True)
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[None, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor)
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
if use_gpu:
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
device_count={'GPU': 0})
else:
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TEST.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
epoch_nums = int(math.ceil(len(image_path_list) / batch_size))
for epoch in range(epoch_nums):
image_path_epoch = image_path_list[epoch * batch_size:(epoch + 1) *
batch_size]
image_list_epoch = [
cv2.imread(tmp, cv2.IMREAD_COLOR) for tmp in image_path_epoch
]
image_vis_list = image_list_epoch
image_list_epoch = [
cv2.resize(tmp, (512, 256), interpolation=cv2.INTER_LINEAR)
for tmp in image_list_epoch
]
image_list_epoch = [tmp - VGG_MEAN for tmp in image_list_epoch]
t_start = time.time()
binary_seg_images, instance_seg_images = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: image_list_epoch})
t_cost = time.time() - t_start
log.info(
'[Epoch:{:d}] Predict {:d} images: total_cost_time {:.5f}s mean_cost_time {:.5f}s'
.format(epoch + 1, len(image_path_epoch), t_cost,
t_cost / len(image_path_epoch)))
cluster_time = []
for index, binary_seg_image in enumerate(binary_seg_images):
t_start = time.time()
binary_seg_image = postprocessor.postprocess(binary_seg_image)
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image,
instance_seg_ret=instance_seg_images[index])
cluster_time.append(time.time() - t_start)
mask_image = cv2.resize(mask_image,
(image_vis_list[index].shape[1],
image_vis_list[index].shape[0]),
interpolation=cv2.INTER_LINEAR)
# 批量保存预测结果图
mask_image = cv2.addWeighted(image_vis_list[index], 1.0,
mask_image, 1.0, 0)
image_name = ops.split(image_path_epoch[index])[1]
image_save_path = ops.join(save_dir, image_name)
cv2.imwrite(image_save_path, mask_image)
log.info(
'[Epoch:{:d}] Cluster {:d} images: total_cost_time {:.5f}s mean_cost_time {:.5f}'
.format(epoch + 1, len(image_path_epoch), np.sum(cluster_time),
np.mean(cluster_time)))
sess.close()
return
def predict_lanenet(gt_image, lanenet_weights):
"""
:param gt_image:
:param lanenet_weights:
:return:
"""
lanenet_image = gt_image - VGG_MEAN
# Step1, predict from lanenet
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant(False, tf.bool)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='enet')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
saver = tf.train.Saver()
# Set sess configuration
sess_config = tf.ConfigProto(device_count={'GPU': 1})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
with tf.Session(config=sess_config) as sess:
saver.restore(sess=sess, save_path=lanenet_weights)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [lanenet_image]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
t_start = time.time()
mask = np.random.randn(binary_seg_image[0].shape[0],
binary_seg_image[0].shape[1]) > 0.5
bi = binary_seg_image[0] * mask
mask_image, lane_coordinate, cluster_index, labels = cluster.get_lane_mask(
binary_seg_ret=bi,
instance_seg_ret=instance_seg_image[0],
gt_image=gt_image)
t_cost = time.time() - t_start
log.info('单张图像车道线聚类耗时: {:.5f}s'.format(t_cost))
print(instance_seg_image.shape)
for i in range(4):
instance_seg_image[0][:, :,
i] = minmax_scale(instance_seg_image[0][:, :,
i])
embedding_image = np.array(instance_seg_image[0], np.uint8)
cv2.imwrite('./out/predict_binary.png', binary_seg_image[0] * 255)
cv2.imwrite('./out/predict_lanenet.png', mask_image)
cv2.imwrite('./out/predict_instance.png', embedding_image)
sess.close()
return lane_coordinate, cluster_index, labels
import tensorflow as tf
import uff
from lanenet_model import lanenet_merge_model
if __name__ == '__main__':
sess = tf.Session()
with sess.as_default():
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(
input_tensor=input_tensor, name='lanenet_model')
with open('convert_before.txt', 'w') as f:
for op in tf.get_default_graph().get_operations():
f.write(op.name + '\n')
saver = tf.train.Saver()
saver.restore(
sess,
'Tusimple_Lanenet_Model_Weights/tusimple_lanenet_vgg_2018-10-19-13-33-56.ckpt-200000'
)
print('stored!')
g_def = tf.get_default_graph().as_graph_def()
g_def_freezed = tf.graph_util.convert_variables_to_constants(
sess, g_def,
def test_lanenet_batch(self, batch_size=2, use_gpu=1):
"""
:param image_dir:
:param weights_path:
:param batch_size:
:param use_gpu:
:param save_dir:
:return:
"""
assert ops.exists(self.path), '{:s} not exist'.format(self.path)
assert ops.exists(self.save_path), '{:s} not exist'.format(
self.save_path)
assert ops.exists(
self.save_processed_video_path), '{:s} not exist'.format(
self.save_processed_video_path)
log.info('Start getting the image file path...')
image_path_list = sorted(
glob.glob('{:s}/**/*.jpg'.format(self.path), recursive=True) +
glob.glob('{:s}/**/*.png'.format(self.path), recursive=True) +
glob.glob('{:s}/**/*.jpeg'.format(self.path), recursive=True))
input_tensor = tf.placeholder(
dtype=tf.float32, shape=[2, 352, 640, 3],
name='input_tensor') # 2, 640, 352, 3 #None, 256, 512, 3
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(
input_tensor=input_tensor, name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
log.info('GPU detected, processing on GPU now..')
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=self.weights_path)
epoch_nums = int(math.ceil(len(image_path_list) / batch_size))
for epoch in range(epoch_nums):
log.info(
'[Epoch:{:d}] Start image reading and preprocessing...'.
format(epoch))
t_start = time.time()
image_path_epoch = image_path_list[epoch *
batch_size:(epoch + 1) *
batch_size]
image_list_epoch = [
cv2.imread(tmp, cv2.IMREAD_COLOR)
for tmp in image_path_epoch
]
image_vis_list = image_list_epoch
image_list_epoch = [
cv2.resize(tmp, (640, 352), interpolation=cv2.INTER_LINEAR)
for tmp in image_list_epoch
]
image_list_epoch = [tmp - VGG_MEAN for tmp in image_list_epoch]
t_cost = time.time() - t_start
log.info(
'[Epoch:{:d}] Pretreatment{:d}Image, total time consuming: {:.5f}s, Average Time per Sheet: {:.5f}'
.format(epoch, len(image_path_epoch), t_cost,
t_cost / len(image_path_epoch)))
t_start = time.time()
binary_seg_images, instance_seg_images = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: image_list_epoch})
t_cost = time.time() - t_start
log.info(
'[Epoch:{:d}] prediction{:d}Image lane line, total time consuming: {:.5f}s, Average Time per Sheet: {:.5f}s'
.format(epoch, len(image_path_epoch), t_cost,
t_cost / len(image_path_epoch)))
cluster_time = []
for index, binary_seg_image in enumerate(binary_seg_images):
t_start = time.time()
binary_seg_image = postprocessor.postprocess(
binary_seg_image)
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image,
instance_seg_ret=instance_seg_images[index])
cluster_time.append(time.time() - t_start)
mask_image = cv2.resize(mask_image,
(image_vis_list[index].shape[1],
image_vis_list[index].shape[0]),
interpolation=cv2.INTER_LINEAR)
if self.save_path is None:
plt.ion()
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis_list[index][:, :, (2, 1, 0)])
plt.pause(3.0)
plt.show()
plt.ioff()
if self.save_path is not None:
mask_image = cv2.addWeighted(image_vis_list[index],
1.0, mask_image, 1.0, 0)
image_name = ops.split(image_path_epoch[index])[1]
image_save_path = ops.join(self.save_path, image_name)
cv2.imwrite(image_save_path, mask_image)
log.info(
'[Epoch:{:d}] Get on {:d}Image lane line clustering, total time consuming: {:.5f}s, Average Time per Sheet: {:.5f}'
.format(epoch, len(image_path_epoch), np.sum(cluster_time),
np.mean(cluster_time)))
sess.close()
return
def train_net(dataset_dir, weights_path=None, net_flag='vgg'):
"""
:param dataset_dir:
:param net_flag: choose which base network to use
:param weights_path:
:return:
"""
train_dataset_file = ops.join(dataset_dir, 'train_gt.txt')
val_dataset_file = ops.join(dataset_dir, 'val_gt.txt')
assert ops.exists(train_dataset_file)
train_dataset = lanenet_data_processor.DataSet(train_dataset_file)
val_dataset = lanenet_data_processor.DataSet(val_dataset_file)
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[
CFG.TRAIN.BATCH_SIZE,
CFG.TRAIN.IMG_HEIGHT,
CFG.TRAIN.IMG_WIDTH, 3
],
name='input_tensor')
instance_label_tensor = tf.placeholder(dtype=tf.int64,
shape=[
CFG.TRAIN.BATCH_SIZE,
CFG.TRAIN.IMG_HEIGHT,
CFG.TRAIN.IMG_WIDTH
],
name='instance_input_label')
existence_label_tensor = tf.placeholder(dtype=tf.float32,
shape=[CFG.TRAIN.BATCH_SIZE, 4],
name='existence_input_label')
phase = tf.placeholder(dtype=tf.string, shape=None, name='net_phase')
net = lanenet_merge_model.LaneNet(net_flag=net_flag, phase=phase)
# calculate the loss
compute_ret = net.compute_loss(input_tensor=input_tensor,
binary_label=instance_label_tensor,
existence_label=existence_label_tensor,
name='lanenet_loss')
total_loss = compute_ret['total_loss']
instance_loss = compute_ret['instance_seg_loss']
existence_loss = compute_ret['existence_pre_loss']
existence_logits = compute_ret['existence_logits']
# calculate the accuracy
out_logits = compute_ret['instance_seg_logits']
out_logits = tf.nn.softmax(logits=out_logits)
out_logits_out = tf.argmax(out_logits, axis=-1)
out = tf.argmax(out_logits, axis=-1)
out = tf.expand_dims(out, axis=-1)
idx = tf.where(tf.equal(instance_label_tensor, 1))
pix_cls_ret = tf.gather_nd(out, idx)
accuracy = tf.count_nonzero(pix_cls_ret)
accuracy = tf.divide(accuracy, tf.cast(tf.shape(pix_cls_ret)[0], tf.int64))
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(CFG.TRAIN.LEARNING_RATE,
global_step,
5000,
0.96,
staircase=True)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(
loss=total_loss,
var_list=tf.trainable_variables(),
global_step=global_step)
# Set tf saver
saver = tf.train.Saver()
model_save_dir = 'model/culane_lanenet/culane_scnn'
if not ops.exists(model_save_dir):
os.makedirs(model_save_dir)
train_start_time = time.strftime('%Y-%m-%d-%H-%M-%S',
time.localtime(time.time()))
model_name = 'culane_lanenet_{:s}_{:s}.ckpt'.format(
net_flag, str(train_start_time))
model_save_path = ops.join(model_save_dir, model_name)
# Set sess configuration
sess_config = tf.ConfigProto(device_count={'GPU':
4}) # device_count={'GPU': 1}
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
# Set the training parameters
train_epochs = CFG.TRAIN.EPOCHS
log.info('Global configuration is as follows:')
log.info(CFG)
with sess.as_default():
tf.train.write_graph(
graph_or_graph_def=sess.graph,
logdir='',
name='{:s}/lanenet_model.pb'.format(model_save_dir))
if weights_path is None:
log.info('Training from scratch')
init = tf.global_variables_initializer()
sess.run(init)
else:
log.info('Restore model from last model checkpoint {:s}'.format(
weights_path))
saver.restore(sess=sess, save_path=weights_path)
# 加载预训练参数
if net_flag == 'vgg' and weights_path is None:
pretrained_weights = np.load('./data/vgg16.npy',
encoding='latin1').item()
for vv in tf.trainable_variables():
weights_key = vv.name.split('/')[-3]
try:
weights = pretrained_weights[weights_key][0]
_op = tf.assign(vv, weights)
sess.run(_op)
except Exception as e:
continue
train_cost_time_mean = []
train_instance_loss_mean = []
train_existence_loss_mean = []
train_accuracy_mean = []
val_cost_time_mean = []
val_instance_loss_mean = []
val_existence_loss_mean = []
val_accuracy_mean = []
for epoch in range(train_epochs):
# training part
t_start = time.time()
gt_imgs, instance_gt_labels, existence_gt_labels = train_dataset.next_batch(
CFG.TRAIN.BATCH_SIZE)
gt_imgs = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_LINEAR) for tmp in gt_imgs
]
gt_imgs = [tmp - VGG_MEAN for tmp in gt_imgs]
instance_gt_labels = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_NEAREST)
for tmp in instance_gt_labels
]
phase_train = 'train'
_, c, train_accuracy, train_instance_loss, train_existence_loss, binary_seg_img = \
sess.run([optimizer, total_loss,
accuracy,
instance_loss,
existence_loss,
out_logits_out],
feed_dict={input_tensor: gt_imgs,
instance_label_tensor: instance_gt_labels,
existence_label_tensor: existence_gt_labels,
phase: phase_train})
cost_time = time.time() - t_start
train_cost_time_mean.append(cost_time)
train_instance_loss_mean.append(train_instance_loss)
train_existence_loss_mean.append(train_existence_loss)
train_accuracy_mean.append(train_accuracy)
# validation part
gt_imgs_val, instance_gt_labels_val, existence_gt_labels_val \
= val_dataset.next_batch(CFG.TRAIN.VAL_BATCH_SIZE)
gt_imgs_val = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_LINEAR)
for tmp in gt_imgs_val
]
gt_imgs_val = [tmp - VGG_MEAN for tmp in gt_imgs_val]
instance_gt_labels_val = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_NEAREST)
for tmp in instance_gt_labels_val
]
phase_val = 'test'
t_start_val = time.time()
c_val, val_accuracy, val_instance_loss, val_existence_loss = \
sess.run([total_loss, accuracy, instance_loss, existence_loss],
feed_dict={input_tensor: gt_imgs_val,
instance_label_tensor: instance_gt_labels_val,
existence_label_tensor: existence_gt_labels_val,
phase: phase_val})
cost_time_val = time.time() - t_start_val
val_cost_time_mean.append(cost_time_val)
val_instance_loss_mean.append(val_instance_loss)
val_existence_loss_mean.append(val_existence_loss)
val_accuracy_mean.append(val_accuracy)
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
print(
'Epoch: {:d} loss_ins= {:6f} ({:6f}) loss_ext= {:6f} ({:6f}) accuracy= {:6f} ({:6f})'
' mean_time= {:5f}s '.format(
epoch + 1, train_instance_loss,
np.mean(train_instance_loss_mean),
train_existence_loss,
np.mean(train_existence_loss_mean), train_accuracy,
np.mean(train_accuracy_mean),
np.mean(train_cost_time_mean))) # log.info
if epoch % CFG.TRAIN.TEST_DISPLAY_STEP == 0:
print('Epoch_Val: {:d} loss_ins= {:6f} ({:6f}) '
'loss_ext= {:6f} ({:6f}) accuracy= {:6f} ({:6f})'
'mean_time= {:5f}s '.format(
epoch + 1, val_instance_loss,
np.mean(val_instance_loss_mean), val_existence_loss,
np.mean(val_existence_loss_mean), val_accuracy,
np.mean(val_accuracy_mean),
np.mean(val_cost_time_mean)))
if epoch % 500 == 0:
train_cost_time_mean.clear()
train_instance_loss_mean.clear()
train_existence_loss_mean.clear()
train_accuracy_mean.clear()
val_cost_time_mean.clear()
val_instance_loss_mean.clear()
val_existence_loss_mean.clear()
val_accuracy_mean.clear()
if epoch % 2000 == 0:
saver.save(sess=sess,
save_path=model_save_path,
global_step=epoch)
sess.close()
return
def train_net(dataset_dir, weights_path=None, net_flag='vgg'):
"""
:param dataset_dir:
:param net_flag: choose which base network to use
:param weights_path:
:return:
"""
train_dataset_file = ops.join(dataset_dir, 'train.txt')
val_dataset_file = ops.join(dataset_dir, 'val.txt')
assert ops.exists(train_dataset_file)
train_dataset = lanenet_data_processor.DataSet(train_dataset_file)
val_dataset = lanenet_data_processor.DataSet(val_dataset_file)
input_tensor = tf.placeholder(
dtype=tf.float32,
shape=[None, CFG.TRAIN.IMG_HEIGHT, CFG.TRAIN.IMG_WIDTH, 3],
name='input_tensor')
binary_label_tensor = tf.placeholder(
dtype=tf.int64,
shape=[None, CFG.TRAIN.IMG_HEIGHT, CFG.TRAIN.IMG_WIDTH, 1],
name='binary_input_label')
instance_label_tensor = tf.placeholder(
dtype=tf.float32,
shape=[None, CFG.TRAIN.IMG_HEIGHT, CFG.TRAIN.IMG_WIDTH],
name='instance_input_label')
phase = tf.placeholder(dtype=tf.string, shape=None, name='net_phase')
# net = lanenet_instance_segmentation.LaneNetInstanceSeg(net_flag=net_flag, phase=phase)
net = lanenet_merge_model.LaneNet(net_flag=net_flag, phase=phase)
# calculate the loss
compute_ret = net.compute_loss(input_tensor=input_tensor,
binary_label=binary_label_tensor,
instance_label=instance_label_tensor,
name='lanenet_loss')
total_loss = compute_ret['total_loss']
binary_seg_loss = compute_ret['binary_seg_loss']
disc_loss = compute_ret['discriminative_loss']
pix_embedding = compute_ret['instance_seg_logits']
# calculate the accuracy
out_logits = compute_ret['binary_seg_logits']
out_logits = tf.nn.softmax(logits=out_logits)
out_logits_out = tf.argmax(out_logits, axis=-1)
out = tf.argmax(out_logits, axis=-1)
out = tf.expand_dims(out, axis=-1)
accuracy = tf.add(binary_label_tensor, -1 * out)
accuracy = tf.count_nonzero(accuracy, axis=[1, 2, 3])
accuracy = tf.add(
tf.constant(1, dtype=tf.float64),
-1 * tf.divide(accuracy, CFG.TRAIN.IMG_HEIGHT * CFG.TRAIN.IMG_WIDTH))
accuracy = tf.reduce_mean(accuracy, axis=0)
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(CFG.TRAIN.LEARNING_RATE,
global_step,
5000,
0.96,
staircase=True)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(
loss=total_loss,
var_list=tf.trainable_variables(),
global_step=global_step)
# Set tf saver
saver = tf.train.Saver()
model_save_dir = 'model/kitti_lanenet'
if not ops.exists(model_save_dir):
os.makedirs(model_save_dir)
train_start_time = time.strftime('%Y-%m-%d-%H-%M-%S',
time.localtime(time.time()))
model_name = 'kitti_lanenet_{:s}_{:s}.ckpt'.format(net_flag,
str(train_start_time))
model_save_path = ops.join(model_save_dir, model_name)
# Set tf summary
tboard_save_path = 'tboard/kitti_lanenet/{:s}'.format(net_flag)
if not ops.exists(tboard_save_path):
os.makedirs(tboard_save_path)
train_cost_scalar = tf.summary.scalar(name='train_cost', tensor=total_loss)
val_cost_scalar = tf.summary.scalar(name='val_cost', tensor=total_loss)
train_accuracy_scalar = tf.summary.scalar(name='train_accuracy',
tensor=accuracy)
val_accuracy_scalar = tf.summary.scalar(name='val_accuracy',
tensor=accuracy)
train_binary_seg_loss_scalar = tf.summary.scalar(
name='train_binary_seg_loss', tensor=binary_seg_loss)
val_binary_seg_loss_scalar = tf.summary.scalar(name='val_binary_seg_loss',
tensor=binary_seg_loss)
train_instance_seg_loss_scalar = tf.summary.scalar(
name='train_instance_seg_loss', tensor=disc_loss)
val_instance_seg_loss_scalar = tf.summary.scalar(
name='val_instance_seg_loss', tensor=disc_loss)
learning_rate_scalar = tf.summary.scalar(name='learning_rate',
tensor=learning_rate)
train_merge_summary_op = tf.summary.merge([
train_accuracy_scalar, train_cost_scalar, learning_rate_scalar,
train_binary_seg_loss_scalar, train_instance_seg_loss_scalar
])
val_merge_summary_op = tf.summary.merge([
val_accuracy_scalar, val_cost_scalar, val_binary_seg_loss_scalar,
val_instance_seg_loss_scalar
])
# Set sess configuration
sess_config = tf.ConfigProto(device_count={'GPU': 1})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
summary_writer = tf.summary.FileWriter(tboard_save_path)
summary_writer.add_graph(sess.graph)
# Set the training parameters
train_epochs = CFG.TRAIN.EPOCHS
log.info('Global configuration is as follows:')
log.info(CFG)
with sess.as_default():
tf.train.write_graph(
graph_or_graph_def=sess.graph,
logdir='',
name='{:s}/lanenet_model.pb'.format(model_save_dir))
if weights_path is None:
log.info('Training from scratch')
init = tf.global_variables_initializer()
sess.run(init)
else:
log.info('Restore model from last model checkpoint {:s}'.format(
weights_path))
saver.restore(sess=sess, save_path=weights_path)
# 加载预训练参数
if net_flag == 'vgg':
pretrained_weights = np.load(
'/home/baidu/Silly_Project/ICode/baidu/beec/semantic-road-estimation/data/vgg16.npy',
encoding='latin1').item()
for vv in tf.trainable_variables():
weights_key = vv.name.split('/')[-3]
try:
weights = pretrained_weights[weights_key][0]
_op = tf.assign(vv, weights)
sess.run(_op)
except Exception as e:
continue
train_cost_time_mean = []
val_cost_time_mean = []
for epoch in range(train_epochs):
# training part
t_start = time.time()
gt_imgs, binary_gt_labels, instance_gt_labels = train_dataset.next_batch(
CFG.TRAIN.BATCH_SIZE)
gt_imgs = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_LINEAR) for tmp in gt_imgs
]
gt_imgs = [tmp - VGG_MEAN for tmp in gt_imgs]
binary_gt_labels = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_NEAREST)
for tmp in binary_gt_labels
]
binary_gt_labels = [
np.expand_dims(tmp, axis=-1) for tmp in binary_gt_labels
]
instance_gt_labels = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_NEAREST)
for tmp in instance_gt_labels
]
phase_train = 'train'
_, c, train_accuracy, train_summary, binary_loss, instance_loss, embedding, binary_seg_img = \
sess.run([optimizer, total_loss,
accuracy,
train_merge_summary_op,
binary_seg_loss,
disc_loss,
pix_embedding,
out_logits_out],
feed_dict={input_tensor: gt_imgs,
binary_label_tensor: binary_gt_labels,
instance_label_tensor: instance_gt_labels,
phase: phase_train})
if math.isnan(c) or math.isnan(binary_loss) or math.isnan(
instance_loss):
log.error('cost is: {:.5f}'.format(c))
log.error('binary cost is: {:.5f}'.format(binary_loss))
log.error('instance cost is: {:.5f}'.format(instance_loss))
cv2.imwrite('nan_image.png', gt_imgs[0] + VGG_MEAN)
cv2.imwrite('nan_instance_label.png', instance_gt_labels[0])
cv2.imwrite('nan_binary_label.png', binary_gt_labels[0] * 255)
cv2.imwrite('nan_embedding.png', embedding[0])
return
if epoch % 100 == 0:
cv2.imwrite('image.png', gt_imgs[0] + VGG_MEAN)
cv2.imwrite('binary_label.png', binary_gt_labels[0] * 255)
cv2.imwrite('instance_label.png', instance_gt_labels[0])
cv2.imwrite('binary_seg_img.png', binary_seg_img[0] * 255)
cv2.imwrite('embedding.png', embedding[0])
cost_time = time.time() - t_start
train_cost_time_mean.append(cost_time)
summary_writer.add_summary(summary=train_summary,
global_step=epoch)
# validation part
gt_imgs_val, binary_gt_labels_val, instance_gt_labels_val \
= val_dataset.next_batch(CFG.TRAIN.VAL_BATCH_SIZE)
gt_imgs_val = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_LINEAR)
for tmp in gt_imgs_val
]
gt_imgs_val = [tmp - VGG_MEAN for tmp in gt_imgs_val]
binary_gt_labels_val = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp) for tmp in binary_gt_labels_val
]
binary_gt_labels_val = [
np.expand_dims(tmp, axis=-1) for tmp in binary_gt_labels_val
]
instance_gt_labels_val = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_NEAREST)
for tmp in instance_gt_labels_val
]
phase_val = 'test'
t_start_val = time.time()
c_val, val_summary, val_accuracy, val_binary_seg_loss, val_instance_seg_loss = \
sess.run([total_loss, val_merge_summary_op, accuracy, binary_seg_loss, disc_loss],
feed_dict={input_tensor: gt_imgs_val,
binary_label_tensor: binary_gt_labels_val,
instance_label_tensor: instance_gt_labels_val,
phase: phase_val})
if epoch % 100 == 0:
cv2.imwrite('test_image.png', gt_imgs_val[0] + VGG_MEAN)
summary_writer.add_summary(val_summary, global_step=epoch)
cost_time_val = time.time() - t_start_val
val_cost_time_mean.append(cost_time_val)
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
log.info(
'Epoch: {:d} total_loss= {:6f} binary_seg_loss= {:6f} instance_seg_loss= {:6f} accuracy= {:6f}'
' mean_cost_time= {:5f}s '.format(
epoch + 1, c, binary_loss, instance_loss,
train_accuracy, np.mean(train_cost_time_mean)))
train_cost_time_mean.clear()
if epoch % CFG.TRAIN.TEST_DISPLAY_STEP == 0:
log.info(
'Epoch_Val: {:d} total_loss= {:6f} binary_seg_loss= {:6f} '
'instance_seg_loss= {:6f} accuracy= {:6f} '
'mean_cost_time= {:5f}s '.format(
epoch + 1, c_val, val_binary_seg_loss,
val_instance_seg_loss, val_accuracy,
np.mean(val_cost_time_mean)))
val_cost_time_mean.clear()
if epoch % 2000 == 0:
saver.save(sess=sess,
save_path=model_save_path,
global_step=epoch)
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
log.info('开始读取图像数据并进行预处理')
t_start = time.time()
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
org_size = (image_vis.shape[1], image_vis.shape[0])
print("org_size: ", org_size)
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
w, h = image.shape[:2]
w_part = int(w / split_num)
h_part = int(h / split_num)
for ii in range(0, split_num):
for jj in range(0, split_num):
print("ii, jj: ", ii, " ", jj)
part_img = image[ii * w_part:(ii + 1) * w_part - 1,
jj * h_part:(jj + 1) * h_part - 1]
part_img = cv2.resize(part_img, (512, 256),
interpolation=cv2.INTER_LINEAR)
part_img = part_img - VGG_MEAN
log.info('图像读取完毕, 耗时: {:.5f}s'.format(time.time() - t_start))
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [part_img]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
binary_seg_image[0] = postprocessor.postprocess(
binary_seg_image[0])
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
for i in range(4):
instance_seg_image[0][:, :, i] = minmax_scale(
instance_seg_image[0][:, :, i])
embedding_image = np.array(instance_seg_image[0], np.uint8)
# plt.figure('mask_image')
# plt.imshow(mask_image[:, :, (2, 1, 0)])
# plt.figure('src_image')
# plt.imshow(image_vis[:, :, (2, 1, 0)])
# plt.figure('instance_image')
# plt.imshow(embedding_image[:, :, (2, 1, 0)], cmap='gray')
# plt.figure('binary_image')
# plt.imshow(binary_seg_image[0] * 255, cmap='gray')
# plt.show()
embedding_img_list.append(embedding_image.copy())
binary_img_list.append(binary_seg_image[0] * 255)
# merge to a complete embedding_image
embedding_image = embedding_img_list[0].copy()
binary_image = binary_img_list[0].copy()
embedding_image_row = None
binary_image_row = None
for i in range(0, split_num):
embedding_image_row = embedding_img_list[i * split_num]
binary_image_row = binary_img_list[i * split_num]
for j in range(0, split_num):
if j > 0:
embedding_image_row = np.hstack(
(embedding_image_row,
embedding_img_list[i * split_num + j]))
binary_image_row = np.hstack(
(binary_image_row, binary_img_list[i * split_num + j]))
if i == 0:
embedding_image = embedding_image_row.copy()
binary_image = binary_image_row.copy()
else:
embedding_image = np.vstack((embedding_image, embedding_image_row))
binary_image = np.vstack((binary_image, binary_image_row))
#binary_image_org_size = cv2.resize(binary_image, org_size, interpolation=cv2.INTER_LINEAR)
cv2.imwrite("binary_image_org_size.jpg", binary_image)
instance_image_org_size = cv2.resize(embedding_image[:, :, (2, 1, 0)],
org_size,
interpolation=cv2.INTER_LINEAR)
cv2.imwrite("instance_image_org_size.jpg", instance_image_org_size)
instance_image_org_size_red = cv2.resize(embedding_image[:, :, 0],
org_size,
interpolation=cv2.INTER_LINEAR)
cv2.imwrite("instance_image_org_size_red.jpg", instance_image_org_size_red)
instance_image_org_size_green = cv2.resize(embedding_image[:, :, 1],
org_size,
interpolation=cv2.INTER_LINEAR)
cv2.imwrite("instance_image_org_size_green.jpg",
instance_image_org_size_green)
instance_image_org_size_blue = cv2.resize(embedding_image[:, :, 2],
org_size,
interpolation=cv2.INTER_LINEAR)
cv2.imwrite("instance_image_org_size_blue.jpg",
instance_image_org_size_blue)
# Remember -> OpenCV stores things in BGR order
lowerBound_blue = np.array((33), dtype=np.uint8, ndmin=1)
upperBound_blue = np.array((213), dtype=np.uint8, ndmin=1)
lowerBound_red = np.array((148), dtype=np.uint8, ndmin=1)
upperBound_red = np.array((208), dtype=np.uint8, ndmin=1)
# this gives you the mask for those in the ranges you specified,
# but you want the inverse, so we'll add bitwise_not...
cv_rgb_thresh_blue = cv2.inRange(instance_image_org_size_blue,
lowerBound_blue, upperBound_blue)
cv_rgb_thresh_blue = cv2.bitwise_not(cv_rgb_thresh_blue)
cv2.imwrite("instance_image_blue_thresh.jpg", cv_rgb_thresh_blue)
th3 = cv2.adaptiveThreshold(instance_image_org_size_blue, 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY, 13, 0)
cv2.imwrite("instance_image_blue_adaptive_thresh.jpg", th3)
cv_rgb_thresh_red = cv2.inRange(instance_image_org_size_red,
lowerBound_red, upperBound_red)
cv_rgb_thresh_red = cv2.bitwise_not(cv_rgb_thresh_red)
cv2.imwrite("instance_image_red_thresh.jpg", cv_rgb_thresh_red)
cv_rgb_thresh_mix = cv2.bitwise_or(cv_rgb_thresh_blue, cv_rgb_thresh_red)
cv2.imwrite("instance_image_mix_thresh.jpg", cv_rgb_thresh_mix)
#instance_image_org_gray = cv2.cvtColor(instance_image_org_size, cv2.COLOR_BGR2GRAY)
#cv2.imwrite("instance_image_org_gray.jpg", instance_image_org_gray)
sess.close()
return
def test_lanenet_batch(image_dir, weights_path, batch_size, use_gpu, save_dir=None):
"""
:param image_dir:
:param weights_path:
:param batch_size:
:param use_gpu:
:param save_dir:
:return:
"""
assert ops.exists(image_dir), '{:s} not exist'.format(image_dir)
log.info('开始获取图像文件路径...')
image_path_list = glob.glob('{:s}/**/*.jpg'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.png'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.jpeg'.format(image_dir), recursive=True)
input_tensor = tf.placeholder(dtype=tf.float32, shape=[None, 256, 512, 3], name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor, name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
epoch_nums = int(math.ceil(len(image_path_list) / batch_size))
for epoch in range(epoch_nums):
log.info('[Epoch:{:d}] 开始图像读取和预处理...'.format(epoch))
t_start = time.time()
image_path_epoch = image_path_list[epoch * batch_size:(epoch + 1) * batch_size]
image_list_epoch = [cv2.imread(tmp, cv2.IMREAD_COLOR) for tmp in image_path_epoch]
image_vis_list = image_list_epoch
image_list_epoch = [cv2.resize(tmp, (512, 256), interpolation=cv2.INTER_LINEAR)
for tmp in image_list_epoch]
image_list_epoch = [tmp - VGG_MEAN for tmp in image_list_epoch]
t_cost = time.time() - t_start
log.info('[Epoch:{:d}] 预处理{:d}张图像, 共耗时: {:.5f}s, 平均每张耗时: {:.5f}'.format(
epoch, len(image_path_epoch), t_cost, t_cost / len(image_path_epoch)))
t_start = time.time()
binary_seg_images, instance_seg_images = sess.run(
[binary_seg_ret, instance_seg_ret], feed_dict={input_tensor: image_list_epoch})
t_cost = time.time() - t_start
log.info('[Epoch:{:d}] 预测{:d}张图像车道线, 共耗时: {:.5f}s, 平均每张耗时: {:.5f}s'.format(
epoch, len(image_path_epoch), t_cost, t_cost / len(image_path_epoch)))
cluster_time = []
for index, binary_seg_image in enumerate(binary_seg_images):
t_start = time.time()
# 一些传统图像处理方法,对二值化图进行处理
binary_seg_image = postprocessor.postprocess(binary_seg_image)
# 将二值化图和实例分割图进行融合,聚类
mask_image = cluster.get_lane_mask(binary_seg_ret=binary_seg_image,
instance_seg_ret=instance_seg_images[index])
cluster_time.append(time.time() - t_start)
mask_image = cv2.resize(mask_image, (image_vis_list[index].shape[1],
image_vis_list[index].shape[0]),
interpolation=cv2.INTER_LINEAR)
if save_dir is None:
plt.ion()
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis_list[index][:, :, (2, 1, 0)])
plt.pause(3.0)
plt.show()
plt.ioff()
if save_dir is not None:
mask_image = cv2.addWeighted(image_vis_list[index], 1.0, mask_image, 1.0, 0)
image_name = ops.split(image_path_epoch[index])[1]
image_save_path = ops.join(save_dir, image_name)
cv2.imwrite(image_save_path, mask_image)
log.info('[Epoch:{:d}] 进行{:d}张图像车道线聚类, 共耗时: {:.5f}s, 平均每张耗时: {:.5f}'.format(
epoch, len(image_path_epoch), np.sum(cluster_time), np.mean(cluster_time)))
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu, image_list, batch_size,
save_dir):
"""
:param image_path:
:param weights_path: ***
:param use_gpu:
:return:
"""
print("6666666666666")
global total_img
test_dataset = lanenet_data_processor_test.DataSet(image_path, batch_size)
input_tensor = tf.placeholder(dtype=tf.string,
shape=[None],
name='input_tensor')
imgs = tf.map_fn(test_dataset.process_img, input_tensor, dtype=tf.float32)
phase_tensor = tf.constant('test', tf.string) # str常量
net = lanenet_merge_model.LaneNet()
binary_seg_ret, instance_seg_ret = net.test_inference(
imgs, phase_tensor, 'lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
print(len(final_var)) # 85
# Pass the variables as a list:
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver.restore(
sess=sess, save_path=weights_path
) #.ckpt //save_path: Path where parameters were previously saved.
# There is a mismatch between the graph and the checkpoint being loaded.
print("Model restored.")
#print(len(image_list) / batch_size) #1.0
for i in range(int(math.ceil(len(image_list) / batch_size))):
print("i: ", i)
paths = test_dataset.next_batch()
#print(paths) # 1.jpg ~ 8.jpg
instance_seg_image, existence_output = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: paths})
#print('instance_seg_image shape: ',instance_seg_image.shape)
#image_list_epoch = [cv2.imread(tmp, cv2.IMREAD_COLOR) for tmp in paths]
#image_list_epoch = [tmp - VGG_MEAN for tmp in image_list_epoch]
#instance_seg_image, existence_output = sess.run([binary_seg_ret, instance_seg_ret],
#feed_dict={input_tensor: image_list_epoch})
for cnt, image_name in enumerate(paths):
total_img = np.zeros([288, 800])
#print(image_name)
parent_path = os.path.dirname(image_name)
#print(parent_path) /Users/wutong/Downloads/test_set/clips/0601/1494453197736907986
# //Users/wutong/Downloads/train_set/clips/0313-1/8580/20.jpg
ph = parent_path.split('/')
directory = os.path.join(save_dir, 'cable_pic', ph[-1])
if not os.path.exists(directory):
os.makedirs(directory)
file_exist = open(
os.path.join(
directory,
os.path.basename(image_name)[:-3] + 'exist.txt'), 'w')
for cnt_img in range(4): # 4 lines
cv2.imwrite(
os.path.join(
directory,
os.path.basename(image_name)[:-4] + '_' +
str(cnt_img + 1) + '_avg.png'),
(instance_seg_image[cnt, :, :, cnt_img + 1] *
255).astype(int))
if existence_output[
cnt,
cnt_img] > 0.5: # >0.5 suppose that have a line
#file_exist.write('%s ' % existence_output[cnt, cnt_img])
file_exist.write('1 ')
total_img += (
instance_seg_image[cnt, :, :, cnt_img + 1] *
255).astype(int)
else:
file_exist.write('0 ')
cv2.imwrite(
os.path.join(
directory,
os.path.basename(image_name)[:-4] + '_' +
'total_img.png'), total_img)
file_exist.close()
sess.close()
return
def test_lanenet_batch(image_dir,
weights_path,
batch_size,
use_gpu,
save_dir=None,
encoder="vgg"):
"""
:param image_dir:
:param weights_path:
:param batch_size:
:param use_gpu:
:param save_dir:
:return:
"""
assert ops.exists(image_dir), '{:s} not exist'.format(image_dir)
log.info('开始获取图像文件路径...')
image_path_list = glob.glob('{:s}/**/*.jpg'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.png'.format(image_dir), recursive=True) + \
glob.glob('{:s}/**/*.jpeg'.format(image_dir), recursive=True)
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[None, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag=encoder)
binary_seg_ret, instance_seg_ret, prob_seg_ret = net.inference(
input_tensor=input_tensor, name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
# Set tf saver
# if weights_path is not None:
# var_map = restore_from_classification_checkpoint_fn("")
# available_var_map = (get_variables_available_in_checkpoint(
# var_map, weights_path, include_global_step=False))
#
# saver = tf.train.Saver(available_var_map)
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
# sess_config = tf.ConfigProto(device_count={'CPU': 0})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 1})
# sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
ignore_labels = cv2.imread(
'/media/remus/datasets/AVMSnapshots/AVM/ignore_labels.png')
ignore_labels = cv2.cvtColor(ignore_labels, cv2.COLOR_BGR2GRAY)
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
epoch_nums = int(math.ceil(len(image_path_list) / batch_size))
for epoch in range(epoch_nums):
log.info('[Epoch:{:d}] starts image reading and preprocessing...'.
format(epoch))
t_start = time.time()
image_path_epoch = image_path_list[epoch * batch_size:(epoch + 1) *
batch_size]
image_list_epoch = [
cv2.imread(tmp, cv2.IMREAD_COLOR) for tmp in image_path_epoch
]
image_vis_list = image_list_epoch
image_list_epoch = [
cv2.resize(tmp, (512, 256), interpolation=cv2.INTER_LINEAR)
for tmp in image_list_epoch
]
if encoder == "mobilenet":
image_list_epoch = [
tmp / 128.0 - 1.0 for tmp in image_list_epoch
]
else:
image_list_epoch = [tmp - VGG_MEAN for tmp in image_list_epoch]
t_cost = time.time() - t_start
log.info(
'[Epoch:{:d}] preprocesses {:d} images, total time: {:.5f}s, average time per sheet: {:.5f}'
.format(epoch, len(image_path_epoch), t_cost,
t_cost / len(image_path_epoch)))
t_start = time.time()
binary_seg_images, instance_seg_images, prob_seg_images = sess.run(
[binary_seg_ret, instance_seg_ret, prob_seg_ret],
feed_dict={input_tensor: image_list_epoch})
t_cost = time.time() - t_start
log.info(
'[Epoch:{:d}] predicts {:d} image lane lines, total time: {:.5f}s, average time per sheet: {:.5f}s'
.format(epoch, len(image_path_epoch), t_cost,
t_cost / len(image_path_epoch)))
cluster_time = []
for index, binary_seg_image in enumerate(binary_seg_images):
t_start = time.time()
binary_seg_image[ignore_labels == 0] = 0
binary_seg_image = postprocessor.postprocess(binary_seg_image)
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image,
instance_seg_ret=instance_seg_images[index])
cluster_time.append(time.time() - t_start)
mask_image = cv2.resize(mask_image,
(image_vis_list[index].shape[1],
image_vis_list[index].shape[0]),
interpolation=cv2.INTER_NEAREST)
_instance_seg_images = np.copy(instance_seg_images)
prob_seg_image = prob_seg_images[index, :, :, 1]
for i in range(4):
_instance_seg_images[index][:, :, i] = minmax_scale(
instance_seg_images[index][:, :, i])
_embedding_image = np.array(_instance_seg_images[index],
np.uint8)
if save_dir is None:
plt.ion()
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis_list[index][:, :, (2, 1, 0)])
plt.pause(3.0)
plt.show()
plt.ioff()
if save_dir is not None:
mask_image = cv2.addWeighted(image_vis_list[index], 1.0,
mask_image, 1.0, 0)
image_name = ops.split(image_path_epoch[index])[1]
image_save_path = ops.join(save_dir + "/image", image_name)
mask_save_path = ops.join(save_dir + "/mask", image_name)
prob_save_path = ops.join(save_dir + "/prob", image_name)
embedding_save_path = ops.join(save_dir + "/embedding",
image_name)
cv2.imwrite(mask_save_path, binary_seg_image * 255)
cv2.imwrite(prob_save_path, prob_seg_image * 255)
cv2.imwrite(image_save_path, mask_image)
cv2.imwrite(embedding_save_path,
_embedding_image[:, :, (2, 1, 0)])
# cv2.imwrite(embedding_save_path + "_", _embedding_image[:, :, (3, 2, 1)])
log.info(
'[Epoch:{:d}] performs {:d} image lane line clustering, which takes a total of time: {:.5f}s, average time per sheet: {:.5f}'
.format(epoch, len(image_path_epoch), np.sum(cluster_time),
np.mean(cluster_time)))
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu, image_list, batch_size):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
test_dataset = lanenet_data_processor_test.DataSet(image_path)
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[batch_size, 288, 800, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver.restore(sess=sess, save_path=weights_path)
for i in range(int(len(image_list) / batch_size)):
print(i)
gt_imgs = test_dataset.next_batch(CFG.TRAIN.BATCH_SIZE)
gt_imgs = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_CUBIC) for tmp in gt_imgs
]
gt_imgs = [(tmp - VGG_MEAN) for tmp in gt_imgs]
instance_seg_image, existence_output = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: gt_imgs})
for cnt in range(batch_size):
image_name = image_list[i * batch_size + cnt]
image_prefix = image_name[:-10]
directory = 'predicts_SCNN_test_final/vgg_SCNN_DULR_w9' + image_prefix
if not os.path.exists(directory):
os.makedirs(directory)
file_exist = open(
directory + image_name[-10:-4] + '.exist.txt', 'w')
for cnt_img in range(4):
cv2.imwrite(
directory + image_name[-10:-4] + '_' +
str(cnt_img + 1) + '_avg.png',
(instance_seg_image[cnt, :, :, cnt_img + 1] *
255).astype(int))
if existence_output[cnt, cnt_img] > 0.5:
file_exist.write('1 ')
else:
file_exist.write('0 ')
file_exist.close()
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu, image_list, batch_size,
save_dir):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
src_image = cv2.imread('./00330.jpg')
#print(src_image.shape[0], src_image.shape[1])
#src_image = cv2.resize(src_image, (1640,590), interpolation=cv2.INTER_AREA)
w, h = src_image.shape[:2]
test_dataset = lanenet_data_processor_test.DataSet(image_path, batch_size)
input_tensor = tf.placeholder(dtype=tf.string,
shape=[None],
name='input_tensor')
imgs = tf.map_fn(test_dataset.process_img, input_tensor, dtype=tf.float32)
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet()
binary_seg_ret, instance_seg_ret = net.test_inference(
imgs, phase_tensor, 'lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver.restore(sess=sess, save_path=weights_path)
for i in range(math.ceil(len(image_list) / batch_size)):
#print(i)
paths = test_dataset.next_batch()
instance_seg_image, existence_output = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: paths})
for cnt, image_name in enumerate(paths):
#print(image_name)
parent_path = os.path.dirname(image_name)
directory = os.path.join(save_dir, 'vgg_SCNN_DULR_w9',
parent_path)
if not os.path.exists(directory):
os.makedirs(directory)
file_exist = open(
os.path.join(
directory,
os.path.basename(image_name)[:-3] + 'exist.txt'), 'w')
for cnt_img in range(4):
coordinate_tmp = np.zeros((1, 30))
#print(coordinate_tmp)
img = (instance_seg_image[cnt, :, :, cnt_img + 1] *
255).astype(int)
#print(w, h)
for i in range(30):
#print(i)
lineId = math.ceil(288 - i * 20 / w * 288) - 1
#print(lineId)
#print(img.shape)
img_line = img[lineId]
#print(type(img_line))a = list(a)
value = np.max(img_line)
id = np.where(img_line == value)
#print(id[0])
#Wprint()
#print(value, id[0][0])
if (value / 255 > 0.3):
coordinate_tmp[0][i] = id[0][0]
if np.sum(coordinate_tmp > 0) < 2:
coordinate_tmp = np.zeros((1, 30))
#print(coordinate_tmp)
#print(np.sum(coordinate_tmp>0))
for i in range(30):
if coordinate_tmp[0][i] > 0:
cv2.circle(src_image,
(int(coordinate_tmp[0][i] * h / 800),
int(w - i * 20)), 6, (0, 0, 255), -1)
#line = score(lineId,:)
#[value, id] = max(line)
#if double(value)/255 > thr
# coordinate(i) = id
#end
#end
cv2.imwrite(
os.path.join(
directory,
os.path.basename(image_name)[:-4] + '_' +
str(cnt_img + 1) + '_avg.png'),
(instance_seg_image[cnt, :, :, cnt_img + 1] *
255).astype(int))
if existence_output[cnt, cnt_img] > 0.5:
print('\n\n\n')
print(existence_output[cnt, cnt_img])
print('\n\n\n')
file_exist.write('1 ')
else:
file_exist.write('0 ')
file_exist.close()
cv2.imshow('img', src_image)
cv2.waitKey(0)
sess.close()
return
def train_net(dataset_dir, weights_path=None, net_flag='vgg'):
"""
:param dataset_dir:
:param net_flag: choose which base network to use
:param weights_path:
:return:
"""
train_dataset_file = ops.join(dataset_dir, 'train_gt.txt')
val_dataset_file = ops.join(dataset_dir, 'val_gt.txt')
assert ops.exists(train_dataset_file)
train_dataset = lanenet_data_processor.DataSet(train_dataset_file)
val_dataset = lanenet_data_processor.DataSet(val_dataset_file)
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[
CFG.TRAIN.BATCH_SIZE,
CFG.TRAIN.IMG_HEIGHT,
CFG.TRAIN.IMG_WIDTH, 3
],
name='input_tensor')
instance_label_tensor = tf.placeholder(dtype=tf.int64,
shape=[
CFG.TRAIN.BATCH_SIZE,
CFG.TRAIN.IMG_HEIGHT,
CFG.TRAIN.IMG_WIDTH
],
name='instance_input_label')
existence_label_tensor = tf.placeholder(dtype=tf.float32,
shape=[CFG.TRAIN.BATCH_SIZE, 4],
name='existence_input_label')
phase = tf.placeholder(dtype=tf.string, shape=None, name='net_phase')
net = lanenet_merge_model.LaneNet(net_flag=net_flag, phase=phase)
# calculate the loss
compute_ret = net.compute_loss(input_tensor=input_tensor,
binary_label=instance_label_tensor,
existence_label=existence_label_tensor,
name='lanenet_loss')
total_loss = compute_ret['total_loss']
instance_loss = compute_ret['instance_seg_loss']
existence_loss = compute_ret['existence_pre_loss']
existence_logits = compute_ret['existence_logits']
# calculate the accuracy
out_logits = compute_ret['instance_seg_logits']
out_logits_ref = out_logits
out_logits = tf.nn.softmax(logits=out_logits)
out_logits_out = tf.argmax(out_logits, axis=-1) # 8 x 288 x 800
pred_0 = tf.count_nonzero(
tf.multiply(tf.cast(tf.equal(instance_label_tensor, 0), tf.int64),
tf.cast(tf.equal(out_logits_out, 0), tf.int64)))
pred_1 = tf.count_nonzero(
tf.multiply(tf.cast(tf.equal(instance_label_tensor, 1), tf.int64),
tf.cast(tf.equal(out_logits_out, 1), tf.int64)))
pred_2 = tf.count_nonzero(
tf.multiply(tf.cast(tf.equal(instance_label_tensor, 2), tf.int64),
tf.cast(tf.equal(out_logits_out, 2), tf.int64)))
pred_3 = tf.count_nonzero(
tf.multiply(tf.cast(tf.equal(instance_label_tensor, 3), tf.int64),
tf.cast(tf.equal(out_logits_out, 3), tf.int64)))
pred_4 = tf.count_nonzero(
tf.multiply(tf.cast(tf.equal(instance_label_tensor, 4), tf.int64),
tf.cast(tf.equal(out_logits_out, 4), tf.int64)))
gt_all = tf.count_nonzero(
tf.cast(tf.greater(instance_label_tensor, 0), tf.int64))
gt_back = tf.count_nonzero(
tf.cast(tf.equal(instance_label_tensor, 0), tf.int64))
pred_all = tf.add(tf.add(tf.add(pred_1, pred_2), pred_3), pred_4)
accuracy = tf.divide(pred_all, gt_all)
accuracy_back = tf.divide(pred_0, gt_back)
# Compute mIoU of Lanes
overlap_1 = pred_1
union_1 = tf.add(
tf.count_nonzero(tf.cast(tf.equal(instance_label_tensor, 1),
tf.int64)),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 1), tf.int64)))
union_1 = tf.subtract(union_1, overlap_1)
IoU_1 = tf.divide(overlap_1, union_1)
overlap_2 = pred_2
union_2 = tf.add(
tf.count_nonzero(tf.cast(tf.equal(instance_label_tensor, 2),
tf.int64)),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 2), tf.int64)))
union_2 = tf.subtract(union_2, overlap_2)
IoU_2 = tf.divide(overlap_2, union_2)
overlap_3 = pred_3
union_3 = tf.add(
tf.count_nonzero(tf.cast(tf.equal(instance_label_tensor, 3),
tf.int64)),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 3), tf.int64)))
union_3 = tf.subtract(union_3, overlap_3)
IoU_3 = tf.divide(overlap_3, union_3)
overlap_4 = pred_4
union_4 = tf.add(
tf.count_nonzero(tf.cast(tf.equal(instance_label_tensor, 4),
tf.int64)),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 4), tf.int64)))
union_4 = tf.subtract(union_4, overlap_4)
IoU_4 = tf.divide(overlap_4, union_4)
IoU = tf.reduce_mean(tf.stack([IoU_1, IoU_2, IoU_3, IoU_4]))
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.polynomial_decay(CFG.TRAIN.LEARNING_RATE,
global_step,
90100,
power=0.9)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.MomentumOptimizer(
learning_rate=learning_rate,
momentum=0.9).minimize(loss=total_loss,
var_list=tf.trainable_variables(),
global_step=global_step)
# Set tf saver
saver = tf.train.Saver()
model_save_dir = 'model/culane_lanenet/culane_scnn'
if not ops.exists(model_save_dir):
os.makedirs(model_save_dir)
train_start_time = time.strftime('%Y-%m-%d-%H-%M-%S',
time.localtime(time.time()))
model_name = 'culane_lanenet_{:s}_{:s}.ckpt'.format(
net_flag, str(train_start_time))
model_save_path = ops.join(model_save_dir, model_name)
# Set sess configuration
sess_config = tf.ConfigProto(device_count={'GPU':
4}) # device_count={'GPU': 1}
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
# Set the training parameters
train_epochs = CFG.TRAIN.EPOCHS
log.info('Global configuration is as follows:')
log.info(CFG)
with sess.as_default():
if weights_path is None:
log.info('Training from scratch')
init = tf.global_variables_initializer()
sess.run(init)
else:
log.info('Restore model from last model checkpoint {:s}'.format(
weights_path))
saver.restore(sess=sess, save_path=weights_path)
# 加载预训练参数
if net_flag == 'vgg' and weights_path is None:
pretrained_weights = np.load('./data/vgg16.npy',
encoding='latin1').item()
for vv in tf.trainable_variables():
weights_key = vv.name.split('/')[-3]
try:
weights = pretrained_weights[weights_key][0]
_op = tf.assign(vv, weights)
sess.run(_op)
except Exception as e:
continue
train_cost_time_mean = []
train_instance_loss_mean = []
train_existence_loss_mean = []
train_accuracy_mean = []
train_accuracy_back_mean = []
val_cost_time_mean = []
val_instance_loss_mean = []
val_existence_loss_mean = []
val_accuracy_mean = []
val_accuracy_back_mean = []
val_IoU_mean = []
for epoch in range(train_epochs):
# training part
t_start = time.time()
gt_imgs, instance_gt_labels, existence_gt_labels = train_dataset.next_batch(
CFG.TRAIN.BATCH_SIZE)
gt_imgs = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_CUBIC) for tmp in gt_imgs
]
gt_imgs = [(tmp - VGG_MEAN) for tmp in gt_imgs]
instance_gt_labels = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_NEAREST)
for tmp in instance_gt_labels
]
phase_train = 'train'
_, c, train_accuracy, train_accuracy_back, train_instance_loss, train_existence_loss, binary_seg_img = \
sess.run([optimizer, total_loss,
accuracy, accuracy_back,
instance_loss,
existence_loss,
out_logits_out],
feed_dict={input_tensor: gt_imgs,
instance_label_tensor: instance_gt_labels,
existence_label_tensor: existence_gt_labels,
phase: phase_train})
cost_time = time.time() - t_start
train_cost_time_mean.append(cost_time)
train_instance_loss_mean.append(train_instance_loss)
train_existence_loss_mean.append(train_existence_loss)
train_accuracy_mean.append(train_accuracy)
train_accuracy_back_mean.append(train_accuracy_back)
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
print(
'Epoch: {:d} loss_ins= {:6f} ({:6f}) loss_ext= {:6f} ({:6f}) accuracy= {:6f} ({:6f}) accuracy_back= {:6f} ({:6f})'
' mean_time= {:5f}s '.format(
epoch + 1, train_instance_loss,
np.mean(train_instance_loss_mean),
train_existence_loss,
np.mean(train_existence_loss_mean), train_accuracy,
np.mean(train_accuracy_mean), train_accuracy_back,
np.mean(train_accuracy_back_mean),
np.mean(train_cost_time_mean)))
if epoch % 500 == 0:
train_cost_time_mean.clear()
train_instance_loss_mean.clear()
train_existence_loss_mean.clear()
train_accuracy_mean.clear()
train_accuracy_back_mean.clear()
if epoch % 1000 == 0:
saver.save(sess=sess,
save_path=model_save_path,
global_step=epoch)
if epoch % 10000 != 0 or epoch == 0:
continue
for epoch_val in range(int(9675 / 8.0)):
# validation part
gt_imgs_val, instance_gt_labels_val, existence_gt_labels_val \
= val_dataset.next_batch(CFG.TRAIN.VAL_BATCH_SIZE)
gt_imgs_val = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH,
CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_CUBIC)
for tmp in gt_imgs_val
]
gt_imgs_val = [(tmp - VGG_MEAN) for tmp in gt_imgs_val]
instance_gt_labels_val = [
cv2.resize(tmp,
dsize=(CFG.TRAIN.IMG_WIDTH,
CFG.TRAIN.IMG_HEIGHT),
dst=tmp,
interpolation=cv2.INTER_NEAREST)
for tmp in instance_gt_labels_val
]
phase_val = 'test'
t_start_val = time.time()
c_val, val_accuracy, val_accuracy_back, val_IoU, val_instance_loss, val_existence_loss = \
sess.run([total_loss, accuracy, accuracy_back, IoU, instance_loss, existence_loss],
feed_dict={input_tensor: gt_imgs_val,
instance_label_tensor: instance_gt_labels_val,
existence_label_tensor: existence_gt_labels_val,
phase: phase_val})
cost_time_val = time.time() - t_start_val
val_cost_time_mean.append(cost_time_val)
val_instance_loss_mean.append(val_instance_loss)
val_existence_loss_mean.append(val_existence_loss)
val_accuracy_mean.append(val_accuracy)
val_accuracy_back_mean.append(val_accuracy_back)
val_IoU_mean.append(val_IoU)
if epoch_val % 1 == 0:
print(
'Epoch_Val: {:d} loss_ins= {:6f} ({:6f}) '
'loss_ext= {:6f} ({:6f}) accuracy= {:6f} ({:6f}) accuracy_back= {:6f} ({:6f}) mIoU= {:6f} ({:6f})'
'mean_time= {:5f}s '.format(
epoch_val + 1, val_instance_loss,
np.mean(val_instance_loss_mean),
val_existence_loss,
np.mean(val_existence_loss_mean), val_accuracy,
np.mean(val_accuracy_mean), val_accuracy_back,
np.mean(val_accuracy_back_mean), val_IoU,
np.mean(val_IoU_mean),
np.mean(val_cost_time_mean)))
val_cost_time_mean.clear()
val_instance_loss_mean.clear()
val_existence_loss_mean.clear()
val_accuracy_mean.clear()
val_accuracy_back_mean.clear()
val_IoU_mean.clear()
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
log.info('开始读取图像数据并进行预处理')
t_start = time.time()
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256), interpolation=cv2.INTER_LINEAR)
image = image - VGG_MEAN
log.info('图像读取完毕, 耗时: {:.5f}s'.format(time.time() - t_start))
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant(False, tf.bool)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='enet')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor,
name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
for i in range(1):
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
# 删除一些比较小的联通区域
# binary_seg_image[0] = postprocessor.postprocess(binary_seg_image[0])
t_start = time.time()
mask_image, _, _, _ = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
t_cost = time.time() - t_start
log.info('单张图像车道线聚类耗时: {:.5f}s'.format(t_cost))
print(instance_seg_image.shape)
for i in range(4):
instance_seg_image[0][:, :,
i] = minmax_scale(instance_seg_image[0][:, :,
i])
embedding_image = np.array(instance_seg_image[0], np.uint8)
cv2.imwrite('./out/out_bin_img.png', binary_seg_image[0] * 255)
cv2.imwrite('./out/out_mask_img.png', mask_image)
cv2.imwrite('./out/out_ori_img.png', image_vis)
cv2.imwrite('./out/out_ins_img.png', embedding_image)
sess.close()
return
def test_lanenet(image_path, weights_path, use_gpu):
"""
:param image_path:一张待测试的图片路径
:param weights_path:训练好的权重路径
:param use_gpu:是否使用gpu
:return:
"""
assert ops.exists(image_path), '{:s} not exist'.format(image_path)
log.info('开始读取图像数据并进行预处理')
t_start = time.time()
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256), interpolation=cv2.INTER_LINEAR)# 使用线性插值
image = image - VGG_MEAN# 三通道减去均值
log.info('图像读取完毕, 耗时: {:.5f}s'.format(time.time() - t_start))
input_tensor = tf.placeholder(dtype=tf.float32, shape=[1, 256, 512, 3], name='input_tensor')
phase_tensor = tf.constant('test', tf.string)# 初始化为测试
# 实例化主干网络
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
binary_seg_ret, instance_seg_ret = net.inference(input_tensor=input_tensor, name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
# 加载训练好的权重
saver.restore(sess=sess, save_path=weights_path)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run([binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
binary_seg_image[0] = postprocessor.postprocess(binary_seg_image[0])
mask_image = cluster.get_lane_mask(binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
for i in range(4):
# 获取矩阵的最大值和最小值并归一化到[0,255]
instance_seg_image[0][:, :, i] = minmax_scale(instance_seg_image[0][:, :, i])
embedding_image = np.array(instance_seg_image[0], np.uint8)# 转换成图片的显示格式uint8(1, 256, 512, 4)
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis[:, :, (2, 1, 0)])
plt.figure('instance_image')
plt.imshow(embedding_image[:, :, (2, 1, 0)])# (256, 512, 3)
plt.figure('binary_image')
plt.imshow(binary_seg_image[0] * 255, cmap='gray')
plt.show()
sess.close()
return
def test_lanenet(video_path, weights_path, use_gpu, output_path=''):
#def detect_video(yolo, output_path=""):
import cv2
from timeit import default_timer as timer
from PIL import Image, ImageFont, ImageDraw
print(video_path)
vid = cv2.VideoCapture(video_path)
#vid = cv2.VideoCapture(0)
if not vid.isOpened():
raise IOError("Couldn't open webcam or video")
video_FourCC = int(vid.get(cv2.CAP_PROP_FOURCC))
video_fps = vid.get(cv2.CAP_PROP_FPS)
video_size = (int(vid.get(cv2.CAP_PROP_FRAME_WIDTH)),
int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT)))
isOutput = True if output_path != "" else False
if isOutput:
print("!!! TYPE:", type(output_path), type(video_FourCC),
type(video_fps), type(video_size))
out = cv2.VideoWriter(output_path, video_FourCC, video_fps, video_size)
accum_time = 0
curr_fps = 0
fps = "FPS: ??"
prev_time = timer()
while True:
return_value, frame = vid.read()
tf.reset_default_graph()
image = Image.fromarray(frame)
image = cv2.cvtColor(np.asarray(image), cv2.COLOR_RGB2BGR)
log.info('开始读取图像数据并进行预处理')
t_start = time.time()
#image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image_vis = image
image = cv2.resize(image, (512, 256), interpolation=cv2.INTER_LINEAR)
image = image - VGG_MEAN
log.info('图像读取完毕, 耗时: {:.5f}s'.format(time.time() - t_start))
input_tensor = tf.placeholder(dtype=tf.float32,
shape=[1, 256, 512, 3],
name='input_tensor')
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet(phase=phase_tensor, net_flag='vgg')
#tf.reset_default_graph() # zj添加 参考网址：https://blog.csdn.net/mr_brooks/article/details/80393396
binary_seg_ret, instance_seg_ret = net.inference(
input_tensor=input_tensor, name='lanenet_model')
cluster = lanenet_cluster.LaneNetCluster()
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
saver = tf.train.Saver()
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'CPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
t_start = time.time()
binary_seg_image, instance_seg_image = sess.run(
[binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: [image]})
t_cost = time.time() - t_start
log.info('单张图像车道线预测耗时: {:.5f}s'.format(t_cost))
binary_seg_image[0] = postprocessor.postprocess(
binary_seg_image[0])
mask_image = cluster.get_lane_mask(
binary_seg_ret=binary_seg_image[0],
instance_seg_ret=instance_seg_image[0])
for i in range(4):
instance_seg_image[0][:, :, i] = minmax_scale(
instance_seg_image[0][:, :, i])
embedding_image = np.array(instance_seg_image[0], np.uint8)
'''
plt.figure('mask_image')
plt.imshow(mask_image[:, :, (2, 1, 0)])
plt.figure('src_image')
plt.imshow(image_vis[:, :, (2, 1, 0)])
plt.figure('instance_image')
plt.imshow(embedding_image[:, :, (2, 1, 0)])
plt.figure('binary_image')
plt.imshow(binary_seg_image[0] * 255, cmap='gray')
plt.show()
'''
result = np.asarray(embedding_image[:, :, (2, 1, 0)])
curr_time = timer()
exec_time = curr_time - prev_time
prev_time = curr_time
accum_time = accum_time + exec_time
curr_fps = curr_fps + 1
if accum_time > 1:
accum_time = accum_time - 1
fps = "FPS: " + str(curr_fps)
curr_fps = 0
cv2.putText(result,
text=fps,
org=(3, 15),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=0.50,
color=(255, 0, 0),
thickness=2)
cv2.namedWindow("result", cv2.WINDOW_NORMAL)
cv2.imshow("result", result)
if isOutput:
out.write(result)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
sess.close()
return