def run(self, image_path):
# 读入图片数据
img, filename = self._load_img(image_path)
# 输出预测的结果
predictions_op = self._init_net(img=img)
sess = tf.Session(config=self.config)
sess.run(tf.global_variables_initializer())
# 加载模型
ckpt = tf.train.get_checkpoint_state(self.log_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=tf.global_variables())
loader.restore(sess, ckpt.model_checkpoint_path)
Tools.print_info("Restored model parameters from {}".format(
ckpt.model_checkpoint_path))
else:
Tools.print_info('No checkpoint file found.')
# 运行
predictions = sess.run(predictions_op)
msk = decode_labels(predictions, num_classes=self.num_classes)
im = Image.fromarray(msk[0])
im.save(os.path.join(self.save_dir, filename))
Tools.print_info('over : result save in {}'.format(
os.path.join(self.save_dir, filename)))
pass
def main():
args = get_arguments()
if args.img_path[-4] != '.':
files = GetAllFilesListRecusive(args.img_path,
['.jpg', '.jpeg', '.png', '.bmp'])
else:
files = [args.img_path]
shape = INPUT_SIZE.split(',')
shape = (int(shape[0]), int(shape[1]), 3)
if args.pb_file == '':
sess, pred, x = load_from_checkpoint(shape, args.snapshots_dir,
args.model)
else:
sess, pred, x, label_colors, label_names = load_from_pb(
shape, args.pb_file)
if args.measure_time:
calculate_perfomance(sess, x, pred, shape, args.runs, args.batch_size)
quit()
for path in files:
img, filename = load_img(path)
orig_img = cv2.imread(path)
if args.pb_file != '':
img = np.expand_dims(img, axis=0)
t = time.time()
preds = sess.run(pred, feed_dict={x: img})
print('time: ', time.time() - t)
print('output shape: ', preds.shape)
msk = decode_labels(preds,
num_classes=len(label_colors),
label_colours=label_colors)
im = msk[0]
im = cv2.resize(im, (orig_img.shape[1], orig_img.shape[0]),
interpolation=cv2.INTER_NEAREST)
print('im', im.shape)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
#img = cv2.cvtColor(orig_img, cv2.COLOR_RGB2BGR)
img = orig_img
if args.weighted:
indx = (im == [0, 0, 0])
print(im.shape, img.shape)
im = cv2.addWeighted(im, 0.8, img, 0.2, 0)
im[indx] = img[indx]
cv2.imwrite(args.save_dir + filename.replace('.jpg', '.png'), im)
def main():
args = get_arguments()
if args.dataset == 'cityscapes':
num_classes = cityscapes_class
else:
num_classes = ADE20k_class
img, filename = load_img(args.img_path)
shape = img.shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
model = model_config[args.model]
net = model({'data': img_tf},
num_classes=num_classes,
filter_scale=args.filter_scale)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=n_shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, shape, num_classes)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
model_path = model_paths[args.model]
if args.model == 'others':
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=tf.global_variables())
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
else:
net.load(model_path, sess)
print('Restore from {}'.format(model_path))
preds = sess.run(pred, feed_dict={x: img})
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
misc.imsave(args.save_dir + filename, preds[0])
def __init__(self):
self.shape = IMG_SHAPE
self.num_classes = sun_class
self.show_classes = num_class
output_graph_path = PB_PATH
# with tf.Graph().as_default():
output_graph_def = tf.GraphDef()
with open(output_graph_path, 'rb') as f:
output_graph_def.ParseFromString(f.read())
_ = tf.import_graph_def(output_graph_def, name="")
self.sess = tf.Session()
tf.initialize_all_variables().run(session=self.sess)
self.img_ph = self.sess.graph.get_tensor_by_name("Placeholder:0")
raw_output = self.sess.graph.get_tensor_by_name("final_out:0")
raw_output_up = tf.image.resize_bilinear(raw_output,
size=self.shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
self.shape[0],
self.shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
self.pred = decode_labels(raw_output_up, self.shape, self.show_classes,
self.num_classes)
def main():
args = get_arguments()
num_classes = sun_class
show_classes = args.num_class
output_graph_path = args.pb_path
imgs = []
filenames = []
if os.path.isdir(args.img_path):
file_paths = glob.glob(os.path.join(args.img_path, '*'))
for file_path in file_paths:
ext = file_path.split('.')[-1].lower()
if ext == 'png' or ext == 'jpg':
img, filename = load_img(file_path)
imgs.append(img)
filenames.append(filename)
else:
img, filename = load_img(args.img_path)
img = preprocess(img)
imgs.append(img)
filenames.append(filename)
shape = imgs[0].shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
x, n_shape = check_input(x)
with tf.Graph().as_default():
output_graph_def = tf.GraphDef()
with open(output_graph_path, 'rb') as f:
output_graph_def.ParseFromString(f.read())
_ = tf.import_graph_def(output_graph_def, name="")
with tf.Session() as sess:
tf.initialize_all_variables().run()
img = sess.graph.get_tensor_by_name("Placeholder:0")
raw_output = sess.graph.get_tensor_by_name("final_out:0")
raw_output_up = tf.image.resize_bilinear(raw_output,
size=n_shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(
raw_output_up, 0, 0, shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, shape, show_classes,
num_classes)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
for i in trange(len(imgs), desc='Inference', leave=True):
start_time = timeit.default_timer()
raw_output_up_temp = sess.run(raw_output,
feed_dict={img: [imgs[i]]})
preds = sess.run(pred, feed_dict={img: [imgs[i]]})
misc.imsave(args.save_dir + filenames[i], preds[0])
def main():
args = get_arguments()
img, filename = load_img(args.img_path)
shape = img.shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
# Create network.
net = ICNet_BN({'data': img_tf}, num_classes=num_classes)
# Predictions.
raw_output = net.layers['conv6_cls']
output = tf.image.resize_bilinear(raw_output, tf.shape(img_tf)[1:3, ])
output = tf.argmax(output, dimension=3)
pred = tf.expand_dims(output, dim=3)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
ckpt = tf.train.get_checkpoint_state(args.snapshots_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
preds = sess.run(pred, feed_dict={x: img})
# print(preds.shape)
# s = preds.flatten()
# print(set(s))
# print((s == 0).sum())
# print((s == 1).sum())
# print((s == 2).sum())
msk = decode_labels(preds, num_classes=num_classes)
im = Image.fromarray(msk[0])
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im.save(args.save_dir + filename.replace('.jpg', '.png'))
def main():
args = get_arguments()
img, filename = load_img(args.img_path)
shape = img.shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
# Create network.
net = ICNet({'data': img_tf}, num_classes=num_classes)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True)
raw_output_up = tf.squeeze(raw_output_up, axis=0)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, dimension=2)
pred = tf.expand_dims(raw_output_up, dim=2)
pred = tf.expand_dims(pred, dim=0)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
if args.model == 'train':
print('Restore from train30k model...')
net.load(model_train30k, sess)
elif args.model == 'trainval':
print('Restore from trainval90k model...')
net.load(model_trainval90k, sess)
for i in range(50):
start = time.time()
preds = sess.run(pred, feed_dict={x: img})
end = time.time()
print('Inf time = %f' % (end - start))
msk = decode_labels(preds, num_classes=num_classes)
im = Image.fromarray(msk[0])
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im.save(args.save_dir + filename)
def _load_model(self, shape):
self.x = tf.placeholder(dtype=tf.float32, shape=shape)
img_tf = self.preprocess(self.x)
self.img_tf, self.n_shape = self.check_input(img_tf)
self.model = model_config[self.model_name]
print(self.model)
net = self.model({'data': img_tf},
num_classes=self.num_classes,
filter_scale=self.filter_scale)
# net = self.model({'data': img_tf}, num_classes=self.num_classes)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=self.n_shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
self.pred = decode_labels(raw_output_up, shape, self.num_classes)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
init = tf.global_variables_initializer()
self.sess.run(init)
# model_path = model_paths[self.model_name]
model_path = snapshot_dir
if self.model_name == 'others':
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=tf.global_variables())
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
self.load(loader, self.sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
else:
# model path must be a model
net.load(self.model_path, self.sess)
print('Restore from {}'.format(model_path))
def main():
args = get_arguments()
img, filename = load_img(args.img_path)
img = preprocess(img)
# Create network.
net = ICNet({'data': img}, num_classes=num_classes)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=input_size,
align_corners=True)
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
if args.model == 'train':
print('Restore from train30k model...')
net.load(model_train30k, sess)
elif args.model == 'trainval':
print('Restore from trainval90k model...')
net.load(model_trainval90k, sess)
preds = sess.run(pred)
msk = decode_labels(preds, num_classes=num_classes)
im = Image.fromarray(msk[0])
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im.save(args.save_dir + filename)
def predict(self):
self.predict_setup()
self.sess.run(tf.global_variables_initializer())
self.sess.run(tf.local_variables_initializer())
checkpointfile = self.parameters.modeldir + '/model.ckpt-' + str(
self.parameters.valid_step)
self.load(self.loader, checkpointfile)
threads = tf.train.start_queue_runners(coord=self.coord,
sess=self.sess)
image_list, _ = read_labeled_image_list('',
self.parameters.test_data_list)
for step in range(self.parameters.test_num_steps):
preds = self.sess.run(self.pred)
img_name = image_list[step].split('/')[2].split('.')[0]
im = Image.fromarray(preds[0, :, :, 0], mode='L')
filename = '/%s_mask.png' % (img_name)
im.save(self.parameters.out_dir + '/prediction' + filename)
if self.parameters.visual:
msk = decode_labels(preds,
num_classes=self.parameters.num_classes)
im = Image.fromarray(msk[0], mode='RGB')
filename = '/%s_mask_visual.png' % (img_name)
im.save(self.parameters.out_dir + '/visual_prediction' +
filename)
if step % 100 == 0:
print('step {:d}'.format(step))
print('The output files has been saved to {}'.format(
self.parameters.out_dir))
self.coord.request_stop()
self.coord.join(threads)
def evaluate(config, evaluation_set='val', plot_confusionMatrix=False):
# --------------------------------------------------------------------
# init network
# --------------------------------------------------------------------
tf.compat.v1.reset_default_graph()
# define input placeholders
input_placeholder = {}
input_placeholder.update(
{'is_training': tf.compat.v1.placeholder(dtype=tf.bool, shape=())})
if config.ARCHITECTURE == 'semantic_segmentation':
batch_size = config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH
# Search for available GPUs: the result is a list of device ids like `['/gpu:0', '/gpu:1']`
devices = get_available_gpus()
print("found devices: ", devices)
num_GPU = len(devices)
if (num_GPU) == 0:
num_GPU = 1 # CPU support!
# min 1 sample should be applied on a GPU
if (config.BATCH_SIZE < num_GPU):
num_GPU = config.BATCH_SIZE
image_placeholder = []
label_placeholder = []
for iter in range(num_GPU):
if (iter == (num_GPU - 1)):
batch_size_local = batch_size - (num_GPU - 1) * (batch_size //
num_GPU)
else:
batch_size_local = batch_size // num_GPU
print('batch_size /gpu:{} : {}'.format(iter, num_GPU))
image_placeholder.append(
tf.compat.v1.placeholder(
dtype=tf.float32,
shape=(batch_size_local,
config.DATASET_TRAIN.INPUT_SIZE[0],
config.DATASET_TRAIN.INPUT_SIZE[1],
config.DATASET_TRAIN.NUM_CHANNELS)))
label_placeholder.append(
tf.compat.v1.placeholder(
dtype=tf.float32,
shape=(batch_size_local,
config.DATASET_TRAIN.INPUT_SIZE[0],
config.DATASET_TRAIN.INPUT_SIZE[1], 1)))
input_placeholder.update({'image_batch': image_placeholder})
input_placeholder.update({'label_batch': label_placeholder})
else:
print(
'[ERROR] network architecture does not exist!!! Please check your spelling!'
)
raise NotImplementedError
# load network architecture
if config.ARCHITECTURE == 'semantic_segmentation':
model = get_model(config.MODEL)
net = model(
{
'data': input_placeholder['image_batch'],
'is_training': input_placeholder['is_training']
},
is_training=input_placeholder['is_training'],
evaluation=tf.logical_not(input_placeholder['is_training']),
#is_inference=True,
num_classes=config.DATASET_TRAIN.NUM_CLASSES,
filter_scale=config.FILTER_SCALE,
timeSequence=config.TIMESEQUENCE_LENGTH,
variant=config.MODEL_VARIANT)
else:
print(
'[ERROR] network architecture does not exist!!! Please check your spelling!'
)
raise NotImplementedError
# --------------------------------------------------------------------
# determine evaluation metric
# --------------------------------------------------------------------
if config.ARCHITECTURE == 'semantic_segmentation':
list_raw_gt = []
list_pred_flattern_mIoU = []
for iter_gpu in range(len(input_placeholder['image_batch'])):
with tf.device('/gpu:%d' % iter_gpu):
if config.MODEL == 'SegNet_BN' or config.MODEL == 'SegNet_BN_encoder' or config.MODEL == 'SegNet_BN_decoder' or config.MODEL == 'SegNet_BN_encoderDecoder':
raw_output = net.layers['output'][iter_gpu]
raw_output_up = tf.argmax(raw_output,
axis=3,
output_type=tf.int32)
raw_pred_mIoU = tf.expand_dims(raw_output_up, dim=3)
else: # ICNet
ori_shape = config.DATASET_TRAIN.INPUT_SIZE #??
raw_output = net.layers['output'][iter_gpu]
raw_output_up = tf.compat.v1.image.resize_bilinear(
raw_output, size=ori_shape[:2], align_corners=True)
raw_output_up = tf.argmax(raw_output_up,
axis=3,
output_type=tf.int32)
raw_pred_mIoU = tf.expand_dims(raw_output_up, dim=3)
# determine mIoU
if config.USAGE_TIMESEQUENCES: # evaluate only last image of time sequence
pred_of_interest = np.array(
range(config.BATCH_SIZE), dtype=np.int32
) * config.TIMESEQUENCE_LENGTH + config.TIMESEQUENCE_LENGTH - 1
pred_flatten_mIoU = tf.reshape(
tf.gather(raw_pred_mIoU, pred_of_interest), [
-1,
])
raw_gt = tf.reshape(
tf.gather(input_placeholder['label_batch'][iter_gpu],
pred_of_interest), [
-1,
])
else: # evaluate all images of batch size
pred_flatten_mIoU = tf.reshape(raw_pred_mIoU, [
-1,
])
raw_gt = tf.reshape(
input_placeholder['label_batch'][iter_gpu], [
-1,
])
list_raw_gt.append(raw_gt)
list_pred_flattern_mIoU.append(pred_flatten_mIoU)
# combine output of different GPUs
with tf.device('/gpu:%d' % 0):
all_raw_gt = tf.reshape(tf.concat(list_raw_gt, -1), [
-1,
])
all_pred_flatten_mIoU = tf.reshape(
tf.concat(list_pred_flattern_mIoU, -1), [
-1,
])
indices_mIoU = tf.squeeze(
tf.where(
tf.less_equal(raw_gt,
config.DATASET_TRAIN.NUM_CLASSES - 1)), 1)
gt_mIoU = tf.cast(tf.gather(raw_gt, indices_mIoU), tf.int32)
pred_mIoU = tf.gather(pred_flatten_mIoU, indices_mIoU)
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(
pred_mIoU, gt_mIoU, num_classes=config.DATASET_VAL.NUM_CLASSES)
# create colored image
pred_color = decode_labels(pred_flatten_mIoU,
config.DATASET_VAL.INPUT_SIZE[0:2],
config.DATASET_VAL.NUM_CLASSES)
# deterimine confusing matrix
if plot_confusionMatrix:
# Create an accumulator variable to hold the counts
confusion = tf.Variable(tf.zeros([
config.DATASET_VAL.NUM_CLASSES, config.DATASET_VAL.NUM_CLASSES
],
dtype=tf.int64),
name='confusion',
collections=[
tf.compat.v1.GraphKeys.LOCAL_VARIABLES
])
# Compute a per-batch confusion
batch_confusion = tf.math.confusion_matrix(
tf.reshape(gt_mIoU, [-1]),
tf.reshape(pred_mIoU, [-1]),
num_classes=config.DATASET_VAL.NUM_CLASSES,
name='batch_confusion')
# Create the update op for doing a "+=" accumulation on the batch
confusion_update = confusion.assign(
confusion + tf.cast(batch_confusion, dtype=tf.int64))
# -----------------------------------------
# init session
# -----------------------------------------
# Set up tf session and initialize variables.
sessConfig = tf.compat.v1.ConfigProto()
sessConfig.gpu_options.allow_growth = True
# use only a fraction of gpu memory, otherwise the TensorRT-test has not enough free GPU memory for execution
sessConfig.gpu_options.per_process_gpu_memory_fraction = 0.5
sess = tf.compat.v1.Session(config=sessConfig)
init = tf.compat.v1.global_variables_initializer()
local_init = tf.compat.v1.local_variables_initializer()
sess.run(init)
sess.run(local_init)
# load checkpoint file
print(config.EVALUATION.MODELPATH)
ckpt = tf.compat.v1.train.get_checkpoint_state(config.EVALUATION.MODELPATH)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.compat.v1.train.Saver(
var_list=tf.compat.v1.global_variables())
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
# `sess.graph` provides access to the graph used in a <a href="./../api_docs/python/tf/Session"><code>tf.Session</code></a>.
writer = tf.compat.v1.summary.FileWriter("/tmp/tensorflow_graph",
tf.compat.v1.get_default_graph())
# --------------------------------------------------------------------
# Evaluate - Iterate over training steps.
# --------------------------------------------------------------------
# evaluate training or validation set
if evaluation_set == "val":
imagereader_val = ImageReader(config.IMAGEREADER.VAL,
config.DATASET_VAL, config.BATCH_SIZE,
config.TIMESEQUENCE_LENGTH)
elif evaluation_set == "train":
imagereader_val = ImageReader(config.IMAGEREADER.VAL,
config.DATASET_TRAIN, config.BATCH_SIZE,
config.TIMESEQUENCE_LENGTH)
elif evaluation_set == "test":
imagereader_val = ImageReader(config.IMAGEREADER.VAL,
config.DATASET_TEST, config.BATCH_SIZE,
config.TIMESEQUENCE_LENGTH)
elif evaluation_set == "all":
imagereader_val = ImageReader(config.IMAGEREADER.VAL,
config.DATASET_ALL, config.BATCH_SIZE,
config.TIMESEQUENCE_LENGTH)
else:
print("Dataset {} does not exist!".format(evaluation_set))
filename_memory = ""
filename_count = 0
average_inference_time = 0
# --------------------------------------
# perform evaluation - semantic segmentation
# --------------------------------------
if config.ARCHITECTURE == 'semantic_segmentation':
if config.TIMESEQUENCES_SLIDINGWINDOW: # use time sequences
for step in trange(
int(imagereader_val._dataset_amount -
config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH + 1),
desc='inference',
leave=True):
#start_time = time.time()
training_batch = imagereader_val.getNextMinibatch()
feed_dict = {input_placeholder['is_training']: False}
for iter_GPU in range(len(input_placeholder['image_batch'])):
num_GPU = len(input_placeholder['image_batch'])
batch_size = training_batch['blob_data'].shape[0]
batch_size_local = batch_size // num_GPU
if (iter_GPU == (num_GPU - 1)):
batch_size_act = batch_size - (num_GPU - 1) * (
batch_size // num_GPU)
else:
batch_size_act = batch_size // num_GPU
feed_dict.update({
input_placeholder['image_batch'][iter_GPU]:
training_batch['blob_data'][iter_GPU *
batch_size_local:iter_GPU *
batch_size_local +
batch_size_act, :, :, :],
input_placeholder['label_batch'][iter_GPU]:
training_batch['blob_label']
[iter_GPU *
batch_size_local:iter_GPU * batch_size_local +
batch_size_act, :, :, :]
})
start_time = time.time()
if plot_confusionMatrix:
sess.run([update_op, confusion_update],
feed_dict=feed_dict)
else:
sess.run([update_op], feed_dict=feed_dict)
duration = time.time() - start_time
average_inference_time += duration
# save image
prediction = sess.run([pred_color], feed_dict=feed_dict)
predi = np.array(
prediction[0][0, :, :, :]).astype(dtype=np.uint8)
img = cv2.imread(imagereader_val._dataset[step][0])
if imagereader_val._configDataset.USE_IMAGE_ROI:
img = img[imagereader_val._configDataset.IMAGE_ROI_MIN_X:
imagereader_val._configDataset.IMAGE_ROI_MAX_X,
imagereader_val._configDataset.
IMAGE_ROI_MIN_Y:imagereader_val._configDataset.
IMAGE_ROI_MAX_Y, :]
cv2.addWeighted(predi, config.INFERENCE.OVERLAPPING_IMAGE, img,
1 - config.INFERENCE.OVERLAPPING_IMAGE, 0, img)
buff = imagereader_val._dataset[step][-1]
buff = buff.split('/')
filename = buff[-1].split('.')[0]
if filename_memory == buff[-1].split('.')[0]:
filename_count += 1
filename = buff[-1].split('.')[0] + "_" + str(
filename_count) + ".png"
else:
filename_memory = buff[-1].split('.')[0]
filename = buff[-1].split('.')[0] + "_0.png"
filename_count = 0
cv2.imwrite(config.INFERENCE.SAVEDIR_IMAGES + filename,
predi[:, :, (2, 1, 0)])
cv2.imwrite(config.INFERENCE.SAVEDIR_IMAGES + filename,
img[:, :, (2, 1, 0)])
# determine average time
average_inference_time = average_inference_time / int(
imagereader_val._dataset_amount -
config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH + 1)
else: # do not use time sequences (normal evaluation)
# TODO parameters for flickering evaluation
flickering_sum = 0
flickering_img_size = 0
flickering_sum1 = 0
flickering_img_size1 = 0
for step in trange(
int(imagereader_val._dataset_amount /
(config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH)),
desc='inference',
leave=True):
training_batch = imagereader_val.getNextMinibatch()
feed_dict = {input_placeholder['is_training']: False}
for iter_GPU in range(len(input_placeholder['image_batch'])):
num_GPU = len(input_placeholder['image_batch'])
batch_size = training_batch['blob_data'].shape[0]
batch_size_local = batch_size // num_GPU
if (iter_GPU == (num_GPU - 1)):
batch_size_act = batch_size - (num_GPU - 1) * (
batch_size // num_GPU)
else:
batch_size_act = batch_size // num_GPU
# for depth images, if result should be shown in camera image
if True:
feed_dict.update({
input_placeholder['image_batch'][iter_GPU]:
training_batch['blob_data']
[iter_GPU *
batch_size_local:iter_GPU * batch_size_local +
batch_size_act, :, :, :],
input_placeholder['label_batch'][iter_GPU]:
training_batch['blob_label']
[iter_GPU *
batch_size_local:iter_GPU * batch_size_local +
batch_size_act, :, :, :]
})
else:
training_batch['blob_data'][:, :, :,
1] = training_batch[
'blob_data'][:, :, :,
-1]
training_batch['blob_data'][:, :, :,
2] = training_batch[
'blob_data'][:, :, :,
-1]
feed_dict.update({
input_placeholder['image_batch'][iter_GPU]:
training_batch['blob_data']
[iter_GPU *
batch_size_local:iter_GPU * batch_size_local +
batch_size_act, :, :, 1:4],
input_placeholder['label_batch'][iter_GPU]:
training_batch['blob_label']
[iter_GPU *
batch_size_local:iter_GPU * batch_size_local +
batch_size_act, :, :, :]
})
start_time = time.time()
if plot_confusionMatrix:
sess.run([update_op, confusion_update],
feed_dict=feed_dict)
else:
sess.run([update_op], feed_dict=feed_dict)
duration = time.time() - start_time
# skip the first 50 images
if step >= 50:
average_inference_time += duration
# --------------------------
# save image
# --------------------------
prediction = sess.run([pred_color, raw_output_up],
feed_dict=feed_dict)
predi = np.array(
prediction[0][0, :, :, :]).astype(dtype=np.uint8)
predi_id = np.array(prediction[1][-1,
...]).astype(dtype=np.uint8)
data_index = step * config.TIMESEQUENCE_LENGTH + config.TIMESEQUENCE_LENGTH - 1
buff = imagereader_val._dataset[data_index][-1]
buff = buff.split('/')
filename = buff[-1].split('.')[0]
if filename_memory == buff[-1].split('.')[0]:
filename_count += 1
filename = buff[-1].split('.')[0] + "_" + str(
filename_count).zfill(6) + ".png"
else:
filename_memory = buff[-1].split('.')[0]
filename = buff[-1].split('.')[0] + "_000000.png"
filename_count = 0
img = cv2.imread(imagereader_val._dataset[data_index][0])
if imagereader_val._configDataset.USE_IMAGE_ROI:
img = img[imagereader_val._configDataset.IMAGE_ROI_MIN_X:
imagereader_val._configDataset.IMAGE_ROI_MAX_X,
imagereader_val._configDataset.
IMAGE_ROI_MIN_Y:imagereader_val._configDataset.
IMAGE_ROI_MAX_Y, :]
# label images for video
cv2.addWeighted(predi, config.INFERENCE.OVERLAPPING_IMAGE, img,
1 - config.INFERENCE.OVERLAPPING_IMAGE, 0, img)
cv2.imwrite(
config.INFERENCE.SAVEDIR_IMAGES + 'pred_' + filename,
predi[:, :, (2, 1, 0)])
cv2.imwrite(
config.INFERENCE.SAVEDIR_IMAGES + 'overlay_' + filename,
img[:, :, (2, 1, 0)])
# --------------------------
# flickering evaluation
# --------------------------
# to measure flickering between non-ground-truth classes, multiply it with the predicted result (add 1, since True * class 0 = 0!)
diff_img = np.array(
(predi_id != training_batch['blob_label'][-1, :, :, 0]),
np.float32) * np.array(
training_batch['blob_label'][-1, :, :, 0] + 1,
np.float32)
diff_img1 = np.array(predi_id, np.float32)
if step > 0: # skip step = 0, since there is no reference image
flickering = np.sum((diff_img != prediction_old))
flickering_sum += flickering
flickering_img_size += predi_id.shape[0] * predi_id.shape[1]
flickering1 = np.sum((diff_img1 != prediction_old1))
flickering_sum1 += flickering1
flickering_img_size1 += predi_id.shape[0] * predi_id.shape[
1]
prediction_old = diff_img
prediction_old1 = diff_img1
# determine average time
average_inference_time = average_inference_time / int(
imagereader_val._dataset_amount /
(config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH) - 50)
mIoU_value = sess.run(mIoU)
print('flickering_sum: ', flickering_sum)
print('FP: ', float(flickering_sum) / float(flickering_img_size))
print('--------------')
print('flickering_sum1: ', flickering_sum1)
print('FIP: ', float(flickering_sum1) / float(flickering_img_size1))
print('--------------')
print(
float(flickering_sum) / float(flickering_img_size),
float(flickering_sum1) / float(flickering_img_size1))
print('--------------')
if plot_confusionMatrix:
confusion_matrix = sess.run(confusion)
# print Accuracy:
np.set_printoptions(linewidth=np.inf) #150)
acc_value = float(np.sum(np.diag(confusion_matrix))) / float(
np.sum(confusion_matrix))
else:
print(
'[ERROR] network architecture does not exist!!! Please check your spelling!'
)
raise NotImplementedError
# --------------------------------------------
# create optimized pb-model
# --------------------------------------------
if config.FREEZEINFERENCEGRAPH.MODE:
output_nodes = config.FREEZEINFERENCEGRAPH.OUTPUT_NODE_NAMES.split(',')
input_nodes = config.FREEZEINFERENCEGRAPH.INPUT_NODE_NAMES.split(',')
frozen_graph_def = tf.compat.v1.graph_util.convert_variables_to_constants(
sess,
tf.compat.v1.get_default_graph().as_graph_def(), output_nodes)
# Write tensorrt graph def to pb file for use in C++
output_graph_path = config.EVALUATION.MODELPATH + '/model.pb'
with tf.io.gfile.GFile(output_graph_path, "wb") as f:
f.write(frozen_graph_def.SerializeToString())
transforms = [
'remove_nodes(op=Identity)', 'merge_duplicate_nodes',
'strip_unused_nodes', 'fold_constants(ignore_errors=true)',
'fold_batch_norms', 'remove_device'
]
optimized_graph_def = TransformGraph(frozen_graph_def, input_nodes,
output_nodes, transforms)
print('inference model saved in ', output_graph_path)
# ------------------------------------
# apply TensorRT
# ------------------------------------
if config.FREEZEINFERENCEGRAPH.MODE:
config_TensorRT = tf.ConfigProto()
#config_TensorRT.gpu_options.per_process_gpu_memory_fraction = 0.5
config_TensorRT.gpu_options.allow_growth = True
converter = trt.TrtGraphConverter(input_graph_def=optimized_graph_def,
session_config=config_TensorRT,
max_workspace_size_bytes=4000000000,
nodes_blacklist=output_nodes,
is_dynamic_op=False,
precision_mode='FP16')
converted_graph_def = converter.convert()
# Write tensorrt graph def to pb file for use in C++
output_graph_TensorRT_path = config.EVALUATION.MODELPATH + '/tensorrt_model.pb'
with tf.io.gfile.GFile(output_graph_TensorRT_path, "wb") as f:
f.write(converted_graph_def.SerializeToString())
print('TensorRT-model saved in ', output_graph_TensorRT_path)
# --------------------------------------------
# close session
# --------------------------------------------
writer.close()
sess.close()
tf.compat.v1.reset_default_graph()
# ------------------------------------------
# define inference-function for model.pb
# ------------------------------------------
def determineInferenceTime(path2frozenmodel):
# We load the protobuf file from the disk and parse it to retrieve the unserialized graph_def
with tf.gfile.GFile(path2frozenmodel, "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
for node in graph_def.node:
if node.op == 'RefSwitch':
node.op = 'Switch'
for index in xrange(len(node.input)):
if 'moving_' in node.input[index]:
node.input[index] = node.input[index] + '/read'
elif node.op == 'AssignSub':
node.op = 'Sub'
if 'use_locking' in node.attr: del node.attr['use_locking']
elif node.op == 'AssignAdd':
node.op = 'Add'
if 'use_locking' in node.attr: del node.attr['use_locking']
elif node.op == 'AssignMovingAvg':
node.op = 'MovingAvg'
if 'use_locking' in node.attr: del node.attr['use_locking']
# Then, we import the graph_def into a new Graph and returns it
with tf.Graph().as_default() as graph:
# The name var will prefix every op/nodes in your graph
# Since we load everything in a new graph, this is not needed
tf.import_graph_def(graph_def, name="")
# print output node names
if config.FREEZEINFERENCEGRAPH.PRINT_OUTPUT_NODE_NAMES:
for op in graph.get_operations():
print(str(op.name))
##### init network
if config.ARCHITECTURE == 'semantic_segmentation':
image_tensor = graph.get_tensor_by_name('Placeholder_1:0')
ouput_tensor = graph.get_tensor_by_name('ArgMax:0')
# init session # Note: we don't nee to initialize/restore anything. There is no Variables in this graph, only hardcoded constants
sess_inf = tf.compat.v1.Session(graph=graph)
average_inference_time_frozenModel = 0.0
for step in trange(
int(imagereader_val._dataset_amount /
(config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH)),
desc='inference',
leave=True):
# get images from datastet
training_batch = imagereader_val.getNextMinibatch()
if config.ARCHITECTURE == 'semantic_segmentation':
feed_dict = {image_tensor: training_batch['blob_data']}
# apply inference
start_time = time.time()
if config.ARCHITECTURE == 'semantic_segmentation':
sess_inf.run(ouput_tensor, feed_dict=feed_dict)
duration_inf = time.time() - start_time
# skip the first 50 images
if step >= 50:
average_inference_time_frozenModel += duration_inf
sess_inf.close()
average_inference_time_frozenModel = average_inference_time_frozenModel / int(
imagereader_val._dataset_amount /
(config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH) - 50)
return average_inference_time_frozenModel
# ------------------------------------------
# test model - optimized model
# ------------------------------------------
if config.FREEZEINFERENCEGRAPH.MODE:
### apply optimized model (model.pb)
path2frozenmodel_opt = config.EVALUATION.MODELPATH + '/model.pb'
average_inference_time_opt = determineInferenceTime(
path2frozenmodel_opt)
### apply TensorRT model (tensorrt_model.pb)
path2frozenmodel_tensorrt = config.EVALUATION.MODELPATH + '/tensorrt_model.pb'
average_inference_time_tensorrt = determineInferenceTime(
path2frozenmodel_tensorrt)
print('average time optimized model: {:.2f} ms'.format(
average_inference_time_opt * 1000.0))
print('average time TensorRT: {:.2f} ms'.format(
average_inference_time_tensorrt * 1000.0))
# --------------------------------------------------------------------
# Show results
# --------------------------------------------------------------------
print('average time: {:.2f} ms'.format(average_inference_time * 1000.0))
if plot_confusionMatrix and config.ARCHITECTURE == 'semantic_segmentation':
# determine class-wise IoU
buff = 0.0
print('-------------------------------------------------------------')
for iter in xrange(config.DATASET_VAL.NUM_CLASSES):
if np.sum(
confusion_matrix[iter, :]) == 0: # avoid division by zero
IoU = 0.0
else:
IoU = 100.0 * confusion_matrix[iter, iter] / (
np.sum(confusion_matrix[iter, :]) +
np.sum(confusion_matrix[:, iter]) -
confusion_matrix[iter, iter])
buff = buff + IoU
print('{}: {}'.format(config.DATASET_VAL.CLASSES[iter], IoU))
print('-------------------------------------------------------------')
print('dataset: {} - {}'.format(config.DATASET_NAME,
config.DATASET_WEATHER))
print('Accuracy: {}'.format(acc_value))
print('mIoU: {}'.format(mIoU_value))
print('average time: {:.2f} ms'.format(average_inference_time *
1000.0))
print('-------------------------------------------------------------')
if plot_confusionMatrix and config.ARCHITECTURE == 'semantic_segmentation':
print('-------------------------------------------------------------')
print(confusion_matrix)
print('-------------------------------------------------------------')
#plot_confusion_matrix(confusion_matrix, config.DATASET_VAL.CLASSES)
return mIoU_value
def main():
args = get_arguments()
img, filename = load_img(args.img_path)
shape = img.shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
# Create network.
if args.model[-2:] == 'bn':
net = ICNet_BN({'data': img_tf}, num_classes=num_classes)
elif args.model == 'others':
net = ICNet_BN({'data': img_tf}, num_classes=num_classes)
else:
net = ICNet({'data': img_tf}, num_classes=num_classes)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=n_shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, shape, num_classes)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
if args.model == 'train':
print('Restore from train30k model...')
net.load(model_train30k, sess)
elif args.model == 'trainval':
print('Restore from trainval90k model...')
net.load(model_trainval90k, sess)
elif args.model == 'train_bn':
print('Restore from train30k bnnomerge model...')
net.load(model_train30k_bn, sess)
elif args.model == 'trainval_bn':
print('Restore from trainval90k bnnomerge model...')
net.load(model_trainval90k_bn, sess)
else:
ckpt = tf.train.get_checkpoint_state(snapshot_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
preds = sess.run(pred, feed_dict={x: img})
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
misc.imsave(args.save_dir + filename, preds[0])
def main():
args = get_arguments()
if args.dataset == 'cityscapes':
num_classes = cityscapes_class
else:
num_classes = ADE20k_class
# Read images from directory (size must be the same) or single input file
imgs = []
filenames = []
if os.path.isdir(args.img_path):
file_paths = glob.glob(os.path.join(args.img_path, '*'))
for file_path in file_paths:
ext = file_path.split('.')[-1].lower()
if ext == 'png' or ext == 'jpg':
img, filename = load_img(file_path)
imgs.append(img)
filenames.append(filename)
else:
img, filename = load_img(args.img_path)
imgs.append(img)
filenames.append(filename)
shape = imgs[0].shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
model = model_config[args.model]
net = model({'data': img_tf},
num_classes=num_classes,
filter_scale=args.filter_scale)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=n_shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, shape,
num_classes) # this is the labelled mask in RGB.
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
model_path = model_paths[args.model]
if args.model == 'others':
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=tf.global_variables())
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
else:
net.load(model_path, sess)
print('Restore from {}'.format(model_path))
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
for i in trange(len(imgs), desc='Inference', leave=True):
start_time = timeit.default_timer()
preds = sess.run(pred, feed_dict={x: imgs[i]})
elapsed = timeit.default_timer() - start_time
print('inference time: {}'.format(elapsed))
misc.imsave(os.path.join(args.save_dir, filenames[i]), preds[0])
def main():
args = get_arguments()
if args.dataset == 'cityscapes':
num_classes = cityscapes_class
elif args.dataset == 'osm':
num_classes = osm_class
elif args.dataset == 'osm_nores':
num_classes = 5 #due to wrong slurm call osm_nores_class
elif args.dataset == 'ade20k':
num_classes = ADE20k_class
elif args.dataset == 'kaggle_dstl':
num_classes = dstl_class
elif args.dataset == 'vaihingen':
num_classes = vaihingen_class
else:
num_classes = osm_class
# Read images from directory (size must be the same) or single input file
imgs = []
filenames = []
gt_imgs = {}
if os.path.isdir(args.img_path):
file_paths = glob.glob(os.path.join(args.img_path, '*'))
for file_path in file_paths:
ext = file_path.split('.')[-1].lower()
if ext == 'png' or ext == 'jpg':
img, filename = load_img(file_path)
imgs.append(img)
filenames.append(filename)
if args.gt_dir:
filename = os.path.basename(filename)
no_ext = filename.split('.png')[0]
gt_path = args.gt_dir+'/'+'GT_'+no_ext+'.'+ext
img_gt, filename_gt = load_img(gt_path, cmode="L")
gt_imgs[filename] = img_gt
else:
img, filename = load_img(args.img_path)
imgs.append(img)
filenames.append(filename)
shape = imgs[0].shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
model = model_config[args.model]
net = model({'data': img_tf}, num_classes=num_classes, filter_scale=args.filter_scale)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1])
before_argmax = raw_output_up
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, shape, num_classes, args.dataset)
# get certainity
#
before_argmax = tf.reduce_max(before_argmax, axis=3)
cert = get_certainity(before_argmax, shape)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
model_path = model_paths[args.model]
if args.model_path:
model_path = args.model_path
if args.model == 'others':
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=tf.global_variables())
load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
else:
net.load(model_path, sess)
print('Restore from {}'.format(model_path))
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
save_dir_name = args.save_dir# + '/' + filenames[0])
# if not os.path.exists(save_dir_name):
# os.makedirs(save_dir_name)
for i in trange(len(imgs), desc='Inference', leave=True):
start_time = timeit.default_timer()
preds, certs = sess.run([pred, cert], feed_dict={x: imgs[i]})
elapsed = timeit.default_timer() - start_time
print('inference time: {}'.format(elapsed))
base_name = os.path.basename(args.save_dir + '/' + filenames[i])
subdir = os.path.join(save_dir_name, str(i))
print(subdir)
print('or '+save_dir_name+"/"+str(i))
if not os.path.exists(subdir):
os.makedirs(subdir)
if gt_imgs[base_name] is not None:
gt = get_coloredGT(gt_imgs[base_name], num_classes, args.dataset)
misc.imsave(subdir + '/' + 'gt_' + base_name, gt)
misc.imsave(subdir + '/' + 'sat_' + base_name, imgs[i])
misc.imsave(subdir + '/' + 'pred_' + base_name, preds[0])
misc.imsave(subdir + '/' + 'cert_' + base_name, certs[0])
def run_on_video(video_filename,
out_filename,
model_path,
num_classes,
save_to='simple',
canvas_size=(1600, 800),
alpha=0.8,
beta=0.2,
output_size=(1280, 720),
step=1):
'''
save_to: simple, double_screen or weighted
'''
input_size = (int(INPUT_SIZE.split(',')[0]), int(INPUT_SIZE.split(',')[0]))
x = tf.placeholder(dtype=tf.float32,
shape=(int(input_size[0]), int(input_size[1]), 3))
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
net = ICNet_BN({'data': img_tf}, num_classes=num_classes)
raw_output = net.layers['conv6']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=n_shape,
align_corners=True)
#raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, INPUT_SIZE[0], INPUT_SIZE[1])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
print('model_path', model_path)
ckpt = tf.train.latest_checkpoint(model_path)
if len(ckpt):
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, ckpt)
######
cap = cv2.VideoCapture(video_filename)
out_cap = None
if save_to == 'double_screen':
out_cap = cv2.VideoWriter(out_filename.replace('.mp4', '.avi'),
cv2.VideoWriter_fourcc(*"MJPG"), 60,
(canvas_size[0], canvas_size[1]))
elif save_to == 'weighted':
out_cap = cv2.VideoWriter(out_filename.replace('.mp4', '.avi'),
cv2.VideoWriter_fourcc(*"MJPG"), 60,
(output_size[0], output_size[1]))
# Check if camera opened successfully
if cap.isOpened() == False:
print("Error opening video stream or file")
return
frame_num = 0
zf = None
while (cap.isOpened()):
# Capture frame-by-frame
ret, image = cap.read()
frame_num = frame_num + 1
if frame_num % step != 0:
continue
print('Processing frame', frame_num)
if out_cap == None and save_to != 'images':
out_cap = cv2.VideoWriter(out_filename.replace('.mp4', '.avi'),
cv2.VideoWriter_fourcc(*'MJPG'), 60,
(image.shape[1], image.shape[0]))
elif save_to == 'images' and zf == None:
zipfile_name = out_filename.replace('.avi', '.zip')
original_shape = image.shape
if image.shape[2] == 1:
image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
if image.shape[2] == 4:
image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (input_size[0], input_size[1]))
preds = sess.run(pred, feed_dict={x: image})
msk = decode_labels(preds, num_classes=num_classes)
frame = msk[0]
#cv2.imshow('1', frame)
#cv2.waitKey(0)
#print(frame.shape)
if save_to == 'double_screen':
canvas = np.zeros((canvas_size[1], canvas_size[0], 3),
dtype=np.uint8)
#frame_orig = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
frame_orig = cv2.resize(
image, (int(canvas_size[0] / 2), int(canvas_size[1])))
#frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame = cv2.resize(frame,
(int(canvas_size[0] / 2), int(canvas_size[1])))
canvas[:, 0:int(canvas_size[0] / 2), :] = frame_orig
canvas[:, int(canvas_size[0] / 2):, :] = frame
#cv2.imshow('1', frame)
#cv2.waitKey(0)
canvas = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB)
print('canvas shape', canvas.shape)
out_cap.write(canvas)
elif save_to == 'simple':
frame = cv2.resize(frame, (original_shape[1], original_shape[0]),
interpolation=cv2.INTER_NEAREST)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
#cv2.imshow('1', frame)
#cv2.waitKey(0)
out_cap.write(frame)
elif save_to == 'images':
frame = cv2.resize(frame, (original_shape[1], original_shape[0]),
interpolation=cv2.INTER_NEAREST)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (original_shape[1], original_shape[0]),
interpolation=cv2.INTER_NEAREST)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
cv2.imwrite('/tmp/1.png', frame)
#cv2.imwrite('/tmp/1_orig.png', image)
zf = zipfile.ZipFile(zipfile_name, "a", zipfile.ZIP_DEFLATED)
name = 'frame_' + '%08d' % frame_num + '.png'
orig_name = 'frame_orig_' + '%08d' % frame_num + '.png'
zf.write('/tmp/1.png', name)
#zf.write('/tmp/1_orig.png', orig_name)
zf.close()
elif save_to == 'weighted':
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
frame = cv2.resize(frame, output_size)
image = cv2.resize(image, output_size)
frame = cv2.addWeighted(image, alpha, frame, beta, 0)
out_cap.write(frame)
elif save_to == 'perspective':
preds = preds.squeeze()
img, mask = getCutedRoad(image, preds)
mask = np.expand_dims(mask, axis=0)
mask = np.expand_dims(mask, axis=3)
msk = decode_labels(mask, num_classes=num_classes)
f = msk[0]
# h, w = frame.shape[:2]
# src = np.float32([[x1, y1], # br
# [x0, y1], # bl
# [x0, y0], # tl
# [x1, y0]]) # tr
# dst = np.float32([[w, h], # br
# [0, h], # bl
# [0, 0], # tl
# [w, 0]]) # tr
# M = cv2.getPerspectiveTransform(src, dst)
# Minv = cv2.getPerspectiveTransform(dst, src)
# warped = cv2.warpPerspective(image, M, (w, h), flags=cv2.INTER_LINEAR)
# cv2.imshow('1', warped)
#resized = cv2.resize(img[y0 : y1, x0 : x1], input_size, interpolation = cv2.INTER_NEAREST)
print((preds == 2).sum() / (preds.shape[0] * preds.shape[1]))
mask = np.array(mask)
mask = mask.squeeze()
print((mask == 2).sum() / (mask.shape[0] * mask.shape[1]))
cv2.imshow(
'2',
cv2.resize(img, input_size, interpolation=cv2.INTER_NEAREST))
cv2.imshow(
'3', cv2.resize(f, input_size,
interpolation=cv2.INTER_NEAREST))
cv2.imshow('4', image)
cv2.waitKey(0)
#quit()
cap.release()
out_cap.release()
zf.close()
def main():
num_classes = cityscapes_class
MODEL = 'trainval'
root_dir = "D:/ANNOTATION/CityScape/leftImg8bit/demoVideo/stuttgart_00"
images = os.listdir(root_dir)
# shape = img.shape[0:2]
# x = tf.placeholder(dtype=tf.float32, shape=img.shape)
shape = (H_RES, W_RES)
x = tf.placeholder(dtype=tf.float32, shape=(H_RES, W_RES, C_RES))
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
model = model_config[MODEL]
net = model({'data': img_tf}, num_classes=num_classes, filter_scale=1)
print(net.inputs)
for k, v in net.layers.items():
print("{:30} => {:}".format(k, v))
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=n_shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, shape, num_classes)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
model_path = model_paths[MODEL]
net.load(model_path, sess)
print('Restore from {}'.format(model_path))
i = 0
speed = 1
while i < len(images):
#img_read = misc.imread(root_dir + '/' + images[i], mode='RGB')
img_read = cv2.imread(root_dir + '/' + images[i])
img_read = cv2.resize(img_read, (W_RES, H_RES))
preds = sess.run(pred, feed_dict={x: img_read})
input = cv2.cvtColor(img_read, cv2.COLOR_BGR2RGB)
out = cv2.cvtColor(cv2.convertScaleAbs(preds[0]), cv2.COLOR_BGR2RGB)
out = cv2.addWeighted(input, 0.3, out, 0.7, 0.0)
cv2.imshow("out", out)
key = cv2.waitKey(5)
if key == 27:
break
elif key == 56:
i += 100
i += speed
print("frame= %d key=%d" % (i, key))
def main():
global atn
global preds
args = Args(
img_path='D:/val/',
model='others',
dataset='antenna',
filter_scale=1,
)
args.weight_path = './snapshots/3wDataSet/model.ckpt-400000'
# args.weight_path = './snapshots/3wDataSet/model.ckpt-300000'
# args = get_arguments()
if args.dataset == 'cityscapes':
num_classes = cityscapes_class
elif args.dataset == 'antenna':
num_classes = antenna_classes
else:
num_classes = ADE20k_class
# Read images from directory (size must be the same) or single input file
imgs = []
ori_imgs = []
filenames = []
# [2048, 1024],[1280, 720],[3840,2160]
load_shape = [2048, 1024]
if os.path.isdir(args.img_path):
file_paths = glob.glob(os.path.join(args.img_path, '*'))
for file_path in file_paths:
pic_files = glob.glob(os.path.join(file_path, '*'))
for pic_file in pic_files:
ext = pic_file.split('jpg')
if len(ext) > 2:
continue
img, filename = load_img(pic_file, load_shape)
imgs.append(img)
ori_imgs.append(img)
filenames.append(filename.replace('\\', '/'))
print(filenames)
else:
img, filename = load_img(args.img_path)
imgs.append(img)
filenames.append(filename)
shape = imgs[0].shape[0:2]
x = tf.placeholder(dtype=tf.float32, shape=img.shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
model = model_config[args.model]
net = model({'data': img_tf}, num_classes=num_classes, filter_scale=args.filter_scale)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, shape, num_classes)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
model_path = model_paths[args.model]
if args.model == 'others':
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=tf.global_variables())
load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
# load(loader, sess, ckpt.model_checkpoint_path)
load(loader, sess, args.weight_path)
else:
print('No checkpoint file found.')
else:
net.load(model_path, sess)
print('Restore from {}'.format(model_path))
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
val_time = []
det_dict = {1: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 12: 0, 15: 0}
fit_dict = {1: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 12: 0, 15: 0}
for i in trange(len(imgs), desc='Inference', leave=True):
# print(filenames[i].split('\\'))
act_deg = int(filenames[i].split('/')[1])
fit_count = 0
det_count = 0
start_time = timeit.default_timer()
preds = sess.run(pred, feed_dict={x: imgs[i]})
os.makedirs(args.save_dir + filenames[i].replace(filenames[i].split("/")[-1], ''), exist_ok=True)
misc.imsave(args.save_dir + filenames[i].split('.')[0] + '_Pred.jpg', preds[0])
# misc.imsave(args.save_dir + filenames[i].split('.')[0] + 'Ori.jpg', ori_imgs[i])
# print(preds[0][0][0])
p = np.array(np.where(preds[0] != [90, 90, 90])[:-1])
p = np.dstack((p[0], p[1]))[0]
if p.shape[0] == 0:
continue
uni_p = np.unique(p, axis=0)
atns = []
atn = Antenna()
atn.l_line.append(uni_p[0])
# 遍历所有标记为天线类别的像素点
for p_i in range(1, len(uni_p)):
point = uni_p[p_i]
last_point = uni_p[p_i-1]
if not same_atn(point, last_point):
atn.r_line.append(last_point)
# 每次插完右边点检查是否换行
if not same_row(point, last_point):
# 成对插入天线左右边像素点
row = atn_2_row(atn)
atn.clear()
# 第一次换行时初始化天线对象列表atns
if not atns:
atns = init_atns(row, preds)
row.clear()
atn.l_line.append(point)
continue
# 天线左右边像素点组合遍历所有天线对象最后一个点作差，相差最小则插入天线对象
for atns_i in range(len(atns)):
if not row:
break
dis = np.abs(np.array(row) - atns[atns_i].last_point())
m_dis = np.argmin(np.sum(np.sum(dis, axis=1), axis=1))
if np.abs(row[m_dis][0][0] - atns[atns_i].last_point()[0][0]) == 1 \
and np.abs(row[m_dis][0][1] - atns[atns_i].last_point()[0][1]) <= 1\
and np.abs(row[m_dis][1][1] - atns[atns_i].last_point()[1][1]) <= 1:
# print(np.abs(row[m_dis][0][1] - atns[atns_i].last_point()[0][1]), np.abs(row[m_dis][1][1] - atns[atns_i].last_point()[1][1]))
atns[atns_i].append(row[m_dis], preds, atns_i)
row.pop(m_dis)
if row:
for row_j in range(0, len(row)):
atn_tmp = Antenna()
atn_tmp.append(row[row_j], preds, len(atns))
atns.append(atn_tmp)
row.clear()
atn.l_line.append(point)
# 遍历天线对象，去除左右边拟合计算下倾角
atns = [atns[atns_i] for atns_i in range(0, len(atns)) if len(atns[atns_i].l_line)>75]
for k in range(len(atns)):
a = np.array(atns[k].l_line)
# 天线左边
x1 = a[:, :1].flatten()
dis = int(len(x1)/10)
x1 = x1[dis:-dis]
y = a[:, 1:].flatten()[dis:-dis]
z1 = np.polyfit(x1, y, 1) # 一次多项式拟合，相当于线性拟合
p1 = np.poly1d(z1)
y1 = p1(x1).astype(np.int)
# 天线右边
b = np.array(atns[k].r_line)
x2 = b[:, :1].flatten()[dis:-dis]
y2 = b[:, 1:].flatten()[dis:-dis]
z2 = np.polyfit(x2, y2, 1)
p2 = np.poly1d(z2)
yl2 = p2(x2).astype(np.int)
# plt.figure()
# plt.scatter(x1, y, 25, "red")
# plt.scatter(x2, y2, 25, "green")
#
# plt.plot(x1, y1, 'blue')
# plt.plot(x2, yl2, 'yellow')
# plt.show()
cur_img = np.array(ori_imgs[i], dtype='float32')
ori_imgs[i] = cv2.line(cur_img, (y1[0], x1[0]), (y1[-1], x1[-1]), (199, 0, 69), 3)
ori_imgs[i] = cv2.line(cur_img, (yl2[0], x2[0]), (yl2[-1], x2[-1]), (199, 0, 69), 3)
# l_degree = (x1[-1] - x1[0])/(y1[-1] - y1[0])
l_degree = (math.atan2((x1[-1] - x1[0]), (y1[-1] - y1[0])))*180/math.pi
# r_degree = (x2[-1] - x2[0])/(yl2[-1] - yl2[0])
r_degree = (math.atan2((x2[-1] - x2[0]), (yl2[-1] - yl2[0])))*180/math.pi
degree = abs(round((l_degree+r_degree)/2-90, 2))
acc_dis = abs(degree - act_deg)
if acc_dis < 3:
det_count = det_count + 1
if acc_dis < 1:
fit_count = fit_count + 1
# print(l_degree,r_degree, degree)
ori_imgs[i] = cv2.putText(cur_img, str(degree), (yl2[0], x2[0]), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 3)
# print((y1[0], x[0]), (y1[-1], x[-1]))
# print((yl2[0], x2[0]), (yl2[-1], x2[-1]))
overlayed_img = cv2.addWeighted(np.array(ori_imgs[i], dtype='float32'), 0.7, preds[0], 0.3, 0)
# os.makedirs(args.save_dir + filenames[i].split('Screen')[0], exist_ok=True)
fit_dict[act_deg] += fit_count
det_dict[act_deg] += det_count
# print(args.save_dir + filenames[i], det_count, fit_count)
misc.imsave(args.save_dir + filenames[i], overlayed_img)
print(args.save_dir + filenames[i])
elapsed = timeit.default_timer() - start_time
print('inference time: {}'.format(elapsed))
val_time.append(elapsed)
print('mean_val_time:', np.mean(val_time[1:]))
print('detection:', det_dict, '%.3f' % (sum(det_dict.values())/158*100) + "%")
print('fitting:', fit_dict, '%.3f' % (sum(fit_dict.values())/158*100) + "%")
print(val_time)
def main():
args = get_arguments()
if args.img_path[-4] != '.':
files = GetAllFilesListRecusive(args.img_path,
['.jpg', '.jpeg', '.png'])
else:
files = [args.img_path]
shape = INPUT_SIZE.split(',')
shape = (int(shape[0]), int(shape[1]), 3)
x = tf.placeholder(dtype=tf.float32, shape=shape)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
# Create network.
net = ICNet_BN({'data': img_tf},
is_training=False,
num_classes=num_classes)
# Predictions.
raw_output = net.layers['conv6_cls']
output = tf.image.resize_bilinear(raw_output, tf.shape(img_tf)[1:3, ])
output = tf.argmax(output, dimension=3)
pred = tf.expand_dims(output, dim=3)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
restore_var = tf.global_variables()
ckpt = tf.train.get_checkpoint_state(args.snapshots_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
for path in files:
img, filename = load_img(path)
preds = sess.run(pred, feed_dict={x: img})
msk = decode_labels(preds, num_classes=num_classes)
im = msk[0]
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if args.weighted:
indx = (im == [0, 0, 0])
im = cv2.addWeighted(im, 0.7, img, 0.7, -15)
im[indx] = img[indx]
cv2.imwrite(args.save_dir + filename.replace('.jpg', '.png'), im)
img_tf = preprocess(x)
img_tf, n_shape = check_input(img_tf)
# Create network.
net = ICNet({'data': img_tf}, num_classes=num_classes, filter_scale=1)
raw_output = net.layers['conv6_cls']
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=n_shape,
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
img.shape[0], img.shape[1])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = decode_labels(raw_output_up, img.shape, num_classes)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
net.load(model_path, sess)
print('Restore from {}'.format(model_path))
preds = sess.run(pred, feed_dict={x: img})
plt.figure(1, [15, 30])
def main():
args = get_arguments()
img, filename = load_img(args.img_path)
img_shape = tf.shape(img)
h, w = (tf.maximum(crop_size[0],
img_shape[0]), tf.maximum(crop_size[1], img_shape[1]))
img = preprocess(img, h, w)
# Create network.
net = PSPNet({'data': img}, is_training=False, num_classes=num_classes)
with tf.variable_scope('', reuse=True):
flipped_img = tf.image.flip_left_right(tf.squeeze(img))
flipped_img = tf.expand_dims(flipped_img, dim=0)
net2 = PSPNet({'data': flipped_img},
is_training=False,
num_classes=num_classes)
raw_output = net.layers['conv6']
if args.flipped_eval:
flipped_output = tf.image.flip_left_right(
tf.squeeze(net2.layers['conv6']))
flipped_output = tf.expand_dims(flipped_output, dim=0)
raw_output = tf.add_n([raw_output, flipped_output])
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=[h, w],
align_corners=True)
raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0,
img_shape[0], img_shape[1])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
restore_var = tf.global_variables()
ckpt = tf.train.get_checkpoint_state(args.model)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
preds = sess.run(pred)
msk = decode_labels(preds, num_classes=num_classes)
im = Image.fromarray(msk[0])
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im.save(args.save_dir + filename)
img_list.sort()
seg_list = grab_all_entries('/local/Data/NIPS18/kitti_raw_data_seg',
'_compact.npy')
seg_list.sort()
assert img_list.size == seg_list.size
# create dataset loader
dataloader = tf.data.Dataset.from_tensor_slices((img_list, seg_list))
dataloader = dataloader.map(load_func)
iterator = dataloader.make_initializable_iterator()
next_element = iterator.get_next()
sess = tf.Session()
sess.run(iterator.initializer)
label_map = decode_labels(tf.argmax(next_element[1], axis=2),
img_shape=[256, 512],
num_classes=19)
while True:
try:
(image, prob_mask), mask = sess.run([next_element, label_map])
plt.clf()
plt.subplot(131)
plt.imshow(image.astype(np.uint8))
plt.subplot(132)
plt.imshow(mask[0, ...].astype(np.uint8))
plt.subplot(133)
blend = cv2.addWeighted(image, 0.7, mask[0, ...], 0.3, 0)
plt.imshow(blend.astype(np.uint8))
plt.show()
except tf.errors.OutOfRangeError:
