def run(self, image_path): # 读入图片数据 img, filename = self._load_img(image_path) # 输出预测的结果 predictions_op = self._init_net(img=img) sess = tf.Session(config=self.config) sess.run(tf.global_variables_initializer()) # 加载模型 ckpt = tf.train.get_checkpoint_state(self.log_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=tf.global_variables()) loader.restore(sess, ckpt.model_checkpoint_path) Tools.print_info("Restored model parameters from {}".format( ckpt.model_checkpoint_path)) else: Tools.print_info('No checkpoint file found.') # 运行 predictions = sess.run(predictions_op) msk = decode_labels(predictions, num_classes=self.num_classes) im = Image.fromarray(msk[0]) im.save(os.path.join(self.save_dir, filename)) Tools.print_info('over : result save in {}'.format( os.path.join(self.save_dir, filename))) pass
def main(): args = get_arguments() if args.img_path[-4] != '.': files = GetAllFilesListRecusive(args.img_path, ['.jpg', '.jpeg', '.png', '.bmp']) else: files = [args.img_path] shape = INPUT_SIZE.split(',') shape = (int(shape[0]), int(shape[1]), 3) if args.pb_file == '': sess, pred, x = load_from_checkpoint(shape, args.snapshots_dir, args.model) else: sess, pred, x, label_colors, label_names = load_from_pb( shape, args.pb_file) if args.measure_time: calculate_perfomance(sess, x, pred, shape, args.runs, args.batch_size) quit() for path in files: img, filename = load_img(path) orig_img = cv2.imread(path) if args.pb_file != '': img = np.expand_dims(img, axis=0) t = time.time() preds = sess.run(pred, feed_dict={x: img}) print('time: ', time.time() - t) print('output shape: ', preds.shape) msk = decode_labels(preds, num_classes=len(label_colors), label_colours=label_colors) im = msk[0] im = cv2.resize(im, (orig_img.shape[1], orig_img.shape[0]), interpolation=cv2.INTER_NEAREST) print('im', im.shape) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR) #img = cv2.cvtColor(orig_img, cv2.COLOR_RGB2BGR) img = orig_img if args.weighted: indx = (im == [0, 0, 0]) print(im.shape, img.shape) im = cv2.addWeighted(im, 0.8, img, 0.2, 0) im[indx] = img[indx] cv2.imwrite(args.save_dir + filename.replace('.jpg', '.png'), im)
def main(): args = get_arguments() if args.dataset == 'cityscapes': num_classes = cityscapes_class else: num_classes = ADE20k_class img, filename = load_img(args.img_path) shape = img.shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) model = model_config[args.model] net = model({'data': img_tf}, num_classes=num_classes, filter_scale=args.filter_scale) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, shape, num_classes) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() model_path = model_paths[args.model] if args.model == 'others': ckpt = tf.train.get_checkpoint_state(model_path) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=tf.global_variables()) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') else: net.load(model_path, sess) print('Restore from {}'.format(model_path)) preds = sess.run(pred, feed_dict={x: img}) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) misc.imsave(args.save_dir + filename, preds[0])
def __init__(self): self.shape = IMG_SHAPE self.num_classes = sun_class self.show_classes = num_class output_graph_path = PB_PATH # with tf.Graph().as_default(): output_graph_def = tf.GraphDef() with open(output_graph_path, 'rb') as f: output_graph_def.ParseFromString(f.read()) _ = tf.import_graph_def(output_graph_def, name="") self.sess = tf.Session() tf.initialize_all_variables().run(session=self.sess) self.img_ph = self.sess.graph.get_tensor_by_name("Placeholder:0") raw_output = self.sess.graph.get_tensor_by_name("final_out:0") raw_output_up = tf.image.resize_bilinear(raw_output, size=self.shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, self.shape[0], self.shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) self.pred = decode_labels(raw_output_up, self.shape, self.show_classes, self.num_classes)
def main(): args = get_arguments() num_classes = sun_class show_classes = args.num_class output_graph_path = args.pb_path imgs = [] filenames = [] if os.path.isdir(args.img_path): file_paths = glob.glob(os.path.join(args.img_path, '*')) for file_path in file_paths: ext = file_path.split('.')[-1].lower() if ext == 'png' or ext == 'jpg': img, filename = load_img(file_path) imgs.append(img) filenames.append(filename) else: img, filename = load_img(args.img_path) img = preprocess(img) imgs.append(img) filenames.append(filename) shape = imgs[0].shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) x, n_shape = check_input(x) with tf.Graph().as_default(): output_graph_def = tf.GraphDef() with open(output_graph_path, 'rb') as f: output_graph_def.ParseFromString(f.read()) _ = tf.import_graph_def(output_graph_def, name="") with tf.Session() as sess: tf.initialize_all_variables().run() img = sess.graph.get_tensor_by_name("Placeholder:0") raw_output = sess.graph.get_tensor_by_name("final_out:0") raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box( raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, shape, show_classes, num_classes) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) for i in trange(len(imgs), desc='Inference', leave=True): start_time = timeit.default_timer() raw_output_up_temp = sess.run(raw_output, feed_dict={img: [imgs[i]]}) preds = sess.run(pred, feed_dict={img: [imgs[i]]}) misc.imsave(args.save_dir + filenames[i], preds[0])
def main(): args = get_arguments() img, filename = load_img(args.img_path) shape = img.shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) # Create network. net = ICNet_BN({'data': img_tf}, num_classes=num_classes) # Predictions. raw_output = net.layers['conv6_cls'] output = tf.image.resize_bilinear(raw_output, tf.shape(img_tf)[1:3, ]) output = tf.argmax(output, dimension=3) pred = tf.expand_dims(output, dim=3) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() ckpt = tf.train.get_checkpoint_state(args.snapshots_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) preds = sess.run(pred, feed_dict={x: img}) # print(preds.shape) # s = preds.flatten() # print(set(s)) # print((s == 0).sum()) # print((s == 1).sum()) # print((s == 2).sum()) msk = decode_labels(preds, num_classes=num_classes) im = Image.fromarray(msk[0]) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) im.save(args.save_dir + filename.replace('.jpg', '.png'))
def main(): args = get_arguments() img, filename = load_img(args.img_path) shape = img.shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) # Create network. net = ICNet({'data': img_tf}, num_classes=num_classes) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.squeeze(raw_output_up, axis=0) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, dimension=2) pred = tf.expand_dims(raw_output_up, dim=2) pred = tf.expand_dims(pred, dim=0) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() if args.model == 'train': print('Restore from train30k model...') net.load(model_train30k, sess) elif args.model == 'trainval': print('Restore from trainval90k model...') net.load(model_trainval90k, sess) for i in range(50): start = time.time() preds = sess.run(pred, feed_dict={x: img}) end = time.time() print('Inf time = %f' % (end - start)) msk = decode_labels(preds, num_classes=num_classes) im = Image.fromarray(msk[0]) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) im.save(args.save_dir + filename)
def _load_model(self, shape): self.x = tf.placeholder(dtype=tf.float32, shape=shape) img_tf = self.preprocess(self.x) self.img_tf, self.n_shape = self.check_input(img_tf) self.model = model_config[self.model_name] print(self.model) net = self.model({'data': img_tf}, num_classes=self.num_classes, filter_scale=self.filter_scale) # net = self.model({'data': img_tf}, num_classes=self.num_classes) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=self.n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) self.pred = decode_labels(raw_output_up, shape, self.num_classes) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) init = tf.global_variables_initializer() self.sess.run(init) # model_path = model_paths[self.model_name] model_path = snapshot_dir if self.model_name == 'others': ckpt = tf.train.get_checkpoint_state(model_path) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=tf.global_variables()) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) self.load(loader, self.sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') else: # model path must be a model net.load(self.model_path, self.sess) print('Restore from {}'.format(model_path))
def main(): args = get_arguments() img, filename = load_img(args.img_path) img = preprocess(img) # Create network. net = ICNet({'data': img}, num_classes=num_classes) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=input_size, align_corners=True) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() if args.model == 'train': print('Restore from train30k model...') net.load(model_train30k, sess) elif args.model == 'trainval': print('Restore from trainval90k model...') net.load(model_trainval90k, sess) preds = sess.run(pred) msk = decode_labels(preds, num_classes=num_classes) im = Image.fromarray(msk[0]) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) im.save(args.save_dir + filename)
def predict(self): self.predict_setup() self.sess.run(tf.global_variables_initializer()) self.sess.run(tf.local_variables_initializer()) checkpointfile = self.parameters.modeldir + '/model.ckpt-' + str( self.parameters.valid_step) self.load(self.loader, checkpointfile) threads = tf.train.start_queue_runners(coord=self.coord, sess=self.sess) image_list, _ = read_labeled_image_list('', self.parameters.test_data_list) for step in range(self.parameters.test_num_steps): preds = self.sess.run(self.pred) img_name = image_list[step].split('/')[2].split('.')[0] im = Image.fromarray(preds[0, :, :, 0], mode='L') filename = '/%s_mask.png' % (img_name) im.save(self.parameters.out_dir + '/prediction' + filename) if self.parameters.visual: msk = decode_labels(preds, num_classes=self.parameters.num_classes) im = Image.fromarray(msk[0], mode='RGB') filename = '/%s_mask_visual.png' % (img_name) im.save(self.parameters.out_dir + '/visual_prediction' + filename) if step % 100 == 0: print('step {:d}'.format(step)) print('The output files has been saved to {}'.format( self.parameters.out_dir)) self.coord.request_stop() self.coord.join(threads)
def evaluate(config, evaluation_set='val', plot_confusionMatrix=False): # -------------------------------------------------------------------- # init network # -------------------------------------------------------------------- tf.compat.v1.reset_default_graph() # define input placeholders input_placeholder = {} input_placeholder.update( {'is_training': tf.compat.v1.placeholder(dtype=tf.bool, shape=())}) if config.ARCHITECTURE == 'semantic_segmentation': batch_size = config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH # Search for available GPUs: the result is a list of device ids like `['/gpu:0', '/gpu:1']` devices = get_available_gpus() print("found devices: ", devices) num_GPU = len(devices) if (num_GPU) == 0: num_GPU = 1 # CPU support! # min 1 sample should be applied on a GPU if (config.BATCH_SIZE < num_GPU): num_GPU = config.BATCH_SIZE image_placeholder = [] label_placeholder = [] for iter in range(num_GPU): if (iter == (num_GPU - 1)): batch_size_local = batch_size - (num_GPU - 1) * (batch_size // num_GPU) else: batch_size_local = batch_size // num_GPU print('batch_size /gpu:{} : {}'.format(iter, num_GPU)) image_placeholder.append( tf.compat.v1.placeholder( dtype=tf.float32, shape=(batch_size_local, config.DATASET_TRAIN.INPUT_SIZE[0], config.DATASET_TRAIN.INPUT_SIZE[1], config.DATASET_TRAIN.NUM_CHANNELS))) label_placeholder.append( tf.compat.v1.placeholder( dtype=tf.float32, shape=(batch_size_local, config.DATASET_TRAIN.INPUT_SIZE[0], config.DATASET_TRAIN.INPUT_SIZE[1], 1))) input_placeholder.update({'image_batch': image_placeholder}) input_placeholder.update({'label_batch': label_placeholder}) else: print( '[ERROR] network architecture does not exist!!! Please check your spelling!' ) raise NotImplementedError # load network architecture if config.ARCHITECTURE == 'semantic_segmentation': model = get_model(config.MODEL) net = model( { 'data': input_placeholder['image_batch'], 'is_training': input_placeholder['is_training'] }, is_training=input_placeholder['is_training'], evaluation=tf.logical_not(input_placeholder['is_training']), #is_inference=True, num_classes=config.DATASET_TRAIN.NUM_CLASSES, filter_scale=config.FILTER_SCALE, timeSequence=config.TIMESEQUENCE_LENGTH, variant=config.MODEL_VARIANT) else: print( '[ERROR] network architecture does not exist!!! Please check your spelling!' ) raise NotImplementedError # -------------------------------------------------------------------- # determine evaluation metric # -------------------------------------------------------------------- if config.ARCHITECTURE == 'semantic_segmentation': list_raw_gt = [] list_pred_flattern_mIoU = [] for iter_gpu in range(len(input_placeholder['image_batch'])): with tf.device('/gpu:%d' % iter_gpu): if config.MODEL == 'SegNet_BN' or config.MODEL == 'SegNet_BN_encoder' or config.MODEL == 'SegNet_BN_decoder' or config.MODEL == 'SegNet_BN_encoderDecoder': raw_output = net.layers['output'][iter_gpu] raw_output_up = tf.argmax(raw_output, axis=3, output_type=tf.int32) raw_pred_mIoU = tf.expand_dims(raw_output_up, dim=3) else: # ICNet ori_shape = config.DATASET_TRAIN.INPUT_SIZE #?? raw_output = net.layers['output'][iter_gpu] raw_output_up = tf.compat.v1.image.resize_bilinear( raw_output, size=ori_shape[:2], align_corners=True) raw_output_up = tf.argmax(raw_output_up, axis=3, output_type=tf.int32) raw_pred_mIoU = tf.expand_dims(raw_output_up, dim=3) # determine mIoU if config.USAGE_TIMESEQUENCES: # evaluate only last image of time sequence pred_of_interest = np.array( range(config.BATCH_SIZE), dtype=np.int32 ) * config.TIMESEQUENCE_LENGTH + config.TIMESEQUENCE_LENGTH - 1 pred_flatten_mIoU = tf.reshape( tf.gather(raw_pred_mIoU, pred_of_interest), [ -1, ]) raw_gt = tf.reshape( tf.gather(input_placeholder['label_batch'][iter_gpu], pred_of_interest), [ -1, ]) else: # evaluate all images of batch size pred_flatten_mIoU = tf.reshape(raw_pred_mIoU, [ -1, ]) raw_gt = tf.reshape( input_placeholder['label_batch'][iter_gpu], [ -1, ]) list_raw_gt.append(raw_gt) list_pred_flattern_mIoU.append(pred_flatten_mIoU) # combine output of different GPUs with tf.device('/gpu:%d' % 0): all_raw_gt = tf.reshape(tf.concat(list_raw_gt, -1), [ -1, ]) all_pred_flatten_mIoU = tf.reshape( tf.concat(list_pred_flattern_mIoU, -1), [ -1, ]) indices_mIoU = tf.squeeze( tf.where( tf.less_equal(raw_gt, config.DATASET_TRAIN.NUM_CLASSES - 1)), 1) gt_mIoU = tf.cast(tf.gather(raw_gt, indices_mIoU), tf.int32) pred_mIoU = tf.gather(pred_flatten_mIoU, indices_mIoU) mIoU, update_op = tf.contrib.metrics.streaming_mean_iou( pred_mIoU, gt_mIoU, num_classes=config.DATASET_VAL.NUM_CLASSES) # create colored image pred_color = decode_labels(pred_flatten_mIoU, config.DATASET_VAL.INPUT_SIZE[0:2], config.DATASET_VAL.NUM_CLASSES) # deterimine confusing matrix if plot_confusionMatrix: # Create an accumulator variable to hold the counts confusion = tf.Variable(tf.zeros([ config.DATASET_VAL.NUM_CLASSES, config.DATASET_VAL.NUM_CLASSES ], dtype=tf.int64), name='confusion', collections=[ tf.compat.v1.GraphKeys.LOCAL_VARIABLES ]) # Compute a per-batch confusion batch_confusion = tf.math.confusion_matrix( tf.reshape(gt_mIoU, [-1]), tf.reshape(pred_mIoU, [-1]), num_classes=config.DATASET_VAL.NUM_CLASSES, name='batch_confusion') # Create the update op for doing a "+=" accumulation on the batch confusion_update = confusion.assign( confusion + tf.cast(batch_confusion, dtype=tf.int64)) # ----------------------------------------- # init session # ----------------------------------------- # Set up tf session and initialize variables. sessConfig = tf.compat.v1.ConfigProto() sessConfig.gpu_options.allow_growth = True # use only a fraction of gpu memory, otherwise the TensorRT-test has not enough free GPU memory for execution sessConfig.gpu_options.per_process_gpu_memory_fraction = 0.5 sess = tf.compat.v1.Session(config=sessConfig) init = tf.compat.v1.global_variables_initializer() local_init = tf.compat.v1.local_variables_initializer() sess.run(init) sess.run(local_init) # load checkpoint file print(config.EVALUATION.MODELPATH) ckpt = tf.compat.v1.train.get_checkpoint_state(config.EVALUATION.MODELPATH) if ckpt and ckpt.model_checkpoint_path: loader = tf.compat.v1.train.Saver( var_list=tf.compat.v1.global_variables()) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') # `sess.graph` provides access to the graph used in a <a href="./../api_docs/python/tf/Session"><code>tf.Session</code></a>. writer = tf.compat.v1.summary.FileWriter("/tmp/tensorflow_graph", tf.compat.v1.get_default_graph()) # -------------------------------------------------------------------- # Evaluate - Iterate over training steps. # -------------------------------------------------------------------- # evaluate training or validation set if evaluation_set == "val": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_VAL, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) elif evaluation_set == "train": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_TRAIN, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) elif evaluation_set == "test": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_TEST, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) elif evaluation_set == "all": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_ALL, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) else: print("Dataset {} does not exist!".format(evaluation_set)) filename_memory = "" filename_count = 0 average_inference_time = 0 # -------------------------------------- # perform evaluation - semantic segmentation # -------------------------------------- if config.ARCHITECTURE == 'semantic_segmentation': if config.TIMESEQUENCES_SLIDINGWINDOW: # use time sequences for step in trange( int(imagereader_val._dataset_amount - config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH + 1), desc='inference', leave=True): #start_time = time.time() training_batch = imagereader_val.getNextMinibatch() feed_dict = {input_placeholder['is_training']: False} for iter_GPU in range(len(input_placeholder['image_batch'])): num_GPU = len(input_placeholder['image_batch']) batch_size = training_batch['blob_data'].shape[0] batch_size_local = batch_size // num_GPU if (iter_GPU == (num_GPU - 1)): batch_size_act = batch_size - (num_GPU - 1) * ( batch_size // num_GPU) else: batch_size_act = batch_size // num_GPU feed_dict.update({ input_placeholder['image_batch'][iter_GPU]: training_batch['blob_data'][iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :], input_placeholder['label_batch'][iter_GPU]: training_batch['blob_label'] [iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :] }) start_time = time.time() if plot_confusionMatrix: sess.run([update_op, confusion_update], feed_dict=feed_dict) else: sess.run([update_op], feed_dict=feed_dict) duration = time.time() - start_time average_inference_time += duration # save image prediction = sess.run([pred_color], feed_dict=feed_dict) predi = np.array( prediction[0][0, :, :, :]).astype(dtype=np.uint8) img = cv2.imread(imagereader_val._dataset[step][0]) if imagereader_val._configDataset.USE_IMAGE_ROI: img = img[imagereader_val._configDataset.IMAGE_ROI_MIN_X: imagereader_val._configDataset.IMAGE_ROI_MAX_X, imagereader_val._configDataset. IMAGE_ROI_MIN_Y:imagereader_val._configDataset. IMAGE_ROI_MAX_Y, :] cv2.addWeighted(predi, config.INFERENCE.OVERLAPPING_IMAGE, img, 1 - config.INFERENCE.OVERLAPPING_IMAGE, 0, img) buff = imagereader_val._dataset[step][-1] buff = buff.split('/') filename = buff[-1].split('.')[0] if filename_memory == buff[-1].split('.')[0]: filename_count += 1 filename = buff[-1].split('.')[0] + "_" + str( filename_count) + ".png" else: filename_memory = buff[-1].split('.')[0] filename = buff[-1].split('.')[0] + "_0.png" filename_count = 0 cv2.imwrite(config.INFERENCE.SAVEDIR_IMAGES + filename, predi[:, :, (2, 1, 0)]) cv2.imwrite(config.INFERENCE.SAVEDIR_IMAGES + filename, img[:, :, (2, 1, 0)]) # determine average time average_inference_time = average_inference_time / int( imagereader_val._dataset_amount - config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH + 1) else: # do not use time sequences (normal evaluation) # TODO parameters for flickering evaluation flickering_sum = 0 flickering_img_size = 0 flickering_sum1 = 0 flickering_img_size1 = 0 for step in trange( int(imagereader_val._dataset_amount / (config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH)), desc='inference', leave=True): training_batch = imagereader_val.getNextMinibatch() feed_dict = {input_placeholder['is_training']: False} for iter_GPU in range(len(input_placeholder['image_batch'])): num_GPU = len(input_placeholder['image_batch']) batch_size = training_batch['blob_data'].shape[0] batch_size_local = batch_size // num_GPU if (iter_GPU == (num_GPU - 1)): batch_size_act = batch_size - (num_GPU - 1) * ( batch_size // num_GPU) else: batch_size_act = batch_size // num_GPU # for depth images, if result should be shown in camera image if True: feed_dict.update({ input_placeholder['image_batch'][iter_GPU]: training_batch['blob_data'] [iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :], input_placeholder['label_batch'][iter_GPU]: training_batch['blob_label'] [iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :] }) else: training_batch['blob_data'][:, :, :, 1] = training_batch[ 'blob_data'][:, :, :, -1] training_batch['blob_data'][:, :, :, 2] = training_batch[ 'blob_data'][:, :, :, -1] feed_dict.update({ input_placeholder['image_batch'][iter_GPU]: training_batch['blob_data'] [iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, 1:4], input_placeholder['label_batch'][iter_GPU]: training_batch['blob_label'] [iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :] }) start_time = time.time() if plot_confusionMatrix: sess.run([update_op, confusion_update], feed_dict=feed_dict) else: sess.run([update_op], feed_dict=feed_dict) duration = time.time() - start_time # skip the first 50 images if step >= 50: average_inference_time += duration # -------------------------- # save image # -------------------------- prediction = sess.run([pred_color, raw_output_up], feed_dict=feed_dict) predi = np.array( prediction[0][0, :, :, :]).astype(dtype=np.uint8) predi_id = np.array(prediction[1][-1, ...]).astype(dtype=np.uint8) data_index = step * config.TIMESEQUENCE_LENGTH + config.TIMESEQUENCE_LENGTH - 1 buff = imagereader_val._dataset[data_index][-1] buff = buff.split('/') filename = buff[-1].split('.')[0] if filename_memory == buff[-1].split('.')[0]: filename_count += 1 filename = buff[-1].split('.')[0] + "_" + str( filename_count).zfill(6) + ".png" else: filename_memory = buff[-1].split('.')[0] filename = buff[-1].split('.')[0] + "_000000.png" filename_count = 0 img = cv2.imread(imagereader_val._dataset[data_index][0]) if imagereader_val._configDataset.USE_IMAGE_ROI: img = img[imagereader_val._configDataset.IMAGE_ROI_MIN_X: imagereader_val._configDataset.IMAGE_ROI_MAX_X, imagereader_val._configDataset. IMAGE_ROI_MIN_Y:imagereader_val._configDataset. IMAGE_ROI_MAX_Y, :] # label images for video cv2.addWeighted(predi, config.INFERENCE.OVERLAPPING_IMAGE, img, 1 - config.INFERENCE.OVERLAPPING_IMAGE, 0, img) cv2.imwrite( config.INFERENCE.SAVEDIR_IMAGES + 'pred_' + filename, predi[:, :, (2, 1, 0)]) cv2.imwrite( config.INFERENCE.SAVEDIR_IMAGES + 'overlay_' + filename, img[:, :, (2, 1, 0)]) # -------------------------- # flickering evaluation # -------------------------- # to measure flickering between non-ground-truth classes, multiply it with the predicted result (add 1, since True * class 0 = 0!) diff_img = np.array( (predi_id != training_batch['blob_label'][-1, :, :, 0]), np.float32) * np.array( training_batch['blob_label'][-1, :, :, 0] + 1, np.float32) diff_img1 = np.array(predi_id, np.float32) if step > 0: # skip step = 0, since there is no reference image flickering = np.sum((diff_img != prediction_old)) flickering_sum += flickering flickering_img_size += predi_id.shape[0] * predi_id.shape[1] flickering1 = np.sum((diff_img1 != prediction_old1)) flickering_sum1 += flickering1 flickering_img_size1 += predi_id.shape[0] * predi_id.shape[ 1] prediction_old = diff_img prediction_old1 = diff_img1 # determine average time average_inference_time = average_inference_time / int( imagereader_val._dataset_amount / (config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH) - 50) mIoU_value = sess.run(mIoU) print('flickering_sum: ', flickering_sum) print('FP: ', float(flickering_sum) / float(flickering_img_size)) print('--------------') print('flickering_sum1: ', flickering_sum1) print('FIP: ', float(flickering_sum1) / float(flickering_img_size1)) print('--------------') print( float(flickering_sum) / float(flickering_img_size), float(flickering_sum1) / float(flickering_img_size1)) print('--------------') if plot_confusionMatrix: confusion_matrix = sess.run(confusion) # print Accuracy: np.set_printoptions(linewidth=np.inf) #150) acc_value = float(np.sum(np.diag(confusion_matrix))) / float( np.sum(confusion_matrix)) else: print( '[ERROR] network architecture does not exist!!! Please check your spelling!' ) raise NotImplementedError # -------------------------------------------- # create optimized pb-model # -------------------------------------------- if config.FREEZEINFERENCEGRAPH.MODE: output_nodes = config.FREEZEINFERENCEGRAPH.OUTPUT_NODE_NAMES.split(',') input_nodes = config.FREEZEINFERENCEGRAPH.INPUT_NODE_NAMES.split(',') frozen_graph_def = tf.compat.v1.graph_util.convert_variables_to_constants( sess, tf.compat.v1.get_default_graph().as_graph_def(), output_nodes) # Write tensorrt graph def to pb file for use in C++ output_graph_path = config.EVALUATION.MODELPATH + '/model.pb' with tf.io.gfile.GFile(output_graph_path, "wb") as f: f.write(frozen_graph_def.SerializeToString()) transforms = [ 'remove_nodes(op=Identity)', 'merge_duplicate_nodes', 'strip_unused_nodes', 'fold_constants(ignore_errors=true)', 'fold_batch_norms', 'remove_device' ] optimized_graph_def = TransformGraph(frozen_graph_def, input_nodes, output_nodes, transforms) print('inference model saved in ', output_graph_path) # ------------------------------------ # apply TensorRT # ------------------------------------ if config.FREEZEINFERENCEGRAPH.MODE: config_TensorRT = tf.ConfigProto() #config_TensorRT.gpu_options.per_process_gpu_memory_fraction = 0.5 config_TensorRT.gpu_options.allow_growth = True converter = trt.TrtGraphConverter(input_graph_def=optimized_graph_def, session_config=config_TensorRT, max_workspace_size_bytes=4000000000, nodes_blacklist=output_nodes, is_dynamic_op=False, precision_mode='FP16') converted_graph_def = converter.convert() # Write tensorrt graph def to pb file for use in C++ output_graph_TensorRT_path = config.EVALUATION.MODELPATH + '/tensorrt_model.pb' with tf.io.gfile.GFile(output_graph_TensorRT_path, "wb") as f: f.write(converted_graph_def.SerializeToString()) print('TensorRT-model saved in ', output_graph_TensorRT_path) # -------------------------------------------- # close session # -------------------------------------------- writer.close() sess.close() tf.compat.v1.reset_default_graph() # ------------------------------------------ # define inference-function for model.pb # ------------------------------------------ def determineInferenceTime(path2frozenmodel): # We load the protobuf file from the disk and parse it to retrieve the unserialized graph_def with tf.gfile.GFile(path2frozenmodel, "rb") as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) for node in graph_def.node: if node.op == 'RefSwitch': node.op = 'Switch' for index in xrange(len(node.input)): if 'moving_' in node.input[index]: node.input[index] = node.input[index] + '/read' elif node.op == 'AssignSub': node.op = 'Sub' if 'use_locking' in node.attr: del node.attr['use_locking'] elif node.op == 'AssignAdd': node.op = 'Add' if 'use_locking' in node.attr: del node.attr['use_locking'] elif node.op == 'AssignMovingAvg': node.op = 'MovingAvg' if 'use_locking' in node.attr: del node.attr['use_locking'] # Then, we import the graph_def into a new Graph and returns it with tf.Graph().as_default() as graph: # The name var will prefix every op/nodes in your graph # Since we load everything in a new graph, this is not needed tf.import_graph_def(graph_def, name="") # print output node names if config.FREEZEINFERENCEGRAPH.PRINT_OUTPUT_NODE_NAMES: for op in graph.get_operations(): print(str(op.name)) ##### init network if config.ARCHITECTURE == 'semantic_segmentation': image_tensor = graph.get_tensor_by_name('Placeholder_1:0') ouput_tensor = graph.get_tensor_by_name('ArgMax:0') # init session # Note: we don't nee to initialize/restore anything. There is no Variables in this graph, only hardcoded constants sess_inf = tf.compat.v1.Session(graph=graph) average_inference_time_frozenModel = 0.0 for step in trange( int(imagereader_val._dataset_amount / (config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH)), desc='inference', leave=True): # get images from datastet training_batch = imagereader_val.getNextMinibatch() if config.ARCHITECTURE == 'semantic_segmentation': feed_dict = {image_tensor: training_batch['blob_data']} # apply inference start_time = time.time() if config.ARCHITECTURE == 'semantic_segmentation': sess_inf.run(ouput_tensor, feed_dict=feed_dict) duration_inf = time.time() - start_time # skip the first 50 images if step >= 50: average_inference_time_frozenModel += duration_inf sess_inf.close() average_inference_time_frozenModel = average_inference_time_frozenModel / int( imagereader_val._dataset_amount / (config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH) - 50) return average_inference_time_frozenModel # ------------------------------------------ # test model - optimized model # ------------------------------------------ if config.FREEZEINFERENCEGRAPH.MODE: ### apply optimized model (model.pb) path2frozenmodel_opt = config.EVALUATION.MODELPATH + '/model.pb' average_inference_time_opt = determineInferenceTime( path2frozenmodel_opt) ### apply TensorRT model (tensorrt_model.pb) path2frozenmodel_tensorrt = config.EVALUATION.MODELPATH + '/tensorrt_model.pb' average_inference_time_tensorrt = determineInferenceTime( path2frozenmodel_tensorrt) print('average time optimized model: {:.2f} ms'.format( average_inference_time_opt * 1000.0)) print('average time TensorRT: {:.2f} ms'.format( average_inference_time_tensorrt * 1000.0)) # -------------------------------------------------------------------- # Show results # -------------------------------------------------------------------- print('average time: {:.2f} ms'.format(average_inference_time * 1000.0)) if plot_confusionMatrix and config.ARCHITECTURE == 'semantic_segmentation': # determine class-wise IoU buff = 0.0 print('-------------------------------------------------------------') for iter in xrange(config.DATASET_VAL.NUM_CLASSES): if np.sum( confusion_matrix[iter, :]) == 0: # avoid division by zero IoU = 0.0 else: IoU = 100.0 * confusion_matrix[iter, iter] / ( np.sum(confusion_matrix[iter, :]) + np.sum(confusion_matrix[:, iter]) - confusion_matrix[iter, iter]) buff = buff + IoU print('{}: {}'.format(config.DATASET_VAL.CLASSES[iter], IoU)) print('-------------------------------------------------------------') print('dataset: {} - {}'.format(config.DATASET_NAME, config.DATASET_WEATHER)) print('Accuracy: {}'.format(acc_value)) print('mIoU: {}'.format(mIoU_value)) print('average time: {:.2f} ms'.format(average_inference_time * 1000.0)) print('-------------------------------------------------------------') if plot_confusionMatrix and config.ARCHITECTURE == 'semantic_segmentation': print('-------------------------------------------------------------') print(confusion_matrix) print('-------------------------------------------------------------') #plot_confusion_matrix(confusion_matrix, config.DATASET_VAL.CLASSES) return mIoU_value
def main(): args = get_arguments() img, filename = load_img(args.img_path) shape = img.shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) # Create network. if args.model[-2:] == 'bn': net = ICNet_BN({'data': img_tf}, num_classes=num_classes) elif args.model == 'others': net = ICNet_BN({'data': img_tf}, num_classes=num_classes) else: net = ICNet({'data': img_tf}, num_classes=num_classes) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, shape, num_classes) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() if args.model == 'train': print('Restore from train30k model...') net.load(model_train30k, sess) elif args.model == 'trainval': print('Restore from trainval90k model...') net.load(model_trainval90k, sess) elif args.model == 'train_bn': print('Restore from train30k bnnomerge model...') net.load(model_train30k_bn, sess) elif args.model == 'trainval_bn': print('Restore from trainval90k bnnomerge model...') net.load(model_trainval90k_bn, sess) else: ckpt = tf.train.get_checkpoint_state(snapshot_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) preds = sess.run(pred, feed_dict={x: img}) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) misc.imsave(args.save_dir + filename, preds[0])
def main(): args = get_arguments() if args.dataset == 'cityscapes': num_classes = cityscapes_class else: num_classes = ADE20k_class # Read images from directory (size must be the same) or single input file imgs = [] filenames = [] if os.path.isdir(args.img_path): file_paths = glob.glob(os.path.join(args.img_path, '*')) for file_path in file_paths: ext = file_path.split('.')[-1].lower() if ext == 'png' or ext == 'jpg': img, filename = load_img(file_path) imgs.append(img) filenames.append(filename) else: img, filename = load_img(args.img_path) imgs.append(img) filenames.append(filename) shape = imgs[0].shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) model = model_config[args.model] net = model({'data': img_tf}, num_classes=num_classes, filter_scale=args.filter_scale) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, shape, num_classes) # this is the labelled mask in RGB. # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) model_path = model_paths[args.model] if args.model == 'others': ckpt = tf.train.get_checkpoint_state(model_path) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=tf.global_variables()) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') else: net.load(model_path, sess) print('Restore from {}'.format(model_path)) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) for i in trange(len(imgs), desc='Inference', leave=True): start_time = timeit.default_timer() preds = sess.run(pred, feed_dict={x: imgs[i]}) elapsed = timeit.default_timer() - start_time print('inference time: {}'.format(elapsed)) misc.imsave(os.path.join(args.save_dir, filenames[i]), preds[0])
def main(): args = get_arguments() if args.dataset == 'cityscapes': num_classes = cityscapes_class elif args.dataset == 'osm': num_classes = osm_class elif args.dataset == 'osm_nores': num_classes = 5 #due to wrong slurm call osm_nores_class elif args.dataset == 'ade20k': num_classes = ADE20k_class elif args.dataset == 'kaggle_dstl': num_classes = dstl_class elif args.dataset == 'vaihingen': num_classes = vaihingen_class else: num_classes = osm_class # Read images from directory (size must be the same) or single input file imgs = [] filenames = [] gt_imgs = {} if os.path.isdir(args.img_path): file_paths = glob.glob(os.path.join(args.img_path, '*')) for file_path in file_paths: ext = file_path.split('.')[-1].lower() if ext == 'png' or ext == 'jpg': img, filename = load_img(file_path) imgs.append(img) filenames.append(filename) if args.gt_dir: filename = os.path.basename(filename) no_ext = filename.split('.png')[0] gt_path = args.gt_dir+'/'+'GT_'+no_ext+'.'+ext img_gt, filename_gt = load_img(gt_path, cmode="L") gt_imgs[filename] = img_gt else: img, filename = load_img(args.img_path) imgs.append(img) filenames.append(filename) shape = imgs[0].shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) model = model_config[args.model] net = model({'data': img_tf}, num_classes=num_classes, filter_scale=args.filter_scale) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) before_argmax = raw_output_up raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, shape, num_classes, args.dataset) # get certainity # before_argmax = tf.reduce_max(before_argmax, axis=3) cert = get_certainity(before_argmax, shape) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() model_path = model_paths[args.model] if args.model_path: model_path = args.model_path if args.model == 'others': ckpt = tf.train.get_checkpoint_state(model_path) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=tf.global_variables()) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') else: net.load(model_path, sess) print('Restore from {}'.format(model_path)) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) save_dir_name = args.save_dir# + '/' + filenames[0]) # if not os.path.exists(save_dir_name): # os.makedirs(save_dir_name) for i in trange(len(imgs), desc='Inference', leave=True): start_time = timeit.default_timer() preds, certs = sess.run([pred, cert], feed_dict={x: imgs[i]}) elapsed = timeit.default_timer() - start_time print('inference time: {}'.format(elapsed)) base_name = os.path.basename(args.save_dir + '/' + filenames[i]) subdir = os.path.join(save_dir_name, str(i)) print(subdir) print('or '+save_dir_name+"/"+str(i)) if not os.path.exists(subdir): os.makedirs(subdir) if gt_imgs[base_name] is not None: gt = get_coloredGT(gt_imgs[base_name], num_classes, args.dataset) misc.imsave(subdir + '/' + 'gt_' + base_name, gt) misc.imsave(subdir + '/' + 'sat_' + base_name, imgs[i]) misc.imsave(subdir + '/' + 'pred_' + base_name, preds[0]) misc.imsave(subdir + '/' + 'cert_' + base_name, certs[0])
def run_on_video(video_filename, out_filename, model_path, num_classes, save_to='simple', canvas_size=(1600, 800), alpha=0.8, beta=0.2, output_size=(1280, 720), step=1): ''' save_to: simple, double_screen or weighted ''' input_size = (int(INPUT_SIZE.split(',')[0]), int(INPUT_SIZE.split(',')[0])) x = tf.placeholder(dtype=tf.float32, shape=(int(input_size[0]), int(input_size[1]), 3)) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) net = ICNet_BN({'data': img_tf}, num_classes=num_classes) raw_output = net.layers['conv6'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) #raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, INPUT_SIZE[0], INPUT_SIZE[1]) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() print('model_path', model_path) ckpt = tf.train.latest_checkpoint(model_path) if len(ckpt): loader = tf.train.Saver(var_list=restore_var) load(loader, sess, ckpt) ###### cap = cv2.VideoCapture(video_filename) out_cap = None if save_to == 'double_screen': out_cap = cv2.VideoWriter(out_filename.replace('.mp4', '.avi'), cv2.VideoWriter_fourcc(*"MJPG"), 60, (canvas_size[0], canvas_size[1])) elif save_to == 'weighted': out_cap = cv2.VideoWriter(out_filename.replace('.mp4', '.avi'), cv2.VideoWriter_fourcc(*"MJPG"), 60, (output_size[0], output_size[1])) # Check if camera opened successfully if cap.isOpened() == False: print("Error opening video stream or file") return frame_num = 0 zf = None while (cap.isOpened()): # Capture frame-by-frame ret, image = cap.read() frame_num = frame_num + 1 if frame_num % step != 0: continue print('Processing frame', frame_num) if out_cap == None and save_to != 'images': out_cap = cv2.VideoWriter(out_filename.replace('.mp4', '.avi'), cv2.VideoWriter_fourcc(*'MJPG'), 60, (image.shape[1], image.shape[0])) elif save_to == 'images' and zf == None: zipfile_name = out_filename.replace('.avi', '.zip') original_shape = image.shape if image.shape[2] == 1: image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) if image.shape[2] == 4: image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) image = cv2.resize(image, (input_size[0], input_size[1])) preds = sess.run(pred, feed_dict={x: image}) msk = decode_labels(preds, num_classes=num_classes) frame = msk[0] #cv2.imshow('1', frame) #cv2.waitKey(0) #print(frame.shape) if save_to == 'double_screen': canvas = np.zeros((canvas_size[1], canvas_size[0], 3), dtype=np.uint8) #frame_orig = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) frame_orig = cv2.resize( image, (int(canvas_size[0] / 2), int(canvas_size[1]))) #frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame = cv2.resize(frame, (int(canvas_size[0] / 2), int(canvas_size[1]))) canvas[:, 0:int(canvas_size[0] / 2), :] = frame_orig canvas[:, int(canvas_size[0] / 2):, :] = frame #cv2.imshow('1', frame) #cv2.waitKey(0) canvas = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB) print('canvas shape', canvas.shape) out_cap.write(canvas) elif save_to == 'simple': frame = cv2.resize(frame, (original_shape[1], original_shape[0]), interpolation=cv2.INTER_NEAREST) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) #cv2.imshow('1', frame) #cv2.waitKey(0) out_cap.write(frame) elif save_to == 'images': frame = cv2.resize(frame, (original_shape[1], original_shape[0]), interpolation=cv2.INTER_NEAREST) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) image = cv2.resize(image, (original_shape[1], original_shape[0]), interpolation=cv2.INTER_NEAREST) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) cv2.imwrite('/tmp/1.png', frame) #cv2.imwrite('/tmp/1_orig.png', image) zf = zipfile.ZipFile(zipfile_name, "a", zipfile.ZIP_DEFLATED) name = 'frame_' + '%08d' % frame_num + '.png' orig_name = 'frame_orig_' + '%08d' % frame_num + '.png' zf.write('/tmp/1.png', name) #zf.write('/tmp/1_orig.png', orig_name) zf.close() elif save_to == 'weighted': frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) frame = cv2.resize(frame, output_size) image = cv2.resize(image, output_size) frame = cv2.addWeighted(image, alpha, frame, beta, 0) out_cap.write(frame) elif save_to == 'perspective': preds = preds.squeeze() img, mask = getCutedRoad(image, preds) mask = np.expand_dims(mask, axis=0) mask = np.expand_dims(mask, axis=3) msk = decode_labels(mask, num_classes=num_classes) f = msk[0] # h, w = frame.shape[:2] # src = np.float32([[x1, y1], # br # [x0, y1], # bl # [x0, y0], # tl # [x1, y0]]) # tr # dst = np.float32([[w, h], # br # [0, h], # bl # [0, 0], # tl # [w, 0]]) # tr # M = cv2.getPerspectiveTransform(src, dst) # Minv = cv2.getPerspectiveTransform(dst, src) # warped = cv2.warpPerspective(image, M, (w, h), flags=cv2.INTER_LINEAR) # cv2.imshow('1', warped) #resized = cv2.resize(img[y0 : y1, x0 : x1], input_size, interpolation = cv2.INTER_NEAREST) print((preds == 2).sum() / (preds.shape[0] * preds.shape[1])) mask = np.array(mask) mask = mask.squeeze() print((mask == 2).sum() / (mask.shape[0] * mask.shape[1])) cv2.imshow( '2', cv2.resize(img, input_size, interpolation=cv2.INTER_NEAREST)) cv2.imshow( '3', cv2.resize(f, input_size, interpolation=cv2.INTER_NEAREST)) cv2.imshow('4', image) cv2.waitKey(0) #quit() cap.release() out_cap.release() zf.close()
def main(): num_classes = cityscapes_class MODEL = 'trainval' root_dir = "D:/ANNOTATION/CityScape/leftImg8bit/demoVideo/stuttgart_00" images = os.listdir(root_dir) # shape = img.shape[0:2] # x = tf.placeholder(dtype=tf.float32, shape=img.shape) shape = (H_RES, W_RES) x = tf.placeholder(dtype=tf.float32, shape=(H_RES, W_RES, C_RES)) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) model = model_config[MODEL] net = model({'data': img_tf}, num_classes=num_classes, filter_scale=1) print(net.inputs) for k, v in net.layers.items(): print("{:30} => {:}".format(k, v)) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, shape, num_classes) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() model_path = model_paths[MODEL] net.load(model_path, sess) print('Restore from {}'.format(model_path)) i = 0 speed = 1 while i < len(images): #img_read = misc.imread(root_dir + '/' + images[i], mode='RGB') img_read = cv2.imread(root_dir + '/' + images[i]) img_read = cv2.resize(img_read, (W_RES, H_RES)) preds = sess.run(pred, feed_dict={x: img_read}) input = cv2.cvtColor(img_read, cv2.COLOR_BGR2RGB) out = cv2.cvtColor(cv2.convertScaleAbs(preds[0]), cv2.COLOR_BGR2RGB) out = cv2.addWeighted(input, 0.3, out, 0.7, 0.0) cv2.imshow("out", out) key = cv2.waitKey(5) if key == 27: break elif key == 56: i += 100 i += speed print("frame= %d key=%d" % (i, key))
def main(): global atn global preds args = Args( img_path='D:/val/', model='others', dataset='antenna', filter_scale=1, ) args.weight_path = './snapshots/3wDataSet/model.ckpt-400000' # args.weight_path = './snapshots/3wDataSet/model.ckpt-300000' # args = get_arguments() if args.dataset == 'cityscapes': num_classes = cityscapes_class elif args.dataset == 'antenna': num_classes = antenna_classes else: num_classes = ADE20k_class # Read images from directory (size must be the same) or single input file imgs = [] ori_imgs = [] filenames = [] # [2048, 1024],[1280, 720],[3840,2160] load_shape = [2048, 1024] if os.path.isdir(args.img_path): file_paths = glob.glob(os.path.join(args.img_path, '*')) for file_path in file_paths: pic_files = glob.glob(os.path.join(file_path, '*')) for pic_file in pic_files: ext = pic_file.split('jpg') if len(ext) > 2: continue img, filename = load_img(pic_file, load_shape) imgs.append(img) ori_imgs.append(img) filenames.append(filename.replace('\\', '/')) print(filenames) else: img, filename = load_img(args.img_path) imgs.append(img) filenames.append(filename) shape = imgs[0].shape[0:2] x = tf.placeholder(dtype=tf.float32, shape=img.shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) model = model_config[args.model] net = model({'data': img_tf}, num_classes=num_classes, filter_scale=args.filter_scale) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, shape[0], shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, shape, num_classes) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() model_path = model_paths[args.model] if args.model == 'others': ckpt = tf.train.get_checkpoint_state(model_path) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=tf.global_variables()) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) # load(loader, sess, ckpt.model_checkpoint_path) load(loader, sess, args.weight_path) else: print('No checkpoint file found.') else: net.load(model_path, sess) print('Restore from {}'.format(model_path)) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) val_time = [] det_dict = {1: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 12: 0, 15: 0} fit_dict = {1: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 12: 0, 15: 0} for i in trange(len(imgs), desc='Inference', leave=True): # print(filenames[i].split('\\')) act_deg = int(filenames[i].split('/')[1]) fit_count = 0 det_count = 0 start_time = timeit.default_timer() preds = sess.run(pred, feed_dict={x: imgs[i]}) os.makedirs(args.save_dir + filenames[i].replace(filenames[i].split("/")[-1], ''), exist_ok=True) misc.imsave(args.save_dir + filenames[i].split('.')[0] + '_Pred.jpg', preds[0]) # misc.imsave(args.save_dir + filenames[i].split('.')[0] + 'Ori.jpg', ori_imgs[i]) # print(preds[0][0][0]) p = np.array(np.where(preds[0] != [90, 90, 90])[:-1]) p = np.dstack((p[0], p[1]))[0] if p.shape[0] == 0: continue uni_p = np.unique(p, axis=0) atns = [] atn = Antenna() atn.l_line.append(uni_p[0]) # 遍历所有标记为天线类别的像素点 for p_i in range(1, len(uni_p)): point = uni_p[p_i] last_point = uni_p[p_i-1] if not same_atn(point, last_point): atn.r_line.append(last_point) # 每次插完右边点检查是否换行 if not same_row(point, last_point): # 成对插入天线左右边像素点 row = atn_2_row(atn) atn.clear() # 第一次换行时初始化天线对象列表atns if not atns: atns = init_atns(row, preds) row.clear() atn.l_line.append(point) continue # 天线左右边像素点组合遍历所有天线对象最后一个点作差,相差最小则插入天线对象 for atns_i in range(len(atns)): if not row: break dis = np.abs(np.array(row) - atns[atns_i].last_point()) m_dis = np.argmin(np.sum(np.sum(dis, axis=1), axis=1)) if np.abs(row[m_dis][0][0] - atns[atns_i].last_point()[0][0]) == 1 \ and np.abs(row[m_dis][0][1] - atns[atns_i].last_point()[0][1]) <= 1\ and np.abs(row[m_dis][1][1] - atns[atns_i].last_point()[1][1]) <= 1: # print(np.abs(row[m_dis][0][1] - atns[atns_i].last_point()[0][1]), np.abs(row[m_dis][1][1] - atns[atns_i].last_point()[1][1])) atns[atns_i].append(row[m_dis], preds, atns_i) row.pop(m_dis) if row: for row_j in range(0, len(row)): atn_tmp = Antenna() atn_tmp.append(row[row_j], preds, len(atns)) atns.append(atn_tmp) row.clear() atn.l_line.append(point) # 遍历天线对象,去除左右边拟合计算下倾角 atns = [atns[atns_i] for atns_i in range(0, len(atns)) if len(atns[atns_i].l_line)>75] for k in range(len(atns)): a = np.array(atns[k].l_line) # 天线左边 x1 = a[:, :1].flatten() dis = int(len(x1)/10) x1 = x1[dis:-dis] y = a[:, 1:].flatten()[dis:-dis] z1 = np.polyfit(x1, y, 1) # 一次多项式拟合,相当于线性拟合 p1 = np.poly1d(z1) y1 = p1(x1).astype(np.int) # 天线右边 b = np.array(atns[k].r_line) x2 = b[:, :1].flatten()[dis:-dis] y2 = b[:, 1:].flatten()[dis:-dis] z2 = np.polyfit(x2, y2, 1) p2 = np.poly1d(z2) yl2 = p2(x2).astype(np.int) # plt.figure() # plt.scatter(x1, y, 25, "red") # plt.scatter(x2, y2, 25, "green") # # plt.plot(x1, y1, 'blue') # plt.plot(x2, yl2, 'yellow') # plt.show() cur_img = np.array(ori_imgs[i], dtype='float32') ori_imgs[i] = cv2.line(cur_img, (y1[0], x1[0]), (y1[-1], x1[-1]), (199, 0, 69), 3) ori_imgs[i] = cv2.line(cur_img, (yl2[0], x2[0]), (yl2[-1], x2[-1]), (199, 0, 69), 3) # l_degree = (x1[-1] - x1[0])/(y1[-1] - y1[0]) l_degree = (math.atan2((x1[-1] - x1[0]), (y1[-1] - y1[0])))*180/math.pi # r_degree = (x2[-1] - x2[0])/(yl2[-1] - yl2[0]) r_degree = (math.atan2((x2[-1] - x2[0]), (yl2[-1] - yl2[0])))*180/math.pi degree = abs(round((l_degree+r_degree)/2-90, 2)) acc_dis = abs(degree - act_deg) if acc_dis < 3: det_count = det_count + 1 if acc_dis < 1: fit_count = fit_count + 1 # print(l_degree,r_degree, degree) ori_imgs[i] = cv2.putText(cur_img, str(degree), (yl2[0], x2[0]), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 3) # print((y1[0], x[0]), (y1[-1], x[-1])) # print((yl2[0], x2[0]), (yl2[-1], x2[-1])) overlayed_img = cv2.addWeighted(np.array(ori_imgs[i], dtype='float32'), 0.7, preds[0], 0.3, 0) # os.makedirs(args.save_dir + filenames[i].split('Screen')[0], exist_ok=True) fit_dict[act_deg] += fit_count det_dict[act_deg] += det_count # print(args.save_dir + filenames[i], det_count, fit_count) misc.imsave(args.save_dir + filenames[i], overlayed_img) print(args.save_dir + filenames[i]) elapsed = timeit.default_timer() - start_time print('inference time: {}'.format(elapsed)) val_time.append(elapsed) print('mean_val_time:', np.mean(val_time[1:])) print('detection:', det_dict, '%.3f' % (sum(det_dict.values())/158*100) + "%") print('fitting:', fit_dict, '%.3f' % (sum(fit_dict.values())/158*100) + "%") print(val_time)
def main(): args = get_arguments() if args.img_path[-4] != '.': files = GetAllFilesListRecusive(args.img_path, ['.jpg', '.jpeg', '.png']) else: files = [args.img_path] shape = INPUT_SIZE.split(',') shape = (int(shape[0]), int(shape[1]), 3) x = tf.placeholder(dtype=tf.float32, shape=shape) img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) # Create network. net = ICNet_BN({'data': img_tf}, is_training=False, num_classes=num_classes) # Predictions. raw_output = net.layers['conv6_cls'] output = tf.image.resize_bilinear(raw_output, tf.shape(img_tf)[1:3, ]) output = tf.argmax(output, dimension=3) pred = tf.expand_dims(output, dim=3) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) restore_var = tf.global_variables() ckpt = tf.train.get_checkpoint_state(args.snapshots_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) for path in files: img, filename = load_img(path) preds = sess.run(pred, feed_dict={x: img}) msk = decode_labels(preds, num_classes=num_classes) im = msk[0] if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) if args.weighted: indx = (im == [0, 0, 0]) im = cv2.addWeighted(im, 0.7, img, 0.7, -15) im[indx] = img[indx] cv2.imwrite(args.save_dir + filename.replace('.jpg', '.png'), im)
img_tf = preprocess(x) img_tf, n_shape = check_input(img_tf) # Create network. net = ICNet({'data': img_tf}, num_classes=num_classes, filter_scale=1) raw_output = net.layers['conv6_cls'] # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=n_shape, align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, img.shape[0], img.shape[1]) raw_output_up = tf.argmax(raw_output_up, axis=3) pred = decode_labels(raw_output_up, img.shape, num_classes) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) net.load(model_path, sess) print('Restore from {}'.format(model_path)) preds = sess.run(pred, feed_dict={x: img}) plt.figure(1, [15, 30])
def main(): args = get_arguments() img, filename = load_img(args.img_path) img_shape = tf.shape(img) h, w = (tf.maximum(crop_size[0], img_shape[0]), tf.maximum(crop_size[1], img_shape[1])) img = preprocess(img, h, w) # Create network. net = PSPNet({'data': img}, is_training=False, num_classes=num_classes) with tf.variable_scope('', reuse=True): flipped_img = tf.image.flip_left_right(tf.squeeze(img)) flipped_img = tf.expand_dims(flipped_img, dim=0) net2 = PSPNet({'data': flipped_img}, is_training=False, num_classes=num_classes) raw_output = net.layers['conv6'] if args.flipped_eval: flipped_output = tf.image.flip_left_right( tf.squeeze(net2.layers['conv6'])) flipped_output = tf.expand_dims(flipped_output, dim=0) raw_output = tf.add_n([raw_output, flipped_output]) # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=[h, w], align_corners=True) raw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, img_shape[0], img_shape[1]) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) restore_var = tf.global_variables() ckpt = tf.train.get_checkpoint_state(args.model) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') preds = sess.run(pred) msk = decode_labels(preds, num_classes=num_classes) im = Image.fromarray(msk[0]) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) im.save(args.save_dir + filename)
img_list.sort() seg_list = grab_all_entries('/local/Data/NIPS18/kitti_raw_data_seg', '_compact.npy') seg_list.sort() assert img_list.size == seg_list.size # create dataset loader dataloader = tf.data.Dataset.from_tensor_slices((img_list, seg_list)) dataloader = dataloader.map(load_func) iterator = dataloader.make_initializable_iterator() next_element = iterator.get_next() sess = tf.Session() sess.run(iterator.initializer) label_map = decode_labels(tf.argmax(next_element[1], axis=2), img_shape=[256, 512], num_classes=19) while True: try: (image, prob_mask), mask = sess.run([next_element, label_map]) plt.clf() plt.subplot(131) plt.imshow(image.astype(np.uint8)) plt.subplot(132) plt.imshow(mask[0, ...].astype(np.uint8)) plt.subplot(133) blend = cv2.addWeighted(image, 0.7, mask[0, ...], 0.3, 0) plt.imshow(blend.astype(np.uint8)) plt.show() except tf.errors.OutOfRangeError: