def evaluate(generator, action, threshold=0.05):
"""
Evaluate the model and saves results
:param generator: generator for validation dataset
:param action: eval the original or quantized model
:param threshold: Score Threshold
:return:
"""
in_tensor = "input_1:0"
out_tensor = [
'filtered_detections/map/TensorArrayStack/TensorArrayGatherV3:0',
'filtered_detections/map/TensorArrayStack_1/TensorArrayGatherV3:0',
'filtered_detections/map/TensorArrayStack_2/TensorArrayGatherV3:0'
]
with tf.Session() as new_sess:
if action == 'eval_original':
saver = tf.train.import_meta_graph('./original_model.ckpt.meta')
saver.restore(new_sess, './original_model.ckpt')
else:
new_quantsim = load_checkpoint('./quantzied_sim.ckpt',
'orig_quantsim_model')
new_sess = new_quantsim.session
model = TFRunWrapper(new_sess, in_tensor, out_tensor)
evaluate_coco(generator, model, threshold)
def main(config, args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# make save path if it doesn't exist
if args.save_path is not None and not os.path.exists(args.save_path):
os.makedirs(args.save_path)
# create the generator
generator = create_generator(args, config)
# load the model
print('Loading model, this may take a second...')
model = models.load_model(args.model,
backbone_name=args.backbone,
convert=args.convert_model)
# print model summary
# print(model.summary())
# start evaluation
if args.dataset_type == 'coco':
from keras_retinanet.utils.coco_eval import evaluate_coco
evaluate_coco(generator, model, args.score_threshold)
else:
average_precisions = evaluate(generator,
model,
iou_threshold=args.iou_threshold,
score_threshold=args.score_threshold,
max_detections=args.max_detections,
save_path=args.save_path)
# print evaluation
present_classes = 0
precision = 0
for label, (average_precision,
num_annotations) in average_precisions.items():
print('{:.0f} instances of class'.format(num_annotations),
generator.label_to_name(label),
'with average precision: {:.4f}'.format(average_precision))
if num_annotations > 0:
present_classes += 1
precision += average_precision
print('mAP: {:.4f}'.format(precision / present_classes))
def on_epoch_end(self, epoch, logs={}):
evaluate_coco(self.generator, self.model, self.threshold)
return parser.parse_args()
if __name__ == '__main__':
# parse arguments
args = parse_args()
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# create the model
print('Loading model, this may take a second...')
model = keras.models.load_model(args.model, custom_objects=custom_objects)
# create image data generator object
test_image_data_generator = keras.preprocessing.image.ImageDataGenerator()
# create a generator for testing data
test_generator = CocoGenerator(
args.coco_path,
args.set,
test_image_data_generator,
)
evaluate_coco(test_generator, model, args.score_threshold)