def evaluate_model_fast(model_name,path):
model = keras.models.load_model(model_name)
images,labels,files = train_model.load_images_classes(path)
predictions = []
errors = []
num_classes = len(train_model.get_classes())
confusion_matrix = np.zeros((num_classes,num_classes))
print("Evaluating model (fast version) on %i instances..." % images.shape[0])
start = time.time()
prediction_probs = model.predict(images)
print("Elased time: %f seconds" % (time.time()-start))
for i in range(0,images.shape[0]):
predicted_class = np.argmax(prediction_probs[i,:])
predictions.append(predicted_class)
label = int(labels[i])
confusion_matrix[label,predicted_class] += 1
if label == predicted_class:
errors.append(0)
else:
save_error(label,predicted_class,files[i])
errors.append(1)
print_statistics(confusion_matrix)
def evaluate_model(model_name, path):
model = keras.models.load_model(model_name)
images,labels,files = train_model.load_images_classes(path)
predictions = []
errors = []
num_classes = len(train_model.get_classes())
confusion_matrix = np.zeros((num_classes,num_classes))
print("Evaluating model...")
if os.path.isdir("./errors"):
shutil.rmtree("./errors")
for i in range(0,images.shape[0]):
if i % 1000 == 0:
print(i)
prediction_probs = predict_image_set_with_augmentation(images[i:i+1,:,:,:],model)[0]
predicted_class = np.argmax(prediction_probs)
predictions.append(predicted_class)
label = int(labels[i])
confusion_matrix[label,predicted_class] += 1
if label == predicted_class:
errors.append(0)
else:
#save_error(label,predicted_class,files[i])
errors.append(1)
print_statistics(confusion_matrix)
def annotate_and_save_per_class(annotations, d, image_file, model_name, precision_mode):
base_image_file = image_file.replace('Results/seeds/annotations_trt', 'Data')
img = Image.open(base_image_file)
classes = train_model.get_classes()
labels = load_labels(image_file)
classes_image = []
for c in range(0,len(classes)):
new_img = img.copy()
draw = ImageDraw.Draw(new_img, mode='RGBA')
for i in range(0,len(annotations)):
color,predicted_class = utils.get_color_probs(annotations[i][0])
classes_image.append(predicted_class)
if predicted_class==c:
x,y = (annotations[i][1], annotations[i][2])
draw.rectangle((x-d/2, y-d/2, x+d/2, y+d/2), fill=color)
for (c_name,x,y) in labels:
solid_color = utils.get_color(classes.index(c_name),255)
draw.ellipse((x-6, y-6, x+6, y+6), fill=solid_color)
new_image_file = image_file.replace('.' + image_format, '_annotated_by_' + precision_mode + '_' + model_name + '_for_class_%s.jpg' % classes[c])
new_img.save(new_image_file)
print("Saved %s" % new_image_file)
return classes_image
def print_statistics(confusion_matrix):
accuracy = np.trace(confusion_matrix)/np.sum(confusion_matrix)
print("Accuracy = %.4f (on %i test instances)" % (accuracy,np.sum(confusion_matrix)))
classes = train_model.get_classes()
for i in range(0,len(classes)):
recall_class = confusion_matrix[i,i]/np.sum(confusion_matrix[i,:])
precision_class = confusion_matrix[i,i]/np.sum(confusion_matrix[:,i])
print("Class %s precision/recall: %0.4f/%0.4f" % (classes[i],precision_class,recall_class))
print("Confusion matrix (rows = true class, columns = predicted class):\n")
for i in range(0,6):
for j in range(0,6):
s = "%i" % int(confusion_matrix[i,j])
while len(s) < 6:
s = " "+s
sys.stdout.write(s)
sys.stdout.write("\n")
def annotations_from_inference(
inference_results_fp='./Inferences/trt_model_predictions.npy',
image_fp='./Data/201x201_TestSetMidFull/DSC03534.JPG',
save_annotations=False):
""" Takes a (numpy) array and creates annotations from it.
@Args
inference_results_fp (str) : ./path/to/inference.npy
image_fp (str) : ./path/to/image.jpg
@Returns
"""
img = Image.open(image_fp)
sx, sy = img.size
inference_results = np.load(inference_results_fp)
print("Computing annotations...")
annotations = []
d = 4
for x in range(100, sx - 101, d):
for y in range(100, sy - 101, d):
x0 = int(round(float(x - 100) / 4) + 15)
y0 = int(round(float(y - 100) / 4) + 15)
probs_flex = np.squeeze(inference_results[0, y0, x0, :])
annotations.append((probs_flex, x, y))
if save_annotations:
annotate_and_save(annotations, d, image_fp)
annotate_and_save_per_class(annotations, d, image_fp)
labels = load_labels(image_fp)
confusion_matrix = np.zeros((6, 6))
for (c_name, x, y) in labels:
if 100 <= x < sx - 101 and 100 <= y < sy - 101:
x0 = int(round(float(x - 100) / 4) + 15)
y0 = int(round(float(y - 100) / 4) + 15)
probs_flex = np.squeeze(inference_results[0, y0, x0, :])
predicted_class = np.argmax(probs_flex)
c = train_model.get_classes().index(c_name)
confusion_matrix[c, predicted_class] += 1
evaluate_model.print_statistics(confusion_matrix)
return confusion_matrix
def predict_image(model, image_file, model_name='', precision_mode='FP32'):
img = Image.open(image_file)
sx,sy = img.size
print("Predicting for image %s (%i x %i pixel)" % (image_file, sx, sy))
d = 4
annotations = []
for x in range(0, sx-201, d):
crops = []
print(x)
for y in range(0, sy-201, d):
crop = img.crop((x, y, x+201, y+201))
crops.append((crop, x+100, y+100))
crops_np = np.zeros((len(crops), 201, 201, 3))
for i in range(0,len(crops)):
crops_np[i, :, :, :] = np.array(crops[i][0])/255.0
print("Predicting on %i crops" % crops_np.shape[0])
start = time.time()
predictions = model.predict(crops_np)
elapsed = time.time() - start
print("Took %f seconds per crop" % (elapsed/float(crops_np.shape[0])))
for i in range(0,len(crops)):
annotations.append((predictions[i, :],crops[i][1], crops[i][2]))
annotate_and_save(annotations, d, image_file, model_name, precision_mode)
annotate_and_save_per_class(annotations, d, image_file, model_name, precision_mode)
labels = load_labels(image_file)
confusion_matrix = np.zeros((num_classes,num_classes))
for (c_name,x,y) in labels:
if 100 <= x < sx-101 and 100 <= y < sy-201:
crop = img.crop((x-100, y-100, x+101, y+101))
crop_np = np.array(crop)/255.0
probs = model.predict(crop_np.reshape(1, 201, 201, 3))
predicted_class = np.argmax(probs)
c = train_model.get_classes().index(c_name)
confusion_matrix[c, predicted_class] += 1
return confusion_matrix
def save_error(label,prediction,file):
classes = train_model.get_classes()
path = "./errors/%s_classified_as_%s" % (classes[label],classes[prediction])
if not os.path.isdir(path):
os.makedirs(path)
shutil.copyfile(file,path+"/"+os.path.basename(file))
def main(seed, filter_, num_classes, setup, model_name, images_dir, precision_mode, test):
""" The main method is responsible for running the generation of confusion matrices,
the time taken for inferences, and save them. It iterates over all seeds and filters,
in our case, seeds ranges from 1 to 5, and filters from 2-4 to 16-32.
Args:
seed (str): The seed that generated the model loaded
filter_ (list): List of strings, containing the first and second filter used, such as ['2','4']
num_classes (int): Number of classes of our problem.
setup (dict): Dictionary that maps the model setup to its filter size
model_name (str): Name of model to be used, eg `resnet_manual_highres_center_only_f1_2_f2_4`
images_dir (str): Path to the directory where the images are located
precision_mode (str): precision_mode (str): either 'FP32' or 'FP16'
Returns:
None
"""
f1, f2 = filter_
model_name = 'flex_random_seed_{}_resnet_manual_highres_center_only_f1_{}_f2_{}'.format(seed, f1, f2)
frozen_graph_filepath = './Models/Frozen_graphs/{}_{}/'.format(f1,f2) + model_name + '_frozen_graph.pb'
frozen_graph, x_tensor, y_tensor = trt_frozen_graph_and_tensors(
model_name=model_name,
frozen_graph_filepath=frozen_graph_filepath,
precision_mode=precision_mode
)
elapsed_time_full_dataset = []
sum_of_confusion_matrices = np.zeros((6, 6))
with tf.compat.v1.Session(graph=frozen_graph) as sess:
for image_file in [img for img in os.listdir(images_dir) if img.endswith('.JPG')]:
img = Image.open(images_dir + image_file)
sx,sy = img.size
print("Image size is %i x %i" % (sx,sy)) # sx = 4912, sy = 3264
print("Loading image %s" % image_file)
img_np = np.array(img)/255.0
del img
print("Predicting for image %s (%i x %i pixel)" % (image_file,sx,sy))
start = time.time()
predictions_flex = sess.run(y_tensor, feed_dict={x_tensor:np.expand_dims(img_np, 0)})
elapsed = time.time() - start
elapsed_time_full_dataset.append(elapsed)
del img_np #deleting afterwards to not take the deleting time into account
print("Prediction took %f seconds (inference on full image)" % elapsed)
print("Merging predictions")
# merge the predictions on the quarter images
predictions_flex_combined = np.zeros(predictions_flex.shape)
elapsed = time.time()-start
if embedded_version:
print("Prediction took %f seconds (inference on split up image)" % elapsed)
if embedded_version:
predictions_flex = predictions_flex_combined
if save_annotations:
print("Computing annotations...")
annotations = []
d = 4
for x in range(100, sx-101, d):
for y in range(100, sy-101, d):
x0 = int(round(float(x-100)/4) + 15)
y0 = int(round(float(y-100)/4) + 15)
probs_flex = np.squeeze(predictions_flex[0, y0, x0, :])
annotations.append((probs_flex, x, y))
if test: # add a prefix for test to not replace real experiments
model_name = 'TEST_' + model_name
# saving annotations
annotation_dir = images_dir.replace('Data', 'Results/seeds/annotations_trt') + image_file
annotate_and_save(annotations, d, annotation_dir, model_name, precision_mode)
classes_image = annotate_and_save_per_class(
annotations,
d,
annotation_dir,
model_name,
precision_mode
)
labels = load_labels(annotation_dir)
confusion_matrix = np.zeros((num_classes, num_classes))
for (c_name, x, y) in labels:
if 100 <= x < sx-101 and 100 <= y < sy-101:
x0 = int(round(float(x-100)/4) + 15 )
y0 = int(round(float(y-100)/4) + 15)
probs_flex = np.squeeze(predictions_flex[0, y0, x0, :])
predicted_class = np.argmax(probs_flex)
c = train_model.get_classes().index(c_name)
confusion_matrix[c, predicted_class] += 1
print(confusion_matrix)
sum_of_confusion_matrices += confusion_matrix
print(sum_of_confusion_matrices)
sum_of_cm_fp = './Results/seeds/preds_trt/{}/{}_{}/sum_of_cm_'\
.format(precision_mode.lower(), f1,f2) + model_name + '_fp32.npy'
elapsed_time_fp = './Results/seeds/elapsed_trt/{}/{}_{}/time_taken_'\
.format(precision_mode.lower(), f1,f2) + model_name + '_fp32.npy'
np.save(sum_of_cm_fp, sum_of_confusion_matrices)
np.save(elapsed_time_fp, elapsed_time_full_dataset)
tf.reset_default_graph()
def predict_image_highres(frozen_graph, x_tensor, y_tensor, image_file, model_name='', precision_mode='FP32'):
img = Image.open(image_file)
sx,sy = img.size
print("Image size is %i x %i" % (sx, sy)) # sx = 4912, sy = 3264
print("Loading image %s" % image_file)
img_np = np.array(img)/255.0
print(img_np.shape)
if embedded_version:
img = None
print("Predicting for image %s (%i x %i pixel)" % (image_file, sx, sy))
if not embedded_version:
with tf.Session(graph=frozen_graph) as sess:
start = time.time()
predictions_flex = sess.run(y_tensor, feed_dict={x_tensor:np.expand_dims(img_np, 0)})
print('predictions_flex.shape')
print(predictions_flex.shape)
elapsed = time.time() - start
del img_np
print("Prediction took %f seconds (inference on full image)" % elapsed)
print("Merging predictions")
# merge the predictions on the quarter images
predictions_flex_combined = np.zeros(predictions_flex.shape)
elapsed = time.time() - start
if embedded_version:
print("Prediction took %f seconds (inference on split up image)" % elapsed)
if embedded_version:
predictions_flex = predictions_flex_combined
if save_annotations:
print("Computing annotations...")
annotations = []
d = 4
for x in range(100, sx-101, d):
for y in range(100, sy-101, d):
x0 = int(round(float(x-100)/4) + 15)
y0 = int(round(float(y-100)/4) + 15)
probs_flex = np.squeeze(predictions_flex[0, y0, x0, :])
annotations.append((probs_flex, x, y))
annotate_and_save(annotations, d, image_file, model_name, precision_mode)
classes_image = annotate_and_save_per_class(annotations, d, image_file, model_name, precision_mode)
labels = load_labels(image_file)
confusion_matrix = np.zeros((num_classes, num_classes))
for (c_name, x, y) in labels:
if 100 <= x < sx-101 and 100 <= y < sy-101:
x0 = int(round(float(x-100)/4)+15)
y0 = int(round(float(y-100)/4)+15)
probs_flex = np.squeeze(predictions_flex[0, y0, x0, :])
predicted_class = np.argmax(probs_flex)
c = train_model.get_classes().index(c_name)
confusion_matrix[c, predicted_class] += 1
if save_annotations:
return confusion_matrix, classes_image
else:
return confusion_matrix