def evaluate_model_fast(model_name,path): model = keras.models.load_model(model_name) images,labels,files = train_model.load_images_classes(path) predictions = [] errors = [] num_classes = len(train_model.get_classes()) confusion_matrix = np.zeros((num_classes,num_classes)) print("Evaluating model (fast version) on %i instances..." % images.shape[0]) start = time.time() prediction_probs = model.predict(images) print("Elased time: %f seconds" % (time.time()-start)) for i in range(0,images.shape[0]): predicted_class = np.argmax(prediction_probs[i,:]) predictions.append(predicted_class) label = int(labels[i]) confusion_matrix[label,predicted_class] += 1 if label == predicted_class: errors.append(0) else: save_error(label,predicted_class,files[i]) errors.append(1) print_statistics(confusion_matrix)
def evaluate_model(model_name, path): model = keras.models.load_model(model_name) images,labels,files = train_model.load_images_classes(path) predictions = [] errors = [] num_classes = len(train_model.get_classes()) confusion_matrix = np.zeros((num_classes,num_classes)) print("Evaluating model...") if os.path.isdir("./errors"): shutil.rmtree("./errors") for i in range(0,images.shape[0]): if i % 1000 == 0: print(i) prediction_probs = predict_image_set_with_augmentation(images[i:i+1,:,:,:],model)[0] predicted_class = np.argmax(prediction_probs) predictions.append(predicted_class) label = int(labels[i]) confusion_matrix[label,predicted_class] += 1 if label == predicted_class: errors.append(0) else: #save_error(label,predicted_class,files[i]) errors.append(1) print_statistics(confusion_matrix)
def annotate_and_save_per_class(annotations, d, image_file, model_name, precision_mode): base_image_file = image_file.replace('Results/seeds/annotations_trt', 'Data') img = Image.open(base_image_file) classes = train_model.get_classes() labels = load_labels(image_file) classes_image = [] for c in range(0,len(classes)): new_img = img.copy() draw = ImageDraw.Draw(new_img, mode='RGBA') for i in range(0,len(annotations)): color,predicted_class = utils.get_color_probs(annotations[i][0]) classes_image.append(predicted_class) if predicted_class==c: x,y = (annotations[i][1], annotations[i][2]) draw.rectangle((x-d/2, y-d/2, x+d/2, y+d/2), fill=color) for (c_name,x,y) in labels: solid_color = utils.get_color(classes.index(c_name),255) draw.ellipse((x-6, y-6, x+6, y+6), fill=solid_color) new_image_file = image_file.replace('.' + image_format, '_annotated_by_' + precision_mode + '_' + model_name + '_for_class_%s.jpg' % classes[c]) new_img.save(new_image_file) print("Saved %s" % new_image_file) return classes_image
def print_statistics(confusion_matrix): accuracy = np.trace(confusion_matrix)/np.sum(confusion_matrix) print("Accuracy = %.4f (on %i test instances)" % (accuracy,np.sum(confusion_matrix))) classes = train_model.get_classes() for i in range(0,len(classes)): recall_class = confusion_matrix[i,i]/np.sum(confusion_matrix[i,:]) precision_class = confusion_matrix[i,i]/np.sum(confusion_matrix[:,i]) print("Class %s precision/recall: %0.4f/%0.4f" % (classes[i],precision_class,recall_class)) print("Confusion matrix (rows = true class, columns = predicted class):\n") for i in range(0,6): for j in range(0,6): s = "%i" % int(confusion_matrix[i,j]) while len(s) < 6: s = " "+s sys.stdout.write(s) sys.stdout.write("\n")
def annotations_from_inference( inference_results_fp='./Inferences/trt_model_predictions.npy', image_fp='./Data/201x201_TestSetMidFull/DSC03534.JPG', save_annotations=False): """ Takes a (numpy) array and creates annotations from it. @Args inference_results_fp (str) : ./path/to/inference.npy image_fp (str) : ./path/to/image.jpg @Returns """ img = Image.open(image_fp) sx, sy = img.size inference_results = np.load(inference_results_fp) print("Computing annotations...") annotations = [] d = 4 for x in range(100, sx - 101, d): for y in range(100, sy - 101, d): x0 = int(round(float(x - 100) / 4) + 15) y0 = int(round(float(y - 100) / 4) + 15) probs_flex = np.squeeze(inference_results[0, y0, x0, :]) annotations.append((probs_flex, x, y)) if save_annotations: annotate_and_save(annotations, d, image_fp) annotate_and_save_per_class(annotations, d, image_fp) labels = load_labels(image_fp) confusion_matrix = np.zeros((6, 6)) for (c_name, x, y) in labels: if 100 <= x < sx - 101 and 100 <= y < sy - 101: x0 = int(round(float(x - 100) / 4) + 15) y0 = int(round(float(y - 100) / 4) + 15) probs_flex = np.squeeze(inference_results[0, y0, x0, :]) predicted_class = np.argmax(probs_flex) c = train_model.get_classes().index(c_name) confusion_matrix[c, predicted_class] += 1 evaluate_model.print_statistics(confusion_matrix) return confusion_matrix
def predict_image(model, image_file, model_name='', precision_mode='FP32'): img = Image.open(image_file) sx,sy = img.size print("Predicting for image %s (%i x %i pixel)" % (image_file, sx, sy)) d = 4 annotations = [] for x in range(0, sx-201, d): crops = [] print(x) for y in range(0, sy-201, d): crop = img.crop((x, y, x+201, y+201)) crops.append((crop, x+100, y+100)) crops_np = np.zeros((len(crops), 201, 201, 3)) for i in range(0,len(crops)): crops_np[i, :, :, :] = np.array(crops[i][0])/255.0 print("Predicting on %i crops" % crops_np.shape[0]) start = time.time() predictions = model.predict(crops_np) elapsed = time.time() - start print("Took %f seconds per crop" % (elapsed/float(crops_np.shape[0]))) for i in range(0,len(crops)): annotations.append((predictions[i, :],crops[i][1], crops[i][2])) annotate_and_save(annotations, d, image_file, model_name, precision_mode) annotate_and_save_per_class(annotations, d, image_file, model_name, precision_mode) labels = load_labels(image_file) confusion_matrix = np.zeros((num_classes,num_classes)) for (c_name,x,y) in labels: if 100 <= x < sx-101 and 100 <= y < sy-201: crop = img.crop((x-100, y-100, x+101, y+101)) crop_np = np.array(crop)/255.0 probs = model.predict(crop_np.reshape(1, 201, 201, 3)) predicted_class = np.argmax(probs) c = train_model.get_classes().index(c_name) confusion_matrix[c, predicted_class] += 1 return confusion_matrix
def save_error(label,prediction,file): classes = train_model.get_classes() path = "./errors/%s_classified_as_%s" % (classes[label],classes[prediction]) if not os.path.isdir(path): os.makedirs(path) shutil.copyfile(file,path+"/"+os.path.basename(file))
def main(seed, filter_, num_classes, setup, model_name, images_dir, precision_mode, test): """ The main method is responsible for running the generation of confusion matrices, the time taken for inferences, and save them. It iterates over all seeds and filters, in our case, seeds ranges from 1 to 5, and filters from 2-4 to 16-32. Args: seed (str): The seed that generated the model loaded filter_ (list): List of strings, containing the first and second filter used, such as ['2','4'] num_classes (int): Number of classes of our problem. setup (dict): Dictionary that maps the model setup to its filter size model_name (str): Name of model to be used, eg `resnet_manual_highres_center_only_f1_2_f2_4` images_dir (str): Path to the directory where the images are located precision_mode (str): precision_mode (str): either 'FP32' or 'FP16' Returns: None """ f1, f2 = filter_ model_name = 'flex_random_seed_{}_resnet_manual_highres_center_only_f1_{}_f2_{}'.format(seed, f1, f2) frozen_graph_filepath = './Models/Frozen_graphs/{}_{}/'.format(f1,f2) + model_name + '_frozen_graph.pb' frozen_graph, x_tensor, y_tensor = trt_frozen_graph_and_tensors( model_name=model_name, frozen_graph_filepath=frozen_graph_filepath, precision_mode=precision_mode ) elapsed_time_full_dataset = [] sum_of_confusion_matrices = np.zeros((6, 6)) with tf.compat.v1.Session(graph=frozen_graph) as sess: for image_file in [img for img in os.listdir(images_dir) if img.endswith('.JPG')]: img = Image.open(images_dir + image_file) sx,sy = img.size print("Image size is %i x %i" % (sx,sy)) # sx = 4912, sy = 3264 print("Loading image %s" % image_file) img_np = np.array(img)/255.0 del img print("Predicting for image %s (%i x %i pixel)" % (image_file,sx,sy)) start = time.time() predictions_flex = sess.run(y_tensor, feed_dict={x_tensor:np.expand_dims(img_np, 0)}) elapsed = time.time() - start elapsed_time_full_dataset.append(elapsed) del img_np #deleting afterwards to not take the deleting time into account print("Prediction took %f seconds (inference on full image)" % elapsed) print("Merging predictions") # merge the predictions on the quarter images predictions_flex_combined = np.zeros(predictions_flex.shape) elapsed = time.time()-start if embedded_version: print("Prediction took %f seconds (inference on split up image)" % elapsed) if embedded_version: predictions_flex = predictions_flex_combined if save_annotations: print("Computing annotations...") annotations = [] d = 4 for x in range(100, sx-101, d): for y in range(100, sy-101, d): x0 = int(round(float(x-100)/4) + 15) y0 = int(round(float(y-100)/4) + 15) probs_flex = np.squeeze(predictions_flex[0, y0, x0, :]) annotations.append((probs_flex, x, y)) if test: # add a prefix for test to not replace real experiments model_name = 'TEST_' + model_name # saving annotations annotation_dir = images_dir.replace('Data', 'Results/seeds/annotations_trt') + image_file annotate_and_save(annotations, d, annotation_dir, model_name, precision_mode) classes_image = annotate_and_save_per_class( annotations, d, annotation_dir, model_name, precision_mode ) labels = load_labels(annotation_dir) confusion_matrix = np.zeros((num_classes, num_classes)) for (c_name, x, y) in labels: if 100 <= x < sx-101 and 100 <= y < sy-101: x0 = int(round(float(x-100)/4) + 15 ) y0 = int(round(float(y-100)/4) + 15) probs_flex = np.squeeze(predictions_flex[0, y0, x0, :]) predicted_class = np.argmax(probs_flex) c = train_model.get_classes().index(c_name) confusion_matrix[c, predicted_class] += 1 print(confusion_matrix) sum_of_confusion_matrices += confusion_matrix print(sum_of_confusion_matrices) sum_of_cm_fp = './Results/seeds/preds_trt/{}/{}_{}/sum_of_cm_'\ .format(precision_mode.lower(), f1,f2) + model_name + '_fp32.npy' elapsed_time_fp = './Results/seeds/elapsed_trt/{}/{}_{}/time_taken_'\ .format(precision_mode.lower(), f1,f2) + model_name + '_fp32.npy' np.save(sum_of_cm_fp, sum_of_confusion_matrices) np.save(elapsed_time_fp, elapsed_time_full_dataset) tf.reset_default_graph()
def predict_image_highres(frozen_graph, x_tensor, y_tensor, image_file, model_name='', precision_mode='FP32'): img = Image.open(image_file) sx,sy = img.size print("Image size is %i x %i" % (sx, sy)) # sx = 4912, sy = 3264 print("Loading image %s" % image_file) img_np = np.array(img)/255.0 print(img_np.shape) if embedded_version: img = None print("Predicting for image %s (%i x %i pixel)" % (image_file, sx, sy)) if not embedded_version: with tf.Session(graph=frozen_graph) as sess: start = time.time() predictions_flex = sess.run(y_tensor, feed_dict={x_tensor:np.expand_dims(img_np, 0)}) print('predictions_flex.shape') print(predictions_flex.shape) elapsed = time.time() - start del img_np print("Prediction took %f seconds (inference on full image)" % elapsed) print("Merging predictions") # merge the predictions on the quarter images predictions_flex_combined = np.zeros(predictions_flex.shape) elapsed = time.time() - start if embedded_version: print("Prediction took %f seconds (inference on split up image)" % elapsed) if embedded_version: predictions_flex = predictions_flex_combined if save_annotations: print("Computing annotations...") annotations = [] d = 4 for x in range(100, sx-101, d): for y in range(100, sy-101, d): x0 = int(round(float(x-100)/4) + 15) y0 = int(round(float(y-100)/4) + 15) probs_flex = np.squeeze(predictions_flex[0, y0, x0, :]) annotations.append((probs_flex, x, y)) annotate_and_save(annotations, d, image_file, model_name, precision_mode) classes_image = annotate_and_save_per_class(annotations, d, image_file, model_name, precision_mode) labels = load_labels(image_file) confusion_matrix = np.zeros((num_classes, num_classes)) for (c_name, x, y) in labels: if 100 <= x < sx-101 and 100 <= y < sy-101: x0 = int(round(float(x-100)/4)+15) y0 = int(round(float(y-100)/4)+15) probs_flex = np.squeeze(predictions_flex[0, y0, x0, :]) predicted_class = np.argmax(probs_flex) c = train_model.get_classes().index(c_name) confusion_matrix[c, predicted_class] += 1 if save_annotations: return confusion_matrix, classes_image else: return confusion_matrix