def evaluate(logger, graph, database_dir, test_data_dir, num_similar, treshold): """ Evaluates the classifier frozen in graph using the images of database_dir for comparison and the images of test_image_path for evaluation. The parameters num_similar nad treshold describe how many of the most similar images should be considered for construction of the convex hull and which images to consider similar at all. """ evaluator = Evaluator(logger, graph, database_dir, NN_START_RELIABILITY, NN_RELIABILITY_DELTA) iterations = 1 nn_reliability = 100 datasetStart = datetime.datetime.now() sum_classification_time = 0 min_classification_time = 100000 max_classification_time = 0 sum_prediction_time = 0 min_prediction_time = 100000 max_prediction_time = 0 sum_error = 0 min_error = 100000 max_error = 0 # Write CSV file header HEADER = "Data set;N;D_N;D_%;C_Min;C_Avg;C_Max;P_Min;P_Avg;P_Max;E_Min;E_Avg;E_Max;TP;FP\n" if not os.path.isfile(CSV_FILE): with open(CSV_FILE, "a") as myfile: myfile.write(HEADER) # Start estimation logger.info("[Main] Base NN reliability is: " + str(nn_reliability)) differences = [] for filename in sorted(os.listdir(test_data_dir)): if filename.endswith(".png"): path = test_data_dir + filename failure, error, c_time, p_time = evaluator.evaluate_nn_for_image(path, IM_RESIZE_HEIGHT, IM_RESIZE_WIDTH) if not failure is None: differences.append(failure) min_classification_time = min(min_classification_time, c_time) max_classification_time = max(max_classification_time, c_time) sum_classification_time = sum_classification_time + c_time min_prediction_time = min(min_prediction_time, p_time) max_prediction_time = max(max_prediction_time, p_time) sum_prediction_time = sum_prediction_time + p_time# min_error = min(min_error, error) max_error = max(max_error, error) sum_error = sum_error + error logger.info("[Main] NN reliability after seeing " + str(iterations) + " files is now: " + str(evaluator.get_nn_reliability())) iterations += 1 datasetDuration = (datetime.datetime.now() - datasetStart).total_seconds() difference_quota = float(len(differences)) / iterations avg_classification_time = float(sum_classification_time) / iterations avg_prediction_time = float(sum_prediction_time) / iterations avg_error = float(sum_error) / iterations logger.info("[Main] Resulting NN reliability is: " + str(evaluator.get_nn_reliability())) logger.info("[Main] NN and predictor differ in " + str(difference_quota) + " %: " + str(differences)) logger.info("[Main] Overall data set processing time = " + str(datasetDuration) + " s") logger.info("[Main] Classfication times (min,mean,max)s = (" + str(min_classification_time) + ", " + str(avg_classification_time) + ", " + str(max_classification_time) + ")") logger.info("[Main] Prediction times (min,mean,max)s = (" + str(min_prediction_time) + ", " + str(avg_prediction_time) + ", " + str(max_prediction_time) + ")") logger.info("[Main] Absolute errors (min,mean,max)px = (" + str(min_error) + ", " + str(avg_error) + ", " + str(max_error) + ")") line = test_data_dir + ";" \ + str(iterations-1) \ + ";" + str(len(differences)) \ + ";" + str(difference_quota) \ + ";" + str(min_classification_time) \ + ";" + str(avg_classification_time) \ + ";" + str(max_classification_time) \ + ";" + str(min_prediction_time) \ + ";" + str(avg_prediction_time) \ + ";" + str(max_prediction_time) \ + ";" + str(min_error) \ + ";" + str(avg_error) \ + ";" + str(max_error) \ + "\n" logger.info("[Main] For CSV <" + line + ">") with open(CSV_FILE, "a") as myfile: myfile.write(line) if len(differences) > 0: # Report differences failure_dir = test_data_dir.replace('/test-data','/tmp') # Maybe create output dir if not os.path.exists(failure_dir): os.makedirs(failure_dir) logger.info("[Main] Reporting " + str(len(differences)) + " differences to " + failure_dir) for diff in differences: path = diff[0] expectation = diff[1] x = diff[2] # Load differencing image img = cv2.imread(path,0) # Annotate and save image failure_path = path.replace('.png', '.error.png').replace('/test-data','/tmp') logger.info("[Main] Reporting failed image to " + failure_path) cv2.circle(img, (x,0), 5, (255,255,255), -1) font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(img,expectation,(30,260), font, 1,(255,255,255), 2) cv2.imwrite(failure_path, img)
def evaluate(logger, graph, database_dir, test_data_dir, num_similar, treshold): """ Evaluates the classifier frozen in graph using the images of database_dir for comparison and the images of test_image_path for evaluation. The parameters num_similar nad treshold describe how many of the most similar images should be considered for construction of the convex hull and which images to consider similar at all. """ evaluator = Evaluator(logger, graph, database_dir, NN_START_RELIABILITY, NN_RELIABILITY_DELTA) iterations = 1 nn_reliability = 100 datasetStart = datetime.datetime.now() sum_classification_time = 0 min_classification_time = 100000 max_classification_time = 0 sum_prediction_time = 0 min_prediction_time = 100000 max_prediction_time = 0 sum_error = 0 min_error = 100000 max_error = 0 # Write CSV file header HEADER = "Data set;N;D_N;D_%;C_Min;C_Avg;C_Max;P_Min;P_Avg;P_Max;E_Min;E_Avg;E_Max;TP;FP\n" if not os.path.isfile(CSV_FILE): with open(CSV_FILE, "a") as myfile: myfile.write(HEADER) # Start estimation logger.info("[Main] Base NN reliability is: " + str(nn_reliability)) differences = [] for filename in sorted(os.listdir(test_data_dir)): if filename.endswith(".png"): path = test_data_dir + filename failure, error, c_time, p_time = evaluator.evaluate_nn_for_image( path, IM_RESIZE_HEIGHT, IM_RESIZE_WIDTH) if not failure is None: differences.append(failure) min_classification_time = min(min_classification_time, c_time) max_classification_time = max(max_classification_time, c_time) sum_classification_time = sum_classification_time + c_time min_prediction_time = min(min_prediction_time, p_time) max_prediction_time = max(max_prediction_time, p_time) sum_prediction_time = sum_prediction_time + p_time # min_error = min(min_error, error) max_error = max(max_error, error) sum_error = sum_error + error logger.info("[Main] NN reliability after seeing " + str(iterations) + " files is now: " + str(evaluator.get_nn_reliability())) iterations += 1 datasetDuration = (datetime.datetime.now() - datasetStart).total_seconds() difference_quota = float(len(differences)) / iterations avg_classification_time = float(sum_classification_time) / iterations avg_prediction_time = float(sum_prediction_time) / iterations avg_error = float(sum_error) / iterations logger.info("[Main] Resulting NN reliability is: " + str(evaluator.get_nn_reliability())) logger.info("[Main] NN and predictor differ in " + str(difference_quota) + " %: " + str(differences)) logger.info("[Main] Overall data set processing time = " + str(datasetDuration) + " s") logger.info("[Main] Classfication times (min,mean,max)s = (" + str(min_classification_time) + ", " + str(avg_classification_time) + ", " + str(max_classification_time) + ")") logger.info("[Main] Prediction times (min,mean,max)s = (" + str(min_prediction_time) + ", " + str(avg_prediction_time) + ", " + str(max_prediction_time) + ")") logger.info("[Main] Absolute errors (min,mean,max)px = (" + str(min_error) + ", " + str(avg_error) + ", " + str(max_error) + ")") line = test_data_dir + ";" \ + str(iterations-1) \ + ";" + str(len(differences)) \ + ";" + str(difference_quota) \ + ";" + str(min_classification_time) \ + ";" + str(avg_classification_time) \ + ";" + str(max_classification_time) \ + ";" + str(min_prediction_time) \ + ";" + str(avg_prediction_time) \ + ";" + str(max_prediction_time) \ + ";" + str(min_error) \ + ";" + str(avg_error) \ + ";" + str(max_error) \ + "\n" logger.info("[Main] For CSV <" + line + ">") with open(CSV_FILE, "a") as myfile: myfile.write(line) if len(differences) > 0: # Report differences failure_dir = test_data_dir.replace('/test-data', '/tmp') # Maybe create output dir if not os.path.exists(failure_dir): os.makedirs(failure_dir) logger.info("[Main] Reporting " + str(len(differences)) + " differences to " + failure_dir) for diff in differences: path = diff[0] expectation = diff[1] x = diff[2] # Load differencing image img = cv2.imread(path, 0) # Annotate and save image failure_path = path.replace('.png', '.error.png').replace( '/test-data', '/tmp') logger.info("[Main] Reporting failed image to " + failure_path) cv2.circle(img, (x, 0), 5, (255, 255, 255), -1) font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(img, expectation, (30, 260), font, 1, (255, 255, 255), 2) cv2.imwrite(failure_path, img)