def evaluate(logger, graph, database_dir, test_data_dir, num_similar, treshold):
""" Evaluates the classifier frozen in graph using the images of
database_dir for comparison and the images of test_image_path
for evaluation. The parameters num_similar nad treshold describe
how many of the most similar images should be considered for
construction of the convex hull and which images to consider
similar at all.
"""
evaluator = Evaluator(logger, graph, database_dir, NN_START_RELIABILITY, NN_RELIABILITY_DELTA)
iterations = 1
nn_reliability = 100
datasetStart = datetime.datetime.now()
sum_classification_time = 0
min_classification_time = 100000
max_classification_time = 0
sum_prediction_time = 0
min_prediction_time = 100000
max_prediction_time = 0
sum_error = 0
min_error = 100000
max_error = 0
# Write CSV file header
HEADER = "Data set;N;D_N;D_%;C_Min;C_Avg;C_Max;P_Min;P_Avg;P_Max;E_Min;E_Avg;E_Max;TP;FP\n"
if not os.path.isfile(CSV_FILE):
with open(CSV_FILE, "a") as myfile:
myfile.write(HEADER)
# Start estimation
logger.info("[Main] Base NN reliability is: " + str(nn_reliability))
differences = []
for filename in sorted(os.listdir(test_data_dir)):
if filename.endswith(".png"):
path = test_data_dir + filename
failure, error, c_time, p_time = evaluator.evaluate_nn_for_image(path, IM_RESIZE_HEIGHT, IM_RESIZE_WIDTH)
if not failure is None:
differences.append(failure)
min_classification_time = min(min_classification_time, c_time)
max_classification_time = max(max_classification_time, c_time)
sum_classification_time = sum_classification_time + c_time
min_prediction_time = min(min_prediction_time, p_time)
max_prediction_time = max(max_prediction_time, p_time)
sum_prediction_time = sum_prediction_time + p_time#
min_error = min(min_error, error)
max_error = max(max_error, error)
sum_error = sum_error + error
logger.info("[Main] NN reliability after seeing " + str(iterations) + " files is now: " + str(evaluator.get_nn_reliability()))
iterations += 1
datasetDuration = (datetime.datetime.now() - datasetStart).total_seconds()
difference_quota = float(len(differences)) / iterations
avg_classification_time = float(sum_classification_time) / iterations
avg_prediction_time = float(sum_prediction_time) / iterations
avg_error = float(sum_error) / iterations
logger.info("[Main] Resulting NN reliability is: " + str(evaluator.get_nn_reliability()))
logger.info("[Main] NN and predictor differ in " + str(difference_quota) + " %: " + str(differences))
logger.info("[Main] Overall data set processing time = " + str(datasetDuration) + " s")
logger.info("[Main] Classfication times (min,mean,max)s = (" + str(min_classification_time) + ", " + str(avg_classification_time) + ", " + str(max_classification_time) + ")")
logger.info("[Main] Prediction times (min,mean,max)s = (" + str(min_prediction_time) + ", " + str(avg_prediction_time) + ", " + str(max_prediction_time) + ")")
logger.info("[Main] Absolute errors (min,mean,max)px = (" + str(min_error) + ", " + str(avg_error) + ", " + str(max_error) + ")")
line = test_data_dir + ";" \
+ str(iterations-1) \
+ ";" + str(len(differences)) \
+ ";" + str(difference_quota) \
+ ";" + str(min_classification_time) \
+ ";" + str(avg_classification_time) \
+ ";" + str(max_classification_time) \
+ ";" + str(min_prediction_time) \
+ ";" + str(avg_prediction_time) \
+ ";" + str(max_prediction_time) \
+ ";" + str(min_error) \
+ ";" + str(avg_error) \
+ ";" + str(max_error) \
+ "\n"
logger.info("[Main] For CSV <" + line + ">")
with open(CSV_FILE, "a") as myfile:
myfile.write(line)
if len(differences) > 0:
# Report differences
failure_dir = test_data_dir.replace('/test-data','/tmp')
# Maybe create output dir
if not os.path.exists(failure_dir):
os.makedirs(failure_dir)
logger.info("[Main] Reporting " + str(len(differences)) + " differences to " + failure_dir)
for diff in differences:
path = diff[0]
expectation = diff[1]
x = diff[2]
# Load differencing image
img = cv2.imread(path,0)
# Annotate and save image
failure_path = path.replace('.png', '.error.png').replace('/test-data','/tmp')
logger.info("[Main] Reporting failed image to " + failure_path)
cv2.circle(img, (x,0), 5, (255,255,255), -1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img,expectation,(30,260), font, 1,(255,255,255), 2)
cv2.imwrite(failure_path, img)
def evaluate(logger, graph, database_dir, test_data_dir, num_similar,
treshold):
""" Evaluates the classifier frozen in graph using the images of
database_dir for comparison and the images of test_image_path
for evaluation. The parameters num_similar nad treshold describe
how many of the most similar images should be considered for
construction of the convex hull and which images to consider
similar at all.
"""
evaluator = Evaluator(logger, graph, database_dir, NN_START_RELIABILITY,
NN_RELIABILITY_DELTA)
iterations = 1
nn_reliability = 100
datasetStart = datetime.datetime.now()
sum_classification_time = 0
min_classification_time = 100000
max_classification_time = 0
sum_prediction_time = 0
min_prediction_time = 100000
max_prediction_time = 0
sum_error = 0
min_error = 100000
max_error = 0
# Write CSV file header
HEADER = "Data set;N;D_N;D_%;C_Min;C_Avg;C_Max;P_Min;P_Avg;P_Max;E_Min;E_Avg;E_Max;TP;FP\n"
if not os.path.isfile(CSV_FILE):
with open(CSV_FILE, "a") as myfile:
myfile.write(HEADER)
# Start estimation
logger.info("[Main] Base NN reliability is: " + str(nn_reliability))
differences = []
for filename in sorted(os.listdir(test_data_dir)):
if filename.endswith(".png"):
path = test_data_dir + filename
failure, error, c_time, p_time = evaluator.evaluate_nn_for_image(
path, IM_RESIZE_HEIGHT, IM_RESIZE_WIDTH)
if not failure is None:
differences.append(failure)
min_classification_time = min(min_classification_time, c_time)
max_classification_time = max(max_classification_time, c_time)
sum_classification_time = sum_classification_time + c_time
min_prediction_time = min(min_prediction_time, p_time)
max_prediction_time = max(max_prediction_time, p_time)
sum_prediction_time = sum_prediction_time + p_time #
min_error = min(min_error, error)
max_error = max(max_error, error)
sum_error = sum_error + error
logger.info("[Main] NN reliability after seeing " +
str(iterations) + " files is now: " +
str(evaluator.get_nn_reliability()))
iterations += 1
datasetDuration = (datetime.datetime.now() - datasetStart).total_seconds()
difference_quota = float(len(differences)) / iterations
avg_classification_time = float(sum_classification_time) / iterations
avg_prediction_time = float(sum_prediction_time) / iterations
avg_error = float(sum_error) / iterations
logger.info("[Main] Resulting NN reliability is: " +
str(evaluator.get_nn_reliability()))
logger.info("[Main] NN and predictor differ in " + str(difference_quota) +
" %: " + str(differences))
logger.info("[Main] Overall data set processing time = " +
str(datasetDuration) + " s")
logger.info("[Main] Classfication times (min,mean,max)s = (" +
str(min_classification_time) + ", " +
str(avg_classification_time) + ", " +
str(max_classification_time) + ")")
logger.info("[Main] Prediction times (min,mean,max)s = (" +
str(min_prediction_time) + ", " + str(avg_prediction_time) +
", " + str(max_prediction_time) + ")")
logger.info("[Main] Absolute errors (min,mean,max)px = (" +
str(min_error) + ", " + str(avg_error) + ", " +
str(max_error) + ")")
line = test_data_dir + ";" \
+ str(iterations-1) \
+ ";" + str(len(differences)) \
+ ";" + str(difference_quota) \
+ ";" + str(min_classification_time) \
+ ";" + str(avg_classification_time) \
+ ";" + str(max_classification_time) \
+ ";" + str(min_prediction_time) \
+ ";" + str(avg_prediction_time) \
+ ";" + str(max_prediction_time) \
+ ";" + str(min_error) \
+ ";" + str(avg_error) \
+ ";" + str(max_error) \
+ "\n"
logger.info("[Main] For CSV <" + line + ">")
with open(CSV_FILE, "a") as myfile:
myfile.write(line)
if len(differences) > 0:
# Report differences
failure_dir = test_data_dir.replace('/test-data', '/tmp')
# Maybe create output dir
if not os.path.exists(failure_dir):
os.makedirs(failure_dir)
logger.info("[Main] Reporting " + str(len(differences)) +
" differences to " + failure_dir)
for diff in differences:
path = diff[0]
expectation = diff[1]
x = diff[2]
# Load differencing image
img = cv2.imread(path, 0)
# Annotate and save image
failure_path = path.replace('.png', '.error.png').replace(
'/test-data', '/tmp')
logger.info("[Main] Reporting failed image to " + failure_path)
cv2.circle(img, (x, 0), 5, (255, 255, 255), -1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img, expectation, (30, 260), font, 1, (255, 255, 255),
2)
cv2.imwrite(failure_path, img)
