def vehicle_tracker(cf):
with log_context(cf.log_file):
logger = logging.getLogger(__name__)
logger.info(' ---> Init test: ' + cf.test_name + ' <---')
if cf.save_results:
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
image_list = get_image_list_changedetection_dataset(
cf.dataset_path, 'in', cf.first_image, cf.image_type, cf.nr_images)
background_img_list = image_list[:len(image_list) // 2]
foreground_img_list = image_list[(len(image_list) // 2):]
mean, variance = background_modeling.multivariative_gaussian_modelling(
background_img_list, cf.color_space)
# Instantiate tracker for multi-object tracking
kalman_init_params = {
'init_estimate_error': cf.init_estimate_error,
'motion_model_noise': cf.motion_model_noise,
'measurement_noise': cf.measurement_noise
}
multi_tracker = MultiTracker(cf.cost_of_non_assignment,
cf.invisible_too_long,
cf.min_age_threshold, kalman_init_params)
for image_path in tqdm(foreground_img_list):
foreground, mean, variance = background_modeling.adaptive_foreground_estimation_color(
image_path, mean, variance, cf.alpha, cf.rho, cf.color_space)
foreground = foreground_improving.hole_filling(
foreground, cf.four_connectivity)
foreground = foreground_improving.remove_small_regions(
foreground, cf.area_filtering_P)
foreground = foreground_improving.image_opening(
foreground, cf.opening_strel, cf.opening_strel_size)
foreground = foreground_improving.image_closing(
foreground, cf.closing_strel, cf.closing_strel_size)
image_gray = skio.imread(image_path, as_grey=True)
bboxes, centroids = detection.detectObjects(image_gray, foreground)
# Tracking
multi_tracker.predict_new_locations_of_tracks()
multi_tracker.detection_to_track_assignment(centroids)
multi_tracker.update_assigned_tracks(bboxes, centroids)
multi_tracker.update_unassigned_tracks()
multi_tracker.delete_lost_tracks()
multi_tracker.create_new_tracks(bboxes, centroids)
if cf.save_results:
image_name = os.path.basename(image_path)
image_name = os.path.splitext(image_name)[0]
save_path = os.path.join(
cf.results_path, image_name + '.' + cf.result_image_type)
image = cv.imread(image_path)
visualization.display_tracking_results(image,
multi_tracker.tracks,
foreground, save_path)
logger.info(' ---> Finish test: ' + cf.test_name + ' <---')
def visualize_all_tracks(cf):
with log_context(cf.log_file):
logger = logging.getLogger(__name__)
logger.info(' ---> Init test: ' + cf.test_name + ' <---')
if cf.save_results:
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
image_list = get_image_list_changedetection_dataset(
cf.dataset_path, 'in', cf.first_image, cf.image_type, cf.nr_images)
background_img_list = image_list[:len(image_list) // 2]
foreground_img_list = image_list[(len(image_list) // 2):]
mean, variance = background_modeling.multivariative_gaussian_modelling(
background_img_list, cf.color_space)
# Instantiate tracker for multi-object tracking
#tracker = Tracker(cf.distance_threshold, cf.max_frames_to_skip, cf.max_trace_length, 0, cf)
tracks = [] # Create an empty array of tracks.
nextId = 1 # ID of the next track
for image_path in foreground_img_list:
foreground, mean, variance = background_modeling.adaptive_foreground_estimation_color(
image_path, mean, variance, cf.alpha, cf.rho, cf.color_space)
foreground = foreground_improving.hole_filling(
foreground, cf.four_connectivity)
foreground = foreground_improving.remove_small_regions(
foreground, cf.area_filtering_P)
foreground = foreground_improving.image_opening(
foreground, cf.opening_strel, cf.opening_strel_size)
foreground = foreground_improving.image_closing(
foreground, cf.closing_strel, cf.closing_strel_size)
bboxes, centroids = detection.detectObjects(image_path, foreground)
multi_tracking.predictNewLocationsOfTracks(tracks)
assignments, unassignedTracks, unassignedDetections = multi_tracking.detectionToTrackAssignment(
tracks, centroids, cf.costOfNonAssignment)
multi_tracking.updateAssignedTracks(tracks, bboxes, centroids,
assignments)
multi_tracking.updateUnassignedTracks(tracks, unassignedTracks)
multi_tracking.createNewTracks(tracks, bboxes, centroids,
unassignedDetections)
img = cv.imread(
cf.dataset_path +
'/in000023.jpg') # in000023 for traffic and in000471 for highway
if tracks != list():
# Display the objects. If an object has not been detected
# in this frame, display its predicted bounding box.
if tracks != list():
for track in tracks:
car_colour = CAR_COLOURS[track.id % len(CAR_COLOURS)]
# for point in track.positions:
# cv.circle(img, (int(point[0]), int(point[1])), 5, car_colour, 1)
cv.polylines(img, [np.int32(track.positions)], False,
(0, 0, 0), 1)
'''for point in track.predictions:
vrtx = np.array([[point[0]-5, point[1]-5], [point[0], point[1]+5], [point[0]+5, point[1]-5]],
np.int32)
cv.polylines(img, [vrtx], True, car_colour, 1)
# cv.rectangle(img, (int(point[0]) - 2, int(point[1]) - 2),
# (int(point[0]) + 2, int(point[1]) + 2), car_colour, 1)'''
cv.polylines(img, [np.int32(track.predictions)], False,
car_colour, 1)
cv.imwrite(cf.results_path + '/all_tracks.jpg', img)
logger.info(' ---> Finish test: ' + cf.test_name + ' <---')
def foreground_estimation(cf):
if cf.AUC_area_filtering:
""" TASK 2 """
with log_context(cf.log_file):
# Get a list with input images filenames
image_list = get_image_list_changedetection_dataset(
cf.dataset_path, 'in', cf.first_image, cf.image_type, cf.nr_images
)
# Get a list with groung truth images filenames
gt_list = get_image_list_changedetection_dataset(
cf.gt_path, 'gt', cf.first_image, cf.gt_image_type, cf.nr_images
)
background_img_list = image_list[:len(image_list) // 2]
foreground_img_list = image_list[(len(image_list) // 2):]
foreground_gt_list = gt_list[(len(image_list) // 2):]
auc_pr, pixels_range, best_pixels, best_alpha = foreground_improving.area_filtering_auc_vs_pixels(
cf, background_img_list, foreground_img_list, foreground_gt_list
)
visualization.aux_plot_auc_vs_pixels(auc_pr, pixels_range, cf.output_folder)
# Save auc_pr as a pickle
auc_pr_path = os.path.join(cf.output_folder, '{}_AUC_vs_pixels.pkl'.format(cf.dataset_name))
with open(auc_pr_path, 'w') as fd:
pickle.dump(auc_pr, fd)
if cf.save_results:
mkdirs(cf.results_path)
mean, variance = background_modeling.multivariative_gaussian_modelling(background_img_list,
cf.color_space)
for (image, gt) in zip(foreground_img_list, foreground_gt_list):
foreground, mean, variance = background_modeling.adaptive_foreground_estimation_color(
image, mean, variance, best_alpha, cf.rho, cf.color_space
)
foreground = foreground_improving.hole_filling(foreground, cf.four_connectivity)
foreground = foreground_improving.remove_small_regions(foreground, best_pixels)
fore = np.array(foreground, dtype='uint8') * 255
image_name = os.path.basename(image)
image_name = os.path.splitext(image_name)[0]
cv.imwrite(
os.path.join(cf.results_path, 'task2_' + image_name + '.' + cf.result_image_type),
fore)
else:
with log_context(cf.log_file):
logger = logging.getLogger(__name__)
# Get a list with input images filenames
image_list = get_image_list_changedetection_dataset(
cf.dataset_path, 'in', cf.first_image, cf.image_type, cf.nr_images
)
# Get a list with groung truth images filenames
gt_list = get_image_list_changedetection_dataset(
cf.gt_path, 'gt', cf.first_image, cf.gt_image_type, cf.nr_images
)
background_img_list = image_list[:len(image_list) // 2]
foreground_img_list = image_list[(len(image_list) // 2):]
foreground_gt_list = gt_list[(len(image_list) // 2):]
# Task 1
mean_back, variance_back = background_modeling.multivariative_gaussian_modelling(background_img_list,
cf.color_space)
alpha_range = np.linspace(cf.evaluate_alpha_range[0], cf.evaluate_alpha_range[1], num=cf.evaluate_alpha_values)
precision = []
recall = []
F1_score = []
i = 1
for alpha in alpha_range:
tp = 0
fp = 0
tn = 0
fn = 0
mean = np.copy(mean_back)
variance = np.copy(variance_back)
for (image, gt) in zip(foreground_img_list, foreground_gt_list):
gt_img = cv.imread(gt, cv.IMREAD_GRAYSCALE)
foreground, mean, variance = background_modeling.adaptive_foreground_estimation_color(
image, mean, variance, alpha, cf.rho, cf.color_space
)
foreground = foreground_improving.hole_filling(foreground, cf.four_connectivity)
if cf.area_filtering:
# Area Filtering
foreground = foreground_improving.remove_small_regions(foreground, cf.area_filtering_P)
if cf.task_name == 'task3':
foreground = foreground_improving.image_opening(foreground, cf.opening_strel,
cf.opening_strel_size)
foreground = foreground_improving.image_closing(foreground, cf.closing_strel,
cf.closing_strel_size)
if cf.shadow_remove:
shadow, highlight = foreground_improving.shadow_detection(cf, mean, image, foreground)
foreground = foreground - shadow - highlight
foreground = np.array(foreground, dtype='uint8')
tp_temp, fp_temp, tn_temp, fn_temp = seg_metrics.evaluate_single_image(foreground,
gt_img)
tp += tp_temp
fp += fp_temp
tn += tn_temp
fn += fn_temp
pre = tp / (tp + fp) if (tp + fp) > 0 else 0.0
rec = tp / (tp + fn) if (tp + fn) > 0 else 0.0
f1 = 2 * pre * rec / (pre + rec + EPSILON)
precision.append(tp / (tp + fp) if (tp + fp) > 0 else 0.0)
recall.append(tp / (tp + fn) if (tp + fn) > 0 else 0.0)
F1_score.append(f1)
logger.info('[{} of {}]\talpha: {}. F1-score {} '.format(i, len(alpha_range), alpha, f1))
i += 1
best_f1_score = max(F1_score)
index_alpha = F1_score.index(best_f1_score)
best_alpha = alpha_range[index_alpha]
logger.info('Best alpha: {:.3f}'.format(best_alpha))
logger.info('Best F1-score: {:.3f}'.format(best_f1_score))
visualization.plot_metrics_vs_threshold(precision, recall, F1_score, alpha_range,
cf.output_folder)
colors = {
'highway': 'blue',
'fall': 'green',
'traffic': 'orange',
}
color = colors.get(cf.dataset_name, 'blue')
auc_pr = visualization.plot_precision_recall_curve(precision, recall, cf.output_folder, color=color)
logger.info('Best alpha: {:.3f}'.format(best_alpha))
logger.info('Best F1-score: {:.3f}'.format(best_f1_score))
logger.info('AUC: {:.3f}'.format(auc_pr))
if cf.save_results:
mean = np.copy(mean_back)
variance = np.copy(variance_back)
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
for image in foreground_img_list:
foreground, mean, variance = background_modeling.adaptive_foreground_estimation_color(
image, mean, variance, best_alpha, cf.rho, cf.color_space
)
foreground = foreground_improving.hole_filling(foreground, cf.four_connectivity)
if cf.area_filtering:
# Area Filtering
foreground = foreground_improving.remove_small_regions(foreground, cf.area_filtering_P)
if cf.task_name == 'task3':
foreground = foreground_improving.image_opening(foreground, cf.opening_strel, cf.opening_strel_size)
foreground = foreground_improving.image_closing(foreground, cf.closing_strel, cf.closing_strel_size)
image_name = os.path.basename(image)
image_name = os.path.splitext(image_name)[0]
if cf.shadow_remove:
shadow, highlight = foreground_improving.shadow_detection(cf, mean, image, foreground)
foreground = foreground - shadow - highlight
shadow_comp = np.stack([foreground, shadow, highlight], axis=2)
cv.imwrite(os.path.join(cf.results_path, 'rgb_' + image_name + '.' + cf.result_image_type),
shadow_comp * 255)
cv.imwrite(os.path.join(cf.results_path, 'shadow_' + image_name + '.' + cf.result_image_type),
foreground*255)
fore = np.array(foreground, dtype='uint8') * 255
cv.imwrite(os.path.join(cf.results_path, image_name + '.' + cf.result_image_type), fore)
def road_statistics(cf):
with log_context(cf.log_file):
logger = logging.getLogger(__name__)
logger.info(' ---> Init test: ' + cf.test_name + ' <---')
if cf.save_results:
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
image_list = get_image_list_changedetection_dataset(
cf.dataset_path, cf.input_prefix, cf.first_image, cf.image_type,
cf.nr_images)
background_img_list = get_image_list_changedetection_dataset(
cf.dataset_path, cf.input_prefix, cf.first_back, cf.image_type,
cf.nr_back)
visualization.visualize_lanes(cv.imread(
background_img_list[0]), cf.lanes, (background_img_list[0].replace(
'input', 'results').replace(cf.input_prefix, 'lane')))
visualization.visualize_roi(
cv.imread(background_img_list[0]), cf.roi_speed,
(background_img_list[0].replace('input', 'results').replace(
cf.input_prefix, 'roi')))
mean, variance = background_modeling.multivariative_gaussian_modelling(
background_img_list, cf.color_space)
# Instantiate tracker for multi-object tracking
kalman_init_params = {
'init_estimate_error': cf.init_estimate_error,
'motion_model_noise': cf.motion_model_noise,
'measurement_noise': cf.measurement_noise
}
multi_tracker = MultiTracker(cf.cost_of_non_assignment,
cf.invisible_too_long,
cf.min_age_threshold, kalman_init_params)
lanes = [Lane() for _ in range(len(cf.lanes))]
# Tracking folder to save results
tracking_folder = os.path.join(cf.results_path, 'tracking')
mkdirs(tracking_folder)
for n, image_path in tqdm(enumerate(image_list)):
image = cv.imread(image_path)
foreground, mean, variance = background_modeling.adaptive_foreground_estimation_color(
image, mean, variance, cf.alpha, cf.rho, cf.color_space)
foreground = foreground_improving.hole_filling(
foreground, cf.four_connectivity)
foreground = foreground_improving.remove_small_regions(
foreground, cf.area_filtering_P)
foreground = foreground_improving.image_opening(
foreground, cf.opening_strel, cf.opening_strel_size)
foreground = foreground_improving.image_closing(
foreground, cf.closing_strel, cf.closing_strel_size)
image_gray = skio.imread(image_path, as_grey=True)
bboxes, centroids = detection.detectObjects(image_gray, foreground)
# Tracking
multi_tracker.detection_to_track_assignment(centroids)
multi_tracker.update_assigned_tracks(bboxes, centroids)
multi_tracker.update_unassigned_tracks()
multi_tracker.delete_lost_tracks()
multi_tracker.create_new_tracks(bboxes, centroids)
multi_tracker.predict_new_locations_of_tracks()
tracks = multi_tracker.tracks
if n % cf.update_speed == 0:
for lane in lanes:
lane.current_vehicles = 0
for track in tracks:
if isinstance(cf.pixels_meter, list):
if track.lane != -1:
pix_meter = cf.pixels_meter[track.lane]
else:
pix_meter = sum(cf.pixels_meter) / len(
cf.pixels_meter)
else:
pix_meter = cf.pixels_meter
if traffic_parameters.is_inside_speed_roi(
track.positions[-1], cf.roi_speed):
# Compute speed using running average
try:
run_avg_alpha = cf.speed_estimate_running_avg
except AttributeError:
run_avg_alpha = 0.2 # Default value
traffic_parameters.speed(track,
pix_meter,
cf.frames_second,
dt=cf.update_speed,
alpha=run_avg_alpha)
track.speeds.append(track.current_speed)
# Detect and assign lane
vehicle_lane = traffic_parameters.lane_detection(
track.positions[-1], cf.lanes)
if vehicle_lane != -1 and track.lane == -1:
track.lane = vehicle_lane
lanes[vehicle_lane].total_vehicles += 1
# Update speeds of the lane assigned to this track
if track.lane != -1 and track.current_speed != 0:
# noinspection PyTypeChecker
lanes[track.lane].sum_vehicles += 1
# noinspection PyTypeChecker
lanes[track.
lane].sum_velocities += track.current_speed
for lane in lanes:
if lane.current_vehicles > cf.high_density:
lane.density = 'high'
else:
lane.density = 'low'
if cf.save_results:
image_name = os.path.basename(image_path)
image_name = os.path.splitext(image_name)[0]
save_path = os.path.join(
cf.results_path, image_name + '.' + cf.result_image_type)
image = image.astype('uint8')
image_copy = image.copy()
visualization.display_speed_results(image, tracks,
cf.max_speed, lanes,
save_path, cf.roi_speed,
cf.margin)
save_path = os.path.join(
tracking_folder,
image_name + '_tracking.' + cf.result_image_type)
visualization.display_tracking_results(image_copy, tracks,
foreground, save_path)
for n, lane in enumerate(lanes):
average_velocity = lane.sum_velocities / lane.sum_vehicles
if n + 1 == 1:
logger.info(
'{}st lane: a total of {} vehicles have passed with an average velocity of {} km/h'
.format(n + 1, lane.total_vehicles, average_velocity))
elif n + 1 == 2:
logger.info(
'{}nd lane: a total of {} vehicles have passed with an average velocity of {} km/h'
.format(n + 1, lane.total_vehicles, average_velocity))
elif n + 1 == 3:
logger.info(
'{}rd lane: a total of {} vehicles have passed with an average velocity of {} km/h'
.format(n + 1, lane.total_vehicles, average_velocity))
else:
logger.info(
'{}th lane: a total of {} vehicles have passed with an average velocity of {} km/h'
.format(n + 1, lane.total_vehicles, average_velocity))
logger.info(' ---> Finish test: ' + cf.test_name + ' <---')
def background_estimation(cf):
logger = logging.getLogger(__name__)
# Get a list with input images filenames
image_list = get_image_list_changedetection_dataset(
cf.dataset_path, 'in', cf.first_image, cf.image_type, cf.nr_images)
# Get a list with groung truth images filenames
gt_list = get_image_list_changedetection_dataset(cf.gt_path, 'gt',
cf.first_image,
cf.gt_image_type,
cf.nr_images)
background_img_list = image_list[:len(image_list) // 2]
foreground_img_list = image_list[(len(image_list) // 2):]
foreground_gt_list = gt_list[(len(image_list) // 2):]
if cf.plot_back_model:
output_path = os.path.join(cf.output_folder, 'gaussian_model')
if not os.path.exists(output_path):
os.makedirs(output_path)
visualization.plot_back_evolution(background_img_list, cf.first_image,
output_path)
if cf.evaluate_foreground:
logger.info('Running foreground evaluation')
if cf.color_images:
mean, variance = background_modeling.multivariative_gaussian_modelling(
background_img_list, cf.color_space)
else:
# Model with a single Gaussian
mean, variance = background_modeling.single_gaussian_modelling(
background_img_list)
alpha_range = np.linspace(cf.evaluate_alpha_range[0],
cf.evaluate_alpha_range[1],
num=cf.evaluate_alpha_values)
precision, recall, f1_score, false_positive_rate = seg_metrics.evaluate_foreground_estimation(
cf.modelling_method, foreground_img_list, foreground_gt_list, mean,
variance, alpha_range, cf.rho, cf.color_images, cf.color_space)
best_f1_score = max(f1_score)
index_alpha = f1_score.index(best_f1_score)
best_alpha = alpha_range[index_alpha]
logger.info('Best alpha: {:.3f}'.format(best_alpha))
logger.info('Best F1-score: {:.3f}'.format(best_f1_score))
visualization.plot_metrics_vs_threshold(precision, recall, f1_score,
alpha_range, cf.output_folder)
colors = {
'highway': 'blue',
'fall': 'green',
'traffic': 'orange',
}
color = colors.get(cf.dataset_name, 'blue')
auc_pr = visualization.plot_precision_recall_curve(precision,
recall,
cf.output_folder,
color=color)
logger.info("AUC: {}".format(auc_pr))
for alpha_value, prec, rec, f1 in zip(alpha_range, precision, recall,
f1_score):
logger.info(
'[alpha={:.2f}] precision={} recall={} f1={}'.format(
alpha_value, prec, rec, f1))
else:
if cf.color_images:
mean, variance = background_modeling.multivariative_gaussian_modelling(
background_img_list, cf.color_space)
else:
# Model with a single Gaussian
mean, variance = background_modeling.single_gaussian_modelling(
background_img_list)
if cf.modelling_method == 'non-adaptive':
logger.info('Running single Gaussian background estimation')
if cf.save_results:
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
for image in foreground_img_list:
if cf.color_images:
foreground = background_modeling.foreground_estimation_color(
image, mean, variance, cf.alpha, cf.color_space)
else:
foreground = background_modeling.foreground_estimation(
image, mean, variance, cf.alpha)
image_name = os.path.basename(image)
image_name = os.path.splitext(image_name)[0]
fore = np.array(foreground, dtype='uint8') * 255
cv.imwrite(
os.path.join(cf.results_path,
image_name + '.' + cf.result_image_type),
fore)
elif cf.modelling_method == 'adaptive':
logger.info(
'Running adaptive single Gaussian background estimation')
if cf.find_best_parameters:
# Grid search over rho and alpha parameter space
logger.info(
'Finding best alpha and rho parameters for adaptive Gaussian model.'
)
alpha_range = np.linspace(cf.evaluate_alpha_range[0],
cf.evaluate_alpha_range[1],
cf.evaluate_alpha_values)
rho_range = np.linspace(cf.evaluate_rho_range[0],
cf.evaluate_rho_range[1],
cf.evaluate_rho_values)
num_iterations = len(rho_range) * len(alpha_range)
logger.info('Running {} iterations'.format(num_iterations))
score_grid = np.zeros(
(len(alpha_range), len(rho_range))) # score = F1-score
precision_grid = np.zeros_like(
score_grid) # for representacion purposes
recall_grid = np.zeros_like(
score_grid) # for representation purposes
best_parameters = dict(alpha=-1, rho=-1)
max_score = 0
for i, (alpha, rho) in enumerate(
itertools.product(alpha_range, rho_range)):
logger.info('[{} of {}]\talpha={:.2f}\trho={:.2f}'.format(
i + 1, num_iterations, alpha, rho))
# Indices in parameter grid
i_idx = np.argwhere(alpha_range == alpha)
j_idx = np.argwhere(rho_range == rho)
# Compute evaluation metrics for this combination of parameters
_, _, _, _, precision, recall, f1_score, fpr = seg_metrics.evaluate_list_foreground_estimation(
cf.modelling_method, foreground_img_list,
foreground_gt_list, mean, variance, alpha, rho,
cf.color_images, cf.color_space)
# Store them in the array
score_grid[i_idx, j_idx] = f1_score
precision_grid[i_idx, j_idx] = precision
recall_grid[i_idx, j_idx] = recall
# Compare and select best parameters according to best score
if f1_score > max_score:
max_score = f1_score
best_parameters = dict(alpha=alpha, rho=rho)
logger.info('Finished grid search')
logger.info('Best parameters: alpha={alpha}\trho={rho}'.format(
**best_parameters))
logger.info('Best F1-score: {:.3f}'.format(max_score))
visualization.plot_adaptive_gaussian_grid_search(
score_grid,
alpha_range,
rho_range,
best_parameters,
best_score=max_score,
metric='F1-score',
sequence_name=cf.dataset_name)
if cf.save_results:
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
for image in foreground_img_list:
if cf.color_images:
foreground, mean, variance = background_modeling.adaptive_foreground_estimation_color(
image, mean, variance, cf.alpha, cf.rho,
cf.color_space)
else:
foreground, mean, variance = background_modeling.adaptive_foreground_estimation(
image, mean, variance, cf.alpha, cf.rho)
image_name = os.path.basename(image)
image_name = os.path.splitext(image_name)[0]
fore = np.array(foreground, dtype='uint8') * 255
cv.imwrite(
os.path.join(
cf.results_path, 'ADAPTIVE_' + image_name + '.' +
cf.result_image_type), fore)
def optical_flow(cf):
with log_context(cf.log_file):
logger = logging.getLogger(__name__)
logger.info(' ---> Init test: ' + cf.test_name + ' <---')
if cf.dataset_name == 'kitti':
# Get a list with input images filenames
image_list = get_sequence_list_kitti_dataset(cf.dataset_path, cf.image_sequence, cf.image_type)
# Get a list with ground truth images filenames
gt_list = get_gt_list_kitti_dataset(cf.gt_path, cf.image_sequence, cf.image_type)
current_image = image_list[1]
previous_image = image_list[0]
if cf.compensation == 'backward':
reference_image = current_image
search_image = previous_image
else:
reference_image = previous_image
search_image = current_image
ref_img_data = cv.imread(reference_image, cv.IMREAD_GRAYSCALE)
search_img_data = cv.imread(search_image, cv.IMREAD_GRAYSCALE)
if cf.optimize_block_matching:
# Placeholder to store results
optimization_results = dict()
# GT flow
optical_flow_gt = cv.imread(gt_list[0], cv.IMREAD_UNCHANGED)
# Grid search
total_iters = len(cf.block_size_range) * len(cf.search_area_range)
for i, (block_size, area_search) in enumerate(
itertools.product(cf.block_size_range, cf.search_area_range)
):
logger.info('[{} / {}] block_size={}\t area_search={}'.format(
i + 1, total_iters, block_size, area_search
))
_, _, dense_optical_flow, total_time = of.exhaustive_search_block_matching(
ref_img_data, search_img_data, block_size, area_search, cf.dfd_norm_type,
)
msen, pepn, squared_errors, pixel_errors, valid_pixels = of_metrics.flow_errors_MSEN_PEPN(
dense_optical_flow, optical_flow_gt
)
optimization_results[i] = {
'block_size': block_size,
'area_search': area_search,
'msen': msen,
'pepn': pepn,
'execution_time': total_time,
}
# Save results every 10 iterations, in case the experiment stops for unknown reasons
if i % 10 == 0:
output_path = os.path.join(cf.output_folder, '{}_ebma_optimization.pkl'.format(cf.dataset_name))
with open(output_path, 'w') as fd:
pickle.dump(optimization_results, fd)
elif cf.sota_opt_flow:
if cf.sota_opt_flow_option == 'opencv':
dense_optical_flow = of.opencv_optflow(
ref_img_data, search_img_data, cf.block_size)
if dense_optical_flow is None:
logger.info('OpenCV version not supported')
sys.exit()
# Evaluate the optical flow
if cf.evaluate:
optical_flow_gt = cv.imread(gt_list[0], cv.IMREAD_UNCHANGED)
msen, pepn, squared_errors, pixel_errors, valid_pixels = of_metrics.flow_errors_MSEN_PEPN(
dense_optical_flow, optical_flow_gt
)
logger.info('Mean Squared Error: {}'.format(msen))
logger.info('Percentage of Erroneous Pixels: {}'.format(pepn))
# Histogram
visualization.plot_histogram_msen(msen, np.ravel(squared_errors[valid_pixels]),
cf.image_sequence,
cf.output_folder)
# Image
visualization.plot_msen_image(image_list[1], squared_errors, pixel_errors, valid_pixels,
cf.image_sequence, cf.output_folder)
if cf.plot_optical_flow:
# Quiver plot
output_path = os.path.join(cf.output_folder, 'optical_flow_opencv_{}.png'.format(cf.image_sequence))
im = cv.imread(image_list[1], cv.IMREAD_GRAYSCALE)
visualization.plot_optical_flow(im, dense_optical_flow, cf.optical_flow_downsample,
cf.image_sequence, output_path, is_ndarray=True)
# HSV plot
output_path = os.path.join(cf.output_folder,
'optical_flow_hsv_{}.png'.format(cf.image_sequence))
visualization.plot_optical_flow_hsv(im, dense_optical_flow, cf.image_sequence, output_path,
is_ndarray=True)
output_path = os.path.join(cf.output_folder,
'optical_flow_middlebury_opencv_{}.png'.format(cf.image_sequence))
middlebury = visualization.flow_to_image(dense_optical_flow)
plt.figure(figsize=(10, 5), dpi=200)
plt.imshow(middlebury)
plt.axis('off')
plt.show(block=False)
plt.savefig(output_path)
plt.close()
logger.info(' ---> Finish test: ' + cf.test_name + ' <---')
elif cf.sota_opt_flow_option == 'flownet2':
# Evaluate the optical flow
if cf.evaluate:
# Groun-truth optical flow
optical_flow_gt = cv.imread(gt_list[0], cv.IMREAD_UNCHANGED)
# Reference image
ref_img = cv.imread(image_list[0], cv.IMREAD_GRAYSCALE)
# Load pre-computed optical flows with different architecture and training procedures
pre_compute_of_folder = cf.output_folder
# FlowNet variants
flownet_variants = ('css', 's', 'S', 'SD')
for fnet_variant in flownet_variants:
logger.info('Evaluating FlowNet2-{}'.format(fnet_variant))
flow_path = os.path.join(pre_compute_of_folder,
'flownet2_{}_{}_10.flo'.format(fnet_variant, cf.image_sequence))
optical_flow_data = of.read_flo_flow(flow_path)
msen, pepn, squared_errors, pixel_errors, valid_pixels = of_metrics.flow_errors_MSEN_PEPN(
optical_flow_data, optical_flow_gt
)
logger.info('Mean Squared Error: {}'.format(msen))
logger.info('Percentage of Erroneous Pixels: {}'.format(pepn))
# Histogram
savefig_path = os.path.join(pre_compute_of_folder, 'flownet2_{}_msen_hist_{}.png'.format(
fnet_variant, cf.image_sequence
))
visualization.plot_histogram_msen(
msen, np.ravel(squared_errors[valid_pixels]), cf.image_sequence, savefig_path
)
# Image
savefig_path = os.path.join(pre_compute_of_folder, 'flownet2_{}_msen_im_{}.png'.format(
fnet_variant, cf.image_sequence
))
visualization.plot_msen_image(
image_list[0], squared_errors, pixel_errors, valid_pixels, cf.image_sequence,
savefig_path
)
if cf.plot_optical_flow:
# Quiver plot
output_path = os.path.join(
cf.output_folder, 'flownet_{}_optical_flow_{}.png'.format(
fnet_variant, cf.image_sequence
)
)
visualization.plot_optical_flow(
ref_img, optical_flow_data, cf.optical_flow_downsample, cf.image_sequence,
output_path, is_ndarray=True
)
# HSV plot
output_path = os.path.join(
cf.output_folder, 'flownet_{}_optical_flow_hsv_{}.png'.format(
fnet_variant, cf.image_sequence
)
)
visualization.plot_optical_flow_hsv(
ref_img, optical_flow_data, cf.image_sequence, output_path, is_ndarray=True
)
output_path = os.path.join(cf.output_folder,
'flownet_{}_optical_flow_middlebury_{}.png'.format(
fnet_variant, cf.image_sequence))
middlebury = visualization.flow_to_image(optical_flow_data)
plt.figure(figsize=(10, 5), dpi=200)
plt.imshow(middlebury)
plt.axis('off')
plt.show(block=False)
plt.savefig(output_path)
plt.close()
else:
raise ValueError('cv.sota_opt_flow_option {!r} not supported'.format(cf.sota_opt_flow_option))
else:
# Run Task 1: Block Matching Algorithm
predicted_image, optical_flow, dense_optical_flow, _ = of.exhaustive_search_block_matching(
ref_img_data, search_img_data, cf.block_size, cf.search_area, cf.dfd_norm_type, verbose=False
)
# Evaluate the optical flow
if cf.evaluate:
optical_flow_gt = cv.imread(gt_list[0], cv.IMREAD_UNCHANGED)
msen, pepn, squared_errors, pixel_errors, valid_pixels = of_metrics.flow_errors_MSEN_PEPN(
dense_optical_flow, optical_flow_gt
)
logger.info('Mean Squared Error: {}'.format(msen))
logger.info('Percentage of Erroneous Pixels: {}'.format(pepn))
if cf.plot_prediction:
output_path = os.path.join(cf.output_folder, 'optical_flow_prediction_{}.png'.format(
cf.image_sequence
))
plt.imshow(predicted_image, cmap='gray')
plt.show(block=False)
plt.savefig(output_path)
plt.close()
# Histogram
visualization.plot_histogram_msen(msen, np.ravel(squared_errors[valid_pixels]), cf.image_sequence,
cf.output_folder)
# Image
visualization.plot_msen_image(image_list[1], squared_errors, pixel_errors, valid_pixels,
cf.image_sequence, cf.output_folder)
if cf.plot_optical_flow:
# Quiver plot
output_path = os.path.join(cf.output_folder, 'optical_flow_{}.png'.format(cf.image_sequence))
im = cv.imread(image_list[1], cv.IMREAD_GRAYSCALE)
visualization.plot_optical_flow(im, dense_optical_flow, cf.optical_flow_downsample,
cf.image_sequence, output_path, is_ndarray=True)
# HSV plot
output_path = os.path.join(cf.output_folder, 'optical_flow_hsv_{}.png'.format(cf.image_sequence))
visualization.plot_optical_flow_hsv(im, dense_optical_flow, cf.image_sequence, output_path,
is_ndarray=True)
elif cf.dataset_name == 'traffic':
image_list = get_image_list_changedetection_dataset(cf.dataset_path, 'in', cf.first_image, cf.image_type,
cf.nr_images)
gt_list = get_image_list_changedetection_dataset(cf.gt_path, 'gt', cf.first_image, cf.gt_image_type,
cf.nr_images)
if cf.save_results:
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
if cf.sota_video_stab:
of.video_stabilization_sota2(image_list, cf.output_folder)
'''prev_to_cur_transform = []
previous_image = image_list[0]
prev_image = cv.imread(previous_image, cv.IMREAD_GRAYSCALE)
prev_corner = cv.goodFeaturesToTrack(prev_image, maxCorners=200, qualityLevel=0.01, minDistance=30.0,
blockSize=3)
for idx in range(1, len(image_list)):
current_image = image_list[idx]
previous_image = image_list[idx - 1]
curr_image = cv.imread(current_image, cv.IMREAD_GRAYSCALE)
prev_image = cv.imread(previous_image, cv.IMREAD_GRAYSCALE)
prev_to_cur_transform = of.video_stabilization_sota(prev_image, curr_image,
prev_to_cur_transform,
prev_corner)
# convert list of transforms to array
prev_to_cur_transform = np.array(prev_to_cur_transform)
# cumsum of all transforms for trajectory
trajectory = np.cumsum(prev_to_cur_transform, axis=0)
# convert trajectory array to df
trajectory = pd.DataFrame(trajectory)
# rolling mean to smooth
smoothed_trajectory = trajectory.rolling(window=30, center=False).mean()
# back fill nas caused by smoothing
smoothed_trajectory = smoothed_trajectory.fillna(method='bfill')
# new set of prev to cur transform, removing trajectory and replacing w/smoothed
new_prev_to_cur_transform = prev_to_cur_transform + (smoothed_trajectory - trajectory)
# initialize transformation matrix
T = np.zeros((2, 3))
# convert transform df to array
new_prev_to_cur_transform = np.array(new_prev_to_cur_transform)
for k in range(len(image_list) - 1):
cur = cv.imread(image_list[k])
T[0, 0] = np.cos(new_prev_to_cur_transform[k][2])
T[0, 1] = -np.sin(new_prev_to_cur_transform[k][2])
T[1, 0] = np.sin(new_prev_to_cur_transform[k][2])
T[1, 1] = np.cos(new_prev_to_cur_transform[k][2])
T[0, 2] = new_prev_to_cur_transform[k][0]
T[1, 2] = new_prev_to_cur_transform[k][1]
# apply saved transform (resource: http://nghiaho.com/?p=2208)
rect_image = cv.warpAffine(cur, T, (cur.shape[0], cur.shape[1]))
if cf.save_results:
image_name = os.path.basename(image_list[k])
image_name = os.path.splitext(image_name)[0]
cv.imwrite(os.path.join(cf.results_path, image_name + '.' + cf.result_image_type), rect_image)'''
else:
acc_u = 0
acc_v = 0
# Running average of directions
previous_u = 0
previous_v = 0
if cf.save_results:
# Create folder to store histograms
histogram_folder = os.path.join(cf.results_path, 'histograms')
mkdirs(histogram_folder)
for idx in range(1, len(image_list)):
current_image = image_list[idx]
previous_image = image_list[idx - 1]
if cf.compensation == 'backward':
reference_image = current_image
search_image = previous_image
else:
reference_image = previous_image
search_image = current_image
ref_img_data = cv.imread(reference_image, cv.IMREAD_GRAYSCALE)
search_img_data = cv.imread(search_image, cv.IMREAD_GRAYSCALE)
save_path = os.path.join(cf.output_folder, '{}_{}_{}_{}.pkl'.format(
idx, cf.block_size, cf.search_area, cf.compensation
))
if cf.sota_opt_flow and cf.sota_opt_flow_option == 'opencv':
dense_flow = of.opencv_optflow(
ref_img_data, search_img_data, cf.block_size)
else:
try:
with open(save_path, 'rb') as file_flow:
dense_flow = pickle.load(file_flow)
except Exception:
with open(save_path, 'wb') as fd:
_, opt_flow, dense_flow, _ = of.exhaustive_search_block_matching(
ref_img_data, search_img_data, cf.block_size, cf.search_area, cf.dfd_norm_type,
verbose=False)
pickle.dump(dense_flow, fd)
image_data = cv.imread(gt_list[idx])
# Params
strategy = 'max' # 'max', 'trimmed_mean', 'background_block'
if strategy == 'background_blocks':
center_positions = [(30, 290), (210, 30)]
neighborhood = 20
additional_params = {
'center_positions': center_positions,
'neighborhood': neighborhood,
}
else:
additional_params = dict()
running_avg = 0
# Run
rect_image, acc_direction, previous_direction = of.video_stabilization(
image_data, dense_flow, cf.compensation, strategy, cf.search_area,
(acc_u, acc_v), (previous_u, previous_v), running_avg, **additional_params
)
acc_u, acc_v = acc_direction
previous_u, previous_v = previous_direction
if cf.save_results:
image_name = os.path.basename(current_image)
image_name = os.path.splitext(image_name)[0]
cv.imwrite(os.path.join(cf.results_path, image_name + '.' + cf.result_image_type), rect_image)
if cf.save_plots:
if strategy == 'background_blocks':
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111)
plt.imshow(image_data)
for center_position, color in zip(center_positions, itertools.cycle(['r', 'y', 'b'])):
ax.scatter(center_position[1], center_position[0], s=8, marker='x', color=color,
label='Pixel position ({0}, {1})'.format(*center_position))
rectangle_center = (center_position[1] - neighborhood,
center_position[0] - neighborhood)
ax.add_patch(
patches.Rectangle(
rectangle_center,
neighborhood * 2,
neighborhood * 2,
fill=False, color=color
)
)
plt.legend()
plt.axis('off')
plot_path = os.path.join(
histogram_folder,
image_name + '_block_markers.' + cf.result_image_type
)
plt.savefig(plot_path)
plt.close()
# Save histogram of directions
hist_path = os.path.join(histogram_folder, image_name + '_hist_2d.' + cf.result_image_type)
visualization.plot_optical_flow_histogram(dense_flow, cf.search_area, hist_path)
logger.info(' ---> Finish test: ' + cf.test_name + ' <---')
elif cf.dataset_name == 'ski_video':
image_list = get_image_list_ski_video_dataset(cf.dataset_path, cf.first_image, cf.image_type, cf.nr_images)
if cf.save_results:
logger.info('Saving results in {}'.format(cf.results_path))
mkdirs(cf.results_path)
acc_u = 0
acc_v = 0
# Running average of directions
previous_u = 0
previous_v = 0
if cf.save_results:
# Create folder to store histograms
histogram_folder = os.path.join(cf.results_path, 'histograms')
mkdirs(histogram_folder)
for idx in range(1, len(image_list)):
current_image = image_list[idx]
previous_image = image_list[idx - 1]
if cf.compensation == 'backward':
reference_image = current_image
search_image = previous_image
else:
reference_image = previous_image
search_image = current_image
ref_img_data = cv.imread(reference_image, cv.IMREAD_GRAYSCALE)
search_img_data = cv.imread(search_image, cv.IMREAD_GRAYSCALE)
save_path = os.path.join(cf.output_folder, '{}_{}_{}_{}.pkl'.format(
idx, cf.block_size, cf.search_area, cf.compensation
))
try:
with open(save_path, 'rb') as file_flow:
dense_flow = pickle.load(file_flow)
except Exception:
with open(save_path, 'wb') as fd:
_, opt_flow, dense_flow, _ = of.exhaustive_search_block_matching(
ref_img_data, search_img_data, cf.block_size, cf.search_area, cf.dfd_norm_type,
verbose=False)
pickle.dump(dense_flow, fd)
image_data = cv.imread(current_image, cv.IMREAD_COLOR)
# Params
strategy = 'background_blocks' # 'max', 'trimmed_mean', 'background_block'
if strategy == 'background_blocks':
center_positions = [(210, 30)]
# center_positions = [(30, 290), (210, 30)]
neighborhood = 20
additional_params = {
'center_positions': center_positions,
'neighborhood': neighborhood,
}
else:
additional_params = dict()
running_avg = 0
# Run
rect_image, acc_direction, previous_direction = of.video_stabilization(
image_data, dense_flow, cf.compensation, strategy, cf.search_area,
(acc_u, acc_v), (previous_u, previous_v), running_avg, **additional_params
)
acc_u, acc_v = acc_direction
previous_u, previous_v = previous_direction
if cf.save_results:
image_name = os.path.basename(current_image)
image_name = os.path.splitext(image_name)[0]
cv.imwrite(os.path.join(cf.results_path, image_name + '.' + cf.result_image_type), rect_image)
if cf.save_plots:
if strategy == 'background_blocks':
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111)
plt.imshow(image_data)
for center_position, color in zip(center_positions, itertools.cycle(['r', 'y', 'b'])):
ax.scatter(center_position[1], center_position[0], s=8, marker='x', color=color,
label='Pixel position ({0}, {1})'.format(*center_position))
rectangle_center = (center_position[1] - neighborhood,
center_position[0] - neighborhood)
ax.add_patch(
patches.Rectangle(
rectangle_center,
neighborhood * 2,
neighborhood * 2,
fill=False, color=color
)
)
plt.legend()
plt.axis('off')
plot_path = os.path.join(
histogram_folder,
image_name + '_block_markers.' + cf.result_image_type
)
plt.savefig(plot_path)
plt.close()
# Save histogram of directions
hist_path = os.path.join(histogram_folder, image_name + '_hist_2d.' + cf.result_image_type)
visualization.plot_optical_flow_histogram(dense_flow, cf.search_area, hist_path)
# Save optical flow plot
opt_flow_path = os.path.join(histogram_folder, image_name + '_flow.' + cf.result_image_type)
visualization.plot_optical_flow(
image_data, dense_flow, 16, 'ski-video-{}'.format(image_name),
opt_flow_path, is_ndarray=True
)
logger.info(' ---> Finish test: ' + cf.test_name + ' <---')