def __init__(self, global_shape, patch_res, map_count, modes, gauss_mu_range, gauss_sig_scaling):
self.global_shape = global_shape
self.patch_res = patch_res
self.map_count = map_count
self.modes = modes
self.gauss_mu_range = gauss_mu_range
self.gauss_sig_scaling = gauss_sig_scaling
self.current_patch_index = -1
self.patch_boundingboxes = image_utils.compute_patch_boundingboxes(self.global_shape, stride=self.patch_res, patch_res=self.patch_res)
self.disp_maps = None
self.create_new_disp_maps()
def sample_patches(params):
raw_dirpath, tile_info, ds_fac, size, count, seed = params
im_filepath = read.get_image_filepath(raw_dirpath, tile_info["city"],
tile_info["number"])
im_size = image_utils.get_image_size(im_filepath)
polygon_list = read.load_polygons(raw_dirpath, read.POLYGON_DIRNAME,
tile_info["city"], tile_info["number"])
# Rescale data
corrected_factor = ds_fac * REFERENCE_PIXEL_SIZE / tile_info["pixelsize"]
scale_factor = 1 / corrected_factor
im_size = (int(np.round(im_size[0] * scale_factor)),
int(np.round(im_size[1] * scale_factor)))
ds_polygon_list = polygon_utils.rescale_polygon(polygon_list,
1 / corrected_factor)
bbox_list = image_utils.compute_patch_boundingboxes(im_size, size, size)
random.seed(seed)
random.shuffle(bbox_list)
# Sample <count> patches in tile, making sure there is at least a polygon inside
sampled_bbox_list = []
for bbox in bbox_list:
bbox_polygon_list = polygon_utils.filter_polygons_in_bounding_box(
ds_polygon_list, bbox)
if 1 <= len(bbox_polygon_list):
sampled_bbox_list.append(bbox)
if count <= len(sampled_bbox_list):
break
tile_info["bbox_list"] = sampled_bbox_list
tile_info["scale_factor"] = scale_factor
return tile_info
def inference(self, image_array, ori_gt_array, checkpoints_dir):
"""
Runs inference on image_array and ori_gt_array with model checkpoint in checkpoints_dir
:param image_array:
:param ori_gt_array:
:param checkpoints_dir:
:return:
"""
spatial_shape = image_array.shape[:2]
# Format inputs
image_array = image_array[:, :, :3] # Remove alpha channel if any
image_array = (image_array / 255) * (self.image_dynamic_range[1] - self.image_dynamic_range[0]) + \
self.image_dynamic_range[0]
ori_gt_array = ori_gt_array / 255
padding = (self.input_res - self.output_res) // 2
# Init displacement field and segmentation image
complete_pred_field_map = np.zeros(
(spatial_shape[0] - 2 * padding, spatial_shape[1] - 2 * padding, self.disp_output_channels))
complete_segmentation_image = np.zeros(
(spatial_shape[0] - 2 * padding, spatial_shape[1] - 2 * padding, self.seg_output_channels))
# visualization.init_figures(["example"])
# Iterate over every patch and predict displacement field for this patch
patch_boundingboxes = image_utils.compute_patch_boundingboxes(spatial_shape,
stride=self.output_res,
patch_res=self.input_res)
batch_boundingboxes_list = list(python_utils.split_list_into_chunks(patch_boundingboxes, self.batch_size, pad=True))
# Saver
saver = tf.train.Saver(save_relative_paths=True)
with tf.Session() as sess:
# Restore checkpoint
restore_checkpoint_success = self.restore_checkpoint(sess, saver, checkpoints_dir)
if not restore_checkpoint_success:
sys.exit('No checkpoint found in {}'.format(checkpoints_dir))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# Loop over every batch
for batch_index, batch_boundingboxes in enumerate(batch_boundingboxes_list):
if batch_index % 10 == 0:
print("Processing batch {}/{}"
.format(batch_index + 1, len(batch_boundingboxes_list)))
# Form batch
batch_image_list = []
batch_ori_gt_list = []
for boundingbox in batch_boundingboxes:
patch_image = image_array[boundingbox[0]:boundingbox[2],
boundingbox[1]:boundingbox[3], :]
patch_ori_gt = ori_gt_array[boundingbox[0]:boundingbox[2],
boundingbox[1]:boundingbox[3], :]
batch_image_list.append(patch_image)
batch_ori_gt_list.append(patch_ori_gt)
batch_image = np.stack(batch_image_list, axis=0)
batch_ori_gt = np.stack(batch_ori_gt_list, axis=0)
if self.add_seg_output:
batch_pred_disp_field_map, batch_pred_seg = sess.run([self.level_0_disp_pred, self.level_0_seg_pred], feed_dict={
self.input_image: batch_image,
self.input_disp_polygon_map: batch_ori_gt,
self.keep_prob: 1.0
})
else:
batch_pred_disp_field_map = sess.run(
self.level_0_disp_pred, feed_dict={
self.input_image: batch_image,
self.input_disp_polygon_map: batch_ori_gt,
self.keep_prob: 1.0
})
batch_pred_seg = np.zeros((batch_pred_disp_field_map.shape[0], batch_pred_disp_field_map.shape[1], batch_pred_disp_field_map.shape[2], self.seg_output_channels))
# Fill complete outputs
for batch_index, boundingbox in enumerate(batch_boundingboxes):
patch_pred_disp_field_map = batch_pred_disp_field_map[batch_index]
patch_pred_seg = batch_pred_seg[batch_index]
# print("--- patch_pred_seg: ---")
# print(patch_pred_seg[:, :, 0])
# print("---")
# print(patch_pred_seg[:, :, 1])
# print("---")
# print(patch_pred_seg[:, :, 2])
# print("---")
# print(patch_pred_seg[:, :, 3])
# print("---")
# # visualization.init_figures(["example", "example 2"])
# visualization.init_figures(["example"])
# patch_image = image_array[boundingbox[0]:boundingbox[2],
# boundingbox[1]:boundingbox[3], :]
# patch_image = (patch_image - self.image_dynamic_range[0]) / (
# self.image_dynamic_range[1] - self.image_dynamic_range[0])
# visualization.plot_seg("example", patch_image, patch_pred_seg)
padded_boundingbox = image_utils.padded_boundingbox(boundingbox, padding)
translated_padded_boundingbox = [x - padding for x in padded_boundingbox]
complete_pred_field_map[
translated_padded_boundingbox[0]:translated_padded_boundingbox[2],
translated_padded_boundingbox[1]:translated_padded_boundingbox[3], :] = patch_pred_disp_field_map
complete_segmentation_image[
translated_padded_boundingbox[0]:translated_padded_boundingbox[2],
translated_padded_boundingbox[1]:translated_padded_boundingbox[3],
:] = patch_pred_seg
# visualization.plot_seg("example 2", patch_image, complete_segmentation_image[
# translated_padded_boundingbox[0]:translated_padded_boundingbox[2],
# translated_padded_boundingbox[1]:translated_padded_boundingbox[3],
# :])
# visualization.plot_example("example",
# patch_image[0],
# patch_ori_gt[0],
# patch_pred_disp_field_map[0],
# patch_ori_gt[0])
coord.request_stop()
coord.join(threads)
# De-normalize field map
complete_pred_field_map = complete_pred_field_map / self.disp_map_dynamic_range_fac # Within [-1, 1]
complete_pred_field_map = complete_pred_field_map * self.disp_max_abs_value # Within [-config.DISP_MAX_ABS_VALUE, config.DISP_MAX_ABS_VALUE]
# # De-normalize segmentation image
# complete_segmentation_image = complete_segmentation_image * 255
# complete_segmentation_image = complete_segmentation_image.astype(dtype=np.uint8)
return complete_pred_field_map, complete_segmentation_image
def compute_patch_gradients(self, ori_image, polygon_map_array,
checkpoints_dir):
"""
Runs inference on image_array and ori_gt_array with model checkpoint in checkpoints_dir
:param image_array:
:param ori_gt_array:
:param checkpoints_dir:
:return:
"""
spatial_shape = ori_image.shape[:2]
# Format inputs
image = ori_image[:, :, :3] # Remove alpha channel if any
image = (image / 255) * (self.image_dynamic_range[1] - self.image_dynamic_range[0]) + \
self.image_dynamic_range[0]
polygon_map_array = polygon_map_array / 255
# Init patch_gradient_list
patch_info_list = []
# Iterate over every patch and compute all gradients for this patch
patch_bbox_list = image_utils.compute_patch_boundingboxes(
spatial_shape, stride=self.input_res, patch_res=self.input_res)
y_x = self.level_0_disp_pred[:, :, :, 0]
y_y = self.level_0_disp_pred[:, :, :, 1]
xs = tf.trainable_variables() # All trainable variables
grad_x_ops = tf.gradients(y_x, xs, name='gradients')
grad_y_ops = tf.gradients(y_y, xs, name='gradients')
grad_x_op = [
grad_x_op for grad_x_op in grad_x_ops if grad_x_op is not None
]
grad_y_op = [
grad_y_op for grad_y_op in grad_y_ops if grad_y_op is not None
]
# Saver
saver = tf.train.Saver(save_relative_paths=True)
with tf.Session() as sess:
# Restore checkpoint
restore_checkpoint_success = self.restore_checkpoint(
sess, saver, checkpoints_dir)
if not restore_checkpoint_success:
sys.exit('No checkpoint found in {}'.format(checkpoints_dir))
# Loop over every patch
for index, bbox in enumerate(
tqdm(patch_bbox_list, desc="Computing patch gradients")):
patch_image = image[bbox[0]:bbox[2], bbox[1]:bbox[3], :]
patch_polygon_map = polygon_map_array[bbox[0]:bbox[2],
bbox[1]:bbox[3], :]
batch_image = np.expand_dims(patch_image, axis=0)
batch_polygon_map = np.expand_dims(patch_polygon_map, axis=0)
feed_dict = {
self.input_image: batch_image,
self.input_disp_polygon_map: batch_polygon_map,
self.keep_prob: 1.0
}
patch_grads_x, patch_grads_y = sess.run([grad_x_op, grad_y_op],
feed_dict=feed_dict)
patch_ori_image = ori_image[bbox[0]:bbox[2],
bbox[1]:bbox[3], :]
patch_info = {
"bbox": bbox,
"image": patch_ori_image,
"grads": {
"x": patch_grads_x,
"y": patch_grads_y,
},
}
patch_info_list.append(patch_info)
return patch_info_list
def process_sample_into_patches(patch_stride, patch_res, image, gt_polygon_map, disp_field_maps, disp_polygon_maps,
gt_polygons=None, disp_polygons_list=None):
"""
Crops all inputs to patches generated with patch_stride and patch_res
:param patch_stride:
:param patch_res:
:param image:
:param gt_polygon_map:
:param disp_field_maps:
:param disp_polygon_maps:
:param gt_polygons:
:param disp_polygons_list:
:return:
"""
include_polygons = gt_polygons is not None and disp_polygons_list is not None
patches = []
patch_boundingboxes = image_utils.compute_patch_boundingboxes(image.shape[0:2],
stride=patch_stride,
patch_res=patch_res)
# print(patch_boundingboxes)
for patch_boundingbox in patch_boundingboxes:
# Crop image
patch_image = image[patch_boundingbox[0]:patch_boundingbox[2], patch_boundingbox[1]:patch_boundingbox[3], :]
if include_polygons:
patch_gt_polygons, \
patch_disp_polygons_array = polygon_utils.prepare_polygons_for_tfrecord(gt_polygons, disp_polygons_list,
patch_boundingbox)
else:
patch_gt_polygons = patch_disp_polygons_array = None
patch_gt_polygon_map = gt_polygon_map[patch_boundingbox[0]:patch_boundingbox[2],
patch_boundingbox[1]:patch_boundingbox[3], :]
patch_disp_field_maps = disp_field_maps[:,
patch_boundingbox[0]:patch_boundingbox[2],
patch_boundingbox[1]:patch_boundingbox[3], :]
patch_disp_polygon_maps_array = disp_polygon_maps[:,
patch_boundingbox[0]:patch_boundingbox[2],
patch_boundingbox[1]:patch_boundingbox[3], :]
# Filter out patches based on presence of polygon and area ratio inside inner patch =
patch_inner_res = 2 * patch_stride
patch_padding = (patch_res - patch_inner_res) // 2
inner_patch_gt_polygon_map_corners = patch_gt_polygon_map[patch_padding:-patch_padding,
patch_padding:-patch_padding, 2]
if np.sum(inner_patch_gt_polygon_map_corners) \
and (not include_polygons or (include_polygons and patch_gt_polygons is not None)):
assert patch_image.shape[0] == patch_image.shape[
1], "image should be square otherwise tile_res cannot be defined"
tile_res = patch_image.shape[0]
disp_map_count = patch_disp_polygon_maps_array.shape[0]
patches.append({
"tile_res": tile_res,
"disp_map_count": disp_map_count,
"image": patch_image,
"gt_polygons": patch_gt_polygons,
"disp_polygons": patch_disp_polygons_array,
"gt_polygon_map": patch_gt_polygon_map,
"disp_field_maps": patch_disp_field_maps,
"disp_polygon_maps": patch_disp_polygon_maps_array,
})
return patches
