def fit_image(f1, f2, P):
tx = 0
ty = 0
xy1 = matrix_transform([0, 0], np.linalg.inv(P))
xy2 = matrix_transform([0, f2.shape[0]], np.linalg.inv(P))
xy3 = matrix_transform([f2.shape[1], f2.shape[0]], np.linalg.inv(P))
xy4 = matrix_transform([f2.shape[1], 0], np.linalg.inv(P))
x_min = min(xy1[0][0], xy2[0][0], xy3[0][0], xy4[0][0], 0)
x_max = max(xy1[0][0], xy2[0][0], xy3[0][0], xy4[0][0], f1.shape[1])
y_min = min(xy1[0][1], xy2[0][1], xy3[0][1], xy4[0][1], 0)
y_max = max(xy1[0][1], xy2[0][1], xy3[0][1], xy4[0][1], f1.shape[0])
x = int(x_max - x_min)
y = int(y_max - y_min)
if y_min < 0:
ty = y_min
if x_min < 0:
tx = x_min
f_stitched = warp(f2, P, output_shape=(y, x))
M, N = f1.shape[:2]
f_stitched[0:M, 0:N, :] = f1
tform = SimilarityTransform(translation=(tx, ty))
warped = warp(f_stitched, tform)
plt.imshow(warped)
plt.axis('off')
plt.show()
def rotate_coordinates(coords, angle, rotation_centre, imsize, resize=False):
"""Rotate coordinates in the image"""
rot_centre = np.asanyarray(rotation_centre)
angle = math.radians(angle)
rot_matrix = np.array([
[math.cos(angle), math.sin(angle), 0],
[-math.sin(angle), math.cos(angle), 0],
[0, 0, 1],
])
coords = transform.matrix_transform(coords - rot_centre,
rot_matrix) + rot_centre
if resize:
rows, cols = imsize[0], imsize[1]
corners = np.array(
[[0, 0], [0, rows - 1], [cols - 1, rows - 1], [cols - 1, 0]],
dtype=np.float32)
if rotation_centre is not None:
corners = (
transform.matrix_transform(corners - rot_centre, rot_matrix) +
rot_centre)
x_shift = min(corners[:, 0])
y_shift = min(corners[:, 1])
coords -= np.array([x_shift, y_shift])
return coords
def get_error(P, h_inliers, h_model, b_error, b_model):
inliers = np.array([])
for i in h_model:
new_pos = matrix_transform([i[0], i[1]], P)
error = np.linalg.norm([i[2], i[3]] - new_pos)
if error < t:
inliers = np.append(inliers, i)
if (inliers.shape[0] / 4) > d:
inliers = np.append(inliers, h_inliers)
inliers = inliers.reshape(((int(inliers.shape[0] / 4), 4)))
P2 = perspectiveTransformMatrix(inliers)
for i in h_model:
new_pos = matrix_transform([i[0], i[1]], P)
error += np.linalg.norm([i[2], i[3]] - new_pos)
error = error / inliers.shape[0]
if error < b_error:
print("Found better match")
b_error = error
b_model = P2
return b_model, b_error
def transform_point(point_list, transform_parameters):
scale_x, scale_y = transform_parameters['zy'], transform_parameters['zx']
translate_x_px, translate_y_px = transform_parameters[
'ty'], transform_parameters['tx']
rotate = transform_parameters['theta']
shear = transform_parameters['shear']
flip_horizontal = transform_parameters.get('flip_horizontal', False)
flip_vertical = transform_parameters.get('flip_vertical', False)
if scale_x != 1.0 or scale_y != 1.0 or translate_x_px != 0 or translate_y_px != 0 or rotate != 0 \
or shear != 0:
matrix_to_topleft = skimage_tf.SimilarityTransform(
translation=[-0.5, -0.5])
matrix_transforms = skimage_tf.AffineTransform(
scale=(scale_x, scale_y),
translation=(translate_x_px, translate_y_px),
rotation=math.radians(rotate),
shear=math.radians(shear))
matrix_to_center = skimage_tf.SimilarityTransform(
translation=[0.5, 0.5])
matrix = (matrix_to_topleft + matrix_transforms + matrix_to_center)
point_list = skimage_tf.matrix_transform(point_list, matrix.params)
if flip_horizontal or flip_vertical:
matrix_to_topleft = skimage_tf.SimilarityTransform(
translation=[-0.5, -0.5])
point_list = skimage_tf.matrix_transform(point_list,
matrix_to_topleft.params)
if flip_horizontal:
point_list = [(-x, y) for x, y in point_list]
if flip_vertical:
point_list = [(x, -y) for x, y in point_list]
matrix_to_center = skimage_tf.SimilarityTransform(
translation=[0.5, 0.5])
point_list = skimage_tf.matrix_transform(point_list,
matrix_to_center.params)
return point_list
def infer(edge_image, edge_lengths, mu, phi, sigma2,
update_slice=slice(None),
scale_estimate=None,
rotation=0,
translation=(0, 0)):
# edge_points = np.array(np.where(edge_image)).T
# edge_points[:, [0, 1]] = edge_points[:, [1, 0]]
# edge_score = edge_image.shape[0] * np.exp(-edge_lengths[edge_image] / (0.25 * edge_image.shape[0])).reshape(-1, 1)
# edge_points = np.concatenate((edge_points, edge_score), axis=1)
#
# edge_nn = NearestNeighbors(n_neighbors=1).fit(edge_points)
edge_near = scipy.ndimage.distance_transform_edt(~edge_image)
edge_near_blur = gaussian(edge_near, 2)
Gy, Gx = np.gradient(edge_near_blur)
mag = np.sqrt(np.power(Gy, 2) + np.power(Gx, 2))
if scale_estimate is None:
scale_estimate = min(edge_image.shape) * 4
mu = (mu.reshape(-1, 2) - mu.reshape(-1, 2).mean(axis=0)).reshape(-1, 1)
average_distance = np.sqrt(np.power(mu.reshape(-1, 2), 2).sum(axis=1)).mean()
scale_estimate /= average_distance * np.sqrt(2)
h = np.zeros((phi.shape[1], 1))
psi = SimilarityTransform(scale=scale_estimate, rotation=rotation, translation=translation)
while True:
w = (mu + phi @ h).reshape(-1, 2)
image_points = matrix_transform(w, psi.params)[update_slice, :]
image_points = np.concatenate((image_points, np.zeros((image_points.shape[0], 1))), axis=1)
# closest_edge_point_indices = edge_nn.kneighbors(image_points)[1].flatten()
# closest_edge_points = edge_points[closest_edge_point_indices, :2]
closest_edge_points = gradient_step(Gy, Gx, mag, image_points)
w = mu.reshape(-1, 2)
psi = estimate_transform('similarity', w[update_slice, :], closest_edge_points)
image_points = matrix_transform(w, psi.params)[update_slice, :]
image_points = np.concatenate((image_points, np.zeros((image_points.shape[0], 1))), axis=1)
# closest_edge_point_indices = edge_nn.kneighbors(image_points)[1].flatten()
# closest_edge_points = edge_points[closest_edge_point_indices, :2]
closest_edge_points = gradient_step(Gy, Gx, mag, image_points)
mu_slice = mu.reshape(-1, 2)[update_slice, :].reshape(-1, 1)
K = phi.shape[-1]
phi_full = phi.reshape(-1, 2, K)
phi_slice = phi_full[update_slice, :].reshape(-1, K)
h = update_h(sigma2, phi_slice, closest_edge_points, mu_slice, psi)
w = (mu + phi @ h).reshape(-1, 2)
image_points = matrix_transform(w, psi.params)
update_slice = yield image_points, closest_edge_points
def infer(edge_image,
edge_lengths,
mu,
phi,
sigma2,
update_slice=slice(None),
scale_estimate=None,
rotation=0,
translation=(0, 0)):
edge_near = scipy.ndimage.distance_transform_edt(~edge_image)
edge_near_blur = gaussian(edge_near, 2)
Gy, Gx = np.gradient(edge_near_blur)
mag = np.sqrt(np.power(Gy, 2) + np.power(Gx, 2))
if scale_estimate is None:
scale_estimate = min(edge_image.shape) * 4
mu = (mu.reshape(-1, 2) - mu.reshape(-1, 2).mean(axis=0)).reshape(-1, 1)
average_distance = np.sqrt(np.power(mu.reshape(-1, 2),
2).sum(axis=1)).mean()
scale_estimate /= average_distance * np.sqrt(2)
h = np.zeros((phi.shape[1], 1))
psi = SimilarityTransform(scale=scale_estimate,
rotation=rotation,
translation=translation)
while True:
w = (mu + phi @ h).reshape(-1, 2)
image_points = matrix_transform(w, psi.params)[update_slice, :]
image_points = np.concatenate(
(image_points, np.zeros((image_points.shape[0], 1))), axis=1)
closest_edge_points = gradient_step(Gy, Gx, mag, image_points)
w = mu.reshape(-1, 2)
psi = estimate_transform('similarity', w[update_slice, :],
closest_edge_points)
image_points = matrix_transform(w, psi.params)[update_slice, :]
image_points = np.concatenate(
(image_points, np.zeros((image_points.shape[0], 1))), axis=1)
closest_edge_points = gradient_step(Gy, Gx, mag, image_points)
mu_slice = mu.reshape(-1, 2)[update_slice, :].reshape(-1, 1)
K = phi.shape[-1]
phi_full = phi.reshape(-1, 2, K)
phi_slice = phi_full[update_slice, :].reshape(-1, K)
h = update_h(sigma2, phi_slice, closest_edge_points, mu_slice, psi)
w = (mu + phi @ h).reshape(-1, 2)
image_points = matrix_transform(w, psi.params)
update_slice = yield image_points, closest_edge_points, h, psi
def crop_img(self, fullimg, depth, orig_coords, orig_labels):
import numpy as np
import skimage.transform as T
[crop_height, crop_width, mat] = self.transform_matrix(depth)
transform_mat = np.linalg.inv(mat)
L.debug('Final transform matrix\n%s', transform_mat)
cropped_img = T.warp(fullimg, transform_mat, output_shape=np.round((crop_height, crop_width)), order=3)
# Bicubic, default is 1, bilinear
cropped_coords = T.matrix_transform(orig_coords, mat)
L.debug('Original coords %s', orig_coords)
L.debug('Transformed coords %s', cropped_coords)
# T.matrix_transform() # Use this to tranform coordinate
index_mapping = self.get_joint_mapping(orig_labels)
# index_mapping = np.array([6, 1, 3, 2, 0, 5, 8, 10, 12, 11, 9, 7, 13, 4])
# The order for LSP dataset
# PC format
# The order is defined in here: http://www.comp.leeds.ac.uk/mat4saj/lsp.html
L.debug('Transformed labels %s', [orig_labels[v] for v in index_mapping])
resorted_cropped_coords = cropped_coords[index_mapping, :]
L.debug('Transformed LSP coords\n%s', resorted_cropped_coords)
# L.debug('Transformed joint labels %s', [labels[i] for i in index_mapping])
return [cropped_img, resorted_cropped_coords]
def transform_to_template(shape, template, transformation_type='similarity'):
"""Returns a transformed shape that is as close as possible to the given
template under a certain type of transformation.
Args:
shape (ndarray): An n x 2 array where each row represents a vertex in
the shape. The first column is the x-coordinate and
the second column is the y-coordinate.
template (ndarray): Another shape corresponding to the first input.
It must have the same array shape and type, and
corresponding rows must respresent corresponding
vertices. For example, the vertex respresented by
row **i** in the *input* will try to match as
closesly as possible to the vertex represented by
row **i** in the *template*.
transformation_type (str): The type of transformation to use when
fitting the shape to the template. The
string must be one of the ones specified by
`skimage.transform.estimate_transform`_.
Returns:
ndarray: Transformed shape of the same type and array shape as the
input shape.
.. _skimage.transform.estimate_transform: http://scikit-image.org/docs/dev/api/skimage.transform.html#skimage.transform.estimate_transform
"""
transformation = estimate_transform(transformation_type, shape, template)
return matrix_transform(shape, transformation.params)
def video_to_court(xs, ys, homography):
'''
Input: list of x, list of y, homography
Output: transformed x and y
'''
xst, yst = zip(*tf.matrix_transform(zip(xs,ys),homography))
return xst, yst
def test_find_transform_givensources(self):
from skimage.transform import estimate_transform, matrix_transform
source = np.array([
[1.4, 2.2],
[5.3, 1.0],
[3.7, 1.5],
[10.1, 9.6],
[1.3, 10.2],
[7.1, 2.0],
])
nsrc = source.shape[0]
scale = 1.5 # scaling parameter
alpha = np.pi / 8.0 # rotation angle
mm = scale * np.array([[np.cos(alpha), -np.sin(alpha)],
[np.sin(alpha), np.cos(alpha)]])
tx, ty = 2.0, 1.0 # translation parameters
transl = np.array([nsrc * [tx], nsrc * [ty]])
dest = (mm.dot(source.T) + transl).T
t_true = estimate_transform("similarity", source, dest)
# disorder dest points so they don't match the order of source
np.random.shuffle(dest)
t, (src_pts, dst_pts) = aa.find_transform(source, dest)
self.assertLess(t_true.scale - t.scale, 1e-10)
self.assertLess(t_true.rotation - t.rotation, 1e-10)
self.assertLess(np.linalg.norm(t_true.translation - t.translation),
1e-10)
self.assertEqual(src_pts.shape[0], dst_pts.shape[0])
self.assertEqual(src_pts.shape[1], 2)
self.assertEqual(dst_pts.shape[1], 2)
dst_pts_test = matrix_transform(src_pts, t.params)
self.assertLess(np.linalg.norm(dst_pts_test - dst_pts), 1e-10)
def align(self, sub, centroids):
# Align sub to current keystars, updating its status.
min_inliers = 4
# choose warp model to project existing keystars so they lie closer to centroids (in theory)
keys = self.keystars
if self.warp_model:
keys = matrix_transform(self.keystars, self.warp_model.params)
else:
keys = self.keystars
# find closest matching star to each transformed keystar
nstars = min(len(keys), len(centroids))
matched_stars = np.zeros((nstars, 2))
for i, (x1, y1) in enumerate(keys):
if i < nstars:
matched_stars[i, :] = centroids[np.argmin([
(x1 - x2)**2 + (y1 - y2)**2 for x2, y2 in centroids
])]
# do we have enough matched stars?
if len(matched_stars) < self.min_stars:
sub.status = 'nalign'
else:
# apply RANSAC to find which matching pairs best fitting Euclidean model
# can throw a warning in cases where no inliers (bug surely) which we ignore
with warnings.catch_warnings():
warnings.simplefilter('ignore')
warp_model, inliers = ransac(
(np.array(self.keystars[:nstars]), matched_stars),
EuclideanTransform,
4,
.5,
max_trials=100)
# managed to align
if (inliers is not None) and (sum(inliers) >= min_inliers):
# update warp model
self.warp_model = warp_model
sub.image = warp(sub.image,
self.warp_model,
order=3,
preserve_range=True)
self.align_count += 1
# change to select if it was nalign before; if reject before, leave it as reject
if sub.status == 'nalign':
sub.status = 'select'
else:
sub.status = 'nalign'
# return inverse transform of centroids (for platesolver)
if self.warp_model:
return self.warp_model.inverse(centroids)
return None
def _augment_keypoints(self, keypoints_on_images, random_state, parents,
hooks):
result = []
nb_images = len(keypoints_on_images)
scale_samples, translate_samples, rotate_samples, shear_samples, cval_samples, mode_samples, order_samples = self._draw_samples(
nb_images, random_state)
for i, keypoints_on_image in enumerate(keypoints_on_images):
height, width = keypoints_on_image.height, keypoints_on_image.width
shift_x = int(width / 2.0)
shift_y = int(height / 2.0)
scale_x, scale_y = scale_samples[0][i], scale_samples[1][i]
translate_x, translate_y = translate_samples[0][
i], translate_samples[1][i]
#assert isinstance(translate_x, (float, int))
#assert isinstance(translate_y, (float, int))
if ia.is_single_float(translate_y):
translate_y_px = int(
round(translate_y * keypoints_on_image.shape[0]))
else:
translate_y_px = translate_y
if ia.is_single_float(translate_x):
translate_x_px = int(
round(translate_x * keypoints_on_image.shape[1]))
else:
translate_x_px = translate_x
rotate = rotate_samples[i]
shear = shear_samples[i]
#cval = cval_samples[i]
#mode = mode_samples[i]
#order = order_samples[i]
if scale_x != 1.0 or scale_y != 1.0 or translate_x_px != 0 or translate_y_px != 0 or rotate != 0 or shear != 0:
matrix_to_topleft = tf.SimilarityTransform(
translation=[-shift_x, -shift_y])
matrix_transforms = tf.AffineTransform(
scale=(scale_x, scale_y),
translation=(translate_x_px, translate_y_px),
rotation=math.radians(rotate),
shear=math.radians(shear))
matrix_to_center = tf.SimilarityTransform(
translation=[shift_x, shift_y])
matrix = (matrix_to_topleft + matrix_transforms +
matrix_to_center)
coords = keypoints_on_image.get_coords_array()
#print("coords", coords)
#print("matrix", matrix.params)
coords_aug = tf.matrix_transform(coords, matrix.params)
#print("coords before", coords)
#print("coordsa ftre", coords_aug, np.around(coords_aug).astype(np.int32))
result.append(
ia.KeypointsOnImage.from_coords_array(
np.around(coords_aug).astype(np.int32),
shape=keypoints_on_image.shape))
else:
result.append(keypoints_on_image)
return result
def similarity(edge_image, mu, phi, sigma2, h, psi):
height, width = edge_image.shape
edge_distance = scipy.ndimage.distance_transform_edt(~edge_image)
w = (mu + phi @ h).reshape(-1, 2)
image_points = matrix_transform(w, psi.params)
closest_distances = scipy.interpolate.interp2d(range(width), range(height), edge_distance)
K = h.size
noise = scipy.stats.multivariate_normal(mean=np.zeros(K), cov=np.eye(K))
if noise.pdf(h.flatten()) == 0:
print(h.flatten())
noise = np.log(noise.pdf(h.flatten()))
return -closest_distances(image_points[:, 0], image_points[:, 1]).sum() / sigma2 + noise
def apply_random_transformation(background_size, segmented_box, config_params):
"""apply a random transformation to 2D coordinates nomalized to image size"""
# translate object coordinates to the object center's frame, i.e. whitens
whitened_coords_norm = segmented_box.segmented_coords_norm - (
segmented_box.x_center_norm, segmented_box.y_center_norm)
# then generate a random rotation around the z-axis (perpendicular to the image plane), and limit the object scale
# to maximum (default) 50% of the background image, i.e. the normalized largest dimension of the object must be at
# most 0.5. To put it simply, scale the objects down if they're too big.
# TODO(minhnh) add shear
max_scale = config_params.max_scale
if segmented_box.max_dimension_norm > config_params.max_obj_size_in_bg:
max_scale = config_params.max_obj_size_in_bg / segmented_box.max_dimension_norm
random_rot_angle = np.random.uniform(0, np.pi)
rand_scale = np.random.uniform(config_params.min_scale,
config_params.max_scale)
# generate a random translation within the image boundaries for whitened, normalized coordinates, taking into
# account the maximum allowed object dimension. After this translation, the normalized coordinates should
# stay within [margin, 1-margin] for each dimension
scaled_max_dimension = segmented_box.max_dimension_norm * max_scale
low_norm_bound, high_norm_bound = ((scaled_max_dimension / 2) +
config_params.margin,
1 - config_params.margin -
(scaled_max_dimension / 2))
random_translation_x = np.random.uniform(
low_norm_bound, high_norm_bound) * background_size[1]
random_translation_y = np.random.uniform(
low_norm_bound, high_norm_bound) * background_size[0]
# create the transformation matrix for the generated rotation, translation and scale
if np.random.uniform() < config_params.prob_rand_rotation:
tf_matrix = SimilarityTransform(
rotation=random_rot_angle,
scale=rand_scale * min(background_size),
translation=(random_translation_x, random_translation_y)).params
else:
tf_matrix = SimilarityTransform(
scale=rand_scale * min(background_size),
translation=(random_translation_x, random_translation_y)).params
# apply transformation
transformed_coords = matrix_transform(whitened_coords_norm, tf_matrix)
# we clip the object coordinates so that they are within the image boundaries
transformed_coords[np.where(transformed_coords[:, 0] < 0), 0] = 0
transformed_coords[np.where(transformed_coords[:, 0] > background_size[1]),
0] = background_size[1] - 1
transformed_coords[np.where(transformed_coords[:, 1] < 0), 1] = 0
transformed_coords[np.where(transformed_coords[:, 1] > background_size[0]),
1] = background_size[0] - 1
return transformed_coords
def transform(self, T):
"""
Applies a skimage Transformation to self.x and self.y
The original coords are stored at self.x_orig and self.y_orig
The new coordinates can also be found at self.x_t and self.y_t
Arguments:
T -- A skimage Transformation to apply
"""
self.x_orig = self.x
self.y_orig = self.y
self.x, self.y = matrix_transform([self.x, self.y], T.params)[0]
self.x_t, self.y_t = self.x, self.y
self.pos = [self.x, self.y]
def similarity(edge_image, mu, phi, sigma2, h, psi):
height, width = edge_image.shape
edge_distance = scipy.ndimage.distance_transform_edt(~edge_image)
w = (mu + phi @ h).reshape(-1, 2)
image_points = matrix_transform(w, psi.params)
closest_distances = scipy.interpolate.interp2d(range(width), range(height),
edge_distance)
K = h.size
noise = scipy.stats.multivariate_normal(mean=np.zeros(K), cov=np.eye(K))
if noise.pdf(h.flatten()) == 0:
print(h.flatten())
noise = np.log(noise.pdf(h.flatten()))
return -closest_distances(image_points[:, 0],
image_points[:, 1]).sum() / sigma2
def check_if_findtransform_ok(self, numstars):
"""Helper function to test find_transform with common test code
for 3, 4, 5, and 6 stars"""
from skimage.transform import estimate_transform, matrix_transform
if numstars > 6:
raise NotImplementedError
# x and y of stars in the ref frame (int's)
self.star_refx = np.array([100, 120, 400, 400, 200, 200])[:numstars]
self.star_refy = np.array([150, 200, 200, 320, 210, 350])[:numstars]
self.num_stars = numstars
# Fluxes of stars
self.star_f = np.array(numstars * [700.0])
(
self.image,
self.image_ref,
self.star_ref_pos,
self.star_new_pos,
) = simulate_image_pair(
shape=(self.h, self.w),
translation=(self.x_offset, self.y_offset),
rot_angle_deg=50.0,
num_stars=self.num_stars,
star_refx=self.star_refx,
star_refy=self.star_refy,
star_flux=self.star_f,
)
source = self.star_ref_pos
dest = self.star_new_pos.copy()
t_true = estimate_transform("similarity", source, dest)
# disorder dest points so they don't match the order of source
np.random.shuffle(dest)
t, (src_pts, dst_pts) = aa.find_transform(source, dest)
self.assertLess(t_true.scale - t.scale, 1e-10)
self.assertLess(t_true.rotation - t.rotation, 1e-10)
self.assertLess(np.linalg.norm(t_true.translation - t.translation),
1.0)
self.assertEqual(src_pts.shape[0], dst_pts.shape[0])
self.assertLessEqual(src_pts.shape[0], source.shape[0])
self.assertEqual(src_pts.shape[1], 2)
self.assertEqual(dst_pts.shape[1], 2)
dst_pts_test = matrix_transform(src_pts, t.params)
self.assertLess(np.linalg.norm(dst_pts_test - dst_pts), 1.0)
def transform_coords(t, coords_in, centre_patch):
def tform_centred_rec(radian, translation, center):
tform1 = skt.SimilarityTransform(translation=translation)
tform2 = skt.SimilarityTransform(translation=center)
tform3 = skt.SimilarityTransform(rotation=radian)
tform4 = skt.SimilarityTransform(translation=-center)
tform = tform4 + tform3 + tform2 + tform1
return tform
param = t.get_params()
rot_radian = -param[0]
tx = param[2]
ty = param[1]
# transform the transformed patch coordinates back
tform_patch_rec = tform_centred_rec(radian=rot_radian,
translation=(tx, ty),
center=centre_patch)
coords_out = skt.matrix_transform(coords_in, tform_patch_rec.params)
return coords_out
def transform(postX, postY, transX, transY, rot):
"""Coordinate transform function.
Args:
postX (list): X coordinates to transform (1D).
postY (list): Y coordinates to transform (1D).
transX (float): Translation value in X dimension.
transY (float): Translation value in Y dimension.
rot (float): Rotation value in degree.
Returns:
transformed (list): Transformed coordinates (2D).
"""
tform = tf.SimilarityTransform(scale=1,
rotation=rot,
translation=(transX, transY))
transformed = tf.matrix_transform(np.transpose([postX, postY]),
tform.params)
return transformed
def get_im_coor(path, json): import math import cv2 import numpy as np img = cv2.imread(path) rows = img.shape[0]# height y columns = img.shape[1]# width x # get the head and tail coordinates if(np.size(json.annotations) == 0): return None, None,None, None x1 = json.annotations[0][u'x'] x2 = json.annotations[1][u'x'] y1 = json.annotations[0][u'y'] y2 = json.annotations[1][u'y'] # find appropriate placement x1_ = np.maximum(0, np.minimum(x1,x2)-50) x2_ = np.minimum(columns, np.maximum(x1,x2)+50) y1_ = np.maximum(0, np.minimum(y1,y2)-50) y2_ = np.minimum(rows, np.maximum(y1,y2)+50) target = 139 resized = cv2.resize(img, (target, target), cv2.INTER_LINEAR) height, width = img.shape[:2] x_ratio = float(target)/float(width) y_ratio = float(target)/float(height) x1_r = x1_*x_ratio y1_r = y1_*y_ratio x2_r = x2_*x_ratio y2_r = y2_*y_ratio # data augmentation: M = np.array([[ 0.85, -0.15, 18. ],[ -0.15, 0.85, 18. ]]) M1 = np.concatenate((M, np.array([[0,0,1]])),axis = 0) # get the transformed image dst = cv2.warpAffine(resized, M, (target, target)) coord = np.array([[x1_r,y1_r],[x2_r,y2_r]]) coord1 = matrix_transform(coord, M1) # resized x0, y0, w, h resized_array = np.array([x1_r, y1_r, x2_r-x1_r, y2_r-y1_r]) dst_array = np.array([coord1[0][0], coord1[0][1], coord1[1][0]-coord1[0][0], coord1[1][1]-coord1[0][1]]) return resized, resized_array, dst, dst_array
def apply_random_transformation(background_size,
segmented_box,
margin=0.1,
max_obj_size_in_bg=0.4):
"""apply a random transformation to 2D coordinates nomalized to image size"""
orig_coords_norm = segmented_box.segmented_coords_homog_norm[:, :2]
# translate object coordinates to the object center's frame, i.e. whitens
whitened_coords_norm = orig_coords_norm - (segmented_box.x_center_norm,
segmented_box.y_center_norm)
# then generate a random rotation around the z-axis (perpendicular to the image plane), and limit the object scale
# to maximum (default) 50% of the background image, i.e. the normalized largest dimension of the object must be at
# most 0.5. To put it simply, scale the objects down if they're too big.
# TODO(minhnh) add shear
max_scale = 1.
if segmented_box.max_dimension_norm > max_obj_size_in_bg:
max_scale = max_obj_size_in_bg / segmented_box.max_dimension_norm
random_rot_angle = np.random.uniform(0, np.pi)
rand_scale = np.random.uniform(0.5 * max_scale, max_scale)
# generate a random translation within the image boundaries for whitened, normalized coordinates, taking into
# account the maximum allowed object dimension. After this translation, the normalized coordinates should
# stay within [margin, 1-margin] for each dimension
scaled_max_dimension = segmented_box.max_dimension_norm * max_scale
low_norm_bound, high_norm_bound = ((scaled_max_dimension / 2) + margin,
1 - margin - (scaled_max_dimension / 2))
random_translation_x = np.random.uniform(low_norm_bound, high_norm_bound)
random_translation_y = np.random.uniform(low_norm_bound, high_norm_bound)
# create the transformation matrix for the generated rotation, translation and scale
tf_matrix = SimilarityTransform(rotation=random_rot_angle,
scale=rand_scale,
translation=(random_translation_x,
random_translation_y)).params
# apply transformation
transformed_coords_norm = matrix_transform(whitened_coords_norm, tf_matrix)
return transformed_coords_norm
def find_transform(source,
target,
max_control_points=50,
detection_sigma=5,
min_area=5):
"""Estimate the transform between ``source`` and ``target``.
Return a SimilarityTransform object ``T`` that maps pixel x, y indices from
the source image s = (x, y) into the target (destination) image t = (x, y).
T contains parameters of the tranformation: ``T.rotation``,
``T.translation``, ``T.scale``, ``T.params``.
Args:
source (array-like): Either a numpy array of the source image to be
transformed or an interable of (x, y) coordinates of the target
control points.
target (array-like): Either a numpy array of the target (destination)
image or an interable of (x, y) coordinates of the target
control points.
max_control_points: The maximum number of control point-sources to find
the transformation.
detection_sigma: Factor of background std-dev above which is considered
a detection. This value is ignored if input are not images.
min_area: Minimum number of connected pixels to be considered a source.
This value is ignored if input are not images.
Returns:
The transformation object and a tuple of corresponding star positions
in source and target.::
T, (source_pos_array, target_pos_array)
Raises:
TypeError: If input type of ``source`` or ``target`` is not supported.
ValueError: If it cannot find more than 3 stars on any input.
"""
from scipy.spatial import KDTree
try:
if len(source[0]) == 2:
# Assume it's a list of (x, y) pairs
source_controlp = _np.array(source)[:max_control_points]
else:
# Assume it's a 2D image
source_controlp = _find_sources(
source, detection_sigma=detection_sigma,
min_area=min_area)[:max_control_points]
except Exception:
raise TypeError("Input type for source not supported.")
try:
if len(target[0]) == 2:
# Assume it's a list of (x, y) pairs
target_controlp = _np.array(target)[:max_control_points]
else:
# Assume it's a 2D image
target_controlp = _find_sources(
target, detection_sigma=detection_sigma,
min_area=min_area)[:max_control_points]
except Exception:
raise TypeError("Input type for target not supported.")
# Check for low number of reference points
if len(source_controlp) < 3:
raise ValueError("Reference stars in source image are less than the "
"minimum value (3).")
if len(target_controlp) < 3:
raise ValueError("Reference stars in target image are less than the "
"minimum value (3).")
source_invariants, source_asterisms = _generate_invariants(source_controlp)
source_invariant_tree = KDTree(source_invariants)
target_invariants, target_asterisms = _generate_invariants(target_controlp)
target_invariant_tree = KDTree(target_invariants)
# r = 0.1 is the maximum search distance, 0.1 is an empirical value that
# returns about the same number of matches than inputs
# matches_list is a list of lists such that for each element
# source_invariant_tree.data[i], matches_list[i] is a list of the indices
# of its neighbors in target_invariant_tree.data
matches_list = source_invariant_tree.query_ball_tree(target_invariant_tree,
r=0.1)
# matches unravels the previous list of matches into pairs of source and
# target control point matches.
# matches is a (N, 3, 2) array. N sets of similar corresponding triangles.
# 3 indices for a triangle in ref
# and the 3 indices for the corresponding triangle in target;
matches = []
# t1 is an asterism in source, t2 in target
for t1, t2_list in zip(source_asterisms, matches_list):
for t2 in target_asterisms[t2_list]:
matches.append(list(zip(t1, t2)))
matches = _np.array(matches)
inv_model = _MatchTransform(source_controlp, target_controlp)
n_invariants = len(matches)
max_iter = n_invariants
# Set the minimum matches to be between 1 and 10 asterisms
min_matches = max(1, min(10, int(n_invariants * MIN_MATCHES_FRACTION)))
if (len(source_controlp) == 3
or len(target_controlp) == 3) and len(matches) == 1:
best_t = inv_model.fit(matches)
inlier_ind = _np.arange(len(matches)) # All of the indices
else:
best_t, inlier_ind = _ransac(matches, inv_model, 1, max_iter,
PIXEL_TOL, min_matches)
triangle_inliers = matches[inlier_ind]
d1, d2, d3 = triangle_inliers.shape
inl_arr = triangle_inliers.reshape(d1 * d2, d3)
inl_unique = set(tuple(pair) for pair in inl_arr)
# In the next, multiple assignements to the same source point s are removed
# We keep the pair (s, t) with the lowest reprojection error.
inl_dict = {}
for s_i, t_i in inl_unique:
# calculate error
s_vertex = source_controlp[s_i]
t_vertex = target_controlp[t_i]
t_vertex_pred = matrix_transform(s_vertex, best_t.params)
error = _np.linalg.norm(t_vertex_pred - t_vertex)
# if s_i not in dict, or if its error is smaller than previous error
if s_i not in inl_dict or (error < inl_dict[s_i][1]):
inl_dict[s_i] = (t_i, error)
inl_arr_unique = _np.array([[s_i, t_i]
for s_i, (t_i, e) in inl_dict.items()])
s, d = inl_arr_unique.T
return best_t, (source_controlp[s], target_controlp[d])
def align_crop_5pts_skimage(img,
src_landmarks,
mean_landmarks=_DEFAULT_MEAN_LANDMARKS,
crop_size=512,
face_factor=0.7,
landmark_factor=0.35,
align_type='similarity',
order=3,
mode='edge'):
"""Align and crop a face image by 5 landmarks.
Arguments:
img : Face image to be aligned and cropped.
src_landmarks : 5 landmarks:
[[left_eye_x, left_eye_y],
[right_eye_x, right_eye_y],
[nose_x, nose_y],
[left_mouth_x, left_mouth_y],
[right_mouth_x, right_mouth_y]].
mean_landmarks : Mean shape, should be normalized in [-0.5, 0.5].
crop_size : Output image size.
face_factor : The factor of face area relative to the output image.
landmark_factor : The factor of landmarks' area relative to the face.
align_type : 'similarity' or 'affine'.
order : The order of interpolation. The order has to be in the range 0-5:
- 0: Nearest-neighbor
- 1: Bi-linear
- 2: Bi-quadratic
- 3: Bi-cubic
- 4: Bi-quartic
- 5: Bi-quintic
mode : One of ['constant', 'edge', 'symmetric', 'reflect', 'wrap'].
Points outside the boundaries of the input are filled according
to the given mode.
"""
import skimage.transform as transform
# check
assert align_type in [
'affine', 'similarity'
], 'Invalid `align_type`! Allowed: %s!' % ['affine', 'similarity']
assert order in [0, 1, 2, 3, 4,
5], 'Invalid `order`! Allowed: %s!' % [0, 1, 2, 3, 4, 5]
assert mode in ['constant', 'edge', 'symmetric', 'reflect',
'wrap'], 'Invalid `mode`! Allowed: %s!' % [
'constant', 'edge', 'symmetric', 'reflect', 'wrap'
]
# move
move = np.array([img.shape[1] // 2, img.shape[0] // 2])
# pad border
v_border = img.shape[0] - crop_size
w_border = img.shape[1] - crop_size
if v_border < 0:
v_half = (-v_border + 1) // 2
img = np.pad(img, ((v_half, v_half), (0, 0), (0, 0)), mode=mode)
src_landmarks += np.array([0, v_half])
move += np.array([0, v_half])
if w_border < 0:
w_half = (-w_border + 1) // 2
img = np.pad(img, ((0, 0), (w_half, w_half), (0, 0)), mode=mode)
src_landmarks += np.array([w_half, 0])
move += np.array([w_half, 0])
# estimate transform matrix
mean_landmarks -= np.array([mean_landmarks[0, :] + mean_landmarks[1, :]
]) / 2.0 # middle point of eyes as center
trg_landmarks = mean_landmarks * (crop_size * face_factor *
landmark_factor) + move
tform = transform.estimate_transform(align_type, trg_landmarks,
src_landmarks)
# fix the translation to match the middle point of eyes
trg_mid = (trg_landmarks[0, :] + trg_landmarks[1, :]) / 2.0
src_mid = (src_landmarks[0, :] + src_landmarks[1, :]) / 2.0
new_trg_mid = transform.matrix_transform([trg_mid], tform.params)[0]
tform.params[:2, 2] += src_mid - new_trg_mid
# warp image by given transform
output_shape = (crop_size // 2 + move[1] + 1, crop_size // 2 + move[0] + 1)
img_align = transform.warp(img,
tform,
output_shape=output_shape,
order=order,
mode=mode)
# crop
img_crop = img_align[-crop_size:, -crop_size:]
return img_crop
def test_matrix_transform():
tform = AffineTransform(scale=(0.1, 0.5), rotation=2)
assert_equal(tform(SRC), matrix_transform(SRC, tform.params))
def getNewMarkers(model_robust, markers):
mat = model_robust.params
markers1 = transform.matrix_transform(markers, mat)
return markers1
def evaluate_model(data_root, model_name, i_model='0500', display=None):
# data_root='../Datasets/Balvan_patches/fold1/patch_tlevel4/'
# model_name='balvan_fold1_us_b2a'
# i_model='1500'
# dataset-specific variables
if 'eliceiri' in model_name:
img_root = '../Datasets/HighRes_Splits/WSI'
w = 834
o = 608
elif 'balvan' in model_name:
img_root = '../Datasets/Balvan_1to4tiles'
w = 300 # patch width
o = w // 2 # offset: upper-left corner of patch
method = 'VXM'
gan_name = ''
mode = model_name.split('_')[-1]
coords_ref = np.array(([0, 0], [0, w], [w, w], [w, 0]))
centre_patch = np.array((w, w)) / 2. - 0.5
dir_A = data_root + 'A/test/'
dir_B = data_root + 'B/test/'
supervision = model_name.split('_')[-2]
assert supervision in ['su', 'us'], "supervision must be in ['su', 'us']"
supervision_dict = {'su': 'supervised', 'us': 'unsupervised'}
assert mode in ['a2b', 'b2a', 'a2a',
'b2b'], "mode must be in ['a2b', 'b2a', 'a2a', 'b2b']"
if mode == 'a2b':
dir_src = dir_A
dir_tar = dir_B
elif mode == 'b2a':
dir_src = dir_B
dir_tar = dir_A
elif mode == 'a2a':
dir_src = dir_A
dir_tar = dir_A
elif mode == 'b2b':
dir_src = dir_B
dir_tar = dir_B
load_model_file = glob(f'./models/{model_name}/{i_model}*')[-1]
# device handling
if args.gpu and (args.gpu != '-1'):
device = '/gpu:' + args.gpu
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
tf.keras.backend.set_session(tf.Session(config=config))
else:
device = '/cpu:0'
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
# load the affine model
with tf.device(device):
model = vxm.networks.VxmAffine.load(load_model_file)
suffix_src = '_' + os.listdir(dir_src)[0].split('_')[-1]
name_srcs = set([name[:-len(suffix_src)] for name in os.listdir(dir_src)])
suffix_tar = '_' + os.listdir(dir_tar)[0].split('_')[-1]
name_tars = set([name[:-len(suffix_tar)] for name in os.listdir(dir_tar)])
f_names = name_srcs & name_tars
f_names = list(f_names)
f_names.sort()
df = pd.read_csv(data_root + 'info_test.csv', index_col='Filename')
cnt_disp = 0
for f_name in tqdm(f_names):
# extract transformed patch coordinates
coords_trans = df.loc[f_name, [
'X1_Trans', 'Y1_Trans', 'X2_Trans', 'Y2_Trans', 'X3_Trans',
'Y3_Trans', 'X4_Trans', 'Y4_Trans'
]].to_numpy().reshape((4, 2))
# load test pair
img_src = load_image(dir_src +
f"{f_name}_T.{suffix_src.split('.')[-1]}")
img_tar = load_image(dir_tar +
f"{f_name}_R.{suffix_tar.split('.')[-1]}")
if img_src.ndim == 2:
img_src = img_src[np.newaxis, ..., np.newaxis]
if img_tar.ndim == 2:
img_tar = img_tar[np.newaxis, ..., np.newaxis]
img_src = img_src.astype('float') / 255
img_tar = img_tar.astype('float') / 255
# register
with tf.device(device):
affine = model.register(img_src, img_tar)
affine_matrix = np.concatenate(
[affine.squeeze().reshape(
(2, 3)), np.zeros((1, 3))], 0) + np.eye(3)
# coords_rec = skt.matrix_transform(coords_trans, affine_matrix)
tform = skt.SimilarityTransform(affine_matrix)
tform_patch_rec = tform_centred_rec(radian=tform.rotation,
translation=(tform.translation[0],
tform.translation[1]),
center=centre_patch)
coords_rec = skt.matrix_transform(coords_trans, tform_patch_rec.params)
# img_rec = vxm.tf.utils.transform(img_src, affine, rescale=1.0)
# calculate error
disp_error = dist_coords(coords_rec, coords_ref)
result = {
'X1_Recover': coords_rec[0][0],
'Y1_Recover': coords_rec[0][1],
'X2_Recover': coords_rec[1][0],
'Y2_Recover': coords_rec[1][1],
'X3_Recover': coords_rec[2][0],
'Y3_Recover': coords_rec[2][1],
'X4_Recover': coords_rec[3][0],
'Y4_Recover': coords_rec[3][1],
'Error': np.mean(disp_error)
}
# update result
df.loc[f_name, [
'X1_Recover', 'Y1_Recover', 'X2_Recover', 'Y2_Recover',
'X3_Recover', 'Y3_Recover', 'X4_Recover', 'Y4_Recover', 'Error'
]] = result
# display patch outline in original image
if display:
if cnt_disp < display:
suffix = os.path.basename(
os.listdir(f'{img_root}/A/')[0]).split('.')[-1]
imgB = skio.imread(f"{img_root}/B/{f_name}.{suffix}")
dispdirB = f'{data_root}/display/B/test'
if not os.path.exists(dispdirB):
os.makedirs(dispdirB)
if 'eliceiri' in model_name:
imgB_disp = imgB
elif 'balvan' in model_name:
imgB_disp = np.pad(imgB, w // 2, mode='reflect')
if len(imgB_disp.shape) == 2:
imgB_disp = np.repeat(imgB_disp.reshape(
imgB_disp.shape[0], imgB_disp.shape[1], 1),
3,
axis=-1)
imgB_disp = cv2.polylines(imgB_disp,
pts=[(o + coords_ref).reshape(
(-1, 1, 2))],
isClosed=True,
color=(0, 255, 0),
thickness=2)
imgB_disp = cv2.polylines(
imgB_disp,
pts=[np.int32(o + coords_trans).reshape((-1, 1, 2))],
isClosed=True,
color=(0, 0, 255),
thickness=2)
imgB_disp = cv2.polylines(
imgB_disp,
pts=[np.int32(o + coords_rec).reshape((-1, 1, 2))],
isClosed=True,
color=(255, 0, 0),
thickness=2)
# skio.imshow(imgB_disp)
skio.imsave(
f'{dispdirB}/{f_name}_{method+gan_name}_{mode}_{supervision}.{suffix}',
imgB_disp)
cnt_disp += 1
# img_rec = vxm.tf.utils.transform(img_src, affine, rescale=1.0)
# skio.imshow(img_rec[0, ..., 0])
else:
return
df.to_csv(data_root + f'results_{method}_{mode}_{supervision}.csv')
return
def transform(postX, postY,transX, transY, rot):
tform = tf.SimilarityTransform(scale=1, rotation=rot, translation=(transX, transY))
transformed = tf.matrix_transform(np.transpose([postX, postY]), tform.params)
return transformed
def compute_homography_and_warp(image,
vp1,
vp2,
trajectories,
corners,
method="posture",
output_dir=""):
height, width, _ = image.shape
# Find Projective Transform
vanishing_line = np.cross(vp1, vp2)
H = np.eye(3)
H[2] = vanishing_line / vanishing_line[2]
H = H / H[2, 2] # As h32 needs to be 1 in projection
final_homography = H
# If trajectories are not provided, we cannot rely on them for affine correction
if trajectories is not None:
# Determine a and b for the affine component of the homography
intersections, mean_intersection = extract_circular_points(
trajectories, H, method, output_dir)
# for mean_intersection in intersections:
A = np.eye(3, 3)
if len(mean_intersection) > 0:
a = mean_intersection[0]
b = mean_intersection[1]
A[0, 0] = 1 / b
A[0, 1] = -a / b
final_homography = np.dot(A, H)
# region Translation and scaling operations
# The image corners are transformed by the current matrix to determine the
# endpoints of the resulting wrapped image. Each column is a corner in homogenous
# coordinates
image_corners = np.array([[0, 0, width, width], [0, height, 0, height],
[1, 1, 1, 1]])
# Apply the current transformation
cords = np.dot(final_homography, image_corners)
# Normalize the points
cords = cords[:2] / cords[2]
# Now, the lines contain x and y coordinates of all points of the model
# The smallest ones are translated to 0-0 borders by determining the min
tx = min(0, cords[0].min())
ty = min(0, cords[1].min())
# Considering the applied transformation, determine the farthest points to the
# upper left corner
max_x = int(cords[0].max() - tx)
max_y = int(cords[1].max() - ty)
T = np.array([[1, 0, -tx], [0, 1, -ty], [0, 0, 1]])
final_homography = np.dot(T, final_homography)
S = np.array([[width / max_x, 0, 0], [0, height / max_y, 0], [0, 0, 1]])
final_homography = np.dot(S, final_homography)
# We end up with a image that has the same size but perspective corrected
# We don't clamp the result as losing information is not an option
# endregion
warped_img = transform.warp(image,
np.linalg.inv(final_homography),
clip=False,
output_shape=(height, width))
transformed_corners = transform.matrix_transform(corners, final_homography)
return warped_img, transformed_corners, final_homography
def make_patches(
img_root,
target_root,
fold=None,
t_level=1,
# trans_min=0, trans_max=20, rot_min=0, rot_max=5,
mode='train',
display=None):
# img_root='../Datasets/HighRes_Splits/WSI'
# target_root='../Datasets/Eliceiri_patches'
# trans_min=0
# trans_max=20
# rot_min=0
# rot_max=0
# mode='train'
w = 834
o = 608
coords_ref = np.array(([0, 0], [0, w], [w, w], [w, 0]))
centre_patch = np.array((w, w)) / 2. - 0.5
step_trans = 20
step_rot = 5
trans_min = step_trans * (t_level - 1)
trans_max = step_trans * t_level
rot_min = step_rot * (t_level - 1)
rot_max = step_rot * t_level
#modalities = {'MI':'SHG', 'WB':'BF'}
tardirA = f'{target_root}/patch_tlevel{t_level}/A/{mode}'
tardirB = f'{target_root}/patch_tlevel{t_level}/B/{mode}'
if not os.path.exists(tardirA):
os.makedirs(tardirA)
if not os.path.exists(tardirB):
os.makedirs(tardirB)
f_names = set([
'_'.join(name.split('_')[:-1])
for name in os.listdir(f'{img_root}/{mode}')
])
f_names = list(f_names)
f_names.sort()
# csv information
header = [
'ReferenceImage', 'Method', 'X1_Ref', 'Y1_Ref', 'X2_Ref', 'Y2_Ref',
'X3_Ref', 'Y3_Ref', 'X4_Ref', 'Y4_Ref', 'X1_Trans', 'Y1_Trans',
'X2_Trans', 'Y2_Trans', 'X3_Trans', 'Y3_Trans', 'X4_Trans', 'Y4_Trans',
'X1_Recover', 'Y1_Recover', 'X2_Recover', 'Y2_Recover', 'X3_Recover',
'Y3_Recover', 'X4_Recover', 'Y4_Recover', 'Displacement', 'Tx', 'Ty',
'AngleDegree', 'AngleRad', 'Error', 'DisplacementCategory'
]
df = pd.DataFrame(index=f_names, columns=header)
df.index.set_names('Filename', inplace=True)
if display is not None:
cnt_disp = 0
for f_name in tqdm(f_names):
# load WSI
if mode == 'train':
f_nameA = f_name + '_MI.tif'
f_nameB = f_name + '_WB.tif'
else:
f_nameA = f_name + '_SHG.tif'
f_nameB = f_name + '_BF.tif'
imgA = skio.imread(f'{img_root}/{mode}/{f_nameA}')
imgB = skio.imread(f'{img_root}/{mode}/{f_nameB}')
# random transformation parameters
rot_degree = random.choice(
(random.uniform(-rot_max,
-rot_min), random.uniform(rot_min, rot_max)))
tx = random.choice(
(random.uniform(-trans_max,
-trans_min), random.uniform(trans_min, trans_max)))
ty = random.choice(
(random.uniform(-trans_max,
-trans_min), random.uniform(trans_min, trans_max)))
rot_radian = np.deg2rad(rot_degree)
# transform WSI
centre_img = np.array((imgA.shape[0], imgA.shape[1])) / 2. - 0.5
tform_img = tform_centred(radian=rot_radian,
translation=(tx, ty),
center=centre_img)
imgA_trans = np.asarray(skt.warp(imgA, tform_img, preserve_range=True),
dtype=np.uint8)
imgB_trans = np.asarray(skt.warp(imgB, tform_img, preserve_range=True),
dtype=np.uint8)
# crop patches
patchA_ref = imgA[o:o + w, o:o + w]
patchB_ref = imgB[o:o + w, o:o + w]
patchA_trans = imgA_trans[o:o + w, o:o + w]
patchB_trans = imgB_trans[o:o + w, o:o + w]
# transform patch coordinates
tform_patch = tform_centred(radian=rot_radian,
translation=(tx, ty),
center=centre_patch)
coords_trans = skt.matrix_transform(coords_ref, tform_patch.params)
# calculate distance
dist = dist_coords(coords_trans, coords_ref)
# write csv line
line = {
'X1_Ref': coords_ref[0][0],
'Y1_Ref': coords_ref[0][1],
'X2_Ref': coords_ref[1][0],
'Y2_Ref': coords_ref[1][1],
'X3_Ref': coords_ref[2][0],
'Y3_Ref': coords_ref[2][1],
'X4_Ref': coords_ref[3][0],
'Y4_Ref': coords_ref[3][1],
'X1_Trans': coords_trans[0][0],
'Y1_Trans': coords_trans[0][1],
'X2_Trans': coords_trans[1][0],
'Y2_Trans': coords_trans[1][1],
'X3_Trans': coords_trans[2][0],
'Y3_Trans': coords_trans[2][1],
'X4_Trans': coords_trans[3][0],
'Y4_Trans': coords_trans[3][1],
'Displacement': np.mean(dist),
'Tx': tx,
'Ty': ty,
'AngleDegree': rot_degree,
'AngleRad': rot_radian
}
df.loc[f_name] = line
# save patches
skio.imsave(f'{tardirA}/{f_name}_R.tif', patchA_ref)
skio.imsave(f'{tardirA}/{f_name}_T.tif', patchA_trans)
skio.imsave(f'{tardirB}/{f_name}_R.tif', patchB_ref)
skio.imsave(f'{tardirB}/{f_name}_T.tif', patchB_trans)
# display patch outline in original image
if display is not None and cnt_disp < display:
dispdirA = f'{target_root}/patch_tlevel{t_level}/display/A/{mode}'
dispdirB = f'{target_root}/patch_tlevel{t_level}/display/B/{mode}'
if not os.path.exists(dispdirA):
os.makedirs(dispdirA)
if not os.path.exists(dispdirB):
os.makedirs(dispdirB)
imgA = np.repeat(imgA.reshape(imgA.shape[0], imgA.shape[1], 1),
3,
axis=-1)
imgA = cv2.polylines(imgA,
pts=[(o + coords_ref).reshape((-1, 1, 2))],
isClosed=True,
color=(0, 255, 0),
thickness=3)
imgA = cv2.polylines(
imgA,
pts=[np.int32(o + coords_trans).reshape((-1, 1, 2))],
isClosed=True,
color=(0, 0, 255),
thickness=3)
skio.imsave(f'{dispdirA}/{f_name}_display.tif', imgA)
imgB = cv2.polylines(imgB,
pts=[(o + coords_ref).reshape((-1, 1, 2))],
isClosed=True,
color=(0, 255, 0),
thickness=3)
imgB = cv2.polylines(
imgB,
pts=[np.int32(o + coords_trans).reshape((-1, 1, 2))],
isClosed=True,
color=(0, 0, 255),
thickness=3)
# imgB = cv2.polylines(imgB, pts=[np.int32(o+coords_rec).reshape((-1,1,2))], isClosed=True, color=(255,0,0), thickness=3)
skio.imsave(f'{dispdirB}/{f_name}_display.tif', imgB)
cnt_disp += 1
df.to_csv(f'{target_root}/patch_tlevel{t_level}/info_{mode}.csv')
def make_patches(img_root,
target_root,
fold=1,
t_level=1,
mode='train',
display=None):
# img_root='./Datasets/Zurich_tiles'
# target_root='./Datasets/Zurich_patches'
# fold=1
# t_level=2
# mode='test'
w = 300 # patch width
o = 0 # upper-left corner of patch
coords_ref = np.array(([0, 0], [0, w], [w, w], [w, 0]))
centre_patch = np.array((w, w)) / 2. - 0.5
step_trans = 7
step_rot = 5
trans_min = step_trans * (t_level - 1)
trans_max = step_trans * t_level
rot_min = step_rot * (t_level - 1)
rot_max = step_rot * t_level
#modalities = {'IR':'A', 'RGB':'B'}
tardirA = f'{target_root}/fold{fold}/patch_tlevel{t_level}/A/{mode}'
tardirB = f'{target_root}/fold{fold}/patch_tlevel{t_level}/B/{mode}'
if not os.path.exists(tardirA):
os.makedirs(tardirA)
if not os.path.exists(tardirB):
os.makedirs(tardirB)
ids_train, ids_test = split_zurich_data(fold)
if mode == 'train':
f_names = [
os.path.basename(f_path).split('.')[0] for id_img in ids_train
for f_path in glob(f'{img_root}/A/zh{id_img}_*')
]
elif mode == 'test':
f_names = [
os.path.basename(f_path).split('.')[0] for id_img in ids_test
for f_path in glob(f'{img_root}/A/zh{id_img}_*')
]
f_names = list(f_names)
f_names.sort()
# csv information
header = [
'ReferenceImage', 'Method', 'X1_Ref', 'Y1_Ref', 'X2_Ref', 'Y2_Ref',
'X3_Ref', 'Y3_Ref', 'X4_Ref', 'Y4_Ref', 'X1_Trans', 'Y1_Trans',
'X2_Trans', 'Y2_Trans', 'X3_Trans', 'Y3_Trans', 'X4_Trans', 'Y4_Trans',
'X1_Recover', 'Y1_Recover', 'X2_Recover', 'Y2_Recover', 'X3_Recover',
'Y3_Recover', 'X4_Recover', 'Y4_Recover', 'Displacement',
'RelativeDisplacement', 'Tx', 'Ty', 'AngleDegree', 'AngleRad', 'Error',
'DisplacementCategory'
]
df = pd.DataFrame(index=f_names, columns=header)
df.index.set_names('Filename', inplace=True)
if display is not None:
cnt_disp = 0
for f_name in tqdm(f_names):
# load original images
suffix = os.path.basename(
os.listdir(f'{img_root}/A/')[0]).split('.')[-1]
imgA = skio.imread(f"{img_root}/A/{f_name}.{suffix}")
imgB = skio.imread(f"{img_root}/B/{f_name}.{suffix}")
# random transformation parameters
rot_degree = random.choice(
(random.uniform(-rot_max,
-rot_min), random.uniform(rot_min, rot_max)))
tx = random.choice(
(random.uniform(-trans_max,
-trans_min), random.uniform(trans_min, trans_max)))
ty = random.choice(
(random.uniform(-trans_max,
-trans_min), random.uniform(trans_min, trans_max)))
rot_radian = np.deg2rad(rot_degree)
# transform original images
centre_img = np.array((imgA.shape[0], imgA.shape[1])) / 2. - 0.5
tform_img = tform_centred(radian=rot_radian,
translation=(tx, ty),
center=centre_img)
imgA_trans = np.asarray(skt.warp(imgA,
tform_img,
mode='reflect',
preserve_range=True),
dtype=np.uint8)
imgB_trans = np.asarray(skt.warp(imgB,
tform_img,
mode='reflect',
preserve_range=True),
dtype=np.uint8)
# crop patches
patchA_ref = imgA[o:o + w, o:o + w]
patchB_ref = imgB[o:o + w, o:o + w]
patchA_trans = imgA_trans[o:o + w, o:o + w]
patchB_trans = imgB_trans[o:o + w, o:o + w]
# transform patch coordinates
tform_patch = tform_centred(radian=rot_radian,
translation=(tx, ty),
center=centre_patch)
coords_trans = skt.matrix_transform(coords_ref, tform_patch.params)
# calculate distance
dist_array = dist_coords(coords_trans, coords_ref)
dist = np.mean(dist_array)
# write csv line
line = {
'X1_Ref': coords_ref[0][0],
'Y1_Ref': coords_ref[0][1],
'X2_Ref': coords_ref[1][0],
'Y2_Ref': coords_ref[1][1],
'X3_Ref': coords_ref[2][0],
'Y3_Ref': coords_ref[2][1],
'X4_Ref': coords_ref[3][0],
'Y4_Ref': coords_ref[3][1],
'X1_Trans': coords_trans[0][0],
'Y1_Trans': coords_trans[0][1],
'X2_Trans': coords_trans[1][0],
'Y2_Trans': coords_trans[1][1],
'X3_Trans': coords_trans[2][0],
'Y3_Trans': coords_trans[2][1],
'X4_Trans': coords_trans[3][0],
'Y4_Trans': coords_trans[3][1],
'Displacement': dist,
'RelativeDisplacement': dist / w,
'Tx': tx,
'Ty': ty,
'AngleDegree': rot_degree,
'AngleRad': rot_radian
}
df.loc[f_name] = line
# save patches
skio.imsave(f'{tardirA}/{f_name}_R.{suffix}', patchA_ref)
skio.imsave(f'{tardirA}/{f_name}_T.{suffix}', patchA_trans)
skio.imsave(f'{tardirB}/{f_name}_R.{suffix}', patchB_ref)
skio.imsave(f'{tardirB}/{f_name}_T.{suffix}', patchB_trans)
# display patch outline in original image
if display is not None and cnt_disp < display:
dispdirA = f'{target_root}/fold{fold}/patch_tlevel{t_level}/display/A/{mode}'
dispdirB = f'{target_root}/fold{fold}/patch_tlevel{t_level}/display/B/{mode}'
if not os.path.exists(dispdirA):
os.makedirs(dispdirA)
if not os.path.exists(dispdirB):
os.makedirs(dispdirB)
if len(imgA.shape) == 2:
imgA_disp = np.pad(imgA, w // 2, mode='reflect')
imgA_disp = np.repeat(imgA_disp.reshape(
imgA_disp.shape[0], imgA_disp.shape[1], 1),
3,
axis=-1)
else:
imgA_disp = np.pad(imgA, ((w // 2, w // 2), (w // 2, w // 2),
(0, 0)),
mode='reflect')
imgA_disp = cv2.polylines(imgA_disp,
pts=[(w // 2 + coords_ref).reshape(
(-1, 1, 2))],
isClosed=True,
color=(0, 255, 0),
thickness=2)
imgA_disp = cv2.polylines(
imgA_disp,
pts=[np.int32(w // 2 + coords_trans).reshape((-1, 1, 2))],
isClosed=True,
color=(0, 0, 255),
thickness=2)
skio.imsave(f'{dispdirA}/{f_name}_display.{suffix}', imgA_disp)
if len(imgB.shape) == 2:
imgB_disp = np.pad(imgB, w // 2, mode='reflect')
imgB_disp = np.repeat(imgB_disp.reshape(
imgB_disp.shape[0], imgB_disp.shape[1], 1),
3,
axis=-1)
else:
imgB_disp = np.pad(imgB, ((w // 2, w // 2), (w // 2, w // 2),
(0, 0)),
mode='reflect')
imgB_disp = cv2.polylines(imgB_disp,
pts=[(w // 2 + coords_ref).reshape(
(-1, 1, 2))],
isClosed=True,
color=(0, 255, 0),
thickness=2)
imgB_disp = cv2.polylines(
imgB_disp,
pts=[np.int32(w // 2 + coords_trans).reshape((-1, 1, 2))],
isClosed=True,
color=(0, 0, 255),
thickness=2)
# imgB_disp = cv2.polylines(imgB_disp, pts=[np.int32(w//2+coords_rec).reshape((-1,1,2))], isClosed=True, color=(255,0,0), thickness=2)
skio.imsave(f'{dispdirB}/{f_name}_display.{suffix}', imgB_disp)
cnt_disp += 1
df.to_csv(
f'{target_root}/fold{fold}/patch_tlevel{t_level}/info_{mode}.csv')
def align_ecc(img,
img_ref,
method='ecc',
mode='affine',
coords=None,
rescale=False,
use_gradient=True):
try:
import cv2
except ModuleNotFoundError:
print('It seems OpenCV is not install. Please do so by running:'
'pip install opencv-python-headless')
if rescale:
img0 = rescale_intensity(img_ref,
in_range='image',
out_range='float32').astype('float32')
img1 = rescale_intensity(img, in_range='image',
out_range='float32').astype('float32')
else:
img0 = img_ref.astype('float32')
img1 = img.astype('float32')
if use_gradient:
def get_gradient(im):
# Calculate the x and y gradients using Sobel operator
grad_x = cv2.Sobel(im, cv2.CV_32F, 1, 0, ksize=1)
grad_y = cv2.Sobel(im, cv2.CV_32F, 0, 1, ksize=1)
# Combine the two gradients
grad = cv2.addWeighted(np.absolute(grad_x), 0.5,
np.absolute(grad_y), 0.5, 0)
return grad
img0 = get_gradient(img0)
img1 = get_gradient(img1)
shift = register_translation(img0, img1, 10)
print('Found init shift: {}'.format(shift[0]))
warp_matrix = np.eye(2, 3, dtype=np.float32)
warp_matrix[:, 2] = -shift[0][::-1]
# warp_matrix[:,2] = -shift[0]
number_of_iterations = 1000000
termination_eps = 1e-6
ecc_mode = {
'affine': cv2.MOTION_AFFINE,
'translation': cv2.MOTION_TRANSLATION
}
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT,
number_of_iterations, termination_eps)
# print(warp_matrix)
(cc, warp_matrix) = cv2.findTransformECC(img0, img1, warp_matrix,
ecc_mode[mode], criteria)
# print(warp_matrix)
img_x = cv2.warpAffine(img1, cv2.invertAffineTransform(warp_matrix),
img1.shape)
imgref_x = cv2.warpAffine(img0, warp_matrix, img0.shape)
# make a scikit-image/ndimage-compatible output transform matrix (y and x flipped!)
trans_matrix = np.vstack((np.hstack(
(np.rot90(warp_matrix[:2, :2],
2), np.flipud(warp_matrix[:, 2:]))), [0, 0, 1]))
if coords is not None:
coords_x = matrix_transform(coords, trans_matrix)
else:
coords_x = []
return trans_matrix, coords_x, img_x, imgref_x
def test_matrix_transform():
tform = AffineTransform(scale=(0.1, 0.5), rotation=2)
assert_equal(tform(SRC), matrix_transform(SRC, tform.params))
# transform coordinates
response = requests.get(
"https://d24h2xsgaj29mf.cloudfront.net/raw/spatial_transcriptomics_stahl_2016/"
"Layer1_BC_transformation.txt")
transform = np.array([
float(v) for v in response.content.decode().strip().split()
]).reshape(3, 3).T
x, y = zip(*[map(float, v.split('x')) for v in data.index])
xy = np.hstack([
np.array(x)[:, None],
np.array(y)[:, None],
])
transformed = matrix_transform(xy, transform)
dims = (MATRIX_AXES.REGIONS.value, MATRIX_AXES.FEATURES.value)
coords = {
MATRIX_REQUIRED_REGIONS.REGION_ID:
(MATRIX_AXES.REGIONS, np.arange(data.shape[0])),
MATRIX_REQUIRED_REGIONS.X_REGION: (MATRIX_AXES.REGIONS, transformed[:, 0]),
MATRIX_REQUIRED_REGIONS.Y_REGION: (MATRIX_AXES.REGIONS, transformed[:, 1]),
MATRIX_REQUIRED_FEATURES.GENE_NAME: (MATRIX_AXES.FEATURES, data.columns)
}
data = da.from_array(data.values, chunks=MATRIX_CHUNK_SIZE)
matrix = starspace.Matrix.from_expression_data(data=data,
coords=coords,
dims=dims,
name="matrix",
# find the transformation matrix
# we only use rigid body, gives: x shift, y shift, rotation
sreg = StackReg(StackReg.RIGID_BODY)
tmat = sreg.register(ref=ref_img, mov=mov_img)
# if average_over_experiments:
# out_img = sreg.transform(mov=mov_img, tmat=tmat)
# ref_img = np.mean([ref_img, out_img], axis=0)
# apply the matrix to the old ROIs
# this is essentially just:
# src = np.vstack((x, y, np.ones_like(x)))
# dst = src.T @ matrix.T
mov_points = np.copy(ref_points)
mov_points[:, 0:2] = tf.matrix_transform(coords=ref_points[:, 0:2],
matrix=tmat)
# save in netcals image format. import via "load roi (legacy)"
x, y, i = mov_points[:].T
ut.save_rois(fname=mov_roi_saveto_list[idx],
roi_id=i,
x=x,
y=y,
roi_width=roi_width)
# plot to check if alignment worked and compare
if plt is not None:
mov_mask = np.zeros(shape=(width, height, 4), dtype="uint8")
for point in mov_points:
ut.paint_roi(point[0], point[1], roi_width, mov_mask, channel=2)
