def get_next_center(k,
stop_temporal,
c1_prev,
c2_prev,
img_current,
params_dict,
model,
template_init,
c1_init,
c2_init,
logger=None,
est_c1=None,
est_c2=None,
c1_hist=None,
c2_hist=None):
xax, yax = find_template_pixel(c1_prev, c2_prev, params_dict['width'],
img_current.shape[1], img_current.shape[0])
template_current = img_current[np.ravel(yax), np.ravel(xax)].reshape(
1, len(yax), len(xax))
current_centers = np.asarray([c1_prev, c2_prev]).reshape(1, 2)
pred = model.predict(x=[template_current, template_init, current_centers])
c1, c2 = pred[0, 0], pred[0, 1]
if est_c1 is not None and not stop_temporal:
c1_temp = est_c1.predict(c1_hist.reshape(1, -1))
c2_temp = est_c2.predict(c2_hist.reshape(1, -1))
if np.sqrt((c1_temp - c1)**2 + (c2_temp - c2)**2) > 1000:
if logger is None:
print('WARN: using temporal pred')
else:
logger.info('WARN: using temporal pred')
#logger.info('temp {}, {}'.format(c1_temp, c2_temp))
#logger.info('net {}, {}'.format(c1, c2))
c1, c2 = np.mean([c1_temp, c1]), np.mean([c2_temp, c2])
if ((np.abs(c1 - c1_init) > 30) or (np.abs(c2 - c2_init) > 30)):
k += 1
else:
k = 0
if ((k > 50) or (c1 > img_current.shape[1]) or (c2 > img_current.shape[0])
or (c1 < 0) or (c2 < 0)):
#logger.info('WARN: absurd prediction - remove temporal model')
stop_temporal = True
#logger.info('keep init')
c1, c2 = c1_init, c2_init
k = 0
try:
assert (np.abs(c1 - c1_prev) < 6)
assert (np.abs(c2 - c2_prev) < 6)
except AssertionError:
print(np.abs(c1 - c1_prev))
print(np.abs(c2 - c2_prev))
return c1, c2, stop_temporal, k
def __init__(self,
data_dir,
list_dir,
batch_size,
width_template=60,
resolution_df=None,
shuffle=True,
type='train',
upsample=True):
self.upsample = upsample
self.type = type
self.data_dir = data_dir
self.list_dir = list_dir
self.batch_size = batch_size
self.width_template = width_template
self.resolution_df = None
self.list_imgs = []
self.list_init_x = []
self.list_init_y = []
self.orig_labels_x = []
self.orig_labels_y = []
self.list_imgs_init = []
self.list_res_x = []
self.list_res_y = []
self.resolution_df = pd.read_csv(os.path.join(data_dir,
'resolution.csv'),
sep=',\s+',
decimal='.')
for subfolder in self.list_dir:
res_x = self.resolution_df.loc[self.resolution_df['scan'] ==
subfolder, 'res_x'].values[0]
res_y = self.resolution_df.loc[self.resolution_df['scan'] ==
subfolder, 'res_y'].values[0]
current_dir = os.path.join(self.data_dir, subfolder)
annotation_dir = os.path.join(current_dir, 'Annotation')
img_dir = os.path.join(current_dir, 'Data')
list_imgs = [
os.path.join(img_dir, dI) for dI in os.listdir(img_dir)
if (dI.endswith('png') and not dI.startswith('.'))
]
list_label_files = [
os.path.join(annotation_dir, dI)
for dI in os.listdir(annotation_dir)
if (dI.endswith('txt') and not dI.startswith('.'))
]
list_imgs.sort() # very important to keep the index order
list_label_files.sort()
for label_file in list_label_files:
df = pd.read_csv(label_file,
header=None,
names=['id', 'x', 'y'],
sep='\s+')
n_obs = len(df)
if self.upsample:
df['x_newres'] = df['x'] * res_x / 0.4
df['y_newres'] = df['y'] * res_y / 0.4
else:
df['x_newres'] = df['x']
df['y_newres'] = df['y']
c1_init, c2_init = df[['x_newres', 'y_newres']].values[0]
try:
img_init = np.asarray(
Image.open(
os.path.join(data_dir, subfolder, 'Data',
"{:04d}.png".format(int(1)))))
except FileNotFoundError:
img_init = np.asarray(
Image.open(
os.path.join(data_dir, subfolder, 'Data',
"{:05d}.png".format(int(1)))))
if np.isnan(c1_init):
print(label_file)
print(df.head())
print(df[['x', 'y']])
print(df[['x_newres', 'y_newres']])
print(c1_init, c2_init)
print(img_init.shape[1], img_init.shape[0])
xax, yax = find_template_pixel(c1_init, c2_init, 300,
img_init.shape[1],
img_init.shape[0])
template_big = img_init[np.ravel(yax), np.ravel(xax)]
listid = df.id.values[1:n_obs].astype(int).tolist()
big_array = np.asarray(
parmap.map(return_orig_pairs, listid, df, self.data_dir,
subfolder))
if self.type == 'train':
other_big_array = np.asarray(
parmap.starmap(return_rdm_pairs,
zip(listid, range(1, n_obs)), df,
self.data_dir, subfolder))
other_big_array2 = np.asarray(
parmap.starmap(return_rdm_pairs,
zip(listid, range(1, n_obs)), df,
self.data_dir, subfolder))
other_big_array3 = np.asarray(
parmap.starmap(return_rdm_pairs,
zip(listid, range(1, n_obs)), df,
self.data_dir, subfolder))
self.orig_labels_x = np.append(self.orig_labels_x, [
big_array[:, 0].astype(float),
other_big_array[:, 0].astype(float),
other_big_array2[:, 0].astype(float),
other_big_array3[:, 0].astype(float)
])
self.orig_labels_y = np.append(self.orig_labels_y, [
big_array[:, 1].astype(float),
other_big_array[:, 1].astype(float),
other_big_array2[:, 1].astype(float),
other_big_array3[:, 1].astype(float)
])
self.list_init_x = np.append(self.list_init_x, [
big_array[:, 2].astype(float),
other_big_array[:, 2].astype(float),
other_big_array2[:, 2].astype(float),
other_big_array3[:, 2].astype(float)
])
self.list_init_y = np.append(self.list_init_y, [
big_array[:, 3].astype(float),
other_big_array[:, 3].astype(float),
other_big_array2[:, 3].astype(float),
other_big_array3[:, 3].astype(float)
])
self.list_imgs = np.append(self.list_imgs, [
big_array[:, 5], other_big_array[:, 5],
other_big_array2[:, 5], other_big_array3[:, 5]
])
self.list_imgs_init = np.append(self.list_imgs_init, [
big_array[:, 4], other_big_array[:, 4],
other_big_array2[:, 4], other_big_array3[:, 4]
])
self.list_res_x = np.append(
self.list_res_x, np.repeat(res_x, 4 * (n_obs - 1)))
self.list_res_y = np.append(
self.list_res_y, np.repeat(res_y, 4 * (n_obs - 1)))
else:
self.orig_labels_x = np.append(
self.orig_labels_x, big_array[:, 0].astype(float))
self.orig_labels_y = np.append(
self.orig_labels_y, big_array[:, 1].astype(float))
self.list_init_x = np.append(self.list_init_x,
big_array[:, 2].astype(float))
self.list_init_y = np.append(self.list_init_y,
big_array[:, 3].astype(float))
self.list_imgs = np.append(self.list_imgs, big_array[:, 5])
self.list_imgs_init = np.append(self.list_imgs_init,
big_array[:, 4])
self.list_res_x = np.append(self.list_res_x,
np.repeat(res_x, (n_obs - 1)))
self.list_res_y = np.append(self.list_res_y,
np.repeat(res_y, (n_obs - 1)))
print(self.list_imgs.shape)
print(self.orig_labels_x.shape)
self.shuffle = shuffle
self.u_x_list = np.random.randn(len(self.orig_labels_x)) * 5
self.u_y_list = np.random.randn(len(self.orig_labels_y)) * 5
if self.type == 'val':
self.shuffle = False # don't shuffle if test set.
self.on_epoch_end()
def __data_generation(self, indexes):
'Generates data containing batch_size samples'
batch_orig_centers = np.zeros((len(indexes), 2))
batch_centers = np.zeros((len(indexes), 2))
batch_orig_centers[:, 0] = self.orig_labels_x[indexes]
batch_orig_centers[:, 1] = self.orig_labels_y[indexes]
batch_labels = np.zeros((len(indexes), 2))
batch_imgs = np.zeros(
(len(indexes), self.width_template + 1, self.width_template + 1))
batch_imgs_init = np.zeros(
(len(indexes), self.width_template + 1, self.width_template + 1))
try:
for i, idx in enumerate(indexes):
img = np.asarray(Image.open(self.list_imgs[idx]))
img = prepare_input_img(img, self.list_res_x[idx],
self.list_res_y[idx], self.upsample)
img_init = np.asarray(Image.open(self.list_imgs_init[idx]))
img_init = prepare_input_img(img_init, self.list_res_x[idx],
self.list_res_y[idx],
self.upsample)
c1_init = self.list_init_x[idx]
c2_init = self.list_init_y[idx]
# print(c1_init, c2_init)
xax, yax = find_template_pixel(c1_init, c2_init,
self.width_template,
img_init.shape[1],
img_init.shape[0])
try:
batch_imgs_init[i, :, :] = img_init[
np.ravel(yax),
np.ravel(xax)].reshape(self.width_template + 1,
self.width_template + 1)
except:
print(c1_init, c2_init)
print(self.list_imgs_init[idx])
# true location
c1, c2 = self.orig_labels_x[idx], self.orig_labels_y[idx]
# perturbed center of the template
# N(0,10) i.e. 95% perturbation are in -20;20.
u_x, u_y = self.u_x_list[idx], self.u_y_list[idx]
c1_perturbed, c2_perturbed = c1 - u_x, c2 - u_y
batch_centers[i] = [c1_perturbed, c2_perturbed]
# labels is the coord wrt to the center of
# the pixel so here c1 = c1_perturbed - 2
# label_x = -2 i.e. c1 = c1_perturbed + label
batch_labels[i] = [c1, c2]
xax, yax = find_template_pixel(c1_perturbed, c2_perturbed,
self.width_template,
img_init.shape[1],
img_init.shape[0])
batch_imgs[i] = img[np.ravel(yax),
np.ravel(xax)].reshape(
self.width_template + 1,
self.width_template + 1)
except:
print('ERROR')
print(self.list_res_x[idx], self.list_res_y[idx])
print(i)
print(idx)
print(c1, c2)
print(np.max(np.ravel(yax)))
print(np.max(np.ravel(xax)))
raise
return ([batch_imgs, batch_imgs_init, batch_centers], batch_labels)
def run_global_cv(fold_iterator,
data_dir,
checkpoint_dir,
logger,
params_dict,
upsample=True):
eucl_dist_per_fold = []
pixel_dist_per_fold = []
for traindirs, testdirs in fold_iterator:
# TRAIN LOCAL PREDICTION MODEL
# Generators
logger.info('############ FOLD #############')
logger.info('Training folders are {}'.format(traindirs))
training_generator = DataLoader(data_dir,
traindirs,
32,
width_template=params_dict['width'],
upsample=upsample)
validation_generator = DataLoader(data_dir,
testdirs,
32,
width_template=params_dict['width'],
type='val',
upsample=upsample)
model, est_c1, est_c2 = train(traindirs, data_dir, upsample,
params_dict, checkpoint_dir, logger,
validation_generator)
# PREDICT WITH GLOBAL MATCHING + LOCAL MODEL ON TEST SET
curr_fold_dist = []
curr_fold_pix = []
for k, testfolder in enumerate(testdirs):
res_x, res_y = training_generator.resolution_df.loc[
training_generator.resolution_df['scan'] == testfolder,
['res_x', 'res_y']].values[0]
annotation_dir = os.path.join(data_dir, testfolder, 'Annotation')
img_dir = os.path.join(data_dir, testfolder, 'Data')
list_imgs = [
os.path.join(img_dir, dI) for dI in os.listdir(img_dir)
if (dI.endswith('png') and not dI.startswith('.'))
]
list_label_files = [
dI for dI in os.listdir(annotation_dir)
if (dI.endswith('txt') and not dI.startswith('.'))
]
print(list_label_files)
try:
img_init = np.asarray(
Image.open(os.path.join(img_dir, "{:04d}.png".format(1))))
except FileNotFoundError:
img_init = np.asarray(
Image.open(os.path.join(img_dir, "{:05d}.png".format(1))))
img_init = prepare_input_img(img_init, res_x, res_y, upsample)
for j, label_file in enumerate(list_label_files):
print(label_file)
img_current = img_init
df = pd.read_csv(os.path.join(annotation_dir, label_file),
header=None,
names=['id', 'x', 'y'],
sep='\s+')
if upsample:
df['x_newres'] = df['x'] * res_x / 0.4
df['y_newres'] = df['y'] * res_y / 0.4
else:
df['x_newres'] = df['x']
df['y_newres'] = df['y']
c1_init, c2_init = df.loc[
df['id'] == 1, ['x_newres', 'y_newres']].values[0, :]
a, b = np.nonzero(img_init[:, 20:(len(img_init) - 20)])
if upsample:
list_centers = [[
c1_init * 0.4 / res_x, c2_init * 0.4 / res_y
]]
else:
list_centers = [[c1_init, c2_init]]
xax, yax = find_template_pixel(c1_init, c2_init,
params_dict['width'],
img_init.shape[1],
img_init.shape[0])
template_init = img_init[np.ravel(yax),
np.ravel(xax)].reshape(
1, len(yax), len(xax))
c1, c2 = c1_init, c2_init
stop_temporal = False
k = 0
for i in range(2, len(list_imgs) + 1):
if i % 100 == 0:
print(i)
img_prev = img_current
try:
img_current = np.asarray(
Image.open(
os.path.join(img_dir, "{:04d}.png".format(i))))
except FileNotFoundError:
img_current = np.asarray(
Image.open(
os.path.join(img_dir, "{:05d}.png".format(i))))
img_current = prepare_input_img(img_current, res_x, res_y,
upsample)
if i > 5:
tmp = list_centers[-10:].reshape(-1, 2)
assert tmp.shape[0] == 5
c1, c2, stop_temporal, k = get_next_center(
k, stop_temporal, c1, c2, img_prev, img_current,
params_dict, model, template_init, c1_init,
c2_init, logger, est_c1, est_c2, tmp[:, 0], tmp[:,
1])
else:
c1, c2, stop_temporal, k = get_next_center(
k, stop_temporal, c1, c2, img_prev, img_current,
params_dict, model, template_init, c1_init,
c2_init, logger)
# project back in init coords
if upsample:
c1_orig_coords = c1 * 0.4 / res_x
c2_orig_coords = c2 * 0.4 / res_y
else:
c1_orig_coords = c1
c2_orig_coords = c2
list_centers = np.append(list_centers,
[c1_orig_coords, c2_orig_coords])
if i in df.id.values:
true = df.loc[df['id'] == i, ['x', 'y']].values[0]
diff_x = np.abs(c1_orig_coords - true[0])
diff_y = np.abs(c2_orig_coords - true[1])
if upsample:
dist = np.sqrt(diff_x**2 + diff_y**2)
logger.info(
'ID {} : euclidean dist diff {}'.format(
i, dist * 0.4))
else:
dist = np.sqrt((res_x * diff_x)**2 +
(diff_y * res_y)**2)
logger.info(
'ID {} : euclidean dist diff {}'.format(
i, dist))
if dist > 10:
# logger.info(
# 'Bad dist - maxNCC was {}'.format(maxNCC))
logger.info('True {},{}'.format(true[0], true[1]))
logger.info('Pred {},{}'.format(
c1_orig_coords, c2_orig_coords))
idx = df.id.values.astype(int)
list_centers = list_centers.reshape(-1, 2)
df_preds = list_centers[idx - 1]
df_true = df[['x', 'y']].values
absolute_diff = np.mean(np.abs(df_preds - df_true))
pix_dist = np.mean(
np.sqrt((df_preds[:, 0] - df_true[:, 0])**2 +
(df_preds[:, 1] - df_true[:, 1])**2))
dist = compute_euclidean_distance(df_preds, df_true)
curr_fold_dist.append(dist)
curr_fold_pix.append(pix_dist)
logger.info(
'======== Test Feature {} ======='.format(label_file))
logger.info('Pixel distance is {}'.format(pix_dist))
logger.info('Euclidean distance in mm {}'.format(dist))
logger.info('Mean absolute difference in pixels {}'.format(
absolute_diff))
pred_df = pd.DataFrame()
pred_df['idx'] = range(1, len(list_centers) + 1)
pred_df['c1'] = list_centers[:, 0]
pred_df['c2'] = list_centers[:, 1]
pred_df.to_csv(os.path.join(checkpoint_dir,
'{}.txt'.format(label_file)),
header=False,
index=False)
eucl_dist_per_fold = np.append(eucl_dist_per_fold,
np.mean(curr_fold_dist))
pixel_dist_per_fold = np.append(pixel_dist_per_fold,
np.mean(curr_fold_pix))
logger.info('EUCLIDEAN DISTANCE CURRENT FOLD {}'.format(
eucl_dist_per_fold[-1]))
logger.info('PIXEL DISTANCE CURRENT FOLD {}'.format(
pixel_dist_per_fold[-1]))
logger.info('================= END RESULTS =================')
logger.info('Mean euclidean distance in mm {} (std {})'.format(
np.mean(eucl_dist_per_fold), np.std(eucl_dist_per_fold)))
def predict(testdirs,
checkpoint_dir,
data_dir,
params_dict,
upsample=False,
resolution_df=None):
model = create_model(params_dict['width'] + 1,
params_dict['h1'],
params_dict['h2'],
params_dict['h3'],
embed_size=params_dict['embed_size'],
drop_out_rate=params_dict['dropout_rate'],
use_batch_norm=params_dict['use_batchnorm'])
model.load_weights(os.path.join(checkpoint_dir, 'model.h5'))
est_c1 = load(os.path.join(checkpoint_dir, 'est_c1.joblib'))
est_c2 = load(os.path.join(checkpoint_dir, 'est_c2.joblib'))
for testfolder in testdirs:
res_x, res_y = None, None
if upsample:
res_x, res_y = resolution_df.loc[resolution_df['scan'] ==
testfolder,
['res_x', 'res_y']].values[0]
annotation_dir = os.path.join(data_dir, testfolder, 'Annotation')
img_dir = os.path.join(data_dir, testfolder, 'Data')
list_imgs = [
os.path.join(img_dir, dI) for dI in os.listdir(img_dir)
if (dI.endswith('png') and not dI.startswith('.'))
]
list_label_files = [
dI for dI in os.listdir(annotation_dir)
if (dI.endswith('txt') and not dI.startswith('.'))
]
try:
img_init = np.asarray(
Image.open(os.path.join(img_dir, "{:04d}.png".format(1))))
except FileNotFoundError:
img_init = np.asarray(
Image.open(os.path.join(img_dir, "{:05d}.png".format(1))))
img_init = prepare_input_img(img_init, res_x, res_y, upsample)
for j, label_file in enumerate(list_label_files):
print(label_file)
img_current = img_init
df = pd.read_csv(os.path.join(annotation_dir, label_file),
header=None,
names=['id', 'x', 'y'],
sep='\s+')
if upsample:
df['x_newres'] = df['x'] * res_x / 0.4
df['y_newres'] = df['y'] * res_y / 0.4
else:
df['x_newres'] = df['x']
df['y_newres'] = df['y']
c1_init, c2_init = df.loc[df['id'] == 1,
['x_newres', 'y_newres']].values[0, :]
a, b = np.nonzero(img_init[:, 20:(len(img_init) - 20)])
if upsample:
list_centers = [[c1_init * 0.4 / res_x, c2_init * 0.4 / res_y]]
else:
list_centers = [[c1_init, c2_init]]
xax, yax = find_template_pixel(c1_init, c2_init,
params_dict['width'],
img_init.shape[1],
img_init.shape[0])
template_init = img_init[np.ravel(yax),
np.ravel(xax)].reshape(
1, len(yax), len(xax))
c1, c2 = c1_init, c2_init
k = 0
stop_temporal = False
for i in range(2, len(list_imgs) + 1):
if i % 100 == 0:
print(i)
try:
img_current = np.asarray(
Image.open(
os.path.join(img_dir, "{:04d}.png".format(i))))
except FileNotFoundError:
img_current = np.asarray(
Image.open(
os.path.join(img_dir, "{:05d}.png".format(i))))
img_current = prepare_input_img(img_current, res_x, res_y,
upsample)
if i > 5:
tmp = list_centers[-10:].reshape(-1, 2)
assert tmp.shape[0] == 5
c1, c2, stop_temporal, k = get_next_center(
k, stop_temporal, c1, c2, img_current, params_dict,
model, template_init, c1_init, c2_init, None, est_c1,
est_c2, tmp[:, 0], tmp[:, 1])
else:
c1, c2, stop_temporal, k = get_next_center(
k, stop_temporal, c1, c2, img_current, params_dict,
model, template_init, c1_init, c2_init, None)
# project back in init coords
if upsample:
c1_orig_coords = c1 * 0.4 / res_x
c2_orig_coords = c2 * 0.4 / res_y
else:
c1_orig_coords = c1
c2_orig_coords = c2
list_centers = np.append(list_centers,
[c1_orig_coords, c2_orig_coords])
list_centers = list_centers.reshape(-1, 2)
pred_df = pd.DataFrame()
pred_df['idx'] = range(1, len(list_centers) + 1)
pred_df['c1'] = list_centers[:, 0]
pred_df['c2'] = list_centers[:, 1]
pred_df.to_csv(os.path.join(checkpoint_dir,
'{}'.format(label_file)),
header=False,
index=False)