def evaluate_dataset(self, image_dataset, frame_ids, out_dir):
total_predictions = total_annotations = total_matches = 0
for idx in frame_ids:
frame = image_dataset.all_frames[idx]
predicted_bboxes = self.predict_complete(frame.img_data)
predicted_bboxes = image_utils.group_bboxes(predicted_bboxes)
predicted_annotations = [
image_utils.get_cell_center(r) for r in predicted_bboxes
]
gt_annotations = [(ann[0], ann[1]) for ann in frame.annotations]
matches = metric_utils.get_matches(predicted_annotations,
gt_annotations)
total_matches += len(matches)
total_predictions += len(predicted_annotations)
total_annotations += len(gt_annotations)
recall_f, precision_f, f1_f = metric_utils.score_detections(
predicted_annotations, frame.annotations, matches)
logger.info('Processed frame:{}, precision:{}, recall:{}, f1:{}',
frame.img_id, precision_f, recall_f, f1_f)
annotated_img = image_utils.get_annotated_img(
frame.img_data, predicted_annotations, (15, 15))
cv2.imwrite(os.path.join(out_dir, frame.img_id),
image_utils.normalize(annotated_img))
# plt.imshow(image_utils.get_annotated_img(frame.img_data, predicted_annotations,(15,15)))
# plt.show()
precision = total_matches * 1.0 / total_predictions
recall = total_matches * 1.0 / total_annotations
f1 = 2 * precision * recall / (precision + recall)
return precision, recall, f1
def load_image_data(self):
'''
Reads the annotation file and create frame objects for all image frames.
'''
logger.info('Loading data from directory:{}'.format(self.base_dir))
all_annotations = {}
with open(self.annotation_file, 'r') as fr:
for line in fr:
frame, x, y, s, _ = list(map(int, line.split()))
if frame not in all_annotations:
all_annotations[frame] = []
all_annotations[frame].append((x, y, s))
all_files = os.listdir(self.base_dir)
all_files = [fn for fn in all_files if fn.endswith(self.file_suffix)]
all_files.sort(key=lambda x: filename_to_id(x))
# since image files are listed sequentially in annotation file
roi = self.get_global_bounds(all_annotations)
frame_infos = []
total_annotations = 0
for i, fn in enumerate(all_files):
annotations = all_annotations[i] if i in all_annotations else []
frame_info = FrameInfo(self.base_dir, fn, roi, annotations)
frame_infos.append(frame_info)
total_annotations += len(annotations)
logger.info('Total frames loaded:{}, total annotations:{}',
len(frame_infos), total_annotations)
return frame_infos
def decorator(*args, **kwargs):
start_time = time.time()
result = func(*args, **kwargs)
end_time = time.time()
logger.info("Execution time : {}() = {}sec".format(
func.__name__, end_time - start_time))
return result
def on_epoch_end(self, epoch, logs={}):
for idx in self.image_dataset.validation_idx[:1]:
frame = self.image_dataset.all_frames[idx]
patches = frame.sequential_patches(self.patch_size, self.step_size)
for i, patch in enumerate(patches):
input_img = np.expand_dims(patch.get_img(), axis=0)
response_map = np.squeeze(self.model.predict(input_img))
logger.info('Frame:{} response min:{}, max:{}', frame.img_id,
np.min(response_map), np.max(response_map))
np.save(
os.path.join(
self.outdir,
'response_map_{}_{}.npy'.format(frame.img_id, i)),
response_map)
# if __name__ == '__main__':
# dataset_dir = '/data/lrz/hm-cell-tracking/sequences_A549/annotations'
# checkpoint_dir = '/data/training/detectnet'
# out_dir = os.path.join(checkpoint_dir, 'out')
# batch_size = 1
# samples_per_epoc = 2
# nb_epocs = 500
# nb_validation_samples = 1
# patch_size = (224, 224)
# grid_size = (32, 32)
# no_classes = 2
# image_dataset = ImageDataset(dataset_dir)
# detector = Detectnet(ResNet50, 'activation_49') # activation_48
# model = detector.fully_convolutional(no_classes)
#
# callback = OutputWriter(image_dataset, out_dir, patch_size, (200,200))
# callback.model = model
# callback.on_epoch_end(None)
def build_model(self):
input = Input(batch_shape=self.input_shape, name='input_1')
base_model = VGG16(input_tensor=input)
last_layer_name = 'block3_pool'
base_model_out = base_model.get_layer(last_layer_name).output
model = Model(input=base_model.input, output=base_model_out)
no_features = base_model_out.get_shape()[3].value
model = Convolution2D(no_features, 1, 1, border_mode='same', activation='relu')(model.output)
model = Dropout(0.5)(model)
class_out = Deconvolution2D(self.out_channels, 8, 8, output_shape=self.output_shape, subsample=(8, 8),
activation='sigmoid', name='class_out')(model)
model = Model(base_model.input, output=class_out)
optimizer = Adam(lr=self.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=optimizer,
loss={'class_out': 'binary_crossentropy'}, metrics=['binary_accuracy'])
for layer in model.layers:
layer.trainable = False
if layer.name == last_layer_name:
break
if self.weight_file:
logger.info('Loading weights from :{}', self.weight_file)
model.load_weights(weight_file)
model.summary()
return model
def calc_channel_mean(self):
c1_frames = [frame.img_data[:, :, 0] for frame in self.all_frames]
c2_frames = [frame.img_data[:, :, 1] for frame in self.all_frames]
c3_frames = [frame.img_data[:, :, 2] for frame in self.all_frames]
channel_mean = np.mean(c1_frames), np.mean(c2_frames), np.mean(
c3_frames)
logger.info('Channel mean:{}', channel_mean)
return channel_mean
def __init__(self, input_shape, no_classes, grid_size, weight_file=None):
self.input_shape = input_shape
self.no_classes = no_classes
self.grid_size = grid_size
self.weight_file = weight_file
self.model = self.build_model()
if self.weight_file:
self.model.load_weights(self.weight_file)
logger.info('Loaded model weights from:{}', self.weight_file)
def split_dataset(self, test_ratio, validation_ratio):
'''
Splits the dataset into training, testing and validation sets.
'''
idx = range(self.dataset_size)
idx = np.random.permutation(idx)
training_data_size = int(self.dataset_size * (1 - test_ratio))
validation_size = int(training_data_size * validation_ratio)
self.validation_idx = idx[:validation_size]
self.training_idx = idx[validation_size:training_data_size]
self.testing_idx = idx[training_data_size:]
logger.info('Total dataset size:{}, training:{}, validation:{}, test:{}', self.dataset_size,
len(self.training_idx), len(self.validation_idx), len(self.testing_idx))
def load_image_data(self):
'''
Reads the annotation file and create frame objects for all image frames.
'''
logger.info('Loading data from directory:{}'.format(self.base_dir))
all_annotations = {}
all_files = os.listdir(self.base_dir)
all_files = [fn for fn in all_files if fn.endswith(self.file_suffix)]
all_files.sort(key=lambda x: filename_to_id(x))
# since image files are listed sequentially in annotation file
global_bounds_of_images = self.get_global_bounds_from_images(
self.base_dir, all_files)
trees = gp.read_file(self.annotation_file)
filtered_trees = trees[trees['geometry'].apply(
lambda g: self.bound_contains_point(global_bounds_of_images, g))]
filtered_trees.dropna(inplace=True)
# The ['KRONE_DM'] is divided by 0.20, as each pixel is 0.20 * 0.20 cm and KRONE_DM is in metres
# For Sanju: The third parameter is the size right????
filtered_trees_info = filtered_trees.apply(
lambda row:
(row['geometry'].x, row['geometry'].y, int(row['KRONE_DM'] / 0.2)),
axis=1).values
for (x1, y1, s) in filtered_trees_info:
# TODO: Fix this hack when running on larger dataset!
frame = int((x1 / 1000) - 565)
if frame not in all_annotations:
all_annotations[frame] = []
# x1 and y1 are divided by 0.2 to convert to pixel index
all_annotations[frame].append((int((x1 % 1000) / 0.2), 5000 - int(
(y1 % 1000) / 0.2), s))
roi = self.get_global_bounds(all_annotations)
frame_infos = []
total_annotations = 0
for i, fn in enumerate(all_files):
annotations = all_annotations[i] if i in all_annotations else []
frame_info = FrameInfo(self.base_dir, fn, roi, annotations)
frame_infos.append(frame_info)
total_annotations += len(annotations)
logger.info('Total frames loaded:{}, total annotations:{}',
len(frame_infos), total_annotations)
return frame_infos
def split_dataset(self, test_ratio, validation_ratio):
'''
Splits the dataset into training, testing and validation sets.
'''
all_files = os.listdir(self.base_dir)
idx = [fn for fn in all_files if fn.startswith(self.file_prefix)]
training_data_size = int(self.dataset_size * (1 - test_ratio))
validation_size = int(training_data_size * validation_ratio)
self.validation_idx = idx[:validation_size]
self.training_idx = idx[validation_size:training_data_size]
self.testing_idx = idx[training_data_size:]
logger.info(
'Total dataset size:{}, training:{}, validation:{}, test:{}',
self.dataset_size, len(self.training_idx),
len(self.validation_idx), len(self.testing_idx))
def grid_dataset_generator(self,
batch_size,
patch_size=(224, 224),
grid_size=(28, 28),
dataset='training',
sampling_type='random'):
'''
Training data generator for grid based predictions.
'''
all_patches = []
sample_idx = self.dataset_idx(dataset)
for idx in sample_idx:
frame = self.all_frames[idx]
if sampling_type == 'random':
all_patches.extend(frame.get_random_patches(patch_size, 1))
else:
all_patches.extend(
frame.sequential_patches(patch_size, (200, 200)))
logger.info('Total patches:{}', len(all_patches))
while True:
# Randomly select patches from sequential patches ...
patches_idx = np.random.randint(0, len(sample_idx), batch_size)
class_batch = []
bbout_batch = []
input_batch = []
for idx in patches_idx:
frame = self.all_frames[sample_idx[idx]]
# patch = frame.get_random_patches(patch_size, 1)[0]
patch = frame.sequential_patches(patch_size, (200, 200))[0]
label_map, bbox_map = self.grid_ground_truth(patch, grid_size)
# plt.figure(1), plt.imshow(np.squeeze(label_map))
# plt.figure(2), plt.imshow(patch.annotated_img())
# plt.show()
class_batch.append(label_map)
bbout_batch.append(bbox_map)
input_batch.append(patch.get_img())
yield ({
'input_1': np.array(input_batch)
}, {
'class_out': np.array(class_batch),
'bb_out': np.array(bbout_batch)
})
def evaluate_model(dataset_dir, weight_file, out_dir, file_ext='.png'):
"""
Evaluates model using all images from dataset_dir.
Usage: python -m detection.detectors.detectnet evaluate-model '/data/lrz/hm-cell-tracking/annotations/in/' \
'/data/training/detectnet/model_checkpoints/model.hdf5' /data/training/detectnet/eval/ --file-ext '.jpg'
:param dataset_dir:
:param weight_file:
:param file_ext:
:return:
"""
batch_size = 1
no_classes = 1
grid_size = (16, 16)
detector = Detectnet([batch_size, 224, 224, 3], no_classes, grid_size,
weight_file)
dataset = ImageDataset(dataset_dir, file_ext, normalize=False)
precision, recall, f1 = detector.evaluate_dataset(
dataset, range(len(dataset.all_frames)), out_dir)
logger.info("Precision:{}, recall:{}, f1:{}", precision, recall, f1)
def build_model(self):
'''
The top layers after base model are fully connected.
'''
last_layer_name = 'activation_11'
base_model = ResNet50(input_tensor=self.input_shape)
base_model_out = base_model.get_layer(last_layer_name).output
model = Model(input=base_model.input, output=base_model_out)
class_out = Convolution2D(self.no_classes, 1, 1, border_mode='same', activation='sigmoid')(model)
bb_out = Convolution2D(4, 1, 1, border_mode='same')(model)
model = Model(base_model.input, output=[class_out, bb_out])
optimizer = Adam(lr=1e-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=optimizer,
loss={'class_out': 'binary_crossentropy', 'bb_out': 'mean_squared_error'})
logger.info('Compiled fc with output:{}', model.output)
return model
def build_model(self):
'''
The top layers after base model are fully convolutional.
'''
input_tensor = Input(batch_shape=self.input_shape)
last_layer_name = 'activation_23'
base_model = ResNet50(input_tensor=input_tensor)
base_model_out = base_model.get_layer(last_layer_name).output
model = Model(input=base_model.input, output=base_model_out)
model = Convolution2D(128, 3, 3, border_mode='same',
activation='relu')(model.output)
model = Dropout(0.5)(model)
# model = MaxPooling2D((2, 2))(model)
class_out = Convolution2D(self.no_classes,
1,
1,
border_mode='same',
activation='sigmoid',
name='class_out')(model)
bb_out = Convolution2D(4, 1, 1, border_mode='same',
name='bb_out')(model)
model = Model(base_model.input, output=[class_out, bb_out])
optimizer = Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
model.compile(optimizer=optimizer,
loss={
'class_out': 'binary_crossentropy',
'bb_out': 'mean_absolute_error'
})
logger.info('Compiled fc with output:{}', model.output)
# model.summary()
return model
def predict_all(model, input_dir, out_dir, file_pattern):
batch_size = 1
if model == 'resnet':
detector = FCNResnet50([batch_size, 224, 224, 3],
1e-3,
1,
weight_file='weights/fcn_resnet.hdf5')
step_size = (160, 160)
elif model == 'unet':
detector = UNet([batch_size, 252, 252, 3],
1e-3,
1,
weight_file='weights/unet.hdf5')
step_size = (180, 180)
all_files = glob.glob(input_dir + file_pattern)
for fn in all_files:
logger.info('Processing file:{}', fn)
base_filename = os.path.basename(fn)
image = cv2.imread(fn)
response_map = detector.predict_complete(image, step_size)
rm_norm = normalize(response_map)
out_path = out_dir + 'response_map_{}'.format(base_filename)
logger.info('Writing response map at :{}', out_path)
cv2.imwrite(out_path, rm_norm)
def grid_patch_dataset_generator(self,
batch_size,
patch_size=(224, 224),
grid_size=(28, 28),
nb_objects=5,
dataset='training'):
'''
Training data generator for grid patch based predictions where nb_objects labels and bounding boxes are
predicted at each grid.
'''
all_patches = []
sample_idx = self.dataset_idx(dataset)
for idx in sample_idx:
all_patches.extend(self.all_frames[idx].sequential_patches(
patch_size, (200, 200)))
logger.info('Total patches:{}', len(all_patches))
while True:
# Randomly select patches from sequential patches ...
patches_idx = np.random.randint(0, len(all_patches), batch_size)
class_batch = []
bbout_batch = []
input_batch = []
for idx in patches_idx:
patch = all_patches[idx]
label_map, bbox_map = self.grid_patch_ground_truth(
patch, grid_size, nb_objects)
class_batch.append(label_map)
bbout_batch.append(bbox_map)
input_batch.append(patch.get_img())
yield ({
'input_1': np.array(input_batch)
}, {
'class_out': np.array(class_batch),
'bb_out': np.array(bbout_batch)
})
def normalize_frames(self):
for frame in self.all_frames:
frame.img_data -= frame.img_data.mean() / (frame.img_data.std() +
1e-8)
logger.info('Normalized frames with channel mean')
def get_predictions(self, dataset, frame_idx, base_dir):
total_predictions = total_annotations = total_matches = 0
frame_metrics = []
all_predictions = []
all_fp = []
all_fn = []
for idx in frame_idx:
frame = dataset.all_frames[idx]
response_map = self.predict_complete(frame.img_data)
actual_annotations = np.array([(ann[0], ann[1])
for ann in frame.annotations])
predicted_annotations = local_maxima(response_map, 20, 0.4)
matches = get_matches(predicted_annotations, actual_annotations)
logger.info('Frame:{} Predicted:{}, actual:{}, matches:{}',
frame.img_id, len(predicted_annotations),
len(frame.annotations), len(matches))
total_annotations += len(frame.annotations)
total_matches += len(matches)
total_predictions += len(predicted_annotations)
false_pos = set(range(
len(predicted_annotations))) - {match[0]
for match in matches}
false_neg = set(range(
len(actual_annotations))) - {match[1]
for match in matches}
false_neg_ann = [frame.annotations[idx] for idx in false_neg]
false_pos_ann = [predicted_annotations[idx] for idx in false_pos]
all_predictions.extend([[frame.img_id] + ann.tolist()
for ann in predicted_annotations])
all_fp.extend([[frame.img_id] + ann.tolist()
for ann in false_pos_ann])
all_fn.extend([[frame.img_id] + list(ann)
for ann in false_neg_ann])
rm_norm = normalize(response_map)
cv2.imwrite(
os.path.join(base_dir, 'response_map_{}'.format(frame.img_id)),
rm_norm)
# predicted_img = get_annotated_img(frame.img_data, predicted_annotations, (15, 15))
# false_neg_img = get_annotated_img(frame.img_data, false_neg_ann, (15, 15))
# fals_pos_img = get_annotated_img(frame.img_data, false_pos_ann, (15, 15))
# cv2.imwrite(os.path.join(base_dir, 'fn_{}'.format(frame.img_id)), false_neg_img)
# cv2.imwrite(os.path.join(base_dir, 'fp_{}'.format(frame.img_id)), fals_pos_img)
# cv2.imwrite(os.path.join(base_dir, 'prediction_{}'.format(frame.img_id)), predicted_img)
recall_f, precision_f, f1_f = score_detections(
predicted_annotations, frame.annotations, matches)
# img_id, total_ann, total_predictions, total_matches, recall, precision, f1
frame_metric = [
frame.img_id,
len(frame.annotations),
len(predicted_annotations),
len(matches), recall_f, precision_f, f1_f
]
frame_metrics.append(frame_metric)
write_to_file(all_predictions, os.path.join(base_dir,
'predictions.csv'))
write_to_file(all_fp, os.path.join(base_dir, 'fp.csv'))
write_to_file(all_fn, os.path.join(base_dir, 'fn.csv'))
write_to_file(frame_metrics, os.path.join(base_dir, 'score.csv'))
logger.info('Total matches:{}, predictions:{}, actual:{}',
total_matches, total_predictions, total_annotations)
precision = total_matches * 1.0 / total_predictions
recall = total_matches * 1.0 / total_annotations
f1 = 2 * precision * recall / (precision + recall)
logger.info('Precision:{}, recall:{}, F1:{}', precision, recall, f1)