def evaluate_dataset(self, image_dataset, frame_ids, out_dir): total_predictions = total_annotations = total_matches = 0 for idx in frame_ids: frame = image_dataset.all_frames[idx] predicted_bboxes = self.predict_complete(frame.img_data) predicted_bboxes = image_utils.group_bboxes(predicted_bboxes) predicted_annotations = [ image_utils.get_cell_center(r) for r in predicted_bboxes ] gt_annotations = [(ann[0], ann[1]) for ann in frame.annotations] matches = metric_utils.get_matches(predicted_annotations, gt_annotations) total_matches += len(matches) total_predictions += len(predicted_annotations) total_annotations += len(gt_annotations) recall_f, precision_f, f1_f = metric_utils.score_detections( predicted_annotations, frame.annotations, matches) logger.info('Processed frame:{}, precision:{}, recall:{}, f1:{}', frame.img_id, precision_f, recall_f, f1_f) annotated_img = image_utils.get_annotated_img( frame.img_data, predicted_annotations, (15, 15)) cv2.imwrite(os.path.join(out_dir, frame.img_id), image_utils.normalize(annotated_img)) # plt.imshow(image_utils.get_annotated_img(frame.img_data, predicted_annotations,(15,15))) # plt.show() precision = total_matches * 1.0 / total_predictions recall = total_matches * 1.0 / total_annotations f1 = 2 * precision * recall / (precision + recall) return precision, recall, f1
def load_image_data(self): ''' Reads the annotation file and create frame objects for all image frames. ''' logger.info('Loading data from directory:{}'.format(self.base_dir)) all_annotations = {} with open(self.annotation_file, 'r') as fr: for line in fr: frame, x, y, s, _ = list(map(int, line.split())) if frame not in all_annotations: all_annotations[frame] = [] all_annotations[frame].append((x, y, s)) all_files = os.listdir(self.base_dir) all_files = [fn for fn in all_files if fn.endswith(self.file_suffix)] all_files.sort(key=lambda x: filename_to_id(x)) # since image files are listed sequentially in annotation file roi = self.get_global_bounds(all_annotations) frame_infos = [] total_annotations = 0 for i, fn in enumerate(all_files): annotations = all_annotations[i] if i in all_annotations else [] frame_info = FrameInfo(self.base_dir, fn, roi, annotations) frame_infos.append(frame_info) total_annotations += len(annotations) logger.info('Total frames loaded:{}, total annotations:{}', len(frame_infos), total_annotations) return frame_infos
def decorator(*args, **kwargs): start_time = time.time() result = func(*args, **kwargs) end_time = time.time() logger.info("Execution time : {}() = {}sec".format( func.__name__, end_time - start_time)) return result
def on_epoch_end(self, epoch, logs={}): for idx in self.image_dataset.validation_idx[:1]: frame = self.image_dataset.all_frames[idx] patches = frame.sequential_patches(self.patch_size, self.step_size) for i, patch in enumerate(patches): input_img = np.expand_dims(patch.get_img(), axis=0) response_map = np.squeeze(self.model.predict(input_img)) logger.info('Frame:{} response min:{}, max:{}', frame.img_id, np.min(response_map), np.max(response_map)) np.save( os.path.join( self.outdir, 'response_map_{}_{}.npy'.format(frame.img_id, i)), response_map) # if __name__ == '__main__': # dataset_dir = '/data/lrz/hm-cell-tracking/sequences_A549/annotations' # checkpoint_dir = '/data/training/detectnet' # out_dir = os.path.join(checkpoint_dir, 'out') # batch_size = 1 # samples_per_epoc = 2 # nb_epocs = 500 # nb_validation_samples = 1 # patch_size = (224, 224) # grid_size = (32, 32) # no_classes = 2 # image_dataset = ImageDataset(dataset_dir) # detector = Detectnet(ResNet50, 'activation_49') # activation_48 # model = detector.fully_convolutional(no_classes) # # callback = OutputWriter(image_dataset, out_dir, patch_size, (200,200)) # callback.model = model # callback.on_epoch_end(None)
def build_model(self): input = Input(batch_shape=self.input_shape, name='input_1') base_model = VGG16(input_tensor=input) last_layer_name = 'block3_pool' base_model_out = base_model.get_layer(last_layer_name).output model = Model(input=base_model.input, output=base_model_out) no_features = base_model_out.get_shape()[3].value model = Convolution2D(no_features, 1, 1, border_mode='same', activation='relu')(model.output) model = Dropout(0.5)(model) class_out = Deconvolution2D(self.out_channels, 8, 8, output_shape=self.output_shape, subsample=(8, 8), activation='sigmoid', name='class_out')(model) model = Model(base_model.input, output=class_out) optimizer = Adam(lr=self.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model.compile(optimizer=optimizer, loss={'class_out': 'binary_crossentropy'}, metrics=['binary_accuracy']) for layer in model.layers: layer.trainable = False if layer.name == last_layer_name: break if self.weight_file: logger.info('Loading weights from :{}', self.weight_file) model.load_weights(weight_file) model.summary() return model
def calc_channel_mean(self): c1_frames = [frame.img_data[:, :, 0] for frame in self.all_frames] c2_frames = [frame.img_data[:, :, 1] for frame in self.all_frames] c3_frames = [frame.img_data[:, :, 2] for frame in self.all_frames] channel_mean = np.mean(c1_frames), np.mean(c2_frames), np.mean( c3_frames) logger.info('Channel mean:{}', channel_mean) return channel_mean
def __init__(self, input_shape, no_classes, grid_size, weight_file=None): self.input_shape = input_shape self.no_classes = no_classes self.grid_size = grid_size self.weight_file = weight_file self.model = self.build_model() if self.weight_file: self.model.load_weights(self.weight_file) logger.info('Loaded model weights from:{}', self.weight_file)
def split_dataset(self, test_ratio, validation_ratio): ''' Splits the dataset into training, testing and validation sets. ''' idx = range(self.dataset_size) idx = np.random.permutation(idx) training_data_size = int(self.dataset_size * (1 - test_ratio)) validation_size = int(training_data_size * validation_ratio) self.validation_idx = idx[:validation_size] self.training_idx = idx[validation_size:training_data_size] self.testing_idx = idx[training_data_size:] logger.info('Total dataset size:{}, training:{}, validation:{}, test:{}', self.dataset_size, len(self.training_idx), len(self.validation_idx), len(self.testing_idx))
def load_image_data(self): ''' Reads the annotation file and create frame objects for all image frames. ''' logger.info('Loading data from directory:{}'.format(self.base_dir)) all_annotations = {} all_files = os.listdir(self.base_dir) all_files = [fn for fn in all_files if fn.endswith(self.file_suffix)] all_files.sort(key=lambda x: filename_to_id(x)) # since image files are listed sequentially in annotation file global_bounds_of_images = self.get_global_bounds_from_images( self.base_dir, all_files) trees = gp.read_file(self.annotation_file) filtered_trees = trees[trees['geometry'].apply( lambda g: self.bound_contains_point(global_bounds_of_images, g))] filtered_trees.dropna(inplace=True) # The ['KRONE_DM'] is divided by 0.20, as each pixel is 0.20 * 0.20 cm and KRONE_DM is in metres # For Sanju: The third parameter is the size right???? filtered_trees_info = filtered_trees.apply( lambda row: (row['geometry'].x, row['geometry'].y, int(row['KRONE_DM'] / 0.2)), axis=1).values for (x1, y1, s) in filtered_trees_info: # TODO: Fix this hack when running on larger dataset! frame = int((x1 / 1000) - 565) if frame not in all_annotations: all_annotations[frame] = [] # x1 and y1 are divided by 0.2 to convert to pixel index all_annotations[frame].append((int((x1 % 1000) / 0.2), 5000 - int( (y1 % 1000) / 0.2), s)) roi = self.get_global_bounds(all_annotations) frame_infos = [] total_annotations = 0 for i, fn in enumerate(all_files): annotations = all_annotations[i] if i in all_annotations else [] frame_info = FrameInfo(self.base_dir, fn, roi, annotations) frame_infos.append(frame_info) total_annotations += len(annotations) logger.info('Total frames loaded:{}, total annotations:{}', len(frame_infos), total_annotations) return frame_infos
def split_dataset(self, test_ratio, validation_ratio): ''' Splits the dataset into training, testing and validation sets. ''' all_files = os.listdir(self.base_dir) idx = [fn for fn in all_files if fn.startswith(self.file_prefix)] training_data_size = int(self.dataset_size * (1 - test_ratio)) validation_size = int(training_data_size * validation_ratio) self.validation_idx = idx[:validation_size] self.training_idx = idx[validation_size:training_data_size] self.testing_idx = idx[training_data_size:] logger.info( 'Total dataset size:{}, training:{}, validation:{}, test:{}', self.dataset_size, len(self.training_idx), len(self.validation_idx), len(self.testing_idx))
def grid_dataset_generator(self, batch_size, patch_size=(224, 224), grid_size=(28, 28), dataset='training', sampling_type='random'): ''' Training data generator for grid based predictions. ''' all_patches = [] sample_idx = self.dataset_idx(dataset) for idx in sample_idx: frame = self.all_frames[idx] if sampling_type == 'random': all_patches.extend(frame.get_random_patches(patch_size, 1)) else: all_patches.extend( frame.sequential_patches(patch_size, (200, 200))) logger.info('Total patches:{}', len(all_patches)) while True: # Randomly select patches from sequential patches ... patches_idx = np.random.randint(0, len(sample_idx), batch_size) class_batch = [] bbout_batch = [] input_batch = [] for idx in patches_idx: frame = self.all_frames[sample_idx[idx]] # patch = frame.get_random_patches(patch_size, 1)[0] patch = frame.sequential_patches(patch_size, (200, 200))[0] label_map, bbox_map = self.grid_ground_truth(patch, grid_size) # plt.figure(1), plt.imshow(np.squeeze(label_map)) # plt.figure(2), plt.imshow(patch.annotated_img()) # plt.show() class_batch.append(label_map) bbout_batch.append(bbox_map) input_batch.append(patch.get_img()) yield ({ 'input_1': np.array(input_batch) }, { 'class_out': np.array(class_batch), 'bb_out': np.array(bbout_batch) })
def evaluate_model(dataset_dir, weight_file, out_dir, file_ext='.png'): """ Evaluates model using all images from dataset_dir. Usage: python -m detection.detectors.detectnet evaluate-model '/data/lrz/hm-cell-tracking/annotations/in/' \ '/data/training/detectnet/model_checkpoints/model.hdf5' /data/training/detectnet/eval/ --file-ext '.jpg' :param dataset_dir: :param weight_file: :param file_ext: :return: """ batch_size = 1 no_classes = 1 grid_size = (16, 16) detector = Detectnet([batch_size, 224, 224, 3], no_classes, grid_size, weight_file) dataset = ImageDataset(dataset_dir, file_ext, normalize=False) precision, recall, f1 = detector.evaluate_dataset( dataset, range(len(dataset.all_frames)), out_dir) logger.info("Precision:{}, recall:{}, f1:{}", precision, recall, f1)
def build_model(self): ''' The top layers after base model are fully connected. ''' last_layer_name = 'activation_11' base_model = ResNet50(input_tensor=self.input_shape) base_model_out = base_model.get_layer(last_layer_name).output model = Model(input=base_model.input, output=base_model_out) class_out = Convolution2D(self.no_classes, 1, 1, border_mode='same', activation='sigmoid')(model) bb_out = Convolution2D(4, 1, 1, border_mode='same')(model) model = Model(base_model.input, output=[class_out, bb_out]) optimizer = Adam(lr=1e-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model.compile(optimizer=optimizer, loss={'class_out': 'binary_crossentropy', 'bb_out': 'mean_squared_error'}) logger.info('Compiled fc with output:{}', model.output) return model
def build_model(self): ''' The top layers after base model are fully convolutional. ''' input_tensor = Input(batch_shape=self.input_shape) last_layer_name = 'activation_23' base_model = ResNet50(input_tensor=input_tensor) base_model_out = base_model.get_layer(last_layer_name).output model = Model(input=base_model.input, output=base_model_out) model = Convolution2D(128, 3, 3, border_mode='same', activation='relu')(model.output) model = Dropout(0.5)(model) # model = MaxPooling2D((2, 2))(model) class_out = Convolution2D(self.no_classes, 1, 1, border_mode='same', activation='sigmoid', name='class_out')(model) bb_out = Convolution2D(4, 1, 1, border_mode='same', name='bb_out')(model) model = Model(base_model.input, output=[class_out, bb_out]) optimizer = Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model.compile(optimizer=optimizer, loss={ 'class_out': 'binary_crossentropy', 'bb_out': 'mean_absolute_error' }) logger.info('Compiled fc with output:{}', model.output) # model.summary() return model
def predict_all(model, input_dir, out_dir, file_pattern): batch_size = 1 if model == 'resnet': detector = FCNResnet50([batch_size, 224, 224, 3], 1e-3, 1, weight_file='weights/fcn_resnet.hdf5') step_size = (160, 160) elif model == 'unet': detector = UNet([batch_size, 252, 252, 3], 1e-3, 1, weight_file='weights/unet.hdf5') step_size = (180, 180) all_files = glob.glob(input_dir + file_pattern) for fn in all_files: logger.info('Processing file:{}', fn) base_filename = os.path.basename(fn) image = cv2.imread(fn) response_map = detector.predict_complete(image, step_size) rm_norm = normalize(response_map) out_path = out_dir + 'response_map_{}'.format(base_filename) logger.info('Writing response map at :{}', out_path) cv2.imwrite(out_path, rm_norm)
def grid_patch_dataset_generator(self, batch_size, patch_size=(224, 224), grid_size=(28, 28), nb_objects=5, dataset='training'): ''' Training data generator for grid patch based predictions where nb_objects labels and bounding boxes are predicted at each grid. ''' all_patches = [] sample_idx = self.dataset_idx(dataset) for idx in sample_idx: all_patches.extend(self.all_frames[idx].sequential_patches( patch_size, (200, 200))) logger.info('Total patches:{}', len(all_patches)) while True: # Randomly select patches from sequential patches ... patches_idx = np.random.randint(0, len(all_patches), batch_size) class_batch = [] bbout_batch = [] input_batch = [] for idx in patches_idx: patch = all_patches[idx] label_map, bbox_map = self.grid_patch_ground_truth( patch, grid_size, nb_objects) class_batch.append(label_map) bbout_batch.append(bbox_map) input_batch.append(patch.get_img()) yield ({ 'input_1': np.array(input_batch) }, { 'class_out': np.array(class_batch), 'bb_out': np.array(bbout_batch) })
def normalize_frames(self): for frame in self.all_frames: frame.img_data -= frame.img_data.mean() / (frame.img_data.std() + 1e-8) logger.info('Normalized frames with channel mean')
def get_predictions(self, dataset, frame_idx, base_dir): total_predictions = total_annotations = total_matches = 0 frame_metrics = [] all_predictions = [] all_fp = [] all_fn = [] for idx in frame_idx: frame = dataset.all_frames[idx] response_map = self.predict_complete(frame.img_data) actual_annotations = np.array([(ann[0], ann[1]) for ann in frame.annotations]) predicted_annotations = local_maxima(response_map, 20, 0.4) matches = get_matches(predicted_annotations, actual_annotations) logger.info('Frame:{} Predicted:{}, actual:{}, matches:{}', frame.img_id, len(predicted_annotations), len(frame.annotations), len(matches)) total_annotations += len(frame.annotations) total_matches += len(matches) total_predictions += len(predicted_annotations) false_pos = set(range( len(predicted_annotations))) - {match[0] for match in matches} false_neg = set(range( len(actual_annotations))) - {match[1] for match in matches} false_neg_ann = [frame.annotations[idx] for idx in false_neg] false_pos_ann = [predicted_annotations[idx] for idx in false_pos] all_predictions.extend([[frame.img_id] + ann.tolist() for ann in predicted_annotations]) all_fp.extend([[frame.img_id] + ann.tolist() for ann in false_pos_ann]) all_fn.extend([[frame.img_id] + list(ann) for ann in false_neg_ann]) rm_norm = normalize(response_map) cv2.imwrite( os.path.join(base_dir, 'response_map_{}'.format(frame.img_id)), rm_norm) # predicted_img = get_annotated_img(frame.img_data, predicted_annotations, (15, 15)) # false_neg_img = get_annotated_img(frame.img_data, false_neg_ann, (15, 15)) # fals_pos_img = get_annotated_img(frame.img_data, false_pos_ann, (15, 15)) # cv2.imwrite(os.path.join(base_dir, 'fn_{}'.format(frame.img_id)), false_neg_img) # cv2.imwrite(os.path.join(base_dir, 'fp_{}'.format(frame.img_id)), fals_pos_img) # cv2.imwrite(os.path.join(base_dir, 'prediction_{}'.format(frame.img_id)), predicted_img) recall_f, precision_f, f1_f = score_detections( predicted_annotations, frame.annotations, matches) # img_id, total_ann, total_predictions, total_matches, recall, precision, f1 frame_metric = [ frame.img_id, len(frame.annotations), len(predicted_annotations), len(matches), recall_f, precision_f, f1_f ] frame_metrics.append(frame_metric) write_to_file(all_predictions, os.path.join(base_dir, 'predictions.csv')) write_to_file(all_fp, os.path.join(base_dir, 'fp.csv')) write_to_file(all_fn, os.path.join(base_dir, 'fn.csv')) write_to_file(frame_metrics, os.path.join(base_dir, 'score.csv')) logger.info('Total matches:{}, predictions:{}, actual:{}', total_matches, total_predictions, total_annotations) precision = total_matches * 1.0 / total_predictions recall = total_matches * 1.0 / total_annotations f1 = 2 * precision * recall / (precision + recall) logger.info('Precision:{}, recall:{}, F1:{}', precision, recall, f1)