def predict_multiple_concat(configs, datasets, model_names, epoch = None,
augment_flips = False, augment_scale = False,
param_dict = {},
use_semantic = False,
nms_threshold = 0.3, voting_threshold = 0.5,
img_pad = 0, dilate = False,
save_predictions = False, create_submission = True):
ImageId = []
EncodedPixels = []
for _config, model_name, dataset in zip(configs, model_names, datasets):
_ImageId, _EncodedPixels = predict_model(_config, dataset, model_name = model_name, epoch = epoch,
augment_flips = augment_flips, augment_scale = augment_scale,
param_dict = param_dict,
nms_threshold = nms_threshold, voting_threshold = voting_threshold,
img_pad = img_pad, dilate = dilate,
save_predictions = save_predictions, create_submission = False)
ImageId += _ImageId
EncodedPixels += _EncodedPixels
if create_submission:
submission_filename = os.path.join(
submissions_dir, '_'.join(('submission',
configs[0].NAME,
str(epoch), datetime.datetime.now().strftime('%Y%m%d%H%M%S'), '.csv')))
f.write2csv(submission_filename, ImageId, EncodedPixels)
return submission_filename
return ImageId, EncodedPixels
def correct_submission(submission_file, use_test_dir=test_dir):
submission_data = extract_data(submission_file)
submission_filenames = submission_data[0]
submission_rles = submission_data[1]
assert len(np.unique(submission_filenames)) == 3019
problem_files = []
ImageId = []
EncodedPixels = []
for i in tqdm(range(len(submission_filenames) - 1, -1, -1)):
this_file = submission_filenames[i]
test_img = load_img(os.path.join(use_test_dir, this_file, 'images',
''.join((this_file, '.png'))),
greyscale=True)
mask_rles = submission_rles[np.argwhere(
submission_filenames == this_file).reshape(-1, )][0]
masks = np.stack([
run_length_decode(
rle, test_img.shape[1], test_img.shape[0], 1, index_offset=1).T
for rle in mask_rles
],
axis=-1)
if np.any(np.sum(masks, axis=-1) > 1):
#plot_multiple_images([np.sum(masks, axis = -1), np.sum(masks, axis = -1) > 1])
problem_files.append(this_file)
assert np.array_equal(np.unique(np.sum(masks, axis=-1)),
np.array([0, 2]))
n_masks = masks.shape[-1]
masks = list(np.moveaxis(masks, -1, 0))
masks = remove_overlaps(masks)
assert len(masks) == n_masks / 2
mask_rles = [f.run_length_encoding(m) for m in masks]
ImageId.extend([this_file] * len(mask_rles))
EncodedPixels.extend(list(mask_rles))
f.write2csv(submission_file.replace('.csv', '_CORRECTED.csv'), ImageId,
EncodedPixels)
return problem_files
def main(model_paths, cluster, output_dir, test_dir):
# test dataset
dataset_test = DSBDataset()
dataset_test.load_bowl(test_dir)
dataset_test.prepare()
# Recreate the model in inference mode
ROOT_DIR = os.getcwd()
MODEL_DIR = os.path.join(ROOT_DIR, "logs")
model = modellib.MaskRCNN(mode="inference",
config=inference_config,
model_dir=MODEL_DIR)
for model_path in tqdm(model_paths):
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
name, ext = os.path.splitext(os.path.basename(model_path))
output = []
sample_submission = pd.read_csv(
'../dataset/DSB/stage1_sample_submission.csv')
ImageId = []
EncodedPixels = []
for image_id in sample_submission.ImageId:
image_path = os.path.join(test_dir, image_id, 'images',
image_id + '.png')
original_image = cv2.imread(image_path)
results = model.detect([original_image], verbose=0)
r = results[0]
masks = r['masks']
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap2(
masks, image_id, r['scores'])
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
# save csv
df = f.write2csv(ImageId, EncodedPixels)
if cluster:
df = f.tocluster(df, cluster)
if not os.path.exists('./submit/{}'.format(output_dir)):
os.mkdir('./submit/{}'.format(output_dir))
df.to_csv('./submit/{}/{}.csv'.format(output_dir, name),
index=False,
columns=['ImageId', 'EncodedPixels'])
model_path = model.find_last()[1]
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
dataset_test = BowlDataset()
dataset_test.load_bowl('stage1_test')
dataset_test.prepare()
output = []
sample_submission = pd.read_csv('stage1_sample_submission.csv')
ImageId = []
EncodedPixels = []
for image_id in tqdm(sample_submission.ImageId):
image_path = os.path.join('stage1_test', image_id, 'images',
image_id + '.png')
original_image = cv2.imread(image_path)
results = model.detect([original_image], verbose=0)
r = results[0]
masks = r['masks']
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap2(
masks, image_id, r['scores'])
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
f.write2csv('submission_v2.csv', ImageId, EncodedPixels)
'''
ImageId = []
EncodedPixels = []
for i, image_id in enumerate(dataset.image_ids):
print(i)
# Load image
image = dataset.load_image(image_id)
# Run detection
r = model.detect([image], verbose=0)[0]
# post
semantic = dataset.load_semantic(image_id)
for j in range(r['masks'].shape[-1] - 1):
r['masks'][:, :, j] = dilation(r['masks'][:, :, j], 2)
r['masks'][:, :, j] = r['masks'][:, :, j] * semantic
masks = r['masks'] #[H, W, N] instance binary masks
img_name = dataset.image_info[image_id]['name']
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap2(
masks, img_name, r['scores'])
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
f.write2csv('results/' + 'MaskRCNN' + '_test.csv', ImageId, EncodedPixels)
print(len(np.unique(ImageId)))
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
dataset_test = BowlDataset()
dataset_test.load_bowl('stage1_val')
dataset_test.prepare()
output = []
sample_submission = pd.read_csv('sub_validation.csv')
ImageId = []
EncodedPixels = []
Scores_all = []
for image_id in tqdm(sample_submission.ImageId):
image_path = os.path.join('stage1_val', image_id, 'images',
image_id + '.png')
original_image = cv2.imread(image_path)
if original_image.shape[2] != 3:
original_image = original_image[:, :, :3]
results = model.detect([original_image], verbose=0)
r = results[0]
masks = r['masks']
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap2(
masks, image_id, r['scores'])
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
f.write2csv('sub_resnet101_Adam_new_FPN_val.csv', ImageId, EncodedPixels)
Scores_all=[]
for image_id in tqdm(sample_submission.ImageId):
image_path = os.path.join('stage1_test', image_id, 'images', image_id + '.png')
original_image = skimage.io.imread(image_path)
if original_image.shape[2] != 3:
original_image = original_image[:, :, :3]
results = model.detect([original_image], verbose=0)
r = results[0]
masks = r['masks']
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap2(masks, image_id, r['scores'])
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
f.write2csv('submission_v13_coco_newpost_from_44.csv', ImageId, EncodedPixels)
# for image_id in tqdm(sample_submission.ImageId):
# image_path = os.path.join('stage1_train', image_id, 'images', image_id + '.png')
#
# original_image = skimage.io.imread(image_path)
# if original_image.shape[2] != 3:
# original_image = original_image[:, :, :3]
# results = model.detect([original_image], verbose=0)
# r = results[0]
#
# masks = r['masks']
# ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap2(masks, image_id, r['scores'])
# ImageId += ImageId_batch
# EncodedPixels += EncodedPixels_batch
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
dataset_test = BowlDataset()
dataset_test.load_bowl('stage1_test')
dataset_test.prepare()
output = []
sample_submission = pd.read_csv('stage1_sample_submission.csv')
ImageId = []
EncodedPixels = []
for image_id in tqdm(sample_submission.ImageId):
image_path = os.path.join('stage1_test', image_id, 'images',
image_id + '.png')
original_image = cv2.imread(image_path)
if original_image.ndim != 3:
original_image = skimage.color.gray2rgb(image)
results = model.detect([original_image], verbose=0)
r = results[0]
masks = r['masks']
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap2(
masks, image_id, r['scores'])
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
f.write2csv('submission_v2_' + ep + '_epochs.csv', ImageId, EncodedPixels)
def predict_voting(configs, datasets, model_names, epochs = None,
augment_flips = False, augment_scale = False,
param_dict = {},
use_semantic = False,
nms_threshold = 0.3, voting_threshold = 0.5,
img_pad = 0, dilate = False,
save_predictions = False, create_submission = True):
"""
Predicts an ensemble over multiple models via voting
Presently assumes that augment_flips/scale/param_dict/threshold/use_semantic are the same
for all models you want to ensemble. Need to reformat to make these specific to each model.
Allows for cases where a single model is made up of multiple submodels that apply to different images.
"""
# Generalise the format of configs and datasets to cater for cases where a single model set may be
# made up of multiple models/datasets
configs = [_config if isinstance(_config, list) else [_config] for _config in configs]
datasets = [dataset if isinstance(dataset, list) else [dataset] for dataset in datasets]
model_names = [model_name if isinstance(model_name, list) else [model_name] for model_name in model_names]
epochs = [epoch if isinstance(epoch, list) else [epoch] for epoch in epochs] if epochs is not None else [[None for d in dataset] for dataset in datasets]
config_batch_sizes = [[c.BATCH_SIZE for c in _config] for _config in configs]
batch_size = max([max([b for b in _config_batch_size]) for _config_batch_size in config_batch_sizes])
# Create the models
models = [[create_model(c, m, e) for c, e, m in zip(_config, epoch, model_name)] for _config, epoch, model_name in zip(configs, epochs, model_names)]
# Create a mapping for each model of image_path: model index
model_infos = merge_model_info(datasets)
# Make sure that you have a full set of model mappings for each model set
assert np.all([len(m) == len(model_infos[0]) for m in model_infos[1:]])
img_paths = list(model_infos[0].keys())
img_paths.sort()
img_paths = np.array(img_paths)
n_images = len(img_paths)
# Set up holders for the submission rles which you will accumulate
ImageId = []
EncodedPixels = []
list_fn_apply = [] + (['apply_flips_rotations'] if augment_flips else []) + (['apply_scaling'] if augment_scale else [])
# NB: we need to predict in batches of _config.BATCH_SIZE
# as there are layers within the model that have strides dependent on this.
for i in tqdm(range(0, n_images, batch_size)):
batch_img_paths = img_paths[i : (i + batch_size)]
if len(batch_img_paths) != batch_size:
batch_img_paths = np.append(batch_img_paths, batch_img_paths[:(i + batch_size - len(img_paths))])
images, images_idx = gather_images(datasets, batch_img_paths)
images_model_set = [[model[_idx] for _idx in idx] for model, idx in zip(models, images_idx)]
configs_model_set = [[_config[_idx] for _idx in idx] for _config, idx in zip(configs, images_idx)]
identical_idx = [np.all([id == _idx[0] for id in _idx]) for _idx in images_idx]
# Run detection
res = []
for model, _images, _config, same_model in zip(images_model_set, images, configs_model_set, identical_idx):
# Check if we can run the whole batch through with one model
if same_model and _config[0].BATCH_SIZE == batch_size:
# Run detection
if len(list_fn_apply) > 0:
r = maskrcnn_detect_augmentations(_config[0], model[0], _images, list_fn_apply,
threshold = nms_threshold, voting_threshold = voting_threshold,
param_dict = param_dict,
use_nms = False, use_semantic = use_semantic)
else:
r = maskrcnn_detect(_config[0], model[0], _images, param_dict = param_dict, use_semantic = use_semantic)
else:
# The batch needs to be split into individual models
r = []
for _model, c, img in zip(model, _config, _images):
# Artifically expand the batch if required by batch_size
batch_img = [img] if c.BATCH_SIZE == 1 else [img] * c.BATCH_SIZE
# Run detection
if len(list_fn_apply) > 0:
prediction = maskrcnn_detect_augmentations(c, _model, batch_img, list_fn_apply,
threshold = nms_threshold, voting_threshold = voting_threshold,
param_dict = param_dict,
use_nms = False, use_semantic = use_semantic)
else:
prediction = maskrcnn_detect(c, _model, batch_img, param_dict = param_dict, use_semantic = use_semantic)
prediction = prediction[0]
r.append(prediction)
# r now contains the results for the images in the batch
res.append(r)
# Reduce to N images
for j, idx in enumerate(range(i, i + batch_size)):
if idx < n_images:
# Get masks via voting
# First reshape masks so that they can be concatenated:
for r in res:
r[j]['masks'] = np.moveaxis(r[j]['masks'], -1, 0)
if use_semantic:
# semantic_masks is flat. We need to expand to the r[j]['masks'] dimensions
r[j]['semantic_masks'] = np.stack([r[j]['semantic_masks']] * max(1, r[j]['masks'].shape[0]), axis = 0)
# Concatenate
img_results = du.concatenate_list_of_dicts([r[j] for r in res])
# Reduce via voting
img_results = reduce_via_voting(img_results, nms_threshold, voting_threshold, param_dict, use_semantic = use_semantic, n_votes = len(models))
# Reshape
img_results['masks'] = np.moveaxis(img_results['masks'], 0, -1)
img_results['class_ids'] = img_results['class_ids'].reshape(-1, )
img_results['scores'] = img_results['scores'].reshape(-1, )
img_name = os.path.splitext(os.path.split(batch_img_paths[j])[-1])[0]
# Create submission rle entry
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap_threshold(img_results['masks'], img_name, img_results['scores'])
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
# Print interim update
f.write2csv(os.path.join(submissions_dir, '_'.join(('submission_ensemble_interim', '.csv'))), ImageId, EncodedPixels)
if create_submission:
submission_filename = os.path.join(
submissions_dir,
'_'.join(
('submission_ensemble', datetime.datetime.now().strftime('%Y%m%d%H%M%S'), '.csv')))
f.write2csv(submission_filename, ImageId, EncodedPixels)
def predict_model(_config, dataset, model_name='MaskRCNN', epoch = None,
augment_flips = False, augment_scale = False,
param_dict = {},
nms_threshold = 0.3, voting_threshold = 0.5,
use_semantic = False,
img_pad = 0, dilate = False,
save_predictions = False, create_submission = True):
# Create save_dir
if save_predictions:
save_dir = os.path.join(data_dir, _config.NAME, '_'.join(('submission', datetime.datetime.now().strftime('%Y%m%d%H%M%S'))))
os.makedirs(save_dir)
# Recreate the model in inference mode
model = create_model(_config, model_name, epoch)
ImageId = []
EncodedPixels = []
list_fn_apply = [] + (['apply_flips_rotations'] if augment_flips else []) + (['apply_scaling'] if augment_scale else [])
if dilate:
n_dilate = param_dict['n_dilate'] if 'n_dilate' in param_dict else 1
# NB: we need to predict in batches of _config.BATCH_SIZE
# as there are layers within the model that have strides dependent on this.
for i in tqdm(range(0, len(dataset.image_ids), _config.BATCH_SIZE)):
# Load image
images = []
N = 0
for idx in range(i, i + _config.BATCH_SIZE):
if idx < len(dataset.image_ids):
N += 1
if img_pad > 0:
img = dataset.load_image(dataset.image_ids[idx])
images.append(np.stack([np.pad(img[:, :, i], img_pad, mode = 'reflect') for i in range(img.shape[-1])], axis = -1))
else:
images.append(dataset.load_image(dataset.image_ids[idx]))
else:
images.append(images[-1])
# Run detection
if len(list_fn_apply) > 0:
r = maskrcnn_detect_augmentations(_config, model, images, list_fn_apply,
threshold = nms_threshold, voting_threshold = voting_threshold,
param_dict = param_dict,
use_nms = False, use_semantic = use_semantic)
else:
r = maskrcnn_detect(_config, model, images, param_dict = param_dict, use_semantic = use_semantic)
# Reduce to N images
for j, idx in enumerate(range(i, i + _config.BATCH_SIZE)):
if j < N:
masks = r[j]['masks'] #[H, W, N] instance binary masks
scores = r[j]['scores']
boxes = r[j]['rois']
if img_pad > 0:
if use_semantic:
r[j]['semantic_masks'] = r[j]['semantic_masks'][img_pad : -img_pad, img_pad : -img_pad]
masks = masks[img_pad : -img_pad, img_pad : -img_pad]
valid = np.sum(masks, axis = (0, 1)) > 0
masks = masks[:, :, valid]
r[j]['masks'] = masks
r[j]['scores'] = r[j]['scores'][valid]
r[j]['class_ids'] = r[j]['class_ids'][valid]
r[j]['rois'] = r[j]['rois'][valid]
if dilate:
# Dilate masks within boundary box perimeters
box_labels = du.maskrcnn_boxes_to_labels(boxes, scores, masks.shape[:2])
dilated_masks = []
for i in range(masks.shape[-1]):
dilated_mask = scipy.ndimage.morphology.binary_dilation(masks[:, :, i], iterations = n_dilate)
#from visualize import plot_multiple_images, image_with_masks;
#plot_multiple_images([image_with_masks(masks[:, :, i], [box_labels == (i + 1)]), np.multiply(box_labels == (i + 1), dilated_mask)])
dilated_masks.append(np.multiply(box_labels == (i + 1), dilated_mask))
masks = np.stack(dilated_masks, axis = -1)
img_name = dataset.image_info[idx]['name']
ImageId_batch, EncodedPixels_batch = f.numpy2encoding_no_overlap_threshold(masks, img_name, scores)
ImageId += ImageId_batch
EncodedPixels += EncodedPixels_batch
if False:
class_names = ['background', 'nucleus']
visualize.display_instances((images[j] * 255).astype(np.uint8), r[j]['rois'], r[j]['masks'], r[j]['class_ids'], class_names, r[j]['scores'], figsize = (8, 8))
if save_predictions:
# Extract final masks from EncodedPixels_batch here and save
# using filename: (mosaic_id)_(mosaic_position)_(img_name).npy
save_model_predictions(save_dir, EncodedPixels_batch, masks.shape[:2], dataset.image_info[idx])
if create_submission:
submission_filename = os.path.join(submissions_dir, '_'.join(('submission', _config.NAME, str(epoch), datetime.datetime.now().strftime('%Y%m%d%H%M%S'), '.csv')))
f.write2csv(submission_filename, ImageId, EncodedPixels)
return submission_filename
return ImageId, EncodedPixels