def save_data(image, labels, image_id=None, out_path=None): """ Save splited images and labels :param image: image to save :param labels: label to save :param image_id: name for image to save :param out_path: path for saved images :return: """ if image.shape[0] == labels.shape[0]: fold = image.shape[0] else: fold = None print("Number of splits mismatch") for f in range(0, fold): split_image_id = "%s-%s" % (image_id, f+1) out_images_dir = make_dir(os.path.join(out_path, r'{}/images'.format(split_image_id)), top_dir=DATASET_DIR) out_masks_dir = make_dir(os.path.join(out_path, r'{}/masks'.format(split_image_id)), top_dir=DATASET_DIR) imsave(os.path.join(out_images_dir, r'{}.png'.format(split_image_id)), image[f, :, :]) for l in range(0, labels.shape[1]): label = labels[f, l, :, :] if np.max(label) > 0: imsave(os.path.join(out_masks_dir, r'mask_{}.png'.format(l+1)), label)
def select_data(in_path,out_path,prefix="bin"): dirs.make_dir(out_path) all_files=dirs.get_files(in_path) prefix_files=[f for f in all_files if files.extract_prefix(str(f))==prefix] out_files=[out_path.create(path_i.get_name()) for path_i in prefix_files] for in_file,out_file in zip(in_prefix_files,out_prefix_files): shutil.copyfile(in_file, out_file) print(in_file)
def action_cats(action_path,out_path): dirs.make_dir(out_path) action_paths=dirs.get_files(action_path,dirs=True) actions=[ read_action(path_i,False) for path_i in action_paths] cats={} for action_i in actions: cats[action_i.cat]=out_path.create(action_i.cat) for cat_i in cats.keys(): dirs.make_dir(cats[cat_i]) for action_i in actions: name=action_i.name cat_path=cats[action_i.cat] dst_path=cat_path.create(name) print(str(dst_path)) action_i.save(dst_path)
def save_labels(labels, out_dir, model_name, image_id): """ Save labels (masks) after postprocessing pipeline :param labels: label image :param out_dir: output directory :param model_name: name of the model, used for this labels predition :param image_id: :return: """ warnings.filterwarnings("ignore") # Warnings off # Make output directories dir_to_save = r'{}/{}'.format(out_dir, model_name) # Path to return current_image_dir = r'{}/{}'.format(dir_to_save, image_id) make_dir(current_image_dir) for i, image in enumerate(labels): imsave(os.path.join(OUT_FILES, current_image_dir, r'{}.png'.format(i)), image)
def sum_masks(images_dir, image_id): warnings.filterwarnings("ignore") # Make output directories image_dir = os.path.join(ROOT_DIR, r'{}/{}'.format(images_dir, image_id)) model_name = images_dir.replace(dir_splitter, "/").split("/")[-1] out_dir = r'out_files/images/postproc/sum_masks/{}/{}'.format( model_name, image_id) make_dir(out_dir) image_ids = next(os.walk(image_dir))[2] image_ids.sort(key=lambda x: int(x.split(".")[0])) image = skimage.io.imread(r'{}/{}'.format(image_dir, image_ids[0])) height, width = image.shape labels = np.zeros((height, width), np.uint16) for img_id in image_ids: img = imread(r'{}/{}'.format(image_dir, img_id)) labels[img > 0] = 255 imsave(os.path.join(ROOT_DIR, out_dir, r'{}.png'.format(image_id)), labels)
def submit_predict(model, config, params, model_name, val_score, test_ids): """ Predicting and submission generate pipeline: 1. Predict test data 2. Postprocessing predicted data 3. Encode submit :param model: pretrained model :param config: inference configurations :param params: parameters dictionary :param model_name: parameters dictionary :param val_score: mean IoU score for the validation set :param test_ids: array of test ids :return: """ params['predict_type'] = 'test' # Predict data print('\nStep 1 of 3: Predicting test data... ') predict_images_dir = predict(model=model, config=config, params=params, model_name=model_name, images_ids=test_ids) # Data postprocessing print('\nStep 2 of 3: Test data postprocessing... ') postproc_out_dir = r'out_files/images/postproc' postproc_model_name = predict_images_dir.replace(dir_splitter, "/").split("/")[-1] predict_images_ids = next(os.walk(os.path.join(OUT_FILES, predict_images_dir)))[1] for image_id in tqdm(predict_images_ids, total=len(predict_images_ids)): labels = dpost.read_labels(predict_images_dir, image_id) morfling_labels = dpost.morfling(labels=labels) removed_instances_labels = dpost.remove_small_instances(labels=morfling_labels) overlap_fix_labels = dpost.overlapping_fix(labels=removed_instances_labels) dpost.save_labels(labels=overlap_fix_labels, out_dir=postproc_out_dir, model_name=postproc_model_name, image_id=image_id) print('\nStep 3 of 3: Creating submit file... ') images_to_encode = os.path.join(OUT_FILES, r'{}/{}'.format(postproc_out_dir, postproc_model_name)) submit_path = make_dir('sub/{}-V{}'.format(postproc_model_name, val_score)) config.save_to_file(os.path.join(submit_path)) esfmr.create_submit(files_path=images_to_encode, model_name=postproc_model_name, submit_path=submit_path)
def predict(config, params, model_name, images_ids): """ Predict masks for test images :param config: model configure class instance :param params: parameters dictionary :param model_name: the name of the model :param images_ids: ids of images to predict (test_ids or val_ids) :return: path to the folder with predicted files """ # Create model object in "inference" mode model = modellib.MaskRCNN(mode="inference", config=config, model_dir=MODEL_DIR) # Choose the weights for predicting model_path = "" if params['epoch_for_predict'].isdigit(): model_path = os.path.join(model.find_last()[0], r'mask_rcnn_cell_00{}.h5'.format(params['epoch_for_predict'])) elif params['epoch_for_predict'] == 'last': model_path = model.find_last()[1] elif params['epoch_for_predict'] == 'path': model_path = os.path.join(ROOT_DIR, params['path_to_weights_for_predict']) assert model_path != "", "Provide path to trained weights" print("Loading weights from ", model_path) model.load_weights(model_path, by_name=True) # Choose the type of predicting dataset: # - val (validation) - predict validation dataset for model validation score estimation; # - test - predict test dataset. predict_type = params['predict_type'] pred_files_dir = "" images_dir = "" if predict_type == 'test': # Create folder for predicted files relative_preds_path = r'out_files\images\predict\{}-{}_pep'.format(model_name, params['epoch_for_predict']) pred_files_dir = make_dir(relative_preds_path) images_dir = TEST_DATASET_DIR elif predict_type == 'val': # Create folder for predicted files relative_preds_path = r'out_files\images\predict_val\{}-{}_pep'.format(model_name, params['epoch_for_predict']) pred_files_dir = make_dir(relative_preds_path) images_dir = DATASET_DIR assert pred_files_dir != "", "Provide path to predict files" assert images_dir != "", "Provide path to source files" # Save config file config.save_to_file(os.path.join(pred_files_dir, '{}.csv'.format(model_name))) # Predict images for image_id in tqdm(images_ids, total=len(images_ids)): # Create folder for image image_dir = os.path.join(pred_files_dir, image_id) if not os.path.exists(image_dir): os.mkdir(image_dir) # Read test image test_image = skimage.io.imread(os.path.join(images_dir, r'{}\images\{}.png'.format(image_id, image_id))) # If grayscale. Convert to RGB for consistency. if test_image.ndim != 3: test_image = skimage.color.gray2rgb(test_image) elif test_image.shape[2] > 3: test_image = test_image[:, :, :3] # Predict mask for the given image results = model.detect([test_image], verbose=0) r = results[0] # Save predicted masks for i in range(r['masks'].shape[2]): import warnings warnings.filterwarnings("ignore") r['masks'][:, :, i] *= 255 skimage.io.imsave('{}\{}.png'.format(image_dir, i), r['masks'][:, :, i]) return pred_files_dir
dpost.save_labels(labels=overlap_fix_labels, out_dir=postproc_out_dir, process_type='remove_small_obj', model_name=postproc_model_name, image_id=image_id) # Check IoU score for validation data or encode submit for test data if params['predict_type'] == 'val': print('\n' * 2 + '-' * 30 + ' Check validation score... ' + '-' * 30 + '\n') pred_images_dir = r'out_files/images/postproc_val/remove_small_obj/{}'.format(postproc_model_name) val_ids = next(os.walk(os.path.join(ROOT_DIR, pred_images_dir)))[1] image_ids = val_ids true_images_dir = r'data/images/train' p = 0 for image_id in tqdm(image_ids, total=len(image_ids)): p += metric.calculate_image_iou(image_id=image_id, true_images_dir=true_images_dir, pred_images_dir=pred_images_dir) mean_p = p / len(image_ids) print('\n\nTotal IoU for validation set: {:1.3f}'.format(mean_p)) elif params['predict_type'] == 'test': print('\n' * 2 + '-' * 30 + ' Creating submit file... ' + '-' * 30 + '\n') images_to_encode = os.path.join(ROOT_DIR, r'out_files/images/postproc/remove_small_obj/{}'.format(postproc_model_name)) submit_path = make_dir('sub/{}'.format(postproc_model_name)) config.save_to_file(os.path.join(submit_path, 'config.csv')) esfmr.create_submit(files_path=images_to_encode, model_name=postproc_model_name, submit_path=submit_path)
ids = next(os.walk(os.path.join(files_path, '{}'.format(image_id)))) for i, image_index in enumerate(ids[2]): pred_image = imread(os.path.join(ids[0], image_index)) preds_test_images.append(pred_image) for idx, img_id in enumerate(ids[2]): rle = list(prob_to_rles(preds_test_images[idx])) rles.extend(rle) new_test_ids.extend([image_id] * len(rle)) # Create submission DataFrame sub = pd.DataFrame() sub['ImageId'] = new_test_ids sub['EncodedPixels'] = pd.Series(rles).apply( lambda x: ' '.join(str(y) for y in x)) sub.to_csv(os.path.join(submit_path, r'{}_sub.csv'.format(model_name)), index=False) if __name__ == '__main__': print('-' * 30 + ' Creating submit file... ' + '-' * 30) images_to_encode = os.path.join( ROOT_DIR, r'out_files/images/postproc/remove_small_obj/mrcnn-60_ep-0.2_vs-coco_iw-heads_l-24_pep' ) model_name = images_to_encode.replace(dir_splitter, "/").split("/")[-1] submit_path = make_dir('sub/{}'.format(model_name)) create_submit(files_path=images_to_encode, model_name=model_name, submit_path=submit_path)
os.mkdir(path) np.save(os.path.join(path, r'X_train_normalize.npy'), splited_train_images) # np.save(os.path.join(path, r'Y_train.npy'), splited_train_labels) # np.save(os.path.join(path, r'Y_train_contours.npy'), splited_train_contours) def make_test_npy(test_ids, save_dir): images_to_concatenate = [] for id in tqdm(test_ids, total=len(test_ids)): image = da.read_test_image(id) normalize_image = da.normalization(image) split_image = da.split(normalize_image) images_to_concatenate.append(split_image) splited_test_images = np.concatenate(images_to_concatenate, axis=0) path = os.path.join(save_dir, r'{}_{}_split'.format(da.SPLIT_SIZE, da.SPLIT_SIZE)) if not os.path.exists(path): os.mkdir(path) np.save(os.path.join(path, r'X_test_normalize.npy'), splited_test_images) if __name__ == '__main__': train_ids, test_ids = de.get_id() save_dir = make_dir(r'out_files/npy') make_train_npy(train_ids, save_dir) make_test_npy(test_ids, save_dir)
def transform_action(action_path,out_path): print(out_path) dirs.make_dir(out_path) action=read_action(action_path,False) new_imgs=action.apply(func) [imgs.save_img(out_path,img_i) for img_i in self.frames]
def save(self,out_path): print(str(out_path)) dirs.make_dir(out_path) [imgs.save_img(out_path,img_i) for img_i in self.frames]