def save_result(complnet_path,dataset_path): #------------------------------------------------------------- compl_model = load_compl_model(complnet_path, (None,None,1)) global cnet_path, index cnet_path = complnet_path index = 0 #------------------------------------------------------------- paths = list(utils.file_paths(dataset_path)) mask_paths = list(filter(lambda s: 'mask' in s, paths)) #answer_paths = list(filter(lambda s: 'clean' in s, paths)) answer_paths = list(filter(lambda s: not('mask' in s), paths)) #print(len(answer_paths),answer_paths,flush=True) test_infos, similarities, errors, masked_ssims, full_ssims = [],[],[],[],[] for path_tup in path_tuples(answer_paths, mask_paths): similarity, error, masked_ssim, full_ssim\ = scores(compl_model, *path_tup2img_tup(*path_tup)) test_infos.append(path_tup) similarities.append(np.asscalar(similarity)) errors.append(np.asscalar(error)) masked_ssims.append(np.asscalar(masked_ssim)) full_ssims.append(np.asscalar(full_ssim)) #------------------------------------------------------------- result = {'name' : (complnet_path.replace(os.sep,'_') + '+' + dataset_path.replace(os.sep,'_')), 'cnet_path' : complnet_path, 'dataset_path' : dataset_path, 'mse ratio similarity mean' : sum(similarities) / len(similarities), 'mse ratio error mean' : sum(errors) / len(similarities), 'masked ssim mean' : sum(masked_ssims) / len(similarities), 'full ssim mean' : sum(full_ssims) / len(similarities), 'origin,answer,mask paths' : test_infos, 'similarities' : similarities, 'errors' : errors, 'masked_ssims' : masked_ssims, 'full_ssims' : full_ssims} with open(result['name']+'.yml','w') as f: f.write(yaml.dump(result)) print('{}'.format(result['name']),end='|') print('mse ratio similarity mean = {:f} ({:f}%)'\ .format(result['mse ratio similarity mean'], result['mse ratio similarity mean']*100),end='|') print('mse ratio error mean = {:f} ({:f}%)'\ .format(result['mse ratio error mean'], result['mse ratio error mean']*100),end='|') print('masked ssim mean = {:f}'.format(result['masked ssim mean']),end='|') print('full ssim mean = {:f}'.format(result['full ssim mean']),flush=True)
make list of pair of image:mask shuffle the list and then move image, masks into train/valid/test NOTE: the names of the masks are changed to the same as the image. [synopsys] python crops_dir2dataset_dir.py crops_dir label_str ex) python crops_dir2dataset_dir.py 35crops _mask_ ''') crops_dir = sys.argv[1] crops_dir = pathlib.Path(crops_dir).parts[0] label_str = sys.argv[2] all_paths = list(utils.file_paths(crops_dir)) img_paths = sorted(filter(lambda p: label_str not in p, all_paths)) mask_paths = sorted(filter(lambda p: label_str in p, all_paths)) img_mask_pairs = list(zip(img_paths, mask_paths)) random.shuffle(img_mask_pairs) # Make directory structure train_img_dir = pjoin(pjoin(crops_dir, 'train'), 'image') train_label_dir = pjoin(pjoin(crops_dir, 'train'), 'label') valid_img_dir = pjoin(pjoin(crops_dir, 'valid'), 'image') valid_label_dir = pjoin(pjoin(crops_dir, 'valid'), 'label') test_img_dir = pjoin(pjoin(crops_dir, 'test'), 'image') test_label_dir = pjoin(pjoin(crops_dir, 'test'), 'label')
r = bgr_img[:, :, 2] #calculate and leave difference set of lower priority color #(remove intersection) r_minus_b = (r != b).astype(np.uint8) * r bgr_img[:, :, 2] = r_minus_b r_minus_g = (r != g).astype(np.uint8) * r bgr_img[:, :, 2] = r_minus_g g_minus_b = (g != b).astype(np.uint8) * g bgr_img[:, :, 1] = g_minus_b #cv2.imshow('intersection removed',bgr_img); cv2.waitKey(0) return bgr_img if __name__ == '__main__': mask_dir = sys.argv[1] mask_paths = list(utils.file_paths(mask_dir)) check_img = cv2.imread(mask_paths[0]) if (np.array_equal(check_img[:, :, 0], check_img[:, :, 1]) and np.array_equal(check_img[:, :, 1], check_img[:, :, 2])): img_type = cv2.IMREAD_GRAYSCALE else: img_type = cv2.IMREAD_COLOR leaving_channels = sys.argv[2] if len(sys.argv) == 3 else 'rgb' masks \ = pipe(cmap(lambda path: cv2.imread(path, img_type)), cfilter(lambda img: img is not None), cmap(binarization), cmap(dilation),
history.history['val_loss'])), val_acc=list( map(np.asscalar, history.history['val_mean_iou'])), test_loss=np.asscalar(test_metrics[0]), test_acc=np.asscalar(test_metrics[1]), train_time=train_time_str, ))) modulo = 2**num_maxpool evaluator.eval_and_save_result( dataset_dir, save_model_path, eval_result_dirpath, files_2b_copied=[history_path, experiment_yml_path], num_filters=num_filters, num_maxpool=num_maxpool, modulo=modulo) #-------------------------------------------------------------------- if __name__ == '__main__': with open('experiment_log', 'w') as log: for experiment_path in human_sorted(file_paths(sys.argv[1])): try: timer = ElapsedTimer(experiment_path) main(experiment_path) log.write(timer.elapsed_time()) except AssertionError as error: print(str(error)) log.write(str(error))
def main(src_imgs_path, dataset_name, num_crop, crop_size, chk_size): if num_crop != 0: rand_sqr_crop = img2rand_sqr_crops(crop_size) gen \ = pipe(utils.file_paths, cmap(lambda path: cv2.imread(path)), cfilter(lambda img: img is not None), cfilter(lambda img: is_cuttable(img, crop_size)), cmap(utils.slice1channel), cflatMap(crepeat(num_crop)), cmap(lambda img: rand_sqr_crop(img)), cmap(lambda img: (img / 255).astype(np.float32)), lambda imgs: split_every(chk_size, imgs)) else: print('!') num_crop = 100 # big enough value.. gen \ = pipe(utils.file_paths, cmap(lambda path: cv2.imread(path)), cfilter(lambda img: img is not None), cfilter(lambda img: is_cuttable(img, crop_size)), cmap(utils.slice1channel), cflatMap(lambda img: img2sqr_crops(img, crop_size)), cmap(lambda img: (img / 255).astype(np.float32)), lambda imgs: split_every(chk_size, imgs)) print(src_imgs_path) expected_num_imgs = len(list(utils.file_paths(src_imgs_path))) * num_crop print('-------------- SUMARY --------------') print(' dataset name = ', dataset_name) print(' size of crop = ', crop_size) print(' num crops per img = ', num_crop) print(' expected num imgs = ', expected_num_imgs) print(' chunk size = ', chk_size) f = h5py.File(dataset_name, 'w') timer = utils.ElapsedTimer() #------------------------------------------------------------- f.create_dataset('images', (expected_num_imgs, crop_size, crop_size, 1), maxshape=(None, crop_size, crop_size, 1), chunks=(chk_size, crop_size, crop_size, 1)) mean = 0 num_img_elems = (crop_size**2) for chk_no, chunk in tqdm(enumerate(gen(src_imgs_path)), total=expected_num_imgs // chk_size): beg_idx = chk_no * chk_size f['images'][beg_idx:beg_idx + len(chunk)] = chunk mean = iter_mean(mean, beg_idx * num_img_elems, np.sum(chunk), len(chunk) * num_img_elems) f.create_dataset('mean_pixel_value', data=mean) last_chunk_size = len(chunk) actual_num_img = chk_no * chk_size + last_chunk_size if actual_num_img != expected_num_imgs: print(expected_num_imgs, ' != ', actual_num_img) print('dataset resized!') f['images'].resize((actual_num_img, crop_size, crop_size, 1)) # [mean test code] #li = list(flatten(gen(src_imgs_path))) #real_mean = np.mean(li) #print('real MEAN:', real_mean) #print(len(li)) #print('saved MEAN:', f['mean_pixel_value'][()]) #------------------------------------------------------------- f.close() print('------------------------------------') print('dataset generated successfully.') msg = timer.elapsed_time() ''' import mailing mailing.send_mail_to_kur( 'Dataset generated successfully.',msg ) ''' # [load test code] f = h5py.File(dataset_name, 'r') #------------------------------------------------------------- print('f', f['images'].shape) print('loaded MEAN:', f['mean_pixel_value'][()]) #for i in range(f['images'].shape[0] ): #cv2.imshow('img',f['images'][i]);cv2.waitKey(0) cv2.imshow('img', f['images'][-1]) cv2.waitKey(0) #------------------------------------------------------------- f.close() ''' ''' '''
parser.add_argument('-n', '--num_crop', type=int, help='number of crops per 1 original image.') parser.add_argument('-c', '--chk_size', type=int, help='size of chunk of data for performance.') if __name__ == '__main__': #unittest.main() args = parser.parse_args() src_imgs_path = args.src_imgs_path#'H:\\DATA2\\f' dataset_name = args.dataset_name#'gray128.h5' num_crop = args.num_crop# 3 crop_size = args.crop_size#128 chk_size = args.chk_size#100 #00 print(src_imgs_path) expected_num_imgs = len(list(utils.file_paths(src_imgs_path))) * num_crop print('-------------- SUMARY --------------') print(' dataset name = ', dataset_name) print(' size of crop = ', crop_size) print(' num crops per img = ', num_crop) print(' expected num imgs = ', expected_num_imgs) print(' chunk size = ', chk_size) img2_128x128crop = img2sqr_crop(crop_size) gen = pipe(utils.file_paths, cmap(lambda path: cv2.imread(path)), cfilter(lambda img: img is not None), cfilter(lambda img: is_cuttable(img, crop_size)), cmap(slice1channel), cflatMap(crepeat(num_crop)), cmap(lambda img: img2_128x128crop(img)),
def main(complnet_dir,dataset_dir): complnet_paths = utils.file_paths(complnet_dir) #complnet_paths = list(human_sorted(complnet_paths)) complnet_paths = list(reversed(human_sorted(complnet_paths))) for complnet_path in tqdm(complnet_paths): save_result(complnet_path, dataset_dir)