def aggregate_and_save(folders_to_agg, output_path):
master_tiles = sorted(
list([p.name for p in folders_to_agg[0].iterdir() if not p.is_dir()]))
if not output_path.exists():
output_path.mkdir(parents=True)
for tile_name in master_tiles:
# Skip merged files generated when `to_submission` job was exectuted on predictions
if "merged" in tile_name:
continue
ref_fp = folders_to_agg[0] / tile_name
assert ref_fp.exists(), f"{ref_fp.as_posix()}"
img = read_image(ref_fp, dtype='uint8')
agg_img = np.zeros((len(folders_to_agg), img.shape[0], img.shape[1]),
dtype='uint8')
agg_img[0, :, :] = img
for i, f in enumerate(folders_to_agg[1:]):
fp = f / tile_name
assert fp.exists(), f"{fp.as_posix()}"
img = read_image(fp, dtype='uint8')
agg_img[i + 1, :, :] = img
agg_img = np.sum(agg_img, axis=0)
agg_img = (agg_img >= (len(folders_to_agg) - 1)).astype('uint8')
with rio.open(ref_fp, 'r') as src:
profile = src.profile
profile.update(dtype=rio.uint8, count=1)
filepath = (output_path / tile_name).as_posix()
with rio.open(filepath, 'w', **profile) as dst:
dst.write(agg_img.astype(rio.uint8), 1)
def read_nimg_sqrt_mask(image, mask, **kwargs):
img = read_image(image.as_posix(), dtype='float32')
mins = img.reshape((-1, img.shape[2])).min(axis=0)[None, None, :]
maxs = img.reshape((-1, img.shape[2])).max(axis=0)[None, None, :]
nimg = (img - mins) / (maxs - mins + 1e-15)
nimg = np.power(nimg, 0.2) - 0.5
kwargs['image'] = nimg
kwargs['mask'] = read_image(mask, dtype='uint8')
return kwargs
def read_img_as_5b_with_mask(image, mask, **kwargs):
"""Method to read image and mask, transform image to 5 channels:
(3 original bands, vv - vh, sqrt(b1^2 + b2^2))
"""
img3b = read_image(image.as_posix(), dtype='float32')
img5b = np.empty((img3b.shape[0], img3b.shape[1], 5), dtype=img3b.dtype)
img5b[:, :, (0, 1, 2)] = img3b
img5b[:, :, 3] = img3b[:, :, 1] - img3b[:, :, 0]
img5b[:, :, 4] = np.linalg.norm(img3b[:, :, (0, 1)], axis=-1)
kwargs['image'] = img5b
kwargs['mask'] = read_image(mask, dtype='uint8')
return kwargs
def worker_task(pred_file, gt_path):
fname = pred_file.name
parent = pred_file.parent.name
gt_file = gt_path / parent / fname
if not gt_file.exists():
gt_file = gt_path / (parent.replace("_3b_", "_vh_")) / fname
assert gt_file.exists(), "File is not found {}".format(gt_file.as_posix())
y_pred = read_image(pred_file, dtype='int')
y_true = read_image(gt_file, dtype='int')
return confusion_matrix(y_true.ravel(), y_pred.ravel(), labels=[0, 1])
def read_img_5b_in_db(image, **kwargs):
img = read_image(image.as_posix(), dtype='float32')
img = linear_to_decibel(img)
img5b = np.empty((img.shape[0], img.shape[1], 5), dtype=img.dtype)
img5b[:, :, (0, 1, 2)] = img
img5b[:, :, 3] = img[:, :, 1] * img[:, :, 0]
img5b[:, :, 4] = (img[:, :, 1] + 1.0) / (img[:, :, 0] + 1.0)
kwargs['image'] = img5b
return kwargs
def aggregate_and_save(folders_to_agg, filepath):
path = folders_to_agg[0]
ref_image_fp = path / "merged.tif"
img = read_image(ref_image_fp, dtype='uint8')
agg_img = np.zeros((len(folders_to_agg), img.shape[0], img.shape[1]),
dtype='uint8')
agg_img[0, :, :] = img
for i, path in enumerate(folders_to_agg[1:]):
mask_fp = path / "merged.tif"
img = read_image(mask_fp, dtype='uint8')
agg_img[i + 1, :, :] = img
agg_img = np.sum(agg_img, axis=0)
agg_img = (agg_img >= (len(folders_to_agg) - 1)).astype('uint8')
with rio.open(ref_image_fp, 'r') as src:
profile = src.profile
profile.update(dtype=rio.uint8, count=1)
with rio.open(filepath, 'w', **profile) as dst:
dst.write(agg_img.astype(rio.uint8), 1)
def run(input_path, output_path):
train_dataset = UnoSatTiles(input_path)
data = [None] * len(train_dataset)
for i, dp in tqdm.tqdm(enumerate(train_dataset), total=len(train_dataset)):
img = read_image(dp['image'], dtype='float32')
mask = read_image(dp['mask'], dtype='uint8')
img_ravel = img.reshape((-1, img.shape[-1]))
img_min = tuple(img_ravel.min(axis=0))
img_max = tuple(img_ravel.max(axis=0))
img_mean = tuple(img_ravel.mean(axis=0))
img_std = tuple(img_ravel.std(axis=0))
target_ratio = (mask > 0).sum() / (mask.shape[0] * mask.shape[1])
fold_index = [f in dp['image'].as_posix() for f in folds].index(True)
skip = sum(img_min) == sum(img_max)
if img_min[0] < -10000:
skip = True
data[i] = [
i, img_min, img_max, img_mean, img_std, target_ratio, fold_index,
skip, dp['image']
]
df = pd.DataFrame(data,
columns=[
"index", "img_min", "img_max", "img_mean", "img_std",
"target_ratio", "fold_index", "skip", "img_path"
])
output_fp = output_path / "tile_stats.csv"
df.to_csv(output_fp, index=False)
def read_img_5b_in_db_with_mask(image, mask, **kwargs):
kwargs = read_img_5b_in_db(image, **kwargs)
kwargs['mask'] = read_image(mask, dtype='uint8')
return kwargs
def read_img_in_db(image, **kwargs):
img = read_image(image.as_posix(), dtype='float32')
img = linear_to_decibel(img)
kwargs['image'] = img
return kwargs
def read_img_mask(image, mask, **kwargs):
kwargs['image'] = read_image(image.as_posix(), dtype='float32')
kwargs['mask'] = read_image(mask, dtype='uint8')
return kwargs
def read_img_only(image, **kwargs):
kwargs['image'] = read_image(image.as_posix(), dtype='float32')
return kwargs
def read_img_2log_with_mask(image, mask, **kwargs):
img = read_image(image.as_posix(), dtype='float32')
img[img == 0] = np.exp(-5.0)
kwargs['image'] = np.log(img ** 2)
kwargs['mask'] = read_image(mask, dtype='uint8')
return kwargs