def test_L8CCA__len__(
self,
batch_size,
expected
):
data_gen = DG_L8CCA(
img_paths=IMG_PATHS_L8CCA,
batch_size=batch_size,
shuffle=False
)
assert data_gen.__len__() == expected
def get_img_pred(path: Path,
model: keras.Model,
batch_size: int,
bands: Tuple[int] = (4, 3, 2, 5),
bands_names: Tuple[str] = ("red", "green", "blue", "nir"),
resize: bool = False,
normalize=True,
standardize=False,
patch_size: int = 384) -> Tuple[np.ndarray, float]:
"""
Generates prediction for a given image.
:param path: path containing directories with image channels.
:param model: trained model to make predictions.
:param batch_size: size of generated batches, only one batch is loaded
to memory at a time.
:param bands: band numbers to load
:param bands_names: names of the bands to load. Should have the same number
of elements as bands.
:param resize: whether to resize loaded img to gt.
:param normalize: whether to normalize the image.
:param standardize: whether to standardize the image.
:param patch_size: size of the image patches.
:return: prediction for a given image along with evaluation time.
"""
testgen = DG_L8CCA(img_paths=[path],
batch_size=batch_size,
bands=bands,
bands_names=bands_names,
resize=resize,
normalize=normalize,
standardize=standardize,
shuffle=False)
tbeg = time.time()
preds = model.predict_generator(testgen)
scene_time = time.time() - tbeg
print(f"Scene prediction took { scene_time } seconds")
img_height, img_width, _ = testgen.img_shapes[0]
preds = rearrange(
preds,
"(r c) dr dc b -> r c dr dc b",
r=int(img_height / patch_size),
c=int(img_width / patch_size),
)
img = np.full((img_height, img_width, 1), np.inf)
for row in range(preds.shape[0]):
for column in range(preds.shape[1]):
img[row * patch_size:(row + 1) * patch_size, column *
patch_size:(column + 1) * patch_size, ] = preds[row, column]
return img, scene_time
def test__generate_gt_patches(
self,
patch_size,
expected
):
data_gen = DG_L8CCA(
img_paths=IMG_PATHS_L8CCA,
batch_size=1,
with_gt=True,
patch_size=patch_size,
shuffle=False
)
data_gen._generate_gt_patches()
assert data_gen.gt_patches.shape == expected
def test_L8CCA__getitem__(
self,
batch_size,
x_expected,
y_expected
):
data_gen = DG_L8CCA(
img_paths=IMG_PATHS_L8CCA,
batch_size=batch_size,
with_gt=True,
shuffle=False
)
x, y = data_gen.__getitem__(0)
assert x.shape == x_expected
assert y.shape == y_expected
def test__partition_data(
self,
data_part,
expected
):
data_gen = DG_L8CCA(
img_paths=IMG_PATHS_L8CCA,
batch_size=1,
data_part=data_part,
shuffle=False
)
old_patches_num = len(data_gen._patches_indexes)
data_gen._partition_data()
new_patches_num = len(data_gen._patches_indexes)
assert abs(new_patches_num - int(expected * old_patches_num)) <= 2
def test__generate_img_patches(
self,
patch_size,
bands,
bands_names,
expected
):
data_gen = DG_L8CCA(
img_paths=IMG_PATHS_L8CCA,
batch_size=1,
patch_size=patch_size,
bands=bands,
bands_names=bands_names,
shuffle=False
)
data_gen._generate_img_patches(bands, bands_names)
assert data_gen.patches.shape == expected
def test_datagen_to_gt_array(self):
data_38Cloud = DG_38Cloud(
files=FILES_38CLOUD,
batch_size=1,
shuffle=False
)
gt_38Cloud = datagen_to_gt_array(data_38Cloud)
assert gt_38Cloud.shape == (3, 384, 384, 1)
np.testing.assert_array_equal(np.unique(gt_38Cloud), [0, 1])
data_L8CCA = DG_L8CCA(
img_paths=IMG_PATHS_L8CCA,
batch_size=1,
with_gt=True,
patch_size=100,
bands=(4, 3),
bands_names=("red", "green"),
shuffle=False
)
gt_L8CCA = datagen_to_gt_array(data_L8CCA)
assert gt_L8CCA.shape == (5928, 100, 100, 1)
np.testing.assert_array_equal(np.unique(gt_L8CCA), [0, 1])
def main(
run_name: str,
train_ids: List[Tuple[str]],
test_ids: List[Tuple[str]],
dpath: str,
rpath: str,
vids: Tuple[str],
mlflow: bool,
train_size: float,
snow_imgs: List[str],
batch_size: int,
thr: float,
learning_rate: float,
bn_momentum: float,
epochs: int,
stopping_patience: int,
bands: Tuple[int] = (8, ),
bands_names: Tuple[str] = ("panchromatic", ),
):
"""
Train and test the U-Net model using L8CCA panchromatic images.
:param run_name: name of the run.
:param train_ids: IDs of the training images.
:param test_ids: IDs of the testing images.
:param dpath: path to dataset.
:param rpath: path to directory where results and artifacts should be
logged (randomly named directory will be created to store the
results).
:param vids: tuple of ids of images which should be used to create
visualisations. If contains '*' visualisations will be
created for all images in the datasets.
:param mlflow: whether to use mlflow
:param train_size: proportion of the training set
(the rest goes to validation set).
:param snow_imgs: list of snow images IDs for testing.
:param batch_size: size of generated batches, only one batch is loaded
to memory at a time.
:param thr: threshold for determining whether pixels contain the clouds
(if None, threshold will be determined automatically).
:param learning_rate: learning rate for training.
:param bn_momentum: momentum of the batch normalization layer.
:param epochs: number of epochs.
:param stopping_patience: patience param for early stopping.
:param bands: band numbers to load
:param bands_names: names of the bands to load. Should have the same number
of elements as bands.
"""
dpath, rpath = make_paths(dpath, rpath)
rpath = rpath / uuid.uuid4().hex
rpath.mkdir(parents=True, exist_ok=False)
print(f"Working dir: {os.getcwd()}, artifacts dir: {rpath}", flush=True)
if mlflow:
setup_mlflow(run_name)
params = dict(locals())
del params["train_ids"]
del params["test_ids"]
log_params(params)
set_tags({"train_ids": train_ids, "test_ids": test_ids})
train_paths = [dpath / id_[0] / id_[1] for id_ in train_ids]
traingen = DG_L8CCA(img_paths=train_paths,
batch_size=batch_size,
data_part=(0., train_size),
with_gt=True,
bands=bands,
bands_names=bands_names,
resize=True,
normalize=False,
standardize=True,
shuffle=True)
valgen = DG_L8CCA(img_paths=train_paths,
batch_size=batch_size,
data_part=(train_size, 1.),
with_gt=True,
bands=bands,
bands_names=bands_names,
resize=True,
normalize=False,
standardize=True,
shuffle=True)
model, auto_thr = train_model(
traingen=traingen,
valgen=valgen,
rpath=rpath,
bn_momentum=bn_momentum,
learning_rate=learning_rate,
stopping_patience=stopping_patience,
epochs=epochs,
mlflow=mlflow,
)
print("Finished training and validation, starting evaluation.", flush=True)
thr = auto_thr if thr is None else thr
evaluate_model(dataset_name="L8CCA",
model=model,
thr=thr,
dpath=dpath,
rpath=rpath / "vis",
vids=vids,
batch_size=batch_size,
img_ids=[id_[1] for id_ in test_ids],
snow_imgs=snow_imgs,
mlflow=mlflow,
bands=bands,
bands_names=bands_names,
resize=True,
normalize=False,
standardize=True)
if mlflow:
log_param("threshold", thr)
log_artifacts(rpath)
end_run()