def main():
parser = argparse.ArgumentParser(description='Mask L4B raw maps')
parser.add_argument('l4b_path', metavar='l4b-path', help='The L4B path')
parser.add_argument('l4a_path', metavar='l4a-path', help='The L4A path')
args = parser.parse_args()
tile_id_re = re.compile(r"/crop_type_map_([0-9a-zA-Z]+)\.tif$")
steps = []
for tile_crop_map in glob.glob(
os.path.join(args.l4b_path, "crop_type_map_*.tif")):
m = tile_id_re.search(tile_crop_map, re.IGNORECASE)
if not m:
continue
tile_id = m.group(1)
tile_mask = os.path.join(args.l4a_path,
"crop-mask-segmented-{}.tif".format(tile_id))
if not os.path.exists(tile_mask):
tile_mask = os.path.join(args.l4a_path,
"crop_mask_map_{}.tif".format(tile_id))
if not os.path.exists(tile_mask):
tile_mask = None
if tile_mask is None:
print("No crop mask found for tile {}".format(tile_id))
continue
print("Using crop mask {} for tile {}".format(tile_mask, tile_id))
tile_crop_map_masked = os.path.join(
args.l4b_path, "crop_type_map_masked_{}.tif".format(tile_id))
step_args = [
"otbcli_BandMath", "-exp", "im2b1 == 0 ? 0 : im1b1", "-il",
tile_crop_map, tile_mask, "-out",
format_otb_filename(tile_crop_map_masked,
compression='DEFLATE'), "int16"
]
steps.append([
Step("Mask by crop mask " + tile_id, step_args),
Step("Nodata_" + tile_id,
["gdal_edit.py", "-a_nodata", -10000, tile_crop_map_masked])
])
pool = multiprocessing.dummy.Pool(multiprocessing.cpu_count())
pool.map(run_step_list, steps)
pool.close()
pool.join()
def postprocess_tile(self, tile):
if tile.crop_mask is not None:
tile_crop_map = self.get_output_path("crop_type_map_{}.tif",
tile.id)
tile_crop_map_masked = self.get_output_path(
"crop_type_map_masked_{}.tif", tile.id)
step_args = [
"otbcli_BandMath", "-progress", "false", "-exp",
"im2b1 == 0 ? 0 : im1b1", "-il", tile_crop_map, tile.crop_mask,
"-out",
format_otb_filename(tile_crop_map_masked,
compression='DEFLATE'), "int16"
]
run_step(Step("Mask by crop mask " + tile.id, step_args))
run_step(
Step("Nodata_" + tile.id, [
"gdal_edit.py", "-a_nodata", -10000, tile_crop_map_masked
]))
def postprocess_tile(self, tile):
if self.args.skip_segmentation:
return
tile_crop_mask = self.get_tile_classification_output(tile)
if not os.path.exists(tile_crop_mask):
print(
"Skipping post-processing for tile {} due to missing raw mask".
format(tile.id))
return
tile_ndvi = self.get_output_path("ndvi-{}.tif", tile.id)
tile_pca = self.get_output_path("pca-{}.tif", tile.id)
tile_smoothed = self.get_output_path("smoothed-{}.tif", tile.id)
tile_smoothed_spatial = self.get_output_path("smoothed-spatial-{}.tif",
tile.id)
tile_segmentation = self.get_output_path("segmentation-{}.tif",
tile.id)
tile_segmentation_merged = self.get_output_path(
"segmentation-merged-{}.tif", tile.id)
tile_segmented = self.get_output_path("crop-mask-segmented-{}.tif",
tile.id)
if not self.args.reuse_segmentation:
needs_segmentation_merged = True
needs_segmentation = True
needs_tile_smoothed = True
needs_tile_smoothed_spatial = True
needs_pca = True
needs_ndvi = True
else:
if os.path.exists(tile_segmented):
return
needs_segmentation_merged = not os.path.exists(
tile_segmentation_merged)
needs_segmentation = needs_segmentation_merged and not os.path.exists(
tile_segmentation)
needs_tile_smoothed = needs_segmentation and not os.path.exists(
tile_smoothed)
needs_tile_smoothed_spatial = needs_segmentation and not os.path.exists(
tile_smoothed_spatial)
needs_pca = (needs_tile_smoothed or needs_tile_smoothed_spatial
) and not os.path.exists(tile_pca)
needs_ndvi = needs_pca and not os.path.exists(tile_ndvi)
if self.args.main_mission_segmentation:
tile_descriptors = tile.get_mission_descriptor_paths(
self.args.mission)
else:
tile_descriptors = tile.get_descriptor_paths()
step_args = [
"otbcli", "NDVISeries", self.args.buildfolder, "-progress",
"false", "-mission", self.args.mission.name, "-pixsize",
self.args.pixsize, "-mode", "gapfill", "-out", tile_ndvi
]
step_args += ["-il"] + tile_descriptors
step_args += ["-sp"] + self.args.sp
# if self.args.main_mission_segmentation:
# step_args += ["-mode", "gapfillmain"]
# else:
# step_args += ["-mode", "gapfill"]
if not needs_ndvi:
print("Skipping NDVI extraction for tile {}".format(tile.id))
else:
run_step(Step("NDVI Series " + tile.id, step_args))
step_args = [
"otbcli", "PrincipalComponentAnalysis", self.args.buildfolder,
"-progress", "false", "-nbcomp", self.args.nbcomp, "-bv", -10000,
"-in", tile_ndvi, "-out", tile_pca
]
if not needs_pca:
print("Skipping PCA for tile {}".format(tile.id))
else:
run_step(Step("NDVI PCA " + tile.id, step_args))
if not self.args.keepfiles:
os.remove(tile_ndvi)
step_args = [
"otbcli_MeanShiftSmoothing", "-progress", "false", "-in", tile_pca,
"-modesearch", 0, "-spatialr", self.args.spatialr, "-ranger",
self.args.ranger, "-maxiter", 20, "-fout", tile_smoothed,
"-foutpos", tile_smoothed_spatial
]
if not needs_tile_smoothed and not needs_tile_smoothed_spatial:
print("Skipping mean-shift smoothing for tile {}".format(tile.id))
else:
run_step(Step("Mean-Shift Smoothing " + tile.id, step_args))
if not self.args.keepfiles:
os.remove(tile_pca)
step_args = [
"otbcli_LSMSSegmentation", "-progress", "false", "-in",
tile_smoothed, "-inpos", tile_smoothed_spatial, "-spatialr",
self.args.spatialr, "-ranger", self.args.ranger, "-minsize", 0,
"-tilesizex", 1024, "-tilesizey", 1024, "-tmpdir",
self.args.tmpfolder, "-out",
format_otb_filename(tile_segmentation,
compression='DEFLATE'), "uint32"
]
if not needs_segmentation:
print("Skipping segmentation for tile {}".format(tile.id))
else:
run_step(Step("Segmentation " + tile.id, step_args))
step_args = [
"otbcli_LSMSSmallRegionsMerging", "-progress", "false", "-in",
tile_smoothed, "-inseg", tile_segmentation, "-minsize",
self.args.minsize, "-tilesizex", 1024, "-tilesizey", 1024, "-out",
format_otb_filename(tile_segmentation_merged,
compression='DEFLATE'), "uint32"
]
if not needs_segmentation_merged:
print("Skipping small regions merging for tile {}".format(tile.id))
else:
run_step(Step("Small regions merging " + tile.id, step_args))
if not self.args.keepfiles:
os.remove(tile_smoothed)
os.remove(tile_smoothed_spatial)
os.remove(tile_segmentation)
step_args = [
"otbcli", "MajorityVoting", self.args.buildfolder, "-progress",
"false", "-nodatasegvalue", 0, "-nodataclassifvalue", -10000,
"-minarea", self.args.minarea, "-inclass", tile_crop_mask,
"-inseg", tile_segmentation_merged, "-rout",
format_otb_filename(tile_segmented, compression='DEFLATE'), "int16"
]
run_step(Step("Majority voting " + tile.id, step_args))
if not self.args.keepfiles and not self.args.reuse_segmentation:
os.remove(tile_segmentation_merged)
def classify_tile(self, tile):
models = []
model_ids = []
days = []
statistics = []
for stratum in tile.strata:
area_model = self.get_output_path("model-{}.txt", stratum.id)
area_days = self.get_output_path("days-{}.txt", stratum.id)
area_statistics = self.get_output_path("statistics-{}.xml",
stratum.id)
models.append(area_model)
model_ids.append(stratum.id)
days.append(area_days)
statistics.append(area_statistics)
if len(models) == 0:
print("Skipping classification for tile {} due to stratum filter".
format(tile.id))
return
if not self.single_stratum:
tile_model_mask = self.get_output_path("model-mask-{}.tif",
tile.id)
run_step(
Step("Rasterize model mask", [
"otbcli_Rasterization", "-progress", "false", "-mode",
"attribute", "-mode.attribute.field", "ID", "-in",
self.args.filtered_strata, "-im", tile.reference_raster,
"-out",
format_otb_filename(tile_model_mask,
compression='DEFLATE'), "uint8"
]))
tile_crop_mask_uncompressed = self.get_output_path(
"crop_mask_map_{}_uncompressed.tif", tile.id)
if self.args.refp is not None:
step_args = [
"otbcli", "CropMaskImageClassifier", self.args.buildfolder,
"-progress", "false", "-mission", self.args.mission.name,
"-pixsize", self.args.pixsize, "-bv", -10000, "-nodatalabel",
-10000, "-bm", "true" if self.args.bm else "false", "-out",
tile_crop_mask_uncompressed, "-indays"
] + days
if self.args.red_edge:
step_args += ["-rededge", "true"]
step_args += ["-model"] + models
step_args += ["-il"] + tile.get_descriptor_paths()
if self.args.classifier == "svm":
step_args += ["-imstat"] + statistics
if not self.single_stratum:
step_args += ["-mask", tile_model_mask]
step_args += ["-modelid"] + model_ids
else:
tile_spectral_features = self.get_output_path(
"spectral-features-{}.tif", tile.id)
step_args = [
"otbcli", "MultiModelImageClassifier", self.args.buildfolder,
"-progress", "false", "-in", tile_spectral_features, "-out",
tile_crop_mask_uncompressed
]
step_args += ["-model"] + models
if not self.single_stratum:
step_args += ["-mask", tile_model_mask]
step_args += ["-modelid"] + model_ids
run_step(
Step("ImageClassifier_{}".format(tile.id), step_args, retry=True))
if not self.args.keepfiles:
if not self.single_stratum:
os.remove(tile_model_mask)
if self.args.refp is None:
tile_spectral_features = self.get_output_path(
"spectral-features-{}.tif", tile.id)
os.remove(tile_spectral_features)
tile_crop_mask_map = self.get_tile_classification_output(tile)
step_args = [
"otbcli_Convert", "-progress", "false", "-in",
tile_crop_mask_uncompressed, "-out",
format_otb_filename(tile_crop_mask_map,
compression='DEFLATE'), "int16"
]
run_step(Step("Compression_{}".format(tile.id), step_args))
if not self.args.keepfiles:
os.remove(tile_crop_mask_uncompressed)
def train_stratum(self, stratum):
area_model = self.get_output_path("model-{}.txt", stratum.id)
area_confmatout = self.get_output_path(
"confusion-matrix-training-{}.csv", stratum.id)
if self.args.refp is not None:
features_shapefile = self.get_output_path("features-{}.shp",
stratum.id)
split_features(stratum, self.args.refp, self.args.outdir)
area_training_polygons = self.get_output_path(
"training_polygons-{}.shp", stratum.id)
area_validation_polygons = self.get_output_path(
"validation_polygons-{}.shp", stratum.id)
area_statistics = self.get_output_path("statistics-{}.xml",
stratum.id)
area_days = self.get_output_path("days-{}.txt", stratum.id)
area_descriptors = []
area_prodpertile = []
for tile in stratum.tiles:
area_descriptors += tile.get_descriptor_paths()
area_prodpertile.append(len(tile.descriptors))
run_step(
Step("SampleSelection", [
"otbcli", "SampleSelection", self.args.buildfolder, "-ref",
features_shapefile, "-ratio", self.args.ratio, "-seed",
self.args.rseed, "-nofilter", "true", "-tp",
area_training_polygons, "-vp", area_validation_polygons
]))
step_args = [
"otbcli", "CropMaskTrainImagesClassifier",
self.args.buildfolder, "-progress", "false", "-mission",
self.args.mission.name, "-nodatalabel", -10000, "-pixsize",
self.args.pixsize, "-outdays", area_days, "-mode",
self.args.trm, "-io.vd", area_training_polygons, "-rand",
self.args.rseed, "-sample.bm", 0, "-io.confmatout",
area_confmatout, "-io.out", area_model, "-sample.mt",
self.args.nbtrsample, "-sample.mv", 10, "-sample.vfn", "CROP",
"-sample.vtr", 0.01, "-window", self.args.window, "-bm",
"true" if self.args.bm else "false", "-classifier",
self.args.classifier
]
if self.args.red_edge:
step_args += ["-rededge", "true"]
step_args += ["-sp"] + self.args.sp
step_args += ["-prodpertile"] + area_prodpertile
step_args += ["-il"] + area_descriptors
if self.args.classifier == "rf":
step_args += [
"-classifier.rf.nbtrees", self.args.rfnbtrees,
"-classifier.rf.min", self.args.rfmin,
"-classifier.rf.max", self.args.rfmax
]
else:
step_args += [
"-classifier.svm.k", "rbf", "-classifier.svm.opt", 1,
"-imstat", area_statistics
]
run_step(Step("TrainImagesClassifier", step_args))
else:
for tile in stratum.tiles:
tile_reference_trimmed = self.get_output_path(
"reference-trimmed-{}.tif", tile.id)
tile_stratum_reference_trimmed = self.get_output_path(
"reference-trimmed-{}-{}.tif", stratum.id, tile.id)
self.rasterize_tile_mask(stratum, tile)
stratum_tile_mask = self.get_stratum_tile_mask(stratum, tile)
step_args = [
"otbcli_BandMath", "-progress", "false", "-exp",
"im1b1 > 0 ? im2b1 : -10000", "-il", stratum_tile_mask,
tile_reference_trimmed, "-out",
format_otb_filename(tile_stratum_reference_trimmed,
compression='DEFLATE'), "int16"
]
run_step(Step("BandMath_" + str(tile.id), step_args))
area_model = self.get_output_path("model-{}.txt", stratum.id)
area_confmatout = self.get_output_path(
"confusion-matrix-training-{}.csv", stratum.id)
if self.args.classifier == "svm":
files = []
for tile in stratum.tiles:
tile_spectral_features = self.get_output_path(
"spectral-features-{}.tif", tile.id)
tile_stratum_spectral_features = self.get_output_path(
"spectral-features-{}-{}.tif", stratum.id, tile.id)
stratum_tile_mask = self.get_stratum_tile_mask(
stratum, tile)
step_args = [
"otbcli_BandMath", "-progress", "false", "-exp",
"im1b1 > 0 ? im2b1 : -10000", "-il", stratum_tile_mask,
tile_spectral_features, "-out",
format_otb_filename(tile_stratum_spectral_features,
compression='DEFLATE'), "int16"
]
run_step(Step("BandMath_" + str(tile.id), step_args))
files.append(tile_stratum_spectral_features)
step_args = [
"otbcli_ComputeImagesStatistics", "-bv", -10000, "-out",
area_statistics, "-il"
] + files
step_args = [
"otbcli", "TrainImagesClassifierNew", self.args.buildfolder,
"-nodatalabel", -10000, "-rand", self.args.rseed, "-sample.bm",
0, "-io.confmatout", area_confmatout, "-io.out", area_model,
"-sample.mt", self.args.nbtrsample, "-sample.mv", 1000,
"-sample.vfn", "CROP", "-sample.vtr", 0.01, "-classifier",
self.args.classifier
]
if self.args.classifier == "rf":
step_args += [
"-classifier.rf.nbtrees", self.args.rfnbtrees,
"-classifier.rf.min", self.args.rfmin,
"-classifier.rf.max", self.args.rfmax
]
else:
step_args += [
"-classifier.svm.k", "rbf", "-classifier.svm.opt", 1,
"-imstat", area_statistics
]
step_args.append("-io.rs")
for tile in stratum.tiles:
tile_stratum_reference_trimmed = self.get_output_path(
"reference-trimmed-{}-{}.tif", stratum.id, tile.id)
step_args.append(tile_stratum_reference_trimmed)
step_args.append("-io.il")
for tile in stratum.tiles:
tile_spectral_features = self.get_output_path(
"spectral-features-{}.tif", tile.id)
step_args.append(tile_spectral_features)
run_step(Step("TrainImagesClassifier", step_args))
