def clip_with_aoi(src):
dst = os.path.join(MVI_RESULTS_DIR, os.path.basename(src))
logger.info("Clip %s with AOI to %s", src, dst)
os.makedirs(MVI_RESULTS_DIR, exist_ok=True)
if not os.path.exists(dst):
run_command(
"{gdal_bin_path}/gdalwarp -co COMPRESS=DEFLATE -co TILED=YES -of GTiff -cutline {aoi} -crop_to_cutline {src} {dst}".format(
gdal_bin_path=settings.GDAL_BIN_PATH, aoi=AOI_PATH, src=src, dst=dst
)
)
def clip_srtm_to_extent():
logger.info("Clip SRTM to extent")
srtm_clipped_path = os.path.join(MODIS_VI_TASKS_DATA_DIR, "srtm_dem_clipped.tif")
if not os.path.exists(srtm_clipped_path):
run_command(
"{gdal_bin_path}/gdalwarp -of GTiff -cutline {aoi} -crop_to_cutline {src} {dst}".format(
gdal_bin_path=settings.GDAL_BIN_PATH,
aoi=EXTENT_PATH,
src=SRTM_DEM_PATH,
dst=srtm_clipped_path,
)
)
return srtm_clipped_path
def extract_subdatasets_as_gtiffs(files, tif_dir):
logger.info("Extract subdatasets as GeoTIFFs")
for src in files:
name, _ = os.path.splitext(os.path.basename(src))
dst = os.path.join(tif_dir, name)
if not os.path.exists(f"{dst}_ndvi.tif"):
run_command(
f"gdal_translate "
f'HDF4_EOS:EOS_GRID:{src}:MODIS_Grid_16DAY_250m_500m_VI:"250m 16 days NDVI" '
f"{dst}_ndvi.tif")
if not os.path.exists(f"{dst}_pixelrel.tif"):
run_command(
f"gdal_translate "
f'HDF4_EOS:EOS_GRID:{src}:MODIS_Grid_16DAY_250m_500m_VI:"250m 16 days pixel reliability" '
f"{dst}_pixelrel.tif")
def create_raster_tiles(raster, *, levels):
gdal2tiles = settings.GDAL2TILES_BIN_PATH
n_jobs = settings.GDAL2TILES_NUM_JOBS
media_dir = settings.MEDIA_ROOT
rasters_dir = os.path.join(media_dir, "rasters")
tiles_dir = os.path.join(media_dir, "tiles")
src = raster.file.path
dst = os.path.join(tiles_dir, raster.path())
zoom_range = f"{levels[0]}-{levels[1]}"
cmd = f"{gdal2tiles} --processes {n_jobs} -w none -n -z {zoom_range} {src} {dst}"
# Make sure output directory does not exist
if os.path.exists(dst):
shutil.rmtree(dst)
run_command(cmd)
def create_masks(date_from, date_to):
period_s = f'{date_from.strftime("%Y%m")}-{date_to.strftime("%Y%m")}'
logger.info("Polygonize mask")
src_path = os.path.join(
MVI_RESULTS_DIR, "{}_vegetation_cloud_mask.tif".format(period_s)
)
dst_path = os.path.join(
MVI_RESULTS_DIR, "{}_vegetation_cloud_mask.geojson".format(period_s)
)
run_command(
'{gdal_bin_path}/gdal_polygonize.py {src} {dst} -b 1 -f "GeoJSON" DN'.format(
gdal_bin_path=settings.GDAL_BIN_PATH, src=src_path, dst=dst_path
)
)
logging.info("Reproject to epsg:4326")
data = gpd.read_file(dst_path)
data_proj = data.copy()
data_proj["geometry"] = data_proj["geometry"].to_crs(epsg=4326)
data_proj.to_file(dst_path)
logger.info("Load vegetation mask to DB")
create_vegetation_masks(dst_path, date_to)
def download_and_process(date_from, date_to):
os.makedirs(MVI_RAW_DIR, exist_ok=True)
os.makedirs(MVI_TIF_DIR, exist_ok=True)
modis_filenames = download_modis_vi_images(
date_from=date_from,
date_to=date_to,
output_dir=MVI_RAW_DIR,
username=settings.MODIS_USER,
password=settings.MODIS_PASS,
)
if not modis_filenames:
logger.error("No MODIS files for this period!")
return
extract_subdatasets_as_gtiffs(modis_filenames, MVI_TIF_DIR)
srtm_clipped_path = clip_srtm_to_extent()
lomas_mask = calculate_srtm_mask(srtm_clipped_path)
logger.info("Clip NDVI to extent")
os.makedirs(MVI_CLIP_DIR, exist_ok=True)
name, _ = os.path.splitext(os.path.basename(modis_filenames[0]))
ndvi_path = glob(os.path.join(MVI_TIF_DIR, f"{name}_ndvi.tif"))[0]
ndvi_clipped_path = os.path.join(MVI_CLIP_DIR, os.path.basename(ndvi_path))
if os.path.exists(ndvi_clipped_path):
os.unlink(ndvi_clipped_path)
run_command(
"{gdal_bin_path}/gdalwarp -of GTiff -cutline {extent} -crop_to_cutline {src} {dst}".format(
gdal_bin_path=settings.GDAL_BIN_PATH,
extent=EXTENT_PATH,
src=ndvi_path,
dst=ndvi_clipped_path,
)
)
logger.info("Superimpose clipped SRTM and NDVI rasters to align them")
run_command(
"{otb_bin_path}/otbcli_Superimpose -inr {inr} -inm {inm} -out {out}".format(
otb_bin_path=settings.OTB_BIN_PATH,
inr=srtm_clipped_path,
inm=ndvi_clipped_path,
out=ndvi_clipped_path,
)
)
with rasterio.open(ndvi_clipped_path) as src:
modis_ndvi = src.read(1)
modis_meta = src.profile.copy()
modis_meta["nodata"] = 0
modis_meta["dtype"] = np.uint8
logger.info("Build final vegetation mask")
vegetacion_mask = modis_ndvi > THRESHOLD
verde_mask = vegetacion_mask & lomas_mask
verde = np.copy(modis_ndvi)
verde[~verde_mask] = 0
logger.info("Build scaled NDVI mask")
verde_rango = np.copy(verde)
verde_rango[(verde >= (0.2 / FACTOR_ESCALA)) & (verde < (0.4 / FACTOR_ESCALA))] = 1
verde_rango[(verde >= (0.4 / FACTOR_ESCALA)) & (verde < (0.6 / FACTOR_ESCALA))] = 2
verde_rango[(verde >= (0.6 / FACTOR_ESCALA)) & (verde < (0.8 / FACTOR_ESCALA))] = 3
verde_rango[verde >= (0.8 / FACTOR_ESCALA)] = 4
verde_rango[verde < 0] = 0
verde_rango = verde_rango.astype(dtype=np.uint8)
period_s = f'{date_from.strftime("%Y%m")}-{date_to.strftime("%Y%m")}'
logger.info("Write masked NDVI raster")
os.makedirs(MVI_MASK_DIR, exist_ok=True)
ndvi_path = os.path.join(MVI_MASK_DIR, "{}_ndvi.tif".format(period_s))
meta = modis_meta.copy()
meta.update(dtype=verde.dtype)
with rasterio.open(ndvi_path, "w", **meta) as dst:
dst.write(verde, 1)
verde[verde_mask] = 1
verde = verde.astype(dtype=np.uint8)
logger.info("Write vegetation mask raster")
os.makedirs(MVI_MASK_DIR, exist_ok=True)
vegetation_mask_path = os.path.join(
MVI_MASK_DIR, "{}_vegetation_mask.tif".format(period_s)
)
with rasterio.open(vegetation_mask_path, "w", **modis_meta) as dst:
dst.write(verde, 1)
# Cloud mask
logger.info("Clip pixel reliability raster to extent")
name, _ = os.path.splitext(os.path.basename(modis_filenames[0]))
pixelrel_path = glob(os.path.join(MVI_TIF_DIR, f"{name}_pixelrel.tif"))[0]
pixelrel_clipped_path = os.path.join(MVI_CLIP_DIR, os.path.basename(pixelrel_path))
if os.path.exists(pixelrel_clipped_path):
os.unlink(pixelrel_clipped_path)
run_command(
"{gdal_bin_path}/gdalwarp -of GTiff -cutline {extent} -crop_to_cutline {src} {dst}".format(
gdal_bin_path=settings.GDAL_BIN_PATH,
extent=EXTENT_PATH,
src=pixelrel_path,
dst=pixelrel_clipped_path,
)
)
logger.info("Superimpose pixel rel raster to SRTM raster")
run_command(
"{otb_bin_path}/otbcli_Superimpose -inr {inr} -inm {inm} -out {out}".format(
otb_bin_path=settings.OTB_BIN_PATH,
inr=srtm_clipped_path,
inm=pixelrel_clipped_path,
out=pixelrel_clipped_path,
)
)
logger.info("Build cloud mask from pixel reliability raster")
with rasterio.open(pixelrel_clipped_path) as cloud_src:
clouds = cloud_src.read(1)
# In clouds 2 is snow/ice and 3 are clouds, and -1 is not processed data
cloud_mask = np.copy(clouds)
cloud_mask[(clouds == 2) | (clouds == 3) | (clouds == -1)] = 1
cloud_mask[(clouds != 2) & (clouds != 3)] = 0
cloud_mask = cloud_mask.astype(np.uint8)
logger.info("Write cloud mask raster")
cloud_mask_path = os.path.join(MVI_MASK_DIR, "{}_cloud_mask.tif".format(period_s))
with rasterio.open(cloud_mask_path, "w", **modis_meta) as dst:
dst.write(cloud_mask, 1)
# vegetation_range_path = os.path.join(
# MVI_MASK_DIR, "{}_vegetation_range.tif".format(period_s)
# )
# with rasterio.open(vegetation_range_path, "w", **modis_meta) as dst:
# dst.write(verde_rango, 1)
logger.info("Create a mask with both vegetation and clouds")
verde[cloud_mask == 1] = 2
veg_cloud_mask_path = os.path.join(
MVI_MASK_DIR, "{}_vegetation_cloud_mask.tif".format(period_s)
)
with rasterio.open(veg_cloud_mask_path, "w", **modis_meta) as dst:
dst.write(verde, 1)
# Clip to AOI all rasters into RESULTS_DIR
clip_with_aoi(ndvi_path)
clip_with_aoi(vegetation_mask_path)
clip_with_aoi(cloud_mask_path)
clip_with_aoi(veg_cloud_mask_path)
return True