def create_consolidated_water_mask(self):
sentinel2_mask_path=os.path.join(self.output_directory, 'sentinel2_water_mask.tif')
sentinel2_mask_ds=gdal.Open(sentinel2_mask_path)
sentinel2_mask_nodata_value=sentinel2_mask_ds.GetRasterBand(1).GetNoDataValue()
sentinel2_mask_array=sentinel2_mask_ds.GetRasterBand(1).ReadAsArray()
sentinel2_mask_ds=None
landsat_mask_path=os.path.join(self.output_directory, 'landsat_water_mask.tif')
landsat_mask_ds=gdal.Warp('MEM', landsat_mask_path, options=gdal.WarpOptions(format = 'MEM', xRes = 10, yRes = -10))
landsat_mask_nodata_value=landsat_mask_ds.GetRasterBand(1).GetNoDataValue()
landsat_mask_array=landsat_mask_ds.GetRasterBand(1).ReadAsArray()
landsat_mask_ds=None
sentinel2_mask_array[sentinel2_mask_array==sentinel2_mask_nodata_value]=landsat_mask_array
save_array_as_gtiff(sentinel2_mask_array, os.path.join(self.output_directory, 'sentinel2_landsat_water_mask.tif'), gtiff_path=os.path.join(self.output_directory, 'sentinel2_water_mask.tif'), dtype='uint', nodata_value=landsat_mask_nodata_value)
def save_indices(self, output_folder):
array = self.get_NDVI_as_array()
save_array_as_gtiff(array,
output_folder + '/NDVI.tif',
dataset=self.nir)
array = self.get_NDWI_as_array()
save_array_as_gtiff(array,
output_folder + '/NDWI.tif',
dataset=self.nir)
#array=self.get_MNDWI_as_array()
#save_array_as_gtiff(array, output_folder+'/MNDWI.tif', dataset=self.nir)
array = self.get_WRI_as_array()
save_array_as_gtiff(array,
output_folder + '/WRI.tif',
dataset=self.nir)
array = self.get_AWEI_as_array()
save_array_as_gtiff(array,
output_folder + '/AWEI.tif',
dataset=self.nir)
array = None
def classify_dataset(self, landsat=False, sentinel2=False, bands_using=True, indices_using=True):
if sentinel2==True:
images_collection=[]
if indices_using==True:
indices=os.listdir(os.path.join(self.output_directory, 'sentinel2_indices'))
for index in indices:
images_collection.append(os.path.join(self.output_directory, 'sentinel2_indices', index))
if bands_using==True:
bands=os.listdir(os.path.join(self.output_directory, 'sentinel2'))
for band in bands:
if band!='cloud_mask.tif':
images_collection.append(os.path.join(self.output_directory, 'sentinel2', band))
a=WatershesBasedClassifier(images_collection, base_image_index=2)
a.get_classified_segmentation(os.path.join(self.output_directory, 'sentinel2_class.tif'), mode='raster', window_size=500, statistical_indicators=['mean', 'min', 'max'])
a=None
for index in indices:
if index=='NDWI.tif':
ds=gdal.Open(os.path.join(self.output_directory, 'sentinel2_indices', index))
array=np.array(ds.GetRasterBand(1).ReadAsArray())
ds=None
ndwi_bin_array=get_binary_classified_array(array)
save_array_as_gtiff(ndwi_bin_array, os.path.join(self.output_directory, 'ndwi_bin_array.tif'), gtiff_path=os.path.join(self.output_directory, 'sentinel2_class.tif'))
array=None
if 'NDVI' in index:
ds=gdal.Open(os.path.join(self.output_directory, 'sentinel2_indices', index))
array=np.array(ds.GetRasterBand(1).ReadAsArray())
ds=None
ndvi_bin_array=get_binary_classified_array(array)
save_array_as_gtiff(ndvi_bin_array, os.path.join(self.output_directory, 'ndvi_bin_array.tif'), gtiff_path=os.path.join(self.output_directory, 'sentinel2_class.tif'))
array=None
claster_ds=gdal.Open(os.path.join(self.output_directory, 'sentinel2_class.tif'))
claster_array=np.array(claster_ds.GetRasterBand(1).ReadAsArray())
claster_ds=None
new_claster_array=get_binary_array_from_clasters(claster_array, [ndwi_bin_array, reverse_binary_array(ndvi_bin_array)])
mask_path=os.path.join(self.output_directory, 'sentinel2', 'nir.tif')
mask_ds=gdal.Open(mask_path)
mask_array=mask_ds.getRasterBand(1).ReadAsArray()
mask_nodata=mask_ds.getRasterBand(1).GetNoDataValue()
mask_ds=None
new_claster_array[mask_array==mask_nodata]=5
mask_array=None
ndwi_bin_array=None
ndvi_bin_array=None
save_array_as_gtiff(new_claster_array, os.path.join(self.output_directory, 'sentinel2_water_mask.tif'), gtiff_path=os.path.join(self.output_directory, 'sentinel2_class.tif'), dtype='uint', nodata_value=5)
if landsat==True:
images_collection=[]
if indices_using==True:
indices=os.listdir(os.path.join(self.output_directory, 'landsat_indices'))
for index in indices:
images_collection.append(os.path.join(self.output_directory, 'landsat_indices', index))
if bands_using==True:
bands=os.listdir(os.path.join(self.output_directory, 'landsat'))
for band in bands:
if band!='cloud_mask.tif':
images_collection.append(os.path.join(self.output_directory, 'landsat', band))
a=WatershesBasedClassifier(images_collection, base_image_index=2)
a.get_classified_segmentation(os.path.join(self.output_directory, 'landsat_class.tif'), mode='raster', window_size=500, statistical_indicators=['mean', 'min', 'max'])
a=None
for index in indices:
if index=='NDWI.tif':
ds=gdal.Open(os.path.join(self.output_directory, 'landsat_indices', index))
array=np.array(ds.GetRasterBand(1).ReadAsArray())
ds=None
ndwi_bin_array=get_binary_classified_array(array)
save_array_as_gtiff(ndwi_bin_array, os.path.join(self.output_directory, 'ndwi_bin_array.tif'), gtiff_path=os.path.join(self.output_directory, 'landsat_class.tif'), dtype='uint')
array=None
if 'NDVI' in index:
ds=gdal.Open(os.path.join(self.output_directory, 'landsat_indices', index))
array=np.array(ds.GetRasterBand(1).ReadAsArray())
ds=None
ndvi_bin_array=get_binary_classified_array(array)
save_array_as_gtiff(ndvi_bin_array, os.path.join(self.output_directory, 'ndvi_bin_array.tif'), gtiff_path=os.path.join(self.output_directory, 'landsat_class.tif'), dtype='uint')
array=None
claster_ds=gdal.Open(os.path.join(self.output_directory, 'landsat_class.tif'))
claster_array=np.array(claster_ds.GetRasterBand(1).ReadAsArray())
claster_ds=None
new_claster_array=get_binary_array_from_clasters(claster_array, [ndwi_bin_array, reverse_binary_array(ndvi_bin_array)])
mask_path=os.path.join(self.output_directory, 'landsat', 'nir.tif')
mask_ds=gdal.Open(mask_path)
mask_array=mask_ds.getRasterBand(1).ReadAsArray()
mask_nodata=mask_ds.getRasterBand(1).GetNoDataValue()
mask_ds=None
new_claster_array[mask_array==mask_nodata]=5
mask_array=None
ndwi_bin_array=None
ndvi_bin_array=None
save_array_as_gtiff(new_claster_array, os.path.join(self.output_directory, 'landsat_water_mask.tif'), gtiff_path=os.path.join(self.output_directory, 'landsat_class.tif'), dtype='uint', nodata_value=5)
def save_landsat_prepared_images(self, output_folder):
if self.landsat_folder != None:
landsat_files = os.listdir(
os.path.join(self.input_folder, self.landsat_folder))
for file in landsat_files:
if 'b3' in file.lower().split('.')[0].split('_'):
band_B3 = os.path.join(self.input_folder,
self.landsat_folder, file)
print('band_B3: ' + file)
if 'b4' in file.lower().split('.')[0].split('_'):
band_B4 = os.path.join(self.input_folder,
self.landsat_folder, file)
print('band_B4: ' + file)
if 'b5' in file.lower().split('.')[0].split('_'):
band_B5 = os.path.join(self.input_folder,
self.landsat_folder, file)
print('band_B5: ' + file)
if 'b6' in file.lower().split('.')[0].split('_'):
band_B6 = os.path.join(self.input_folder,
self.landsat_folder, file)
print('band_B6: ' + file)
if 'b7' in file.lower().split('.')[0].split('_'):
band_B7 = os.path.join(self.input_folder,
self.landsat_folder, file)
print('band_B7: ' + file)
if 'MTL' in file.split('.')[0].split('_'):
MTL_file = os.path.join(self.input_folder,
self.landsat_folder, file)
print('MTL_file: ' + file)
if 'ANG' in file.split('.')[0].split('_'):
ANG_file = os.path.join(self.input_folder,
self.landsat_folder, file)
print('ANG_file: ' + file)
bands_list = [band_B3, band_B4, band_B5, band_B6, band_B7]
try:
(os.mkdir(os.path.join(self.input_folder, 'temp')))
except Exception:
pass
if self.landsat_correction_method == 'srem' and self.usgs_util_path == None:
raise Exception(
"For using SREM correction indicate usgs_util_path!")
if self.landsat_correction_method == 'srem' and self.usgs_util_path != None:
srem = SREMPyLandsat(mode='landsat-usgs-utils')
for band in bands_list:
if band != band_B6 and band != band_B7:
data = {
'band': band,
'metadata': MTL_file,
'angles_file': ANG_file,
'usgs_util_path': self.usgs_util_path,
'temp_dir': os.path.join(self.input_folder,
'temp/'),
'cygwin_bash_exe_path': None
}
srem.set_data(data)
sr = srem.get_srem_surface_reflectance_as_array()
if band == band_B3:
filename = os.path.join(output_folder, 'grn.tif')
if band == band_B4:
filename = os.path.join(output_folder, 'red.tif')
if band == band_B5:
filename = os.path.join(output_folder, 'nir.tif')
srem.save_array_as_gtiff(sr, filename)
else:
if band == band_B6:
filename = os.path.join(output_folder, 'mir.tif')
if band == band_B7:
filename = os.path.join(output_folder, 'swir.tif')
pre_image = LandsatBandCalibrator(band, MTL_file)
corrected_array = pre_image.get_dos_corrected_reflectance_as_array(
)
pre_image.save_array_as_gtiff(corrected_array,
filename)
if self.landsat_correction_method == 'dos':
for band in bands_list:
print('Running DOS: ' + band)
if band == band_B3:
filename = os.path.join(output_folder, 'grn.tif')
if band == band_B4:
filename = os.path.join(output_folder, 'red.tif')
if band == band_B5:
filename = os.path.join(output_folder, 'nir.tif')
if band == band_B6:
filename = os.path.join(output_folder, 'mir.tif')
if band == band_B7:
filename = os.path.join(output_folder, 'swir.tif')
pre_image = LandsatBandCalibrator(band, MTL_file)
corrected_array = pre_image.get_dos_corrected_reflectance_as_array(
)
pre_image.save_array_as_gtiff(corrected_array, filename)
pre_image = None
if self.landsat_cloud_fmask == True:
output_cloud_path = os.path.join(output_folder,
'cloud_mask.tif')
input_directory = os.path.join(self.input_folder,
self.landsat_folder)
print('Running FMASK')
cmd = '%s -o %s --scenedir %s --cloudbufferdistance %s --cloudprobthreshold %s --shadowbufferdistance %s' % (
FMASK_EXECUTABLE_PATH_LANDSAT, output_cloud_path,
input_directory, 30, 75.0, 0)
print("cmd: " + cmd)
os.system(cmd)
os.remove(output_cloud_path + '.aux.xml')
mask_ds = gdal.Open(output_cloud_path)
mask_array = np.array(mask_ds.GetRasterBand(1).ReadAsArray())
mask_ds = None
output_files_list = os.listdir(output_folder)
for file in output_files_list:
if file != 'cloud_mask.tif':
ds = gdal.Open(os.path.join(output_folder, file))
output_array = np.array(
ds.GetRasterBand(1).ReadAsArray())
output_array[mask_array == 2] = np.nan
output_array[mask_array == 3] = np.nan
save_array_as_gtiff(output_array,
os.path.join(output_folder, file),
dataset=ds)
mask_array = None
ds = None
output_array = None
shutil.rmtree(os.path.join(self.input_folder, 'temp'))
try:
os.remove(output_cloud_path + '.aux.xml')
except Exception as e:
print(e)
pass
def save_sentinel2_prepared_images(self, output_folder):
if self.sentinel2_L2A_folder == None and self.sentinel2_L1C_folder != None:
if self.sen2cor_util_path == None:
raise Exception(
'For processing Sentinel-2 1C-level product sen2cor_util_path is required'
)
cmd = '%s %s' % (self.sen2cor_util_path,
os.path.join(self.input_folder,
self.sentinel2_L1C_folder))
#после демоверсии можно посмотреть на встроенную возможность использования ЦМР,
#которую sen2cor сам же скачивает http://data_public:[email protected]/srtm/tiles/GeoTIFF/
#но придется конфигурацию sen2cor менять пользователю, а не программно
#поэтому пока без ЦМР
print("cmd: " + cmd)
os.system(cmd)
self.sentinel2_L2A_folder = self.sentinel2_L1C_folder.replace(
'MSIL1C', 'MSIL2A')
if self.sentinel2_L2A_folder != None:
granule_into_main = os.path.join(self.input_folder,
self.sentinel2_L2A_folder,
'GRANULE')
images_dir = os.path.join(
os.listdir(granule_into_main)[0], 'IMG_DATA')
if self.sentinel2_resolution == 20:
print('Output spatial resolution - 20')
m_images_folder = os.path.join(granule_into_main, images_dir,
'R20m')
for file in os.listdir(m_images_folder):
if 'B03' in file.split('_') or 'B04' in file.split(
'_') or 'B8A' in file.split(
'_') or 'B11' in file.split(
'_') or 'B12' in file.split('_'):
new_file = file.replace('B03', '__grn')
new_file = new_file.replace('B04', '__red')
new_file = new_file.replace('B8A', '__nir')
new_file = new_file.replace('B11', '__mir')
new_file = new_file.replace('B12', '__swir')
new_file = new_file.split('___')[-1]
new_file = new_file.replace('_20m', '')
gdal.Warp(
output_folder + '/' + new_file.split('.')[0] +
'.tif', os.path.join(m_images_folder, file))
if self.sentinel2_resolution == 10:
print('Output spatial resolution - 10')
m_images_folder = os.path.join(granule_into_main, images_dir,
'R10m')
for file in os.listdir(m_images_folder):
if 'B03' in file.split('_') or 'B04' in file.split(
'_') or 'B08' in file.split('_'):
new_file = file.replace('B03', '__grn')
new_file = new_file.replace('B04', '__red')
new_file = new_file.replace('B08', '__nir')
new_file = new_file.split('___')[-1]
new_file = new_file.replace('_10m', '')
gdal.Warp(
output_folder + '/' + new_file.split('.')[0] +
'.tif', os.path.join(m_images_folder, file))
mm_images_folder = os.path.join(granule_into_main, images_dir,
'R20m')
for file in os.listdir(mm_images_folder):
if 'B11' in file.split('_') or 'B12' in file.split('_'):
new_file = file.replace('B11', '__mir')
new_file = new_file.replace('B12', '__swir')
new_file = new_file.split('___')[-1]
gdal.Warp(output_folder + '/' +
new_file.replace('_20m.jp2', '.tif'),
os.path.join(mm_images_folder, file),
xRes=10,
yRes=-10,
resampleAlg='bilinear')
if self.sentinel2_cloud != None:
if self.sentinel2_cloud == 'native_2A_level':
cloud_mask_folder = os.path.join(granule_into_main, images_dir,
'R20m')
for file in os.listdir(cloud_mask_folder):
if 'SCL' in file.split('_'):
cloud_mask_file = os.path.join(cloud_mask_folder, file)
cloud_flags = [3, 8, 9, 10, 11]
if self.sentinel2_resolution == 10:
gdal.Warp(os.path.join(output_folder, 'cloud_mask.tif'),
cloud_mask_file,
xRes=10,
yRes=-10)
cloud_mask_file = os.path.join(output_folder,
'cloud_mask.tif')
if self.sentinel2_cloud == 'fmask':
if self.sentinel2_L1C_folder == None:
raise Exception(
'Fmask is applicable only to 1C-level product')
cloud_mask_file = os.path.join(output_folder, 'cloud_mask.tif')
print('Running FMASK')
cmd = '%s -o %s --safedir %s --cloudbufferdistance %s --cloudprobthreshold %s --shadowbufferdistance %s' % (
FMASK_EXECUTABLE_PATH_SENTINEL, cloud_mask_file,
os.path.join(self.input_folder,
self.sentinel2_L1C_folder), 30, 75.0, 0)
print("cmd: " + cmd)
os.system(cmd)
try:
os.remove(cloud_mask_file + '.aux.xml')
except Exception:
pass
cloud_flags = [2, 3]
if self.sentinel2_resolution == 10:
gdal.Warp(os.path.join(output_folder, 'cloud_mask.tif'),
cloud_mask_file,
xRes=10,
yRes=-10)
cloud_mask_file = os.path.join(output_folder,
'cloud_mask.tif')
print('del .aux.xml')
try:
os.remove(cloud_mask_file + '.aux.xml')
except Exception:
pass
if self.sentinel2_cloud == 's2cloudless':
print('Running s2cloudless')
if self.sentinel2_resolution == 10:
resolution = '10m'
if self.sentinel2_resolution == 20:
resolution = '20m'
cloud_mask_file = os.path.join(output_folder, 'cloud_mask.tif')
u = S2CloudDetectorUtil(os.path.join(
self.input_folder, self.sentinel2_L1C_folder),
resolution=resolution)
u.detect_clouds()
u.export_to_gtiff(output_gtiff=cloud_mask_file, mode='mask')
u = None
cloud_flags = [1]
mask_ds = gdal.Open(cloud_mask_file)
mask_array = np.array(mask_ds.GetRasterBand(1).ReadAsArray())
mask_ds = None
for file in os.listdir(output_folder):
if file != 'cloud_mask.tif':
print('Masking file: ' + file)
ds = gdal.Open(os.path.join(output_folder, file))
output_array = np.array(ds.GetRasterBand(1).ReadAsArray())
for flag in cloud_flags:
output_array[mask_array == flag] = 32767
save_array_as_gtiff(output_array,
os.path.join(output_folder, file),
dataset=ds,
dtype='int')
new_ds = gdal.Open(os.path.join(output_folder, file),
GA_Update)
new_ds.GetRasterBand(1).SetNoDataValue(32767)
mask_array = None
ds = None
new_ds = None
output_array = None