def clip(self, in_stack_file, out_clipped_file, nodata=0, process_bar=True):
"""
Clipping the stack file only if is activated selected area or shape area,
else return the original image
"""
if not self.clipping_with_aoi and not self.clipping_with_shape:
return in_stack_file
if process_bar:
update_process_bar(self.process_bar, 24, self.process_status,
self.tr("Clipping the reflective stack..."))
if self.clipping_with_aoi:
tmp_aoi = os.path.join(self.tmp_dir, "aoi_tmp_{}.gpkg".format(datetime.now().strftime("%y%m%d_%H%M%S")))
QgsVectorFileWriter.writeAsVectorFormat(self.aoi_features, tmp_aoi, "System", self.aoi_features.crs(), "GPKG")
shape_layer = load_layer(tmp_aoi, add_to_legend=False)
self.do_clipping_with_shape(in_stack_file, shape_layer, tmp_aoi, out_clipped_file, False, nodata)
unload_layer(tmp_aoi)
try: os.remove(tmp_aoi)
except: pass
if self.clipping_with_shape:
shape_layer = get_layer_by_name(os.path.splitext(os.path.basename(self.shape_path))[0])
if shape_layer is None:
shape_layer = load_layer(self.shape_path, add_to_legend=False)
self.do_clipping_with_shape(in_stack_file, shape_layer, os.path.abspath(self.shape_path),
out_clipped_file, self.crop_to_cutline, nodata)
unload_layer(self.shape_path)
else:
self.do_clipping_with_shape(in_stack_file, shape_layer, os.path.abspath(self.shape_path),
out_clipped_file, self.crop_to_cutline, nodata)
return out_clipped_file
def do_cloud_qa_l457(self, cloud_qa_file, checked_items, specific_values=[]):
# tmp file for cloud
self.cloud_qa = os.path.join(self.tmp_dir, "cloud_qa_{}.tif".format(datetime.now().strftime('%H%M%S')))
update_process_bar(self.process_bar, 50, self.process_status,
self.tr("Making the Cloud QA filter..."))
########################################
# clipping the QA Mask (only if is activated selected area or shape area)
self.cloud_qa_clip_file = os.path.join(self.tmp_dir, "cloud_qa_clip.tif")
self.cloud_qa_for_process = self.clip(cloud_qa_file, self.cloud_qa_clip_file)
########################################
# convert selected items to binary and decimal values
values_combinations = []
# bits not used or not fill
static_bits = [6, 7]
# generate the values combinations for one bit items selected
cloud_qa_items_1b = {"Dark Dense Vegetation (bit 0)": [0], "Cloud (bit 1)": [1], "Cloud Shadow (bit 2)": [2],
"Adjacent to cloud (bit 3)": [3], "Snow (bit 4)": [4], "Water (bit 5)": [5]}
for item, bits in cloud_qa_items_1b.items():
binary = [0] * 8
if checked_items[item]:
binary[(len(binary) - 1) - bits[0]] = 1
values_combinations += list(binary_combination(binary, static_bits + bits))
# add the specific values
if specific_values:
values_combinations += specific_values
# delete duplicates
values_combinations = list(set(values_combinations))
# only left the values inside the image
values_combinations = check_values_in_image(self.cloud_qa_for_process, values_combinations)
filter_values = ",".join(["A==" + str(x) for x in values_combinations])
not_filter_values = ",".join(["A!=" + str(x) for x in values_combinations])
########################################
# do QA Mask filter
gdal_calc.Calc(calc="1*(numpy.all([{nfv}], axis=0)) + 7*(numpy.any([{fv}], axis=0))".format(fv=filter_values,
nfv=not_filter_values),
A=self.cloud_qa_for_process, outfile=self.cloud_qa, type="Byte", NoDataValue=1)
# unset the nodata, leave the 1 (valid fields)
cmd = ['gdal_edit' if platform.system() == 'Windows' else 'gdal_edit.py', '"{}"'.format(self.cloud_qa), '-unsetnodata']
call(" ".join(cmd), shell=True)
# save final result of masking
self.cloud_masking_files.append(self.cloud_qa)
### ending process
update_process_bar(self.process_bar, 100, self.process_status,
self.tr("DONE"))
def clip(self,
in_stack_file,
out_clipped_file,
nodata=0,
process_bar=True):
"""
Clipping the stack file only if is activated selected area or shape area,
else return the original image
"""
if not self.clipping_with_aoi and not self.clipping_with_shape:
return in_stack_file
if process_bar:
update_process_bar(self.process_bar, 24, self.process_status,
self.tr("Clipping the reflective stack..."))
if self.clipping_with_aoi:
tmp_memory_file = Path(tempfile.gettempdir(),
"memory_layer_aoi.gpkg")
QgsVectorFileWriter.writeAsVectorFormat(self.aoi_features,
str(tmp_memory_file),
"System",
self.aoi_features.crs(),
"GPKG")
load_layer(str(tmp_memory_file), add_to_legend=False)
self.do_clipping_with_shape(in_stack_file, str(tmp_memory_file),
out_clipped_file, False, nodata)
unload_layer(str(tmp_memory_file))
tmp_memory_file.unlink()
if self.clipping_with_shape:
if not get_layer_by_name(
os.path.splitext(os.path.basename(self.shape_path))[0]):
load_layer(self.shape_path, add_to_legend=False)
self.do_clipping_with_shape(in_stack_file,
os.path.abspath(self.shape_path),
out_clipped_file,
self.crop_to_cutline, nodata)
unload_layer(self.shape_path)
else:
self.do_clipping_with_shape(in_stack_file,
os.path.abspath(self.shape_path),
out_clipped_file,
self.crop_to_cutline, nodata)
return out_clipped_file
def do_blue_band(self, bb_threshold):
# tmp file for cloud
self.cloud_bb_file = os.path.join(
self.tmp_dir,
"cloud_bb_{}.tif".format(datetime.now().strftime('%H%M%S')))
update_process_bar(self.process_bar, 50, self.process_status,
self.tr("Making the blue band filter..."))
########################################
# select the Blue Band
if self.landsat_version in [4, 5, 7]:
# get the reflective file names bands
self.blue_band_file = os.path.join(
self.input_dir, self.mtl_file['FILE_NAME_BAND_1'])
if self.landsat_version in [8]:
# get the reflective file names bands
self.blue_band_file = os.path.join(
self.input_dir, self.mtl_file['FILE_NAME_BAND_2'])
# fix file name
self.blue_band_file = get_prefer_name(self.blue_band_file)
########################################
# clipping the Blue Band (only if is activated selected area or shape area)
self.blue_band_clip_file = os.path.join(self.tmp_dir,
"blue_band_clip.tif")
self.blue_band_for_process = self.clip(self.blue_band_file,
self.blue_band_clip_file)
########################################
# do blue band filter
gdal_calc.Calc(calc="1*(A<{threshold})+6*(A>={threshold})".format(
threshold=bb_threshold),
A=self.blue_band_for_process,
outfile=self.cloud_bb_file,
type="Byte")
# save final result of masking
self.cloud_masking_files.append(self.cloud_bb_file)
### ending process
update_process_bar(self.process_bar, 100, self.process_status,
self.tr("DONE"))
def do_blue_band(self, bb_threshold):
# tmp file for cloud
self.cloud_bb_file = os.path.join(self.tmp_dir, "cloud_bb_{}.tif".format(datetime.now().strftime('%H%M%S')))
update_process_bar(self.process_bar, 50, self.process_status,
self.tr("Making the blue band filter..."))
########################################
# select the Blue Band
if self.landsat_version in [4, 5, 7]:
# get the reflective file names bands
self.blue_band_file = os.path.join(self.input_dir, self.mtl_file['FILE_NAME_BAND_1'])
if self.landsat_version in [8]:
# get the reflective file names bands
self.blue_band_file = os.path.join(self.input_dir, self.mtl_file['FILE_NAME_BAND_2'])
# fix file name
self.blue_band_file = get_prefer_name(self.blue_band_file)
########################################
# clipping the Blue Band (only if is activated selected area or shape area)
self.blue_band_clip_file = os.path.join(self.tmp_dir, "blue_band_clip.tif")
self.blue_band_for_process = self.clip(self.blue_band_file, self.blue_band_clip_file)
########################################
# do blue band filter
cmd = ['gdal_calc' if platform.system() == 'Windows' else 'gdal_calc.py', '--quiet', '--overwrite',
'--calc "1*(A<{threshold})+6*(A>={threshold})"'.format(threshold=bb_threshold),
'-A {}'.format(self.blue_band_for_process), '--outfile "{}"'.format(self.cloud_bb_file),
'--type="Byte"', '--co COMPRESS=PACKBITS']
call(" ".join(cmd), shell=True)
# save final result of masking
self.cloud_masking_files.append(self.cloud_bb_file)
### ending process
update_process_bar(self.process_bar, 100, self.process_status,
self.tr("DONE"))
def do_pixel_qa(self, pixel_qa_file, checked_items, specific_values=[]):
"""
http://landsat.usgs.gov/qualityband.php
"""
# tmp file for Pixel QA
self.pixel_qa = os.path.join(self.tmp_dir, "pixel_qa_{}.tif".format(datetime.now().strftime('%H%M%S')))
update_process_bar(self.process_bar, 50, self.process_status,
self.tr("Making the Pixel QA filter..."))
########################################
# clipping the QA Mask (only if is activated selected area or shape area)
self.pixel_qa_clip_file = os.path.join(self.tmp_dir, "pixel_qa_clip.tif")
self.pixel_qa_for_process = self.clip(pixel_qa_file, self.pixel_qa_clip_file)
########################################
# convert selected items to binary and decimal values
values_combinations = []
# bits not used or not fill
if self.landsat_version in [4, 5, 7]:
static_bits = [0, 1, 8, 9, 10, 11, 12, 13, 14, 15]
if self.landsat_version in [8]:
static_bits = [0, 1, 11, 12, 13, 14, 15]
# generate the values combinations for one bit items selected
pixel_qa_items_1b = {"Water (bit 2)": [2], "Cloud Shadow (bit 3)": [3],
"Snow (bit 4)": [4], "Cloud (bit 5)": [5]}
if self.landsat_version in [8]: # add to L8
pixel_qa_items_1b["Terrain Occlusion (bit 10)"] = [10]
for item, bits in pixel_qa_items_1b.items():
binary = [0]*16
if checked_items[item]:
binary[(len(binary) - 1) - bits[0]] = 1
values_combinations += list(binary_combination(binary, static_bits + bits))
# generate the values combinations for two bits items selected
if self.landsat_version in [4, 5, 7]:
pixel_qa_items_2b = {"Cloud Confidence (bits 6-7)": [6, 7]}
if self.landsat_version in [8]:
pixel_qa_items_2b = {"Cloud Confidence (bits 6-7)": [6, 7], "Cirrus Confidence (bits 8-9)": [8, 9]}
levels = {"0% None": [0, 0], "0-33% Low": [0, 1],
"34-66% Medium": [1, 0], "67-100% High": [1, 1]}
for item, bits in pixel_qa_items_2b.items():
if item in checked_items.keys():
for level in checked_items[item]:
binary = [0] * 16
binary[bits[0]:bits[1]+1] = (levels[level])[::-1]
binary.reverse()
values_combinations += list(binary_combination(binary, static_bits + bits))
# add the specific values
if specific_values:
values_combinations += specific_values
# delete duplicates
values_combinations = list(set(values_combinations))
# only left the values inside the image
values_combinations = check_values_in_image(self.pixel_qa_for_process, values_combinations)
filter_values = ",".join(["A==" + str(x) for x in values_combinations])
not_filter_values = ",".join(["A!=" + str(x) for x in values_combinations])
########################################
# do QA Mask filter
gdal_calc.Calc(calc="1*(numpy.all([{nfv}], axis=0)) + 9*(numpy.any([{fv}], axis=0))".format(fv=filter_values, nfv=not_filter_values),
A=self.pixel_qa_for_process, outfile=self.pixel_qa, type="Byte", NoDataValue=1)
# unset nodata
cmd = ['gdal_edit' if platform.system() == 'Windows' else 'gdal_edit.py',
'"{}"'.format(self.pixel_qa), '-unsetnodata']
call(" ".join(cmd), shell=True)
# save final result of masking
self.cloud_masking_files.append(self.pixel_qa)
### ending process
update_process_bar(self.process_bar, 100, self.process_status,
self.tr("DONE"))
def do_aerosol_l8(self, aerosol_file, checked_items, specific_values=[]):
# tmp file for cloud
self.aerosol = os.path.join(self.tmp_dir, "aerosol_{}.tif".format(datetime.now().strftime('%H%M%S')))
update_process_bar(self.process_bar, 50, self.process_status,
self.tr("Making the Aerosol filter..."))
########################################
# clipping the QA Mask (only if is activated selected area or shape area)
self.aerosol_clip_file = os.path.join(self.tmp_dir, "aerosol_clip.tif")
self.aerosol_for_process = self.clip(aerosol_file, self.aerosol_clip_file)
########################################
# convert selected items to binary and decimal values
values_combinations = []
# bits not used or not fill
static_bits = [0, 4, 5]
# generate the values combinations for one bit items selected
aerosol_items_1b = {"Aerosol Retrieval - Valid (bit 1)": [1],
"Aerosol Retrieval - Interpolated (bit 2)": [2],
"Water Pixel (bit 3)": [3]}
for item, bits in aerosol_items_1b.items():
binary = [0]*8
if checked_items[item]:
binary[(len(binary) - 1) - bits[0]] = 1
values_combinations += list(binary_combination(binary, static_bits + bits))
# generate the values combinations for two bits items selected
aerosol_items_2b = {"Aerosol Content (bits 6-7)": [6, 7]}
levels = {"Climatology content": [0, 0], "Low content": [0, 1],
"Average content": [1, 0], "High content": [1, 1]}
for item, bits in aerosol_items_2b.items():
if item in checked_items.keys():
for level in checked_items[item]:
binary = [0]*8
binary[bits[0]:bits[1]+1] = (levels[level])[::-1]
binary.reverse()
values_combinations += list(binary_combination(binary, static_bits + bits))
# add the specific values
if specific_values:
values_combinations += specific_values
# delete duplicates
values_combinations = list(set(values_combinations))
# only left the values inside the image
values_combinations = check_values_in_image(self.aerosol_for_process, values_combinations)
filter_values = ",".join(["A=="+str(x) for x in values_combinations])
not_filter_values = ",".join(["A!="+str(x) for x in values_combinations])
########################################
# do QA Mask filter
gdal_calc.Calc(calc="1*(numpy.all([{nfv}], axis=0)) + 8*(numpy.any([{fv}], axis=0))".format(fv=filter_values, nfv=not_filter_values),
A=self.aerosol_for_process, outfile=self.aerosol, type="Byte", NoDataValue=1)
# unset nodata
cmd = ['gdal_edit' if platform.system() == 'Windows' else 'gdal_edit.py',
'"{}"'.format(self.aerosol), '-unsetnodata']
call(" ".join(cmd), shell=True)
# save final result of masking
self.cloud_masking_files.append(self.aerosol)
### ending process
update_process_bar(self.process_bar, 100, self.process_status,
self.tr("DONE"))
def do_fmask(self, filters_enabled, min_cloud_size=0, cloud_prob_thresh=0.225, cloud_buffer_size=4,
shadow_buffer_size=6, cirrus_prob_ratio=0.04, nir_fill_thresh=0.02, swir2_thresh=0.03,
whiteness_thresh=0.7, swir2_water_test=0.03, nir_snow_thresh=0.11, green_snow_thresh=0.1):
########################################
# reflective bands stack
# tmp file for reflective bands stack
self.reflective_stack_file = os.path.join(self.tmp_dir, "reflective_stack.tif")
if not os.path.isfile(self.reflective_stack_file):
update_process_bar(self.process_bar, 10, self.process_status,
self.tr("Making reflective bands stack..."))
gdal_merge.main(["", "-separate", "-of", "GTiff", "-o",
self.reflective_stack_file] + self.reflective_bands)
########################################
# thermal bands stack
# tmp file for reflective bands stack
self.thermal_stack_file = os.path.join(self.tmp_dir, "thermal_stack.tif")
if not os.path.isfile(self.thermal_stack_file):
update_process_bar(self.process_bar, 20, self.process_status,
self.tr("Making thermal bands stack..."))
gdal_merge.main(["", "-separate", "-of", "GTiff", "-o",
self.thermal_stack_file] + self.thermal_bands)
########################################
# clipping the reflective bands stack (only if is activated selected area or shape area)
self.reflective_stack_clip_file = os.path.join(self.tmp_dir, "reflective_stack_clip.tif")
self.reflective_stack_for_process = self.clip(self.reflective_stack_file, self.reflective_stack_clip_file)
########################################
# clipping the thermal bands stack (only if is activated selected area or shape area)
self.thermal_stack_clip_file = os.path.join(self.tmp_dir, "thermal_stack_clip.tif")
self.thermal_stack_for_process = self.clip(self.thermal_stack_file, self.thermal_stack_clip_file)
########################################
# estimates of per-pixel angles for sun
# and satellite azimuth and zenith
#
# fmask_usgsLandsatMakeAnglesImage.py
# tmp file for angles
self.angles_file = os.path.join(self.tmp_dir, "angles.tif")
update_process_bar(self.process_bar, 30, self.process_status,
self.tr("Making fmask angles file..."))
mtlInfo = config.readMTLFile(self.mtl_path)
imgInfo = fileinfo.ImageInfo(self.reflective_stack_for_process)
corners = landsatangles.findImgCorners(self.reflective_stack_for_process, imgInfo)
nadirLine = landsatangles.findNadirLine(corners)
extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo)
satAzimuth = landsatangles.satAzLeftRight(nadirLine)
landsatangles.makeAnglesImage(self.reflective_stack_for_process, self.angles_file,
nadirLine, extentSunAngles, satAzimuth, imgInfo)
########################################
# saturation mask
#
# fmask_usgsLandsatSaturationMask.py
# tmp file for angles
self.saturationmask_file = os.path.join(self.tmp_dir, "saturationmask.tif")
update_process_bar(self.process_bar, 40, self.process_status,
self.tr("Making saturation mask file..."))
if self.landsat_version == 4:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 5:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 7:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 8:
sensor = config.FMASK_LANDSAT8
# needed so the saturation function knows which
# bands are visible etc.
fmaskConfig = config.FmaskConfig(sensor)
saturationcheck.makeSaturationMask(fmaskConfig, self.reflective_stack_for_process,
self.saturationmask_file)
########################################
# top of Atmosphere reflectance
#
# fmask_usgsLandsatTOA.py
# tmp file for toa
self.toa_file = os.path.join(self.tmp_dir, "toa.tif")
update_process_bar(self.process_bar, 50, self.process_status,
self.tr("Making top of Atmosphere ref..."))
landsatTOA.makeTOAReflectance(self.reflective_stack_for_process, self.mtl_path,
self.angles_file, self.toa_file)
########################################
# cloud mask
#
# fmask_usgsLandsatStacked.py
# tmp file for cloud
self.cloud_fmask_file = os.path.join(self.tmp_dir, "cloud_fmask_{}.tif".format(datetime.now().strftime('%H%M%S')))
update_process_bar(self.process_bar, 70, self.process_status,
self.tr("Making cloud mask with fmask..."))
# 1040nm thermal band should always be the first (or only) band in a
# stack of Landsat thermal bands
thermalInfo = config.readThermalInfoFromLandsatMTL(self.mtl_path)
anglesInfo = config.AnglesFileInfo(self.angles_file, 3, self.angles_file,
2, self.angles_file, 1, self.angles_file, 0)
if self.landsat_version == 4:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 5:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 7:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 8:
sensor = config.FMASK_LANDSAT8
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(self.toa_file)
fmaskFilenames.setThermalFile(self.thermal_stack_for_process)
fmaskFilenames.setOutputCloudMaskFile(self.cloud_fmask_file)
fmaskFilenames.setSaturationMask(self.saturationmask_file) # TODO: optional
fmaskConfig = config.FmaskConfig(sensor)
fmaskConfig.setThermalInfo(thermalInfo)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(False)
fmaskConfig.setVerbose(True)
fmaskConfig.setTempDir(self.tmp_dir)
# Set the settings fmask filters from widget to FmaskConfig
fmaskConfig.setMinCloudSize(min_cloud_size)
fmaskConfig.setEqn17CloudProbThresh(cloud_prob_thresh)
fmaskConfig.setCloudBufferSize(int(cloud_buffer_size))
fmaskConfig.setShadowBufferSize(int(shadow_buffer_size))
fmaskConfig.setCirrusProbRatio(cirrus_prob_ratio)
fmaskConfig.setEqn19NIRFillThresh(nir_fill_thresh)
fmaskConfig.setEqn1Swir2Thresh(swir2_thresh)
fmaskConfig.setEqn2WhitenessThresh(whiteness_thresh)
fmaskConfig.setEqn7Swir2Thresh(swir2_water_test)
fmaskConfig.setEqn20NirSnowThresh(nir_snow_thresh)
fmaskConfig.setEqn20GreenSnowThresh(green_snow_thresh)
# set to 1 for all Fmask filters disabled
if filters_enabled["Fmask Cloud"]:
fmask.OUTCODE_CLOUD = 2
else:
fmask.OUTCODE_CLOUD = 1
if filters_enabled["Fmask Shadow"]:
fmask.OUTCODE_SHADOW = 3
else:
fmask.OUTCODE_SHADOW = 1
if filters_enabled["Fmask Snow"]:
fmask.OUTCODE_SNOW = 4
else:
fmask.OUTCODE_SNOW = 1
if filters_enabled["Fmask Water"]:
fmask.OUTCODE_WATER = 5
else:
fmask.OUTCODE_WATER = 1
# process Fmask
fmask.doFmask(fmaskFilenames, fmaskConfig)
# save final result of masking
self.cloud_masking_files.append(self.cloud_fmask_file)
### ending fmask process
update_process_bar(self.process_bar, 100, self.process_status,
self.tr("DONE"))