def test_load_txt_mtl_2(): txtmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.txt') jsonmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.json') for k in jsonmtl['L1_METADATA_FILE'].keys(): assert k in txtmtl['L1_METADATA_FILE'] assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
def test_load_txt_mtl_1(): txtmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.txt') jsonmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.json') for k in jsonmtl['L1_METADATA_FILE'].keys(): assert k in txtmtl['L1_METADATA_FILE'] assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
def test_data(): mtl1 = toa_utils._load_mtl('tests/data/LC81060712016134LGN00_MTL.json') mtl2 = toa_utils._load_mtl('tests/data/LC80430302016140LGN00_MTL.json') mtl3 = toa_utils._load_mtl('tests/data/LC82290902015304LGN00_MTL.json') mtl4 = toa_utils._load_mtl('tests/data/LC80100202015018LGN00_MTL.json') return mtl1, mtl2, mtl3, mtl4
def processBandSet(i, row, dfTemp): mtl = toa_utils._load_mtl(image_path + dfTemp['image_id'][0] + '_MTL.txt') metadata = mtl['L1_METADATA_FILE'] for b in range(1, 10, 1): dfTemp['REFLECTANCE_MULT_BAND_{}'.format(b)] = metadata[ 'RADIOMETRIC_RESCALING']['REFLECTANCE_MULT_BAND_{}'.format(b)] dfTemp['REFLECTANCE_ADD_BAND_{}'.format(b)] = metadata[ 'RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_{}'.format(b)] for b in range(10, 12, 1): dfTemp['K1_CONSTANT_BAND_{}'.format(b)] = metadata[ 'TIRS_THERMAL_CONSTANTS']['K1_CONSTANT_BAND_{}'.format(b)] dfTemp['K2_CONSTANT_BAND_{}'.format(b)] = metadata[ 'TIRS_THERMAL_CONSTANTS']['K2_CONSTANT_BAND_{}'.format(b)] dfTemp['RADIANCE_MULT_BAND_{}'.format(b)] = metadata[ 'RADIOMETRIC_RESCALING']['RADIANCE_MULT_BAND_{}'.format(b)] dfTemp['RADIANCE_ADD_BAND_{}'.format(b)] = metadata[ 'RADIOMETRIC_RESCALING']['RADIANCE_ADD_BAND_{}'.format(b)] dfTemp['SUN_ELEVATION'] = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] dfTemp['DATE_ACQUIRED'] = date_collected = metadata['PRODUCT_METADATA'][ 'DATE_ACQUIRED'] dfTemp['SCENE_CENTER_TIME'] = metadata['PRODUCT_METADATA'][ 'SCENE_CENTER_TIME'] for elem in pollutant: dfTemp[elem] = row[elem] for bandNo in bands: dfTemp = processBand(bandNo, dfTemp) return dfTemp
def calculate_landsat_reflectance(src_path, src_mtl, dst_path, creation_options, band, dst_dtype, processes): """ Parameters ------------ src_path: string src_mtl: string Returns --------- out: None Output is written to dst_path """ mtl = toa_utils._load_mtl(src_mtl) M = toa_utils._load_mtl_key(mtl, [ 'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_' ], band) A = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'], band) E = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES', 'SUN_ELEVATION']) dst_dtype = np.__dict__[dst_dtype] with rio.open(src_path) as src: dst_profile = src.profile.copy() src_nodata = src.nodata for co in creation_options: dst_profile[co] = creation_options[co] dst_profile['dtype'] = dst_dtype global_args = { 'A': A, 'M': M, 'E': E, 'src_nodata': 0, 'dst_dtype': dst_dtype } with riomucho.RioMucho([src_path], dst_path, _reflectance_worker, options=dst_profile, global_args=global_args) as rm: rm.run(processes)
def calculate_landsat_reflectance(src_path, src_mtl, dst_path, creation_options, band, dst_dtype, processes): """ Parameters ------------ src_path: string src_mtl: string Returns --------- out: None Output is written to dst_path """ mtl = toa_utils._load_mtl(src_mtl) M = toa_utils._load_mtl_key(mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'], band) A = toa_utils._load_mtl_key(mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'], band) E = toa_utils._load_mtl_key(mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES','SUN_ELEVATION']) dst_dtype = np.__dict__[dst_dtype] with rio.open(src_path) as src: dst_profile = src.profile.copy() src_nodata = src.nodata for co in creation_options: dst_profile[co] = creation_options[co] dst_profile['dtype'] = dst_dtype global_args = { 'A': A, 'M': M, 'E': E, 'src_nodata': 0, 'dst_dtype': dst_dtype } with riomucho.RioMucho([src_path], dst_path, _reflectance_worker, options=dst_profile, global_args=global_args) as rm: rm.run(processes)
def dn2toa(self, platform, mtl_file=None, wavelengths=None): """This method converts digital numbers to top of atmosphere reflectance, like described here: https://www.usgs.gov/land-resources/nli/landsat/using-usgs-landsat-level-1-data-product :param platform: image platform, possible Platform.Landsat[5, 7, 8] or Platform.Sentinel2 (<enum 'Platform'>). :param mtl_file: path to Landsat MTL file that holds the band specific rescale factors (str). :param wavelengths: like ["Blue", "Green", "Red", "NIR", "SWIR1", "TIRS", "SWIR2"] for Landsat-5 (list of str). """ if platform in [ Platform.Landsat5, Platform.Landsat7, Platform.Landsat8, ]: if mtl_file is None: raise AttributeError( f"'mtl_file' has to be set if platform is {platform}.") else: # get rescale factors from mtl file mtl = toa_utils._load_mtl( str(mtl_file)) # no obvious reason not to call this metadata = mtl["L1_METADATA_FILE"] sun_elevation = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] toa = [] for idx, b in enumerate( self._lookup_bands(platform, wavelengths)): if (platform == Platform.Landsat8 and b in ["10", "11"]) or (platform != Platform.Landsat8 and b.startswith("6")): if platform == Platform.Landsat8: thermal_conversion_constant1 = metadata[ "TIRS_THERMAL_CONSTANTS"][ f"K1_CONSTANT_BAND_{b}"] thermal_conversion_constant2 = metadata[ "TIRS_THERMAL_CONSTANTS"][ f"K2_CONSTANT_BAND_{b}"] else: thermal_conversion_constant1 = metadata[ "THERMAL_CONSTANTS"][f"K1_CONSTANT_BAND_{b}"] thermal_conversion_constant2 = metadata[ "THERMAL_CONSTANTS"][f"K2_CONSTANT_BAND_{b}"] multiplicative_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"RADIANCE_MULT_BAND_{b}"] additive_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"RADIANCE_ADD_BAND_{b}"] # rescale thermal bands toa.append( brightness_temp.brightness_temp( self.__arr[idx, :, :], ML=multiplicative_rescaling_factors, AL=additive_rescaling_factors, K1=thermal_conversion_constant1, K2=thermal_conversion_constant2, )) continue # rescale reflectance bands multiplicative_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"REFLECTANCE_MULT_BAND_{b}"] additive_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{b}"] toa.append( reflectance.reflectance( self.__arr[idx, :, :], MR=multiplicative_rescaling_factors, AR=additive_rescaling_factors, E=sun_elevation, )) self.__arr = np.array(np.stack(toa, axis=0)) elif platform == Platform.Sentinel2: self.__arr = self.__arr.astype(np.float32) / 10000.0 else: raise AttributeError( f"Cannot convert dn2toa. Platform {platform} not supported [Landsat-5, Landsat-7, Landsat-8, " f"Sentinel-2]. ") self.__update_dataset(self.dataset.crs, self.dataset.transform, nodata=self.dataset.nodata)
def test_load_mtl(): src_mtl = 'tests/data/LC80100202015018LGN00_MTL.json' mtl = _load_mtl(src_mtl) assert isinstance(mtl, dict)
cval.append(-1000000) cval.append(1000000) cval.append(-1000000.0) cval.append(1000000.0) dfTemp = pd.DataFrame([cval, cval], columns=ccol) pd.reset_option('mode.chained_assignment') totC = [] step = 50 bin_range = np.arange(0, 65535 + step, step) for i, row in df.iterrows(): image_id = row['displayId'] if os.path.isfile(image_path + image_id + '_MTL.txt'): mtl = toa_utils._load_mtl(image_path + image_id + '_MTL.txt') metadata = mtl['L1_METADATA_FILE'] for fe in fileExt: srcFile = image_path + image_id + fe band = int(fe[2:-4]) if os.path.isfile(srcFile): with rasterio.open(srcFile) as bandImg: band_in = bandImg.read(1) band_in.resize((band_in.shape[0] * band_in.shape[1]), refcheck=False) for scope in range(0, 2, 1): if scope == 1: band_in = band_in[np.where(band_in > 0)] band_in = band_in[np.where(band_in < 0xFFFF)] max = band_in.max()
classifier.summary() # ------------------------------------------------------------------------------------------------------- dst_data = np.empty(rescaler[0] * rescaler[1]) print(time.strftime("%d.%m.%Y %H:%M:%S"), 'Starting data preparation for', os.path.split(model)[-1][:-3]) logFile.write("{} {}\n".format(time.strftime("%d.%m.%Y %H:%M:%S"), 'Starting data preparation')) # ------------------------------------------------------------------------------------------------------- # prepare vector for learning Xinput = np.empty([rescaler[0] * rescaler[1], 17]) for band in range(0, 12, 1): print(time.strftime("%d.%m.%Y %H:%M:%S"), 'Building band: B{}'.format(band)) # metadata mtl = toa_utils._load_mtl(file_path + displayId + '_MTL.txt') metadata = mtl['L1_METADATA_FILE'] src_data[band].resize( (src_data[band].shape[0] * src_data[band].shape[1]), refcheck=False) if (band < 9): MR = metadata['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_MULT_BAND_{}'.format(band + 1)] AR = metadata['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_ADD_BAND_{}'.format(band + 1)] SE = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] # REFLECTANCE TOA Xinput[:, band + 6] = src_data[band] Xinput[:, band + 6] = (MR * Xinput[:, band + 6] + AR) / np.sin( np.deg2rad(SE)) elif band < 11:
def calculate_landsat_brightness_temperature(src_path, src_mtl, dst_path, temp_scale, creation_options, band, dst_dtype, processes): """Parameters ------------ src_path: list list of src_paths(strings) src_mtl: string mtl file path dst_path: string destination file path rescale_factor: float [default] float(55000.0/2**16) rescale post-TOA tifs to 55,000 or to full 16-bit creation_options: dictionary rio.options.creation_options band: list list of integers dst_dtype: strings [default] uint16 destination data dtype Returns --------- out: None Output is written to dst_path """ mtl = toa_utils._load_mtl(src_mtl) M = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_MULT_BAND_'], band) A = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_ADD_BAND_'], band) K1 = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'TIRS_THERMAL_CONSTANTS', 'K1_CONSTANT_BAND_'], band) K2 = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'TIRS_THERMAL_CONSTANTS', 'K2_CONSTANT_BAND_'], band) dst_dtype = np.__dict__[dst_dtype] with rio.open(src_path) as src: dst_profile = src.profile.copy() src_nodata = src.nodata for co in creation_options: dst_profile[co] = creation_options[co] dst_profile['dtype'] = dst_dtype global_args = { 'M': M, 'A': A, 'K1': K1, 'K2': K2, 'src_nodata': 0, 'temp_scale': temp_scale, 'dst_dtype': dst_dtype } with riomucho.RioMucho([src_path], dst_path, _brightness_temp_worker, options=dst_profile, global_args=global_args) as rm: rm.run(processes)
def calculate_landsat_radiance(src_path, src_mtl, dst_path, rescale_factor, creation_options, band, dst_dtype, processes, clip=True): """ Parameters ------------ src_path: strings src_mtl: string dst_path: string rescale_factor: float creation_options: dict bands: list dst_dtype: string processes: integer pixel_sunangle: boolean clip: boolean Returns --------- None Output is written to dst_path """ mtl = toa_utils._load_mtl(src_mtl) M = toa_utils._load_mtl_key(mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_MULT_BAND_'], band) A = toa_utils._load_mtl_key(mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_ADD_BAND_'], band) rescale_factor = toa_utils.normalize_scale(rescale_factor, dst_dtype) dst_dtype = np.__dict__[dst_dtype] with rasterio.open(src_path) as src: dst_profile = src.profile.copy() src_nodata = src.nodata for co in creation_options: dst_profile[co] = creation_options[co] dst_profile['dtype'] = dst_dtype global_args = { 'A': A, 'M': M, 'src_nodata': src_nodata, 'rescale_factor': rescale_factor, 'clip': clip, 'dst_dtype': dst_dtype } with riomucho.RioMucho([src_path], dst_path, _radiance_worker, options=dst_profile, global_args=global_args) as rm: rm.run(processes)
def calculate_landsat_reflectance(src_paths, src_mtl, dst_path, rescale_factor, creation_options, bands, dst_dtype, processes, pixel_sunangle, clip=True): """ Parameters ------------ src_paths: list of strings src_mtl: string dst_path: string rescale_factor: float creation_options: dict bands: list dst_dtype: string processes: integer pixel_sunangle: boolean clip: boolean Returns --------- None Output is written to dst_path """ mtl = toa_utils._load_mtl(src_mtl) metadata = mtl['L1_METADATA_FILE'] M = [metadata['RADIOMETRIC_RESCALING'] ['REFLECTANCE_MULT_BAND_{}'.format(b)] for b in bands] A = [metadata['RADIOMETRIC_RESCALING'] ['REFLECTANCE_ADD_BAND_{}'.format(b)] for b in bands] E = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] date_collected = metadata['PRODUCT_METADATA']['DATE_ACQUIRED'] time_collected_utc = metadata['PRODUCT_METADATA']['SCENE_CENTER_TIME'] rescale_factor = toa_utils.normalize_scale(rescale_factor, dst_dtype) dst_dtype = np.__dict__[dst_dtype] for src_path in src_paths: with rasterio.open(src_path) as src: dst_profile = src.profile.copy() src_nodata = src.nodata for co in creation_options: dst_profile[co] = creation_options[co] dst_profile['dtype'] = dst_dtype global_args = { 'A': A, 'M': M, 'E': E, 'src_nodata': src_nodata, 'src_crs': dst_profile['crs'], 'dst_dtype': dst_dtype, 'rescale_factor': rescale_factor, 'clip': clip, 'pixel_sunangle': pixel_sunangle, 'date_collected': date_collected, 'time_collected_utc': time_collected_utc, 'bands': len(bands) } dst_profile.update(count=len(bands)) if len(bands) == 3: dst_profile.update(photometric='rgb') else: dst_profile.update(photometric='minisblack') with riomucho.RioMucho(list(src_paths), dst_path, _reflectance_worker, options=dst_profile, global_args=global_args, mode='manual_read') as rm: rm.run(processes)
# load image raster = gdal.Open(my_file, gdal.GA_ReadOnly) array = raster.GetRasterBand(1).ReadAsArray() msk_array = np.ma.masked_where(((array == 0) | (array == 65535)), array) #msk_array = np.ma.masked_equal(array, value = 65535) print('Raster Projection:\n', raster.GetProjection()) print('Raster GeoTransform:\n', raster.GetGeoTransform()) mlt = None # find file & load lon lat cooridanes from image for fdir in os.walk(GetImagesPath()): my_txt_file = os.path.join( fdir[0], fname[0] + '_' + fname[1] + '_' + fname[2] + '_' + fname[3] + '_' + fname[4] + '_' + fname[5] + '_' + fname[6] + '_MTL.txt') if os.path.isfile(my_txt_file): mtl = toa_utils._load_mtl(my_txt_file) break metadata = mtl['L1_METADATA_FILE'] CORNER_UL_LON_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UL_LON_PRODUCT'] CORNER_LL_LAT_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_LL_LAT_PRODUCT'] CORNER_UR_LON_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UR_LON_PRODUCT'] CORNER_UR_LAT_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UR_LAT_PRODUCT'] # map coordinates ul_lon = math.floor(CORNER_UL_LON_PRODUCT) - map_margin ll_lat = math.floor(CORNER_LL_LAT_PRODUCT) - map_margin ur_lon = math.ceil(CORNER_UR_LON_PRODUCT) + map_margin ur_lat = math.ceil(CORNER_UR_LAT_PRODUCT) + map_margin print('Map coordinates : {},{},{},{}'.format(ul_lon, ll_lat, ur_lon, ur_lat)) #ul_lon = 0.0 #ll_lat = 40.0