Пример #1
0
def test_load_txt_mtl_2():
    txtmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.txt')
    jsonmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.json')

    for k in jsonmtl['L1_METADATA_FILE'].keys():
        assert k in txtmtl['L1_METADATA_FILE']
        assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
Пример #2
0
def test_load_txt_mtl_1():
	txtmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.txt')
	jsonmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.json')

	for k in jsonmtl['L1_METADATA_FILE'].keys():
		assert k in txtmtl['L1_METADATA_FILE']
		assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
Пример #3
0
def test_load_txt_mtl_1():
    txtmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.txt')
    jsonmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.json')

    for k in jsonmtl['L1_METADATA_FILE'].keys():
        assert k in txtmtl['L1_METADATA_FILE']
        assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
Пример #4
0
def test_load_txt_mtl_2():
	txtmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.txt')
	jsonmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.json')

	for k in jsonmtl['L1_METADATA_FILE'].keys():
		assert k in txtmtl['L1_METADATA_FILE']
		assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
Пример #5
0
def test_data():
    mtl1 = toa_utils._load_mtl('tests/data/LC81060712016134LGN00_MTL.json')
    mtl2 = toa_utils._load_mtl('tests/data/LC80430302016140LGN00_MTL.json')
    mtl3 = toa_utils._load_mtl('tests/data/LC82290902015304LGN00_MTL.json')
    mtl4 = toa_utils._load_mtl('tests/data/LC80100202015018LGN00_MTL.json')

    return mtl1, mtl2, mtl3, mtl4
def processBandSet(i, row, dfTemp):
    mtl = toa_utils._load_mtl(image_path + dfTemp['image_id'][0] + '_MTL.txt')
    metadata = mtl['L1_METADATA_FILE']
    for b in range(1, 10, 1):
        dfTemp['REFLECTANCE_MULT_BAND_{}'.format(b)] = metadata[
            'RADIOMETRIC_RESCALING']['REFLECTANCE_MULT_BAND_{}'.format(b)]
        dfTemp['REFLECTANCE_ADD_BAND_{}'.format(b)] = metadata[
            'RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_{}'.format(b)]
    for b in range(10, 12, 1):
        dfTemp['K1_CONSTANT_BAND_{}'.format(b)] = metadata[
            'TIRS_THERMAL_CONSTANTS']['K1_CONSTANT_BAND_{}'.format(b)]
        dfTemp['K2_CONSTANT_BAND_{}'.format(b)] = metadata[
            'TIRS_THERMAL_CONSTANTS']['K2_CONSTANT_BAND_{}'.format(b)]
        dfTemp['RADIANCE_MULT_BAND_{}'.format(b)] = metadata[
            'RADIOMETRIC_RESCALING']['RADIANCE_MULT_BAND_{}'.format(b)]
        dfTemp['RADIANCE_ADD_BAND_{}'.format(b)] = metadata[
            'RADIOMETRIC_RESCALING']['RADIANCE_ADD_BAND_{}'.format(b)]
    dfTemp['SUN_ELEVATION'] = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
    dfTemp['DATE_ACQUIRED'] = date_collected = metadata['PRODUCT_METADATA'][
        'DATE_ACQUIRED']
    dfTemp['SCENE_CENTER_TIME'] = metadata['PRODUCT_METADATA'][
        'SCENE_CENTER_TIME']

    for elem in pollutant:
        dfTemp[elem] = row[elem]

    for bandNo in bands:
        dfTemp = processBand(bandNo, dfTemp)
    return dfTemp
Пример #7
0
def calculate_landsat_reflectance(src_path, src_mtl, dst_path,
                                  creation_options, band, dst_dtype,
                                  processes):
    """
    Parameters
    ------------
    src_path: string
    src_mtl: string

    Returns
    ---------
    out: None
        Output is written to dst_path
    """
    mtl = toa_utils._load_mtl(src_mtl)

    M = toa_utils._load_mtl_key(mtl, [
        'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'
    ], band)
    A = toa_utils._load_mtl_key(
        mtl,
        ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'],
        band)
    E = toa_utils._load_mtl_key(
        mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES', 'SUN_ELEVATION'])

    dst_dtype = np.__dict__[dst_dtype]

    with rio.open(src_path) as src:
        dst_profile = src.profile.copy()

        src_nodata = src.nodata

        for co in creation_options:
            dst_profile[co] = creation_options[co]

        dst_profile['dtype'] = dst_dtype

    global_args = {
        'A': A,
        'M': M,
        'E': E,
        'src_nodata': 0,
        'dst_dtype': dst_dtype
    }

    with riomucho.RioMucho([src_path],
                           dst_path,
                           _reflectance_worker,
                           options=dst_profile,
                           global_args=global_args) as rm:

        rm.run(processes)
Пример #8
0
def calculate_landsat_reflectance(src_path, src_mtl, dst_path, creation_options, band, dst_dtype, processes):
    """
    Parameters
    ------------
    src_path: string
    src_mtl: string

    Returns
    ---------
    out: None
        Output is written to dst_path
    """
    mtl = toa_utils._load_mtl(src_mtl)

    M = toa_utils._load_mtl_key(mtl,
        ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'],
        band)
    A = toa_utils._load_mtl_key(mtl,
        ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'],
        band)
    E = toa_utils._load_mtl_key(mtl, 
        ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES','SUN_ELEVATION'])

    dst_dtype = np.__dict__[dst_dtype]

    with rio.open(src_path) as src:
        dst_profile = src.profile.copy()

        src_nodata = src.nodata

        for co in creation_options:
            dst_profile[co] = creation_options[co]

        dst_profile['dtype'] = dst_dtype

    global_args = {
        'A': A,
        'M': M,
        'E': E,
        'src_nodata': 0,
        'dst_dtype': dst_dtype
    }

    with riomucho.RioMucho([src_path], dst_path, _reflectance_worker,
        options=dst_profile,
        global_args=global_args) as rm:

        rm.run(processes)
Пример #9
0
    def dn2toa(self, platform, mtl_file=None, wavelengths=None):
        """This method converts digital numbers to top of atmosphere reflectance, like described here:
        https://www.usgs.gov/land-resources/nli/landsat/using-usgs-landsat-level-1-data-product

        :param platform: image platform, possible Platform.Landsat[5, 7, 8] or Platform.Sentinel2 (<enum 'Platform'>).
        :param mtl_file: path to Landsat MTL file that holds the band specific rescale factors (str).
        :param wavelengths: like ["Blue", "Green", "Red", "NIR", "SWIR1", "TIRS", "SWIR2"] for Landsat-5 (list of str).
        """
        if platform in [
                Platform.Landsat5,
                Platform.Landsat7,
                Platform.Landsat8,
        ]:
            if mtl_file is None:
                raise AttributeError(
                    f"'mtl_file' has to be set if platform is {platform}.")
            else:
                # get rescale factors from mtl file
                mtl = toa_utils._load_mtl(
                    str(mtl_file))  # no obvious reason not to call this
                metadata = mtl["L1_METADATA_FILE"]
                sun_elevation = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
                toa = []

                for idx, b in enumerate(
                        self._lookup_bands(platform, wavelengths)):
                    if (platform == Platform.Landsat8 and b
                            in ["10", "11"]) or (platform != Platform.Landsat8
                                                 and b.startswith("6")):
                        if platform == Platform.Landsat8:
                            thermal_conversion_constant1 = metadata[
                                "TIRS_THERMAL_CONSTANTS"][
                                    f"K1_CONSTANT_BAND_{b}"]
                            thermal_conversion_constant2 = metadata[
                                "TIRS_THERMAL_CONSTANTS"][
                                    f"K2_CONSTANT_BAND_{b}"]
                        else:
                            thermal_conversion_constant1 = metadata[
                                "THERMAL_CONSTANTS"][f"K1_CONSTANT_BAND_{b}"]
                            thermal_conversion_constant2 = metadata[
                                "THERMAL_CONSTANTS"][f"K2_CONSTANT_BAND_{b}"]
                        multiplicative_rescaling_factors = metadata[
                            "RADIOMETRIC_RESCALING"][f"RADIANCE_MULT_BAND_{b}"]
                        additive_rescaling_factors = metadata[
                            "RADIOMETRIC_RESCALING"][f"RADIANCE_ADD_BAND_{b}"]

                        # rescale thermal bands
                        toa.append(
                            brightness_temp.brightness_temp(
                                self.__arr[idx, :, :],
                                ML=multiplicative_rescaling_factors,
                                AL=additive_rescaling_factors,
                                K1=thermal_conversion_constant1,
                                K2=thermal_conversion_constant2,
                            ))
                        continue

                    # rescale reflectance bands
                    multiplicative_rescaling_factors = metadata[
                        "RADIOMETRIC_RESCALING"][f"REFLECTANCE_MULT_BAND_{b}"]
                    additive_rescaling_factors = metadata[
                        "RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{b}"]
                    toa.append(
                        reflectance.reflectance(
                            self.__arr[idx, :, :],
                            MR=multiplicative_rescaling_factors,
                            AR=additive_rescaling_factors,
                            E=sun_elevation,
                        ))

                self.__arr = np.array(np.stack(toa, axis=0))
        elif platform == Platform.Sentinel2:
            self.__arr = self.__arr.astype(np.float32) / 10000.0
        else:
            raise AttributeError(
                f"Cannot convert dn2toa. Platform {platform} not supported [Landsat-5, Landsat-7, Landsat-8, "
                f"Sentinel-2]. ")

        self.__update_dataset(self.dataset.crs,
                              self.dataset.transform,
                              nodata=self.dataset.nodata)
Пример #10
0
def test_load_mtl():
    src_mtl = 'tests/data/LC80100202015018LGN00_MTL.json'
    mtl = _load_mtl(src_mtl)
    assert isinstance(mtl, dict)
    cval.append(-1000000)
    cval.append(1000000)
    cval.append(-1000000.0)
    cval.append(1000000.0)

dfTemp = pd.DataFrame([cval, cval], columns=ccol)
pd.reset_option('mode.chained_assignment')

totC = []
step = 50
bin_range = np.arange(0, 65535 + step, step)

for i, row in df.iterrows():
    image_id = row['displayId']
    if os.path.isfile(image_path + image_id + '_MTL.txt'):
        mtl = toa_utils._load_mtl(image_path + image_id + '_MTL.txt')
        metadata = mtl['L1_METADATA_FILE']

        for fe in fileExt:
            srcFile = image_path + image_id + fe
            band = int(fe[2:-4])
            if os.path.isfile(srcFile):
                with rasterio.open(srcFile) as bandImg:
                    band_in = bandImg.read(1)
                    band_in.resize((band_in.shape[0] * band_in.shape[1]),
                                   refcheck=False)
                    for scope in range(0, 2, 1):
                        if scope == 1:
                            band_in = band_in[np.where(band_in > 0)]
                            band_in = band_in[np.where(band_in < 0xFFFF)]
                        max = band_in.max()
    classifier.summary()

    # -------------------------------------------------------------------------------------------------------
    dst_data = np.empty(rescaler[0] * rescaler[1])
    print(time.strftime("%d.%m.%Y %H:%M:%S"), 'Starting data preparation for',
          os.path.split(model)[-1][:-3])
    logFile.write("{} {}\n".format(time.strftime("%d.%m.%Y %H:%M:%S"),
                                   'Starting data preparation'))
    # -------------------------------------------------------------------------------------------------------
    # prepare vector for learning
    Xinput = np.empty([rescaler[0] * rescaler[1], 17])
    for band in range(0, 12, 1):
        print(time.strftime("%d.%m.%Y %H:%M:%S"),
              'Building band: B{}'.format(band))
        # metadata
        mtl = toa_utils._load_mtl(file_path + displayId + '_MTL.txt')
        metadata = mtl['L1_METADATA_FILE']
        src_data[band].resize(
            (src_data[band].shape[0] * src_data[band].shape[1]),
            refcheck=False)
        if (band < 9):
            MR = metadata['RADIOMETRIC_RESCALING'][
                'REFLECTANCE_MULT_BAND_{}'.format(band + 1)]
            AR = metadata['RADIOMETRIC_RESCALING'][
                'REFLECTANCE_ADD_BAND_{}'.format(band + 1)]
            SE = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
            # REFLECTANCE TOA
            Xinput[:, band + 6] = src_data[band]
            Xinput[:, band + 6] = (MR * Xinput[:, band + 6] + AR) / np.sin(
                np.deg2rad(SE))
        elif band < 11:
Пример #13
0
def calculate_landsat_brightness_temperature(src_path, src_mtl, dst_path,
                                             temp_scale, creation_options,
                                             band, dst_dtype, processes):
    """Parameters
    ------------
    src_path: list
              list of src_paths(strings)
    src_mtl: string
             mtl file path
    dst_path: string
              destination file path
    rescale_factor: float [default] float(55000.0/2**16)
                    rescale post-TOA tifs to 55,000 or to full 16-bit
    creation_options: dictionary
                      rio.options.creation_options
    band: list
          list of integers
    dst_dtype: strings [default] uint16
               destination data dtype

    Returns
    ---------
    out: None
        Output is written to dst_path
    """
    mtl = toa_utils._load_mtl(src_mtl)

    M = toa_utils._load_mtl_key(
        mtl,
        ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_MULT_BAND_'],
        band)
    A = toa_utils._load_mtl_key(
        mtl,
        ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_ADD_BAND_'],
        band)

    K1 = toa_utils._load_mtl_key(
        mtl,
        ['L1_METADATA_FILE', 'TIRS_THERMAL_CONSTANTS', 'K1_CONSTANT_BAND_'],
        band)
    K2 = toa_utils._load_mtl_key(
        mtl,
        ['L1_METADATA_FILE', 'TIRS_THERMAL_CONSTANTS', 'K2_CONSTANT_BAND_'],
        band)

    dst_dtype = np.__dict__[dst_dtype]

    with rio.open(src_path) as src:
        dst_profile = src.profile.copy()

        src_nodata = src.nodata

        for co in creation_options:
            dst_profile[co] = creation_options[co]

        dst_profile['dtype'] = dst_dtype

    global_args = {
        'M': M,
        'A': A,
        'K1': K1,
        'K2': K2,
        'src_nodata': 0,
        'temp_scale': temp_scale,
        'dst_dtype': dst_dtype
    }

    with riomucho.RioMucho([src_path],
                           dst_path,
                           _brightness_temp_worker,
                           options=dst_profile,
                           global_args=global_args) as rm:

        rm.run(processes)
Пример #14
0
def calculate_landsat_radiance(src_path, src_mtl, dst_path, rescale_factor,
                               creation_options, band, dst_dtype, processes,
                               clip=True):
    """
    Parameters
    ------------
    src_path: strings
    src_mtl: string
    dst_path: string
    rescale_factor: float
    creation_options: dict
    bands: list
    dst_dtype: string
    processes: integer
    pixel_sunangle: boolean
    clip: boolean

    Returns
    ---------
    None
        Output is written to dst_path
    """
    mtl = toa_utils._load_mtl(src_mtl)

    M = toa_utils._load_mtl_key(mtl,
                                ['L1_METADATA_FILE',
                                 'RADIOMETRIC_RESCALING',
                                 'RADIANCE_MULT_BAND_'],
                                band)
    A = toa_utils._load_mtl_key(mtl,
                                ['L1_METADATA_FILE',
                                 'RADIOMETRIC_RESCALING',
                                 'RADIANCE_ADD_BAND_'],
                                band)

    rescale_factor = toa_utils.normalize_scale(rescale_factor, dst_dtype)

    dst_dtype = np.__dict__[dst_dtype]

    with rasterio.open(src_path) as src:
        dst_profile = src.profile.copy()

        src_nodata = src.nodata

        for co in creation_options:
            dst_profile[co] = creation_options[co]

        dst_profile['dtype'] = dst_dtype

    global_args = {
        'A': A,
        'M': M,
        'src_nodata': src_nodata,
        'rescale_factor': rescale_factor,
        'clip': clip,
        'dst_dtype': dst_dtype
        }

    with riomucho.RioMucho([src_path],
                           dst_path,
                           _radiance_worker,
                           options=dst_profile,
                           global_args=global_args) as rm:

        rm.run(processes)
Пример #15
0
def calculate_landsat_reflectance(src_paths, src_mtl, dst_path, rescale_factor,
                                  creation_options, bands, dst_dtype,
                                  processes, pixel_sunangle, clip=True):
    """
    Parameters
    ------------
    src_paths: list of strings
    src_mtl: string
    dst_path: string
    rescale_factor: float
    creation_options: dict
    bands: list
    dst_dtype: string
    processes: integer
    pixel_sunangle: boolean
    clip: boolean

    Returns
    ---------
    None
        Output is written to dst_path
    """
    mtl = toa_utils._load_mtl(src_mtl)
    metadata = mtl['L1_METADATA_FILE']

    M = [metadata['RADIOMETRIC_RESCALING']
         ['REFLECTANCE_MULT_BAND_{}'.format(b)]
         for b in bands]
    A = [metadata['RADIOMETRIC_RESCALING']
         ['REFLECTANCE_ADD_BAND_{}'.format(b)]
         for b in bands]

    E = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
    date_collected = metadata['PRODUCT_METADATA']['DATE_ACQUIRED']
    time_collected_utc = metadata['PRODUCT_METADATA']['SCENE_CENTER_TIME']

    rescale_factor = toa_utils.normalize_scale(rescale_factor, dst_dtype)

    dst_dtype = np.__dict__[dst_dtype]

    for src_path in src_paths:
        with rasterio.open(src_path) as src:
            dst_profile = src.profile.copy()
            src_nodata = src.nodata

            for co in creation_options:
                dst_profile[co] = creation_options[co]

            dst_profile['dtype'] = dst_dtype

    global_args = {
        'A': A,
        'M': M,
        'E': E,
        'src_nodata': src_nodata,
        'src_crs': dst_profile['crs'],
        'dst_dtype': dst_dtype,
        'rescale_factor': rescale_factor,
        'clip': clip,
        'pixel_sunangle': pixel_sunangle,
        'date_collected': date_collected,
        'time_collected_utc': time_collected_utc,
        'bands': len(bands)
    }

    dst_profile.update(count=len(bands))

    if len(bands) == 3:
        dst_profile.update(photometric='rgb')
    else:
        dst_profile.update(photometric='minisblack')

    with riomucho.RioMucho(list(src_paths),
                           dst_path,
                           _reflectance_worker,
                           options=dst_profile,
                           global_args=global_args,
                           mode='manual_read') as rm:

        rm.run(processes)
Пример #16
0
def test_load_mtl():
	src_mtl = 'tests/data/LC80100202015018LGN00_MTL.json'
	mtl = _load_mtl(src_mtl)
	assert isinstance(mtl, dict)
# load image
raster = gdal.Open(my_file, gdal.GA_ReadOnly)
array = raster.GetRasterBand(1).ReadAsArray()
msk_array = np.ma.masked_where(((array == 0) | (array == 65535)), array)
#msk_array = np.ma.masked_equal(array, value = 65535)
print('Raster Projection:\n', raster.GetProjection())
print('Raster GeoTransform:\n', raster.GetGeoTransform())

mlt = None
# find file & load lon lat cooridanes from image
for fdir in os.walk(GetImagesPath()):
    my_txt_file = os.path.join(
        fdir[0], fname[0] + '_' + fname[1] + '_' + fname[2] + '_' + fname[3] +
        '_' + fname[4] + '_' + fname[5] + '_' + fname[6] + '_MTL.txt')
    if os.path.isfile(my_txt_file):
        mtl = toa_utils._load_mtl(my_txt_file)
        break
metadata = mtl['L1_METADATA_FILE']
CORNER_UL_LON_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UL_LON_PRODUCT']
CORNER_LL_LAT_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_LL_LAT_PRODUCT']
CORNER_UR_LON_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UR_LON_PRODUCT']
CORNER_UR_LAT_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UR_LAT_PRODUCT']
# map coordinates
ul_lon = math.floor(CORNER_UL_LON_PRODUCT) - map_margin
ll_lat = math.floor(CORNER_LL_LAT_PRODUCT) - map_margin
ur_lon = math.ceil(CORNER_UR_LON_PRODUCT) + map_margin
ur_lat = math.ceil(CORNER_UR_LAT_PRODUCT) + map_margin
print('Map coordinates : {},{},{},{}'.format(ul_lon, ll_lat, ur_lon, ur_lat))

#ul_lon = 0.0
#ll_lat = 40.0