def test_config_set_res(self):
with gw.config.update(ref_res=100):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(src.gw.celly, 100)
with gw.open(l8_224078_20200518) as src:
self.assertEqual(src.gw.celly, 30)
def test_with_config(self):
with gw.config.update():
with gw.open(l8_224078_20200518) as src:
self.assertTrue(src.gw.config['with_config'])
with gw.open(l8_224078_20200518) as src:
self.assertFalse(src.gw.config['with_config'])
def test_single_image_single_band(self):
with gw.open(l8_224078_20200518_B2) as src:
df = gw.extract(src, aoi, band_names=['blue'])
self.assertTrue(
np.allclose(df.blue.values, l8_224078_20200518_B2_values))
def test_config_bands_set_none(self):
with gw.config.update(sensor=None):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(','.join(map(str, src.band.values.tolist())),
'1,2,3')
def test_config_bands_set(self):
with gw.config.update(sensor='bgr'):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(','.join(src.band.values.tolist()),
'blue,green,red')
def test_config_bands_set_override(self):
with gw.config.update(sensor='bgr'):
with gw.open(l8_224078_20200518, band_names=['b1', 'b2',
'b3']) as src:
self.assertEqual(','.join(src.band.values.tolist()),
'b1,b2,b3')
def test_transform(self):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(
'epsg:4326',
src.gw.transform_crs(dst_crs=4326,
dst_width=src.gw.ncols,
dst_height=src.gw.nrows,
coords_only=True).crs.to_string().lower())
def test_single_image_multi_band(self):
with gw.open(l8_224078_20200518) as src:
df = gw.extract(src, aoi, band_names=['blue', 'green', 'red'])
self.assertTrue(
np.allclose(df.blue.values, l8_224078_20200518_values[0, :]))
self.assertTrue(
np.allclose(df.green.values, l8_224078_20200518_values[1, :]))
self.assertTrue(
np.allclose(df.red.values, l8_224078_20200518_values[2, :]))
def test_warnings_ignore(self):
with LogCapture() as log:
with gw.config.update(sensor='l7', ignore_warnings=True):
with gw.open(l8_224078_20200518) as src:
pass
self.assertNotIn((
'geowombat.backends.xarray_', 'WARNING',
" The new bands, ['blue', 'green', 'red', 'nir', 'swir1', 'swir2'], do not match the sensor bands, [1, 2, 3]."
), log)
def test_config_defaults(self):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(src.gw.config['scale_factor'], 1)
for config_default in ['with_config', 'ignore_warnings']:
self.assertFalse(src.gw.config[config_default])
for config_default in [
'sensor', 'nodata', 'ref_image', 'ref_bounds', 'ref_crs',
'ref_res', 'ref_tar', 'compress'
]:
self.assertIsNone(src.gw.config[config_default])
def coregister(target, reference, **kwargs):
"""
Co-registers an image, or images, using AROSICS.
While the required inputs are DataArrays, the intermediate results are stored as NumPy arrays.
Therefore, memory usage is constrained to the size of the input data. Dask is not used for any of the
computation in this function.
Args:
target (DataArray or str): The target ``xarray.DataArray`` or file name to co-register to ``reference``.
reference (DataArray or str): The reference ``xarray.DataArray`` or file name used to co-register ``target``.
kwargs (Optional[dict]): Keyword arguments passed to ``arosics``.
Reference:
https://pypi.org/project/arosics
Returns:
``xarray.DataArray``
Example:
>>> import geowombat as gw
>>>
>>> # Co-register a single image to a reference image
>>> with gw.open('target.tif') as tar, gw.open('reference.tif') as ref:
>>> results = gw.coregister(tar, ref, q=True, ws=(512, 512), max_shift=3, CPUs=4)
>>>
>>> # or
>>>
>>> results = gw.coregister('target.tif', 'reference.tif', q=True, ws=(512, 512), max_shift=3, CPUs=4)
"""
import geowombat as gw_
if not AROSICS_INSTALLED:
logger.exception(
'\nAROSICS must be installed to co-register data.\nSee https://pypi.org/project/arosics for details'
)
raise NameError
if isinstance(reference, str):
if not os.path.isfile(reference):
logger.exception(' The reference file does not exist.')
raise OSError
with gw_.open(reference) as reference:
pass
if isinstance(target, str):
if not os.path.isfile(target):
logger.exception(' The target file does not exist.')
raise OSError
with gw_.open(target) as target:
pass
cr = arosics.COREG(reference.filename, target.filename, **kwargs)
try:
cr.calculate_spatial_shifts()
except:
logger.warning(' Could not co-register the data.')
return target
shift_info = cr.correct_shifts()
left = shift_info['updated geotransform'][0]
top = shift_info['updated geotransform'][3]
transform = (target.gw.cellx, 0.0, left, 0.0, -target.gw.celly, top)
target.attrs['transform'] = transform
data = shift_info['arr_shifted'].transpose(2, 0, 1)
ycoords = np.linspace(
top - target.gw.cellyh,
top - target.gw.cellyh - (data.shape[1] * target.gw.celly),
data.shape[1])
xcoords = np.linspace(
left + target.gw.cellxh,
left + target.gw.cellxh + (data.shape[2] * target.gw.cellx),
data.shape[2])
return xr.DataArray(data=da.from_array(data,
chunks=(target.gw.band_chunks,
target.gw.row_chunks,
target.gw.col_chunks)),
dims=('band', 'y', 'x'),
coords={
'band': target.band.values.tolist(),
'y': ycoords,
'x': xcoords
},
attrs=target.attrs)
def test_has_band(self):
with gw.open(l8_224078_20200518) as src:
self.assertTrue(src.gw.has_band)
def test_height(self):
with gw.open(l8_224078_20200518) as src, rio.open(
l8_224078_20200518) as rsrc:
self.assertEqual(src.gw.ncols, rsrc.width)
def download_cube(self,
sensors,
date_range,
bounds,
bands,
crs=None,
out_bounds=None,
outdir='.',
ref_res=None,
l57_angles_path=None,
l8_angles_path=None,
**kwargs):
"""
Downloads a cube of Landsat and/or Sentinel 2 imagery
Args:
sensors (str or list): The sensors, or sensor, to download.
date_range (list): The date range, given as [date1, date2], where the date format is yyyy-mm-dd.
bounds (GeoDataFrame, list, or tuple): The geometry bounds (in WGS84 lat/lon) that define the cube extent
to download. If given as a ``GeoDataFrame``, only the first ``DataFrame`` record will be used.
If given as a ``tuple`` or a ``list``, the order should be (left, bottom, right, top).
bands (str or list): The bands to download.
crs (Optional[str or object]): The output CRS. If ``bounds`` is a ``GeoDataFrame``, the CRS is taken
from the object.
out_bounds (Optional[list or tuple]): The output bounds in ``crs``. If not given, the bounds are
taken from ``bounds``.
outdir (Optional[str]): The output directory.
ref_res (Optional[tuple]): A reference cell resolution.
l57_angles_path (str): The path to the Landsat 5 and 7 angles bin.
l8_angles_path (str): The path to the Landsat 8 angles bin.
kwargs (Optional[dict]): Keyword arguments passed to ``to_raster``.
Examples:
>>> from geowombat.util import GeoDownloads
>>> gdl = GeoDownloads()
>>>
>>> # Download a Landsat 7 panchromatic cube
>>> gdl.download_cube(['l7'],
>>> ['2010-01-01', '2010-02-01'],
>>> (-91.57, 40.37, -91.46, 40.42),
>>> ['pan'],
>>> crs="+proj=aea +lat_1=-5 +lat_2=-42 +lat_0=-32 +lon_0=-60 +x_0=0 +y_0=0 +ellps=aust_SA +units=m +no_defs")
>>>
>>> # Download a Landsat 7, 8 and Sentinel 2 cube of the visible spectrum
>>> gdl.download_cube(['l7', 'l8', 's2'],
>>> ['2017-01-01', '2018-01-01'],
>>> (-91.57, 40.37, -91.46, 40.42),
>>> ['blue', 'green', 'red'],
>>> crs={'init': 'epsg:102033'},
>>> readxsize=1024,
>>> readysize=1024,
>>> n_workers=1,
>>> n_threads=8)
"""
# TODO: parameterize
# kwargs = dict(readxsize=1024,
# readysize=1024,
# verbose=1,
# separate=False,
# n_workers=1,
# n_threads=8,
# n_chunks=100,
# overwrite=False)
angle_infos = dict()
rt = RadTransforms()
br = BRDF()
la = LinearAdjustments()
main_path = Path(outdir)
outdir_brdf = main_path.joinpath('brdf')
if not main_path.is_dir():
main_path.mkdir()
if not outdir_brdf.is_dir():
outdir_brdf.mkdir()
if isinstance(sensors, str):
sensors = [sensors]
status = Path(outdir).joinpath('status.txt')
if not status.is_file():
with open(status.as_posix(), mode='w') as tx:
pass
# Get bounds from geometry
if isinstance(bounds, tuple) or isinstance(bounds, list):
bounds = Polygon([
(bounds[0], bounds[3]), # upper left
(bounds[2], bounds[3]), # upper right
(bounds[2], bounds[1]), # lower right
(bounds[0], bounds[1]), # lower left
(bounds[0], bounds[3])
]) # upper left
bounds = gpd.GeoDataFrame([0],
geometry=[bounds],
crs={'init': 'epsg:4326'})
bounds_object = bounds.geometry.values[0]
if not out_bounds:
# Project the bounds
out_bounds = bounds.to_crs(crs).bounds.values[0].tolist()
# Get WRS file
data_bin = os.path.realpath(os.path.dirname(__file__))
data_dir = Path(data_bin).joinpath('../data')
shp_dict = dict()
if ('l7' in sensors) or ('l8' in sensors):
path_tar = Path(data_dir).joinpath('wrs2.tar.gz')
path_shp = Path(data_dir).joinpath('wrs2_descending.shp')
wrs = os.path.realpath(path_shp.as_posix())
if not path_shp.is_file():
with tarfile.open(os.path.realpath(path_tar.as_posix()),
mode='r:gz') as tf:
tf.extractall(data_dir.as_posix())
df_wrs = gpd.read_file(wrs)
df_wrs = df_wrs[df_wrs.geometry.intersects(bounds_object)]
if df_wrs.empty:
logger.warning(' The geometry bounds is empty.')
return
shp_dict['wrs'] = df_wrs
if 's2' in sensors:
path_tar = Path(data_dir).joinpath('mgrs.tar.gz')
path_shp = Path(data_dir).joinpath('sentinel2_grid.shp')
mgrs = os.path.realpath(path_shp.as_posix())
if not path_shp.is_file():
with tarfile.open(os.path.realpath(path_tar.as_posix()),
mode='r:gz') as tf:
tf.extractall(data_dir.as_posix())
df_mgrs = gpd.read_file(mgrs)
df_mgrs = df_mgrs[df_mgrs.geometry.intersects(bounds_object)]
if df_mgrs.empty:
logger.warning(' The geometry bounds is empty.')
return
shp_dict['mgrs'] = df_mgrs
dt1 = datetime.strptime(date_range[0], '%Y-%m-%d')
dt2 = datetime.strptime(date_range[1], '%Y-%m-%d')
year = dt1.year
month = dt1.month
months = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
if month < dt2.month:
month_range = months[months.index(month):months.index(dt2.month) +
1]
else:
month_range = months[
months.index(month):] + months[:months.index(dt2.month) + 1]
while True:
if year > dt2.year:
break
for m in month_range:
yearmonth_query = '{:d}{:02d}'.format(year, m)
for sensor in sensors:
band_associations = self.associations[sensor]
# TODO: get path/row and MGRS from geometry
# location = '21/H/UD' # or '225/083'
if sensor.lower() == 's2':
locations = [
'{}/{}/{}'.format(dfrow.Name[:2], dfrow.Name[2],
dfrow.Name[3:])
for dfi, dfrow in shp_dict['mgrs'].iterrows()
]
else:
locations = [
'{:03d}/{:03d}'.format(int(dfrow.PATH),
int(dfrow.ROW))
for dfi, dfrow in shp_dict['wrs'].iterrows()
]
for location in locations:
if sensor.lower() == 's2':
query = '{LOCATION}/*{YM}*.SAFE/GRANULE/*'.format(
LOCATION=location, YM=yearmonth_query)
else:
query = '{LOCATION}/*{PATHROW}_{YM}*_T*'.format(
LOCATION=location,
PATHROW=location.replace('/', ''),
YM=yearmonth_query)
# Query and list available files on the GCP
self.list_gcp(sensor, query)
if not self.search_dict:
logger.warning(
' No results found for {SENSOR} at location {LOC}, year {YEAR:d}, month {MONTH:d}.'
.format(SENSOR=sensor,
LOC=location,
YEAR=year,
MONTH=m))
continue
# Download data
if sensor.lower() == 's2':
load_bands = sorted([
'B{:02d}'.format(band_associations[bd])
if bd != 'rededge' else 'B{:02d}A'.format(
band_associations[bd]) for bd in bands
])
search_wildcards = ['MTD_TL.xml'] + [
bd + '.jp2' for bd in load_bands
]
file_info = self.download_gcp(
sensor,
outdir=outdir,
search_wildcards=search_wildcards,
check_file=status.as_posix(),
verbose=1)
# Reorganize the dictionary to combine bands and metadata
new_dict_ = dict()
for finfo_key, finfo_dict in file_info.items():
sub_dict_ = dict()
if 'meta' in finfo_dict:
key = finfo_dict['meta'].name
sub_dict_['meta'] = finfo_dict['meta']
for finfo_key_, finfo_dict_ in file_info.items(
):
if 'meta' not in finfo_dict_:
for bdkey_, bdinfo_ in finfo_dict_.items(
):
if '_'.join(
bdinfo_.name.split('_')
[:-1]) in key:
sub_dict_[bdkey_] = bdinfo_
new_dict_[finfo_key] = sub_dict_
file_info = new_dict_
else:
del_keys = [
k for k, v in self.search_dict.items()
if 'gap_mask' in k
]
for dk in del_keys:
del self.search_dict[dk]
load_bands = sorted([
'B{:d}'.format(band_associations[bd])
for bd in bands
])
search_wildcards = [
'ANG.txt', 'MTL.txt', 'BQA.TIF'
] + [bd + '.TIF' for bd in load_bands]
file_info = self.download_gcp(
sensor,
outdir=outdir,
search_wildcards=search_wildcards,
check_file=status.as_posix(),
verbose=1)
logger.info(' Finished downloading files')
# Create pixel angle files
# TODO: this can be run in parallel
for finfo_key, finfo_dict in file_info.items():
brdfp = '_'.join(
Path(finfo_dict['meta'].name).name.split('_')
[:-1])
logger.info(' Processing {} ...'.format(brdfp))
out_brdf = outdir_brdf.joinpath(brdfp +
'.tif').as_posix()
if os.path.isfile(out_brdf):
logger.warning(
' The output BRDF file, {}, already exists.'
.format(brdfp))
continue
with open(status.as_posix(), mode='r') as tx:
lines = tx.readlines()
if sensor == 's2':
outdir_angles = main_path.joinpath(
'angles_{}'.format(
Path(finfo_dict['meta'].name).name.
replace('_MTD_TL.xml', '')))
else:
outdir_angles = main_path.joinpath(
'angles_{}'.format(
Path(finfo_dict['meta'].name).name.
replace('_MTL.txt', '')))
outdir_angles.mkdir(parents=True, exist_ok=True)
ref_file = finfo_dict[load_bands[0]].name
if sensor.lower() == 's2':
angle_info = sentinel_pixel_angles(
finfo_dict['meta'].name,
ref_file,
outdir_angles.as_posix(),
nodata=-32768,
overwrite=False,
verbose=1)
if ' '.join(
bands
) == 'blue green red nir1 nir2 nir3 nir rededge swir1 swir2':
rad_sensor = 's2'
elif ' '.join(
bands
) == 'blue green red nir swir1 swir2':
rad_sensor = 's2l7'
elif ' '.join(bands) == 'blue green red nir':
rad_sensor = 's210'
else:
rad_sensor = 's2'
bandpass_sensor = angle_info.sensor
else:
meta = rt.get_landsat_coefficients(
finfo_dict['meta'].name)
angle_info = landsat_pixel_angles(
finfo_dict['angle'].name,
ref_file,
outdir_angles.as_posix(),
meta.sensor,
l57_angles_path=l57_angles_path,
l8_angles_path=l8_angles_path,
verbose=1)
if (len(bands) == 1) and (bands[0] == 'pan'):
rad_sensor = sensor + bands[0]
else:
if (len(bands) == 6) and (meta.sensor
== 'l8'):
rad_sensor = 'l8l7'
elif (len(bands)
== 7) and (meta.sensor
== 'l8') and ('pan'
in bands):
rad_sensor = 'l8l7mspan'
elif (len(bands)
== 7) and (meta.sensor
== 'l7') and ('pan'
in bands):
rad_sensor = 'l7mspan'
else:
rad_sensor = meta.sensor
bandpass_sensor = sensor
# Get band names from user
load_bands_names = [
finfo_dict[bd].name for bd in load_bands
]
with gw.config.update(sensor=rad_sensor,
ref_bounds=out_bounds,
ref_crs=crs,
ref_res=ref_res if ref_res
else load_bands_names[-1]):
with gw.open(angle_info.sza,
resampling='cubic') as sza, \
gw.open(angle_info.vza,
resampling='cubic') as vza, \
gw.open(angle_info.saa,
resampling='cubic') as saa, \
gw.open(angle_info.vaa,
resampling='cubic') as vaa:
with gw.open(
load_bands_names,
band_names=bands,
stack_dim='band',
resampling='cubic') as data: #, \
# gw.open(finfo_dict['qa'].name,
# band_names=['qa']) as qa:
# Setup the mask
# if sensor.lower() != 's2':
#
# if sensor.lower() == 'l8':
# qa_sensor = 'l8-c1'
# else:
# qa_sensor = 'l-c1'
# mask = QAMasker(qa,
# qa_sensor,
# mask_items=['clear',
# 'fill',
# 'shadow',
# 'cloudconf',
# 'cirrusconf',
# 'snowiceconf'],
# confidence_level='maybe').to_mask()
if sensor.lower() == 's2':
# The S-2 data are in TOAR (0-10000)
toar_scaled = (data * 0.0001).clip(
0, 1).astype('float64')
toar_scaled.attrs = data.attrs.copy(
)
# Convert TOAR to surface reflectance
sr = rt.toar_to_sr(
toar_scaled, sza, saa, vza,
vaa, sensor)
else:
# Convert DN to surface reflectance
sr = rt.dn_to_sr(data,
sza,
saa,
vza,
vaa,
sensor=rad_sensor,
meta=meta)
# BRDF normalization
sr_brdf = br.norm_brdf(
sr,
sza,
saa,
vza,
vaa,
sensor=rad_sensor,
wavelengths=data.band.values.
tolist(),
out_range=10000.0,
nodata=65535)
if bandpass_sensor.lower() in [
'l5', 'l7', 's2a', 's2b'
]:
# Linear adjust to Landsat 8
sr_brdf = la.bandpass(
sr_brdf,
bandpass_sensor.lower(),
to='l8',
scale_factor=0.0001)
# Mask non-clear pixels
# attrs = sr_brdf.attrs
# sr_brdf = xr.where(mask.sel(band='mask') < 2, sr_brdf, 65535)
# sr_brdf = sr_brdf.transpose('band', 'y', 'x')
# sr_brdf.attrs = attrs
attrs = sr_brdf.attrs.copy()
sr_brdf = sr_brdf.clip(
0, 10000).astype('uint16')
sr_brdf.attrs = attrs.copy()
sr_brdf.gw.to_raster(
out_brdf, **kwargs)
angle_infos[finfo_key] = angle_info
shutil.rmtree(outdir_angles.as_posix())
for k, v in finfo_dict.items():
os.remove(v.name)
lines.append(finfo_dict['meta'].name + '\n')
with open(status.as_posix(), mode='r+') as tx:
tx.writelines(lines)
year += 1
def test_has_no_band_coord(self):
with gw.open(l8_224078_20200518) as src:
self.assertFalse(src.drop_vars('band').gw.has_band_coord)
def test_row_chunks(self):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(src.gw.row_chunks, 256)
def test_row_chunks_set(self):
with gw.open(l8_224078_20200518, chunks=64) as src:
self.assertEqual(src.gw.row_chunks, 64)
def test_open_multiple(self):
with gw.open([l8_224078_20200518, l8_224078_20200518],
stack_dim='time') as src:
self.assertEqual(src.gw.ntime, 2)
def test_time_chunks(self):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(src.gw.time_chunks, 1)
def test_width(self):
with gw.open(l8_224078_20200518) as src, rio.open(
l8_224078_20200518) as rsrc:
self.assertEqual(src.gw.nrows, rsrc.height)
def test_dtype(self):
with gw.open(l8_224078_20200518, dtype='float64') as src:
self.assertEqual(src.dtype, 'float64')
def test_open_path(self):
with gw.open(Path(l8_224078_20200518)) as src:
self.assertEqual(src.gw.nbands, 3)
def test_has_no_time_coord(self):
with gw.open([l8_224078_20200518, l8_224078_20200518],
stack_dim='time') as src:
self.assertFalse(src.drop_vars('time').gw.has_time_coord)
def test_has_time(self):
with gw.open([l8_224078_20200518, l8_224078_20200518],
stack_dim='time') as src:
self.assertTrue(src.gw.has_time)
def test_open_type_xarray(self):
with gw.open(l8_224078_20200518) as src:
self.assertIsInstance(src, xr.DataArray)
def test_count(self):
with gw.open(l8_224078_20200518) as src, rio.open(
l8_224078_20200518) as rsrc:
self.assertEqual(src.gw.nbands, rsrc.count)
def test_config_bands(self):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(','.join(map(str, src.band.values.tolist())),
'1,2,3')
def test_open_type_dask(self):
with gw.open(l8_224078_20200518) as src:
self.assertIsInstance(src.data, dask.array.core.Array)
def get_mean_altitude(data,
out_dir,
username=None,
key_file=None,
code_file=None,
chunks=512,
n_jobs=1,
delete_downloaded=False):
if not username:
username = data.gw.nasa_earthdata_user
if not key_file:
key_file = data.gw.nasa_earthdata_key
if not code_file:
code_file = data.gw.nasa_earthdata_code
if not username or not key_file or not code_file:
logger.exception(
' The NASA EarthData username, secret key file, and secret code file must be provided to download SRTM data.'
)
raise AttributeError
if not Path(out_dir).is_dir():
Path(out_dir).mkdir(parents=True, exist_ok=True)
srtm_grid_path_temp = Path(
out_dir) / f'srtm30m_bounding_boxes_{_random_id(9)}.gpkg'
shutil.copy(str(srtm30m_bounding_boxes), str(srtm_grid_path_temp))
srtm_df = gpd.read_file(srtm_grid_path_temp)
srtm_grid_path_temp.unlink()
srtm_df_int = srtm_df[srtm_df.geometry.intersects(
data.gw.geodataframe.to_crs(epsg=4326).geometry.values[0])]
nedd = NASAEarthdataDownloader(username, key_file, code_file)
hgt_files = []
zip_paths = []
for dfn in srtm_df_int.dataFile.values.tolist():
zip_file = f"{out_dir}/NASADEM_HGT_{dfn.split('.')[0].lower()}.zip"
nedd.download_srtm(dfn.split('.')[0].lower(), zip_file)
src_zip = f"zip+file://{zip_file}!/{Path(zip_file).stem.split('_')[-1]}.hgt"
hgt_files.append(Path(zip_file))
zip_paths.append(src_zip)
if len(zip_paths) == 1:
zip_paths = zip_paths[0]
mosaic = False
else:
mosaic = True
with gw.open(zip_paths, mosaic=mosaic, chunks=chunks) as src:
mean_elev = src.transpose('band', 'y', 'x')\
.mean().data\
.compute(num_workers=n_jobs)
if delete_downloaded:
for fn in hgt_files:
fn.unlink()
return mean_elev
def test_crs(self):
with gw.open(l8_224078_20200518) as src:
self.assertEqual(src.crs, '+init=epsg:32621')
