def load_tiff(filename, crs=None, apply_transform=False, nan_nodata=False, **kwargs):
"""
Returns an RGB or Image element loaded from a geotiff file.
The data is loaded using xarray and rasterio. If a crs attribute
is present on the loaded data it will attempt to decode it into
a cartopy projection otherwise it will default to a non-geographic
HoloViews element.
Parameters
----------
filename: string
Filename pointing to geotiff file to load
crs: Cartopy CRS or EPSG string (optional)
Overrides CRS inferred from the data
apply_transform: boolean
Whether to apply affine transform if defined on the data
nan_nodata: boolean
If data contains nodata values convert them to NaNs
**kwargs:
Keyword arguments passed to the HoloViews/GeoViews element
Returns
-------
element: Image/RGB/QuadMesh element
"""
try:
import xarray as xr
except:
raise ImportError('Loading tiffs requires xarray to be installed')
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
da = xr.open_rasterio(filename)
return from_xarray(da, crs, apply_transform, nan_nodata, **kwargs)
def test_resample_methods():
"""Assert that an error is raised when incorrect upsample and/or downsample
methods are provided to cvs.raster().
"""
with xr.open_rasterio(TEST_RASTER_PATH) as src:
try:
cvs.raster(src, upsample_method='santaclaus', downsample_method='toothfairy')
except ValueError:
pass
else:
assert False
try:
cvs.raster(src, upsample_method='honestlawyer')
except ValueError:
pass
else:
assert False
try:
cvs.raster(src, downsample_method='tenantfriendlylease')
except ValueError:
pass
else:
assert False
def locate_vs(vs_file, pix_nb=20, acc_min=1_000_000):
with open(vs_file) as f:
virtual_stations = f.read().split()
da = xr.open_rasterio('../data/hydrosheds/acc.vrt')
pix_deg = 1 / 1200
width2 = pix_deg * pix_nb
station_l, lat_l, lon_l, new_lat_l, new_lon_l = [], [], [], [], []
for vs in tqdm(virtual_stations):
url = f'http://hydroweb.theia-land.fr/hydroweb/view/{vs}?lang=en&basin=AMAZONAS'
df, lat, lon = get_vs(url)
max_lat, max_lon = np.nan, np.nan
for pix_nb2 in range(1, pix_nb+1):
# look further and further away from original lat/lon
width = pix_nb2 * pix_deg
acc = da.loc[1, lat+width:lat-width, lon-width:lon+width]
max_acc = acc.where(acc==acc.max(), drop=True)
if max_acc.values[0, 0] >= acc_min:
# consider only rivers with a minimum accumulated flow
max_lat = max_acc.y.values[0]
max_lon = max_acc.x.values[0]
break
station_l.append(vs)
lat_l.append(lat)
lon_l.append(lon)
new_lat_l.append(max_lat)
new_lon_l.append(max_lon)
return DataFrame({'station':station_l, 'lat':lat_l, 'lon':lon_l, 'new_lat':new_lat_l, 'new_lon':new_lon_l})
def test_dask_distributed_rasterio_integration_test(loop):
with create_tmp_geotiff() as (tmp_file, expected):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
da_tiff = xr.open_rasterio(tmp_file, chunks={'band': 1})
assert isinstance(da_tiff.data, da.Array)
actual = da_tiff.compute()
assert_allclose(actual, expected)
def test_calc_res():
"""Assert that resolution is calculated correctly when using the xarray
rasterio backend.
"""
with xr.open_rasterio(TEST_RASTER_PATH) as src:
xr_res = ds.utils.calc_res(src)
with rasterio.open(TEST_RASTER_PATH) as src:
rio_res = src.res
assert np.allclose(xr_res, rio_res)
def test_calc_bbox():
"""Assert that bounding boxes are calculated correctly when using the xarray
rasterio backend.
"""
with xr.open_rasterio(TEST_RASTER_PATH) as src:
xr_res = ds.utils.calc_res(src)
xr_bounds = ds.utils.calc_bbox(src.x.values, src.y.values, xr_res)
with rasterio.open(TEST_RASTER_PATH) as src:
rio_bounds = src.bounds
assert np.allclose(xr_bounds, rio_bounds, atol=1.0) # allow for absolute diff of 1.0
def test_out_of_bounds_return_correct_size():
with xr.open_rasterio(TEST_RASTER_PATH) as src:
cvs = ds.Canvas(plot_width=2,
plot_height=2,
x_range=[1e10, 1e20],
y_range=[1e10, 1e20])
try:
cvs.raster(src)
except ValueError:
pass
else:
assert False
def __init__(self, m, label):
self.m = m
self.label = label
self.width = 0.1
self.coord = None
self.io = None
self.s = None
self.p = None
self.show_flow = False
self.show_menu = False
self.da = xr.open_rasterio('../data/hydrosheds/acc.vrt')
self.marker = None
def test_partial_extent_with_layer_returns_correct_size():
with xr.open_rasterio(TEST_RASTER_PATH) as src:
res = ds.utils.calc_res(src)
left, bottom, right, top = ds.utils.calc_bbox(src.x.values, src.y.values, res)
half_width = (right - left) / 2
half_height = (top - bottom) / 2
cvs = ds.Canvas(plot_width=512,
plot_height=256,
x_range=[left-half_width, left+half_width],
y_range=[bottom-half_height, bottom+half_height])
agg = cvs.raster(src, layer=1)
assert agg.shape == (256, 512)
assert agg is not None
def load_bgids(dirc, indstr, loc_str):
indices_list = ['s6m', 'mndwi', 'msavi', 'whi']
n_ids = len(indices_list)
print("Start loading long term mean of indices")
for i, indname in enumerate(indices_list):
fname = dirc + '/' + loc_str + '_' + indname + '_' + indstr + '.tif'
dbs = xr.open_rasterio(fname)
if i == 0:
irow = dbs['y'].size
icol = dbs['x'].size
bgids = np.zeros((n_ids, irow * icol), dtype=np.float32)
bgids[i] = dbs.data[0].flatten()
dbs.close()
return bgids
def test_platecarree(self):
with create_tmp_geotiff(
8,
10,
1,
transform_args=[1, 2, 0.5, 2.0],
crs="+proj=latlong",
open_kwargs={"nodata": -9765},
) as (tmp_file, expected):
with xr.open_rasterio(tmp_file) as rioda:
assert_allclose(rioda, expected)
assert rioda.attrs["scales"] == (1.0,)
assert rioda.attrs["offsets"] == (0.0,)
assert isinstance(rioda.attrs["descriptions"], tuple)
assert isinstance(rioda.attrs["units"], tuple)
assert isinstance(rioda.attrs["crs"], str)
assert isinstance(rioda.attrs["res"], tuple)
assert isinstance(rioda.attrs["is_tiled"], np.uint8)
assert isinstance(rioda.rio._cached_transform(), Affine)
np.testing.assert_array_equal(rioda.attrs["nodatavals"], [-9765.0])
def house_radiance():
""" Radiance image of a house and foliage.
Radiances calculated by the VTT software from the `house_raw`
dataset.
CC0
Returns
-------
house_raw : xarray.DataArray
(4, 400, 400) radiance cube with metadata.
"""
res = open_rasterio(osp.join(data_dir, 'house_crop_4b_RAD.dat'))
res = res.to_dataset(name=c.radiance_data)
res = res.reset_coords()
return res
def test_make_src_affine(modis_reproject):
with xarray.open_dataarray(modis_reproject["input"],
autoclose=True) as xdi, xarray.open_rasterio(
modis_reproject["input"]) as xri:
# check the transform
attribute_transform = tuple(xdi.attrs["transform"])
attribute_transform_func = tuple(xdi.rio.transform())
calculated_transform = tuple(xdi.rio.transform(recalc=True))
# delete the transform to ensure it is not being used
del xdi.attrs["transform"]
calculated_transform_check = tuple(xdi.rio.transform())
calculated_transform_check2 = tuple(xdi.rio.transform())
rio_transform = xri.attrs["transform"]
assert_array_equal(attribute_transform, attribute_transform_func)
assert_array_equal(calculated_transform, calculated_transform_check)
assert_array_equal(calculated_transform, calculated_transform_check2)
assert_array_equal(attribute_transform, calculated_transform)
assert_array_equal(calculated_transform[:6], rio_transform)
def _open_files(self, files):
import xarray as xr
das = [
xr.open_rasterio(f, chunks=self.chunks, **self._kwargs)
for f in files
]
out = xr.concat(das, dim=self.dim)
coords = {}
if self.pattern:
coords = {
k: xr.concat([
xr.DataArray(np.full(das[i].sizes.get(self.dim, 1), v),
dims=self.dim) for i, v in enumerate(values)
],
dim=self.dim)
for k, values in reverse_formats(self.pattern, files).items()
}
return out.assign_coords(**coords).chunk(self.chunks)
def test_rasterio_vrt(self):
# tmp_file default crs is UTM: CRS({'init': 'epsg:32618'}
with create_tmp_geotiff() as (tmp_file, expected):
with rasterio.open(tmp_file) as src:
with rasterio.vrt.WarpedVRT(src, crs="epsg:4326") as vrt:
expected_shape = (vrt.width, vrt.height)
expected_crs = vrt.crs
expected_res = vrt.res
# Value of single pixel in center of image
lon, lat = vrt.xy(vrt.width // 2, vrt.height // 2)
expected_val = next(vrt.sample([(lon, lat)]))
with xr.open_rasterio(vrt) as da:
actual_shape = (da.sizes["x"], da.sizes["y"])
actual_crs = da.crs
actual_res = da.res
actual_val = da.sel(dict(x=lon, y=lat), method="nearest").data
assert actual_crs == expected_crs
assert actual_res == expected_res
assert actual_shape == expected_shape
assert expected_val.all() == actual_val.all()
def get_SWISS_ref_grid(
basepath='/mnt/CEPH_PROJECTS/ADO/SM/reference_data/swiss_model/'):
# initiate sm stack
tmp = xr.open_rasterio(basepath + 'fcp_20190101.tif')
# del tmp['band']
# coordinate transformation
tmp.attrs['crs'] = 'EPSG:21781'
ny, nx = len(tmp['y']), len(tmp['x'])
x, y = np.meshgrid(tmp['x'], tmp['y'])
lon, lat = transform(tmp.crs, {'init': 'EPSG:4326'}, x.flatten(),
y.flatten())
lon = np.asarray(lon).reshape((ny, nx))
lat = np.asarray(lat).reshape((ny, nx))
tmp.coords['lon'] = (('y', 'x'), lon)
tmp.coords['lat'] = (('y', 'x'), lat)
ref_grid = xr.DataArray(np.full((len(lat[:, 0]), len(lon[0, :])), -9999.0),
coords=[('lat', lat[:, 0]), ('lon', lon[0, :])],
name='ref_grid')
tmp.close()
return ref_grid
def rasterio_to_xarray(arr,
meta,
tmp_dir='.',
fil_name=None,
chunks=None,
out=False,
*args,
**kwargs):
if fil_name is None:
fil_name = str(uuid.uuid4())
tmp_path = os.path.join(tmp_dir, '%s' % fil_name)
with rio.open(tmp_path, 'w', **meta) as fil2:
fil2.write(arr)
ret = xr.open_rasterio(tmp_path, chunks=chunks)
ret = clean_raster_xarray(ret)
if chunks is None and not out:
os.remove(tmp_path)
return ret, tmp_path
def _open_data_source(datasource, mask=None, offset=0):
"""
Open data source and return data array needed for plotting.
Parameters
----------
datasource: string
A data array or the path to a data file to be opened with rasterio.
mask: None
Not implemented yet.
offset: scalar, optional
Substract this number to the data. Mostly used to fix data stored as
unsigned integers.
"""
if isinstance(datasource, xr.DataArray):
darray = datasource
else:
darray = xr.open_rasterio(datasource)
darray = darray.where(~darray.isin(darray.nodatavals))
darray = darray.squeeze() - offset
return darray
def readClassArray(outdir, dates, classtag):
"""
@params
outdir (str): location of the rasterized images
dates ([str]): list of dates of dates of the raster in format %Y%m%d
classtag (str): tag of the rasterized image name
"""
classdates = dates #list([str(t)[:4]+str(t)[5:7]+str(t)[8:10] for t in dates]) ## this was for np.datetime64
arlist = [
xr.open_rasterio(f) for f in list(
[outdir + 'class_' + x + classtag + '.tif' for x in classdates])
]
time = xr.Variable('time',
pd.DatetimeIndex([pd.Timestamp(f) for f in dates]))
data_array = xr.concat(arlist,
dim=time).isel(band=0).drop('band').transpose(
'time', 'y', 'x') #.values
data_array.name = classtag
return data_array.astype(np.uint8, copy=False) #mask.astype(np.int16)
def load_tiff(filename,
crs=None,
apply_transform=False,
nan_nodata=False,
**kwargs):
"""
Returns an RGB or Image element loaded from a geotiff file.
The data is loaded using xarray and rasterio. If a crs attribute
is present on the loaded data it will attempt to decode it into
a cartopy projection otherwise it will default to a non-geographic
HoloViews element.
Arguments
---------
filename: string
Filename pointing to geotiff file to load
crs: Cartopy CRS or EPSG string (optional)
Overrides CRS inferred from the data
apply_transform: boolean
Whether to apply affine transform if defined on the data
nan_nodata: boolean
If data contains nodata values convert them to NaNs
**kwargs:
Keyword arguments passed to the HoloViews/GeoViews element
Returns
-------
element: Image/RGB/QuadMesh element
"""
try:
import xarray as xr
except:
raise ImportError('Loading tiffs requires xarray to be installed')
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
da = xr.open_rasterio(filename)
return from_xarray(da, crs, apply_transform, nan_nodata, **kwargs)
def load_tif(filepath: Path, start_date: datetime,
days_per_timestep: int) -> xr.DataArray:
r"""
The sentinel files exported from google earth have all the timesteps
concatenated together. This function loads a tif files and splits the
timesteps
"""
# this mirrors the eo-learn approach
# also, we divide by 10,000, to remove the scaling factor
# https://developers.google.com/earth-engine/datasets/catalog/COPERNICUS_S2
da = xr.open_rasterio(filepath).rename("FEATURES") / 10000
da_split_by_time: List[xr.DataArray] = []
bands_per_timestep = len(BANDS)
num_bands = len(da.band)
assert (
num_bands % bands_per_timestep == 0
), f"Total number of bands not divisible by the expected bands per timestep"
cur_band = 0
while cur_band + bands_per_timestep <= num_bands:
time_specific_da = da.isel(
band=slice(cur_band, cur_band + bands_per_timestep))
time_specific_da["band"] = range(bands_per_timestep)
da_split_by_time.append(time_specific_da)
cur_band += bands_per_timestep
timesteps = [
start_date + timedelta(days=days_per_timestep) * i
for i in range(len(da_split_by_time))
]
combined = xr.concat(da_split_by_time, pd.Index(timesteps,
name="time"))
combined.attrs["band_descriptions"] = BANDS
return combined
def test_chunks(self):
with create_tmp_geotiff(8,
10,
3,
transform_args=[1, 2, 0.5, 2.0],
crs="+proj=latlong") as (tmp_file, expected):
# Chunk at open time
with xr.open_rasterio(tmp_file, chunks=(1, 2, 2)) as actual:
import dask.array as da
assert isinstance(actual.data, da.Array)
assert "open_rasterio" in actual.data.name
# do some arithmetic
ac = actual.mean()
ex = expected.mean()
assert_allclose(ac, ex)
ac = actual.sel(band=1).mean(dim="x")
ex = expected.sel(band=1).mean(dim="x")
assert_allclose(ac, ex)
def fix_mukey(mukey_tif):
"""The mukey tiff might come out with some odd values around the edges,
which makes viewing it a bit tricky. This fixes that."""
mukey_tif = os.path.expanduser(mukey_tif)
# For the profile
profile = rasterio.open(mukey_tif).profile
# For the data array
mukey = xr.open_rasterio(mukey_tif, chunks=(1, 5000, 5000))
# Change everything under 0 to the nan value
array = mukey.data
array[array < 0] = profile["nodata"]
with Client():
array = array.compute()
# save
with rasterio.Env():
with rasterio.open(mukey_tif, "w", **profile) as file:
file.write(array[0].astype(rasterio.int32), 1)
def geoimread(fname, roi_x=None, roi_y=None, roi_crs=None, buffer=0, band=0):
"""Reads a sub-region of a geotiff/geojp2/ that overlaps a region of interest (roi)
This is a simple wrapper of xarray functionality.
Parameters:
fname (str) : file.
roi_x, roi_y (List[float]) : region of interest.
roi_crs (dict) : ROI coordinate reference system, in rasterio dict format (if different from scene coordinates)
buffer (float) : adds a buffer around the region of interest.
band (int) : which bands to read.
Returns
xr.core.dataarray.DataArray : The cropped scene.
"""
da = xr.open_rasterio(fname)
if roi_x is not None:
if not hasattr(roi_x, "__len__"):
roi_x = [roi_x]
roi_y = [roi_y]
if roi_crs is not None:
if str(roi_crs) == "LL":
roi_crs = {"init": "EPSG:4326"}
# remove nans
roi_x = np.array(roi_x)
roi_y = np.array(roi_y)
ix = ~np.isnan(roi_x + roi_y)
roi_x, roi_y = transform(src_crs=roi_crs,
dst_crs=da.crs,
xs=roi_x[ix],
ys=roi_y[ix])
rows = (da.y > np.min(roi_y) - buffer) & (da.y <
np.max(roi_y) + buffer)
cols = (da.x > np.min(roi_x) - buffer) & (da.x <
np.max(roi_x) + buffer)
return da[band, rows, cols].squeeze()
else:
return da[band, :, :].squeeze()
def __init__(self,
data=None,
chunks=None,
row_chunks=None,
col_chunks=None,
padding=None,
scheduler='threads',
get_ray=False,
n_workers=1,
n_chunks=None):
self.chunks = 512 if not isinstance(chunks, int) else chunks
if isinstance(data, list):
self.data_list = data
self.data = xr.open_rasterio(self.data_list[0])
else:
self.data = data
self.row_chunks = row_chunks
self.col_chunks = col_chunks
self.padding = padding
self.n = None
self.scheduler = scheduler
self.get_ray = get_ray
self.executor = _EXEC_DICT[scheduler]
self.n_workers = n_workers
self.n_chunks = n_chunks
self._in_session = False
self.windows = None
self.slices = None
self.n_windows = None
if not isinstance(self.n_chunks, int):
self.n_chunks = self.n_workers * 50
if not isinstance(data, list):
self._setup()
def make_da():
"""Clips are automatically taken from clipped_files folder by default. We stronlgy recommand to clip datasets to avoid RAM overquota.
Raster datasets are concatenated along a new dimension (time) extracted from TIFFTAG_DATETIME attribute of S1 data arrays, and then transposed to be properly sliced along the new time dimension to create stacked layers.
Return: data array of clipped images concatenated along time dimension (`xarray.DataArray`)
"""
clipped_files = sorted(glob.glob("clipped_files/*.tiff", recursive=True))
if len(clipped_files) > 0:
data_sets, titles = [], []
for f in clipped_files:
tmp = xarray.open_rasterio(f).isel(band=0)
data_sets.append(tmp)
titles.append(tmp.attrs["TIFFTAG_DATETIME"])
dc = xarray.concat(
[d for d in data_sets],
pd.Index([d.attrs["TIFFTAG_DATETIME"] for d in data_sets]))
dc = dc.rename({"concat_dim": "UTC"})
dc = dc.sortby("UTC")
return dc
else:
warnings.warn("No tiff clips found in clipped_files folder")
return None
def _get_schema(self):
xarr = xr.open_rasterio(self.path)
ds2 = xr.Dataset({'raster': xarr})
metadata = {
'dims': dict(ds2.dims),
'data_vars':
{k: list(ds2[k].coords)
for k in ds2.data_vars.keys()},
'coords': tuple(ds2.coords.keys()),
'array': 'raster'
}
atts = ['transform', 'crs', 'res', 'is_tiled', 'nodatavals']
for att in atts:
if att in xarr.attrs:
metadata[att] = xarr.attrs[att]
if att == 'nodatavals' and self.with_nodata is True:
metadata[att] = np.nan
return Schema(datashape=None,
dtype=str(xarr.dtype),
shape=xarr.shape,
npartitions=1,
extra_metadata=metadata)
def _open_data(self, path):
if self.engine == 'xarray':
return xr.open_dataset(path, backend_kwargs=self.xr_kwargs)
elif self.engine == 'opendap':
try:
if self.session:
return xr.open_dataset(
xr.backends.PydapDataStore.open(path,
session=self.session))
else:
return xr.open_dataset(path)
except ConnectionRefusedError as e:
raise e
except Exception as e:
print('Unexpected Error')
raise e
elif self.engine == 'auth-opendap':
return xr.open_dataset(
xr.backends.PydapDataStore.open(path,
session=setup_session(
self.user,
self.pswd,
check_url=path)))
elif self.engine == 'netcdf4':
return nc.Dataset(path, 'r')
elif self.engine == 'cfgrib':
return xr.open_dataset(path,
engine='cfgrib',
backend_kwargs=self.xr_kwargs)
elif self.engine == 'pygrib':
a = pygrib.open(path)
return a.read()
elif self.engine == 'h5py':
return h5py.File(path, 'r')
elif self.engine == 'rasterio':
return xr.open_rasterio(path)
else:
raise ValueError(
f'Unable to open file, unsupported engine: {self.engine}')
def read_sar(src, dsname=None, chunks=(20, 500, 500), keep_DN=False):
'''Returns a SAR power data array time series stack from the Amplitude scale data stack'''
try:
da = xr.open_rasterio(src, chunks=chunks)
if not dsname:
da.name = os.path.splitext(os.path.basename(src))[0]
da.name = dsname
da['band'] = pd.DatetimeIndex(da.descriptions)
da = da.rename({'band': 'time'})
#da.set_index(time="time")
if not keep_DN:
# Convert to power
attrs = da.attrs.copy()
CAL = np.power(10, -8.3)
da = (np.power(da.astype(np.float32), 2) * CAL)
da = da.where(da > 0) # Sets values <= 0 to nan
da.attrs = attrs
return da
except Exception as e:
raise ValueError(e)
def reproject(content):
with rio.MemoryFile() as memfile:
memfile.write(content)
with memfile.open() as src:
transform, width, height = rio_warp.calculate_default_transform(
src.crs, crs, src.width, src.height, *src.bounds
)
kwargs = src.meta.copy()
kwargs.update(
{"crs": crs, "transform": transform, "width": width, "height": height}
)
with rio.vrt.WarpedVRT(src, **kwargs) as vrt:
if crs != src.crs:
for i in range(1, src.count + 1):
rio_warp.reproject(
source=rio.band(src, i),
destination=rio.band(vrt, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
crs=crs,
resampling=resampling,
)
da = xr.open_rasterio(vrt)
try:
da = da.squeeze("band", drop=True)
except ValueError:
pass
da.name = "elevation"
da.attrs["transform"] = transform
da.attrs["res"] = (transform[0], transform[4])
da.attrs["bounds"] = tuple(vrt.bounds)
da.attrs["nodatavals"] = vrt.nodatavals
da.attrs["crs"] = vrt.crs.to_string()
return da
def load_raster(file_path,
xmin=None,
ymin=None,
xmax=None,
ymax=None,
chunks=None,
layername='data'):
if file_path.endswith('.tif'):
arr = xr.open_rasterio(expanduser(file_path),
chunks={
'y': 512,
'x': 512
})
if hasattr(arr, 'nodatavals'):
if np.issubdtype(arr.data.dtype, np.integer):
arr.data = arr.data.astype('f8')
for val in arr.nodatavals:
arr.data[arr.data == val] = np.nan
arr.name = file_path
elif file_path.endswith('.nc'):
# TODO: add chunk parameter to config
arr = xr.open_dataset(file_path, chunks={
'x': 512,
'y': 512
})[layername]
arr['name'] = file_path
else:
raise TypeError(f"Unable to load raster {file_path}")
return arr
def run_int(input_file,
output_file,
chunks=(512, 512),
name='data',
fill_value=-9999):
arr = xr.open_rasterio(input_file)
arr = squeeze(arr, 'band')
arr = orient_array(arr)
arr = flip_coords(arr, dim='y') # do we need this?
arr = reproject_raster(arr, epsg=3857)
dataset = xr.Dataset({name: (['y', 'x'], arr.chunk(chunks))},
coords={
'x': arr.coords['x'],
'y': arr.coords['y']
})
dataset.attrs = dict(name=name)
dataset.to_netcdf(
output_file,
encoding={'data': {
'dtype': 'int16',
'_FillValue': fill_value
}})
def read(self):
"""Read the image"""
data = xr.open_rasterio(self.finfo["filename"],
chunks=(1, CHUNK_SIZE, CHUNK_SIZE))
attrs = data.attrs.copy()
# Create area definition
if hasattr(data, 'crs'):
self.area = self.get_geotiff_area_def(data.crs)
# Rename to SatPy convention
data = data.rename({'band': 'bands'})
# Rename bands to [R, G, B, A], or a subset of those
data['bands'] = BANDS[data.bands.size]
# Mask data if alpha channel is present
try:
data = mask_image_data(data)
except ValueError as err:
logger.warning(err)
data.attrs = attrs
self.file_content['image'] = data
def process_topographic_variables_file(path_to_file: str):
path_to_file = pathlib.Path(path_to_file)
names = ('elevation', 'tpi_500', 'we_derivative', 'sn_derivative', 'slope',
'aspect')
if all(
(path_to_file.parent / f"topo_{name}.nc").exists() for name in names):
print("Already processed all topo files")
return
dem = xr.open_rasterio(path_to_file)
dem = dem.isel(band=0, drop=True)
ind_nans, dem = helpers.fill_na(dem)
# TPI 500m
scale_meters = 500
scale_pixel, res_meters = helpers.scale_to_pixel(scale_meters, dem)
tpi = topo.tpi(dem, scale_pixel)
# Gradient
gradient = topo.gradient(dem, 1 / 4 * scale_pixel, res_meters)
# Norm and second the direction of Ridge index
variables = (dem, tpi, *gradient)
for data, name in zip(variables, names):
da = xr.DataArray(data, coords=dem.coords, name=name)
filename = f"topo_{name}.nc"
da.to_dataset().to_netcdf(pathlib.Path(path_to_file.parent, filename))
def to_ds(search, username, api_key):
chirps_ds = []
dates = []
datasets = []
for index, row in search.iterrows():
# read the vsicurl geotiff
da = xr.open_rasterio(get_vsi_url(row['enclosure'], username, api_key))
# remove the band dimension
da = da.squeeze().drop(labels='band')
# add the variable
da = da.to_dataset(name='rainfall')
dates.append(row['startdate'])
datasets.append(da)
ds = xr.concat(datasets, dim=pd.DatetimeIndex(dates, name='date'))
return ds
def get_demz_tiff(filin, lon, lat):
import pyproj
# Read the data
da = xr.open_rasterio(filin)
# Rasterio works with 1D arrays (but still need to pass whole mesh, flattened)
# convert lon/lat to xy using intrinsic crs, then generate additional dimension for
# advanced xarray interpolation
# print(da.crs)
p = pyproj.Proj(da.crs)
xi, yi = p(lon, lat, inverse=False)
xi = xr.DataArray(xi, dims="z")
yi = xr.DataArray(yi, dims="z")
if debug:
print(da)
print("x,y len:", len(xi), len(yi))
da_interp = da.interp(x=xi, y=yi)
return da_interp.data * 1.e-3 # convert to km for compatibility with grd
def test_clip_geojson():
with xarray.open_rasterio(
os.path.join(TEST_COMPARE_DATA_DIR,
"small_dem_3m_merged.tif")) as xdi:
# get subset for testing
subset = xdi.isel(x=slice(150, 160), y=slice(100, 150))
comp_subset = subset.isel(x=slice(1, None), y=slice(1, None))
# add transform for test
comp_subset.attrs["transform"] = tuple(
comp_subset.rio.transform(recalc=True))
# add grid mapping for test
comp_subset.attrs["grid_mapping"] = "spatial_ref"
comp_subset.coords["spatial_ref"] = 0
# make sure nodata exists for test
comp_subset.attrs["_FillValue"] = comp_subset.attrs["nodatavals"][0]
geometries = [{
"type":
"Polygon",
"coordinates": [[
[subset.x.values[0], subset.y.values[-1]],
[subset.x.values[0], subset.y.values[0]],
[subset.x.values[-1], subset.y.values[0]],
[subset.x.values[-1], subset.y.values[-1]],
[subset.x.values[0], subset.y.values[-1]],
]],
}]
# test data array
clipped = xdi.rio.clip(geometries, subset.rio.crs)
_assert_xarrays_equal(clipped, comp_subset)
# test dataset
clipped_ds = xdi.to_dataset(name="test_data").rio.clip(
geometries, subset.rio.crs)
comp_subset_ds = comp_subset.to_dataset(name="test_data")
_assert_xarrays_equal(clipped_ds, comp_subset_ds)
def merge_tifs_into_ds(
root_fld: str,
tifs: List[str],
rename_dict: Optional[Dict] = None,
tifs_min_num=8,
) -> xr.Dataset:
"""
Will be replaced with dc.load(gm_6month) once they've been produced.
use os.walk to get the all files under a folder, it just merge the half year tifs.
We need combine two half-year tifs ds and add (calculated indices, rainfall, and slope)
@param tifs: tifs with the bands
@param root_fld: the parent folder for the sub_fld
@param tifs_min_num: geo-median tifs is 16 a tile idx
@param rename_dict: we can put the rename dictionary here
@return:
"""
assert len(tifs) > tifs_min_num
cache = []
for tif in tifs:
if tif.endswith(".tif"):
band_name = re.search(r"_([A-Za-z0-9]+).tif", tif).groups()[0]
if band_name in ["rgba", "COUNT"]:
continue
band_array = assign_crs(
xr.open_rasterio(osp.join(root_fld, tif))
.squeeze()
.to_dataset(name=band_name),
crs="epsg:6933",
)
cache.append(band_array)
# clean up output
output = xr.merge(cache).squeeze()
output = output.drop(["band"])
return output.rename(rename_dict) if rename_dict else output
def read_hydro_metadata(path=hydro_path, gis_path=gis_path, plot=True):
import pandas as pd
import geopandas as gpd
import xarray as xr
df = pd.read_excel(hydro_path / 'hydro_stations_metadata.xlsx', header=4)
# drop last row:
df.drop(df.tail(1).index, inplace=True) # drop last n rows
df.columns = ['id', 'name', 'active', 'agency', 'type', 'X', 'Y', 'area']
df.loc[:, 'active'][df['active'] == 'פעילה'] = 1
df.loc[:, 'active'][df['active'] == 'לא פעילה'] = 0
df.loc[:, 'active'][df['active'] == 'לא פעילה זמנית'] = 0
df['active'] = df['active'].astype(float)
df = df[~df.X.isnull()]
df = df[~df.Y.isnull()]
# now, geopandas part:
geo_df = gpd.GeoDataFrame(df,
geometry=gpd.points_from_xy(df.X, df.Y),
crs={'init': 'epsg:2039'})
# geo_df.crs = {'init': 'epsg:2039'}
geo_df = geo_df.to_crs({'init': 'epsg:4326'})
isr_dem = xr.open_rasterio(gis_path / 'israel_dem.tif')
alt_list = []
for index, row in geo_df.iterrows():
lat = row.geometry.y
lon = row.geometry.x
alt = isr_dem.sel(band=1, x=lon, y=lat, method='nearest').values.item()
alt_list.append(float(alt))
geo_df['alt'] = alt_list
geo_df['lat'] = geo_df.geometry.y
geo_df['lon'] = geo_df.geometry.x
isr = gpd.read_file(gis_path / 'Israel_and_Yosh.shp')
isr.crs = {'init': 'epsg:4326'}
geo_df = gpd.sjoin(geo_df, isr, op='within')
if plot:
ax = isr.plot()
geo_df.plot(ax=ax, edgecolor='black', legend=True)
return geo_df
def test_raster_aggregate_without_overviews():
with xr.open_rasterio(TEST_RASTER_PATH) as src:
agg = cvs.raster(src, use_overviews=False)
assert agg is not None
import pytest
rasterio = pytest.importorskip("rasterio")
from os import path
import datashader as ds
import xarray as xr
import numpy as np
BASE_PATH = path.split(__file__)[0]
DATA_PATH = path.abspath(path.join(BASE_PATH, 'data'))
TEST_RASTER_PATH = path.join(DATA_PATH, 'world.rgb.tif')
with xr.open_rasterio(TEST_RASTER_PATH) as src:
res = ds.utils.calc_res(src)
left, bottom, right, top = ds.utils.calc_bbox(src.x.values, src.y.values, res)
cvs = ds.Canvas(plot_width=2,
plot_height=2,
x_range=(left, right),
y_range=(bottom, top))
def test_raster_aggregate_default():
with xr.open_rasterio(TEST_RASTER_PATH) as src:
agg = cvs.raster(src)
assert agg is not None
def test_raster_aggregate_nearest():
with xr.open_rasterio(TEST_RASTER_PATH) as src:
agg = cvs.raster(src, upsample_method='nearest')
============================
imshow() and map projections
============================
Using rasterio's projection information for more accurate plots.
This example extends :ref:`recipes.rasterio` and plots the image in the
original map projection instead of relying on pcolormesh and a map
transformation.
"""
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import xarray as xr
# Read the data
url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif'
da = xr.open_rasterio(url)
# The data is in UTM projection. We have to set it manually until
# https://github.com/SciTools/cartopy/issues/813 is implemented
crs = ccrs.UTM('18N')
# Plot on a map
ax = plt.subplot(projection=crs)
da.plot.imshow(ax=ax, rgb='band', transform=crs)
ax.coastlines('10m', color='r')
plt.show()
def test_raster_aggregate_nearest():
with xr.open_rasterio(TEST_RASTER_PATH) as src:
agg = cvs.raster(src, upsample_method='nearest')
assert agg is not None
transformation.
"""
import os
import urllib.request
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import xarray as xr
# Download the file from rasterio's repository
url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif'
urllib.request.urlretrieve(url, 'RGB.byte.tif')
# Read the data
da = xr.open_rasterio('RGB.byte.tif')
# The data is in UTM projection. We have to set it manually until
# https://github.com/SciTools/cartopy/issues/813 is implemented
crs = ccrs.UTM('18N')
# Plot on a map
ax = plt.subplot(projection=crs)
da.plot.imshow(ax=ax, rgb='band', transform=crs)
ax.coastlines('10m', color='r')
plt.show()
# Delete the file
os.remove('RGB.byte.tif')
def test_raster_aggregate_default():
with xr.open_rasterio(TEST_RASTER_PATH) as src:
agg = cvs.raster(src)
assert agg is not None
