def test_iter_args_winfuncs():
wins = [
((0, 6), (3, 6)),
((2, 4), (1, 5))]
assert windows.intersect(*wins) == windows.intersect(wins)
assert windows.intersection(*wins) == windows.intersection(wins)
assert windows.union(*wins) == windows.union(wins)
def test_iter_args_winfuncs():
wins = [
((0, 6), (3, 6)),
((2, 4), (1, 5))]
assert windows.intersect(*wins) == windows.intersect(wins)
assert windows.intersection(*wins) == windows.intersection(wins)
assert windows.union(*wins) == windows.union(wins)
def test_window_union(recwarn):
data = [((0, 6), (3, 6)), ((2, 4), (1, 5))]
warnings.simplefilter('always')
old = window_union(data)
assert len(recwarn) == 1
assert recwarn.pop(DeprecationWarning)
new = windows.union(data)
assert len(recwarn) == 0
assert old == new
def test_union_boundless_above():
"""Windows entirely above a dataset form a proper union"""
uw = union(
Window(col_off=0, row_off=-10, width=2, height=2),
Window(col_off=0, row_off=-8.5, width=2, height=2.5),
)
assert uw.row_off == -10
assert uw.height == 4
assert uw.width == 2
assert uw.col_off == 0
def test_window_union(recwarn):
data = [
((0, 6), (3, 6)),
((2, 4), (1, 5))]
warnings.simplefilter('always')
old = window_union(data)
assert len(recwarn) == 1
assert recwarn.pop(DeprecationWarning)
new = windows.union(data)
assert len(recwarn) == 0
assert old == new
def _union_blocks(
dst: DatasetWriter, min_i: int, min_j: int, max_i: int, max_j: int
) -> Window:
"""Loops over selected blocks of data source and merges their windows
into one."""
windows: List[Window] = list()
for i in range(min_i, max_i):
for j in range(min_j, max_j):
windows.append(dst.block_window(1, i, j))
return union(*windows)
def to_netcdf(
files,
output,
variable,
dtype,
src_crs,
x_name,
y_name,
z_name,
datetime_pattern,
netcdf3,
compress,
packed,
xy_dtype,
# z_dtype,
calendar,
autocrop):
"""
Convert rasters to NetCDF and stack them according to a dimension.
X and Y dimension names will be named according to the source projection (lon, lat if geographic projection, x, y
otherwise) unless specified.
Will overwrite an existing NetCDF file.
Only the first band of the input will be turned into a NetCDF file.
"""
# TODO: add format string template to this to parse out components
filenames = list(glob.glob(files))
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='files',
param_hint='FILES')
z_values = []
if datetime_pattern is not None:
datetimes = (datetime.strptime(x, datetime_pattern) for x in filenames)
# Sort both datimes and filenames by datetimes
z_values, filenames = [
list(x)
for x in zip(*sorted(zip(datetimes, filenames), key=itemgetter(0)))
]
items = tuple(enumerate(filenames))
has_z = len(filenames) > 1
if has_z and not z_name:
raise click.BadParameter('Required when > 1 input file',
param='--z',
param_hint='--z')
if src_crs:
src_crs = CRS.from_string(src_crs)
template_ds = rasterio.open(filenames[0])
src_crs = template_ds.crs or src_crs
if not src_crs:
raise click.BadParameter(
'Required when no CRS information available in source files',
param='--src-crs',
param_hint='--src-crs')
prj = Proj(**src_crs.to_dict())
bounds = template_ds.bounds
width = template_ds.width
height = template_ds.height
window = None
src_dtype = numpy.dtype(template_ds.dtypes[0])
dtype = numpy.dtype(dtype) if dtype else src_dtype
if dtype == src_dtype:
fill_value = template_ds.nodata
if src_dtype.kind in ('u', 'i'):
# nodata always comes from rasterio as floating point
fill_value = int(fill_value)
else:
fill_value = get_fill_value(dtype)
x_name = x_name or ('lon' if src_crs.is_geographic else 'x')
y_name = y_name or ('lat' if src_crs.is_geographic else 'y')
var_kwargs = {'fill_value': fill_value}
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
with Dataset(output, 'w', format=format) as out:
if packed or autocrop:
mins = []
maxs = []
windows = []
click.echo('Inspecting input datasets...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True)
if packed:
mins.append(data.min())
maxs.append(data.max())
if autocrop:
data_window = get_data_window(data)
if data_window != ((0, height), (0, width)):
windows.append(data_window)
if packed:
min_value = min(mins)
max_value = max(maxs)
scale, offset = get_pack_atts(dtype, min_value, max_value)
if autocrop and windows:
window = union(windows)
bounds = template_ds.window_bounds(window)
height = window[0][1] - window[0][0]
width = window[1][1] - window[1][0]
coords = SpatialCoordinateVariables.from_bbox(BBox(bounds, prj), width,
height, xy_dtype)
coords.add_to_dataset(out, x_name, y_name, zlib=compress)
var_dimensions = [y_name, x_name]
shape = list(coords.shape)
if has_z:
shape.insert(0, len(filenames))
out.createDimension(z_name, shape[0])
var_dimensions.insert(0, z_name)
if z_values:
dates = DateVariable(numpy.array(z_values),
units_start_date=z_values[0],
calendar=calendar)
dates.add_to_dataset(out, z_name)
click.echo('Creating {0}:{1} with shape {2}'.format(
output, variable, shape))
out_var = out.createVariable(variable,
dtype,
dimensions=var_dimensions,
zlib=compress,
**var_kwargs)
set_crs(out, variable, prj, set_proj4_att=True)
if packed:
out_var.setncattr('scale_factor', scale)
out_var.setncattr('add_offset', offset)
click.echo('Copying data from input files...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True, window=window)
if has_z:
out_var[index, :] = data
else:
out_var[:] = data
out.sync()
def test_window_union():
"""Window union works."""
window = union(Window(0, 0, 1, 1), Window(1, 1, 2, 2))
assert window == Window.from_slices((0, 3), (0, 3))
def window_union(data):
warnings.warn("Deprecated; Use rasterio.windows instead", DeprecationWarning)
return windows.union(data)
def test_window_union():
assert windows.union(((0, 6), (3, 6)),
((2, 4), (1, 5))) == windows.Window.from_ranges(
(0, 6), (1, 6))
def window_union(data):
warnings.warn("Deprecated; Use rasterio.windows instead",
DeprecationWarning)
return windows.union(data)
def test_window_union():
assert windows.union(
windows.Window(0, 3, 3, 6),
windows.Window(2, 1, 2, 4)
) == windows.Window(0, 1, 4, 8)
def test_window_union():
assert windows.union(
((0, 6), (3, 6)),
((2, 4), (1, 5))
) == windows.Window.from_ranges((0, 6), (1, 6))
def test_window_union():
assert windows.union(windows.Window(0, 3, 3, 6),
windows.Window(2, 1, 2,
4)) == windows.Window(0, 1, 4, 8)
def read_records(self, file_name, range_tracker):
from rasterio.io import MemoryFile
from rasterio.features import shapes
from rasterio.windows import union
from shapely.geometry import shape
from fiona import transform
import json
total_bytes = self.estimate_size()
next_pos = range_tracker.start_position()
end_pos = range_tracker.stop_position()
def split_points_unclaimed(stop_pos):
return 0 if stop_pos <= next_pos else iobase.RangeTracker.SPLIT_POINTS_UNKNOWN
range_tracker.set_split_points_unclaimed_callback(
split_points_unclaimed)
with self.open_file(file_name) as f, MemoryFile(
f.read()) as m, m.open() as src:
is_wgs84, src_crs = _GeoSourceUtils.validate_crs(
src.crs, self.in_epsg)
block_windows = list([win for ji, win in src.block_windows()])
num_windows = len(block_windows)
window_bytes = math.floor(total_bytes / num_windows)
i = 0
logging.info(
json.dumps(
{
'msg': 'read_records',
'file_name': file_name,
'file_size_mb': round(total_bytes / 1048576),
'block_windows': num_windows,
'merge_blocks': self.merge_blocks,
'include_nodata': self.include_nodata,
'src_profile': src.profile
},
default=str))
while range_tracker.try_claim(next_pos):
i = math.ceil(next_pos / window_bytes)
end_i = math.floor(end_pos / window_bytes)
if i >= num_windows:
break
actual_merge_blocks = max(1, min(self.merge_blocks, end_i - i))
slice_end = min(i + actual_merge_blocks, num_windows)
window_slice = block_windows[i:slice_end]
cur_window = union(window_slice)
cur_transform = src.window_transform(cur_window)
block_mask = src.read_masks(self.band_number,
window=cur_window)
if self.include_nodata:
block = src.read(self.band_number, window=cur_window)
else:
block = src.read(self.band_number,
window=cur_window,
masked=True)
logging.debug(
json.dumps(
{
'msg': 'read_records.try_claim',
'file_name': file_name,
'band_number': self.band_number,
'next_pos': next_pos,
'i': i,
'window': cur_window
},
default=str))
for (g, v) in shapes(block,
block_mask,
transform=cur_transform):
if not is_wgs84 or self.skip_reproject:
geom = transform.transform_geom(
src_crs, 'epsg:4326', g)
else:
geom = g
if self.centroid_only:
geom = shape(geom).centroid.__geo_interface__
yield (v, geom)
next_pos = next_pos + (window_bytes * actual_merge_blocks)
def test_window_union():
"""Window union works."""
window = union(Window(0, 0, 1, 1), Window(1, 1, 2, 2))
assert window == Window.from_slices((0, 3), (0, 3))
def to_netcdf(
files,
output,
variable,
dtype,
src_crs,
x_name,
y_name,
z_name,
datetime_pattern,
netcdf3,
compress,
packed,
xy_dtype,
# z_dtype,
calendar,
autocrop):
"""
Convert rasters to NetCDF and stack them according to a dimension.
X and Y dimension names will be named according to the source projection (lon, lat if geographic projection, x, y
otherwise) unless specified.
Will overwrite an existing NetCDF file.
Only the first band of the input will be turned into a NetCDF file.
"""
# TODO: add format string template to this to parse out components
filenames = list(glob.glob(files))
if not filenames:
raise click.BadParameter('No files found matching that pattern', param='files', param_hint='FILES')
z_values = []
if datetime_pattern is not None:
datetimes = (datetime.strptime(x, datetime_pattern) for x in filenames)
# Sort both datimes and filenames by datetimes
z_values, filenames = [list(x) for x in zip(*sorted(zip(datetimes, filenames), key=itemgetter(0)))]
items = tuple(enumerate(filenames))
has_z = len(filenames) > 1
if has_z and not z_name:
raise click.BadParameter('Required when > 1 input file', param='--z', param_hint='--z')
if src_crs:
src_crs = CRS.from_string(src_crs)
template_ds = rasterio.open(filenames[0])
src_crs = template_ds.crs or src_crs
if not src_crs:
raise click.BadParameter('Required when no CRS information available in source files', param='--src-crs',
param_hint='--src-crs')
prj = Proj(**src_crs.to_dict())
bounds = template_ds.bounds
width = template_ds.width
height = template_ds.height
window = None
src_dtype = numpy.dtype(template_ds.dtypes[0])
dtype = numpy.dtype(dtype) if dtype else src_dtype
if dtype == src_dtype:
fill_value = template_ds.nodata
if src_dtype.kind in ('u', 'i'):
# nodata always comes from rasterio as floating point
fill_value = int(fill_value)
else:
fill_value = get_fill_value(dtype)
x_name = x_name or ('lon' if src_crs.is_geographic else 'x')
y_name = y_name or ('lat' if src_crs.is_geographic else 'y')
var_kwargs = {
'fill_value': fill_value
}
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
with Dataset(output, 'w', format=format) as out:
if packed or autocrop:
mins = []
maxs = []
windows = []
click.echo('Inspecting input datasets...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True)
if packed:
mins.append(data.min())
maxs.append(data.max())
if autocrop:
data_window = get_data_window(data)
if data_window != ((0, height), (0, width)):
windows.append(data_window)
if packed:
min_value = min(mins)
max_value = max(maxs)
scale, offset = get_pack_atts(dtype, min_value, max_value)
if autocrop and windows:
window = union(windows)
bounds = template_ds.window_bounds(window)
height = window[0][1] - window[0][0]
width = window[1][1] - window[1][0]
coords = SpatialCoordinateVariables.from_bbox(BBox(bounds, prj), width, height, xy_dtype)
coords.add_to_dataset(out, x_name, y_name, zlib=compress)
var_dimensions = [y_name, x_name]
shape = list(coords.shape)
if has_z:
shape.insert(0, len(filenames))
out.createDimension(z_name, shape[0])
var_dimensions.insert(0, z_name)
if z_values:
dates = DateVariable(numpy.array(z_values),
units_start_date=z_values[0], calendar=calendar)
dates.add_to_dataset(out, z_name)
click.echo('Creating {0}:{1} with shape {2}'.format(output, variable, shape))
out_var = out.createVariable(variable, dtype, dimensions=var_dimensions,
zlib=compress, **var_kwargs)
set_crs(out, variable, prj, set_proj4_att=True)
if packed:
out_var.setncattr('scale_factor', scale)
out_var.setncattr('add_offset', offset)
click.echo('Copying data from input files...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True, window=window)
if has_z:
out_var[index, :] = data
else:
out_var[:] = data
out.sync()
def test_window_union():
assert windows.union(((0, 6), (3, 6)),
((2, 4), (1, 5))) == ((0, 6), (1, 6))
def test_window_union():
assert windows.union(
((0, 6), (3, 6)),
((2, 4), (1, 5))
) == ((0, 6), (1, 6))
