def test_iter_args_winfuncs(): wins = [ ((0, 6), (3, 6)), ((2, 4), (1, 5))] assert windows.intersect(*wins) == windows.intersect(wins) assert windows.intersection(*wins) == windows.intersection(wins) assert windows.union(*wins) == windows.union(wins)
def test_window_union(recwarn): data = [((0, 6), (3, 6)), ((2, 4), (1, 5))] warnings.simplefilter('always') old = window_union(data) assert len(recwarn) == 1 assert recwarn.pop(DeprecationWarning) new = windows.union(data) assert len(recwarn) == 0 assert old == new
def test_union_boundless_above(): """Windows entirely above a dataset form a proper union""" uw = union( Window(col_off=0, row_off=-10, width=2, height=2), Window(col_off=0, row_off=-8.5, width=2, height=2.5), ) assert uw.row_off == -10 assert uw.height == 4 assert uw.width == 2 assert uw.col_off == 0
def test_window_union(recwarn): data = [ ((0, 6), (3, 6)), ((2, 4), (1, 5))] warnings.simplefilter('always') old = window_union(data) assert len(recwarn) == 1 assert recwarn.pop(DeprecationWarning) new = windows.union(data) assert len(recwarn) == 0 assert old == new
def _union_blocks( dst: DatasetWriter, min_i: int, min_j: int, max_i: int, max_j: int ) -> Window: """Loops over selected blocks of data source and merges their windows into one.""" windows: List[Window] = list() for i in range(min_i, max_i): for j in range(min_j, max_j): windows.append(dst.block_window(1, i, j)) return union(*windows)
def to_netcdf( files, output, variable, dtype, src_crs, x_name, y_name, z_name, datetime_pattern, netcdf3, compress, packed, xy_dtype, # z_dtype, calendar, autocrop): """ Convert rasters to NetCDF and stack them according to a dimension. X and Y dimension names will be named according to the source projection (lon, lat if geographic projection, x, y otherwise) unless specified. Will overwrite an existing NetCDF file. Only the first band of the input will be turned into a NetCDF file. """ # TODO: add format string template to this to parse out components filenames = list(glob.glob(files)) if not filenames: raise click.BadParameter('No files found matching that pattern', param='files', param_hint='FILES') z_values = [] if datetime_pattern is not None: datetimes = (datetime.strptime(x, datetime_pattern) for x in filenames) # Sort both datimes and filenames by datetimes z_values, filenames = [ list(x) for x in zip(*sorted(zip(datetimes, filenames), key=itemgetter(0))) ] items = tuple(enumerate(filenames)) has_z = len(filenames) > 1 if has_z and not z_name: raise click.BadParameter('Required when > 1 input file', param='--z', param_hint='--z') if src_crs: src_crs = CRS.from_string(src_crs) template_ds = rasterio.open(filenames[0]) src_crs = template_ds.crs or src_crs if not src_crs: raise click.BadParameter( 'Required when no CRS information available in source files', param='--src-crs', param_hint='--src-crs') prj = Proj(**src_crs.to_dict()) bounds = template_ds.bounds width = template_ds.width height = template_ds.height window = None src_dtype = numpy.dtype(template_ds.dtypes[0]) dtype = numpy.dtype(dtype) if dtype else src_dtype if dtype == src_dtype: fill_value = template_ds.nodata if src_dtype.kind in ('u', 'i'): # nodata always comes from rasterio as floating point fill_value = int(fill_value) else: fill_value = get_fill_value(dtype) x_name = x_name or ('lon' if src_crs.is_geographic else 'x') y_name = y_name or ('lat' if src_crs.is_geographic else 'y') var_kwargs = {'fill_value': fill_value} format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4' with Dataset(output, 'w', format=format) as out: if packed or autocrop: mins = [] maxs = [] windows = [] click.echo('Inspecting input datasets...') with click.progressbar(items) as iter: for index, filename in iter: with rasterio.open(filename) as src: data = src.read(1, masked=True) if packed: mins.append(data.min()) maxs.append(data.max()) if autocrop: data_window = get_data_window(data) if data_window != ((0, height), (0, width)): windows.append(data_window) if packed: min_value = min(mins) max_value = max(maxs) scale, offset = get_pack_atts(dtype, min_value, max_value) if autocrop and windows: window = union(windows) bounds = template_ds.window_bounds(window) height = window[0][1] - window[0][0] width = window[1][1] - window[1][0] coords = SpatialCoordinateVariables.from_bbox(BBox(bounds, prj), width, height, xy_dtype) coords.add_to_dataset(out, x_name, y_name, zlib=compress) var_dimensions = [y_name, x_name] shape = list(coords.shape) if has_z: shape.insert(0, len(filenames)) out.createDimension(z_name, shape[0]) var_dimensions.insert(0, z_name) if z_values: dates = DateVariable(numpy.array(z_values), units_start_date=z_values[0], calendar=calendar) dates.add_to_dataset(out, z_name) click.echo('Creating {0}:{1} with shape {2}'.format( output, variable, shape)) out_var = out.createVariable(variable, dtype, dimensions=var_dimensions, zlib=compress, **var_kwargs) set_crs(out, variable, prj, set_proj4_att=True) if packed: out_var.setncattr('scale_factor', scale) out_var.setncattr('add_offset', offset) click.echo('Copying data from input files...') with click.progressbar(items) as iter: for index, filename in iter: with rasterio.open(filename) as src: data = src.read(1, masked=True, window=window) if has_z: out_var[index, :] = data else: out_var[:] = data out.sync()
def test_window_union(): """Window union works.""" window = union(Window(0, 0, 1, 1), Window(1, 1, 2, 2)) assert window == Window.from_slices((0, 3), (0, 3))
def window_union(data): warnings.warn("Deprecated; Use rasterio.windows instead", DeprecationWarning) return windows.union(data)
def test_window_union(): assert windows.union(((0, 6), (3, 6)), ((2, 4), (1, 5))) == windows.Window.from_ranges( (0, 6), (1, 6))
def test_window_union(): assert windows.union( windows.Window(0, 3, 3, 6), windows.Window(2, 1, 2, 4) ) == windows.Window(0, 1, 4, 8)
def test_window_union(): assert windows.union( ((0, 6), (3, 6)), ((2, 4), (1, 5)) ) == windows.Window.from_ranges((0, 6), (1, 6))
def test_window_union(): assert windows.union(windows.Window(0, 3, 3, 6), windows.Window(2, 1, 2, 4)) == windows.Window(0, 1, 4, 8)
def read_records(self, file_name, range_tracker): from rasterio.io import MemoryFile from rasterio.features import shapes from rasterio.windows import union from shapely.geometry import shape from fiona import transform import json total_bytes = self.estimate_size() next_pos = range_tracker.start_position() end_pos = range_tracker.stop_position() def split_points_unclaimed(stop_pos): return 0 if stop_pos <= next_pos else iobase.RangeTracker.SPLIT_POINTS_UNKNOWN range_tracker.set_split_points_unclaimed_callback( split_points_unclaimed) with self.open_file(file_name) as f, MemoryFile( f.read()) as m, m.open() as src: is_wgs84, src_crs = _GeoSourceUtils.validate_crs( src.crs, self.in_epsg) block_windows = list([win for ji, win in src.block_windows()]) num_windows = len(block_windows) window_bytes = math.floor(total_bytes / num_windows) i = 0 logging.info( json.dumps( { 'msg': 'read_records', 'file_name': file_name, 'file_size_mb': round(total_bytes / 1048576), 'block_windows': num_windows, 'merge_blocks': self.merge_blocks, 'include_nodata': self.include_nodata, 'src_profile': src.profile }, default=str)) while range_tracker.try_claim(next_pos): i = math.ceil(next_pos / window_bytes) end_i = math.floor(end_pos / window_bytes) if i >= num_windows: break actual_merge_blocks = max(1, min(self.merge_blocks, end_i - i)) slice_end = min(i + actual_merge_blocks, num_windows) window_slice = block_windows[i:slice_end] cur_window = union(window_slice) cur_transform = src.window_transform(cur_window) block_mask = src.read_masks(self.band_number, window=cur_window) if self.include_nodata: block = src.read(self.band_number, window=cur_window) else: block = src.read(self.band_number, window=cur_window, masked=True) logging.debug( json.dumps( { 'msg': 'read_records.try_claim', 'file_name': file_name, 'band_number': self.band_number, 'next_pos': next_pos, 'i': i, 'window': cur_window }, default=str)) for (g, v) in shapes(block, block_mask, transform=cur_transform): if not is_wgs84 or self.skip_reproject: geom = transform.transform_geom( src_crs, 'epsg:4326', g) else: geom = g if self.centroid_only: geom = shape(geom).centroid.__geo_interface__ yield (v, geom) next_pos = next_pos + (window_bytes * actual_merge_blocks)
def to_netcdf( files, output, variable, dtype, src_crs, x_name, y_name, z_name, datetime_pattern, netcdf3, compress, packed, xy_dtype, # z_dtype, calendar, autocrop): """ Convert rasters to NetCDF and stack them according to a dimension. X and Y dimension names will be named according to the source projection (lon, lat if geographic projection, x, y otherwise) unless specified. Will overwrite an existing NetCDF file. Only the first band of the input will be turned into a NetCDF file. """ # TODO: add format string template to this to parse out components filenames = list(glob.glob(files)) if not filenames: raise click.BadParameter('No files found matching that pattern', param='files', param_hint='FILES') z_values = [] if datetime_pattern is not None: datetimes = (datetime.strptime(x, datetime_pattern) for x in filenames) # Sort both datimes and filenames by datetimes z_values, filenames = [list(x) for x in zip(*sorted(zip(datetimes, filenames), key=itemgetter(0)))] items = tuple(enumerate(filenames)) has_z = len(filenames) > 1 if has_z and not z_name: raise click.BadParameter('Required when > 1 input file', param='--z', param_hint='--z') if src_crs: src_crs = CRS.from_string(src_crs) template_ds = rasterio.open(filenames[0]) src_crs = template_ds.crs or src_crs if not src_crs: raise click.BadParameter('Required when no CRS information available in source files', param='--src-crs', param_hint='--src-crs') prj = Proj(**src_crs.to_dict()) bounds = template_ds.bounds width = template_ds.width height = template_ds.height window = None src_dtype = numpy.dtype(template_ds.dtypes[0]) dtype = numpy.dtype(dtype) if dtype else src_dtype if dtype == src_dtype: fill_value = template_ds.nodata if src_dtype.kind in ('u', 'i'): # nodata always comes from rasterio as floating point fill_value = int(fill_value) else: fill_value = get_fill_value(dtype) x_name = x_name or ('lon' if src_crs.is_geographic else 'x') y_name = y_name or ('lat' if src_crs.is_geographic else 'y') var_kwargs = { 'fill_value': fill_value } format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4' with Dataset(output, 'w', format=format) as out: if packed or autocrop: mins = [] maxs = [] windows = [] click.echo('Inspecting input datasets...') with click.progressbar(items) as iter: for index, filename in iter: with rasterio.open(filename) as src: data = src.read(1, masked=True) if packed: mins.append(data.min()) maxs.append(data.max()) if autocrop: data_window = get_data_window(data) if data_window != ((0, height), (0, width)): windows.append(data_window) if packed: min_value = min(mins) max_value = max(maxs) scale, offset = get_pack_atts(dtype, min_value, max_value) if autocrop and windows: window = union(windows) bounds = template_ds.window_bounds(window) height = window[0][1] - window[0][0] width = window[1][1] - window[1][0] coords = SpatialCoordinateVariables.from_bbox(BBox(bounds, prj), width, height, xy_dtype) coords.add_to_dataset(out, x_name, y_name, zlib=compress) var_dimensions = [y_name, x_name] shape = list(coords.shape) if has_z: shape.insert(0, len(filenames)) out.createDimension(z_name, shape[0]) var_dimensions.insert(0, z_name) if z_values: dates = DateVariable(numpy.array(z_values), units_start_date=z_values[0], calendar=calendar) dates.add_to_dataset(out, z_name) click.echo('Creating {0}:{1} with shape {2}'.format(output, variable, shape)) out_var = out.createVariable(variable, dtype, dimensions=var_dimensions, zlib=compress, **var_kwargs) set_crs(out, variable, prj, set_proj4_att=True) if packed: out_var.setncattr('scale_factor', scale) out_var.setncattr('add_offset', offset) click.echo('Copying data from input files...') with click.progressbar(items) as iter: for index, filename in iter: with rasterio.open(filename) as src: data = src.read(1, masked=True, window=window) if has_z: out_var[index, :] = data else: out_var[:] = data out.sync()
def test_window_union(): assert windows.union(((0, 6), (3, 6)), ((2, 4), (1, 5))) == ((0, 6), (1, 6))
def test_window_union(): assert windows.union( ((0, 6), (3, 6)), ((2, 4), (1, 5)) ) == ((0, 6), (1, 6))