def raise_low_pixels(memfile: MemoryFile,
max_no_raise: float = 0.0,
max_brightness: int = 255,
noisy: bool = False) -> MemoryFile:
"""Detect very low (but above sea level) elevations in a raster and
increase the elevation of the relevant pixels such that, when the
file is converted to a greyscale image, those pixels will be given
a value of 1, rather than rounded down to 0.
"""
with memfile.open() as src:
data = src.read(1)
# Floor all values at 0
data *= (data > 0)
max_elev = data.max()
min_visible = math.ceil(
max_elev / max_brightness
) # Minimum value that will be rounded to 1 in a greyscale image
print(
f'Raising pixels of {max_no_raise} < elevation < {min_visible} to {min_visible}.'
)
if noisy:
# Add a small value to each affected pixel's elevation, which is proportionate to that pixel's original
# elevation. This ensures that fixed areas do not have a uniform elevation (unless they originally had
# a uniform elevation), so that they are not picked up by Microdem as lakes.
min_visible = (data / 10) + min_visible
data = np.where((data > max_no_raise) & (data < min_visible),
min_visible, data)
dst_memfile = MemoryFile()
kwargs = src.profile.copy()
with dst_memfile.open(**kwargs) as dst:
dst.write(data, 1)
return dst_memfile
def _open_and_stack_bands(self, coordinate, date, bands):
"""Loads band file rasters at path specified by arguments and stacks
bands into single raster
Args:
coordinate (object): coordinate information - to be precised in child class
date (object): date information - to be precised in child class
bands (list[str]): list of band files to load
Returns:
type: rasterio.io.DatasetReader
"""
# Load metadata from first band - assume similar for other bands
meta = self.get_meta(coordinate=coordinate, date=date, band=bands[0])
meta.update({'count': len(bands)})
# Create temporary in-memory file
memory_file = MemoryFile()
# Write new scene containing all bands from directory specified in kwargs
with memory_file.open(**meta) as target_raster:
for idx, band in enumerate(bands):
with self(coordinate=coordinate, date=date,
bands=[band]) as source_raster:
target_raster.write_band(idx + 1, source_raster.read(1))
return memory_file.open()
def resample(memfile: MemoryFile, scale_factor: float) -> MemoryFile:
"""Resample raster by a factor of scale_factor.
scale_factor > 1: Upsample
scale factor < 1: Downsample
"""
print(f'Resampling raster with scaling factor of {scale_factor}.')
with memfile.open() as src:
print(f'Source raster has shape {src.shape}.')
# resample data to target shape
height = int(src.height * scale_factor)
width = int(src.width * scale_factor)
data = src.read(out_shape=(src.count, height, width),
resampling=Resampling.bilinear)
# scale image transform
transform = src.transform * src.transform.scale(width, height)
kwargs = src.profile.copy()
kwargs.update({'height': height, 'width': width})
dst_memfile = MemoryFile()
with dst_memfile.open(**kwargs) as dst:
for i in range(1, src.count + 1):
dst.write(data)
print(f'Resampled raster has shape {dst.shape}.')
return dst_memfile
def reproject_raster(memfile: MemoryFile,
dst_crs: str,
src_crs: str = WGS84) -> MemoryFile:
"""Reproject raster with CRS src_crs to new CRS dst_crs."""
print(f'Reprojecting raster from {src_crs} to {dst_crs}.')
with memfile.open() as src:
print(f'Source raster has shape {src.shape}.')
transform, width, height = calculate_default_transform(
src_crs, dst_crs, src.width, src.height, *src.bounds)
kwargs = src.profile.copy()
kwargs.update({
'crs': dst_crs,
'transform': transform,
'width': width,
'height': height
})
dst_memfile = MemoryFile()
with dst_memfile.open(**kwargs) as dst:
for i in range(1, src.count + 1):
reproject(source=rasterio.band(src, i),
destination=rasterio.band(dst, i),
src_transform=src.transform,
src_crs=src_crs,
dst_transform=transform,
dst_crs=dst_crs,
resampling=Resampling.bilinear)
print(f'Reprojected raster has shape {dst.shape}.')
return dst_memfile
def __update_dataset(self, crs, transform, nodata=None):
"""Update dataset without writing to file after it theoretically changed.
:param crs: crs of the dataset
:param transform: transform of the dataset
:param nodata: nodata value, optional
:return: file in memory, open as dataset
"""
meta = {
"driver": "GTiff",
"dtype": self.__arr.dtype,
"nodata": nodata,
"height": self.__arr.shape[-2],
"width": self.__arr.shape[-1],
"count": self.__arr.shape[0],
"crs": crs,
"transform": transform,
}
memfile = MemoryFile()
with memfile.open(**meta) as ds:
ds.write(self.__arr)
self.dataset = memfile.open()
memfile.close()
def set_lakes_to_elev(memfile: MemoryFile,
min_lake_size: int,
fill_lakes_as: int = None,
nodata: int = SRTM_NODATA) -> MemoryFile:
"""Find all lakes in the data for a raster and set the elevation of
the relevant pixels to fill_lakes_as.
"""
if fill_lakes_as is None:
fill_lakes_as = nodata
print(
f'Finding lakes with minimum size of {min_lake_size} and setting elevation to {fill_lakes_as}.'
)
with memfile.open() as src:
data = src.read(1)
lakes = get_all_lakes(data, min_lake_size)
print(f'Found {len(lakes)} lakes.')
for lake in lakes:
for row, col in lake:
data[row, col] = fill_lakes_as
dst_memfile = MemoryFile()
kwargs = src.profile.copy()
with dst_memfile.open(**kwargs) as dst:
dst.write(data, 1)
return dst_memfile
def in_memory_raster(array, meta):
"""Encapsulates image array and metadata as raster instance in reading mode
stored in memory i.e. no actual file written
Args:
array (np.ndarray): raster image content
meta (dict): raster metadata
Returns:
type: rasterio.io.DatasetReader
"""
memory_file = MemoryFile()
with memory_file.open(**meta) as raster:
raster.write(array)
return memory_file.open()
def create(scenes, bands=[4, 3, 2]):
"""Handler."""
_worker = partial(worker, bands=bands)
with futures.ThreadPoolExecutor(max_workers=10) as executor:
responses = executor.map(_worker, scenes)
with contextlib.ExitStack() as stack:
sources = [
stack.enter_context(rasterio.open(scene)) for scene in responses
if scene
]
dest, output_transform = merge(sources, nodata=0)
meta = {
"driver": "GTiff",
"count": 3,
"dtype": np.uint8,
"nodata": 0,
"height": dest.shape[1],
"width": dest.shape[2],
"compress": "JPEG",
"crs": "epsg:3857",
"transform": output_transform,
}
memfile = MemoryFile()
with memfile.open(**meta) as dataset:
dataset.write(dest)
wgs_bounds = transform_bounds(*[dataset.crs, "epsg:4326"] +
list(dataset.bounds),
densify_pts=21)
return memfile, wgs_bounds
def create(tiles):
"""Handler."""
with futures.ThreadPoolExecutor(max_workers=8) as executor:
responses = executor.map(worker, tiles)
with contextlib.ExitStack() as stack:
sources = [
stack.enter_context(rasterio.open(tile)) for tile in responses
if tile
]
dest, output_transform = merge(sources, nodata=-32767)
meta = {
"driver": "GTiff",
"count": 1,
"dtype": np.int16,
"nodata": -32767,
"height": dest.shape[1],
"width": dest.shape[2],
"compress": "DEFLATE",
"crs": "epsg:4326",
"transform": output_transform,
}
memfile = MemoryFile()
with memfile.open(**meta) as dataset:
dataset.write(dest)
return memfile
def to_png(memfile: MemoryFile,
zero_floor: bool = False,
max_brightness: int = 255,
nodata: int = SRTM_NODATA):
"""Save raster as a greyscale PNG file to to_file. If set_negative
is set, any elevation values below zero are set to that value (which
should be in the range 0-255).
"""
print(f'Converting raster to PNG image.')
with memfile.open() as src:
data = src.read(1)
data[data == nodata] = 0
if zero_floor:
data[data < 0] = 0
max_elev = data.max()
min_elev = data.min()
scale_factor = max_brightness / (max_elev - min_elev)
if min_elev > 0:
# If everywhere on the map is above sea level, we should scale
# such that the lowest parts of the map appear slightly above sea level.
floor = min_elev + 1
else:
floor = min_elev
data = ((data - floor) * scale_factor).astype(np.uint8)
im = Image.fromarray(data, mode='L')
width, height = im.size
print(f'Image size is {width}x{height}.')
return im
def test_memory_file_gdal_error_message(capsys):
"""No weird error messages should be seen, see #1659"""
memfile = MemoryFile()
data = numpy.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12],
[13, 14, 15, 16]]).astype('uint8')
west_bound = 0
north_bound = 2
cellsize = 0.5
nodata = -9999
driver = 'AAIGrid'
dtype = data.dtype
shape = data.shape
transform = rasterio.transform.from_origin(west_bound, north_bound,
cellsize, cellsize)
dataset = memfile.open(driver=driver,
width=shape[1],
height=shape[0],
transform=transform,
count=1,
dtype=dtype,
nodata=nodata,
crs='epsg:3226')
dataset.write(data, 1)
dataset.close()
captured = capsys.readouterr()
assert "ERROR 4" not in captured.err
assert "ERROR 4" not in captured.out
class RasterWindowMemoryFile():
"""Context manager around rasterio.io.MemoryFile."""
def __init__(self,
in_tile=None,
in_data=None,
out_profile=None,
out_tile=None,
tags=None):
"""Prepare data & profile."""
out_tile = out_tile or in_tile
validate_write_window_params(in_tile, out_tile, in_data, out_profile)
self.data = extract_from_array(in_raster=in_data,
in_affine=in_tile.affine,
out_tile=out_tile)
# use transform instead of affine
if "affine" in out_profile:
out_profile["transform"] = out_profile.pop("affine")
self.profile = out_profile
self.tags = tags
def __enter__(self):
"""Open MemoryFile, write data and return."""
self.rio_memfile = MemoryFile()
with self.rio_memfile.open(**self.profile) as dst:
dst.write(self.data.astype(self.profile["dtype"], copy=False))
_write_tags(dst, self.tags)
return self.rio_memfile
def __exit__(self, *args):
"""Make sure MemoryFile is closed."""
self.rio_memfile.close()
def reproject_like(this, reproject_this):
dst = MemoryFile()
d_transform, d_width, d_height = calculate_default_transform(
reproject_this.crs, this.crs, reproject_this.width,
reproject_this.height, *reproject_this.bounds)
dst_profile = reproject_this.profile.copy()
dst_profile.update({
"driver": "GTiff",
"crs": this.crs,
"transform": d_transform,
"width": d_width,
"height": d_height,
})
#
with rio.open(dst, 'w', **dst_profile) as dst_file:
reproject(source=reproject_this.read(1),
destination=rio.band(dst_file, 1),
src_transform=reproject_this.transform,
src_crs=reproject_this.crs,
dst_transform=d_transform,
dst_crs=this.crs,
resampling=Resampling.bilinear)
dst.seek(0)
return dst.open()
def fp_reader(fp):
memfile = MemoryFile(fp.read())
dataset = memfile.open()
try:
yield dataset
finally:
dataset.close()
memfile.close()
def fp_reader(fp):
memfile = MemoryFile(fp.read())
dataset = memfile.open(driver=driver, sharing=sharing)
try:
yield dataset
finally:
dataset.close()
memfile.close()
def fp_reader(fp):
memfile = MemoryFile(fp.read())
dataset = memfile.open()
try:
yield dataset
finally:
dataset.close()
memfile.close()
def get_bounding_box_from_file(file):
data = file.read()
try: # Intenta abrir la imagen. De no ser una imagen, se informa al cliente
memfile = MemoryFile(data)
dataset = memfile.open()
return get_bounding_box(dataset)
except rio.errors.RasterioIOError:
response = {'error': 'File is not an image'}
return response
def raise_undersea_land(memfile: MemoryFile, raise_to: int = 1):
"""Raise land with a negative elevation (ie, land that is below sea
level) to raise_to. Probably shouldn't be called before
set_lakes_to_elev, otherwise the newly raised land will be picked up
as a lake by that function.
"""
with memfile.open() as src:
data = src.read(1)
print(f'Raising pixels of elevation < 0 to {raise_to}.')
data = np.where((data < 0), raise_to, data)
dst_memfile = MemoryFile()
kwargs = src.profile.copy()
with dst_memfile.open(**kwargs) as dst:
dst.write(data, 1)
return dst_memfile
def fp_writer(fp):
memfile = MemoryFile()
dataset = memfile.open(driver=driver, width=width, height=height,
count=count, crs=crs, transform=transform,
dtype=dtype, nodata=nodata, **kwargs)
try:
yield dataset
finally:
dataset.close()
memfile.seek(0)
fp.write(memfile.read())
memfile.close()
def fp_writer(fp):
memfile = MemoryFile()
dataset = memfile.open(driver=driver, width=width, height=height,
count=count, crs=crs, transform=transform,
dtype=dtype, nodata=nodata, sharing=sharing, **kwargs)
try:
yield dataset
finally:
dataset.close()
memfile.seek(0)
fp.write(memfile.read())
memfile.close()
def clip(array=None, profile=None, vector=None, raster=None, all_touched=True,
driver='GTiff', masked=True, extent=False):
"""Clip a raster file.
Takes either a ndarray + profile or a path to a raster file.
Also takes a vector as path or GeoDataFrame.
Returns a ndarray + profile.
"""
if type(vector) == str:
# with fiona.open(shp, 'r') as shapefile:
# shapes = [feature['geometry'] for feature in shapefile]
gdf = gpd.read_file(vector)
elif type(vector) == gpd.geodataframe.GeoDataFrame:
gdf = vector
else:
raise ValueError('Please specify a valid vector path or GeoDataFrame.')
if raster is not None:
with rasterio.open(raster) as src:
array = src.read(masked=masked)
profile = src.profile
elif array is None or profile is None:
raise ValueError('Please supply either a raster file or a ndarray and '
'a profile')
if extent:
polygon = bbox_polygon(*gdf.total_bounds)
shapes = [mapping(polygon)]
else:
shapes = [feat['geometry'] for feat in gdf.geometry.__geo_interface__['features']]
memfile = MemoryFile()
with memfile.open(**profile) as mem:
mem.write(array)
dst_array, dst_transform = riomask.mask(mem,
shapes,
all_touched=all_touched,
crop=True,
filled=masked)
dst_profile = profile.copy()
dst_profile.update({"height": dst_array.shape[1],
"width": dst_array.shape[2],
"transform": dst_transform,
"driver": driver})
print('---Clipping successful---')
return dst_array, dst_profile
def remove_sea(memfile: MemoryFile, min_elev: int = 1) -> MemoryFile:
"""Offset elevation data so that lowest value is equal to min_elev.
Useful for when real-world data includes land below sea level but no
sea (eg, an in-land NL map).
By default,min_elev is 1, which means you may need to use this in
conjunction with raise_low_pixels to actually render as land.
"""
with memfile.open() as src:
data = src.read(1)
offset = -(data.min() - min_elev)
print(f'Increasing elevation by {offset}.')
data += offset
dst_memfile = MemoryFile()
kwargs = src.profile.copy()
with dst_memfile.open(**kwargs) as dst:
dst.write(data, 1)
return dst_memfile
def test_memory_file_gdal_error_message(capsys):
"""No weird error messages should be seen, see #1659"""
memfile = MemoryFile()
data = numpy.array([[1,2,3,4],[5,6,7,8],[9,10,11,12],[13,14,15,16]]).astype('uint8')
west_bound = 0; north_bound = 2; cellsize=0.5; nodata = -9999; driver='AAIGrid';
dtype = data.dtype
shape = data.shape
transform = rasterio.transform.from_origin(west_bound, north_bound, cellsize, cellsize)
dataset = memfile.open(driver=driver, width=shape[1], height=shape[0], transform=transform, count=1, dtype=dtype, nodata=nodata, crs='epsg:3226')
dataset.write(data, 1)
dataset.close()
captured = capsys.readouterr()
assert "ERROR 4" not in captured.err
assert "ERROR 4" not in captured.out
class FakeCog(ImageBase):
"""fake cloud optimized geotiff"""
src_img: FakeImage
dst_kwargs: Dict
indexes: Optional[Sequence[int]] = None
nodata: Optional[Union[int, float]] = None
dtype: Optional[str] = None
add_mask: bool = False
overview_level: Optional[int] = None
overview_resampling: str = "nearest"
web_optimized: bool = False
latitude_adjustment: bool = True
resampling: str = "nearest"
config: Optional[Dict] = None
allow_intermediate_compression: bool = False
forward_band_tags: bool = False
quiet: bool = False
temporary_compression: str = "DEFLATE"
def __post_init__(self):
"""post init hook"""
self.memfile = MemoryFile()
cog_translate(
source=self.src_img.handle,
dst_path=self.memfile.name,
dst_kwargs=self.dst_kwargs,
indexes=self.indexes,
nodata=self.nodata,
dtype=self.dtype,
add_mask=self.add_mask,
overview_level=self.overview_level,
overview_resampling=self.overview_resampling,
web_optimized=self.web_optimized,
latitude_adjustment=self.latitude_adjustment,
resampling=self.resampling,
in_memory=True,
config=self.config,
allow_intermediate_compression=self.allow_intermediate_compression,
forward_band_tags=self.forward_band_tags,
quiet=self.quiet,
temporary_compression=self.temporary_compression,
)
self.handle = self.memfile.open()
def close(self):
"""close resources"""
self.handle.close()
self.memfile.close()
def create_tif_file(left: float,
bottom: float,
right: float,
top: float,
to_file: Optional[str] = None,
cache_dir: str = CACHE_DIR,
nodata: int = SRTM_NODATA) -> Union[str, MemoryFile]:
"""Create a TIF file using SRTM data for the box defined by left,
bottom, right, top. If to_file is provided, saves the resulting
file to to_file and returns the path; otherwise, creates a
rasterio.io.MemoryFile and returns that.
"""
os.makedirs(cache_dir, exist_ok=True)
xy = get_all_xy_components(left, bottom, right, top)
zip_fnames = {}
for x, y in xy:
zip_fnames[(x, y)] = ZIP_FNAME.format(x=x, y=y)
zip_fpaths = fetch_all_zips(zip_fnames, cache_dir)
unzip_all(zip_fpaths.values(), cache_dir)
srcs = [
rasterio.open(get_tif_fpath(x, y, cache_dir), 'r', nodata=nodata)
for x, y in xy
]
print(f'Creating TIF file from following files: {[s.name for s in srcs]}.')
#print(f'Heights are: {[s.height for s in srcs]}.')
#print(f'Widths are: {[s.width for s in srcs]}.')
profile = srcs[0].profile
data, transform = rasterio.merge.merge(srcs, (left, bottom, right, top),
nodata=nodata)
for src in srcs:
src.close()
clear_cache(cache_dir, True)
bands, height, width = data.shape # No idea if this is the correct order for height and width, but they are both
# the same so it doesn't matter in this case
profile.update({'height': height, 'width': width, 'transform': transform})
print(f'Created TIF file with dimensions {width}x{height}.')
if to_file:
print(f'Writing TIF file to {to_file}.')
with rasterio.open(to_file, 'w', **profile) as dst:
dst.write(data)
return to_file
else:
memfile = MemoryFile()
with memfile.open(**profile) as dst:
dst.write(data)
return memfile
def write_raster_window(
in_tile=None, in_data=None, out_profile=None, out_tile=None, out_path=None
):
"""
Write a window from a numpy array to an output file.
Parameters
----------
in_tile : ``BufferedTile``
``BufferedTile`` with a data attribute holding NumPy data
in_data : array
out_profile : dictionary
metadata dictionary for rasterio
out_tile : ``Tile``
provides output boundaries; if None, in_tile is used
out_path : string
output path to write to; if output path is "memoryfile", a
rasterio.MemoryFile() is returned
"""
out_tile = in_tile if out_tile is None else out_tile
for t in [in_tile, out_tile]:
if not isinstance(t, BufferedTile):
raise TypeError("in_tile and out_tile must be BufferedTile")
if not isinstance(in_data, ma.MaskedArray):
raise TypeError("in_data must be ma.MaskedArray")
if not isinstance(out_profile, dict):
raise TypeError("out_profile must be a dictionary")
if not isinstance(out_path, six.string_types):
raise TypeError("out_path must be a string")
window_data = extract_from_array(
in_raster=in_data,
in_affine=in_tile.affine,
out_tile=out_tile)
# use transform instead of affine
if "affine" in out_profile:
out_profile["transform"] = out_profile.pop("affine")
# write if there is any band with non-masked data
if window_data.all() is not ma.masked:
if out_path == "memoryfile":
memfile = MemoryFile()
with memfile.open(**out_profile) as dst:
for band, data in enumerate(window_data):
dst.write(data.astype(out_profile["dtype"]), band + 1)
return memfile
else:
with rasterio.open(out_path, 'w', **out_profile) as dst:
for band, data in enumerate(window_data):
dst.write(data.astype(out_profile["dtype"]), band + 1)
def memory_file(data=None, profile=None):
"""
Return a rasterio.io.MemoryFile instance from input.
Parameters
----------
data : array
array to be written
profile : dict
rasterio profile for MemoryFile
"""
memfile = MemoryFile()
with memfile.open(**dict(
profile, width=data.shape[-2], height=data.shape[-1])) as dataset:
dataset.write(data)
return memfile
def make_masked_raster(polygon,
resolution,
bands=1,
all_touched=False,
nodata=-9999.,
crs='+init=epsg:27700',
filename=None):
"""Generates a raster with points outside poly set to nodata. Pixels
are set to be of dimension res x res"""
x = np.arange(polygon.bounds['minx'].values, polygon.bounds['maxx'].values,
resolution)
y = np.arange(polygon.bounds['miny'].values, polygon.bounds['maxy'].values,
resolution)
X, Y = np.meshgrid(x, y)
Z = np.full(X.shape, 1.)
transform = from_origin(x[0] - resolution / 2, y[-1] + resolution / 2,
resolution, resolution)
raster_args = {
'driver': 'GTiff',
'height': Z.shape[0],
'width': Z.shape[1],
'count': bands,
'dtype': Z.dtype,
'crs': polygon.crs,
'transform': transform,
'nodata': nodata
}
if filename is None:
memfile = MemoryFile()
raster = memfile.open(**raster_args)
else:
raster = rasterio.open(filename, 'w+', **raster_args)
for i in range(bands):
raster.write(Z, i + 1)
mask = rasterio.mask.mask(raster,
polygon.geometry,
crop=True,
all_touched=all_touched,
filled=True)
for i in range(bands):
raster.write(mask[0][i], i + 1)
return raster
def _get(self, name, bbox, width, height, time, x, y, r_flag):
"""Return the image value for a location.
Args:
name (str): The image(coverage) name to retrieve from service.
bbox (str): The extent of the image(coverage) to retrieve.
x (int/float): A longitude value according to EPSG:4326.
y (int/float): A latitude value according to EPSG:4326.
"""
output = self.wcs_owslib.getCoverage(identifier=name,
format='GeoTIFF',
bbox=bbox,
crs='EPSG:4326',
time=[time],
width=width,
height=height)
data_array = None
data = output.read()
result = dict()
try:
memfile = MemoryFile(data)
dataset = memfile.open()
values = list(dataset.sample([(x, y)]))
if r_flag:
result['geom'] = dataset.read()
memfile = None
dataset = None
data = None
result['raster_value'] = values[0]
return result
except:
return None
def open(fp, mode='r', driver=None, width=None, height=None, count=None,
crs=None, transform=None, dtype=None, nodata=None, sharing=False,
**kwargs):
"""Open a dataset for reading or writing.
The dataset may be located in a local file, in a resource located by
a URL, or contained within a stream of bytes.
In read ('r') or read/write ('r+') mode, no keyword arguments are
required: these attributes are supplied by the opened dataset.
In write ('w' or 'w+') mode, the driver, width, height, count, and dtype
keywords are strictly required.
Parameters
----------
fp : str, file object, PathLike object, FilePath, or MemoryFile
A filename or URL, a file object opened in binary ('rb') mode, a
Path object, or one of the rasterio classes that provides the
dataset-opening interface (has an open method that returns a
dataset).
mode : str, optional
'r' (read, the default), 'r+' (read/write), 'w' (write), or
'w+' (write/read).
driver : str, optional
A short format driver name (e.g. "GTiff" or "JPEG") or a list of
such names (see GDAL docs at
https://gdal.org/drivers/raster/index.html). In 'w' or 'w+' modes
a single name is required. In 'r' or 'r+' modes the driver can
usually be omitted. Registered drivers will be tried
sequentially until a match is found. When multiple drivers are
available for a format such as JPEG2000, one of them can be
selected by using this keyword argument.
width : int, optional
The number of columns of the raster dataset. Required in 'w' or
'w+' modes, it is ignored in 'r' or 'r+' modes.
height : int, optional
The number of rows of the raster dataset. Required in 'w' or
'w+' modes, it is ignored in 'r' or 'r+' modes.
count : int, optional
The count of dataset bands. Required in 'w' or 'w+' modes, it is
ignored in 'r' or 'r+' modes.
crs : str, dict, or CRS; optional
The coordinate reference system. Required in 'w' or 'w+' modes,
it is ignored in 'r' or 'r+' modes.
transform : Affine instance, optional
Affine transformation mapping the pixel space to geographic
space. Required in 'w' or 'w+' modes, it is ignored in 'r' or
'r+' modes.
dtype : str or numpy dtype
The data type for bands. For example: 'uint8' or
``rasterio.uint16``. Required in 'w' or 'w+' modes, it is
ignored in 'r' or 'r+' modes.
nodata : int, float, or nan; optional
Defines the pixel value to be interpreted as not valid data.
Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+'
modes.
sharing : bool; optional
To reduce overhead and prevent programs from running out of file
descriptors, rasterio maintains a pool of shared low level
dataset handles. When `True` this function will use a shared
handle if one is available. Multithreaded programs must avoid
sharing and should set *sharing* to `False`.
kwargs : optional
These are passed to format drivers as directives for creating or
interpreting datasets. For example: in 'w' or 'w+' modes
a `tiled=True` keyword argument will direct the GeoTIFF format
driver to create a tiled, rather than striped, TIFF.
Returns
-------
A ``DatasetReader`` or ``DatasetWriter`` object.
Examples
--------
To open a GeoTIFF for reading using standard driver discovery and
no directives:
>>> import rasterio
>>> with rasterio.open('example.tif') as dataset:
... print(dataset.profile)
To open a JPEG2000 using only the JP2OpenJPEG driver:
>>> with rasterio.open(
... 'example.jp2', driver='JP2OpenJPEG') as dataset:
... print(dataset.profile)
To create a new 8-band, 16-bit unsigned, tiled, and LZW-compressed
GeoTIFF with a global extent and 0.5 degree resolution:
>>> from rasterio.transform import from_origin
>>> with rasterio.open(
... 'example.tif', 'w', driver='GTiff', dtype='uint16',
... width=720, height=360, count=8, crs='EPSG:4326',
... transform=from_origin(-180.0, 90.0, 0.5, 0.5),
... nodata=0, tiled=True, compress='lzw') as dataset:
... dataset.write(...)
"""
if not isinstance(fp, str):
if not (
hasattr(fp, "read")
or hasattr(fp, "write")
or isinstance(fp, (os.PathLike, MemoryFile, FilePath))
):
raise TypeError("invalid path or file: {0!r}".format(fp))
if mode and not isinstance(mode, str):
raise TypeError("invalid mode: {0!r}".format(mode))
if driver and not isinstance(driver, str):
raise TypeError("invalid driver: {0!r}".format(driver))
if dtype and not check_dtype(dtype):
raise TypeError("invalid dtype: {0!r}".format(dtype))
if nodata is not None:
nodata = float(nodata)
if transform:
transform = guard_transform(transform)
# Check driver/mode blacklist.
if driver and is_blacklisted(driver, mode):
raise RasterioIOError(
"Blacklisted: file cannot be opened by "
"driver '{0}' in '{1}' mode".format(driver, mode))
# If the fp argument is a file-like object and can be adapted by
# rasterio's FilePath we do so. Otherwise, we use a MemoryFile to
# hold fp's contents and store that in an ExitStack attached to the
# dataset object that we will return. When a dataset's close method
# is called, this ExitStack will be unwound and the MemoryFile's
# storage will be cleaned up.
if mode == 'r' and hasattr(fp, 'read'):
if have_vsi_plugin:
return FilePath(fp).open(driver=driver, sharing=sharing, **kwargs)
else:
memfile = MemoryFile(fp.read())
dataset = memfile.open(driver=driver, sharing=sharing, **kwargs)
dataset._env.enter_context(memfile)
return dataset
elif mode in ('w', 'w+') and hasattr(fp, 'write'):
memfile = MemoryFile()
dataset = memfile.open(
driver=driver,
width=width,
height=height,
count=count,
crs=crs,
transform=transform,
dtype=dtype,
nodata=nodata,
sharing=sharing,
**kwargs
)
dataset._env.enter_context(memfile)
# For the writing case we push an extra callback onto the
# ExitStack. It ensures that the MemoryFile's contents are
# copied to the open file object.
def func(*args, **kwds):
memfile.seek(0)
fp.write(memfile.read())
dataset._env.callback(func)
return dataset
# TODO: test for a shared base class or abstract type.
elif isinstance(fp, (FilePath, MemoryFile)):
if mode.startswith("r"):
dataset = fp.open(driver=driver, sharing=sharing, **kwargs)
# Note: FilePath does not support writing and an exception will
# result from this.
elif mode.startswith("w"):
dataset = fp.open(
driver=driver,
width=width,
height=height,
count=count,
crs=crs,
transform=transform,
dtype=dtype,
nodata=nodata,
sharing=sharing,
**kwargs
)
return dataset
# At this point, the fp argument is a string or path-like object
# which can be converted to a string.
else:
raw_dataset_path = os.fspath(fp)
path = _parse_path(raw_dataset_path)
if mode == "r":
dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
elif mode == "r+":
dataset = get_writer_for_path(path, driver=driver)(
path, mode, driver=driver, sharing=sharing, **kwargs
)
elif mode.startswith("w"):
if not driver:
driver = driver_from_extension(path)
writer = get_writer_for_driver(driver)
if writer is not None:
dataset = writer(
path,
mode,
driver=driver,
width=width,
height=height,
count=count,
crs=crs,
transform=transform,
dtype=dtype,
nodata=nodata,
sharing=sharing,
**kwargs
)
else:
raise DriverCapabilityError(
"Writer does not exist for driver: %s" % str(driver)
)
else:
raise DriverCapabilityError(
"mode must be one of 'r', 'r+', or 'w', not %s" % mode)
return dataset
def generate_cog_raster(fileset: Fileset, filename: str, **options):
# 1) Open file & reproject to epsg 4326 in memory
src = rasterio.open(filename)
src_crs = src.crs
crs = rasterio.crs.CRS({"init": "epsg:4326"})
transform, width, height = calculate_default_transform(
src_crs, crs, src.width, src.height, *src.bounds)
kwargs = src.meta.copy()
kwargs.update({
"crs": crs,
"transform": transform,
"width": width,
"height": height
})
memfile = MemoryFile()
with memfile.open(**kwargs) as mem:
for i in range(1, src.count + 1):
reproject(
source=rasterio.band(src, i),
destination=rasterio.band(mem, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
dst_crs=crs,
resampling=Resampling.nearest,
)
# 2) Convert image to COG
# Format creation option (see gdalwarp `-co` option)
output_profile = cog_profiles.get("deflate")
output_profile.update(dict(BIGTIFF="IF_SAFER"))
# Dataset Open option (see gdalwarp `-oo` option)
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=True,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
cog_filename = "/tmp/current_work_file.cog.tif"
with memfile.open() as src_mem:
cog_translate(
src_mem,
cog_filename,
output_profile,
config=config,
in_memory=True,
quiet=True,
use_cog_driver=True,
# web_optimized=True, # TODO: understand why this param hardcode epsg:3857 into the resulting file
**options,
)
# 3) Lookup where to save the new COG file
# Not doing any check, they where already done in validate_data_and_download()
# assume s3 bucket
src_filename = fileset.file_set.first().uri
dst_path = (get_object_key(get_base_dir(src_filename)) + "/" +
get_main_filename(src_filename) + ".cog.tif")
bucket = Bucket.objects.get(name=get_bucket_name(src_filename))
s3_api.upload_file(bucket=bucket,
object_key=dst_path,
src_path=cog_filename)
fileset.metadata["cog_raster_uri"] = f"s3://{bucket.name}/{dst_path}"
fileset.save()
def to_s3(self, bucket, key):
#https://github.com/mapbox/rasterio/issues/899#issuecomment-253133665
memfile = MemoryFile()
with memfile.open(**self.profile) as gtiff:
gtiff.write(self.arr)
s3.put_object(Bucket=bucket, Key=key, Body=memfile)
def create(scene, bands=None, expression=None):
"""Handler."""
scene_params = cbers_parse_scene_id(scene)
cbers_address = f'{CBERS_BUCKET}/{scene_params["key"]}'
if not expression and not bands:
raise Exception("Expression or Bands must be provided")
if bands:
nb_bands = len(bands)
data_type = np.uint8
if nb_bands != 3:
raise Exception("RGB combination only")
if expression:
bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression)))
rgb = expression.split(",")
data_type = np.float32
nb_bands = len(rgb)
bqa = f"{cbers_address}/{scene}_BAND6.tif"
with rasterio.open(bqa) as src:
meta = src.meta
wind = [w for ij, w in src.block_windows(1)]
meta.update(
nodata=0,
count=nb_bands,
interleave="pixel",
compress="DEFLATE",
photometric="MINISBLACK" if nb_bands == 1 else "RGB",
dtype=data_type,
)
memfile = MemoryFile()
with memfile.open(**meta) as dataset:
with contextlib.ExitStack() as stack:
srcs = [
stack.enter_context(
rasterio.open(f"{cbers_address}/{scene}_BAND{band}.tif")
)
for band in bands
]
def get_window(idx, window):
return srcs[idx].read(window=window, boundless=True, indexes=(1))
for window in wind:
_worker = partial(get_window, window=window)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
data = np.stack(list(executor.map(_worker, range(len(bands)))))
if expression:
ctx = {}
for bdx, b in enumerate(bands):
ctx["b{}".format(b)] = data[bdx]
data = np.array(
[
np.nan_to_num(ne.evaluate(bloc.strip(), local_dict=ctx))
for bloc in rgb
]
)
dataset.write(data.astype(data_type), window=window)
return memfile