def raise_low_pixels(memfile: MemoryFile, max_no_raise: float = 0.0, max_brightness: int = 255, noisy: bool = False) -> MemoryFile: """Detect very low (but above sea level) elevations in a raster and increase the elevation of the relevant pixels such that, when the file is converted to a greyscale image, those pixels will be given a value of 1, rather than rounded down to 0. """ with memfile.open() as src: data = src.read(1) # Floor all values at 0 data *= (data > 0) max_elev = data.max() min_visible = math.ceil( max_elev / max_brightness ) # Minimum value that will be rounded to 1 in a greyscale image print( f'Raising pixels of {max_no_raise} < elevation < {min_visible} to {min_visible}.' ) if noisy: # Add a small value to each affected pixel's elevation, which is proportionate to that pixel's original # elevation. This ensures that fixed areas do not have a uniform elevation (unless they originally had # a uniform elevation), so that they are not picked up by Microdem as lakes. min_visible = (data / 10) + min_visible data = np.where((data > max_no_raise) & (data < min_visible), min_visible, data) dst_memfile = MemoryFile() kwargs = src.profile.copy() with dst_memfile.open(**kwargs) as dst: dst.write(data, 1) return dst_memfile
def _open_and_stack_bands(self, coordinate, date, bands): """Loads band file rasters at path specified by arguments and stacks bands into single raster Args: coordinate (object): coordinate information - to be precised in child class date (object): date information - to be precised in child class bands (list[str]): list of band files to load Returns: type: rasterio.io.DatasetReader """ # Load metadata from first band - assume similar for other bands meta = self.get_meta(coordinate=coordinate, date=date, band=bands[0]) meta.update({'count': len(bands)}) # Create temporary in-memory file memory_file = MemoryFile() # Write new scene containing all bands from directory specified in kwargs with memory_file.open(**meta) as target_raster: for idx, band in enumerate(bands): with self(coordinate=coordinate, date=date, bands=[band]) as source_raster: target_raster.write_band(idx + 1, source_raster.read(1)) return memory_file.open()
def resample(memfile: MemoryFile, scale_factor: float) -> MemoryFile: """Resample raster by a factor of scale_factor. scale_factor > 1: Upsample scale factor < 1: Downsample """ print(f'Resampling raster with scaling factor of {scale_factor}.') with memfile.open() as src: print(f'Source raster has shape {src.shape}.') # resample data to target shape height = int(src.height * scale_factor) width = int(src.width * scale_factor) data = src.read(out_shape=(src.count, height, width), resampling=Resampling.bilinear) # scale image transform transform = src.transform * src.transform.scale(width, height) kwargs = src.profile.copy() kwargs.update({'height': height, 'width': width}) dst_memfile = MemoryFile() with dst_memfile.open(**kwargs) as dst: for i in range(1, src.count + 1): dst.write(data) print(f'Resampled raster has shape {dst.shape}.') return dst_memfile
def reproject_raster(memfile: MemoryFile, dst_crs: str, src_crs: str = WGS84) -> MemoryFile: """Reproject raster with CRS src_crs to new CRS dst_crs.""" print(f'Reprojecting raster from {src_crs} to {dst_crs}.') with memfile.open() as src: print(f'Source raster has shape {src.shape}.') transform, width, height = calculate_default_transform( src_crs, dst_crs, src.width, src.height, *src.bounds) kwargs = src.profile.copy() kwargs.update({ 'crs': dst_crs, 'transform': transform, 'width': width, 'height': height }) dst_memfile = MemoryFile() with dst_memfile.open(**kwargs) as dst: for i in range(1, src.count + 1): reproject(source=rasterio.band(src, i), destination=rasterio.band(dst, i), src_transform=src.transform, src_crs=src_crs, dst_transform=transform, dst_crs=dst_crs, resampling=Resampling.bilinear) print(f'Reprojected raster has shape {dst.shape}.') return dst_memfile
def __update_dataset(self, crs, transform, nodata=None): """Update dataset without writing to file after it theoretically changed. :param crs: crs of the dataset :param transform: transform of the dataset :param nodata: nodata value, optional :return: file in memory, open as dataset """ meta = { "driver": "GTiff", "dtype": self.__arr.dtype, "nodata": nodata, "height": self.__arr.shape[-2], "width": self.__arr.shape[-1], "count": self.__arr.shape[0], "crs": crs, "transform": transform, } memfile = MemoryFile() with memfile.open(**meta) as ds: ds.write(self.__arr) self.dataset = memfile.open() memfile.close()
def set_lakes_to_elev(memfile: MemoryFile, min_lake_size: int, fill_lakes_as: int = None, nodata: int = SRTM_NODATA) -> MemoryFile: """Find all lakes in the data for a raster and set the elevation of the relevant pixels to fill_lakes_as. """ if fill_lakes_as is None: fill_lakes_as = nodata print( f'Finding lakes with minimum size of {min_lake_size} and setting elevation to {fill_lakes_as}.' ) with memfile.open() as src: data = src.read(1) lakes = get_all_lakes(data, min_lake_size) print(f'Found {len(lakes)} lakes.') for lake in lakes: for row, col in lake: data[row, col] = fill_lakes_as dst_memfile = MemoryFile() kwargs = src.profile.copy() with dst_memfile.open(**kwargs) as dst: dst.write(data, 1) return dst_memfile
def in_memory_raster(array, meta): """Encapsulates image array and metadata as raster instance in reading mode stored in memory i.e. no actual file written Args: array (np.ndarray): raster image content meta (dict): raster metadata Returns: type: rasterio.io.DatasetReader """ memory_file = MemoryFile() with memory_file.open(**meta) as raster: raster.write(array) return memory_file.open()
def create(scenes, bands=[4, 3, 2]): """Handler.""" _worker = partial(worker, bands=bands) with futures.ThreadPoolExecutor(max_workers=10) as executor: responses = executor.map(_worker, scenes) with contextlib.ExitStack() as stack: sources = [ stack.enter_context(rasterio.open(scene)) for scene in responses if scene ] dest, output_transform = merge(sources, nodata=0) meta = { "driver": "GTiff", "count": 3, "dtype": np.uint8, "nodata": 0, "height": dest.shape[1], "width": dest.shape[2], "compress": "JPEG", "crs": "epsg:3857", "transform": output_transform, } memfile = MemoryFile() with memfile.open(**meta) as dataset: dataset.write(dest) wgs_bounds = transform_bounds(*[dataset.crs, "epsg:4326"] + list(dataset.bounds), densify_pts=21) return memfile, wgs_bounds
def create(tiles): """Handler.""" with futures.ThreadPoolExecutor(max_workers=8) as executor: responses = executor.map(worker, tiles) with contextlib.ExitStack() as stack: sources = [ stack.enter_context(rasterio.open(tile)) for tile in responses if tile ] dest, output_transform = merge(sources, nodata=-32767) meta = { "driver": "GTiff", "count": 1, "dtype": np.int16, "nodata": -32767, "height": dest.shape[1], "width": dest.shape[2], "compress": "DEFLATE", "crs": "epsg:4326", "transform": output_transform, } memfile = MemoryFile() with memfile.open(**meta) as dataset: dataset.write(dest) return memfile
def to_png(memfile: MemoryFile, zero_floor: bool = False, max_brightness: int = 255, nodata: int = SRTM_NODATA): """Save raster as a greyscale PNG file to to_file. If set_negative is set, any elevation values below zero are set to that value (which should be in the range 0-255). """ print(f'Converting raster to PNG image.') with memfile.open() as src: data = src.read(1) data[data == nodata] = 0 if zero_floor: data[data < 0] = 0 max_elev = data.max() min_elev = data.min() scale_factor = max_brightness / (max_elev - min_elev) if min_elev > 0: # If everywhere on the map is above sea level, we should scale # such that the lowest parts of the map appear slightly above sea level. floor = min_elev + 1 else: floor = min_elev data = ((data - floor) * scale_factor).astype(np.uint8) im = Image.fromarray(data, mode='L') width, height = im.size print(f'Image size is {width}x{height}.') return im
def test_memory_file_gdal_error_message(capsys): """No weird error messages should be seen, see #1659""" memfile = MemoryFile() data = numpy.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]).astype('uint8') west_bound = 0 north_bound = 2 cellsize = 0.5 nodata = -9999 driver = 'AAIGrid' dtype = data.dtype shape = data.shape transform = rasterio.transform.from_origin(west_bound, north_bound, cellsize, cellsize) dataset = memfile.open(driver=driver, width=shape[1], height=shape[0], transform=transform, count=1, dtype=dtype, nodata=nodata, crs='epsg:3226') dataset.write(data, 1) dataset.close() captured = capsys.readouterr() assert "ERROR 4" not in captured.err assert "ERROR 4" not in captured.out
class RasterWindowMemoryFile(): """Context manager around rasterio.io.MemoryFile.""" def __init__(self, in_tile=None, in_data=None, out_profile=None, out_tile=None, tags=None): """Prepare data & profile.""" out_tile = out_tile or in_tile validate_write_window_params(in_tile, out_tile, in_data, out_profile) self.data = extract_from_array(in_raster=in_data, in_affine=in_tile.affine, out_tile=out_tile) # use transform instead of affine if "affine" in out_profile: out_profile["transform"] = out_profile.pop("affine") self.profile = out_profile self.tags = tags def __enter__(self): """Open MemoryFile, write data and return.""" self.rio_memfile = MemoryFile() with self.rio_memfile.open(**self.profile) as dst: dst.write(self.data.astype(self.profile["dtype"], copy=False)) _write_tags(dst, self.tags) return self.rio_memfile def __exit__(self, *args): """Make sure MemoryFile is closed.""" self.rio_memfile.close()
def reproject_like(this, reproject_this): dst = MemoryFile() d_transform, d_width, d_height = calculate_default_transform( reproject_this.crs, this.crs, reproject_this.width, reproject_this.height, *reproject_this.bounds) dst_profile = reproject_this.profile.copy() dst_profile.update({ "driver": "GTiff", "crs": this.crs, "transform": d_transform, "width": d_width, "height": d_height, }) # with rio.open(dst, 'w', **dst_profile) as dst_file: reproject(source=reproject_this.read(1), destination=rio.band(dst_file, 1), src_transform=reproject_this.transform, src_crs=reproject_this.crs, dst_transform=d_transform, dst_crs=this.crs, resampling=Resampling.bilinear) dst.seek(0) return dst.open()
def fp_reader(fp): memfile = MemoryFile(fp.read()) dataset = memfile.open() try: yield dataset finally: dataset.close() memfile.close()
def fp_reader(fp): memfile = MemoryFile(fp.read()) dataset = memfile.open(driver=driver, sharing=sharing) try: yield dataset finally: dataset.close() memfile.close()
def get_bounding_box_from_file(file): data = file.read() try: # Intenta abrir la imagen. De no ser una imagen, se informa al cliente memfile = MemoryFile(data) dataset = memfile.open() return get_bounding_box(dataset) except rio.errors.RasterioIOError: response = {'error': 'File is not an image'} return response
def raise_undersea_land(memfile: MemoryFile, raise_to: int = 1): """Raise land with a negative elevation (ie, land that is below sea level) to raise_to. Probably shouldn't be called before set_lakes_to_elev, otherwise the newly raised land will be picked up as a lake by that function. """ with memfile.open() as src: data = src.read(1) print(f'Raising pixels of elevation < 0 to {raise_to}.') data = np.where((data < 0), raise_to, data) dst_memfile = MemoryFile() kwargs = src.profile.copy() with dst_memfile.open(**kwargs) as dst: dst.write(data, 1) return dst_memfile
def fp_writer(fp): memfile = MemoryFile() dataset = memfile.open(driver=driver, width=width, height=height, count=count, crs=crs, transform=transform, dtype=dtype, nodata=nodata, **kwargs) try: yield dataset finally: dataset.close() memfile.seek(0) fp.write(memfile.read()) memfile.close()
def fp_writer(fp): memfile = MemoryFile() dataset = memfile.open(driver=driver, width=width, height=height, count=count, crs=crs, transform=transform, dtype=dtype, nodata=nodata, sharing=sharing, **kwargs) try: yield dataset finally: dataset.close() memfile.seek(0) fp.write(memfile.read()) memfile.close()
def clip(array=None, profile=None, vector=None, raster=None, all_touched=True, driver='GTiff', masked=True, extent=False): """Clip a raster file. Takes either a ndarray + profile or a path to a raster file. Also takes a vector as path or GeoDataFrame. Returns a ndarray + profile. """ if type(vector) == str: # with fiona.open(shp, 'r') as shapefile: # shapes = [feature['geometry'] for feature in shapefile] gdf = gpd.read_file(vector) elif type(vector) == gpd.geodataframe.GeoDataFrame: gdf = vector else: raise ValueError('Please specify a valid vector path or GeoDataFrame.') if raster is not None: with rasterio.open(raster) as src: array = src.read(masked=masked) profile = src.profile elif array is None or profile is None: raise ValueError('Please supply either a raster file or a ndarray and ' 'a profile') if extent: polygon = bbox_polygon(*gdf.total_bounds) shapes = [mapping(polygon)] else: shapes = [feat['geometry'] for feat in gdf.geometry.__geo_interface__['features']] memfile = MemoryFile() with memfile.open(**profile) as mem: mem.write(array) dst_array, dst_transform = riomask.mask(mem, shapes, all_touched=all_touched, crop=True, filled=masked) dst_profile = profile.copy() dst_profile.update({"height": dst_array.shape[1], "width": dst_array.shape[2], "transform": dst_transform, "driver": driver}) print('---Clipping successful---') return dst_array, dst_profile
def remove_sea(memfile: MemoryFile, min_elev: int = 1) -> MemoryFile: """Offset elevation data so that lowest value is equal to min_elev. Useful for when real-world data includes land below sea level but no sea (eg, an in-land NL map). By default,min_elev is 1, which means you may need to use this in conjunction with raise_low_pixels to actually render as land. """ with memfile.open() as src: data = src.read(1) offset = -(data.min() - min_elev) print(f'Increasing elevation by {offset}.') data += offset dst_memfile = MemoryFile() kwargs = src.profile.copy() with dst_memfile.open(**kwargs) as dst: dst.write(data, 1) return dst_memfile
def test_memory_file_gdal_error_message(capsys): """No weird error messages should be seen, see #1659""" memfile = MemoryFile() data = numpy.array([[1,2,3,4],[5,6,7,8],[9,10,11,12],[13,14,15,16]]).astype('uint8') west_bound = 0; north_bound = 2; cellsize=0.5; nodata = -9999; driver='AAIGrid'; dtype = data.dtype shape = data.shape transform = rasterio.transform.from_origin(west_bound, north_bound, cellsize, cellsize) dataset = memfile.open(driver=driver, width=shape[1], height=shape[0], transform=transform, count=1, dtype=dtype, nodata=nodata, crs='epsg:3226') dataset.write(data, 1) dataset.close() captured = capsys.readouterr() assert "ERROR 4" not in captured.err assert "ERROR 4" not in captured.out
class FakeCog(ImageBase): """fake cloud optimized geotiff""" src_img: FakeImage dst_kwargs: Dict indexes: Optional[Sequence[int]] = None nodata: Optional[Union[int, float]] = None dtype: Optional[str] = None add_mask: bool = False overview_level: Optional[int] = None overview_resampling: str = "nearest" web_optimized: bool = False latitude_adjustment: bool = True resampling: str = "nearest" config: Optional[Dict] = None allow_intermediate_compression: bool = False forward_band_tags: bool = False quiet: bool = False temporary_compression: str = "DEFLATE" def __post_init__(self): """post init hook""" self.memfile = MemoryFile() cog_translate( source=self.src_img.handle, dst_path=self.memfile.name, dst_kwargs=self.dst_kwargs, indexes=self.indexes, nodata=self.nodata, dtype=self.dtype, add_mask=self.add_mask, overview_level=self.overview_level, overview_resampling=self.overview_resampling, web_optimized=self.web_optimized, latitude_adjustment=self.latitude_adjustment, resampling=self.resampling, in_memory=True, config=self.config, allow_intermediate_compression=self.allow_intermediate_compression, forward_band_tags=self.forward_band_tags, quiet=self.quiet, temporary_compression=self.temporary_compression, ) self.handle = self.memfile.open() def close(self): """close resources""" self.handle.close() self.memfile.close()
def create_tif_file(left: float, bottom: float, right: float, top: float, to_file: Optional[str] = None, cache_dir: str = CACHE_DIR, nodata: int = SRTM_NODATA) -> Union[str, MemoryFile]: """Create a TIF file using SRTM data for the box defined by left, bottom, right, top. If to_file is provided, saves the resulting file to to_file and returns the path; otherwise, creates a rasterio.io.MemoryFile and returns that. """ os.makedirs(cache_dir, exist_ok=True) xy = get_all_xy_components(left, bottom, right, top) zip_fnames = {} for x, y in xy: zip_fnames[(x, y)] = ZIP_FNAME.format(x=x, y=y) zip_fpaths = fetch_all_zips(zip_fnames, cache_dir) unzip_all(zip_fpaths.values(), cache_dir) srcs = [ rasterio.open(get_tif_fpath(x, y, cache_dir), 'r', nodata=nodata) for x, y in xy ] print(f'Creating TIF file from following files: {[s.name for s in srcs]}.') #print(f'Heights are: {[s.height for s in srcs]}.') #print(f'Widths are: {[s.width for s in srcs]}.') profile = srcs[0].profile data, transform = rasterio.merge.merge(srcs, (left, bottom, right, top), nodata=nodata) for src in srcs: src.close() clear_cache(cache_dir, True) bands, height, width = data.shape # No idea if this is the correct order for height and width, but they are both # the same so it doesn't matter in this case profile.update({'height': height, 'width': width, 'transform': transform}) print(f'Created TIF file with dimensions {width}x{height}.') if to_file: print(f'Writing TIF file to {to_file}.') with rasterio.open(to_file, 'w', **profile) as dst: dst.write(data) return to_file else: memfile = MemoryFile() with memfile.open(**profile) as dst: dst.write(data) return memfile
def write_raster_window( in_tile=None, in_data=None, out_profile=None, out_tile=None, out_path=None ): """ Write a window from a numpy array to an output file. Parameters ---------- in_tile : ``BufferedTile`` ``BufferedTile`` with a data attribute holding NumPy data in_data : array out_profile : dictionary metadata dictionary for rasterio out_tile : ``Tile`` provides output boundaries; if None, in_tile is used out_path : string output path to write to; if output path is "memoryfile", a rasterio.MemoryFile() is returned """ out_tile = in_tile if out_tile is None else out_tile for t in [in_tile, out_tile]: if not isinstance(t, BufferedTile): raise TypeError("in_tile and out_tile must be BufferedTile") if not isinstance(in_data, ma.MaskedArray): raise TypeError("in_data must be ma.MaskedArray") if not isinstance(out_profile, dict): raise TypeError("out_profile must be a dictionary") if not isinstance(out_path, six.string_types): raise TypeError("out_path must be a string") window_data = extract_from_array( in_raster=in_data, in_affine=in_tile.affine, out_tile=out_tile) # use transform instead of affine if "affine" in out_profile: out_profile["transform"] = out_profile.pop("affine") # write if there is any band with non-masked data if window_data.all() is not ma.masked: if out_path == "memoryfile": memfile = MemoryFile() with memfile.open(**out_profile) as dst: for band, data in enumerate(window_data): dst.write(data.astype(out_profile["dtype"]), band + 1) return memfile else: with rasterio.open(out_path, 'w', **out_profile) as dst: for band, data in enumerate(window_data): dst.write(data.astype(out_profile["dtype"]), band + 1)
def memory_file(data=None, profile=None): """ Return a rasterio.io.MemoryFile instance from input. Parameters ---------- data : array array to be written profile : dict rasterio profile for MemoryFile """ memfile = MemoryFile() with memfile.open(**dict( profile, width=data.shape[-2], height=data.shape[-1])) as dataset: dataset.write(data) return memfile
def make_masked_raster(polygon, resolution, bands=1, all_touched=False, nodata=-9999., crs='+init=epsg:27700', filename=None): """Generates a raster with points outside poly set to nodata. Pixels are set to be of dimension res x res""" x = np.arange(polygon.bounds['minx'].values, polygon.bounds['maxx'].values, resolution) y = np.arange(polygon.bounds['miny'].values, polygon.bounds['maxy'].values, resolution) X, Y = np.meshgrid(x, y) Z = np.full(X.shape, 1.) transform = from_origin(x[0] - resolution / 2, y[-1] + resolution / 2, resolution, resolution) raster_args = { 'driver': 'GTiff', 'height': Z.shape[0], 'width': Z.shape[1], 'count': bands, 'dtype': Z.dtype, 'crs': polygon.crs, 'transform': transform, 'nodata': nodata } if filename is None: memfile = MemoryFile() raster = memfile.open(**raster_args) else: raster = rasterio.open(filename, 'w+', **raster_args) for i in range(bands): raster.write(Z, i + 1) mask = rasterio.mask.mask(raster, polygon.geometry, crop=True, all_touched=all_touched, filled=True) for i in range(bands): raster.write(mask[0][i], i + 1) return raster
def _get(self, name, bbox, width, height, time, x, y, r_flag): """Return the image value for a location. Args: name (str): The image(coverage) name to retrieve from service. bbox (str): The extent of the image(coverage) to retrieve. x (int/float): A longitude value according to EPSG:4326. y (int/float): A latitude value according to EPSG:4326. """ output = self.wcs_owslib.getCoverage(identifier=name, format='GeoTIFF', bbox=bbox, crs='EPSG:4326', time=[time], width=width, height=height) data_array = None data = output.read() result = dict() try: memfile = MemoryFile(data) dataset = memfile.open() values = list(dataset.sample([(x, y)])) if r_flag: result['geom'] = dataset.read() memfile = None dataset = None data = None result['raster_value'] = values[0] return result except: return None
def open(fp, mode='r', driver=None, width=None, height=None, count=None, crs=None, transform=None, dtype=None, nodata=None, sharing=False, **kwargs): """Open a dataset for reading or writing. The dataset may be located in a local file, in a resource located by a URL, or contained within a stream of bytes. In read ('r') or read/write ('r+') mode, no keyword arguments are required: these attributes are supplied by the opened dataset. In write ('w' or 'w+') mode, the driver, width, height, count, and dtype keywords are strictly required. Parameters ---------- fp : str, file object, PathLike object, FilePath, or MemoryFile A filename or URL, a file object opened in binary ('rb') mode, a Path object, or one of the rasterio classes that provides the dataset-opening interface (has an open method that returns a dataset). mode : str, optional 'r' (read, the default), 'r+' (read/write), 'w' (write), or 'w+' (write/read). driver : str, optional A short format driver name (e.g. "GTiff" or "JPEG") or a list of such names (see GDAL docs at https://gdal.org/drivers/raster/index.html). In 'w' or 'w+' modes a single name is required. In 'r' or 'r+' modes the driver can usually be omitted. Registered drivers will be tried sequentially until a match is found. When multiple drivers are available for a format such as JPEG2000, one of them can be selected by using this keyword argument. width : int, optional The number of columns of the raster dataset. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes. height : int, optional The number of rows of the raster dataset. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes. count : int, optional The count of dataset bands. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes. crs : str, dict, or CRS; optional The coordinate reference system. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes. transform : Affine instance, optional Affine transformation mapping the pixel space to geographic space. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes. dtype : str or numpy dtype The data type for bands. For example: 'uint8' or ``rasterio.uint16``. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes. nodata : int, float, or nan; optional Defines the pixel value to be interpreted as not valid data. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes. sharing : bool; optional To reduce overhead and prevent programs from running out of file descriptors, rasterio maintains a pool of shared low level dataset handles. When `True` this function will use a shared handle if one is available. Multithreaded programs must avoid sharing and should set *sharing* to `False`. kwargs : optional These are passed to format drivers as directives for creating or interpreting datasets. For example: in 'w' or 'w+' modes a `tiled=True` keyword argument will direct the GeoTIFF format driver to create a tiled, rather than striped, TIFF. Returns ------- A ``DatasetReader`` or ``DatasetWriter`` object. Examples -------- To open a GeoTIFF for reading using standard driver discovery and no directives: >>> import rasterio >>> with rasterio.open('example.tif') as dataset: ... print(dataset.profile) To open a JPEG2000 using only the JP2OpenJPEG driver: >>> with rasterio.open( ... 'example.jp2', driver='JP2OpenJPEG') as dataset: ... print(dataset.profile) To create a new 8-band, 16-bit unsigned, tiled, and LZW-compressed GeoTIFF with a global extent and 0.5 degree resolution: >>> from rasterio.transform import from_origin >>> with rasterio.open( ... 'example.tif', 'w', driver='GTiff', dtype='uint16', ... width=720, height=360, count=8, crs='EPSG:4326', ... transform=from_origin(-180.0, 90.0, 0.5, 0.5), ... nodata=0, tiled=True, compress='lzw') as dataset: ... dataset.write(...) """ if not isinstance(fp, str): if not ( hasattr(fp, "read") or hasattr(fp, "write") or isinstance(fp, (os.PathLike, MemoryFile, FilePath)) ): raise TypeError("invalid path or file: {0!r}".format(fp)) if mode and not isinstance(mode, str): raise TypeError("invalid mode: {0!r}".format(mode)) if driver and not isinstance(driver, str): raise TypeError("invalid driver: {0!r}".format(driver)) if dtype and not check_dtype(dtype): raise TypeError("invalid dtype: {0!r}".format(dtype)) if nodata is not None: nodata = float(nodata) if transform: transform = guard_transform(transform) # Check driver/mode blacklist. if driver and is_blacklisted(driver, mode): raise RasterioIOError( "Blacklisted: file cannot be opened by " "driver '{0}' in '{1}' mode".format(driver, mode)) # If the fp argument is a file-like object and can be adapted by # rasterio's FilePath we do so. Otherwise, we use a MemoryFile to # hold fp's contents and store that in an ExitStack attached to the # dataset object that we will return. When a dataset's close method # is called, this ExitStack will be unwound and the MemoryFile's # storage will be cleaned up. if mode == 'r' and hasattr(fp, 'read'): if have_vsi_plugin: return FilePath(fp).open(driver=driver, sharing=sharing, **kwargs) else: memfile = MemoryFile(fp.read()) dataset = memfile.open(driver=driver, sharing=sharing, **kwargs) dataset._env.enter_context(memfile) return dataset elif mode in ('w', 'w+') and hasattr(fp, 'write'): memfile = MemoryFile() dataset = memfile.open( driver=driver, width=width, height=height, count=count, crs=crs, transform=transform, dtype=dtype, nodata=nodata, sharing=sharing, **kwargs ) dataset._env.enter_context(memfile) # For the writing case we push an extra callback onto the # ExitStack. It ensures that the MemoryFile's contents are # copied to the open file object. def func(*args, **kwds): memfile.seek(0) fp.write(memfile.read()) dataset._env.callback(func) return dataset # TODO: test for a shared base class or abstract type. elif isinstance(fp, (FilePath, MemoryFile)): if mode.startswith("r"): dataset = fp.open(driver=driver, sharing=sharing, **kwargs) # Note: FilePath does not support writing and an exception will # result from this. elif mode.startswith("w"): dataset = fp.open( driver=driver, width=width, height=height, count=count, crs=crs, transform=transform, dtype=dtype, nodata=nodata, sharing=sharing, **kwargs ) return dataset # At this point, the fp argument is a string or path-like object # which can be converted to a string. else: raw_dataset_path = os.fspath(fp) path = _parse_path(raw_dataset_path) if mode == "r": dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs) elif mode == "r+": dataset = get_writer_for_path(path, driver=driver)( path, mode, driver=driver, sharing=sharing, **kwargs ) elif mode.startswith("w"): if not driver: driver = driver_from_extension(path) writer = get_writer_for_driver(driver) if writer is not None: dataset = writer( path, mode, driver=driver, width=width, height=height, count=count, crs=crs, transform=transform, dtype=dtype, nodata=nodata, sharing=sharing, **kwargs ) else: raise DriverCapabilityError( "Writer does not exist for driver: %s" % str(driver) ) else: raise DriverCapabilityError( "mode must be one of 'r', 'r+', or 'w', not %s" % mode) return dataset
def generate_cog_raster(fileset: Fileset, filename: str, **options): # 1) Open file & reproject to epsg 4326 in memory src = rasterio.open(filename) src_crs = src.crs crs = rasterio.crs.CRS({"init": "epsg:4326"}) transform, width, height = calculate_default_transform( src_crs, crs, src.width, src.height, *src.bounds) kwargs = src.meta.copy() kwargs.update({ "crs": crs, "transform": transform, "width": width, "height": height }) memfile = MemoryFile() with memfile.open(**kwargs) as mem: for i in range(1, src.count + 1): reproject( source=rasterio.band(src, i), destination=rasterio.band(mem, i), src_transform=src.transform, src_crs=src.crs, dst_transform=transform, dst_crs=crs, resampling=Resampling.nearest, ) # 2) Convert image to COG # Format creation option (see gdalwarp `-co` option) output_profile = cog_profiles.get("deflate") output_profile.update(dict(BIGTIFF="IF_SAFER")) # Dataset Open option (see gdalwarp `-oo` option) config = dict( GDAL_NUM_THREADS="ALL_CPUS", GDAL_TIFF_INTERNAL_MASK=True, GDAL_TIFF_OVR_BLOCKSIZE="128", ) cog_filename = "/tmp/current_work_file.cog.tif" with memfile.open() as src_mem: cog_translate( src_mem, cog_filename, output_profile, config=config, in_memory=True, quiet=True, use_cog_driver=True, # web_optimized=True, # TODO: understand why this param hardcode epsg:3857 into the resulting file **options, ) # 3) Lookup where to save the new COG file # Not doing any check, they where already done in validate_data_and_download() # assume s3 bucket src_filename = fileset.file_set.first().uri dst_path = (get_object_key(get_base_dir(src_filename)) + "/" + get_main_filename(src_filename) + ".cog.tif") bucket = Bucket.objects.get(name=get_bucket_name(src_filename)) s3_api.upload_file(bucket=bucket, object_key=dst_path, src_path=cog_filename) fileset.metadata["cog_raster_uri"] = f"s3://{bucket.name}/{dst_path}" fileset.save()
def to_s3(self, bucket, key): #https://github.com/mapbox/rasterio/issues/899#issuecomment-253133665 memfile = MemoryFile() with memfile.open(**self.profile) as gtiff: gtiff.write(self.arr) s3.put_object(Bucket=bucket, Key=key, Body=memfile)
def create(scene, bands=None, expression=None): """Handler.""" scene_params = cbers_parse_scene_id(scene) cbers_address = f'{CBERS_BUCKET}/{scene_params["key"]}' if not expression and not bands: raise Exception("Expression or Bands must be provided") if bands: nb_bands = len(bands) data_type = np.uint8 if nb_bands != 3: raise Exception("RGB combination only") if expression: bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) rgb = expression.split(",") data_type = np.float32 nb_bands = len(rgb) bqa = f"{cbers_address}/{scene}_BAND6.tif" with rasterio.open(bqa) as src: meta = src.meta wind = [w for ij, w in src.block_windows(1)] meta.update( nodata=0, count=nb_bands, interleave="pixel", compress="DEFLATE", photometric="MINISBLACK" if nb_bands == 1 else "RGB", dtype=data_type, ) memfile = MemoryFile() with memfile.open(**meta) as dataset: with contextlib.ExitStack() as stack: srcs = [ stack.enter_context( rasterio.open(f"{cbers_address}/{scene}_BAND{band}.tif") ) for band in bands ] def get_window(idx, window): return srcs[idx].read(window=window, boundless=True, indexes=(1)) for window in wind: _worker = partial(get_window, window=window) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.stack(list(executor.map(_worker, range(len(bands))))) if expression: ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] data = np.array( [ np.nan_to_num(ne.evaluate(bloc.strip(), local_dict=ctx)) for bloc in rgb ] ) dataset.write(data.astype(data_type), window=window) return memfile