def tiffs(tmpdir):
data = np.ones((1, 1, 1), 'uint8')
kwargs = {
'count': '1',
'driver': 'GTiff',
'dtype': 'uint8',
'height': 1,
'width': 1
}
kwargs['transform'] = Affine(1, 0, 1, 0, -1, 1)
with rasterio.open(str(tmpdir.join('a-sw.tif')), 'w', **kwargs) as r:
r.write(data * 40)
kwargs['transform'] = Affine(1, 0, 2, 0, -1, 2)
with rasterio.open(str(tmpdir.join('b-ct.tif')), 'w', **kwargs) as r:
r.write(data * 60)
kwargs['transform'] = Affine(2, 0, 3, 0, -2, 4)
with rasterio.open(str(tmpdir.join('c-ne.tif')), 'w', **kwargs) as r:
r.write(data * 90)
kwargs['transform'] = Affine(2, 0, 2, 0, -2, 4)
with rasterio.open(str(tmpdir.join('d-ne.tif')), 'w', **kwargs) as r:
r.write(data * 120)
return tmpdir
def predictFromTreeDEM(fp, dtree, outfn):
hh_msk, hv_msk = readBands(fp)
#print('HH read msk contains nan: '+str(np.isnan(hh_msk).any()))
#print('HV read msk contains nan: '+str(np.isnan(hv_msk).any()))
hh_msk_flat = hh_msk.flatten()
hv_msk_flat = hv_msk.flatten()
hhhvDEM_stacked = np.stack((hh_msk_flat, hv_msk_flat, dem_flat), axis=1)
pred = dtree.predict(hhhvDEM_stacked)
pred_reshaped = pred.reshape(-1, 11908)
meta_outFile = {
'driver':
'GTiff',
'dtype':
'float32',
'nodata':
-9999.0,
'width':
11908,
'height':
12600,
'count':
1,
'crs':
CRS.from_epsg(3413),
'transform':
Affine(10.0, 0.0, -384702.1263054441, 0.0, -10.0, -2122443.806211936)
}
with rio.open(outfn, 'w', **meta_outFile) as outTif:
outTif.write(pred_reshaped, indexes=1)
def get_web_optimized_params(
src_dst,
tilesize=256,
latitude_adjustment: bool = True,
warp_resampling: str = "nearest",
grid_crs=CRS.from_epsg(3857),
) -> Dict:
"""Return VRT parameters for a WebOptimized COG."""
bounds = list(
transform_bounds(src_dst.crs,
CRS.from_epsg(4326),
*src_dst.bounds,
densify_pts=21))
center = [(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2]
lat = 0 if latitude_adjustment else center[1]
_, max_zoom = get_zooms(src_dst, lat=lat, tilesize=tilesize)
extrema = tile_extrema(bounds, max_zoom)
left, _, _, top = mercantile.xy_bounds(extrema["x"]["min"],
extrema["y"]["min"], max_zoom)
vrt_res = _meters_per_pixel(max_zoom, 0, tilesize=tilesize)
vrt_transform = Affine(vrt_res, 0, left, 0, -vrt_res, top)
vrt_width = (extrema["x"]["max"] - extrema["x"]["min"]) * tilesize
vrt_height = (extrema["y"]["max"] - extrema["y"]["min"]) * tilesize
return dict(
crs=grid_crs,
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
resampling=ResamplingEnums[warp_resampling],
)
def preprocess_bands(self, band_stack, subset_box=None):
bands_rescaled = band_stack[0:4].copy()
bands_rescaled[np.isnan(bands_rescaled)] = 0
bands_rescaled = rescale_s2(bands_rescaled)
bands_rescaled[bands_rescaled == 0] = np.nan
band_stack = None
self.lat, self.lon = lat_from_meta(self.meta), lon_from_meta(self.meta)
if subset_box is not None:
ymin, ymax, xmin, xmax = subset_box["ymin"], subset_box[
"ymax"], subset_box["xmin"], subset_box["xmax"]
bands_rescaled = bands_rescaled[:, ymin:ymax, xmin:xmax]
self.lat, self.lon = self.lat[ymin:ymax], self.lon[xmin:xmax]
self.meta["height"], self.meta["width"] = bands_rescaled.shape[
1], bands_rescaled.shape[2]
t = list(self.meta["transform"])
t[2], t[5] = self.lon[0], self.lat[0]
self.meta["transform"] = Affine(t[0], t[1], t[2], t[3], t[4], t[5])
bbox_epsg4326 = list(np.flip(metadata_to_bbox_epsg4326(self.meta)))
osm_mask = self._get_osm_mask(
bbox_epsg4326, "EPSG:" + str(self.meta["crs"].to_epsg()),
bands_rescaled[0], {
"lat": self.lat,
"lon": self.lon
}, dirs["osm"])
if np.count_nonzero(osm_mask) == 0:
raise ValueError("No OSM roads of requested road types in aoi")
bands_rescaled *= osm_mask
bands_rescaled[bands_rescaled == 0] = np.nan
osm_mask = None
self._build_variables(bands_rescaled)
def __init__(self, pixel_size, raster_data=None, **kwargs):
logger.info("Preparing a base dataframe...")
x_min, y_min, x_max, y_max = -180, -90, 180, 90 # global
self.pixel_size = pixel_size
self.x_res = int((x_max - x_min) / self.pixel_size)
self.y_res = int((y_max - y_min) / self.pixel_size)
self.base_layer = RasterEnvironmentalLayer()
self.base_layer.raster_affine = Affine(pixel_size, 0.0, x_min, 0.0,
-pixel_size, y_max)
logger.info("Base layer: Computing world coordinates...")
if raster_data is not None:
all_coordinates = self.base_layer.pixel_to_world_coordinates(
raster_data=raster_data)
else:
all_coordinates = self.base_layer.pixel_to_world_coordinates(
raster_data=np.ones((self.y_res, self.x_res), dtype=np.uint8))
# self.base_dataframe = pd.DataFrame([all_coordinates[0], all_coordinates[1]]).T
# self.base_dataframe.columns = ['decimallatitude', 'decimallongitude']
self.base_dataframe = pd.DataFrame(
columns=['decimallatitude', 'decimallongitude'])
self.base_dataframe['decimallatitude'] = np.array(all_coordinates[0])
self.base_dataframe['decimallongitude'] = np.array(all_coordinates[1])
self.base_dataframe.set_index(['decimallatitude', 'decimallongitude'],
inplace=True,
drop=True)
del all_coordinates
gc.collect()
def write_cog(layout, location, data, filename_template):
"""
Write a GPS tile out as a Cloud-Optimized GeoTIFF
Uses GDAL to write a COG out to disk, utilizing the given layout and location
to generate the GeoTIFF header.
Args:
layout (``gps.LayoutDefinition``): The layout for the source layer
location (``gps.SpatialKey`` or ``gps.SpaceTimeKey``): The cell identifier
in the layer corresponding to the image data
data (``gps.Tile``): The image data
filename_template (str): A pattern giving the destination for the target
image. Contains two '{}' tokens which will be ``.format``ed with the
column and row of the ``location``, in that order. May be an S3 uri or
local path.
"""
bands, w, h = data.cells.shape
nodata = data.no_data_value
dtype = data.cells.dtype
cw, ch = cell_size(layout)
ex = extent_for_cell(layout, location)
overview_level = int(log(w) / log(2) - 8)
with rstr.io.MemoryFile() as memfile:
with memfile.open(driver='GTiff',
count=bands,
width=w,
height=h,
transform=Affine(cw, 0.0, ex.xmin, 0.0, -ch,
ex.ymax),
crs=rstr.crs.CRS.from_epsg(4326),
nodata=nodata,
dtype=dtype,
compress='lzw',
tiled=True) as mem:
windows = list(mem.block_windows(1))
for _, w in windows:
segment = data.cells[:, w.row_off:(w.row_off + w.height),
w.col_off:(w.col_off + w.width)]
mem.write(segment, window=w)
mask_value = np.all(segment != nodata, axis=0).astype(
np.uint8) * 255
mem.write_mask(mask_value, window=w)
overviews = [2**j for j in range(1, overview_level + 1)]
mem.build_overviews(overviews, rwarp.Resampling.nearest)
mem.update_tags(ns='rio_oveview',
resampling=rwarp.Resampling.nearest.value)
uri = urlparse.urlparse(filename_template)
if uri.scheme == 's3':
boto3.resource('s3').Bucket(uri.netloc).upload_fileobj(
memfile,
uri.path.format(location.col, location.row)[1:])
else:
rshutil.copy(mem,
filename_template.format(location.col,
location.row),
copy_src_overviews=True)
def pad(array, transform, pad_width, mode=None, **kwargs):
"""pad array and adjust affine transform matrix.
Parameters
----------
array: ndarray
Numpy ndarray, for best results a 2D array
transform: Affine transform
transform object mapping pixel space to coordinates
pad_width: int
number of pixels to pad array on all four
mode: str or function
define the method for determining padded values
Returns
-------
(array, transform): tuple
Tuple of new array and affine transform
Notes
-----
See numpy docs for details on mode and other kwargs:
http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.pad.html
"""
import numpy as np
transform = guard_transform(transform)
padded_array = np.pad(array, pad_width, mode, **kwargs)
padded_trans = list(transform)
padded_trans[2] -= pad_width * padded_trans[0]
padded_trans[5] -= pad_width * padded_trans[4]
return padded_array, Affine(*padded_trans[:6])
def download_tiles(self, scene, grid):
bbox, shape = grid
x, y = map(lambda coor: int(coor / self.resolution), shape)
if scene.geometry.intersects(bbox):
array = self.request(scene, bbox, (x, y))
diff = bbox.difference(scene.geometry)
if diff.area != 0:
x0, _, _, ye = bbox.bounds
transform = Affine(a=self.resolution,
b=0,
c=x0,
d=0,
e=-self.resolution,
f=ye)
mask = rasterize([(diff, True)],
out_shape=(y, x),
transform=transform,
fill=False,
all_touched=True)
array = numpy.where(mask, self.nodata, array)
else:
array = None
if array is not None:
return array
else:
return self.nodata_tile((x, y))
def read_buffered_window(uri, metadata, px_buffer, cell, resampling):
"""
Read a region from a source image, reprojecting in the process.
Given the metadata for a layer (including a layout), a cell id, and a buffer
size, read a window from the supplied image while reprojecting the source image
to the CRS specified by the metadata.
In concept, we specify a grid in the target CRS using the cell and resolution in
the metadata, padding with px_buffer additional cells on all sides of the cell.
For each cell center, we read from the source image after back-projecting the
target location to the source image's coordinate system and applying the chosen
resampling kernel.
Args:
uri (str): The source image URI; any scheme interpretable by GDAL is allowed
metadata (``gps.Metadata``): The source layer metadata
px_buffer (int): The number of pixels worth of padding that should be added
to the extent to be read and added to the tile.
cell (``gps.SpatialKey`` or ``gps.SpaceTimeKey``): The key of the target
region
resampling (``rasterio.warp.Resampling``): The resampling method to be used
while reading from the source image
Returns:
``gps.Tile``
"""
if ("GDAL_DATA" not in os.environ) and '_GDAL_DATA' in vars():
os.environ["GDAL_DATA"] = _GDAL_DATA
layout = metadata.layout_definition
dest_bounds = buffered_cell_extent(layout, px_buffer, cell)
res = cell_size(metadata.layout_definition)
dest_transform = Affine(res[0], 0, dest_bounds.xmin, 0, -res[1],
dest_bounds.ymax)
dest_width = max(int(ceil((dest_bounds.xmax - dest_bounds.xmin) / res[0])),
1)
dest_height = max(
int(ceil((dest_bounds.ymax - dest_bounds.ymin) / res[1])), 1)
dest_transform, dest_width, dest_height = rwarp.aligned_target(
dest_transform, dest_width, dest_height, res)
with rstr.open(uri) as src:
tile = np.zeros((src.count, layout.tileLayout.tileCols + 2 * px_buffer,
layout.tileLayout.tileRows + 2 * px_buffer))
rwarp.reproject(source=rstr.band(src, list(range(1, src.count + 1))),
destination=tile,
src_transform=src.transform,
src_crs=src.crs,
src_nodata=src.nodata,
dst_transform=dest_transform,
dst_crs=CRS.from_epsg(4326),
dst_nodata=src.nodata,
resampling=resampling,
num_threads=1)
return gps.Tile.from_numpy_array(tile)
def get_params():
return {
'rain': {
'height': 129,
'width': 135,
'affine': Affine(0.25, 0, 66.375, 0, -0.25, 38.625)
},
'tmin': {
'height': 31,
'width': 31,
'affine': Affine(1, 0, 67, 0, -1, 38)
},
'tmax': {
'height': 31,
'width': 31,
'affine': Affine(1, 0, 67, 0, -1, 38)
}
}
def get_web_optimized_params(
src_dst,
zoom_level_strategy: str = "auto",
zoom_level: Optional[int] = None,
aligned_levels: Optional[int] = None,
tms: morecantile.TileMatrixSet = morecantile.tms.get("WebMercatorQuad"),
) -> Dict:
"""Return VRT parameters for a WebOptimized COG."""
if src_dst.crs != tms.rasterio_crs:
with WarpedVRT(src_dst, crs=tms.rasterio_crs) as vrt:
bounds = vrt.bounds
aff = list(vrt.transform)
else:
bounds = src_dst.bounds
aff = list(src_dst.transform)
resolution = max(abs(aff[0]), abs(aff[4]))
if zoom_level is None:
# find max zoom (closest to the raster resolution)
max_zoom = tms.zoom_for_res(
resolution,
max_z=30,
zoom_level_strategy=zoom_level_strategy,
)
else:
max_zoom = zoom_level
# defined the zoom level we want to align the raster
aligned_levels = aligned_levels or 0
base_zoom = max_zoom - aligned_levels
# find new raster bounds (bounds of UL tile / LR tile)
ul_tile = tms._tile(bounds[0], bounds[3], base_zoom)
w, _, _, n = tms.xy_bounds(ul_tile)
# The output resolution should match the TMS resolution at MaxZoom
vrt_res = tms._resolution(tms.matrix(max_zoom))
# Output transform is built from the origin (UL tile) and output resolution
vrt_transform = Affine(vrt_res, 0, w, 0, -vrt_res, n)
lr_tile = tms._tile(bounds[2], bounds[1], base_zoom)
e, _, _, s = tms.xy_bounds(
morecantile.Tile(lr_tile.x + 1, lr_tile.y + 1, lr_tile.z)
)
vrt_width = max(1, round((e - w) / vrt_transform.a))
vrt_height = max(1, round((s - n) / vrt_transform.e))
return dict(
crs=tms.rasterio_crs,
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
)
def temp_to_tif(input, output):
filepath = '/home/sg/Projects/FIrehub-model/Bartsotas/example.tif'
with rasterio.open(filepath) as src:
print(src.crs)
metadata = src.profile
metadata.update(transform=Affine(0.02, 0.0, 19.2, 0.0, 0.02, 34))
with rasterio.open(
'/home/sg/Projects/FIrehub-model/Bartsotas/dataset_07072019/' +
output + '.tif', 'w', **metadata) as dst:
dst.write(input.astype(rasterio.float64), 1)
def test_instantiate_scene(self):
self.assertEqual(self.l7.mtl['L1_METADATA_FILE']['PRODUCT_METADATA']['FILE_NAME_BAND_1'],
'LE07_L1TP_039028_20100702_20160915_01_T1_B1.TIF')
self.assertEqual(self.l7.utm_zone, 12)
self.assertEqual(self.l7.ex_atm_irrad, (1970.0, 1842.0, 1547.0, 1044.0,
255.700, np.nan, 82.06, 1369.00))
self.assertEqual(self.l7.rasterio_geometry['height'], 727)
self.assertEqual(self.l7.rasterio_geometry['driver'], 'GTiff')
self.assertEqual(self.l7.rasterio_geometry['dtype'], 'uint8')
self.assertEqual(self.l7.rasterio_geometry['transform'], Affine(30.0, 0.0, 367035.0,
0.0, -30.0, 5082585.0))
def pad(array, transform, pad_width, mode=None, **kwargs):
"""Returns a padded array and shifted affine transform matrix.
Array is padded using `numpy.pad()`."""
import numpy
transform = guard_transform(transform)
padded_array = numpy.pad(array, pad_width, mode, **kwargs)
padded_trans = list(transform)
padded_trans[2] -= pad_width*padded_trans[0]
padded_trans[5] -= pad_width*padded_trans[4]
return padded_array, Affine(*padded_trans[:6])
def test_instantiate_scene(self):
l8 = Landsat8(self.dirname_cloud)
self.assertEqual(l8.mtl['L1_METADATA_FILE']['PRODUCT_METADATA']['FILE_NAME_BAND_1'],
'LC80400282014193LGN00_B1.TIF')
self.assertEqual(l8.utm_zone, 12)
self.assertEqual(l8.rasterio_geometry['height'], 727)
self.assertEqual(l8.rasterio_geometry['driver'], 'GTiff')
self.assertEqual(l8.rasterio_geometry['dtype'], 'uint16')
self.assertEqual(l8.rasterio_geometry['transform'], Affine(30.0, 0.0, 367035.0,
0.0, -30.0, 5082585.0))
def get_web_optimized_params(
src_dst,
tilesize=256,
warp_resampling: str = "nearest",
zoom_level_strategy: str = "auto",
aligned_levels: Optional[int] = None,
tms: morecantile.TileMatrixSet = morecantile.tms.get("WebMercatorQuad"),
) -> Dict:
"""Return VRT parameters for a WebOptimized COG."""
bounds = list(
transform_bounds(src_dst.crs,
CRS.from_epsg(4326),
*src_dst.bounds,
densify_pts=21))
min_zoom, max_zoom = get_zooms(
src_dst,
tilesize=tilesize,
tms=tms,
zoom_level_strategy=zoom_level_strategy,
)
if aligned_levels is not None:
min_zoom = max_zoom - aligned_levels
ul_tile = tms.tile(bounds[0], bounds[3], min_zoom)
left, _, _, top = tms.xy_bounds(ul_tile.x, ul_tile.y, ul_tile.z)
vrt_res = tms._resolution(tms.matrix(max_zoom))
vrt_transform = Affine(vrt_res, 0, left, 0, -vrt_res, top)
lr_tile = tms.tile(bounds[2], bounds[1], min_zoom)
extrema = {
"x": {
"min": ul_tile.x,
"max": lr_tile.x + 1
},
"y": {
"min": ul_tile.y,
"max": lr_tile.y + 1
},
}
vrt_width = ((extrema["x"]["max"] - extrema["x"]["min"]) * tilesize *
2**(max_zoom - min_zoom))
vrt_height = ((extrema["y"]["max"] - extrema["y"]["min"]) * tilesize *
2**(max_zoom - min_zoom))
return dict(
crs=tms.crs,
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
resampling=ResamplingEnums[warp_resampling],
)
def setUp(self):
self.year = 2014
self.target_profile = {
'driver': 'GTiff',
'dtype': 'uint16',
'nodata': None,
'width': 7671,
'height': 7791,
'count': 1,
'crs': CRS({'init': 'epsg:32612'}),
'transform': Affine(30.0, 0.0, 582285.0, 0.0, -30.0, 5216715.0)
}
self.dir_name_LC8 = os.path.join(TEST_DATA, 'LC80370282013169LGN03')
def test_instantiate_scene(self):
self.assertTrue(self.l5.isdir)
self.assertEqual(self.l5.mtl['L1_METADATA_FILE']['PRODUCT_METADATA']['FILE_NAME_BAND_1'],
'LT05_L1TP_040028_20060706_20160909_01_T1_B1.TIF')
self.assertEqual((self.l5.rasterio_geometry['height'], self.l5.rasterio_geometry['width']),
(727, 727))
self.assertEqual(self.l5.utm_zone, 12)
self.assertEqual(self.l5.ex_atm_irrad, (1958.0, 1827.0, 1551.0,
1036.0, 214.9, np.nan, 80.65))
self.assertEqual(self.l5.rasterio_geometry['height'], 727)
self.assertEqual(self.l5.rasterio_geometry['driver'], 'GTiff')
self.assertEqual(self.l5.rasterio_geometry['dtype'], 'uint16')
self.assertEqual(self.l5.rasterio_geometry['transform'], Affine(30.0, 0.0, 367035.0, 0.0, -30.0, 5082585.0))
def _resample(self):
# home = os.path.expanduser('~')
# resample_path = os.path.join(home, 'images', 'sandbox', 'thredds', 'resamp_twx_{}.tif'.format(var))
resample_path = os.path.join(self.temp_dir, 'resample.tif')
with rasopen(self.mask, 'r') as src:
array = src.read(1)
profile = src.profile
res = src.res
try:
target_affine = self.target_profile['affine']
except KeyError:
target_affine = self.target_profile['transform']
target_res = target_affine.a
res_coeff = res[0] / target_res
new_array = empty(shape=(1, round(array.shape[0] * res_coeff),
round(array.shape[1] * res_coeff)),
dtype=float32)
aff = src.transform
new_affine = Affine(aff.a / res_coeff, aff.b, aff.c, aff.d,
aff.e / res_coeff, aff.f)
profile['transform'] = self.target_profile['transform']
profile['width'] = self.target_profile['width']
profile['height'] = self.target_profile['height']
profile['dtype'] = str(new_array.dtype)
delattr(self, 'mask')
with rasopen(resample_path, 'w', **profile) as dst:
reproject(array,
new_array,
src_transform=aff,
dst_transform=new_affine,
src_crs=src.crs,
dst_crs=src.crs,
resampling=Resampling.nearest)
dst.write(new_array)
with rasopen(resample_path, 'r') as src:
arr = src.read()
return arr
def save_raster(self, array, name):
height, width = array.shape
if self.aoi.crs.to_epsg() == self.epsg:
x, y = self.aoi.upper_left
transform = Affine(a=self.resolution,
b=0,
c=x,
d=0,
e=-self.resolution,
f=y)
else:
src = {'init': f'EPSG:{self.aoi.crs.to_epsg()}'}
dst = {'init': f'EPSG:{str(self.epsg)}'}
left, top = self.aoi.upper_left
bottom = top - height * self.resolution
right = left + width * self.resolution
transform, width, height = calculate_default_transform(
src,
dst,
width,
height,
left=left,
bottom=bottom,
right=right,
top=top,
dst_width=width,
dst_height=height)
profile = {
'driver': 'GTiff',
'dtype': 'float32',
'nodata': self.nodata,
'width': width,
'height': height,
'count': 1,
'crs': f'http://www.opengis.net/def/crs/EPSG/0/{self.epsg}',
'transform': transform
}
path = os.path.join(self.config.get("output"), self.fld_name, 'scenes')
if not os.path.exists(path):
os.makedirs(path, exist_ok=True)
with raster_open(os.path.join(path, name),
"w",
**profile,
compress='lzw') as dest:
dest.write(array, 1)
def array_to_geotif(array_data,
tif_name,
tile_corner,
save_folder=f"intermediate/geotifs",
affine_option='4corners'):
"""Convert an array into a geoTIF, which is a format required to intersect with ROI polygon.
"""
if affine_option == '4corners':
origin = [[1, 1], [1, array_data.shape[2]],
[array_data.shape[1], array_data.shape[2]],
[array_data.shape[1], 1]]
convert = tile_corner
trans_matrix = affine6p.estimate(origin, convert).get_matrix()
transform = Affine(trans_matrix[0][1], trans_matrix[0][0],
trans_matrix[0][2], trans_matrix[1][1],
trans_matrix[1][0], trans_matrix[1][2])
elif affine_option == '4bounds':
west = min(tile_corner[0][0], tile_corner[3][0])
south = min(tile_corner[2][1], tile_corner[3][1])
east = max(tile_corner[1][0], tile_corner[2][0])
north = max(tile_corner[0][1], tile_corner[1][1])
bearing = GetBearing(tile_corner[0], tile_corner[1])
transform = rasterio.transform.from_bounds(
west, south, east, north, array_data.shape[1],
array_data.shape[2]) #* Affine.rotation(bearing)
#FIXME: to be further tested
else:
print(f"Affine option {affine_option} not supported...Aborting")
return
# Save png as geoTIF
with rasterio.open(f"{bucket}/{save_folder}/{tif_name}",
'w',
driver='GTiff',
height=array_data.shape[1],
width=array_data.shape[2],
count=3,
dtype=array_data.dtype,
crs='EPSG:4326',
transform=transform) as dst:
dst.write(array_data)
return transform
def resample(self):
temp_path = os.path.join(self.temp_dir, 'resample.tif')
if self.mask:
ras = self.mask
else:
ras = self.reprojection
with rasopen(ras, 'r') as src:
array = src.read(1)
profile = src.profile
res = src.res
try:
target_affine = self.target_profile['affine']
except KeyError:
target_affine = self.target_profile['transform']
target_res = target_affine.a
res_coeff = res[0] / target_res
new_array = empty(shape=(1, round(array.shape[0] * res_coeff - 2),
round(array.shape[1] * res_coeff)),
dtype=float32)
aff = src.transform
new_affine = Affine(aff.a / res_coeff, aff.b, aff.c, aff.d,
aff.e / res_coeff, aff.f)
profile['transform'] = self.target_profile['transform']
profile['width'] = self.target_profile['width']
profile['height'] = self.target_profile['height']
profile['dtype'] = str(array.dtype)
with rasopen(temp_path, 'w', **profile) as dst:
reproject(array,
new_array,
src_transform=aff,
dst_transform=new_affine,
src_crs=src.crs,
dst_crs=src.crs,
resampling=Resampling.bilinear)
dst.write(new_array)
return new_array
def affine_from_grid(xi, yi):
"""
Retrieve the affine transformation from a gridded set of coordinates.
This function (unlike rasterio.transform functions) can also handle rotated grids
:param xi: 2D numpy-like gridded x-coordinates
:param yi: 2D numpy-like gridded y-coordinates
:return: rasterio.Affine object
"""
xul, yul = xi[0, 0], yi[0, 0]
xcol, ycol = xi[0, 1], yi[0, 1]
xrow, yrow = xi[1, 0], yi[1, 0]
dx_col = xcol - xul
dy_col = ycol - yul
dx_row = xrow - xul
dy_row = yrow - yul
return Affine(dx_col, dy_col, xul, dx_row, dy_row, yul)
def project2wgs(gtiff, output):
with rio.Env():
with rio.open(gtiff) as src:
out_kwargs = src.meta.copy()
out_kwargs['driver'] = 'GTiff'
print(out_kwargs)
res = (0.01, 0.01)
dst_crs = crs.from_string('+units=m +init=epsg:4326')
#dst_width, dst_height = src.width, src.height
xmin, ymin, xmax, ymax = [
-127.8294048826629989, 5.1830409679864857,
-59.0561278820333229, 49.9999999955067977
]
dst_transform = Affine(res[0], 0, xmin, 0, -res[1], ymax)
dst_width = max(int(ceil((xmax - xmin) / res[0])), 1)
dst_height = max(int(ceil((ymax - ymin) / res[1])), 1)
print(dst_transform)
out_kwargs.update({
'crs': dst_crs,
'transform': dst_transform,
'affine': dst_transform,
'width': dst_width,
'height': dst_height
})
print(out_kwargs)
with rio.open(output, 'w', **out_kwargs) as dst:
reproject(source=rio.band(src, 1),
destination=rio.band(dst, 1),
src_transform=src.affine,
src_crs=src.crs,
src_nodata=src.nodata,
dst_transform=out_kwargs['transform'],
dst_crs=out_kwargs['crs'],
dst_nodata=src.nodata,
resampling=0,
num_threads=2)
def test_warped_vrt_dimensions(path_rgb_byte_tif):
"""
A WarpedVRT with target dimensions has the expected dataset
properties.
"""
with rasterio.open(path_rgb_byte_tif) as src:
extent = (-20037508.34, 20037508.34)
size = (2 ** 16) * 256
resolution = (extent[1] - extent[0]) / size
dst_transform = Affine(resolution, 0.0, extent[0],
0.0, -resolution, extent[1])
vrt = WarpedVRT(src, dst_crs='EPSG:3857',
dst_width=size, dst_height=size,
dst_transform=dst_transform)
assert vrt.dst_crs == 'EPSG:3857'
assert vrt.src_nodata == 0.0
assert vrt.dst_nodata == 0.0
assert vrt.resampling == Resampling.nearest
assert vrt.width == size
assert vrt.height == size
assert vrt.transform == dst_transform
assert vrt.warp_extras == {'init_dest': 'NO_DATA'}
def predictFromTreeSnwprb(inFp, dtree, outfn, snwPrbfp=''):
hhhv_stacked = readAndStackBandsInclSnwProb(inFp)
pred = dtree.predict(hhhv_stacked)
pred_reshaped = pred.reshape(-1, 11908)
meta_outFile = {
'driver':
'GTiff',
'dtype':
'float32',
'nodata':
-9999.0,
'width':
11908,
'height':
12600,
'count':
1,
'crs':
CRS.from_epsg(3413),
'transform':
Affine(10.0, 0.0, -384702.1263054441, 0.0, -10.0, -2122443.806211936)
}
with rio.open(outfn, 'w', **meta_outFile) as outTif:
outTif.write(pred_reshaped, indexes=1)
def warp(ctx, files, output, driver, like, dst_crs, dimensions, src_bounds,
dst_bounds, res, resampling, src_nodata, dst_nodata, threads,
check_invert_proj, force_overwrite, creation_options,
target_aligned_pixels):
"""
Warp a raster dataset.
If a template raster is provided using the --like option, the
coordinate reference system, affine transform, and dimensions of
that raster will be used for the output. In this case --dst-crs,
--bounds, --res, and --dimensions options are not applicable and
an exception will be raised.
\b
$ rio warp input.tif output.tif --like template.tif
The output coordinate reference system may be either a PROJ.4 or
EPSG:nnnn string,
\b
--dst-crs EPSG:4326
--dst-crs '+proj=longlat +ellps=WGS84 +datum=WGS84'
or a JSON text-encoded PROJ.4 object.
\b
--dst-crs '{"proj": "utm", "zone": 18, ...}'
If --dimensions are provided, --res and --bounds are not applicable and an
exception will be raised.
Resolution is calculated based on the relationship between the
raster bounds in the target coordinate system and the dimensions,
and may produce rectangular rather than square pixels.
\b
$ rio warp input.tif output.tif --dimensions 100 200 \\
> --dst-crs EPSG:4326
If --bounds are provided, --res is required if --dst-crs is provided
(defaults to source raster resolution otherwise).
\b
$ rio warp input.tif output.tif \\
> --bounds -78 22 -76 24 --res 0.1 --dst-crs EPSG:4326
"""
output, files = resolve_inout(files=files,
output=output,
force_overwrite=force_overwrite)
resampling = Resampling[resampling] # get integer code for method
if not len(res):
# Click sets this as an empty tuple if not provided
res = None
else:
# Expand one value to two if needed
res = (res[0], res[0]) if len(res) == 1 else res
if target_aligned_pixels:
if not res:
raise click.BadParameter(
'--target-aligned-pixels requires a specified resolution')
if src_bounds or dst_bounds:
raise click.BadParameter(
'--target-aligned-pixels cannot be used with '
'--src-bounds or --dst-bounds')
# Check invalid parameter combinations
if like:
invalid_combos = (dimensions, dst_bounds, dst_crs, res)
if any(p for p in invalid_combos if p is not None):
raise click.BadParameter(
"--like cannot be used with any of --dimensions, --bounds, "
"--dst-crs, or --res")
elif dimensions:
invalid_combos = (dst_bounds, res)
if any(p for p in invalid_combos if p is not None):
raise click.BadParameter(
"--dimensions cannot be used with --bounds or --res")
with ctx.obj['env']:
setenv(CHECK_WITH_INVERT_PROJ=check_invert_proj)
with rasterio.open(files[0]) as src:
l, b, r, t = src.bounds
out_kwargs = src.profile.copy()
out_kwargs['driver'] = driver
# Sort out the bounds options.
if src_bounds and dst_bounds:
raise click.BadParameter(
"--src-bounds and destination --bounds may not be "
"specified simultaneously.")
if like:
with rasterio.open(like) as template_ds:
dst_crs = template_ds.crs
dst_transform = template_ds.transform
dst_height = template_ds.height
dst_width = template_ds.width
elif dst_crs is not None:
try:
dst_crs = CRS.from_string(dst_crs)
except ValueError as err:
raise click.BadParameter(str(err),
param='dst_crs',
param_hint='dst_crs')
if dimensions:
# Calculate resolution appropriate for dimensions
# in target.
dst_width, dst_height = dimensions
try:
xmin, ymin, xmax, ymax = transform_bounds(
src.crs, dst_crs, *src.bounds)
except CRSError as err:
raise click.BadParameter(str(err),
param='dst_crs',
param_hint='dst_crs')
dst_transform = Affine(
(xmax - xmin) / float(dst_width), 0, xmin, 0,
(ymin - ymax) / float(dst_height), ymax)
elif src_bounds or dst_bounds:
if not res:
raise click.BadParameter(
"Required when using --bounds.",
param='res',
param_hint='res')
if src_bounds:
try:
xmin, ymin, xmax, ymax = transform_bounds(
src.crs, dst_crs, *src_bounds)
except CRSError as err:
raise click.BadParameter(str(err),
param='dst_crs',
param_hint='dst_crs')
else:
xmin, ymin, xmax, ymax = dst_bounds
dst_transform = Affine(res[0], 0, xmin, 0, -res[1], ymax)
dst_width = max(int(ceil((xmax - xmin) / res[0])), 1)
dst_height = max(int(ceil((ymax - ymin) / res[1])), 1)
elif target_aligned_pixels:
try:
xmin, ymin, xmax, ymax = transform_bounds(
src.crs, dst_crs, *src.bounds)
except CRSError as err:
raise click.BadParameter(str(err),
param='dst_crs',
param_hint='dst_crs')
xmin = floor(xmin / res[0]) * res[0]
xmax = ceil(xmax / res[0]) * res[0]
ymin = floor(ymin / res[1]) * res[1]
ymax = ceil(ymax / res[1]) * res[1]
dst_transform = Affine(res[0], 0, xmin, 0, -res[1], ymax)
dst_width = max(int(ceil((xmax - xmin) / res[0])), 1)
dst_height = max(int(ceil((ymax - ymin) / res[1])), 1)
else:
try:
if src.transform.is_identity and src.gcps:
src_crs = src.gcps[1]
kwargs = {'gcps': src.gcps[0]}
else:
src_crs = src.crs
kwargs = src.bounds._asdict()
dst_transform, dst_width, dst_height = calcdt(
src_crs,
dst_crs,
src.width,
src.height,
resolution=res,
**kwargs)
except CRSError as err:
raise click.BadParameter(str(err),
param='dst_crs',
param_hint='dst_crs')
elif dimensions:
# Same projection, different dimensions, calculate resolution.
dst_crs = src.crs
dst_width, dst_height = dimensions
dst_transform = Affine((r - l) / float(dst_width), 0, l, 0,
(b - t) / float(dst_height), t)
elif src_bounds or dst_bounds:
# Same projection, different dimensions and possibly
# different resolution.
if not res:
res = (src.transform.a, -src.transform.e)
dst_crs = src.crs
xmin, ymin, xmax, ymax = (src_bounds or dst_bounds)
dst_transform = Affine(res[0], 0, xmin, 0, -res[1], ymax)
dst_width = max(int(ceil((xmax - xmin) / res[0])), 1)
dst_height = max(int(ceil((ymax - ymin) / res[1])), 1)
elif res:
# Same projection, different resolution.
dst_crs = src.crs
dst_transform = Affine(res[0], 0, l, 0, -res[1], t)
dst_width = max(int(ceil((r - l) / res[0])), 1)
dst_height = max(int(ceil((t - b) / res[1])), 1)
else:
dst_crs = src.crs
dst_transform = src.transform
dst_width = src.width
dst_height = src.height
# If src_nodata is not None, update the dst metadata NODATA
# value to src_nodata (will be overridden by dst_nodata if it is not None
if src_nodata is not None:
# Update the dst nodata value
out_kwargs.update({'nodata': src_nodata})
# Validate a manually set destination NODATA value
# against the input datatype.
if dst_nodata is not None:
if src_nodata is None and src.meta['nodata'] is None:
raise click.BadParameter(
"--src-nodata must be provided because dst-nodata is not None"
)
else:
# Update the dst nodata value
out_kwargs.update({'nodata': dst_nodata})
# When the bounds option is misused, extreme values of
# destination width and height may result.
if (dst_width < 0 or dst_height < 0 or dst_width > MAX_OUTPUT_WIDTH
or dst_height > MAX_OUTPUT_HEIGHT):
raise click.BadParameter(
"Invalid output dimensions: {0}.".format(
(dst_width, dst_height)))
out_kwargs.update({
'crs': dst_crs,
'transform': dst_transform,
'width': dst_width,
'height': dst_height
})
# Adjust block size if necessary.
if ('blockxsize' in out_kwargs
and dst_width < out_kwargs['blockxsize']):
del out_kwargs['blockxsize']
if ('blockysize' in out_kwargs
and dst_height < out_kwargs['blockysize']):
del out_kwargs['blockysize']
out_kwargs.update(**creation_options)
with rasterio.open(output, 'w', **out_kwargs) as dst:
reproject(source=rasterio.band(src,
list(range(1, src.count + 1))),
destination=rasterio.band(
dst, list(range(1, src.count + 1))),
src_transform=src.transform,
src_crs=src.crs,
src_nodata=src_nodata,
dst_transform=out_kwargs['transform'],
dst_crs=out_kwargs['crs'],
dst_nodata=dst_nodata,
resampling=resampling,
num_threads=threads)
def rasterize(ctx, files, driver, like, bounds, dimensions, res, src_crs,
all_touched, default_value, fill, property):
"""Rasterize GeoJSON into a new or existing raster.
If the output raster exists, rio-rasterize will rasterize feature values
into all bands of that raster. The GeoJSON is assumed to be in the same
coordinate reference system as the output unless --src-crs is provided.
--default_value or property values when using --property must be using a
data type valid for the data type of that raster.
If a template raster is provided using the --like option, the affine
transform and data type from that raster will be used to create the output.
Only a single band will be output.
The GeoJSON is assumed to be in the same coordinate reference system unless
--src-crs is provided.
--default_value or property values when using --property must be using a
data type valid for the data type of that raster.
--driver, --bounds, --dimensions, and --res are ignored when output exists
or --like raster is provided
If the output does not exist and --like raster is not provided, the input
GeoJSON will be used to determine the bounds of the output unless
provided using --bounds.
--dimensions or --res are required in this case.
If --res is provided, the bottom and right coordinates of bounds are
ignored.
Note:
The GeoJSON is not projected to match the coordinate reference system
of the output or --like rasters at this time. This functionality may be
added in the future.
"""
from rasterio._base import is_geographic_crs, is_same_crs
from rasterio.features import rasterize
from rasterio.features import bounds as calculate_bounds
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
files = list(files)
output = files.pop()
input = click.open_file(files.pop(0) if files else '-')
has_src_crs = src_crs is not None
src_crs = src_crs or 'EPSG:4326'
# If values are actually meant to be integers, we need to cast them
# as such or rasterize creates floating point outputs
if default_value == int(default_value):
default_value = int(default_value)
if fill == int(fill):
fill = int(fill)
with rasterio.drivers(CPL_DEBUG=verbosity > 2):
def feature_value(feature):
if property and 'properties' in feature:
return feature['properties'].get(property, default_value)
return default_value
geojson = json.loads(input.read())
if 'features' in geojson:
geometries = []
for f in geojson['features']:
geometries.append((f['geometry'], feature_value(f)))
elif 'geometry' in geojson:
geometries = ((geojson['geometry'], feature_value(geojson)), )
else:
raise click.BadParameter('Invalid GeoJSON',
param=input,
param_hint='input')
geojson_bounds = geojson.get('bbox', calculate_bounds(geojson))
if os.path.exists(output):
with rasterio.open(output, 'r+') as out:
if has_src_crs and not is_same_crs(src_crs, out.crs):
raise click.BadParameter(
'GeoJSON does not match crs of '
'existing output raster',
param='input',
param_hint='input')
if _disjoint_bounds(geojson_bounds, out.bounds):
click.echo(
"GeoJSON outside bounds of existing output "
"raster. Are they in different coordinate "
"reference systems?",
err=True)
meta = out.meta.copy()
result = rasterize(geometries,
out_shape=(meta['height'], meta['width']),
transform=meta.get('affine',
meta['transform']),
all_touched=all_touched,
dtype=meta.get('dtype', None),
default_value=default_value,
fill=fill)
for bidx in range(1, meta['count'] + 1):
data = out.read_band(bidx, masked=True)
# Burn in any non-fill pixels, and update mask accordingly
ne = result != fill
data[ne] = result[ne]
data.mask[ne] = False
out.write_band(bidx, data)
else:
if like is not None:
template_ds = rasterio.open(like)
if has_src_crs and not is_same_crs(src_crs, template_ds.crs):
raise click.BadParameter(
'GeoJSON does not match crs of '
'--like raster',
param='input',
param_hint='input')
if _disjoint_bounds(geojson_bounds, template_ds.bounds):
click.echo(
"GeoJSON outside bounds of --like raster. "
"Are they in different coordinate reference "
"systems?",
err=True)
kwargs = template_ds.meta.copy()
kwargs['count'] = 1
template_ds.close()
else:
bounds = bounds or geojson_bounds
if is_geographic_crs(src_crs):
if (bounds[0] < -180 or bounds[2] > 180 or bounds[1] < -80
or bounds[3] > 80):
raise click.BadParameter(
"Bounds are beyond the valid extent for "
"EPSG:4326.",
ctx,
param=bounds,
param_hint='--bounds')
if dimensions:
width, height = dimensions
res = ((bounds[2] - bounds[0]) / float(width),
(bounds[3] - bounds[1]) / float(height))
else:
if not res:
raise click.BadParameter(
'pixel dimensions are required',
ctx,
param=res,
param_hint='--res')
elif len(res) == 1:
res = (res[0], res[0])
width = max(
int(ceil((bounds[2] - bounds[0]) / float(res[0]))), 1)
height = max(
int(ceil((bounds[3] - bounds[1]) / float(res[1]))), 1)
src_crs = src_crs.upper()
if not src_crs.count('EPSG:'):
raise click.BadParameter(
'invalid CRS. Must be an EPSG code.',
ctx,
param=src_crs,
param_hint='--src_crs')
kwargs = {
'count':
1,
'crs':
src_crs,
'width':
width,
'height':
height,
'transform':
Affine(res[0], 0, bounds[0], 0, -res[1], bounds[3]),
'driver':
driver
}
result = rasterize(geometries,
out_shape=(kwargs['height'], kwargs['width']),
transform=kwargs.get('affine',
kwargs['transform']),
all_touched=all_touched,
dtype=kwargs.get('dtype', None),
default_value=default_value,
fill=fill)
if 'dtype' not in kwargs:
kwargs['dtype'] = result.dtype
kwargs['nodata'] = fill
with rasterio.open(output, 'w', **kwargs) as out:
out.write_band(1, result)
'driver':
'GTiff',
'dtype':
'float32',
'nodata':
0.0,
'width':
len(lon_conus_ease_400m),
'height':
len(lat_conus_ease_400m),
'count':
2,
'crs':
rasterio.crs.CRS.from_dict(init='epsg:6933'),
'transform':
Affine(400.358009339824, 0.0, -12060785.03616676, 0.0, -400.358009339824,
5854435.16975144)
}
# Extract the viirs LST day/night data and write to geotiff files
for idt in range(len(datename_seq_2019)):
viirs_mat_day_daily = []
viirs_mat_night_daily = []
for ifo in range(len(subfolders) - 1): # Exclude the LAI data folder
# LST Day
if len(file_lstd_ind_all[ifo][idt]) != 0:
with gzip.open(file_lstd_all[ifo][file_lstd_ind_all[ifo][idt][0]],
'rb') as file:
viirs_mat_day = np.frombuffer(file.read(), dtype='f')
viirs_mat_day = np.reshape(
viirs_mat_day, (size_viirs_tile, size_viirs_tile)).copy()
viirs_mat_day[viirs_mat_day <= 0] = np.nan
def merge_warped(self, activity):
print('==> start MERGE')
services = self.services
key = activity['ARDfile']
mystart = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
activity['mystart'] = mystart
activity['mystatus'] = 'DONE'
dataset = activity.get('dataset')
# If ARDfile already exists, update activitiesTable and chech if this merge is the last one for the mosaic
if services.s3fileExists(key=key):
efficacy = 0
cloudratio = 100
if activity['band'] == 'quality':
with rasterio.open('{}{}'.format(services.prefix, key)) as src:
mask = src.read(1)
cloudratio, efficacy = getMaskStats(mask)
# Update entry in DynamoDB
activity['myend'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
activity['efficacy'] = '{}'.format(int(efficacy))
activity['cloudratio'] = '{}'.format(int(cloudratio))
services.put_item_kinesis(activity)
key = '{}activities/{}{}.json'.format(activity['dirname'], activity['dynamoKey'], activity['date'])
services.save_file_S3(key, activity)
return
# Lets warp and merge
resx = int(activity['resx'])
resy = int(activity['resy'])
xmin = float(activity['xmin'])
ymax = float(activity['ymax'])
numcol = int(activity['numcol'])
numlin = int(activity['numlin'])
block_size = int(activity['block_size'])
nodata = int(activity['nodata']) if 'nodata' in activity else -9999
transform = Affine(resx, 0, xmin, 0, -resy, ymax)
# Quality band is resampled by nearest, other are bilinear
band = activity['band']
if band == 'quality':
resampling = Resampling.nearest
raster = numpy.zeros((numlin, numcol,), dtype=numpy.uint16)
raster_merge = numpy.zeros((numlin, numcol,), dtype=numpy.uint16)
raster_mask = numpy.ones((numlin, numcol,), dtype=numpy.uint16)
nodata = 0
else:
resampling = Resampling.bilinear
raster = numpy.zeros((numlin, numcol,), dtype=numpy.int16)
raster_merge = numpy.full((numlin, numcol,), fill_value=nodata, dtype=numpy.int16)
# For all files
template = None
for url in activity['links']:
with rasterio.Env(CPL_CURL_VERBOSE=False):
with rasterio.open(url) as src:
kwargs = src.meta.copy()
kwargs.update({
'crs': activity['srs'],
'transform': transform,
'width': numcol,
'height': numlin
})
source_nodata = 0
if src.profile['nodata'] is not None:
source_nodata = src.profile['nodata']
elif 'LC8SR' in dataset:
if band != 'quality':
source_nodata = nodata
else:
source_nodata = 1
elif 'CBERS' in dataset and band != 'quality':
source_nodata = nodata
kwargs.update({
'nodata': source_nodata
})
with MemoryFile() as memfile:
with memfile.open(**kwargs) as dst:
reproject(
source=rasterio.band(src, 1),
destination=raster,
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
dst_crs=activity['srs'],
src_nodata=source_nodata,
dst_nodata=nodata,
resampling=resampling)
if band != 'quality':
valid_data_scene = raster[raster != nodata]
raster_merge[raster != nodata] = valid_data_scene.reshape(numpy.size(valid_data_scene))
else:
raster_merge = raster_merge + raster * raster_mask
raster_mask[raster != nodata] = 0
if template is None:
template = dst.profile
if band != 'quality':
template['dtype'] = 'int16'
template['nodata'] = nodata
# Evaluate cloud cover and efficacy if band is quality
efficacy = 0
cloudratio = 100
if activity['band'] == 'quality':
raster_merge, efficacy, cloudratio = getMask(raster_merge, dataset)
template.update({'dtype': 'uint16'})
# Save merged image on S3
with MemoryFile() as memfile:
template.update({
'compress': 'LZW',
'tiled': True,
'interleave': 'pixel',
'blockxsize': block_size,
'blockysize': block_size
})
with memfile.open(**template) as riodataset:
riodataset.nodata = nodata
riodataset.write_band(1, raster_merge)
riodataset.build_overviews([2, 4, 8, 16, 32, 64], Resampling.nearest)
riodataset.update_tags(ns='rio_overview', resampling='nearest')
services.upload_fileobj_S3(memfile, key, {'ACL': 'public-read'})
# Update entry in DynamoDB
myend = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
activity['myend'] = myend
activity['efficacy'] = '{}'.format(int(efficacy))
activity['cloudratio'] = '{}'.format(int(cloudratio))
activity['raster_size_x'] = '{}'.format(numcol)
activity['raster_size_y'] = '{}'.format(numlin)
activity['block_size'] = '{}'.format(block_size)
services.put_item_kinesis(activity)
key = '{}activities/{}{}.json'.format(activity['dirname'], activity['dynamoKey'], activity['date'])
services.save_file_S3(key, activity)
