def merge_rasters(inrasters, outfile):
"""Merge a collection of rasters together into one raster.
Parameters
----------
inrasters: list
outfile: output raster name
Notes
-----
Output raster format is GeoTiff (.tif extension)
Rasterio uses the affine package to define the raster's
position in space and handle coordinate transformations, etc.
Affine 1.2 apparently had an issue where rasters in geographic coordinates
(lat-lon) would produce an error in the raster.merge.merge method because
the determinant of the transformation matrix was very small (9e-5 x 9e-5 for a 10m dem).
This issue appears to have been fixed in affine 2.0 (pip install affine==2.0 should resolve
the issue).
"""
rasterio = import_rasterio()
from rasterio.merge import merge
srces = []
print('merging:')
for file in inrasters:
srces.append(rasterio.open(file))
print('\t{}'.format(file))
out_image, out_trans = merge(srces)
out_meta = srces[0].meta.copy()
out_meta.update({"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_trans})
for src in srces:
src.close()
with rasterio.open(outfile, "w", **out_meta) as dest:
dest.write(out_image)
print('wrote {}'.format(outfile))
dataset.FlushCache()
dataset = None
# Preparing to add our new tile to the mosaic
src_files_to_mosaic = []
src = rasterio.open('Topo_Slope.tif')
src_files_to_mosaic.append(src)
src = rasterio.open(name)
src_files_to_mosaic.append(src)
os.remove('/Users/jackson/Desktop/Execute/' + tile_id + '.tif')
# Adding to Mosaic
mosaic, out = merge(src_files_to_mosaic,
method='first',
bounds=Extent,
res=Resolution,
nodata=-9999)
mosaic = np.reshape(mosaic, (Height, Width))
# Using GDAL to write the output raster
filename = 'Topo_Slope.tif' # Tiff holding Black Top Hat
driver = gdal.GetDriverByName('GTiff') # Driver for writing geo-tiffs
dataset = driver.Create(filename, Width, Height, 1,
gdal.GDT_Float32) # Creating our tiff file
dataset.GetRasterBand(1).WriteArray(mosaic) # Writing values to our tiff
dataset.SetGeoTransform(Geotransform) # Setting geo-transform
dataset.SetProjection(Projection) # Setting the projection
dataset.FlushCache() # Saving tiff to disk
dataset = None
# Change to directory with files
os.chdir(dir_name)
# List files to be mosaiced
files = os.listdir(dir_name)
# List of open rasterio files
raster_obj = []
for file in files:
src = rasterio.open(file)
raster_obj.append(src)
# Mosaic raster files
mosaic, mosaic_trans = merge(raster_obj)
# Parse EPSG code
epsg_code = int(src.crs.data['init'][5:])
# Reproject into the same coordinate system as raster data
geo = geo.to_crs(epsg=epsg_code)
# Get geometry coordinates for rasterio
coords = getFeatures(geo)
# Clip mosaic to target area
clip_img, clip_transform = mask(raster=mosaic, shapes=coords, crop=True)
# Copy the metadata
clip_meta = src.meta.copy()
def NFIP_PNNL(depth_grids, NFIP_points, out_folder, export_shapefile=False):
""" Calculates coverage and policies by flood depth
Keyword arguments:
depth_grids: list<str> -- array of file path names to the depth grid rasters
NFIP_points: str -- Shapefile containing the NFIP points. Must contain fields: LOW_FLOOR, LOWADJ_GRA, T_COV_BLDG, T_COV_CONT, POL_NO, STATEFP and NAME
out_folder: str -- file path name to directory for outputs
"""
# imports
import geopandas as gpd
import pandas as pd
import rasterio as rio
from rasterio.merge import merge
# from rasterio.plot import show
import matplotlib.pyplot as plt
from shapely.geometry import mapping
import numpy as np
from time import time
import uuid
import os
StateFipsCodes = {
"01": "Alabama",
"02": "Alaska",
"04": "Arizona",
"05": "Arkansas",
"06": "California",
"08": "Colorado",
"09": "Connecticut",
"10": "Delaware",
"11": "District of Columbia",
"12": "Florida",
"13": "Geogia",
"15": "Hawaii",
"16": "Idaho",
"17": "Illinois",
"18": "Indiana",
"19": "Iowa",
"20": "Kansas",
"21": "Kentucky",
"22": "Louisiana",
"23": "Maine",
"24": "Maryland",
"25": "Massachusetts",
"26": "Michigan",
"27": "Minnesota",
"28": "Mississippi",
"29": "Missouri",
"30": "Montana",
"31": "Nebraska",
"32": "Nevada",
"33": "New Hampshire",
"34": "New Jersey",
"35": "New Mexico",
"36": "New York",
"37": "North Carolina",
"38": "North Dakota",
"39": "Ohio",
"40": "Oklahoma",
"41": "Oregon",
"42": "Pennsylvania",
"44": "Rhode Island",
"45": "South Carolina",
"46": "South Dakota",
"47": "Tennessee",
"48": "Texas",
"49": "Utah",
"50": "Vermont",
"51": "Virginia",
"53": "Washington",
"54": "West Virginia",
"55": "Wisconsin",
"56": "Wyoming"
}
if out_folder.endswith('/'):
out_folder = out_folder[0:-1]
t0 = time()
print('Reading shapefile into memory')
t1 =time()
shp = gpd.read_file(NFIP_points)
print(time() - t1)
print('mosaicking rasters')
t1 = time()
raster_files = depth_grids
src_files_to_mosaic = []
for file in raster_files:
src = rio.open(file)
src_files_to_mosaic.append(src)
mosaic, transformation_info = merge(src_files_to_mosaic)
# show(mosaic, cmap='terrain')
# Copy the metadata
out_meta = src.meta.copy()
out_mosiac_fp = out_folder + '/' + str(uuid.uuid1()) + '.tif'
# Update the metadata
out_meta.update({"driver": "GTiff",
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": transformation_info,
"crs": shp.crs
}
)
with rio.open(out_mosiac_fp, "w", **out_meta) as dest:
dest.write(mosaic)
print(time() - t1)
print('Reading mosiacked raster into memory')
t1 = time()
ras = rio.open(out_mosiac_fp)
print(time() - t1)
print('Mapping geometry for processing')
t1 = time()
geoms = list(map(lambda x: mapping(x), shp.geometry.values))
x = list(map(lambda x: x['coordinates'][0], geoms))
y = list(map(lambda x: x['coordinates'][1], geoms))
print(time() - t1)
print('Extracting depths from raster to points')
t1 = time()
depths = []
for val in ras.sample(zip(x, y)):
depths.append(val)
print(time() - t1)
print('Calcualting fields')
t1 = time()
gdf = gpd.GeoDataFrame(data=depths,geometry=shp.geometry)
gdf.columns = ['Depth', gdf.columns[1]]
# Calculate first floor height
gdf['FFH'] = shp['LOW_FLOOR'] = shp['LOWADJ_GRA']
# Handle null values
gdf['FFH'][gdf['FFH'] > 9000] = 1
gdf['FFH'][gdf['FFH'] < 1] = 1
gdf['FFH'][gdf['FFH'] > 17] = 17
# Calculate depth above first floor height
gdf['Depth_Above_FFH'] = gdf['Depth'] - gdf['FFH']
# Calcualte exposure
gdf['Coverage'] = (shp['T_COV_BLDG'] + shp['T_COV_CONT']) * 100
# Policy numbers
gdf['Policy_Number'] = shp['POL_NO']
# Bin depth values
gdf['Depth_Bins'] = '0'
# gdf['Depth_Bins'][gdf['Depth'] == 0] = '0'
gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 0) & (gdf['Depth_Above_FFH'] <= 1)] = '0 to 1'
gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 1) & (gdf['Depth_Above_FFH'] <= 3)] = '1 to 3'
gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 3) & (gdf['Depth_Above_FFH'] <= 6)] = '3 to 6'
gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 6) & (gdf['Depth_Above_FFH'] <= 9)] = '6 to 9'
gdf['Depth_Bins'][gdf['Depth_Above_FFH'] > 9] = '> 9'
# Sorting columns
try:
gdf['State'] = shp['STATEFP_1']
except:
gdf['State'] = shp['STATEFP']
gdf = gdf.replace({'State': StateFipsCodes})
try:
gdf['County'] = shp['NAMELSAD_1']
except:
gdf['County'] = shp['NAME']
# Summarize
pivot_coverage = pd.pivot_table(gdf, values='Coverage', index=['State', 'County'], columns='Depth_Bins', aggfunc=np.sum, margins=True)
pivot_policies = pd.pivot_table(gdf, values='Policy_Number', index=['State', 'County'], columns='Depth_Bins', aggfunc=np.count_nonzero, margins=True)
try:
pivot_coverage.to_excel(out_folder + '/pivot_coverage.xlsx')
pivot_policies.to_excel(out_folder + '/pivot_policies.xlsx')
except:
pivot_coverage.to_csv(out_folder + '/pivot_coverage.csv')
pivot_policies.to_csv(out_folder + '/pivot_policies.csv')
print(time() - t1)
# print('Cleaning')
# t1 = time()
# os.remove(out_mosiac_fp) #unable to bc of file lock
# print(time() - t1)
if export_shapefile:
print('Exporting spatial output')
t1 = time()
gdf.to_file(out_folder + '/' + "output.shp", driver='ESRI Shapefile')
print(time() - t1)
print('Total elapsed time: ' + str(time() - t0))
def test_merge_pathlib_path(tiffs):
inputs = [Path(x) for x in tiffs.listdir()]
inputs.sort()
merge(inputs, res=2)
import rasterio
from rasterio.merge import merge
from pathlib import Path
SRC_FOLDER = 'source_images_oza'
OUT_FOLDER = 'mosaic_images_oza'
src_images = [
rasterio.open(path) for path in (Path.cwd() / SRC_FOLDER).iterdir()
]
mosaic, out_trans = merge(src_images)
out_meta = src_images[0].meta.copy()
out_meta.update({
"driver": "GTiff",
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans,
"crs": "+proj=utm +zone=35 +ellps=GRS80 +units=m +no_defs "
})
with rasterio.open(Path(OUT_FOLDER) / 'mosaic.tif', "w", **out_meta) as dest:
dest.write(mosaic)
def test_merge_tiny_intres(tiffs):
inputs = [str(x) for x in tiffs.listdir()]
inputs.sort()
datasets = [rasterio.open(x) for x in inputs]
merge(datasets, res=2)
def merge_maps(map_list, resolution):
maps = [rasterio.open(map_path) for map_path in map_list]
new_image, new_transform = merge.merge(maps, res=resolution, nodata=0)
return new_image, new_transform
def rsts_to_mosaic(inRasterS, o, api="grass", fformat='.tif'):
"""
Create Mosaic of Raster
"""
if api == 'pygrass':
"""
The GRASS program r.patch allows the user to build a new raster map the size
and resolution of the current region by assigning known data values from
input raster maps to the cells in this region. This is done by filling in
"no data" cells, those that do not yet contain data, contain NULL data, or,
optionally contain 0 data, with the data from the first input map.
Once this is done the remaining holes are filled in by the next input map,
and so on. This program is useful for making a composite raster map layer
from two or more adjacent map layers, for filling in "holes" in a raster map
layer's data (e.g., in digital elevation data), or for updating an older map
layer with more recent data. The current geographic region definition and
mask settings are respected.
The first name listed in the string input=name,name,name, ... is the name of
the first map whose data values will be used to fill in "no data" cells in
the current region. The second through last input name maps will be used,
in order, to supply data values for for the remaining "no data" cells.
"""
from grass.pygrass.modules import Module
m = Module("r.patch",
input=inRasterS,
output=o,
overwrite=True,
run_=False,
quiet=True)
m()
elif api == 'grass':
from glass.pys import execmd
rcmd = execmd("r.patch input={} output={} --overwrite --quiet".format(
",".join(inRasterS), o))
elif api == 'rasterio':
import rasterio
from rasterio.merge import merge
from glass.g.prop import drv_name
from glass.g.prop.prj import get_epsg, epsg_to_wkt
if type(inRasterS) != list:
from glass.pys.oss import lst_ff
rsts = lst_ff(inRasterS, file_format=fformat)
else:
rsts = inRasterS
srcs = [rasterio.open(r) for r in rsts]
mosaic, out_trans = merge(srcs)
out_meta = srcs[0].meta.copy()
out_meta.update({
"driver": drv_name(o),
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans,
"count": 1,
"crs": epsg_to_wkt(get_epsg(rsts[0])),
"compress": 'lzw'
})
with rasterio.open(o, "w", **out_meta) as dest:
dest.write(mosaic)
else:
raise ValueError('api {} is not available'.format(api))
return o
dem_fps = glob.glob(q)
# Files that were found:
dem_fps
# List for the source files
src_files_to_mosaic = []
# Iterate over raster files and add them to source -list in 'read mode'
for fp in dem_fps:
src = rasterio.open(fp)
src_files_to_mosaic.append(src)
# Merge function returns a single mosaic array and the transformation info
bnd = [87.813774, 22.432564, 91.157858, 26.209197]
mosaic, ot = merge(src_files_to_mosaic, bounds=bnd)
# Copy the metadata
out_meta = src.meta.copy()
# Update the metadata
out_meta.update({
"driver": "GTiff",
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": from_origin(bnd[0], bnd[3], 0.00179,
0.00179), #north and west coord
"crs": "EPSG:4326"
})
#~ print(mosaic.shape,out_meta)
print("Mosaicing rasters... ")
def main(sqlite, state, folder):
import pyproj
from shapely.ops import transform
from shapely.geometry import mapping
from shapely.prepared import prep
from rasterio.mask import mask
from rasterio.enums import Resampling
sqlite = r'E:\GeospatialData\Projects\i-love-washington\data\data.sqlite'
state = 'Washington'
folder = "E:\GeospatialData\Projects\i-love-washington\data\mosaic"
# Create the output database table
conn, cur = utils.connect_spatialite(sqlite)
cur.execute('DROP TABLE lake_points')
cur.execute('''
CREATE TABLE IF NOT EXISTS lake_points (
oid INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
z DOUBLE,
path_ INTEGER,
row_ INTEGER)
''')
cur.execute(
"SELECT AddGeometryColumn('lake_points', 'geom',3857, 'POINT', 2)")
cur.execute("DELETE FROM lake_points")
conn.commit()
cur.close(), conn.close()
sql = '''
SELECT name, TRANSFORM(geometry, 3857) geom
FROM multipolygons
WHERE boundary='administrative' AND name='Washington'
'''
df_wa = utils.ogr2pandas(sqlite, sql=sql)
wa_geom = df_wa.iloc[0].geom
wa_buf = wa_geom.buffer(500, 1)
bounds = wa_buf.bounds
# Pull lakes from database
lakes_sql = '''
SELECT name, ST_TRANSFORM(ST_UNION(geometry), 3857) geom, ST_AREA(ST_TRANSFORM(ST_UNION(geometry), 3857)) area
FROM multipolygons
WHERE natural='water'
AND name IS NOT NULL
AND ST_AREA(geometry) > 0.0001
AND (LOWER(name) LIKE '%lake%' OR LOWER(name) LIKE '%pool%' OR LOWER(name) LIKE '%reservoir%')
GROUP BY name
ORDER BY ST_AREA(ST_BUFFER(geometry, -0.001, 10)) DESC
LIMIT 100
'''
df_lakes = utils.ogr2pandas(sqlite, sql=lakes_sql)
# Apply a negative buffer, used for ignoring shoreline temperatures when sampling raster
df_lakes['neg_buf'] = df_lakes['geom'].apply(lambda x: x.buffer(-150, 2))
# Prepare lakes for efficient lookups once
df_lakes['prep'] = df_lakes['geom'].apply(lambda x: prep(x))
# print('Fetching LANDSAT WRS geoms')
# df_wrs = utils.ogr2pandas(
# sqlite,
# 'WRS2',
# columns=['PATH', 'ROW', 'geometry'])
df_wrs = utils.ogr2pandas(
dataset="E:\GeospatialData\Projects\i-love-washington\data\data.sqlite",
sql="""
SELECT PATH, ROW, w.geometry
FROM multipolygons m, wrs2 w
WHERE m.name = 'Washington'
AND ST_INTERSECTS(m.geometry, w.geometry)
""")
for row in df_wrs.to_dict('records'):
logging.warning(f'Starting row {row["row"]}, path {row["path"]}')
# Identify a cloudless scene
mtl_md = landsat._return_candidates(int(row['path']), int(row['row']),
8, 5)
logging.warning(f'Identified {len(mtl_md)} scenes')
for key in mtl_md:
metadata = mtl_md[key]
save_file = utils.os.path.join(
folder, f"{metadata['LANDSAT_PRODUCT_ID']}_FARENHEIT.TIF")
if utils.os.path.exists(save_file):
logging.warning(' Already Exists')
continue
# Pull image data from google into the metadata key 'bytes'
logging.warning('Downloading image')
landsat_bytes = landsat._pull_raster(metadata, 10)
# Read Image
logging.warning('Reading bytes to array')
array, profile = utils.gdalbytes2numpy(landsat_bytes)
logging.warning('Converting to farenheit')
farenheit = landsat.array2farenheit(array, metadata,
-99999).round(1)
# Set profile nodata to -99999 and output type to float32
profile['nodata'] = -99999
profile['dtype'] = 'float32'
# Reproject image bytes folder
logging.warning('Writing to file')
save_file = utils.os.path.join(
folder, f"{metadata['LANDSAT_PRODUCT_ID']}_FARENHEIT.TIF")
utils.reproject_array_to_file(farenheit, profile, 'EPSG:3857',
save_file)
j = utils.json.dumps(metadata, indent=2)
with open(save_file.replace('FARENHEIT.TIF', 'METADATA.json'),
'w') as w:
w.write(j)
tifs = [
rasterio.open(utils.os.path.join(folder, f))
for f in utils.os.listdir(folder) if '_T1_FARENHEIT.TIF' in f
]
bounds = (-13897124.13181157, 5707029.565730883, -13014721.44634677,
6275775.285325819)
import numpy as np
def calc_median(old_data, new_data, old_nodata, new_nodata, **kwargs):
mask = np.logical_and(~old_nodata, ~new_nodata)
old_data[mask] = np.ma.median([old_data[mask], new_data[mask]], axis=0)
mask = np.logical_and(old_nodata, ~new_nodata)
old_data[mask] = new_data[mask]
mosaic, out_trans = merge(datasets=tifs,
bounds=bounds,
precision=1,
method=calc_median)
[tif.close() for tif in tifs]
out_meta = tifs[0].meta.copy()
out_meta.update({
"driver": "GTiff",
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans,
'compress': 'LZW',
'tiled': True
})
out_fp = utils.os.path.join(folder, 'mosaic_median.tif')
with rasterio.open(out_fp, "w", **out_meta) as dest:
dest.write(mosaic)
del mosaic
mosaic = rasterio.open(out_fp)
profile = mosaic.profile
# Iterate through the lakes and export a PNG
for lake in df_lakes.to_dict('records'):
break
geoj = mapping(lake['geom'])
# Mask the landsat image to lake pixels
array, array_transform = mask(src, [geoj], crop=True)
# mosaic, out_trans = merge(
# datasets=tifs,
# bounds=bounds,
# precision=1,
# method='max')
# out_meta = tifs[0].meta.copy()
# out_meta.update({"driver": "GTiff",
# "height": mosaic.shape[1],
# "width": mosaic.shape[2],
# "transform": out_trans,
# 'compress':'LZW',
# 'tiled':True})
# out_fp = utils.os.path.join(folder, 'mosaic_max.tif')
# with rasterio.open(out_fp, "w", **out_meta) as dest:
# dest.write(mosaic)
# # Write VRT
# tifs = [utils.os.path.join(folder, f) for f in utils.os.listdir(folder) if f[-4:].lower() == '.tif']
# args = [
# utils.os.path.join(utils.gdal_path, 'gdalbuildvrt'),
# '-hidenodata', '-srcnodata', '-99999',
# '-r', 'average',
# utils.os.path.join(folder, 'mosaic.vrt')]
# args.extend(tifs)
# p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE)
# stdout, stderr = p.communicate()
# # Write mosaic
# args = [
# utils.os.path.join(utils.gdal_path, 'gdalwarp'),
# '-overwrite',
# '--config', 'GDAL_CACHEMAX', '512',
# '-srcnodata', '-99999',
# '-co', 'TILED=YES',
# '-co', 'COMPRESS=LZW',
# '-r', 'med',
# '-wm', '2000',
# utils.os.path.join(folder, 'mosaic.vrt'),
# utils.os.path.join(folder, 'mosaic.tif'),
# ]
# p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE)
# stdout, stderr = p.communicate()
# # Get the projection of the image
# logging.warning('Identifying projection')
# metadata.update(utils.gdalinfo(metadata['bytes']))
# landsat_crs = pyproj.CRS(metadata['coordinateSystem']['wkt'])
# wgs84 = pyproj.CRS('EPSG:4326')
# project = pyproj.Transformer.from_crs(wgs84, landsat_crs, always_xy=True).transform
# # Filter geometries to those that intersect with the scene, project to the correct UTM, and convert to geojson
# logging.warning('Filtering lakes to those in the scene')
# geoms = [mapping(transform(project, x)) for x in df_lakes['neg_buf'] if x.intersects(row['geom'])]
# logging.warning('Creating raster memfile from image bytes')
# memfile = utils.gdalbytes2numpy(metadata['bytes'], return_memfile=True)
# logging.warning('Masking raster by lakes')
# with memfile.open() as src:
# # Get the metadata of the landsat image
# profile = src.profile
# # Mask the landsat image to lake pixels
# array, array_transform = mask(src, geoms, crop=True)
# # Set profile nodata to -99999 and output type to float32
# profile['nodata'] = 0
# profile['nodata'] = -99999
# profile['dtype'] = 'float32'
# # Create coordinates from valid data
# logging.warning('Creating coordinates')
# df_coords = utils.array2coords(farenheit[0], array_transform, -99999)
# df_coords['row_'] = row['row']
# df_coords['path_'] = row['path']
# # Write coords to database
# logging.warning('Writing coordinates to database')
# conn, cur = utils.connect_spatialite(sqlite)
# cur.executemany(
# f'INSERT INTO lake_points (geom, z, row_, path_) VALUES ( Transform(MakePoint(?, ?, {landsat_crs.to_epsg()}), 3857), ?, ?, ?)',
# df_coords[['x', 'y', 'z', 'row_', 'path_']].itertuples(index=False) )
# conn.commit()
# cur.close(), conn.close()
# Move to top
# # Create a copy of the lakes in the database for editing
# args = [
# utils.os.path.join(utils.gdal_path, 'ogr2ogr'),
# '-append',
# '-t_srs', 'EPSG:3857',
# '-sql', lakes_sql,
# sqlite, sqlite, 'multipolygons',
# '-nln','lakes_3857']
# p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE)
# stdout, stderr = p.communicate()
# args[-1] = 'lakes_3857_modified'
# p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE)
# stdout, stderr = p.communicate()
conn, cur = utils.connect_spatialite(sqlite)
cur.execute("SELECT CreateSpatialIndex('lake_points', 'geom')")
dataset = driver.Create(filename,BTH_Array.shape[1],
BTH_Array.shape[0], 1,gdal.GDT_Float32)
dataset.GetRasterBand(1).WriteArray(BTH_Array)
dataset.SetGeoTransform(T) # Setting geo-transform
dataset.SetProjection(P) # Setting the projection
dataset.FlushCache() # Saving tiff to disk
dataset=None # Clearing variable name
src_files_to_mosaic = [] # Initializing mosaic input
src = rasterio.open('Tile.tif') # Grabbing the BTH results
src_files_to_mosaic.append(src) # Appending BTH results
src = rasterio.open('BTH.tif') # Grabbing results geo tiff
src_files_to_mosaic.append(src) # Appending results geo tiff
mosaic, out = merge(src_files_to_mosaic,
method='first',
bounds=Extent, res=Resolution,
nodata=None) # Mosaic-ing in BTH results
mosaic = np.reshape(mosaic,(Height,Width)) # Reshaping results
filename = 'BTH.tif' # Tiff holding Black Top Hat
driver = gdal.GetDriverByName('GTiff') # Driver for writing geo-tiffs
dataset = driver.Create(filename,Width, Height,
1,gdal.GDT_Float32) # Creating our tiff file
dataset.GetRasterBand(1).WriteArray(mosaic) # Writing values to our tiff
dataset.SetGeoTransform(Geotransform) # Setting geo-transform
dataset.SetProjection(Projection) # Setting the projection
dataset.FlushCache() # Saving tiff to disk
dataset=None # Clearing variable name
os.remove('Tile.tif') # Removing the tile DEM
def _worker(self, job):
"""Load rasters, merge, and save to disk for a given date and db."""
# --- unpack job ---
sanitized_db, date = job
# --- list all satellite image COGT fragments for date ---
y = str(date.year).zfill(2)
m = str(date.month).zfill(2)
d = str(date.day).zfill(2)
db_dir = os.path.join(self.raw_s3_local_path, sanitized_db)
date_dir = os.path.join(db_dir, str(y), str(m), str(d))
files = os.listdir(date_dir)
# --- filter out inapplicable files ---
files = [f for f in files if '.tif' in f]
files = [f for f in files if 'PRODUCT_qa_value' not in f]
# --- open files with rasterio ---
src_files = []
for f in files:
# --- read rasters ---
srcr = rasterio.open(os.path.join(date_dir, f))
src_files.append(srcr)
# --- merge into a single raster --- #TODO: Confirm rasterio behavior for overlaps
mosaic, mosaic_transform = merge(
src_files + [self.blank_world]
) #https://gis.stackexchange.com/questions/360685/creating-daily-mosaics-from-orbiting-satellite-imagery
# --- replace null values ---
null_val = srcr.nodata
mosaic[mosaic == null_val] = np.nan
# --- copy meta data ---
profile = srcr.meta.copy()
profile.update(width=mosaic.shape[2],
height=mosaic.shape[1],
transform=mosaic_transform)
# --- resample in memory ---
if self.resample_shape != None:
with MemoryFile() as memfile:
with memfile.open(**profile) as dataset:
dataset.write(mosaic)
dataset = memfile.open()
mosaic = dataset.read(
out_shape=(
dataset.count, self.resample_shape[0],
self.resample_shape[1]
# int(dataset.height * self.resample_scale),
# int(dataset.width * self.resample_scale)
),
resampling=Resampling.bilinear)
# --- scale image transform ---
mosaic_transform = dataset.transform * dataset.transform.scale(
(dataset.width / mosaic.shape[-1]),
(dataset.height / mosaic.shape[-2]))
del dataset
if self.downcast_dtype != 'float64':
# --- clip ---
mosaic = np.clip(mosaic, np.percentile(mosaic, 1),
np.percentile(mosaic, 99))
# --- normalize ---
downcast_min_val = np.iinfo(self.downcast_dtype).min
downcast_max_val = np.iinfo(self.downcast_dtype).max
Scaler = MinMaxScaler(feature_range=(downcast_min_val,
downcast_max_val))
mosaic = np.squeeze(mosaic)
mosaic = Scaler.fit_transform(mosaic)
# --- downcast ---
mosaic = mosaic.astype(self.downcast_dtype)
# --- Make 3D as rasterio expects ---
mosaic = np.expand_dims(mosaic, axis=0)
# --- update profile again ---
profile.update(width=mosaic.shape[2],
height=mosaic.shape[1],
transform=mosaic_transform,
dtype=self.downcast_dtype,
nodata=0)
# --- save to disk ---
out_db = sanitized_db.split(os.path.sep)[-2].replace('L2', 'L3')
out_filename = f"{out_db}{date.strftime('%Y-%m-%d')}.tif"
out_fp = os.path.join(self.out_file_dir, out_filename)
with rasterio.open(out_fp, "w", **profile) as fp:
fp.write(mosaic)
return self
# Coordinate Reference System
d[0].meta
# In[23]:
# Check Meta Data of original Files
out_meta = src.meta.copy()
out_meta
# In[24]:
mosaic, out_trans = merge(d)
# In[25]:
# Create Metadata of the for the mosaic TIFF
out_meta.update({"driver": "HFA","height":mosaic.shape[1],"width":mosaic.shape[2],"transform": out_trans,})
# In[26]:
# Write the updated DEM to the specified file path
with rasterio.open(strOutTiffPath, "w", **out_meta) as dest:
dest.write(mosaic)
def test_merge_tiny_intres(tiffs):
inputs = [str(x) for x in tiffs.listdir()]
inputs.sort()
datasets = [rasterio.open(x) for x in inputs]
merge(datasets, res=2)
ls = WYcountylist
src_file_to_mosaic = []
outf = 'D:/twu/fireline/travelers/tiffs/merge.tiff'
for county in ls:
print(county)
try:
with rasterio.drivers():
src = rasterio.open(
'D:/twu/fireline/travelers/tiffs/bon_ID_WFHS.tiff')
src_1 = rasterio.open(
'D:/twu/fireline/travelers/tiffs/peop_WA_WFHS.tiff')
#src_file_to_mosaic.append()
src_file_to_mosaic = [src, src_1]
print("open")
mosaic, out_trans = merge(src_file_to_mosaic)
print("merge")
mosaic = mosaic.astype(np.int8)
with rasterio.open(outf,
"w",
driver='GTiff',
dtype="uint8",
nodata=255,
compress="lzw",
height=mosaic.shape[1],
count=1,
width=mosaic.shape[2],
transform=out_trans,
crs="EPSG:4326") as dest:
dest.write(mosaic, 1)
print("done")
def merge_ak_canada(ak_fn, can_fn, template_fn, variable, output_path,
scalevals):
from rasterio.merge import merge
# reproject canada to 3338 and rescale temperature values from ints to floats
dtype = 'float32'
if variable == 'ppt':
scalevals = False
# open alaska AAI grid -- special for precip...
ak = open_ak_precip(ak_fn)
else: # temperature variables
scalevals = True
# open alaska AAI grid
ak = scale_ak_temperature_to_int(ak_fn,
nodata=-9999) # already in 3338
can = reproject_canada_to3338(can_fn, template_fn, dtype=dtype)
# get some output bounds information for the merge...
with rasterio.open(template_fn) as tmp:
bounds = tmp.bounds
mask = tmp.read_masks(1)
# merge 'em
merged, transform = merge([can, ak],
bounds=bounds,
res=(2000, 2000),
nodata=-9999,
precision=5)
# fix some mask mismatches
merged = fill_missing_mask(merged[0, ...], mask)
# update metadata
meta = ak.meta
height, width = merged.shape
meta.update({
'driver': 'GTiff',
'height': height,
'width': width,
'crs': {
'init': 'EPSG:3338'
},
'transform': transform,
'compress': 'lzw',
'dtype': dtype
})
# output filenaming
varname_lookup = {
'tmean': 'tas',
'tmin': 'tasmin',
'tmax': 'tasmax',
'ppt': 'pr'
}
if variable == 'ppt':
metric = 'total'
units = 'mm'
elif variable in ['tmean', 'tmax', 'tmin']:
metric = 'mean'
units = 'C'
# write to disk
try:
if not os.path.exists(output_path):
os.makedirs(output_path)
except:
pass # deal with parallel processing in a clunky way
month = os.path.basename(ak_fn).split('.')[0].split('_')[-1]
if len(month) == 1:
month = '0{}'.format(month)
# scale the temperature PRISM values
if scalevals:
merged[
merged != meta['nodata']] = merged[merged != meta['nodata']] / 10.0
output_filename = os.path.join(
output_path, '{}_{}_{}_akcan_prism_{}_1961_1990.tif'.format(
varname_lookup[variable], metric, units, month))
with rasterio.open(output_filename, 'w', **meta) as out:
out.write(merged, 1)
return output_filename
# Make a search criteria to select the DEM files
search_criteria = "L*.tif"
q = os.path.join(data_dir, search_criteria)
print(q)
# List files that matches the criteria
dem_fps = glob.glob(q)
dem_fps
# Open the source files with rasterio
# len gives the amount of files, charactor etc
src_files_to_mosaic = [ rasterio.open(fp) for fp in dem_fps ]
len(src_files_to_mosaic)
# Merge rasters into mosaic
mosaic, out_trans = merge(datasets=src_files_to_mosaic)
# Plot
show(mosaic, cmap="terrain")
#plt.imshow(mosaic, cmap="terrain")
#plt.colorbar()
# Save the mosaic and update the metadata
# Here the metadata is a copy of one file and needs to be update
out_meta = src_files_to_mosaic[0].meta.copy()
# Update metadata with new dimensions and crs
out_meta.update(
{"height" : mosaic.shape[1],
"width" :mosaic.shape[2],
"transform": out_trans,
def get_mosaic(avail_hu12catchs_group, hu, break_hu, dst_crs):
## Get mosaic of DEMs for each HUC
def append_check(src_files_to_mosaic, var, subdirectory, hu):
## Check each raster's resolution in this HUC
if any(np.float16(i) > 1. for i in var.res):
out_path = os.path.join(subdirectory, "gt1m.err")
Path(out_path).touch()
print('WARNING: >1m raster input for HUC12: ' + str(hu))
sys.stdout.flush()
#del(out_path)
else:
src_res_min_to_mosaic.append(min(var.res))
src_res_max_to_mosaic.append(min(var.res))
src_x_to_mosaic.append(var.res[0])
src_y_to_mosaic.append(var.res[1])
src_files_to_mosaic.append(var)
return (src_files_to_mosaic, src_res_min_to_mosaic,
src_res_max_to_mosaic, src_x_to_mosaic,
src_y_to_mosaic)
## Reproject the mosaic to DEM tiles pertaining to each HUC
#print('INSIDE GET MOSAIC\t',arguments[0])
#print('INSIDE GET MOSAIC\t',flows_key)
#sys.stdout.flush()
dem_fps = list(avail_hu12catchs_group['stampede2name'])
src_files_to_mosaic = []
src_res_min_to_mosaic = []
src_res_max_to_mosaic = []
src_x_to_mosaic = []
src_y_to_mosaic = []
memfile = {}
for fp in dem_fps:
memfile[fp] = MemoryFile()
#del(fp)
for fp in dem_fps:
with rasterio.open(fp) as src:
transform, width, height = calculate_default_transform(
src.crs, dst_crs, src.width, src.height, *src.bounds)
out_meta = src.meta.copy()
out_meta.update({
'crs': dst_crs,
'transform': transform,
'width': width,
'height': height
})
#del(width,height)
## Don't do an expensive reprojection if projection already correct
if src.meta == out_meta:
#del(fp,transform)
src_files_to_mosaic, src_res_min_to_mosaic, src_res_max_to_mosaic, src_x_to_mosaic, src_y_to_mosaic = append_check(
src_files_to_mosaic, src, subdirectory, hu)
else:
dst = memfile[fp].open(**out_meta)
#del(fp,out_meta)
for i in range(1, src.count + 1):
reproject(source=rasterio.band(src, i),
destination=rasterio.band(dst, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=dst.transform,
dst_crs=dst.crs,
resampling=Resampling.nearest)
#del(i)
#del(transform)
src_files_to_mosaic, src_res_min_to_mosaic, src_res_max_to_mosaic, src_x_to_mosaic, src_y_to_mosaic = append_check(
src_files_to_mosaic, dst, subdirectory, hu)
#del(dst)
#del(dem_fps)
if len(src_files_to_mosaic) == 0:
out_path = os.path.join(subdirectory, "allGT1m.err")
Path(out_path).touch()
print('WARNING: Found no <=1m raster input data for HUC12: ' +
str(hu))
sys.stdout.flush()
#del(out_path)
break_hu = True
mosaic_tuple = ()
return (break_hu, mosaic_tuple)
else:
src_files_to_mosaic = pd.DataFrame(
data={
'Files': src_files_to_mosaic,
'min(resolution)': src_res_min_to_mosaic,
'max(resolution)': src_res_max_to_mosaic
})
#del(src_res_min_to_mosaic,src_res_max_to_mosaic)
src_files_to_mosaic.sort_values(
by=['min(resolution)', 'max(resolution)'], inplace=True)
mosaic, out_trans = merge(list(src_files_to_mosaic['Files']),
res=(max(src_x_to_mosaic),
max(src_y_to_mosaic)))
#del(src_x_to_mosaic,src_y_to_mosaic)
for src in src_files_to_mosaic['Files']:
src.close()
#del(src_files_to_mosaic)
out_meta = src.meta.copy()
#del(src)
out_meta.update({
"driver": 'GTiff',
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans,
"crs": dst_crs
})
#del(out_trans)
for keyvalue in memfile.items():
keyvalue[1].close()
#del(keyvalue)
#del(memfile)
mosaic_tuple = (mosaic, out_meta)
return (break_hu, mosaic_tuple)
def test_merge_filenames(tiffs):
inputs = [str(x) for x in tiffs.listdir()]
inputs.sort()
merge(inputs, res=2)
def test_issue2163():
"""Demonstrate fix for issue 2163"""
with rasterio.open("tests/data/float_raster_with_nodata.tif") as src:
data = src.read()
result, transform = merge([src])
assert numpy.allclose(data, result)
