def merge_rasters(inrasters, outfile): """Merge a collection of rasters together into one raster. Parameters ---------- inrasters: list outfile: output raster name Notes ----- Output raster format is GeoTiff (.tif extension) Rasterio uses the affine package to define the raster's position in space and handle coordinate transformations, etc. Affine 1.2 apparently had an issue where rasters in geographic coordinates (lat-lon) would produce an error in the raster.merge.merge method because the determinant of the transformation matrix was very small (9e-5 x 9e-5 for a 10m dem). This issue appears to have been fixed in affine 2.0 (pip install affine==2.0 should resolve the issue). """ rasterio = import_rasterio() from rasterio.merge import merge srces = [] print('merging:') for file in inrasters: srces.append(rasterio.open(file)) print('\t{}'.format(file)) out_image, out_trans = merge(srces) out_meta = srces[0].meta.copy() out_meta.update({"driver": "GTiff", "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_trans}) for src in srces: src.close() with rasterio.open(outfile, "w", **out_meta) as dest: dest.write(out_image) print('wrote {}'.format(outfile))
dataset.FlushCache() dataset = None # Preparing to add our new tile to the mosaic src_files_to_mosaic = [] src = rasterio.open('Topo_Slope.tif') src_files_to_mosaic.append(src) src = rasterio.open(name) src_files_to_mosaic.append(src) os.remove('/Users/jackson/Desktop/Execute/' + tile_id + '.tif') # Adding to Mosaic mosaic, out = merge(src_files_to_mosaic, method='first', bounds=Extent, res=Resolution, nodata=-9999) mosaic = np.reshape(mosaic, (Height, Width)) # Using GDAL to write the output raster filename = 'Topo_Slope.tif' # Tiff holding Black Top Hat driver = gdal.GetDriverByName('GTiff') # Driver for writing geo-tiffs dataset = driver.Create(filename, Width, Height, 1, gdal.GDT_Float32) # Creating our tiff file dataset.GetRasterBand(1).WriteArray(mosaic) # Writing values to our tiff dataset.SetGeoTransform(Geotransform) # Setting geo-transform dataset.SetProjection(Projection) # Setting the projection dataset.FlushCache() # Saving tiff to disk dataset = None
# Change to directory with files os.chdir(dir_name) # List files to be mosaiced files = os.listdir(dir_name) # List of open rasterio files raster_obj = [] for file in files: src = rasterio.open(file) raster_obj.append(src) # Mosaic raster files mosaic, mosaic_trans = merge(raster_obj) # Parse EPSG code epsg_code = int(src.crs.data['init'][5:]) # Reproject into the same coordinate system as raster data geo = geo.to_crs(epsg=epsg_code) # Get geometry coordinates for rasterio coords = getFeatures(geo) # Clip mosaic to target area clip_img, clip_transform = mask(raster=mosaic, shapes=coords, crop=True) # Copy the metadata clip_meta = src.meta.copy()
def NFIP_PNNL(depth_grids, NFIP_points, out_folder, export_shapefile=False): """ Calculates coverage and policies by flood depth Keyword arguments: depth_grids: list<str> -- array of file path names to the depth grid rasters NFIP_points: str -- Shapefile containing the NFIP points. Must contain fields: LOW_FLOOR, LOWADJ_GRA, T_COV_BLDG, T_COV_CONT, POL_NO, STATEFP and NAME out_folder: str -- file path name to directory for outputs """ # imports import geopandas as gpd import pandas as pd import rasterio as rio from rasterio.merge import merge # from rasterio.plot import show import matplotlib.pyplot as plt from shapely.geometry import mapping import numpy as np from time import time import uuid import os StateFipsCodes = { "01": "Alabama", "02": "Alaska", "04": "Arizona", "05": "Arkansas", "06": "California", "08": "Colorado", "09": "Connecticut", "10": "Delaware", "11": "District of Columbia", "12": "Florida", "13": "Geogia", "15": "Hawaii", "16": "Idaho", "17": "Illinois", "18": "Indiana", "19": "Iowa", "20": "Kansas", "21": "Kentucky", "22": "Louisiana", "23": "Maine", "24": "Maryland", "25": "Massachusetts", "26": "Michigan", "27": "Minnesota", "28": "Mississippi", "29": "Missouri", "30": "Montana", "31": "Nebraska", "32": "Nevada", "33": "New Hampshire", "34": "New Jersey", "35": "New Mexico", "36": "New York", "37": "North Carolina", "38": "North Dakota", "39": "Ohio", "40": "Oklahoma", "41": "Oregon", "42": "Pennsylvania", "44": "Rhode Island", "45": "South Carolina", "46": "South Dakota", "47": "Tennessee", "48": "Texas", "49": "Utah", "50": "Vermont", "51": "Virginia", "53": "Washington", "54": "West Virginia", "55": "Wisconsin", "56": "Wyoming" } if out_folder.endswith('/'): out_folder = out_folder[0:-1] t0 = time() print('Reading shapefile into memory') t1 =time() shp = gpd.read_file(NFIP_points) print(time() - t1) print('mosaicking rasters') t1 = time() raster_files = depth_grids src_files_to_mosaic = [] for file in raster_files: src = rio.open(file) src_files_to_mosaic.append(src) mosaic, transformation_info = merge(src_files_to_mosaic) # show(mosaic, cmap='terrain') # Copy the metadata out_meta = src.meta.copy() out_mosiac_fp = out_folder + '/' + str(uuid.uuid1()) + '.tif' # Update the metadata out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": transformation_info, "crs": shp.crs } ) with rio.open(out_mosiac_fp, "w", **out_meta) as dest: dest.write(mosaic) print(time() - t1) print('Reading mosiacked raster into memory') t1 = time() ras = rio.open(out_mosiac_fp) print(time() - t1) print('Mapping geometry for processing') t1 = time() geoms = list(map(lambda x: mapping(x), shp.geometry.values)) x = list(map(lambda x: x['coordinates'][0], geoms)) y = list(map(lambda x: x['coordinates'][1], geoms)) print(time() - t1) print('Extracting depths from raster to points') t1 = time() depths = [] for val in ras.sample(zip(x, y)): depths.append(val) print(time() - t1) print('Calcualting fields') t1 = time() gdf = gpd.GeoDataFrame(data=depths,geometry=shp.geometry) gdf.columns = ['Depth', gdf.columns[1]] # Calculate first floor height gdf['FFH'] = shp['LOW_FLOOR'] = shp['LOWADJ_GRA'] # Handle null values gdf['FFH'][gdf['FFH'] > 9000] = 1 gdf['FFH'][gdf['FFH'] < 1] = 1 gdf['FFH'][gdf['FFH'] > 17] = 17 # Calculate depth above first floor height gdf['Depth_Above_FFH'] = gdf['Depth'] - gdf['FFH'] # Calcualte exposure gdf['Coverage'] = (shp['T_COV_BLDG'] + shp['T_COV_CONT']) * 100 # Policy numbers gdf['Policy_Number'] = shp['POL_NO'] # Bin depth values gdf['Depth_Bins'] = '0' # gdf['Depth_Bins'][gdf['Depth'] == 0] = '0' gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 0) & (gdf['Depth_Above_FFH'] <= 1)] = '0 to 1' gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 1) & (gdf['Depth_Above_FFH'] <= 3)] = '1 to 3' gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 3) & (gdf['Depth_Above_FFH'] <= 6)] = '3 to 6' gdf['Depth_Bins'][(gdf['Depth_Above_FFH'] > 6) & (gdf['Depth_Above_FFH'] <= 9)] = '6 to 9' gdf['Depth_Bins'][gdf['Depth_Above_FFH'] > 9] = '> 9' # Sorting columns try: gdf['State'] = shp['STATEFP_1'] except: gdf['State'] = shp['STATEFP'] gdf = gdf.replace({'State': StateFipsCodes}) try: gdf['County'] = shp['NAMELSAD_1'] except: gdf['County'] = shp['NAME'] # Summarize pivot_coverage = pd.pivot_table(gdf, values='Coverage', index=['State', 'County'], columns='Depth_Bins', aggfunc=np.sum, margins=True) pivot_policies = pd.pivot_table(gdf, values='Policy_Number', index=['State', 'County'], columns='Depth_Bins', aggfunc=np.count_nonzero, margins=True) try: pivot_coverage.to_excel(out_folder + '/pivot_coverage.xlsx') pivot_policies.to_excel(out_folder + '/pivot_policies.xlsx') except: pivot_coverage.to_csv(out_folder + '/pivot_coverage.csv') pivot_policies.to_csv(out_folder + '/pivot_policies.csv') print(time() - t1) # print('Cleaning') # t1 = time() # os.remove(out_mosiac_fp) #unable to bc of file lock # print(time() - t1) if export_shapefile: print('Exporting spatial output') t1 = time() gdf.to_file(out_folder + '/' + "output.shp", driver='ESRI Shapefile') print(time() - t1) print('Total elapsed time: ' + str(time() - t0))
def test_merge_pathlib_path(tiffs): inputs = [Path(x) for x in tiffs.listdir()] inputs.sort() merge(inputs, res=2)
import rasterio from rasterio.merge import merge from pathlib import Path SRC_FOLDER = 'source_images_oza' OUT_FOLDER = 'mosaic_images_oza' src_images = [ rasterio.open(path) for path in (Path.cwd() / SRC_FOLDER).iterdir() ] mosaic, out_trans = merge(src_images) out_meta = src_images[0].meta.copy() out_meta.update({ "driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_trans, "crs": "+proj=utm +zone=35 +ellps=GRS80 +units=m +no_defs " }) with rasterio.open(Path(OUT_FOLDER) / 'mosaic.tif', "w", **out_meta) as dest: dest.write(mosaic)
def test_merge_tiny_intres(tiffs): inputs = [str(x) for x in tiffs.listdir()] inputs.sort() datasets = [rasterio.open(x) for x in inputs] merge(datasets, res=2)
def merge_maps(map_list, resolution): maps = [rasterio.open(map_path) for map_path in map_list] new_image, new_transform = merge.merge(maps, res=resolution, nodata=0) return new_image, new_transform
def rsts_to_mosaic(inRasterS, o, api="grass", fformat='.tif'): """ Create Mosaic of Raster """ if api == 'pygrass': """ The GRASS program r.patch allows the user to build a new raster map the size and resolution of the current region by assigning known data values from input raster maps to the cells in this region. This is done by filling in "no data" cells, those that do not yet contain data, contain NULL data, or, optionally contain 0 data, with the data from the first input map. Once this is done the remaining holes are filled in by the next input map, and so on. This program is useful for making a composite raster map layer from two or more adjacent map layers, for filling in "holes" in a raster map layer's data (e.g., in digital elevation data), or for updating an older map layer with more recent data. The current geographic region definition and mask settings are respected. The first name listed in the string input=name,name,name, ... is the name of the first map whose data values will be used to fill in "no data" cells in the current region. The second through last input name maps will be used, in order, to supply data values for for the remaining "no data" cells. """ from grass.pygrass.modules import Module m = Module("r.patch", input=inRasterS, output=o, overwrite=True, run_=False, quiet=True) m() elif api == 'grass': from glass.pys import execmd rcmd = execmd("r.patch input={} output={} --overwrite --quiet".format( ",".join(inRasterS), o)) elif api == 'rasterio': import rasterio from rasterio.merge import merge from glass.g.prop import drv_name from glass.g.prop.prj import get_epsg, epsg_to_wkt if type(inRasterS) != list: from glass.pys.oss import lst_ff rsts = lst_ff(inRasterS, file_format=fformat) else: rsts = inRasterS srcs = [rasterio.open(r) for r in rsts] mosaic, out_trans = merge(srcs) out_meta = srcs[0].meta.copy() out_meta.update({ "driver": drv_name(o), "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_trans, "count": 1, "crs": epsg_to_wkt(get_epsg(rsts[0])), "compress": 'lzw' }) with rasterio.open(o, "w", **out_meta) as dest: dest.write(mosaic) else: raise ValueError('api {} is not available'.format(api)) return o
dem_fps = glob.glob(q) # Files that were found: dem_fps # List for the source files src_files_to_mosaic = [] # Iterate over raster files and add them to source -list in 'read mode' for fp in dem_fps: src = rasterio.open(fp) src_files_to_mosaic.append(src) # Merge function returns a single mosaic array and the transformation info bnd = [87.813774, 22.432564, 91.157858, 26.209197] mosaic, ot = merge(src_files_to_mosaic, bounds=bnd) # Copy the metadata out_meta = src.meta.copy() # Update the metadata out_meta.update({ "driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": from_origin(bnd[0], bnd[3], 0.00179, 0.00179), #north and west coord "crs": "EPSG:4326" }) #~ print(mosaic.shape,out_meta) print("Mosaicing rasters... ")
def main(sqlite, state, folder): import pyproj from shapely.ops import transform from shapely.geometry import mapping from shapely.prepared import prep from rasterio.mask import mask from rasterio.enums import Resampling sqlite = r'E:\GeospatialData\Projects\i-love-washington\data\data.sqlite' state = 'Washington' folder = "E:\GeospatialData\Projects\i-love-washington\data\mosaic" # Create the output database table conn, cur = utils.connect_spatialite(sqlite) cur.execute('DROP TABLE lake_points') cur.execute(''' CREATE TABLE IF NOT EXISTS lake_points ( oid INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, z DOUBLE, path_ INTEGER, row_ INTEGER) ''') cur.execute( "SELECT AddGeometryColumn('lake_points', 'geom',3857, 'POINT', 2)") cur.execute("DELETE FROM lake_points") conn.commit() cur.close(), conn.close() sql = ''' SELECT name, TRANSFORM(geometry, 3857) geom FROM multipolygons WHERE boundary='administrative' AND name='Washington' ''' df_wa = utils.ogr2pandas(sqlite, sql=sql) wa_geom = df_wa.iloc[0].geom wa_buf = wa_geom.buffer(500, 1) bounds = wa_buf.bounds # Pull lakes from database lakes_sql = ''' SELECT name, ST_TRANSFORM(ST_UNION(geometry), 3857) geom, ST_AREA(ST_TRANSFORM(ST_UNION(geometry), 3857)) area FROM multipolygons WHERE natural='water' AND name IS NOT NULL AND ST_AREA(geometry) > 0.0001 AND (LOWER(name) LIKE '%lake%' OR LOWER(name) LIKE '%pool%' OR LOWER(name) LIKE '%reservoir%') GROUP BY name ORDER BY ST_AREA(ST_BUFFER(geometry, -0.001, 10)) DESC LIMIT 100 ''' df_lakes = utils.ogr2pandas(sqlite, sql=lakes_sql) # Apply a negative buffer, used for ignoring shoreline temperatures when sampling raster df_lakes['neg_buf'] = df_lakes['geom'].apply(lambda x: x.buffer(-150, 2)) # Prepare lakes for efficient lookups once df_lakes['prep'] = df_lakes['geom'].apply(lambda x: prep(x)) # print('Fetching LANDSAT WRS geoms') # df_wrs = utils.ogr2pandas( # sqlite, # 'WRS2', # columns=['PATH', 'ROW', 'geometry']) df_wrs = utils.ogr2pandas( dataset="E:\GeospatialData\Projects\i-love-washington\data\data.sqlite", sql=""" SELECT PATH, ROW, w.geometry FROM multipolygons m, wrs2 w WHERE m.name = 'Washington' AND ST_INTERSECTS(m.geometry, w.geometry) """) for row in df_wrs.to_dict('records'): logging.warning(f'Starting row {row["row"]}, path {row["path"]}') # Identify a cloudless scene mtl_md = landsat._return_candidates(int(row['path']), int(row['row']), 8, 5) logging.warning(f'Identified {len(mtl_md)} scenes') for key in mtl_md: metadata = mtl_md[key] save_file = utils.os.path.join( folder, f"{metadata['LANDSAT_PRODUCT_ID']}_FARENHEIT.TIF") if utils.os.path.exists(save_file): logging.warning(' Already Exists') continue # Pull image data from google into the metadata key 'bytes' logging.warning('Downloading image') landsat_bytes = landsat._pull_raster(metadata, 10) # Read Image logging.warning('Reading bytes to array') array, profile = utils.gdalbytes2numpy(landsat_bytes) logging.warning('Converting to farenheit') farenheit = landsat.array2farenheit(array, metadata, -99999).round(1) # Set profile nodata to -99999 and output type to float32 profile['nodata'] = -99999 profile['dtype'] = 'float32' # Reproject image bytes folder logging.warning('Writing to file') save_file = utils.os.path.join( folder, f"{metadata['LANDSAT_PRODUCT_ID']}_FARENHEIT.TIF") utils.reproject_array_to_file(farenheit, profile, 'EPSG:3857', save_file) j = utils.json.dumps(metadata, indent=2) with open(save_file.replace('FARENHEIT.TIF', 'METADATA.json'), 'w') as w: w.write(j) tifs = [ rasterio.open(utils.os.path.join(folder, f)) for f in utils.os.listdir(folder) if '_T1_FARENHEIT.TIF' in f ] bounds = (-13897124.13181157, 5707029.565730883, -13014721.44634677, 6275775.285325819) import numpy as np def calc_median(old_data, new_data, old_nodata, new_nodata, **kwargs): mask = np.logical_and(~old_nodata, ~new_nodata) old_data[mask] = np.ma.median([old_data[mask], new_data[mask]], axis=0) mask = np.logical_and(old_nodata, ~new_nodata) old_data[mask] = new_data[mask] mosaic, out_trans = merge(datasets=tifs, bounds=bounds, precision=1, method=calc_median) [tif.close() for tif in tifs] out_meta = tifs[0].meta.copy() out_meta.update({ "driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_trans, 'compress': 'LZW', 'tiled': True }) out_fp = utils.os.path.join(folder, 'mosaic_median.tif') with rasterio.open(out_fp, "w", **out_meta) as dest: dest.write(mosaic) del mosaic mosaic = rasterio.open(out_fp) profile = mosaic.profile # Iterate through the lakes and export a PNG for lake in df_lakes.to_dict('records'): break geoj = mapping(lake['geom']) # Mask the landsat image to lake pixels array, array_transform = mask(src, [geoj], crop=True) # mosaic, out_trans = merge( # datasets=tifs, # bounds=bounds, # precision=1, # method='max') # out_meta = tifs[0].meta.copy() # out_meta.update({"driver": "GTiff", # "height": mosaic.shape[1], # "width": mosaic.shape[2], # "transform": out_trans, # 'compress':'LZW', # 'tiled':True}) # out_fp = utils.os.path.join(folder, 'mosaic_max.tif') # with rasterio.open(out_fp, "w", **out_meta) as dest: # dest.write(mosaic) # # Write VRT # tifs = [utils.os.path.join(folder, f) for f in utils.os.listdir(folder) if f[-4:].lower() == '.tif'] # args = [ # utils.os.path.join(utils.gdal_path, 'gdalbuildvrt'), # '-hidenodata', '-srcnodata', '-99999', # '-r', 'average', # utils.os.path.join(folder, 'mosaic.vrt')] # args.extend(tifs) # p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE) # stdout, stderr = p.communicate() # # Write mosaic # args = [ # utils.os.path.join(utils.gdal_path, 'gdalwarp'), # '-overwrite', # '--config', 'GDAL_CACHEMAX', '512', # '-srcnodata', '-99999', # '-co', 'TILED=YES', # '-co', 'COMPRESS=LZW', # '-r', 'med', # '-wm', '2000', # utils.os.path.join(folder, 'mosaic.vrt'), # utils.os.path.join(folder, 'mosaic.tif'), # ] # p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE) # stdout, stderr = p.communicate() # # Get the projection of the image # logging.warning('Identifying projection') # metadata.update(utils.gdalinfo(metadata['bytes'])) # landsat_crs = pyproj.CRS(metadata['coordinateSystem']['wkt']) # wgs84 = pyproj.CRS('EPSG:4326') # project = pyproj.Transformer.from_crs(wgs84, landsat_crs, always_xy=True).transform # # Filter geometries to those that intersect with the scene, project to the correct UTM, and convert to geojson # logging.warning('Filtering lakes to those in the scene') # geoms = [mapping(transform(project, x)) for x in df_lakes['neg_buf'] if x.intersects(row['geom'])] # logging.warning('Creating raster memfile from image bytes') # memfile = utils.gdalbytes2numpy(metadata['bytes'], return_memfile=True) # logging.warning('Masking raster by lakes') # with memfile.open() as src: # # Get the metadata of the landsat image # profile = src.profile # # Mask the landsat image to lake pixels # array, array_transform = mask(src, geoms, crop=True) # # Set profile nodata to -99999 and output type to float32 # profile['nodata'] = 0 # profile['nodata'] = -99999 # profile['dtype'] = 'float32' # # Create coordinates from valid data # logging.warning('Creating coordinates') # df_coords = utils.array2coords(farenheit[0], array_transform, -99999) # df_coords['row_'] = row['row'] # df_coords['path_'] = row['path'] # # Write coords to database # logging.warning('Writing coordinates to database') # conn, cur = utils.connect_spatialite(sqlite) # cur.executemany( # f'INSERT INTO lake_points (geom, z, row_, path_) VALUES ( Transform(MakePoint(?, ?, {landsat_crs.to_epsg()}), 3857), ?, ?, ?)', # df_coords[['x', 'y', 'z', 'row_', 'path_']].itertuples(index=False) ) # conn.commit() # cur.close(), conn.close() # Move to top # # Create a copy of the lakes in the database for editing # args = [ # utils.os.path.join(utils.gdal_path, 'ogr2ogr'), # '-append', # '-t_srs', 'EPSG:3857', # '-sql', lakes_sql, # sqlite, sqlite, 'multipolygons', # '-nln','lakes_3857'] # p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE) # stdout, stderr = p.communicate() # args[-1] = 'lakes_3857_modified' # p = utils.Popen(args, stdout=utils.PIPE, stderr=utils.PIPE) # stdout, stderr = p.communicate() conn, cur = utils.connect_spatialite(sqlite) cur.execute("SELECT CreateSpatialIndex('lake_points', 'geom')")
dataset = driver.Create(filename,BTH_Array.shape[1], BTH_Array.shape[0], 1,gdal.GDT_Float32) dataset.GetRasterBand(1).WriteArray(BTH_Array) dataset.SetGeoTransform(T) # Setting geo-transform dataset.SetProjection(P) # Setting the projection dataset.FlushCache() # Saving tiff to disk dataset=None # Clearing variable name src_files_to_mosaic = [] # Initializing mosaic input src = rasterio.open('Tile.tif') # Grabbing the BTH results src_files_to_mosaic.append(src) # Appending BTH results src = rasterio.open('BTH.tif') # Grabbing results geo tiff src_files_to_mosaic.append(src) # Appending results geo tiff mosaic, out = merge(src_files_to_mosaic, method='first', bounds=Extent, res=Resolution, nodata=None) # Mosaic-ing in BTH results mosaic = np.reshape(mosaic,(Height,Width)) # Reshaping results filename = 'BTH.tif' # Tiff holding Black Top Hat driver = gdal.GetDriverByName('GTiff') # Driver for writing geo-tiffs dataset = driver.Create(filename,Width, Height, 1,gdal.GDT_Float32) # Creating our tiff file dataset.GetRasterBand(1).WriteArray(mosaic) # Writing values to our tiff dataset.SetGeoTransform(Geotransform) # Setting geo-transform dataset.SetProjection(Projection) # Setting the projection dataset.FlushCache() # Saving tiff to disk dataset=None # Clearing variable name os.remove('Tile.tif') # Removing the tile DEM
def _worker(self, job): """Load rasters, merge, and save to disk for a given date and db.""" # --- unpack job --- sanitized_db, date = job # --- list all satellite image COGT fragments for date --- y = str(date.year).zfill(2) m = str(date.month).zfill(2) d = str(date.day).zfill(2) db_dir = os.path.join(self.raw_s3_local_path, sanitized_db) date_dir = os.path.join(db_dir, str(y), str(m), str(d)) files = os.listdir(date_dir) # --- filter out inapplicable files --- files = [f for f in files if '.tif' in f] files = [f for f in files if 'PRODUCT_qa_value' not in f] # --- open files with rasterio --- src_files = [] for f in files: # --- read rasters --- srcr = rasterio.open(os.path.join(date_dir, f)) src_files.append(srcr) # --- merge into a single raster --- #TODO: Confirm rasterio behavior for overlaps mosaic, mosaic_transform = merge( src_files + [self.blank_world] ) #https://gis.stackexchange.com/questions/360685/creating-daily-mosaics-from-orbiting-satellite-imagery # --- replace null values --- null_val = srcr.nodata mosaic[mosaic == null_val] = np.nan # --- copy meta data --- profile = srcr.meta.copy() profile.update(width=mosaic.shape[2], height=mosaic.shape[1], transform=mosaic_transform) # --- resample in memory --- if self.resample_shape != None: with MemoryFile() as memfile: with memfile.open(**profile) as dataset: dataset.write(mosaic) dataset = memfile.open() mosaic = dataset.read( out_shape=( dataset.count, self.resample_shape[0], self.resample_shape[1] # int(dataset.height * self.resample_scale), # int(dataset.width * self.resample_scale) ), resampling=Resampling.bilinear) # --- scale image transform --- mosaic_transform = dataset.transform * dataset.transform.scale( (dataset.width / mosaic.shape[-1]), (dataset.height / mosaic.shape[-2])) del dataset if self.downcast_dtype != 'float64': # --- clip --- mosaic = np.clip(mosaic, np.percentile(mosaic, 1), np.percentile(mosaic, 99)) # --- normalize --- downcast_min_val = np.iinfo(self.downcast_dtype).min downcast_max_val = np.iinfo(self.downcast_dtype).max Scaler = MinMaxScaler(feature_range=(downcast_min_val, downcast_max_val)) mosaic = np.squeeze(mosaic) mosaic = Scaler.fit_transform(mosaic) # --- downcast --- mosaic = mosaic.astype(self.downcast_dtype) # --- Make 3D as rasterio expects --- mosaic = np.expand_dims(mosaic, axis=0) # --- update profile again --- profile.update(width=mosaic.shape[2], height=mosaic.shape[1], transform=mosaic_transform, dtype=self.downcast_dtype, nodata=0) # --- save to disk --- out_db = sanitized_db.split(os.path.sep)[-2].replace('L2', 'L3') out_filename = f"{out_db}{date.strftime('%Y-%m-%d')}.tif" out_fp = os.path.join(self.out_file_dir, out_filename) with rasterio.open(out_fp, "w", **profile) as fp: fp.write(mosaic) return self
# Coordinate Reference System d[0].meta # In[23]: # Check Meta Data of original Files out_meta = src.meta.copy() out_meta # In[24]: mosaic, out_trans = merge(d) # In[25]: # Create Metadata of the for the mosaic TIFF out_meta.update({"driver": "HFA","height":mosaic.shape[1],"width":mosaic.shape[2],"transform": out_trans,}) # In[26]: # Write the updated DEM to the specified file path with rasterio.open(strOutTiffPath, "w", **out_meta) as dest: dest.write(mosaic)
def test_merge_tiny_intres(tiffs): inputs = [str(x) for x in tiffs.listdir()] inputs.sort() datasets = [rasterio.open(x) for x in inputs] merge(datasets, res=2)
ls = WYcountylist src_file_to_mosaic = [] outf = 'D:/twu/fireline/travelers/tiffs/merge.tiff' for county in ls: print(county) try: with rasterio.drivers(): src = rasterio.open( 'D:/twu/fireline/travelers/tiffs/bon_ID_WFHS.tiff') src_1 = rasterio.open( 'D:/twu/fireline/travelers/tiffs/peop_WA_WFHS.tiff') #src_file_to_mosaic.append() src_file_to_mosaic = [src, src_1] print("open") mosaic, out_trans = merge(src_file_to_mosaic) print("merge") mosaic = mosaic.astype(np.int8) with rasterio.open(outf, "w", driver='GTiff', dtype="uint8", nodata=255, compress="lzw", height=mosaic.shape[1], count=1, width=mosaic.shape[2], transform=out_trans, crs="EPSG:4326") as dest: dest.write(mosaic, 1) print("done")
def merge_ak_canada(ak_fn, can_fn, template_fn, variable, output_path, scalevals): from rasterio.merge import merge # reproject canada to 3338 and rescale temperature values from ints to floats dtype = 'float32' if variable == 'ppt': scalevals = False # open alaska AAI grid -- special for precip... ak = open_ak_precip(ak_fn) else: # temperature variables scalevals = True # open alaska AAI grid ak = scale_ak_temperature_to_int(ak_fn, nodata=-9999) # already in 3338 can = reproject_canada_to3338(can_fn, template_fn, dtype=dtype) # get some output bounds information for the merge... with rasterio.open(template_fn) as tmp: bounds = tmp.bounds mask = tmp.read_masks(1) # merge 'em merged, transform = merge([can, ak], bounds=bounds, res=(2000, 2000), nodata=-9999, precision=5) # fix some mask mismatches merged = fill_missing_mask(merged[0, ...], mask) # update metadata meta = ak.meta height, width = merged.shape meta.update({ 'driver': 'GTiff', 'height': height, 'width': width, 'crs': { 'init': 'EPSG:3338' }, 'transform': transform, 'compress': 'lzw', 'dtype': dtype }) # output filenaming varname_lookup = { 'tmean': 'tas', 'tmin': 'tasmin', 'tmax': 'tasmax', 'ppt': 'pr' } if variable == 'ppt': metric = 'total' units = 'mm' elif variable in ['tmean', 'tmax', 'tmin']: metric = 'mean' units = 'C' # write to disk try: if not os.path.exists(output_path): os.makedirs(output_path) except: pass # deal with parallel processing in a clunky way month = os.path.basename(ak_fn).split('.')[0].split('_')[-1] if len(month) == 1: month = '0{}'.format(month) # scale the temperature PRISM values if scalevals: merged[ merged != meta['nodata']] = merged[merged != meta['nodata']] / 10.0 output_filename = os.path.join( output_path, '{}_{}_{}_akcan_prism_{}_1961_1990.tif'.format( varname_lookup[variable], metric, units, month)) with rasterio.open(output_filename, 'w', **meta) as out: out.write(merged, 1) return output_filename
# Make a search criteria to select the DEM files search_criteria = "L*.tif" q = os.path.join(data_dir, search_criteria) print(q) # List files that matches the criteria dem_fps = glob.glob(q) dem_fps # Open the source files with rasterio # len gives the amount of files, charactor etc src_files_to_mosaic = [ rasterio.open(fp) for fp in dem_fps ] len(src_files_to_mosaic) # Merge rasters into mosaic mosaic, out_trans = merge(datasets=src_files_to_mosaic) # Plot show(mosaic, cmap="terrain") #plt.imshow(mosaic, cmap="terrain") #plt.colorbar() # Save the mosaic and update the metadata # Here the metadata is a copy of one file and needs to be update out_meta = src_files_to_mosaic[0].meta.copy() # Update metadata with new dimensions and crs out_meta.update( {"height" : mosaic.shape[1], "width" :mosaic.shape[2], "transform": out_trans,
def get_mosaic(avail_hu12catchs_group, hu, break_hu, dst_crs): ## Get mosaic of DEMs for each HUC def append_check(src_files_to_mosaic, var, subdirectory, hu): ## Check each raster's resolution in this HUC if any(np.float16(i) > 1. for i in var.res): out_path = os.path.join(subdirectory, "gt1m.err") Path(out_path).touch() print('WARNING: >1m raster input for HUC12: ' + str(hu)) sys.stdout.flush() #del(out_path) else: src_res_min_to_mosaic.append(min(var.res)) src_res_max_to_mosaic.append(min(var.res)) src_x_to_mosaic.append(var.res[0]) src_y_to_mosaic.append(var.res[1]) src_files_to_mosaic.append(var) return (src_files_to_mosaic, src_res_min_to_mosaic, src_res_max_to_mosaic, src_x_to_mosaic, src_y_to_mosaic) ## Reproject the mosaic to DEM tiles pertaining to each HUC #print('INSIDE GET MOSAIC\t',arguments[0]) #print('INSIDE GET MOSAIC\t',flows_key) #sys.stdout.flush() dem_fps = list(avail_hu12catchs_group['stampede2name']) src_files_to_mosaic = [] src_res_min_to_mosaic = [] src_res_max_to_mosaic = [] src_x_to_mosaic = [] src_y_to_mosaic = [] memfile = {} for fp in dem_fps: memfile[fp] = MemoryFile() #del(fp) for fp in dem_fps: with rasterio.open(fp) as src: transform, width, height = calculate_default_transform( src.crs, dst_crs, src.width, src.height, *src.bounds) out_meta = src.meta.copy() out_meta.update({ 'crs': dst_crs, 'transform': transform, 'width': width, 'height': height }) #del(width,height) ## Don't do an expensive reprojection if projection already correct if src.meta == out_meta: #del(fp,transform) src_files_to_mosaic, src_res_min_to_mosaic, src_res_max_to_mosaic, src_x_to_mosaic, src_y_to_mosaic = append_check( src_files_to_mosaic, src, subdirectory, hu) else: dst = memfile[fp].open(**out_meta) #del(fp,out_meta) for i in range(1, src.count + 1): reproject(source=rasterio.band(src, i), destination=rasterio.band(dst, i), src_transform=src.transform, src_crs=src.crs, dst_transform=dst.transform, dst_crs=dst.crs, resampling=Resampling.nearest) #del(i) #del(transform) src_files_to_mosaic, src_res_min_to_mosaic, src_res_max_to_mosaic, src_x_to_mosaic, src_y_to_mosaic = append_check( src_files_to_mosaic, dst, subdirectory, hu) #del(dst) #del(dem_fps) if len(src_files_to_mosaic) == 0: out_path = os.path.join(subdirectory, "allGT1m.err") Path(out_path).touch() print('WARNING: Found no <=1m raster input data for HUC12: ' + str(hu)) sys.stdout.flush() #del(out_path) break_hu = True mosaic_tuple = () return (break_hu, mosaic_tuple) else: src_files_to_mosaic = pd.DataFrame( data={ 'Files': src_files_to_mosaic, 'min(resolution)': src_res_min_to_mosaic, 'max(resolution)': src_res_max_to_mosaic }) #del(src_res_min_to_mosaic,src_res_max_to_mosaic) src_files_to_mosaic.sort_values( by=['min(resolution)', 'max(resolution)'], inplace=True) mosaic, out_trans = merge(list(src_files_to_mosaic['Files']), res=(max(src_x_to_mosaic), max(src_y_to_mosaic))) #del(src_x_to_mosaic,src_y_to_mosaic) for src in src_files_to_mosaic['Files']: src.close() #del(src_files_to_mosaic) out_meta = src.meta.copy() #del(src) out_meta.update({ "driver": 'GTiff', "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_trans, "crs": dst_crs }) #del(out_trans) for keyvalue in memfile.items(): keyvalue[1].close() #del(keyvalue) #del(memfile) mosaic_tuple = (mosaic, out_meta) return (break_hu, mosaic_tuple)
def test_merge_filenames(tiffs): inputs = [str(x) for x in tiffs.listdir()] inputs.sort() merge(inputs, res=2)
def test_issue2163(): """Demonstrate fix for issue 2163""" with rasterio.open("tests/data/float_raster_with_nodata.tif") as src: data = src.read() result, transform = merge([src]) assert numpy.allclose(data, result)