def merge_gcp_tiled_results(bucket_path,pattern,region,retries=-1,clean_up=False,cloud_project=None,file_dims=None): land_area = ( ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017") .filterBounds(region) .geometry(100) .buffer(2500, maxError=100) ) grid = geeutils.tile_region(region, intersect_geom=land_area, grid_size=1.0) expected_n = grid.size().getInfo() fcomponents = bucket_path.split("/") bucket = fcomponents[2] fpath = pattern.replace("*","") files = utils.list_gcs_objs(bucket, pattern=pattern, project=cloud_project) if len(files) == expected_n: images = [ee.Image.loadGeoTIFF(file) for file in files] merged = ee.ImageCollection.fromImages(images).mosaic() export_region = region.bounds(maxError=100).getInfo()["coordinates"] # bucket_path, ext = os.path.splitext(output_bucket_path) now = datetime.datetime.now() time_id = now.strftime("%Y%m%d%H%M%s") task = ee.batch.Export.image.toCloudStorage( image=merged, description=f"hydrafloods_merge_gcp_export_{time_id}", bucket=bucket, fileNamePrefix=fpath, region=export_region, scale=10, crs="EPSG:4326", fileDimensions=file_dims, maxPixels=1e13, fileFormat="GeoTIFF", formatOptions={"cloudOptimized": True}, ) task.start() if clean_up: gcsfs.GCSFileSystem.rm(files) elif retries > 0: time.sleep(60*10) merge_gcp_tiled_results(bucket_path,pattern,region,retries=(retries-1)) else: raise RuntimeError(f"could not find all expected tiles to merge...found {len(files)} tiles but expected {expected_n}") return
def export_harmonics( region, start_time, end_time, feature_names=None, label=None, fusion_samples=None, fusion_model_asset=None, output_asset_path=None, output_bucket=None, tile=False, ): if tile: land_area = ( ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017") .filterBounds(region) .geometry(100) .buffer(2500, maxError=100) ) grid = geeutils.tile_region(region, intersect_geom=land_area, grid_size=1.0) n = grid.size().getInfo() grid_list = grid.toList(n) for i in range(n): grid_tile = ee.Feature(grid_list.get(i)).geometry() if output_asset_path is not None: output_tile_path = output_asset_path + f"harmonics_t{i}" export_harmonics( grid_tile, start_time, end_time, feature_names, label, fusion_samples, fusion_model_asset, output_tile_path, output_bucket, tile=False, ) else: if fusion_samples is not None: fusion_model, scaling_dict = ml.random_forest_ee( 25, fusion_samples, feature_names, label, mode="regression" ) elif fusion_model_asset is not None: raise NotImplementedError() else: raise ValueError( "Either 'fusion_samples' or 'fusion_model_path' needs to be defined to run fusion process" ) ds, label = _fuse_dataset( region, start_time, end_time, fusion_model, scaling_dict, target_band="mndwi", ) now = datetime.datetime.now() time_id = now.strftime("%Y%m%d%H%M%s") time_str = now.strftime("%Y-%m-%d %H:%M:%s") scale_factor = 0.0001 # create metadata dict metadata = ee.Dictionary( { "hf_version": hf.__version__, "scale_factor": scale_factor, "fit_time_start": start_time, "fit_time_end": end_time, "execution_time": time_str, } ) harmonic_coefs = timeseries.fit_harmonic_trend( ds, dependent="mndwi", output_err=True ) harmonic_coefs = harmonic_coefs.divide(scale_factor).int32().set(metadata) if output_asset_path is not None: geeutils.export_image( harmonic_coefs, region, output_asset_path, description=f"hydrafloods_harmonic_coefficient_export_{time_id}", scale=10, crs="EPSG:4326", ) elif output_bucket is not None: raise NotImplementedError() else: raise ValueError( "Either 'output_asset_path' or 'output_bucket' needs to be defined to run fusion export process" ) return
def export_daily_surface_water( region, target_date, harmonic_coefs=None, harmonic_collection=None, feature_names=None, label=None, look_back=30, lag=4, output_confidence=False, output_flood = False, fusion_samples=None, fusion_model_asset=None, output_asset_path=None, output_bucket_path=None, initial_threshold=0.1, tile=False, tile_buffer=100000 ): def get_weights(i): i = ee.Number(i) t_diff = ( ee.Number(i).multiply(-1).subtract(lag) ) # calc how many days to adjust ini date new_date = target_date.advance(t_diff, "day") # calculate new date corr_img = ( ds.collection.select(label) .filterDate(new_date, new_date.advance(1, "day")) .qualityMosaic(label) ) time_img = timeseries.get_dummy_img(new_date) harmon_pred = ( timeseries.add_harmonic_coefs(time_img) .multiply(harmonic_coefs) .reduce("sum") ) harmon_diff = harmon_pred.subtract(corr_img).rename("residual") return harmon_diff.set("system:time_start", new_date.millis()) def calc_confidence(i): i = ee.Number(i) # uniform sampling of std dev at 95% confidence interval long_term_seed = i.add(500) short_term_seed = i.add(1000) long_term_random = ( ee.Image.random(long_term_seed).multiply(3.92).subtract(1.96) ) short_term_random = ( ee.Image.random(short_term_seed).multiply(3.92).subtract(1.96) ) lin_sim = lin_pred.add(short_term_random.multiply(linCi)) har_sim = har_pred.add(long_term_random.multiply(harCi)) sim_pred = har_sim.subtract(lin_sim) # random_water = thresholding.bmax_otsu(random_combination,invert=True) # naive estimate of water (>0) return sim_pred.gt(ci_threshold).uint8() if tile: if tile: land_area = ( ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017") .filterBounds(region) .geometry(100) .buffer(2500, maxError=100) ) grid = geeutils.tile_region(region, intersect_geom=land_area, grid_size=1.0) n = grid.size().getInfo() grid_list = grid.toList(n) for i in range(n): if output_asset_path is not None: output_asset_tile = output_asset_path + f"daily_tile{i:05d}" else: output_asset_tile = None if output_bucket_path is not None: output_bucket_tile = output_bucket_path + f"_tile{i:05d}" else: output_bucket_tile = None grid_tile = ee.Feature(grid_list.get(i)).geometry() export_daily_surface_water( grid_tile, target_date, harmonic_coefs, harmonic_collection, feature_names, label, look_back, lag, output_confidence, output_flood, fusion_samples, fusion_model_asset, output_asset_tile, output_bucket_tile, initial_threshold, tile=False, tile_buffer=tile_buffer ) else: end_time = ee.Date(target_date).advance(-(lag - 1), "day") start_time = end_time.advance(-look_back, "day") if fusion_samples is not None: fusion_model, scaling_dict = ml.random_forest_ee( 25, fusion_samples, feature_names, label, mode="regression" ) elif fusion_model_asset is not None: raise NotImplementedError() else: raise ValueError( "Either 'fusion_samples' or 'fusion_model_path' needs to be defined to run fusion process" ) if not isinstance(target_date, ee.Date): target_date = ee.Date(target_date) now = datetime.datetime.now() time_id = now.strftime("%Y%m%d%H%M%s") time_str = now.strftime("%Y-%m-%d %H:%M:%s") if harmonic_coefs is not None: harmonic_coefs = ee.Image(harmonic_coefs) harmonic_coefs = harmonic_coefs.multiply( ee.Image(ee.Number(harmonic_coefs.get("scale_factor"))) ) elif harmonic_collection is not None: harmonic_collection = ee.ImageCollection(harmonic_collection) first = ee.Image(harmonic_collection.first()) harmonic_coefs = harmonic_collection.mosaic().multiply( ee.Image(ee.Number(first.get("scale_factor"))) ) else: raise ValueError( "Either 'harmonic_coefs' or 'harmonic_collection' needs to be defined to run fusion process" ) if output_confidence: harmonic_err = harmonic_coefs.select(".*(x|y|n)$") harmonic_coefs = harmonic_coefs.select("^(c|t|s).*") else: harmonic_coefs = harmonic_coefs.select("^(c|t|s).*") prod_region = region.buffer(tile_buffer,100) ds, label = _fuse_dataset( prod_region, start_time, end_time, fusion_model, scaling_dict, target_band=label, use_viirs=True, use_modis=False, ) dummy_target = timeseries.get_dummy_img(target_date) weights = ee.ImageCollection.fromImages( ee.List.sequence(0, look_back - 1).map(get_weights) ).sort("system:time_start") weights_lr = timeseries.fit_linear_trend( weights, dependent="residual", output_err=output_confidence ) weights_coefs = weights_lr.select("^(c|t).*") lin_pred = ( dummy_target.multiply(weights_coefs).reduce("sum").rename("residual_est") ) har_pred = ( timeseries.add_harmonic_coefs(dummy_target) .multiply(harmonic_coefs) .reduce("sum") ) fused_pred = ( (har_pred.subtract(lin_pred)) .convolve(ee.Kernel.gaussian(2.5)) .rename("fused_product") ) # water,threshold = thresholding.bmax_otsu( # fused_pred, # initial_threshold=0, # grid_size=0.2, # region=region, # invert=True, # reduction_scale=100, # return_threshold=True # ) ci_threshold = thresholding.edge_otsu( fused_pred, initial_threshold=initial_threshold, edge_buffer=300, region=prod_region, invert=True, reduction_scale=200, return_threshold=True, ) # ci_threshold = ee.Number(ee.Algorithms.If(ci_threshold.lt(-0.1),-0.1,ci_threshold)) permanent_water = ( ee.ImageCollection("JRC/GSW1_2/YearlyHistory") .limit(5,"system:time_start",False) .map(lambda x: x.select('waterClass').eq(3)) .sum().unmask(0).gt(0) ) water = fused_pred.gt(ci_threshold).Or(permanent_water).rename("water").uint8() if output_flood: flood = water.select("water").And(permanent_water.Not()).rename("flood") water = water.addBands(flood) if output_confidence: weights_err = weights_lr.select(".*(x|y|n)$") linCi = weights_err.expression( "mse * ((1/n) + ((t-xmean)**2/xr))**(1/2)", { "mse": weights_err.select("residual_y"), "n": weights_err.select("n"), "xmean": weights_err.select("mean_x"), "xr": weights_err.select("residual_x"), "t": dummy_target.select("time"), }, ) harCi = harmonic_err.expression( "mse * ((1/n) + ((t-xmean)**2/xr))**(1/2)", { "mse": harmonic_err.select("residual_y"), "n": harmonic_err.select("n"), "xmean": harmonic_err.select("mean_x"), "xr": harmonic_err.select("residual_x"), "t": dummy_target.select("time"), }, ) # ci_threshold = ee.Number(fused_pred.updateMask(water).reduceRegion( # reducer=ee.Reducer.min(), # geometry=region, # scale=100, # bestEffort=True, # maxPixels=1e6 # ).get('fused_product')) confidence = ( ee.ImageCollection.fromImages( ee.List.sequence(0, 99).map(calc_confidence) ) .reduce(ee.Reducer.mean(), 16) .multiply(100) .uint8() .rename("confidence") ) out_water = ee.Image.cat([confidence, water,]) else: out_water = water # water.uint8().rename("water"), fused_pred = fused_pred.multiply(10000).int16() if output_asset_path is not None: # create metadata dict metadata = ee.Dictionary( { "hf_version": hf.__version__, "system:time_start": target_date.millis(), "system:time_end": target_date.advance(86399, "seconds").millis(), "execution_time": time_str, "lag": lag, "look_back": look_back, } ) geeutils.export_image( out_water.set(metadata.combine({"product": "water"})), region, output_asset_path + "_water", description=f"hydrafloods_water_ee_export_{time_id}", scale=10, crs="EPSG:4326", ) geeutils.export_image( fused_pred.set(metadata.combine({"product": "fusion"})), region, output_asset_path + "_fusion", description=f"hydrafloods_fusion_ee_export_{time_id}", scale=10, crs="EPSG:4326", ) elif output_bucket_path is not None: export_region = region.bounds(maxError=100).getInfo()["coordinates"] bucket_path, ext = os.path.splitext(output_bucket_path) fcomponents = bucket_path.split("/") bucket = fcomponents[2] fpath = fcomponents[3:-1] f_water = "/".join(fpath + [fcomponents[-1] + "_water" + ext]) f_fusion = "/".join(fpath + [fcomponents[-1] + "_fusion" + ext]) water_task = ee.batch.Export.image.toCloudStorage( image=out_water, description=f"hydrafloods_water_gcp_export_{time_id}", bucket=bucket, fileNamePrefix=f_water, region=export_region, scale=10, crs="EPSG:4326", maxPixels=1e13, fileFormat="GeoTIFF", formatOptions={"cloudOptimized": True}, ) water_task.start() fusion_task = ee.batch.Export.image.toCloudStorage( image=fused_pred, description=f"hydrafloods_fusion_gcp_export_{time_id}", bucket=bucket, fileNamePrefix=f_fusion, region=export_region, scale=10, crs="EPSG:4326", maxPixels=1e13, fileFormat="GeoTIFF", formatOptions={"cloudOptimized": True}, ) fusion_task.start() else: raise ValueError( "Either 'output_asset_path' or 'output_bucket_path' needs to be defined to run fusion export process" ) return
def export_daily_surface_water( region, target_date, harmonic_image=None, harmonic_collection=None, feature_names=None, label=None, look_back=30, lag=4, n_cycles=2, include_confidence=False, include_flood=False, export_fusion=False, fusion_samples=None, output_asset_path=None, output_bucket_path=None, initial_threshold=0.1, tile=False, tile_size=1.0, tile_buffer=100000, output_scale=30, ): """Last and repeated step of the daily surface water fusion process. This procedure uses the results from `export_fusion_samples` and `export_surface_water_harmonics` to build a random forest model to predict a water index from SAR imagery and predict water using the harmonic model. This process will correct the harmonic estimate using observed data and export the resulting imagery. args: region (ee.Geometry): geographic region to look for coincident data and sample from target_date (str | datetime.datetime): date to estimate surface water extent for harmonic_image (str, optional): Earth Engine Image asset id of the harmonic model weights exported by `export_surface_water_harmonics`. If left as None then `harmonic_collection` must be defined. default = None harmonic_collection (str, optional): Earth Engine ImageCollection asset id of the harmonic model weights from tile `export_surface_water_harmonics`. If left as None then `harmonic_image` must be defined. default = None feature_names (list[str],): names of feature columns used to calculate `label` from label (str): name of feature column to predict using `feature_names` look_back (int,optional): number of days used to estimate short-term trend in water. default = 30 lag (int, optional): number of days after `target_date` to begin `look_back`. default=4 n_cycles (int, optional): number of interannual cycles to model. default = 2 include_confidence (bool, optional): boolean keyword to specify if a confidence band will be exported with surface water image. If True then confidence will be calculated. default = False include_flood (bool, optional): boolean keyword to specify if a flood band will be exported with surface water image. If True then flood will be calculated based on JRC permanent water data. default = False export_fusion (bool, optional): boolean keyword to specify if the fusion image used to calculate water should be exported as a seperated task. If True then run fusion export task. default = False fusion_samples (str): Earth Engine FeatureCollection asset id of samples to get a data fusion model from. Should be the asset output from `export_fusion_samples` output_asset_path (str): Earth Engine asset id to save estimate water and fusion results to as image. If tile==True, then output_asset_path much be a precreated ImageCollection asset. If left as None then `output_bucket_path` must be specified. default = None output_bucket_path (str): GCP cloud bucket path to save estimate water and fusion results to cloud optimized geotiffs. If tile==True, then multiple file will be created. If left as None then `output_asset_path` must be specified. default = None initial_threshold (float, optional): initial threshold value used in `edge_otsu` thresholding algorithm to segment water from fusion image. default = 0.1 tile (bool, optional): boolean keyword to tile exports. If false will try to calculate harmonic weights as image. If true, it will tile area and recusively call to export smaller areas. If true then expects that `output_asset_path` is an ImageCollection. default = False tile_size (float, optional): resolution in decimal degrees to create tiles over region for smaller exports. Only used if tile==True. default = 1.0 tile_buffer (float,optional): buffer size in meters to buffer tiles to calculate threshold. This is used to ensure running tiled exports produces consistent results at tile seams. default = 100000 output_scale (float, optional): output resolution of harmonic weight image. default = 30 raises: ValueError: if `fusion_samples` is None ValueError: if both`harmonic_image` and `harmonic_collection` is None ValueError: if both 'output_asset_path' and 'output_bucket_path' is None """ def get_residuals(i): """Closure function to calculate residuals of harmonic water estimate compared to observed data. """ i = ee.Number(i) t_diff = (ee.Number(i).multiply(-1).subtract(lag) ) # calc how many days to adjust ini date new_date = target_date.advance(t_diff, "day") # calculate new date corr_img = (ds.collection.select(label).filterDate( new_date, new_date.advance(1, "day")).qualityMosaic(label)) time_img = timeseries.get_dummy_img(new_date) harmon_pred = (timeseries.add_harmonic_coefs(time_img).multiply( harmonic_coefs).reduce("sum")) harmon_diff = harmon_pred.subtract(corr_img).rename("residual") return harmon_diff.set("system:time_start", new_date.millis()) def calc_confidence(i): """Closure function to calculate confidence in water estimate using monte carlo methods and simulating errors in long- and short-term water dynamics """ i = ee.Number(i) # uniform sampling of std dev at 95% confidence interval long_term_seed = i.add(500) short_term_seed = i.add(1000) long_term_random = ee.Image.random(long_term_seed).multiply( 3.92).subtract(1.96) short_term_random = ( ee.Image.random(short_term_seed).multiply(3.92).subtract(1.96)) lin_sim = lin_pred.add(short_term_random.multiply(linCi)) har_sim = har_pred.add(long_term_random.multiply(harCi)) sim_pred = har_sim.subtract(lin_sim) # random_water = thresholding.bmax_otsu(random_combination,invert=True) # naive estimate of water (>0) return sim_pred.gt(ci_threshold).uint8() if tile: if tile: land_area = (ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017"). filterBounds(region).geometry(100).buffer( 2500, maxError=100)) grid = geeutils.tile_region(region, intersect_geom=land_area, grid_size=tile_size) n = grid.size().getInfo() grid_list = grid.toList(n) for i in range(n): if output_asset_path is not None: output_asset_tile = output_asset_path + f"daily_tile{i:05d}" else: output_asset_tile = None if output_bucket_path is not None: output_bucket_tile = output_bucket_path + f"_tile{i:05d}" else: output_bucket_tile = None grid_tile = ee.Feature(grid_list.get(i)).geometry() export_daily_surface_water( region=grid_tile, target_date=target_date, harmonic_image=harmonic_image, harmonic_collection=harmonic_collection, feature_names=feature_names, label=label, look_back=look_back, lag=lag, n_cycles=n_cycles, include_confidence=include_confidence, include_flood=include_flood, export_fusion=export_fusion, fusion_samples=fusion_samples, output_asset_path=output_asset_tile, output_bucket_path=output_bucket_tile, initial_threshold=initial_threshold, tile=False, tile_buffer=tile_buffer, output_scale=output_scale, ) else: if not isinstance(target_date, ee.Date): target_date = ee.Date(target_date) end_time = target_date.advance(-(lag - 1), "day") start_time = end_time.advance(-look_back, "day") if fusion_samples is not None: fusion_model, scaling_dict = ml.random_forest_ee( 25, fusion_samples.limit(10000), feature_names, label, scaling="standard", mode="regression", ) else: raise ValueError( "'fusion_samples' needs to be defined to run fusion process") now = datetime.datetime.now() time_id = now.strftime("%Y%m%d%H%M%s") time_str = now.strftime("%Y-%m-%d %H:%M:%s") if harmonic_image is not None: harmonic_coefs = ee.Image(harmonic_image) harmonic_coefs = harmonic_coefs.multiply( ee.Image(ee.Number(harmonic_coefs.get("scale_factor")))) elif harmonic_collection is not None: harmonic_collection = ee.ImageCollection(harmonic_collection) first = ee.Image(harmonic_collection.first()) harmonic_coefs = harmonic_collection.mosaic().multiply( ee.Image(ee.Number(first.get("scale_factor")))) else: raise ValueError( "Either 'harmonic_image' or 'harmonic_collection' needs to be defined to run fusion process" ) if include_confidence: harmonic_err = harmonic_coefs.select(".*(x|y|n)$") harmonic_coefs = harmonic_coefs.select("^(c|t|s).*") else: harmonic_coefs = harmonic_coefs.select("^(c|t|s).*") prod_region = region.buffer(tile_buffer, 100) ds = _fuse_dataset(region, start_time, end_time, fusion_model, scaling_dict, feature_names, target_band=label, use_viirs=True) dummy_target = timeseries.get_dummy_img(target_date) weights = ee.ImageCollection.fromImages( ee.List.sequence(0, look_back - 1).map(get_residuals)).sort("system:time_start") weights_lr = timeseries.fit_linear_trend(weights, dependent="residual", output_err=include_confidence) weights_coefs = weights_lr.select("^(c|t).*") lin_pred = (dummy_target.multiply(weights_coefs).reduce("sum").rename( "residual_est")) har_pred = (timeseries.add_harmonic_coefs( dummy_target, n_cycles=n_cycles).multiply(harmonic_coefs).reduce("sum")) fused_pred = hf.filtering.p_median( (har_pred.subtract(lin_pred))).rename("fused_product") ci_threshold = thresholding.edge_otsu( fused_pred, initial_threshold=initial_threshold, edge_buffer=300, region=prod_region, invert=True, scale=200, return_threshold=True, ) permanent_water = ( ee.ImageCollection("JRC/GSW1_2/YearlyHistory").filterDate( "1985-01-01", end_time).limit(5, "system:time_start", False). map(lambda x: x.select("waterClass").eq(3)).sum().unmask(0).gt(0)) water = fused_pred.gt(ci_threshold).Or(permanent_water).rename( "water").uint8() if include_flood: flood = water.select("water").And( permanent_water.Not()).rename("flood") water = water.addBands(flood) if include_confidence: weights_err = weights_lr.select(".*(x|y|n)$") linCi = weights_err.expression( "mse * (1 + (1/n) + ((t-xmean)**2/xr))**(1/2)", { "mse": weights_err.select("residual_y"), "n": weights_err.select("n"), "xmean": weights_err.select("mean_x"), "xr": weights_err.select("residual_x"), "t": dummy_target.select("time"), }, ) harCi = harmonic_err.expression( "mse * (1 + (1/n) + ((t-xmean)**2/xr))**(1/2)", { "mse": harmonic_err.select("residual_y"), "n": harmonic_err.select("n"), "xmean": harmonic_err.select("mean_x"), "xr": harmonic_err.select("residual_x"), "t": dummy_target.select("time"), }, ) confidence = (ee.ImageCollection.fromImages( ee.List.sequence(0, 99).map(calc_confidence)).reduce( ee.Reducer.mean(), 16).multiply(100).uint8().rename("confidence")) out_water = ee.Image.cat([ confidence, water, ]) else: out_water = water fused_pred = fused_pred.multiply(10000).int16() if output_asset_path is not None: # create metadata dict metadata = ee.Dictionary({ "hf_version": hf.__version__, "system:time_start": target_date.millis(), "system:time_end": target_date.advance(86399, "seconds").millis(), "execution_time": time_str, "lag": lag, "look_back": look_back, }) geeutils.export_image( out_water.set(metadata.combine({"product": "water"})), region, output_asset_path + "_water", description=f"hydrafloods_water_ee_export_{time_id}", scale=output_scale, crs="EPSG:4326", ) if export_fusion: geeutils.export_image( fused_pred.set(metadata.combine({"product": "fusion"})), region, output_asset_path + "_fusion", description=f"hydrafloods_fusion_ee_export_{time_id}", scale=output_scale, crs="EPSG:4326", ) elif output_bucket_path is not None: export_region = region.bounds( maxError=100).getInfo()["coordinates"] bucket_path, ext = os.path.splitext(output_bucket_path) fcomponents = bucket_path.split("/") bucket = fcomponents[2] fpath = fcomponents[3:-1] # TODO: remove extension from string formulation f_water = "/".join(fpath + [fcomponents[-1] + "_water" + ext]) f_fusion = "/".join(fpath + [fcomponents[-1] + "_fusion" + ext]) water_task = ee.batch.Export.image.toCloudStorage( image=out_water, description=f"hydrafloods_water_gcp_export_{time_id}", bucket=bucket, fileNamePrefix=f_water, region=export_region, scale=output_scale, crs="EPSG:4326", maxPixels=1e13, fileFormat="GeoTIFF", formatOptions={"cloudOptimized": True}, ) water_task.start() if export_fusion: fusion_task = ee.batch.Export.image.toCloudStorage( image=fused_pred, description=f"hydrafloods_fusion_gcp_export_{time_id}", bucket=bucket, fileNamePrefix=f_fusion, region=export_region, scale=output_scale, crs="EPSG:4326", maxPixels=1e13, fileFormat="GeoTIFF", formatOptions={"cloudOptimized": True}, ) fusion_task.start() else: raise ValueError( "Either 'output_asset_path' or 'output_bucket_path' needs to be defined to run fusion export process" ) return
def merge_gcp_tiled_results( bucket_path, pattern, region, retries=-1, clean_up=False, cloud_project=None, file_dims=None, output_scale=30, ): """Helper function to merge tiled surface water estimates from `export_daily_surface_water` into on cloud optimized geotiff on Google Cloud Platform args: bucket_path (str): GCP cloud bucket path to read tiled cloud optimized geotiffs. Will also export merged file to this path. pattern (str): regex string to search for specific files. Useful for selecting files from a specific bucket subdirectory or date in filename region (ee.Geometry): geographic region to export merged data over. region must align with region from the tiled export retries (int, optional): number of retries to search for tiled data. Useful is runtime is unknown and the merge function is run at a set time each day. If less than or equal to zero, no retries will be used. default = -1 clean_up (bool, optional): boolean keyword to delete tile geotiffs after merge is complete. Only use if you are sure the merging is/will be successful. default = False cloud_project (str, optional): name of GCP cloud project name that the cloud storage bucket is part of. If nothing is provided, it will try to use default system GCP credentials. default = None file_dims (int | list[int], optional): the dimensions in pixels of each image file, if the image is too large to fit in a single file. May specify a single number to indicate a square shape, or a list of two dimensions to indicate (width,height). Note that the image will still be clipped to the overall image dimensions. Must be a multiple of shardSize (256). If none, then Earth Engine will automatically estimate dimensions. default = None output_scale (float, optional): output resolution of harmonic weight image. default = 30 """ land_area = (ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017").filterBounds( region).geometry(100).buffer(2500, maxError=100)) grid = geeutils.tile_region(region, intersect_geom=land_area, grid_size=1.0) expected_n = grid.size().getInfo() fcomponents = bucket_path.split("/") bucket = fcomponents[2] fpath = pattern.replace("*", "") files = utils.list_gcs_objs(bucket, pattern=pattern, project=cloud_project) if len(files) == expected_n: images = [ee.Image.loadGeoTIFF(file) for file in files] merged = ee.ImageCollection.fromImages(images).mosaic() export_region = region.bounds(maxError=100).getInfo()["coordinates"] # bucket_path, ext = os.path.splitext(output_bucket_path) now = datetime.datetime.now() time_id = now.strftime("%Y%m%d%H%M%s") task = ee.batch.Export.image.toCloudStorage( image=merged, description=f"hydrafloods_merge_gcp_export_{time_id}", bucket=bucket, fileNamePrefix=fpath, region=export_region, scale=output_scale, crs="EPSG:4326", fileDimensions=file_dims, maxPixels=1e13, fileFormat="GeoTIFF", formatOptions={"cloudOptimized": True}, ) task.start() if clean_up: gcsfs.GCSFileSystem.rm(files) elif retries > 0: time.sleep(60 * 10) merge_gcp_tiled_results(bucket_path, pattern, region, retries=(retries - 1)) else: raise RuntimeError( f"could not find all expected tiles to merge...found {len(files)} tiles but expected {expected_n}" ) return
def export_surface_water_harmonics( region, start_time, end_time, output_asset_path, n_cycles=2, feature_names=None, label=None, fusion_samples=None, tile=False, tile_size=1.0, output_scale=30, ): """Second step of the daily surface water fusion process. This procedure uses samples from `export_fusion_samples` to build a random forest model to predict a water index from SAR imagery. This a time series of optical-SAR fused data is used to calculate a harmonic model for long-term surface water trend and is exported to an Earth Engine asset. args: region (ee.Geometry): geographic region to look for coincident data and sample from start_time (str | datetime.datetime): start time used to look for coincident data end_time (str | datetime.datetime): end time used to look for coincident data output_asset_path (str): Earth Engine asset id to save harmonic model weights to as image. If tile==True, then output_asset_path much be a precreated ImageCollection asset n_cycles (int, optional): number of interannual cycles to model. default = 2 feature_names (list[str],): names of feature columns used to calculate `label` from label (str): name of feature column to predict using `feature_names` fusion_samples (str): Earth Engine FeatureCollection asset id of samples to get a data fusion model from. Should be the asset output from `export_fusion_samples` tile (bool, optional): boolean keyword to tile exports. If false will try to calculate harmonic weights as image. If true, it will tile area and recusively call to export smaller areas. If true then expects that `output_asset_path` is an ImageCollection. default = False tile_size (float, optional): resolution in decimal degrees to create tiles over region for smaller exports. Only used if tile==True. default = 1.0 output_scale (float, optional): output resolution of harmonic weight image. default = 30 """ if tile: land_area = (ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017"). filterBounds(region).geometry(1000).buffer(2500, maxError=1000)) grid = geeutils.tile_region(region, intersect_geom=land_area, grid_size=tile_size) n = grid.size().getInfo() grid_list = grid.toList(n) for i in range(n): grid_tile = ee.Feature(grid_list.get(i)).geometry() if output_asset_path is not None: output_tile_path = output_asset_path + f"harmonics_t{i:05d}" export_surface_water_harmonics( region=grid_tile, start_time=start_time, end_time=end_time, n_cycles=n_cycles, feature_names=feature_names, label=label, fusion_samples=fusion_samples, output_asset_path=output_tile_path, tile=False, tile_size=tile_size, output_scale=output_scale, ) else: if fusion_samples is not None: fusion_model, scaling_dict = ml.random_forest_ee( 25, fusion_samples.limit(10000), feature_names, label, scaling="standard", mode="regression", ) else: raise ValueError( "Either 'fusion_samples' or 'fusion_model_path' needs to be defined to run fusion process" ) ds = _fuse_dataset( region, start_time, end_time, fusion_model, scaling_dict, feature_names, target_band=label, ) now = datetime.datetime.now() time_id = now.strftime("%Y%m%d%H%M%s") time_str = now.strftime("%Y-%m-%d %H:%M:%s") scale_factor = 0.0001 # create metadata dict metadata = ee.Dictionary({ "hf_version": hf.__version__, "scale_factor": scale_factor, "fit_time_start": start_time, "fit_time_end": end_time, "execution_time": time_str, }) harmonic_coefs = timeseries.fit_harmonic_trend(ds, dependent=label, n_cycles=n_cycles, output_err=True) harmonic_coefs = harmonic_coefs.divide(scale_factor).int32().set( metadata) if output_asset_path is not None: geeutils.export_image( harmonic_coefs, region, output_asset_path, description= f"hydrafloods_harmonic_coefficient_export_{time_id}", scale=output_scale, crs="EPSG:4326", ) else: raise ValueError( "'output_asset_path' needs to be defined to run fusion export process" ) return
def bmax_otsu( img, band=None, region=None, scale=90, initial_threshold=0, invert=False, grid_size=0.1, bmax_threshold=0.75, max_boxes=100, seed=7, max_buckets=255, min_bucket_width=0.001, max_raw=1e6, return_threshold=False, ): """Implementation of the B-Max Otsu thresholding algorithm. Detailed explanation of algorithm can be found at https://doi.org/10.3390/rs12152469 args: img (ee.Image): input image to thresholding algorithm band (str | None,optional): band name to use for thresholding, if set to `None` will use first band in image. default = None region (ee.Geometry | None, optional): region to determine threshold, if set to `None` will use img.geometry(). default = None scale (int, optional): scale at which to perform reduction operations, setting higher will prevent OOM errors. default = 90 initial_threshold (float, optional): initial estimate of water/no-water for estimating the probabilities of classes in segment. default = 0 invert (bool, optional): boolean switch to determine if to threshold greater than (True) or less than (False). default = False grid_size (float, optional): size in decimal degrees to tile image/region to check for bimodality. default = 0.1 bmax_threshold (float, optional): value 0-1 to determine if a value of bmax is bimodal or not. default = 0.75 max_boxes (int, optional): maximum number of tiles/boxes to use when determining threshold. default = 100 seed (int, optional): random number generator seed for randomly selected max_boxes. default = 7 max_buckets (int, optional): The maximum number of buckets to use when building a histogram; will be rounded up to a power of 2. default = 255 min_bucket_width (float, optional): The minimum histogram bucket width to allow any power of 2. default = 0.001 max_raw (int, optional): The number of values to accumulate before building the initial histogram. default = 1e6 return_threshold (bool, optional): boolean switch, if set to true then function will return threshold number, else thresholded image. default = False returns: ee.Image: thresholded image based (if return_threshold==False) or threshold value (if return_threshold==True) based on the threshold determined by the algorithm """ def calcBmax(feature): """Closure function to calculate Bmax for each feature covering image """ segment = img initial = segment.lt(initial_threshold) p1 = ee.Number( initial.reduceRegion( reducer=ee.Reducer.mean(), geometry=feature.geometry(), bestEffort=True, scale=scale, ).get(histBand) ) p1 = ee.Number(ee.Algorithms.If(p1, p1, 0.99)) p2 = ee.Number(1).subtract(p1) m = ( segment.updateMask(initial) .rename("m1") .addBands(segment.updateMask(initial.Not()).rename("m2")) ) mReduced = m.reduceRegion( reducer=ee.Reducer.mean(), geometry=feature.geometry(), bestEffort=True, scale=scale, ) m1 = ee.Number(mReduced.get("m1")) m2 = ee.Number(mReduced.get("m2")) m1 = ee.Number(ee.Algorithms.If(m1, m1, -25)) m2 = ee.Number(ee.Algorithms.If(m2, m2, 0)) sigmab = p1.multiply(p2.multiply(m1.subtract(m2).pow(2))) sigmat = ee.Number( segment.reduceRegion( reducer=ee.Reducer.variance(), geometry=feature.geometry(), bestEffort=True, scale=scale, ).get(histBand) ) sigmat = ee.Number(ee.Algorithms.If(sigmat, sigmat, 2)) bmax = sigmab.divide(sigmat) return feature.set({"bmax": bmax}) if band is None: img = img.select([0]) histBand = ee.String(img.bandNames().get(0)) else: histBand = ee.String(band) img = img.select(histBand) if region is None: region = img.geometry() grid = geeutils.tile_region(region, intersect_geom=region, grid_size=0.1) bmaxes = ( grid.map(calcBmax) .filter(ee.Filter.gt("bmax", bmax_threshold)) .randomColumn("random", seed) ) nBoxes = ee.Number(bmaxes.size()) randomThresh = ee.Number(max_boxes).divide(nBoxes) selection = bmaxes.filter(ee.Filter.lt("random", randomThresh)) histogram = img.reduceRegion( ee.Reducer.histogram(max_buckets, min_bucket_width, max_raw) .combine("mean", None, True) .combine("variance", None, True), selection, scale, bestEffort=True, tileScale=16, ) threshold = otsu(histogram.get(histBand.cat("_histogram"))) if return_threshold is True: return ee.Image(threshold) else: water = ee.Image(ee.Algorithms.If(invert, img.gt(threshold), img.lt(threshold))) return water.rename("water").uint8()
def export_fusion_product( region, target_date, harmonic_image=None, harmonic_collection=None, feature_names=None, label=None, look_back=30, lag=4, n_cycles=2, fusion_samples=None, output_asset_path=None, output_bucket_path=None, tile=False, tile_size=1.0, tile_buffer=100000, output_scale=30, ): def get_residuals(i): """Closure function to calculate residuals of harmonic water estimate compared to observed data. """ i = ee.Number(i) t_diff = ( ee.Number(i).multiply(-1).subtract(lag) ) # calc how many days to adjust ini date new_date = target_date.advance(t_diff, "day") # calculate new date corr_img = ( ds.collection.select(label) .filterDate(new_date, new_date.advance(1, "day")) .median() ) # .select(f"^{label}.*",label) time_img = timeseries.get_dummy_img(new_date) harmon_pred = ( timeseries.add_harmonic_coefs(time_img) .multiply(harmonic_coefs) .reduce("sum") ) harmon_diff = harmon_pred.subtract(corr_img).rename("residual") return harmon_diff.set("system:time_start", new_date.millis()) if tile: if tile: land_area = ( ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017") .filterBounds(region) .geometry(100) .buffer(2500, maxError=100) ) grid = geeutils.tile_region( region, intersect_geom=land_area, grid_size=tile_size ) n = grid.size().getInfo() grid_list = grid.toList(n) for i in range(n): if output_asset_path is not None: output_asset_tile = output_asset_path + f"daily_tile{i:05d}" else: output_asset_tile = None if output_bucket_path is not None: output_bucket_tile = output_bucket_path + f"_tile{i:05d}" else: output_bucket_tile = None grid_tile = ee.Feature(grid_list.get(i)).geometry() export_fusion_product( region=grid_tile, target_date=target_date, harmonic_image=harmonic_image, harmonic_collection=harmonic_collection, feature_names=feature_names, label=label, look_back=look_back, lag=lag, n_cycles=n_cycles, fusion_samples=fusion_samples, output_asset_path=output_asset_path, output_bucket_path=output_bucket_path, tile=tile, tile_size=tile_size, tile_buffer=tile_buffer, output_scale=output_scale, ) else: if not isinstance(target_date, ee.Date): target_date = ee.Date(target_date) end_time = target_date.advance(-(lag - 1), "day") start_time = end_time.advance(-look_back, "day") if fusion_samples is not None: fusion_model, scaling_dict = ml.random_forest_ee( 30, fusion_samples, feature_names, label, scaling=None, mode="regression", ) else: raise ValueError( "'fusion_samples' needs to be defined to run fusion process" ) now = datetime.datetime.now() time_id = now.strftime("%Y%m%d%H%M%s") time_str = now.strftime("%Y-%m-%d %H:%M:%s") if harmonic_image is not None: harmonic_coefs = ee.Image(harmonic_image) harmonic_coefs = harmonic_coefs.multiply( ee.Image(ee.Number(harmonic_coefs.get("scale_factor"))) ) elif harmonic_collection is not None: harmonic_collection = ee.ImageCollection(harmonic_collection) first = ee.Image(harmonic_collection.first()) harmonic_coefs = harmonic_collection.mosaic().multiply( ee.Image(ee.Number(first.get("scale_factor"))) ) else: raise ValueError( "Either 'harmonic_image' or 'harmonic_collection' needs to be defined to run fusion process" ) harmonic_coefs = harmonic_coefs.select("^(c|t|s).*") prod_region = region.buffer(tile_buffer, 100) ds = _fuse_dataset( region, start_time, end_time, fusion_model, scaling_dict, feature_names, target_band=label, use_viirs=True, ) dummy_target = timeseries.get_dummy_img(target_date) weights = ee.ImageCollection.fromImages( ee.List.sequence(0, look_back - 1).map(get_residuals) ).sort("system:time_start") weights_lr = timeseries.fit_linear_trend( weights, dependent="residual", output_err=False ) weights_coefs = weights_lr.select("^(c|t).*") lin_pred = ( dummy_target.multiply(weights_coefs).reduce("sum").rename("residual_est") ) har_pred = ( timeseries.add_harmonic_coefs(dummy_target, n_cycles=n_cycles) .multiply(harmonic_coefs) .reduce("sum") ) fused_pred = (har_pred.subtract(lin_pred)).rename("fused_product") fused_pred = fused_pred.multiply(10000).int16() if output_asset_path is not None: # create metadata dict metadata = ee.Dictionary( { "hf_version": hf.__version__, "system:time_start": target_date.millis(), "system:time_end": target_date.advance(86399, "seconds").millis(), "execution_time": time_str, "lag": lag, "look_back": look_back, "scale_factor": 0.0001, } ) geeutils.export_image( fused_pred.set(metadata.combine({"product": "fusion"})), region, output_asset_path + "_fusion", description=f"hydrafloods_fusion_ee_export_{time_id}", scale=output_scale, crs="EPSG:4326", ) elif output_bucket_path is not None: export_region = region.bounds(maxError=100).getInfo()["coordinates"] bucket_path, ext = os.path.splitext(output_bucket_path) fcomponents = bucket_path.split("/") bucket = fcomponents[2] fpath = fcomponents[3:-1] # TODO: remove extension from string formulation f_water = "/".join(fpath + [fcomponents[-1] + "_water" + ext]) f_fusion = "/".join(fpath + [fcomponents[-1] + "_fusion" + ext]) fusion_task = ee.batch.Export.image.toCloudStorage( image=fused_pred, description=f"hydrafloods_fusion_gcp_export_{time_id}", bucket=bucket, fileNamePrefix=f_fusion, region=export_region, scale=output_scale, crs="EPSG:4326", maxPixels=1e13, fileFormat="GeoTIFF", formatOptions={"cloudOptimized": True}, ) fusion_task.start() else: raise ValueError( "Either 'output_asset_path' or 'output_bucket_path' needs to be defined to run fusion export process" ) return
def fuzzy_otsu( img, band=None, region=None, scale=90, initial_threshold=-15.5, grid_size=0.1, max_boxes=20, seed=0, max_buckets=255, min_bucket_width=0.001, max_raw=1e6, ): """Implementation of Otsu thresholding algorithm. Segment the grids into water and land representation using initial threshold and randomly select features to calculate minimum and maximum threshold of the image. Calculate Midpoint of min and max threshold using Fuzzy_Gaussian to map water. args: img (ee.Image): input image to thresholding algorithm band (str | None,optional): band name to use for thresholding, if set to `None` will use first band in image. default = None region (ee.Geometry | None, optional): region to determine threshold, if set to `None` will use img.geometry(). default = None scale (int, optional): scale at which to perform reduction operations, setting higher will prevent OOM errors. default = 90 initial_threshold (float, optional): initial estimate of water/no-water for estimating the probabilities of classes in segment. default = -15.5 grid_size (float, optional): size in decimal degrees to tile image/region to check for bimodality. default = 0.1 max_boxes (int, optional): maximum number of tiles/boxes to use when determining threshold. default = 20 seed (int, optional): random number generator seed for randomly selected max_boxes. default = 7 max_buckets (int, optional): The maximum number of buckets to use when building a histogram; will be rounded up to a power of 2. default = 255 min_bucket_width (float, optional): The minimum histogram bucket width to allow any power of 2. default = 0.001 max_raw (int, optional): The number of values to accumulate before building the initial histogram. default = 1e6 returns: ee.Image: thresholded water image. """ # Function to Segment Land and Water Grids def segment_grid(feature): counts = initImg.reduceRegion( reducer=ee.Reducer.sum(), geometry=feature.geometry(), bestEffort=True, scale=scale, ) return feature.set(counts) if region is None: region = img.geometry() if band is None: img = img.select([0]) histBand = ee.String(img.bandNames().get(0)) else: histBand = ee.String(band) img = img.select(histBand) grid = geeutils.tile_region(region, intersect_geom=region, grid_size=grid_size) # Prepare good representation of water and land initWater = img.lt(initial_threshold).rename("water") initImg = initWater.addBands(initWater.Not().rename("land")) grid = grid.map(segment_grid) # Select water and land grid selection = grid.filter( ee.Filter.And(ee.Filter.gt("water", 1000), ee.Filter.gt("land", 1000))) # Randomly select features selection = selection.randomColumn("random") nBoxes = ee.Number(selection.size()) randomSelection = ee.Number(max_boxes).divide(nBoxes) selection = selection.filter(ee.Filter.lt("random", randomSelection)) # Create histogram for selected features to calculate min threshold histogram = img.reduceRegion( ee.Reducer.histogram(max_buckets, min_bucket_width, max_raw).combine( "mean", None, True).combine("variance", None, True), selection.geometry(), scale, bestEffort=True, tileScale=16, ) min_threshold = otsu(histogram.get(histBand.cat("_histogram"))) # Create histogram to calculate max threshold histogram2 = img.updateMask(img.lt(min_threshold)).reduceRegion( ee.Reducer.histogram(max_buckets, min_bucket_width, max_raw).combine( "mean", None, True).combine("variance", None, True), selection.geometry(), scale, bestEffort=True, tileScale=16, ) max_threshold = otsu(histogram2.get(histBand.cat("_histogram"))) # Calculate Midpoint of min and max threshold using Fuzzy_Gaussian midpoint = ee.Number( ee.Number(min_threshold).add(ee.Number(max_threshold))).divide(2) spread = 0.2 gauss = fuzzy.fuzzy_gaussian(img, midpoint, spread).clip(region) waterImg = img.lt(midpoint) waterImg = waterImg.where(waterImg.eq(0), gauss) return ee.Image(waterImg)
def bmax_otsu( img, band=None, region=None, reduction_scale=90, initial_threshold=0, invert=False, grid_size=0.1, bmax_threshold=0.75, max_boxes=100, seed=7, max_buckets=255, min_bucket_width=0.001, max_raw=1e6, return_threshold=False ): def calcBmax(feature): segment = img # .clip(feature) initial = segment.lt(initial_threshold) p1 = ee.Number( initial.reduceRegion( reducer=ee.Reducer.mean(), geometry=feature.geometry(), bestEffort=True, scale=reduction_scale, ).get(histBand) ) p1 = ee.Number(ee.Algorithms.If(p1, p1, 0.99)) p2 = ee.Number(1).subtract(p1) m = ( segment.updateMask(initial) .rename("m1") .addBands(segment.updateMask(initial.Not()).rename("m2")) ) mReduced = m.reduceRegion( reducer=ee.Reducer.mean(), geometry=feature.geometry(), bestEffort=True, scale=reduction_scale, ) m1 = ee.Number(mReduced.get("m1")) m2 = ee.Number(mReduced.get("m2")) m1 = ee.Number(ee.Algorithms.If(m1, m1, -25)) m2 = ee.Number(ee.Algorithms.If(m2, m2, 0)) sigmab = p1.multiply(p2.multiply(m1.subtract(m2).pow(2))) sigmat = ee.Number( segment.reduceRegion( reducer=ee.Reducer.variance(), geometry=feature.geometry(), bestEffort=True, scale=reduction_scale, ).get(histBand) ) sigmat = ee.Number(ee.Algorithms.If(sigmat, sigmat, 2)) bmax = sigmab.divide(sigmat) return feature.set({"bmax": bmax}) if band is None: img = img.select([0]) histBand = ee.String(img.bandNames().get(0)) else: histBand = ee.String(band) img = img.select(histBand) if region is None: region = img.geometry() grid = geeutils.tile_region(region,intersect_geom=region,grid_size=0.1) bmaxes = ( grid.map(calcBmax) .filter(ee.Filter.gt("bmax", bmax_threshold)) .randomColumn("random", seed) ) nBoxes = ee.Number(bmaxes.size()) randomThresh = ee.Number(max_boxes).divide(nBoxes) selection = bmaxes.filter(ee.Filter.lt("random", randomThresh)) histogram = img.reduceRegion( ee.Reducer.histogram(max_buckets, min_bucket_width, max_raw) .combine("mean", None, True) .combine("variance", None, True), selection, reduction_scale, bestEffort=True, tileScale=16, ) threshold = otsu(histogram.get(histBand.cat("_histogram"))) if return_threshold is True: return ee.Image(threshold) else: water = ee.Image(ee.Algorithms.If(invert, img.gt(threshold), img.lt(threshold))) return water.rename("water").uint8()