def convert_float_to_int( stats: Optional[Dict[str, Any]], source_asset_co: RasterTileSetSourceCreationOptions, ) -> Tuple[RasterTileSetSourceCreationOptions, str]: stats = generate_stats(stats) logger.info("In convert_float_to_int()") assert len(stats.bands) == 1 stats_min = stats.bands[0].min stats_max = stats.bands[0].max value_range = math.fabs(stats_max - stats_min) logger.info( f"stats_min: {stats_min} stats_max: {stats_max} value_range: {value_range}" ) # Shift by 1 (and add 1 later) so any values of zero don't get counted as no_data uint16_max = np.iinfo(np.uint16).max - 1 # Expand or squeeze to fit into a uint16 mult_factor = (uint16_max / value_range) if value_range else 1 logger.info(f"Multiplicative factor: {mult_factor}") if isinstance(source_asset_co.no_data, list): raise RuntimeError("Cannot apply colormap on multi band image") elif source_asset_co.no_data is None: old_no_data: str = "None" elif source_asset_co.no_data == str(np.nan): old_no_data = "np.nan" else: old_no_data = str(source_asset_co.no_data) calc_str = (f"(A != {old_no_data}).astype(bool) * " f"(1 + (A - {stats_min}) * {mult_factor}).astype(np.uint16)") logger.info(f"Resulting calc string: {calc_str}") source_asset_co.data_type = DataType.uint16 source_asset_co.no_data = 0 if source_asset_co.symbology and source_asset_co.symbology.colormap is not None: source_asset_co.symbology.colormap = { (1 + (float(k) - stats_min) * mult_factor): v for k, v in source_asset_co.symbology.colormap.items() } logger.info( f"Resulting colormap: {source_asset_co.symbology.colormap}") return source_asset_co, calc_str
async def no_symbology( dataset: str, version: str, pixel_meaning: str, source_asset_co: RasterTileSetSourceCreationOptions, zoom_level: int, max_zoom: int, jobs_dict: Dict, ) -> Tuple[List[Job], str]: """Skip symbology step.""" if source_asset_co.source_uri: wm_source_uri: str = tile_uri_to_tiles_geojson( get_asset_uri( dataset, version, AssetType.raster_tile_set, source_asset_co.copy(deep=True, update={ "grid": f"zoom_{zoom_level}" }).dict(by_alias=True), "epsg:3857", )) return list(), wm_source_uri else: raise RuntimeError("No source URI set.")
async def test_colormap_symbology_with_intensity( mock_merge_assets, mock_create_intensity_asset, mock_create_colormapped_asset, ): intensity_symbology = { "type": "discrete_intensity", "colormap": { "1": { "red": 102, "green": 134, "blue": 54 } }, } with patch.dict(wm_tile_set_co, {"symbology": intensity_symbology}, clear=False): _ = await colormap_symbology( "umd_regional_primary_forest_2001", "v201901.2", "pixel_meaning", RasterTileSetSourceCreationOptions(**wm_tile_set_co), 12, 12, {12: { "source_reprojection_job": "some_job" }}, ) assert mock_merge_assets.called is True assert mock_create_intensity_asset.called is True assert mock_create_colormapped_asset.called is True
async def create_wm_tile_set_job( dataset: str, version: str, creation_options: RasterTileSetSourceCreationOptions, job_name: str, parents: Optional[List[Job]] = None, use_resampler: bool = False, ) -> Tuple[Job, str]: asset_uri = get_asset_uri( dataset, version, AssetType.raster_tile_set, creation_options.dict(by_alias=True), "epsg:3857", ) # Create an asset record asset_options = AssetCreateIn( asset_type=AssetType.raster_tile_set, asset_uri=asset_uri, is_managed=True, creation_options=creation_options, metadata=RasterTileSetMetadata(), ).dict(by_alias=True) wm_asset_record = await create_asset(dataset, version, **asset_options) logger.debug(f"Created asset for {asset_uri}") # TODO: Consider removing the use_resampler argument and changing this # to "if creation_options.calc is None:" # Make sure to test different scenarios when done! if use_resampler: job = await create_resample_job( dataset, version, creation_options, int(creation_options.grid.strip("zoom_")), job_name, callback_constructor(wm_asset_record.asset_id), parents=parents, ) else: job = await create_pixetl_job( dataset, version, creation_options, job_name, callback_constructor(wm_asset_record.asset_id), parents=parents, ) zoom_level = int(creation_options.grid.strip("zoom_")) job = scale_batch_job(job, zoom_level) return job, asset_uri
async def reproject_to_web_mercator( dataset: str, version: str, source_creation_options: RasterTileSetSourceCreationOptions, zoom_level: int, max_zoom: int, parents: Optional[List[Job]] = None, max_zoom_resampling: Optional[str] = None, max_zoom_calc: Optional[str] = None, use_resampler: bool = False, ) -> Tuple[Job, str]: """Create Tileset reprojected into Web Mercator projection.""" calc = (max_zoom_calc if zoom_level == max_zoom and max_zoom_calc else source_creation_options.calc) resampling = (max_zoom_resampling if zoom_level == max_zoom and max_zoom_resampling else source_creation_options.resampling) source_uri: Optional[List[str]] = get_zoom_source_uri( dataset, version, source_creation_options, zoom_level, max_zoom) # We create RGBA image in a second step, since we cannot easily resample RGBA to next zoom level using PixETL. symbology = None creation_options = source_creation_options.copy( deep=True, update={ "calc": calc, "resampling": resampling, "grid": f"zoom_{zoom_level}", "source_uri": source_uri, "symbology": symbology, }, ) job_name = sanitize_batch_job_name( f"{dataset}_{version}_{source_creation_options.pixel_meaning}_{zoom_level}" ) return await create_wm_tile_set_job( dataset, version, creation_options, job_name, parents, use_resampler=use_resampler, )
async def _create_intensity_asset( dataset: str, version: str, pixel_meaning: str, source_asset_co: RasterTileSetSourceCreationOptions, zoom_level: int, max_zoom: int, jobs_dict: Dict, max_zoom_calc: str, resampling: ResamplingMethod, ) -> Tuple[List[Job], str]: """Create intensity Raster Tile Set asset based on source asset. Create Intensity value layer(s) using provided calc function, resample intensity based on provided resampling method. """ source_uri: Optional[List[str]] = get_zoom_source_uri( dataset, version, source_asset_co, zoom_level, max_zoom) intensity_source_co: RasterTileSetSourceCreationOptions = source_asset_co.copy( deep=True, update={ "source_uri": source_uri, "no_data": None, "pixel_meaning": f"intensity_{pixel_meaning}", "resampling": resampling, }, ) if zoom_level == max_zoom: parent_jobs: Optional[List[Job]] = None else: parent_jobs = [jobs_dict[zoom_level + 1]["intensity_reprojection_job"]] intensity_job, intensity_uri = await reproject_to_web_mercator( dataset, version, intensity_source_co, zoom_level, max_zoom, parent_jobs, max_zoom_resampling=PIXETL_DEFAULT_RESAMPLING, max_zoom_calc=max_zoom_calc, ) jobs_dict[zoom_level]["intensity_reprojection_job"] = intensity_job return [intensity_job], intensity_uri
async def test_colormap_symbology_no_intensity( mock_merge_assets, mock_create_intensity_asset, mock_create_colormapped_asset, ): _ = await colormap_symbology( "umd_regional_primary_forest_2001", "v201901.2", "pixel_meaning", RasterTileSetSourceCreationOptions(**wm_tile_set_co), 12, 12, {12: { "source_reprojection_job": "some_job" }}, ) assert mock_merge_assets.called is False assert mock_create_intensity_asset.called is False assert mock_create_colormapped_asset.called is True
async def raster_tile_cache_asset( dataset: str, version: str, asset_id: UUID, input_data: Dict[str, Any], ) -> ChangeLog: """Generate Raster Tile Cache Assets.""" # TODO: Refactor to be easier to test min_zoom = input_data["creation_options"]["min_zoom"] max_zoom = input_data["creation_options"]["max_zoom"] max_static_zoom = input_data["creation_options"]["max_static_zoom"] implementation = input_data["creation_options"]["implementation"] symbology = input_data["creation_options"]["symbology"] resampling = input_data["creation_options"]["resampling"] # source_asset_id is currently required. Could perhaps make it optional # in the case that the default asset is the only one. source_asset: ORMAsset = await get_asset( input_data["creation_options"]["source_asset_id"] ) # Get the creation options from the original raster tile set asset and # overwrite settings. Make sure source_type and source_driver are set in # case it is an auxiliary asset new_source_uri = [ tile_uri_to_tiles_geojson( get_asset_uri( dataset, version, AssetType.raster_tile_set, source_asset.creation_options, ) ).replace("/geotiff", "/gdal-geotiff") ] # The first thing we do for each zoom level is reproject the source asset # to web-mercator. We don't want the calc string (if any) used to # create the source asset to be applied again to the already transformed # data, so set it to None. source_asset_co = RasterTileSetSourceCreationOptions( # TODO: With python 3.9, we can use the `|` operator here # waiting for https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker/pull/67 **{ **source_asset.creation_options, **{ "source_type": RasterSourceType.raster, "source_driver": RasterDrivers.geotiff, "source_uri": new_source_uri, "calc": None, "resampling": resampling, "compute_stats": False, "compute_histogram": False, "symbology": Symbology(**symbology), "subset": None, }, } ) # If float data type, convert to int in derivative assets for performance # FIXME: Make this work for multi-band inputs max_zoom_calc = None if source_asset_co.data_type == DataType.boolean: pass # So the next line doesn't break elif np.issubdtype(np.dtype(source_asset_co.data_type), np.floating): logger.info("Source datatype is float subtype, converting to int") source_asset_co, max_zoom_calc = convert_float_to_int( source_asset.stats, source_asset_co ) assert source_asset_co.symbology is not None symbology_function = symbology_constructor[source_asset_co.symbology.type].function # We want to make sure that the final RGB asset is named after the # implementation of the tile cache and that the source_asset name is not # already used by another intermediate asset. # TODO: Actually make sure the intermediate assets aren't going to # overwrite any existing assets if symbology_function == no_symbology: source_asset_co.pixel_meaning = implementation else: source_asset_co.pixel_meaning = ( f"{source_asset_co.pixel_meaning}_{implementation}" ) job_list: List[Job] = [] jobs_dict: Dict[int, Dict[str, Job]] = dict() for zoom_level in range(max_zoom, min_zoom - 1, -1): jobs_dict[zoom_level] = dict() if zoom_level == max_zoom: source_reprojection_parent_jobs: List[Job] = [] else: source_reprojection_parent_jobs = [ jobs_dict[zoom_level + 1]["source_reprojection_job"] ] ( source_reprojection_job, source_reprojection_uri, ) = await reproject_to_web_mercator( dataset, version, source_asset_co, zoom_level, max_zoom, source_reprojection_parent_jobs, max_zoom_resampling=PIXETL_DEFAULT_RESAMPLING, max_zoom_calc=max_zoom_calc, use_resampler=max_zoom_calc is None, ) jobs_dict[zoom_level]["source_reprojection_job"] = source_reprojection_job job_list.append(source_reprojection_job) symbology_jobs: List[Job] symbology_uri: str symbology_co = source_asset_co.copy(deep=True) symbology_jobs, symbology_uri = await symbology_function( dataset, version, implementation, symbology_co, zoom_level, max_zoom, jobs_dict, ) job_list += symbology_jobs bit_depth: int = symbology_constructor[source_asset_co.symbology.type].bit_depth if zoom_level <= max_static_zoom: tile_cache_job: Job = await create_tile_cache( dataset, version, symbology_uri, zoom_level, implementation, callback_constructor(asset_id), [*symbology_jobs, source_reprojection_job], bit_depth, ) job_list.append(tile_cache_job) log: ChangeLog = await execute(job_list) return log
async def _merge_assets( dataset: str, version: str, pixel_meaning: str, asset1_uri: str, asset2_uri: str, zoom_level: int, parents: List[Job], calc_str: str = "np.ma.array([A, B, C, D])", band_count: int = 4, ) -> Tuple[List[Job], str]: """Create RGBA-encoded raster tile set from two source assets, potentially using a custom merge function (the default works for 3+1 band sources, such as RGB + Intensity as Alpha)""" encoded_co = RasterTileSetSourceCreationOptions( pixel_meaning=pixel_meaning, data_type=DataType.uint8, # FIXME: Revisit for 16-bit assets band_count=band_count, no_data=None, resampling=ResamplingMethod.nearest, grid=Grid(f"zoom_{zoom_level}"), compute_stats=False, compute_histogram=False, source_type=RasterSourceType.raster, source_driver=RasterDrivers.geotiff, source_uri=[asset1_uri, asset2_uri], calc=calc_str, photometric=PhotometricType.rgb, ) asset_uri = get_asset_uri( dataset, version, AssetType.raster_tile_set, encoded_co.dict(by_alias=True), "epsg:3857", ) logger.debug( f"ATTEMPTING TO CREATE MERGED ASSET WITH THESE CREATION OPTIONS: {encoded_co}" ) # Create an asset record asset_options = AssetCreateIn( asset_type=AssetType.raster_tile_set, asset_uri=asset_uri, is_managed=True, creation_options=encoded_co, metadata=RasterTileSetMetadata(), ).dict(by_alias=True) asset = await create_asset(dataset, version, **asset_options) logger.debug( f"ZOOM LEVEL {zoom_level} MERGED ASSET CREATED WITH ASSET_ID {asset.asset_id}" ) callback = callback_constructor(asset.asset_id) pixetl_job = await create_pixetl_job( dataset, version, encoded_co, job_name=f"merge_assets_zoom_{zoom_level}", callback=callback, parents=parents, ) pixetl_job = scale_batch_job(pixetl_job, zoom_level) return ( [pixetl_job], tile_uri_to_tiles_geojson(asset_uri), )
async def _create_colormapped_asset( dataset: str, version: str, pixel_meaning: str, source_asset_co: RasterTileSetSourceCreationOptions, zoom_level: int, jobs_dict: Dict, ) -> Tuple[List[Job], str]: wm_source_co = source_asset_co.copy(deep=True, update={"grid": f"zoom_{zoom_level}"}) wm_source_uri: str = tile_uri_to_tiles_geojson( get_asset_uri( dataset, version, AssetType.raster_tile_set, wm_source_co.dict(by_alias=True), "epsg:3857", )) colormap_co = wm_source_co.copy( deep=True, update={ "source_uri": [wm_source_uri], "calc": None, "resampling": PIXETL_DEFAULT_RESAMPLING, "pixel_meaning": pixel_meaning, }, ) colormap_asset_uri = get_asset_uri( dataset, version, AssetType.raster_tile_set, colormap_co.dict(by_alias=True), "epsg:3857", ) # Create an asset record colormap_asset_model = AssetCreateIn( asset_type=AssetType.raster_tile_set, asset_uri=colormap_asset_uri, is_managed=True, creation_options=colormap_co, ).dict(by_alias=True) colormap_asset_record = await create_asset(dataset, version, **colormap_asset_model) logger.debug(f"Created asset record for {colormap_asset_uri} " f"with creation options: {colormap_co}") parents = [jobs_dict[zoom_level]["source_reprojection_job"]] job_name = sanitize_batch_job_name( f"{dataset}_{version}_{pixel_meaning}_{zoom_level}") # Apply the colormap gdaldem_job = await create_gdaldem_job( dataset, version, colormap_co, job_name, callback_constructor(colormap_asset_record.asset_id), parents=parents, ) gdaldem_job = scale_batch_job(gdaldem_job, zoom_level) return [gdaldem_job], colormap_asset_uri
async def year_intensity_symbology( dataset: str, version: str, pixel_meaning: str, source_asset_co: RasterTileSetSourceCreationOptions, zoom_level: int, max_zoom: int, jobs_dict: Dict, ) -> Tuple[List[Job], str]: """Create Raster Tile Set asset which combines year raster and intensity raster into one. At native resolution (max_zoom) it will create intensity raster based on given source. For lower zoom levels it will resample higher zoom level tiles using average resampling method. Once intensity raster tile set is created it will combine it with source (year) raster into an RGB-encoded raster. This symbology is used for the Tree Cover Loss dataset. """ intensity_calc_string = "(A > 0) * 255" intensity_jobs, intensity_uri = await _create_intensity_asset( dataset, version, pixel_meaning, source_asset_co, zoom_level, max_zoom, jobs_dict, intensity_calc_string, ResamplingMethod.average, ) # The resulting raster channels are as follows: # 1. Intensity # 2. All zeros # 3. Year # 4. Alpha (which is set to 255 everywhere intensity is >0) merge_calc_string = "np.ma.array([B, np.ma.zeros(A.shape, dtype='uint8'), A, (B > 0) * 255], fill_value=0).astype('uint8')" wm_source_uri: str = get_asset_uri( dataset, version, AssetType.raster_tile_set, source_asset_co.copy(deep=True, update={ "grid": f"zoom_{zoom_level}" }).dict(by_alias=True), "epsg:3857", ) # We also need to depend on the original source reprojection job source_job = jobs_dict[zoom_level]["source_reprojection_job"] merge_jobs, final_asset_uri = await _merge_assets( dataset, version, pixel_meaning, tile_uri_to_tiles_geojson(wm_source_uri), tile_uri_to_tiles_geojson(intensity_uri), zoom_level, [*intensity_jobs, source_job], merge_calc_string, 4, ) return [*intensity_jobs, *merge_jobs], final_asset_uri
async def date_conf_intensity_multi_8_symbology( dataset: str, version: str, pixel_meaning: str, source_asset_co: RasterTileSetSourceCreationOptions, zoom_level: int, max_zoom: int, jobs_dict: Dict, ) -> Tuple[List[Job], str]: """Create a Raster Tile Set asset which combines the earliest detected alerts of three date_conf bands/alert systems (new encoding) with a new derived intensity asset, and the confidences of each of the original alerts. At native resolution (max_zoom) it an "intensity" asset which contains the value 55 everywhere there is data in any of the source bands. For lower zoom levels it resamples the previous zoom level intensity asset using the bilinear resampling method, causing isolated pixels to "fade". Finally the merge function takes the alert with the minimum date of the three bands and encodes its date, confidence, and the intensities into three 8-bit bands according to the formula the front end expects, and also adds a fourth band which encodes the confidences of all three original alert systems. """ # What we want is a value of 55 (max intensity for this scenario) # anywhere there is an alert in any system. intensity_max_calc_string = ( f"np.ma.array((A.data > 0) * {MAX_8_BIT_INTENSITY}, mask=False)") intensity_co = source_asset_co.copy( deep=True, update={ "calc": None, "band_count": 1, "data_type": DataType.uint8, }, ) intensity_jobs, intensity_uri = await _create_intensity_asset( dataset, version, pixel_meaning, intensity_co, zoom_level, max_zoom, jobs_dict, intensity_max_calc_string, ResamplingMethod.bilinear, ) wm_date_conf_uri: str = get_asset_uri( dataset, version, AssetType.raster_tile_set, source_asset_co.copy(deep=True, update={ "grid": f"zoom_{zoom_level}" }).dict(by_alias=True), "epsg:3857", ) merge_calc_string: str = integrated_alerts_merge_calc() # We also need to depend on the original source reprojection job source_job = jobs_dict[zoom_level]["source_reprojection_job"] merge_jobs, final_asset_uri = await _merge_assets( dataset, version, pixel_meaning, tile_uri_to_tiles_geojson(wm_date_conf_uri), tile_uri_to_tiles_geojson(intensity_uri), zoom_level, [*intensity_jobs, source_job], merge_calc_string, ) return [*intensity_jobs, *merge_jobs], final_asset_uri
async def date_conf_intensity_symbology( dataset: str, version: str, pixel_meaning: str, source_asset_co: RasterTileSetSourceCreationOptions, zoom_level: int, max_zoom: int, jobs_dict: Dict, ) -> Tuple[List[Job], str]: """Create a Raster Tile Set asset which is the combination of a date_conf asset and a new derived intensity asset. At native resolution (max_zoom) it creates an "intensity" asset which contains the value 55 everywhere there is data in the source (date_conf) raster. For lower zoom levels it resamples the higher zoom level intensity tiles using the "bilinear" resampling method. Finally the merge function combines the date_conf and intensity assets into a three band RGB-encoded asset suitable for converting to PNGs with gdal2tiles in the final stage of raster_tile_cache_asset """ intensity_co = source_asset_co.copy(deep=True, update={ "calc": None, "band_count": 1, "data_type": DataType.uint8 }) intensity_max_calc_string = f"(A > 0) * {MAX_8_BIT_INTENSITY}" intensity_jobs, intensity_uri = await _create_intensity_asset( dataset, version, pixel_meaning, intensity_co, zoom_level, max_zoom, jobs_dict, intensity_max_calc_string, ResamplingMethod.bilinear, ) wm_date_conf_uri: str = get_asset_uri( dataset, version, AssetType.raster_tile_set, source_asset_co.copy(deep=True, update={ "grid": f"zoom_{zoom_level}" }).dict(by_alias=True), "epsg:3857", ) merge_calc_string: str = date_conf_merge_calc() # We also need to depend on the original source reprojection job source_job = jobs_dict[zoom_level]["source_reprojection_job"] merge_jobs, final_asset_uri = await _merge_assets( dataset, version, pixel_meaning, tile_uri_to_tiles_geojson(wm_date_conf_uri), tile_uri_to_tiles_geojson(intensity_uri), zoom_level, [*intensity_jobs, source_job], merge_calc_string, 3, ) return [*intensity_jobs, *merge_jobs], final_asset_uri
async def colormap_symbology( dataset: str, version: str, pixel_meaning: str, source_asset_co: RasterTileSetSourceCreationOptions, zoom_level: int, max_zoom: int, jobs_dict: Dict, ) -> Tuple[List[Job], str]: """Create an RGB(A) raster with gradient or discrete breakpoint symbology.""" assert source_asset_co.symbology is not None # make mypy happy if source_asset_co.symbology.type in ( ColorMapType.discrete_intensity, ColorMapType.gradient_intensity, ): add_intensity_as_alpha: bool = True colormap_asset_pixel_meaning: str = f"colormap_{pixel_meaning}" else: add_intensity_as_alpha = False colormap_asset_pixel_meaning = pixel_meaning colormap_jobs, colormapped_asset_uri = await _create_colormapped_asset( dataset, version, colormap_asset_pixel_meaning, source_asset_co, zoom_level, jobs_dict, ) # Optionally add intensity as alpha band intensity_jobs: Sequence[Job] = tuple() merge_jobs: Sequence[Job] = tuple() if add_intensity_as_alpha: intensity_co = source_asset_co.copy( deep=True, update={ "calc": None, "data_type": DataType.uint8, }, ) intensity_max_zoom_calc_string = "np.ma.array((~A.mask) * 255)" intensity_jobs, intensity_uri = await _create_intensity_asset( dataset, version, pixel_meaning, intensity_co, zoom_level, max_zoom, jobs_dict, intensity_max_zoom_calc_string, ResamplingMethod.average, ) # We also need to depend on the original source reprojection job source_job = jobs_dict[zoom_level]["source_reprojection_job"] merge_jobs, final_asset_uri = await _merge_assets( dataset, version, pixel_meaning, tile_uri_to_tiles_geojson(colormapped_asset_uri), tile_uri_to_tiles_geojson(intensity_uri), zoom_level, [*colormap_jobs, *intensity_jobs, source_job], ) else: final_asset_uri = colormapped_asset_uri return [*colormap_jobs, *intensity_jobs, *merge_jobs], final_asset_uri