def add_item(self, path_pro: Optional[str], path_tiff: Optional[str], path_img: Optional[str]) -> None: assert path_pro.endswith(".pro") file_name = path_pro.split("\\")[-1].rstrip(".pro") print(file_name) b0, data = parse(path_pro) item: Item = stac.create_item(i_id=file_name, metadata=b0) assets: List[Asset] = [ Asset(href=path_pro, media_type="pro") ] if path_tiff is not None: assets.append(Asset(href=path_tiff, media_type="geotiff")) if path_img is not None: assets.append(Asset(href=path_img, media_type="img")) stac.add_assets(item, assets) catalog = self.root_catalog.get_child(str(b0["b0_common"]["satId"][0])) if catalog is None: extent = Extent(spatial=SpatialExtent([[-180, -90, 180, 90]]), # TODO: Реальный Extent temporal=TemporalExtent([[ datetime.strptime("2009-01-01T00:00:00.000000", "%Y-%m-%dT%H:%M:%S.%f"), None]])) catalog = Collection(id=str(b0["b0_common"]["satId"][0]), title=b0["b0_common"]["satName"][0].decode("utf-8"), description=f"Catalog for satellite " f"{b0['b0_common']['satName'][0].decode('utf-8')}", extent=extent) self.root_catalog.add_child(catalog, catalog.title) # update_collection_extent(item, catalog) catalog.add_item(item)
def test_eo_items_are_heritable(self): item1 = Item(id='test-item-1', geometry=RANDOM_GEOM, bbox=RANDOM_BBOX, datetime=datetime.utcnow(), properties={'key': 'one'}, stac_extensions=['eo', 'commons']) item2 = Item(id='test-item-2', geometry=RANDOM_GEOM, bbox=RANDOM_BBOX, datetime=datetime.utcnow(), properties={'key': 'two'}, stac_extensions=['eo', 'commons']) wv3_bands = [ Band.create(name='Coastal', description='Coastal: 400 - 450 nm', common_name='coastal'), Band.create(name='Blue', description='Blue: 450 - 510 nm', common_name='blue'), Band.create(name='Green', description='Green: 510 - 580 nm', common_name='green'), Band.create(name='Yellow', description='Yellow: 585 - 625 nm', common_name='yellow'), Band.create(name='Red', description='Red: 630 - 690 nm', common_name='red'), Band.create(name='Red Edge', description='Red Edge: 705 - 745 nm', common_name='rededge'), Band.create(name='Near-IR1', description='Near-IR1: 770 - 895 nm', common_name='nir08'), Band.create(name='Near-IR2', description='Near-IR2: 860 - 1040 nm', common_name='nir09') ] spatial_extent = SpatialExtent(bboxes=[RANDOM_BBOX]) temporal_extent = TemporalExtent(intervals=[[item1.datetime, None]]) collection_extent = Extent(spatial=spatial_extent, temporal=temporal_extent) common_properties = { 'eo:bands': [b.to_dict() for b in wv3_bands], 'gsd': 0.3, 'eo:platform': 'Maxar', 'eo:instrument': 'WorldView3' } collection = Collection(id='test', description='test', extent=collection_extent, properties=common_properties, stac_extensions=['commons'], license='CC-BY-SA-4.0') collection.add_items([item1, item2]) with TemporaryDirectory() as tmp_dir: collection.normalize_hrefs(tmp_dir) collection.save(catalog_type=CatalogType.SELF_CONTAINED) read_col = Collection.from_file('{}/collection.json'.format(tmp_dir)) items = list(read_col.get_all_items()) self.assertEqual(len(items), 2) self.assertTrue(items[0].ext.implements('eo')) self.assertTrue(items[1].ext.implements('eo'))
def test_spatial_extent_from_coordinates(self): extent = SpatialExtent.from_coordinates(RANDOM_GEOM['coordinates']) self.assertEqual(len(extent.bboxes), 1) bbox = extent.bboxes[0] self.assertEqual(len(bbox), 4) for x in bbox: self.assertTrue(type(x) is float)
def test_spatial_extent_from_coordinates(self) -> None: extent = SpatialExtent.from_coordinates(ARBITRARY_GEOM["coordinates"]) self.assertEqual(len(extent.bboxes), 1) bbox = extent.bboxes[0] self.assertEqual(len(bbox), 4) for x in bbox: self.assertTrue(type(x) is float)
def build_items(index_geom): """ index_geom: fioan readable file (ex, shapefile) Build the STAC items """ with fiona.open(index_geom) as src: src_crs = Proj(src.crs) dest_crs = Proj("WGS84") extent = box(*src.bounds) project = Transformer.from_proj(src_crs, dest_crs) catalog_bbox = shapely_transform(project.transform, extent) # build spatial extent for collection ortho_collection = GeobaseSTAC.get_child("canada_spot_orthoimages") ortho_collection.extent.spatial = SpatialExtent( [list(catalog_bbox.bounds)]) geobase = GeobaseSpotFTP() count = 0 for f in src: feature_out = f.copy() new_coords = transform_geom(src_crs, dest_crs, f["geometry"]["coordinates"]) feature_out["geometry"]["coordinates"] = new_coords name = feature_out["properties"]["NAME"] sensor = SPOT_SENSOR[name[:2]] new_item = create_item(name, feature_out, ortho_collection) for f in geobase.list_contents(name): # Add data to the asset spot_file = Asset(href=f, title=None, media_type="application/zip") file_key = f[-13:-4] # image type new_item.add_asset(file_key, spot_file) # Add the thumbnail asset new_item.add_asset( key="thumbnail", asset=Asset( href=geobase.get_thumbnail(name), title=None, media_type=MediaType.JPEG, ), ) ortho_collection.add_item(new_item) count += 1 print(f"{count}... {new_item.id}")
def test_supplying_href_in_init_does_not_fail(self) -> None: test_href = "http://example.com/collection.json" spatial_extent = SpatialExtent(bboxes=[ARBITRARY_BBOX]) temporal_extent = TemporalExtent(intervals=[[TEST_DATETIME, None]]) collection_extent = Extent(spatial=spatial_extent, temporal=temporal_extent) collection = Collection( id="test", description="test desc", extent=collection_extent, href=test_href ) self.assertEqual(collection.get_self_href(), test_href)
def test_supplying_href_in_init_does_not_fail(self): test_href = "http://example.com/collection.json" spatial_extent = SpatialExtent(bboxes=[RANDOM_BBOX]) temporal_extent = TemporalExtent(intervals=[[TEST_DATETIME, None]]) collection_extent = Extent(spatial=spatial_extent, temporal=temporal_extent) collection = Collection(id='test', description='test desc', extent=collection_extent, properties={}, href=test_href) self.assertEqual(collection.get_self_href(), test_href)
def test_spatial_allows_single_bbox(self) -> None: temporal_extent = TemporalExtent(intervals=[[TEST_DATETIME, None]]) # Pass in a single BBOX spatial_extent = SpatialExtent(bboxes=ARBITRARY_BBOX) collection_extent = Extent(spatial=spatial_extent, temporal=temporal_extent) collection = Collection( id="test", description="test desc", extent=collection_extent ) # HREF required by validation collection.set_self_href("https://example.com/collection.json") collection.validate()
def test_spatial_allows_single_bbox(self): temporal_extent = TemporalExtent(intervals=[[TEST_DATETIME, None]]) # Pass in a single BBOX spatial_extent = SpatialExtent(bboxes=RANDOM_BBOX) collection_extent = Extent(spatial=spatial_extent, temporal=temporal_extent) collection = Collection(id='test', description='test desc', extent=collection_extent) # HREF required by validation collection.set_self_href('https://example.com/collection.json') collection.validate()
def collection_update_extents(collection): bounds = GeometryCollection( [shape(s.geometry) for s in collection.get_all_items()]).bounds collection.extent.spatial = SpatialExtent(bounds) dates = { i.datetime for i in collection.get_all_items() if isinstance(i.datetime, datetime) } if len(dates) == 1: collection.extent.temporal = TemporalExtent([(next(iter(dates)), None) ]) elif len(dates) > 1: collection.extent.temporal = TemporalExtent([(min(dates), max(dates))]) else: print("WARN: {} has no TemporalExtent. Dates: {}".format( collection.id, dates)) collection.extent.temporal = TemporalExtent([ (datetime(1900, 1, 1, 0, 0, 0), None) ])
} RANDOM_GEOM = { "type": "Polygon", "coordinates": [[[-2.5048828125, 3.8916575492899987], [-1.9610595703125, 3.8916575492899987], [-1.9610595703125, 4.275202171119132], [-2.5048828125, 4.275202171119132], [-2.5048828125, 3.8916575492899987]]] } RANDOM_BBOX = [ RANDOM_GEOM['coordinates'][0][0][0], RANDOM_GEOM['coordinates'][0][0][1], RANDOM_GEOM['coordinates'][0][1][0], RANDOM_GEOM['coordinates'][0][1][1] ] RANDOM_EXTENT = Extent(spatial=SpatialExtent.from_coordinates(RANDOM_GEOM['coordinates']), temporal=TemporalExtent.from_now()) # noqa: E126 class TestCases: @staticmethod def get_path(rel_path): return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', rel_path)) @staticmethod def get_examples_info(): examples = [] info_path = TestCases.get_path('data-files/examples/example-info.csv') with open(TestCases.get_path('data-files/examples/example-info.csv')) as f: for row in csv.reader(f):
def main(): """ # The Data 446 qc'ed chips containing flood events, hand-labeled flood classifications 4385 non-qc'ed chips containing water exported only with sentinel 1 and 2 flood classifications # The Catalog Outline ** We want to generate a root catalog that is all, or only training, or only validation items ** ^^^ Script should support this - Root Catalog - Collection: Sentinel 1 data chips - Item: The Item - Collection: Sentinel 2 data chips - Item: The Item - Collection: Sentinel 1 weak labels - Item: The Item - Collection: Sentinel 2 weak labels - Item: The Item - Collection: Hand labels - Item: The Item - Collection: Permanent water labels - Item: The Item - Collection: Traditional otsu algo labels - Item: The Item ## Alternate catalog structure This structure was considered but rejected in the interest of facilitating collections for each of the label datasets. - Root Catalog - Collection: Sentinel 1 - Catalog: Country - Catalog: Event ID (Note: Catalog will always have the first item. Then it will either have the second item or all the others depending on which dir the first item came from) - Item: (dir: S1 + S1_NoQC) Sentinel 1 data chip - Item: (dir: S1Flood_NoQC) Labels from "weak" classification algorithm applied to S1 - Item: (dir: QC_v2) Labels from hand classification (ORed with item below) - Item: (dir: S1Flood) Labels from traditional Otsu algorithm - Item: (dir: Perm) Labels from perm water dataset (this is a Byte tiff, only 1 or 0 for yes or no perm water) - Collection: Sentinel 2 - Catalog: Country - Catalog: Event ID - Item: (dir: S2 + S2_NoQC) Sentinel 2 data chip - Item: (dir: S2Flood) Labels from traditional Otsu algorithm applied to S2 - Collection: PermJRC - Catalog: Lat 10 - Catalog: Lon 10 - Item: (dir: PermJRC) """ parser = argparse.ArgumentParser( description="Build STAC Catalog for sen1floods11") parser.add_argument("--debug", action="store_true") args = parser.parse_args() debug = args.debug storage = S3Storage("sen1floods11-data") catalog_description = "Bonafilia, D., Tellman, B., Anderson, T., Issenberg, E. 2020. Sen1Floods11: a georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1. The IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) Workshops, 2020, pp. 210-211. Available Open access at: http://openaccess.thecvf.com/content_CVPRW_2020/html/w11/Bonafilia_Sen1Floods11_A_Georeferenced_Dataset_to_Train_and_Test_Deep_Learning_CVPRW_2020_paper.html" # noqa: E501 catalog_title = "A georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1" # noqa: E501 catalog = Catalog("sen1floods11", catalog_description, title=catalog_title) print("Created Catalog {}".format(catalog.id)) # Build Sentinel 1 Collection sentinel1 = Collection( "S1", "Sentinel-1 GRD Chips overlapping labeled data. IW mode, GRD product. See https://developers.google.com/earth-engine/sentinel1 for information on preprocessing", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), ) collection_add_sentinel_chips(sentinel1, storage.ls("S1/"), "s1", debug=debug) collection_add_sentinel_chips(sentinel1, storage.ls("S1_NoQC/"), "s1", debug=debug) collection_update_extents(sentinel1) catalog.add_child(sentinel1) # Build Sentinel 2 Collection sentinel2 = Collection( "S2", "Sentinel-2 MSI L1C chips overlapping labeled data. Contains all spectral bands (1 - 12). Does not contain QA mask.", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), ) collection_add_sentinel_chips(sentinel2, storage.ls("S2/"), "s2", debug=debug) collection_add_sentinel_chips(sentinel2, storage.ls("S2_NoQC/"), "s2", debug=debug) collection_update_extents(sentinel2) catalog.add_child(sentinel2) # Build S1 Weak Labels Collection s1weak_labels = Collection( "S1Flood_NoQC", "Chips of water/nowater labels derived from standard OTSU thresholding of Sentinel-1 VH band overlapping weakly-labeled data.", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( s1weak_labels, catalog, storage.ls("S1Flood_NoQC/"), sentinel1_links_func, "0: Not Water. 1: Water.", LabelType.RASTER, label_classes=[LabelClasses([0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(s1weak_labels) catalog.add_child(s1weak_labels) # Build S2 Weak Labels Collection s2weak_labels = Collection( "NoQC", "Weakly-labeled chips derived from traditional Sentinel-2 Classification", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( s2weak_labels, catalog, storage.ls("NoQC/"), sentinel2_links_func, "-1: No Data / Not Valid. 0: Not Water. 1: Water.", # noqa: E501 LabelType.RASTER, label_classes=[LabelClasses([-1, 0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(s2weak_labels) catalog.add_child(s2weak_labels) # Build Hand Labels Collection hand_labels = Collection( "QC_v2", "446 hand labeled chips of surface water from selected flood events", extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( hand_labels, catalog, storage.ls("QC_v2/"), sentinel1_sentinel2_links_func, "Hand labeled chips containing ground truth. -1: No Data / Not Valid. 0: Not Water. 1: Water.", # noqa: E501 LabelType.RASTER, label_classes=[LabelClasses([-1, 0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(hand_labels) catalog.add_child(hand_labels) # Build Permanent Labels collection permanent_labels = Collection( "Perm", "Permanent water chips generated from the 'transition' layer of the JRC (European Commission Joint Research Centre) dataset", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( permanent_labels, catalog, storage.ls("Perm/"), lambda *_: [ ], # No easy way to map JRC source files to the label chips... "0: Not Water. 1: Water.", LabelType.RASTER, label_classes=[LabelClasses([0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(permanent_labels) catalog.add_child(permanent_labels) # Build Otsu algorithm Labels collection otsu_labels = Collection( "S1Flood", "Chips of water/nowater derived from standard OTSU thresholding of Sentinel-1 VH band overlapping labeled data", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( otsu_labels, catalog, storage.ls("S1Flood/"), sentinel1_links_func, "0: Not Water. 1: Water.", LabelType.RASTER, label_classes=[LabelClasses([0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(otsu_labels) catalog.add_child(otsu_labels) # Save Complete Catalog root_path = "./catalog" catalog.normalize_and_save(root_path, catalog_type=CatalogType.SELF_CONTAINED) print("Saved STAC Catalog {} to {}...".format(catalog.id, root_path))
required=True, help= "Bucket+key path to a directory containing JRC surface water labels", ) args = parser.parse_args() parsed_s3_path = urlparse(args.jrc_monthly_root_s3) bucket = parsed_s3_path.netloc collection_description = ( "JRC Global Monthly Water around the Mississippi river system", ) collection_title = "Global Monthly Water: Mississippi river system" spatial_extent = SpatialExtent([[ 29.038948834106055, -92.72807246278022, 42.55475543734189, -88.02592402528022, ]]) # The JRC water dataset examines imagery from march, 1984 to december, 2019 start_dt = datetime.combine(date(1984, 3, 1), time.min) end_dt = datetime.combine(date(2019, 12, 1), time.min) collection_temporal_extent = TemporalExtent(intervals=[[start_dt, end_dt]]) collection = Collection( id="jrc-monthly-water-mississippi-river", description=collection_description, extent=Extent(spatial_extent, collection_temporal_extent), title=collection_title, )
SpotProviders = [ Provider( "Government of Canada", "Natural Resources Canada Centre for Topographic Information", ["licensor", "processor"], "www.geobase.ca", ), Provider("Sparkgeo", "*****@*****.**", ["processor", "host"], "www.sparkgeo.com"), Provider( "PCI Geomatics", "*****@*****.**", ["processor", "host"], "www.pcigeomatics.com" ), ] SpotExtents = Extent( SpatialExtent([[0.0, 0.0, 0.0, 0.0]]), TemporalExtent( [ [ datetime.strptime("2005-01-01", "%Y-%m-%d"), datetime.strptime("2010-01-01", "%Y-%m-%d"), ] ] ), ) OrthoCollection = Collection( id="canada_spot_orthoimages", description="Orthoimages of Canada 2005-2010", extent=SpotExtents, title=None,
# The extents should be the same, so whichever one is checked last should be fine with rio.open(s3_path) as img: bounds = img.bounds assets.append(Asset(s3_path)) if aggregate_bounds is None: aggregate_bounds = bounds else: aggregate_bounds = rio.coords.BoundingBox( min(bounds.left, aggregate_bounds.left), min(bounds.bottom, aggregate_bounds.bottom), max(bounds.right, aggregate_bounds.right), max(bounds.top, aggregate_bounds.top), ) item_spatial_extent = SpatialExtent( [[bounds.bottom, bounds.left, bounds.top, bounds.right]]) item_extent = Extent(item_spatial_extent, temporal_extent) image_item = Item( "flood_" + flood_id + "_chip_" + group_id, geometry=mapping( Polygon([ [bounds.left, bounds.bottom], [bounds.right, bounds.bottom], [bounds.right, bounds.top], [bounds.left, bounds.top], [bounds.left, bounds.bottom], ])), bbox=[bounds.bottom, bounds.left, bounds.top, bounds.right], datetime=start_time, properties={}, )
os.makedirs("{}/{}".format(root_path, fid), exist_ok=True) with open("{}/{}/{}-usfimr.wkt".format(root_path, fid, fid), "w") as wkt_file: wkt_file.write(shapely_geom.wkt) with open("{}/{}/{}-usfimr.wkb".format(root_path, fid, fid), "wb") as wkb_file: wkb_file.write(shapely_geom.wkb) with open("{}/{}/{}-usfimr.geojson".format(root_path, fid, fid), "w") as geojson_file: geojson_file.write(json.dumps(geom)) overall_extent = Extent( SpatialExtent(running_spatial_extent), TemporalExtent([[running_start_dt, running_end_dt]]), ) root_collection = Collection( id="USFIMR", description= "GloFIMR is an extension of the USFIMR project that commenced in August 2016 with funding from NOAA. The project’s main goal is to provide high-resolution inundation extent maps of flood events to be used by scientists and practitioners for model calibration and flood susceptibility evaluation. The maps are based on analysis of Remote Sensing imagery from a number of Satellite sensors (e.g. Landsat, Sentinel-1, Sentinel-2). The maps are accessible via the online map repository below. The repository is under development and new maps are added upon request.", title="U.S. Flood Inundation Mapping Repository", extent=overall_extent, ) for item in items: root_collection.add_item(item) # Save Complete Catalog
[-1.9610595703125, 3.8916575492899987], [-1.9610595703125, 4.275202171119132], [-2.5048828125, 4.275202171119132], [-2.5048828125, 3.8916575492899987], ]], } ARBITRARY_BBOX: List[float] = [ ARBITRARY_GEOM["coordinates"][0][0][0], ARBITRARY_GEOM["coordinates"][0][0][1], ARBITRARY_GEOM["coordinates"][0][1][0], ARBITRARY_GEOM["coordinates"][0][1][1], ] ARBITRARY_EXTENT = Extent( spatial=SpatialExtent.from_coordinates(ARBITRARY_GEOM["coordinates"]), temporal=TemporalExtent.from_now(), ) class ExampleInfo: def __init__( self, path: str, object_type: pystac.STACObjectType, stac_version: str, extensions: List[str], valid: bool, ) -> None: self.path = path self.object_type = object_type
media_type="image/tiff; application=geotiff", ) huc_item.add_asset(key="catchhuc", asset=catchhuc_asset) hydrogeo_asset = Asset( href="{}/{}/hydrogeo-fulltable-{}.csv".format(args.root_uri, fid, fid), description="Hydraulic property table with the following fields: CatchId, Stage, Number of Cells, SurfaceArea (m2), BedArea (m2), Volume (m3), SLOPE, LENGTHKM, AREASQKM, Roughness, TopWidth (m), WettedPerimeter (m), WetArea (m2), HydraulicRadius (m), Discharge (m3s-1)", media_type="text/csv", ) huc_item.add_asset(key="hydrogeo", asset=hydrogeo_asset) items.append(huc_item) overall_extent = Extent( SpatialExtent( [running_extent[0], running_extent[1], running_extent[2], running_extent[3]] ), TemporalExtent([[version_dt, None]]), ) # Root Collection root_collection = Collection( id="hand_021", description="The continental flood inundation mapping (CFIM) framework is a high-performance computing (HPC)-based computational framework for the Height Above Nearest Drainage (HAND)-based inundation mapping methodology. Using the 10m Digital Elevation Model (DEM) data produced by U.S. Geological Survey (USGS) 3DEP (the 3-D Elevation Program) and the NHDPlus hydrography dataset produced by USGS and the U.S. Environmental Protection Agency (EPA), a hydrological terrain raster called HAND is computed for HUC6 units in the conterminous U.S. (CONUS). The value of each raster cell in HAND is an approximation of the relative elevation between the cell and its nearest water stream. Derived from HAND, a hydraulic property table is established to calculate river geometry properties for each of the 2.7 million river reaches covered by NHDPlus (5.5 million kilometers in total length). This table is a lookup table for water depth given an input stream flow value. Such lookup is available between water depth 0m and 25m at 1-foot interval. The flood inundation map can then be computed by using HAND and this lookup table based on the near real-time water forecast from the National Water Model (NWM) at the National Oceanic and Atmospheric Administration (NOAA).", title="HAND and the Hydraulic Property Table version 0.2.1", extent=overall_extent, license="CC-BY-4.0", ) for item in items: root_collection.add_item(item)