def test_label_classes(self) -> None: # Get label_item = pystac.Item.from_file(self.label_example_1_uri) label_classes = LabelExtension.ext(label_item).label_classes self.assertEqual(len(get_opt(label_classes)), 2) self.assertEqual(get_opt(label_classes)[1].classes, ["three", "four"]) # Set new_classes = [ LabelClasses.create(name="label2", classes=["five", "six"]), LabelClasses.create(name="label", classes=["seven", "eight"]), ] label_ext = LabelExtension.ext(label_item) label_ext.label_classes = new_classes self.assertEqual( [ class_name for lc in label_item.properties["label:classes"] for class_name in lc["classes"] ], ["five", "six", "seven", "eight"], ) self.assertListEqual([lc.name for lc in label_ext.label_classes], ["label2", "label"]) first_lc = label_ext.label_classes[0] self.assertEqual("<ClassObject classes=five,six>", first_lc.__repr__()) label_item.validate()
def test_label_classes_summary(self) -> None: label_classes = [ LabelClasses({ "name": "road_type", "classes": ["1", "2", "3", "4", "5", "6"] }), LabelClasses({ "name": "lane_number", "classes": ["1", "2", "3", "4", "5"] }), LabelClasses({ "name": "paved", "classes": ["0", "1"] }), ] collection = Collection.from_file(self.EXAMPLE_COLLECTION) label_ext_summaries = LabelExtension.summaries(collection, True) label_ext_summaries.label_classes = label_classes summaries = collection.summaries assert summaries is not None label_classes_summary = summaries.get_list("label:classes") assert label_classes_summary is not None self.assertListEqual([lc.to_dict() for lc in label_classes], label_classes_summary) label_classes_summary_ext = label_ext_summaries.label_classes assert label_classes_summary_ext is not None self.assertListEqual(label_classes, label_classes_summary_ext)
def test_case_3() -> Catalog: root_cat = Catalog(id="test3", description="test case 3 catalog", title="test case 3 title") image_item = Item( id="imagery-item", geometry=ARBITRARY_GEOM, bbox=ARBITRARY_BBOX, datetime=datetime.utcnow(), properties={}, ) image_item.add_asset( "ortho", Asset(href="some/geotiff.tiff", media_type=MediaType.GEOTIFF)) overviews = [ LabelOverview.create( "label", counts=[ LabelCount.create("one", 1), LabelCount.create("two", 2) ], ) ] label_item = Item( id="label-items", geometry=ARBITRARY_GEOM, bbox=ARBITRARY_BBOX, datetime=datetime.utcnow(), properties={}, ) LabelExtension.add_to(label_item) label_ext = LabelExtension.ext(label_item) label_ext.apply( label_description="ML Labels", label_type=LabelType.VECTOR, label_properties=["label"], label_classes=[ LabelClasses.create(classes=["one", "two"], name="label") ], label_tasks=["classification"], label_methods=["manual"], label_overviews=overviews, ) label_ext.add_source(image_item, assets=["ortho"]) root_cat.add_item(image_item) root_cat.add_item(label_item) return root_cat
def test_case_3(): root_cat = Catalog(id='test3', description='test case 3 catalog', title='test case 3 title') image_item = Item(id='imagery-item', geometry=RANDOM_GEOM, bbox=RANDOM_BBOX, datetime=datetime.utcnow(), properties={}) image_item.add_asset( 'ortho', Asset(href='some/geotiff.tiff', media_type=MediaType.GEOTIFF)) overviews = [ LabelOverview.create('label', counts=[ LabelCount.create('one', 1), LabelCount.create('two', 2) ]) ] label_item = Item(id='label-items', geometry=RANDOM_GEOM, bbox=RANDOM_BBOX, datetime=datetime.utcnow(), properties={}) label_item.ext.enable(Extensions.LABEL) label_item.ext.label.apply(label_description='ML Labels', label_type='vector', label_properties=['label'], label_classes=[ LabelClasses.create( classes=['one', 'two'], name='label') ], label_tasks=['classification'], label_methods=['manual'], label_overviews=overviews) label_item.ext.label.add_source(image_item, assets=['ortho']) root_cat.add_item(image_item) root_cat.add_item(label_item) return root_cat
def test_full_copy_2(self): with TemporaryDirectory() as tmp_dir: cat = Catalog(id='test', description='test catalog') image_item = Item(id='Imagery', geometry=RANDOM_GEOM, bbox=RANDOM_BBOX, datetime=datetime.utcnow(), properties={}) for key in ['ortho', 'dsm']: image_item.add_asset( key, Asset(href='some/{}.tif'.format(key), media_type=MediaType.GEOTIFF)) label_item = Item(id='Labels', geometry=RANDOM_GEOM, bbox=RANDOM_BBOX, datetime=datetime.utcnow(), properties={}, stac_extensions=[Extensions.LABEL]) label_ext = label_item.ext.label label_ext.apply(label_description='labels', label_type='vector', label_properties=['label'], label_classes=[ LabelClasses.create(classes=['one', 'two'], name='label') ], label_tasks=['classification']) label_ext.add_source(image_item, assets=['ortho']) cat.add_items([image_item, label_item]) cat.normalize_hrefs( os.path.join(tmp_dir, 'catalog-full-copy-2-source')) cat.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED) cat2 = cat.full_copy() cat2.normalize_hrefs( os.path.join(tmp_dir, 'catalog-full-copy-2-dest')) cat2.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED) self.check_catalog(cat, 'source') self.check_catalog(cat2, 'dest')
def create_label_item(item): # Assumes the GEOJSON labels are in the # same location as the image img_href = item.assets['ortho'].href label_href = '{}.geojson'.format(os.path.splitext(img_href)[0]) label_item = Item(id='Labels', geometry=item.geometry, bbox=item.bbox, datetime=datetime.utcnow(), properties={}) label_item.ext.enable(Extensions.LABEL) label_ext = label_item.ext.label label_ext.apply( label_description='labels', label_type='vector', label_properties=['label'], label_classes=[LabelClasses.create(classes=['one', 'two'], name='label')], label_tasks=['classification']) label_ext.add_source(item, assets=['ortho']) label_ext.add_geojson_labels(label_href) return [item, label_item]
def main(): """ # The Data 446 qc'ed chips containing flood events, hand-labeled flood classifications 4385 non-qc'ed chips containing water exported only with sentinel 1 and 2 flood classifications # The Catalog Outline ** We want to generate a root catalog that is all, or only training, or only validation items ** ^^^ Script should support this - Root Catalog - Collection: Sentinel 1 data chips - Item: The Item - Collection: Sentinel 2 data chips - Item: The Item - Collection: Sentinel 1 weak labels - Item: The Item - Collection: Sentinel 2 weak labels - Item: The Item - Collection: Hand labels - Item: The Item - Collection: Permanent water labels - Item: The Item - Collection: Traditional otsu algo labels - Item: The Item ## Alternate catalog structure This structure was considered but rejected in the interest of facilitating collections for each of the label datasets. - Root Catalog - Collection: Sentinel 1 - Catalog: Country - Catalog: Event ID (Note: Catalog will always have the first item. Then it will either have the second item or all the others depending on which dir the first item came from) - Item: (dir: S1 + S1_NoQC) Sentinel 1 data chip - Item: (dir: S1Flood_NoQC) Labels from "weak" classification algorithm applied to S1 - Item: (dir: QC_v2) Labels from hand classification (ORed with item below) - Item: (dir: S1Flood) Labels from traditional Otsu algorithm - Item: (dir: Perm) Labels from perm water dataset (this is a Byte tiff, only 1 or 0 for yes or no perm water) - Collection: Sentinel 2 - Catalog: Country - Catalog: Event ID - Item: (dir: S2 + S2_NoQC) Sentinel 2 data chip - Item: (dir: S2Flood) Labels from traditional Otsu algorithm applied to S2 - Collection: PermJRC - Catalog: Lat 10 - Catalog: Lon 10 - Item: (dir: PermJRC) """ parser = argparse.ArgumentParser( description="Build STAC Catalog for sen1floods11") parser.add_argument("--debug", action="store_true") args = parser.parse_args() debug = args.debug storage = S3Storage("sen1floods11-data") catalog_description = "Bonafilia, D., Tellman, B., Anderson, T., Issenberg, E. 2020. Sen1Floods11: a georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1. The IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) Workshops, 2020, pp. 210-211. Available Open access at: http://openaccess.thecvf.com/content_CVPRW_2020/html/w11/Bonafilia_Sen1Floods11_A_Georeferenced_Dataset_to_Train_and_Test_Deep_Learning_CVPRW_2020_paper.html" # noqa: E501 catalog_title = "A georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1" # noqa: E501 catalog = Catalog("sen1floods11", catalog_description, title=catalog_title) print("Created Catalog {}".format(catalog.id)) # Build Sentinel 1 Collection sentinel1 = Collection( "S1", "Sentinel-1 GRD Chips overlapping labeled data. IW mode, GRD product. See https://developers.google.com/earth-engine/sentinel1 for information on preprocessing", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), ) collection_add_sentinel_chips(sentinel1, storage.ls("S1/"), "s1", debug=debug) collection_add_sentinel_chips(sentinel1, storage.ls("S1_NoQC/"), "s1", debug=debug) collection_update_extents(sentinel1) catalog.add_child(sentinel1) # Build Sentinel 2 Collection sentinel2 = Collection( "S2", "Sentinel-2 MSI L1C chips overlapping labeled data. Contains all spectral bands (1 - 12). Does not contain QA mask.", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), ) collection_add_sentinel_chips(sentinel2, storage.ls("S2/"), "s2", debug=debug) collection_add_sentinel_chips(sentinel2, storage.ls("S2_NoQC/"), "s2", debug=debug) collection_update_extents(sentinel2) catalog.add_child(sentinel2) # Build S1 Weak Labels Collection s1weak_labels = Collection( "S1Flood_NoQC", "Chips of water/nowater labels derived from standard OTSU thresholding of Sentinel-1 VH band overlapping weakly-labeled data.", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( s1weak_labels, catalog, storage.ls("S1Flood_NoQC/"), sentinel1_links_func, "0: Not Water. 1: Water.", LabelType.RASTER, label_classes=[LabelClasses([0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(s1weak_labels) catalog.add_child(s1weak_labels) # Build S2 Weak Labels Collection s2weak_labels = Collection( "NoQC", "Weakly-labeled chips derived from traditional Sentinel-2 Classification", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( s2weak_labels, catalog, storage.ls("NoQC/"), sentinel2_links_func, "-1: No Data / Not Valid. 0: Not Water. 1: Water.", # noqa: E501 LabelType.RASTER, label_classes=[LabelClasses([-1, 0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(s2weak_labels) catalog.add_child(s2weak_labels) # Build Hand Labels Collection hand_labels = Collection( "QC_v2", "446 hand labeled chips of surface water from selected flood events", extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( hand_labels, catalog, storage.ls("QC_v2/"), sentinel1_sentinel2_links_func, "Hand labeled chips containing ground truth. -1: No Data / Not Valid. 0: Not Water. 1: Water.", # noqa: E501 LabelType.RASTER, label_classes=[LabelClasses([-1, 0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(hand_labels) catalog.add_child(hand_labels) # Build Permanent Labels collection permanent_labels = Collection( "Perm", "Permanent water chips generated from the 'transition' layer of the JRC (European Commission Joint Research Centre) dataset", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( permanent_labels, catalog, storage.ls("Perm/"), lambda *_: [ ], # No easy way to map JRC source files to the label chips... "0: Not Water. 1: Water.", LabelType.RASTER, label_classes=[LabelClasses([0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(permanent_labels) catalog.add_child(permanent_labels) # Build Otsu algorithm Labels collection otsu_labels = Collection( "S1Flood", "Chips of water/nowater derived from standard OTSU thresholding of Sentinel-1 VH band overlapping labeled data", # noqa: E501 extent=Extent(SpatialExtent([None, None, None, None]), None), stac_extensions=[Extensions.LABEL], ) label_collection_add_items( otsu_labels, catalog, storage.ls("S1Flood/"), sentinel1_links_func, "0: Not Water. 1: Water.", LabelType.RASTER, label_classes=[LabelClasses([0, 1])], label_tasks=["classification"], debug=debug, ) collection_update_extents(otsu_labels) catalog.add_child(otsu_labels) # Save Complete Catalog root_path = "./catalog" catalog.normalize_and_save(root_path, catalog_type=CatalogType.SELF_CONTAINED) print("Saved STAC Catalog {} to {}...".format(catalog.id, root_path))
def test_label_classes_typing(self) -> None: classes: List[str] = ["foo", "bar"] LabelClasses.create(classes=classes)