예제 #1
0
    def test_label_classes(self) -> None:
        # Get
        label_item = pystac.Item.from_file(self.label_example_1_uri)
        label_classes = LabelExtension.ext(label_item).label_classes

        self.assertEqual(len(get_opt(label_classes)), 2)
        self.assertEqual(get_opt(label_classes)[1].classes, ["three", "four"])

        # Set
        new_classes = [
            LabelClasses.create(name="label2", classes=["five", "six"]),
            LabelClasses.create(name="label", classes=["seven", "eight"]),
        ]

        label_ext = LabelExtension.ext(label_item)
        label_ext.label_classes = new_classes
        self.assertEqual(
            [
                class_name for lc in label_item.properties["label:classes"]
                for class_name in lc["classes"]
            ],
            ["five", "six", "seven", "eight"],
        )

        self.assertListEqual([lc.name for lc in label_ext.label_classes],
                             ["label2", "label"])

        first_lc = label_ext.label_classes[0]
        self.assertEqual("<ClassObject classes=five,six>", first_lc.__repr__())

        label_item.validate()
예제 #2
0
    def test_label_classes_summary(self) -> None:
        label_classes = [
            LabelClasses({
                "name": "road_type",
                "classes": ["1", "2", "3", "4", "5", "6"]
            }),
            LabelClasses({
                "name": "lane_number",
                "classes": ["1", "2", "3", "4", "5"]
            }),
            LabelClasses({
                "name": "paved",
                "classes": ["0", "1"]
            }),
        ]
        collection = Collection.from_file(self.EXAMPLE_COLLECTION)
        label_ext_summaries = LabelExtension.summaries(collection, True)

        label_ext_summaries.label_classes = label_classes

        summaries = collection.summaries
        assert summaries is not None
        label_classes_summary = summaries.get_list("label:classes")
        assert label_classes_summary is not None
        self.assertListEqual([lc.to_dict() for lc in label_classes],
                             label_classes_summary)

        label_classes_summary_ext = label_ext_summaries.label_classes
        assert label_classes_summary_ext is not None
        self.assertListEqual(label_classes, label_classes_summary_ext)
예제 #3
0
    def test_case_3() -> Catalog:
        root_cat = Catalog(id="test3",
                           description="test case 3 catalog",
                           title="test case 3 title")

        image_item = Item(
            id="imagery-item",
            geometry=ARBITRARY_GEOM,
            bbox=ARBITRARY_BBOX,
            datetime=datetime.utcnow(),
            properties={},
        )

        image_item.add_asset(
            "ortho",
            Asset(href="some/geotiff.tiff", media_type=MediaType.GEOTIFF))

        overviews = [
            LabelOverview.create(
                "label",
                counts=[
                    LabelCount.create("one", 1),
                    LabelCount.create("two", 2)
                ],
            )
        ]

        label_item = Item(
            id="label-items",
            geometry=ARBITRARY_GEOM,
            bbox=ARBITRARY_BBOX,
            datetime=datetime.utcnow(),
            properties={},
        )

        LabelExtension.add_to(label_item)
        label_ext = LabelExtension.ext(label_item)
        label_ext.apply(
            label_description="ML Labels",
            label_type=LabelType.VECTOR,
            label_properties=["label"],
            label_classes=[
                LabelClasses.create(classes=["one", "two"], name="label")
            ],
            label_tasks=["classification"],
            label_methods=["manual"],
            label_overviews=overviews,
        )
        label_ext.add_source(image_item, assets=["ortho"])

        root_cat.add_item(image_item)
        root_cat.add_item(label_item)

        return root_cat
예제 #4
0
    def test_case_3():
        root_cat = Catalog(id='test3',
                           description='test case 3 catalog',
                           title='test case 3 title')

        image_item = Item(id='imagery-item',
                          geometry=RANDOM_GEOM,
                          bbox=RANDOM_BBOX,
                          datetime=datetime.utcnow(),
                          properties={})

        image_item.add_asset(
            'ortho',
            Asset(href='some/geotiff.tiff', media_type=MediaType.GEOTIFF))

        overviews = [
            LabelOverview.create('label',
                                 counts=[
                                     LabelCount.create('one', 1),
                                     LabelCount.create('two', 2)
                                 ])
        ]

        label_item = Item(id='label-items',
                          geometry=RANDOM_GEOM,
                          bbox=RANDOM_BBOX,
                          datetime=datetime.utcnow(),
                          properties={})

        label_item.ext.enable(Extensions.LABEL)
        label_item.ext.label.apply(label_description='ML Labels',
                                   label_type='vector',
                                   label_properties=['label'],
                                   label_classes=[
                                       LabelClasses.create(
                                           classes=['one', 'two'],
                                           name='label')
                                   ],
                                   label_tasks=['classification'],
                                   label_methods=['manual'],
                                   label_overviews=overviews)
        label_item.ext.label.add_source(image_item, assets=['ortho'])

        root_cat.add_item(image_item)
        root_cat.add_item(label_item)

        return root_cat
예제 #5
0
    def test_full_copy_2(self):
        with TemporaryDirectory() as tmp_dir:
            cat = Catalog(id='test', description='test catalog')
            image_item = Item(id='Imagery',
                              geometry=RANDOM_GEOM,
                              bbox=RANDOM_BBOX,
                              datetime=datetime.utcnow(),
                              properties={})
            for key in ['ortho', 'dsm']:
                image_item.add_asset(
                    key,
                    Asset(href='some/{}.tif'.format(key),
                          media_type=MediaType.GEOTIFF))

            label_item = Item(id='Labels',
                              geometry=RANDOM_GEOM,
                              bbox=RANDOM_BBOX,
                              datetime=datetime.utcnow(),
                              properties={},
                              stac_extensions=[Extensions.LABEL])
            label_ext = label_item.ext.label
            label_ext.apply(label_description='labels',
                            label_type='vector',
                            label_properties=['label'],
                            label_classes=[
                                LabelClasses.create(classes=['one', 'two'],
                                                    name='label')
                            ],
                            label_tasks=['classification'])
            label_ext.add_source(image_item, assets=['ortho'])

            cat.add_items([image_item, label_item])

            cat.normalize_hrefs(
                os.path.join(tmp_dir, 'catalog-full-copy-2-source'))
            cat.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED)
            cat2 = cat.full_copy()
            cat2.normalize_hrefs(
                os.path.join(tmp_dir, 'catalog-full-copy-2-dest'))
            cat2.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED)

            self.check_catalog(cat, 'source')
            self.check_catalog(cat2, 'dest')
예제 #6
0
        def create_label_item(item):
            # Assumes the GEOJSON labels are in the
            # same location as the image
            img_href = item.assets['ortho'].href
            label_href = '{}.geojson'.format(os.path.splitext(img_href)[0])
            label_item = Item(id='Labels',
                              geometry=item.geometry,
                              bbox=item.bbox,
                              datetime=datetime.utcnow(),
                              properties={})
            label_item.ext.enable(Extensions.LABEL)
            label_ext = label_item.ext.label
            label_ext.apply(
                label_description='labels',
                label_type='vector',
                label_properties=['label'],
                label_classes=[LabelClasses.create(classes=['one', 'two'], name='label')],
                label_tasks=['classification'])
            label_ext.add_source(item, assets=['ortho'])
            label_ext.add_geojson_labels(label_href)

            return [item, label_item]
def main():
    """

# The Data

446 qc'ed chips containing flood events, hand-labeled flood classifications
4385 non-qc'ed chips containing water exported only with sentinel 1 and 2 flood classifications

# The Catalog Outline

** We want to generate a root catalog that is all, or only training, or only validation items **
^^^ Script should support this

- Root Catalog
    - Collection: Sentinel 1 data chips
        - Item: The Item
    - Collection: Sentinel 2 data chips
        - Item: The Item
    - Collection: Sentinel 1 weak labels
        - Item: The Item
    - Collection: Sentinel 2 weak labels
        - Item: The Item
    - Collection: Hand labels
        - Item: The Item
    - Collection: Permanent water labels
        - Item: The Item
    - Collection: Traditional otsu algo labels
        - Item: The Item

## Alternate catalog structure

This structure was considered but rejected in the interest of facilitating collections for each
of the label datasets.

- Root Catalog
    - Collection: Sentinel 1
        - Catalog: Country
            - Catalog: Event ID
                (Note: Catalog will always have the first item. Then it will either have the second
                       item or all the others depending on which dir the first item came from)
                - Item: (dir: S1 + S1_NoQC) Sentinel 1 data chip
                - Item: (dir: S1Flood_NoQC) Labels from "weak" classification algorithm applied to S1
                - Item: (dir: QC_v2) Labels from hand classification (ORed with item below)
                - Item: (dir: S1Flood) Labels from traditional Otsu algorithm
                - Item: (dir: Perm) Labels from perm water dataset (this is a Byte tiff, only 1 or 0
                        for yes or no perm water)
    - Collection: Sentinel 2
        - Catalog: Country
            - Catalog: Event ID
                - Item: (dir: S2 + S2_NoQC) Sentinel 2 data chip
                - Item: (dir: S2Flood) Labels from traditional Otsu algorithm applied to S2
    - Collection: PermJRC
        - Catalog: Lat 10
            - Catalog: Lon 10
                - Item: (dir: PermJRC)
    """
    parser = argparse.ArgumentParser(
        description="Build STAC Catalog for sen1floods11")
    parser.add_argument("--debug", action="store_true")
    args = parser.parse_args()
    debug = args.debug

    storage = S3Storage("sen1floods11-data")

    catalog_description = "Bonafilia, D., Tellman, B., Anderson, T., Issenberg, E. 2020. Sen1Floods11: a georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1. The IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) Workshops, 2020, pp. 210-211. Available Open access at: http://openaccess.thecvf.com/content_CVPRW_2020/html/w11/Bonafilia_Sen1Floods11_A_Georeferenced_Dataset_to_Train_and_Test_Deep_Learning_CVPRW_2020_paper.html"  # noqa: E501
    catalog_title = "A georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1"  # noqa: E501

    catalog = Catalog("sen1floods11", catalog_description, title=catalog_title)
    print("Created Catalog {}".format(catalog.id))

    # Build Sentinel 1 Collection
    sentinel1 = Collection(
        "S1",
        "Sentinel-1 GRD Chips overlapping labeled data. IW mode, GRD product. See https://developers.google.com/earth-engine/sentinel1 for information on preprocessing",  # noqa: E501
        extent=Extent(SpatialExtent([None, None, None, None]), None),
    )
    collection_add_sentinel_chips(sentinel1,
                                  storage.ls("S1/"),
                                  "s1",
                                  debug=debug)
    collection_add_sentinel_chips(sentinel1,
                                  storage.ls("S1_NoQC/"),
                                  "s1",
                                  debug=debug)
    collection_update_extents(sentinel1)
    catalog.add_child(sentinel1)

    # Build Sentinel 2 Collection
    sentinel2 = Collection(
        "S2",
        "Sentinel-2 MSI L1C chips overlapping labeled data. Contains all spectral bands (1 - 12). Does not contain QA mask.",  # noqa: E501
        extent=Extent(SpatialExtent([None, None, None, None]), None),
    )
    collection_add_sentinel_chips(sentinel2,
                                  storage.ls("S2/"),
                                  "s2",
                                  debug=debug)
    collection_add_sentinel_chips(sentinel2,
                                  storage.ls("S2_NoQC/"),
                                  "s2",
                                  debug=debug)
    collection_update_extents(sentinel2)
    catalog.add_child(sentinel2)

    # Build S1 Weak Labels Collection
    s1weak_labels = Collection(
        "S1Flood_NoQC",
        "Chips of water/nowater labels derived from standard OTSU thresholding of Sentinel-1 VH band overlapping weakly-labeled data.",  # noqa: E501
        extent=Extent(SpatialExtent([None, None, None, None]), None),
        stac_extensions=[Extensions.LABEL],
    )
    label_collection_add_items(
        s1weak_labels,
        catalog,
        storage.ls("S1Flood_NoQC/"),
        sentinel1_links_func,
        "0: Not Water. 1: Water.",
        LabelType.RASTER,
        label_classes=[LabelClasses([0, 1])],
        label_tasks=["classification"],
        debug=debug,
    )
    collection_update_extents(s1weak_labels)
    catalog.add_child(s1weak_labels)

    # Build S2 Weak Labels Collection
    s2weak_labels = Collection(
        "NoQC",
        "Weakly-labeled chips derived from traditional Sentinel-2 Classification",  # noqa: E501
        extent=Extent(SpatialExtent([None, None, None, None]), None),
        stac_extensions=[Extensions.LABEL],
    )
    label_collection_add_items(
        s2weak_labels,
        catalog,
        storage.ls("NoQC/"),
        sentinel2_links_func,
        "-1: No Data / Not Valid. 0: Not Water. 1: Water.",  # noqa: E501
        LabelType.RASTER,
        label_classes=[LabelClasses([-1, 0, 1])],
        label_tasks=["classification"],
        debug=debug,
    )
    collection_update_extents(s2weak_labels)
    catalog.add_child(s2weak_labels)

    # Build Hand Labels Collection
    hand_labels = Collection(
        "QC_v2",
        "446 hand labeled chips of surface water from selected flood events",
        extent=Extent(SpatialExtent([None, None, None, None]), None),
        stac_extensions=[Extensions.LABEL],
    )
    label_collection_add_items(
        hand_labels,
        catalog,
        storage.ls("QC_v2/"),
        sentinel1_sentinel2_links_func,
        "Hand labeled chips containing ground truth. -1: No Data / Not Valid. 0: Not Water. 1: Water.",  # noqa: E501
        LabelType.RASTER,
        label_classes=[LabelClasses([-1, 0, 1])],
        label_tasks=["classification"],
        debug=debug,
    )
    collection_update_extents(hand_labels)
    catalog.add_child(hand_labels)

    # Build Permanent Labels collection
    permanent_labels = Collection(
        "Perm",
        "Permanent water chips generated from the 'transition' layer of the JRC (European Commission Joint Research Centre) dataset",  # noqa: E501
        extent=Extent(SpatialExtent([None, None, None, None]), None),
        stac_extensions=[Extensions.LABEL],
    )
    label_collection_add_items(
        permanent_labels,
        catalog,
        storage.ls("Perm/"),
        lambda *_: [
        ],  # No easy way to map JRC source files to the label chips...
        "0: Not Water. 1: Water.",
        LabelType.RASTER,
        label_classes=[LabelClasses([0, 1])],
        label_tasks=["classification"],
        debug=debug,
    )
    collection_update_extents(permanent_labels)
    catalog.add_child(permanent_labels)

    # Build Otsu algorithm Labels collection
    otsu_labels = Collection(
        "S1Flood",
        "Chips of water/nowater derived from standard OTSU thresholding of Sentinel-1 VH band overlapping labeled data",  # noqa: E501
        extent=Extent(SpatialExtent([None, None, None, None]), None),
        stac_extensions=[Extensions.LABEL],
    )
    label_collection_add_items(
        otsu_labels,
        catalog,
        storage.ls("S1Flood/"),
        sentinel1_links_func,
        "0: Not Water. 1: Water.",
        LabelType.RASTER,
        label_classes=[LabelClasses([0, 1])],
        label_tasks=["classification"],
        debug=debug,
    )
    collection_update_extents(otsu_labels)
    catalog.add_child(otsu_labels)

    # Save Complete Catalog
    root_path = "./catalog"
    catalog.normalize_and_save(root_path,
                               catalog_type=CatalogType.SELF_CONTAINED)
    print("Saved STAC Catalog {} to {}...".format(catalog.id, root_path))
예제 #8
0
 def test_label_classes_typing(self) -> None:
     classes: List[str] = ["foo", "bar"]
     LabelClasses.create(classes=classes)