def test_label_classes(self) -> None:
# Get
label_item = pystac.Item.from_file(self.label_example_1_uri)
label_classes = LabelExtension.ext(label_item).label_classes
self.assertEqual(len(get_opt(label_classes)), 2)
self.assertEqual(get_opt(label_classes)[1].classes, ["three", "four"])
# Set
new_classes = [
LabelClasses.create(name="label2", classes=["five", "six"]),
LabelClasses.create(name="label", classes=["seven", "eight"]),
]
label_ext = LabelExtension.ext(label_item)
label_ext.label_classes = new_classes
self.assertEqual(
[
class_name for lc in label_item.properties["label:classes"]
for class_name in lc["classes"]
],
["five", "six", "seven", "eight"],
)
self.assertListEqual([lc.name for lc in label_ext.label_classes],
["label2", "label"])
first_lc = label_ext.label_classes[0]
self.assertEqual("<ClassObject classes=five,six>", first_lc.__repr__())
label_item.validate()
def test_label_classes_summary(self) -> None:
label_classes = [
LabelClasses({
"name": "road_type",
"classes": ["1", "2", "3", "4", "5", "6"]
}),
LabelClasses({
"name": "lane_number",
"classes": ["1", "2", "3", "4", "5"]
}),
LabelClasses({
"name": "paved",
"classes": ["0", "1"]
}),
]
collection = Collection.from_file(self.EXAMPLE_COLLECTION)
label_ext_summaries = LabelExtension.summaries(collection, True)
label_ext_summaries.label_classes = label_classes
summaries = collection.summaries
assert summaries is not None
label_classes_summary = summaries.get_list("label:classes")
assert label_classes_summary is not None
self.assertListEqual([lc.to_dict() for lc in label_classes],
label_classes_summary)
label_classes_summary_ext = label_ext_summaries.label_classes
assert label_classes_summary_ext is not None
self.assertListEqual(label_classes, label_classes_summary_ext)
def test_case_3() -> Catalog:
root_cat = Catalog(id="test3",
description="test case 3 catalog",
title="test case 3 title")
image_item = Item(
id="imagery-item",
geometry=ARBITRARY_GEOM,
bbox=ARBITRARY_BBOX,
datetime=datetime.utcnow(),
properties={},
)
image_item.add_asset(
"ortho",
Asset(href="some/geotiff.tiff", media_type=MediaType.GEOTIFF))
overviews = [
LabelOverview.create(
"label",
counts=[
LabelCount.create("one", 1),
LabelCount.create("two", 2)
],
)
]
label_item = Item(
id="label-items",
geometry=ARBITRARY_GEOM,
bbox=ARBITRARY_BBOX,
datetime=datetime.utcnow(),
properties={},
)
LabelExtension.add_to(label_item)
label_ext = LabelExtension.ext(label_item)
label_ext.apply(
label_description="ML Labels",
label_type=LabelType.VECTOR,
label_properties=["label"],
label_classes=[
LabelClasses.create(classes=["one", "two"], name="label")
],
label_tasks=["classification"],
label_methods=["manual"],
label_overviews=overviews,
)
label_ext.add_source(image_item, assets=["ortho"])
root_cat.add_item(image_item)
root_cat.add_item(label_item)
return root_cat
def test_case_3():
root_cat = Catalog(id='test3',
description='test case 3 catalog',
title='test case 3 title')
image_item = Item(id='imagery-item',
geometry=RANDOM_GEOM,
bbox=RANDOM_BBOX,
datetime=datetime.utcnow(),
properties={})
image_item.add_asset(
'ortho',
Asset(href='some/geotiff.tiff', media_type=MediaType.GEOTIFF))
overviews = [
LabelOverview.create('label',
counts=[
LabelCount.create('one', 1),
LabelCount.create('two', 2)
])
]
label_item = Item(id='label-items',
geometry=RANDOM_GEOM,
bbox=RANDOM_BBOX,
datetime=datetime.utcnow(),
properties={})
label_item.ext.enable(Extensions.LABEL)
label_item.ext.label.apply(label_description='ML Labels',
label_type='vector',
label_properties=['label'],
label_classes=[
LabelClasses.create(
classes=['one', 'two'],
name='label')
],
label_tasks=['classification'],
label_methods=['manual'],
label_overviews=overviews)
label_item.ext.label.add_source(image_item, assets=['ortho'])
root_cat.add_item(image_item)
root_cat.add_item(label_item)
return root_cat
def test_full_copy_2(self):
with TemporaryDirectory() as tmp_dir:
cat = Catalog(id='test', description='test catalog')
image_item = Item(id='Imagery',
geometry=RANDOM_GEOM,
bbox=RANDOM_BBOX,
datetime=datetime.utcnow(),
properties={})
for key in ['ortho', 'dsm']:
image_item.add_asset(
key,
Asset(href='some/{}.tif'.format(key),
media_type=MediaType.GEOTIFF))
label_item = Item(id='Labels',
geometry=RANDOM_GEOM,
bbox=RANDOM_BBOX,
datetime=datetime.utcnow(),
properties={},
stac_extensions=[Extensions.LABEL])
label_ext = label_item.ext.label
label_ext.apply(label_description='labels',
label_type='vector',
label_properties=['label'],
label_classes=[
LabelClasses.create(classes=['one', 'two'],
name='label')
],
label_tasks=['classification'])
label_ext.add_source(image_item, assets=['ortho'])
cat.add_items([image_item, label_item])
cat.normalize_hrefs(
os.path.join(tmp_dir, 'catalog-full-copy-2-source'))
cat.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED)
cat2 = cat.full_copy()
cat2.normalize_hrefs(
os.path.join(tmp_dir, 'catalog-full-copy-2-dest'))
cat2.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED)
self.check_catalog(cat, 'source')
self.check_catalog(cat2, 'dest')
def create_label_item(item):
# Assumes the GEOJSON labels are in the
# same location as the image
img_href = item.assets['ortho'].href
label_href = '{}.geojson'.format(os.path.splitext(img_href)[0])
label_item = Item(id='Labels',
geometry=item.geometry,
bbox=item.bbox,
datetime=datetime.utcnow(),
properties={})
label_item.ext.enable(Extensions.LABEL)
label_ext = label_item.ext.label
label_ext.apply(
label_description='labels',
label_type='vector',
label_properties=['label'],
label_classes=[LabelClasses.create(classes=['one', 'two'], name='label')],
label_tasks=['classification'])
label_ext.add_source(item, assets=['ortho'])
label_ext.add_geojson_labels(label_href)
return [item, label_item]
def main():
"""
# The Data
446 qc'ed chips containing flood events, hand-labeled flood classifications
4385 non-qc'ed chips containing water exported only with sentinel 1 and 2 flood classifications
# The Catalog Outline
** We want to generate a root catalog that is all, or only training, or only validation items **
^^^ Script should support this
- Root Catalog
- Collection: Sentinel 1 data chips
- Item: The Item
- Collection: Sentinel 2 data chips
- Item: The Item
- Collection: Sentinel 1 weak labels
- Item: The Item
- Collection: Sentinel 2 weak labels
- Item: The Item
- Collection: Hand labels
- Item: The Item
- Collection: Permanent water labels
- Item: The Item
- Collection: Traditional otsu algo labels
- Item: The Item
## Alternate catalog structure
This structure was considered but rejected in the interest of facilitating collections for each
of the label datasets.
- Root Catalog
- Collection: Sentinel 1
- Catalog: Country
- Catalog: Event ID
(Note: Catalog will always have the first item. Then it will either have the second
item or all the others depending on which dir the first item came from)
- Item: (dir: S1 + S1_NoQC) Sentinel 1 data chip
- Item: (dir: S1Flood_NoQC) Labels from "weak" classification algorithm applied to S1
- Item: (dir: QC_v2) Labels from hand classification (ORed with item below)
- Item: (dir: S1Flood) Labels from traditional Otsu algorithm
- Item: (dir: Perm) Labels from perm water dataset (this is a Byte tiff, only 1 or 0
for yes or no perm water)
- Collection: Sentinel 2
- Catalog: Country
- Catalog: Event ID
- Item: (dir: S2 + S2_NoQC) Sentinel 2 data chip
- Item: (dir: S2Flood) Labels from traditional Otsu algorithm applied to S2
- Collection: PermJRC
- Catalog: Lat 10
- Catalog: Lon 10
- Item: (dir: PermJRC)
"""
parser = argparse.ArgumentParser(
description="Build STAC Catalog for sen1floods11")
parser.add_argument("--debug", action="store_true")
args = parser.parse_args()
debug = args.debug
storage = S3Storage("sen1floods11-data")
catalog_description = "Bonafilia, D., Tellman, B., Anderson, T., Issenberg, E. 2020. Sen1Floods11: a georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1. The IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) Workshops, 2020, pp. 210-211. Available Open access at: http://openaccess.thecvf.com/content_CVPRW_2020/html/w11/Bonafilia_Sen1Floods11_A_Georeferenced_Dataset_to_Train_and_Test_Deep_Learning_CVPRW_2020_paper.html" # noqa: E501
catalog_title = "A georeferenced dataset to train and test deep learning flood algorithms for Sentinel-1" # noqa: E501
catalog = Catalog("sen1floods11", catalog_description, title=catalog_title)
print("Created Catalog {}".format(catalog.id))
# Build Sentinel 1 Collection
sentinel1 = Collection(
"S1",
"Sentinel-1 GRD Chips overlapping labeled data. IW mode, GRD product. See https://developers.google.com/earth-engine/sentinel1 for information on preprocessing", # noqa: E501
extent=Extent(SpatialExtent([None, None, None, None]), None),
)
collection_add_sentinel_chips(sentinel1,
storage.ls("S1/"),
"s1",
debug=debug)
collection_add_sentinel_chips(sentinel1,
storage.ls("S1_NoQC/"),
"s1",
debug=debug)
collection_update_extents(sentinel1)
catalog.add_child(sentinel1)
# Build Sentinel 2 Collection
sentinel2 = Collection(
"S2",
"Sentinel-2 MSI L1C chips overlapping labeled data. Contains all spectral bands (1 - 12). Does not contain QA mask.", # noqa: E501
extent=Extent(SpatialExtent([None, None, None, None]), None),
)
collection_add_sentinel_chips(sentinel2,
storage.ls("S2/"),
"s2",
debug=debug)
collection_add_sentinel_chips(sentinel2,
storage.ls("S2_NoQC/"),
"s2",
debug=debug)
collection_update_extents(sentinel2)
catalog.add_child(sentinel2)
# Build S1 Weak Labels Collection
s1weak_labels = Collection(
"S1Flood_NoQC",
"Chips of water/nowater labels derived from standard OTSU thresholding of Sentinel-1 VH band overlapping weakly-labeled data.", # noqa: E501
extent=Extent(SpatialExtent([None, None, None, None]), None),
stac_extensions=[Extensions.LABEL],
)
label_collection_add_items(
s1weak_labels,
catalog,
storage.ls("S1Flood_NoQC/"),
sentinel1_links_func,
"0: Not Water. 1: Water.",
LabelType.RASTER,
label_classes=[LabelClasses([0, 1])],
label_tasks=["classification"],
debug=debug,
)
collection_update_extents(s1weak_labels)
catalog.add_child(s1weak_labels)
# Build S2 Weak Labels Collection
s2weak_labels = Collection(
"NoQC",
"Weakly-labeled chips derived from traditional Sentinel-2 Classification", # noqa: E501
extent=Extent(SpatialExtent([None, None, None, None]), None),
stac_extensions=[Extensions.LABEL],
)
label_collection_add_items(
s2weak_labels,
catalog,
storage.ls("NoQC/"),
sentinel2_links_func,
"-1: No Data / Not Valid. 0: Not Water. 1: Water.", # noqa: E501
LabelType.RASTER,
label_classes=[LabelClasses([-1, 0, 1])],
label_tasks=["classification"],
debug=debug,
)
collection_update_extents(s2weak_labels)
catalog.add_child(s2weak_labels)
# Build Hand Labels Collection
hand_labels = Collection(
"QC_v2",
"446 hand labeled chips of surface water from selected flood events",
extent=Extent(SpatialExtent([None, None, None, None]), None),
stac_extensions=[Extensions.LABEL],
)
label_collection_add_items(
hand_labels,
catalog,
storage.ls("QC_v2/"),
sentinel1_sentinel2_links_func,
"Hand labeled chips containing ground truth. -1: No Data / Not Valid. 0: Not Water. 1: Water.", # noqa: E501
LabelType.RASTER,
label_classes=[LabelClasses([-1, 0, 1])],
label_tasks=["classification"],
debug=debug,
)
collection_update_extents(hand_labels)
catalog.add_child(hand_labels)
# Build Permanent Labels collection
permanent_labels = Collection(
"Perm",
"Permanent water chips generated from the 'transition' layer of the JRC (European Commission Joint Research Centre) dataset", # noqa: E501
extent=Extent(SpatialExtent([None, None, None, None]), None),
stac_extensions=[Extensions.LABEL],
)
label_collection_add_items(
permanent_labels,
catalog,
storage.ls("Perm/"),
lambda *_: [
], # No easy way to map JRC source files to the label chips...
"0: Not Water. 1: Water.",
LabelType.RASTER,
label_classes=[LabelClasses([0, 1])],
label_tasks=["classification"],
debug=debug,
)
collection_update_extents(permanent_labels)
catalog.add_child(permanent_labels)
# Build Otsu algorithm Labels collection
otsu_labels = Collection(
"S1Flood",
"Chips of water/nowater derived from standard OTSU thresholding of Sentinel-1 VH band overlapping labeled data", # noqa: E501
extent=Extent(SpatialExtent([None, None, None, None]), None),
stac_extensions=[Extensions.LABEL],
)
label_collection_add_items(
otsu_labels,
catalog,
storage.ls("S1Flood/"),
sentinel1_links_func,
"0: Not Water. 1: Water.",
LabelType.RASTER,
label_classes=[LabelClasses([0, 1])],
label_tasks=["classification"],
debug=debug,
)
collection_update_extents(otsu_labels)
catalog.add_child(otsu_labels)
# Save Complete Catalog
root_path = "./catalog"
catalog.normalize_and_save(root_path,
catalog_type=CatalogType.SELF_CONTAINED)
print("Saved STAC Catalog {} to {}...".format(catalog.id, root_path))
def test_label_classes_typing(self) -> None:
classes: List[str] = ["foo", "bar"]
LabelClasses.create(classes=classes)