def main(): register_s3_io() parser = argparse.ArgumentParser() parser.add_argument( "--mldata-catalog", default= "s3://usfimr-s1-mldata/usfimr-s1-mldata-catalog_seed42/catalog.json", type=str, ) parser.add_argument( "--chip-label-dir", default="s3://jrc-fimr-rasterized-labels/version2", type=str, ) args = parser.parse_args() catalog = pystac.Catalog.from_file(args.mldata_catalog) chip_label_dir = args.chip_label_dir.rstrip("/") + "/" train = catalog.get_child("train") test = catalog.get_child("test") validation = catalog.get_child("validation") mldata_catalog = pystac.Catalog( "usfimr_jrc-s1-mldata-rasterized", "MLData STAC Catalog for usfimr+jrc labels of flood and permanent water with S1 imagery", ) label_items = [] train_collection = pystac.Collection("train", "Training collection", train.extent) for t in train.get_items(): train_collection.add_item(t) label_items.append(construct_label_item(t, chip_label_dir)) test_collection = pystac.Collection("test", "Test collection", test.extent) for t in test.get_items(): test_collection.add_item(t) label_items.append(construct_label_item(t, chip_label_dir)) val_collection = pystac.Collection("validation", "Validation collection", validation.extent) for v in validation.get_items(): val_collection.add_item(v) label_items.append(construct_label_item(v, chip_label_dir)) label_catalog = pystac.Catalog("usfimr_sar_labels_tif", "USFIMR + JRC labels for flood detection") for l in label_items: label_catalog.add_item(l) mldata_catalog.add_child(label_catalog) mldata_catalog.add_child(train_collection) mldata_catalog.add_child(test_collection) mldata_catalog.add_child(val_collection) mldata_catalog.normalize_and_save( "./data/catalog", catalog_type=pystac.CatalogType.SELF_CONTAINED, )
def save_catalog(dbname=DBNAME, catalog_type='SELF_CONTAINED', root_href=''): """ make a STAC catalog for all the data in the database, with collections organized by source, region, and year """ db = StackDB(dbname) summary = db.summary() if summary['Total Items'] < 1: print('No data to save') db.close() return # if we don't get a root reference, assume we will use the current directory if root_href == '': root_href = os.getcwd() # the master catalog catalog = pystac.Catalog('icecharts', 'Weekly Ice Charts from NIC and CIS', catalog_type=catalog_type) for source in summary['Sources']: print(source) sroot_href = '/'.join([root_href, source]) srccat = pystac.Catalog(source + '-icecharts', 'Weekly icecharts from ' + source, catalog_type=catalog_type) for region in summary[source + ' Regions']: print(region) rsroot_href = '/'.join([sroot_href, region]) rgncat = pystac.Catalog('-'.join([source, region, 'icecharts']).strip(), 'Weekly icecharts for ' + region, catalog_type=catalog_type) for yr in range( summary['{0} {1} Date Range'.format(source, region)][0], summary['{0} {1} Date Range'.format(source, region)][1] + 1): print(yr) yrsroot_href = '/'.join([rsroot_href, str(yr)]) collid = ''.join([source, region, str(yr), 'icecharts']).strip() coll = make_collection(db, source, region, str(yr), yrsroot_href, collid, 'Icecharts from ' + source) if coll: rgncat.add_child(coll, title=collid) srccat.add_child(rgncat) catalog.add_child(srccat) catalog.normalize_and_save(root_href, catalog_type=catalog_type) db.close()
def test_from_dict_set_root(self) -> None: path = TestCases.get_path("data-files/examples/hand-0.8.1/collection.json") with open(path) as f: collection_dict = json.load(f) catalog = pystac.Catalog(id="test", description="test desc") collection = Collection.from_dict(collection_dict, root=catalog) self.assertIs(collection.get_root(), catalog)
def test_produces_layout_for_catalog_with_filename(self) -> None: template = "cat/${id}/${description}/${id}.json" strategy = TemplateLayoutStrategy(catalog_template=template) cat = pystac.Catalog(id="test", description="test-desc") href = strategy.get_href(cat, parent_dir="http://example.com") self.assertEqual(href, "http://example.com/cat/test/test-desc/test.json")
def test_produces_layout_for_catalog(self) -> None: strategy = TemplateLayoutStrategy( catalog_template=self.TEST_CATALOG_TEMPLATE) cat = pystac.Catalog(id="test", description="test-desc") href = strategy.get_href(cat, parent_dir="http://example.com") self.assertEqual(href, "http://example.com/cat/test/test-desc/catalog.json")
async def test_fetches_valid_item(app_client, load_test_data: Callable, load_test_collection): coll = load_test_collection in_json = load_test_data("test_item.json") in_item = Item.parse_obj(in_json) resp = await app_client.post( "/collections/{coll.id}/items", json=in_json, ) assert resp.status_code == 200 post_item = Item.parse_obj(resp.json()) assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) resp = await app_client.get(f"/collections/{coll.id}/items/{post_item.id}") assert resp.status_code == 200 item_dict = resp.json() # Mock root to allow validation mock_root = pystac.Catalog(id="test", description="test desc", href="https://example.com") item = pystac.Item.from_dict(item_dict, preserve_dict=False, root=mock_root) item.validate()
def test_produces_layout_for_catalog(self) -> None: strategy = CustomLayoutStrategy( catalog_func=self.get_custom_catalog_func()) cat = pystac.Catalog(id="test", description="test desc") href = strategy.get_href(cat, parent_dir="http://example.com", is_root=True) self.assertEqual(href, "http://example.com/cat/True/test.json")
def test_link_does_not_fail_if_href_is_none(self) -> None: """Test to ensure get_href does not fail when the href is None.""" catalog = pystac.Catalog(id="test", description="test desc") catalog.add_item(self.item) catalog.set_self_href("/some/href") link = catalog.get_single_link("item") assert link is not None self.assertIsNone(link.get_href())
def test_link_does_not_fail_if_href_is_none(self): """Test to ensure get_href does not fail when the href is None.""" catalog = pystac.Catalog(id='test', description='test desc') catalog.add_item(self.item) catalog.set_self_href('/some/href') catalog.make_all_links_relative() link = catalog.get_single_link('item') self.assertIsNone(link.get_href())
def info(args): keys = list(get_resources(args.bucket, delimiter='/')) logger.debug('Querying boundaries for %d keys' % (len(keys))) queue = Process() catalog = pystac.Catalog('3dep', 'A catalog of USGS 3DEP Lidar hosted on AWS s3.', href=f'{args.stac_base_url}catalog.json', stac_extensions=['POINTCLOUD']) base = Path(args.stac_directory) base.mkdir(exist_ok=True, parents=True) count = 0 for k in keys: if count == args.limit and count != 0: break t = Task(args.bucket, k, args.resolution) queue.put(t) # logger.debug(t) count += 1 queue.do(count=20) l = Layer(args) stac_items = [] for r in queue.results: if not r.error: l.add(r) with open(base / f"{r.name}.json", 'w') as f: d = l.add_stac(r).to_dict() json.dump(d, f) link = pystac.Link('item', f'{args.stac_base_url}{r.name}.json') catalog.add_link(link) errors = [] for r in queue.results: if r.error: errors.append(r.error) f = open('errors.json','wb') f.write(json.dumps(errors).encode('utf-8')) f.close() with open(base / "catalog.json", 'w') as f: json.dump(catalog.to_dict(), f)
def create_catalog(suffix: Any, include_href: bool = True) -> pystac.Catalog: return pystac.Catalog( id="test {}".format(suffix), description="test desc {}".format(suffix), href=( "http://example.com/catalog_{}.json".format(suffix) if include_href else None ), )
def test_resolved_self_href(self) -> None: catalog = pystac.Catalog(id="test", description="test desc") with TemporaryDirectory() as temporary_directory: catalog.normalize_and_save(temporary_directory) path = os.path.join(temporary_directory, "catalog.json") catalog = pystac.Catalog.from_file(path) link = catalog.get_single_link(pystac.RelType.SELF) assert link link.resolve_stac_object() self.assertEqual(link.get_absolute_href(), path)
def test_produces_fallback_layout_for_catalog(self) -> None: fallback = BestPracticesLayoutStrategy() strategy = CustomLayoutStrategy( collection_func=self.get_custom_collection_func(), item_func=self.get_custom_item_func(), fallback_strategy=fallback, ) cat = pystac.Catalog(id="test", description="test desc") href = strategy.get_href(cat, parent_dir="http://example.com") expected = fallback.get_href(cat, parent_dir="http://example.com") self.assertEqual(href, expected)
def test_produces_fallback_layout_for_catalog(self) -> None: fallback = BestPracticesLayoutStrategy() strategy = TemplateLayoutStrategy( collection_template=self.TEST_COLLECTION_TEMPLATE, item_template=self.TEST_ITEM_TEMPLATE, fallback_strategy=fallback, ) cat = pystac.Catalog(id="test", description="test desc") href = strategy.get_href(cat, parent_dir="http://example.com") expected = fallback.get_href(cat, parent_dir="http://example.com") self.assertEqual(href, expected)
async def test_returns_valid_links_in_collections(app_client, load_test_data): """Test links from listing collections""" in_json = load_test_data("test_collection.json") resp = await app_client.post( "/collections", json=in_json, ) assert resp.status_code == 200 # Get collection by ID resp = await app_client.get(f"/collections/{in_json['id']}") assert resp.status_code == 200 resp_json = resp.json() # Mock root to allow validation mock_root = pystac.Catalog( id="test", description="test desc", href="https://example.com" ) collection = pystac.Collection.from_dict( resp_json, root=mock_root, preserve_dict=False ) assert collection.validate() # List collections resp = await app_client.get("/collections") assert resp.status_code == 200 resp_json = resp.json() collections = resp_json["collections"] # Find collection in list by ID single_coll = next(coll for coll in collections if coll["id"] == in_json["id"]) is_coll_from_list_valid = False single_coll_mocked_link = dict() if single_coll is not None: single_coll_mocked_link = pystac.Collection.from_dict( single_coll, root=mock_root, preserve_dict=False ) is_coll_from_list_valid = single_coll_mocked_link.validate() assert is_coll_from_list_valid # Check links from the collection GET and list assert [ i for i in collection.to_dict()["links"] if i not in single_coll_mocked_link.to_dict()["links"] ] == []
def main(): register_s3_io() parser = argparse.ArgumentParser() parser.add_argument( "--usfimr-collection", default="s3://usfimr-data/collection.json" ) parser.add_argument("--sar-catalog", required=True, type=str) parser.add_argument("--random-seed", default=42, type=int) args = parser.parse_args() usfimr_collection = pystac.Collection.from_file(args.usfimr_collection) sar_catalog = pystac.Catalog.from_file(args.sar_catalog) mldata_catalog = pystac.Catalog( "usfimr-s1-mldata", "MLData STAC Catalog for usfimr-s1 dataset" ) image_items, labels_collection = collect_items(sar_catalog, usfimr_collection) training, testing, validation = train_test_val_split(image_items, 0.2, 0.2, random_state=args.random_seed) train_collection = pystac.Collection( "train", "train", usfimr_collection.extent ) for t in training: train_collection.add_item(t) test_collection = pystac.Collection( "test", "test", usfimr_collection.extent ) for t in testing: test_collection.add_item(t) val_collection = pystac.Collection( "validation", "validation", usfimr_collection.extent ) for v in validation: val_collection.add_item(v) mldata_catalog.add_child(labels_collection) mldata_catalog.add_child(train_collection) mldata_catalog.add_child(test_collection) mldata_catalog.add_child(val_collection) mldata_catalog.normalize_and_save( "./data/mldata-catalog_seed{}".format(args.random_seed), catalog_type=pystac.CatalogType.SELF_CONTAINED )
def test_returns_valid_collection(app_client, load_test_data): """Test validates fetched collection with jsonschema""" test_collection = load_test_data("test_collection.json") resp = app_client.put("/collections", json=test_collection) assert resp.status_code == 200 resp = app_client.get(f"/collections/{test_collection['id']}") assert resp.status_code == 200 resp_json = resp.json() # Mock root to allow validation mock_root = pystac.Catalog(id="test", description="test desc", href="https://example.com") collection = pystac.Collection.from_dict(resp_json, root=mock_root, preserve_dict=False) collection.validate()
def generate_subcatalogs(self, template, defaults=None, **kwargs): """Walks through the catalog and generates subcatalogs for items based on the template string. See :class:`~pystac.layout.LayoutTemplate` for details on the construction of template strings. This template string will be applied to the items, and subcatalogs will be created that separate and organize the items based on template values. Args: template (str): A template string that can be consumed by a :class:`~pystac.layout.LayoutTemplate` defaults (dict): Default values for the template variables that will be used if the property cannot be found on the item. Returns: [catalog]: List of new catalogs created """ result = [] for child in self.get_children(): result.extend( child.generate_subcatalogs(template, defaults=defaults)) layout_template = LayoutTemplate(template, defaults=defaults) subcat_id_to_cat = {} items = list(self.get_items()) for item in items: item_parts = layout_template.get_template_values(item) curr_parent = self for k, v in item_parts.items(): subcat_id = '{}'.format(v) subcat = subcat_id_to_cat.get(subcat_id) if subcat is None: subcat_desc = 'Catalog of items from {} with {} of {}'.format( curr_parent.id, k, v) subcat = pystac.Catalog(id=subcat_id, description=subcat_desc) curr_parent.add_child(subcat) subcat_id_to_cat[subcat_id] = subcat result.append(subcat) curr_parent = subcat self.remove_item(item.id) curr_parent.add_item(item) return result
def test_returns_valid_item(app_client, load_test_data): """Test validates fetched item with jsonschema""" test_item = load_test_data("test_item.json") resp = app_client.post(f"/collections/{test_item['collection']}/items", json=test_item) assert resp.status_code == 200 get_item = app_client.get( f"/collections/{test_item['collection']}/items/{test_item['id']}") assert get_item.status_code == 200 item_dict = get_item.json() # Mock root to allow validation mock_root = pystac.Catalog(id="test", description="test desc", href="https://example.com") item = pystac.Item.from_dict(item_dict, preserve_dict=False, root=mock_root) item.validate()
def catalog(self): """Check if catalog exists and create it otherwise.""" if self.catalog_path is not None and self._catalog is None: if os.path.isfile(self.catalog_path): os.remove(self.catalog_path) if self.with_bbox: self._catalog = pystac.Collection(id="Sen2Like_catalog" if self.sid is None else self.sid, title="Sen2Like Catalog" if self.title is None else self.title, href=self.catalog_path, description="Catalog containing Sen2Like generated products", extent=pystac.Extent(pystac.SpatialExtent([180, -56, 180, 83]), pystac.TemporalExtent([None, None]))) else: self._catalog = pystac.Catalog(id="Sen2Like_catalog" if self.sid is None else self.sid, title="Sen2Like Catalog" if self.title is None else self.title, href=self.catalog_path, description="Catalog containing Sen2Like generated products") return self._catalog
async def test_returns_valid_collection(app_client, load_test_data): """Test updating a collection which already exists""" in_json = load_test_data("test_collection.json") resp = await app_client.post( "/collections", json=in_json, ) assert resp.status_code == 200 resp = await app_client.get(f"/collections/{in_json['id']}") assert resp.status_code == 200 resp_json = resp.json() # Mock root to allow validation mock_root = pystac.Catalog(id="test", description="test desc", href="https://example.com") collection = pystac.Collection.from_dict(resp_json, root=mock_root, preserve_dict=False) collection.validate()
def main(): df = AvirisClassic.as_df("aviris-flight-lines.csv") collection = pystac.Collection( AvirisClassic.COLLECTION_NAME, AVIRIS_DESCRIPTION, pystac.Extent( spatial=pystac.SpatialExtent([[None, None, None, None]]), temporal=pystac.TemporalExtent( [[datetime(1970, 1, 1, tzinfo=timezone.utc), None]]), ), ) stacframes.df_to(collection, df) df_ng = AvirisNg.as_df("aviris-ng-flight-lines.csv") collection_ng = pystac.Collection( AvirisNg.COLLECTION_NAME, AVIRIS_DESCRIPTION, pystac.Extent( spatial=pystac.SpatialExtent([[None, None, None, None]]), temporal=pystac.TemporalExtent( [[datetime(1970, 1, 1, tzinfo=timezone.utc), None]]), ), ) stacframes.df_to(collection_ng, df_ng) # Normalize before validation to set all the required object links catalog = pystac.Catalog("aviris", AVIRIS_DESCRIPTION) catalog.add_child(collection) catalog.add_child(collection_ng) catalog_path = "./data/catalog" catalog.normalize_hrefs(catalog_path) logger.info("Validating catalog...") catalog.validate_all() logger.info("Saving catalog to {}...".format(catalog_path)) catalog.save(catalog_type=pystac.CatalogType.SELF_CONTAINED) logger.info("Done!")
def test_from_dict_sets_root(self) -> None: param_dict = deepcopy(self.item_collection_dict) catalog = pystac.Catalog(id="test", description="test desc") item_collection = ItemCollection.from_dict(param_dict, root=catalog) for item in item_collection.items: self.assertEqual(item.get_root(), catalog)
def test_produces_layout_for_child_catalog(self) -> None: cat = pystac.Catalog(id="test", description="test desc") href = self.strategy.get_href(cat, parent_dir="http://example.com") self.assertEqual(href, "http://example.com/test/catalog.json")
def create_catalog(suffix, include_href=True): return pystac.Catalog( id='test {}'.format(suffix), description='test desc {}'.format(suffix), href=('http://example.com/catalog_{}.json'.format(suffix) if include_href else None))
def generate_subcatalogs( self, template: str, defaults: Optional[Dict[str, Any]] = None, parent_ids: Optional[List[str]] = None, ) -> List["Catalog"]: """Walks through the catalog and generates subcatalogs for items based on the template string. See :class:`~pystac.layout.LayoutTemplate` for details on the construction of template strings. This template string will be applied to the items, and subcatalogs will be created that separate and organize the items based on template values. Args: template : A template string that can be consumed by a :class:`~pystac.layout.LayoutTemplate` defaults : Default values for the template variables that will be used if the property cannot be found on the item. parent_ids : Optional list of the parent catalogs' identifiers. If the bottom-most subcatalogs already match the template, no subcatalog is added. Returns: [catalog]: List of new catalogs created """ result: List[Catalog] = [] parent_ids = parent_ids or list() parent_ids.append(self.id) for child in self.get_children(): result.extend( child.generate_subcatalogs( template, defaults=defaults, parent_ids=parent_ids.copy() ) ) layout_template = LayoutTemplate(template, defaults=defaults) keep_item_links: List[Link] = [] item_links = [lk for lk in self.links if lk.rel == pystac.RelType.ITEM] for link in item_links: link.resolve_stac_object(root=self.get_root()) item = cast(pystac.Item, link.target) subcat_ids = layout_template.substitute(item).split("/") id_iter = reversed(parent_ids) if all( ["{}".format(id) == next(id_iter, None) for id in reversed(subcat_ids)] ): # Skip items for which the sub-catalog structure already # matches the template. The list of parent IDs can include more # elements on the root side, so compare the reversed sequences. keep_item_links.append(link) continue curr_parent = self for subcat_id in subcat_ids: subcat = curr_parent.get_child(subcat_id) if subcat is None: subcat_desc = "Catalog of items from {} with id {}".format( curr_parent.id, subcat_id ) subcat = pystac.Catalog(id=subcat_id, description=subcat_desc) curr_parent.add_child(subcat) result.append(subcat) curr_parent = subcat # resolve collection link so when added back points to correct location col_link = item.get_single_link(pystac.RelType.COLLECTION) if col_link is not None: col_link.resolve_stac_object() curr_parent.add_item(item) # keep only non-item links and item links that have not been moved elsewhere self.links = [ lk for lk in self.links if lk.rel != pystac.RelType.ITEM ] + keep_item_links return result
def main(): """ Pull Copernicus EU Rapid Mapping Activations data from the GeoRSS feed """ sentinel_oauth_id = os.environ.get("SENTINELHUB_OAUTH_ID") sentinel_oauth_secret = os.environ.get("SENTINELHUB_OAUTH_SECRET") if sentinel_oauth_id is None: raise ValueError("Must set SENTINELHUB_OAUTH_ID") if sentinel_oauth_secret is None: raise ValueError("Must set SENTINELHUB_OAUTH_SECRET") events_xml_url = "https://emergency.copernicus.eu/mapping/activations-rapid/feed" events_xml_file = Path("./data/copernicus-rapid-mapping-activations.xml") if not events_xml_file.is_file(): logger.info("Pulling {}...".format(events_xml_url)) urlretrieve(events_xml_url, str(events_xml_file)) event_xml_dir = Path("./data/event-xml") os.makedirs(event_xml_dir, exist_ok=True) # Generate a list of all unique CEMS products (combination of event, aoi, # monitoring type, revision and version) for all flood events in 2019 and 2020 products = [] events_root = ET.parse(events_xml_file).getroot() for event in events_root.iter("item"): category = event.find("category").text.strip().lower() if category != "flood": continue event_id = event.find("guid").text title = event.find("title").text rss_url = event.find("{http://www.iwg-sem.org/}activationRSS").text logger.info(title) description = event.find("description").text event_dts = re.findall( r"Date\/Time of Event \(UTC\):[</b>\s]*?(\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{2}:\d{2})", description, flags=re.MULTILINE, ) if len(event_dts) != 1: logger.warning("{}: Available event date times {}".format( title, event_dts)) raise AssertionError() event_datetime = datetime.strptime( event_dts[0], "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) if event_datetime < datetime(2019, 1, 1, 0, 0, 0, tzinfo=timezone.utc): continue event_country = event.find( "{http://www.iwg-sem.org/}activationAffectedCountries").text event_xml_file = Path(event_xml_dir, event_id).with_suffix(".xml") if not event_xml_file.is_file(): logger.info("\tPulling {} GeoRSS: {}...".format( event_id, event_xml_file)) urlretrieve(rss_url, event_xml_file) event_root = ET.parse(event_xml_file).getroot() for item in event_root.iter("item"): try: data_type = item.find("{http://www.gdacs.org/}cemsctype").text except AttributeError: data_type = "" try: product_type = item.find( "{http://www.gdacs.org/}cemsptype").text except AttributeError: product_type = "" # Only care about downloading VECTOR data for Delineation product # More info at https://emergency.copernicus.eu/mapping/ems/rapid-mapping-portfolio if not (data_type == "VECTOR" and (product_type == "DEL" or product_type == "GRA")): continue item_url = urlparse(item.find("link").text) _, _, product_id, version_id = item_url.path.lstrip("/").split("/") ( product_event_id, aoi_id, product_type_id, monitoring_type, revision_id, data_type_id, ) = product_id.split("_") # Some sanity checks to ensure we've parsed our product id string correctly assert event_id == product_event_id assert product_type_id == product_type assert data_type_id == "VECTORS" georss_polygon = item.find( "{http://www.georss.org/georss}polygon").text # Split string, group number pairs, convert to float and swap pairs to lon first polygon = Polygon( map( lambda x: (float(x[1]), float(x[0])), grouper(georss_polygon.split(" "), 2), )) event_product = EventProduct( # Rebuild product_id from scratch because we need to include version "_".join([ event_id, aoi_id, product_type_id, monitoring_type, revision_id, version_id, data_type_id, ]), event_id, event_country, aoi_id, event_datetime.timestamp(), polygon, data_type_id, product_type_id, monitoring_type, revision_id, version_id, urlunparse(item_url), ) products.append(event_product) df = gpd.GeoDataFrame(products) geojson_file = "./data/cems-rapid-mapping-flood-products-2019-2020.geojson" logger.info( "Writing GeoJSON of flood event products to {}".format(geojson_file)) df.to_file(geojson_file, driver="GeoJSON") sentinel_session = get_session(sentinel_oauth_id, sentinel_oauth_secret) catalog = pystac.Catalog( "copernicus-rapid-mapping-floods-2019-2020", "Copernicus Rapid Mapping provisions geospatial information within hours or days from the activation in support of emergency management activities immediately following a disaster. Standardised mapping products are provided: e.g. to ascertain the situation before the event (reference product), to roughly identify and assess the most affected locations (first estimate product), assess the geographical extent of the event (delineation product) or to evaluate the intensity and scope of the damage resulting from the event (grading product). This catalog contains a subset of products for flood events from 2019-2020 that intersect with Sentinel 2 L2A Chips.", title="Copernicus Rapid Mapping Floods 2019-2020", ) s2_collection = pystac.Collection( "Sentinel-2-L2A", "Sentinel 2 L2A images corresponding to CEMS rapid mapping floods", pystac.Extent( pystac.SpatialExtent([None, None, None, None]), pystac.TemporalExtent([( # TODO: Make this more specific by looping actual dts # after ingest datetime(2019, 1, 1, 0, 0, 0, tzinfo=timezone.utc), datetime(2020, 12, 31, 23, 59, 59, tzinfo=timezone.utc), )]), ), ) catalog.add_child(s2_collection) # Loop Products grouped by event id, lookup Sentinel 2 matches for each # Product, and create STAC Items in catalog for any matches sorted_products = sorted(products, key=lambda x: x.event_id) for event_id, event_products in groupby(sorted_products, key=lambda x: x.event_id): for p in event_products: event_datetime = datetime.fromtimestamp(p.event_time, tz=timezone.utc) # Check for sentinel 2 results before anything else, so we # don't do unnecessary work. We'll use these results later # after we've created our STAC Item response = stac_search( p.geometry.bounds, "sentinel-2-l2a", event_datetime - timedelta(hours=12), event_datetime + timedelta(hours=12), sentinel_session, ).json() if len(response["features"]) < 1: logger.debug("No Sentinel 2 results for {}".format( p.product_id)) continue event_collection = catalog.get_child(event_id) if event_collection is None: event_collection = pystac.Collection( event_id, "", pystac.Extent( pystac.SpatialExtent([None, None, None, None]), pystac.TemporalExtent([(event_datetime, None)]), ), ) catalog.add_child(event_collection) pystac_item = pystac.Item( p.product_id, mapping(p.geometry), p.geometry.bounds, event_datetime, properties={ "aoi_id": p.aoi_id, "country": p.event_country, "event_id": p.event_id, "product_type": p.product_type, "data_type": p.data_type, "monitoring_type": p.monitoring_type, "revision": p.revision, "version": p.version, }, ) event_collection.add_item(pystac_item) url_link = pystac.Link("alternate", p.product_link, media_type="text/html") pystac_item.add_link(url_link) # Get or create Item in S2 collection for each match from # SentinelHub and add as links to our Product Item for feature in response["features"]: s2_item = s2_collection.get_item(feature["id"]) if s2_item is None: s2_item = pystac.Item.from_dict(feature) s2_collection.add_item(s2_item) s2_link = pystac.Link( "data", s2_item, link_type=pystac.LinkType.RELATIVE).set_owner(pystac_item) pystac_item.add_link(s2_link) logger.info("Created STAC Item {} with {} Sentinel 2 links".format( p.product_id, len(response["features"]))) # Set spatial extents for collection in catalog.get_children(): if not isinstance(collection, pystac.Collection): continue bounds = GeometryCollection( [shape(s.geometry) for s in collection.get_all_items()]).bounds collection.extent.spatial = pystac.SpatialExtent(bounds) catalog_root = "./data/catalog" logger.info("Writing STAC Catalog to {}...".format(catalog_root)) catalog.normalize_and_save(catalog_root, pystac.CatalogType.SELF_CONTAINED)
def test_from_dict_set_root(self) -> None: item_dict = self.get_example_item_dict() catalog = pystac.Catalog(id="test", description="test desc") item = Item.from_dict(item_dict, root=catalog) self.assertIs(item.get_root(), catalog)