def test_landsat_c2_l1(self, loader: Loader) -> None:
"""
Test that a dehydrated item is created properly from a raw item against a
base item from a collection
"""
with open(LANDSAT_COLLECTION) as f:
collection = json.load(f)
loader.load_collections(str(LANDSAT_COLLECTION))
with open(LANDSAT_ITEM) as f:
item = json.load(f)
base_item = cast(
Dict[str, Any],
loader.db.query_one(
"SELECT base_item FROM collections WHERE id=%s;",
(collection["id"], )),
)
assert type(base_item) == dict
dehydrated = self.dehydrate(base_item, item)
# Expect certain keys on base and not on dehydrated
only_base_keys = ["type", "collection", "stac_version"]
assert all(k in base_item for k in only_base_keys)
assert not any(k in dehydrated for k in only_base_keys)
# Expect certain keys on dehydrated and not on base
only_dehydrated_keys = ["id", "bbox", "geometry", "properties"]
assert not any(k in base_item for k in only_dehydrated_keys)
assert all(k in dehydrated for k in only_dehydrated_keys)
# Properties, links should be exactly the same pre- and post-dehydration
assert item["properties"] == dehydrated["properties"]
assert item["links"] == dehydrated["links"]
# Check specific assets are dehydrated correctly
thumbnail = dehydrated["assets"]["thumbnail"]
assert list(thumbnail.keys()) == ["href"]
assert thumbnail["href"] == item["assets"]["thumbnail"]["href"]
# Red asset raster bands have additional `scale` and `offset` keys
red = dehydrated["assets"]["red"]
assert list(red.keys()) == ["href", "eo:bands", "raster:bands"]
assert len(red["raster:bands"]) == 1
assert list(red["raster:bands"][0].keys()) == ["scale", "offset"]
item_red_rb = item["assets"]["red"]["raster:bands"][0]
assert red["raster:bands"] == [{
"scale": item_red_rb["scale"],
"offset": item_red_rb["offset"]
}]
# nir09 asset raster bands does not have a `unit` attribute, which is
# present on base
nir09 = dehydrated["assets"]["nir09"]
assert list(nir09.keys()) == ["href", "eo:bands", "raster:bands"]
assert len(nir09["raster:bands"]) == 1
assert list(nir09["raster:bands"][0].keys()) == ["unit"]
assert nir09["raster:bands"] == [{"unit": DO_NOT_MERGE_MARKER}]
def test_landsat_c2_l1(self, loader: Loader) -> None:
"""Test that a dehydrated item is is equal to the raw item it was dehydrated
from, against the base item of the collection"""
with open(LANDSAT_COLLECTION) as f:
collection = json.load(f)
loader.load_collections(str(LANDSAT_COLLECTION))
with open(LANDSAT_DEHYDRATED_ITEM) as f:
dehydrated = json.load(f)
with open(LANDSAT_ITEM) as f:
raw_item = json.load(f)
base_item = cast(
Dict[str, Any],
loader.db.query_one(
"SELECT base_item FROM collections WHERE id=%s;",
(collection["id"], ),
),
)
assert type(base_item) == dict
hydrated = self.hydrate(base_item, dehydrated)
assert hydrated == raw_item
def test_format_items_keys(loader: Loader) -> None:
"""Test pypgstac items ignore loader."""
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.ignore,
)
items_iter = read_json(str(TEST_ITEMS))
item_json = next(iter(items_iter))
out = loader.format_item(item_json)
# Top level keys expected after format
assert "id" in out
assert "collection" in out
assert "geometry" in out
assert "content" in out
# Special keys expected not to be in the item content
content_json = json.loads(out["content"])
assert "id" not in content_json
assert "collection" not in content_json
assert "geometry" not in content_json
# Ensure bbox is included in content
assert "bbox" in content_json
def test_load_collections_json_duplicates_with_ignore(loader: Loader) -> None:
"""Test pypgstac collections loader."""
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.insert,
)
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.ignore,
)
def test_load_items_incompatible_version(loader: Loader) -> None:
"""Test pypgstac items loader raises an exception for incompatible version."""
with mock.patch("pypgstac.db.PgstacDB.version",
new_callable=mock.PropertyMock) as mock_version:
mock_version.return_value = "dummy"
with pytest.raises(Exception):
loader.load_items(
str(TEST_ITEMS),
insert_mode=Methods.insert,
)
def test_load_items_succeeds(loader: Loader) -> None:
"""Test pypgstac items loader."""
loader.load_collections(
str(TEST_COLLECTIONS),
insert_mode=Methods.upsert,
)
loader.load_items(
str(TEST_ITEMS),
insert_mode=Methods.insert,
)
def test_load_collections_json_duplicates_fails(loader: Loader) -> None:
"""Test pypgstac collections loader."""
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.insert,
)
with pytest.raises(UniqueViolation):
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.insert,
)
def loader(db: PgstacDB) -> Generator:
"""Fixture to get a loader and an empty pgstac."""
db.query("DROP SCHEMA IF EXISTS pgstac CASCADE;")
migrator = Migrate(db)
print(migrator.run_migration())
ldr = Loader(db)
yield ldr
def test_landsat_c2_l1(loader: Loader) -> None:
"""Test that a base item is created when a collection is loaded and that it
is equal to the item_assets of the collection"""
with open(LANDSAT_COLLECTION) as f:
collection = json.load(f)
loader.load_collections(str(LANDSAT_COLLECTION))
base_item = cast(
Dict[str, Any],
loader.db.query_one("SELECT base_item FROM collections WHERE id=%s;",
(collection["id"], )),
)
assert type(base_item) == dict
assert base_item["collection"] == collection["id"]
assert base_item["assets"] == collection["item_assets"]
def test_partition_loads_default(loader: Loader) -> None:
"""Test pypgstac items ignore loader."""
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.ignore,
)
loader.load_items(
str(TEST_ITEMS),
insert_mode=Methods.insert,
)
partitions = loader.db.query_one("""
SELECT count(*) from partitions;
""")
assert partitions == 1
def test_load_dehydrated(loader: Loader) -> None:
"""Test loader for items dumped directly out of item table."""
collections = [
HERE / "data-files" / "hydration" / "collections" /
"chloris-biomass.json",
]
for collection in collections:
loader.load_collections(
str(collection),
insert_mode=Methods.ignore,
)
dehydrated_items = HERE / "data-files" / "load" / "dehydrated.txt"
loader.load_items(str(dehydrated_items),
insert_mode=Methods.insert,
dehydrated=True)
def test_s1_grd_load_and_query(loader: Loader) -> None:
"""Test pypgstac items ignore loader."""
loader.load_collections(
str(S1_GRD_COLLECTION),
insert_mode=Methods.ignore,
)
loader.load_items(str(S1_GRD_ITEM), insert_mode=Methods.insert)
search_body = {
"filter-lang": "cql2-json",
"filter": {
"op":
"and",
"args": [
{
"op": "=",
"args": [{
"property": "collection"
}, "sentinel-1-grd"],
},
{
"op":
"=",
"args": [
{
"property": "id"
},
"S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C", # noqa: E501
],
},
],
},
}
res = next(loader.db.func(
"search",
search_body,
))[0]
item = res["features"][0]
pystac.Item.from_dict(item).validate()
def test_partition_loads_year(loader: Loader) -> None:
"""Test pypgstac items ignore loader."""
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.ignore,
)
if loader.db.connection is not None:
loader.db.connection.execute("""
UPDATE collections SET partition_trunc='year';
""")
loader.load_items(
str(TEST_ITEMS),
insert_mode=Methods.insert,
)
partitions = loader.db.query_one("""
SELECT count(*) from partitions;
""")
assert partitions == 1
def load(
self,
table: Tables,
file: str,
method: Optional[Methods] = Methods.insert,
dehydrated: Optional[bool] = False,
chunksize: Optional[int] = 10000,
) -> None:
"""Load collections or items into PGStac."""
loader = Loader(db=self._db)
if table == "collections":
loader.load_collections(file, method)
if table == "items":
loader.load_items(file, method, dehydrated, chunksize)
def test_load_items_dehydrated_ignore_succeeds(loader: Loader) -> None:
"""Test pypgstac items ignore loader."""
loader.load_collections(
str(TEST_COLLECTIONS),
insert_mode=Methods.ignore,
)
loader.load_items(str(TEST_DEHYDRATED_ITEMS),
insert_mode=Methods.insert,
dehydrated=True)
loader.load_items(str(TEST_DEHYDRATED_ITEMS),
insert_mode=Methods.ignore,
dehydrated=True)
def test_load_collections_json_succeeds(loader: Loader) -> None:
"""Test pypgstac collections loader."""
loader.load_collections(
str(TEST_COLLECTIONS_JSON),
insert_mode=Methods.insert,
)