def test_null_datetime(self) -> None:
item = pystac.Item.from_file(
TestCases.get_path("data-files/item/sample-item.json"))
with self.assertRaises(pystac.STACError):
Item(
"test",
geometry=item.geometry,
bbox=item.bbox,
datetime=None,
properties={},
)
null_dt_item = Item(
"test",
geometry=item.geometry,
bbox=item.bbox,
datetime=None,
properties={
"start_datetime": datetime_to_str(get_opt(item.datetime)),
"end_datetime": datetime_to_str(get_opt(item.datetime)),
},
)
null_dt_item.validate()
def anx_datetime(self, v: Optional[RangeSummary[Datetime]]) -> None:
self._set_summary(
ANX_DATETIME_PROP,
map_opt(
lambda s: RangeSummary(datetime_to_str(s.minimum),
datetime_to_str(s.maximum)),
v,
),
)
def test_update_extents(self) -> None:
catalog = TestCases.test_case_2()
base_collection = catalog.get_child(
"1a8c1632-fa91-4a62-b33e-3a87c2ebdf16")
assert isinstance(base_collection, Collection)
base_extent = base_collection.extent
collection = base_collection.clone()
item1 = Item(
id="test-item-1",
geometry=ARBITRARY_GEOM,
bbox=[-180, -90, 180, 90],
datetime=TEST_DATETIME,
properties={"key": "one"},
stac_extensions=["eo", "commons"],
)
item2 = Item(
id="test-item-1",
geometry=ARBITRARY_GEOM,
bbox=[-180, -90, 180, 90],
datetime=None,
properties={
"start_datetime":
datetime_to_str(datetime(2000, 1, 1, 12, 0, 0, 0)),
"end_datetime":
datetime_to_str(datetime(2000, 2, 1, 12, 0, 0, 0)),
},
stac_extensions=["eo", "commons"],
)
collection.add_item(item1)
collection.update_extent_from_items()
self.assertEqual([[-180, -90, 180, 90]],
collection.extent.spatial.bboxes)
self.assertEqual(len(base_extent.spatial.bboxes[0]),
len(collection.extent.spatial.bboxes[0]))
self.assertNotEqual(base_extent.temporal.intervals,
collection.extent.temporal.intervals)
collection.remove_item("test-item-1")
collection.update_extent_from_items()
self.assertNotEqual([[-180, -90, 180, 90]],
collection.extent.spatial.bboxes)
collection.add_item(item2)
collection.update_extent_from_items()
self.assertEqual(
[[
item2.common_metadata.start_datetime,
base_extent.temporal.intervals[0][1],
]],
collection.extent.temporal.intervals,
)
def unpublished(self, v: Optional[RangeSummary[datetime]]) -> None:
self._set_summary(
UNPUBLISHED_PROP,
map_opt(
lambda s: RangeSummary(datetime_to_str(s.minimum),
datetime_to_str(s.maximum)),
v,
),
)
def expires(self, v: Optional[RangeSummary[datetime]]) -> None:
self._set_summary(
EXPIRES_PROP,
map_opt(
lambda s: RangeSummary(datetime_to_str(s.minimum),
datetime_to_str(s.maximum)),
v,
),
)
def set_created(self, created, asset=None):
"""Set an Item or an Asset created time.
If an Asset is supplied, sets the property on the Asset.
Otherwise sets the Item's value.
"""
if asset is None:
self.properties['created'] = datetime_to_str(created)
else:
asset.properties['created'] = datetime_to_str(created)
def set_end_datetime(self, end_datetime, asset=None):
"""Set an Item or an Asset end_datetime.
If an Asset is supplied, sets the property on the Asset.
Otherwise sets the Item's value.
"""
if asset is None:
self.properties['end_datetime'] = datetime_to_str(end_datetime)
else:
asset.properties['end_datetime'] = datetime_to_str(end_datetime)
def test_from_items(self) -> None:
item1 = Item(
id="test-item-1",
geometry=ARBITRARY_GEOM,
bbox=[-10, -20, 0, -10],
datetime=datetime(2000, 2, 1, 12, 0, 0, 0, tzinfo=tz.UTC),
properties={},
)
item2 = Item(
id="test-item-2",
geometry=ARBITRARY_GEOM,
bbox=[0, -9, 10, 1],
datetime=None,
properties={
"start_datetime":
datetime_to_str(
datetime(2000, 1, 1, 12, 0, 0, 0, tzinfo=tz.UTC)),
"end_datetime":
datetime_to_str(
datetime(2000, 7, 1, 12, 0, 0, 0, tzinfo=tz.UTC)),
},
)
item3 = Item(
id="test-item-2",
geometry=ARBITRARY_GEOM,
bbox=[-5, -20, 5, 0],
datetime=None,
properties={
"start_datetime":
datetime_to_str(
datetime(2000, 12, 1, 12, 0, 0, 0, tzinfo=tz.UTC)),
"end_datetime":
datetime_to_str(
datetime(2001, 1, 1, 12, 0, 0, 0, tzinfo=tz.UTC), ),
},
)
extent = Extent.from_items([item1, item2, item3])
self.assertEqual(len(extent.spatial.bboxes), 1)
self.assertEqual(extent.spatial.bboxes[0], [-10, -20, 10, 1])
self.assertEqual(len(extent.temporal.intervals), 1)
interval = extent.temporal.intervals[0]
self.assertEqual(interval[0],
datetime(2000, 1, 1, 12, 0, 0, 0, tzinfo=tz.UTC))
self.assertEqual(interval[1],
datetime(2001, 1, 1, 12, 0, 0, 0, tzinfo=tz.UTC))
def stac(
input,
input_datetime,
extension,
collection,
property,
id,
asset_name,
asset_href,
asset_mediatype,
output,
):
"""Rasterio stac cli."""
property = property or {}
if not input_datetime:
input_datetime = datetime.datetime.utcnow()
else:
if "/" in input_datetime:
start_datetime, end_datetime = input_datetime.split("/")
property["start_datetime"] = datetime_to_str(
str_to_datetime(start_datetime))
property["end_datetime"] = datetime_to_str(
str_to_datetime(end_datetime))
input_datetime = None
else:
input_datetime = str_to_datetime(input_datetime)
if asset_mediatype and asset_mediatype != "auto":
asset_mediatype = MediaType[asset_mediatype]
extensions = [e for e in extension if e]
item = create_stac_item(
input,
input_datetime=input_datetime,
extensions=extensions,
collection=collection,
properties=property,
id=id,
asset_name=asset_name,
asset_href=asset_href,
asset_media_type=asset_mediatype,
)
if output:
with open(output, "w") as f:
f.write(json.dumps(item.to_dict(), separators=(",", ":")))
else:
click.echo(json.dumps(item.to_dict(), separators=(",", ":")))
def test_update_extents(self):
catalog = TestCases.test_case_2()
base_collection = catalog.get_child(
'1a8c1632-fa91-4a62-b33e-3a87c2ebdf16')
base_extent = base_collection.extent
collection = base_collection.clone()
item1 = Item(id='test-item-1',
geometry=RANDOM_GEOM,
bbox=[-180, -90, 180, 90],
datetime=TEST_DATETIME,
properties={'key': 'one'},
stac_extensions=['eo', 'commons'])
item2 = Item(id='test-item-1',
geometry=RANDOM_GEOM,
bbox=[-180, -90, 180, 90],
datetime=None,
properties={
'start_datetime':
datetime_to_str(datetime(2000, 1, 1, 12, 0, 0, 0)),
'end_datetime':
datetime_to_str(datetime(2000, 2, 1, 12, 0, 0, 0))
},
stac_extensions=['eo', 'commons'])
collection.add_item(item1)
collection.update_extent_from_items()
self.assertEqual([[-180, -90, 180, 90]],
collection.extent.spatial.bboxes)
self.assertEqual(len(base_extent.spatial.bboxes[0]),
len(collection.extent.spatial.bboxes[0]))
self.assertNotEqual(base_extent.temporal.intervals,
collection.extent.temporal.intervals)
collection.remove_item('test-item-1')
collection.update_extent_from_items()
self.assertNotEqual([[-180, -90, 180, 90]],
collection.extent.spatial.bboxes)
collection.add_item(item2)
collection.update_extent_from_items()
self.assertEqual([[
item2.common_metadata.start_datetime,
base_extent.temporal.intervals[0][1]
]], collection.extent.temporal.intervals)
def to_dict(self, include_self_link=True):
links = self.links
if not include_self_link:
links = filter(lambda x: x.rel != 'self', links)
assets = dict(map(lambda x: (x[0], x[1].to_dict()), self.assets.items()))
self.properties['datetime'] = datetime_to_str(self.datetime)
d = {
'type': 'Feature',
'stac_version': STAC_VERSION,
'id': self.id,
'properties': self.properties,
'geometry': self.geometry,
'bbox': self.bbox,
'links': [l.to_dict() for l in links],
'assets': assets
}
if self.stac_extensions is not None:
d['stac_extensions'] = self.stac_extensions
if self.collection_id:
d['collection'] = self.collection_id
return deepcopy(d)
def to_dict(self, include_self_link=True):
links = self.links
if not include_self_link:
links = filter(lambda x: x.rel != 'self', links)
assets = dict(map(lambda x: (x[0], x[1].to_dict()), self.assets.items()))
if self.datetime is not None:
self.properties['datetime'] = datetime_to_str(self.datetime)
else:
self.properties['datetime'] = None
d = {
'type': 'Feature',
'stac_version': pystac.get_stac_version(),
'id': self.id,
'properties': self.properties,
'geometry': self.geometry,
'links': [link.to_dict() for link in links],
'assets': assets
}
if self.bbox is not None:
d['bbox'] = self.bbox
if self.stac_extensions is not None:
d['stac_extensions'] = self.stac_extensions
if self.collection_id:
d['collection'] = self.collection_id
for key in self.extra_fields:
d[key] = self.extra_fields[key]
return deepcopy(d)
def test_from_items(self):
item1 = Item(id='test-item-1',
geometry=RANDOM_GEOM,
bbox=[-10, -20, 0, -10],
datetime=datetime(2000, 2, 1, 12, 0, 0, 0, tzinfo=tz.UTC),
properties={})
item2 = Item(id='test-item-2',
geometry=RANDOM_GEOM,
bbox=[0, -9, 10, 1],
datetime=None,
properties={
'start_datetime':
datetime_to_str(
datetime(2000, 1, 1, 12, 0, 0, 0, tzinfo=tz.UTC)),
'end_datetime':
datetime_to_str(
datetime(2000, 7, 1, 12, 0, 0, 0, tzinfo=tz.UTC))
})
item3 = Item(id='test-item-2',
geometry=RANDOM_GEOM,
bbox=[-5, -20, 5, 0],
datetime=None,
properties={
'start_datetime':
datetime_to_str(
datetime(2000, 12, 1, 12, 0, 0, 0,
tzinfo=tz.UTC)),
'end_datetime':
datetime_to_str(
datetime(2001, 1, 1, 12, 0, 0, 0,
tzinfo=tz.UTC), )
})
extent = Extent.from_items([item1, item2, item3])
self.assertEqual(len(extent.spatial.bboxes), 1)
self.assertEqual(extent.spatial.bboxes[0], [-10, -20, 10, 1])
self.assertEqual(len(extent.temporal.intervals), 1)
interval = extent.temporal.intervals[0]
self.assertEqual(interval[0],
datetime(2000, 1, 1, 12, 0, 0, 0, tzinfo=tz.UTC))
self.assertEqual(interval[1],
datetime(2001, 1, 1, 12, 0, 0, 0, tzinfo=tz.UTC))
def _timestamp_setter(self, timestamp, key, asset=None):
if timestamp is None:
self.item.properties[key] = timestamp
else:
timestamp_str = datetime_to_str(timestamp)
if asset is not None:
asset.properties[key] = timestamp_str
else:
self.item.properties[key] = timestamp_str
def set_datetime(self,
datetime: Datetime,
asset: Optional[Asset] = None) -> None:
"""Set an Item or an Asset datetime.
If an Asset is supplied, sets the property on the Asset.
Otherwise sets the Item's value.
"""
if asset is None:
self.datetime = datetime
else:
asset.extra_fields["datetime"] = datetime_to_str(datetime)
def _convert_value_to_stac_type(key: str, value):
"""
Convert return type as per STAC specification
"""
# In STAC spec, "instruments" have [String] type
if key == "eo:instrument":
return _as_stac_instruments(value)
# Convert the non-default datetimes to a string
elif isinstance(value, datetime.datetime) and key != "datetime":
return datetime_to_str(value)
else:
return value
def to_dict(self):
"""Generate a dictionary representing the JSON of this TemporalExtent.
Returns:
dict: A serializion of the TemporalExtent that can be written out as JSON.
"""
encoded_intervals = []
for i in self.intervals:
start = None
end = None
if i[0]:
start = datetime_to_str(i[0])
if i[1]:
end = datetime_to_str(i[1])
encoded_intervals.append([start, end])
d = {'interval': encoded_intervals}
return deepcopy(d)
def to_dict(self) -> Dict[str, Any]:
"""Generate a dictionary representing the JSON of this TemporalExtent.
Returns:
dict: A serialization of the TemporalExtent that can be written out as JSON.
"""
encoded_intervals: List[List[Optional[str]]] = []
for i in self.intervals:
start = None
end = None
if i[0] is not None:
start = datetime_to_str(i[0])
if i[1] is not None:
end = datetime_to_str(i[1])
encoded_intervals.append([start, end])
d = {"interval": encoded_intervals, **self.extra_fields}
return d
def test_datetime_to_str(self):
cases = (
('timezone naive, assume utc', datetime(2000, 1, 1), '2000-01-01T00:00:00Z'),
('timezone aware, utc', datetime(2000, 1, 1,
tzinfo=timezone.utc), '2000-01-01T00:00:00Z'),
('timezone aware, utc -7', datetime(2000, 1, 1, tzinfo=timezone(timedelta(hours=-7))),
'2000-01-01T00:00:00-07:00'),
)
for title, dt, expected in cases:
with self.subTest(title=title):
got = utils.datetime_to_str(dt)
self.assertEqual(expected, got)
def test_anx_datetime(self) -> None:
collection = self.collection()
summaries_ext = SatExtension.summaries(collection, True)
anx_datetime_range = RangeSummary(
str_to_datetime("2020-01-01T00:00:00.000Z"),
str_to_datetime("2020-01-02T00:00:00.000Z"),
)
summaries_ext.anx_datetime = anx_datetime_range
self.assertEqual(
summaries_ext.anx_datetime,
anx_datetime_range,
)
summaries_dict = collection.to_dict()["summaries"]
self.assertDictEqual(
summaries_dict["sat:anx_datetime"],
{
"minimum": datetime_to_str(anx_datetime_range.minimum),
"maximum": datetime_to_str(anx_datetime_range.maximum),
},
)
def test_unpublished(self) -> None:
collection = self.collection()
summaries_ext = TimestampsExtension.summaries(collection, True)
unpublished_range = RangeSummary(
str_to_datetime("2020-01-01T00:00:00.000Z"),
str_to_datetime("2020-01-02T00:00:00.000Z"),
)
summaries_ext.unpublished = unpublished_range
self.assertEqual(
summaries_ext.unpublished,
unpublished_range,
)
summaries_dict = collection.to_dict()["summaries"]
self.assertDictEqual(
summaries_dict["unpublished"],
{
"minimum": datetime_to_str(unpublished_range.minimum),
"maximum": datetime_to_str(unpublished_range.maximum),
},
)
def to_dict(self,
include_self_link: bool = True,
transform_hrefs: bool = True) -> Dict[str, Any]:
links = self.links
if not include_self_link:
links = [x for x in links if x.rel != pystac.RelType.SELF]
assets = {k: v.to_dict() for k, v in self.assets.items()}
if self.datetime is not None:
self.properties["datetime"] = datetime_to_str(self.datetime)
else:
self.properties["datetime"] = None
d: Dict[str, Any] = {
"type":
"Feature",
"stac_version":
pystac.get_stac_version(),
"id":
self.id,
"properties":
self.properties,
"geometry":
self.geometry,
"links":
[link.to_dict(transform_href=transform_hrefs) for link in links],
"assets":
assets,
}
if self.bbox is not None:
d["bbox"] = self.bbox
if self.stac_extensions is not None:
d["stac_extensions"] = self.stac_extensions
if self.collection_id:
d["collection"] = self.collection_id
for key in self.extra_fields:
d[key] = self.extra_fields[key]
return d
def test_published(self) -> None:
timestamps_item = pystac.Item.from_file(self.example_uri)
# Get
self.assertIn("published", timestamps_item.properties)
timestamps_published = TimestampsExtension.ext(
timestamps_item).published
self.assertIsInstance(timestamps_published, datetime)
self.assertEqual(
datetime_to_str(get_opt(timestamps_published)),
timestamps_item.properties["published"],
)
# Set
TimestampsExtension.ext(
timestamps_item).published = self.sample_datetime
self.assertEqual(self.sample_datetime_str,
timestamps_item.properties["published"])
# Get from Asset
asset_no_prop = timestamps_item.assets["red"]
asset_prop = timestamps_item.assets["blue"]
self.assertEqual(
TimestampsExtension.ext(asset_no_prop).published,
TimestampsExtension.ext(timestamps_item).published,
)
self.assertEqual(
TimestampsExtension.ext(asset_prop).published,
str_to_datetime("2018-11-02T00:00:00Z"),
)
# # Set to Asset
asset_value = str_to_datetime("2019-02-02T00:00:00Z")
TimestampsExtension.ext(asset_no_prop).published = asset_value
self.assertNotEqual(
TimestampsExtension.ext(asset_no_prop).published,
TimestampsExtension.ext(timestamps_item).published,
)
self.assertEqual(
TimestampsExtension.ext(asset_no_prop).published, asset_value)
# Validate
timestamps_item.validate()
def test_updated(self) -> None:
item = self.item.clone()
cm = item.common_metadata
analytic = item.assets["analytic"]
analytic_cm = CommonMetadata(analytic)
thumbnail = item.assets["thumbnail"]
thumbnail_cm = CommonMetadata(thumbnail)
item_value = cm.updated
a2_known_value = utils.str_to_datetime("2017-05-18T13:22:30.040Z")
# Get
self.assertNotEqual(thumbnail_cm.updated, item_value)
self.assertEqual(thumbnail_cm.updated, a2_known_value)
# Set
set_value = utils.str_to_datetime("2014-05-18T13:22:30.040Z")
analytic_cm.updated = set_value
self.assertEqual(analytic_cm.updated, set_value)
self.assertEqual(analytic.to_dict()["updated"],
utils.datetime_to_str(set_value))
def test_published(self):
timestamps_item = pystac.read_file(self.example_uri)
# Get
self.assertIn("published", timestamps_item.properties)
timestamps_published = timestamps_item.ext.timestamps.published
self.assertIsInstance(timestamps_published, datetime)
self.assertEqual(datetime_to_str(timestamps_published),
timestamps_item.properties['published'])
# Set
timestamps_item.ext.timestamps.published = self.sample_datetime
self.assertEqual(self.sample_datetime_str,
timestamps_item.properties['published'])
# Get from Asset
asset_no_prop = timestamps_item.assets['red']
asset_prop = timestamps_item.assets['blue']
self.assertEqual(
timestamps_item.ext.timestamps.get_published(asset_no_prop),
timestamps_item.ext.timestamps.get_published())
self.assertEqual(
timestamps_item.ext.timestamps.get_published(asset_prop),
str_to_datetime("2018-11-02T00:00:00Z"))
# # Set to Asset
asset_value = str_to_datetime("2019-02-02T00:00:00Z")
timestamps_item.ext.timestamps.set_published(asset_value,
asset_no_prop)
self.assertNotEqual(
timestamps_item.ext.timestamps.get_published(asset_no_prop),
timestamps_item.ext.timestamps.get_published())
self.assertEqual(
timestamps_item.ext.timestamps.get_published(asset_no_prop),
asset_value)
# Validate
timestamps_item.validate()
def download_and_cog_chirps(
year: str,
month: str,
s3_dst: str,
day: str = None,
overwrite: bool = False,
slack_url: str = None,
):
# Cleaning and sanity checks
s3_dst = s3_dst.rstrip("/")
# Set up file strings
if day is not None:
# Set up a daily process
in_file = f"chirps-v2.0.{year}.{month}.{day}.tif.gz"
in_href = DAILY_URL_TEMPLATE.format(year=year, in_file=in_file)
in_data = f"/vsigzip//vsicurl/{in_href}"
if not check_for_url_existence(in_href):
log.warning("Couldn't find the gzipped file, trying the .tif")
in_file = f"chirps-v2.0.{year}.{month}.{day}.tif"
in_href = DAILY_URL_TEMPLATE.format(year=year, in_file=in_file)
in_data = f"/vsicurl/{in_href}"
if not check_for_url_existence(in_href):
log.error("Couldn't find the .tif file either, aborting")
sys.exit(1)
file_base = f"{s3_dst}/{year}/{month}/chirps-v2.0_{year}.{month}.{day}"
out_data = f"{file_base}.tif"
out_stac = f"{file_base}.stac-item.json"
start_datetime = f"{year}-{month}-{day}T00:00:00Z"
end_datetime = f"{year}-{month}-{day}T23:59:59Z"
product_name = "rainfall_chirps_daily"
else:
# Set up a monthly process
in_file = f"chirps-v2.0.{year}.{month}.tif.gz"
in_href = MONTHLY_URL_TEMPLATE.format(in_file=in_file)
in_data = f"/vsigzip//vsicurl/{in_href}"
if not check_for_url_existence(in_href):
log.warning("Couldn't find the gzipped file, trying the .tif")
in_file = f"chirps-v2.0.{year}.{month}.tif"
in_href = MONTHLY_URL_TEMPLATE.format(in_file=in_file)
in_data = f"/vsicurl/{in_href}"
if not check_for_url_existence(in_href):
log.error("Couldn't find the .tif file either, aborting")
sys.exit(1)
file_base = f"{s3_dst}/chirps-v2.0_{year}.{month}"
out_data = f"{file_base}.tif"
out_stac = f"{file_base}.stac-item.json"
_, end = calendar.monthrange(int(year), int(month))
start_datetime = f"{year}-{month}-01T00:00:00Z"
end_datetime = f"{year}-{month}-{end}T23:59:59Z"
product_name = "rainfall_chirps_monthly"
# Set to 15 for the STAC metadata
day = 15
try:
# Check if file already exists
log.info(f"Working on {in_file}")
if not overwrite and s3_head_object(out_stac) is not None:
log.warning(f"File {out_stac} already exists. Skipping.")
return
# COG and STAC
with MemoryFile() as mem_dst:
# Creating the COG, with a memory cache and no download. Shiny.
cog_translate(
in_data,
mem_dst.name,
cog_profiles.get("deflate"),
in_memory=True,
nodata=-9999,
)
# Creating the STAC document with appropriate date range
_, end = calendar.monthrange(int(year), int(month))
item = create_stac_item(
mem_dst,
id=str(odc_uuid("chirps", "2.0", [in_file])),
with_proj=True,
input_datetime=datetime(int(year), int(month), int(day)),
properties={
"odc:processing_datetime": datetime_to_str(datetime.now()),
"odc:product": product_name,
"start_datetime": start_datetime,
"end_datetime": end_datetime,
},
)
item.set_self_href(out_stac)
# Manually redo the asset
del item.assets["asset"]
item.assets["rainfall"] = pystac.Asset(
href=out_data,
title="CHIRPS-v2.0",
media_type=pystac.MediaType.COG,
roles=["data"],
)
# Let's add a link to the source
item.add_links([
pystac.Link(
target=in_href,
title="Source file",
rel=pystac.RelType.DERIVED_FROM,
media_type="application/gzip",
)
])
# Dump the data to S3
mem_dst.seek(0)
log.info(f"Writing DATA to: {out_data}")
s3_dump(mem_dst, out_data, ACL="bucket-owner-full-control")
# Write STAC to S3
log.info(f"Writing STAC to: {out_stac}")
s3_dump(
json.dumps(item.to_dict(), indent=2),
out_stac,
ContentType="application/json",
ACL="bucket-owner-full-control",
)
# All done!
log.info(f"Completed work on {in_file}")
except Exception as e:
message = f"Failed to handle {in_file} with error {e}"
if slack_url is not None:
send_slack_notification(slack_url, "Chirps Rainfall Monthly",
message)
log.exception(message)
exit(1)