class MetadataTest(unittest.TestCase):
def setUp(self):
self.url = "http://example.com/metadata"
self.instance = Metadata(url=self.url,
auth=Auth(jwt_token=public_token,
token_info_path=None))
self.instance._raster = Raster(url=self.url, auth=self.instance.auth)
self.match_url = re.compile(self.url)
def mock_response(self, method, json, status=200, **kwargs):
responses.add(method,
self.match_url,
json=json,
status=status,
**kwargs)
def test_expr_serialization(self):
p = self.instance.properties
q = ((0.1 < p.cloud_fraction <= 0.2) &
(p.sat_id == "f00b")) | (p.sat_id == "usa-245")
expected_q = {
"or": [
{
"and": [
{
"range": {
"cloud_fraction": {
"gt": 0.1,
"lte": 0.2
}
}
},
{
"eq": {
"sat_id": "f00b"
}
},
]
},
{
"eq": {
"sat_id": "usa-245"
}
},
]
}
assert q.serialize() == expected_q
def test_expr_contains(self):
p = self.instance.properties
q = p.sat_id.in_(("usa-245", "terra"))
expected_q = {
"or": [{
"eq": {
"sat_id": "usa-245"
}
}, {
"eq": {
"sat_id": "terra"
}
}]
}
assert q.serialize() == expected_q
@responses.activate
def test_paged_search(self):
features = [{"id": "foo"}]
token = "token"
self.mock_response(responses.POST,
json=features,
headers={"x-continuation-token": token})
collection = self.instance.paged_search(limit=100)
assert features == collection.features
assert token == collection.properties.continuation_token
@responses.activate
def test_paged_search_deprecated_args(self):
features = [{"id": "foo"}]
self.mock_response(responses.POST, json=features)
with warnings.catch_warnings(record=True) as w:
collection = self.instance.paged_search(limit=100,
start_time="2017-07-08")
assert 1 == len(w)
assert w[0].category == DeprecationWarning
assert features == collection.features
@responses.activate
def test_paged_search_dltile(self):
features = [{"id": "foo"}]
tile_geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=features)
collection = self.instance.paged_search(
limit=100, dltile="256:16:30.0:15:-11:591")
assert features == collection.features
@responses.activate
def test_paged_search_shapely(self):
features = [{"id": "foo"}]
geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
shape_geom = shape(geom)
self.mock_response(responses.GET, json={"geometry": geom})
self.mock_response(responses.POST, json=features)
collection = self.instance.paged_search(limit=100, geom=shape_geom)
assert features == collection.features
@responses.activate
def test_search(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features)
collection = self.instance.search(limit=2)
req = responses.calls[0].request
assert "storage_state" not in json.loads(req.body.decode("utf-8"))
assert features[:2] == collection.features
@responses.activate
def test_search_storage_state(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features)
collection = self.instance.search(limit=2, storage_state="available")
assert features[:2] == collection.features
req = responses.calls[0].request
assert "storage_state" in json.loads(req.body.decode("utf-8"))
@responses.activate
def test_features(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST,
json=features[:2],
headers={"x-continuation-token": "token"})
self.mock_response(
responses.POST,
json=features[2:],
headers={"x-continuation-token": "token2"},
)
# Note: Unfortunately the client has historically been written in such a way that it always
# expects a token header, even if the end of the search was reached, so an extra request
# with 0 results happens in practice.
self.mock_response(responses.POST,
json=[],
headers={"x-continuation-token": "token3"})
assert features == list(self.instance.features())
req = responses.calls[0].request
assert "storage_state" not in json.loads(req.body.decode("utf-8"))
@responses.activate
def test_summary_default(self):
summary = {"count": 42}
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary()
req = responses.calls[0].request
assert "storage_state" not in json.loads(req.body.decode("utf-8"))
@responses.activate
def test_summary_storage_state(self):
summary = {"count": 42}
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary(storage_state="available")
expected_req = {"date": "acquired", "storage_state": "available"}
req = responses.calls[0].request
assert json.loads(req.body.decode("utf-8")) == expected_req
@responses.activate
def test_summary_dltile(self):
summary = {"count": 42}
tile_geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary(
dltile="256:16:30.0:15:-11:591")
@responses.activate
def test_summary_shapely(self):
summary = {"count": 42}
geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
shape_geom = shape(geom)
self.mock_response(responses.GET, json={"geometry": geom})
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary(geom=shape_geom)
class MetadataTest(unittest.TestCase):
def setUp(self):
self.url = "http://example.com/metadata"
self.instance = Metadata(url=self.url, auth=Auth(jwt_token=public_token, token_info_path=None))
self.instance._raster = Raster(url=self.url, auth=self.instance.auth)
self.match_url = re.compile(self.url)
def mock_response(self, method, json, status=200, **kwargs):
responses.add(method, self.match_url, json=json, status=status, **kwargs)
def test_expr_serialization(self):
p = self.instance.properties
q = ((0.1 < p.cloud_fraction <= 0.2) & (p.sat_id == "f00b")) | (
p.sat_id == "usa-245"
)
expected_q = {
"or": [
{
"and": [
{"range": {"cloud_fraction": {"gt": 0.1, "lte": 0.2}}},
{"eq": {"sat_id": "f00b"}},
]
},
{"eq": {"sat_id": "usa-245"}},
]
}
self.assertEqual(q.serialize(), expected_q)
@responses.activate
def test_paged_search(self):
features = [{"id": "foo"}]
token = "token"
self.mock_response(responses.POST, json=features, headers={"x-continuation-token": token})
collection = self.instance.paged_search(limit=100)
self.assertEqual(features, collection.features)
self.assertEqual(token, collection.properties.continuation_token)
@responses.activate
def test_paged_search_deprecated_args(self):
features = [{"id": "foo"}]
self.mock_response(responses.POST, json=features)
with warnings.catch_warnings(record=True) as w:
collection = self.instance.paged_search(limit=100, start_time="2017-07-08")
self.assertEqual(1, len(w))
self.assertEqual(w[0].category, DeprecationWarning)
self.assertEqual(features, collection.features)
@responses.activate
def test_paged_search_dltile(self):
features = [{"id": "foo"}]
tile_geom = {
"coordinates": [[[-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606]]],
"type": "Polygon"
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=features)
collection = self.instance.paged_search(limit=100, dltile="256:16:30.0:15:-11:591")
self.assertEqual(features, collection.features)
@responses.activate
def test_search(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features)
collection = self.instance.search(limit=2)
self.assertEqual(features[:2], collection.features)
@responses.activate
def test_features(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features[:2], headers={"x-continuation-token": "token"})
self.mock_response(responses.POST, json=features[2:], headers={"x-continuation-token": "token2"})
# Note: Unfortunately the client has historically been written in such a way that it always
# expects a token header, even if the end of the search was reached, so an extra request
# with 0 results happens in practice.
self.mock_response(responses.POST, json=[], headers={"x-continuation-token": "token3"})
self.assertEqual(features, list(self.instance.features()))
@responses.activate
def test_summary(self):
summary = {"count": 42}
self.mock_response(responses.POST, json=summary)
self.assertEqual(summary, self.instance.summary())
@responses.activate
def test_summary_dltile(self):
summary = {"count": 42}
tile_geom = {
"coordinates": [[[-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606]]],
"type": "Polygon"
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=summary)
self.assertEqual(summary, self.instance.summary(dltile="256:16:30.0:15:-11:591"))
def search(aoi,
products=None,
start_datetime=None,
end_datetime=None,
cloud_fraction=None,
limit=100,
sort_field=None,
sort_order='asc',
date_field='acquired',
query=None,
randomize=False,
raster_client=None,
metadata_client=None):
"""
Search for Scenes in the Descartes Labs catalog.
Returns a SceneCollection of Scenes that overlap with an area of interest,
and meet the given search criteria.
Parameters
----------
aoi : GeoJSON-like dict, GeoContext, or object with __geo_interface__
Search for scenes that intersect this area by any amount.
If a GeoContext, a copy is returned as ``ctx``, with missing values filled in.
Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry.
products : str or List[str], optional
Descartes Labs product identifiers
start_datetime : str, datetime-like, optional
Restrict to scenes acquired after this datetime
end_datetime : str, datetime-like, optional
Restrict to scenes acquired before this datetime
cloud_fraction : float, optional
Restrict to scenes that are covered in clouds by less than this fraction
(between 0 and 1)
limit : int, optional
Maximum number of Scenes to return, up to 10000.
sort_field : str, optional
Field name in ``Scene.properties`` by which to order the results
sort_order : str, optional, default 'asc'
``"asc"`` or ``"desc"``
date_field : str, optional, default 'acquired'
The field used when filtering by date
(``"acquired"``, ``"processed"``, ``"published"``)
query : descarteslabs.common.property_filtering.Expression, optional
Expression used to filter Scenes by their properties, built from ``dl.properties``.
>>> query = 150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5
>>> query = dl.properties.sat_id == "Terra"
randomize : bool, default False, optional
Randomize the order of the results.
You may also use an int or str as an explicit seed.
raster_client : Raster, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scenes : SceneCollection
Scenes matching your criteria.
ctx: GeoContext
The given ``aoi`` as a GeoContext (if it isn't one already),
with reasonable default parameters for loading all matching Scenes.
If ``aoi`` was a `GeoContext`, ``ctx`` will be a copy of ``aoi``,
with any properties that were ``None`` assigned the defaults below.
If ``aoi`` was not a `GeoContext`, an `AOI` instance will be created
with ``aoi`` as its geometry, and defaults assigned as described below:
**Default Spatial Parameters:**
* resolution: the finest resolution of any band of all matching scenes
* crs: the most common CRS used of all matching scenes
"""
if isinstance(aoi, geocontext.GeoContext):
ctx = aoi
if ctx.bounds is None and ctx.geometry is None:
raise ValueError(
"Unspecified where to search, "
"since the GeoContext given for ``aoi`` has neither geometry nor bounds set"
)
else:
ctx = geocontext.AOI(geometry=aoi)
if raster_client is None:
raster_client = Raster()
if metadata_client is None:
metadata_client = Metadata()
if isinstance(products, six.string_types):
products = [products]
if isinstance(start_datetime, datetime.datetime):
start_datetime = start_datetime.isoformat()
if isinstance(end_datetime, datetime.datetime):
end_datetime = end_datetime.isoformat()
if limit > MAX_RESULT_WINDOW:
raise ValueError("Limit must be <= {}".format(MAX_RESULT_WINDOW))
metadata_params = dict(products=products,
geom=ctx.__geo_interface__,
start_datetime=start_datetime,
end_datetime=end_datetime,
cloud_fraction=cloud_fraction,
limit=limit,
sort_field=sort_field,
sort_order=sort_order,
date=date_field,
q=query,
randomize=randomize)
metadata = metadata_client.search(**metadata_params)
if products is None:
products = {
meta["properties"]["product"]
for meta in metadata["features"]
}
product_bands = {
product:
Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product))
for product in products
}
scenes = SceneCollection(
(Scene(meta, product_bands[meta["properties"]["product"]])
for meta in metadata["features"]),
raster_client=raster_client)
if len(scenes) > 0:
assign_ctx = {}
if ctx.resolution is None and ctx.shape is None:
resolutions = filter(None,
(b.get("resolution")
for band in six.itervalues(product_bands)
for b in six.itervalues(band)))
try:
assign_ctx["resolution"] = min(resolutions)
except ValueError:
assign_ctx[
"resolution"] = None # from min of an empty sequence; no band defines resolution
if ctx.crs is None:
assign_ctx["crs"] = collections.Counter(
scene.properties["crs"]
for scene in scenes).most_common(1)[0][0]
if len(assign_ctx) > 0:
ctx = ctx.assign(**assign_ctx)
return scenes, ctx
def search(
aoi,
products=None,
start_datetime=None,
end_datetime=None,
cloud_fraction=None,
storage_state=None,
limit=100,
sort_field=None,
sort_order="asc",
date_field="acquired",
query=None,
randomize=False,
raster_client=None,
metadata_client=None,
):
"""
Search for Scenes in the Descartes Labs catalog.
Returns a SceneCollection of Scenes that overlap with an area of interest,
and meet the given search criteria.
Parameters
----------
aoi : GeoJSON-like dict, :class:`~descarteslabs.scenes.geocontext.GeoContext`, or object with __geo_interface__
Search for scenes that intersect this area by any amount.
If a :class:`~descarteslabs.scenes.geocontext.GeoContext`, a copy is returned as ``ctx``, with missing values
filled in. Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry.
products : str or List[str], optional
Descartes Labs product identifiers
start_datetime : str, datetime-like, optional
Restrict to scenes acquired after this datetime
end_datetime : str, datetime-like, optional
Restrict to scenes acquired before this datetime
cloud_fraction : float, optional
Restrict to scenes that are covered in clouds by less than this fraction
(between 0 and 1)
storage_state : str, optional
Filter results based on ``storage_state`` value
(``"available"``, ``"remote"``, or ``None``)
limit : int or None, optional, default 100
Maximum number of Scenes to return, or None for all results.
sort_field : str, optional
Field name in :py:attr:`Scene.properties` by which to order the results
sort_order : str, optional, default 'asc'
``"asc"`` or ``"desc"``
date_field : str, optional, default 'acquired'
The field used when filtering by date
(``"acquired"``, ``"processed"``, ``"published"``)
query : ~descarteslabs.common.property_filtering.filtering.Expression, optional
Expression used to filter Scenes by their properties, built from
:class:`dl.properties <descarteslabs.common.property_filtering.filtering.GenericProperties>`.
You can construct filter expression using the ``==``, ``!=``, ``<``, ``>``,
``<=`` and ``>=`` operators as well as the
:meth:`~descarteslabs.common.property_filtering.filtering.Property.like`
and :meth:`~descarteslabs.common.property_filtering.filtering.Property.in_`
methods. You cannot use the boolean keywords ``and`` and ``or`` because of
Python language limitations; instead you can combine filter expressions
with ``&`` (boolean "and") and ``|`` (boolean "or").
Example:
``150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5``
randomize : bool, default False, optional
Randomize the order of the results.
You may also use an int or str as an explicit seed.
raster_client : Raster, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scenes : `SceneCollection`
Scenes matching your criteria.
ctx: :class:`~descarteslabs.scenes.geocontext.GeoContext`
The given ``aoi`` as a :class:`~descarteslabs.scenes.geocontext.GeoContext` (if it isn't one already),
with reasonable default parameters for loading all matching Scenes.
If ``aoi`` was a :class:`~descarteslabs.scenes.geocontext.GeoContext`, ``ctx`` will be a copy of ``aoi``,
with any properties that were ``None`` assigned the defaults below.
If ``aoi`` was not a :class:`~descarteslabs.scenes.geocontext.GeoContext`, an `AOI` instance will be created
with ``aoi`` as its geometry, and defaults assigned as described below:
**Default Spatial Parameters:**
* resolution: the finest resolution of any band of all matching scenes
* crs: the most common CRS used of all matching scenes
"""
if isinstance(aoi, geocontext.GeoContext):
ctx = aoi
if ctx.bounds is None and ctx.geometry is None:
raise ValueError(
"Unspecified where to search, "
"since the GeoContext given for ``aoi`` has neither geometry nor bounds set"
)
else:
ctx = geocontext.AOI(geometry=aoi)
if raster_client is None:
raster_client = Raster()
if metadata_client is None:
metadata_client = Metadata()
if isinstance(products, six.string_types):
products = [products]
if isinstance(start_datetime, datetime.datetime):
start_datetime = start_datetime.isoformat()
if isinstance(end_datetime, datetime.datetime):
end_datetime = end_datetime.isoformat()
metadata_params = dict(
products=products,
geom=ctx.__geo_interface__,
start_datetime=start_datetime,
end_datetime=end_datetime,
cloud_fraction=cloud_fraction,
storage_state=storage_state,
limit=limit,
sort_field=sort_field,
sort_order=sort_order,
date=date_field,
q=query,
randomize=randomize,
)
metadata = metadata_client.search(**metadata_params)
if products is None:
products = {
meta["properties"]["product"]
for meta in metadata["features"]
}
product_bands = {
product:
Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product))
for product in products
}
scenes = SceneCollection(
(Scene(meta, product_bands[meta["properties"]["product"]])
for meta in metadata["features"]),
raster_client=raster_client,
)
if len(scenes) > 0 and isinstance(ctx, geocontext.AOI):
assign_ctx = {}
if ctx.resolution is None and ctx.shape is None:
resolutions = filter(
None,
(b.get("resolution") for band in six.itervalues(product_bands)
for b in six.itervalues(band)),
)
try:
assign_ctx["resolution"] = min(resolutions)
except ValueError:
assign_ctx[
"resolution"] = None # from min of an empty sequence; no band defines resolution
if ctx.crs is None:
assign_ctx["crs"] = collections.Counter(
scene.properties["crs"]
for scene in scenes).most_common(1)[0][0]
if len(assign_ctx) > 0:
ctx = ctx.assign(**assign_ctx)
return scenes, ctx