def from_id(cls, scene_id, metadata_client=None):
"""
Return the metadata for a Descartes Labs scene ID as a Scene object.
Also returns a :class:`~descarteslabs.scenes.geocontext.GeoContext`
for loading the Scene's original, unwarped data.
Parameters
----------
scene_id: str
Descartes Labs scene ID,
e.g. "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1"
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scene: Scene
Scene instance with metadata loaded from the Descartes Labs catalog
ctx: AOI
A :class:`~descarteslabs.scenes.geocontext.GeoContext`for loading this Scene's original data.
The defaults used are described in `Scene.default_ctx`.
Example
-------
>>> import descarteslabs as dl
>>> scene, ctx = dl.scenes.Scene.from_id("landsat:LC08:PRE:TOAR:meta_LC80260322016197_v1") # doctest: +SKIP
>>> ctx # doctest: +SKIP
AOI(geometry=None,
resolution=15.0,
crs='EPSG:32615',
align_pixels=False,
bounds=(348592.5, 4345567.5, 581632.5, 4582807.5),
bounds_crs='EPSG:32615',
shape=None)
>>> scene.properties.date # doctest: +SKIP
datetime.datetime(2016, 7, 15, 16, 53, 59, 495435)
Raises
------
NotFoundError
If the ``scene_id`` cannot be found in the Descartes Labs catalog
"""
if metadata_client is None:
metadata_client = Metadata()
metadata = metadata_client.get(scene_id)
metadata = {
"type": "Feature",
"geometry": metadata.pop("geometry"),
"id": metadata.pop("id"),
"key": metadata.pop("key"),
"properties": metadata
}
bands = metadata_client.get_bands_by_id(scene_id)
scene = cls(metadata, bands)
return scene, scene.default_ctx()
def setUp(self):
self.url = "http://example.com/metadata"
self.instance = Metadata(url=self.url,
auth=Auth(jwt_token=public_token,
token_info_path=None))
self.instance._raster = Raster(url=self.url, auth=self.instance.auth)
self.match_url = re.compile(self.url)
def __init__(self, url=None, auth=None, metadata=None):
"""The parent Service class implements authentication and exponential
backoff/retry. Override the url parameter to use a different instance
of the backing service.
"""
if auth is None:
auth = Auth()
if metadata is None:
self._metadata = Metadata(auth=auth)
else:
self._metadata = metadata
self._gcs_upload_service = ThirdPartyService()
if url is None:
url = os.environ.get(
"DESCARTESLABS_CATALOG_URL",
"https://platform.descarteslabs.com/metadata/v1/catalog")
super(Catalog, self).__init__(url, auth=auth)
def setUpClass(cls):
cls.instance = Metadata()
import datetime
import collections
import textwrap
import warnings
import shapely.geometry
import numpy as np
from descarteslabs.common.dotdict import DotDict
from descarteslabs.scenes import Scene, geocontext
from descarteslabs.scenes.scene import _strptime_helper
from descarteslabs.client.services.metadata import Metadata
from .mock_data import _metadata_get, _metadata_get_bands, _raster_ndarray
metadata_client = Metadata()
class MockScene(Scene):
"Circumvent __init__ method to create a Scene with arbitrary geometry and properties objects"
def __init__(self, geometry, properties):
self.geometry = DotDict(geometry)
self.properties = DotDict(properties)
class TestScene(unittest.TestCase):
@mock.patch("descarteslabs.client.services.metadata.Metadata.get", _metadata_get)
@mock.patch("descarteslabs.client.services.metadata.Metadata.get_bands_by_id", _metadata_get_bands)
def test_init(self):
scene_id = "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1"
metadata = metadata_client.get(scene_id)
def auth_handler(args):
auth = Auth()
if args.command == 'login':
print(
'Follow this link to login https://iam.descarteslabs.com/auth/login?refresh_token=true&destination=/auth/refresh_token'
) # NOQA
s = input('...then come back here and paste the generated token: ')
if isinstance(s, six.text_type):
s = s.encode('utf-8')
if s:
token_info = json.loads(base64url_decode(s).decode('utf-8'))
token_info_directory = os.path.dirname(DEFAULT_TOKEN_INFO_PATH)
makedirs_if_not_exists(token_info_directory)
os.chmod(token_info_directory,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
with open(DEFAULT_TOKEN_INFO_PATH, 'w+') as fp:
json.dump(token_info, fp)
os.chmod(DEFAULT_TOKEN_INFO_PATH, stat.S_IRUSR | stat.S_IWUSR)
# Get a fresh Auth token
auth = Auth()
keys = Metadata(auth=auth).keys()
name = auth.payload['name']
groups = ', '.join(auth.payload['groups'])
if len(keys):
print('Welcome, %s!' % name)
else:
print(
'Welcome, %s! Your %s role(s) do not permit access to any imagery at this time.'
% (name, groups))
print(
'Contact [email protected] if you believe you received this message in error or have any questions.'
) # NOQA
if args.command == 'token':
print(auth.token)
if args.command == 'name':
auth.token
print(auth.payload['name'])
if args.command == 'groups':
auth.token
print(json.dumps(auth.payload['groups']))
if args.command == 'payload':
auth.token
print(auth.payload)
if args.command == 'env':
auth.token
print('%s=%s' % ('CLIENT_ID', auth.client_id))
print('%s=%s' % ('CLIENT_SECRET', auth.client_secret))
if args.command == 'version':
print(__version__)
class MetadataTest(unittest.TestCase):
def setUp(self):
self.url = "http://example.com/metadata"
self.instance = Metadata(url=self.url,
auth=Auth(jwt_token=public_token,
token_info_path=None))
self.instance._raster = Raster(url=self.url, auth=self.instance.auth)
self.match_url = re.compile(self.url)
def mock_response(self, method, json, status=200, **kwargs):
responses.add(method,
self.match_url,
json=json,
status=status,
**kwargs)
def test_expr_serialization(self):
p = self.instance.properties
q = ((0.1 < p.cloud_fraction <= 0.2) &
(p.sat_id == "f00b")) | (p.sat_id == "usa-245")
expected_q = {
"or": [
{
"and": [
{
"range": {
"cloud_fraction": {
"gt": 0.1,
"lte": 0.2
}
}
},
{
"eq": {
"sat_id": "f00b"
}
},
]
},
{
"eq": {
"sat_id": "usa-245"
}
},
]
}
assert q.serialize() == expected_q
def test_expr_contains(self):
p = self.instance.properties
q = p.sat_id.in_(("usa-245", "terra"))
expected_q = {
"or": [{
"eq": {
"sat_id": "usa-245"
}
}, {
"eq": {
"sat_id": "terra"
}
}]
}
assert q.serialize() == expected_q
@responses.activate
def test_paged_search(self):
features = [{"id": "foo"}]
token = "token"
self.mock_response(responses.POST,
json=features,
headers={"x-continuation-token": token})
collection = self.instance.paged_search(limit=100)
assert features == collection.features
assert token == collection.properties.continuation_token
@responses.activate
def test_paged_search_deprecated_args(self):
features = [{"id": "foo"}]
self.mock_response(responses.POST, json=features)
with warnings.catch_warnings(record=True) as w:
collection = self.instance.paged_search(limit=100,
start_time="2017-07-08")
assert 1 == len(w)
assert w[0].category == DeprecationWarning
assert features == collection.features
@responses.activate
def test_paged_search_dltile(self):
features = [{"id": "foo"}]
tile_geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=features)
collection = self.instance.paged_search(
limit=100, dltile="256:16:30.0:15:-11:591")
assert features == collection.features
@responses.activate
def test_paged_search_shapely(self):
features = [{"id": "foo"}]
geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
shape_geom = shape(geom)
self.mock_response(responses.GET, json={"geometry": geom})
self.mock_response(responses.POST, json=features)
collection = self.instance.paged_search(limit=100, geom=shape_geom)
assert features == collection.features
@responses.activate
def test_search(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features)
collection = self.instance.search(limit=2)
req = responses.calls[0].request
assert "storage_state" not in json.loads(req.body.decode("utf-8"))
assert features[:2] == collection.features
@responses.activate
def test_search_storage_state(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features)
collection = self.instance.search(limit=2, storage_state="available")
assert features[:2] == collection.features
req = responses.calls[0].request
assert "storage_state" in json.loads(req.body.decode("utf-8"))
@responses.activate
def test_features(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST,
json=features[:2],
headers={"x-continuation-token": "token"})
self.mock_response(
responses.POST,
json=features[2:],
headers={"x-continuation-token": "token2"},
)
# Note: Unfortunately the client has historically been written in such a way that it always
# expects a token header, even if the end of the search was reached, so an extra request
# with 0 results happens in practice.
self.mock_response(responses.POST,
json=[],
headers={"x-continuation-token": "token3"})
assert features == list(self.instance.features())
req = responses.calls[0].request
assert "storage_state" not in json.loads(req.body.decode("utf-8"))
@responses.activate
def test_summary_default(self):
summary = {"count": 42}
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary()
req = responses.calls[0].request
assert "storage_state" not in json.loads(req.body.decode("utf-8"))
@responses.activate
def test_summary_storage_state(self):
summary = {"count": 42}
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary(storage_state="available")
expected_req = {"date": "acquired", "storage_state": "available"}
req = responses.calls[0].request
assert json.loads(req.body.decode("utf-8")) == expected_req
@responses.activate
def test_summary_dltile(self):
summary = {"count": 42}
tile_geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary(
dltile="256:16:30.0:15:-11:591")
@responses.activate
def test_summary_shapely(self):
summary = {"count": 42}
geom = {
"coordinates": [[
[-94.01008346640455, 40.992358024242606],
[-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541],
[-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606],
]],
"type":
"Polygon",
}
shape_geom = shape(geom)
self.mock_response(responses.GET, json={"geometry": geom})
self.mock_response(responses.POST, json=summary)
assert summary == self.instance.summary(geom=shape_geom)
def from_id(cls, scene_id, metadata_client=None):
"""
Return the metadata for a Descartes Labs scene ID as a Scene object.
Also returns a GeoContext with reasonable defaults to use
when loading the Scene's ndarray.
Parameters
----------
scene_id: str
Descartes Labs scene ID,
e.g. "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1"
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scene: Scene
Scene instance with metadata loaded from the Descartes Labs catalog
ctx: AOI
A default GeoContext useful for loading this Scene.
These defaults are used:
* bounds: bounds of the Scene's geometry
* resolution: the finest resolution of any band in the scene
* crs: native CRS of the Scene (generally, a UTM CRS)
* align_pixels: True
Example
-------
>>> import descarteslabs as dl
>>> scene, ctx = dl.scenes.Scene.from_id("landsat:LC08:PRE:TOAR:meta_LC80260322016197_v1")
>>> ctx
AOI(geometry=None,
resolution=15,
crs='EPSG:32615',
align_pixels=True,
bounds=(-94.724166, 39.2784859, -92.0686956, 41.3717716),
shape=None)
>>> scene.properties.date
datetime.datetime(2016, 7, 15, 16, 53, 59, 495435)
Raises
------
NotFoundError
If the ``scene_id`` cannot be found in the Descartes Labs catalog
"""
if metadata_client is None:
metadata_client = Metadata()
metadata = metadata_client.get(scene_id)
metadata = {
"type": "Feature",
"geometry": metadata.pop("geometry"),
"id": metadata.pop("id"),
"key": metadata.pop("key"),
"properties": metadata
}
bands = metadata_client.get_bands_by_id(scene_id)
scene = cls(metadata, bands)
# TODO: not sure what default res should be
try:
default_resolution = min(
filter(None,
[b.get("resolution") for b in six.itervalues(bands)]))
except ValueError:
default_resolution = 100
# QUESTION: default bounds will now be in WGS84, not UTM
# indeed, there's no way to give bounds in UTM besides with a DLTile,
# which means you could get off-by-one issues with loading an entire scene
# at native resolution, where the WGS84 bounds result in a slightly differently
# sized raster than native UTM bounds would with reprojection errors
try:
bounds = scene.geometry.bounds
except AttributeError:
xs, ys = zip(*scene.geometry["coordinates"][0])
bounds = (min(xs), min(ys), max(xs), max(ys))
default_ctx = geocontext.AOI(bounds=bounds,
resolution=default_resolution,
crs=scene.properties["crs"])
return scene, default_ctx
# Copyright 2018 Descartes Labs. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # flake8: noqa from descarteslabs.client.auth import Auth from descarteslabs.client.services.metadata import Metadata from descarteslabs.client.services.places import Places from descarteslabs.client.services.raster import Raster descartes_auth = Auth.from_environment_or_token_json() metadata = Metadata(auth=descartes_auth) places = Places(auth=descartes_auth) raster = Raster(auth=descartes_auth) __all__ = ["descartes_auth", "metadata", "places", "raster"]
def search(aoi,
products=None,
start_datetime=None,
end_datetime=None,
cloud_fraction=None,
limit=100,
sort_field=None,
sort_order='asc',
date_field='acquired',
query=None,
randomize=False,
raster_client=None,
metadata_client=None):
"""
Search for Scenes in the Descartes Labs catalog.
Returns a SceneCollection of Scenes that overlap with an area of interest,
and meet the given search criteria.
Parameters
----------
aoi : GeoJSON-like dict, GeoContext, or object with __geo_interface__
Search for scenes that intersect this area by any amount.
If a GeoContext, a copy is returned as ``ctx``, with missing values filled in.
Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry.
products : str or List[str], optional
Descartes Labs product identifiers
start_datetime : str, datetime-like, optional
Restrict to scenes acquired after this datetime
end_datetime : str, datetime-like, optional
Restrict to scenes acquired before this datetime
cloud_fraction : float, optional
Restrict to scenes that are covered in clouds by less than this fraction
(between 0 and 1)
limit : int, optional
Maximum number of Scenes to return, up to 10000.
sort_field : str, optional
Field name in ``Scene.properties`` by which to order the results
sort_order : str, optional, default 'asc'
``"asc"`` or ``"desc"``
date_field : str, optional, default 'acquired'
The field used when filtering by date
(``"acquired"``, ``"processed"``, ``"published"``)
query : descarteslabs.common.property_filtering.Expression, optional
Expression used to filter Scenes by their properties, built from ``dl.properties``.
>>> query = 150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5
>>> query = dl.properties.sat_id == "Terra"
randomize : bool, default False, optional
Randomize the order of the results.
You may also use an int or str as an explicit seed.
raster_client : Raster, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scenes : SceneCollection
Scenes matching your criteria.
ctx: GeoContext
The given ``aoi`` as a GeoContext (if it isn't one already),
with reasonable default parameters for loading all matching Scenes.
If ``aoi`` was a `GeoContext`, ``ctx`` will be a copy of ``aoi``,
with any properties that were ``None`` assigned the defaults below.
If ``aoi`` was not a `GeoContext`, an `AOI` instance will be created
with ``aoi`` as its geometry, and defaults assigned as described below:
**Default Spatial Parameters:**
* resolution: the finest resolution of any band of all matching scenes
* crs: the most common CRS used of all matching scenes
"""
if isinstance(aoi, geocontext.GeoContext):
ctx = aoi
if ctx.bounds is None and ctx.geometry is None:
raise ValueError(
"Unspecified where to search, "
"since the GeoContext given for ``aoi`` has neither geometry nor bounds set"
)
else:
ctx = geocontext.AOI(geometry=aoi)
if raster_client is None:
raster_client = Raster()
if metadata_client is None:
metadata_client = Metadata()
if isinstance(products, six.string_types):
products = [products]
if isinstance(start_datetime, datetime.datetime):
start_datetime = start_datetime.isoformat()
if isinstance(end_datetime, datetime.datetime):
end_datetime = end_datetime.isoformat()
if limit > MAX_RESULT_WINDOW:
raise ValueError("Limit must be <= {}".format(MAX_RESULT_WINDOW))
metadata_params = dict(products=products,
geom=ctx.__geo_interface__,
start_datetime=start_datetime,
end_datetime=end_datetime,
cloud_fraction=cloud_fraction,
limit=limit,
sort_field=sort_field,
sort_order=sort_order,
date=date_field,
q=query,
randomize=randomize)
metadata = metadata_client.search(**metadata_params)
if products is None:
products = {
meta["properties"]["product"]
for meta in metadata["features"]
}
product_bands = {
product:
Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product))
for product in products
}
scenes = SceneCollection(
(Scene(meta, product_bands[meta["properties"]["product"]])
for meta in metadata["features"]),
raster_client=raster_client)
if len(scenes) > 0:
assign_ctx = {}
if ctx.resolution is None and ctx.shape is None:
resolutions = filter(None,
(b.get("resolution")
for band in six.itervalues(product_bands)
for b in six.itervalues(band)))
try:
assign_ctx["resolution"] = min(resolutions)
except ValueError:
assign_ctx[
"resolution"] = None # from min of an empty sequence; no band defines resolution
if ctx.crs is None:
assign_ctx["crs"] = collections.Counter(
scene.properties["crs"]
for scene in scenes).most_common(1)[0][0]
if len(assign_ctx) > 0:
ctx = ctx.assign(**assign_ctx)
return scenes, ctx
class Catalog(Service):
"""The Descartes Labs (DL) Catalog allows you to add georeferenced raster products
into the Descartes Labs Platform. Catalog products can be used in other DL services
like Raster and Tasks and Metadata.
The entrypoint for using catalog is creating a Product using :meth:`Catalog.add_product`.
After creating a product you should add band(s) to it using :meth:`Catalog.add_band`, and
upload imagery using :meth:`Catalog.upload_image`.
Bands define where and how data is encoded into imagery files, and how it should be displayed.
Images define metadata about a specific groups of pixels (in the form of images), their georeferencing, geometry,
coordinate system, and other pertinent information.
"""
UPLOAD_NDARRAY_SUPPORTED_DTYPES = [
"uint8",
"int8",
"uint16",
"int16",
"uint32",
"int32",
"float32",
"float64",
]
TIMEOUT = (9.5, 30)
def __init__(self, url=None, auth=None, metadata=None):
"""The parent Service class implements authentication and exponential
backoff/retry. Override the url parameter to use a different instance
of the backing service.
"""
if auth is None:
auth = Auth()
if metadata is None:
self._metadata = Metadata(auth=auth)
else:
self._metadata = metadata
self._gcs_upload_service = ThirdPartyService()
if url is None:
url = os.environ.get(
"DESCARTESLABS_CATALOG_URL",
"https://platform.descarteslabs.com/metadata/v1/catalog",
)
super(Catalog, self).__init__(url, auth=auth)
def namespace_product(self, product_id):
namespace = self.auth.namespace
if not product_id.startswith(namespace):
product_id = "{}:{}".format(namespace, product_id)
return product_id
def own_products(self):
return self._metadata.products(owner=self.auth.payload["sub"])
def own_bands(self):
return self._metadata.bands(owner=self.auth.payload["sub"])
def _add_core_product(self, product_id, **kwargs):
kwargs["id"] = product_id
check_deprecated_kwargs(kwargs, {
"start_time": "start_datetime",
"end_time": "end_datetime"
})
r = self.session.post("/core/products", json=kwargs)
return r.json()
def get_product(self, product_id, add_namespace=False):
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
r = self.session.get("/products/{}".format(product_id))
return r.json()
def add_product(self,
product_id,
title,
description,
add_namespace=False,
**kwargs):
"""Add a product to your catalog.
:param str product_id: (Required) A unique name for this product. In the created
product a namespace consisting of your user id (e.g.
"ae60fc891312ggadc94ade8062213b0063335a3c:") or your organization id (e.g.,
"yourcompany:") will be prefixed to this, if it doesn't already have one, in
order to make the id globally unique.
:param str title: (Required) Official product title.
:param str description: (Required) Information about the product,
why it exists, and what it provides.
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
:param list(str) read: A list of groups, or user hashes to give read access to.
:param int spectral_bands: Number of spectral bands the product has.
:param list(str) native_bands: A list of the names of the native bands of this product
(most applicable to satellite sensors).
:param str start_datetime: ISO8601 compliant date string, indicating start of product data.
:param str end_datetime: ISO8601 compliant date string, indicating end of product data.
:param str notes: Any notes to relay about the product.
:param str orbit: Type of orbit (satellite only).
:param str processing_level: Way in which raw data has been processed if any.
:param str resolution: Pixel density of the data, provide units.
:param str revisit: How often an AOI can expect updated data.
:param str sensor: Name of the sensor used.
:param str swath: How large an area the sensor captures at a given time.
:param list(str) writers: A list of groups, or user hashes to give read access to.
:return: JSON API representation of the product.
:rtype: dict
"""
for k, v in locals().items():
if k in ["title", "description"]:
if v is None:
raise TypeError("required arg `{}` not provided".format(k))
kwargs[k] = v
check_deprecated_kwargs(kwargs, {
"start_time": "start_datetime",
"end_time": "end_datetime"
})
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
kwargs["id"] = self.namespace_product(product_id)
else:
kwargs["id"] = product_id
r = self.session.post("/products", json=kwargs)
return r.json()
def replace_product(self,
product_id,
title,
description,
add_namespace=False,
set_global_permissions=False,
**kwargs):
"""Replace a product in your catalog with a new version.
:param str product_id: (Required) A unique name for this product.
:param str title: (Required) Official product title.
:param str description: (Required) Information about the product,
why it exists, and what it provides.
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
:param list(str) read: A list of groups, or user hashes to give read access to.
:param int spectral_bands: Number of spectral bands the product has.
:param list(str) native_bands: A list of the names of the native bands of this product
(most applicable to satellite sensors).
:param str start_datetime: ISO8601 compliant date string, indicating start of product data.
:param str end_datetime: ISO8601 compliant date string, indicating end of product data.
:param str notes: Any notes to relay about the product.
:param str orbit: Type of orbit (satellite only).
:param str processing_level: Way in which raw data has been processed if any.
:param str resolution: Pixel density of the data, provide units.
:param str revisit: How often an AOI can expect updated data.
:param str sensor: Name of the sensor used.
:param str swath: How large an area the sensor captures at a given time.
:param list(str) writers: A list of groups, or user hashes to give read access to.
:return: JSON API representation of the product.
:rtype: dict
"""
for k, v in locals().items():
if k in ["title", "description"]:
if v is None:
raise TypeError("required arg `{}` not provided".format(k))
kwargs[k] = v
check_deprecated_kwargs(kwargs, {
"start_time": "start_datetime",
"end_time": "end_datetime"
})
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
params = None
if set_global_permissions is True:
params = {"set_global_permissions": "true"}
r = self.session.put("/products/{}".format(product_id),
json=kwargs,
params=params)
return r.json()
def change_product(self,
product_id,
add_namespace=False,
set_global_permissions=False,
**kwargs):
"""Update a product in your catalog.
:param str product_id: (Required) The ID of the product to change.
:param list(str) read: A list of groups, or user hashes to give read access to.
:param int spectral_bands: Number of spectral bands the product has.
:param list(str) native_bands: A list of the names of the native bands of this product
(most applicable to satellite sensors).
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
:param bool set_global_permissions: Update permissions of all existing bands and products
that belong to this product with the updated permission set
specified in the `read` param. Default to false.
:param str start_datetime: ISO8601 compliant date string, indicating start of product data.
:param str end_datetime: ISO8601 compliant date string, indicating end of product data.
:param str title: Official product title.
:param str description: Information about the product,
why it exists, and what it provides.
:param str notes: Any notes to relay about the product.
:param str orbit: Type of orbit (satellite only).
:param str processing_level: Way in which raw data has been processed if any.
:param str resolution: Pixel density of the data, provide units.
:param str revisit: How often an AOI can expect updated data.
:param str sensor: Name of the sensor used.
:param str swath: How large an area the sensor captures at a given time.
:param list(str) writers: A list of groups, or user hashes to give read access to.
:return: JSON API representation of the product.
:rtype: dict
"""
check_deprecated_kwargs(kwargs, {
"start_time": "start_datetime",
"end_time": "end_datetime"
})
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
params = None
if set_global_permissions is True:
params = {"set_global_permissions": "true"}
r = self.session.patch("/products/{}".format(product_id),
json=kwargs,
params=params)
return r.json()
def remove_product(self, product_id, add_namespace=False, cascade=False):
"""Remove a product from the catalog.
:param str product_id: ID of the product to remove.
:param bool cascade: Force deletion of all the associated bands and images. BEWARE this cannot be undone.
"""
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
params = {"cascade": cascade}
r = self.session.delete("/products/{}".format(product_id),
params=params)
if r.headers["content-type"] == "application/json":
return r.json()
def product_deletion_status(self, deletion_task_id):
"""Get the status of a long running product deletion job.
:param str deletion_task_id: deletion_task ID returned from a call to :meth:`Catalog.remove_product`
with a deletion_token.
:return: document with information about product deletion progress.
:rtype: dict
"""
r = self.session.get(
"/products/deletion_tasks/{}".format(deletion_task_id))
return r.json()
def get_band(self, product_id, name, add_namespace=False):
"""Get a band by name.
:return: JSON API representation of the band.
:rtype: dict
"""
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
r = self.session.get("/products/{}/bands/{}".format(product_id, name))
return r.json()
def _add_core_band(self,
product_id,
name,
srcband=None,
dtype=None,
nbits=None,
data_range=None,
type=None,
**kwargs):
for k, v in locals().items():
if k in [
"name", "data_range", "dtype", "nbits", "srcband", "type"
]:
if v is None:
raise TypeError("required arg `{}` not provided".format(k))
kwargs[k] = v
kwargs["id"] = name
r = self.session.post("/core/products/{}/bands".format(product_id),
json=kwargs)
return r.json()
def add_band(self,
product_id,
name,
add_namespace=False,
srcband=None,
dtype=None,
nbits=None,
data_range=None,
type=None,
**kwargs):
"""Add a data band to an existing product.
:param str product_id: (Required) Product to which this band will belong.
:param str name: (Required) Name of this band.
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
:param int jpx_layer: If data is processed to JPEG2000, which layer is the band in. Defaults to 0.
:param int srcband: (Required) The 1-based index of the band in the jpx_layer specified.
:param int srcfile: If the product was processed into multiple files,
which one is this in. Defaults to 0 (first file).
:param str dtype: (Required) The data type used to store this band e.g Byte or Uint16 or Float32.
:param int nbits: (Required) The number of bits of the dtype used to store this band.
:param list(int) data_range: (Required) A list specifying the min and max values for the data in this band.
:param str type: (Required) The data interpretation of the band. One of ['spectral', 'derived', 'mask', 'class']
:param int nodata: Pixel value indicating no data available.
:param list(str) read: A list of groups, or user hashes to give read access to.
Defaults to the same as the parent product if not specified.
:param str color: The color interpretation of this band.
One of ['Alpha', 'Black', 'Blue', 'Cyan', 'Gray', 'Green', 'Hue',
'Ligntness', 'Magenta', 'Palette', 'Red', 'Saturation',
'Undefined', 'YCbCr_Cb', 'YCbCr_Cr', 'YCbCr_Y', 'Yellow'].
Must be 'Alpha' if `type` is 'mask'.
Must be 'Palette' if `colormap_name` or `colormap` is set.
:param str colormap_name: A named colormap to use for this band, one of
['plasma', 'magma', 'viridis', 'msw', 'inferno']
:param list(list(str)) colormap: A custom colormap to use for this band. Takes a list of lists, where each
nested list is a 4-tuple of rgba values to map pixels whose value is the index
of the tuple. i.e the colormap [["100", "20", "200", "255"]] would map pixels
whose value is 0 in the original band, to the rgba color vector at colormap[0].
Less than 2^nbits 4-tuples may be provided, and omitted values
will default map to black.
:param str data_description: Description of band data.
:param list(float) physical_range: If band represents a physical value e.g reflectance
or radiance what are the possible range of those values.
:param str data_unit: Units of the physical range e.g w/sr for radiance
:param str data_unit_description: Description of the data unit.
:param list(int) default_range: A good default scale for the band to use
when rastering for display purposes.
:param str description: Description of the band.
:param str name_common: Standard name for band
:param str name_vendor: What the vendor calls the band eg. B1
:param int vendor_order: The index of the band in the vendors band tables.
Useful for referencing the band to other processing
properties like surface reflectance.
:param str processing_level: Description of how the band was processed if at all.
:param int res_factor: Scaling of this band relative to the native band resolution.
:param int resolution: Resolution of this band.
:param str resolution_unit: Unit of the resolution.
:param float wavelength_center: Center position of wavelength.
:param float wavelength_fwhm: Full width at half maximum value of the wavelength spread.
:param float wavelength_min: Minimum wavelength this band is sensitive to.
:param float wavelength_max: Maximum wavelength this band is sensitive to.
:param str wavelength_unit: Units the wavelength is expressed in.
:param list(str) writers: A list of groups, or user hashes to give read access to.
:return: JSON API representation of the band.
:rtype: dict
"""
for k, v in locals().items():
if k in [
"name", "data_range", "dtype", "nbits", "srcband", "type"
]:
if v is None:
raise TypeError("required arg `{}` not provided".format(k))
kwargs[k] = v
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
check_deprecated_kwargs(kwargs, {"id": "name"})
r = self.session.post("/products/{}/bands".format(product_id),
json=kwargs)
return r.json()
def replace_band(self,
product_id,
name,
srcband=None,
dtype=None,
nbits=None,
data_range=None,
add_namespace=False,
type=None,
**kwargs):
"""Replaces an existing data band with a new document.
:param str product_id: (Required) Product to which this band will belong.
:param str name: (Required) Name of this band.
.. note::
- See :meth:`Catalog.add_band` for additional kwargs.
:return: JSON API representation of the band.
:rtype: dict
"""
for k, v in locals().items():
if k in ["data_range", "dtype", "nbits", "srcband", "type"]:
if v is None:
raise TypeError("required arg `{}` not provided".format(k))
kwargs[k] = v
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
r = self.session.put("/products/{}/bands/{}".format(product_id, name),
json=kwargs)
return r.json()
def change_band(self, product_id, name, add_namespace=False, **kwargs):
"""Update a data band of a product.
:param str product_id: (Required) Product to which this band belongs.
:param str name: Name or id of band to modify.
.. note::
- See :meth:`Catalog.add_band` for additional kwargs.
:return: JSON API representation of the band.
:rtype: dict
"""
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
r = self.session.patch("/products/{}/bands/{}".format(
product_id, name),
json=kwargs)
return r.json()
def remove_band(self, product_id, name, add_namespace=False):
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
self.session.delete("/products/{}/bands/{}".format(product_id, name))
def get_image(self, product_id, image_id, add_namespace=False):
"""Get a single image metadata entry.
:return: JSON API representation of the image.
:rtype: dict
"""
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
r = self.session.get("/products/{}/images/{}".format(
product_id, image_id))
return r.json()
def add_image(self, product_id, image_id, add_namespace=False, **kwargs):
"""Add an image metadata entry to a product.
:param str product_id: (Required) Product to which this image belongs.
:param str image_id: (Required) New image's id = <product_id>:<image_id>.
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
:param list(str) read: A list of groups, or user hashes to give read access to.
Defaults to the same as the parent product if not specified.
:param int absolute_orbit: Orbit number since mission start.
:param str acquired: Date the imagery was acquired
:param str archived: Date the imagery was archived.
:param float area: Surface area the image covers
:param float azimuth_angle:
:param float azimuth_angle_1:
:param list(float) bits_per_pixel: Average bits of data per pixel per band.
:param float bright_fraction: Fraction of the image that has reflectance greater than .4 in the blue band.
:param str bucket: Name of Google Cloud Bucket. Must be public bucket or Descartes Labs user bucket.
:param str catalog_id:
:param float cirrus_fraction: Fraction of pixel which are distorted by cirrus clouds.
:param float cloud_fraction: Fraction of pixels which are obscured by clouds.
:param float cloud_fraction_0: Fraction of pixels which are obscured by clouds.
:param str cs_code: Spatial coordinate system code eg. `EPSG:4326`
:param datastrip_id: ID of the data strip this image was taken in.
:param float degraded_fraction_0:
:param str descartes_version:
:param str directory: Subdirectory location.
:param float duration: duration of the scan in seconds
:param int duration_days:
:param float earth_sun_distance: Earth sun distance at time of image capture.
:param list(str) files: Names of the files this image is stored in.
:param list(str) file_md5s: File integrity checksums.
:param list(int) file_sizes: Number of bytes of each file
:param float fill_fraction: Fraction of this image which has data.
:param float geolocation_accuracy:
:param str|dict geometry: GeoJSON representation of the image coverage.
:param float geotrans: Geographic Translator values.
:param float gpt_time:
:param str identifier: Vendor scene identifier.
:param int ifg_tdelta_days:
:param float incidence_angle: Sensor incidence angle.
:param str pass: On which pass was this image taken.
:param str processed: Date which this image was processed.
:param str proj4: proj.4 transformation parameters
:param str projcs: Projection coordinate system.
:param str published: Date the image was published.
:param str radiometric_version:
:param list(int) raster_size: Dimensions of image in pixels in (width, height).
:param list(float) reflectance_scale: Scale factors converting TOA radiances to TOA reflectances
:param list(float) reflectance_scale_1:
:param int relative_orbit: Orbit number in revisit cycle.
:param float roll_angle:
:param str safe_id: Standard Archive Format for Europe.
:param str sat_id: Satellite identifier.
:param float scan_gap_interpolation:
:param float solar_azimuth_angle:
:param float solar_azimuth_angle_0:
:param float solar_azimuth_angle_1:
:param float solar_elevation_angle:
:param float solar_elevation_angle_0:
:param float solar_elevation_angle_1:
:param str tile_id:
:param float view_angle:
:param float view_angle_1:
:param list(str) writers: A list of groups, or user hashes to give read access to.
:param dict extra_properties: User defined custom properties for this image.
Only 10 keys are allowed. The dict can only map strings to primitive types (str -> str|float|int).
:return: JSON API representation of the image.
:rtype: dict
"""
check_deprecated_kwargs(kwargs, {
"bpp": "bits_per_pixel",
"key": "image_id"
})
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
kwargs["id"] = image_id
r = self.session.post("/products/{}/images".format(product_id),
json=kwargs)
return r.json()
def replace_image(self,
product_id,
image_id,
add_namespace=False,
**kwargs):
"""Replace image metadata with a new version.
:param str product_id: (Required) Product to which this image belongs.
:param str image_id: (Required) ID of the image to replace.
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
.. note::
- See :meth:`Catalog.add_image` for additional kwargs.
:return: JSON API representation of the image.
:rtype: dict
"""
check_deprecated_kwargs(kwargs, {"bpp": "bits_per_pixel"})
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
r = self.session.put("/products/{}/images/{}".format(
product_id, image_id),
json=kwargs)
return r.json()
def _add_core_image(self, product_id, image_id, **kwargs):
check_deprecated_kwargs(kwargs, {"bpp": "bits_per_pixel"})
kwargs["id"] = image_id
r = self.session.post("/core/products/{}/images".format(product_id),
json=kwargs)
return r.json()
def change_image(self,
product_id,
image_id,
add_namespace=False,
**kwargs):
"""Update an image metadata entry of a product.
:param str product_id: (Required) Product to which this image belongs.
:param str image_id: (Required) ID of the image to modify.
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
.. note::
- See :meth:`Catalog.add_image` for additional kwargs.
:return: JSON API representation of the image.
:rtype: dict
"""
check_deprecated_kwargs(kwargs, {"bpp": "bits_per_pixel"})
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
r = self.session.patch("/products/{}/images/{}".format(
product_id, image_id),
json=kwargs)
return r.json()
def remove_image(self, product_id, image_id, add_namespace=False):
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
self.session.delete("/products/{}/images/{}".format(
product_id, image_id))
def upload_image(self,
files,
product_id,
metadata=None,
multi=False,
image_id=None,
add_namespace=False,
**kwargs):
"""Upload an image for a product you own.
This is an asynchronous operation and you can query for the status
using `Catalog.upload_result()` with the upload_id returned by this
method. The upload id is the image_id, which defaults to the
name of the file to be uploaded.
:param str|file|list(str)|list(file) files: (Required) a reference to the file to upload.
:param str product_id: (Required) The id of the product this image belongs to.
:param dict metadata: Image metadata to use instead of the computed default values.
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
:return: The upload id.
:rtype: str
.. note::
- See :meth:`Catalog.add_image` for additional kwargs.
"""
if metadata is None:
metadata = {}
metadata.update(kwargs)
check_deprecated_kwargs(metadata, {"bpp": "bits_per_pixel"})
if multi is True:
if not hasattr(files, "__iter__"):
raise ValueError(
"Using `multi=True` requires `files` to be iterable")
elif image_id is None:
raise ValueError(
"Using `multi=True` requires `image_id` to be specified")
else:
failed, upload_id, error = self._do_multi_file_upload(
files,
product_id,
image_id,
metadata,
add_namespace=add_namespace)
else:
failed, upload_id, error = self._do_upload(
files,
product_id,
metadata=metadata,
add_namespace=add_namespace)
if failed:
raise error
return upload_id
def upload_ndarray(self,
ndarray,
product_id,
image_id,
proj4=None,
wkt_srs=None,
geotrans=None,
raster_meta=None,
overviews=None,
overview_resampler=None,
add_namespace=False,
**kwargs):
"""Upload an ndarray with georeferencing information.
This is an asynchronous operation and you can query for the status
using `Catalog.upload_result()` with the upload_id returned by this
method. The upload id is the image_id.
:param ndarray ndarray: (Required) A numpy ndarray with image data. If you are providing a multi-band image
it should have 3 dimensions and the 3rd dimension of the array should index the bands. The dtype of the
ndarray must also be one of the following:
['uint8', 'int8', 'uint16', 'int16', 'uint32', 'int32', 'float32', 'float64']
:param str product_id: (Required) The id of the product this image belongs to.
:param str image_id: (Required) Resulting image's id = <product_id>:<image_id>.
:param str proj4: (One of proj4 or wkt_srs is required) A proj4 formatted string representing the
spatial reference system used by the image.
:param str wkt_srs: (One of proj4 or wkt_srs is required) A well known text string representing the
spatial reference system used by the image.
:param list(float) geotrans: (Required) The 6 number geographic transform of the image. Maps pixel coordinates
to coordinates in the specified spatial reference system.
:param dict raster_meta: Metadata returned from the :meth:`descarteslabs.client.services.raster.Raster.ndarray`
request which generated the initial data for the :param ndarray: being uploaded. Passing :param geotrans:
and :param wkt_srs: is unnecessary in this case.
:param list(int) overviews: a list of overview resolution magnification factors i.e [2, 4] would make two
overviews at 2x and 4x the native resolution. Maximum number of overviews allowed is 16.
:param str overview_resampler: Resampler algorithm to use when building overviews. Controls how pixels are
combined to make lower res pixels in overviews. Allowed resampler algorithms are:
['nearest', 'average', 'gauss', 'cubic', 'cubicspline', 'lanczos', 'average_mp',
'average_magphase', 'mode'].
:param bool add_namespace: Add your user namespace to the product_id. *Deprecated*
:return: The upload id.
:rtype: str
.. note::
- See :meth:`Catalog.add_image` for additional kwargs.
- Only one of `proj4` or `wkt_srs` can be provided.
"""
if ndarray.dtype.name not in self.UPLOAD_NDARRAY_SUPPORTED_DTYPES:
raise TypeError("{} is not in supported dtypes {}".format(
ndarray.dtype.name, self.UPLOAD_NDARRAY_SUPPORTED_DTYPES))
metadata = kwargs
metadata.setdefault("process_controls",
{}).update({"upload_type": "ndarray"})
if raster_meta is not None:
geotrans = raster_meta.get("geoTransform")
wkt_srs = raster_meta.get("coordinateSystem", {}).get("wkt")
for arg in ["image_id", "proj4", "wkt_srs", "geotrans"]:
if locals()[arg] is not None:
kwargs[arg] = locals()[arg]
for arg in ["overviews", "overview_resampler"]:
if locals()[arg] is not None:
metadata["process_controls"][arg] = locals()[arg]
with NamedTemporaryFile(delete=False) as tmp:
try:
np.save(tmp, ndarray, allow_pickle=False)
# From tempfile docs:
# Whether the name can be used to open the file a second time,
# while the named temporary file is still open, varies across
# platforms (it can be so used on Unix; it cannot on Windows
# NT or later)
#
# We close the underlying file object so _do_upload can open
# the path again in a cross platform compatible way.
# Cleanup is manual in the finally block.
tmp.close()
failed, upload_id, error = self._do_upload(
tmp.name,
product_id,
metadata=metadata,
add_namespace=add_namespace)
if failed:
raise error
return upload_id
finally:
os.unlink(tmp.name)
def upload_results(
self,
product_id,
limit=100,
offset=None,
status=None,
updated=None,
created=None,
continuation_token=None,
):
"""Get result information for debugging your uploads.
The upload id is included in the ``labels`` property.
:param str product_id: Product ID to get upload results for.
:param int limit: Number of results to get, useful for paging.
:param int offset: Start of results to get, useful for paging.
:param str status: Filter results by status, values are ["SUCCESS", "FAILURE"]
:param str|int updated: Unix timestamp or ISO8601 formatted date for filtering results updated after this time.
:param str|int created: Unix timestamp or ISO8601 formatted date for filtering results created after this time.
:return: A list of upload result objects.
:rtype: list
"""
kwargs = {"limit": limit}
for arg in [
"offset", "status", "updated", "created", "continuation_token"
]:
if locals()[arg] is not None:
kwargs[arg] = locals()[arg]
results = self.session.post("/products/{}/uploads".format(product_id),
json=kwargs)
return results.json()
def iter_upload_results(self,
product_id,
status=None,
updated=None,
created=None):
"""Get result information for debugging your uploads.
The upload id is included in the ``labels`` property.
:param str product_id: Product ID to get upload results for.
:param str status: Filter results by status, values are ["SUCCESS", "FAILURE"]
:param str|int updated: Unix timestamp or ISO8601 formatted date for filtering results updated after this time.
:param str|int created: Unix timestamp or ISO8601 formatted date for filtering results created after this time.
:return: iterator to upload results.
:rtype: generator
"""
continuation_token = None
kwargs = {}
for arg in ["status", "updated", "created"]:
if locals()[arg] is not None:
kwargs[arg] = locals()[arg]
while True:
page = self.upload_results(product_id,
continuation_token=continuation_token,
**kwargs)
for res in page["data"]:
yield res
continuation_token = page["meta"]["continuation_token"]
if continuation_token is None:
break
def upload_result(self, product_id, upload_id):
"""Get one upload result with the processing logs.
This is useful for debugging failed uploads.
:param str product_id: Product ID to get upload result for.
:param str upload_id: ID of specific upload to get a result record of, includes the run logs.
:return: One upload result with run logs.
:rtype: dict
"""
result = self.session.get("/products/{}/uploads/{}".format(
product_id, upload_id))
return result.json()
def _do_multi_file_upload(self,
files,
product_id,
image_id,
metadata,
add_namespace=False):
file_keys = [os.path.basename(_f) for _f in files]
process_controls = metadata.setdefault("process_controls",
{"upload_type": "file"})
multi_file_args = {
"multi_file": {
"image_files": file_keys,
"image_id": image_id
}
}
process_controls.update(multi_file_args)
for _file in files:
failed, upload_id, error = self._do_upload(
_file,
product_id,
metadata=metadata,
add_namespace=add_namespace)
if failed:
break
return failed, upload_id, error
def _do_upload(self,
file_ish,
product_id,
metadata=None,
add_namespace=False):
# kwargs are treated as metadata fields and restricted to primitives
# for the key val pairs.
fd = None
upload_id = None
if add_namespace:
check_deprecated_kwargs(locals(), {"add_namespace": None})
product_id = self.namespace_product(product_id)
if metadata is None:
metadata = {}
metadata.setdefault("process_controls", {"upload_type": "file"})
check_deprecated_kwargs(metadata, {"bpp": "bits_per_pixel"})
if not isinstance(product_id, six.string_types):
raise TypeError("product_id={} is invalid. "
"product_id must be a string.".format(product_id))
if isinstance(file_ish, io.IOBase):
if "b" not in file_ish.mode:
file_ish.close()
file_ish = io.open(file_ish.name, "rb")
fd = file_ish
elif isinstance(file_ish,
six.string_types) and os.path.exists(file_ish):
fd = io.open(file_ish, "rb")
else:
e = Exception("Could not handle file: `{}` pass a valid path "
"or open IOBase file".format(file_ish))
return True, upload_id, e
try:
upload_id = metadata.pop("image_id", None) or os.path.basename(
fd.name)
r = self.session.post(
"/products/{}/images/upload/{}".format(product_id, upload_id),
json=metadata,
)
upload_url = r.text
r = self._gcs_upload_service.session.put(upload_url, data=fd)
except (ServerError, RequestException) as e:
return True, upload_id, e
except NotFoundError as e:
raise NotFoundError(
"Make sure product_id exists in the catalog before \
attempting to upload data. %s" % e.message)
finally:
fd.close()
return False, upload_id, None
from descarteslabs.client.services.raster import Raster
from descarteslabs.client.services.catalog import Catalog
from descarteslabs.client.services.metadata import Metadata
raster = Raster()
catalog = Catalog()
metadata = Metadata()
id_ = '5151d2825f5e29ff129f86d834946363ff3f7e57:modis:09:CREFL_v2_test:2017-01-01-1835_11N_07_MO_09_v2'
scene_meta = metadata.get(id_)
r, meta = raster.ndarray(id_, bands=['red'])
p = catalog.add_product('test_nda_upload',
description="Test uploading georeferenced ndarrays",
title="Test ndarray upload")
band_spec = meta['bands'][0]['description']
catalog.add_band(
p['data']['id'],
'red',
srcband=1,
jpx_layer=0,
**{
k: v
for k, v in band_spec.items()
if k not in ['product', 'id', 'name', 'read', 'owner', 'owner_type']
})
catalog.upload_ndarray(r,
p['data']['id'],
id_[41:],
class MetadataTest(unittest.TestCase):
def setUp(self):
self.url = "http://example.com/metadata"
self.instance = Metadata(url=self.url, auth=Auth(jwt_token=public_token, token_info_path=None))
self.instance._raster = Raster(url=self.url, auth=self.instance.auth)
self.match_url = re.compile(self.url)
def mock_response(self, method, json, status=200, **kwargs):
responses.add(method, self.match_url, json=json, status=status, **kwargs)
def test_expr_serialization(self):
p = self.instance.properties
q = ((0.1 < p.cloud_fraction <= 0.2) & (p.sat_id == "f00b")) | (
p.sat_id == "usa-245"
)
expected_q = {
"or": [
{
"and": [
{"range": {"cloud_fraction": {"gt": 0.1, "lte": 0.2}}},
{"eq": {"sat_id": "f00b"}},
]
},
{"eq": {"sat_id": "usa-245"}},
]
}
self.assertEqual(q.serialize(), expected_q)
@responses.activate
def test_paged_search(self):
features = [{"id": "foo"}]
token = "token"
self.mock_response(responses.POST, json=features, headers={"x-continuation-token": token})
collection = self.instance.paged_search(limit=100)
self.assertEqual(features, collection.features)
self.assertEqual(token, collection.properties.continuation_token)
@responses.activate
def test_paged_search_deprecated_args(self):
features = [{"id": "foo"}]
self.mock_response(responses.POST, json=features)
with warnings.catch_warnings(record=True) as w:
collection = self.instance.paged_search(limit=100, start_time="2017-07-08")
self.assertEqual(1, len(w))
self.assertEqual(w[0].category, DeprecationWarning)
self.assertEqual(features, collection.features)
@responses.activate
def test_paged_search_dltile(self):
features = [{"id": "foo"}]
tile_geom = {
"coordinates": [[[-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606]]],
"type": "Polygon"
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=features)
collection = self.instance.paged_search(limit=100, dltile="256:16:30.0:15:-11:591")
self.assertEqual(features, collection.features)
@responses.activate
def test_search(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features)
collection = self.instance.search(limit=2)
self.assertEqual(features[:2], collection.features)
@responses.activate
def test_features(self):
features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}]
self.mock_response(responses.POST, json=features[:2], headers={"x-continuation-token": "token"})
self.mock_response(responses.POST, json=features[2:], headers={"x-continuation-token": "token2"})
# Note: Unfortunately the client has historically been written in such a way that it always
# expects a token header, even if the end of the search was reached, so an extra request
# with 0 results happens in practice.
self.mock_response(responses.POST, json=[], headers={"x-continuation-token": "token3"})
self.assertEqual(features, list(self.instance.features()))
@responses.activate
def test_summary(self):
summary = {"count": 42}
self.mock_response(responses.POST, json=summary)
self.assertEqual(summary, self.instance.summary())
@responses.activate
def test_summary_dltile(self):
summary = {"count": 42}
tile_geom = {
"coordinates": [[[-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176],
[-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104],
[-94.01008346640455, 40.992358024242606]]],
"type": "Polygon"
}
self.mock_response(responses.GET, json={"geometry": tile_geom})
self.mock_response(responses.POST, json=summary)
self.assertEqual(summary, self.instance.summary(dltile="256:16:30.0:15:-11:591"))
def search(
aoi,
products=None,
start_datetime=None,
end_datetime=None,
cloud_fraction=None,
storage_state=None,
limit=100,
sort_field=None,
sort_order="asc",
date_field="acquired",
query=None,
randomize=False,
raster_client=None,
metadata_client=None,
):
"""
Search for Scenes in the Descartes Labs catalog.
Returns a SceneCollection of Scenes that overlap with an area of interest,
and meet the given search criteria.
Parameters
----------
aoi : GeoJSON-like dict, :class:`~descarteslabs.scenes.geocontext.GeoContext`, or object with __geo_interface__
Search for scenes that intersect this area by any amount.
If a :class:`~descarteslabs.scenes.geocontext.GeoContext`, a copy is returned as ``ctx``, with missing values
filled in. Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry.
products : str or List[str], optional
Descartes Labs product identifiers
start_datetime : str, datetime-like, optional
Restrict to scenes acquired after this datetime
end_datetime : str, datetime-like, optional
Restrict to scenes acquired before this datetime
cloud_fraction : float, optional
Restrict to scenes that are covered in clouds by less than this fraction
(between 0 and 1)
storage_state : str, optional
Filter results based on ``storage_state`` value
(``"available"``, ``"remote"``, or ``None``)
limit : int or None, optional, default 100
Maximum number of Scenes to return, or None for all results.
sort_field : str, optional
Field name in :py:attr:`Scene.properties` by which to order the results
sort_order : str, optional, default 'asc'
``"asc"`` or ``"desc"``
date_field : str, optional, default 'acquired'
The field used when filtering by date
(``"acquired"``, ``"processed"``, ``"published"``)
query : ~descarteslabs.common.property_filtering.filtering.Expression, optional
Expression used to filter Scenes by their properties, built from
:class:`dl.properties <descarteslabs.common.property_filtering.filtering.GenericProperties>`.
You can construct filter expression using the ``==``, ``!=``, ``<``, ``>``,
``<=`` and ``>=`` operators as well as the
:meth:`~descarteslabs.common.property_filtering.filtering.Property.like`
and :meth:`~descarteslabs.common.property_filtering.filtering.Property.in_`
methods. You cannot use the boolean keywords ``and`` and ``or`` because of
Python language limitations; instead you can combine filter expressions
with ``&`` (boolean "and") and ``|`` (boolean "or").
Example:
``150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5``
randomize : bool, default False, optional
Randomize the order of the results.
You may also use an int or str as an explicit seed.
raster_client : Raster, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scenes : `SceneCollection`
Scenes matching your criteria.
ctx: :class:`~descarteslabs.scenes.geocontext.GeoContext`
The given ``aoi`` as a :class:`~descarteslabs.scenes.geocontext.GeoContext` (if it isn't one already),
with reasonable default parameters for loading all matching Scenes.
If ``aoi`` was a :class:`~descarteslabs.scenes.geocontext.GeoContext`, ``ctx`` will be a copy of ``aoi``,
with any properties that were ``None`` assigned the defaults below.
If ``aoi`` was not a :class:`~descarteslabs.scenes.geocontext.GeoContext`, an `AOI` instance will be created
with ``aoi`` as its geometry, and defaults assigned as described below:
**Default Spatial Parameters:**
* resolution: the finest resolution of any band of all matching scenes
* crs: the most common CRS used of all matching scenes
"""
if isinstance(aoi, geocontext.GeoContext):
ctx = aoi
if ctx.bounds is None and ctx.geometry is None:
raise ValueError(
"Unspecified where to search, "
"since the GeoContext given for ``aoi`` has neither geometry nor bounds set"
)
else:
ctx = geocontext.AOI(geometry=aoi)
if raster_client is None:
raster_client = Raster()
if metadata_client is None:
metadata_client = Metadata()
if isinstance(products, six.string_types):
products = [products]
if isinstance(start_datetime, datetime.datetime):
start_datetime = start_datetime.isoformat()
if isinstance(end_datetime, datetime.datetime):
end_datetime = end_datetime.isoformat()
metadata_params = dict(
products=products,
geom=ctx.__geo_interface__,
start_datetime=start_datetime,
end_datetime=end_datetime,
cloud_fraction=cloud_fraction,
storage_state=storage_state,
limit=limit,
sort_field=sort_field,
sort_order=sort_order,
date=date_field,
q=query,
randomize=randomize,
)
metadata = metadata_client.search(**metadata_params)
if products is None:
products = {
meta["properties"]["product"]
for meta in metadata["features"]
}
product_bands = {
product:
Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product))
for product in products
}
scenes = SceneCollection(
(Scene(meta, product_bands[meta["properties"]["product"]])
for meta in metadata["features"]),
raster_client=raster_client,
)
if len(scenes) > 0 and isinstance(ctx, geocontext.AOI):
assign_ctx = {}
if ctx.resolution is None and ctx.shape is None:
resolutions = filter(
None,
(b.get("resolution") for band in six.itervalues(product_bands)
for b in six.itervalues(band)),
)
try:
assign_ctx["resolution"] = min(resolutions)
except ValueError:
assign_ctx[
"resolution"] = None # from min of an empty sequence; no band defines resolution
if ctx.crs is None:
assign_ctx["crs"] = collections.Counter(
scene.properties["crs"]
for scene in scenes).most_common(1)[0][0]
if len(assign_ctx) > 0:
ctx = ctx.assign(**assign_ctx)
return scenes, ctx
