def from_id(cls, scene_id, metadata_client=None): """ Return the metadata for a Descartes Labs scene ID as a Scene object. Also returns a :class:`~descarteslabs.scenes.geocontext.GeoContext` for loading the Scene's original, unwarped data. Parameters ---------- scene_id: str Descartes Labs scene ID, e.g. "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1" metadata_client : Metadata, optional Unneeded in general use; lets you use a specific client instance with non-default auth and parameters. Returns ------- scene: Scene Scene instance with metadata loaded from the Descartes Labs catalog ctx: AOI A :class:`~descarteslabs.scenes.geocontext.GeoContext`for loading this Scene's original data. The defaults used are described in `Scene.default_ctx`. Example ------- >>> import descarteslabs as dl >>> scene, ctx = dl.scenes.Scene.from_id("landsat:LC08:PRE:TOAR:meta_LC80260322016197_v1") # doctest: +SKIP >>> ctx # doctest: +SKIP AOI(geometry=None, resolution=15.0, crs='EPSG:32615', align_pixels=False, bounds=(348592.5, 4345567.5, 581632.5, 4582807.5), bounds_crs='EPSG:32615', shape=None) >>> scene.properties.date # doctest: +SKIP datetime.datetime(2016, 7, 15, 16, 53, 59, 495435) Raises ------ NotFoundError If the ``scene_id`` cannot be found in the Descartes Labs catalog """ if metadata_client is None: metadata_client = Metadata() metadata = metadata_client.get(scene_id) metadata = { "type": "Feature", "geometry": metadata.pop("geometry"), "id": metadata.pop("id"), "key": metadata.pop("key"), "properties": metadata } bands = metadata_client.get_bands_by_id(scene_id) scene = cls(metadata, bands) return scene, scene.default_ctx()
def setUp(self): self.url = "http://example.com/metadata" self.instance = Metadata(url=self.url, auth=Auth(jwt_token=public_token, token_info_path=None)) self.instance._raster = Raster(url=self.url, auth=self.instance.auth) self.match_url = re.compile(self.url)
def __init__(self, url=None, auth=None, metadata=None): """The parent Service class implements authentication and exponential backoff/retry. Override the url parameter to use a different instance of the backing service. """ if auth is None: auth = Auth() if metadata is None: self._metadata = Metadata(auth=auth) else: self._metadata = metadata self._gcs_upload_service = ThirdPartyService() if url is None: url = os.environ.get( "DESCARTESLABS_CATALOG_URL", "https://platform.descarteslabs.com/metadata/v1/catalog") super(Catalog, self).__init__(url, auth=auth)
def setUpClass(cls): cls.instance = Metadata()
import datetime import collections import textwrap import warnings import shapely.geometry import numpy as np from descarteslabs.common.dotdict import DotDict from descarteslabs.scenes import Scene, geocontext from descarteslabs.scenes.scene import _strptime_helper from descarteslabs.client.services.metadata import Metadata from .mock_data import _metadata_get, _metadata_get_bands, _raster_ndarray metadata_client = Metadata() class MockScene(Scene): "Circumvent __init__ method to create a Scene with arbitrary geometry and properties objects" def __init__(self, geometry, properties): self.geometry = DotDict(geometry) self.properties = DotDict(properties) class TestScene(unittest.TestCase): @mock.patch("descarteslabs.client.services.metadata.Metadata.get", _metadata_get) @mock.patch("descarteslabs.client.services.metadata.Metadata.get_bands_by_id", _metadata_get_bands) def test_init(self): scene_id = "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1" metadata = metadata_client.get(scene_id)
def auth_handler(args): auth = Auth() if args.command == 'login': print( 'Follow this link to login https://iam.descarteslabs.com/auth/login?refresh_token=true&destination=/auth/refresh_token' ) # NOQA s = input('...then come back here and paste the generated token: ') if isinstance(s, six.text_type): s = s.encode('utf-8') if s: token_info = json.loads(base64url_decode(s).decode('utf-8')) token_info_directory = os.path.dirname(DEFAULT_TOKEN_INFO_PATH) makedirs_if_not_exists(token_info_directory) os.chmod(token_info_directory, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) with open(DEFAULT_TOKEN_INFO_PATH, 'w+') as fp: json.dump(token_info, fp) os.chmod(DEFAULT_TOKEN_INFO_PATH, stat.S_IRUSR | stat.S_IWUSR) # Get a fresh Auth token auth = Auth() keys = Metadata(auth=auth).keys() name = auth.payload['name'] groups = ', '.join(auth.payload['groups']) if len(keys): print('Welcome, %s!' % name) else: print( 'Welcome, %s! Your %s role(s) do not permit access to any imagery at this time.' % (name, groups)) print( 'Contact [email protected] if you believe you received this message in error or have any questions.' ) # NOQA if args.command == 'token': print(auth.token) if args.command == 'name': auth.token print(auth.payload['name']) if args.command == 'groups': auth.token print(json.dumps(auth.payload['groups'])) if args.command == 'payload': auth.token print(auth.payload) if args.command == 'env': auth.token print('%s=%s' % ('CLIENT_ID', auth.client_id)) print('%s=%s' % ('CLIENT_SECRET', auth.client_secret)) if args.command == 'version': print(__version__)
class MetadataTest(unittest.TestCase): def setUp(self): self.url = "http://example.com/metadata" self.instance = Metadata(url=self.url, auth=Auth(jwt_token=public_token, token_info_path=None)) self.instance._raster = Raster(url=self.url, auth=self.instance.auth) self.match_url = re.compile(self.url) def mock_response(self, method, json, status=200, **kwargs): responses.add(method, self.match_url, json=json, status=status, **kwargs) def test_expr_serialization(self): p = self.instance.properties q = ((0.1 < p.cloud_fraction <= 0.2) & (p.sat_id == "f00b")) | (p.sat_id == "usa-245") expected_q = { "or": [ { "and": [ { "range": { "cloud_fraction": { "gt": 0.1, "lte": 0.2 } } }, { "eq": { "sat_id": "f00b" } }, ] }, { "eq": { "sat_id": "usa-245" } }, ] } assert q.serialize() == expected_q def test_expr_contains(self): p = self.instance.properties q = p.sat_id.in_(("usa-245", "terra")) expected_q = { "or": [{ "eq": { "sat_id": "usa-245" } }, { "eq": { "sat_id": "terra" } }] } assert q.serialize() == expected_q @responses.activate def test_paged_search(self): features = [{"id": "foo"}] token = "token" self.mock_response(responses.POST, json=features, headers={"x-continuation-token": token}) collection = self.instance.paged_search(limit=100) assert features == collection.features assert token == collection.properties.continuation_token @responses.activate def test_paged_search_deprecated_args(self): features = [{"id": "foo"}] self.mock_response(responses.POST, json=features) with warnings.catch_warnings(record=True) as w: collection = self.instance.paged_search(limit=100, start_time="2017-07-08") assert 1 == len(w) assert w[0].category == DeprecationWarning assert features == collection.features @responses.activate def test_paged_search_dltile(self): features = [{"id": "foo"}] tile_geom = { "coordinates": [[ [-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176], [-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104], [-94.01008346640455, 40.992358024242606], ]], "type": "Polygon", } self.mock_response(responses.GET, json={"geometry": tile_geom}) self.mock_response(responses.POST, json=features) collection = self.instance.paged_search( limit=100, dltile="256:16:30.0:15:-11:591") assert features == collection.features @responses.activate def test_paged_search_shapely(self): features = [{"id": "foo"}] geom = { "coordinates": [[ [-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176], [-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104], [-94.01008346640455, 40.992358024242606], ]], "type": "Polygon", } shape_geom = shape(geom) self.mock_response(responses.GET, json={"geometry": geom}) self.mock_response(responses.POST, json=features) collection = self.instance.paged_search(limit=100, geom=shape_geom) assert features == collection.features @responses.activate def test_search(self): features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}] self.mock_response(responses.POST, json=features) collection = self.instance.search(limit=2) req = responses.calls[0].request assert "storage_state" not in json.loads(req.body.decode("utf-8")) assert features[:2] == collection.features @responses.activate def test_search_storage_state(self): features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}] self.mock_response(responses.POST, json=features) collection = self.instance.search(limit=2, storage_state="available") assert features[:2] == collection.features req = responses.calls[0].request assert "storage_state" in json.loads(req.body.decode("utf-8")) @responses.activate def test_features(self): features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}] self.mock_response(responses.POST, json=features[:2], headers={"x-continuation-token": "token"}) self.mock_response( responses.POST, json=features[2:], headers={"x-continuation-token": "token2"}, ) # Note: Unfortunately the client has historically been written in such a way that it always # expects a token header, even if the end of the search was reached, so an extra request # with 0 results happens in practice. self.mock_response(responses.POST, json=[], headers={"x-continuation-token": "token3"}) assert features == list(self.instance.features()) req = responses.calls[0].request assert "storage_state" not in json.loads(req.body.decode("utf-8")) @responses.activate def test_summary_default(self): summary = {"count": 42} self.mock_response(responses.POST, json=summary) assert summary == self.instance.summary() req = responses.calls[0].request assert "storage_state" not in json.loads(req.body.decode("utf-8")) @responses.activate def test_summary_storage_state(self): summary = {"count": 42} self.mock_response(responses.POST, json=summary) assert summary == self.instance.summary(storage_state="available") expected_req = {"date": "acquired", "storage_state": "available"} req = responses.calls[0].request assert json.loads(req.body.decode("utf-8")) == expected_req @responses.activate def test_summary_dltile(self): summary = {"count": 42} tile_geom = { "coordinates": [[ [-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176], [-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104], [-94.01008346640455, 40.992358024242606], ]], "type": "Polygon", } self.mock_response(responses.GET, json={"geometry": tile_geom}) self.mock_response(responses.POST, json=summary) assert summary == self.instance.summary( dltile="256:16:30.0:15:-11:591") @responses.activate def test_summary_shapely(self): summary = {"count": 42} geom = { "coordinates": [[ [-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176], [-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104], [-94.01008346640455, 40.992358024242606], ]], "type": "Polygon", } shape_geom = shape(geom) self.mock_response(responses.GET, json={"geometry": geom}) self.mock_response(responses.POST, json=summary) assert summary == self.instance.summary(geom=shape_geom)
def from_id(cls, scene_id, metadata_client=None): """ Return the metadata for a Descartes Labs scene ID as a Scene object. Also returns a GeoContext with reasonable defaults to use when loading the Scene's ndarray. Parameters ---------- scene_id: str Descartes Labs scene ID, e.g. "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1" metadata_client : Metadata, optional Unneeded in general use; lets you use a specific client instance with non-default auth and parameters. Returns ------- scene: Scene Scene instance with metadata loaded from the Descartes Labs catalog ctx: AOI A default GeoContext useful for loading this Scene. These defaults are used: * bounds: bounds of the Scene's geometry * resolution: the finest resolution of any band in the scene * crs: native CRS of the Scene (generally, a UTM CRS) * align_pixels: True Example ------- >>> import descarteslabs as dl >>> scene, ctx = dl.scenes.Scene.from_id("landsat:LC08:PRE:TOAR:meta_LC80260322016197_v1") >>> ctx AOI(geometry=None, resolution=15, crs='EPSG:32615', align_pixels=True, bounds=(-94.724166, 39.2784859, -92.0686956, 41.3717716), shape=None) >>> scene.properties.date datetime.datetime(2016, 7, 15, 16, 53, 59, 495435) Raises ------ NotFoundError If the ``scene_id`` cannot be found in the Descartes Labs catalog """ if metadata_client is None: metadata_client = Metadata() metadata = metadata_client.get(scene_id) metadata = { "type": "Feature", "geometry": metadata.pop("geometry"), "id": metadata.pop("id"), "key": metadata.pop("key"), "properties": metadata } bands = metadata_client.get_bands_by_id(scene_id) scene = cls(metadata, bands) # TODO: not sure what default res should be try: default_resolution = min( filter(None, [b.get("resolution") for b in six.itervalues(bands)])) except ValueError: default_resolution = 100 # QUESTION: default bounds will now be in WGS84, not UTM # indeed, there's no way to give bounds in UTM besides with a DLTile, # which means you could get off-by-one issues with loading an entire scene # at native resolution, where the WGS84 bounds result in a slightly differently # sized raster than native UTM bounds would with reprojection errors try: bounds = scene.geometry.bounds except AttributeError: xs, ys = zip(*scene.geometry["coordinates"][0]) bounds = (min(xs), min(ys), max(xs), max(ys)) default_ctx = geocontext.AOI(bounds=bounds, resolution=default_resolution, crs=scene.properties["crs"]) return scene, default_ctx
# Copyright 2018 Descartes Labs. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # flake8: noqa from descarteslabs.client.auth import Auth from descarteslabs.client.services.metadata import Metadata from descarteslabs.client.services.places import Places from descarteslabs.client.services.raster import Raster descartes_auth = Auth.from_environment_or_token_json() metadata = Metadata(auth=descartes_auth) places = Places(auth=descartes_auth) raster = Raster(auth=descartes_auth) __all__ = ["descartes_auth", "metadata", "places", "raster"]
def search(aoi, products=None, start_datetime=None, end_datetime=None, cloud_fraction=None, limit=100, sort_field=None, sort_order='asc', date_field='acquired', query=None, randomize=False, raster_client=None, metadata_client=None): """ Search for Scenes in the Descartes Labs catalog. Returns a SceneCollection of Scenes that overlap with an area of interest, and meet the given search criteria. Parameters ---------- aoi : GeoJSON-like dict, GeoContext, or object with __geo_interface__ Search for scenes that intersect this area by any amount. If a GeoContext, a copy is returned as ``ctx``, with missing values filled in. Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry. products : str or List[str], optional Descartes Labs product identifiers start_datetime : str, datetime-like, optional Restrict to scenes acquired after this datetime end_datetime : str, datetime-like, optional Restrict to scenes acquired before this datetime cloud_fraction : float, optional Restrict to scenes that are covered in clouds by less than this fraction (between 0 and 1) limit : int, optional Maximum number of Scenes to return, up to 10000. sort_field : str, optional Field name in ``Scene.properties`` by which to order the results sort_order : str, optional, default 'asc' ``"asc"`` or ``"desc"`` date_field : str, optional, default 'acquired' The field used when filtering by date (``"acquired"``, ``"processed"``, ``"published"``) query : descarteslabs.common.property_filtering.Expression, optional Expression used to filter Scenes by their properties, built from ``dl.properties``. >>> query = 150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5 >>> query = dl.properties.sat_id == "Terra" randomize : bool, default False, optional Randomize the order of the results. You may also use an int or str as an explicit seed. raster_client : Raster, optional Unneeded in general use; lets you use a specific client instance with non-default auth and parameters. metadata_client : Metadata, optional Unneeded in general use; lets you use a specific client instance with non-default auth and parameters. Returns ------- scenes : SceneCollection Scenes matching your criteria. ctx: GeoContext The given ``aoi`` as a GeoContext (if it isn't one already), with reasonable default parameters for loading all matching Scenes. If ``aoi`` was a `GeoContext`, ``ctx`` will be a copy of ``aoi``, with any properties that were ``None`` assigned the defaults below. If ``aoi`` was not a `GeoContext`, an `AOI` instance will be created with ``aoi`` as its geometry, and defaults assigned as described below: **Default Spatial Parameters:** * resolution: the finest resolution of any band of all matching scenes * crs: the most common CRS used of all matching scenes """ if isinstance(aoi, geocontext.GeoContext): ctx = aoi if ctx.bounds is None and ctx.geometry is None: raise ValueError( "Unspecified where to search, " "since the GeoContext given for ``aoi`` has neither geometry nor bounds set" ) else: ctx = geocontext.AOI(geometry=aoi) if raster_client is None: raster_client = Raster() if metadata_client is None: metadata_client = Metadata() if isinstance(products, six.string_types): products = [products] if isinstance(start_datetime, datetime.datetime): start_datetime = start_datetime.isoformat() if isinstance(end_datetime, datetime.datetime): end_datetime = end_datetime.isoformat() if limit > MAX_RESULT_WINDOW: raise ValueError("Limit must be <= {}".format(MAX_RESULT_WINDOW)) metadata_params = dict(products=products, geom=ctx.__geo_interface__, start_datetime=start_datetime, end_datetime=end_datetime, cloud_fraction=cloud_fraction, limit=limit, sort_field=sort_field, sort_order=sort_order, date=date_field, q=query, randomize=randomize) metadata = metadata_client.search(**metadata_params) if products is None: products = { meta["properties"]["product"] for meta in metadata["features"] } product_bands = { product: Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product)) for product in products } scenes = SceneCollection( (Scene(meta, product_bands[meta["properties"]["product"]]) for meta in metadata["features"]), raster_client=raster_client) if len(scenes) > 0: assign_ctx = {} if ctx.resolution is None and ctx.shape is None: resolutions = filter(None, (b.get("resolution") for band in six.itervalues(product_bands) for b in six.itervalues(band))) try: assign_ctx["resolution"] = min(resolutions) except ValueError: assign_ctx[ "resolution"] = None # from min of an empty sequence; no band defines resolution if ctx.crs is None: assign_ctx["crs"] = collections.Counter( scene.properties["crs"] for scene in scenes).most_common(1)[0][0] if len(assign_ctx) > 0: ctx = ctx.assign(**assign_ctx) return scenes, ctx
class Catalog(Service): """The Descartes Labs (DL) Catalog allows you to add georeferenced raster products into the Descartes Labs Platform. Catalog products can be used in other DL services like Raster and Tasks and Metadata. The entrypoint for using catalog is creating a Product using :meth:`Catalog.add_product`. After creating a product you should add band(s) to it using :meth:`Catalog.add_band`, and upload imagery using :meth:`Catalog.upload_image`. Bands define where and how data is encoded into imagery files, and how it should be displayed. Images define metadata about a specific groups of pixels (in the form of images), their georeferencing, geometry, coordinate system, and other pertinent information. """ UPLOAD_NDARRAY_SUPPORTED_DTYPES = [ "uint8", "int8", "uint16", "int16", "uint32", "int32", "float32", "float64", ] TIMEOUT = (9.5, 30) def __init__(self, url=None, auth=None, metadata=None): """The parent Service class implements authentication and exponential backoff/retry. Override the url parameter to use a different instance of the backing service. """ if auth is None: auth = Auth() if metadata is None: self._metadata = Metadata(auth=auth) else: self._metadata = metadata self._gcs_upload_service = ThirdPartyService() if url is None: url = os.environ.get( "DESCARTESLABS_CATALOG_URL", "https://platform.descarteslabs.com/metadata/v1/catalog", ) super(Catalog, self).__init__(url, auth=auth) def namespace_product(self, product_id): namespace = self.auth.namespace if not product_id.startswith(namespace): product_id = "{}:{}".format(namespace, product_id) return product_id def own_products(self): return self._metadata.products(owner=self.auth.payload["sub"]) def own_bands(self): return self._metadata.bands(owner=self.auth.payload["sub"]) def _add_core_product(self, product_id, **kwargs): kwargs["id"] = product_id check_deprecated_kwargs(kwargs, { "start_time": "start_datetime", "end_time": "end_datetime" }) r = self.session.post("/core/products", json=kwargs) return r.json() def get_product(self, product_id, add_namespace=False): if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) r = self.session.get("/products/{}".format(product_id)) return r.json() def add_product(self, product_id, title, description, add_namespace=False, **kwargs): """Add a product to your catalog. :param str product_id: (Required) A unique name for this product. In the created product a namespace consisting of your user id (e.g. "ae60fc891312ggadc94ade8062213b0063335a3c:") or your organization id (e.g., "yourcompany:") will be prefixed to this, if it doesn't already have one, in order to make the id globally unique. :param str title: (Required) Official product title. :param str description: (Required) Information about the product, why it exists, and what it provides. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* :param list(str) read: A list of groups, or user hashes to give read access to. :param int spectral_bands: Number of spectral bands the product has. :param list(str) native_bands: A list of the names of the native bands of this product (most applicable to satellite sensors). :param str start_datetime: ISO8601 compliant date string, indicating start of product data. :param str end_datetime: ISO8601 compliant date string, indicating end of product data. :param str notes: Any notes to relay about the product. :param str orbit: Type of orbit (satellite only). :param str processing_level: Way in which raw data has been processed if any. :param str resolution: Pixel density of the data, provide units. :param str revisit: How often an AOI can expect updated data. :param str sensor: Name of the sensor used. :param str swath: How large an area the sensor captures at a given time. :param list(str) writers: A list of groups, or user hashes to give read access to. :return: JSON API representation of the product. :rtype: dict """ for k, v in locals().items(): if k in ["title", "description"]: if v is None: raise TypeError("required arg `{}` not provided".format(k)) kwargs[k] = v check_deprecated_kwargs(kwargs, { "start_time": "start_datetime", "end_time": "end_datetime" }) if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) kwargs["id"] = self.namespace_product(product_id) else: kwargs["id"] = product_id r = self.session.post("/products", json=kwargs) return r.json() def replace_product(self, product_id, title, description, add_namespace=False, set_global_permissions=False, **kwargs): """Replace a product in your catalog with a new version. :param str product_id: (Required) A unique name for this product. :param str title: (Required) Official product title. :param str description: (Required) Information about the product, why it exists, and what it provides. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* :param list(str) read: A list of groups, or user hashes to give read access to. :param int spectral_bands: Number of spectral bands the product has. :param list(str) native_bands: A list of the names of the native bands of this product (most applicable to satellite sensors). :param str start_datetime: ISO8601 compliant date string, indicating start of product data. :param str end_datetime: ISO8601 compliant date string, indicating end of product data. :param str notes: Any notes to relay about the product. :param str orbit: Type of orbit (satellite only). :param str processing_level: Way in which raw data has been processed if any. :param str resolution: Pixel density of the data, provide units. :param str revisit: How often an AOI can expect updated data. :param str sensor: Name of the sensor used. :param str swath: How large an area the sensor captures at a given time. :param list(str) writers: A list of groups, or user hashes to give read access to. :return: JSON API representation of the product. :rtype: dict """ for k, v in locals().items(): if k in ["title", "description"]: if v is None: raise TypeError("required arg `{}` not provided".format(k)) kwargs[k] = v check_deprecated_kwargs(kwargs, { "start_time": "start_datetime", "end_time": "end_datetime" }) if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) params = None if set_global_permissions is True: params = {"set_global_permissions": "true"} r = self.session.put("/products/{}".format(product_id), json=kwargs, params=params) return r.json() def change_product(self, product_id, add_namespace=False, set_global_permissions=False, **kwargs): """Update a product in your catalog. :param str product_id: (Required) The ID of the product to change. :param list(str) read: A list of groups, or user hashes to give read access to. :param int spectral_bands: Number of spectral bands the product has. :param list(str) native_bands: A list of the names of the native bands of this product (most applicable to satellite sensors). :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* :param bool set_global_permissions: Update permissions of all existing bands and products that belong to this product with the updated permission set specified in the `read` param. Default to false. :param str start_datetime: ISO8601 compliant date string, indicating start of product data. :param str end_datetime: ISO8601 compliant date string, indicating end of product data. :param str title: Official product title. :param str description: Information about the product, why it exists, and what it provides. :param str notes: Any notes to relay about the product. :param str orbit: Type of orbit (satellite only). :param str processing_level: Way in which raw data has been processed if any. :param str resolution: Pixel density of the data, provide units. :param str revisit: How often an AOI can expect updated data. :param str sensor: Name of the sensor used. :param str swath: How large an area the sensor captures at a given time. :param list(str) writers: A list of groups, or user hashes to give read access to. :return: JSON API representation of the product. :rtype: dict """ check_deprecated_kwargs(kwargs, { "start_time": "start_datetime", "end_time": "end_datetime" }) if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) params = None if set_global_permissions is True: params = {"set_global_permissions": "true"} r = self.session.patch("/products/{}".format(product_id), json=kwargs, params=params) return r.json() def remove_product(self, product_id, add_namespace=False, cascade=False): """Remove a product from the catalog. :param str product_id: ID of the product to remove. :param bool cascade: Force deletion of all the associated bands and images. BEWARE this cannot be undone. """ if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) params = {"cascade": cascade} r = self.session.delete("/products/{}".format(product_id), params=params) if r.headers["content-type"] == "application/json": return r.json() def product_deletion_status(self, deletion_task_id): """Get the status of a long running product deletion job. :param str deletion_task_id: deletion_task ID returned from a call to :meth:`Catalog.remove_product` with a deletion_token. :return: document with information about product deletion progress. :rtype: dict """ r = self.session.get( "/products/deletion_tasks/{}".format(deletion_task_id)) return r.json() def get_band(self, product_id, name, add_namespace=False): """Get a band by name. :return: JSON API representation of the band. :rtype: dict """ if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) r = self.session.get("/products/{}/bands/{}".format(product_id, name)) return r.json() def _add_core_band(self, product_id, name, srcband=None, dtype=None, nbits=None, data_range=None, type=None, **kwargs): for k, v in locals().items(): if k in [ "name", "data_range", "dtype", "nbits", "srcband", "type" ]: if v is None: raise TypeError("required arg `{}` not provided".format(k)) kwargs[k] = v kwargs["id"] = name r = self.session.post("/core/products/{}/bands".format(product_id), json=kwargs) return r.json() def add_band(self, product_id, name, add_namespace=False, srcband=None, dtype=None, nbits=None, data_range=None, type=None, **kwargs): """Add a data band to an existing product. :param str product_id: (Required) Product to which this band will belong. :param str name: (Required) Name of this band. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* :param int jpx_layer: If data is processed to JPEG2000, which layer is the band in. Defaults to 0. :param int srcband: (Required) The 1-based index of the band in the jpx_layer specified. :param int srcfile: If the product was processed into multiple files, which one is this in. Defaults to 0 (first file). :param str dtype: (Required) The data type used to store this band e.g Byte or Uint16 or Float32. :param int nbits: (Required) The number of bits of the dtype used to store this band. :param list(int) data_range: (Required) A list specifying the min and max values for the data in this band. :param str type: (Required) The data interpretation of the band. One of ['spectral', 'derived', 'mask', 'class'] :param int nodata: Pixel value indicating no data available. :param list(str) read: A list of groups, or user hashes to give read access to. Defaults to the same as the parent product if not specified. :param str color: The color interpretation of this band. One of ['Alpha', 'Black', 'Blue', 'Cyan', 'Gray', 'Green', 'Hue', 'Ligntness', 'Magenta', 'Palette', 'Red', 'Saturation', 'Undefined', 'YCbCr_Cb', 'YCbCr_Cr', 'YCbCr_Y', 'Yellow']. Must be 'Alpha' if `type` is 'mask'. Must be 'Palette' if `colormap_name` or `colormap` is set. :param str colormap_name: A named colormap to use for this band, one of ['plasma', 'magma', 'viridis', 'msw', 'inferno'] :param list(list(str)) colormap: A custom colormap to use for this band. Takes a list of lists, where each nested list is a 4-tuple of rgba values to map pixels whose value is the index of the tuple. i.e the colormap [["100", "20", "200", "255"]] would map pixels whose value is 0 in the original band, to the rgba color vector at colormap[0]. Less than 2^nbits 4-tuples may be provided, and omitted values will default map to black. :param str data_description: Description of band data. :param list(float) physical_range: If band represents a physical value e.g reflectance or radiance what are the possible range of those values. :param str data_unit: Units of the physical range e.g w/sr for radiance :param str data_unit_description: Description of the data unit. :param list(int) default_range: A good default scale for the band to use when rastering for display purposes. :param str description: Description of the band. :param str name_common: Standard name for band :param str name_vendor: What the vendor calls the band eg. B1 :param int vendor_order: The index of the band in the vendors band tables. Useful for referencing the band to other processing properties like surface reflectance. :param str processing_level: Description of how the band was processed if at all. :param int res_factor: Scaling of this band relative to the native band resolution. :param int resolution: Resolution of this band. :param str resolution_unit: Unit of the resolution. :param float wavelength_center: Center position of wavelength. :param float wavelength_fwhm: Full width at half maximum value of the wavelength spread. :param float wavelength_min: Minimum wavelength this band is sensitive to. :param float wavelength_max: Maximum wavelength this band is sensitive to. :param str wavelength_unit: Units the wavelength is expressed in. :param list(str) writers: A list of groups, or user hashes to give read access to. :return: JSON API representation of the band. :rtype: dict """ for k, v in locals().items(): if k in [ "name", "data_range", "dtype", "nbits", "srcband", "type" ]: if v is None: raise TypeError("required arg `{}` not provided".format(k)) kwargs[k] = v if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) check_deprecated_kwargs(kwargs, {"id": "name"}) r = self.session.post("/products/{}/bands".format(product_id), json=kwargs) return r.json() def replace_band(self, product_id, name, srcband=None, dtype=None, nbits=None, data_range=None, add_namespace=False, type=None, **kwargs): """Replaces an existing data band with a new document. :param str product_id: (Required) Product to which this band will belong. :param str name: (Required) Name of this band. .. note:: - See :meth:`Catalog.add_band` for additional kwargs. :return: JSON API representation of the band. :rtype: dict """ for k, v in locals().items(): if k in ["data_range", "dtype", "nbits", "srcband", "type"]: if v is None: raise TypeError("required arg `{}` not provided".format(k)) kwargs[k] = v if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) r = self.session.put("/products/{}/bands/{}".format(product_id, name), json=kwargs) return r.json() def change_band(self, product_id, name, add_namespace=False, **kwargs): """Update a data band of a product. :param str product_id: (Required) Product to which this band belongs. :param str name: Name or id of band to modify. .. note:: - See :meth:`Catalog.add_band` for additional kwargs. :return: JSON API representation of the band. :rtype: dict """ if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) r = self.session.patch("/products/{}/bands/{}".format( product_id, name), json=kwargs) return r.json() def remove_band(self, product_id, name, add_namespace=False): if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) self.session.delete("/products/{}/bands/{}".format(product_id, name)) def get_image(self, product_id, image_id, add_namespace=False): """Get a single image metadata entry. :return: JSON API representation of the image. :rtype: dict """ if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) r = self.session.get("/products/{}/images/{}".format( product_id, image_id)) return r.json() def add_image(self, product_id, image_id, add_namespace=False, **kwargs): """Add an image metadata entry to a product. :param str product_id: (Required) Product to which this image belongs. :param str image_id: (Required) New image's id = <product_id>:<image_id>. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* :param list(str) read: A list of groups, or user hashes to give read access to. Defaults to the same as the parent product if not specified. :param int absolute_orbit: Orbit number since mission start. :param str acquired: Date the imagery was acquired :param str archived: Date the imagery was archived. :param float area: Surface area the image covers :param float azimuth_angle: :param float azimuth_angle_1: :param list(float) bits_per_pixel: Average bits of data per pixel per band. :param float bright_fraction: Fraction of the image that has reflectance greater than .4 in the blue band. :param str bucket: Name of Google Cloud Bucket. Must be public bucket or Descartes Labs user bucket. :param str catalog_id: :param float cirrus_fraction: Fraction of pixel which are distorted by cirrus clouds. :param float cloud_fraction: Fraction of pixels which are obscured by clouds. :param float cloud_fraction_0: Fraction of pixels which are obscured by clouds. :param str cs_code: Spatial coordinate system code eg. `EPSG:4326` :param datastrip_id: ID of the data strip this image was taken in. :param float degraded_fraction_0: :param str descartes_version: :param str directory: Subdirectory location. :param float duration: duration of the scan in seconds :param int duration_days: :param float earth_sun_distance: Earth sun distance at time of image capture. :param list(str) files: Names of the files this image is stored in. :param list(str) file_md5s: File integrity checksums. :param list(int) file_sizes: Number of bytes of each file :param float fill_fraction: Fraction of this image which has data. :param float geolocation_accuracy: :param str|dict geometry: GeoJSON representation of the image coverage. :param float geotrans: Geographic Translator values. :param float gpt_time: :param str identifier: Vendor scene identifier. :param int ifg_tdelta_days: :param float incidence_angle: Sensor incidence angle. :param str pass: On which pass was this image taken. :param str processed: Date which this image was processed. :param str proj4: proj.4 transformation parameters :param str projcs: Projection coordinate system. :param str published: Date the image was published. :param str radiometric_version: :param list(int) raster_size: Dimensions of image in pixels in (width, height). :param list(float) reflectance_scale: Scale factors converting TOA radiances to TOA reflectances :param list(float) reflectance_scale_1: :param int relative_orbit: Orbit number in revisit cycle. :param float roll_angle: :param str safe_id: Standard Archive Format for Europe. :param str sat_id: Satellite identifier. :param float scan_gap_interpolation: :param float solar_azimuth_angle: :param float solar_azimuth_angle_0: :param float solar_azimuth_angle_1: :param float solar_elevation_angle: :param float solar_elevation_angle_0: :param float solar_elevation_angle_1: :param str tile_id: :param float view_angle: :param float view_angle_1: :param list(str) writers: A list of groups, or user hashes to give read access to. :param dict extra_properties: User defined custom properties for this image. Only 10 keys are allowed. The dict can only map strings to primitive types (str -> str|float|int). :return: JSON API representation of the image. :rtype: dict """ check_deprecated_kwargs(kwargs, { "bpp": "bits_per_pixel", "key": "image_id" }) if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) kwargs["id"] = image_id r = self.session.post("/products/{}/images".format(product_id), json=kwargs) return r.json() def replace_image(self, product_id, image_id, add_namespace=False, **kwargs): """Replace image metadata with a new version. :param str product_id: (Required) Product to which this image belongs. :param str image_id: (Required) ID of the image to replace. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* .. note:: - See :meth:`Catalog.add_image` for additional kwargs. :return: JSON API representation of the image. :rtype: dict """ check_deprecated_kwargs(kwargs, {"bpp": "bits_per_pixel"}) if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) r = self.session.put("/products/{}/images/{}".format( product_id, image_id), json=kwargs) return r.json() def _add_core_image(self, product_id, image_id, **kwargs): check_deprecated_kwargs(kwargs, {"bpp": "bits_per_pixel"}) kwargs["id"] = image_id r = self.session.post("/core/products/{}/images".format(product_id), json=kwargs) return r.json() def change_image(self, product_id, image_id, add_namespace=False, **kwargs): """Update an image metadata entry of a product. :param str product_id: (Required) Product to which this image belongs. :param str image_id: (Required) ID of the image to modify. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* .. note:: - See :meth:`Catalog.add_image` for additional kwargs. :return: JSON API representation of the image. :rtype: dict """ check_deprecated_kwargs(kwargs, {"bpp": "bits_per_pixel"}) if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) r = self.session.patch("/products/{}/images/{}".format( product_id, image_id), json=kwargs) return r.json() def remove_image(self, product_id, image_id, add_namespace=False): if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) self.session.delete("/products/{}/images/{}".format( product_id, image_id)) def upload_image(self, files, product_id, metadata=None, multi=False, image_id=None, add_namespace=False, **kwargs): """Upload an image for a product you own. This is an asynchronous operation and you can query for the status using `Catalog.upload_result()` with the upload_id returned by this method. The upload id is the image_id, which defaults to the name of the file to be uploaded. :param str|file|list(str)|list(file) files: (Required) a reference to the file to upload. :param str product_id: (Required) The id of the product this image belongs to. :param dict metadata: Image metadata to use instead of the computed default values. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* :return: The upload id. :rtype: str .. note:: - See :meth:`Catalog.add_image` for additional kwargs. """ if metadata is None: metadata = {} metadata.update(kwargs) check_deprecated_kwargs(metadata, {"bpp": "bits_per_pixel"}) if multi is True: if not hasattr(files, "__iter__"): raise ValueError( "Using `multi=True` requires `files` to be iterable") elif image_id is None: raise ValueError( "Using `multi=True` requires `image_id` to be specified") else: failed, upload_id, error = self._do_multi_file_upload( files, product_id, image_id, metadata, add_namespace=add_namespace) else: failed, upload_id, error = self._do_upload( files, product_id, metadata=metadata, add_namespace=add_namespace) if failed: raise error return upload_id def upload_ndarray(self, ndarray, product_id, image_id, proj4=None, wkt_srs=None, geotrans=None, raster_meta=None, overviews=None, overview_resampler=None, add_namespace=False, **kwargs): """Upload an ndarray with georeferencing information. This is an asynchronous operation and you can query for the status using `Catalog.upload_result()` with the upload_id returned by this method. The upload id is the image_id. :param ndarray ndarray: (Required) A numpy ndarray with image data. If you are providing a multi-band image it should have 3 dimensions and the 3rd dimension of the array should index the bands. The dtype of the ndarray must also be one of the following: ['uint8', 'int8', 'uint16', 'int16', 'uint32', 'int32', 'float32', 'float64'] :param str product_id: (Required) The id of the product this image belongs to. :param str image_id: (Required) Resulting image's id = <product_id>:<image_id>. :param str proj4: (One of proj4 or wkt_srs is required) A proj4 formatted string representing the spatial reference system used by the image. :param str wkt_srs: (One of proj4 or wkt_srs is required) A well known text string representing the spatial reference system used by the image. :param list(float) geotrans: (Required) The 6 number geographic transform of the image. Maps pixel coordinates to coordinates in the specified spatial reference system. :param dict raster_meta: Metadata returned from the :meth:`descarteslabs.client.services.raster.Raster.ndarray` request which generated the initial data for the :param ndarray: being uploaded. Passing :param geotrans: and :param wkt_srs: is unnecessary in this case. :param list(int) overviews: a list of overview resolution magnification factors i.e [2, 4] would make two overviews at 2x and 4x the native resolution. Maximum number of overviews allowed is 16. :param str overview_resampler: Resampler algorithm to use when building overviews. Controls how pixels are combined to make lower res pixels in overviews. Allowed resampler algorithms are: ['nearest', 'average', 'gauss', 'cubic', 'cubicspline', 'lanczos', 'average_mp', 'average_magphase', 'mode']. :param bool add_namespace: Add your user namespace to the product_id. *Deprecated* :return: The upload id. :rtype: str .. note:: - See :meth:`Catalog.add_image` for additional kwargs. - Only one of `proj4` or `wkt_srs` can be provided. """ if ndarray.dtype.name not in self.UPLOAD_NDARRAY_SUPPORTED_DTYPES: raise TypeError("{} is not in supported dtypes {}".format( ndarray.dtype.name, self.UPLOAD_NDARRAY_SUPPORTED_DTYPES)) metadata = kwargs metadata.setdefault("process_controls", {}).update({"upload_type": "ndarray"}) if raster_meta is not None: geotrans = raster_meta.get("geoTransform") wkt_srs = raster_meta.get("coordinateSystem", {}).get("wkt") for arg in ["image_id", "proj4", "wkt_srs", "geotrans"]: if locals()[arg] is not None: kwargs[arg] = locals()[arg] for arg in ["overviews", "overview_resampler"]: if locals()[arg] is not None: metadata["process_controls"][arg] = locals()[arg] with NamedTemporaryFile(delete=False) as tmp: try: np.save(tmp, ndarray, allow_pickle=False) # From tempfile docs: # Whether the name can be used to open the file a second time, # while the named temporary file is still open, varies across # platforms (it can be so used on Unix; it cannot on Windows # NT or later) # # We close the underlying file object so _do_upload can open # the path again in a cross platform compatible way. # Cleanup is manual in the finally block. tmp.close() failed, upload_id, error = self._do_upload( tmp.name, product_id, metadata=metadata, add_namespace=add_namespace) if failed: raise error return upload_id finally: os.unlink(tmp.name) def upload_results( self, product_id, limit=100, offset=None, status=None, updated=None, created=None, continuation_token=None, ): """Get result information for debugging your uploads. The upload id is included in the ``labels`` property. :param str product_id: Product ID to get upload results for. :param int limit: Number of results to get, useful for paging. :param int offset: Start of results to get, useful for paging. :param str status: Filter results by status, values are ["SUCCESS", "FAILURE"] :param str|int updated: Unix timestamp or ISO8601 formatted date for filtering results updated after this time. :param str|int created: Unix timestamp or ISO8601 formatted date for filtering results created after this time. :return: A list of upload result objects. :rtype: list """ kwargs = {"limit": limit} for arg in [ "offset", "status", "updated", "created", "continuation_token" ]: if locals()[arg] is not None: kwargs[arg] = locals()[arg] results = self.session.post("/products/{}/uploads".format(product_id), json=kwargs) return results.json() def iter_upload_results(self, product_id, status=None, updated=None, created=None): """Get result information for debugging your uploads. The upload id is included in the ``labels`` property. :param str product_id: Product ID to get upload results for. :param str status: Filter results by status, values are ["SUCCESS", "FAILURE"] :param str|int updated: Unix timestamp or ISO8601 formatted date for filtering results updated after this time. :param str|int created: Unix timestamp or ISO8601 formatted date for filtering results created after this time. :return: iterator to upload results. :rtype: generator """ continuation_token = None kwargs = {} for arg in ["status", "updated", "created"]: if locals()[arg] is not None: kwargs[arg] = locals()[arg] while True: page = self.upload_results(product_id, continuation_token=continuation_token, **kwargs) for res in page["data"]: yield res continuation_token = page["meta"]["continuation_token"] if continuation_token is None: break def upload_result(self, product_id, upload_id): """Get one upload result with the processing logs. This is useful for debugging failed uploads. :param str product_id: Product ID to get upload result for. :param str upload_id: ID of specific upload to get a result record of, includes the run logs. :return: One upload result with run logs. :rtype: dict """ result = self.session.get("/products/{}/uploads/{}".format( product_id, upload_id)) return result.json() def _do_multi_file_upload(self, files, product_id, image_id, metadata, add_namespace=False): file_keys = [os.path.basename(_f) for _f in files] process_controls = metadata.setdefault("process_controls", {"upload_type": "file"}) multi_file_args = { "multi_file": { "image_files": file_keys, "image_id": image_id } } process_controls.update(multi_file_args) for _file in files: failed, upload_id, error = self._do_upload( _file, product_id, metadata=metadata, add_namespace=add_namespace) if failed: break return failed, upload_id, error def _do_upload(self, file_ish, product_id, metadata=None, add_namespace=False): # kwargs are treated as metadata fields and restricted to primitives # for the key val pairs. fd = None upload_id = None if add_namespace: check_deprecated_kwargs(locals(), {"add_namespace": None}) product_id = self.namespace_product(product_id) if metadata is None: metadata = {} metadata.setdefault("process_controls", {"upload_type": "file"}) check_deprecated_kwargs(metadata, {"bpp": "bits_per_pixel"}) if not isinstance(product_id, six.string_types): raise TypeError("product_id={} is invalid. " "product_id must be a string.".format(product_id)) if isinstance(file_ish, io.IOBase): if "b" not in file_ish.mode: file_ish.close() file_ish = io.open(file_ish.name, "rb") fd = file_ish elif isinstance(file_ish, six.string_types) and os.path.exists(file_ish): fd = io.open(file_ish, "rb") else: e = Exception("Could not handle file: `{}` pass a valid path " "or open IOBase file".format(file_ish)) return True, upload_id, e try: upload_id = metadata.pop("image_id", None) or os.path.basename( fd.name) r = self.session.post( "/products/{}/images/upload/{}".format(product_id, upload_id), json=metadata, ) upload_url = r.text r = self._gcs_upload_service.session.put(upload_url, data=fd) except (ServerError, RequestException) as e: return True, upload_id, e except NotFoundError as e: raise NotFoundError( "Make sure product_id exists in the catalog before \ attempting to upload data. %s" % e.message) finally: fd.close() return False, upload_id, None
from descarteslabs.client.services.raster import Raster from descarteslabs.client.services.catalog import Catalog from descarteslabs.client.services.metadata import Metadata raster = Raster() catalog = Catalog() metadata = Metadata() id_ = '5151d2825f5e29ff129f86d834946363ff3f7e57:modis:09:CREFL_v2_test:2017-01-01-1835_11N_07_MO_09_v2' scene_meta = metadata.get(id_) r, meta = raster.ndarray(id_, bands=['red']) p = catalog.add_product('test_nda_upload', description="Test uploading georeferenced ndarrays", title="Test ndarray upload") band_spec = meta['bands'][0]['description'] catalog.add_band( p['data']['id'], 'red', srcband=1, jpx_layer=0, **{ k: v for k, v in band_spec.items() if k not in ['product', 'id', 'name', 'read', 'owner', 'owner_type'] }) catalog.upload_ndarray(r, p['data']['id'], id_[41:],
class MetadataTest(unittest.TestCase): def setUp(self): self.url = "http://example.com/metadata" self.instance = Metadata(url=self.url, auth=Auth(jwt_token=public_token, token_info_path=None)) self.instance._raster = Raster(url=self.url, auth=self.instance.auth) self.match_url = re.compile(self.url) def mock_response(self, method, json, status=200, **kwargs): responses.add(method, self.match_url, json=json, status=status, **kwargs) def test_expr_serialization(self): p = self.instance.properties q = ((0.1 < p.cloud_fraction <= 0.2) & (p.sat_id == "f00b")) | ( p.sat_id == "usa-245" ) expected_q = { "or": [ { "and": [ {"range": {"cloud_fraction": {"gt": 0.1, "lte": 0.2}}}, {"eq": {"sat_id": "f00b"}}, ] }, {"eq": {"sat_id": "usa-245"}}, ] } self.assertEqual(q.serialize(), expected_q) @responses.activate def test_paged_search(self): features = [{"id": "foo"}] token = "token" self.mock_response(responses.POST, json=features, headers={"x-continuation-token": token}) collection = self.instance.paged_search(limit=100) self.assertEqual(features, collection.features) self.assertEqual(token, collection.properties.continuation_token) @responses.activate def test_paged_search_deprecated_args(self): features = [{"id": "foo"}] self.mock_response(responses.POST, json=features) with warnings.catch_warnings(record=True) as w: collection = self.instance.paged_search(limit=100, start_time="2017-07-08") self.assertEqual(1, len(w)) self.assertEqual(w[0].category, DeprecationWarning) self.assertEqual(features, collection.features) @responses.activate def test_paged_search_dltile(self): features = [{"id": "foo"}] tile_geom = { "coordinates": [[[-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176], [-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104], [-94.01008346640455, 40.992358024242606]]], "type": "Polygon" } self.mock_response(responses.GET, json={"geometry": tile_geom}) self.mock_response(responses.POST, json=features) collection = self.instance.paged_search(limit=100, dltile="256:16:30.0:15:-11:591") self.assertEqual(features, collection.features) @responses.activate def test_search(self): features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}] self.mock_response(responses.POST, json=features) collection = self.instance.search(limit=2) self.assertEqual(features[:2], collection.features) @responses.activate def test_features(self): features = [{"id": "foo"}, {"id": "bar"}, {"id": "baz"}] self.mock_response(responses.POST, json=features[:2], headers={"x-continuation-token": "token"}) self.mock_response(responses.POST, json=features[2:], headers={"x-continuation-token": "token2"}) # Note: Unfortunately the client has historically been written in such a way that it always # expects a token header, even if the end of the search was reached, so an extra request # with 0 results happens in practice. self.mock_response(responses.POST, json=[], headers={"x-continuation-token": "token3"}) self.assertEqual(features, list(self.instance.features())) @responses.activate def test_summary(self): summary = {"count": 42} self.mock_response(responses.POST, json=summary) self.assertEqual(summary, self.instance.summary()) @responses.activate def test_summary_dltile(self): summary = {"count": 42} tile_geom = { "coordinates": [[[-94.01008346640455, 40.992358024242606], [-93.90737611136569, 40.99321227969176], [-93.908445279927, 41.0710332380541], [-94.01127360818097, 41.070176651899104], [-94.01008346640455, 40.992358024242606]]], "type": "Polygon" } self.mock_response(responses.GET, json={"geometry": tile_geom}) self.mock_response(responses.POST, json=summary) self.assertEqual(summary, self.instance.summary(dltile="256:16:30.0:15:-11:591"))
def search( aoi, products=None, start_datetime=None, end_datetime=None, cloud_fraction=None, storage_state=None, limit=100, sort_field=None, sort_order="asc", date_field="acquired", query=None, randomize=False, raster_client=None, metadata_client=None, ): """ Search for Scenes in the Descartes Labs catalog. Returns a SceneCollection of Scenes that overlap with an area of interest, and meet the given search criteria. Parameters ---------- aoi : GeoJSON-like dict, :class:`~descarteslabs.scenes.geocontext.GeoContext`, or object with __geo_interface__ Search for scenes that intersect this area by any amount. If a :class:`~descarteslabs.scenes.geocontext.GeoContext`, a copy is returned as ``ctx``, with missing values filled in. Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry. products : str or List[str], optional Descartes Labs product identifiers start_datetime : str, datetime-like, optional Restrict to scenes acquired after this datetime end_datetime : str, datetime-like, optional Restrict to scenes acquired before this datetime cloud_fraction : float, optional Restrict to scenes that are covered in clouds by less than this fraction (between 0 and 1) storage_state : str, optional Filter results based on ``storage_state`` value (``"available"``, ``"remote"``, or ``None``) limit : int or None, optional, default 100 Maximum number of Scenes to return, or None for all results. sort_field : str, optional Field name in :py:attr:`Scene.properties` by which to order the results sort_order : str, optional, default 'asc' ``"asc"`` or ``"desc"`` date_field : str, optional, default 'acquired' The field used when filtering by date (``"acquired"``, ``"processed"``, ``"published"``) query : ~descarteslabs.common.property_filtering.filtering.Expression, optional Expression used to filter Scenes by their properties, built from :class:`dl.properties <descarteslabs.common.property_filtering.filtering.GenericProperties>`. You can construct filter expression using the ``==``, ``!=``, ``<``, ``>``, ``<=`` and ``>=`` operators as well as the :meth:`~descarteslabs.common.property_filtering.filtering.Property.like` and :meth:`~descarteslabs.common.property_filtering.filtering.Property.in_` methods. You cannot use the boolean keywords ``and`` and ``or`` because of Python language limitations; instead you can combine filter expressions with ``&`` (boolean "and") and ``|`` (boolean "or"). Example: ``150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5`` randomize : bool, default False, optional Randomize the order of the results. You may also use an int or str as an explicit seed. raster_client : Raster, optional Unneeded in general use; lets you use a specific client instance with non-default auth and parameters. metadata_client : Metadata, optional Unneeded in general use; lets you use a specific client instance with non-default auth and parameters. Returns ------- scenes : `SceneCollection` Scenes matching your criteria. ctx: :class:`~descarteslabs.scenes.geocontext.GeoContext` The given ``aoi`` as a :class:`~descarteslabs.scenes.geocontext.GeoContext` (if it isn't one already), with reasonable default parameters for loading all matching Scenes. If ``aoi`` was a :class:`~descarteslabs.scenes.geocontext.GeoContext`, ``ctx`` will be a copy of ``aoi``, with any properties that were ``None`` assigned the defaults below. If ``aoi`` was not a :class:`~descarteslabs.scenes.geocontext.GeoContext`, an `AOI` instance will be created with ``aoi`` as its geometry, and defaults assigned as described below: **Default Spatial Parameters:** * resolution: the finest resolution of any band of all matching scenes * crs: the most common CRS used of all matching scenes """ if isinstance(aoi, geocontext.GeoContext): ctx = aoi if ctx.bounds is None and ctx.geometry is None: raise ValueError( "Unspecified where to search, " "since the GeoContext given for ``aoi`` has neither geometry nor bounds set" ) else: ctx = geocontext.AOI(geometry=aoi) if raster_client is None: raster_client = Raster() if metadata_client is None: metadata_client = Metadata() if isinstance(products, six.string_types): products = [products] if isinstance(start_datetime, datetime.datetime): start_datetime = start_datetime.isoformat() if isinstance(end_datetime, datetime.datetime): end_datetime = end_datetime.isoformat() metadata_params = dict( products=products, geom=ctx.__geo_interface__, start_datetime=start_datetime, end_datetime=end_datetime, cloud_fraction=cloud_fraction, storage_state=storage_state, limit=limit, sort_field=sort_field, sort_order=sort_order, date=date_field, q=query, randomize=randomize, ) metadata = metadata_client.search(**metadata_params) if products is None: products = { meta["properties"]["product"] for meta in metadata["features"] } product_bands = { product: Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product)) for product in products } scenes = SceneCollection( (Scene(meta, product_bands[meta["properties"]["product"]]) for meta in metadata["features"]), raster_client=raster_client, ) if len(scenes) > 0 and isinstance(ctx, geocontext.AOI): assign_ctx = {} if ctx.resolution is None and ctx.shape is None: resolutions = filter( None, (b.get("resolution") for band in six.itervalues(product_bands) for b in six.itervalues(band)), ) try: assign_ctx["resolution"] = min(resolutions) except ValueError: assign_ctx[ "resolution"] = None # from min of an empty sequence; no band defines resolution if ctx.crs is None: assign_ctx["crs"] = collections.Counter( scene.properties["crs"] for scene in scenes).most_common(1)[0][0] if len(assign_ctx) > 0: ctx = ctx.assign(**assign_ctx) return scenes, ctx