def setUp(self):
self.url = "http://example.com/metadata"
self.instance = Metadata(url=self.url,
auth=Auth(jwt_token=public_token,
token_info_path=None))
self.instance._raster = Raster(url=self.url, auth=self.instance.auth)
self.match_url = re.compile(self.url)
def from_id(cls, scene_id, metadata_client=None):
"""
Return the metadata for a Descartes Labs scene ID as a Scene object.
Also returns a :class:`~descarteslabs.scenes.geocontext.GeoContext`
for loading the Scene's original, unwarped data.
Parameters
----------
scene_id: str
Descartes Labs scene ID,
e.g. "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1"
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scene: Scene
Scene instance with metadata loaded from the Descartes Labs catalog
ctx: AOI
A :class:`~descarteslabs.scenes.geocontext.GeoContext`for loading this Scene's original data.
The defaults used are described in `Scene.default_ctx`.
Example
-------
>>> import descarteslabs as dl
>>> scene, ctx = dl.scenes.Scene.from_id("landsat:LC08:PRE:TOAR:meta_LC80260322016197_v1") # doctest: +SKIP
>>> ctx # doctest: +SKIP
AOI(geometry=None,
resolution=15.0,
crs='EPSG:32615',
align_pixels=False,
bounds=(348592.5, 4345567.5, 581632.5, 4582807.5),
bounds_crs='EPSG:32615',
shape=None)
>>> scene.properties.date # doctest: +SKIP
datetime.datetime(2016, 7, 15, 16, 53, 59, 495435)
Raises
------
NotFoundError
If the ``scene_id`` cannot be found in the Descartes Labs catalog
"""
if metadata_client is None:
metadata_client = Metadata()
metadata = metadata_client.get(scene_id)
metadata = {
"type": "Feature",
"geometry": metadata.pop("geometry"),
"id": metadata.pop("id"),
"key": metadata.pop("key"),
"properties": metadata
}
bands = metadata_client.get_bands_by_id(scene_id)
scene = cls(metadata, bands)
return scene, scene.default_ctx()
def __init__(self, url=None, auth=None, metadata=None):
"""The parent Service class implements authentication and exponential
backoff/retry. Override the url parameter to use a different instance
of the backing service.
"""
if auth is None:
auth = Auth()
if metadata is None:
self._metadata = Metadata(auth=auth)
else:
self._metadata = metadata
self._gcs_upload_service = ThirdPartyService()
if url is None:
url = os.environ.get(
"DESCARTESLABS_CATALOG_URL",
"https://platform.descarteslabs.com/metadata/v1/catalog")
super(Catalog, self).__init__(url, auth=auth)
def setUpClass(cls):
cls.instance = Metadata()
import datetime
import collections
import textwrap
import warnings
import shapely.geometry
import numpy as np
from descarteslabs.common.dotdict import DotDict
from descarteslabs.scenes import Scene, geocontext
from descarteslabs.scenes.scene import _strptime_helper
from descarteslabs.client.services.metadata import Metadata
from .mock_data import _metadata_get, _metadata_get_bands, _raster_ndarray
metadata_client = Metadata()
class MockScene(Scene):
"Circumvent __init__ method to create a Scene with arbitrary geometry and properties objects"
def __init__(self, geometry, properties):
self.geometry = DotDict(geometry)
self.properties = DotDict(properties)
class TestScene(unittest.TestCase):
@mock.patch("descarteslabs.client.services.metadata.Metadata.get", _metadata_get)
@mock.patch("descarteslabs.client.services.metadata.Metadata.get_bands_by_id", _metadata_get_bands)
def test_init(self):
scene_id = "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1"
metadata = metadata_client.get(scene_id)
def auth_handler(args):
auth = Auth()
if args.command == 'login':
print(
'Follow this link to login https://iam.descarteslabs.com/auth/login?refresh_token=true&destination=/auth/refresh_token'
) # NOQA
s = input('...then come back here and paste the generated token: ')
if isinstance(s, six.text_type):
s = s.encode('utf-8')
if s:
token_info = json.loads(base64url_decode(s).decode('utf-8'))
token_info_directory = os.path.dirname(DEFAULT_TOKEN_INFO_PATH)
makedirs_if_not_exists(token_info_directory)
os.chmod(token_info_directory,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
with open(DEFAULT_TOKEN_INFO_PATH, 'w+') as fp:
json.dump(token_info, fp)
os.chmod(DEFAULT_TOKEN_INFO_PATH, stat.S_IRUSR | stat.S_IWUSR)
# Get a fresh Auth token
auth = Auth()
keys = Metadata(auth=auth).keys()
name = auth.payload['name']
groups = ', '.join(auth.payload['groups'])
if len(keys):
print('Welcome, %s!' % name)
else:
print(
'Welcome, %s! Your %s role(s) do not permit access to any imagery at this time.'
% (name, groups))
print(
'Contact [email protected] if you believe you received this message in error or have any questions.'
) # NOQA
if args.command == 'token':
print(auth.token)
if args.command == 'name':
auth.token
print(auth.payload['name'])
if args.command == 'groups':
auth.token
print(json.dumps(auth.payload['groups']))
if args.command == 'payload':
auth.token
print(auth.payload)
if args.command == 'env':
auth.token
print('%s=%s' % ('CLIENT_ID', auth.client_id))
print('%s=%s' % ('CLIENT_SECRET', auth.client_secret))
if args.command == 'version':
print(__version__)
def from_id(cls, scene_id, metadata_client=None):
"""
Return the metadata for a Descartes Labs scene ID as a Scene object.
Also returns a GeoContext with reasonable defaults to use
when loading the Scene's ndarray.
Parameters
----------
scene_id: str
Descartes Labs scene ID,
e.g. "landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1"
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scene: Scene
Scene instance with metadata loaded from the Descartes Labs catalog
ctx: AOI
A default GeoContext useful for loading this Scene.
These defaults are used:
* bounds: bounds of the Scene's geometry
* resolution: the finest resolution of any band in the scene
* crs: native CRS of the Scene (generally, a UTM CRS)
* align_pixels: True
Example
-------
>>> import descarteslabs as dl
>>> scene, ctx = dl.scenes.Scene.from_id("landsat:LC08:PRE:TOAR:meta_LC80260322016197_v1")
>>> ctx
AOI(geometry=None,
resolution=15,
crs='EPSG:32615',
align_pixels=True,
bounds=(-94.724166, 39.2784859, -92.0686956, 41.3717716),
shape=None)
>>> scene.properties.date
datetime.datetime(2016, 7, 15, 16, 53, 59, 495435)
Raises
------
NotFoundError
If the ``scene_id`` cannot be found in the Descartes Labs catalog
"""
if metadata_client is None:
metadata_client = Metadata()
metadata = metadata_client.get(scene_id)
metadata = {
"type": "Feature",
"geometry": metadata.pop("geometry"),
"id": metadata.pop("id"),
"key": metadata.pop("key"),
"properties": metadata
}
bands = metadata_client.get_bands_by_id(scene_id)
scene = cls(metadata, bands)
# TODO: not sure what default res should be
try:
default_resolution = min(
filter(None,
[b.get("resolution") for b in six.itervalues(bands)]))
except ValueError:
default_resolution = 100
# QUESTION: default bounds will now be in WGS84, not UTM
# indeed, there's no way to give bounds in UTM besides with a DLTile,
# which means you could get off-by-one issues with loading an entire scene
# at native resolution, where the WGS84 bounds result in a slightly differently
# sized raster than native UTM bounds would with reprojection errors
try:
bounds = scene.geometry.bounds
except AttributeError:
xs, ys = zip(*scene.geometry["coordinates"][0])
bounds = (min(xs), min(ys), max(xs), max(ys))
default_ctx = geocontext.AOI(bounds=bounds,
resolution=default_resolution,
crs=scene.properties["crs"])
return scene, default_ctx
# Copyright 2018 Descartes Labs. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # flake8: noqa from descarteslabs.client.auth import Auth from descarteslabs.client.services.metadata import Metadata from descarteslabs.client.services.places import Places from descarteslabs.client.services.raster import Raster descartes_auth = Auth.from_environment_or_token_json() metadata = Metadata(auth=descartes_auth) places = Places(auth=descartes_auth) raster = Raster(auth=descartes_auth) __all__ = ["descartes_auth", "metadata", "places", "raster"]
def search(aoi,
products=None,
start_datetime=None,
end_datetime=None,
cloud_fraction=None,
limit=100,
sort_field=None,
sort_order='asc',
date_field='acquired',
query=None,
randomize=False,
raster_client=None,
metadata_client=None):
"""
Search for Scenes in the Descartes Labs catalog.
Returns a SceneCollection of Scenes that overlap with an area of interest,
and meet the given search criteria.
Parameters
----------
aoi : GeoJSON-like dict, GeoContext, or object with __geo_interface__
Search for scenes that intersect this area by any amount.
If a GeoContext, a copy is returned as ``ctx``, with missing values filled in.
Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry.
products : str or List[str], optional
Descartes Labs product identifiers
start_datetime : str, datetime-like, optional
Restrict to scenes acquired after this datetime
end_datetime : str, datetime-like, optional
Restrict to scenes acquired before this datetime
cloud_fraction : float, optional
Restrict to scenes that are covered in clouds by less than this fraction
(between 0 and 1)
limit : int, optional
Maximum number of Scenes to return, up to 10000.
sort_field : str, optional
Field name in ``Scene.properties`` by which to order the results
sort_order : str, optional, default 'asc'
``"asc"`` or ``"desc"``
date_field : str, optional, default 'acquired'
The field used when filtering by date
(``"acquired"``, ``"processed"``, ``"published"``)
query : descarteslabs.common.property_filtering.Expression, optional
Expression used to filter Scenes by their properties, built from ``dl.properties``.
>>> query = 150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5
>>> query = dl.properties.sat_id == "Terra"
randomize : bool, default False, optional
Randomize the order of the results.
You may also use an int or str as an explicit seed.
raster_client : Raster, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scenes : SceneCollection
Scenes matching your criteria.
ctx: GeoContext
The given ``aoi`` as a GeoContext (if it isn't one already),
with reasonable default parameters for loading all matching Scenes.
If ``aoi`` was a `GeoContext`, ``ctx`` will be a copy of ``aoi``,
with any properties that were ``None`` assigned the defaults below.
If ``aoi`` was not a `GeoContext`, an `AOI` instance will be created
with ``aoi`` as its geometry, and defaults assigned as described below:
**Default Spatial Parameters:**
* resolution: the finest resolution of any band of all matching scenes
* crs: the most common CRS used of all matching scenes
"""
if isinstance(aoi, geocontext.GeoContext):
ctx = aoi
if ctx.bounds is None and ctx.geometry is None:
raise ValueError(
"Unspecified where to search, "
"since the GeoContext given for ``aoi`` has neither geometry nor bounds set"
)
else:
ctx = geocontext.AOI(geometry=aoi)
if raster_client is None:
raster_client = Raster()
if metadata_client is None:
metadata_client = Metadata()
if isinstance(products, six.string_types):
products = [products]
if isinstance(start_datetime, datetime.datetime):
start_datetime = start_datetime.isoformat()
if isinstance(end_datetime, datetime.datetime):
end_datetime = end_datetime.isoformat()
if limit > MAX_RESULT_WINDOW:
raise ValueError("Limit must be <= {}".format(MAX_RESULT_WINDOW))
metadata_params = dict(products=products,
geom=ctx.__geo_interface__,
start_datetime=start_datetime,
end_datetime=end_datetime,
cloud_fraction=cloud_fraction,
limit=limit,
sort_field=sort_field,
sort_order=sort_order,
date=date_field,
q=query,
randomize=randomize)
metadata = metadata_client.search(**metadata_params)
if products is None:
products = {
meta["properties"]["product"]
for meta in metadata["features"]
}
product_bands = {
product:
Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product))
for product in products
}
scenes = SceneCollection(
(Scene(meta, product_bands[meta["properties"]["product"]])
for meta in metadata["features"]),
raster_client=raster_client)
if len(scenes) > 0:
assign_ctx = {}
if ctx.resolution is None and ctx.shape is None:
resolutions = filter(None,
(b.get("resolution")
for band in six.itervalues(product_bands)
for b in six.itervalues(band)))
try:
assign_ctx["resolution"] = min(resolutions)
except ValueError:
assign_ctx[
"resolution"] = None # from min of an empty sequence; no band defines resolution
if ctx.crs is None:
assign_ctx["crs"] = collections.Counter(
scene.properties["crs"]
for scene in scenes).most_common(1)[0][0]
if len(assign_ctx) > 0:
ctx = ctx.assign(**assign_ctx)
return scenes, ctx
from descarteslabs.client.services.raster import Raster
from descarteslabs.client.services.catalog import Catalog
from descarteslabs.client.services.metadata import Metadata
raster = Raster()
catalog = Catalog()
metadata = Metadata()
id_ = '5151d2825f5e29ff129f86d834946363ff3f7e57:modis:09:CREFL_v2_test:2017-01-01-1835_11N_07_MO_09_v2'
scene_meta = metadata.get(id_)
r, meta = raster.ndarray(id_, bands=['red'])
p = catalog.add_product('test_nda_upload',
description="Test uploading georeferenced ndarrays",
title="Test ndarray upload")
band_spec = meta['bands'][0]['description']
catalog.add_band(
p['data']['id'],
'red',
srcband=1,
jpx_layer=0,
**{
k: v
for k, v in band_spec.items()
if k not in ['product', 'id', 'name', 'read', 'owner', 'owner_type']
})
catalog.upload_ndarray(r,
p['data']['id'],
id_[41:],
def search(
aoi,
products=None,
start_datetime=None,
end_datetime=None,
cloud_fraction=None,
storage_state=None,
limit=100,
sort_field=None,
sort_order="asc",
date_field="acquired",
query=None,
randomize=False,
raster_client=None,
metadata_client=None,
):
"""
Search for Scenes in the Descartes Labs catalog.
Returns a SceneCollection of Scenes that overlap with an area of interest,
and meet the given search criteria.
Parameters
----------
aoi : GeoJSON-like dict, :class:`~descarteslabs.scenes.geocontext.GeoContext`, or object with __geo_interface__
Search for scenes that intersect this area by any amount.
If a :class:`~descarteslabs.scenes.geocontext.GeoContext`, a copy is returned as ``ctx``, with missing values
filled in. Otherwise, the returned ``ctx`` will be an `AOI`, with this as its geometry.
products : str or List[str], optional
Descartes Labs product identifiers
start_datetime : str, datetime-like, optional
Restrict to scenes acquired after this datetime
end_datetime : str, datetime-like, optional
Restrict to scenes acquired before this datetime
cloud_fraction : float, optional
Restrict to scenes that are covered in clouds by less than this fraction
(between 0 and 1)
storage_state : str, optional
Filter results based on ``storage_state`` value
(``"available"``, ``"remote"``, or ``None``)
limit : int or None, optional, default 100
Maximum number of Scenes to return, or None for all results.
sort_field : str, optional
Field name in :py:attr:`Scene.properties` by which to order the results
sort_order : str, optional, default 'asc'
``"asc"`` or ``"desc"``
date_field : str, optional, default 'acquired'
The field used when filtering by date
(``"acquired"``, ``"processed"``, ``"published"``)
query : ~descarteslabs.common.property_filtering.filtering.Expression, optional
Expression used to filter Scenes by their properties, built from
:class:`dl.properties <descarteslabs.common.property_filtering.filtering.GenericProperties>`.
You can construct filter expression using the ``==``, ``!=``, ``<``, ``>``,
``<=`` and ``>=`` operators as well as the
:meth:`~descarteslabs.common.property_filtering.filtering.Property.like`
and :meth:`~descarteslabs.common.property_filtering.filtering.Property.in_`
methods. You cannot use the boolean keywords ``and`` and ``or`` because of
Python language limitations; instead you can combine filter expressions
with ``&`` (boolean "and") and ``|`` (boolean "or").
Example:
``150 < dl.properties.azimuth_angle < 160 & dl.properties.cloud_fraction < 0.5``
randomize : bool, default False, optional
Randomize the order of the results.
You may also use an int or str as an explicit seed.
raster_client : Raster, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
metadata_client : Metadata, optional
Unneeded in general use; lets you use a specific client instance
with non-default auth and parameters.
Returns
-------
scenes : `SceneCollection`
Scenes matching your criteria.
ctx: :class:`~descarteslabs.scenes.geocontext.GeoContext`
The given ``aoi`` as a :class:`~descarteslabs.scenes.geocontext.GeoContext` (if it isn't one already),
with reasonable default parameters for loading all matching Scenes.
If ``aoi`` was a :class:`~descarteslabs.scenes.geocontext.GeoContext`, ``ctx`` will be a copy of ``aoi``,
with any properties that were ``None`` assigned the defaults below.
If ``aoi`` was not a :class:`~descarteslabs.scenes.geocontext.GeoContext`, an `AOI` instance will be created
with ``aoi`` as its geometry, and defaults assigned as described below:
**Default Spatial Parameters:**
* resolution: the finest resolution of any band of all matching scenes
* crs: the most common CRS used of all matching scenes
"""
if isinstance(aoi, geocontext.GeoContext):
ctx = aoi
if ctx.bounds is None and ctx.geometry is None:
raise ValueError(
"Unspecified where to search, "
"since the GeoContext given for ``aoi`` has neither geometry nor bounds set"
)
else:
ctx = geocontext.AOI(geometry=aoi)
if raster_client is None:
raster_client = Raster()
if metadata_client is None:
metadata_client = Metadata()
if isinstance(products, six.string_types):
products = [products]
if isinstance(start_datetime, datetime.datetime):
start_datetime = start_datetime.isoformat()
if isinstance(end_datetime, datetime.datetime):
end_datetime = end_datetime.isoformat()
metadata_params = dict(
products=products,
geom=ctx.__geo_interface__,
start_datetime=start_datetime,
end_datetime=end_datetime,
cloud_fraction=cloud_fraction,
storage_state=storage_state,
limit=limit,
sort_field=sort_field,
sort_order=sort_order,
date=date_field,
q=query,
randomize=randomize,
)
metadata = metadata_client.search(**metadata_params)
if products is None:
products = {
meta["properties"]["product"]
for meta in metadata["features"]
}
product_bands = {
product:
Scene._scenes_bands_dict(metadata_client.get_bands_by_product(product))
for product in products
}
scenes = SceneCollection(
(Scene(meta, product_bands[meta["properties"]["product"]])
for meta in metadata["features"]),
raster_client=raster_client,
)
if len(scenes) > 0 and isinstance(ctx, geocontext.AOI):
assign_ctx = {}
if ctx.resolution is None and ctx.shape is None:
resolutions = filter(
None,
(b.get("resolution") for band in six.itervalues(product_bands)
for b in six.itervalues(band)),
)
try:
assign_ctx["resolution"] = min(resolutions)
except ValueError:
assign_ctx[
"resolution"] = None # from min of an empty sequence; no band defines resolution
if ctx.crs is None:
assign_ctx["crs"] = collections.Counter(
scene.properties["crs"]
for scene in scenes).most_common(1)[0][0]
if len(assign_ctx) > 0:
ctx = ctx.assign(**assign_ctx)
return scenes, ctx