def request(self, method, url, **kwargs): if self.timeout and "timeout" not in kwargs: kwargs["timeout"] = self.timeout if "headers" not in kwargs: kwargs['headers'] = {} kwargs['headers']['X-Request-Group'] = uuid.uuid4().hex resp = super(WrappedSession, self).request( method, self.base_url + url, **kwargs ) if resp.status_code >= 200 and resp.status_code < 400: return resp elif resp.status_code == 400: raise BadRequestError(resp.text) elif resp.status_code == 404: text = resp.text if not text: text = "404 {} {}".format(method, url) raise NotFoundError(text) elif resp.status_code == 409: raise ConflictError(resp.text) elif resp.status_code == 429: raise RateLimitError(resp.text) elif resp.status_code == 504: raise GatewayTimeoutError( "Your request timed out on the server. " "Consider reducing the complexity of your request." ) else: raise ServerError(resp.text)
def request(self, method, url, **kwargs): if self.timeout and self.ATTR_TIMEOUT not in kwargs: kwargs[self.ATTR_TIMEOUT] = self.timeout if self.ATTR_HEADERS not in kwargs: kwargs[self.ATTR_HEADERS] = {} kwargs[self.ATTR_HEADERS][HttpHeaderKeys.RequestGroup] = uuid.uuid4().hex resp = super(WrappedSession, self).request( method, self.base_url + url, **kwargs ) if resp.status_code >= 200 and resp.status_code < 400: return resp elif resp.status_code == 400: raise BadRequestError(resp.text) elif resp.status_code == 404: text = resp.text if not text: text = "404 {} {}".format(method, url) raise NotFoundError(text) elif resp.status_code == 409: raise ConflictError(resp.text) elif resp.status_code == 422: raise BadRequestError(resp.text) elif resp.status_code == 429: raise RateLimitError( resp.text, retry_after=resp.headers.get(HttpHeaderKeys.RetryAfter) ) elif resp.status_code == 504: raise GatewayTimeoutError( "Your request timed out on the server. " "Consider reducing the complexity of your request." ) else: raise ServerError(resp.text)
def request(self, method, url, **kwargs): if self.timeout and 'timeout' not in kwargs: kwargs['timeout'] = self.timeout resp = super(WrappedSession, self).request(method, self.base_url + url, **kwargs) if resp.status_code >= 200 and resp.status_code < 400: return resp elif resp.status_code == 400: raise BadRequestError(resp.text) elif resp.status_code == 404: raise NotFoundError("404 %s %s" % (method, url)) elif resp.status_code == 409: raise ConflictError(resp.text) elif resp.status_code == 429: raise RateLimitError(resp.text) elif resp.status_code == 504: raise GatewayTimeoutError( "Your request timed out on the server. " "Consider reducing the complexity of your request.") else: raise ServerError(resp.text)
def request(self, method, url, **kwargs): """Sends an HTTP request and emits Descartes Labs specific errors. Parameters ---------- method: str The HTTP method to use. url: str The URL to send the request to. kwargs: dict Additional arguments. See `requests.request <https://requests.readthedocs.io/en/master/api/#requests.request>`_. Returns ------- Response A :py:class:`request.Response` object. Raises ------ BadRequestError Either a 400 or 422 HTTP response status code was encountered. NotFoundError A 404 HTTP response status code was encountered. ProxyAuthenticationRequiredError A 407 HTTP response status code was encountered and the resulting :py:meth:`handle_proxy_authentication` did not indicate that the proxy authentication was handled. ConflictError A 409 HTTP response status code was encountered. RateLimitError A 429 HTTP response status code was encountered. GatewayTimeoutError A 504 HTTP response status code was encountered. ServerError Any HTTP response status code larger than 400 that was not covered above is returned as a ServerError. The original HTTP response status code can be found in the attribute :py:attr:`original_status`. """ if self.timeout and self.ATTR_TIMEOUT not in kwargs: kwargs[self.ATTR_TIMEOUT] = self.timeout if self.ATTR_HEADERS not in kwargs: kwargs[self.ATTR_HEADERS] = {} kwargs[self.ATTR_HEADERS][ HttpHeaderKeys.RequestGroup] = uuid.uuid4().hex resp = super(Session, self).request(method, self.base_url + url, **kwargs) if (resp.status_code >= HttpStatusCode.Ok and resp.status_code < HttpStatusCode.BadRequest): return resp elif resp.status_code == HttpStatusCode.BadRequest: raise BadRequestError(resp.text) elif resp.status_code == HttpStatusCode.NotFound: text = resp.text if not text: text = "{} {} {}".format(HttpStatusCode.NotFound, method, url) raise NotFoundError(text) elif resp.status_code == HttpStatusCode.ProxyAuthenticationRequired: if not self.handle_proxy_authentication(method, url, **kwargs): raise ProxyAuthenticationRequiredError() elif resp.status_code == HttpStatusCode.Conflict: raise ConflictError(resp.text) elif resp.status_code == HttpStatusCode.UnprocessableEntity: raise BadRequestError(resp.text) elif resp.status_code == HttpStatusCode.TooManyRequests: raise RateLimitError(resp.text, retry_after=resp.headers.get( HttpHeaderKeys.RetryAfter)) elif resp.status_code == HttpStatusCode.GatewayTimeout: raise GatewayTimeoutError( "Your request timed out on the server. " "Consider reducing the complexity of your request.") else: # The whole error hierarchy has some problems. Originally a ClientError # could be thrown by our client libraries, but any HTTP error was a # ServerError. That changed and HTTP errors below 500 became ClientErrors. # That means that this actually should be split in ClientError for # status < 500 and ServerError for status >= 500, but that might break # things. So instead, we'll add the original status. server_error = ServerError(resp.text) server_error.original_status = resp.status_code raise server_error
def mosaic( self, bands, ctx, mask_nodata=True, mask_alpha=None, bands_axis=0, resampler="near", processing_level=None, scaling=None, data_type=None, raster_info=False, ): """ Load bands from all scenes, combining them into a single 3D ndarray and optionally masking invalid data. Where multiple scenes overlap, only data from the scene that comes last in the SceneCollection is used. If the selected bands and scenes have different data types the resulting ndarray has the most general of those data types. See `Scene.ndarray() <descarteslabs.scenes.scene.Scene.ndarray>` for details on data type conversions. Parameters ---------- bands : str or Sequence[str] Band names to load. Can be a single string of band names separated by spaces (``"red green blue"``), or a sequence of band names (``["red", "green", "blue"]``). If the alpha band is requested, it must be last in the list to reduce rasterization errors. ctx : :class:`~descarteslabs.scenes.geocontext.GeoContext` A :class:`~descarteslabs.scenes.geocontext.GeoContext` to use when loading each Scene mask_nodata : bool, default True Whether to mask out values in each band that equal that band's ``nodata`` sentinel value. mask_alpha : bool or str or None, default None Whether to mask pixels in all bands where the alpha band of all scenes is 0. Provide a string to use an alternate band name for masking. If the alpha band is available for all scenes in the collection and ``mask_alpha`` is None, ``mask_alpha`` is set to True. If not, mask_alpha is set to False. bands_axis : int, default 0 Axis along which bands should be located in the returned array. If 0, the array will have shape ``(band, y, x)``, if -1, it will have shape ``(y, x, band)``. It's usually easier to work with bands as the outermost axis, but when working with large arrays, or with many arrays concatenated together, NumPy operations aggregating each xy point across bands can be slightly faster with bands as the innermost axis. raster_info : bool, default False Whether to also return a dict of information about the rasterization of the scenes, including the coordinate system WKT and geotransform matrix. Generally only useful if you plan to upload data derived from this scene back to the Descartes catalog, or use it with GDAL. resampler : str, default "near" Algorithm used to interpolate pixel values when scaling and transforming the image to its new resolution or SRS. Possible values are ``near`` (nearest-neighbor), ``bilinear``, ``cubic``, ``cubicsplice``, ``lanczos``, ``average``, ``mode``, ``max``, ``min``, ``med``, ``q1``, ``q3``. processing_level : str, optional How the processing level of the underlying data should be adjusted. Possible values are ``toa`` (top of atmosphere) and ``surface``. For products that support it, ``surface`` applies Descartes Labs' general surface reflectance algorithm to the output. scaling : None, str, list, dict Band scaling specification. Please see :meth:`scaling_parameters` for a full description of this parameter. data_type : None, str Output data type. Please see :meth:`scaling_parameters` for a full description of this parameter. Returns ------- arr : ndarray Returned array's shape will be ``(band, y, x)`` if ``bands_axis`` is 0, and ``(y, x, band)`` if ``bands_axis`` is -1. If ``mask_nodata`` or ``mask_alpha`` is True, arr will be a masked array. The data type ("dtype") of the array is the most general of the data types among the scenes being rastered. raster_info : dict If ``raster_info=True``, a raster information dict is also returned. Raises ------ ValueError If requested bands are unavailable, or band names are not given or are invalid. If not all required parameters are specified in the :class:`~descarteslabs.scenes.geocontext.GeoContext`. If the SceneCollection is empty. `NotFoundError` If a Scene's ID cannot be found in the Descartes Labs catalog `BadRequestError` If the Descartes Labs platform is given unrecognized parameters """ if len(self) == 0: raise ValueError("This SceneCollection is empty") if not (-3 < bands_axis < 3): raise ValueError( "Invalid bands_axis; axis {} would not exist in a 3D array". format(bands_axis)) bands = Scene._bands_to_list(bands) alpha_band_name = "alpha" if isinstance(mask_alpha, six.string_types): alpha_band_name = mask_alpha elif mask_alpha is None: mask_alpha = self._collection_has_alpha(alpha_band_name) if mask_alpha: try: alpha_i = bands.index(alpha_band_name) except ValueError: bands.append(alpha_band_name) drop_alpha = True scaling = _scaling.append_alpha_scaling(scaling) else: if alpha_i != len(bands) - 1: raise ValueError( "Alpha must be the last band in order to reduce rasterization errors" ) drop_alpha = False scales, data_type = _scaling.multiproduct_scaling_parameters( self._product_band_properties(), bands, scaling, data_type) raster_params = ctx.raster_params full_raster_args = dict( inputs=[scene.properties["id"] for scene in self], order="gdal", bands=bands, scales=scales, data_type=data_type, resampler=resampler, processing_level=processing_level, **raster_params) try: arr, info = self._raster_client.ndarray(**full_raster_args) except NotFoundError: raise NotFoundError( "Some or all of these IDs don't exist in the Descartes catalog: {}" .format(full_raster_args["inputs"])) except BadRequestError as e: msg = ( "Error with request:\n" "{err}\n" "For reference, dl.Raster.ndarray was called with these arguments:\n" "{args}") msg = msg.format(err=e, args=json.dumps(full_raster_args, indent=2)) six.raise_from(BadRequestError(msg), None) if len(arr.shape) == 2: # if only 1 band requested, still return a 3d array arr = arr[np.newaxis] if mask_nodata or mask_alpha: if mask_alpha: alpha = arr[-1] if drop_alpha: arr = arr[:-1] bands.pop(-1) mask = np.zeros_like(arr, dtype=bool) if mask_nodata: # collect all possible nodata values per band, # in case different products have different nodata values for the same-named band # QUESTION: is this overkill? band_nodata_values = collections.defaultdict(set) for scene in self: scene_bands = scene.properties["bands"] for bandname in bands: band_nodata_values[bandname].add( scene_bands[bandname].get("nodata")) for i, bandname in enumerate(bands): for nodata in band_nodata_values[bandname]: if nodata is not None: mask[i] |= arr[i] == nodata if mask_alpha: mask |= alpha == 0 arr = np.ma.MaskedArray(arr, mask, copy=False) if bands_axis != 0: arr = np.moveaxis(arr, 0, bands_axis) if raster_info: return arr, info else: return arr
def ndarray(self, bands, ctx, mask_nodata=True, mask_alpha=True, bands_axis=0, raster_info=False, resampler="near", processing_level=None, raster_client=None ): """ Load bands from this scene as an ndarray, optionally masking invalid data. Parameters ---------- bands : str or Sequence[str] Band names to load. Can be a single string of band names separated by spaces (``"red green blue derived:ndvi"``), or a sequence of band names (``["red", "green", "blue", "derived:ndvi"]``). Names must be keys in ``self.properties.bands``. If the alpha band is requested, it must be last in the list to reduce rasterization errors. ctx : `GeoContext` A `GeoContext` to use when loading this Scene mask_nodata : bool, default True Whether to mask out values in each band that equal that band's ``nodata`` sentinel value. mask_alpha : bool, default True Whether to mask pixels in all bands where the alpha band is 0. bands_axis : int, default 0 Axis along which bands should be located in the returned array. If 0, the array will have shape ``(band, y, x)``, if -1, it will have shape ``(y, x, band)``. It's usually easier to work with bands as the outermost axis, but when working with large arrays, or with many arrays concatenated together, NumPy operations aggregating each xy point across bands can be slightly faster with bands as the innermost axis. raster_info : bool, default False Whether to also return a dict of information about the rasterization of the scene, including the coordinate system WKT and geotransform matrix. Generally only useful if you plan to upload data derived from this scene back to the Descartes catalog, or use it with GDAL. resampler : str, default "near" Algorithm used to interpolate pixel values when scaling and transforming the image to its new resolution or CRS. Possible values are ``near`` (nearest-neighbor), ``bilinear``, ``cubic``, ``cubicsplice``, ``lanczos``, ``average``, ``mode``, ``max``, ``min``, ``med``, ``q1``, ``q3``. processing_level : str, optional How the processing level of the underlying data should be adjusted. Possible values are ``toa`` (top of atmosphere) and ``surface``. For products that support it, ``surface`` applies Descartes Labs' general surface reflectance algorithm to the output. raster_client : Raster, optional Unneeded in general use; lets you use a specific client instance with non-default auth and parameters. Returns ------- arr : ndarray Returned array's shape will be ``(band, y, x)`` if bands_axis is 0, ``(y, x, band)`` if bands_axis is -1 If ``mask_nodata`` or ``mask_alpha`` is True, arr will be a masked array. raster_info : dict If ``raster_info=True``, a raster information dict is also returned. Example ------- >>> import descarteslabs as dl >>> scene, ctx = dl.scenes.Scene.from_id("landsat:LC08:PRE:TOAR:meta_LC80270312016188_v1") # doctest: +SKIP >>> arr = scene.ndarray("red green blue", ctx) # doctest: +SKIP >>> type(arr) # doctest: +SKIP <class 'numpy.ma.core.MaskedArray'> >>> arr.shape # doctest: +SKIP (3, 15960, 15696) >>> red_band = arr[0] # doctest: +SKIP Raises ------ ValueError If requested bands are unavailable. If band names are not given or are invalid. If the requested bands have different dtypes. NotFoundError If a Scene's ID cannot be found in the Descartes Labs catalog BadRequestError If the Descartes Labs platform is given invalid parameters """ if raster_client is None: raster_client = Raster() if not (-3 < bands_axis < 3): raise ValueError("Invalid bands_axis; axis {} would not exist in a 3D array".format(bands_axis)) bands = self._bands_to_list(bands) common_data_type = self._common_data_type_of_bands(bands) self_bands = self.properties["bands"] if mask_alpha: if "alpha" not in self_bands: raise ValueError( "Cannot mask alpha: no alpha band for the product '{}'. " "Try setting 'mask_alpha=False'.".format(self.properties["product"]) ) try: alpha_i = bands.index("alpha") except ValueError: bands.append("alpha") drop_alpha = True else: if alpha_i != len(bands) - 1: raise ValueError("Alpha must be the last band in order to reduce rasterization errors") drop_alpha = False raster_params = ctx.raster_params full_raster_args = dict( inputs=self.properties["id"], order="gdal", bands=bands, scales=None, data_type=common_data_type, resampler=resampler, processing_level=processing_level, **raster_params ) try: arr, info = raster_client.ndarray(**full_raster_args) except NotFoundError: six.raise_from( NotFoundError("'{}' does not exist in the Descartes catalog".format(self.properties["id"])), None ) except BadRequestError as e: msg = ("Error with request:\n" "{err}\n" "For reference, dl.Raster.ndarray was called with these arguments:\n" "{args}") msg = msg.format(err=e, args=json.dumps(full_raster_args, indent=2)) six.raise_from(BadRequestError(msg), None) if len(arr.shape) == 2: # if only 1 band requested, still return a 3d array arr = arr[np.newaxis] if mask_nodata or mask_alpha: if mask_alpha: alpha = arr[-1] if drop_alpha: arr = arr[:-1] bands.pop(-1) mask = np.zeros_like(arr, dtype=bool) if mask_nodata: for i, bandname in enumerate(bands): nodata = self_bands[bandname].get('nodata') if nodata is not None: mask[i] = arr[i] == nodata if mask_alpha: mask |= alpha == 0 arr = np.ma.MaskedArray(arr, mask, copy=False) if bands_axis != 0: arr = np.moveaxis(arr, 0, bands_axis) if raster_info: return arr, info else: return arr
def _download(inputs, bands_list, ctx, dtype, dest, format, raster_client=None): """ Download inputs as an image file and save to file or path-like `dest`. Code shared by Scene.download and SceneCollection.download_mosaic """ if raster_client is None: raster_client = Raster() if dest is None: if len(inputs) == 0: raise ValueError("No inputs given to download") bands_str = "-".join(bands_list) if len(inputs) == 1: # default filename for a single scene dest = "{id}-{bands}.{ext}".format(id=inputs[0], bands=bands_str, ext=format) else: # default filename for a mosaic dest = "mosaic-{bands}.{ext}".format(bands=bands_str, ext=format) # Create any intermediate directories if _is_path_like(dest): dirname = os.path.dirname(dest) if dirname != "" and not os.path.exists(dirname): os.makedirs(dirname) format = _format_from_path(dest) else: format = _get_format(format) raster_params = ctx.raster_params full_raster_args = dict(inputs=inputs, bands=bands_list, scales=None, data_type=dtype, output_format=format, save=False, **raster_params) try: result = raster_client.raster(**full_raster_args) except NotFoundError as e: if len(inputs) == 1: msg = "'{}' does not exist in the Descartes catalog".format( inputs[0]) else: msg = "Some or all of these IDs don't exist in the Descartes catalog: {}".format( inputs) six.raise_from(NotFoundError(msg), None) except BadRequestError as e: msg = ( "Error with request:\n" "{err}\n" "For reference, dl.Raster.raster was called with these arguments:\n" "{args}") msg = msg.format(err=e, args=json.dumps(full_raster_args, indent=2)) six.raise_from(BadRequestError(msg), None) # `result["files"]` should be a dict mapping {default_filename: bytestring} filenames = list(result["files"].keys()) if len(filenames) == 0: raise RuntimeError("Unexpected missing results from raster call") elif len(filenames) > 1: raise RuntimeError( "Unexpected multiple files returned from single raster call: {}". format(filenames)) else: file = result["files"][filenames[0]] if _is_path_like(dest): with open(dest, "wb") as f: f.write(file) return dest else: # `dest` is a file-like object try: dest.write(file) except Exception as e: raise TypeError( "Unable to write to the file-like object {} provided as `dest`:\n{}" .format(dest, e))
def test_raster_bad_request(self, mock_raster, mock_makedirs, mock_open): mock_raster.side_effect = BadRequestError("what is a foo") with self.assertRaisesRegexp(BadRequestError, "Error with request"): self.download("file.tif")