def requests_wmts_tile( self, tile: mercantile.Tile, layer: str, date: str, img_format: str = "jpg" ) -> requests.Response: tile_url = self.wmts_url + self.wmts_endpoint.format( layer=layer, date=date, x=tile.x, y=tile.y, zoom=tile.z, img_format=img_format, ) logger.debug(tile_url) try: wmts_response = requests.get(tile_url) logger.info(f"response returned: {wmts_response.status_code}") wmts_response.raise_for_status() except requests.exceptions.ConnectionError as conn_err: logger.error("Network related error occured") raise UP42Error( SupportedErrors.API_CONNECTION_ERROR, str(conn_err) ) from conn_err except requests.exceptions.HTTPError as err: logger.error("HTTP error occured") raise UP42Error(SupportedErrors.API_CONNECTION_ERROR, str(err)) from err return wmts_response
def assert_input_params(self): if not self.params.__dict__["clip_to_aoi"]: if self.params.bbox or self.params.contains or self.params.intersects: raise UP42Error( SupportedErrors.INPUT_PARAMETERS_ERROR, "When clip_to_aoi is set to False, bbox, contains and intersects must be set to null.", ) else: if (self.params.bbox is None and self.params.contains is None and self.params.intersects is None): raise UP42Error( SupportedErrors.INPUT_PARAMETERS_ERROR, "When clip_to_aoi set to True, you MUST define one of bbox, contains or intersect.", )
def process(self, input_fc: FeatureCollection) -> FeatureCollection: if not input_fc.features: raise UP42Error(SupportedErrors.NO_INPUT_ERROR) output_fc = FeatureCollection([]) for feat in input_fc["features"]: logger.info(f"Processing {feat}...") input_path = Path("/tmp/input/") / Path(get_data_path(feat)) with rio.open(input_path) as src: src_win = WindowsUtil(src) ( output_name, output_path, ) = get_output_filename_and_path(input_path.name, postfix="processed") dst_meta = src.meta.copy() with rio.open(output_path, "w", **dst_meta) as dst: for win in src_win.windows_regular(): exp = src.read(window=win)**self.exponent dst.write(exp, window=win) out_feat = Feature(bbox=feat.bbox, geometry=feat.geometry) out_feat["properties"] = self.get_metadata(feat) out_feat = set_data_path(out_feat, output_name) logger.info(f"Processed {out_feat}...") output_fc.features.append(out_feat) return output_fc
def process(self, input_fc: FeatureCollection) -> FeatureCollection: """ This method takes the raster data at 10, 20, and 60 m resolutions and by applying data_final method creates the input data for the the convolutional neural network. It returns 10 m resolution for all the bands in 20 and 60 m resolutions. Args: input_fc: geojson FeatureCollection of all input images """ self.assert_input_params() output_jsonfile = self.get_final_json() LOGGER.info("Started process...") for feature in input_fc.features: LOGGER.info(f"Processing feature {feature}") path_to_input_img = feature["properties"]["up42.data_path"] path_to_output_img = Path( path_to_input_img).stem + "_superresolution.tif" try: subprocess.run( f"python3 src/inference.py {path_to_input_img} {path_to_output_img}", check=True, shell=True, ) except subprocess.CalledProcessError as e: raise UP42Error(SupportedErrors(e.returncode)) from e self.save_output_json(output_jsonfile, self.output_dir) return output_jsonfile
def assert_input_params(self): if not self.params.clip_to_aoi: if self.params.bbox or self.params.contains or self.params.intersects: raise UP42Error( SupportedErrors.WRONG_INPUT_ERROR, "When clip_to_aoi is set to False, bbox, contains " "and intersects must be set to null.", ) else: if (self.params.bbox is None and self.params.contains is None and self.params.intersects is None): raise UP42Error( SupportedErrors.WRONG_INPUT_ERROR, "When clip_to_aoi set to True, you MUST define the same " "coordinates in bbox, contains or intersect for both " "the S1 and SNAP blocks.", )
def process(self, input_fc: FeatureCollection) -> FeatureCollection: """ Given the necessary parameters and a feature collection describing the input datasets, run K-means clustering for each input data set and create output feature collection :param metadata: A GeoJSON FeatureCollection describing all input datasets :return: A GeoJSON FeatureCollection describing all output datasets """ if not input_fc.features: raise UP42Error(SupportedErrors.NO_INPUT_ERROR) results = [] # type: List[Feature] for feature in input_fc.features: try: path_to_input_img = feature["properties"]["up42.data_path"] path_to_output_img = Path( path_to_input_img).stem + "_kmeans.tif" except KeyError as err: raise UP42Error( SupportedErrors.NO_INPUT_ERROR, "up42.data_path was not specified in the metadata.", ) from err out_feature = feature.copy() out_feature["properties"]["up42.data_path"] = path_to_output_img try: self.run_kmeans_clustering( "/tmp/input/" + path_to_input_img, "/tmp/output/" + path_to_output_img, ) results.append(out_feature) except UP42Error as e: logger.warning(e) logger.warning( f"{path_to_input_img} is too large to process, skipping..." ) if not results: raise UP42Error(SupportedErrors.NO_OUTPUT_ERROR) return FeatureCollection(results)
def raise_if_too_large(input_ds: rio.DatasetReader, max_size_bytes: int = 31621877760): """Raises UP42Error if input dataset allocation in memory expected_size max_size_bytes. Xlarge machine has 31621877760 bytes memory Parameters ---------- input_ds : rio.DatasetReader A raster dataset. max_size_bytes : int Maximum allowed size of dataset in bytes (usually memory in machine) Raises ------- UP42Error When estimated input dataset allocation in memory exceedes max_size_bytes """ if input_ds.meta["dtype"] == "uint8": multiplier = 8 elif input_ds.meta["dtype"] == "uint16": multiplier = 16 elif input_ds.meta["dtype"] == "float32": multiplier = 32 else: multiplier = 16 # Calculate expected_size in bytes expected_size = (input_ds.shape[0] * input_ds.shape[1] * input_ds.count * multiplier) / 8 # KMeansClustering algorithm uses at least x4 size of image in bytes in memory # Add x4 buffer for safety expected_size *= 4 * 4 logger.info(f"expected_size is {expected_size}") if expected_size > max_size_bytes: logger.info( f"expected_size {expected_size} is more than max_size_bytes {max_size_bytes}" ) raise UP42Error( SupportedErrors.WRONG_INPUT_ERROR, "Dataset is too large! Please select a smaller AOI.", )
def fetch(self, query: STACQuery, dry_run: bool = False) -> FeatureCollection: query.set_param_if_not_exists("zoom_level", self.default_zoom_level) query.set_param_if_not_exists("imagery_layers", [self.default_imagery_layer]) # Get the list of tiles that cover the query AOI. Sorted by (y, x) in ascending order try: tile_list = list( filter_tiles_intersect_with_geometry( tiles=mercantile.tiles(*query.bounds(), zooms=query.zoom_level, truncate=True), geometry=query.geometry(), )) except MercantileError as mercerr: raise UP42Error( SupportedErrors.INPUT_PARAMETERS_ERROR) from mercerr output_features: List[Feature] = [] date_list = extract_query_dates(query) logger.debug(f"Checking layer {query.imagery_layers}") are_valid, invalid, valid_imagery_layers = self.api.validate_imagery_layers( query.imagery_layers, query.bounds()) if are_valid: logger.debug(f"Layers {query.imagery_layers} OK!") else: raise UP42Error( SupportedErrors.INPUT_PARAMETERS_ERROR, f"Invalid Layers. {invalid} have invalid names." f"{invalid} are layer bounds, search should be within this.", ) for query_date in date_list: self.api.get_layer_bands_count(tile_list, valid_imagery_layers, query_date) for layer in valid_imagery_layers: feature_id: str = str(uuid.uuid4()) return_poly = tiles_to_geom(tile_list) feature = Feature(id=feature_id, bbox=return_poly.bounds, geometry=return_poly) try: self.api.write_quicklook(layer, return_poly.bounds, query_date, feature_id) except requests.exceptions.HTTPError: continue if not dry_run: # Fetch tiles and patch them together img_filename = self.get_final_merged_image( tile_list, valid_imagery_layers, query_date, feature_id) self.api.post_process(img_filename, valid_imagery_layers) to_cog(img_filename, forward_band_tags=True) set_data_path(feature, f"{feature_id}.tif") logger.debug(feature) output_features.append(feature) logger.debug(f"Saving {len(output_features)} result features") return FeatureCollection(list(output_features))
def process(self, input_fc: FeatureCollection): """ Main wrapper method to facilitate snap processing per feature """ polarisations: List = self.params.polarisations or ["VV"] self.assert_input_params() results: List[Feature] = [] out_dict: dict = {} for in_feature in input_fc.get("features"): coordinate = in_feature["bbox"] self.assert_dem(coordinate) try: processed_graphs = self.process_snap(in_feature, polarisations) LOGGER.info("SNAP processing is finished!") if not processed_graphs: LOGGER.debug("No processed images returned, will continue") continue out_feature = copy.deepcopy(in_feature) processed_tif_uuid = out_feature.properties["up42.data_path"] out_path = f"/tmp/output/{processed_tif_uuid}/" if not os.path.exists(out_path): os.mkdir(out_path) for out_polarisation in processed_graphs: # Besides the path we only need to change the capabilities shutil.move( ("%s.tif" % out_polarisation), ("%s%s.tif" % (out_path, out_polarisation.split("_")[-1])), ) del out_feature["properties"]["up42.data_path"] set_data_path(out_feature, processed_tif_uuid + ".tif") results.append(out_feature) out_dict[processed_tif_uuid] = { "id": processed_tif_uuid, "z": [i.split("_")[-1] for i in processed_graphs], "out_path": out_path, } Path(__file__).parent.joinpath( "template/" "snap_polarimetry_graph_%s.xml" % "copy").unlink() except WrongPolarizationError: LOGGER.error( f"WrongPolarizationError: some or all of the polarisations " f"({polarisations}) don't exist in this product " f"({self.safe_file_name(in_feature),}), skipping.", ) continue if not results: raise UP42Error( SupportedErrors.NO_OUTPUT_ERROR, "The used input parameters don't result in any output " "when applied to the provided input images.", ) for out_id in out_dict: # pylint: disable=consider-using-dict-items my_out_path = out_dict[out_id]["out_path"] out_id_z = out_dict[out_id]["z"] if self.params.mask is not None: self.post_process(my_out_path, out_id_z) self.rename_final_stack(my_out_path, out_id_z) result_fc = FeatureCollection(results) if self.params.clip_to_aoi: result_fc = update_extents(result_fc) return result_fc