def deploy_keras_model(dltile, src_product_id, dest_product_id): import tensorflow as tf import numpy as np catalog_client = dl.Catalog() raster_client = dl.Raster() metadata_client = dl.Metadata() # NOTE substitute your own trained model here. model = tf.keras.applications.resnet50.ResNet50() scene = metadata_client.search(src_product_id, geom=raster_client.dltile(dltile), limit=1)['features'][0]['id'] tile, meta = raster_client.ndarray(scene, bands=['red', 'green', 'blue'], scales=[[0, 255]] * 3, ot='Byte', dltile=dltile) # resnet50 expects the shape of the input array to be 4 dimensional, which allows for batch # predictions. tile_for_prediction = tile[np.newaxis, :] pred = model.predict(tile_for_prediction) # get predicted class with a simple maximum of class probabilities. class_ = np.argmax(pred, 1) # create a new raster of the tile area with one channel of the prediction from the model. image = np.full(tile.shape[:-1], class_, dtype=np.uint16) # upload a tile of this "prediction" to catalog image_id = ':'.join([src_product_id, dltile.replace(':', '_')]) catalog_client.upload_ndarray(image, dest_product_id, image_id, raster_meta=meta)
def __init__(self, aoi, item_type=ItemType.PSScene4Band, product_id=None, title=None, description=None, overwrite=False, start_datetime=None, end_datetime=None, cloud_fraction=1, limit=None, order_id=None, api_key=None): self._planet = p.ClientV1(api_key=api_key) self._catalog = dl.Catalog() self._metadata = dl.Metadata() self._auth = dl.Auth() self._order_id = order_id self._title=title self._description=description self.stats = None self._running = False self._items = [] self.uploads = [] if self._running: raise Exception('Already processing') else: self._running = True self._start_datetime = start_datetime self._end_datetime = end_datetime self._cloud_fraction = cloud_fraction self._limit =limit self._get_items(aoi, [item_type.name]) self._init_product(product_id, item_type=item_type, overwrite=overwrite) item_ids = [item['id'] for item in self._items] scenes = clip_and_download(aoi, item_ids, item_type.name, item_type.bundle, api_key, order_id=self._order_id) for scene_id, scene in scenes.items(): with open(scene['metadata.json']) as meta_file: metadata = json.load(meta_file)['properties'] with open(scene['3B_AnalyticMS_metadata_clip.xml']) as xml_file: xml_meta = xmltodict.parse(xml_file.read()) for band in xml_meta['ps:EarthObservation']['gml:resultOf']['ps:EarthObservationResult']['ps:bandSpecificMetadata']: metadata[f"band_{band['ps:bandNumber']}_radiometricScaleFactor"] = band['ps:radiometricScaleFactor'] metadata[f"band_{band['ps:bandNumber']}_reflectanceCoefficient"] = band['ps:reflectanceCoefficient'] self._upload_image([str(scene[str(file_key)]) for file_key in item_type.files] , metadata, scene_id)
def run_model(model_version, dltile_key, output_product_id): import descarteslabs as dl import numpy as np import pickle import os # Load the model, optionally downloading it from DL storage model_filename = "water_model_{}".format(model_version) if not os.path.exists(model_filename): try: dl.Storage().get_file(model_filename, model_filename) except dl.client.exceptions.NotFoundError: print("Could not find {} in DL storage".format(model_filename)) return with open(model_filename, "rb") as f: classifier = pickle.load(f) # Find input scenes for the dltile dltile = dl.scenes.DLTile.from_key(dltile_key) input_scenes, ctx = dl.scenes.search( dltile, products=["sentinel-1:GRD"], start_datetime="2019-03-01", end_datetime="2019-05-01", query=getattr(dl.properties, "pass") == "ASCENDING", ) print("Input scenes:", input_scenes) input_stack = input_scenes.stack("vv vh", ctx, bands_axis=-1) # Predict water_prediction = classifier.predict(input_stack.reshape(-1, 2)) water_prediction = water_prediction.reshape(input_stack[:, :, :, 0].shape) # Aggregate the predictions into a single raster for the tile composite_prediction = water_prediction.mean(axis=0) # Discretize our prediction from 0.0-1.0 to 0-255 composite_prediction = (composite_prediction * 255).astype(np.uint8) print("Uploading output to the catalog with shape", composite_prediction.shape) # Upload the result back to catalog upload_id = dl.Catalog().upload_ndarray( composite_prediction, output_product_id, dltile.key, geotrans=ctx.geotrans, proj4=ctx.proj4, ) print("Upload task id:", upload_id)
def create_product(product_name): catalog = dl.Catalog() # Create a product in the catalog. We will upload output rasters from the # model to this product for storage and further consumption. product = catalog.add_product( product_name, title="Water Classification Example", description="Water Classification Example", ) # Configure a band for our classification output catalog.add_band( product["data"]["id"], name="water", type="mask", srcband=1, # 1-based index dtype="Byte", nbits=8, data_range=[0, 255], ) print("Created product", product["data"]["id"])
"""Simple example of how to create a product, then add some bands and imagery. We will use the included file `building_mask.tif` as an example of some imagery you might want to upload with the catalog. """ import descarteslabs as dl import os from random import randint from time import sleep catalog_client = dl.Catalog() metadata_client = dl.Metadata() raster_client = dl.Raster() # First step, create a product, which is a descriptive document you use to # group related images. product_id = catalog_client.add_product( "building_mask:osm:v0", title="OSM Building Mask", description= "Rasterized OSM building footprints from vector data. Quality varies regionally", )["data"]["id"] # Next we need to add bands. The core function of a band is to tell us how data # is encoded in the imagery that you are going to upload. For these building # masks there is only one file per scene, and each scene has one 8 bit band. band_id = catalog_client.add_band( product_id=product_id, # id of the product we just created. name="footprint", # this is a unique name to describe what the band encodes.
def deploy(model_version, output_product): deploy_aoi = { "type": "Polygon", "coordinates": [[ [-99.24164417538321, 26.138411465362807], [-93.37666136803256, 26.138411465362807], [-93.37666136803256, 31.060649553995205], [-99.24164417538321, 31.060649553995205], [-99.24164417538321, 26.138411465362807], ]], } # Make sure the output product exists try: dl.Catalog().get_product(output_product) except dl.client.exceptions.NotFoundError: print("Output product {} does not exist".format(output_product)) return # Decompose our AOI into 1024x1024 pixel tiles at 90m resolution in UTM tiles = dl.scenes.DLTile.from_shape(deploy_aoi, resolution=90.0, tilesize=1024, pad=0) # Register our prediction function in the Tasks environment. # # We specify the resource requirements per worker (1 CPU & 2GB of RAM), # the environment (container with Python 3.7), and any extra PyPI # requirements (descarteslabs client and scikit-learn). tasks = dl.Tasks() run_model_remotely = tasks.create_function( run_model, name="example water model deployment", image= "us.gcr.io/dl-ci-cd/images/tasks/public/py3.7/default:v2019.05.29", cpu=1.0, memory="2Gi", requirements=[ "descarteslabs[complete]==0.19.0", "scikit-learn==0.21.1" ], ) # Create a list with arguments of each invocation of run_model task_arguments = [(model_version, dltile.key, output_product) for dltile in tiles] results = run_model_remotely.map(*zip(*task_arguments)) print("Submitted {} tasks to task group {}...".format( len(tiles), run_model_remotely.group_id)) # Iterate through task results as they complete. # # If some of the tasks failed, we will print the console output and the # arguments of that invocation. # # Note that this is for informational purposes only, and the tasks will # continue running if the script is interrupted at this point. You can use # https://monitor.descarteslabs.com/ to see the status of all running # task groups. for i, task in enumerate(as_completed(results, show_progress=False)): percent_complete = 100.0 * i / len(results) print( "Progress update: {} completed out of {} ({:.2f}%) - last task took {:.2f}sec to {}" .format( i + 1, len(results), percent_complete, task.runtime, "succeed" if task.is_success else "fail", )) if not task.is_success: print("\nTASK FAILURE with arguments {}:\n{}".format( task.args, task.log.decode())) # Clean up the task group tasks.delete_group_by_id(run_model_remotely.group_id)
""" ================================================== Upload ndarray to new product ================================================== This example demonstrates how to create a product in our Catalog and upload an example scene. """ import descarteslabs as dl import numpy as np catalog = dl.Catalog() ################################################ # Create a product entry in our Catalog product = catalog.add_product( "Paris_final_3", title="Simple Image Upload_final_3", description="An example of creating a product, adding the visible band range, and ingesting a single scene.", ) # Maintain the product id to upload scenes product_id = product["data"]["id"] ################################################ # Add band information to the product # This is a necessary step, and requires the user # to know a bit about the data to be ingested bands = ["red", "green", "blue"] for val, band in enumerate(bands):