def deploy(model_version, output_product):
deploy_aoi = {
"type":
"Polygon",
"coordinates": [[
[-99.24164417538321, 26.138411465362807],
[-93.37666136803256, 26.138411465362807],
[-93.37666136803256, 31.060649553995205],
[-99.24164417538321, 31.060649553995205],
[-99.24164417538321, 26.138411465362807],
]],
}
# Make sure the output product exists
try:
dl.Catalog().get_product(output_product)
except dl.client.exceptions.NotFoundError:
print("Output product {} does not exist".format(output_product))
return
# Decompose our AOI into 1024x1024 pixel tiles at 90m resolution in UTM
tiles = dl.scenes.DLTile.from_shape(deploy_aoi,
resolution=90.0,
tilesize=1024,
pad=0)
# Register our prediction function in the Tasks environment.
#
# We specify the resource requirements per worker (1 CPU & 2GB of RAM),
# the environment (container with Python 3.7), and any extra PyPI
# requirements (descarteslabs client and scikit-learn).
tasks = dl.Tasks()
run_model_remotely = tasks.create_function(
run_model,
name="example water model deployment",
image=
"us.gcr.io/dl-ci-cd/images/tasks/public/py3.7/default:v2019.05.29",
cpu=1.0,
memory="2Gi",
requirements=[
"descarteslabs[complete]==0.19.0", "scikit-learn==0.21.1"
],
)
# Create a list with arguments of each invocation of run_model
task_arguments = [(model_version, dltile.key, output_product)
for dltile in tiles]
results = run_model_remotely.map(*zip(*task_arguments))
print("Submitted {} tasks to task group {}...".format(
len(tiles), run_model_remotely.group_id))
# Iterate through task results as they complete.
#
# If some of the tasks failed, we will print the console output and the
# arguments of that invocation.
#
# Note that this is for informational purposes only, and the tasks will
# continue running if the script is interrupted at this point. You can use
# https://monitor.descarteslabs.com/ to see the status of all running
# task groups.
for i, task in enumerate(as_completed(results, show_progress=False)):
percent_complete = 100.0 * i / len(results)
print(
"Progress update: {} completed out of {} ({:.2f}%) - last task took {:.2f}sec to {}"
.format(
i + 1,
len(results),
percent_complete,
task.runtime,
"succeed" if task.is_success else "fail",
))
if not task.is_success:
print("\nTASK FAILURE with arguments {}:\n{}".format(
task.args, task.log.decode()))
# Clean up the task group
tasks.delete_group_by_id(run_model_remotely.group_id)
tile_for_prediction = tile[np.newaxis, :]
pred = model.predict(tile_for_prediction)
# get predicted class with a simple maximum of class probabilities.
class_ = np.argmax(pred, 1)
# create a new raster of the tile area with one channel of the prediction from the model.
image = np.full(tile.shape[:-1], class_, dtype=np.uint16)
# upload a tile of this "prediction" to catalog
image_id = ':'.join([src_product_id, dltile.replace(':', '_')])
catalog_client.upload_ndarray(image,
dest_product_id,
image_id,
raster_meta=meta)
if __name__ == '__main__':
tasks = dl.Tasks()
catalog_client = dl.Catalog()
raster_client = dl.Raster()
async_function = tasks.create_function(
deploy_keras_model,
image='us.gcr.io/dl-ci-cd/images/tasks/public/py2/default:v2018.06.20',
name="deploy-resnet",
)
print('task group id', async_function.group_id)
try:
prod = catalog_client.get_product(':'.join(
[catalog_client.auth.namespace, 'resnet-predictions']))
except dl.exceptions.NotFoundError:
prod = catalog_client.add_product(
'resnet-predictions',
def download(AOI,
product_id,
resolution=10.,
start_datetime=None,
end_datetime=None,
bands=None,
destination='dl.tif',
tile_size=None,
run_async=False):
"""
Download GeoTiff mosaics as tiles or single images
Parameters
----------
AOI: GeoJSON Geometry, required
product_id: str, required
resolution: float, optional
Resolution in meters
bands: list, optional default: None
List of band names to include, None for all bands
destination: str, required default: dl.tif
Output file to write into
tile_size: int, optional default: None
Tile size in pixels to break the download into. Defaults to None - no tiling
run_async: bool, optional default: False
Run asyncronously on DL tasks, helpful if many tiles are required an Result is passed
through DL Storage to the client and stored behind a key with the same name as the output path
Returns
-------
path: str
Local path to file downloaded
"""
if run_async:
async_func = dl.Tasks().create_function(
'dlutils.downloader.download_async',
image=
f'us.gcr.io/dl-ci-cd/images/tasks/public/py3.7/default:v2019.08.08-7-g062b0653',
name=f'Downloading {product_id}',
cpu=8,
maximum_concurrency=1,
mem='32Gi',
task_timeout=60 * 120,
include_modules=['dlutils'],
requirements=['tqdm'])
job_id = uuid4().hex
else:
async_func = download_async
job_id = None
if run_async:
print(f"""
Assembling {destination.split('/')[-1]} on DLTasks ... this will take a few minutes."""
)
result = async_func(AOI,
product_id,
resolution,
destination,
start_datetime=start_datetime,
end_datetime=end_datetime,
tile_size=tile_size,
bands=bands,
run_async=run_async,
job_id=job_id)
if run_async:
if result.status == 'SUCCESS' and result.result is not None:
print(f'Downloading {result.result} to {destination}')
dl.Storage().get_file(result.result, destination)
dl.Storage().delete(result.result)
print('Done.')
else:
print(result.status)
dl.Tasks().delete_group_by_id(async_func.group_id)
else:
print(f'Saved to {result}')