def deploy_keras_model(dltile, src_product_id, dest_product_id):
import tensorflow as tf
import numpy as np
from descarteslabs.catalog import Image, properties as p
raster_client = dl.Raster()
# NOTE substitute your own trained model here.
model = tf.keras.applications.resnet50.ResNet50()
scene = next(Image.search().filter(
(p.product_id == src_product_id)).intersects(
raster_client.dltile(dltile)).limit(1))
tile, meta = raster_client.ndarray(
scene.id,
bands=["red", "green", "blue"],
scales=[[0, 255]] * 3,
ot="Byte",
dltile=dltile,
)
# resnet50 expects the shape of the input array to be 4 dimensional, which allows for batch
# predictions.
tile_for_prediction = tile[np.newaxis, :]
pred = model.predict(tile_for_prediction)
# get predicted class with a simple maximum of class probabilities.
class_ = np.argmax(pred, 1)
# create a new raster of the tile area with one channel of the prediction from the model.
image_ndarray = np.full(tile.shape[:-1], class_, dtype=np.uint16)
# upload a tile of this "prediction" to catalog
image_name = "-".join(
[src_product_id.replace(":", "_"),
dltile.replace(":", "_")])
image = Image(name=image_name,
product_id=dest_product_id,
acquired=scene.acquired)
image.upload_ndarray(image_ndarray, raster_meta=meta)
def deploy_keras_model(dltile, src_product_id, dest_product_id):
import tensorflow as tf
import numpy as np
catalog_client = dl.Catalog()
raster_client = dl.Raster()
metadata_client = dl.Metadata()
# NOTE substitute your own trained model here.
model = tf.keras.applications.resnet50.ResNet50()
scene = metadata_client.search(src_product_id,
geom=raster_client.dltile(dltile),
limit=1)['features'][0]['id']
tile, meta = raster_client.ndarray(scene,
bands=['red', 'green', 'blue'],
scales=[[0, 255]] * 3,
ot='Byte',
dltile=dltile)
# resnet50 expects the shape of the input array to be 4 dimensional, which allows for batch
# predictions.
tile_for_prediction = tile[np.newaxis, :]
pred = model.predict(tile_for_prediction)
# get predicted class with a simple maximum of class probabilities.
class_ = np.argmax(pred, 1)
# create a new raster of the tile area with one channel of the prediction from the model.
image = np.full(tile.shape[:-1], class_, dtype=np.uint16)
# upload a tile of this "prediction" to catalog
image_id = ':'.join([src_product_id, dltile.replace(':', '_')])
catalog_client.upload_ndarray(image,
dest_product_id,
image_id,
raster_meta=meta)
def __init__(self, path_to_cities_geom, path_to_cities_info):
self.path_to_cities_geom = path_to_cities_geom
self.path_to_cities_info = path_to_cities_info
self.cities_geometry = gpd.read_file(self.path_to_cities_geom)
if os.path.exists(os.path.join(self.path_to_cities_info)):
with open(os.path.join(self.path_to_cities_info), 'r') as infile:
self.cities_info = json.loads(infile.read())
else:
self.cities_info = defaultdict(dict)
self.raster_client = dl.Raster()
self.metadata_client = dl.Metadata()
self.pixel_area = 225
self.tree_area = 50
self.tree_profit = 1.5
def images_from_fids(aoi, fids):
raster_client = dl.Raster()
images = list()
for f in fids:
feat_id = feat["id"]
arr, meta = raster_client.ndarray(
f,
cutline=aoi,
bands=["ndvi", "alpha"],
scales=[[0, 65535, -1, 1], None],
data_type="Float32",
)
images.append(arr)
return images, meta['wgs84Extent']
"""Simple example of how to create a product, then add
some bands and imagery. We will use the included file `building_mask.tif`
as an example of some imagery you might want to upload with the catalog.
"""
import descarteslabs as dl
import os
from random import randint
from time import sleep
catalog_client = dl.Catalog()
metadata_client = dl.Metadata()
raster_client = dl.Raster()
# First step, create a product, which is a descriptive document you use to
# group related images.
product_id = catalog_client.add_product(
"building_mask:osm:v0",
title="OSM Building Mask",
description=
"Rasterized OSM building footprints from vector data. Quality varies regionally",
)["data"]["id"]
# Next we need to add bands. The core function of a band is to tell us how data
# is encoded in the imagery that you are going to upload. For these building
# masks there is only one file per scene, and each scene has one 8 bit band.
band_id = catalog_client.add_band(
product_id=product_id, # id of the product we just created.
name="footprint", # this is a unique name to describe what the band encodes.
def DL_downloader(version, pts, ii_ps, CONFIG, mp_idx, destinations):
import descarteslabs as dl
raster_client = dl.Raster()
if 'gcp' in destinations:
gcp_client = GCPClient(CONFIG['gcp_credentials_path'],
CONFIG['gcp_storage_bucket'], version)
if 'azure' in destinations:
azure_client = AzureClient(CONFIG['azure_path'],
version,
make_container=False)
logger_mp = logging.getLogger(f'DL_{mp_idx}')
def _save_thumbnail(arr, path):
fig, ax = plt.subplots(1, 1, figsize=(6, 6))
ax.imshow(arr)
ax.axis('off')
fig.savefig(path, bbox_inches='tight', pad_inches=0)
plt.close()
TLs = {}
for idx, pt in pts.iterrows():
# make the path
if not os.path.exists(
os.path.join(CONFIG['DATA_ROOT'], version, str(idx))):
os.makedirs(os.path.join(CONFIG['DATA_ROOT'], version, str(idx)))
print(f'checking pt {idx}...', end='')
# get the name
if 'local' in destinations:
name_root = os.path.join(
CONFIG['DATA_ROOT'], version, str(idx), '_'.join([
str(idx), 'DL', pt['DL_S2'].split(':')[2][0:10],
str(pt['lon']),
str(pt['lat'])
]))
else:
name_root = os.path.join(
CONFIG['DATA_ROOT'], 'tmp', '_'.join([
str(idx), 'DL', pt['DL_S2'].split(':')[2][0:10],
str(pt['lon']),
str(pt['lat'])
]))
# check the work done and skip if necessary
checkers = []
if 'gcp' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json',
'_tile.json', '_S2thumb.png', '_S1thumb.png'
]:
checkers.append(gcp_client.check(name_root + ext))
if 'azure' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json',
'_tile.json', '_S2thumb.png', '_S1thumb.png'
]:
checkers.append(azure_client.check(name_root + ext))
if 'local' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json',
'_tile.json', '_S2thumb.png', '_S1thumb.png'
]:
checkers.append(os.path.exists(name_root + ext))
if not np.product(checkers):
# if all checkers true, do everything
# get the tile
tile = raster_client.dltile_from_latlon(pt['lat'], pt['lon'],
CONFIG['resolution'],
CONFIG['patch_size'], 0)
try:
try:
# get the arrays
S2_arr, S2_meta = raster_client.ndarray(
pt['DL_S2'],
bands=CONFIG['DL']['S2_bands'],
scales=[(0, 10000, 0, 10000)] *
(len(CONFIG['DL']['S2_bands']) - 1) + [(0, 1, 0, 1)],
data_type='Float32',
dltile=tile.properties.key,
processing_level='surface')
except:
# search the dl catalog
S2_min_date = dt.strptime(pt['DL_S2'].split(':')[2][0:10],
'%Y-%m-%d') - timedelta(days=1)
S2_max_date = S2_min_date + timedelta(days=3)
S2_scenes, S2_ctx = dl.scenes.search(
aoi=tile.geometry,
products='sentinel-2:L1C',
start_datetime=S2_min_date,
end_datetime=S2_max_date)
_ids = [s._dict()['properties']['id'] for s in S2_scenes]
print('S2 backup search', pt['DL_S2'],
[(_id, fuzz.ratio(_id, pt['DL_S2']))
for _id in _ids])
_id = max(_ids,
key=lambda _id: fuzz.ratio(_id, pt['DL_S2']))
S2_arr, S2_meta = raster_client.ndarray(
_id,
bands=CONFIG['DL']['S2_bands'],
scales=[(0, 10000, 0, 10000)] *
(len(CONFIG['DL']['S2_bands']) - 1) + [(0, 1, 0, 1)],
data_type='Float32',
dltile=tile.properties.key,
processing_level='surface')
try:
S1_arr, S1_meta = raster_client.ndarray(
pt['DL_S1'],
bands=CONFIG['DL']['S1_bands'],
scales=[(0, 255, 0, 255)] *
len(CONFIG['DL']['S1_bands']),
data_type='Float32',
dltile=tile.properties.key,
)
except:
# search the dl catalog
S1_min_date = dt.strptime(pt['DL_S1'].split('_')[1],
'%Y-%m-%d') - timedelta(days=1)
S1_max_date = S1_min_date + timedelta(days=2)
S1_scenes, S1_ctx = dl.scenes.search(
aoi=tile.geometry,
products='sentinel-1:GRD',
start_datetime=S1_min_date,
end_datetime=S1_max_date)
_ids = [s._dict()['properties']['id'] for s in S1_scenes]
print('S1 backup search', pt['DL_S1'],
[(_id, fuzz.ratio(_id, pt['DL_S1']))
for _id in _ids])
_id = max(_ids,
key=lambda _id: fuzz.ratio(_id, pt['DL_S1']))
S1_arr, S1_meta = raster_client.ndarray(
_id,
bands=CONFIG['DL']['S1_bands'],
scales=[(0, 255, 0, 255)] *
len(CONFIG['DL']['S1_bands']),
data_type='Float32',
dltile=tile.properties.key,
)
np.savez(name_root + '_S2arr.npz',
arr=S2_arr.astype(np.float32))
json.dump(S2_meta, open(name_root + '_S2meta.json', 'w'))
np.savez(name_root + '_S1arr.npz',
arr=S1_arr.astype(np.float32))
json.dump(S1_meta, open(name_root + '_S1meta.json', 'w'))
json.dump(tile, open(name_root + '_tile.json', 'w'))
_save_thumbnail(
(S2_arr[:, :,
(3, 2, 1)].astype(np.float32) / 10000).clip(0, 1),
name_root + '_S2thumb.png')
_save_thumbnail((np.stack([
S1_arr[:, :, 0],
np.zeros(S1_arr.shape[0:2]), S1_arr[:, :, 1]
]) / 255 * 2.5).clip(0, 1).transpose([1, 2, 0]),
name_root + '_S1thumb.png')
if 'gcp' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz',
'_S1meta.json', '_tile.json', '_S2thumb.png',
'_S1thumb.png'
]:
gcp_client.upload(name_root + ext)
if 'azure' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz',
'_S1meta.json', '_tile.json', '_S2thumb.png',
'_S1thumb.png'
]:
azure_client.upload(name_root + ext)
if 'local' not in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz',
'_S1meta.json', '_tile.json', '_S2thumb.png',
'_S1thumb.png'
]:
os.remove(name_root + ext)
print(
f'done pt {idx}, S2_min: {S2_arr.min()}, S2_max: {S2_arr.max()}, S1_min: {S1_arr.min()}, S1_max: {S1_arr.max()}, {pt["DL_S2"]},{pt["lon"]},{pt["lat"]}'
)
with open(
os.path.join(os.getcwd(), 'logs', f'{version}_dl.log'),
"a") as f:
f.write(f"{idx}\n")
except Exception as e:
print('Error!')
print(e)
TLs[idx] = dict(tile)
else:
print(f'got pt {idx} already')
with open(os.path.join(os.getcwd(), 'logs', f'{version}_dl.log'),
"a") as f:
f.write(f"{idx}\n")
return TLs, ii_ps
def DL_CLC_downloader(version, pts, ii_ps, CONFIG, mp_idx, TLs, destinations):
import descarteslabs as dl
import warnings
warnings.filterwarnings("ignore")
raster_client = dl.Raster()
if 'gcp' in destinations:
gcp_client = GCPClient(CONFIG['gcp_credentials_path'],
CONFIG['gcp_storage_bucket'], version)
if 'azure' in destinations:
azure_client = AzureClient(CONFIG['azure_path'],
version,
make_container=False)
logger_mp = logging.getLogger(f'LC_{mp_idx}')
colmap = json.load(open(CONFIG['DL_LC']['legend_json'], 'r'))
def _save_LC_thumbnail(arr, path):
# change arr to colmap
im_arr = np.zeros((*arr.shape, 3))
for kk in colmap.keys():
im_arr[arr == int(kk), 0] = colmap[kk]['color'][0]
im_arr[arr == int(kk), 1] = colmap[kk]['color'][1]
im_arr[arr == int(kk), 2] = colmap[kk]['color'][2]
fig, ax = plt.subplots(1, 1, figsize=(6, 6))
#print (im_arr.min(), im_arr.max())
ax.imshow(im_arr.astype(int).clip(0, 255))
ax.axis('off')
fig.savefig(path, bbox_inches='tight', pad_inches=0)
plt.close()
for idx, pt in pts.iterrows():
# make the path
if not os.path.exists(
os.path.join(CONFIG['DATA_ROOT'], version, str(idx))):
os.makedirs(os.path.join(CONFIG['DATA_ROOT'], version, str(idx)))
if 'local' in destinations:
name_root = os.path.join(
CONFIG['DATA_ROOT'], version, str(idx), '_'.join([
str(idx), 'LC', pt['DL_S2'].split(':')[2][0:10],
str(pt['lon']),
str(pt['lat'])
]))
else:
name_root = os.path.join(
CONFIG['DATA_ROOT'], 'tmp', '_'.join([
str(idx), 'LC', pt['DL_S2'].split(':')[2][0:10],
str(pt['lon']),
str(pt['lat'])
]))
# check the work done and skip if necessary
checkers = []
if 'gcp' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json',
'_tile.json', '_S2thumb.png', '_S1thumb.png'
]:
checkers.append(gcp_client.check(name_root + ext))
if 'azure' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json',
'_tile.json', '_S2thumb.png', '_S1thumb.png'
]:
checkers.append(azure_client.check(name_root + ext))
if 'local' in destinations:
for ext in [
'_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json',
'_tile.json', '_S2thumb.png', '_S1thumb.png'
]:
checkers.append(os.path.exists(name_root + ext))
if not np.product(checkers):
# get the tile
tile = TLs[str(idx)]
try:
LC_scenes, LC_ctx = dl.scenes.search(
aoi=tile['geometry'],
products=CONFIG['DL_LC']['LC_product'],
start_datetime=CONFIG['DL_LC']['LC_start_date'],
end_datetime=CONFIG['DL_LC']['LC_end_date'])
# mosaic
for ii_s, s in enumerate(LC_scenes):
#print (s.properties['id'])
s_arr, LC_meta = raster_client.ndarray(
s.properties['id'],
bands=CONFIG['DL_LC']['LC_bands'],
scales=[(0, 255, 0, 255)] *
len(CONFIG['DL_LC']['LC_bands']),
data_type='Byte',
dltile=tile['properties']['key'],
)
if ii_s == 0:
LC_arr = s_arr
LC_arr[LC_arr >= 48] = 0
else:
LC_arr[LC_arr == 0] = s_arr[LC_arr == 0]
LC_arr[LC_arr >= 48] = 0
np.savez(name_root + '_LCarr.npz', arr=LC_arr.astype(np.uint8))
json.dump(LC_meta, open(name_root + '_LCmeta.json', 'w'))
_save_LC_thumbnail(LC_arr, name_root + '_LCthumb.png')
if 'gcp' in destinations:
for ext in ['_LCarr.npz', '_LCmeta.json', '_LCthumb.png']:
gcp_client.upload(name_root + ext)
if 'azure' in destinations:
for ext in ['_LCarr.npz', '_LCmeta.json', '_LCthumb.png']:
azure_client.upload(name_root + ext)
if 'local' not in destinations:
for ext in ['_LCarr.npz', '_LCmeta.json', '_LCthumb.png']:
os.remove(name_root + ext)
print(
f'done pt {idx}, arr_min: {LC_arr.min()}, arr_max: {LC_arr.max()}, latlon: {pt["lon"]},{pt["lat"]}'
)
with open(
os.path.join(os.getcwd(), 'logs',
f'{version}_clc.log'), "a") as f:
f.write(f"{idx}\n")
except Exception as e:
print('Error!')
print(e)
else:
print(f'got pt {idx} already')
with open(os.path.join(os.getcwd(), 'logs', f'{version}_clc.log'),
"a") as f:
f.write(f"{idx}\n")
return ii_ps
def download_cell(job_name, job_loc, cell_id, start_date, end_date):
'''
downloads the data for a given cell.
job_name: name of the job folder where rois file is present.
job_loc: location of the job folder
cell_id: id of the cell to be processed. pass '-1' to process all
'''
def cloud_mask(fpath):
'''
internal function. creates the cloud mask using s2cloudless package
'''
fname = fpath.split('/')[-1]
aband_list = [
'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A',
'B09', 'B10', 'B11', 'B12', 'BQA', 'BQA', 'BQA', 'BQA'
]
cband_list = [
'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11',
'B12'
]
ds = gdal.Open(fpath, 0)
ctr = 0
for cband in cband_list:
cind = aband_list.index(cband)
cdata = ds.GetRasterBand(cind + 1).ReadAsArray()
if ctr == 0:
ndata = np.zeros(
(cdata.shape[0], cdata.shape[1], len(cband_list)))
ndata[:, :, ctr] = cdata
ctr = ctr + 1
ndata = ndata.astype(float)
ndata = ndata * 1.0 / 10000
ndata = np.expand_dims(ndata, 0)
# print(time.time()-start_time)
cloud_detector = S2PixelCloudDetector(threshold=0.4,
average_over=4,
dilation_size=2)
cloud_matrix = cloud_detector.get_cloud_probability_maps(
np.array(ndata))
cloud_matrix = np.squeeze(cloud_matrix)
cloud_matrix = np.round(cloud_matrix * 100).astype(np.uint16)
ctr = 0
for cband in aband_list:
cind = aband_list.index(cband)
cdata = ds.GetRasterBand(cind + 1).ReadAsArray()
if ctr == 0:
odata = np.zeros(
(cdata.shape[0], cdata.shape[1], len(aband_list) + 1))
odata[:, :, ctr] = cdata
ctr = ctr + 1
odata[:, :, -1] = cloud_matrix
odata = odata.astype(np.uint16)
CreateGeoTiffs(odata, fpath, fpath, gdal.GDT_UInt16)
import descarteslabs as dl
raster_client = dl.Raster()
driver = ogr.GetDriverByName("ESRI Shapefile")
job_dir = job_loc + job_name + '/'
roi_file = job_dir + 'rois.shp'
cds = driver.Open(roi_file, 0)
cdl = cds.GetLayer()
if cell_id != '-1':
cdl.SetAttributeFilter("RPOINT_ID='" + cell_id + "'")
for cfeature in cdl:
cgeom = cfeature.GetGeometryRef()
minX, maxX, minY, maxY = cgeom.GetEnvelope() #bounding box of the box
lat_min = minY
lat_max = maxY
lon_min = minX
lon_max = maxX
cur_id = cfeature.GetField('RPOINT_ID')
# create a separate directory for each cell
out_path = job_dir + 'Cells/' #prefix + '-' + curID + '/'
if os.path.isdir(out_path) == False:
os.mkdir(out_path)
out_path = out_path + 'data-' + cur_id + '/'
if os.path.isdir(out_path) == False:
os.mkdir(out_path)
# converting cell boundaries to geometry
aoi_geometry = {
'type':
'Polygon',
'coordinates':
(((lon_min, lat_min), (lon_max, lat_min), (lon_max, lat_max),
(lon_min, lat_max), (lon_min, lat_min)), )
}
#["coastal-aerosol", "blue", "green", "red", "red-edge","red-edge-2","red-edge-3","nir","red-edge-4","water-vapor","cirrus","swir1","swir2","bright-mask","cirrus-cloud-mask","cloud-mask","opaque-cloud-mask"]
#bands = ['B01','B02','B03','B04','B05','B06','B07','B08','B8A','B09','B10','B11','B12','BQA','BQA','BQA','BQA','BCM']
scenes, geoctx = dl.scenes.search(aoi_geometry,
products=["sentinel-2:L1C"],
start_datetime=start_date,
end_datetime=end_date,
limit=None)
print('downloading ' + str(len(scenes)) + ' images for ' + cur_id)
bar = progressbar.ProgressBar(maxval=len(scenes),
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage()
])
bar.start()
for i in range(0, len(scenes)):
curid = scenes[i].properties.id
curname = scenes[i].properties.identifier
# print(curname)
if os.path.isfile(out_path + curname + '.tif') == True:
continue
raster_file = raster_client.raster(
inputs=curid,
bands=[
"coastal-aerosol", "blue", "green", "red", "red-edge",
"red-edge-2", "red-edge-3", "nir", "red-edge-4",
"water-vapor", "cirrus", "swir1", "swir2", "bright-mask",
"cirrus-cloud-mask", "cloud-mask", "opaque-cloud-mask"
],
data_type='UInt16',
align_pixels=True,
cutline=aoi_geometry,
save=True,
outfile_basename=out_path + curname,
output_format='GTiff',
resolution=10)
#add the cloud mask as an additional band in the end
cloud_mask(out_path + curname + '.tif')
bar.update(i + 1)
bar.finish()