def deploy_keras_model(dltile, src_product_id, dest_product_id): import tensorflow as tf import numpy as np from descarteslabs.catalog import Image, properties as p raster_client = dl.Raster() # NOTE substitute your own trained model here. model = tf.keras.applications.resnet50.ResNet50() scene = next(Image.search().filter( (p.product_id == src_product_id)).intersects( raster_client.dltile(dltile)).limit(1)) tile, meta = raster_client.ndarray( scene.id, bands=["red", "green", "blue"], scales=[[0, 255]] * 3, ot="Byte", dltile=dltile, ) # resnet50 expects the shape of the input array to be 4 dimensional, which allows for batch # predictions. tile_for_prediction = tile[np.newaxis, :] pred = model.predict(tile_for_prediction) # get predicted class with a simple maximum of class probabilities. class_ = np.argmax(pred, 1) # create a new raster of the tile area with one channel of the prediction from the model. image_ndarray = np.full(tile.shape[:-1], class_, dtype=np.uint16) # upload a tile of this "prediction" to catalog image_name = "-".join( [src_product_id.replace(":", "_"), dltile.replace(":", "_")]) image = Image(name=image_name, product_id=dest_product_id, acquired=scene.acquired) image.upload_ndarray(image_ndarray, raster_meta=meta)
def deploy_keras_model(dltile, src_product_id, dest_product_id): import tensorflow as tf import numpy as np catalog_client = dl.Catalog() raster_client = dl.Raster() metadata_client = dl.Metadata() # NOTE substitute your own trained model here. model = tf.keras.applications.resnet50.ResNet50() scene = metadata_client.search(src_product_id, geom=raster_client.dltile(dltile), limit=1)['features'][0]['id'] tile, meta = raster_client.ndarray(scene, bands=['red', 'green', 'blue'], scales=[[0, 255]] * 3, ot='Byte', dltile=dltile) # resnet50 expects the shape of the input array to be 4 dimensional, which allows for batch # predictions. tile_for_prediction = tile[np.newaxis, :] pred = model.predict(tile_for_prediction) # get predicted class with a simple maximum of class probabilities. class_ = np.argmax(pred, 1) # create a new raster of the tile area with one channel of the prediction from the model. image = np.full(tile.shape[:-1], class_, dtype=np.uint16) # upload a tile of this "prediction" to catalog image_id = ':'.join([src_product_id, dltile.replace(':', '_')]) catalog_client.upload_ndarray(image, dest_product_id, image_id, raster_meta=meta)
def __init__(self, path_to_cities_geom, path_to_cities_info): self.path_to_cities_geom = path_to_cities_geom self.path_to_cities_info = path_to_cities_info self.cities_geometry = gpd.read_file(self.path_to_cities_geom) if os.path.exists(os.path.join(self.path_to_cities_info)): with open(os.path.join(self.path_to_cities_info), 'r') as infile: self.cities_info = json.loads(infile.read()) else: self.cities_info = defaultdict(dict) self.raster_client = dl.Raster() self.metadata_client = dl.Metadata() self.pixel_area = 225 self.tree_area = 50 self.tree_profit = 1.5
def images_from_fids(aoi, fids): raster_client = dl.Raster() images = list() for f in fids: feat_id = feat["id"] arr, meta = raster_client.ndarray( f, cutline=aoi, bands=["ndvi", "alpha"], scales=[[0, 65535, -1, 1], None], data_type="Float32", ) images.append(arr) return images, meta['wgs84Extent']
"""Simple example of how to create a product, then add some bands and imagery. We will use the included file `building_mask.tif` as an example of some imagery you might want to upload with the catalog. """ import descarteslabs as dl import os from random import randint from time import sleep catalog_client = dl.Catalog() metadata_client = dl.Metadata() raster_client = dl.Raster() # First step, create a product, which is a descriptive document you use to # group related images. product_id = catalog_client.add_product( "building_mask:osm:v0", title="OSM Building Mask", description= "Rasterized OSM building footprints from vector data. Quality varies regionally", )["data"]["id"] # Next we need to add bands. The core function of a band is to tell us how data # is encoded in the imagery that you are going to upload. For these building # masks there is only one file per scene, and each scene has one 8 bit band. band_id = catalog_client.add_band( product_id=product_id, # id of the product we just created. name="footprint", # this is a unique name to describe what the band encodes.
def DL_downloader(version, pts, ii_ps, CONFIG, mp_idx, destinations): import descarteslabs as dl raster_client = dl.Raster() if 'gcp' in destinations: gcp_client = GCPClient(CONFIG['gcp_credentials_path'], CONFIG['gcp_storage_bucket'], version) if 'azure' in destinations: azure_client = AzureClient(CONFIG['azure_path'], version, make_container=False) logger_mp = logging.getLogger(f'DL_{mp_idx}') def _save_thumbnail(arr, path): fig, ax = plt.subplots(1, 1, figsize=(6, 6)) ax.imshow(arr) ax.axis('off') fig.savefig(path, bbox_inches='tight', pad_inches=0) plt.close() TLs = {} for idx, pt in pts.iterrows(): # make the path if not os.path.exists( os.path.join(CONFIG['DATA_ROOT'], version, str(idx))): os.makedirs(os.path.join(CONFIG['DATA_ROOT'], version, str(idx))) print(f'checking pt {idx}...', end='') # get the name if 'local' in destinations: name_root = os.path.join( CONFIG['DATA_ROOT'], version, str(idx), '_'.join([ str(idx), 'DL', pt['DL_S2'].split(':')[2][0:10], str(pt['lon']), str(pt['lat']) ])) else: name_root = os.path.join( CONFIG['DATA_ROOT'], 'tmp', '_'.join([ str(idx), 'DL', pt['DL_S2'].split(':')[2][0:10], str(pt['lon']), str(pt['lat']) ])) # check the work done and skip if necessary checkers = [] if 'gcp' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: checkers.append(gcp_client.check(name_root + ext)) if 'azure' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: checkers.append(azure_client.check(name_root + ext)) if 'local' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: checkers.append(os.path.exists(name_root + ext)) if not np.product(checkers): # if all checkers true, do everything # get the tile tile = raster_client.dltile_from_latlon(pt['lat'], pt['lon'], CONFIG['resolution'], CONFIG['patch_size'], 0) try: try: # get the arrays S2_arr, S2_meta = raster_client.ndarray( pt['DL_S2'], bands=CONFIG['DL']['S2_bands'], scales=[(0, 10000, 0, 10000)] * (len(CONFIG['DL']['S2_bands']) - 1) + [(0, 1, 0, 1)], data_type='Float32', dltile=tile.properties.key, processing_level='surface') except: # search the dl catalog S2_min_date = dt.strptime(pt['DL_S2'].split(':')[2][0:10], '%Y-%m-%d') - timedelta(days=1) S2_max_date = S2_min_date + timedelta(days=3) S2_scenes, S2_ctx = dl.scenes.search( aoi=tile.geometry, products='sentinel-2:L1C', start_datetime=S2_min_date, end_datetime=S2_max_date) _ids = [s._dict()['properties']['id'] for s in S2_scenes] print('S2 backup search', pt['DL_S2'], [(_id, fuzz.ratio(_id, pt['DL_S2'])) for _id in _ids]) _id = max(_ids, key=lambda _id: fuzz.ratio(_id, pt['DL_S2'])) S2_arr, S2_meta = raster_client.ndarray( _id, bands=CONFIG['DL']['S2_bands'], scales=[(0, 10000, 0, 10000)] * (len(CONFIG['DL']['S2_bands']) - 1) + [(0, 1, 0, 1)], data_type='Float32', dltile=tile.properties.key, processing_level='surface') try: S1_arr, S1_meta = raster_client.ndarray( pt['DL_S1'], bands=CONFIG['DL']['S1_bands'], scales=[(0, 255, 0, 255)] * len(CONFIG['DL']['S1_bands']), data_type='Float32', dltile=tile.properties.key, ) except: # search the dl catalog S1_min_date = dt.strptime(pt['DL_S1'].split('_')[1], '%Y-%m-%d') - timedelta(days=1) S1_max_date = S1_min_date + timedelta(days=2) S1_scenes, S1_ctx = dl.scenes.search( aoi=tile.geometry, products='sentinel-1:GRD', start_datetime=S1_min_date, end_datetime=S1_max_date) _ids = [s._dict()['properties']['id'] for s in S1_scenes] print('S1 backup search', pt['DL_S1'], [(_id, fuzz.ratio(_id, pt['DL_S1'])) for _id in _ids]) _id = max(_ids, key=lambda _id: fuzz.ratio(_id, pt['DL_S1'])) S1_arr, S1_meta = raster_client.ndarray( _id, bands=CONFIG['DL']['S1_bands'], scales=[(0, 255, 0, 255)] * len(CONFIG['DL']['S1_bands']), data_type='Float32', dltile=tile.properties.key, ) np.savez(name_root + '_S2arr.npz', arr=S2_arr.astype(np.float32)) json.dump(S2_meta, open(name_root + '_S2meta.json', 'w')) np.savez(name_root + '_S1arr.npz', arr=S1_arr.astype(np.float32)) json.dump(S1_meta, open(name_root + '_S1meta.json', 'w')) json.dump(tile, open(name_root + '_tile.json', 'w')) _save_thumbnail( (S2_arr[:, :, (3, 2, 1)].astype(np.float32) / 10000).clip(0, 1), name_root + '_S2thumb.png') _save_thumbnail((np.stack([ S1_arr[:, :, 0], np.zeros(S1_arr.shape[0:2]), S1_arr[:, :, 1] ]) / 255 * 2.5).clip(0, 1).transpose([1, 2, 0]), name_root + '_S1thumb.png') if 'gcp' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: gcp_client.upload(name_root + ext) if 'azure' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: azure_client.upload(name_root + ext) if 'local' not in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: os.remove(name_root + ext) print( f'done pt {idx}, S2_min: {S2_arr.min()}, S2_max: {S2_arr.max()}, S1_min: {S1_arr.min()}, S1_max: {S1_arr.max()}, {pt["DL_S2"]},{pt["lon"]},{pt["lat"]}' ) with open( os.path.join(os.getcwd(), 'logs', f'{version}_dl.log'), "a") as f: f.write(f"{idx}\n") except Exception as e: print('Error!') print(e) TLs[idx] = dict(tile) else: print(f'got pt {idx} already') with open(os.path.join(os.getcwd(), 'logs', f'{version}_dl.log'), "a") as f: f.write(f"{idx}\n") return TLs, ii_ps
def DL_CLC_downloader(version, pts, ii_ps, CONFIG, mp_idx, TLs, destinations): import descarteslabs as dl import warnings warnings.filterwarnings("ignore") raster_client = dl.Raster() if 'gcp' in destinations: gcp_client = GCPClient(CONFIG['gcp_credentials_path'], CONFIG['gcp_storage_bucket'], version) if 'azure' in destinations: azure_client = AzureClient(CONFIG['azure_path'], version, make_container=False) logger_mp = logging.getLogger(f'LC_{mp_idx}') colmap = json.load(open(CONFIG['DL_LC']['legend_json'], 'r')) def _save_LC_thumbnail(arr, path): # change arr to colmap im_arr = np.zeros((*arr.shape, 3)) for kk in colmap.keys(): im_arr[arr == int(kk), 0] = colmap[kk]['color'][0] im_arr[arr == int(kk), 1] = colmap[kk]['color'][1] im_arr[arr == int(kk), 2] = colmap[kk]['color'][2] fig, ax = plt.subplots(1, 1, figsize=(6, 6)) #print (im_arr.min(), im_arr.max()) ax.imshow(im_arr.astype(int).clip(0, 255)) ax.axis('off') fig.savefig(path, bbox_inches='tight', pad_inches=0) plt.close() for idx, pt in pts.iterrows(): # make the path if not os.path.exists( os.path.join(CONFIG['DATA_ROOT'], version, str(idx))): os.makedirs(os.path.join(CONFIG['DATA_ROOT'], version, str(idx))) if 'local' in destinations: name_root = os.path.join( CONFIG['DATA_ROOT'], version, str(idx), '_'.join([ str(idx), 'LC', pt['DL_S2'].split(':')[2][0:10], str(pt['lon']), str(pt['lat']) ])) else: name_root = os.path.join( CONFIG['DATA_ROOT'], 'tmp', '_'.join([ str(idx), 'LC', pt['DL_S2'].split(':')[2][0:10], str(pt['lon']), str(pt['lat']) ])) # check the work done and skip if necessary checkers = [] if 'gcp' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: checkers.append(gcp_client.check(name_root + ext)) if 'azure' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: checkers.append(azure_client.check(name_root + ext)) if 'local' in destinations: for ext in [ '_S2arr.npz', '_S2meta.json', '_S1arr.npz', '_S1meta.json', '_tile.json', '_S2thumb.png', '_S1thumb.png' ]: checkers.append(os.path.exists(name_root + ext)) if not np.product(checkers): # get the tile tile = TLs[str(idx)] try: LC_scenes, LC_ctx = dl.scenes.search( aoi=tile['geometry'], products=CONFIG['DL_LC']['LC_product'], start_datetime=CONFIG['DL_LC']['LC_start_date'], end_datetime=CONFIG['DL_LC']['LC_end_date']) # mosaic for ii_s, s in enumerate(LC_scenes): #print (s.properties['id']) s_arr, LC_meta = raster_client.ndarray( s.properties['id'], bands=CONFIG['DL_LC']['LC_bands'], scales=[(0, 255, 0, 255)] * len(CONFIG['DL_LC']['LC_bands']), data_type='Byte', dltile=tile['properties']['key'], ) if ii_s == 0: LC_arr = s_arr LC_arr[LC_arr >= 48] = 0 else: LC_arr[LC_arr == 0] = s_arr[LC_arr == 0] LC_arr[LC_arr >= 48] = 0 np.savez(name_root + '_LCarr.npz', arr=LC_arr.astype(np.uint8)) json.dump(LC_meta, open(name_root + '_LCmeta.json', 'w')) _save_LC_thumbnail(LC_arr, name_root + '_LCthumb.png') if 'gcp' in destinations: for ext in ['_LCarr.npz', '_LCmeta.json', '_LCthumb.png']: gcp_client.upload(name_root + ext) if 'azure' in destinations: for ext in ['_LCarr.npz', '_LCmeta.json', '_LCthumb.png']: azure_client.upload(name_root + ext) if 'local' not in destinations: for ext in ['_LCarr.npz', '_LCmeta.json', '_LCthumb.png']: os.remove(name_root + ext) print( f'done pt {idx}, arr_min: {LC_arr.min()}, arr_max: {LC_arr.max()}, latlon: {pt["lon"]},{pt["lat"]}' ) with open( os.path.join(os.getcwd(), 'logs', f'{version}_clc.log'), "a") as f: f.write(f"{idx}\n") except Exception as e: print('Error!') print(e) else: print(f'got pt {idx} already') with open(os.path.join(os.getcwd(), 'logs', f'{version}_clc.log'), "a") as f: f.write(f"{idx}\n") return ii_ps
def download_cell(job_name, job_loc, cell_id, start_date, end_date): ''' downloads the data for a given cell. job_name: name of the job folder where rois file is present. job_loc: location of the job folder cell_id: id of the cell to be processed. pass '-1' to process all ''' def cloud_mask(fpath): ''' internal function. creates the cloud mask using s2cloudless package ''' fname = fpath.split('/')[-1] aband_list = [ 'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12', 'BQA', 'BQA', 'BQA', 'BQA' ] cband_list = [ 'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12' ] ds = gdal.Open(fpath, 0) ctr = 0 for cband in cband_list: cind = aband_list.index(cband) cdata = ds.GetRasterBand(cind + 1).ReadAsArray() if ctr == 0: ndata = np.zeros( (cdata.shape[0], cdata.shape[1], len(cband_list))) ndata[:, :, ctr] = cdata ctr = ctr + 1 ndata = ndata.astype(float) ndata = ndata * 1.0 / 10000 ndata = np.expand_dims(ndata, 0) # print(time.time()-start_time) cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=2) cloud_matrix = cloud_detector.get_cloud_probability_maps( np.array(ndata)) cloud_matrix = np.squeeze(cloud_matrix) cloud_matrix = np.round(cloud_matrix * 100).astype(np.uint16) ctr = 0 for cband in aband_list: cind = aband_list.index(cband) cdata = ds.GetRasterBand(cind + 1).ReadAsArray() if ctr == 0: odata = np.zeros( (cdata.shape[0], cdata.shape[1], len(aband_list) + 1)) odata[:, :, ctr] = cdata ctr = ctr + 1 odata[:, :, -1] = cloud_matrix odata = odata.astype(np.uint16) CreateGeoTiffs(odata, fpath, fpath, gdal.GDT_UInt16) import descarteslabs as dl raster_client = dl.Raster() driver = ogr.GetDriverByName("ESRI Shapefile") job_dir = job_loc + job_name + '/' roi_file = job_dir + 'rois.shp' cds = driver.Open(roi_file, 0) cdl = cds.GetLayer() if cell_id != '-1': cdl.SetAttributeFilter("RPOINT_ID='" + cell_id + "'") for cfeature in cdl: cgeom = cfeature.GetGeometryRef() minX, maxX, minY, maxY = cgeom.GetEnvelope() #bounding box of the box lat_min = minY lat_max = maxY lon_min = minX lon_max = maxX cur_id = cfeature.GetField('RPOINT_ID') # create a separate directory for each cell out_path = job_dir + 'Cells/' #prefix + '-' + curID + '/' if os.path.isdir(out_path) == False: os.mkdir(out_path) out_path = out_path + 'data-' + cur_id + '/' if os.path.isdir(out_path) == False: os.mkdir(out_path) # converting cell boundaries to geometry aoi_geometry = { 'type': 'Polygon', 'coordinates': (((lon_min, lat_min), (lon_max, lat_min), (lon_max, lat_max), (lon_min, lat_max), (lon_min, lat_min)), ) } #["coastal-aerosol", "blue", "green", "red", "red-edge","red-edge-2","red-edge-3","nir","red-edge-4","water-vapor","cirrus","swir1","swir2","bright-mask","cirrus-cloud-mask","cloud-mask","opaque-cloud-mask"] #bands = ['B01','B02','B03','B04','B05','B06','B07','B08','B8A','B09','B10','B11','B12','BQA','BQA','BQA','BQA','BCM'] scenes, geoctx = dl.scenes.search(aoi_geometry, products=["sentinel-2:L1C"], start_datetime=start_date, end_datetime=end_date, limit=None) print('downloading ' + str(len(scenes)) + ' images for ' + cur_id) bar = progressbar.ProgressBar(maxval=len(scenes), widgets=[ progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage() ]) bar.start() for i in range(0, len(scenes)): curid = scenes[i].properties.id curname = scenes[i].properties.identifier # print(curname) if os.path.isfile(out_path + curname + '.tif') == True: continue raster_file = raster_client.raster( inputs=curid, bands=[ "coastal-aerosol", "blue", "green", "red", "red-edge", "red-edge-2", "red-edge-3", "nir", "red-edge-4", "water-vapor", "cirrus", "swir1", "swir2", "bright-mask", "cirrus-cloud-mask", "cloud-mask", "opaque-cloud-mask" ], data_type='UInt16', align_pixels=True, cutline=aoi_geometry, save=True, outfile_basename=out_path + curname, output_format='GTiff', resolution=10) #add the cloud mask as an additional band in the end cloud_mask(out_path + curname + '.tif') bar.update(i + 1) bar.finish()