def get_time_series(aoi, start_date=None, end_date=None, bands=['B8'], satellite='Landsat', sensor=None, out_dir='', api='devseed', mirror='gcloud', cloud_masks=False, check_empty=False, parallel_downloads=multiprocessing.cpu_count()): """ Main function: crop and download a time series of Sentinel-2 images. Args: aoi (geojson.Polygon): area of interest start_date (datetime.datetime, optional): start of the time range end_date (datetime.datetime, optional): end of the time range bands (list, optional): list of bands satellite (str, optional): either L1...L8 sensor (str, optional): MSS, TM, ETM, OLI see https://landsat.usgs.gov/what-are-band-designations-landsat-satellites out_dir (str, optional): path where to store the downloaded crops api (str, optional): either devseed (default), scihub, planet or gcloud mirror (str, optional): either 'aws' or 'gcloud' cloud_masks (bool, optional): if True, cloud masks are downloaded and cloudy images are discarded check_empty (bool, optional): if True, QA masks are downloaded and empty images are discarded parallel_downloads (int): number of parallel gml files downloads """ # check access to the selected search api and download mirror check_args(api, mirror) # default date range if end_date is None: end_date = datetime.date.today() if start_date is None: start_date = end_date - datetime.timedelta(91) # 3 months # list available images images = search(aoi, start_date, end_date, satellite, sensor, api=api) # download crops download(images, bands, aoi, mirror, out_dir, parallel_downloads) # discard images that failed to download images = [i for i in images if bands_files_are_valid(i, bands, api, out_dir)] # embed all metadata as GeoTIFF tags in the image files for img in images: metadata = vars(img) metadata['downloaded_by'] = 'TSD on {}'.format(datetime.datetime.now().isoformat()) for b in bands: filepath = os.path.join(out_dir, '{}_band_{}.tif'.format(img.filename, b)) utils.set_geotif_metadata_items(filepath, metadata) if cloud_masks: # discard images that are totally covered by clouds read_cloud_masks(images, bands, parallel_downloads, out_dir=out_dir) if check_empty: # discard images that are totally empty read_empty_images(images, bands, parallel_downloads, out_dir=out_dir)
def get_time_series(aoi=None, start_date=None, end_date=None, bands=["B04"], tile_id=None, title=None, relative_orbit_number=None, out_dir="", api="stac", mirror="aws", product_type="L2A", cloud_masks=False, parallel_downloads=multiprocessing.cpu_count(), satellite_angles=False, no_crop=False, timeout=60): """ Main function: crop and download a time series of Sentinel-2 images. Args: aoi (geojson.Polygon): area of interest start_date (datetime.datetime, optional): start of the time range end_date (datetime.datetime, optional): end of the time range bands (list, optional): list of bands tile_id (str): MGRS tile identifier, e.g. "31TCJ" title (str): product title, e.g. "S2A_MSIL1C_20160105T143732_N0201_R096_T19KGT_20160105T143758" relative_orbit_number (int): relative orbit number, from 1 to 143 out_dir (str, optional): path where to store the downloaded crops api (str, optional): either stac (default), scihub, planet or gcloud mirror (str, optional): either 'aws' (default) or 'gcloud' product_type (str, optional): either 'L1C' or 'L2A' (default) cloud_masks (bool, optional): if True, cloud masks are downloaded and cloudy images are discarded parallel_downloads (int): number of parallel gml files downloads satellite_angles (bool): whether or not to download satellite zenith and azimuth angles and include them in metadata no_crop (bool): if True, download original JP2 files rather than crops timeout (scalar, optional): timeout for images download, in seconds """ # list available images images = search(aoi, start_date, end_date, relative_orbit_number=relative_orbit_number, tile_id=tile_id, title=title, product_type=product_type, api=api) # download crops download(images, bands, aoi, mirror, out_dir, parallel_downloads, no_crop, timeout) # discard images that failed to download images = [i for i in images if bands_files_are_valid(i, bands, out_dir)] if satellite_angles: # retrieve satellite elevation and azimuth angles for img in images: img.get_satellite_angles() # embed all metadata as GeoTIFF tags in the image files for img in images: img['downloaded_by'] = 'TSD on {}'.format( datetime.datetime.now().isoformat()) for b in bands: filepath = os.path.join(out_dir, '{}_band_{}.tif'.format(img.filename, b)) utils.set_geotif_metadata_items(filepath, img) if cloud_masks: # discard images that are totally covered by clouds read_cloud_masks(aoi, images, bands, mirror, parallel_downloads, out_dir=out_dir)
def get_time_series(aoi, start_date=None, end_date=None, item_types=['PSScene3Band'], asset_type='analytic', out_dir='', parallel_downloads=multiprocessing.cpu_count(), clip_and_ship=True, no_crop=False, satellite_id=None, item_id=None, search_type='contains', remove_duplicates=True): """ Main function: crop and download Planet images. """ # list available images items = search_planet.search(aoi, start_date, end_date, item_types=item_types, satellite_id=satellite_id, item_id=item_id, search_type=search_type, remove_duplicates=remove_duplicates) print('Found {} images'.format(len(items))) # list the requested asset for each available (and allowed) image print('Listing available {} assets...'.format(asset_type), flush=True, end=' ') assets = parallel.run_calls(get_item_asset_info, items, extra_args=(asset_type, ), pool_type='threads', nb_workers=parallel_downloads, timeout=600) # remove 'None' (ie not allowed) assets and corresponding items items = [i for (i, a) in zip(items, assets) if a] assets = [a for a in assets if a] print('Have permissions for {} images'.format(len(items))) # activate the allowed assets print('Requesting activation of {} images...'.format(len(assets)), flush=True, end=' ') parallel.run_calls(request_activation, assets, pool_type='threads', nb_workers=parallel_downloads, timeout=600) # warn user about quota usage n = len(assets) if clip_and_ship: a = n * area.area(aoi) else: a = sum(area.area(i['geometry']) for i in items) print('Your current quota usage is {}'.format(get_quota()), flush=True) print('Downloading these {} images will increase it by {:.3f} km²'.format( n, a / 1e6), flush=True) # build filenames ext = 'zip' if clip_and_ship else 'tif' out_dir = os.path.abspath(os.path.expanduser(out_dir)) fnames = [ os.path.join(out_dir, '{}.{}'.format(fname_from_metadata(i), ext)) for i in items ] if clip_and_ship: print('Requesting clip of {} images...'.format(len(assets)), flush=True, end=' ') clips = parallel.run_calls(request_clip, list(zip(items, assets)), extra_args=(aoi, ), pool_type='threads', nb_workers=parallel_downloads, timeout=3600) # remove clips that were rejected ok = [i for i, x in enumerate(clips) if x] clips = [clips[i] for i in range(len(clips)) if i in ok] fnames = [fnames[i] for i in range(len(fnames)) if i in ok] print('Downloading {} clips...'.format(len(clips)), end=' ', flush=True) parallel.run_calls(download_clip, list(zip(clips, fnames)), pool_type='threads', nb_workers=parallel_downloads, timeout=3600) elif no_crop: # download full images os.makedirs(out_dir, exist_ok=True) print('Downloading {} full images...'.format(len(assets)), end=' ') parallel.run_calls(download_asset, list(zip(fnames, assets)), pool_type='threads', nb_workers=parallel_downloads, timeout=1200) else: if asset_type in [ 'udm', 'visual', 'analytic', 'analytic_dn', 'analytic_sr' ]: aoi_type = 'utm_rectangle' aoi = utils.utm_bbx(aoi) else: aoi_type = 'lonlat_polygon' # download crops with gdal through vsicurl os.makedirs(out_dir, exist_ok=True) print('Downloading {} crops...'.format(len(assets)), end=' ') parallel.run_calls(download_crop, list(zip(fnames, assets)), extra_args=(aoi, aoi_type), pool_type='threads', nb_workers=parallel_downloads, timeout=300) # embed some metadata in the image files for f, img in zip(fnames, items): # embed some metadata as gdal geotiff tags if os.path.isfile(f): utils.set_geotif_metadata_items( f, metadata_from_metadata_dict(img))
def get_time_series(aoi, start_date=None, end_date=None, bands=['B04'], out_dir='', api='devseed', mirror='gcloud', product_type=None, cloud_masks=False, parallel_downloads=multiprocessing.cpu_count(), satellite_angles=False): """ Main function: crop and download a time series of Sentinel-2 images. Args: aoi (geojson.Polygon): area of interest start_date (datetime.datetime, optional): start of the time range end_date (datetime.datetime, optional): end of the time range bands (list, optional): list of bands out_dir (str, optional): path where to store the downloaded crops api (str, optional): either devseed (default), scihub, planet or gcloud mirror (str, optional): either 'aws' or 'gcloud' product_type (str, optional): either 'L1C' or 'L2A' cloud_masks (bool, optional): if True, cloud masks are downloaded and cloudy images are discarded parallel_downloads (int): number of parallel gml files downloads satellite_angles (bool): whether or not to download satellite zenith and azimuth angles and include them in metadata """ # check access to the selected search api and download mirror check_args(api, mirror, product_type) # default date range if end_date is None: end_date = datetime.datetime.now() if start_date is None: start_date = end_date - datetime.timedelta(91) # 3 months # list available images images = search(aoi, start_date, end_date, product_type=product_type, api=api) # download crops download(images, bands, aoi, mirror, out_dir, parallel_downloads) # discard images that failed to download images = [ i for i in images if bands_files_are_valid(i, bands, api, out_dir) ] if satellite_angles: # retrieve satellite elevation and azimuth angles for img in images: img.get_satellite_angles() # embed all metadata as GeoTIFF tags in the image files for img in images: img['downloaded_by'] = 'TSD on {}'.format( datetime.datetime.now().isoformat()) for b in bands: filepath = os.path.join(out_dir, '{}_band_{}.tif'.format(img.filename, b)) utils.set_geotif_metadata_items(filepath, img) if cloud_masks: # discard images that are totally covered by clouds read_cloud_masks(aoi, images, bands, mirror, parallel_downloads, out_dir=out_dir)