def download_crops(imgs,
aoi,
mirror,
out_dir,
parallel_downloads,
timeout=600):
"""
Download a timeseries of crops with GDAL VSI feature.
Args:
imgs (list): list of images
aoi (geojson.Polygon): area of interest
mirror (str): either 'peps', 'aws' or 'scihub'
out_dir (str): path where to store the downloaded crops
parallel_downloads (int): number of parallel downloads
"""
print('Building {} {} download urls...'.format(len(imgs), mirror), end=' ')
if mirror == 'scihub':
parallel.run_calls(
s1_metadata_parser.Sentinel1Image.build_scihub_links,
imgs,
pool_type='threads',
nb_workers=parallel_downloads)
else:
parallel.run_calls(s1_metadata_parser.Sentinel1Image.build_s3_links,
imgs,
pool_type='threads',
timeout=timeout,
nb_workers=parallel_downloads)
# convert aoi coords from (lon, lat) to UTM
coords = utils.utm_bbx(
aoi, r=60) # round to multiples of 60m to match Sentinel-2 grid
crops_args = []
nb_removed = 0
for img in imgs:
if not img.urls[mirror]: # then it cannot be downloaded
nb_removed = nb_removed + 1
continue
for p in img.polarisations:
fname = os.path.join(out_dir, '{}_{}.tif'.format(img.filename, p))
crops_args.append((fname, img.urls[mirror][p], *coords))
if nb_removed:
print('Removed {} image(s) with invalid urls'.format(nb_removed))
out_dir = os.path.abspath(os.path.expanduser(out_dir))
os.makedirs(out_dir, exist_ok=True)
print(
'Downloading {} crops ({} images with 1 or 2 polarisations)...'.format(
len(crops_args),
len(imgs) - nb_removed),
end=' ')
parallel.run_calls(utils.crop_with_gdalwarp,
crops_args,
pool_type='processes',
nb_workers=parallel_downloads)
def download(imgs, bands, aoi, mirror, out_dir, parallel_downloads):
"""
Download a timeseries of crops with GDAL VSI feature.
Args:
imgs (list): list of images
bands (list): list of bands
aoi (geojson.Polygon): area of interest
mirror (str): either 'aws' or 'gcloud'
out_dir (str): path where to store the downloaded crops
parallel_downloads (int): number of parallel downloads
"""
coords = utils.utm_bbx(aoi)
crops_args = []
for img in imgs:
for b in set(bands + ['BQA']):
fname = os.path.join(out_dir, '{}_band_{}.tif'.format(img.filename, b))
crops_args.append((fname, img.urls[mirror][b]))
out_dir = os.path.abspath(os.path.expanduser(out_dir))
os.makedirs(out_dir, exist_ok=True)
print('Downloading {} crops ({} images with {} bands)...'.format(len(crops_args),
len(imgs),
len(bands) +1),
end=' ')
parallel.run_calls(utils.crop_with_gdal_translate, crops_args,
extra_args=(*coords,), pool_type='threads',
nb_workers=parallel_downloads)
def download(imgs, bands, aoi, mirror, out_dir, parallel_downloads):
"""
Download a timeseries of crops with GDAL VSI feature.
Args:
imgs (list): list of images
bands (list): list of bands
aoi (geojson.Polygon): area of interest
mirror (str): either 'aws' or 'gcloud'
out_dir (str): path where to store the downloaded crops
parallel_downloads (int): number of parallel downloads
"""
print('Building {} {} download urls...'.format(len(imgs), mirror), end=' ')
if mirror == 'gcloud':
parallel.run_calls(s2_metadata_parser.Sentinel2Image.build_gs_links,
imgs,
pool_type='threads',
nb_workers=parallel_downloads)
else:
parallel.run_calls(s2_metadata_parser.Sentinel2Image.build_s3_links,
imgs,
pool_type='threads',
nb_workers=parallel_downloads)
crops_args = []
nb_removed = 0
for img in imgs:
if not img.urls[mirror]: # then it cannot be downloaded
nb_removed = nb_removed + 1
continue
# convert aoi coords from (lon, lat) to UTM in the zone of the image
coords = utils.utm_bbx(aoi, epsg=int(
img.epsg), r=60) # round to multiples of 60 (B01 resolution)
for b in bands:
fname = os.path.join(out_dir,
'{}_band_{}.tif'.format(img.filename, b))
crops_args.append((fname, img.urls[mirror][b], *coords))
if nb_removed:
print('Removed {} image(s) with invalid urls'.format(nb_removed))
out_dir = os.path.abspath(os.path.expanduser(out_dir))
os.makedirs(out_dir, exist_ok=True)
print('Downloading {} crops ({} images with {} bands)...'.format(
len(crops_args),
len(imgs) - nb_removed, len(bands)),
end=' ')
parallel.run_calls(utils.rasterio_geo_crop,
crops_args,
extra_args=('UInt16', ),
pool_type='threads',
nb_workers=parallel_downloads)
def download(imgs,
bands,
aoi,
mirror,
out_dir,
parallel_downloads,
no_crop=False,
timeout=60):
"""
Download a timeseries of crops with GDAL VSI feature.
Args:
imgs (list): list of images
bands (list): list of bands
aoi (geojson.Polygon): area of interest
mirror (str): either 'aws' or 'gcloud'
out_dir (str): path where to store the downloaded crops
parallel_downloads (int): number of parallel downloads
no_crop (bool): don't crop but instead download the original JP2 files
"""
if mirror == "gcloud":
parallel.run_calls(s2_metadata_parser.Sentinel2Image.build_gs_links,
imgs,
pool_type='threads',
verbose=False,
nb_workers=parallel_downloads,
timeout=timeout)
elif mirror == "aws":
parallel.run_calls(s2_metadata_parser.Sentinel2Image.build_s3_links,
imgs,
pool_type='threads',
verbose=False,
nb_workers=parallel_downloads,
timeout=timeout)
else:
raise ValueError(f"Unknown mirror {mirror}")
crops_args = []
nb_removed = 0
for img in imgs:
if not img.urls[mirror]: # then it cannot be downloaded
nb_removed = nb_removed + 1
continue
# convert aoi coords from (lon, lat) to UTM in the zone of the image
coords = ()
if aoi is not None:
coords = utils.utm_bbx(aoi, epsg=int(
img.epsg), r=60) # round to multiples of 60 (B01 resolution)
for b in bands:
fname = os.path.join(out_dir,
'{}_band_{}.tif'.format(img.filename, b))
crops_args.append((fname, img.urls[mirror][b], *coords))
if nb_removed:
print('Removed {} image(s) with invalid urls'.format(nb_removed))
out_dir = os.path.abspath(os.path.expanduser(out_dir))
os.makedirs(out_dir, exist_ok=True)
print('Downloading {} crops ({} images with {} bands)...'.format(
len(crops_args),
len(imgs) - nb_removed, len(bands)))
if no_crop or (aoi is None): # download original files
for fname, url, *_ in crops_args:
ext = url.split(".")[-1] # jp2, TIF, ...
utils.download(url, fname.replace(".tif", f".{ext}"))
else: # download crops
parallel.run_calls(utils.rasterio_geo_crop,
crops_args,
pool_type='threads',
nb_workers=parallel_downloads)
def get_time_series(aoi,
start_date=None,
end_date=None,
item_types=['PSScene3Band'],
asset_type='analytic',
out_dir='',
parallel_downloads=multiprocessing.cpu_count(),
clip_and_ship=True,
no_crop=False,
satellite_id=None,
item_id=None,
search_type='contains',
remove_duplicates=True):
"""
Main function: crop and download Planet images.
"""
# list available images
items = search_planet.search(aoi,
start_date,
end_date,
item_types=item_types,
satellite_id=satellite_id,
item_id=item_id,
search_type=search_type,
remove_duplicates=remove_duplicates)
print('Found {} images'.format(len(items)))
# list the requested asset for each available (and allowed) image
print('Listing available {} assets...'.format(asset_type),
flush=True,
end=' ')
assets = parallel.run_calls(get_item_asset_info,
items,
extra_args=(asset_type, ),
pool_type='threads',
nb_workers=parallel_downloads,
timeout=600)
# remove 'None' (ie not allowed) assets and corresponding items
items = [i for (i, a) in zip(items, assets) if a]
assets = [a for a in assets if a]
print('Have permissions for {} images'.format(len(items)))
# activate the allowed assets
print('Requesting activation of {} images...'.format(len(assets)),
flush=True,
end=' ')
parallel.run_calls(request_activation,
assets,
pool_type='threads',
nb_workers=parallel_downloads,
timeout=600)
# warn user about quota usage
n = len(assets)
if clip_and_ship:
a = n * area.area(aoi)
else:
a = sum(area.area(i['geometry']) for i in items)
print('Your current quota usage is {}'.format(get_quota()), flush=True)
print('Downloading these {} images will increase it by {:.3f} km²'.format(
n, a / 1e6),
flush=True)
# build filenames
ext = 'zip' if clip_and_ship else 'tif'
out_dir = os.path.abspath(os.path.expanduser(out_dir))
fnames = [
os.path.join(out_dir, '{}.{}'.format(fname_from_metadata(i), ext))
for i in items
]
if clip_and_ship:
print('Requesting clip of {} images...'.format(len(assets)),
flush=True,
end=' ')
clips = parallel.run_calls(request_clip,
list(zip(items, assets)),
extra_args=(aoi, ),
pool_type='threads',
nb_workers=parallel_downloads,
timeout=3600)
# remove clips that were rejected
ok = [i for i, x in enumerate(clips) if x]
clips = [clips[i] for i in range(len(clips)) if i in ok]
fnames = [fnames[i] for i in range(len(fnames)) if i in ok]
print('Downloading {} clips...'.format(len(clips)),
end=' ',
flush=True)
parallel.run_calls(download_clip,
list(zip(clips, fnames)),
pool_type='threads',
nb_workers=parallel_downloads,
timeout=3600)
elif no_crop: # download full images
os.makedirs(out_dir, exist_ok=True)
print('Downloading {} full images...'.format(len(assets)), end=' ')
parallel.run_calls(download_asset,
list(zip(fnames, assets)),
pool_type='threads',
nb_workers=parallel_downloads,
timeout=1200)
else:
if asset_type in [
'udm', 'visual', 'analytic', 'analytic_dn', 'analytic_sr'
]:
aoi_type = 'utm_rectangle'
aoi = utils.utm_bbx(aoi)
else:
aoi_type = 'lonlat_polygon'
# download crops with gdal through vsicurl
os.makedirs(out_dir, exist_ok=True)
print('Downloading {} crops...'.format(len(assets)), end=' ')
parallel.run_calls(download_crop,
list(zip(fnames, assets)),
extra_args=(aoi, aoi_type),
pool_type='threads',
nb_workers=parallel_downloads,
timeout=300)
# embed some metadata in the image files
for f, img in zip(fnames,
items): # embed some metadata as gdal geotiff tags
if os.path.isfile(f):
utils.set_geotif_metadata_items(
f, metadata_from_metadata_dict(img))