def download_by_id(item_id):
PATH_RESULT = os.path.join(DOWNLOAD_LOCATION, item_id)
if not os.path.exists(PATH_RESULT):
os.makedirs(PATH_RESULT)
# activate assets
print("Downloading product id {}".format(item_id))
assets = client.get_assets_by_id(ITEM_TYPE, item_id).get()
for product in PRODUCTS:
try:
client.activate(assets["{}".format(product)])
except:
print("The Product {} isn't avaible for id: {}".format(product, item_id))
next
# wait until activation completes
while True:
assets = client.get_assets_by_id(ITEM_TYPE, item_id).get()
if all(['location' in assets["{}".format(product)] for product in PRODUCTS]):
break
print("Waiting for the order to be available...")
time.sleep(10)
print("Trying again...")
# Download product.
for product in PRODUCTS:
cd = client.download(assets[product],
callback=api.write_to_file(PATH_RESULT))
file_name = cd.get_body().name
print("{}: {}".format(product, file_name))
file = open(LOG_IMAGES, "a")
file.write(file_name + "\n")
file.close()
print("Finish, all is ok!!!")
def execute(args):
'''
This script executes the planet API to download images.
Current capabilities only allow a single geographic region of interest
to be obvserved, however, multiple different asset types are allowed.
Date filters are all checked using REGEX as well as cloud cover percentage
Currently I utilize command line arguments to give user ability to control
item types (satellite source) and asset types (visual, etc)
TODOS:
1. How to handle requests that are invalid
-especially when the asset type is not available
2. How to handle long activation times
3. Finalize input structure for arguments
'''
client = api.ClientV1(api_key=API_KEY)
start_date = ArgsInputs.get_start_date(args.start_year, args.start_month,
args.start_day)
percent_cloud = ArgsInputs.get_cloud_percentage(args.cloud_percent)
regional_filters = Filters.create_regional_filter(args.geo_json_path,
start_date,
percent_cloud)
req = Requests.create_search_request(regional_filters, args.item_types)
response = client.quick_search(req)
path = os.path.join(args.output_dir, 'generated_results.json')
print("Generated Results JSON")
with open(path, 'w') as f:
response.json_encode(f, indent=True)
print("Begin Item Breakdown")
callback = api.write_to_file(directory=args.output_dir)
for item in response.items_iter(5):
print(f"Item ID: {item['id']}")
assets = client.get_assets(item).get()
for asset_type in args.asset_type:
asset = assets.get(asset_type)
activation_status = asset.get('status')
while activation_status == 'inactive':
print("Activating.....")
client.activate(asset)
activation_status = asset.get('status')
time.sleep(25)
body = client.download(asset, callback=callback)
body.wait()
print("Item Downloaded")
print("\n")
def download_quads(mosaic_name, quad_ids, dest):
"""
Download quad geotiffs
"""
quad_ids = read(quad_ids, split=True)
cl = client()
futures = call_and_wrap(cl.fetch_mosaic_quad_geotiffs, mosaic_name,
quad_ids, api.write_to_file(dest))
complete(futures, check_futures, cl)
def download_quads(mosaic_name, quad_ids, dest):
"""
Download quad geotiffs
"""
quad_ids = read(quad_ids, split=True)
cl = client()
futures = call_and_wrap(cl.fetch_mosaic_quad_geotiffs, mosaic_name,
quad_ids, api.write_to_file(dest))
complete(futures, check_futures, cl)
def fetch_scene_thumbnails(scene_ids, scene_type, size, fmt, dest):
'''Fetch scene thumbnail(s)'''
if len(scene_ids) == 0:
src = click.open_file('-')
if not src.isatty():
scene_ids = map(lambda s: s.strip(), src.readlines())
futures = client().fetch_scene_thumbnails(scene_ids, scene_type, size, fmt,
api.write_to_file(dest))
check_futures(futures)
def fetch_scene_thumbnails(scene_ids, scene_type, size, fmt, dest):
'''Fetch scene thumbnail(s)'''
if len(scene_ids) == 0:
src = click.open_file('-')
if not src.isatty():
scene_ids = map(lambda s: s.strip(), src.readlines())
futures = client().fetch_scene_thumbnails(scene_ids, scene_type, size, fmt,
api.write_to_file(dest))
check_futures(futures)
def downloaditems(client, available_collects, dest):
for item in available_collects:
print(item['id'])
assets = client.get_assets(item).get()
activation = client.activate(assets['analytic'])
# wait for activation
assets = client.get_assets(item).get()
callback = api.write_to_file(dest)
body = client.download(assets['analytic'], callback=callback)
body. await ()
def fetch_scene_thumbnails(scene_ids, scene_type, size, fmt, dest):
'''Fetch scene thumbnail(s)'''
scene_ids = read(scene_ids or '@-', split=True)
if not scene_ids:
return
cl = client()
futures = cl.fetch_scene_thumbnails(scene_ids, scene_type, size, fmt,
api.write_to_file(dest))
complete(futures, check_futures, cl)
def fetch_scene_thumbnails(scene_ids, scene_type, size, fmt, dest):
'''Fetch scene thumbnail(s)'''
scene_ids = read(scene_ids or '@-', split=True)
if not scene_ids:
return
cl = client()
futures = cl.fetch_scene_thumbnails(scene_ids, scene_type, size, fmt,
api.write_to_file(dest))
complete(futures, check_futures, cl)
def active_and_downlaod_asset(item,asset_key,save_dir):
assets = client.get_assets(item).get()
asset = assets.get(asset_key)
# activate
activation = client.activate(asset)
print(activation.response.status_code)
asset_activated = False
while asset_activated == False:
# Get asset and its activation status
assets = client.get_assets(item).get() # need to used client to get the status
asset = assets.get(asset_key)
asset_status = asset["status"]
# If asset is already active, we are done
if asset_status == 'active':
asset_activated = True
print("Asset is active and ready to download")
# Still activating. Wait and check again.
else:
print("...Still waiting for asset activation...")
time.sleep(3)
output_stream = sys.stdout
def download_progress(start=None,wrote=None,total=None, finish=None): #result,skip=None
# print(start,wrote,total,finish)
# if total:
# # print('received: %.2f K'%(float(total)/1024.0))
# output_stream.write('received: %.2f K'%(float(total)/1024.0))
# output_stream.flush()
# if total:
# if finish is None:
# print('received: %.2f K'%(float(total)/1024.0), end='\r')
# else:
# print('received: %.2f K' % (float(total) / 1024.0))
pass
callback = api.write_to_file(directory=save_dir + '/', callback=download_progress) # save_dir + '/' #
body = client.download(assets[asset_key], callback=callback)
# if body._body.name == '':
# basic.outputlogMessage('Warning, the body name is missed, set as the asset key and id: item id: %s, asset %s'%(item['id'],asset_key))
# body._body.name = item['id']+'_'+asset_key # AttributeError: can't set attribute
body.await()
return True
def download_a_asset_from_server(item, assets, asset_key, save_dir):
'''
download a asset from the server
:param item: the item
:param assets: assets from get_assets_from_server
:param asset_key: the name of the asset
:param save_dir: save dir
:return: True if successful, Flase otherwise
'''
proc_id = multiprocessing.current_process().pid
print('Process: %d, start downloading %s (id: %s)' %
(proc_id, asset_key, item['id']))
output_stream = sys.stdout
def download_progress(start=None,
wrote=None,
total=None,
finish=None): #result,skip=None
# print(start,wrote,total,finish)
# if total:
# # print('received: %.2f K'%(float(total)/1024.0))
# output_stream.write('received: %.2f K'%(float(total)/1024.0))
# output_stream.flush()
# if total:
# if finish is None:
# print('received: %.2f K'%(float(total)/1024.0), end='\r')
# else:
# print('received: %.2f K' % (float(total) / 1024.0))
pass
callback = api.write_to_file(
directory=save_dir + '/',
callback=download_progress) # save_dir + '/' #
body = client.download(assets[asset_key], callback=callback)
# body.await() for version 1.1.0
try:
body.wait() # for version > 1.4.2
except APIException as e:
output_planetAPI_error(
'An APIException occurs when try to download %s (id: %s)' %
(asset_key, item['id']))
output_planetAPI_error(str(e))
raise Exception("rate limit error or other API errors")
# return False # return a large number
except:
raise APIException
return True
def fetch_lat_lon(image_dir, lat_lon_list, count = 1, scene_type = 'rapideye', product = 'analytic'):
"""
Download a single rapideye image of each coordinate in a list of coordinates.
:param image_directory: The directory where the images should be downloaded to.
:param lat_lon_list: A list of coordinate tuples in the form of (lat, lon)
:param scene_type: The type of satellite that the images should be from ('ortho' or 'rapideye')
:param product: The image type. 'analytic' or 'visual'.
:return:
"""
points = [geojson.Point([lon, lat]) for lat, lon in lat_lon_list]
intersects = [geojson.dumps(point) for point in points]
sceneIDs = []
url = "https://api.planet.com/v0/scenes/rapideye/"
for intersect in intersects:
params = {
"cloud_cover.estimated.lte": 0,
"intersects":intersect,
}
data = requests.get(url, params=params, auth=(const.API_KEY, ''))
scenes_data = data.json()["features"]
if not scenes_data:
print "No scenes available for these coordinates: ", intersect
continue
counter = 0
for scene in scenes_data:
if counter == count: break
sceneIDs.append(scene['id'])
print scene['id']
counter += 1
print "Downloading scene list!"
planet_client = api.Client(api_key = const.API_KEY)
try:
callback = api.write_to_file(image_dir)
bodies = planet_client.fetch_scene_geotiffs(scene_ids = sceneIDs, product = product, scene_type=scene_type, callback=callback)
for b in bodies:
b.await()
except Exception, e:
print "Download failed: %s"%e
def download(scenes_file, product, path):
with open(scenes_file) as f:
scenes = json.load(f)
assets = [
planet.get_assets(scene).get()[product] for scene in scenes["features"]
]
for asset in assets:
click.echo(asset)
planet.download(asset, api.write_to_file(path)). await ()
files = [pth for pth in os.listdir(path) if pth.endswith(".tif")]
try:
os.mkdir(os.path.join(path, "processed"))
except FileExistsError:
pass
for pth in tqdm(files):
dataset = rasterio.open(os.path.join(path, pth))
mask = dataset.read(4)
data = dataset.read((1, 2, 3)).astype(np.int16)
for i in range(data.shape[0]):
data[i] = data[i] - 9999 * (~mask > 0)
new_dataset = rasterio.open(os.path.join(path, "processed", pth),
'w',
driver='GTiff',
height=dataset.shape[0],
width=dataset.shape[1],
count=3,
dtype=str(data.dtype),
crs=dataset.crs,
transform=dataset.affine,
nodata=-9999)
new_dataset.write(data, [1, 2, 3])
dataset.close()
new_dataset.close()
def activate_and_download_asset_type(asset_type):
activated = False
while not activated:
dataset = \
session.get(
("https://api.planet.com/data/v1/item-types/" +
"{}/items/{}/assets/").format(item['properties']['item_type'], item['id']))
# extract the activation url from the item for the desired asset
item_activation_url = dataset.json(
)[asset_type]["_links"]["activate"]
# request activation
response = session.post(item_activation_url)
activated = (response.status_code == 204)
if not activated:
print("Waiting for activation of: ", item['id'])
import time
time.sleep(30.0)
asset = client.get_assets(item).get()[asset_type]
callback = api.write_to_file(directory=output_directory,
callback=None,
overwrite=True)
name = client.download(asset, callback=callback).wait().name
return name, item
def download_image():
client = api.ClientV1(api_key=API_KEY)
# defining the Area of Interest
aoi = {
"type": "Polygon",
"coordinates": GEOMETRY,
}
# build a filter for the AOI
query = api.filters.and_filter(
api.filters.geom_filter(aoi),
api.filters.range_filter('cloud_cover', gt=0),
api.filters.range_filter('cloud_cover', lt=CLOUD_COVER))
# we are requesting PlanetScope 3 Band imagery
item_types = ['PSScene3Band']
request = api.filters.build_search_request(query, item_types)
# this will cause an exception if there are any API related errors
results = client.quick_search(request)
# creates the output directory if not already exists
if not os.path.exists(SAVE_DIR):
os.mkdir(SAVE_DIR)
# items_iter returns an iterator over API response pages
for item in results.items_iter(NUM_IMG):
# each item is a GeoJSON feature
print('downloading tile {}'.format(item['id']))
assets = client.get_assets(item).get()
activation = client.activate(assets['visual'])
callback = api.write_to_file(directory=SAVE_DIR)
body = client.download(assets['visual'], callback=callback)
body. await ()
#Activate requests (tell Planet that be ready to start image downloading)
assets_list = []
for ids, item in zip(scene_id, item_type):
assets_list.append(client.get_assets_by_id(id=ids, item_type=item).get())
#Activate the assets and get the status (204 is the good code ;])
activation_list = [
client.activate(x['analytic']) for x in assets_list
if 'analytics' in x.keys()
]
for e, i in enumerate(activation_list):
print(i.response.status_code)
#Download images! (mnt/data)
callback = api.write_to_file(directory="/mnt/data/shared/planet/imgs_ts")
body = []
for i in assets_list:
if "analytic" in i.keys():
body.append(client.download(i['analytic'], callback=callback))
# Image processing
#Load requests
planet_torres_strait = []
with open('/mnt/data/shared/planet/results_planet_torres_strait.txt',
'r') as json_file:
for line in json_file:
planet_torres_strait.append(eval(line))
#Get images that were downloaded
WAIT = WAIT + 10
assets = client.get_assets(item).get()
if WAIT > 30:
WAIT = WAIT -10
# wait for activation
#assets = client.get_assets(item).get()
if DEBUG:
print("\n{}\n".format(sceneType[0]))
pprint(assets[sceneType[0]])
print("\n{}\n".format(sceneType[1]))
pprint(assets[sceneType[1]])
pprint(activation[0])
pprint(activation[1])
callback = api.write_to_file(directory=targetDir)
#body = client.download(assets[sceneType[0]], callback=callback)
bodyxml = client.download(assets[sceneType[1]], callback=callback)
#body.await()
bodyxml.await()
#def warpToFile(targetDir, identifier, filename, poly, asset,targetSRID):
warpToFile(targetDir, row['identifier'], "{}_{}".format(item['id'],sceneType[0]), poly, assets[sceneType[0]], TARGETSRID)
else:
print("\t {1} !! {0} already exists! Skipping!{2}".format(targetDir, RED, NORMAL))
[-122.54, 37.81]
]
]
}"""
# will pick up api_key via environment variable PL_API_KEY
# but can be specified using `api_key` named argument
client = api.Client()
# collect all scenes here
scenes = []
print 'loading scenes'
# get `count` number of scenes, for this example, use 1 to verify paging
scene = client.get_scenes_list(count=1)
# we'll use 3 `pages` of results
for s in scene.iter(pages=3):
scenes.extend(s.get()['features'])
assert len(scenes) == 3
ids = [f['id'] for f in scenes]
print 'fetching tiffs for'
print '\n'.join(ids)
results = client.fetch_scene_thumbnails(ids,
callback=api.write_to_file(dest_dir))
# results are async objects and we have to ensure they all process
map(lambda r: r.await(), results)
def sync(destination, scene_type, limit):
'''Synchronize a directory to a specified AOI'''
if not path.exists(destination) or not path.isdir(destination):
raise click.ClickException('destination must exist and be a directory')
aoi_file = path.join(destination, 'aoi.geojson')
if not path.exists(aoi_file):
raise click.ClickException(
'provide an aoi.geojson file in "%s"' % destination
)
aoi = None
with open(aoi_file) as fp:
aoi = fp.read()
sync_file = path.join(destination, 'sync.json')
if path.exists(sync_file):
with open(sync_file) as fp:
sync = json.loads(fp.read())
else:
sync = {}
filters = {}
if 'latest' in sync:
filters['acquired.gt'] = sync['latest']
start_time = time.time()
transferred = 0
_client = client()
res = call_and_wrap(_client.get_scenes_list, scene_type=scene_type,
intersects=aoi, count=100, order_by='acquired asc',
**filters)
click.echo('total scenes to fetch: %s' % res.get()['count'])
if limit > 0:
click.echo('limiting to %s' % limit)
counter = type('counter', (object,),
{'remaining': res.get()['count'] if limit < 1 else limit})()
latest = None
def progress_callback(arg):
if not isinstance(arg, int):
counter.remaining -= 1
click.echo('downloaded %s, remaining %s' %
(arg.name, counter.remaining))
write_callback = api.write_to_file(destination, progress_callback)
for page in res.iter():
features = page.get()['features'][:counter.remaining]
if not features:
break
ids = [f['id'] for f in features]
futures = _client.fetch_scene_geotiffs(
ids, scene_type, callback=write_callback
)
for f in features:
metadata = path.join(destination, '%s_metadata.json' % f['id'])
with open(metadata, 'wb') as fp:
fp.write(json.dumps(f, indent=2))
check_futures(futures)
transferred += total_bytes(futures)
recent = max([
api.strp_timestamp(f['properties']['acquired']) for f in features]
)
latest = max(latest, recent) if latest else recent
if counter.remaining <= 0:
break
if latest:
sync['latest'] = api.strf_timestamp(latest)
with open(sync_file, 'wb') as fp:
fp.write(json.dumps(sync, indent=2))
if transferred:
summarize_throughput(transferred, start_time)
def sync(destination, scene_type, limit):
'''Synchronize a directory to a specified AOI'''
if not path.exists(destination) or not path.isdir(destination):
raise click.ClickException('destination must exist and be a directory')
aoi_file = path.join(destination, 'aoi.geojson')
if not path.exists(aoi_file):
raise click.ClickException('provide an aoi.geojson file in "%s"' %
destination)
aoi = None
with open(aoi_file) as fp:
aoi = fp.read()
sync_file = path.join(destination, 'sync.json')
if path.exists(sync_file):
with open(sync_file) as fp:
sync = json.loads(fp.read())
else:
sync = {}
filters = {}
if 'latest' in sync:
filters['acquired.gt'] = sync['latest']
start_time = time.time()
transferred = 0
_client = client()
res = call_and_wrap(_client.get_scenes_list,
scene_type=scene_type,
intersects=aoi,
count=100,
order_by='acquired asc',
**filters)
click.echo('total scenes to fetch: %s' % res.get()['count'])
if limit > 0:
click.echo('limiting to %s' % limit)
counter = type('counter', (object, ),
{'remaining': res.get()['count'] if limit < 1 else limit})()
latest = None
def progress_callback(arg):
if not isinstance(arg, int):
counter.remaining -= 1
click.echo('downloaded %s, remaining %s' %
(arg.name, counter.remaining))
write_callback = api.write_to_file(destination, progress_callback)
for page in res.iter():
features = page.get()['features'][:counter.remaining]
if not features:
break
ids = [f['id'] for f in features]
futures = _client.fetch_scene_geotiffs(ids,
scene_type,
callback=write_callback)
for f in features:
metadata = path.join(destination, '%s_metadata.json' % f['id'])
with open(metadata, 'wb') as fp:
fp.write(json.dumps(f, indent=2))
check_futures(futures)
transferred += total_bytes(futures)
recent = max([
api.strp_timestamp(f['properties']['acquired']) for f in features
])
latest = max(latest, recent) if latest else recent
if counter.remaining <= 0:
break
if latest:
sync['latest'] = api.strf_timestamp(latest)
with open(sync_file, 'wb') as fp:
fp.write(json.dumps(sync, indent=2))
if transferred:
summarize_throughput(transferred, start_time)
def fetch_images(image_directory, scene_type = 'rapideye', product = 'analytic', filters = {}):
"""
:param image_directory: The directory where the original images should be saved to.
:param scene_type: The satellite type that images should be fetched from.
:param product: The type of imagery that will be downloaded. "analytic" is not pre-processed, "visual" is true-color.
:param filters: Any filters for the search.
:return:
"""
if not os.path.isdir(image_directory):
raise OSError("The specified path " + image_directory + " does not point to a directory!")
if not filters:
start = datetime.datetime(year=2011, month=1, day=1, tzinfo=pytz.utc).isoformat()
end = datetime.datetime(year=2015, month=12, day=1, tzinfo=pytz.utc).isoformat()
filters = {
# Your filters here, for example:
# Get images with estimated 0% cloud cover
"cloud_cover.estimated.lte": 0,
"acquired.gte": start,
"acquired.lte": end
}
next_url = "https://api.planet.com/v0/scenes/" + scene_type + "/?" + urllib.urlencode(filters)
scene_data_pages = []
scene_IDs = []
scene_count = 0
print "Searching for %s images that comply with the given filters." % scene_type
print "Scene List:"
while next_url:
# Note: you don't have to pass the filters in again here,
# here, they will always be included in data.links.next
r = requests.get(next_url, auth=(const.API_KEY, ''))
r.raise_for_status()
data = r.json()
scenes_data = data["features"]
scene_data_pages.append(scenes_data)
for scene in scenes_data:
scene_IDs.append(str(scene['id']))
print str(scene['id'])
scene_count += len(scenes_data)
next_url = data["links"].get("next", None)
print '\n%s total results' % scene_count
print "Downloading scene list!"
planet_client = api.Client(api_key = const.API_KEY)
callback = api.write_to_file(image_directory)
bodies = planet_client.fetch_scene_geotiffs(scene_IDs, scene_type = scene_type, product = product, callback=callback)
# await the completion of the asynchronous downloads, this is where
# any exception handling should be performed
for b in bodies:
b.await()
####
for i in item_type:
request = api.filters.build_search_request(PB_filter, [i],
name=None,
interval='day')
results = client.quick_search(request)
print i
for item in results.items_iter(limit):
print item['id']
dataset = \
session.get(
("https://api.planet.com/data/v1/item-types/" +
"{}/items/{}/assets/").format(i, item['id']))
# extract the activation url from the item for the desired asset
item_activation_url = dataset.json()[asset_type]["_links"]["activate"]
# request activation
response = session.post(item_activation_url)
print response.status_code
while response.status_code != 204:
time.sleep(30)
response = session.post(item_activation_url)
response.status_code = response.status_code
print response.status_code
assets = client.get_assets(item).get()
callback = api.write_to_file(directory=ddir,
callback=None,
overwrite=True)
body = client.download(assets[asset_type], callback=callback)
body. await ()
#
#collect all scenes here
scenes = []
print 'loading scenes'
# get `count` number of scenes, for this example, use 1 to verify paging
filters = {'cloud_cover.estimated.lt': 0.1}
scene = client.get_scenes_list(scene_type= 'ortho', intersects=aoi, **filters)
# we'll use 3 `pages` of results
#for s in scene.iter(pages=1):
# j = scenes.extend(s.get()['features'])
ids = [f['id'] for f in scenes]
print 'fetching tiffs for'
print '\n'.join(ids)
results = client.fetch_scene_thumbnails(ids, callback=api.write_to_file(outdir))
# results are async objects and we have to ensure they all process
#map(lambda r: r.await(), results)
#callback = api.write_to_file(directory='/home/jasondavis/planet_api/output') # change this
#callbacks = client.fetch_scene_geotiffs(ids, callback=callback)
#for cb in callbacks:
# body = cb.await()
# print 'downloaded', body.name, 'size=%s bytes' % len(body)
def activate_and_download_asset(item, asset_key, save_dir):
'''
active a asset of a item and download it
:param item: the item
:param asset_key: the name of the asset
:param save_dir: save dir
:return: True if successful, Flase otherwise
'''
assets = client.get_assets(item).get()
asset = assets.get(asset_key)
# activate
activation = client.activate(asset)
print(activation.response.status_code)
if int(activation.response.status_code) == 401:
basic.outputlogMessage(
'The account does not have permissions to download this file')
return False
asset_activated = False
while asset_activated == False:
# Get asset and its activation status
assets = client.get_assets(
item).get() # need to used client to get the status
asset = assets.get(asset_key)
asset_status = asset["status"]
# If asset is already active, we are done
if asset_status == 'active':
asset_activated = True
print("Asset is active and ready to download")
# Still activating. Wait and check again.
else:
print("...Still waiting for asset activation...")
time.sleep(3)
output_stream = sys.stdout
def download_progress(start=None,
wrote=None,
total=None,
finish=None): #result,skip=None
# print(start,wrote,total,finish)
# if total:
# # print('received: %.2f K'%(float(total)/1024.0))
# output_stream.write('received: %.2f K'%(float(total)/1024.0))
# output_stream.flush()
# if total:
# if finish is None:
# print('received: %.2f K'%(float(total)/1024.0), end='\r')
# else:
# print('received: %.2f K' % (float(total) / 1024.0))
pass
callback = api.write_to_file(
directory=save_dir + '/',
callback=download_progress) # save_dir + '/' #
body = client.download(assets[asset_key], callback=callback)
# body.await() for version 1.1.0
body.wait() # for version > 1.4.2
return True