def test_catalog_search_point(): c = Catalog(gbdx) lat = 40.0149856 lng = -105.2705456 results = c.search_point(lat,lng) assert results['stats']['totalRecords'] == 310
def test_catalog_search_point(self):
c = Catalog()
lat = 40.0149856
lng = -105.2705456
results = c.search_point(lat, lng)
self.assertEqual(len(results), 499)
def test_catalog_search_wkt_and_startDate():
c = Catalog(gbdx)
results = c.search(
searchAreaWkt=
"POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))",
startDate='2012-01-01T00:00:00.000Z')
assert len(results) == 317
def test_catalog_search_point():
c = Catalog(gbdx)
lat = 40.0149856
lng = -105.2705456
results = c.search_point(lat, lng)
assert results['stats']['totalRecords'] == 310
def test_catalog_search_startDate_and_endDate_only_less_than_one_week_apart(self):
c = Catalog()
results = c.search(startDate='2008-01-01T00:00:00.000Z',
endDate='2008-01-03T00:00:00.000Z')
assert len(results) == 643
def test_catalog_search_wkt_and_endDate(self):
c = Catalog()
results = c.search(
searchAreaWkt=
"POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))",
endDate='2012-01-01T00:00:00.000Z')
assert len(results) == 92
def test_catalog_search_startDate_and_endDate_only_less_than_one_week_apart():
c = Catalog(gbdx)
results = c.search(startDate='2008-01-01T00:00:00.000Z',
endDate='2008-01-03T00:00:00.000Z')
assert len(results) == 759
def test_catalog_search_point(self):
c = Catalog()
lat = 40.0149856
lng = -105.2705456
results = c.search_point(lat, lng)
self.assertEqual(len(results),499)
def test_catalog_search_huge_aoi(self):
"""
Search an AOI the size of utah, broken into multiple smaller searches
"""
c = Catalog(self.gbdx)
results = c.search(searchAreaWkt = "POLYGON((-113.88427734375 40.36642741921034,-110.28076171875 40.36642741921034,-110.28076171875 37.565262680889965,-113.88427734375 37.565262680889965,-113.88427734375 40.36642741921034))")
assert len(results) == 2736
def test_catalog_search_huge_aoi(self):
"""
Search an AOI the size of utah, broken into multiple smaller searches
"""
c = Catalog()
results = c.search(searchAreaWkt = "POLYGON((-113.88427734375 40.36642741921034,-110.28076171875 40.36642741921034,-110.28076171875 37.565262680889965,-113.88427734375 37.565262680889965,-113.88427734375 40.36642741921034))")
assert len(results) == 1000 # we will max out the paging limit of the vector service
def test_catalog_get_record(self):
c = Catalog()
catid = '1040010019B4A600'
record = c.get(catid)
self.assertEqual(record['identifier'], '1040010019B4A600')
assert 'DigitalGlobeAcquisition' in record['type']
self.assertTrue('inEdges' not in list(record.keys()))
def test_catalog_get_record(self):
c = Catalog()
catid = '1040010019B4A600'
record = c.get(catid)
self.assertEqual(record['identifier'], '1040010019B4A600')
assert 'DigitalGlobeAcquisition' in record['type']
self.assertTrue('inEdges' not in list(record.keys()))
def test_catalog_get_record_with_relationships(self):
c = Catalog(self.gbdx)
catid = '1040010019B4A600'
record = c.get(catid, includeRelationships=True)
self.assertEqual(record['identifier'], '1040010019B4A600')
self.assertEqual(record['type'], 'DigitalGlobeAcquisition')
self.assertTrue('inEdges' in list(record.keys()))
def test_catalog_get_record_with_relationships(self):
c = Catalog(self.gbdx)
catid = '1040010019B4A600'
record = c.get(catid, includeRelationships=True)
self.assertEqual(record['identifier'], '1040010019B4A600')
self.assertEqual(record['type'], 'DigitalGlobeAcquisition')
self.assertTrue('inEdges' in list(record.keys()))
def test_catalog_get_record_with_relationships(self):
"""
includeRelationships doesn't do anything anymore. This is now a test of backward compatibility.
"""
c = Catalog()
catid = '1040010019B4A600'
record = c.get(catid, includeRelationships=True)
self.assertEqual(record['identifier'], '1040010019B4A600')
assert 'DigitalGlobeAcquisition' in record['type']
def test_catalog_search_types1(self):
c = Catalog()
types = [ "LandsatAcquisition" ]
results = c.search(types=types,
searchAreaWkt="POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
for result in results:
assert 'LandsatAcquisition' in result['type']
def test_catalog_search_startDate_and_endDate_only_more_than_one_week_apart(self):
c = Catalog(self.gbdx)
try:
results = c.search(startDate='2004-01-01T00:00:00.000Z',
endDate='2012-01-01T00:00:00.000Z')
except Exception as e:
pass
else:
raise Exception('failed test')
def test_catalog_search_startDate_and_endDate_only_more_than_one_week_apart():
c = Catalog(gbdx)
try:
results = c.search(startDate='2004-01-01T00:00:00.000Z',
endDate='2012-01-01T00:00:00.000Z')
except Exception as e:
pass
else:
raise Exception('failed test')
def test_catalog_get_record_with_relationships(self):
"""
includeRelationships doesn't do anything anymore. This is now a test of backward compatibility.
"""
c = Catalog()
catid = '1040010019B4A600'
record = c.get(catid, includeRelationships=True)
self.assertEqual(record['identifier'], '1040010019B4A600')
assert 'DigitalGlobeAcquisition' in record['type']
def __init__(self, **kwargs):
""" Construct the Idaho interface class.
Returns:
An instance of the Idaho interface class.
"""
interface = Auth(**kwargs)
self.base_url = '%s/catalog/v2' % interface.root_url
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog()
self.logger = interface.logger
def test_catalog_search_filters2(self):
c = Catalog()
filters = [
"sensorPlatformName = 'WORLDVIEW03'"
]
results = c.search(filters=filters,
searchAreaWkt="POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
for result in results:
assert result['properties']['sensorPlatformName'] in ['WORLDVIEW03']
def test_catalog_search_types1(self):
c = Catalog()
types = ["LandsatAcquisition"]
results = c.search(
types=types,
searchAreaWkt=
"POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
for result in results:
assert 'LandsatAcquisition' in result['type']
def test_catalog_search_huge_aoi():
"""
Search an AOI the size of utah, broken into multiple smaller searches
"""
c = Catalog(gbdx)
results = c.search(
searchAreaWkt=
"POLYGON((-113.88427734375 40.36642741921034,-110.28076171875 40.36642741921034,-110.28076171875 37.565262680889965,-113.88427734375 37.565262680889965,-113.88427734375 40.36642741921034))"
)
assert len(results) == 2736
def test_catalog_search_filters2():
c = Catalog(gbdx)
filters = ["sensorPlatformName = 'WORLDVIEW03'"]
results = c.search(
filters=filters,
searchAreaWkt=
"POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
for result in results:
assert result['properties']['sensorPlatformName'] in ['WORLDVIEW03']
def __init__(self, interface):
''' Construct the Idaho interface class
Args:
connection (gbdx_session): A reference to the GBDX Connection.
Returns:
An instance of the Idaho interface class.
'''
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog(interface)
self.logger = interface.logger
def test_catalog_search_huge_aoi(self):
"""
Search an AOI the size of utah, broken into multiple smaller searches
"""
c = Catalog()
results = c.search(
searchAreaWkt=
"POLYGON((-113.88427734375 40.36642741921034,-110.28076171875 40.36642741921034,-110.28076171875 37.565262680889965,-113.88427734375 37.565262680889965,-113.88427734375 40.36642741921034))"
)
assert len(
results
) == 1000 # we will max out the paging limit of the vector service
def __init__(self, **kwargs):
interface = Auth(**kwargs)
self.gbdx_connection = interface.gbdx_connection
self.root_url = interface.root_url
self.logger = interface.logger
# create and store an instance of the GBDX s3 client
self.s3 = S3()
# create and store an instance of the GBDX Ordering Client
self.ordering = Ordering()
# create and store an instance of the GBDX Catalog Client
self.catalog = Catalog()
# create and store an instance of the GBDX Workflow Client
self.workflow = Workflow()
# create and store an instance of the Idaho Client
self.idaho = Idaho()
self.vectors = Vectors()
self.catalog_image = CatalogImage
self.idaho_image = IdahoImage
self.task_registry = TaskRegistry()
def __init__(self, **kwargs):
interface = Auth(**kwargs)
self.gbdx_connection = interface.gbdx_connection
self.root_url = interface.root_url
self.logger = interface.logger
# create and store an instance of the GBDX s3 client
self.s3 = S3()
# create and store an instance of the GBDX Ordering Client
self.ordering = Ordering()
# create and store an instance of the GBDX Catalog Client
self.catalog = Catalog()
# create and store an instance of the GBDX Workflow Client
self.workflow = Workflow()
# create and store an instance of the Idaho Client
self.idaho = Idaho()
self.vectors = Vectors()
self.catalog_image = CatalogImage
self.idaho_image = IdahoImage
self.landsat_image = LandsatImage
self.sentinel2 = Sentinel2
self.tms_image = TmsImage
self.dem_image = DemImage
self.wv03_vnir = WV03_VNIR
self.wv02 = WV02
self.ge01 = GE01
self.s3_image = S3Image
self.task_registry = TaskRegistry()
def test_catalog_search_filters1(self):
c = Catalog()
filters = [
"(sensorPlatformName = 'WORLDVIEW01' OR sensorPlatformName ='QUICKBIRD02')",
"cloudCover < 10",
"offNadirAngle < 10"
]
results = c.search(startDate='2008-01-01T00:00:00.000Z',
endDate='2012-01-03T00:00:00.000Z',
filters=filters,
searchAreaWkt="POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
for result in results:
assert result['properties']['sensorPlatformName'] in ['WORLDVIEW01','QUICKBIRD02']
assert float(result['properties']['cloudCover']) < 10
assert float(result['properties']['offNadirAngle']) < 10
def __init__(self, **kwargs):
""" Construct the Idaho interface class.
Returns:
An instance of the Idaho interface class.
"""
interface = Auth(**kwargs)
self.base_url = '%s/catalog/v2' % interface.root_url
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog()
self.logger = interface.logger
def test_catalog_search_filters1(self):
c = Catalog()
filters = [
"(sensorPlatformName = 'WORLDVIEW01' OR sensorPlatformName ='QUICKBIRD02')",
"cloudCover < 10", "offNadirAngle < 10"
]
results = c.search(
startDate='2008-01-01T00:00:00.000Z',
endDate='2012-01-03T00:00:00.000Z',
filters=filters,
searchAreaWkt=
"POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
for result in results:
assert result['properties']['sensorPlatformName'] in [
'WORLDVIEW01', 'QUICKBIRD02'
]
assert float(result['properties']['cloudCover']) < 10
assert float(result['properties']['offNadirAngle']) < 10
def __init__(self, interface):
''' Construct the Idaho interface class
Args:
connection (gbdx_session): A reference to the GBDX Connection.
Returns:
An instance of the Idaho interface class.
'''
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog(interface)
self.logger = interface.logger
def __init__(self, **kwargs):
host = kwargs.get('host') if kwargs.get('host') else 'geobigdata.io'
self.root_url = 'https://%s' % host
if (kwargs.get('username') and kwargs.get('password')
and kwargs.get('client_id') and kwargs.get('client_secret')):
self.gbdx_connection = gbdx_auth.session_from_kwargs(**kwargs)
elif kwargs.get('gbdx_connection'):
# Pass in a custom gbdx connection object, for testing purposes
self.gbdx_connection = kwargs.get('gbdx_connection')
else:
# This will throw an exception if your .ini file is not set properly
self.gbdx_connection = gbdx_auth.get_session(
kwargs.get('config_file'))
# create a logger
# for now, just log to the console. We'll replace all the 'print' statements
# with at least logger.info or logger.debug statements
# later, we can log to a service, file, or some other aggregator
self.logger = logging.getLogger('gbdxtools')
self.logger.setLevel(logging.ERROR)
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.ERROR)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
console_handler.setFormatter(formatter)
self.logger.addHandler(console_handler)
self.logger.info('Logger initialized')
# create and store an instance of the GBDX s3 client
self.s3 = S3(self)
# create and store an instance of the GBDX Ordering Client
self.ordering = Ordering(self)
# create and store an instance of the GBDX Catalog Client
self.catalog = Catalog(self)
# create and store an instance of the GBDX Workflow Client
self.workflow = Workflow(self)
# create and store an instance of the Idaho Client
self.idaho = Idaho(self)
self.vectors = Vectors(self)
self.task_registry = TaskRegistry(self)
def test_catalog_get_data_location_nonexistent_catid(self):
c = Catalog()
s3path = c.get_data_location(
catalog_id='nonexistent_asdfasdfasdfdfasffds')
assert s3path == None
def test_catalog_get_data_location_Landsat(self):
c = Catalog()
s3path = c.get_data_location(catalog_id='LC81740532014364LGN00')
assert s3path == 's3://landsat-pds/L8/174/053/LC81740532014364LGN00'
def test_catalog_get_data_location_DG(self):
c = Catalog()
s3path = c.get_data_location(catalog_id='1030010045539700')
assert s3path == 's3://receiving-dgcs-tdgplatform-com/055158926010_01_003'
class Idaho(object):
def __init__(self, interface):
''' Construct the Idaho interface class
Args:
connection (gbdx_session): A reference to the GBDX Connection.
Returns:
An instance of the Idaho interface class.
'''
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog(interface)
self.logger = interface.logger
def get_idaho_by_catid_and_aoi(self, catid, aoiWKT):
''' Retrieves the IDAHO image records associated with a given catid.
Args:
catid (str): The source catalog ID from the platform catalog.
cataoiWKTid (str): The well known text of the area of interest.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
'''
self.logger.debug('Retrieving IDAHO metadata')
# use the footprint to get the IDAHO ID
url = 'https://geobigdata.io/catalog/v1/search'
body = {"filters": ["vendorDatasetIdentifier3 = '%s'" % catid],
"types": ["IDAHOImage"],
"searchAreaWkt": aoiWKT}
headers = {'Content-Type': 'application/json'}
r = self.gbdx_connection.post(url, data=json.dumps(body), headers=headers)
r.raise_for_status()
if r.status_code == 200:
results = r.json()
numresults = len(results['results'])
self.logger.debug('%s IDAHO images found associated with catid %s'
% (numresults, catid))
return results
def get_images_by_catid(self, catid):
''' Retrieves the IDAHO image records associated with a given catid.
Args:
catid (str): The source catalog ID from the platform catalog.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
'''
self.logger.debug('Retrieving IDAHO metadata')
# get the footprint of the catid's strip
footprint = self.catalog.get_strip_footprint_wkt(catid)
if not footprint:
self.logger.debug('''Cannot get IDAHO metadata for strip %s,
footprint not found''' % catid)
return None
return self.get_idaho_by_catid_and_aoi(catid, footprint)
def describe_images(self, idaho_image_results):
''' Describe a set of IDAHO images, as returned in catalog search results.
Args:
idaho_image_results (dict): IDAHO image result set as returned from
the catalog.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
'''
results = idaho_image_results['results']
# filter only idaho images:
results = [r for r in results if r['type']=='IDAHOImage']
self.logger.debug('Describing %s IDAHO images.' % len(results))
# figure out which catids are represented in this set of images
catids = set([r['properties']['vendorDatasetIdentifier3'] for r in results])
description = {}
for catid in catids:
# images associated with a single catid
description[catid] = {}
description[catid]['parts'] = {}
images = [r for r in results if r['properties']['vendorDatasetIdentifier3'] == catid]
for image in images:
description[catid]['sensorPlatformName'] = image['properties']['sensorPlatformName']
part = int(image['properties']['vendorDatasetIdentifier2'][-3:])
color = image['properties']['colorInterpretation']
bucket = image['properties']['imageBucketName']
id = image['identifier']
boundstr = image['properties']['imageBoundsWGS84']
try:
description[catid]['parts'][part]
except:
description[catid]['parts'][part] = {}
description[catid]['parts'][part][color] = {}
description[catid]['parts'][part][color]['id'] = id
description[catid]['parts'][part][color]['bucket'] = bucket
description[catid]['parts'][part][color]['boundstr'] = boundstr
return description
def create_leaflet_viewer(self, idaho_image_results, output_filename):
'''Create a leaflet viewer html file for viewing idaho images
Args:
idaho_image_results (dict): IDAHO image result set as returned from
the catalog.
output_filename (str): where to save an output html file
'''
description = self.describe_images(idaho_image_results)
if len(description) > 0:
functionstring = ''
for catid, images in description.items():
for partnum, part in images['parts'].items():
num_images = len(list(part.keys()))
partname = None
if num_images == 1:
# there is only one image, use the PAN
partname = [p for p in list(part.keys())][0]
pan_image_id = ''
elif num_images == 2:
# there are two images in this part, use the multi (or pansharpen)
partname = [p for p in list(part.keys()) if p is not 'PAN'][0]
pan_image_id = part['PAN']['id']
if not partname:
self.logger.debug("Cannot find part for idaho image.")
continue
bandstr = {
'RGBN': '0,1,2',
'WORLDVIEW_8_BAND': '4,2,1',
'PAN': '0'
}.get(partname, '0,1,2')
part_boundstr_wkt = part[partname]['boundstr']
part_polygon = geometry.from_wkt(part_boundstr_wkt)
bucketname = part[partname]['bucket']
image_id = part[partname]['id']
W, S, E, N = part_polygon.bounds
functionstring += "addLayerToMap('%s','%s',%s,%s,%s,%s,'%s');\n" % (bucketname, image_id, W,S,E,N, pan_image_id)
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
with open(os.path.join(__location__, 'leafletmap_template.html'), 'r') as htmlfile:
data=htmlfile.read().decode("utf8")
data = data.replace('FUNCTIONSTRING',functionstring)
data = data.replace('CENTERLAT',str(S))
data = data.replace('CENTERLON',str(W))
data = data.replace('BANDS',bandstr)
data = data.replace('TOKEN',self.gbdx_connection.access_token)
with codecs.open(output_filename,'w','utf8') as outputfile:
self.logger.debug("Saving %s" % output_filename)
outputfile.write(data)
else:
print("No items returned.")
def get_idaho_chip_url(self, bucket_name, idaho_id, center_lat, center_lon,
resolution=None, pan_id=None, format='tif', bands=None):
'''Gets the URL for an orthorectified IDAHO chip.
Args:
bucket_name (str): The S3 bucket name.
idaho_id (str): The IDAHO ID of the chip.
center_lat (str): The latitude of the center of the desired chip.
center_lon (str): The longitude of the center of the desired chip.
output_folder (str): The folder the chip should be output to.
resolution (str): output resolution in meters (default None = native resolution)
pan_id (str): The associated PAN ID for pan sharpening a multispectral image
format (str): File format. Defaults to 'tif'.
bands (str): band string. Defaults to None.
Returns:
URL (str)
'''
# form request
access_token = self.gbdx_connection.access_token
url = ('http://idaho.geobigdata.io/'
'v1/chip/centroid/' + bucket_name + '/' + idaho_id + '?lat='
+ str(center_lat) + '&long=' + str(center_lon) +
'&format=' + format + '&token='+access_token)
if pan_id:
url += '&panId=' + pan_id
if resolution:
url += '&resolution=' + str(resolution)
if bands:
url += '&bands=' + bands
return url
def get_idaho_chip(self, bucket_name, idaho_id, center_lat, center_lon,
output_folder, resolution=None, pan_id=None):
'''Downloads an orthorectified IDAHO chip.
Args:
bucket_name (str): The S3 bucket name.
idaho_id (str): The IDAHO ID of the chip.
center_lat (str): The latitude of the center of the desired chip.
center_lon (str): The longitude of the center of the desired chip.
output_folder (str): The folder the chip should be output to.
resolution (str): output resolution in meters (default None = native resolution)
pan_id (str): The associated PAN ID for pan sharpening a multispectral image
Returns:
Confirmation (str) that tile processing was done.
'''
print('Retrieving IDAHO chip')
url = self.get_idaho_chip_url(bucket_name, idaho_id, center_lat, center_lon, resolution, pan_id)
r = requests.get(url)
if r.status_code == 200:
# form output path
file_path = os.path.join(output_folder, idaho_id+'.tif')
with open(file_path, 'wb') as the_file:
the_file.write(r.content)
elif r.status_code == 404:
print('IDAHO ID not found: %s' % idaho_id)
r.raise_for_status()
else:
print('There was a problem retrieving IDAHO ID: %s' % idaho_id)
r.raise_for_status()
def view_idaho_tiles_by_bbox(self, catId, bbox, output_filename):
'''Retrieve and view just the IDAHO chips in a particular bounding box
for a catID.
Args:
catid (str): The source catalog ID from the platform catalog.
bbox (list): List of coords: minx(W), miny(S), maxx(E), maxy(N).
output_filename (str): a Leaflet Viewer file showing the IDAHO
images as tiles.
'''
minx, miny, maxx, maxy = bbox
#validate bbox values
if (minx > maxx):
print ('The west value is not less than the east value.')
exit
if (miny > maxy):
print ('The south value is not less than the north value.')
exit
#create bbox polygon
bp1 = Point(minx, miny)
bp2 = Point(minx, maxy)
bp3 = Point(maxx, maxy)
bp4 = Point(maxx, miny)
bbox_polygon = Polygon(bp1, bp2, bp3, bp4)
#get IDAHO image results: parts
idaho_image_results = self.get_images_by_catid(catId)
description = self.describe_images(idaho_image_results)
tile_count = 0
for catid, images in description.items():
functionstring = ''
for partnum, part in images['parts'].items():
num_images = len(list(part.keys()))
partname = None
if num_images == 1:
# there is only one image, use the PAN
partname = [p for p in list(part.keys()) if p.upper() == 'PAN'][0]
pan_image_id = ''
elif num_images == 2:
# there are two images in this part, use the multi (or pansharpen)
partname = [p for p in list(part.keys()) if p is not 'PAN'][0]
pan_image_id = part['PAN']['id']
if not partname:
print("Cannot find part for idaho image.")
continue
bandstr = {
'RGBN': '0,1,2',
'WORLDVIEW_8_BAND': '4,2,1',
'PAN': '0'
}.get(partname, '0,1,2')
part_boundstr_wkt = part[partname]['boundstr']
part_polygon = geometry.from_wkt(part_boundstr_wkt)
bucketname = part[partname]['bucket']
image_id = part[partname]['id']
W, S, E, N = part_polygon.bounds
pp1, pp2, pp3, pp4 = Point(W, S), Point(W, N), Point(E, N), Point(E, S)
part_bbox_polygon = Polygon(pp1, pp2, pp3, pp4)
if (bbox_polygon.intersection(part_bbox_polygon)):
functionstring += ("addLayerToMap('%s','%s',%s,%s,%s,%s,'%s');\n" %
(bucketname, image_id, W,S,E,N, pan_image_id))
tile_count += 1
print ('There were ' + str(tile_count) + ' IDAHO images found to ' +
'intersect with the provided bounding box.')
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(os.path.realpath('__file__'))))
with open(os.path.join(__location__, 'leafletmap_template.html'), 'r') as htmlfile:
data=htmlfile.read().decode("utf8")
data = data.replace('FUNCTIONSTRING',functionstring)
data = data.replace('CENTERLAT',str(S + old_div((N-S),2)))
data = data.replace('CENTERLON',str(W + old_div((E-W),2)))
data = data.replace('BANDS',bandstr)
data = data.replace('TOKEN',self.gbdx_connection.access_token)
with codecs.open(output_filename,'w','utf8') as outputfile:
print("Saving %s" % output_filename)
outputfile.write(data)
def download_idaho_tiles_by_bbox(self, catId, bbox, resolution, outputfolder):
'''Retrieve and view just the IDAHO chips in a particular bounding box
for a catID.
Args:
catid (str): The source catalog ID from the platform catalog.
bbox (list): List of coords: minx(W), miny(S), maxx(E), maxy(N).
resolution (str): The desired floating point resolution of the tiles.
outputfolder (str): The desired output location of the IDAHO tiles.
'''
minx, miny, maxx, maxy = bbox
#validate bbox values
if (minx > maxx):
print ('The west value is not less than the east value.')
exit
if (miny > maxy):
print ('The south value is not less than the north value.')
exit
#create bbox polygon
bp1 = Point(minx, miny)
bp2 = Point(minx, maxy)
bp3 = Point(maxx, maxy)
bp4 = Point(maxx, miny)
bbox_polygon = Polygon(bp1, bp2, bp3, bp4)
#get IDAHO image results: parts
idaho_image_results = self.get_images_by_catid(catId)
description = self.describe_images(idaho_image_results)
tile_count = 0
for catid, images in description.items():
for partnum, part in images['parts'].items():
num_images = len(list(part.keys()))
partname = None
if num_images == 1:
# there is only one image, use the PAN
partname = [p for p in list(part.keys()) if p.upper() == 'PAN'][0]
elif num_images == 2:
# there are two images in this part, use the multi (or pansharpen)
partname = [p for p in list(part.keys()) if p is not 'PAN'][0]
if not partname:
print("Cannot find part for idaho image.")
continue
part_boundstr_wkt = part[partname]['boundstr']
part_polygon = geometry.from_wkt(part_boundstr_wkt)
bucketname = part[partname]['bucket']
image_id = part[partname]['id']
W, S, E, N = part_polygon.bounds
pp1, pp2, pp3, pp4 = Point(W, S), Point(W, N), Point(E, N), Point(E, S)
part_bbox_polygon = Polygon(pp1, pp2, pp3, pp4)
if (bbox_polygon.intersection(part_bbox_polygon)):
center_lat = (S + old_div((N-S),2))
center_lon = (W + old_div((E-W),2))
print(center_lat, center_lon)
self.get_idaho_chip(bucket_name=bucketname,
idaho_id=image_id,
center_lat=str(center_lat),
center_lon=str(center_lon),
resolution=resolution,
output_folder=outputfolder)
tile_count+=1
print ('There were ' + str(tile_count) + ' IDAHO images downloaded that ' +
'intersect with the provided bounding box.')
class Idaho():
def __init__(self, interface):
''' Construct the Idaho interface class
Args:
connection (gbdx_session): A reference to the GBDX Connection.
Returns:
An instance of the Idaho interface class.
'''
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog(interface)
self.logger = interface.logger
def get_images_by_catid(self, catid):
''' Retrieves the IDAHO image records associated with a given catid.
Args:
catid (str): The source catalog ID from the platform catalog.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
'''
self.logger.debug('Retrieving IDAHO metadata')
# get the footprint of the catid's strip
footprint = self.catalog.get_strip_footprint_wkt(catid)
if not footprint:
self.logger.debug('Cannot get IDAHO metadata for strip %s, footprint not found' % catid)
return None
# use the footprint to get the IDAHO ID
url = 'https://geobigdata.io/catalog/v1/search'
body = {"startDate": None,
"filters": ["vendorDatasetIdentifier3 = '%s'" % catid],
"endDate": None,
"types": ["IDAHOImage"],
"searchAreaWkt": footprint}
headers = {'Content-Type': 'application/json'}
r = self.gbdx_connection.post(url, data=json.dumps(body), headers=headers)
r.raise_for_status()
if r.status_code == 200:
results = r.json()
numresults = len(results['results'])
self.logger.debug('%s IDAHO images found associated with catid %s' % (numresults, catid))
return results
def describe_images(self, idaho_image_results):
''' Describe a set of IDAHO images, as returned in catalog search results.
Args:
idaho_image_results (dict): IDAHO image result set as returned from
the catalog.
Returns:
results (json): the full catalog-search response for IDAHO images
within the catID.
'''
results = idaho_image_results['results']
# filter only idaho images:
results = [r for r in results if r['type']=='IDAHOImage']
self.logger.debug('Describing %s IDAHO images.' % len(results))
# figure out which catids are represented in this set of images
catids = set([r['properties']['vendorDatasetIdentifier3'] for r in results])
description = {}
for catid in catids:
# images associated with a single catid
description[catid] = {}
description[catid]['parts'] = {}
description[catid]['sensorPlatformName'] = results[0]['properties']['sensorPlatformName']
images = [r for r in results if r['properties']['vendorDatasetIdentifier3'] == catid]
for image in images:
part = int(image['properties']['vendorDatasetIdentifier2'][-3:])
color = image['properties']['colorInterpretation']
bucket = image['properties']['imageBucketName']
id = image['identifier']
boundstr = image['properties']['imageBoundsWGS84']
try:
description[catid]['parts'][part]
except:
description[catid]['parts'][part] = {}
description[catid]['parts'][part][color] = {}
description[catid]['parts'][part][color]['id'] = id
description[catid]['parts'][part][color]['bucket'] = bucket
description[catid]['parts'][part][color]['boundstr'] = boundstr
return description
def get_tiles_by_zxy(self, catID, z, x, y, outputFolder):
'''Retrieves IDAHO tiles of a given catID for a particular z, x, and
y. The z, x, and y must be known ahead of time and must intersect
the strip boundaries of the particular catID to return content.
Args:
catID (str): The source catalog ID from the platform catalog.
Returns:
Confirmation (str) that tile processing was done.
'''
self.logger.debug('Retrieving IDAHO tiles')
# get the bucket name and IDAHO ID of each tile within the catID
locations = self.get_tile_locations(catID)
access_token = self.gbdx_connection.access_token
for location in locations:
bucket_name = location[0]['imageBucketName']
idaho_id = location[1]
# form request
url = ('http://idahotms-env.us-west-2.elasticbeanstalk.com/'
'v1/tile/' + bucket_name + '/' + idaho_id + '/' + str(z)
+ '/' + str(x) + '/' + str(y) + '?token=' + access_token)
body = {"token": access_token}
r = self.gbdx_connection.get(url, data=json.dumps(body), stream=True)
r.raise_for_status()
# form output path
file_path = os.path.join(outputFolder,
catID + '-'.join(map(str, [z, x, y])) + '.tif')
# save returned image
i = Image.open(StringIO(r.content))
saved = i.save(file_path)
if saved == None:
return 'Retrieval complete; please check output folder.'
else:
return 'There was a problem saving the file at ' + file_path + '.'
def create_leaflet_viewer(self, idaho_image_results, outputfilename):
'''Create a leaflet viewer html file for viewing idaho images
Args:
idaho_image_results (dict): IDAHO image result set as returned from the catalog.
outputfilename (str): where to save an output html file
'''
description = self.describe_images(idaho_image_results)
for catid, images in description.iteritems():
functionstring = ''
for partnum, part in images['parts'].iteritems():
num_images = len(part.keys())
partname = None
if num_images == 1:
# there is only one image, use the PAN
partname = [p for p in part.keys() if p.upper() == 'PAN'][0]
pan_image_id = ''
elif num_images == 2:
# there are two images in this part, use the multi (or pansharpen)
partname = [p for p in part.keys() if p is not 'PAN'][0]
pan_image_id = part['PAN']['id']
if not partname:
self.logger.debug("Cannot find part for idaho image.")
continue
bandstr = {
'RGBN': '0,1,2',
'WORLDVIEW_8_BAND': '4,3,2',
'PAN': '0'
}.get(partname, '0,1,2')
part_boundstr_wkt = part[partname]['boundstr']
part_polygon = geometry.from_wkt(part_boundstr_wkt)
bucketname = part[partname]['bucket']
image_id = part[partname]['id']
W, S, E, N = part_polygon.bounds
functionstring += "addLayerToMap('%s','%s',%s,%s,%s,%s,'%s');\n" % (bucketname, image_id, W,S,E,N, pan_image_id)
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
with open(os.path.join(__location__, 'leafletmap_template.html'), 'r') as htmlfile:
data=htmlfile.read().decode("utf8")
data = data.replace('FUNCTIONSTRING',functionstring)
data = data.replace('CENTERLAT',str(S))
data = data.replace('CENTERLON',str(W))
data = data.replace('BANDS',bandstr)
data = data.replace('TOKEN',self.gbdx_connection.access_token)
with codecs.open(outputfilename,'w','utf8') as outputfile:
self.logger.debug("Saving %s" % outputfilename)
outputfile.write(data)
def test_catalog_search_address():
c = Catalog(gbdx)
results = c.search_address('Boulder, CO')
assert results['stats']['totalRecords'] == 310
def test_catalog_get_address_coords(self):
c = Catalog()
lat, lng = c.get_address_coords('Boulder, CO')
self.assertTrue(lat == 40.0149856)
self.assertTrue(lng == -105.2705456)
def test_catalog_get_data_location_catid_with_no_data(self):
c = Catalog()
s3path = c.get_data_location(catalog_id='1010010011AD6E00')
assert s3path == None
class Idaho(object):
def __init__(self, **kwargs):
""" Construct the Idaho interface class.
Returns:
An instance of the Idaho interface class.
"""
interface = Auth(**kwargs)
self.base_url = '%s/catalog/v2' % interface.root_url
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog()
self.logger = interface.logger
def get_images_by_catid_and_aoi(self, catid, aoi_wkt):
""" Retrieves the IDAHO image records associated with a given catid.
Args:
catid (str): The source catalog ID from the platform catalog.
aoi_wkt (str): The well known text of the area of interest.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
"""
self.logger.debug('Retrieving IDAHO metadata')
# use the footprint to get the IDAHO id
url = '%s/search' % self.base_url
body = {"filters": ["catalogID = '%s'" % catid],
"types": ["IDAHOImage"],
"searchAreaWkt": aoi_wkt}
r = self.gbdx_connection.post(url, data=json.dumps(body))
r.raise_for_status()
if r.status_code == 200:
results = r.json()
numresults = len(results['results'])
self.logger.debug('%s IDAHO images found associated with catid %s'
% (numresults, catid))
return results
def get_images_by_catid(self, catid):
""" Retrieves the IDAHO image records associated with a given catid.
Args:
catid (str): The source catalog ID from the platform catalog.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
"""
self.logger.debug('Retrieving IDAHO metadata')
# get the footprint of the catid's strip
footprint = self.catalog.get_strip_footprint_wkt(catid)
# try to convert from multipolygon to polygon:
try:
footprint = from_wkt(footprint).geoms[0].wkt
except:
pass
if not footprint:
self.logger.debug("""Cannot get IDAHO metadata for strip %s,
footprint not found""" % catid)
return None
return self.get_images_by_catid_and_aoi(catid=catid,
aoi_wkt=footprint)
def describe_images(self, idaho_image_results):
"""Describe the result set of a catalog search for IDAHO images.
Args:
idaho_image_results (dict): Result set of catalog search.
Returns:
results (json): The full catalog-search response for IDAHO images
corresponding to the given catID.
"""
results = idaho_image_results['results']
# filter only idaho images:
results = [r for r in results if 'IDAHOImage' in r['type']]
self.logger.debug('Describing %s IDAHO images.' % len(results))
# figure out which catids are represented in this set of images
catids = set([r['properties']['catalogID'] for r in results])
description = {}
for catid in catids:
# images associated with a single catid
description[catid] = {}
description[catid]['parts'] = {}
images = [r for r in results if r['properties']['catalogID'] == catid]
for image in images:
description[catid]['sensorPlatformName'] = image['properties']['sensorPlatformName']
part = int(image['properties']['vendorDatasetIdentifier'].split(':')[1][-3:])
color = image['properties']['colorInterpretation']
bucket = image['properties']['tileBucketName']
identifier = image['identifier']
boundstr = image['properties']['footprintWkt']
try:
description[catid]['parts'][part]
except:
description[catid]['parts'][part] = {}
description[catid]['parts'][part][color] = {}
description[catid]['parts'][part][color]['id'] = identifier
description[catid]['parts'][part][color]['bucket'] = bucket
description[catid]['parts'][part][color]['boundstr'] = boundstr
return description
def get_chip(self, coordinates, catid, chip_type='PAN', chip_format='TIF', filename='chip.tif'):
"""Downloads a native resolution, orthorectified chip in tif format
from a user-specified catalog id.
Args:
coordinates (list): Rectangle coordinates in order West, South, East, North.
West and East are longitudes, North and South are latitudes.
The maximum chip size is (2048 pix)x(2048 pix)
catid (str): The image catalog id.
chip_type (str): 'PAN' (panchromatic), 'MS' (multispectral), 'PS' (pansharpened).
'MS' is 4 or 8 bands depending on sensor.
chip_format (str): 'TIF' or 'PNG'
filename (str): Where to save chip.
Returns:
True if chip is successfully downloaded; else False.
"""
def t2s1(t):
# Tuple to string 1
return str(t).strip('(,)').replace(',', '')
def t2s2(t):
# Tuple to string 2
return str(t).strip('(,)').replace(' ', '')
if len(coordinates) != 4:
print('Wrong coordinate entry')
return False
W, S, E, N = coordinates
box = ((W, S), (W, N), (E, N), (E, S), (W, S))
box_wkt = 'POLYGON ((' + ','.join([t2s1(corner) for corner in box]) + '))'
# get IDAHO images which intersect box
results = self.get_images_by_catid_and_aoi(catid=catid, aoi_wkt=box_wkt)
description = self.describe_images(results)
pan_id, ms_id, num_bands = None, None, 0
for catid, images in description.items():
for partnum, part in images['parts'].items():
if 'PAN' in part.keys():
pan_id = part['PAN']['id']
bucket = part['PAN']['bucket']
if 'WORLDVIEW_8_BAND' in part.keys():
ms_id = part['WORLDVIEW_8_BAND']['id']
num_bands = 8
bucket = part['WORLDVIEW_8_BAND']['bucket']
elif 'RGBN' in part.keys():
ms_id = part['RGBN']['id']
num_bands = 4
bucket = part['RGBN']['bucket']
# specify band information
band_str = ''
if chip_type == 'PAN':
band_str = pan_id + '?bands=0'
elif chip_type == 'MS':
band_str = ms_id + '?'
elif chip_type == 'PS':
if num_bands == 8:
band_str = ms_id + '?bands=4,2,1&panId=' + pan_id
elif num_bands == 4:
band_str = ms_id + '?bands=0,1,2&panId=' + pan_id
# specify location information
location_str = '&upperLeft={}&lowerRight={}'.format(t2s2((W, N)), t2s2((E, S)))
service_url = 'https://idaho.geobigdata.io/v1/chip/bbox/' + bucket + '/'
url = service_url + band_str + location_str
url += '&format=' + chip_format + '&token=' + self.gbdx_connection.access_token
r = requests.get(url)
if r.status_code == 200:
with open(filename, 'wb') as f:
f.write(r.content)
return True
else:
print('Cannot download chip')
return False
def get_tms_layers(self,
catid,
bands='4,2,1',
gamma=1.3,
highcutoff=0.98,
lowcutoff=0.02,
brightness=1.0,
contrast=1.0):
"""Get list of urls and bounding boxes corrsponding to idaho images for a given catalog id.
Args:
catid (str): Catalog id
bands (str): Bands to display, separated by commas (0-7).
gamma (float): gamma coefficient. This is for on-the-fly pansharpening.
highcutoff (float): High cut off coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
lowcutoff (float): Low cut off coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
brightness (float): Brightness coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
contrast (float): Contrast coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
Returns:
urls (list): TMS urls.
bboxes (list of tuples): Each tuple is (W, S, E, N) where (W,S,E,N) are the bounds of the corresponding idaho part.
"""
description = self.describe_images(self.get_images_by_catid(catid))
service_url = 'http://idaho.geobigdata.io/v1/tile/'
urls, bboxes = [], []
for catid, images in description.items():
for partnum, part in images['parts'].items():
if 'PAN' in part.keys():
pan_id = part['PAN']['id']
if 'WORLDVIEW_8_BAND' in part.keys():
ms_id = part['WORLDVIEW_8_BAND']['id']
ms_partname = 'WORLDVIEW_8_BAND'
elif 'RGBN' in part.keys():
ms_id = part['RGBN']['id']
ms_partname = 'RGBN'
if ms_id:
if pan_id:
band_str = ms_id + '/{z}/{x}/{y}?bands=' + bands + '&panId=' + pan_id
else:
band_str = ms_id + '/{z}/{x}/{y}?bands=' + bands
bbox = from_wkt(part[ms_partname]['boundstr']).bounds
elif not ms_id and pan_id:
band_str = pan_id + '/{z}/{x}/{y}?bands=0'
bbox = from_wkt(part['PAN']['boundstr']).bounds
else:
continue
bboxes.append(bbox)
# Get the bucket. It has to be the same for all entries in the part.
bucket = part[list(part.keys())[0]]['bucket']
# Get the token
token = self.gbdx_connection.access_token
# Assemble url
url = (service_url + bucket + '/'
+ band_str
+ """&gamma={}
&highCutoff={}
&lowCutoff={}
&brightness={}
&contrast={}
&token={}""".format(gamma,
highcutoff,
lowcutoff,
brightness,
contrast,
token))
urls.append(url)
return urls, bboxes
def create_leaflet_viewer(self, idaho_image_results, filename):
"""Create a leaflet viewer html file for viewing idaho images.
Args:
idaho_image_results (dict): IDAHO image result set as returned from
the catalog.
filename (str): Where to save output html file.
"""
description = self.describe_images(idaho_image_results)
if len(description) > 0:
functionstring = ''
for catid, images in description.items():
for partnum, part in images['parts'].items():
num_images = len(list(part.keys()))
partname = None
if num_images == 1:
# there is only one image, use the PAN
partname = [p for p in list(part.keys())][0]
pan_image_id = ''
elif num_images == 2:
# there are two images in this part, use the multi (or pansharpen)
partname = [p for p in list(part.keys()) if p is not 'PAN'][0]
pan_image_id = part['PAN']['id']
if not partname:
self.logger.debug("Cannot find part for idaho image.")
continue
bandstr = {
'RGBN': '0,1,2',
'WORLDVIEW_8_BAND': '4,2,1',
'PAN': '0'
}.get(partname, '0,1,2')
part_boundstr_wkt = part[partname]['boundstr']
part_polygon = from_wkt(part_boundstr_wkt)
bucketname = part[partname]['bucket']
image_id = part[partname]['id']
W, S, E, N = part_polygon.bounds
functionstring += "addLayerToMap('%s','%s',%s,%s,%s,%s,'%s');\n" % (
bucketname, image_id, W, S, E, N, pan_image_id)
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
try:
with open(os.path.join(__location__, 'leafletmap_template.html'), 'r') as htmlfile:
data = htmlfile.read().decode("utf8")
except AttributeError:
with open(os.path.join(__location__, 'leafletmap_template.html'), 'r') as htmlfile:
data = htmlfile.read()
data = data.replace('FUNCTIONSTRING', functionstring)
data = data.replace('CENTERLAT', str(S))
data = data.replace('CENTERLON', str(W))
data = data.replace('BANDS', bandstr)
data = data.replace('TOKEN', self.gbdx_connection.access_token)
with codecs.open(filename, 'w', 'utf8') as outputfile:
self.logger.debug("Saving %s" % filename)
outputfile.write(data)
else:
print('No items returned.')
class Idaho(object):
def __init__(self, **kwargs):
""" Construct the Idaho interface class.
Returns:
An instance of the Idaho interface class.
"""
interface = Auth(**kwargs)
self.base_url = '%s/catalog/v2' % interface.root_url
self.gbdx_connection = interface.gbdx_connection
self.catalog = Catalog()
self.logger = interface.logger
def get_images_by_catid_and_aoi(self, catid, aoi_wkt):
""" Retrieves the IDAHO image records associated with a given catid.
Args:
catid (str): The source catalog ID from the platform catalog.
aoi_wkt (str): The well known text of the area of interest.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
"""
self.logger.debug('Retrieving IDAHO metadata')
# use the footprint to get the IDAHO id
url = '%s/search' % self.base_url
body = {
"filters": ["catalogID = '%s'" % catid],
"types": ["IDAHOImage"],
"searchAreaWkt": aoi_wkt
}
r = self.gbdx_connection.post(url, data=json.dumps(body))
r.raise_for_status()
if r.status_code == 200:
results = r.json()
numresults = len(results['results'])
self.logger.debug(
'%s IDAHO images found associated with catid %s' %
(numresults, catid))
return results
def get_images_by_catid(self, catid):
""" Retrieves the IDAHO image records associated with a given catid.
Args:
catid (str): The source catalog ID from the platform catalog.
Returns:
results (json): The full catalog-search response for IDAHO images
within the catID.
"""
self.logger.debug('Retrieving IDAHO metadata')
# get the footprint of the catid's strip
footprint = self.catalog.get_strip_footprint_wkt(catid)
# try to convert from multipolygon to polygon:
try:
footprint = from_wkt(footprint).geoms[0].wkt
except:
pass
if not footprint:
self.logger.debug("""Cannot get IDAHO metadata for strip %s,
footprint not found""" % catid)
return None
return self.get_images_by_catid_and_aoi(catid=catid, aoi_wkt=footprint)
def describe_images(self, idaho_image_results):
"""Describe the result set of a catalog search for IDAHO images.
Args:
idaho_image_results (dict): Result set of catalog search.
Returns:
results (json): The full catalog-search response for IDAHO images
corresponding to the given catID.
"""
results = idaho_image_results['results']
# filter only idaho images:
results = [r for r in results if 'IDAHOImage' in r['type']]
self.logger.debug('Describing %s IDAHO images.' % len(results))
# figure out which catids are represented in this set of images
catids = set([r['properties']['catalogID'] for r in results])
description = {}
for catid in catids:
# images associated with a single catid
description[catid] = {}
description[catid]['parts'] = {}
images = [
r for r in results if r['properties']['catalogID'] == catid
]
for image in images:
description[catid]['sensorPlatformName'] = image['properties'][
'sensorPlatformName']
part = int(image['properties']
['vendorDatasetIdentifier'].split(':')[1][-3:])
color = image['properties']['colorInterpretation']
bucket = image['properties']['tileBucketName']
identifier = image['identifier']
boundstr = image['properties']['footprintWkt']
try:
description[catid]['parts'][part]
except:
description[catid]['parts'][part] = {}
description[catid]['parts'][part][color] = {}
description[catid]['parts'][part][color]['id'] = identifier
description[catid]['parts'][part][color]['bucket'] = bucket
description[catid]['parts'][part][color]['boundstr'] = boundstr
return description
def get_chip(self,
coordinates,
catid,
chip_type='PAN',
chip_format='TIF',
filename='chip.tif'):
"""Downloads a native resolution, orthorectified chip in tif format
from a user-specified catalog id.
Args:
coordinates (list): Rectangle coordinates in order West, South, East, North.
West and East are longitudes, North and South are latitudes.
The maximum chip size is (2048 pix)x(2048 pix)
catid (str): The image catalog id.
chip_type (str): 'PAN' (panchromatic), 'MS' (multispectral), 'PS' (pansharpened).
'MS' is 4 or 8 bands depending on sensor.
chip_format (str): 'TIF' or 'PNG'
filename (str): Where to save chip.
Returns:
True if chip is successfully downloaded; else False.
"""
def t2s1(t):
# Tuple to string 1
return str(t).strip('(,)').replace(',', '')
def t2s2(t):
# Tuple to string 2
return str(t).strip('(,)').replace(' ', '')
if len(coordinates) != 4:
print('Wrong coordinate entry')
return False
W, S, E, N = coordinates
box = ((W, S), (W, N), (E, N), (E, S), (W, S))
box_wkt = 'POLYGON ((' + ','.join([t2s1(corner)
for corner in box]) + '))'
# get IDAHO images which intersect box
results = self.get_images_by_catid_and_aoi(catid=catid,
aoi_wkt=box_wkt)
description = self.describe_images(results)
pan_id, ms_id, num_bands = None, None, 0
for catid, images in description.items():
for partnum, part in images['parts'].items():
if 'PAN' in part.keys():
pan_id = part['PAN']['id']
if 'WORLDVIEW_8_BAND' in part.keys():
ms_id = part['WORLDVIEW_8_BAND']['id']
num_bands = 8
elif 'RGBN' in part.keys():
ms_id = part['RGBN']['id']
num_bands = 4
# specify band information
band_str = ''
if chip_type == 'PAN':
band_str = pan_id + '?bands=0'
elif chip_type == 'MS':
band_str = ms_id + '?'
elif chip_type == 'PS':
if num_bands == 8:
band_str = ms_id + '?bands=4,2,1&panId=' + pan_id
elif num_bands == 4:
band_str = ms_id + '?bands=0,1,2&panId=' + pan_id
# specify location information
location_str = '&upperLeft={}&lowerRight={}'.format(
t2s2((W, N)), t2s2((E, S)))
service_url = 'https://idaho.geobigdata.io/v1/chip/bbox/idaho-images/'
url = service_url + band_str + location_str
url += '&format=' + chip_format + '&token=' + self.gbdx_connection.access_token
r = requests.get(url)
if r.status_code == 200:
with open(filename, 'wb') as f:
f.write(r.content)
return True
else:
print('Cannot download chip')
return False
def get_tms_layers(self,
catid,
bands='4,2,1',
gamma=1.3,
highcutoff=0.98,
lowcutoff=0.02,
brightness=1.0,
contrast=1.0):
"""Get list of urls and bounding boxes corrsponding to idaho images for a given catalog id.
Args:
catid (str): Catalog id
bands (str): Bands to display, separated by commas (0-7).
gamma (float): gamma coefficient. This is for on-the-fly pansharpening.
highcutoff (float): High cut off coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
lowcutoff (float): Low cut off coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
brightness (float): Brightness coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
contrast (float): Contrast coefficient (0.0 to 1.0). This is for on-the-fly pansharpening.
Returns:
urls (list): TMS urls.
bboxes (list of tuples): Each tuple is (W, S, E, N) where (W,S,E,N) are the bounds of the
corresponding idaho part.
"""
description = self.describe_images(self.get_images_by_catid(catid))
service_url = 'http://idaho.geobigdata.io/v1/tile/'
urls, bboxes = [], []
for catid, images in description.items():
for partnum, part in images['parts'].items():
if 'PAN' in part.keys():
pan_id = part['PAN']['id']
if 'WORLDVIEW_8_BAND' in part.keys():
ms_id = part['WORLDVIEW_8_BAND']['id']
ms_partname = 'WORLDVIEW_8_BAND'
elif 'RGBN' in part.keys():
ms_id = part['RGBN']['id']
ms_partname = 'RGBN'
if ms_id:
if pan_id:
band_str = ms_id + '/{z}/{x}/{y}?bands=' + bands + '&panId=' + pan_id
else:
band_str = ms_id + '/{z}/{x}/{y}?bands=' + bands
bbox = from_wkt(part[ms_partname]['boundstr']).bounds
elif not ms_id and pan_id:
band_str = pan_id + '/{z}/{x}/{y}?bands=0'
bbox = from_wkt(part['PAN']['boundstr']).bounds
else:
continue
bboxes.append(bbox)
# Get the bucket. It has to be the same for all entries in the part.
bucket = part[list(part.keys())[0]]['bucket']
# Get the token
token = self.gbdx_connection.access_token
# Assemble url
url = (service_url + bucket + '/' + band_str + """&gamma={}
&highCutoff={}
&lowCutoff={}
&brightness={}
&contrast={}
&token={}""".format(
gamma, highcutoff, lowcutoff, brightness, contrast, token))
urls.append(url)
return urls, bboxes
def create_leaflet_viewer(self, idaho_image_results, filename):
"""Create a leaflet viewer html file for viewing idaho images.
Args:
idaho_image_results (dict): IDAHO image result set as returned from
the catalog.
filename (str): Where to save output html file.
"""
description = self.describe_images(idaho_image_results)
if len(description) > 0:
functionstring = ''
for catid, images in description.items():
for partnum, part in images['parts'].items():
num_images = len(list(part.keys()))
partname = None
if num_images == 1:
# there is only one image, use the PAN
partname = [p for p in list(part.keys())][0]
pan_image_id = ''
elif num_images == 2:
# there are two images in this part, use the multi (or pansharpen)
partname = [
p for p in list(part.keys()) if p is not 'PAN'
][0]
pan_image_id = part['PAN']['id']
if not partname:
self.logger.debug("Cannot find part for idaho image.")
continue
bandstr = {
'RGBN': '0,1,2',
'WORLDVIEW_8_BAND': '4,2,1',
'PAN': '0'
}.get(partname, '0,1,2')
part_boundstr_wkt = part[partname]['boundstr']
part_polygon = from_wkt(part_boundstr_wkt)
bucketname = part[partname]['bucket']
image_id = part[partname]['id']
W, S, E, N = part_polygon.bounds
functionstring += "addLayerToMap('%s','%s',%s,%s,%s,%s,'%s');\n" % (
bucketname, image_id, W, S, E, N, pan_image_id)
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
try:
with open(
os.path.join(__location__, 'leafletmap_template.html'),
'r') as htmlfile:
data = htmlfile.read().decode("utf8")
except AttributeError:
with open(
os.path.join(__location__, 'leafletmap_template.html'),
'r') as htmlfile:
data = htmlfile.read()
data = data.replace('FUNCTIONSTRING', functionstring)
data = data.replace('CENTERLAT', str(S))
data = data.replace('CENTERLON', str(W))
data = data.replace('BANDS', bandstr)
data = data.replace('TOKEN', self.gbdx_connection.access_token)
with codecs.open(filename, 'w', 'utf8') as outputfile:
self.logger.debug("Saving %s" % filename)
outputfile.write(data)
else:
print('No items returned.')
def test_catalog_search_wkt_and_endDate(self):
c = Catalog()
results = c.search(searchAreaWkt="POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))",
endDate='2012-01-01T00:00:00.000Z')
assert len(results) == 92
def test_catalog_search_wkt_only(self):
c = Catalog()
results = c.search(searchAreaWkt="POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
assert len(results) == 508
def test_catalog_search_address(self):
c = Catalog()
results = c.search_address('Boulder, CO')
self.assertEqual(len(results), 499)
def test_catalog_search_address():
c = Catalog(gbdx)
results = c.search_address('Boulder, CO')
assert results['stats']['totalRecords'] == 310
def test_catalog_get_data_location_catid_with_no_data(self):
c = Catalog()
s3path = c.get_data_location(catalog_id='1010010011AD6E00')
assert s3path == None
def test_init(self):
c = Catalog()
self.assertTrue(isinstance(c, Catalog))
def test_catalog_search_startDate_and_endDate_only_more_than_one_week_apart(
self):
c = Catalog()
results = c.search(startDate='2004-01-01T00:00:00.000Z',
endDate='2012-01-01T00:00:00.000Z')
assert len(results) == 1000
def test_catalog_get_address_coords():
c = Catalog(gbdx)
lat, lng = c.get_address_coords('Boulder, CO')
assert lat == 40.0149856
assert lng == -105.2705456
def test_catalog_get_address_coords(self):
c = Catalog()
lat, lng = c.get_address_coords('Boulder, CO')
self.assertTrue(lat == 40.0149856)
self.assertTrue(lng == -105.2705456)
def test_catalog_get_data_location_DG(self):
c = Catalog()
s3path = c.get_data_location(catalog_id='1030010045539700')
assert s3path == 's3://receiving-dgcs-tdgplatform-com/055158926010_01_003'
def test_catalog_search_address(self):
c = Catalog()
results = c.search_address('Boulder, CO')
self.assertEqual(len(results), 499)
def test_catalog_search_wkt_only(self):
c = Catalog()
results = c.search(
searchAreaWkt=
"POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))")
assert len(results) == 508
def test_catalog_get_data_location_Landsat(self):
c = Catalog()
s3path = c.get_data_location(catalog_id='LC81740532014364LGN00')
assert s3path == 's3://landsat-pds/L8/174/053/LC81740532014364LGN00'
def test_catalog_get_data_location_nonexistent_catid(self):
c = Catalog()
s3path = c.get_data_location(catalog_id='nonexistent_asdfasdfasdfdfasffds')
assert s3path == None
def test_catalog_search_wkt_and_startDate(): c = Catalog(gbdx) results = c.search(searchAreaWkt="POLYGON ((30.1 9.9, 30.1 10.1, 29.9 10.1, 29.9 9.9, 30.1 9.9))", startDate='2012-01-01T00:00:00.000Z') assert len(results) == 317
