def _find_parts(cat_id, band_type):
vectors = Vectors()
aoi = wkt.dumps(box(-180, -90, 180, 90))
query = "item_type:IDAHOImage AND attributes.catalogID:{} " \
"AND attributes.colorInterpretation:{}".format(cat_id, band_types[band_type])
return sorted(vectors.query(aoi, query=query),
key=lambda x: x['properties']['id'])
def vector_services_query(query, aoi=None, **kwargs):
vectors = Vectors()
if not aoi:
aoi = wkt.dumps(box(-180, -90, 180, 90))
_parts = sorted(vectors.query(aoi, query=query, **kwargs),
key=lambda x: x['properties']['id'])
return _parts
def _build_standard_products(idaho_id,
proj,
bucket=None,
gsd=None,
acomp=False):
if bucket is None:
vectors = Vectors()
aoi = wkt.dumps(box(-180, -90, 180, 90))
query = "item_type:IDAHOImage AND id:{}".format(idaho_id)
result = vectors.query(aoi, query=query)
if len(result):
bucket = result[0]["properties"]["attributes"][
"tileBucketName"]
dn_op = ipe.IdahoRead(bucketName=bucket,
imageId=idaho_id,
objectStore="S3")
params = ortho_params(proj, gsd=gsd)
graph = {
"1b":
dn_op,
"ortho":
ipe.Orthorectify(dn_op, **params),
"acomp":
ipe.Format(ipe.Orthorectify(ipe.Acomp(dn_op), **params),
dataType="4"),
"toa_reflectance":
ipe.Format(ipe.Orthorectify(ipe.TOAReflectance(dn_op), **params),
dataType="4")
}
return graph
def test_vectors_search_index(self):
v = Vectors()
aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))"
results = v.query(aoi,
query="item_type:WV03",
index="vector-dgcatalog-2016")
assert len(results) == 140
def _image_by_type(cls, cat_id, **kwargs):
vectors = Vectors()
aoi = wkt.dumps(box(-180, -90, 180, 90))
query = "item_type:GBDXCatalogRecord AND attributes.catalogID:{}".format(cat_id)
query += " AND NOT item_type:IDAHOImage AND NOT item_type:DigitalGlobeAcquisition"
result = vectors.query(aoi, query=query, count=1)
if len(result) == 0:
raise Exception('Could not find a catalog entry for the given id: {}'.format(cat_id))
else:
return cls._image_class(cat_id, result[0], **kwargs)
def _find_parts(cat_id, band_type):
def vendor_id(rec):
_id = rec['properties']['attributes']['vendorDatasetIdentifier']
return _id.split(':')[1].split('_')[0]
vectors = Vectors()
aoi = wkt.dumps(box(-180, -90, 180, 90))
query = "item_type:IDAHOImage AND attributes.catalogID:{} " \
"AND attributes.colorInterpretation:{}".format(cat_id, band_types[band_type])
_parts = sorted(vectors.query(aoi, query=query), key=lambda x: x['properties']['id'])
if not len(_parts):
raise MissingIdahoImages('Unable to find IDAHO imagery in the catalog: {}'.format(query))
_id = vendor_id(_parts[0])
return [p for p in _parts if vendor_id(p) == _id]
def _image_by_type(cls, cat_id, **kwargs):
if cat_id is None:
try:
return IpeImage(GraphMeta(**kwargs))
except KeyError:
raise ValueError("Catalog Images must be initiated by a Catalog Id or an RDA Graph Id")
vectors = Vectors()
aoi = wkt.dumps(box(-180, -90, 180, 90))
query = "item_type:GBDXCatalogRecord AND attributes.catalogID:{}".format(cat_id)
query += " AND NOT item_type:IDAHOImage AND NOT item_type:DigitalGlobeAcquisition"
result = vectors.query(aoi, query=query, count=1)
if len(result) == 0:
raise Exception('Could not find a catalog entry for the given id: {}'.format(cat_id))
else:
return cls._image_class(cat_id, result[0], **kwargs)
def test_vectors_search(self):
v = Vectors(self.gbdx)
aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))"
results = v.query(aoi, query="item_type:WV03")
assert len(results) == 100
def test_vectors_search_index(self):
v = Vectors()
aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))"
results = v.query(aoi, query="item_type:WV03", index="vector-dgcatalog-2016", count=1000)
assert len(results) == 140
def test_vectors_search_count_equal_to_num_records(self):
v = Vectors()
aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))"
results = v.query(aoi, query="item_type:WV03", index=None, count=310)
assert len(results) == 310
def test_vectors_search_paging(self):
v = Vectors()
aoi = "POLYGON ((180 -90, 180 90, -180 90, -180 -90, 180 -90))"
results = v.query(aoi, query="item_type:WV03_VNIR", index=None, count=1010)
assert len(results) == 1010
class CatalogImage(IpeImage):
"""
Catalog Image Class
Collects metadata on all image parts and groups pan and ms bands from idaho
Inherits from IpeImage and represents a mosiac data set of the full catalog strip
"""
_properties = None
def __init__(self,
cat_id,
band_type="MS",
node="toa_reflectance",
**kwargs):
self.interface = Auth()
self.vectors = Vectors()
self._gid = cat_id
self._band_type = band_type
self._pansharpen = kwargs.get('pansharpen', False)
self._acomp = kwargs.get('acomp', False)
if self._pansharpen:
self._node_id = 'pansharpened'
else:
self._node_id = node
self._level = kwargs.get('level', 0)
if 'proj' in kwargs:
self._proj = kwargs['proj']
if '_ipe_graphs' in kwargs:
self._ipe_graphs = kwargs['_ipe_graphs']
else:
self._ipe_graphs = self._init_graphs()
super(CatalogImage, self).__init__(self._ipe_graphs,
cat_id,
node=self._node_id,
**kwargs)
def _query_vectors(self, query, aoi=None):
if aoi is None:
aoi = "POLYGON((-180.0 90.0,180.0 90.0,180.0 -90.0,-180.0 -90.0,-180.0 90.0))"
try:
return self.vectors.query(aoi, query=query)
except Exception as err:
raise Exception(
'Unable to query for image properties, the service may be currently down.',
err)
@property
def properties(self):
if self._properties is None:
query = 'item_type:DigitalGlobeAcquisition AND attributes.catalogID:{}'.format(
self._gid)
self._properties = self._query_vectors(query)
return self._properties
@property
def metadata(self):
meta = {}
query = 'item_type:IDAHOImage AND attributes.catalogID:{}'.format(
self._gid)
results = self._query_vectors(query)
grouped = defaultdict(list)
for idaho in results:
vid = idaho['properties']['attributes']['vendorDatasetIdentifier']
grouped[vid].append(idaho)
meta['parts'] = []
for key, parts in grouped.items():
part = {}
for p in parts:
attrs = p['properties']['attributes']
part[attrs['colorInterpretation']] = {
'properties': attrs,
'geometry': shape(p['geometry'])
}
meta['parts'].append(part)
return meta
def aoi(self, **kwargs):
pansharp = False
if self._pansharpen and 'pansharpen' not in kwargs:
pansharp = True
bounds = self._parse_geoms(**kwargs)
if bounds is None:
print(
'AOI bounds not found. Must specify a bbox, wkt, or geojson geometry.'
)
return
cfg = self._aoi_config(bounds, **kwargs)
return DaskImage(**cfg)
def _init_graphs(self):
graph = {}
ids = []
if self._node_id == 'pansharpened' and self._pansharpen:
return self._pansharpen_graph()
else:
for part in self.metadata['parts']:
for k, p in part.items():
if k == band_types[self._band_type]:
_id = p['properties']['idahoImageId']
graph[_id] = ipe.Orthorectify(
ipe.IdahoRead(bucketName="idaho-images",
imageId=_id,
objectStore="S3"),
**ortho_params(self._proj))
return self._mosaic(graph)
def _pansharpen_graph(self):
pan_graph = {}
ms_graph = {}
for part in self.metadata['parts']:
for k, p in part.items():
_id = p['properties']['idahoImageId']
if k == 'PAN':
pan_graph[_id] = ipe.Orthorectify(
ipe.IdahoRead(bucketName="idaho-images",
imageId=_id,
objectStore="S3"),
**ortho_params(self._proj))
else:
ms_graph[_id] = ipe.Orthorectify(
ipe.IdahoRead(bucketName="idaho-images",
imageId=_id,
objectStore="S3"),
**ortho_params(self._proj))
pan_mosaic = self._mosaic(pan_graph, suffix='-pan')
pan = ipe.Format(ipe.MultiplyConst(pan_mosaic['toa_reflectance-pan'],
constants=json.dumps([1000])),
dataType="1")
ms_mosaic = self._mosaic(ms_graph, suffix='-ms')
ms = ipe.Format(ipe.MultiplyConst(ms_mosaic['toa_reflectance-ms'],
constants=json.dumps([1000] * 8)),
dataType="1")
return {
'ms_mosaic': ms_mosaic,
'pan_mosiac': pan_mosaic,
'pansharpened': ipe.LocallyProjectivePanSharpen(ms, pan)
}
def _mosaic(self, graph, suffix=''):
mosaic = ipe.GeospatialMosaic(*graph.values())
idaho_id = list(graph.keys())[0]
meta = requests.get(
'http://idaho.timbr.io/{}.json'.format(idaho_id)).json()
gains_offsets = calc_toa_gain_offset(meta['properties'])
radiance_scales, reflectance_scales, radiance_offsets = zip(
*gains_offsets)
radiance = ipe.AddConst(ipe.MultiplyConst(ipe.Format(mosaic,
dataType="4"),
constants=radiance_scales),
constants=radiance_offsets)
toa = ipe.MultiplyConst(radiance, constants=reflectance_scales)
graph.update({
"mosaic{}".format(suffix): mosaic,
"radiance{}".format(suffix): radiance,
"toa_reflectance{}".format(suffix): toa
})
return graph
