def _find_parts(cat_id, band_type): vectors = Vectors() aoi = wkt.dumps(box(-180, -90, 180, 90)) query = "item_type:IDAHOImage AND attributes.catalogID:{} " \ "AND attributes.colorInterpretation:{}".format(cat_id, band_types[band_type]) return sorted(vectors.query(aoi, query=query), key=lambda x: x['properties']['id'])
def vector_services_query(query, aoi=None, **kwargs): vectors = Vectors() if not aoi: aoi = wkt.dumps(box(-180, -90, 180, 90)) _parts = sorted(vectors.query(aoi, query=query, **kwargs), key=lambda x: x['properties']['id']) return _parts
def _build_standard_products(idaho_id, proj, bucket=None, gsd=None, acomp=False): if bucket is None: vectors = Vectors() aoi = wkt.dumps(box(-180, -90, 180, 90)) query = "item_type:IDAHOImage AND id:{}".format(idaho_id) result = vectors.query(aoi, query=query) if len(result): bucket = result[0]["properties"]["attributes"][ "tileBucketName"] dn_op = ipe.IdahoRead(bucketName=bucket, imageId=idaho_id, objectStore="S3") params = ortho_params(proj, gsd=gsd) graph = { "1b": dn_op, "ortho": ipe.Orthorectify(dn_op, **params), "acomp": ipe.Format(ipe.Orthorectify(ipe.Acomp(dn_op), **params), dataType="4"), "toa_reflectance": ipe.Format(ipe.Orthorectify(ipe.TOAReflectance(dn_op), **params), dataType="4") } return graph
def test_vectors_search_index(self): v = Vectors() aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))" results = v.query(aoi, query="item_type:WV03", index="vector-dgcatalog-2016") assert len(results) == 140
def _image_by_type(cls, cat_id, **kwargs): vectors = Vectors() aoi = wkt.dumps(box(-180, -90, 180, 90)) query = "item_type:GBDXCatalogRecord AND attributes.catalogID:{}".format(cat_id) query += " AND NOT item_type:IDAHOImage AND NOT item_type:DigitalGlobeAcquisition" result = vectors.query(aoi, query=query, count=1) if len(result) == 0: raise Exception('Could not find a catalog entry for the given id: {}'.format(cat_id)) else: return cls._image_class(cat_id, result[0], **kwargs)
def _find_parts(cat_id, band_type): def vendor_id(rec): _id = rec['properties']['attributes']['vendorDatasetIdentifier'] return _id.split(':')[1].split('_')[0] vectors = Vectors() aoi = wkt.dumps(box(-180, -90, 180, 90)) query = "item_type:IDAHOImage AND attributes.catalogID:{} " \ "AND attributes.colorInterpretation:{}".format(cat_id, band_types[band_type]) _parts = sorted(vectors.query(aoi, query=query), key=lambda x: x['properties']['id']) if not len(_parts): raise MissingIdahoImages('Unable to find IDAHO imagery in the catalog: {}'.format(query)) _id = vendor_id(_parts[0]) return [p for p in _parts if vendor_id(p) == _id]
def _image_by_type(cls, cat_id, **kwargs): if cat_id is None: try: return IpeImage(GraphMeta(**kwargs)) except KeyError: raise ValueError("Catalog Images must be initiated by a Catalog Id or an RDA Graph Id") vectors = Vectors() aoi = wkt.dumps(box(-180, -90, 180, 90)) query = "item_type:GBDXCatalogRecord AND attributes.catalogID:{}".format(cat_id) query += " AND NOT item_type:IDAHOImage AND NOT item_type:DigitalGlobeAcquisition" result = vectors.query(aoi, query=query, count=1) if len(result) == 0: raise Exception('Could not find a catalog entry for the given id: {}'.format(cat_id)) else: return cls._image_class(cat_id, result[0], **kwargs)
def test_vectors_search(self): v = Vectors(self.gbdx) aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))" results = v.query(aoi, query="item_type:WV03") assert len(results) == 100
def test_vectors_search_index(self): v = Vectors() aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))" results = v.query(aoi, query="item_type:WV03", index="vector-dgcatalog-2016", count=1000) assert len(results) == 140
def test_vectors_search_count_equal_to_num_records(self): v = Vectors() aoi = "POLYGON((17.75390625 25.418470119273117,24.08203125 25.418470119273117,24.08203125 19.409611549990895,17.75390625 19.409611549990895,17.75390625 25.418470119273117))" results = v.query(aoi, query="item_type:WV03", index=None, count=310) assert len(results) == 310
def test_vectors_search_paging(self): v = Vectors() aoi = "POLYGON ((180 -90, 180 90, -180 90, -180 -90, 180 -90))" results = v.query(aoi, query="item_type:WV03_VNIR", index=None, count=1010) assert len(results) == 1010
class CatalogImage(IpeImage): """ Catalog Image Class Collects metadata on all image parts and groups pan and ms bands from idaho Inherits from IpeImage and represents a mosiac data set of the full catalog strip """ _properties = None def __init__(self, cat_id, band_type="MS", node="toa_reflectance", **kwargs): self.interface = Auth() self.vectors = Vectors() self._gid = cat_id self._band_type = band_type self._pansharpen = kwargs.get('pansharpen', False) self._acomp = kwargs.get('acomp', False) if self._pansharpen: self._node_id = 'pansharpened' else: self._node_id = node self._level = kwargs.get('level', 0) if 'proj' in kwargs: self._proj = kwargs['proj'] if '_ipe_graphs' in kwargs: self._ipe_graphs = kwargs['_ipe_graphs'] else: self._ipe_graphs = self._init_graphs() super(CatalogImage, self).__init__(self._ipe_graphs, cat_id, node=self._node_id, **kwargs) def _query_vectors(self, query, aoi=None): if aoi is None: aoi = "POLYGON((-180.0 90.0,180.0 90.0,180.0 -90.0,-180.0 -90.0,-180.0 90.0))" try: return self.vectors.query(aoi, query=query) except Exception as err: raise Exception( 'Unable to query for image properties, the service may be currently down.', err) @property def properties(self): if self._properties is None: query = 'item_type:DigitalGlobeAcquisition AND attributes.catalogID:{}'.format( self._gid) self._properties = self._query_vectors(query) return self._properties @property def metadata(self): meta = {} query = 'item_type:IDAHOImage AND attributes.catalogID:{}'.format( self._gid) results = self._query_vectors(query) grouped = defaultdict(list) for idaho in results: vid = idaho['properties']['attributes']['vendorDatasetIdentifier'] grouped[vid].append(idaho) meta['parts'] = [] for key, parts in grouped.items(): part = {} for p in parts: attrs = p['properties']['attributes'] part[attrs['colorInterpretation']] = { 'properties': attrs, 'geometry': shape(p['geometry']) } meta['parts'].append(part) return meta def aoi(self, **kwargs): pansharp = False if self._pansharpen and 'pansharpen' not in kwargs: pansharp = True bounds = self._parse_geoms(**kwargs) if bounds is None: print( 'AOI bounds not found. Must specify a bbox, wkt, or geojson geometry.' ) return cfg = self._aoi_config(bounds, **kwargs) return DaskImage(**cfg) def _init_graphs(self): graph = {} ids = [] if self._node_id == 'pansharpened' and self._pansharpen: return self._pansharpen_graph() else: for part in self.metadata['parts']: for k, p in part.items(): if k == band_types[self._band_type]: _id = p['properties']['idahoImageId'] graph[_id] = ipe.Orthorectify( ipe.IdahoRead(bucketName="idaho-images", imageId=_id, objectStore="S3"), **ortho_params(self._proj)) return self._mosaic(graph) def _pansharpen_graph(self): pan_graph = {} ms_graph = {} for part in self.metadata['parts']: for k, p in part.items(): _id = p['properties']['idahoImageId'] if k == 'PAN': pan_graph[_id] = ipe.Orthorectify( ipe.IdahoRead(bucketName="idaho-images", imageId=_id, objectStore="S3"), **ortho_params(self._proj)) else: ms_graph[_id] = ipe.Orthorectify( ipe.IdahoRead(bucketName="idaho-images", imageId=_id, objectStore="S3"), **ortho_params(self._proj)) pan_mosaic = self._mosaic(pan_graph, suffix='-pan') pan = ipe.Format(ipe.MultiplyConst(pan_mosaic['toa_reflectance-pan'], constants=json.dumps([1000])), dataType="1") ms_mosaic = self._mosaic(ms_graph, suffix='-ms') ms = ipe.Format(ipe.MultiplyConst(ms_mosaic['toa_reflectance-ms'], constants=json.dumps([1000] * 8)), dataType="1") return { 'ms_mosaic': ms_mosaic, 'pan_mosiac': pan_mosaic, 'pansharpened': ipe.LocallyProjectivePanSharpen(ms, pan) } def _mosaic(self, graph, suffix=''): mosaic = ipe.GeospatialMosaic(*graph.values()) idaho_id = list(graph.keys())[0] meta = requests.get( 'http://idaho.timbr.io/{}.json'.format(idaho_id)).json() gains_offsets = calc_toa_gain_offset(meta['properties']) radiance_scales, reflectance_scales, radiance_offsets = zip( *gains_offsets) radiance = ipe.AddConst(ipe.MultiplyConst(ipe.Format(mosaic, dataType="4"), constants=radiance_scales), constants=radiance_offsets) toa = ipe.MultiplyConst(radiance, constants=reflectance_scales) graph.update({ "mosaic{}".format(suffix): mosaic, "radiance{}".format(suffix): radiance, "toa_reflectance{}".format(suffix): toa }) return graph