def load_disk_data(self, format: str, glob_pattern: str, options: dict,
viewing_parameters: dict) -> object:
if format != 'GTiff':
raise NotImplementedError(
"The format is not supported by the backend: " + format)
date_regex = options['date_regex']
if glob_pattern.startswith("hdfs:"):
kerberos()
from_date = normalize_date(viewing_parameters.get("from", None))
to_date = normalize_date(viewing_parameters.get("to", None))
left = viewing_parameters.get("left", None)
right = viewing_parameters.get("right", None)
top = viewing_parameters.get("top", None)
bottom = viewing_parameters.get("bottom", None)
srs = viewing_parameters.get("srs", None)
band_indices = viewing_parameters.get("bands")
sc = gps.get_spark_context()
gateway = JavaGateway(
eager_load=True, gateway_parameters=sc._gateway.gateway_parameters)
jvm = gateway.jvm
extent = jvm.geotrellis.vector.Extent(float(left), float(bottom), float(right), float(top)) \
if left is not None and right is not None and top is not None and bottom is not None else None
pyramid = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_disk(glob_pattern, date_regex) \
.pyramid_seq(extent, srs, from_date, to_date)
temporal_tiled_raster_layer = jvm.geopyspark.geotrellis.TemporalTiledRasterLayer
option = jvm.scala.Option
levels = {
pyramid.apply(index)._1(): TiledRasterLayer(
LayerType.SPACETIME,
temporal_tiled_raster_layer(
option.apply(pyramid.apply(index)._1()),
pyramid.apply(index)._2()))
for index in range(0, pyramid.size())
}
image_collection = GeotrellisTimeSeriesImageCollection(
pyramid=gps.Pyramid(levels),
service_registry=self._service_registry,
metadata={})
return image_collection.band_filter(
band_indices) if band_indices else image_collection
def load_collection(
self, collection_id: str,
viewing_parameters: dict) -> 'GeotrellisTimeSeriesImageCollection':
logger.info("Creating layer for {c} with viewingParameters {v}".format(
c=collection_id, v=viewing_parameters))
# TODO is it necessary to do this kerberos stuff here?
kerberos()
metadata = CollectionMetadata(
self.get_collection_metadata(collection_id, strip_private=False))
layer_source_info = metadata.get("_vito", "data_source", default={})
layer_source_type = layer_source_info.get("type", "Accumulo").lower()
logger.info("Layer source type: {s!r}".format(s=layer_source_type))
import geopyspark as gps
from_date = normalize_date(viewing_parameters.get("from", None))
to_date = normalize_date(viewing_parameters.get("to", None))
left = viewing_parameters.get("left", None)
right = viewing_parameters.get("right", None)
top = viewing_parameters.get("top", None)
bottom = viewing_parameters.get("bottom", None)
srs = viewing_parameters.get("srs", None)
bands = viewing_parameters.get("bands", None)
band_indices = [metadata.get_band_index(b)
for b in bands] if bands else None
logger.info("band_indices: {b!r}".format(b=band_indices))
# TODO: avoid this `still_needs_band_filter` ugliness.
# Also see https://github.com/Open-EO/openeo-geopyspark-driver/issues/29
still_needs_band_filter = False
pysc = gps.get_spark_context()
extent = None
gateway = JavaGateway(
eager_load=True,
gateway_parameters=pysc._gateway.gateway_parameters)
jvm = gateway.jvm
spatial_bounds_present = left is not None and right is not None and top is not None and bottom is not None
if spatial_bounds_present:
extent = jvm.geotrellis.vector.Extent(float(left), float(bottom),
float(right), float(top))
elif ConfigParams().require_bounds:
raise ProcessGraphComplexityException
else:
srs = "EPSG:4326"
extent = jvm.geotrellis.vector.Extent(-180.0, -90.0, 180.0, 90.0)
def accumulo_pyramid():
pyramidFactory = jvm.org.openeo.geotrellisaccumulo.PyramidFactory(
"hdp-accumulo-instance",
','.join(ConfigParams().zookeepernodes))
if layer_source_info.get("split", False):
pyramidFactory.setSplitRanges(True)
accumulo_layer_name = layer_source_info['data_id']
nonlocal still_needs_band_filter
still_needs_band_filter = bool(band_indices)
return pyramidFactory.pyramid_seq(accumulo_layer_name, extent, srs,
from_date, to_date)
def s3_pyramid():
endpoint = layer_source_info['endpoint']
region = layer_source_info['region']
bucket_name = layer_source_info['bucket_name']
nonlocal still_needs_band_filter
still_needs_band_filter = bool(band_indices)
return jvm.org.openeo.geotrelliss3.PyramidFactory(endpoint, region, bucket_name) \
.pyramid_seq(extent, srs, from_date, to_date)
def s3_jp2_pyramid():
endpoint = layer_source_info['endpoint']
region = layer_source_info['region']
return jvm.org.openeo.geotrelliss3.Jp2PyramidFactory(endpoint, region) \
.pyramid_seq(extent, srs, from_date, to_date, band_indices)
def file_s2_radiometry_pyramid():
return jvm.org.openeo.geotrellis.file.Sentinel2RadiometryPyramidFactory() \
.pyramid_seq(extent, srs, from_date, to_date, band_indices)
def file_s2_pyramid():
oscars_collection_id = layer_source_info['oscars_collection_id']
oscars_link_titles = metadata.band_names
root_path = layer_source_info['root_path']
filtered_link_titles = [
oscars_link_titles[i] for i in band_indices
] if band_indices else oscars_link_titles
return jvm.org.openeo.geotrellis.file.Sentinel2PyramidFactory(
oscars_collection_id, filtered_link_titles,
root_path).pyramid_seq(extent, srs, from_date, to_date)
def geotiff_pyramid():
glob_pattern = layer_source_info['glob_pattern']
date_regex = layer_source_info['date_regex']
new_pyramid_factory = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_disk(
glob_pattern, date_regex)
return self._geotiff_pyramid_factories.setdefault(collection_id, new_pyramid_factory) \
.pyramid_seq(extent, srs, from_date, to_date)
def file_s1_coherence_pyramid():
return jvm.org.openeo.geotrellis.file.Sentinel1CoherencePyramidFactory() \
.pyramid_seq(extent, srs, from_date, to_date, band_indices)
def sentinel_hub_s1_pyramid():
return jvm.org.openeo.geotrellissentinelhub.S1PyramidFactory(layer_source_info.get('uuid')) \
.pyramid_seq(extent, srs, from_date, to_date, band_indices)
def sentinel_hub_s2_l1c_pyramid():
return jvm.org.openeo.geotrellissentinelhub.S2L1CPyramidFactory(layer_source_info.get('uuid')) \
.pyramid_seq(extent, srs, from_date, to_date, band_indices)
def sentinel_hub_s2_l2a_pyramid():
return jvm.org.openeo.geotrellissentinelhub.S2L2APyramidFactory(layer_source_info.get('uuid')) \
.pyramid_seq(extent, srs, from_date, to_date, band_indices)
def sentinel_hub_l8_pyramid():
return jvm.org.openeo.geotrellissentinelhub.L8PyramidFactory(layer_source_info.get('uuid')) \
.pyramid_seq(extent, srs, from_date, to_date, band_indices)
logger.info("loading pyramid {s}".format(s=layer_source_type))
if layer_source_type == 's3':
pyramid = s3_pyramid()
elif layer_source_type == 's3-jp2':
pyramid = s3_jp2_pyramid()
elif layer_source_type == 'file-s2-radiometry':
pyramid = file_s2_radiometry_pyramid()
elif layer_source_type == 'file-s2':
pyramid = file_s2_pyramid()
elif layer_source_type == 'geotiff':
pyramid = geotiff_pyramid()
elif layer_source_type == 'file-s1-coherence':
pyramid = file_s1_coherence_pyramid()
elif layer_source_type == 'sentinel-hub-s1':
pyramid = sentinel_hub_s1_pyramid()
elif layer_source_type == 'sentinel-hub-s2-l1c':
pyramid = sentinel_hub_s2_l1c_pyramid()
elif layer_source_type == 'sentinel-hub-s2-l2a':
pyramid = sentinel_hub_s2_l2a_pyramid()
elif layer_source_type == 'sentinel-hub-l8':
pyramid = sentinel_hub_l8_pyramid()
else:
pyramid = accumulo_pyramid()
temporal_tiled_raster_layer = jvm.geopyspark.geotrellis.TemporalTiledRasterLayer
option = jvm.scala.Option
levels = {
pyramid.apply(index)._1(): TiledRasterLayer(
LayerType.SPACETIME,
temporal_tiled_raster_layer(
option.apply(pyramid.apply(index)._1()),
pyramid.apply(index)._2()))
for index in range(0, pyramid.size())
}
image_collection = GeotrellisTimeSeriesImageCollection(
pyramid=gps.Pyramid(levels),
service_registry=self._service_registry,
metadata=metadata)
if still_needs_band_filter:
# TODO: avoid this `still_needs_band_filter` ugliness.
# Also see https://github.com/Open-EO/openeo-geopyspark-driver/issues/29
image_collection = image_collection.band_filter(band_indices)
return image_collection