def recursive_lookup(collection, suffix, used_ids, subsets):
# get all EO objects related to this collection, excluding
# those already searched
eo_objects = models.EOObject.objects.filter(
collections__in=[collection.pk]
).exclude(
pk__in=used_ids
).order_by("begin_time", "end_time")
# apply subsets
eo_objects = subsets.filter(eo_objects)
selection = LayerSelection()
# append all retrived EO objects, either as a coverage of
# the real type, or as a subgroup.
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
selection.append(eo_object.cast(), eo_object.identifier)
elif models.iscollection(eo_object):
selection.extend(recursive_lookup(
eo_object, suffix, used_ids, subsets
))
else:
pass
return selection
def recursive_lookup(collection, suffix, used_ids, subsets):
# get all EO objects related to this collection, excluding
# those already searched
eo_objects = models.EOObject.objects.filter(
collections__in=[collection.pk]).exclude(
pk__in=used_ids).order_by("begin_time", "end_time")
# apply subsets
eo_objects = subsets.filter(eo_objects)
selection = LayerSelection()
# append all retrived EO objects, either as a coverage of
# the real type, or as a subgroup.
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
selection.append(eo_object.cast(),
eo_object.identifier)
elif models.iscollection(eo_object):
selection.extend(
recursive_lookup(eo_object, suffix, used_ids,
subsets))
else:
pass
return selection
def _recursive_lookup(collection):
""" Search recursively through the nested collections
and find the relevant EOObjects."""
eo_objects = subsets.filter(models.EOObject.objects\
.filter(collections__in=[collection.pk])\
.exclude(pk__in=used_ids)\
.order_by("begin_time", "end_time", "identifier")\
.prefetch_related('metadata_items'))
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
selection.append(eo_object.cast(), _get_alias(eo_object))
elif models.iscollection(eo_object):
_recursive_lookup(eo_object)
else:
pass # TODO: Reporting of invalid EOObjects (?)
def recursive_lookup(collection, used_ids, suffix):
eo_objects = models.EOObject.objects.filter(
collections__in=[collection.pk]
).exclude(
pk__in=used_ids
)
result = []
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
result.append((eo_object.cast(), suffix))
elif models.iscollection(eo_object):
result.extend(
recursive_lookup(eo_object, used_ids, suffix)
)
else:
pass
return result
def recursive_lookup(collection, used_ids, suffix):
eo_objects = models.EOObject.objects.filter(
collections__in=[collection.pk]
).exclude(
pk__in=used_ids
)
result = []
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
result.append((eo_object.cast(), suffix))
elif models.iscollection(eo_object):
result.extend(
recursive_lookup(eo_object, used_ids, suffix)
)
else:
pass
return result
def lookup_layers(layers, subsets, suffixes=None):
""" Performs a layer lookup for the given layer names. Applies the given
subsets and looks up all layers with the given suffixes. Returns a
hierarchy of ``LayerSelection`` objects.
"""
suffix_related_ids = {}
root_group = LayerSelection(None)
suffixes = suffixes or (None, )
logger.debug(str(suffixes))
for layer_name in layers:
for suffix in suffixes:
if not suffix:
identifier = layer_name
elif layer_name.endswith(suffix):
identifier = layer_name[:-len(suffix)]
else:
continue
# TODO: nasty, nasty bug... dunno where
eo_objects = models.EOObject.objects.filter(identifier=identifier)
if len(eo_objects):
eo_object = eo_objects[0]
break
else:
raise LayerNotDefined(layer_name)
if models.iscollection(eo_object):
# recursively iterate over all sub-collections and collect all
# coverages
used_ids = suffix_related_ids.setdefault(suffix, set())
def recursive_lookup(collection, suffix, used_ids, subsets):
# get all EO objects related to this collection, excluding
# those already searched
eo_objects = models.EOObject.objects.filter(
collections__in=[collection.pk]).exclude(
pk__in=used_ids).order_by("begin_time", "end_time")
# apply subsets
eo_objects = subsets.filter(eo_objects)
selection = LayerSelection()
# append all retrived EO objects, either as a coverage of
# the real type, or as a subgroup.
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
selection.append(eo_object.cast(),
eo_object.identifier)
elif models.iscollection(eo_object):
selection.extend(
recursive_lookup(eo_object, suffix, used_ids,
subsets))
else:
pass
return selection
root_group.append(
LayerSelection(
eo_object, suffix,
recursive_lookup(eo_object, suffix, used_ids, subsets)))
elif models.iscoverage(eo_object):
# Add a layer selection for the coverage with the suffix
selection = LayerSelection(None, suffix=suffix)
if subsets.matches(eo_object):
selection.append(eo_object.cast(), eo_object.identifier)
else:
selection.append(None, eo_object.identifier)
root_group.append(selection)
return root_group
def synchronize(collection, recursive=False, force=False):
""" Synchronizes a :class:`eoxserver.resources.coverages.models.Collection`
and all its data sources with their respective storages.
:param collection: either the ID of a collection or the collection model
itself. internally the collection will always be
cast to its actual type.
:param recursive: perform a recursive synchronization for all
sub-collections of the specified one. by default this
is not performed.
:param force: force the re-registration of already inserted datasets.
by default, existing datasets will be preserved and not
refreshed.
"""
# allow both model and identifier
if isinstance(collection, basestring):
collection = models.Collection.objects.get(identifier=collection)
collection = collection.cast()
if models.iscoverage(collection):
overrides = {"range_type_name": collection.range_type.name}
else:
overrides = {}
if recursive:
synchronize(
models.Collection.objects.filter(collections__in=[collection.pk]),
recursive
)
logger.info("Synchronizing collection %s" % collection)
all_paths = []
for data_source in collection.data_sources.all():
all_paths.extend(
_expand_data_source(data_source)
)
with CacheContext() as cache:
# loop over all paths and check if there is already a dataset registered
# for it.
registered_pks = []
for paths in all_paths:
for filename, data_item, semantic in paths:
exists = backends.DataItem.objects.filter(
package=data_item.package, storage=data_item.storage,
location=filename
).exists()
if not exists:
break
if not exists:
logger.info("Creating new dataset.")
for registrator in RegistratorComponent(env).registrators:
# TODO: select registrator
pass
# translate for registrator
items = [
(d.storage or d.package, location, semantic, d.format)
for location, d, semantic in paths
]
dataset = registrator.register(
items, overrides, cache
)
collection.insert(dataset)
registered_pks.append(dataset.pk)
# loop over all coverages in this collection. if any is not represented
# by its referenced file, delete it. Skip the just added datasets for
# convenience (as we know that they must have referenced files)
contained = models.Coverage.objects.filter(
collections__in=[collection.pk]
).exclude(pk__in=registered_pks)
for coverage in contained:
data_items = tuple(coverage.data_items.all())
all_exists = True
# loop over all its data items
for existing_data_item in data_items:
for paths in all_paths:
for filename, data_item, semantic in paths:
if existing_data_item.location == filename and \
existing_data_item.semantic == semantic:
pass
else:
logger.info(
"Dataset '%s' is missing its file '%s'." % (
coverage.identifier,
existing_data_item.location
))
all_exists = False
break
if not all_exists:
break
if not all_exists:
break
if not all_exists:
logger.info("Deleting dataset '%s'." % coverage.identifier)
coverage.delete()
def lookup_layers(layers, subsets, suffixes=None):
""" Performs a layer lookup for the given layer names. Applies the given
subsets and looks up all layers with the given suffixes. Returns a
hierarchy of ``LayerSelection`` objects.
"""
suffix_related_ids = {}
root_group = LayerSelection(None)
suffixes = suffixes or (None,)
logger.debug(str(suffixes))
for layer_name in layers:
for suffix in suffixes:
if not suffix:
identifier = layer_name
elif layer_name.endswith(suffix):
identifier = layer_name[:-len(suffix)]
else:
continue
# TODO: nasty, nasty bug... dunno where
eo_objects = models.EOObject.objects.filter(
identifier=identifier
)
if len(eo_objects):
eo_object = eo_objects[0]
break
else:
raise LayerNotDefined(layer_name)
if models.iscollection(eo_object):
# recursively iterate over all sub-collections and collect all
# coverages
used_ids = suffix_related_ids.setdefault(suffix, set())
def recursive_lookup(collection, suffix, used_ids, subsets):
# get all EO objects related to this collection, excluding
# those already searched
eo_objects = models.EOObject.objects.filter(
collections__in=[collection.pk]
).exclude(
pk__in=used_ids
).order_by("begin_time", "end_time")
# apply subsets
eo_objects = subsets.filter(eo_objects)
selection = LayerSelection()
# append all retrived EO objects, either as a coverage of
# the real type, or as a subgroup.
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
selection.append(eo_object.cast(), eo_object.identifier)
elif models.iscollection(eo_object):
selection.extend(recursive_lookup(
eo_object, suffix, used_ids, subsets
))
else:
pass
return selection
root_group.append(
LayerSelection(
eo_object, suffix,
recursive_lookup(eo_object, suffix, used_ids, subsets)
)
)
elif models.iscoverage(eo_object):
# Add a layer selection for the coverage with the suffix
selection = LayerSelection(None, suffix=suffix)
if subsets.matches(eo_object):
selection.append(eo_object.cast(), eo_object.identifier)
else:
selection.append(None, eo_object.identifier)
root_group.append(selection)
return root_group
def synchronize(collection, recursive=False, force=False):
""" Synchronizes a :class:`eoxserver.resources.coverages.models.Collection`
and all its data sources with their respective storages. Synchronization
means to compare the contents of a collection with the contents of the
file system referenced by each of the datasources.
:param collection: either the ID of a collection or the collection model
itself. Internally the collection will always be
cast to its actual type.
:param recursive: perform a recursive synchronization for all
sub-collections of the specified one. By default this
is not performed.
:param force: force the re-registration of already inserted datasets.
By default, existing datasets will be preserved and not
refreshed.
:returns: a two-tuple: the number of newly registered and deleted stale
datasets.
"""
# allow both model and identifier
if isinstance(collection, basestring):
collection = models.Collection.objects.get(identifier=collection)
collection = collection.cast()
if models.iscoverage(collection):
overrides = {"range_type_name": collection.range_type.name}
else:
overrides = {}
if recursive:
synchronize(
models.Collection.objects.filter(collections__in=[collection.pk]),
recursive)
logger.info("Synchronizing collection %s" % collection)
# expand all filesystem globs to actually existing paths
all_paths = []
for data_source in collection.data_sources.all():
all_paths.extend(_expand_data_source(data_source))
deleted_count = 0
with CacheContext() as cache:
# loop over all paths and check if there is already a dataset registered
# for it.
registered_pks = []
for paths in all_paths:
for filename, data_item, semantic in paths:
exists = backends.DataItem.objects.filter(
package=data_item.package,
storage=data_item.storage,
location=filename).exists()
if not exists:
break
if not exists:
logger.info("Creating new dataset.")
for registrator in RegistratorComponent(env).registrators:
# TODO: select registrator
pass
# translate for registrator
items = [(d.storage or d.package, location, semantic, d.format)
for location, d, semantic in paths]
dataset = registrator.register(items, overrides, cache)
collection.insert(dataset)
registered_pks.append(dataset.pk)
# loop over all coverages in this collection. if any is not represented
# by its referenced file, delete it. Skip the just added datasets for
# convenience (as we know that they must have referenced files)
# TODO: large exclusions like this run into problems with SQLite
# contained = models.Coverage.objects.filter(
# collections__in=[collection.pk]
# ).exclude(pk__in=registered_pks)
# TODO: temporary (but slow) workaround
registered_pks = set(registered_pks)
contained = list(c for c in models.Coverage.objects.filter(
collections__in=[collection.pk]) if c.pk not in registered_pks)
for coverage in contained:
data_items = tuple(coverage.data_items.all())
# loop over all data items of the coverage and check if it was found
# in the filesystem lookup
for data_item in data_items:
found = False
# loop over all paths and check if the data item was still found
# in the filesystem
for paths in all_paths:
if found:
break
for filename, _, semantic in paths:
if data_item.location == filename and \
data_item.semantic == semantic:
found = True
break
# if the data item as not found in the paths on the filesystem,
# delete the model
if not found:
logger.info("Deleting dataset '%s'." % coverage.identifier)
coverage.delete()
deleted_count += 1
break
return len(registered_pks), deleted_count
class WCS20DescribeEOCoverageSetHandler(Component):
implements(ServiceHandlerInterface)
implements(GetServiceHandlerInterface)
implements(PostServiceHandlerInterface)
service = "WCS"
versions = ("2.0.0", "2.0.1")
request = "DescribeEOCoverageSet"
index = 20
def get_decoder(self, request):
if request.method == "GET":
return WCS20DescribeEOCoverageSetKVPDecoder(request.GET)
elif request.method == "POST":
return WCS20DescribeEOCoverageSetXMLDecoder(request.body)
@property
def constraints(self):
reader = WCSEOConfigReader(get_eoxserver_config())
return {
"CountDefault": reader.paging_count_default
}
def handle(self, request):
decoder = self.get_decoder(request)
eo_ids = decoder.eo_ids
containment = decoder.containment
if not containment:
containment = "overlaps"
count_default = self.constraints["CountDefault"]
count = decoder.count
if count_default is not None:
count = min(count, count_default)
try:
subsets = Subsets(
decoder.subsets,
crs="http://www.opengis.net/def/crs/EPSG/0/4326",
allowed_types=Trim
)
except ValueError, e:
raise InvalidSubsettingException(str(e))
inc_dss_section = decoder.section_included("DatasetSeriesDescriptions")
inc_cov_section = decoder.section_included("CoverageDescriptions")
if len(eo_ids) == 0:
raise
# fetch a list of all requested EOObjects
available_ids = models.EOObject.objects.filter(
identifier__in=eo_ids
).values_list("identifier", flat=True)
# match the requested EOIDs against the available ones. If any are
# requested, that are not available, raise and exit.
failed = [ eo_id for eo_id in eo_ids if eo_id not in available_ids ]
if failed:
raise NoSuchDatasetSeriesOrCoverageException(failed)
collections_qs = subsets.filter(models.Collection.objects.filter(
identifier__in=eo_ids
), containment="overlaps")
# create a set of all indirectly referenced containers by iterating
# recursively. The containment is set to "overlaps", to also include
# collections that might have been excluded with "contains" but would
# have matching coverages inserted.
def recursive_lookup(super_collection, collection_set):
sub_collections = models.Collection.objects.filter(
collections__in=[super_collection.pk]
).exclude(
pk__in=map(lambda c: c.pk, collection_set)
)
sub_collections = subsets.filter(sub_collections, "overlaps")
# Add all to the set
collection_set |= set(sub_collections)
for sub_collection in sub_collections:
recursive_lookup(sub_collection, collection_set)
collection_set = set(collections_qs)
for collection in set(collection_set):
recursive_lookup(collection, collection_set)
collection_pks = map(lambda c: c.pk, collection_set)
# Get all either directly referenced coverages or coverages that are
# within referenced containers. Full subsetting is applied here.
coverages_qs = subsets.filter(models.Coverage.objects.filter(
Q(identifier__in=eo_ids) | Q(collections__in=collection_pks)
), containment=containment)
# save a reference before limits are applied to obtain the full number
# of matched coverages.
coverages_no_limit_qs = coverages_qs
num_collections = len(
filter(lambda c: not models.iscoverage(c), collection_set)
)
# compute how many (if any) coverages can be retrieved. This depends on
# the "count" parameter and default setting. Also, if we already
# exceeded the count, limit the number of dataset series aswell
if inc_dss_section:
displayed_collections = num_collections
else:
displayed_collections = 0
if displayed_collections < count and inc_cov_section:
coverages_qs = coverages_qs.order_by("identifier")[:count - displayed_collections]
elif displayed_collections == count or not inc_cov_section:
coverages_qs = []
else:
coverages_qs = []
collection_set = sorted(collection_set, key=lambda c: c.identifier)[:count]
# get a number of coverages that *would* have been included, but are not
# because of the count parameter
count_all_coverages = coverages_no_limit_qs.count()
# if containment is "contains" we need to check all collections again
if containment == "contains":
collection_set = filter(lambda c: subsets.matches(c), collection_set)
coverages = set()
dataset_series = set()
# finally iterate over everything that has been retrieved and get
# a list of dataset series and coverages to be encoded into the response
for eo_object in chain(coverages_qs, collection_set):
if inc_cov_section and issubclass(eo_object.real_type, models.Coverage):
coverages.add(eo_object.cast())
elif inc_dss_section and issubclass(eo_object.real_type, models.DatasetSeries):
dataset_series.add(eo_object.cast())
else:
# TODO: what to do here?
pass
# TODO: coverages should be sorted
#coverages = sorted(coverages, )
#encoder = WCS20CoverageDescriptionXMLEncoder()
#return encoder.encode(coverages)
# TODO: remove this at some point
encoder = WCS20EOXMLEncoder()
return (
encoder.serialize(
encoder.encode_eo_coverage_set_description(
sorted(dataset_series, key=lambda s: s.identifier),
sorted(coverages, key=lambda c: c.identifier),
count_all_coverages + num_collections
), pretty_print=True
),
encoder.content_type
)
def lookup_layers(layers, subsets, suffixes=None):
""" Performs a layer lookup for the given layer names. Applies the given
subsets and looks up all layers with the given suffixes. Returns a
list of ``LayerSelection`` objects.
"""
suffixes = suffixes or (None,)
logger.debug("Tested suffixes: %s", suffixes)
# ------------------------------------------------------------------------
# local closures:
def _lookup_eoobject(layer_name):
""" Search an EOObject matching the given layer name. """
for suffix in suffixes:
if not suffix:
identifier = layer_name
elif layer_name.endswith(suffix):
identifier = layer_name[:-len(suffix)]
else: # no match found
continue
eo_objects = list(models.EOObject.objects.filter(identifier=identifier))
if len(eo_objects) > 0:
return eo_objects[0], suffix
raise LayerNotDefined(layer_name)
def _get_wms_view(eo_object):
""" Get an EOObject used for the WMS view. If there is no WMS view
object the closure returns the input object.
"""
try:
md_item = eo_object.metadata_items.get(semantic="wms_view")
return models.EOObject.objects.get(identifier=md_item.value)
except ObjectDoesNotExist:
return eo_object
def _get_alias(eo_object):
""" Get an EOObject alias, i.e., an identifier of the EOObject
the given EOOobject provides the WMS view to.
"""
try:
return eo_object.metadata_items.get(semantic="wms_alias").value
except ObjectDoesNotExist:
return None
def _recursive_lookup(collection):
""" Search recursively through the nested collections
and find the relevant EOObjects."""
eo_objects = subsets.filter(models.EOObject.objects\
.filter(collections__in=[collection.pk])\
.exclude(pk__in=used_ids)\
.order_by("begin_time", "end_time", "identifier")\
.prefetch_related('metadata_items'))
for eo_object in eo_objects:
used_ids.add(eo_object.pk)
if models.iscoverage(eo_object):
selection.append(eo_object.cast(), _get_alias(eo_object))
elif models.iscollection(eo_object):
_recursive_lookup(eo_object)
else:
pass # TODO: Reporting of invalid EOObjects (?)
# ------------------------------------------------------------------------
selections = []
# NOTE: The lookup is performed on a set of unique layer names. This has
# no effect on the rendering order of the layer as this is determined
# by the WMS request handled by the mapserver.
for layer_name in set(layers):
# get an EOObject and suffix matching the layer_name
eoo_src, suffix = _lookup_eoobject(layer_name)
# get EOObject holding the WMS view
eoo_wms = _get_wms_view(eoo_src)
# prepare the final EOObject(s) selection
selection = LayerSelection(eoo_src, suffix)
if models.iscollection(eoo_wms): # EOObject is a collection
used_ids = set()
# recursively look-up the coverages
_recursive_lookup(eoo_wms)
elif models.iscoverage(eoo_wms): # EOObject is a coverage
# append to the selection if the coverage matches the subset
if subsets.matches(eoo_wms):
selection.append(eoo_wms.cast(), eoo_src.identifier)
else:
pass # TODO: Reporting of invalid EOObjects (?)
selections.append(selection)
return selections