def recursive_lookup(collection, suffix, used_ids, subsets): # get all EO objects related to this collection, excluding # those already searched eo_objects = models.EOObject.objects.filter( collections__in=[collection.pk] ).exclude( pk__in=used_ids ).order_by("begin_time", "end_time") # apply subsets eo_objects = subsets.filter(eo_objects) selection = LayerSelection() # append all retrived EO objects, either as a coverage of # the real type, or as a subgroup. for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): selection.append(eo_object.cast(), eo_object.identifier) elif models.iscollection(eo_object): selection.extend(recursive_lookup( eo_object, suffix, used_ids, subsets )) else: pass return selection
def recursive_lookup(collection, suffix, used_ids, subsets): # get all EO objects related to this collection, excluding # those already searched eo_objects = models.EOObject.objects.filter( collections__in=[collection.pk]).exclude( pk__in=used_ids).order_by("begin_time", "end_time") # apply subsets eo_objects = subsets.filter(eo_objects) selection = LayerSelection() # append all retrived EO objects, either as a coverage of # the real type, or as a subgroup. for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): selection.append(eo_object.cast(), eo_object.identifier) elif models.iscollection(eo_object): selection.extend( recursive_lookup(eo_object, suffix, used_ids, subsets)) else: pass return selection
def _recursive_lookup(collection): """ Search recursively through the nested collections and find the relevant EOObjects.""" eo_objects = subsets.filter(models.EOObject.objects\ .filter(collections__in=[collection.pk])\ .exclude(pk__in=used_ids)\ .order_by("begin_time", "end_time", "identifier")\ .prefetch_related('metadata_items')) for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): selection.append(eo_object.cast(), _get_alias(eo_object)) elif models.iscollection(eo_object): _recursive_lookup(eo_object) else: pass # TODO: Reporting of invalid EOObjects (?)
def recursive_lookup(collection, used_ids, suffix): eo_objects = models.EOObject.objects.filter( collections__in=[collection.pk] ).exclude( pk__in=used_ids ) result = [] for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): result.append((eo_object.cast(), suffix)) elif models.iscollection(eo_object): result.extend( recursive_lookup(eo_object, used_ids, suffix) ) else: pass return result
def recursive_lookup(collection, used_ids, suffix): eo_objects = models.EOObject.objects.filter( collections__in=[collection.pk] ).exclude( pk__in=used_ids ) result = [] for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): result.append((eo_object.cast(), suffix)) elif models.iscollection(eo_object): result.extend( recursive_lookup(eo_object, used_ids, suffix) ) else: pass return result
def lookup_layers(layers, subsets, suffixes=None): """ Performs a layer lookup for the given layer names. Applies the given subsets and looks up all layers with the given suffixes. Returns a hierarchy of ``LayerSelection`` objects. """ suffix_related_ids = {} root_group = LayerSelection(None) suffixes = suffixes or (None, ) logger.debug(str(suffixes)) for layer_name in layers: for suffix in suffixes: if not suffix: identifier = layer_name elif layer_name.endswith(suffix): identifier = layer_name[:-len(suffix)] else: continue # TODO: nasty, nasty bug... dunno where eo_objects = models.EOObject.objects.filter(identifier=identifier) if len(eo_objects): eo_object = eo_objects[0] break else: raise LayerNotDefined(layer_name) if models.iscollection(eo_object): # recursively iterate over all sub-collections and collect all # coverages used_ids = suffix_related_ids.setdefault(suffix, set()) def recursive_lookup(collection, suffix, used_ids, subsets): # get all EO objects related to this collection, excluding # those already searched eo_objects = models.EOObject.objects.filter( collections__in=[collection.pk]).exclude( pk__in=used_ids).order_by("begin_time", "end_time") # apply subsets eo_objects = subsets.filter(eo_objects) selection = LayerSelection() # append all retrived EO objects, either as a coverage of # the real type, or as a subgroup. for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): selection.append(eo_object.cast(), eo_object.identifier) elif models.iscollection(eo_object): selection.extend( recursive_lookup(eo_object, suffix, used_ids, subsets)) else: pass return selection root_group.append( LayerSelection( eo_object, suffix, recursive_lookup(eo_object, suffix, used_ids, subsets))) elif models.iscoverage(eo_object): # Add a layer selection for the coverage with the suffix selection = LayerSelection(None, suffix=suffix) if subsets.matches(eo_object): selection.append(eo_object.cast(), eo_object.identifier) else: selection.append(None, eo_object.identifier) root_group.append(selection) return root_group
def synchronize(collection, recursive=False, force=False): """ Synchronizes a :class:`eoxserver.resources.coverages.models.Collection` and all its data sources with their respective storages. :param collection: either the ID of a collection or the collection model itself. internally the collection will always be cast to its actual type. :param recursive: perform a recursive synchronization for all sub-collections of the specified one. by default this is not performed. :param force: force the re-registration of already inserted datasets. by default, existing datasets will be preserved and not refreshed. """ # allow both model and identifier if isinstance(collection, basestring): collection = models.Collection.objects.get(identifier=collection) collection = collection.cast() if models.iscoverage(collection): overrides = {"range_type_name": collection.range_type.name} else: overrides = {} if recursive: synchronize( models.Collection.objects.filter(collections__in=[collection.pk]), recursive ) logger.info("Synchronizing collection %s" % collection) all_paths = [] for data_source in collection.data_sources.all(): all_paths.extend( _expand_data_source(data_source) ) with CacheContext() as cache: # loop over all paths and check if there is already a dataset registered # for it. registered_pks = [] for paths in all_paths: for filename, data_item, semantic in paths: exists = backends.DataItem.objects.filter( package=data_item.package, storage=data_item.storage, location=filename ).exists() if not exists: break if not exists: logger.info("Creating new dataset.") for registrator in RegistratorComponent(env).registrators: # TODO: select registrator pass # translate for registrator items = [ (d.storage or d.package, location, semantic, d.format) for location, d, semantic in paths ] dataset = registrator.register( items, overrides, cache ) collection.insert(dataset) registered_pks.append(dataset.pk) # loop over all coverages in this collection. if any is not represented # by its referenced file, delete it. Skip the just added datasets for # convenience (as we know that they must have referenced files) contained = models.Coverage.objects.filter( collections__in=[collection.pk] ).exclude(pk__in=registered_pks) for coverage in contained: data_items = tuple(coverage.data_items.all()) all_exists = True # loop over all its data items for existing_data_item in data_items: for paths in all_paths: for filename, data_item, semantic in paths: if existing_data_item.location == filename and \ existing_data_item.semantic == semantic: pass else: logger.info( "Dataset '%s' is missing its file '%s'." % ( coverage.identifier, existing_data_item.location )) all_exists = False break if not all_exists: break if not all_exists: break if not all_exists: logger.info("Deleting dataset '%s'." % coverage.identifier) coverage.delete()
def lookup_layers(layers, subsets, suffixes=None): """ Performs a layer lookup for the given layer names. Applies the given subsets and looks up all layers with the given suffixes. Returns a hierarchy of ``LayerSelection`` objects. """ suffix_related_ids = {} root_group = LayerSelection(None) suffixes = suffixes or (None,) logger.debug(str(suffixes)) for layer_name in layers: for suffix in suffixes: if not suffix: identifier = layer_name elif layer_name.endswith(suffix): identifier = layer_name[:-len(suffix)] else: continue # TODO: nasty, nasty bug... dunno where eo_objects = models.EOObject.objects.filter( identifier=identifier ) if len(eo_objects): eo_object = eo_objects[0] break else: raise LayerNotDefined(layer_name) if models.iscollection(eo_object): # recursively iterate over all sub-collections and collect all # coverages used_ids = suffix_related_ids.setdefault(suffix, set()) def recursive_lookup(collection, suffix, used_ids, subsets): # get all EO objects related to this collection, excluding # those already searched eo_objects = models.EOObject.objects.filter( collections__in=[collection.pk] ).exclude( pk__in=used_ids ).order_by("begin_time", "end_time") # apply subsets eo_objects = subsets.filter(eo_objects) selection = LayerSelection() # append all retrived EO objects, either as a coverage of # the real type, or as a subgroup. for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): selection.append(eo_object.cast(), eo_object.identifier) elif models.iscollection(eo_object): selection.extend(recursive_lookup( eo_object, suffix, used_ids, subsets )) else: pass return selection root_group.append( LayerSelection( eo_object, suffix, recursive_lookup(eo_object, suffix, used_ids, subsets) ) ) elif models.iscoverage(eo_object): # Add a layer selection for the coverage with the suffix selection = LayerSelection(None, suffix=suffix) if subsets.matches(eo_object): selection.append(eo_object.cast(), eo_object.identifier) else: selection.append(None, eo_object.identifier) root_group.append(selection) return root_group
def synchronize(collection, recursive=False, force=False): """ Synchronizes a :class:`eoxserver.resources.coverages.models.Collection` and all its data sources with their respective storages. Synchronization means to compare the contents of a collection with the contents of the file system referenced by each of the datasources. :param collection: either the ID of a collection or the collection model itself. Internally the collection will always be cast to its actual type. :param recursive: perform a recursive synchronization for all sub-collections of the specified one. By default this is not performed. :param force: force the re-registration of already inserted datasets. By default, existing datasets will be preserved and not refreshed. :returns: a two-tuple: the number of newly registered and deleted stale datasets. """ # allow both model and identifier if isinstance(collection, basestring): collection = models.Collection.objects.get(identifier=collection) collection = collection.cast() if models.iscoverage(collection): overrides = {"range_type_name": collection.range_type.name} else: overrides = {} if recursive: synchronize( models.Collection.objects.filter(collections__in=[collection.pk]), recursive) logger.info("Synchronizing collection %s" % collection) # expand all filesystem globs to actually existing paths all_paths = [] for data_source in collection.data_sources.all(): all_paths.extend(_expand_data_source(data_source)) deleted_count = 0 with CacheContext() as cache: # loop over all paths and check if there is already a dataset registered # for it. registered_pks = [] for paths in all_paths: for filename, data_item, semantic in paths: exists = backends.DataItem.objects.filter( package=data_item.package, storage=data_item.storage, location=filename).exists() if not exists: break if not exists: logger.info("Creating new dataset.") for registrator in RegistratorComponent(env).registrators: # TODO: select registrator pass # translate for registrator items = [(d.storage or d.package, location, semantic, d.format) for location, d, semantic in paths] dataset = registrator.register(items, overrides, cache) collection.insert(dataset) registered_pks.append(dataset.pk) # loop over all coverages in this collection. if any is not represented # by its referenced file, delete it. Skip the just added datasets for # convenience (as we know that they must have referenced files) # TODO: large exclusions like this run into problems with SQLite # contained = models.Coverage.objects.filter( # collections__in=[collection.pk] # ).exclude(pk__in=registered_pks) # TODO: temporary (but slow) workaround registered_pks = set(registered_pks) contained = list(c for c in models.Coverage.objects.filter( collections__in=[collection.pk]) if c.pk not in registered_pks) for coverage in contained: data_items = tuple(coverage.data_items.all()) # loop over all data items of the coverage and check if it was found # in the filesystem lookup for data_item in data_items: found = False # loop over all paths and check if the data item was still found # in the filesystem for paths in all_paths: if found: break for filename, _, semantic in paths: if data_item.location == filename and \ data_item.semantic == semantic: found = True break # if the data item as not found in the paths on the filesystem, # delete the model if not found: logger.info("Deleting dataset '%s'." % coverage.identifier) coverage.delete() deleted_count += 1 break return len(registered_pks), deleted_count
class WCS20DescribeEOCoverageSetHandler(Component): implements(ServiceHandlerInterface) implements(GetServiceHandlerInterface) implements(PostServiceHandlerInterface) service = "WCS" versions = ("2.0.0", "2.0.1") request = "DescribeEOCoverageSet" index = 20 def get_decoder(self, request): if request.method == "GET": return WCS20DescribeEOCoverageSetKVPDecoder(request.GET) elif request.method == "POST": return WCS20DescribeEOCoverageSetXMLDecoder(request.body) @property def constraints(self): reader = WCSEOConfigReader(get_eoxserver_config()) return { "CountDefault": reader.paging_count_default } def handle(self, request): decoder = self.get_decoder(request) eo_ids = decoder.eo_ids containment = decoder.containment if not containment: containment = "overlaps" count_default = self.constraints["CountDefault"] count = decoder.count if count_default is not None: count = min(count, count_default) try: subsets = Subsets( decoder.subsets, crs="http://www.opengis.net/def/crs/EPSG/0/4326", allowed_types=Trim ) except ValueError, e: raise InvalidSubsettingException(str(e)) inc_dss_section = decoder.section_included("DatasetSeriesDescriptions") inc_cov_section = decoder.section_included("CoverageDescriptions") if len(eo_ids) == 0: raise # fetch a list of all requested EOObjects available_ids = models.EOObject.objects.filter( identifier__in=eo_ids ).values_list("identifier", flat=True) # match the requested EOIDs against the available ones. If any are # requested, that are not available, raise and exit. failed = [ eo_id for eo_id in eo_ids if eo_id not in available_ids ] if failed: raise NoSuchDatasetSeriesOrCoverageException(failed) collections_qs = subsets.filter(models.Collection.objects.filter( identifier__in=eo_ids ), containment="overlaps") # create a set of all indirectly referenced containers by iterating # recursively. The containment is set to "overlaps", to also include # collections that might have been excluded with "contains" but would # have matching coverages inserted. def recursive_lookup(super_collection, collection_set): sub_collections = models.Collection.objects.filter( collections__in=[super_collection.pk] ).exclude( pk__in=map(lambda c: c.pk, collection_set) ) sub_collections = subsets.filter(sub_collections, "overlaps") # Add all to the set collection_set |= set(sub_collections) for sub_collection in sub_collections: recursive_lookup(sub_collection, collection_set) collection_set = set(collections_qs) for collection in set(collection_set): recursive_lookup(collection, collection_set) collection_pks = map(lambda c: c.pk, collection_set) # Get all either directly referenced coverages or coverages that are # within referenced containers. Full subsetting is applied here. coverages_qs = subsets.filter(models.Coverage.objects.filter( Q(identifier__in=eo_ids) | Q(collections__in=collection_pks) ), containment=containment) # save a reference before limits are applied to obtain the full number # of matched coverages. coverages_no_limit_qs = coverages_qs num_collections = len( filter(lambda c: not models.iscoverage(c), collection_set) ) # compute how many (if any) coverages can be retrieved. This depends on # the "count" parameter and default setting. Also, if we already # exceeded the count, limit the number of dataset series aswell if inc_dss_section: displayed_collections = num_collections else: displayed_collections = 0 if displayed_collections < count and inc_cov_section: coverages_qs = coverages_qs.order_by("identifier")[:count - displayed_collections] elif displayed_collections == count or not inc_cov_section: coverages_qs = [] else: coverages_qs = [] collection_set = sorted(collection_set, key=lambda c: c.identifier)[:count] # get a number of coverages that *would* have been included, but are not # because of the count parameter count_all_coverages = coverages_no_limit_qs.count() # if containment is "contains" we need to check all collections again if containment == "contains": collection_set = filter(lambda c: subsets.matches(c), collection_set) coverages = set() dataset_series = set() # finally iterate over everything that has been retrieved and get # a list of dataset series and coverages to be encoded into the response for eo_object in chain(coverages_qs, collection_set): if inc_cov_section and issubclass(eo_object.real_type, models.Coverage): coverages.add(eo_object.cast()) elif inc_dss_section and issubclass(eo_object.real_type, models.DatasetSeries): dataset_series.add(eo_object.cast()) else: # TODO: what to do here? pass # TODO: coverages should be sorted #coverages = sorted(coverages, ) #encoder = WCS20CoverageDescriptionXMLEncoder() #return encoder.encode(coverages) # TODO: remove this at some point encoder = WCS20EOXMLEncoder() return ( encoder.serialize( encoder.encode_eo_coverage_set_description( sorted(dataset_series, key=lambda s: s.identifier), sorted(coverages, key=lambda c: c.identifier), count_all_coverages + num_collections ), pretty_print=True ), encoder.content_type )
def lookup_layers(layers, subsets, suffixes=None): """ Performs a layer lookup for the given layer names. Applies the given subsets and looks up all layers with the given suffixes. Returns a list of ``LayerSelection`` objects. """ suffixes = suffixes or (None,) logger.debug("Tested suffixes: %s", suffixes) # ------------------------------------------------------------------------ # local closures: def _lookup_eoobject(layer_name): """ Search an EOObject matching the given layer name. """ for suffix in suffixes: if not suffix: identifier = layer_name elif layer_name.endswith(suffix): identifier = layer_name[:-len(suffix)] else: # no match found continue eo_objects = list(models.EOObject.objects.filter(identifier=identifier)) if len(eo_objects) > 0: return eo_objects[0], suffix raise LayerNotDefined(layer_name) def _get_wms_view(eo_object): """ Get an EOObject used for the WMS view. If there is no WMS view object the closure returns the input object. """ try: md_item = eo_object.metadata_items.get(semantic="wms_view") return models.EOObject.objects.get(identifier=md_item.value) except ObjectDoesNotExist: return eo_object def _get_alias(eo_object): """ Get an EOObject alias, i.e., an identifier of the EOObject the given EOOobject provides the WMS view to. """ try: return eo_object.metadata_items.get(semantic="wms_alias").value except ObjectDoesNotExist: return None def _recursive_lookup(collection): """ Search recursively through the nested collections and find the relevant EOObjects.""" eo_objects = subsets.filter(models.EOObject.objects\ .filter(collections__in=[collection.pk])\ .exclude(pk__in=used_ids)\ .order_by("begin_time", "end_time", "identifier")\ .prefetch_related('metadata_items')) for eo_object in eo_objects: used_ids.add(eo_object.pk) if models.iscoverage(eo_object): selection.append(eo_object.cast(), _get_alias(eo_object)) elif models.iscollection(eo_object): _recursive_lookup(eo_object) else: pass # TODO: Reporting of invalid EOObjects (?) # ------------------------------------------------------------------------ selections = [] # NOTE: The lookup is performed on a set of unique layer names. This has # no effect on the rendering order of the layer as this is determined # by the WMS request handled by the mapserver. for layer_name in set(layers): # get an EOObject and suffix matching the layer_name eoo_src, suffix = _lookup_eoobject(layer_name) # get EOObject holding the WMS view eoo_wms = _get_wms_view(eoo_src) # prepare the final EOObject(s) selection selection = LayerSelection(eoo_src, suffix) if models.iscollection(eoo_wms): # EOObject is a collection used_ids = set() # recursively look-up the coverages _recursive_lookup(eoo_wms) elif models.iscoverage(eoo_wms): # EOObject is a coverage # append to the selection if the coverage matches the subset if subsets.matches(eoo_wms): selection.append(eoo_wms.cast(), eoo_src.identifier) else: pass # TODO: Reporting of invalid EOObjects (?) selections.append(selection) return selections