Exemplo n.º 1
0
            def recursive_lookup(collection, suffix, used_ids, subsets):
                # get all EO objects related to this collection, excluding 
                # those already searched
                eo_objects = models.EOObject.objects.filter(
                    collections__in=[collection.pk]
                ).exclude(
                    pk__in=used_ids
                ).order_by("begin_time", "end_time")
                # apply subsets
                eo_objects = subsets.filter(eo_objects)

                selection = LayerSelection()

                # append all retrived EO objects, either as a coverage of 
                # the real type, or as a subgroup.
                for eo_object in eo_objects:
                    used_ids.add(eo_object.pk)

                    if models.iscoverage(eo_object):
                        selection.append(eo_object.cast(), eo_object.identifier)
                    elif models.iscollection(eo_object):
                        selection.extend(recursive_lookup(
                            eo_object, suffix, used_ids, subsets
                        ))
                    else: 
                        pass

                return selection
Exemplo n.º 2
0
            def recursive_lookup(collection, suffix, used_ids, subsets):
                # get all EO objects related to this collection, excluding
                # those already searched
                eo_objects = models.EOObject.objects.filter(
                    collections__in=[collection.pk]).exclude(
                        pk__in=used_ids).order_by("begin_time", "end_time")
                # apply subsets
                eo_objects = subsets.filter(eo_objects)

                selection = LayerSelection()

                # append all retrived EO objects, either as a coverage of
                # the real type, or as a subgroup.
                for eo_object in eo_objects:
                    used_ids.add(eo_object.pk)

                    if models.iscoverage(eo_object):
                        selection.append(eo_object.cast(),
                                         eo_object.identifier)
                    elif models.iscollection(eo_object):
                        selection.extend(
                            recursive_lookup(eo_object, suffix, used_ids,
                                             subsets))
                    else:
                        pass

                return selection
Exemplo n.º 3
0
 def _recursive_lookup(collection):
     """ Search recursively through the nested collections
         and find the relevant EOObjects."""
     eo_objects = subsets.filter(models.EOObject.objects\
                     .filter(collections__in=[collection.pk])\
                     .exclude(pk__in=used_ids)\
                     .order_by("begin_time", "end_time", "identifier")\
                     .prefetch_related('metadata_items'))
     for eo_object in eo_objects:
         used_ids.add(eo_object.pk)
         if models.iscoverage(eo_object):
             selection.append(eo_object.cast(), _get_alias(eo_object))
         elif models.iscollection(eo_object):
             _recursive_lookup(eo_object)
         else:
             pass # TODO: Reporting of invalid EOObjects (?)
Exemplo n.º 4
0
            def recursive_lookup(collection, used_ids, suffix):
                eo_objects = models.EOObject.objects.filter(
                    collections__in=[collection.pk]
                ).exclude(
                    pk__in=used_ids
                )

                result = []
                for eo_object in eo_objects:
                    used_ids.add(eo_object.pk)

                    if models.iscoverage(eo_object):
                        result.append((eo_object.cast(), suffix))
                    elif models.iscollection(eo_object):
                        result.extend(
                            recursive_lookup(eo_object, used_ids, suffix)
                        )
                    else:
                        pass

                return result
Exemplo n.º 5
0
            def recursive_lookup(collection, used_ids, suffix):
                eo_objects = models.EOObject.objects.filter(
                    collections__in=[collection.pk]
                ).exclude(
                    pk__in=used_ids
                )

                result = []
                for eo_object in eo_objects:
                    used_ids.add(eo_object.pk)

                    if models.iscoverage(eo_object):
                        result.append((eo_object.cast(), suffix))
                    elif models.iscollection(eo_object):
                        result.extend(
                            recursive_lookup(eo_object, used_ids, suffix)
                        )
                    else:
                        pass

                return result
Exemplo n.º 6
0
def lookup_layers(layers, subsets, suffixes=None):
    """ Performs a layer lookup for the given layer names. Applies the given
        subsets and looks up all layers with the given suffixes. Returns a
        hierarchy of ``LayerSelection`` objects.
    """
    suffix_related_ids = {}
    root_group = LayerSelection(None)
    suffixes = suffixes or (None, )
    logger.debug(str(suffixes))

    for layer_name in layers:
        for suffix in suffixes:
            if not suffix:
                identifier = layer_name
            elif layer_name.endswith(suffix):
                identifier = layer_name[:-len(suffix)]
            else:
                continue

            # TODO: nasty, nasty bug... dunno where
            eo_objects = models.EOObject.objects.filter(identifier=identifier)
            if len(eo_objects):
                eo_object = eo_objects[0]
                break
        else:
            raise LayerNotDefined(layer_name)

        if models.iscollection(eo_object):
            # recursively iterate over all sub-collections and collect all
            # coverages

            used_ids = suffix_related_ids.setdefault(suffix, set())

            def recursive_lookup(collection, suffix, used_ids, subsets):
                # get all EO objects related to this collection, excluding
                # those already searched
                eo_objects = models.EOObject.objects.filter(
                    collections__in=[collection.pk]).exclude(
                        pk__in=used_ids).order_by("begin_time", "end_time")
                # apply subsets
                eo_objects = subsets.filter(eo_objects)

                selection = LayerSelection()

                # append all retrived EO objects, either as a coverage of
                # the real type, or as a subgroup.
                for eo_object in eo_objects:
                    used_ids.add(eo_object.pk)

                    if models.iscoverage(eo_object):
                        selection.append(eo_object.cast(),
                                         eo_object.identifier)
                    elif models.iscollection(eo_object):
                        selection.extend(
                            recursive_lookup(eo_object, suffix, used_ids,
                                             subsets))
                    else:
                        pass

                return selection

            root_group.append(
                LayerSelection(
                    eo_object, suffix,
                    recursive_lookup(eo_object, suffix, used_ids, subsets)))

        elif models.iscoverage(eo_object):
            # Add a layer selection for the coverage with the suffix
            selection = LayerSelection(None, suffix=suffix)
            if subsets.matches(eo_object):
                selection.append(eo_object.cast(), eo_object.identifier)
            else:
                selection.append(None, eo_object.identifier)

            root_group.append(selection)

    return root_group
Exemplo n.º 7
0
def synchronize(collection, recursive=False, force=False):
    """ Synchronizes a :class:`eoxserver.resources.coverages.models.Collection`
        and all its data sources with their respective storages.

        :param collection: either the ID of a collection or the collection model
                           itself. internally the collection will always be
                           cast to its actual type.
        :param recursive: perform a recursive synchronization for all
                          sub-collections of the specified one. by default this
                          is not performed.
        :param force: force the re-registration of already inserted datasets.
                      by default, existing datasets will be preserved and not
                      refreshed.
    """

    # allow both model and identifier
    if isinstance(collection, basestring):
        collection = models.Collection.objects.get(identifier=collection)

    collection = collection.cast()
    if models.iscoverage(collection):
        overrides = {"range_type_name": collection.range_type.name}
    else:
        overrides = {}

    if recursive:
        synchronize(
            models.Collection.objects.filter(collections__in=[collection.pk]),
            recursive
        )

    logger.info("Synchronizing collection %s" % collection)

    all_paths = []
    for data_source in collection.data_sources.all():
        all_paths.extend(
            _expand_data_source(data_source)
        )

    with CacheContext() as cache:
        # loop over all paths and check if there is already a dataset registered
        # for it.
        registered_pks = []
        for paths in all_paths:
            for filename, data_item, semantic in paths:
                exists = backends.DataItem.objects.filter(
                    package=data_item.package, storage=data_item.storage,
                    location=filename
                ).exists()
                if not exists:
                    break

            if not exists:
                logger.info("Creating new dataset.")
                for registrator in RegistratorComponent(env).registrators:
                    # TODO: select registrator
                    pass

                # translate for registrator
                items = [
                    (d.storage or d.package, location, semantic, d.format)
                    for location, d, semantic in paths
                ]
                dataset = registrator.register(
                    items, overrides, cache
                )
                collection.insert(dataset)
                registered_pks.append(dataset.pk)

        # loop over all coverages in this collection. if any is not represented
        # by its referenced file, delete it. Skip the just added datasets for
        # convenience (as we know that they must have referenced files)
        contained = models.Coverage.objects.filter(
            collections__in=[collection.pk]
        ).exclude(pk__in=registered_pks)

        for coverage in contained:
            data_items = tuple(coverage.data_items.all())
            all_exists = True

            # loop over all its data items
            for existing_data_item in data_items:
                for paths in all_paths:
                    for filename, data_item, semantic in paths:
                        if existing_data_item.location == filename and \
                                existing_data_item.semantic == semantic:
                            pass
                        else:
                            logger.info(
                                "Dataset '%s' is missing its file '%s'." % (
                                    coverage.identifier,
                                    existing_data_item.location
                                ))
                            all_exists = False
                            break
                    if not all_exists:
                        break
                if not all_exists:
                    break

            if not all_exists:
                logger.info("Deleting dataset '%s'." % coverage.identifier)
                coverage.delete()
Exemplo n.º 8
0
def lookup_layers(layers, subsets, suffixes=None):
    """ Performs a layer lookup for the given layer names. Applies the given 
        subsets and looks up all layers with the given suffixes. Returns a 
        hierarchy of ``LayerSelection`` objects.
    """
    suffix_related_ids = {}
    root_group = LayerSelection(None)
    suffixes = suffixes or (None,)
    logger.debug(str(suffixes))


    for layer_name in layers:
        for suffix in suffixes:
            if not suffix:
                identifier = layer_name
            elif layer_name.endswith(suffix):
                identifier = layer_name[:-len(suffix)]
            else:
                continue
            
            # TODO: nasty, nasty bug... dunno where
            eo_objects = models.EOObject.objects.filter(
                identifier=identifier
            )
            if len(eo_objects):
                eo_object = eo_objects[0]
                break
        else:
            raise LayerNotDefined(layer_name)

        if models.iscollection(eo_object):
            # recursively iterate over all sub-collections and collect all
            # coverages

            used_ids = suffix_related_ids.setdefault(suffix, set())

            def recursive_lookup(collection, suffix, used_ids, subsets):
                # get all EO objects related to this collection, excluding 
                # those already searched
                eo_objects = models.EOObject.objects.filter(
                    collections__in=[collection.pk]
                ).exclude(
                    pk__in=used_ids
                ).order_by("begin_time", "end_time")
                # apply subsets
                eo_objects = subsets.filter(eo_objects)

                selection = LayerSelection()

                # append all retrived EO objects, either as a coverage of 
                # the real type, or as a subgroup.
                for eo_object in eo_objects:
                    used_ids.add(eo_object.pk)

                    if models.iscoverage(eo_object):
                        selection.append(eo_object.cast(), eo_object.identifier)
                    elif models.iscollection(eo_object):
                        selection.extend(recursive_lookup(
                            eo_object, suffix, used_ids, subsets
                        ))
                    else: 
                        pass

                return selection

            root_group.append(
                LayerSelection(
                    eo_object, suffix,
                    recursive_lookup(eo_object, suffix, used_ids, subsets)
                )
            )

        elif models.iscoverage(eo_object):
            # Add a layer selection for the coverage with the suffix
            selection = LayerSelection(None, suffix=suffix)
            if subsets.matches(eo_object):
                selection.append(eo_object.cast(), eo_object.identifier)
            else:
                selection.append(None, eo_object.identifier)

            root_group.append(selection)

    return root_group
Exemplo n.º 9
0
def synchronize(collection, recursive=False, force=False):
    """ Synchronizes a :class:`eoxserver.resources.coverages.models.Collection`
        and all its data sources with their respective storages. Synchronization
        means to compare the contents of a collection with the contents of the
        file system referenced by each of the datasources.

        :param collection: either the ID of a collection or the collection model
                           itself. Internally the collection will always be
                           cast to its actual type.
        :param recursive: perform a recursive synchronization for all
                          sub-collections of the specified one. By default this
                          is not performed.
        :param force: force the re-registration of already inserted datasets.
                      By default, existing datasets will be preserved and not
                      refreshed.
        :returns: a two-tuple: the number of newly registered and deleted stale
                  datasets.
    """

    # allow both model and identifier
    if isinstance(collection, basestring):
        collection = models.Collection.objects.get(identifier=collection)

    collection = collection.cast()
    if models.iscoverage(collection):
        overrides = {"range_type_name": collection.range_type.name}
    else:
        overrides = {}

    if recursive:
        synchronize(
            models.Collection.objects.filter(collections__in=[collection.pk]),
            recursive)

    logger.info("Synchronizing collection %s" % collection)

    # expand all filesystem globs to actually existing paths
    all_paths = []
    for data_source in collection.data_sources.all():
        all_paths.extend(_expand_data_source(data_source))

    deleted_count = 0

    with CacheContext() as cache:
        # loop over all paths and check if there is already a dataset registered
        # for it.
        registered_pks = []
        for paths in all_paths:
            for filename, data_item, semantic in paths:
                exists = backends.DataItem.objects.filter(
                    package=data_item.package,
                    storage=data_item.storage,
                    location=filename).exists()
                if not exists:
                    break

            if not exists:
                logger.info("Creating new dataset.")
                for registrator in RegistratorComponent(env).registrators:
                    # TODO: select registrator
                    pass

                # translate for registrator
                items = [(d.storage or d.package, location, semantic, d.format)
                         for location, d, semantic in paths]
                dataset = registrator.register(items, overrides, cache)
                collection.insert(dataset)
                registered_pks.append(dataset.pk)

        # loop over all coverages in this collection. if any is not represented
        # by its referenced file, delete it. Skip the just added datasets for
        # convenience (as we know that they must have referenced files)

        # TODO: large exclusions like this run into problems with SQLite
        # contained = models.Coverage.objects.filter(
        #     collections__in=[collection.pk]
        # ).exclude(pk__in=registered_pks)

        # TODO: temporary (but slow) workaround
        registered_pks = set(registered_pks)
        contained = list(c for c in models.Coverage.objects.filter(
            collections__in=[collection.pk]) if c.pk not in registered_pks)

        for coverage in contained:
            data_items = tuple(coverage.data_items.all())

            # loop over all data items of the coverage and check if it was found
            # in the filesystem lookup
            for data_item in data_items:

                found = False
                # loop over all paths and check if the data item was still found
                # in the filesystem
                for paths in all_paths:
                    if found:
                        break

                    for filename, _, semantic in paths:
                        if data_item.location == filename and \
                                data_item.semantic == semantic:
                            found = True
                            break

                # if the data item as not found in the paths on the filesystem,
                # delete the model
                if not found:
                    logger.info("Deleting dataset '%s'." % coverage.identifier)
                    coverage.delete()
                    deleted_count += 1
                    break

        return len(registered_pks), deleted_count
Exemplo n.º 10
0
class WCS20DescribeEOCoverageSetHandler(Component):
    implements(ServiceHandlerInterface)
    implements(GetServiceHandlerInterface)
    implements(PostServiceHandlerInterface)

    service = "WCS"
    versions = ("2.0.0", "2.0.1")
    request = "DescribeEOCoverageSet"

    index = 20

    def get_decoder(self, request):
        if request.method == "GET":
            return WCS20DescribeEOCoverageSetKVPDecoder(request.GET)
        elif request.method == "POST":
            return WCS20DescribeEOCoverageSetXMLDecoder(request.body)

    @property
    def constraints(self):
        reader = WCSEOConfigReader(get_eoxserver_config())
        return {
            "CountDefault": reader.paging_count_default
        }

    def handle(self, request):
        decoder = self.get_decoder(request)
        eo_ids = decoder.eo_ids
        
        containment = decoder.containment
        if not containment:
            containment = "overlaps"

        count_default = self.constraints["CountDefault"]
        count = decoder.count
        if count_default is not None:
            count = min(count, count_default)

        try:
            subsets = Subsets(
                decoder.subsets, 
                crs="http://www.opengis.net/def/crs/EPSG/0/4326",
                allowed_types=Trim
            )
        except ValueError, e:
            raise InvalidSubsettingException(str(e))

        inc_dss_section = decoder.section_included("DatasetSeriesDescriptions")
        inc_cov_section = decoder.section_included("CoverageDescriptions")

        if len(eo_ids) == 0:
            raise

        # fetch a list of all requested EOObjects
        available_ids = models.EOObject.objects.filter(
            identifier__in=eo_ids
        ).values_list("identifier", flat=True)

        # match the requested EOIDs against the available ones. If any are
        # requested, that are not available, raise and exit.
        failed = [ eo_id for eo_id in eo_ids if eo_id not in available_ids ]
        if failed:
            raise NoSuchDatasetSeriesOrCoverageException(failed)

        collections_qs = subsets.filter(models.Collection.objects.filter(
            identifier__in=eo_ids
        ), containment="overlaps")

        # create a set of all indirectly referenced containers by iterating
        # recursively. The containment is set to "overlaps", to also include 
        # collections that might have been excluded with "contains" but would 
        # have matching coverages inserted.

        def recursive_lookup(super_collection, collection_set):
            sub_collections = models.Collection.objects.filter(
                collections__in=[super_collection.pk]
            ).exclude(
                pk__in=map(lambda c: c.pk, collection_set)
            )
            sub_collections = subsets.filter(sub_collections, "overlaps")

            # Add all to the set
            collection_set |= set(sub_collections)

            for sub_collection in sub_collections:
                recursive_lookup(sub_collection, collection_set)

        collection_set = set(collections_qs)
        for collection in set(collection_set):
            recursive_lookup(collection, collection_set)

        collection_pks = map(lambda c: c.pk, collection_set)

        # Get all either directly referenced coverages or coverages that are
        # within referenced containers. Full subsetting is applied here.

        coverages_qs = subsets.filter(models.Coverage.objects.filter(
            Q(identifier__in=eo_ids) | Q(collections__in=collection_pks)
        ), containment=containment)

        # save a reference before limits are applied to obtain the full number
        # of matched coverages.
        coverages_no_limit_qs = coverages_qs

        
        num_collections = len(
            filter(lambda c: not models.iscoverage(c), collection_set)
        )

        # compute how many (if any) coverages can be retrieved. This depends on
        # the "count" parameter and default setting. Also, if we already 
        # exceeded the count, limit the number of dataset series aswell

        if inc_dss_section:
            displayed_collections = num_collections
        else:
            displayed_collections = 0

        if displayed_collections < count and inc_cov_section:
            coverages_qs = coverages_qs.order_by("identifier")[:count - displayed_collections]
        elif displayed_collections == count or not inc_cov_section:
            coverages_qs = []
        else:
            coverages_qs = []
            collection_set = sorted(collection_set, key=lambda c: c.identifier)[:count]

        # get a number of coverages that *would* have been included, but are not
        # because of the count parameter
        count_all_coverages = coverages_no_limit_qs.count()

        # if containment is "contains" we need to check all collections again
        if containment == "contains":
            collection_set = filter(lambda c: subsets.matches(c), collection_set)

        coverages = set()
        dataset_series = set()

        # finally iterate over everything that has been retrieved and get
        # a list of dataset series and coverages to be encoded into the response
        for eo_object in chain(coverages_qs, collection_set):
            if inc_cov_section and issubclass(eo_object.real_type, models.Coverage):
                coverages.add(eo_object.cast())
            elif inc_dss_section and issubclass(eo_object.real_type, models.DatasetSeries):
                dataset_series.add(eo_object.cast())

            else:
                # TODO: what to do here?
                pass

        # TODO: coverages should be sorted
        #coverages = sorted(coverages, ) 

        #encoder = WCS20CoverageDescriptionXMLEncoder()
        #return encoder.encode(coverages)

        # TODO: remove this at some point
        encoder = WCS20EOXMLEncoder()

        return (
            encoder.serialize(
                encoder.encode_eo_coverage_set_description(
                    sorted(dataset_series, key=lambda s: s.identifier), 
                    sorted(coverages, key=lambda c: c.identifier), 
                    count_all_coverages + num_collections
                ), pretty_print=True
            ),
            encoder.content_type
        )
Exemplo n.º 11
0
def lookup_layers(layers, subsets, suffixes=None):
    """ Performs a layer lookup for the given layer names. Applies the given
        subsets and looks up all layers with the given suffixes. Returns a
        list of ``LayerSelection`` objects.
    """
    suffixes = suffixes or (None,)
    logger.debug("Tested suffixes: %s", suffixes)

    # ------------------------------------------------------------------------
    # local closures:

    def _lookup_eoobject(layer_name):
        """ Search an EOObject matching the given layer name. """
        for suffix in suffixes:
            if not suffix:
                identifier = layer_name
            elif layer_name.endswith(suffix):
                identifier = layer_name[:-len(suffix)]
            else: # no match found
                continue
            eo_objects = list(models.EOObject.objects.filter(identifier=identifier))
            if len(eo_objects) > 0:
                return eo_objects[0], suffix
        raise LayerNotDefined(layer_name)

    def _get_wms_view(eo_object):
        """ Get an EOObject used for the WMS view. If there is no WMS view
            object the closure returns the input object.
        """
        try:
            md_item = eo_object.metadata_items.get(semantic="wms_view")
            return models.EOObject.objects.get(identifier=md_item.value)
        except ObjectDoesNotExist:
            return eo_object

    def _get_alias(eo_object):
        """ Get an EOObject alias, i.e., an identifier of the EOObject
            the given EOOobject provides the WMS view to.
        """
        try:
            return eo_object.metadata_items.get(semantic="wms_alias").value
        except ObjectDoesNotExist:
            return None

    def _recursive_lookup(collection):
        """ Search recursively through the nested collections
            and find the relevant EOObjects."""
        eo_objects = subsets.filter(models.EOObject.objects\
                        .filter(collections__in=[collection.pk])\
                        .exclude(pk__in=used_ids)\
                        .order_by("begin_time", "end_time", "identifier")\
                        .prefetch_related('metadata_items'))
        for eo_object in eo_objects:
            used_ids.add(eo_object.pk)
            if models.iscoverage(eo_object):
                selection.append(eo_object.cast(), _get_alias(eo_object))
            elif models.iscollection(eo_object):
                _recursive_lookup(eo_object)
            else:
                pass # TODO: Reporting of invalid EOObjects (?)

    # ------------------------------------------------------------------------

    selections = []

    # NOTE: The lookup is performed on a set of unique layer names. This has
    #       no effect on the rendering order of the layer as this is determined
    #       by the WMS request handled by the mapserver.

    for layer_name in set(layers):
        # get an EOObject and suffix matching the layer_name
        eoo_src, suffix = _lookup_eoobject(layer_name)

        # get EOObject holding the WMS view
        eoo_wms = _get_wms_view(eoo_src)

        # prepare the final EOObject(s) selection
        selection = LayerSelection(eoo_src, suffix)

        if models.iscollection(eoo_wms): # EOObject is a collection
            used_ids = set()
            # recursively look-up the coverages
            _recursive_lookup(eoo_wms)

        elif models.iscoverage(eoo_wms): # EOObject is a coverage
            # append to the selection if the coverage matches the subset
            if subsets.matches(eoo_wms):
                selection.append(eoo_wms.cast(), eoo_src.identifier)

        else:
            pass # TODO: Reporting of invalid EOObjects (?)

        selections.append(selection)

    return selections