Esempio n. 1
0
    def list_grs_schemas(self):
        """Retrieve a list of available Grid Schema on Brazil Data Cube database."""
        schemas = GridRefSys.query().all()

        return [
            dict(**Serializer.serialize(schema), crs=schema.crs)
            for schema in schemas
        ], 200
Esempio n. 2
0
    def get_grs_schema(self, grs_id):
        """Retrieve a Grid Schema definition with tiles associated."""
        schema = GridRefSys.query().filter(GridRefSys.id == grs_id).first()

        if schema is None:
            return 'GRS {} not found.'.format(grs_id), 404

        geom_table = schema.geom_table
        tiles = db.session.query(
            geom_table.c.tile,
            func.ST_AsGeoJSON(func.ST_Transform(geom_table.c.geom, 4326), 6, 3).cast(sqlalchemy.JSON).label('geom_wgs84')
        ).all()

        dump_grs = Serializer.serialize(schema)
        dump_grs['tiles'] = [dict(id=t.tile, geom_wgs84=t.geom_wgs84) for t in tiles]

        return dump_grs, 200
Esempio n. 3
0
    def get_grs_schema(cls,
                       grs_id,
                       bbox: Tuple[float, float, float, float] = None,
                       tiles=None):
        """Retrieve a Grid Schema definition with tiles associated."""
        schema: GridRefSys = GridRefSys.query().filter(
            GridRefSys.id == grs_id).first()

        if schema is None:
            return 'GRS {} not found.'.format(grs_id), 404

        geom_table = schema.geom_table
        srid_column = get_srid_column(geom_table.c, default_srid=4326)
        where = []
        if bbox is not None:
            x_min, y_min, x_max, y_max = bbox
            where.append(
                func.ST_Intersects(
                    func.ST_MakeEnvelope(x_min, y_min, x_max, y_max, 4326),
                    func.ST_Transform(
                        func.ST_SetSRID(geom_table.c.geom, srid_column),
                        4326)))

        if tiles:
            where.append(geom_table.c.tile.in_(tiles))

        tiles = db.session.query(
            geom_table.c.tile,
            func.ST_AsGeoJSON(
                func.ST_Transform(
                    func.ST_SetSRID(geom_table.c.geom, srid_column), 4326), 6,
                3).cast(
                    sqlalchemy.JSON).label('geom_wgs84')).filter(*where).all()

        dump_grs = Serializer.serialize(schema)
        dump_grs['tiles'] = [
            dict(id=t.tile, geom_wgs84=t.geom_wgs84) for t in tiles
        ]

        return dump_grs, 200
Esempio n. 4
0
    def _create_cube_definition(cls, cube_id: str, params: dict) -> dict:
        """Create a data cube definition.

        Basically, the definition consists in `Collection` and `Band` attributes.

        Note:
            It does not try to create when data cube already exists.

        Args:
            cube_id - Data cube
            params - Dict of required values to create data cube. See @validators.py

        Returns:
            A serialized data cube information.
        """
        cube_parts = get_cube_parts(cube_id)

        function = cube_parts.composite_function

        cube_id = cube_parts.datacube

        cube = Collection.query().filter(
            Collection.name == cube_id,
            Collection.version == params['version']).first()

        grs = GridRefSys.query().filter(
            GridRefSys.name == params['grs']).first()

        if grs is None:
            abort(404, f'Grid {params["grs"]} not found.')

        cube_function = CompositeFunction.query().filter(
            CompositeFunction.alias == function).first()

        if cube_function is None:
            abort(404, f'Function {function} not found.')

        data = dict(name='Meter', symbol='m')
        resolution_meter, _ = get_or_create_model(ResolutionUnit,
                                                  defaults=data,
                                                  symbol='m')

        mime_type, _ = get_or_create_model(MimeType,
                                           defaults=dict(name=COG_MIME_TYPE),
                                           name=COG_MIME_TYPE)

        if cube is None:
            cube = Collection(
                name=cube_id,
                title=params['title'],
                temporal_composition_schema=params['temporal_composition']
                if function != 'IDT' else None,
                composite_function_id=cube_function.id,
                grs=grs,
                _metadata=params['metadata'],
                description=params['description'],
                collection_type='cube',
                is_public=params.get('public', True),
                version=params['version'])

            cube.save(commit=False)

            bands = []

            default_bands = (CLEAR_OBSERVATION_NAME.lower(),
                             TOTAL_OBSERVATION_NAME.lower(),
                             PROVENANCE_NAME.lower())

            band_map = dict()

            for band in params['bands']:
                name = band['name'].strip()

                if name in default_bands:
                    continue

                is_not_cloud = params['quality_band'] != band['name']

                if band['name'] == params['quality_band']:
                    data_type = 'uint8'
                else:
                    data_type = band['data_type']

                band_model = Band(name=name,
                                  common_name=band['common_name'],
                                  collection=cube,
                                  min_value=0,
                                  max_value=10000 if is_not_cloud else 4,
                                  nodata=-9999 if is_not_cloud else 255,
                                  scale=0.0001 if is_not_cloud else 1,
                                  data_type=data_type,
                                  resolution_x=params['resolution'],
                                  resolution_y=params['resolution'],
                                  resolution_unit_id=resolution_meter.id,
                                  description='',
                                  mime_type_id=mime_type.id)

                if band.get('metadata'):
                    band_model._metadata = cls._validate_band_metadata(
                        deepcopy(band['metadata']), band_map)

                band_model.save(commit=False)
                bands.append(band_model)

                band_map[name] = band_model

                if band_model._metadata:
                    for _band_origin_id in band_model._metadata['expression'][
                            'bands']:
                        band_provenance = BandSRC(band_src_id=_band_origin_id,
                                                  band_id=band_model.id)
                        band_provenance.save(commit=False)

            quicklook = Quicklook(
                red=band_map[params['bands_quicklook'][0]].id,
                green=band_map[params['bands_quicklook'][1]].id,
                blue=band_map[params['bands_quicklook'][2]].id,
                collection=cube)

            quicklook.save(commit=False)

        # Create default Cube Bands
        if function != 'IDT':
            _ = cls.get_or_create_band(cube.id,
                                       **CLEAR_OBSERVATION_ATTRIBUTES,
                                       resolution_unit_id=resolution_meter.id,
                                       resolution_x=params['resolution'],
                                       resolution_y=params['resolution'])
            _ = cls.get_or_create_band(cube.id,
                                       **TOTAL_OBSERVATION_ATTRIBUTES,
                                       resolution_unit_id=resolution_meter.id,
                                       resolution_x=params['resolution'],
                                       resolution_y=params['resolution'])

            if function == 'STK':
                _ = cls.get_or_create_band(
                    cube.id,
                    **PROVENANCE_ATTRIBUTES,
                    resolution_unit_id=resolution_meter.id,
                    resolution_x=params['resolution'],
                    resolution_y=params['resolution'])

        if params.get('is_combined') and function != 'MED':
            _ = cls.get_or_create_band(cube.id,
                                       **DATASOURCE_ATTRIBUTES,
                                       resolution_unit_id=resolution_meter.id,
                                       resolution_x=params['resolution'],
                                       resolution_y=params['resolution'])

        return CollectionForm().dump(cube)
 def get_grid_by_name(grid_name: str):
     """Try to get a grid, otherwise raises 404."""
     grid = GridRefSys.query().filter(GridRefSys.name == grid_name).first()
     if not grid:
         raise NotFound(f'Grid {grid_name} not found.')
     return grid
Esempio n. 6
0
def get_collection_items(
    collection_id=None,
    roles=None,
    item_id=None,
    bbox=None,
    datetime=None,
    ids=None,
    collections=None,
    intersects=None,
    page=1,
    limit=10,
    query=None,
    **kwargs,
) -> Pagination:
    """Retrieve a list of collection items based on filters.

    :param collection_id: Single Collection ID to include in the search for items.
                          Only Items in one of the provided Collection will be searched, defaults to None
    :type collection_id: str, optional
    :param item_id: item identifier, defaults to None
    :type item_id: str, optional
    :param bbox: bounding box for intersection [west, north, east, south], defaults to None
    :type bbox: list, optional
    :param datetime: Single date+time, or a range ('/' seperator), formatted to RFC 3339, section 5.6.
                     Use double dots '..' for open date ranges, defaults to None. If the start or end date of an image
                     generated by a temporal composition intersects the given datetime or range it will be included in the
                     result.
    :type datetime: str, optional
    :param ids: Array of Item ids to return. All other filter parameters that further restrict the
                number of search results are ignored, defaults to None
    :type ids: list, optional
    :param collections: Array of Collection IDs to include in the search for items.
                        Only Items in one of the provided Collections will be searched, defaults to None
    :type collections: list, optional
    :param intersects: Searches items by performing intersection between their geometry and provided GeoJSON geometry.
                       All GeoJSON geometry types must be supported., defaults to None
    :type intersects: dict, optional
    :param page: The page offset of results, defaults to 1
    :type page: int, optional
    :param limit: The maximum number of results to return (page size), defaults to 10
    :type limit: int, optional
    :return: list of collectio items
    :rtype: list
    """
    columns = [
        func.concat(Collection.name, "-",
                    Collection.version).label("collection"),
        Collection.collection_type,
        Collection._metadata.label("meta"),
        Item._metadata.label("item_meta"),
        Item.name.label("item"),
        Item.id,
        Item.collection_id,
        Item.start_date.label("start"),
        Item.end_date.label("end"),
        Item.assets,
        Item.created,
        Item.updated,
        cast(Item.cloud_cover, Float).label("cloud_cover"),
        func.ST_AsGeoJSON(Item.geom).label("geom"),
        func.ST_XMin(Item.geom).label("xmin"),
        func.ST_XMax(Item.geom).label("xmax"),
        func.ST_YMin(Item.geom).label("ymin"),
        func.ST_YMax(Item.geom).label("ymax"),
        Tile.name.label("tile"),
    ]

    if roles is None:
        roles = []

    where = [
        Collection.id == Item.collection_id,
        or_(Collection.is_public.is_(True),
            Collection.id.in_([int(r.split(":")[0]) for r in roles])),
    ]

    collections_where = _where_collections(collection_id, collections)
    collections_where.append(
        or_(Collection.is_public.is_(True),
            Collection.id.in_([int(r.split(":")[0]) for r in roles])))
    outer_join = [(Tile, [Item.tile_id == Tile.id])]
    _geom_tables = []
    _collections = Collection.query().filter(*collections_where).all()
    if bbox or intersects:
        grids = GridRefSys.query().filter(
            GridRefSys.id.in_([c.grid_ref_sys_id
                               for c in _collections])).all()
        for grid in grids:
            geom_table = grid.geom_table
            if geom_table is None:
                continue
            _geom_tables.append(geom_table)

    if ids is not None:
        if isinstance(ids, str):
            ids = ids.split(",")
        where += [Item.name.in_(ids)]
    else:
        where += _where_collections(collection_id, collections)

        if item_id is not None:
            where += [Item.name.like(item_id)]

        if query:
            filters = create_query_filter(query)
            if filters:
                where += filters

        if intersects is not None:
            # Intersect with native grid if there is
            geom_expr = func.ST_GeomFromGeoJSON(str(intersects))
            grids_where, joins = intersect_grids(geom_expr,
                                                 geom_tables=_geom_tables)

            where += grids_where
            outer_join += joins
        elif bbox is not None:
            try:
                if isinstance(bbox, str):
                    bbox = bbox.split(",")

                bbox = [float(x) for x in bbox]

                if bbox[0] == bbox[2] or bbox[1] == bbox[3]:
                    raise InvalidBoundingBoxError("")

                geom_expr = func.ST_MakeEnvelope(bbox[0], bbox[1], bbox[2],
                                                 bbox[3],
                                                 func.ST_SRID(Item.geom))
                grid_where, joins = intersect_grids(geom_expr,
                                                    geom_tables=_geom_tables)

                where += grid_where
                outer_join += joins
            except (ValueError, InvalidBoundingBoxError) as e:
                abort(400, f"'{bbox}' is not a valid bbox.")

        if datetime is not None:
            if "/" in datetime:
                matches_open = ("..", "")
                time_start, time_end = datetime.split("/")
                if time_start in matches_open:  # open start
                    date_filter = [
                        or_(Item.start_date <= time_end,
                            Item.end_date <= time_end)
                    ]
                elif time_end in matches_open:  # open end
                    date_filter = [
                        or_(Item.start_date >= time_start,
                            Item.end_date >= time_start)
                    ]
                else:  # closed range
                    date_filter = [
                        or_(
                            and_(Item.start_date >= time_start,
                                 Item.start_date <= time_end),
                            and_(Item.end_date >= time_start,
                                 Item.end_date <= time_end),
                            and_(Item.start_date < time_start,
                                 Item.end_date > time_end),
                        )
                    ]
            else:
                date_filter = [
                    and_(Item.start_date <= datetime,
                         Item.end_date >= datetime)
                ]
            where += date_filter

    query = session.query(*columns)
    for entity, join_conditions in outer_join:
        query = query.outerjoin(entity, *join_conditions)

    try:
        query = query.filter(*where).order_by(Item.start_date.desc(), Item.id)
        result = query.paginate(page=int(page),
                                per_page=int(limit),
                                error_out=False,
                                max_per_page=BDC_STAC_MAX_LIMIT)

        return result
    except Exception as err:
        msg = str(err)
        if hasattr(err, "orig"):
            msg = str(err.orig)
        abort(400, msg.rstrip())
Esempio n. 7
0
    def check_scenes(cls, collections: str, start_date: datetime, end_date: datetime,
                     catalog: str = None, dataset: str = None,
                     grid: str = None, tiles: list = None, bbox: list = None, catalog_kwargs=None, only_tiles=False):
        """Check for the scenes in remote provider and compares with the Collection Builder."""
        bbox_list = []
        if grid and tiles:
            grid = GridRefSys.query().filter(GridRefSys.name == grid).first_or_404(f'Grid "{grid}" not found.')
            geom_table = grid.geom_table

            rows = db.session.query(
                geom_table.c.tile,
                func.ST_Xmin(func.ST_Transform(geom_table.c.geom, 4326)).label('xmin'),
                func.ST_Ymin(func.ST_Transform(geom_table.c.geom, 4326)).label('ymin'),
                func.ST_Xmax(func.ST_Transform(geom_table.c.geom, 4326)).label('xmax'),
                func.ST_Ymax(func.ST_Transform(geom_table.c.geom, 4326)).label('ymax'),
            ).filter(geom_table.c.tile.in_(tiles)).all()
            for row in rows:
                bbox_list.append((row.tile, (row.xmin, row.ymin, row.xmax, row.ymax)))
        else:
            bbox_list.append(('', bbox))

        instance, provider = get_provider(catalog)

        collection_map = dict()
        collection_ids = list()

        for _collection in collections:
            collection, version = _collection.split('-')

            collection = Collection.query().filter(
                Collection.name == collection,
                Collection.version == version
            ).first_or_404(f'Collection "{collection}-{version}" not found.')

            collection_ids.append(collection.id)
            collection_map[_collection] = collection

        options = dict(start_date=start_date, end_date=end_date)
        if catalog_kwargs:
            options.update(catalog_kwargs)

        redis = current_app.redis
        output = dict(
            collections={cname: dict(total_scenes=0, total_missing=0, missing_external=[]) for cname in collections}
        )

        items = {cid: set() for cid in collection_ids}
        external_scenes = set()

        for tile, _bbox in bbox_list:
            with redis.pipeline() as pipe:
                if only_tiles:
                    entry = tile
                    options['tile'] = tile
                else:
                    options['bbox'] = _bbox
                    entry = _bbox

                periods = _generate_periods(start_date.replace(tzinfo=None), end_date.replace(tzinfo=None))

                for period_start, period_end in periods:
                    _items = db.session.query(Item.name, Item.collection_id).filter(
                        Item.collection_id.in_(collection_ids),
                        func.ST_Intersects(
                            func.ST_MakeEnvelope(
                                *_bbox, func.ST_SRID(Item.geom)
                            ),
                            Item.geom
                        ),
                        or_(
                            and_(Item.start_date >= period_start, Item.start_date <= period_end),
                            and_(Item.end_date >= period_start, Item.end_date <= period_end),
                            and_(Item.start_date < period_start, Item.end_date > period_end),
                        )
                    ).order_by(Item.name).all()

                    for item in _items:
                        items[item.collection_id].add(item.name)

                    options['start_date'] = period_start.strftime('%Y-%m-%d')
                    options['end_date'] = period_end.strftime('%Y-%m-%d')

                    key = f'scenes:{catalog}:{dataset}:{period_start.strftime("%Y%m%d")}_{period_end.strftime("%Y%m%d")}_{entry}'

                    pipe.get(key)
                    provider_scenes = []

                    if not redis.exists(key):
                        provider_scenes = provider.search(dataset, **options)
                        provider_scenes = [s.scene_id for s in provider_scenes]

                        pipe.set(key, json.dumps(provider_scenes))

                    external_scenes = external_scenes.union(set(provider_scenes))

                cached_scenes = pipe.execute()

                for cache in cached_scenes:
                    # When cache is True, represents set the value were cached.
                    if cache is not None and cache is not True:
                        external_scenes = external_scenes.union(set(json.loads(cache)))

        output['total_external'] = len(external_scenes)
        for _collection_name, _collection in collection_map.items():
            _items = set(items[_collection.id])
            diff = list(external_scenes.difference(_items))

            output['collections'][_collection_name]['total_scenes'] = len(_items)
            output['collections'][_collection_name]['total_missing'] = len(diff)
            output['collections'][_collection_name]['missing_external'] = diff

            for cname, _internal_collection in collection_map.items():
                if cname != _collection_name:
                    diff = list(_items.difference(set(items[_internal_collection.id])))
                    output['collections'][_collection_name][f'total_missing_{cname}'] = len(diff)
                    output['collections'][_collection_name][f'missing_{cname}'] = diff

        return output