예제 #1
0
class DatabaseTest(unittest.TestCase):
    """Tests ConfigManager and local/remote config access."""
    def setUp(self):
        self.session = Context().getSession()

    def tearDown(self):
        all_data = self.session.query(Test_Table).delete()
        self.session.commit()
        Context().closeSession()

    def testCreateData(self):
        for x in range(O):
            obj = Test_Table(
                key=''.join(
                    random.SystemRandom().choice(string.ascii_uppercase +
                                                 string.digits)
                    for _ in range(M)),
                val=''.join(
                    random.SystemRandom().choice(string.ascii_uppercase +
                                                 string.digits)
                    for _ in range(N)))
            try:
                self.session.add(obj)
                self.session.commit()
            except:
                self.session.rollback()
                #raise
            finally:
                self.session.close()
                Context().closeSession()

        self.assertEqual(self.session.query(Test_Table).count(), O)
        Context().closeSession()
예제 #2
0
 def add_dataset(self, obj):
     session = Context().getSession()
     ds_exists = session.query(Catalog_Dataset).filter(
         Catalog_Dataset.entity_id == obj.entity_id).filter(
             Catalog_Dataset.tile_identifier == obj.tile_identifier).filter(
                 Catalog_Dataset.acq_time == obj.acq_time)
     if not ds_exists.count():
         try:
             c = Catalog_Dataset(**dict(obj))
             session.add(c)
             session.commit()
         except:
             session.rollback()
             raise
         finally:
             session.close()
             Context().closeSession()
     return True
예제 #3
0
class Persistance:
    def __init__(self):
        self.logger = logging.getLogger('eoss.' + __name__)
        self.session = Context().getSession()
        self.engine = Context().get_engine()

    #@region.cache_on_arguments()
    def get_sensors(self, group):
        sensor_agg = func.array_agg(SensorAggregation.sensor,
                                    type_=String).label('sensor_agg')
        level_agg = func.array_agg(SensorAggregation.level,
                                   type_=String).label('level_agg')
        return self.session.query(
            SensorAggregation.aggregation_name, sensor_agg,
            level_agg).filter(SensorAggregation.aggregation_type ==
                              group).distinct().group_by(
                                  SensorAggregation.aggregation_name).order_by(
                                      SensorAggregation.aggregation_name)

    def delete_dataset(self, entity_id):
        self.session.query(Catalog_Dataset).filter(
            Catalog_Dataset.entity_id == entity_id).delete(
                synchronize_session=False)
        self.session.commit()

    #@region.cache_on_arguments()
    def get_dataset(self, entity_id):
        ds = self.session.query(Catalog_Dataset).filter(
            Catalog_Dataset.entity_id == entity_id).all()
        return ds

    def get_dataset_by_sensor_and_date(self, sensor, acq_date):
        ds = self.session.query(Catalog_Dataset).filter(
            Catalog_Dataset.sensor == sensor).filter(
                func.DATE(Catalog_Dataset.acq_time) == func.DATE(
                    acq_date)).all()
        print ds
        return ds

    def get_observation_coverage(self, reference_type_id, last_days=2):
        struct = dict()
        struct['geojson'] = list()
        struct['attr'] = list()
        sql = text(
            'select r.ref_name, st_asgeojson(st_centroid(r.geom)), max(ds.acq_time::date), count(ds.*) from catalogue.spatialreference r, CATALOGue.global_catalog ds where r.referencetype_id = %d and ds.tile_identifier = r.ref_name and ds.acq_time::date > now()::date-%d group by r.ref_name, st_asgeojson(st_centroid(r.geom)) order by r.ref_name'
            % (reference_type_id, last_days))

        sql = text(
            'select r.ref_name, st_asgeojson(r.geom), max(ds.acq_time::date), count(ds.*) from catalogue.spatialreference r, CATALOGue.global_catalog ds where r.referencetype_id = %d and ds.tile_identifier = r.ref_name and ds.acq_time::date > now()::date-%d group by r.ref_name, st_asgeojson(r.geom) order by r.ref_name'
            % (reference_type_id, last_days))
        result = self.engine.execute(sql)
        for r in result:
            struct['geojson'].append(ujson.loads(r[1]))
            struct['attr'].append({
                'tile_id': r[0],
                'count': r[3],
                'last_observation': r[2].isoformat(),
            })
        return struct

    def add_dataset(self, obj):
        session = Context().getSession()
        ds_exists = session.query(Catalog_Dataset).filter(
            Catalog_Dataset.entity_id == obj.entity_id).filter(
                Catalog_Dataset.tile_identifier == obj.tile_identifier).filter(
                    Catalog_Dataset.acq_time == obj.acq_time)
        if not ds_exists.count():
            try:
                c = Catalog_Dataset(**dict(obj))
                session.add(c)
                session.commit()
            except:
                session.rollback()
                raise
            finally:
                session.close()
                Context().closeSession()
        return True

    #@region.cache_on_arguments()
    def get_all_sensor_aggregations(self):
        return self.session.query(SensorAggregation).all()

    @region.cache_on_arguments()
    def get_all_tilegrid(self):
        return self.session.query(
            Spatial_Reference.ref_name,
            geoalchemy2.functions.ST_AsGeoJSON(Spatial_Reference.geom)).all()

    def get_reference_by_sensorgrid(self, ref_id, ref_type_id, sensor_grid):
        sat_grid = aliased(Spatial_Reference)
        ref_obj = aliased(Spatial_Reference)
        return self.session.query(sat_grid.ref_name).filter(
            sat_grid.geom.ST_Intersects(ref_obj.geom)).filter(
                ref_obj.referencetype_id == ref_type_id).filter(
                    ref_obj.ref_id == ref_id).filter(
                        sat_grid.referencetype_id == sensor_grid)

    def get_reference(self, ref_id, ref_type_id):
        return self.session.query(Spatial_Reference).filter(
            Spatial_Reference.referencetype_id == ref_type_id).filter(
                Spatial_Reference.ref_id == ref_id)

    def get_referencebyaoi(self, wkt, sensor_grid):
        sat_grid = aliased(Spatial_Reference)
        wkt = WKTElement(wkt, srid=4326)
        return self.session.query(sat_grid.ref_name).filter(
            sat_grid.referencetype_id == sensor_grid).filter(
                sat_grid.geom.ST_Intersects(wkt))

    def find_dataset(self, dates_filter, sensors_filter, grid_list,
                     joint_gridset, clouds):
        if len(sensors_filter) > 0 and len(dates_filter) > 0 and len(
                grid_list[11]) > 0:
            query = self.session.query(Catalog_Dataset).filter(
                Catalog_Dataset.daynight ==
                'day').filter(Catalog_Dataset.clouds <= clouds).filter(
                    or_(*sensors_filter)).filter(or_(*dates_filter)).filter(
                        Catalog_Dataset.tile_identifier.in_(joint_gridset))
        elif len(sensors_filter) == 0 and len(dates_filter) > 0 and len(
                grid_list[11]) > 0:
            query = self.session.query(Catalog_Dataset).filter(
                Catalog_Dataset.daynight == 'day').filter(
                    Catalog_Dataset.clouds <= clouds).filter(
                        or_(*dates_filter)).filter(
                            Catalog_Dataset.tile_identifier.in_(joint_gridset))
        elif len(sensors_filter) > 0 and len(dates_filter) > 0 and len(
                grid_list[11]) == 0:
            query = self.session.query(Catalog_Dataset).filter(
                Catalog_Dataset.daynight == 'day').filter(
                    Catalog_Dataset.clouds <= clouds).filter(
                        or_(*sensors_filter)).filter(or_(*dates_filter))
        elif len(sensors_filter) == 0 and len(dates_filter) > 0 and len(
                grid_list[11]) == 0:
            query = self.session.query(Catalog_Dataset).filter(
                Catalog_Dataset.daynight == 'day').filter(
                    Catalog_Dataset.clouds <= clouds).filter(
                        or_(*dates_filter))

        return query

    @region.cache_on_arguments()
    def get_tile_geom(self, tiles_list):
        return self.session.query(
            Spatial_Reference.ref_name,
            geoalchemy2.functions.ST_AsGeoJSON(Spatial_Reference.geom)).filter(
                Spatial_Reference.ref_name.in_(tiles_list))

    @region.cache_on_arguments()
    def get_all_reference_types(self):
        return self.session.query(Spatial_Reference_type).all()

    @region.cache_on_arguments()
    def get_all_references(self):
        return self.session.query(Spatial_Reference.ref_name,
                                  Spatial_Reference.referencetype_id,
                                  Spatial_Reference.ref_id).distinct().all()

    def get_reference_by_groupid_polygon(self, group_id, polygon_wkt):
        return self.session.query(
            Spatial_Reference,
            geoalchemy2.functions.ST_AsGeoJSON(Spatial_Reference.geom),
            geoalchemy2.functions.ST_AsGeoJSON(
                geoalchemy2.functions.ST_Envelope(
                    Spatial_Reference.geom))).filter(
                        Spatial_Reference.referencetype_id == group_id).filter(
                            Spatial_Reference.geom.ST_Intersects('SRID=4326;' +
                                                                 polygon_wkt))

    def get_reference_by_groupid_reference_id(self, group_id, reference_id):
        return self.session.query(
            Spatial_Reference,
            geoalchemy2.functions.ST_AsGeoJSON(Spatial_Reference.geom),
            geoalchemy2.functions.ST_AsGeoJSON(
                geoalchemy2.functions.ST_Envelope(
                    Spatial_Reference.geom))).filter(
                        Spatial_Reference.referencetype_id == group_id).filter(
                            Spatial_Reference.ref_id == reference_id)

    def get_reference_by_groupid_reference_name(self, group_id,
                                                reference_name):
        return self.session.query(
            Spatial_Reference,
            geoalchemy2.functions.ST_AsGeoJSON(Spatial_Reference.geom),
            geoalchemy2.functions.ST_AsGeoJSON(
                geoalchemy2.functions.ST_Envelope(
                    Spatial_Reference.geom))).filter(
                        Spatial_Reference.referencetype_id == group_id).filter(
                            Spatial_Reference.ref_name == reference_name)

    @region.cache_on_arguments()
    def get_selected_references(self, REGISTERED_REF_TYPES):
        return self.session.query(
            Spatial_Reference.ref_name, Spatial_Reference.referencetype_id,
            Spatial_Reference.ref_id).filter(
                Spatial_Reference.referencetype_id.in_(
                    REGISTERED_REF_TYPES)).distinct().all()

    def get_dataset_tiles_date_clouds(self, sensors_filter, dates_filter,
                                      tiles_list):
        if len(sensors_filter) > 0 and len(dates_filter) > 0:
            query = self.session.query(
                Catalog_Dataset.tile_identifier, Catalog_Dataset.acq_time,
                Catalog_Dataset.clouds).filter(or_(*sensors_filter)).filter(
                    or_(*dates_filter)).filter(
                        Catalog_Dataset.tile_identifier.in_(
                            tiles_list)).order_by(
                                Catalog_Dataset.tile_identifier,
                                Catalog_Dataset.acq_time)
        elif len(sensors_filter) == 0 and len(dates_filter) > 0:
            query = self.session.query(
                Catalog_Dataset.tile_identifier, Catalog_Dataset.acq_time,
                Catalog_Dataset.clouds).filter(or_(*dates_filter)).filter(
                    Catalog_Dataset.tile_identifier.in_(tiles_list)).order_by(
                        Catalog_Dataset.tile_identifier,
                        Catalog_Dataset.acq_time)
        return query