def execute(self, sql, args=None): logger.info(self.mogrify(sql, args)) try: psycopg2.extensions.cursor.execute(self, sql, args) except Exception, exc: logger.error("%s: %s" % (exc.__class__.__name__, exc)) raise
def __exit__(self, exception_type, exception_value, traceback): if exception_value: logger.error("Errors were encountered, rolling back session") self._session.rollback() else: self._session.commit() self._session.close() logger.info("Closed db session")
def one(self, entity_type, filterr): query = self._session.query(entity_type).filter_by(**filterr) try: entity = query.one() except NoResultFound: logger.warn("No results found for %s %s" % (entity_type, filterr)) entity = None except MultipleResultsFound: logger.error("MultipleResultsFound") entity = query.first() except Exception: logger.error(sys.exc_info()) entity = None return entity
def insert_raster(sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None): raster_format = sqa.one(DataFormat, filterr={"name": "RASTER"}) vector_format = sqa.one(DataFormat, filterr={"name": "VECTOR"}) if (srid_in != srid_out) and datagranule.dataformat_id == raster_format.id: raise Exception( "Cannot reproject raster tiles, ensure srid_in and srid_out are same" ) if datagranule.dataformat_id == raster_format.id: tile_count = 0 mask_wkt = None if not mask is None: mask_wkt = sqa.AsEWKT(mask.geom, srid_in) for tile in ras.tile_generator(block_size=block_size): tile_wkt = tile["raster_wkt"] extent = tile["extent_wkt"] rt = RasterTile(datagranule=datagranule, rast=tile_wkt) rt.extent = extent if mask_wkt: if sqa.Intersects(extent, mask_wkt): sqa.insert(rt) tile_count += 1 else: sqa.insert(rt) tile_count += 1 if tile_count and tile_count % 100 == 0: logger.info("Inserted %d tiles" % tile_count) logger.info("Inserted %d tiles" % tile_count) return tile_count elif datagranule.dataformat_id == vector_format.id: batch_size = 1000 ogrds = ogr.Open(config.ogr_connection_string()) srs_in = get_srs(srid_in) if srid_in != srid_out: srs_out = get_srs(srid_out) transform = osr.CoordinateTransformation(srs_in, srs_out) layer = ogrds.CreateLayer(str(datagranule.table_name), srs_out, ogr.wkbPolygon, ['OVERWRITE=YES']) else: transform = None layer = ogrds.CreateLayer(str(datagranule.table_name), srs_in, ogr.wkbPolygon, ['OVERWRITE=YES']) logger.info("Created layer %s" % str(datagranule.table_name)) value_field = ogr.FieldDefn(str(datagranule.variable.name), ogr.OFTReal) layer.CreateField(value_field) layerDefn = layer.GetLayerDefn() layer.StartTransaction() count = 0 #create mask to filter ingest geometries mask_geom = None if not mask is None: mask_wkt = sqa.AsWKT(mask.geom, srid_in) mask_geom = ogr.CreateGeometryFromWkt(mask_wkt) for items in ras.vector_generator(block_size=block_size): for item in items: poly, value = item[0], item[1] if not mask_geom is None: if not mask_geom.Intersects(poly): continue feature = ogr.Feature(layerDefn) if transform: try: poly.Transform(transform) feature.SetGeometry(poly) except: logger.error("Reprojection failed") else: feature.SetGeometry(poly) feature.SetField(str(datagranule.variable.name), value) layer.CreateFeature(feature) count += 1 if count % batch_size == 0: logger.info("inserted features - %d" % count) layer.CommitTransaction() if count % batch_size != 0: logger.info("inserted features - %d" % count) layer.CommitTransaction() return count else: raise Exception("Format should be one of raster or vector")
def insert_raster(sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None): raster_format = sqa.one(DataFormat, filterr={"name": "RASTER"}) vector_format = sqa.one(DataFormat, filterr={"name": "VECTOR"}) if (srid_in != srid_out) and datagranule.dataformat_id == raster_format.id: raise Exception("Cannot reproject raster tiles, ensure srid_in and srid_out are same") if datagranule.dataformat_id == raster_format.id: tile_count = 0 mask_wkt = None if not mask is None: mask_wkt = sqa.AsEWKT(mask.geom, srid_in) for tile in ras.tile_generator(block_size=block_size): tile_wkt = tile["raster_wkt"] extent = tile["extent_wkt"] rt = RasterTile(datagranule=datagranule, rast=tile_wkt) rt.extent = extent if mask_wkt: if sqa.Intersects(extent, mask_wkt): sqa.insert(rt) tile_count += 1 else: sqa.insert(rt) tile_count += 1 if tile_count and tile_count % 100 == 0: logger.info("Inserted %d tiles" % tile_count) logger.info("Inserted %d tiles" % tile_count) return tile_count elif datagranule.dataformat_id == vector_format.id: batch_size = 1000 ogrds = ogr.Open(config.ogr_connection_string()) srs_in = get_srs(srid_in) if srid_in != srid_out: srs_out = get_srs(srid_out) transform = osr.CoordinateTransformation(srs_in, srs_out) layer = ogrds.CreateLayer(str(datagranule.table_name), srs_out, ogr.wkbPolygon, ['OVERWRITE=YES']) else: transform = None layer = ogrds.CreateLayer(str(datagranule.table_name), srs_in, ogr.wkbPolygon, ['OVERWRITE=YES']) logger.info("Created layer %s" % str(datagranule.table_name)) value_field = ogr.FieldDefn(str(datagranule.variable.name), ogr.OFTReal) layer.CreateField(value_field) layerDefn = layer.GetLayerDefn() layer.StartTransaction() count = 0 #create mask to filter ingest geometries mask_geom = None if not mask is None: mask_wkt = sqa.AsWKT(mask.geom, srid_in) mask_geom = ogr.CreateGeometryFromWkt(mask_wkt) for items in ras.vector_generator(block_size=block_size): for item in items: poly, value = item[0], item[1] if not mask_geom is None: if not mask_geom.Intersects(poly): continue feature = ogr.Feature(layerDefn) if transform: try: poly.Transform(transform) feature.SetGeometry(poly) except: logger.error("Reprojection failed") else: feature.SetGeometry(poly) feature.SetField(str(datagranule.variable.name), value) layer.CreateFeature(feature) count += 1 if count % batch_size == 0: logger.info("inserted features - %d" % count) layer.CommitTransaction() if count % batch_size != 0: logger.info("inserted features - %d" % count) layer.CommitTransaction() return count else: raise Exception("Format should be one of raster or vector")